1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/count_zeros.h> 9 #include <linux/kernel.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/bpf.h> 13 #include <linux/btf.h> 14 #include <linux/bpf_verifier.h> 15 #include <linux/filter.h> 16 #include <net/netlink.h> 17 #include <linux/file.h> 18 #include <linux/vmalloc.h> 19 #include <linux/stringify.h> 20 #include <linux/bsearch.h> 21 #include <linux/sort.h> 22 #include <linux/perf_event.h> 23 #include <linux/ctype.h> 24 #include <linux/error-injection.h> 25 #include <linux/bpf_lsm.h> 26 #include <linux/security.h> 27 #include <linux/verification.h> 28 #include <linux/keyctl.h> 29 #include <linux/btf_ids.h> 30 #include <linux/poison.h> 31 #include <linux/module.h> 32 #include <linux/cpumask.h> 33 #include <linux/cnum.h> 34 #include <linux/gcd.h> 35 #include <linux/bpf_mem_alloc.h> 36 #include <net/xdp.h> 37 #include <linux/trace_events.h> 38 #include <linux/kallsyms.h> 39 40 #include "diagnostics.h" 41 #include "disasm.h" 42 43 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 44 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 45 [_id] = & _name ## _verifier_ops, 46 #define BPF_MAP_TYPE(_id, _ops) 47 #define BPF_LINK_TYPE(_id, _name) 48 #include <linux/bpf_types.h> 49 #undef BPF_PROG_TYPE 50 #undef BPF_MAP_TYPE 51 #undef BPF_LINK_TYPE 52 }; 53 54 enum bpf_features { 55 BPF_FEAT_RDONLY_CAST_TO_VOID = 0, 56 BPF_FEAT_STREAMS = 1, 57 __MAX_BPF_FEAT, 58 }; 59 60 struct bpf_mem_alloc bpf_global_percpu_ma; 61 static bool bpf_global_percpu_ma_set; 62 63 /* bpf_check() is a static code analyzer that walks eBPF program 64 * instruction by instruction and updates register/stack state. 65 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 66 * 67 * The first pass is depth-first-search to check that the program is a DAG. 68 * It rejects the following programs: 69 * - larger than BPF_MAXINSNS insns 70 * - if loop is present (detected via back-edge) 71 * - unreachable insns exist (shouldn't be a forest. program = one function) 72 * - out of bounds or malformed jumps 73 * The second pass is all possible path descent from the 1st insn. 74 * Since it's analyzing all paths through the program, the length of the 75 * analysis is limited to 64k insn, which may be hit even if total number of 76 * insn is less then 4K, but there are too many branches that change stack/regs. 77 * Number of 'branches to be analyzed' is limited to 1k 78 * 79 * On entry to each instruction, each register has a type, and the instruction 80 * changes the types of the registers depending on instruction semantics. 81 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 82 * copied to R1. 83 * 84 * All registers are 64-bit. 85 * R0 - return register 86 * R1-R5 argument passing registers 87 * R6-R9 callee saved registers 88 * R10 - frame pointer read-only 89 * 90 * At the start of BPF program the register R1 contains a pointer to bpf_context 91 * and has type PTR_TO_CTX. 92 * 93 * Verifier tracks arithmetic operations on pointers in case: 94 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 95 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 96 * 1st insn copies R10 (which has FRAME_PTR) type into R1 97 * and 2nd arithmetic instruction is pattern matched to recognize 98 * that it wants to construct a pointer to some element within stack. 99 * So after 2nd insn, the register R1 has type PTR_TO_STACK 100 * (and -20 constant is saved for further stack bounds checking). 101 * Meaning that this reg is a pointer to stack plus known immediate constant. 102 * 103 * Most of the time the registers have SCALAR_VALUE type, which 104 * means the register has some value, but it's not a valid pointer. 105 * (like pointer plus pointer becomes SCALAR_VALUE type) 106 * 107 * When verifier sees load or store instructions the type of base register 108 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 109 * four pointer types recognized by check_mem_access() function. 110 * 111 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 112 * and the range of [ptr, ptr + map's value_size) is accessible. 113 * 114 * registers used to pass values to function calls are checked against 115 * function argument constraints. 116 * 117 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 118 * It means that the register type passed to this function must be 119 * PTR_TO_STACK and it will be used inside the function as 120 * 'pointer to map element key' 121 * 122 * For example the argument constraints for bpf_map_lookup_elem(): 123 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 124 * .arg1_type = ARG_CONST_MAP_PTR, 125 * .arg2_type = ARG_PTR_TO_MAP_KEY, 126 * 127 * ret_type says that this function returns 'pointer to map elem value or null' 128 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 129 * 2nd argument should be a pointer to stack, which will be used inside 130 * the helper function as a pointer to map element key. 131 * 132 * On the kernel side the helper function looks like: 133 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 134 * { 135 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 136 * void *key = (void *) (unsigned long) r2; 137 * void *value; 138 * 139 * here kernel can access 'key' and 'map' pointers safely, knowing that 140 * [key, key + map->key_size) bytes are valid and were initialized on 141 * the stack of eBPF program. 142 * } 143 * 144 * Corresponding eBPF program may look like: 145 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 146 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 147 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 148 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 149 * here verifier looks at prototype of map_lookup_elem() and sees: 150 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 151 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 152 * 153 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 154 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 155 * and were initialized prior to this call. 156 * If it's ok, then verifier allows this BPF_CALL insn and looks at 157 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 158 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 159 * returns either pointer to map value or NULL. 160 * 161 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 162 * insn, the register holding that pointer in the true branch changes state to 163 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 164 * branch. See check_cond_jmp_op(). 165 * 166 * After the call R0 is set to return type of the function and registers R1-R5 167 * are set to NOT_INIT to indicate that they are no longer readable. 168 * 169 * The following reference types represent a potential reference to a kernel 170 * resource which, after first being allocated, must be checked and freed by 171 * the BPF program: 172 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 173 * 174 * When the verifier sees a helper call return a reference type, it allocates a 175 * pointer id for the reference and stores it in the current function state. 176 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 177 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 178 * passes through a NULL-check conditional. For the branch wherein the state is 179 * changed to CONST_IMM, the verifier releases the reference. 180 * 181 * For each helper function that allocates a reference, such as 182 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 183 * bpf_sk_release(). When a reference type passes into the release function, 184 * the verifier also releases the reference. If any unchecked or unreleased 185 * reference remains at the end of the program, the verifier rejects it. 186 */ 187 188 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 189 struct bpf_verifier_stack_elem { 190 /* verifier state is 'st' 191 * before processing instruction 'insn_idx' 192 * and after processing instruction 'prev_insn_idx' 193 */ 194 struct bpf_verifier_state st; 195 int insn_idx; 196 int prev_insn_idx; 197 struct bpf_verifier_stack_elem *next; 198 /* length of verifier log at the time this state was pushed on stack */ 199 u32 log_pos; 200 u64 diag_log_pos; 201 }; 202 203 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 204 #define BPF_COMPLEXITY_LIMIT_STATES 64 205 206 #define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512 207 208 #define BPF_PRIV_STACK_MIN_SIZE 64 209 210 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); 211 static int __release_reference_nomark(struct bpf_verifier_state *state, int id); 212 static int release_reference_nomark(struct bpf_verifier_env *env, int id); 213 static int release_reference(struct bpf_verifier_env *env, int id); 214 static int invalidate_reference(struct bpf_verifier_env *env, int id); 215 static u32 reg_lifetime_id(struct bpf_verifier_env *env, 216 const struct bpf_reg_state *reg); 217 static void invalidate_non_owning_refs(struct bpf_verifier_env *env); 218 static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); 219 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); 220 static bool is_tracing_prog_type(enum bpf_prog_type type); 221 static int ref_set_non_owning(struct bpf_verifier_env *env, 222 struct bpf_reg_state *reg); 223 static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); 224 static inline bool in_sleepable_context(struct bpf_verifier_env *env); 225 static bool in_rcu_callback(struct bpf_verifier_env *env); 226 static const char *non_sleepable_context_description(struct bpf_verifier_env *env); 227 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); 228 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); 229 230 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 231 struct bpf_map *map, 232 bool unpriv, bool poison) 233 { 234 unpriv |= bpf_map_ptr_unpriv(aux); 235 aux->map_ptr_state.unpriv = unpriv; 236 aux->map_ptr_state.poison = poison; 237 aux->map_ptr_state.map_ptr = map; 238 } 239 240 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 241 { 242 bool poisoned = bpf_map_key_poisoned(aux); 243 244 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 245 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 246 } 247 248 static void update_ref_obj(struct ref_obj_desc *ref_obj, struct bpf_reg_state *reg) 249 { 250 ref_obj->id = reg->id; 251 ref_obj->parent_id = reg->parent_id; 252 ref_obj->cnt++; 253 } 254 255 static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *ref_obj) 256 { 257 if (ref_obj->cnt > 1) { 258 verifier_bug(env, "function expects only one referenced object but got %d\n", 259 ref_obj->cnt); 260 return -EFAULT; 261 } 262 263 return 0; 264 } 265 266 struct bpf_kfunc_meta { 267 struct btf *btf; 268 const struct btf_type *proto; 269 const char *name; 270 const u32 *flags; 271 s32 id; 272 }; 273 274 struct btf *btf_vmlinux; 275 276 typedef struct argno { 277 int argno; 278 } argno_t; 279 280 static argno_t argno_from_reg(u32 regno) 281 { 282 return (argno_t){ .argno = regno }; 283 } 284 285 static argno_t argno_from_arg(u32 arg) 286 { 287 return (argno_t){ .argno = -arg }; 288 } 289 290 static int reg_from_argno(argno_t a) 291 { 292 if (a.argno >= 0) 293 return a.argno; 294 if (a.argno >= -MAX_BPF_FUNC_REG_ARGS) 295 return -a.argno; 296 return -1; 297 } 298 299 static int arg_from_argno(argno_t a) 300 { 301 if (a.argno < 0) 302 return -a.argno; 303 return -1; 304 } 305 306 static int arg_idx_from_argno(argno_t a) 307 { 308 return arg_from_argno(a) - 1; 309 } 310 311 /* Normalize helper register numbers and kfunc argument numbers to ABI slots. */ 312 static u32 arg_slot_from_argno(argno_t a) 313 { 314 return abs(a.argno) - 1; 315 } 316 317 static const char *btf_type_name(const struct btf *btf, u32 id) 318 { 319 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 320 } 321 322 static DEFINE_MUTEX(bpf_verifier_lock); 323 static DEFINE_MUTEX(btf_vmlinux_lock); 324 static DEFINE_MUTEX(bpf_percpu_ma_lock); 325 326 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 327 { 328 struct bpf_verifier_env *env = private_data; 329 va_list args; 330 331 if (!bpf_verifier_log_needed(&env->log)) 332 return; 333 334 va_start(args, fmt); 335 bpf_verifier_vlog(&env->log, fmt, args); 336 va_end(args); 337 } 338 339 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 340 struct bpf_reg_state *reg, 341 struct bpf_retval_range range, const char *ctx, 342 const char *reg_name) 343 { 344 bool unknown = true; 345 346 verbose(env, "%s the register %s has", ctx, reg_name); 347 if (reg_smin(reg) > S64_MIN) { 348 verbose(env, " smin=%lld", reg_smin(reg)); 349 unknown = false; 350 } 351 if (reg_smax(reg) < S64_MAX) { 352 verbose(env, " smax=%lld", reg_smax(reg)); 353 unknown = false; 354 } 355 if (unknown) 356 verbose(env, " unknown scalar value"); 357 verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); 358 } 359 360 static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) 361 { 362 enum bpf_reg_type type; 363 364 type = reg->type; 365 if (type_may_be_null(type)) 366 return false; 367 368 /* 369 * The types below guarantee a non-NULL base, an unbounded offset can 370 * still wrap base + offset to zero. 371 */ 372 if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF) 373 return false; 374 375 type = base_type(type); 376 return type == PTR_TO_SOCKET || 377 type == PTR_TO_TCP_SOCK || 378 type == PTR_TO_XDP_SOCK || 379 type == PTR_TO_BUF || 380 type == PTR_TO_MAP_VALUE || 381 type == PTR_TO_MAP_KEY || 382 type == PTR_TO_SOCK_COMMON || 383 (type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) || 384 (type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) || 385 type == CONST_PTR_TO_MAP; 386 } 387 388 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 389 { 390 struct btf_record *rec = NULL; 391 struct btf_struct_meta *meta; 392 393 if (reg->type == PTR_TO_MAP_VALUE) { 394 rec = reg->map_ptr->record; 395 } else if (type_is_ptr_alloc_obj(reg->type)) { 396 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 397 if (meta) 398 rec = meta->record; 399 } 400 return rec; 401 } 402 403 bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) 404 { 405 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; 406 407 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; 408 } 409 410 static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) 411 { 412 const struct btf_type *func, *func_proto; 413 const struct btf *btf = env->prog->aux->btf; 414 u32 btf_id; 415 416 if (!btf || !env->prog->aux->func_info) 417 return NULL; 418 419 btf_id = env->prog->aux->func_info[subprog].type_id; 420 421 /* Both already validated by prepare_btf_func() at prog load. */ 422 func = btf_type_by_id(btf, btf_id); 423 func_proto = btf_type_by_id(btf, func->type); 424 425 return btf_type_skip_modifiers(btf, func_proto->type, NULL); 426 } 427 428 static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) 429 { 430 const struct btf_type *type = subprog_ret_type(env, subprog); 431 432 return type && btf_type_is_void(type); 433 } 434 435 static u32 ret_regs_cnt(u32 size) 436 { 437 return size > 8 && size <= 16 ? 2 : 1; 438 } 439 440 /* Registers holding a function return value, in order. See ret_regs_cnt(). */ 441 static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; 442 443 static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) 444 { 445 const struct btf *btf = env->prog->aux->btf; 446 const struct btf_type *type; 447 int subprog; 448 u32 size; 449 450 if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) 451 return 0; 452 453 /* 454 * Skip the main program: its return value is the program's exit code, 455 * read out of R0, so it never uses the register pair. An extension does 456 * have a real prototype for subprog 0, but btf_check_func_type_match() 457 * refuses to replace a function returning more than 8 bytes. 458 */ 459 for (subprog = 1; subprog < env->subprog_cnt; subprog++) { 460 type = subprog_ret_type(env, subprog); 461 if (!type || btf_type_is_void(type)) 462 continue; 463 if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, 464 "cannot size return type of subprog %d", subprog)) 465 return -EFAULT; 466 if (ret_regs_cnt(size) > 1) { 467 subprog_info(env, subprog)->ret_reg_pair = true; 468 env->prog->jit_required = 1; 469 } 470 } 471 472 return 0; 473 } 474 475 const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) 476 { 477 struct bpf_func_info *info; 478 479 if (!env->prog->aux->func_info) 480 return ""; 481 482 info = &env->prog->aux->func_info[subprog]; 483 return btf_type_name(env->prog->aux->btf, info->type_id); 484 } 485 486 void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) 487 { 488 struct bpf_subprog_info *info = subprog_info(env, subprog); 489 490 info->is_cb = true; 491 info->is_async_cb = true; 492 info->is_exception_cb = true; 493 } 494 495 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) 496 { 497 return subprog_info(env, subprog)->is_exception_cb; 498 } 499 500 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 501 { 502 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK); 503 } 504 505 static bool type_is_rdonly_mem(u32 type) 506 { 507 return type & MEM_RDONLY; 508 } 509 510 static bool is_acquire_function(enum bpf_func_id func_id, 511 const struct bpf_map *map) 512 { 513 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 514 515 if (func_id == BPF_FUNC_sk_lookup_tcp || 516 func_id == BPF_FUNC_sk_lookup_udp || 517 func_id == BPF_FUNC_skc_lookup_tcp || 518 func_id == BPF_FUNC_ringbuf_reserve || 519 func_id == BPF_FUNC_kptr_xchg) 520 return true; 521 522 if (func_id == BPF_FUNC_map_lookup_elem && 523 (map_type == BPF_MAP_TYPE_SOCKMAP || 524 map_type == BPF_MAP_TYPE_SOCKHASH)) 525 return true; 526 527 return false; 528 } 529 530 static bool is_ptr_cast_function(enum bpf_func_id func_id) 531 { 532 return func_id == BPF_FUNC_tcp_sock || 533 func_id == BPF_FUNC_sk_fullsock || 534 func_id == BPF_FUNC_skc_to_tcp_sock || 535 func_id == BPF_FUNC_skc_to_tcp6_sock || 536 func_id == BPF_FUNC_skc_to_udp6_sock || 537 func_id == BPF_FUNC_skc_to_mptcp_sock || 538 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 539 func_id == BPF_FUNC_skc_to_tcp_request_sock; 540 } 541 542 static bool is_sync_callback_calling_kfunc(u32 btf_id); 543 static bool is_async_callback_calling_kfunc(u32 btf_id); 544 static bool is_call_rcu_kfunc(u32 btf_id); 545 static bool is_callback_calling_kfunc(u32 btf_id); 546 547 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); 548 static bool is_task_work_add_kfunc(u32 func_id); 549 550 static bool is_sync_callback_calling_function(enum bpf_func_id func_id) 551 { 552 return func_id == BPF_FUNC_for_each_map_elem || 553 func_id == BPF_FUNC_find_vma || 554 func_id == BPF_FUNC_loop || 555 func_id == BPF_FUNC_user_ringbuf_drain; 556 } 557 558 static bool is_async_callback_calling_function(enum bpf_func_id func_id) 559 { 560 return func_id == BPF_FUNC_timer_set_callback; 561 } 562 563 static bool is_callback_calling_function(enum bpf_func_id func_id) 564 { 565 return is_sync_callback_calling_function(func_id) || 566 is_async_callback_calling_function(func_id); 567 } 568 569 bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn) 570 { 571 return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) || 572 (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm)); 573 } 574 575 bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn) 576 { 577 return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) || 578 (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm)); 579 } 580 581 static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn *insn) 582 { 583 /* bpf_timer callbacks are never sleepable. */ 584 if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) 585 return false; 586 587 /* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */ 588 if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) 589 return false; 590 591 /* bpf_wq and bpf_task_work callbacks are always sleepable. */ 592 if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && 593 (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) 594 return true; 595 596 verifier_bug(env, "unhandled async callback in is_async_cb_sleepable"); 597 return false; 598 } 599 600 /* From here on the maps of the program count it as a sleepable user. */ 601 static void mark_sleepable_async_cb(struct bpf_verifier_env *env) 602 { 603 int i; 604 605 if (bpf_prog_runs_sleepable(env->prog)) 606 return; 607 608 env->prog->aux->sleepable_async_cb = true; 609 for (i = 0; i < env->used_map_cnt; i++) 610 atomic64_inc(&env->used_maps[i]->sleepable_refcnt); 611 } 612 613 bool bpf_is_may_goto_insn(struct bpf_insn *insn) 614 { 615 return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; 616 } 617 618 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 619 { 620 int allocated_slots = bpf_stack_nr_slots(state); 621 622 /* 623 * We need to check that slots between [spi - nr_slots + 1, spi] are 624 * within [0, allocated_stack). 625 * 626 * Please note that the spi grows downwards. For example, a dynptr 627 * takes the size of two stack slots; the first slot will be at 628 * spi and the second slot will be at spi - 1. 629 */ 630 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 631 } 632 633 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 634 const char *obj_kind, int nr_slots) 635 { 636 int off, spi; 637 638 if (!tnum_is_const(reg->var_off)) { 639 verbose(env, "%s has to be at a constant offset\n", obj_kind); 640 return -EINVAL; 641 } 642 643 off = reg->var_off.value; 644 if (off >= 0 || off % BPF_REG_SIZE) { 645 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 646 return -EINVAL; 647 } 648 649 spi = bpf_get_spi(off); 650 if (spi + 1 < nr_slots) { 651 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 652 return -EINVAL; 653 } 654 655 if (!is_spi_bounds_valid(bpf_func(env, reg), spi, nr_slots)) 656 return -ERANGE; 657 return spi; 658 } 659 660 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 661 { 662 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); 663 } 664 665 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) 666 { 667 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); 668 } 669 670 static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 671 { 672 return stack_slot_obj_get_spi(env, reg, "irq_flag", 1); 673 } 674 675 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 676 { 677 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 678 case DYNPTR_TYPE_LOCAL: 679 return BPF_DYNPTR_TYPE_LOCAL; 680 case DYNPTR_TYPE_RINGBUF: 681 return BPF_DYNPTR_TYPE_RINGBUF; 682 case DYNPTR_TYPE_SKB: 683 return BPF_DYNPTR_TYPE_SKB; 684 case DYNPTR_TYPE_XDP: 685 return BPF_DYNPTR_TYPE_XDP; 686 case DYNPTR_TYPE_SKB_META: 687 return BPF_DYNPTR_TYPE_SKB_META; 688 case DYNPTR_TYPE_FILE: 689 return BPF_DYNPTR_TYPE_FILE; 690 default: 691 return BPF_DYNPTR_TYPE_INVALID; 692 } 693 } 694 695 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type) 696 { 697 switch (type) { 698 case BPF_DYNPTR_TYPE_LOCAL: 699 return DYNPTR_TYPE_LOCAL; 700 case BPF_DYNPTR_TYPE_RINGBUF: 701 return DYNPTR_TYPE_RINGBUF; 702 case BPF_DYNPTR_TYPE_SKB: 703 return DYNPTR_TYPE_SKB; 704 case BPF_DYNPTR_TYPE_XDP: 705 return DYNPTR_TYPE_XDP; 706 case BPF_DYNPTR_TYPE_SKB_META: 707 return DYNPTR_TYPE_SKB_META; 708 case BPF_DYNPTR_TYPE_FILE: 709 return DYNPTR_TYPE_FILE; 710 default: 711 return 0; 712 } 713 } 714 715 static bool dynptr_type_referenced(enum bpf_dynptr_type type) 716 { 717 return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE; 718 } 719 720 static void __mark_dynptr_reg(struct bpf_reg_state *reg, 721 enum bpf_dynptr_type type, 722 bool first_slot, int id, int parent_id); 723 724 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, 725 struct bpf_reg_state *sreg1, 726 struct bpf_reg_state *sreg2, 727 enum bpf_dynptr_type type, int parent_id) 728 { 729 int id = ++env->id_gen; 730 731 __mark_dynptr_reg(sreg1, type, true, id, parent_id); 732 __mark_dynptr_reg(sreg2, type, false, id, parent_id); 733 } 734 735 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, 736 struct bpf_reg_state *reg, 737 enum bpf_dynptr_type type) 738 { 739 __mark_dynptr_reg(reg, type, true, ++env->id_gen, 0); 740 } 741 742 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 743 struct bpf_func_state *state, int spi); 744 745 static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env, 746 struct bpf_func_state *state, 747 int first_spi, int last_spi) 748 { 749 int spi, err; 750 751 for (spi = first_spi; spi <= last_spi; spi++) { 752 err = destroy_if_dynptr_stack_slot(env, state, spi); 753 if (err) 754 return err; 755 } 756 757 return 0; 758 } 759 760 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 761 enum bpf_arg_type arg_type, int insn_idx, 762 struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) 763 { 764 struct bpf_func_state *state = bpf_func(env, reg); 765 int spi, i, err, parent_id = 0; 766 enum bpf_dynptr_type type; 767 768 spi = dynptr_get_spi(env, reg); 769 if (spi < 0) 770 return spi; 771 772 /* We cannot assume both spi and spi - 1 belong to the same dynptr, 773 * hence we need to destroy dynptrs in both slots, 774 * to ensure that for the following example: 775 * [d1][d1][d2][d2] 776 * spi 3 2 1 0 777 * So marking spi = 2 should lead to destruction of both d1 and d2. In 778 * case they do belong to same dynptr, second call won't see slot_type 779 * as STACK_DYNPTR and will simply skip destruction. 780 */ 781 err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi); 782 if (err) 783 return err; 784 785 for (i = 0; i < BPF_REG_SIZE; i++) { 786 bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR; 787 bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR; 788 } 789 790 type = arg_to_dynptr_type(arg_type); 791 if (type == BPF_DYNPTR_TYPE_INVALID) 792 return -EINVAL; 793 794 if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */ 795 err = validate_ref_obj(env, ref_obj); 796 if (err) 797 return err; 798 799 /* All non-clone constructors take their backing object in R1. */ 800 parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]); 801 802 if (dynptr_type_referenced(type)) { 803 int id; 804 805 /* 806 * Create an intermediate reference that tracks the referenced 807 * object for the referenced dynptr. Freeing a referenced dynptr 808 * through helpers/kfuncs will invalidate all clones. 809 */ 810 id = acquire_reference(env, insn_idx, parent_id); 811 if (id < 0) 812 return id; 813 814 parent_id = id; 815 } 816 } else { /* bpf_dynptr_clone() */ 817 parent_id = dynptr->parent_id; 818 } 819 820 mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr, 821 &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id); 822 823 return 0; 824 } 825 826 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack) 827 { 828 int i; 829 830 for (i = 0; i < BPF_REG_SIZE; i++) { 831 stack[0].slot_type[i] = STACK_INVALID; 832 stack[1].slot_type[i] = STACK_INVALID; 833 } 834 835 bpf_mark_reg_not_init(env, &stack[0].spilled_ptr); 836 bpf_mark_reg_not_init(env, &stack[1].spilled_ptr); 837 } 838 839 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 840 { 841 struct bpf_func_state *state = bpf_func(env, reg); 842 int spi; 843 844 spi = dynptr_get_spi(env, reg); 845 if (spi < 0) 846 return spi; 847 848 /* 849 * For referenced dynptr, release the parent ref which cascades to 850 * all clones and derived slices. For non-referenced dynptr, only 851 * the dynptr and slices derived from it will be invalidated. 852 */ 853 reg = &bpf_stack_slot(state, spi)->spilled_ptr; 854 return release_reference(env, dynptr_type_referenced(reg->dynptr.type) 855 ? reg->parent_id 856 : reg->id); 857 } 858 859 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 860 struct bpf_reg_state *reg); 861 862 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 863 { 864 if (!env->allow_ptr_leaks) 865 bpf_mark_reg_not_init(env, reg); 866 else 867 __mark_reg_unknown(env, reg); 868 } 869 870 static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) 871 { 872 struct bpf_stack_state *stack; 873 struct bpf_func_state *state; 874 struct bpf_reg_state *reg; 875 int ref_cnt = 0; 876 877 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, 1 << STACK_DYNPTR, ({ 878 if (!stack || stack->slot_type[0] != STACK_DYNPTR) 879 continue; 880 if (!stack->spilled_ptr.dynptr.first_slot) 881 continue; 882 if (stack->spilled_ptr.parent_id == v_parent_id) 883 ref_cnt++; 884 })); 885 886 return ref_cnt; 887 } 888 889 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 890 struct bpf_func_state *state, int spi) 891 { 892 struct bpf_stack_state *slot = bpf_stack_slot(state, spi); 893 int err = 0; 894 895 /* We always ensure that STACK_DYNPTR is never set partially, 896 * hence just checking for slot_type[0] is enough. This is 897 * different for STACK_SPILL, where it may be only set for 898 * 1 byte, so code has to use is_spilled_reg. 899 */ 900 if (slot->slot_type[0] != STACK_DYNPTR) 901 return 0; 902 903 /* Reposition spi to first slot */ 904 if (!slot->spilled_ptr.dynptr.first_slot) { 905 spi = spi + 1; 906 slot = bpf_stack_slot(state, spi); 907 } 908 909 /* 910 * A referenced dynptr can be overwritten only if there is at 911 * least one other dynptr sharing the same virtual ref parent, 912 * ensuring the reference can still be properly released. 913 */ 914 if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) && 915 dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) { 916 verbose(env, "cannot overwrite referenced dynptr\n"); 917 bpf_diag_res( 918 env, env->insn_idx, "referenced dynptr overwrite", 919 "This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.", 920 "Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot."); 921 return -EINVAL; 922 } 923 924 /* Invalidate the dynptr and any derived slices */ 925 err = release_reference(env, slot->spilled_ptr.id); 926 if (!err) { 927 mark_stack_slot_scratched(env, spi); 928 mark_stack_slot_scratched(env, spi - 1); 929 } 930 931 return err; 932 } 933 934 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 935 { 936 int spi; 937 938 if (reg->type == CONST_PTR_TO_DYNPTR) 939 return false; 940 941 spi = dynptr_get_spi(env, reg); 942 943 /* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an 944 * error because this just means the stack state hasn't been updated yet. 945 * We will do check_mem_access to check and update stack bounds later. 946 */ 947 if (spi < 0 && spi != -ERANGE) 948 return false; 949 950 /* We don't need to check if the stack slots are marked by previous 951 * dynptr initializations because we allow overwriting existing unreferenced 952 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls 953 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are 954 * touching are completely destructed before we reinitialize them for a new 955 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early 956 * instead of delaying it until the end where the user will get "Unreleased 957 * reference" error. 958 */ 959 return true; 960 } 961 962 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 963 { 964 struct bpf_func_state *state = bpf_func(env, reg); 965 struct bpf_stack_state *slot; 966 int i, spi; 967 968 /* This already represents first slot of initialized bpf_dynptr. 969 * 970 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to 971 * check_func_arg_reg_off's logic, so we don't need to check its 972 * offset and alignment. 973 */ 974 if (reg->type == CONST_PTR_TO_DYNPTR) 975 return true; 976 977 spi = dynptr_get_spi(env, reg); 978 if (spi < 0) 979 return false; 980 slot = bpf_stack_slot(state, spi); 981 if (!slot->spilled_ptr.dynptr.first_slot) 982 return false; 983 984 for (i = 0; i < BPF_REG_SIZE; i++) { 985 if (slot->slot_type[i] != STACK_DYNPTR || 986 bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR) 987 return false; 988 } 989 990 return true; 991 } 992 993 static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 994 { 995 struct bpf_func_state *state; 996 int spi; 997 998 if (reg->type == CONST_PTR_TO_DYNPTR) 999 return reg->dynptr.type; 1000 1001 spi = dynptr_get_spi(env, reg); 1002 if (spi < 0) 1003 return BPF_DYNPTR_TYPE_INVALID; 1004 state = bpf_func(env, reg); 1005 return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; 1006 } 1007 1008 static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) 1009 { 1010 return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; 1011 } 1012 1013 /* The data of the other types is contiguous: bpf_dynptr_slice() does not use the buffer. */ 1014 static bool dynptr_slice_may_be_buffer(enum bpf_dynptr_type type) 1015 { 1016 return type != BPF_DYNPTR_TYPE_LOCAL && type != BPF_DYNPTR_TYPE_RINGBUF && 1017 type != BPF_DYNPTR_TYPE_SKB_META; 1018 } 1019 1020 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1021 enum bpf_arg_type arg_type) 1022 { 1023 /* 1024 * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr. 1025 * Test the flags rather than the whole arg_type, which may carry 1026 * unrelated ones such as PTR_MAYBE_NULL. 1027 */ 1028 if (!(arg_type & DYNPTR_TYPE_FLAG_MASK)) 1029 return true; 1030 1031 return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); 1032 } 1033 1034 static void __mark_reg_known_zero(struct bpf_reg_state *reg); 1035 1036 static bool in_rcu_cs(struct bpf_verifier_env *env); 1037 1038 static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta); 1039 1040 static int mark_stack_slots_iter(struct bpf_verifier_env *env, 1041 struct bpf_call_arg_meta *meta, 1042 struct bpf_reg_state *reg, int insn_idx, 1043 struct btf *btf, u32 btf_id, int nr_slots) 1044 { 1045 struct bpf_func_state *state = bpf_func(env, reg); 1046 int spi, i, j, id; 1047 1048 spi = iter_get_spi(env, reg, nr_slots); 1049 if (spi < 0) 1050 return spi; 1051 1052 id = acquire_reference(env, insn_idx, 0); 1053 if (id < 0) 1054 return id; 1055 1056 for (i = 0; i < nr_slots; i++) { 1057 struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); 1058 struct bpf_reg_state *st = &slot->spilled_ptr; 1059 1060 __mark_reg_known_zero(st); 1061 st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ 1062 if (is_kfunc_rcu_protected(meta)) { 1063 if (in_rcu_cs(env)) 1064 st->type |= MEM_RCU; 1065 else 1066 st->type |= PTR_UNTRUSTED; 1067 } 1068 st->id = i == 0 ? id : 0; 1069 st->iter.btf = btf; 1070 st->iter.btf_id = btf_id; 1071 st->iter.state = BPF_ITER_STATE_ACTIVE; 1072 st->iter.depth = 0; 1073 1074 for (j = 0; j < BPF_REG_SIZE; j++) 1075 slot->slot_type[j] = STACK_ITER; 1076 1077 mark_stack_slot_scratched(env, spi - i); 1078 } 1079 1080 return 0; 1081 } 1082 1083 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, 1084 struct bpf_reg_state *reg, int nr_slots) 1085 { 1086 struct bpf_func_state *state = bpf_func(env, reg); 1087 int spi, i, j, err; 1088 1089 spi = iter_get_spi(env, reg, nr_slots); 1090 if (spi < 0) 1091 return spi; 1092 1093 for (i = 0; i < nr_slots; i++) { 1094 struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); 1095 struct bpf_reg_state *st = &slot->spilled_ptr; 1096 1097 if (i == 0) { 1098 err = release_reference(env, st->id); 1099 if (err) 1100 return err; 1101 } 1102 1103 bpf_mark_reg_not_init(env, st); 1104 1105 for (j = 0; j < BPF_REG_SIZE; j++) 1106 slot->slot_type[j] = STACK_INVALID; 1107 1108 mark_stack_slot_scratched(env, spi - i); 1109 } 1110 1111 return 0; 1112 } 1113 1114 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, 1115 struct bpf_reg_state *reg, int nr_slots) 1116 { 1117 struct bpf_func_state *state = bpf_func(env, reg); 1118 int spi, i, j; 1119 1120 /* For -ERANGE (i.e. spi not falling into allocated stack slots), we 1121 * will do check_mem_access to check and update stack bounds later, so 1122 * return true for that case. 1123 */ 1124 spi = iter_get_spi(env, reg, nr_slots); 1125 if (spi == -ERANGE) 1126 return true; 1127 if (spi < 0) 1128 return false; 1129 1130 for (i = 0; i < nr_slots; i++) { 1131 struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); 1132 1133 for (j = 0; j < BPF_REG_SIZE; j++) 1134 if (slot->slot_type[j] == STACK_ITER) 1135 return false; 1136 } 1137 1138 return true; 1139 } 1140 1141 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1142 struct btf *btf, u32 btf_id, int nr_slots) 1143 { 1144 struct bpf_func_state *state = bpf_func(env, reg); 1145 int spi, i, j; 1146 1147 spi = iter_get_spi(env, reg, nr_slots); 1148 if (spi < 0) 1149 return -EINVAL; 1150 1151 for (i = 0; i < nr_slots; i++) { 1152 struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); 1153 struct bpf_reg_state *st = &slot->spilled_ptr; 1154 1155 if (st->type & PTR_UNTRUSTED) 1156 return -EPROTO; 1157 /* only main (first) slot has id set */ 1158 if (i == 0 && !st->id) 1159 return -EINVAL; 1160 if (i != 0 && st->id) 1161 return -EINVAL; 1162 if (st->iter.btf != btf || st->iter.btf_id != btf_id) 1163 return -EINVAL; 1164 1165 for (j = 0; j < BPF_REG_SIZE; j++) 1166 if (slot->slot_type[j] != STACK_ITER) 1167 return -EINVAL; 1168 } 1169 1170 return 0; 1171 } 1172 1173 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); 1174 static int release_irq_state(struct bpf_verifier_env *env, int id); 1175 1176 static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, 1177 struct bpf_call_arg_meta *meta, 1178 struct bpf_reg_state *reg, int insn_idx, 1179 int kfunc_class) 1180 { 1181 struct bpf_func_state *state = bpf_func(env, reg); 1182 struct bpf_stack_state *slot; 1183 struct bpf_reg_state *st; 1184 int spi, i, id; 1185 1186 spi = irq_flag_get_spi(env, reg); 1187 if (spi < 0) 1188 return spi; 1189 1190 id = acquire_irq_state(env, insn_idx); 1191 if (id < 0) 1192 return id; 1193 1194 slot = bpf_stack_slot(state, spi); 1195 st = &slot->spilled_ptr; 1196 1197 __mark_reg_known_zero(st); 1198 st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ 1199 st->id = id; 1200 st->irq.kfunc_class = kfunc_class; 1201 1202 for (i = 0; i < BPF_REG_SIZE; i++) 1203 slot->slot_type[i] = STACK_IRQ_FLAG; 1204 1205 mark_stack_slot_scratched(env, spi); 1206 return 0; 1207 } 1208 1209 static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1210 int kfunc_class) 1211 { 1212 struct bpf_func_state *state = bpf_func(env, reg); 1213 struct bpf_stack_state *slot; 1214 struct bpf_reg_state *st; 1215 int spi, i, err; 1216 1217 spi = irq_flag_get_spi(env, reg); 1218 if (spi < 0) 1219 return spi; 1220 1221 slot = bpf_stack_slot(state, spi); 1222 st = &slot->spilled_ptr; 1223 1224 if (st->irq.kfunc_class != kfunc_class) { 1225 const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; 1226 const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; 1227 const char *reason; 1228 1229 verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", 1230 flag_kfunc, used_kfunc); 1231 reason = bpf_diag_fmt(env, 1232 "This IRQ flag was saved by %s IRQ kfuncs, but the restore call " 1233 "belongs to the %s IRQ kfunc family. Save and restore operations " 1234 "must use the same family.", 1235 flag_kfunc, used_kfunc); 1236 bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason, 1237 "Restore the flag with the matching IRQ restore kfunc for the save " 1238 "operation that created it.", 1239 bpf_diag_irq_depth(env->cur_state)); 1240 return -EINVAL; 1241 } 1242 1243 err = release_irq_state(env, st->id); 1244 WARN_ON_ONCE(err && err != -EACCES); 1245 if (err) { 1246 int insn_idx = 0; 1247 1248 for (int i = 0; i < env->cur_state->acquired_refs; i++) { 1249 if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) { 1250 insn_idx = env->cur_state->refs[i].insn_idx; 1251 break; 1252 } 1253 } 1254 1255 verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", 1256 env->cur_state->active_irq_id, insn_idx); 1257 bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order", 1258 "IRQ-disabled regions must be restored in last-in, first-out order, " 1259 "but this restore does not match the currently active IRQ flag.", 1260 "Restore nested IRQ flags in the reverse order they were saved.", 1261 bpf_diag_irq_depth(env->cur_state)); 1262 return err; 1263 } 1264 1265 bpf_mark_reg_not_init(env, st); 1266 1267 for (i = 0; i < BPF_REG_SIZE; i++) 1268 slot->slot_type[i] = STACK_INVALID; 1269 1270 mark_stack_slot_scratched(env, spi); 1271 return 0; 1272 } 1273 1274 static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1275 { 1276 struct bpf_func_state *state = bpf_func(env, reg); 1277 struct bpf_stack_state *slot; 1278 int spi, i; 1279 1280 /* For -ERANGE (i.e. spi not falling into allocated stack slots), we 1281 * will do check_mem_access to check and update stack bounds later, so 1282 * return true for that case. 1283 */ 1284 spi = irq_flag_get_spi(env, reg); 1285 if (spi == -ERANGE) 1286 return true; 1287 if (spi < 0) 1288 return false; 1289 1290 slot = bpf_stack_slot(state, spi); 1291 1292 for (i = 0; i < BPF_REG_SIZE; i++) 1293 if (slot->slot_type[i] == STACK_IRQ_FLAG) 1294 return false; 1295 return true; 1296 } 1297 1298 static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1299 { 1300 struct bpf_func_state *state = bpf_func(env, reg); 1301 struct bpf_stack_state *slot; 1302 struct bpf_reg_state *st; 1303 int spi, i; 1304 1305 spi = irq_flag_get_spi(env, reg); 1306 if (spi < 0) 1307 return -EINVAL; 1308 1309 slot = bpf_stack_slot(state, spi); 1310 st = &slot->spilled_ptr; 1311 1312 if (!st->id) 1313 return -EINVAL; 1314 1315 for (i = 0; i < BPF_REG_SIZE; i++) 1316 if (slot->slot_type[i] != STACK_IRQ_FLAG) 1317 return -EINVAL; 1318 return 0; 1319 } 1320 1321 /* Check if given stack slot is "special": 1322 * - spilled register state (STACK_SPILL); 1323 * - dynptr state (STACK_DYNPTR); 1324 * - iter state (STACK_ITER). 1325 * - irq flag state (STACK_IRQ_FLAG) 1326 */ 1327 static bool is_stack_slot_special(const struct bpf_stack_state *stack) 1328 { 1329 enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1]; 1330 1331 switch (type) { 1332 case STACK_SPILL: 1333 case STACK_DYNPTR: 1334 case STACK_ITER: 1335 case STACK_IRQ_FLAG: 1336 return true; 1337 case STACK_INVALID: 1338 case STACK_POISON: 1339 case STACK_MISC: 1340 case STACK_ZERO: 1341 return false; 1342 default: 1343 WARN_ONCE(1, "unknown stack slot type %d\n", type); 1344 return true; 1345 } 1346 } 1347 1348 /* The reg state of a pointer or a bounded scalar was saved when 1349 * it was spilled to the stack. 1350 */ 1351 1352 /* 1353 * Mark stack slot as STACK_MISC, unless it is already: 1354 * - STACK_INVALID, in which case they are equivalent. 1355 * - STACK_ZERO, in which case we preserve more precise STACK_ZERO. 1356 * - STACK_POISON, which truly forbids access to the slot. 1357 * Regardless of allow_ptr_leaks setting (i.e., privileged or unprivileged 1358 * mode), we won't promote STACK_INVALID to STACK_MISC. In privileged case it is 1359 * unnecessary as both are considered equivalent when loading data and pruning, 1360 * in case of unprivileged mode it will be incorrect to allow reads of invalid 1361 * slots. 1362 */ 1363 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype) 1364 { 1365 if (*stype == STACK_ZERO) 1366 return; 1367 if (*stype == STACK_INVALID || *stype == STACK_POISON) 1368 return; 1369 *stype = STACK_MISC; 1370 } 1371 1372 static void scrub_spilled_slot(u8 *stype) 1373 { 1374 if (*stype != STACK_INVALID && *stype != STACK_POISON) 1375 *stype = STACK_MISC; 1376 } 1377 1378 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1379 * small to hold src. This is different from krealloc since we don't want to preserve 1380 * the contents of dst. 1381 * 1382 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1383 * not be allocated. 1384 */ 1385 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1386 { 1387 size_t alloc_bytes; 1388 void *orig = dst; 1389 size_t bytes; 1390 1391 if (ZERO_OR_NULL_PTR(src)) 1392 goto out; 1393 1394 if (unlikely(check_mul_overflow(n, size, &bytes))) 1395 return NULL; 1396 1397 alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes)); 1398 dst = krealloc(orig, alloc_bytes, flags); 1399 if (!dst) { 1400 kfree(orig); 1401 return NULL; 1402 } 1403 1404 memcpy(dst, src, bytes); 1405 out: 1406 return dst ? dst : ZERO_SIZE_PTR; 1407 } 1408 1409 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1410 * small to hold new_n items. new items are zeroed out if the array grows. 1411 * 1412 * Contrary to krealloc_array, does not free arr if new_n is zero. 1413 */ 1414 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1415 { 1416 size_t alloc_size; 1417 void *new_arr; 1418 1419 if (!new_n || old_n == new_n) 1420 goto out; 1421 1422 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1423 new_arr = krealloc(arr, alloc_size, GFP_KERNEL_ACCOUNT); 1424 if (!new_arr) { 1425 kfree(arr); 1426 return NULL; 1427 } 1428 arr = new_arr; 1429 1430 if (new_n > old_n) 1431 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1432 1433 out: 1434 return arr ? arr : ZERO_SIZE_PTR; 1435 } 1436 1437 static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src) 1438 { 1439 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1440 sizeof(struct bpf_reference_state), GFP_KERNEL_ACCOUNT); 1441 if (!dst->refs) 1442 return -ENOMEM; 1443 1444 dst->acquired_refs = src->acquired_refs; 1445 dst->active_locks = src->active_locks; 1446 dst->active_preempt_locks = src->active_preempt_locks; 1447 dst->active_rcu_locks = src->active_rcu_locks; 1448 dst->active_irq_id = src->active_irq_id; 1449 dst->active_lock_id = src->active_lock_id; 1450 dst->active_lock_ptr = src->active_lock_ptr; 1451 return 0; 1452 } 1453 1454 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1455 { 1456 size_t n = bpf_stack_nr_slots(src); 1457 1458 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1459 GFP_KERNEL_ACCOUNT); 1460 if (!dst->stack) 1461 return -ENOMEM; 1462 1463 dst->allocated_stack = src->allocated_stack; 1464 1465 /* copy stack args state */ 1466 n = src->out_stack_arg_cnt; 1467 if (n) { 1468 dst->stack_arg_regs = copy_array(dst->stack_arg_regs, src->stack_arg_regs, n, 1469 sizeof(struct bpf_reg_state), 1470 GFP_KERNEL_ACCOUNT); 1471 if (!dst->stack_arg_regs) 1472 return -ENOMEM; 1473 } 1474 1475 dst->out_stack_arg_cnt = src->out_stack_arg_cnt; 1476 return 0; 1477 } 1478 1479 static int resize_reference_state(struct bpf_verifier_state *state, size_t n) 1480 { 1481 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1482 sizeof(struct bpf_reference_state)); 1483 if (!state->refs) 1484 return -ENOMEM; 1485 1486 state->acquired_refs = n; 1487 return 0; 1488 } 1489 1490 /* Possibly update state->allocated_stack to be at least size bytes. Also 1491 * possibly update the function's high-water mark in its bpf_subprog_info. 1492 */ 1493 int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) 1494 { 1495 size_t old_n = bpf_stack_nr_slots(state), n; 1496 1497 /* The stack size is always a multiple of BPF_REG_SIZE. */ 1498 size = round_up(size, BPF_REG_SIZE); 1499 n = size / BPF_REG_SIZE; 1500 1501 if (old_n >= n) 1502 return 0; 1503 1504 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1505 if (!state->stack) 1506 return -ENOMEM; 1507 1508 state->allocated_stack = size; 1509 1510 /* update known max for given subprogram */ 1511 if (env->subprog_info[state->subprogno].stack_depth < size) 1512 env->subprog_info[state->subprogno].stack_depth = size; 1513 1514 return 0; 1515 } 1516 1517 static int grow_stack_arg_slots(struct bpf_verifier_env *env, 1518 struct bpf_func_state *state, int cnt) 1519 { 1520 size_t old_n = state->out_stack_arg_cnt; 1521 1522 if (old_n >= cnt) 1523 return 0; 1524 1525 state->stack_arg_regs = realloc_array(state->stack_arg_regs, old_n, cnt, 1526 sizeof(struct bpf_reg_state)); 1527 if (!state->stack_arg_regs) 1528 return -ENOMEM; 1529 1530 state->out_stack_arg_cnt = cnt; 1531 return 0; 1532 } 1533 1534 /* Acquire a pointer id from the env and update the state->refs to include 1535 * this new pointer reference. 1536 * On success, returns a valid pointer id to associate with the register 1537 * On failure, returns a negative errno. 1538 */ 1539 static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1540 { 1541 struct bpf_verifier_state *state = env->cur_state; 1542 int new_ofs = state->acquired_refs; 1543 int err; 1544 1545 err = resize_reference_state(state, state->acquired_refs + 1); 1546 if (err) 1547 return NULL; 1548 state->refs[new_ofs].insn_idx = insn_idx; 1549 1550 return &state->refs[new_ofs]; 1551 } 1552 1553 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id) 1554 { 1555 struct bpf_reference_state *s; 1556 1557 s = acquire_reference_state(env, insn_idx); 1558 if (!s) 1559 return -ENOMEM; 1560 s->type = REF_TYPE_PTR; 1561 s->id = ++env->id_gen; 1562 s->parent_id = parent_id; 1563 bpf_diag_record_ref_acquire(env, insn_idx, s->id); 1564 return s->id; 1565 } 1566 1567 static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, 1568 int id, void *ptr) 1569 { 1570 struct bpf_verifier_state *state = env->cur_state; 1571 struct bpf_reference_state *s; 1572 1573 s = acquire_reference_state(env, insn_idx); 1574 if (!s) 1575 return -ENOMEM; 1576 s->type = type; 1577 s->id = id; 1578 s->ptr = ptr; 1579 1580 state->active_locks++; 1581 state->active_lock_id = id; 1582 state->active_lock_ptr = ptr; 1583 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true, 1584 state->active_locks); 1585 return 0; 1586 } 1587 1588 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) 1589 { 1590 struct bpf_verifier_state *state = env->cur_state; 1591 struct bpf_reference_state *s; 1592 1593 s = acquire_reference_state(env, insn_idx); 1594 if (!s) 1595 return -ENOMEM; 1596 s->type = REF_TYPE_IRQ; 1597 s->id = ++env->id_gen; 1598 1599 state->active_irq_id = s->id; 1600 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true, 1601 bpf_diag_irq_depth(state)); 1602 return s->id; 1603 } 1604 1605 static void release_reference_state(struct bpf_verifier_state *state, int idx) 1606 { 1607 int last_idx; 1608 size_t rem; 1609 1610 /* IRQ state requires the relative ordering of elements remaining the 1611 * same, since it relies on the refs array to behave as a stack, so that 1612 * it can detect out-of-order IRQ restore. Hence use memmove to shift 1613 * the array instead of swapping the final element into the deleted idx. 1614 */ 1615 last_idx = state->acquired_refs - 1; 1616 rem = state->acquired_refs - idx - 1; 1617 if (last_idx && idx != last_idx) 1618 memmove(&state->refs[idx], &state->refs[idx + 1], sizeof(*state->refs) * rem); 1619 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1620 state->acquired_refs--; 1621 return; 1622 } 1623 1624 static bool find_reference_state(struct bpf_verifier_state *state, int id) 1625 { 1626 int i; 1627 1628 for (i = 0; i < state->acquired_refs; i++) { 1629 if (state->refs[i].type != REF_TYPE_PTR) 1630 continue; 1631 if (state->refs[i].id == id) 1632 return true; 1633 } 1634 1635 return false; 1636 } 1637 1638 static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) 1639 { 1640 return find_reference_state(env->cur_state, reg->id); 1641 } 1642 1643 static u32 reg_lifetime_id(struct bpf_verifier_env *env, 1644 const struct bpf_reg_state *reg) 1645 { 1646 return reg_is_referenced(env, reg) ? reg->id : reg->parent_id; 1647 } 1648 1649 static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) 1650 { 1651 struct bpf_verifier_state *state = env->cur_state; 1652 void *prev_ptr = NULL; 1653 u32 prev_id = 0; 1654 int i; 1655 1656 for (i = 0; i < state->acquired_refs; i++) { 1657 if (state->refs[i].type == type && state->refs[i].id == id && 1658 state->refs[i].ptr == ptr) { 1659 release_reference_state(state, i); 1660 state->active_locks--; 1661 /* Reassign active lock (id, ptr). */ 1662 state->active_lock_id = prev_id; 1663 state->active_lock_ptr = prev_ptr; 1664 bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK, 1665 false, state->active_locks); 1666 return 0; 1667 } 1668 if (state->refs[i].type & REF_TYPE_LOCK_MASK) { 1669 prev_id = state->refs[i].id; 1670 prev_ptr = state->refs[i].ptr; 1671 } 1672 } 1673 return -EINVAL; 1674 } 1675 1676 static int release_irq_state(struct bpf_verifier_env *env, int id) 1677 { 1678 struct bpf_verifier_state *state = env->cur_state; 1679 u32 prev_id = 0; 1680 int i; 1681 1682 if (id != state->active_irq_id) 1683 return -EACCES; 1684 1685 for (i = 0; i < state->acquired_refs; i++) { 1686 if (state->refs[i].type != REF_TYPE_IRQ) 1687 continue; 1688 if (state->refs[i].id == id) { 1689 release_reference_state(state, i); 1690 state->active_irq_id = prev_id; 1691 bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ, 1692 false, bpf_diag_irq_depth(state)); 1693 return 0; 1694 } else { 1695 prev_id = state->refs[i].id; 1696 } 1697 } 1698 return -EINVAL; 1699 } 1700 1701 static struct bpf_reference_state *find_lock_state(struct bpf_verifier_state *state, enum ref_state_type type, 1702 int id, void *ptr) 1703 { 1704 int i; 1705 1706 for (i = 0; i < state->acquired_refs; i++) { 1707 struct bpf_reference_state *s = &state->refs[i]; 1708 1709 if (!(s->type & type)) 1710 continue; 1711 1712 if (s->id == id && s->ptr == ptr) 1713 return s; 1714 } 1715 return NULL; 1716 } 1717 1718 static void free_func_state(struct bpf_func_state *state) 1719 { 1720 if (!state) 1721 return; 1722 kfree(state->loop_stack); 1723 kfree(state->stack_arg_regs); 1724 kfree(state->stack); 1725 kfree(state); 1726 } 1727 1728 void bpf_clear_jmp_history(struct bpf_verifier_state *state) 1729 { 1730 kfree(state->jmp_history); 1731 state->jmp_history = NULL; 1732 state->jmp_history_cnt = 0; 1733 } 1734 1735 void bpf_free_verifier_state(struct bpf_verifier_state *state, 1736 bool free_self) 1737 { 1738 int i; 1739 1740 for (i = 0; i <= state->curframe; i++) { 1741 free_func_state(state->frame[i]); 1742 state->frame[i] = NULL; 1743 } 1744 kfree(state->refs); 1745 bpf_clear_jmp_history(state); 1746 if (free_self) 1747 kfree(state); 1748 } 1749 1750 /* copy verifier state from src to dst growing dst stack space 1751 * when necessary to accommodate larger src stack 1752 */ 1753 static int copy_func_state(struct bpf_func_state *dst, 1754 const struct bpf_func_state *src) 1755 { 1756 memcpy(dst, src, offsetof(struct bpf_func_state, stack)); 1757 /* Instruction accounting is path-local, not part of verifier state. */ 1758 dst->insns_subtotal = 0; 1759 dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt, 1760 sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT); 1761 if (!dst->loop_stack) 1762 return -ENOMEM; 1763 return copy_stack_state(dst, src); 1764 } 1765 1766 int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, 1767 const struct bpf_verifier_state *src) 1768 { 1769 struct bpf_func_state *dst; 1770 int i, err; 1771 1772 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1773 src->jmp_history_cnt, sizeof(*dst_state->jmp_history), 1774 GFP_KERNEL_ACCOUNT); 1775 if (!dst_state->jmp_history) 1776 return -ENOMEM; 1777 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1778 1779 /* if dst has more stack frames then src frame, free them, this is also 1780 * necessary in case of exceptional exits using bpf_throw. 1781 */ 1782 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1783 free_func_state(dst_state->frame[i]); 1784 dst_state->frame[i] = NULL; 1785 } 1786 err = copy_reference_state(dst_state, src); 1787 if (err) 1788 return err; 1789 dst_state->speculative = src->speculative; 1790 dst_state->in_sleepable = src->in_sleepable; 1791 dst_state->unwound = src->unwound; 1792 dst_state->curframe = src->curframe; 1793 dst_state->branches = src->branches; 1794 dst_state->parent = src->parent; 1795 dst_state->first_insn_idx = src->first_insn_idx; 1796 dst_state->last_insn_idx = src->last_insn_idx; 1797 dst_state->dfs_depth = src->dfs_depth; 1798 dst_state->callback_unroll_depth = src->callback_unroll_depth; 1799 dst_state->may_goto_depth = src->may_goto_depth; 1800 dst_state->equal_state = src->equal_state; 1801 for (i = 0; i <= src->curframe; i++) { 1802 dst = dst_state->frame[i]; 1803 if (!dst) { 1804 dst = kzalloc_obj(*dst, GFP_KERNEL_ACCOUNT); 1805 if (!dst) 1806 return -ENOMEM; 1807 dst_state->frame[i] = dst; 1808 } 1809 err = copy_func_state(dst, src->frame[i]); 1810 if (err) 1811 return err; 1812 } 1813 return 0; 1814 } 1815 1816 static u32 state_htab_size(struct bpf_verifier_env *env) 1817 { 1818 return env->prog->len; 1819 } 1820 1821 struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx) 1822 { 1823 struct bpf_verifier_state *cur = env->cur_state; 1824 struct bpf_func_state *state = cur->frame[cur->curframe]; 1825 1826 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 1827 } 1828 1829 static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b) 1830 { 1831 int fr; 1832 1833 if (a->curframe != b->curframe) 1834 return false; 1835 1836 for (fr = a->curframe; fr >= 0; fr--) 1837 if (a->frame[fr]->callsite != b->frame[fr]->callsite) 1838 return false; 1839 1840 return true; 1841 } 1842 1843 void bpf_free_backedges(struct bpf_scc_visit *visit) 1844 { 1845 struct bpf_scc_backedge *backedge, *next; 1846 1847 for (backedge = visit->backedges; backedge; backedge = next) { 1848 bpf_free_verifier_state(&backedge->state, false); 1849 next = backedge->next; 1850 kfree(backedge); 1851 } 1852 visit->backedges = NULL; 1853 } 1854 1855 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1856 int *insn_idx, bool pop_log) 1857 { 1858 struct bpf_verifier_state *cur = env->cur_state; 1859 struct bpf_verifier_stack_elem *elem, *head = env->head; 1860 int err; 1861 1862 if (env->head == NULL) 1863 return -ENOENT; 1864 1865 if (cur) { 1866 err = bpf_copy_verifier_state(cur, &head->st); 1867 if (err) 1868 return err; 1869 bpf_diag_event_log_restore(env, head->diag_log_pos); 1870 } 1871 if (pop_log) 1872 bpf_vlog_reset(&env->log, head->log_pos); 1873 if (insn_idx) 1874 *insn_idx = head->insn_idx; 1875 if (prev_insn_idx) 1876 *prev_insn_idx = head->prev_insn_idx; 1877 elem = head->next; 1878 bpf_free_verifier_state(&head->st, false); 1879 kfree(head); 1880 env->head = elem; 1881 env->stack_size--; 1882 return 0; 1883 } 1884 1885 static bool error_recoverable_with_nospec(int err) 1886 { 1887 /* Should only return true for non-fatal errors that are allowed to 1888 * occur during speculative verification. For these we can insert a 1889 * nospec and the program might still be accepted. Do not include 1890 * something like ENOMEM because it is likely to re-occur for the next 1891 * architectural path once it has been recovered-from in all speculative 1892 * paths. 1893 */ 1894 return err == -EPERM || err == -EACCES || err == -EINVAL; 1895 } 1896 1897 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1898 int insn_idx, int prev_insn_idx, 1899 bool speculative) 1900 { 1901 struct bpf_verifier_state *cur = env->cur_state; 1902 struct bpf_verifier_stack_elem *elem; 1903 int err; 1904 1905 elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); 1906 if (!elem) 1907 return ERR_PTR(-ENOMEM); 1908 1909 elem->insn_idx = insn_idx; 1910 elem->prev_insn_idx = prev_insn_idx; 1911 elem->next = env->head; 1912 elem->log_pos = env->log.end_pos; 1913 elem->diag_log_pos = bpf_diag_event_log_save(env); 1914 env->head = elem; 1915 env->stack_size++; 1916 err = bpf_copy_verifier_state(&elem->st, cur); 1917 if (err) 1918 return ERR_PTR(-ENOMEM); 1919 elem->st.speculative |= speculative; 1920 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1921 verbose(env, "The sequence of %d jumps is too complex.\n", 1922 env->stack_size); 1923 return ERR_PTR(-E2BIG); 1924 } 1925 if (elem->st.parent) { 1926 ++elem->st.parent->branches; 1927 /* WARN_ON(branches > 2) technically makes sense here, 1928 * but 1929 * 1. speculative states will bump 'branches' for non-branch 1930 * instructions 1931 * 2. is_state_visited() heuristics may decide not to create 1932 * a new state for a sequence of branches and all such current 1933 * and cloned states will be pointing to a single parent state 1934 * which might have large 'branches' count. 1935 */ 1936 } 1937 return &elem->st; 1938 } 1939 1940 static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno) 1941 { 1942 char *buf = env->tmp_arg_name; 1943 int len = sizeof(env->tmp_arg_name); 1944 int arg, regno = reg_from_argno(argno); 1945 1946 if (regno >= 0) { 1947 snprintf(buf, len, "R%d", regno); 1948 } else { 1949 arg = arg_from_argno(argno); 1950 snprintf(buf, len, "*(R11-%u)", (arg - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE); 1951 } 1952 1953 return buf; 1954 } 1955 1956 static const int caller_saved[CALLER_SAVED_REGS] = { 1957 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1958 }; 1959 1960 static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, 1961 struct bpf_reg_state *regs) 1962 { 1963 int i; 1964 1965 for (i = 1; i < CALLER_SAVED_REGS; i++) { 1966 bpf_diag_record_scrub(env, ®s[caller_saved[i]], 1967 BPF_DIAG_MOD_CALLER_SAVED); 1968 } 1969 } 1970 1971 static void reg_step_reset(struct bpf_reg_state *reg) 1972 { 1973 reg->base = 0; 1974 reg->step = 1; 1975 } 1976 1977 static void scalar_step_add(struct bpf_reg_state *dst_reg, 1978 const struct bpf_reg_state *a, 1979 const struct bpf_reg_state *b) 1980 { 1981 const struct bpf_reg_state *reg; 1982 s64 amount, tmp; 1983 1984 if (tnum_is_const(b->var_off)) { 1985 reg = a; 1986 amount = (s64)b->var_off.value; 1987 } else if (tnum_is_const(a->var_off)) { 1988 reg = b; 1989 amount = (s64)a->var_off.value; 1990 } else { 1991 reg_step_reset(dst_reg); 1992 return; 1993 } 1994 1995 /* 1996 * Let x be a possible signed value of the register, 1997 * d = amount, s = reg->step, and b = reg->base. 1998 * Then x mod s = b, with s > 0 and 0 <= b < s. 1999 * If the addition causes signed underflow, the resulting value 2000 * is x + d + 2**64. 2001 * 2002 * Taking the result modulo s and substituting x mod s = b gives: 2003 * (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1) 2004 * Keeping reg->step = s and updating reg->base to (b + d) mod s 2005 * requires (1) to equal (b + d) mod s, which holds if and only 2006 * if 2**64 mod s = 0. 2007 * Signed overflow subtracts 2**64 and has the same requirement. 2008 * 2009 * Conservatively reset the stride if either wrap is possible. 2010 */ 2011 if (check_add_overflow(reg_smin(reg), amount, &tmp) || 2012 check_add_overflow(reg_smax(reg), amount, &tmp)) { 2013 reg_step_reset(dst_reg); 2014 return; 2015 } 2016 2017 dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step; 2018 dst_reg->step = reg->step; 2019 } 2020 2021 static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount) 2022 { 2023 u16 step; 2024 s64 tmp; 2025 2026 /* 2027 * Using the same notation as scalar_step_add(), with d > 0, 2028 * the scaled value satisfies (x * d) mod (s * d) = b * d. 2029 * If the multiplication wraps, its signed result can be represented 2030 * as x * d - n * 2**64 for some nonzero integer n. 2031 * 2032 * Taking the result modulo s * d gives: 2033 * (x * d - n * 2**64) mod (s * d) = 2034 * (b * d - (n * 2**64 mod (s * d))) mod (s * d) (1) 2035 * Updating dst_reg->base to b * d and dst_reg->step to s * d 2036 * requires (1) to equal b * d, which holds if and only if 2037 * n * 2**64 mod (s * d) = 0. 2038 * 2039 * Conservatively reset the stride if a wrap is possible. 2040 */ 2041 if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) || 2042 check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) || 2043 check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) { 2044 reg_step_reset(dst_reg); 2045 return; 2046 } 2047 /* 2048 * The above established that `step = dst_reg->step * amount` 2049 * does not overflow, therefore the base < step invariant holds 2050 * after assignments below. 2051 */ 2052 dst_reg->base = dst_reg->base * amount; 2053 dst_reg->step = step; 2054 } 2055 2056 /* 2057 * Let x be a possible signed value of the register, 2058 * s = reg->step, and b = reg->base. 2059 * Then x mod s = b, with s > 0 and 0 <= b < s. 2060 * Truncating to N bits and then zero- or sign-extending x 2061 * is equivalent to computing x + k * 2**N for some integer k. 2062 * For example, sign-extending the low 8 bits of x = 255 can 2063 * be expressed as 255 - 2**8 = -1. 2064 * 2065 * Taking the result modulo s and substituting x mod s = b gives: 2066 * (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s 2067 * Keeping reg->base = b and reg->step = s requires this to equal b, 2068 * which holds if and only if k * 2**N mod s = 0. 2069 * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0, 2070 * but (-1) mod 3 = 2, so the original base and step no longer hold. 2071 * 2072 * Conservatively reset the stride unless all possible values fit in 2073 * the range where the conversion leaves them unchanged. 2074 * Call before updating the register's bounds. Requires 0 < bits < 64. 2075 */ 2076 static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits) 2077 { 2078 if (reg_umax(reg) >= (1ULL << bits)) 2079 reg_step_reset(reg); 2080 } 2081 2082 /* This helper doesn't clear reg->id */ 2083 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 2084 { 2085 reg->var_off = tnum_const(imm); 2086 reg->r64 = cnum64_from_urange(imm, imm); 2087 reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); 2088 reg_step_reset(reg); 2089 } 2090 2091 /* Mark the unknown part of a register (variable offset or scalar value) as 2092 * known to have the value @imm. 2093 */ 2094 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 2095 { 2096 /* Clear off and union(map_ptr, range) */ 2097 memset(((u8 *)reg) + sizeof(reg->type), 0, 2098 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 2099 reg->id = 0; 2100 reg->parent_id = 0; 2101 reg->map_uid = 0; 2102 ___mark_reg_known(reg, imm); 2103 } 2104 2105 void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm) 2106 { 2107 __mark_reg_known(reg, imm); 2108 reg->type = SCALAR_VALUE; 2109 } 2110 2111 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 2112 { 2113 reg->var_off = tnum_const_subreg(reg->var_off, imm); 2114 reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); 2115 } 2116 2117 /* Mark the 'variable offset' part of a register as zero. This should be 2118 * used only on registers holding a pointer type. 2119 */ 2120 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 2121 { 2122 __mark_reg_known(reg, 0); 2123 } 2124 2125 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 2126 { 2127 __mark_reg_known(reg, 0); 2128 reg->type = SCALAR_VALUE; 2129 /* all scalars are assumed imprecise initially (unless unprivileged, 2130 * in which case everything is forced to be precise) 2131 */ 2132 reg->precise = !env->bpf_capable; 2133 } 2134 2135 static void mark_reg_known_zero(struct bpf_verifier_env *env, 2136 struct bpf_reg_state *regs, u32 regno) 2137 { 2138 __mark_reg_known_zero(regs + regno); 2139 } 2140 2141 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, 2142 bool first_slot, int id, int parent_id) 2143 { 2144 /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for 2145 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply 2146 * set it unconditionally as it is ignored for STACK_DYNPTR anyway. 2147 */ 2148 __mark_reg_known_zero(reg); 2149 reg->type = CONST_PTR_TO_DYNPTR; 2150 /* Give each dynptr a unique id to uniquely associate slices to it. */ 2151 reg->id = id; 2152 reg->parent_id = parent_id; 2153 reg->dynptr.type = type; 2154 reg->dynptr.first_slot = first_slot; 2155 } 2156 2157 /* 2158 * Refine the return type of the bpf_map_lookup_elem() for special map types: 2159 * map-in-map, xskmap, sockmap and sockhash. 2160 */ 2161 static void refine_map_lookup_value(struct bpf_reg_state *reg) 2162 { 2163 enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; 2164 const struct bpf_map *map = reg->map_ptr; 2165 2166 if (map->inner_map_meta) { 2167 reg->type = CONST_PTR_TO_MAP | maybe_null; 2168 reg->map_ptr = map->inner_map_meta; 2169 /* 2170 * transfer reg's id which is unique for every map_lookup_elem 2171 * as UID of the inner map. 2172 */ 2173 reg->map_uid = reg->id; 2174 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 2175 reg->type = PTR_TO_XDP_SOCK | maybe_null; 2176 reg->map_uid = 0; 2177 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 2178 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 2179 reg->type = PTR_TO_SOCKET | maybe_null; 2180 reg->map_uid = 0; 2181 } 2182 } 2183 2184 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 2185 { 2186 reg->type &= ~PTR_MAYBE_NULL; 2187 } 2188 2189 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno, 2190 struct btf_field_graph_root *ds_head) 2191 { 2192 __mark_reg_known(®s[regno], ds_head->node_offset); 2193 regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC; 2194 regs[regno].btf = ds_head->btf; 2195 regs[regno].btf_id = ds_head->value_btf_id; 2196 } 2197 2198 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 2199 { 2200 return type_is_pkt_pointer(reg->type); 2201 } 2202 2203 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 2204 { 2205 return reg_is_pkt_pointer(reg) || 2206 reg->type == PTR_TO_PACKET_END; 2207 } 2208 2209 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg) 2210 { 2211 return base_type(reg->type) == PTR_TO_MEM && 2212 (reg->type & 2213 (DYNPTR_TYPE_SKB | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META)); 2214 } 2215 2216 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 2217 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 2218 enum bpf_reg_type which) 2219 { 2220 /* The register can already have a range from prior markings. 2221 * This is fine as long as it hasn't been advanced from its 2222 * origin. 2223 */ 2224 return reg->type == which && 2225 reg->id == 0 && 2226 tnum_equals_const(reg->var_off, 0); 2227 } 2228 2229 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 2230 { 2231 reg->r32 = CNUM32_UNBOUNDED; 2232 } 2233 2234 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 2235 { 2236 reg->r64 = CNUM64_UNBOUNDED; 2237 } 2238 2239 /* Reset the min/max bounds of a register */ 2240 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 2241 { 2242 __mark_reg64_unbounded(reg); 2243 __mark_reg32_unbounded(reg); 2244 } 2245 2246 static void reset_reg64_and_tnum(struct bpf_reg_state *reg) 2247 { 2248 __mark_reg64_unbounded(reg); 2249 reg->var_off = tnum_unknown; 2250 } 2251 2252 static void reset_reg32_and_tnum(struct bpf_reg_state *reg) 2253 { 2254 __mark_reg32_unbounded(reg); 2255 reg->var_off = tnum_unknown; 2256 } 2257 2258 static struct cnum32 cnum32_from_tnum(struct tnum tnum) 2259 { 2260 tnum = tnum_subreg(tnum); 2261 if ((tnum.mask & S32_MIN) || (tnum.value & S32_MIN)) 2262 /* min signed is max(sign bit) | min(other bits) */ 2263 /* max signed is min(sign bit) | max(other bits) */ 2264 return cnum32_from_srange(tnum.value | (tnum.mask & S32_MIN), 2265 tnum.value | (tnum.mask & S32_MAX)); 2266 else 2267 return cnum32_from_urange(tnum.value, (tnum.value | tnum.mask)); 2268 } 2269 2270 static struct cnum64 cnum64_from_tnum(struct tnum tnum) 2271 { 2272 if ((tnum.mask & S64_MIN) || (tnum.value & S64_MIN)) 2273 /* min signed is max(sign bit) | min(other bits) */ 2274 /* max signed is min(sign bit) | max(other bits) */ 2275 return cnum64_from_srange(tnum.value | (tnum.mask & S64_MIN), 2276 tnum.value | (tnum.mask & S64_MAX)); 2277 else 2278 return cnum64_from_urange(tnum.value, (tnum.value | tnum.mask)); 2279 } 2280 2281 static void __update_reg32_bounds(struct bpf_reg_state *reg) 2282 { 2283 cnum32_intersect_with(®->r32, cnum32_from_tnum(reg->var_off)); 2284 } 2285 2286 static void __update_reg64_bounds(struct bpf_reg_state *reg) 2287 { 2288 u64 tnum_next, tmax; 2289 bool umin_in_tnum; 2290 2291 cnum64_intersect_with(®->r64, cnum64_from_tnum(reg->var_off)); 2292 2293 /* Check if u64 and tnum overlap in a single value */ 2294 tnum_next = tnum_step(reg->var_off, reg_umin(reg)); 2295 umin_in_tnum = (reg_umin(reg) & ~reg->var_off.mask) == reg->var_off.value; 2296 tmax = reg->var_off.value | reg->var_off.mask; 2297 if (umin_in_tnum && tnum_next > reg_umax(reg)) { 2298 /* The u64 range and the tnum only overlap in umin. 2299 * u64: ---[xxxxxx]----- 2300 * tnum: --xx----------x- 2301 */ 2302 ___mark_reg_known(reg, reg_umin(reg)); 2303 } else if (!umin_in_tnum && tnum_next == tmax) { 2304 /* The u64 range and the tnum only overlap in the maximum value 2305 * represented by the tnum, called tmax. 2306 * u64: ---[xxxxxx]----- 2307 * tnum: xx-----x-------- 2308 */ 2309 ___mark_reg_known(reg, tmax); 2310 } else if (!umin_in_tnum && tnum_next <= reg_umax(reg) && 2311 tnum_step(reg->var_off, tnum_next) > reg_umax(reg)) { 2312 /* The u64 range and the tnum only overlap in between umin 2313 * (excluded) and umax. 2314 * u64: ---[xxxxxx]----- 2315 * tnum: xx----x-------x- 2316 */ 2317 ___mark_reg_known(reg, tnum_next); 2318 } 2319 } 2320 2321 static void __update_reg_bounds(struct bpf_reg_state *reg) 2322 { 2323 __update_reg32_bounds(reg); 2324 __update_reg64_bounds(reg); 2325 } 2326 2327 static void deduce_bounds_32_from_64(struct bpf_reg_state *reg) 2328 { 2329 cnum32_intersect_with(®->r32, cnum32_from_cnum64(reg->r64)); 2330 } 2331 2332 static void deduce_bounds_64_from_32(struct bpf_reg_state *reg) 2333 { 2334 reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32); 2335 } 2336 2337 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 2338 { 2339 deduce_bounds_32_from_64(reg); 2340 deduce_bounds_64_from_32(reg); 2341 } 2342 2343 /* Attempts to improve var_off based on unsigned min/max information */ 2344 static void __reg_bound_offset(struct bpf_reg_state *reg) 2345 { 2346 struct tnum var64_off = tnum_intersect(reg->var_off, 2347 tnum_range(reg_umin(reg), 2348 reg_umax(reg))); 2349 struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), 2350 tnum_range(reg_u32_min(reg), 2351 reg_u32_max(reg))); 2352 u32 trailing_zero_bits; 2353 u16 base = reg->base; 2354 u16 step = reg->step; 2355 2356 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 2357 2358 if (base == 0) 2359 trailing_zero_bits = count_trailing_zeros(step); 2360 else 2361 trailing_zero_bits = min(count_trailing_zeros(base), 2362 count_trailing_zeros(step)); 2363 reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits)); 2364 } 2365 2366 static void __update_bounds_from_step(struct bpf_reg_state *reg) 2367 { 2368 reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step); 2369 deduce_bounds_32_from_64(reg); 2370 } 2371 2372 static bool range_bounds_violation(struct bpf_reg_state *reg); 2373 2374 static void reg_bounds_sync(struct bpf_reg_state *reg) 2375 { 2376 /* If the input reg_state is invalid, we can exit early */ 2377 if (range_bounds_violation(reg)) 2378 return; 2379 /* We might have learned new bounds from the var_off. */ 2380 __update_reg_bounds(reg); 2381 /* We might have learned something about the sign bit. */ 2382 __reg_deduce_bounds(reg); 2383 __reg_deduce_bounds(reg); 2384 /* We might have learned some bits from the bounds. */ 2385 __reg_bound_offset(reg); 2386 /* Intersecting with the old var_off might have improved our bounds 2387 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 2388 * then new var_off is (0; 0x7f...fc) which improves our umax. 2389 */ 2390 __update_reg_bounds(reg); 2391 /* 2392 * Both tnum and base+step define a sequence of points, 2393 * potentially w/o intersection, or intersecting after N 2394 * tnum_step()'s, where N can be large. On the other hand, 2395 * step+base *must* be in sync with r64 in order for 2396 * array traversal logic starting at reg_smin() to be sound. 2397 * Hence, place this as a final step in the reg_bounds_sync(). 2398 */ 2399 __update_bounds_from_step(reg); 2400 } 2401 2402 static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) 2403 { 2404 if (!tnum_is_const(reg->var_off)) 2405 return false; 2406 2407 return !cnum64_is_const(reg->r64) || reg->r64.base != reg->var_off.value; 2408 } 2409 2410 static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg) 2411 { 2412 if (!tnum_subreg_is_const(reg->var_off)) 2413 return false; 2414 2415 return !cnum32_is_const(reg->r32) || reg->r32.base != tnum_subreg(reg->var_off).value; 2416 } 2417 2418 static bool range_bounds_violation(struct bpf_reg_state *reg) 2419 { 2420 return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64); 2421 } 2422 2423 static int reg_bounds_sanity_check(struct bpf_verifier_env *env, 2424 struct bpf_reg_state *reg, const char *ctx) 2425 { 2426 const char *msg; 2427 2428 if (range_bounds_violation(reg)) { 2429 msg = "range bounds violation"; 2430 goto out; 2431 } 2432 2433 if (const_tnum_range_mismatch(reg)) { 2434 msg = "const tnum out of sync with range bounds"; 2435 goto out; 2436 } 2437 2438 if (const_tnum_range_mismatch_32(reg)) { 2439 msg = "const subreg tnum out of sync with range bounds"; 2440 goto out; 2441 } 2442 2443 if (reg->step == 0 || reg->base >= reg->step) { 2444 msg = "invalid base/step"; 2445 goto out; 2446 } 2447 2448 return 0; 2449 out: 2450 verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " 2451 "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u", 2452 ctx, msg, 2453 reg->r64.base, reg->r64.size, 2454 reg->r32.base, reg->r32.size, 2455 reg->var_off.value, reg->var_off.mask, 2456 reg->base, reg->step); 2457 if (env->test_reg_invariants) 2458 return -EFAULT; 2459 __mark_reg_unbounded(reg); 2460 reg_step_reset(reg); 2461 return 0; 2462 } 2463 2464 /* Mark a register as having a completely unknown (scalar) value. */ 2465 void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) 2466 { 2467 memset(reg, 0, sizeof(*reg)); 2468 reg->type = SCALAR_VALUE; 2469 reg->var_off = tnum_unknown; 2470 __mark_reg_unbounded(reg); 2471 reg_step_reset(reg); 2472 } 2473 2474 /* Mark a register as having a completely unknown (scalar) value, 2475 * initialize .precise as true when not bpf capable. 2476 */ 2477 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 2478 struct bpf_reg_state *reg) 2479 { 2480 bpf_mark_reg_unknown_imprecise(reg); 2481 reg->precise = !env->bpf_capable; 2482 } 2483 2484 static void mark_reg_unknown(struct bpf_verifier_env *env, 2485 struct bpf_reg_state *regs, u32 regno) 2486 { 2487 __mark_reg_unknown(env, regs + regno); 2488 } 2489 2490 static int __mark_reg_s32_range(struct bpf_verifier_env *env, 2491 struct bpf_reg_state *regs, 2492 u32 regno, 2493 s32 s32_min, 2494 s32 s32_max) 2495 { 2496 struct bpf_reg_state *reg = regs + regno; 2497 2498 reg_set_srange32(reg, 2499 max_t(s32, reg_s32_min(reg), s32_min), 2500 min_t(s32, reg_s32_max(reg), s32_max)); 2501 reg_set_srange64(reg, 2502 max_t(s64, reg_smin(reg), s32_min), 2503 min_t(s64, reg_smax(reg), s32_max)); 2504 2505 reg_bounds_sync(reg); 2506 2507 return reg_bounds_sanity_check(env, reg, "s32_range"); 2508 } 2509 2510 void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, 2511 struct bpf_reg_state *reg) 2512 { 2513 __mark_reg_unknown(env, reg); 2514 reg->type = NOT_INIT; 2515 } 2516 2517 static int mark_btf_ld_reg(struct bpf_verifier_env *env, 2518 struct bpf_reg_state *regs, u32 regno, 2519 enum bpf_reg_type reg_type, 2520 struct btf *btf, u32 btf_id, 2521 enum bpf_type_flag flag) 2522 { 2523 switch (reg_type) { 2524 case SCALAR_VALUE: 2525 mark_reg_unknown(env, regs, regno); 2526 return 0; 2527 case PTR_TO_BTF_ID: 2528 mark_reg_known_zero(env, regs, regno); 2529 regs[regno].type = PTR_TO_BTF_ID | flag; 2530 regs[regno].btf = btf; 2531 regs[regno].btf_id = btf_id; 2532 if (type_may_be_null(flag)) 2533 regs[regno].id = ++env->id_gen; 2534 return 0; 2535 case PTR_TO_MEM: 2536 mark_reg_known_zero(env, regs, regno); 2537 regs[regno].type = PTR_TO_MEM | flag; 2538 regs[regno].mem_size = 0; 2539 return 0; 2540 default: 2541 verifier_bug(env, "unexpected reg_type %d in %s\n", reg_type, __func__); 2542 return -EFAULT; 2543 } 2544 } 2545 2546 static void init_reg_state(struct bpf_verifier_env *env, 2547 struct bpf_func_state *state) 2548 { 2549 struct bpf_reg_state *regs = state->regs; 2550 int i; 2551 2552 for (i = 0; i < MAX_BPF_REG; i++) { 2553 bpf_mark_reg_not_init(env, ®s[i]); 2554 } 2555 2556 /* frame pointer */ 2557 regs[BPF_REG_FP].type = PTR_TO_STACK; 2558 mark_reg_known_zero(env, regs, BPF_REG_FP); 2559 regs[BPF_REG_FP].frameno = state->frameno; 2560 } 2561 2562 static struct bpf_retval_range retval_range(s32 minval, s32 maxval) 2563 { 2564 /* 2565 * return_32bit is set to false by default and set explicitly 2566 * by the caller when necessary. 2567 */ 2568 return (struct bpf_retval_range){ minval, maxval, false }; 2569 } 2570 2571 static void init_func_state(struct bpf_verifier_env *env, 2572 struct bpf_func_state *state, 2573 int callsite, int frameno, int subprogno) 2574 { 2575 state->callsite = callsite; 2576 state->frameno = frameno; 2577 bpf_diag_init_frame(env, state); 2578 state->subprogno = subprogno; 2579 state->callback_ret_range = retval_range(0, 0); 2580 init_reg_state(env, state); 2581 mark_verifier_state_scratched(env); 2582 } 2583 2584 /* Similar to push_stack(), but for async callbacks */ 2585 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 2586 int insn_idx, int prev_insn_idx, 2587 int subprog, bool is_sleepable) 2588 { 2589 struct bpf_verifier_stack_elem *elem; 2590 struct bpf_func_state *frame; 2591 2592 elem = kzalloc_obj(struct bpf_verifier_stack_elem, GFP_KERNEL_ACCOUNT); 2593 if (!elem) 2594 return ERR_PTR(-ENOMEM); 2595 2596 elem->insn_idx = insn_idx; 2597 elem->prev_insn_idx = prev_insn_idx; 2598 elem->next = env->head; 2599 elem->log_pos = env->log.end_pos; 2600 elem->diag_log_pos = bpf_diag_event_log_save(env); 2601 env->head = elem; 2602 env->stack_size++; 2603 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 2604 verbose(env, 2605 "The sequence of %d jumps is too complex for async cb.\n", 2606 env->stack_size); 2607 return ERR_PTR(-E2BIG); 2608 } 2609 /* Unlike push_stack() do not bpf_copy_verifier_state(). 2610 * The caller state doesn't matter. 2611 * This is async callback. It starts in a fresh stack. 2612 * Initialize it similar to do_check_common(). 2613 */ 2614 elem->st.branches = 1; 2615 elem->st.in_sleepable = is_sleepable; 2616 frame = kzalloc_obj(*frame, GFP_KERNEL_ACCOUNT); 2617 if (!frame) 2618 return ERR_PTR(-ENOMEM); 2619 init_func_state(env, frame, 2620 BPF_MAIN_FUNC /* callsite */, 2621 0 /* frameno within this callchain */, 2622 subprog /* subprog number within this prog */); 2623 elem->st.frame[0] = frame; 2624 return &elem->st; 2625 } 2626 2627 /* Find subprogram that contains instruction at 'off' */ 2628 struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off) 2629 { 2630 struct bpf_subprog_info *vals = env->subprog_info; 2631 int l, r, m; 2632 2633 if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0) 2634 return NULL; 2635 2636 l = 0; 2637 r = env->subprog_cnt - 1; 2638 while (l < r) { 2639 m = l + (r - l + 1) / 2; 2640 if (vals[m].start <= off) 2641 l = m; 2642 else 2643 r = m - 1; 2644 } 2645 return &vals[l]; 2646 } 2647 2648 /* Find subprogram that starts exactly at 'off' */ 2649 int bpf_find_subprog(struct bpf_verifier_env *env, int off) 2650 { 2651 struct bpf_subprog_info *p; 2652 2653 p = bpf_find_containing_subprog(env, off); 2654 if (!p || p->start != off) 2655 return -ENOENT; 2656 return p - env->subprog_info; 2657 } 2658 2659 static int add_subprog(struct bpf_verifier_env *env, int off) 2660 { 2661 struct bpf_subprog_info *subprog = env->subprog_info; 2662 int insn_cnt = env->prog->len; 2663 int i; 2664 2665 if (off >= insn_cnt || off < 0) { 2666 verbose(env, "call to invalid destination\n"); 2667 return -EINVAL; 2668 } 2669 i = bpf_find_subprog(env, off); 2670 if (i >= 0) 2671 return i; 2672 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 2673 verbose(env, "too many subprograms\n"); 2674 return -E2BIG; 2675 } 2676 /* 2677 * Determine subprog starts. The end is one before the next starts. 2678 * The array stays sorted: subprogs that start after 'off' move up. 2679 */ 2680 for (i = env->subprog_cnt++; i > 0 && subprog[i - 1].start > off; i--) 2681 subprog[i] = subprog[i - 1]; 2682 subprog[i] = (struct bpf_subprog_info){ .start = off, .exit_idx = U32_MAX }; 2683 return i; 2684 } 2685 2686 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) 2687 { 2688 struct bpf_prog_aux *aux = env->prog->aux; 2689 struct btf *btf = aux->btf; 2690 const struct btf_type *t; 2691 u32 main_btf_id, id; 2692 const char *name; 2693 int ret, i; 2694 2695 /* Non-zero func_info_cnt implies valid btf */ 2696 if (!aux->func_info_cnt) 2697 return 0; 2698 main_btf_id = aux->func_info[0].type_id; 2699 2700 t = btf_type_by_id(btf, main_btf_id); 2701 if (!t) { 2702 verbose(env, "invalid btf id for main subprog in func_info\n"); 2703 return -EINVAL; 2704 } 2705 2706 name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:"); 2707 if (IS_ERR(name)) { 2708 ret = PTR_ERR(name); 2709 /* If there is no tag present, there is no exception callback */ 2710 if (ret == -ENOENT) 2711 ret = 0; 2712 else if (ret == -EEXIST) 2713 verbose(env, "multiple exception callback tags for main subprog\n"); 2714 return ret; 2715 } 2716 2717 ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC); 2718 if (ret < 0) { 2719 verbose(env, "exception callback '%s' could not be found in BTF\n", name); 2720 return ret; 2721 } 2722 id = ret; 2723 t = btf_type_by_id(btf, id); 2724 if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 2725 verbose(env, "exception callback '%s' must have global linkage\n", name); 2726 return -EINVAL; 2727 } 2728 ret = 0; 2729 for (i = 0; i < aux->func_info_cnt; i++) { 2730 if (aux->func_info[i].type_id != id) 2731 continue; 2732 ret = aux->func_info[i].insn_off; 2733 /* Further func_info and subprog checks will also happen 2734 * later, so assume this is the right insn_off for now. 2735 */ 2736 if (!ret) { 2737 verbose(env, "invalid exception callback insn_off in func_info: 0\n"); 2738 ret = -EINVAL; 2739 } 2740 } 2741 if (!ret) { 2742 verbose(env, "exception callback type id not found in func_info\n"); 2743 ret = -EINVAL; 2744 } 2745 return ret; 2746 } 2747 2748 #define MAX_KFUNC_BTFS 256 2749 2750 struct bpf_kfunc_btf { 2751 struct btf *btf; 2752 struct module *module; 2753 u16 offset; 2754 }; 2755 2756 struct bpf_kfunc_btf_tab { 2757 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 2758 u32 nr_descs; 2759 }; 2760 2761 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 2762 { 2763 const struct bpf_kfunc_desc *d0 = a; 2764 const struct bpf_kfunc_desc *d1 = b; 2765 2766 /* func_id is not greater than BTF_MAX_TYPE */ 2767 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 2768 } 2769 2770 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 2771 { 2772 const struct bpf_kfunc_btf *d0 = a; 2773 const struct bpf_kfunc_btf *d1 = b; 2774 2775 return d0->offset - d1->offset; 2776 } 2777 2778 static struct bpf_kfunc_desc * 2779 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 2780 { 2781 struct bpf_kfunc_desc desc = { 2782 .func_id = func_id, 2783 .offset = offset, 2784 }; 2785 struct bpf_kfunc_desc_tab *tab; 2786 2787 tab = prog->aux->kfunc_tab; 2788 return bsearch(&desc, tab->descs, tab->nr_base_descs, 2789 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 2790 } 2791 2792 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, 2793 u16 desc_idx, u8 **func_addr) 2794 { 2795 struct bpf_kfunc_desc_tab *tab; 2796 const struct bpf_kfunc_desc *desc; 2797 2798 tab = prog->aux->kfunc_tab; 2799 if (desc_idx >= tab->nr_descs) 2800 return -EFAULT; 2801 desc = &tab->descs[desc_idx]; 2802 if (desc->func_id != func_id) 2803 return -EFAULT; 2804 2805 *func_addr = (u8 *)desc->addr; 2806 return 0; 2807 } 2808 2809 #define BPF_FD_SLOT_BTF 1UL 2810 2811 static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map) 2812 { 2813 slot->val = (unsigned long)map; 2814 } 2815 2816 static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf) 2817 { 2818 slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF; 2819 } 2820 2821 static struct bpf_map *fd_slot_map(struct bpf_fd_array slot) 2822 { 2823 if (slot.val & BPF_FD_SLOT_BTF) 2824 return NULL; 2825 return (struct bpf_map *)slot.val; 2826 } 2827 2828 static struct btf *fd_slot_btf(struct bpf_fd_array slot) 2829 { 2830 if (!(slot.val & BPF_FD_SLOT_BTF)) 2831 return NULL; 2832 return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF); 2833 } 2834 2835 static struct btf * 2836 fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx) 2837 { 2838 struct btf *btf; 2839 2840 if (idx >= env->fd_array_cnt) { 2841 verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n", 2842 idx, env->fd_array_cnt); 2843 return ERR_PTR(-EINVAL); 2844 } 2845 btf = fd_slot_btf(env->fd_array[idx]); 2846 if (!btf) { 2847 verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx); 2848 return ERR_PTR(-EINVAL); 2849 } 2850 btf_get(btf); 2851 return btf; 2852 } 2853 2854 static struct btf * 2855 fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx) 2856 { 2857 struct btf *btf; 2858 int btf_fd; 2859 2860 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw, 2861 (size_t)idx * sizeof(btf_fd), sizeof(btf_fd))) 2862 return ERR_PTR(-EFAULT); 2863 btf = btf_get_by_fd(btf_fd); 2864 if (IS_ERR(btf)) { 2865 verbose(env, "invalid module BTF fd specified\n"); 2866 return btf; 2867 } 2868 return btf; 2869 } 2870 2871 static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx) 2872 { 2873 if (env->signature) { 2874 verbose(env, "signed program cannot bind any BTF\n"); 2875 return ERR_PTR(-EACCES); 2876 } 2877 if (env->fd_array) 2878 return fd_array_get_btf_continuous(env, idx); 2879 if (!bpfptr_is_null(env->fd_array_raw)) 2880 return fd_array_get_btf_sparse(env, idx); 2881 2882 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 2883 return ERR_PTR(-EPROTO); 2884 } 2885 2886 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 2887 s16 offset) 2888 { 2889 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 2890 struct bpf_kfunc_btf_tab *tab; 2891 struct bpf_kfunc_btf *b; 2892 struct module *mod; 2893 struct btf *btf; 2894 2895 tab = env->prog->aux->kfunc_btf_tab; 2896 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 2897 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 2898 if (!b) { 2899 if (tab->nr_descs == MAX_KFUNC_BTFS) { 2900 verbose(env, "too many different module BTFs\n"); 2901 return ERR_PTR(-E2BIG); 2902 } 2903 2904 btf = fd_array_get_btf(env, offset); 2905 if (IS_ERR(btf)) 2906 return btf; 2907 if (!btf_is_module(btf)) { 2908 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 2909 btf_put(btf); 2910 return ERR_PTR(-EINVAL); 2911 } 2912 2913 mod = btf_try_get_module(btf); 2914 if (!mod) { 2915 btf_put(btf); 2916 return ERR_PTR(-ENXIO); 2917 } 2918 2919 b = &tab->descs[tab->nr_descs++]; 2920 b->btf = btf; 2921 b->module = mod; 2922 b->offset = offset; 2923 2924 /* sort() reorders entries by value, so b may no longer point 2925 * to the right entry after this 2926 */ 2927 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2928 kfunc_btf_cmp_by_off, NULL); 2929 } else { 2930 btf = b->btf; 2931 } 2932 2933 return btf; 2934 } 2935 2936 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2937 { 2938 if (!tab) 2939 return; 2940 2941 while (tab->nr_descs--) { 2942 module_put(tab->descs[tab->nr_descs].module); 2943 btf_put(tab->descs[tab->nr_descs].btf); 2944 } 2945 kfree(tab); 2946 } 2947 2948 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2949 { 2950 if (offset) { 2951 if (offset < 0) { 2952 /* In the future, this can be allowed to increase limit 2953 * of fd index into fd_array, interpreted as u16. 2954 */ 2955 verbose(env, "negative offset disallowed for kernel module function call\n"); 2956 return ERR_PTR(-EINVAL); 2957 } 2958 2959 return __find_kfunc_desc_btf(env, offset); 2960 } 2961 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2962 } 2963 2964 static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) 2965 { 2966 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 2967 struct bpf_kfunc_btf_tab *tab; 2968 struct bpf_kfunc_btf *b; 2969 2970 if (!offset) 2971 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2972 if (offset < 0) 2973 return ERR_PTR(-EINVAL); 2974 2975 tab = env->prog->aux->kfunc_btf_tab; 2976 if (!tab) 2977 return ERR_PTR(-ENOENT); 2978 2979 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 2980 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 2981 return b ? b->btf : ERR_PTR(-ENOENT); 2982 } 2983 2984 #define KF_IMPL_SUFFIX "_impl" 2985 2986 static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, 2987 struct btf *btf, 2988 const char *func_name) 2989 { 2990 const struct btf_type *func; 2991 char buf[KSYM_NAME_LEN]; 2992 s32 impl_id; 2993 int len; 2994 2995 len = snprintf(buf, sizeof(buf), "%s%s", func_name, KF_IMPL_SUFFIX); 2996 if (len < 0 || len >= sizeof(buf)) { 2997 bpf_log(log, "function name %s%s is too long\n", 2998 func_name, KF_IMPL_SUFFIX); 2999 return NULL; 3000 } 3001 3002 impl_id = btf_find_by_name_kind(btf, buf, BTF_KIND_FUNC); 3003 if (impl_id <= 0) { 3004 bpf_log(log, "cannot find function %s in BTF\n", buf); 3005 return NULL; 3006 } 3007 3008 func = btf_type_by_id(btf, impl_id); 3009 3010 return btf_type_by_id(btf, func->type); 3011 } 3012 3013 static int fetch_kfunc_meta(struct bpf_verifier_env *env, 3014 s32 func_id, 3015 s16 offset, 3016 struct bpf_kfunc_meta *kfunc) 3017 { 3018 const struct btf_type *func, *func_proto; 3019 const char *func_name; 3020 u32 *kfunc_flags; 3021 struct btf *btf; 3022 3023 if (func_id <= 0) { 3024 verbose(env, "invalid kernel function btf_id %d\n", func_id); 3025 return -EINVAL; 3026 } 3027 3028 btf = find_kfunc_desc_btf(env, offset); 3029 if (IS_ERR(btf)) { 3030 verbose(env, "failed to find BTF for kernel function\n"); 3031 return PTR_ERR(btf); 3032 } 3033 3034 /* 3035 * Note that kfunc_flags may be NULL at this point, which 3036 * means that we couldn't find func_id in any relevant 3037 * kfunc_id_set. This most likely indicates an invalid kfunc 3038 * call. However we don't fail with an error here, 3039 * and let the caller decide what to do with NULL kfunc->flags. 3040 */ 3041 kfunc_flags = btf_kfunc_flags(btf, func_id, env->prog); 3042 3043 func = btf_type_by_id(btf, func_id); 3044 if (!func || !btf_type_is_func(func)) { 3045 verbose(env, "kernel btf_id %d is not a function\n", func_id); 3046 return -EINVAL; 3047 } 3048 3049 func_name = btf_name_by_offset(btf, func->name_off); 3050 3051 /* 3052 * An actual prototype of a kfunc with KF_IMPLICIT_ARGS flag 3053 * can be found through the counterpart _impl kfunc. 3054 */ 3055 if (kfunc_flags && (*kfunc_flags & KF_IMPLICIT_ARGS)) 3056 func_proto = find_kfunc_impl_proto(&env->log, btf, func_name); 3057 else 3058 func_proto = btf_type_by_id(btf, func->type); 3059 3060 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 3061 verbose(env, "kernel function btf_id %d does not have a valid func_proto\n", 3062 func_id); 3063 return -EINVAL; 3064 } 3065 3066 memset(kfunc, 0, sizeof(*kfunc)); 3067 kfunc->btf = btf; 3068 kfunc->id = func_id; 3069 kfunc->name = func_name; 3070 kfunc->proto = func_proto; 3071 kfunc->flags = kfunc_flags; 3072 3073 return 0; 3074 } 3075 3076 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 3077 const struct btf_func_model *fm, struct bpf_func_proto *proto); 3078 3079 int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) 3080 { 3081 struct bpf_call_arg_meta meta; 3082 struct bpf_kfunc_btf_tab *btf_tab; 3083 struct btf_func_model func_model; 3084 struct bpf_kfunc_desc_tab *tab; 3085 struct bpf_prog_aux *prog_aux; 3086 struct bpf_kfunc_meta kfunc; 3087 struct bpf_kfunc_desc *desc; 3088 unsigned long addr; 3089 int err; 3090 3091 prog_aux = env->prog->aux; 3092 tab = prog_aux->kfunc_tab; 3093 btf_tab = prog_aux->kfunc_btf_tab; 3094 if (!tab) { 3095 if (!btf_vmlinux) { 3096 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 3097 return -ENOTSUPP; 3098 } 3099 3100 if (!env->prog->jit_requested) { 3101 verbose(env, "JIT is required for calling kernel function\n"); 3102 return -ENOTSUPP; 3103 } 3104 3105 if (!bpf_jit_supports_kfunc_call()) { 3106 verbose(env, "JIT does not support calling kernel function\n"); 3107 return -ENOTSUPP; 3108 } 3109 3110 if (!env->prog->gpl_compatible) { 3111 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 3112 return -EINVAL; 3113 } 3114 3115 tab = kzalloc_obj(*tab, GFP_KERNEL_ACCOUNT); 3116 if (!tab) 3117 return -ENOMEM; 3118 prog_aux->kfunc_tab = tab; 3119 } 3120 3121 env->prog->jit_required = 1; 3122 3123 /* func_id == 0 is always invalid, but instead of returning an error, be 3124 * conservative and wait until the code elimination pass before returning 3125 * error, so that invalid calls that get pruned out can be in BPF programs 3126 * loaded from userspace. It is also required that offset be untouched 3127 * for such calls. 3128 */ 3129 if (!func_id && !offset) 3130 return 0; 3131 3132 if (!btf_tab && offset) { 3133 btf_tab = kzalloc_obj(*btf_tab, GFP_KERNEL_ACCOUNT); 3134 if (!btf_tab) 3135 return -ENOMEM; 3136 prog_aux->kfunc_btf_tab = btf_tab; 3137 } 3138 3139 if (find_kfunc_desc(env->prog, func_id, offset)) 3140 return 0; 3141 3142 if (tab->nr_base_descs == MAX_KFUNC_DESCS) { 3143 verbose(env, "too many different kernel function calls\n"); 3144 return -E2BIG; 3145 } 3146 if (tab->nr_descs != tab->nr_base_descs) { 3147 verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", 3148 tab->nr_descs, tab->nr_base_descs); 3149 return -EFAULT; 3150 } 3151 3152 err = fetch_kfunc_meta(env, func_id, offset, &kfunc); 3153 if (err) 3154 return err; 3155 3156 addr = kallsyms_lookup_name(kfunc.name); 3157 if (!addr) { 3158 verbose(env, "cannot find address for kernel function %s\n", kfunc.name); 3159 return -EINVAL; 3160 } 3161 3162 if (bpf_dev_bound_kfunc_id(func_id)) { 3163 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); 3164 if (err) 3165 return err; 3166 } 3167 3168 err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); 3169 if (err) 3170 return err; 3171 if (func_model.ret_size > 8) { 3172 if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { 3173 verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", 3174 kfunc.name); 3175 return -EOPNOTSUPP; 3176 } 3177 if (!bpf_jit_supports_kfunc_ret_reg_pair()) { 3178 verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", 3179 kfunc.name); 3180 return -EOPNOTSUPP; 3181 } 3182 } 3183 3184 memset(&meta, 0, sizeof(meta)); 3185 meta.btf = kfunc.btf; 3186 meta.func_id = kfunc.id; 3187 meta.func_proto = kfunc.proto; 3188 meta.func_name = kfunc.name; 3189 meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; 3190 3191 tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); 3192 if (!tab) 3193 return -ENOMEM; 3194 prog_aux->kfunc_tab = tab; 3195 3196 desc = &tab->descs[tab->nr_descs]; 3197 memset(desc, 0, sizeof(*desc)); 3198 3199 err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); 3200 if (err) 3201 return err; 3202 3203 desc->func_id = func_id; 3204 desc->offset = offset; 3205 desc->addr = addr; 3206 desc->func_model = func_model; 3207 tab->nr_descs++; 3208 tab->nr_base_descs++; 3209 sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), 3210 kfunc_desc_cmp_by_id_off, NULL); 3211 return 0; 3212 } 3213 3214 static int add_subprogs(struct bpf_verifier_env *env) 3215 { 3216 struct bpf_subprog_info *subprog = env->subprog_info; 3217 int i, ret, insn_cnt = env->prog->len, ex_cb_insn; 3218 struct bpf_insn *insn = env->prog->insnsi; 3219 const char *operation, *suggestion; 3220 3221 /* Add entry function. */ 3222 ret = add_subprog(env, 0); 3223 if (ret) 3224 return ret; 3225 3226 for (i = 0; i < insn_cnt; i++, insn++) { 3227 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 3228 continue; 3229 3230 if (!env->bpf_capable) { 3231 if (bpf_pseudo_func(insn)) { 3232 operation = "BPF function reference"; 3233 suggestion = "Load this program with the required capability, or avoid BPF function references in unprivileged programs."; 3234 } else { 3235 operation = "BPF-to-BPF function call"; 3236 suggestion = "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."; 3237 } 3238 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 3239 bpf_diag_policy( 3240 env, i, operation, 3241 "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", 3242 suggestion); 3243 return -EPERM; 3244 } 3245 3246 ret = add_subprog(env, i + insn->imm + 1); 3247 if (ret < 0) 3248 return ret; 3249 } 3250 3251 /* 3252 * func_info describes all functions of the program. Those that are 3253 * referenced only from data, e.g. from a table of functions to be 3254 * called via callx, are not seen by the loop above. They are possible 3255 * callees that have to be known upfront as well. 3256 */ 3257 if (env->bpf_capable) { 3258 struct bpf_prog_aux *aux = env->prog->aux; 3259 3260 for (i = 1; i < aux->func_info_cnt; i++) { 3261 ret = add_subprog(env, aux->func_info[i].insn_off); 3262 if (ret < 0) 3263 return ret; 3264 } 3265 } 3266 3267 ret = bpf_find_exception_callback_insn_off(env); 3268 if (ret < 0) 3269 return ret; 3270 ex_cb_insn = ret; 3271 3272 /* If ex_cb_insn > 0, this means that the main program has a subprog 3273 * marked using BTF decl tag to serve as the exception callback. 3274 */ 3275 if (ex_cb_insn) { 3276 ret = add_subprog(env, ex_cb_insn); 3277 if (ret < 0) 3278 return ret; 3279 for (i = 1; i < env->subprog_cnt; i++) { 3280 if (env->subprog_info[i].start != ex_cb_insn) 3281 continue; 3282 env->exception_callback_subprog = i; 3283 bpf_mark_subprog_exc_cb(env, i); 3284 break; 3285 } 3286 } 3287 3288 /* Add a fake 'exit' subprog which could simplify subprog iteration 3289 * logic. 'subprog_cnt' should not be increased. 3290 */ 3291 subprog[env->subprog_cnt].start = insn_cnt; 3292 3293 if (env->log.level & BPF_LOG_LEVEL2) 3294 for (i = 0; i < env->subprog_cnt; i++) 3295 verbose(env, "func#%d @%d\n", i, subprog[i].start); 3296 3297 return 0; 3298 } 3299 3300 static int add_kfuncs(struct bpf_verifier_env *env) 3301 { 3302 struct bpf_insn *insn = env->prog->insnsi; 3303 int i, ret, insn_cnt = env->prog->len; 3304 3305 for (i = 0; i < insn_cnt; i++, insn++) { 3306 if (!bpf_pseudo_kfunc_call(insn)) 3307 continue; 3308 3309 if (!env->bpf_capable) { 3310 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 3311 bpf_diag_policy( 3312 env, i, "kernel function call", 3313 "calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN", 3314 "Load this program with the required capability, or avoid kernel function calls in unprivileged programs."); 3315 return -EPERM; 3316 } 3317 3318 ret = bpf_add_kfunc_call(env, insn->imm, insn->off); 3319 if (ret < 0) 3320 return ret; 3321 } 3322 3323 return 0; 3324 } 3325 3326 static void find_subprog_properties(struct bpf_verifier_env *env) 3327 { 3328 struct bpf_subprog_info *subprog = env->subprog_info; 3329 struct bpf_insn *insn = env->prog->insnsi; 3330 int cur_subprog; 3331 3332 for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { 3333 int i; 3334 3335 for (i = subprog[cur_subprog].start; 3336 i < subprog[cur_subprog + 1].start; i++) { 3337 u8 code = insn[i].code; 3338 3339 if (code == (BPF_JMP | BPF_CALL) && 3340 insn[i].src_reg == 0 && 3341 insn[i].imm == BPF_FUNC_tail_call) { 3342 subprog[cur_subprog].has_tail_call = true; 3343 subprog[cur_subprog].tail_call_reachable = true; 3344 } 3345 if (BPF_CLASS(code) == BPF_LD && 3346 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 3347 subprog[cur_subprog].has_ld_abs = true; 3348 if (bpf_is_callx(&insn[i])) 3349 env->has_callx = true; 3350 } 3351 } 3352 } 3353 3354 static int check_subprogs(struct bpf_verifier_env *env) 3355 { 3356 int i, subprog_start, subprog_end, off, cur_subprog = 0; 3357 struct bpf_subprog_info *subprog = env->subprog_info; 3358 struct bpf_insn *insn = env->prog->insnsi; 3359 int insn_cnt = env->prog->len; 3360 3361 /* now check that all jumps are within the same subprog */ 3362 subprog_start = subprog[cur_subprog].start; 3363 subprog_end = subprog[cur_subprog + 1].start; 3364 for (i = 0; i < insn_cnt; i++) { 3365 u8 code = insn[i].code; 3366 3367 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 3368 goto next; 3369 if (BPF_OP(code) == BPF_CALL) 3370 goto next; 3371 if (BPF_OP(code) == BPF_EXIT) { 3372 subprog[cur_subprog].exit_idx = i; 3373 goto next; 3374 } 3375 if (insn_is_gotox(&insn[i])) 3376 goto next; 3377 off = i + bpf_jmp_offset(&insn[i]) + 1; 3378 if (off < subprog_start || off >= subprog_end) { 3379 verbose(env, "jump out of range from insn %d to %d\n", i, off); 3380 bpf_diag_program_structure( 3381 env, i, "jump out of range", 3382 "Keep branch targets within the same subprogram, or use an explicit subprogram call.", 3383 "Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. " 3384 "A branch target must stay inside the same subprogram.", 3385 i, off, cur_subprog, subprog_start, subprog_end - 1); 3386 return -EINVAL; 3387 } 3388 next: 3389 if (i == subprog_end - 1) { 3390 /* 3391 * To avoid fall-through from one subprog into another, 3392 * the last insn of the subprog should be either exit 3393 * or unconditional jump back or bpf_throw call. 3394 */ 3395 if (code != (BPF_JMP | BPF_EXIT) && 3396 code != (BPF_JMP32 | BPF_JA) && 3397 code != (BPF_JMP | BPF_JA) && 3398 !insn_is_gotox(&insn[i])) { 3399 verbose(env, "last insn is not an exit or jmp\n"); 3400 bpf_diag_program_structure( 3401 env, i, "subprogram can fall through", 3402 "End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.", 3403 "Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.", 3404 cur_subprog, i); 3405 return -EINVAL; 3406 } 3407 subprog_start = subprog_end; 3408 cur_subprog++; 3409 if (cur_subprog < env->subprog_cnt) 3410 subprog_end = subprog[cur_subprog + 1].start; 3411 } 3412 } 3413 return 0; 3414 } 3415 3416 /* 3417 * The callee of callx is known to the main verification pass only, which 3418 * records the 'caller' -> 'callee' edge of the call graph for the checks 3419 * that follow it: absence of recursion and the maximum stack depth. 3420 */ 3421 static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) 3422 { 3423 u32 cnt = env->subprog_cnt; 3424 3425 if (!env->callx_edges) { 3426 env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), 3427 GFP_KERNEL_ACCOUNT); 3428 if (!env->callx_edges) 3429 return -ENOMEM; 3430 } 3431 __set_bit(caller * cnt + callee, env->callx_edges); 3432 return 0; 3433 } 3434 3435 /* 3436 * Return the first subprog with the number >= 'from' that 'caller' calls 3437 * via callx, or -1 when there is none. 3438 */ 3439 static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) 3440 { 3441 u32 cnt = env->subprog_cnt; 3442 unsigned long bit, end = (caller + 1) * cnt; 3443 3444 if (!env->callx_edges || from >= cnt) 3445 return -1; 3446 bit = find_next_bit(env->callx_edges, end, caller * cnt + from); 3447 return bit < end ? bit - caller * cnt : -1; 3448 } 3449 3450 /* 3451 * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit. 3452 * JITs don't emit a branch for it. unwind_exit finds the pad by its return 3453 * address. So unwind_goto has to follow the call and nothing may jump to it. 3454 */ 3455 static int check_unwind_insns(struct bpf_verifier_env *env) 3456 { 3457 struct bpf_insn *insn = env->prog->insnsi; 3458 bool seen = false; 3459 int i; 3460 3461 for (i = 0; i < env->prog->len; i++, insn++) { 3462 if (bpf_is_unwind_exit(insn)) { 3463 bpf_find_containing_subprog(env, i)->can_unwind = true; 3464 seen = true; 3465 } 3466 if (!bpf_is_unwind_goto(insn)) 3467 continue; 3468 seen = true; 3469 if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) { 3470 verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n", 3471 i); 3472 return -EINVAL; 3473 } 3474 if (bpf_is_jump_target(env, i)) { 3475 verbose(env, "jump to unwind_goto at insn %d\n", i); 3476 return -EINVAL; 3477 } 3478 } 3479 if (!seen) 3480 return 0; 3481 if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) { 3482 verbose(env, "JIT does not support unwinding\n"); 3483 return -ENOTSUPP; 3484 } 3485 env->prog->jit_required = 1; 3486 env->prog->aux->can_unwind = env->subprog_info[0].can_unwind; 3487 return 0; 3488 } 3489 3490 /* 3491 * Sort subprogs in topological order so that leaf subprogs come first and 3492 * their callers come later. This is a DFS post-order traversal of the call 3493 * graph. Scan only reachable instructions (those in the computed postorder) of 3494 * the current subprog to discover callees (direct subprogs and sync 3495 * callbacks). 3496 * 3497 * The callees of callx are not known before the main verification pass. 3498 * When callx is used the sort is repeated after it with the recorded callx 3499 * edges added to the call graph to reject recursion through indirect calls. 3500 */ 3501 static int sort_subprogs_topo(struct bpf_verifier_env *env) 3502 { 3503 struct bpf_subprog_info *si = env->subprog_info; 3504 int *insn_postorder = env->cfg.insn_postorder; 3505 struct bpf_insn *insn = env->prog->insnsi; 3506 int cnt = env->subprog_cnt; 3507 int *dfs_stack = NULL; 3508 int top = 0, order = 0; 3509 int i, ret = 0; 3510 u8 *color = NULL; 3511 3512 color = kvzalloc_objs(*color, cnt, GFP_KERNEL_ACCOUNT); 3513 dfs_stack = kvmalloc_objs(*dfs_stack, cnt, GFP_KERNEL_ACCOUNT); 3514 if (!color || !dfs_stack) { 3515 ret = -ENOMEM; 3516 goto out; 3517 } 3518 3519 /* 3520 * DFS post-order traversal. 3521 * Color values: 0 = unvisited, 1 = on stack, 2 = done. 3522 */ 3523 for (i = 0; i < cnt; i++) { 3524 if (color[i]) 3525 continue; 3526 color[i] = 1; 3527 dfs_stack[top++] = i; 3528 3529 while (top > 0) { 3530 int cur = dfs_stack[top - 1]; 3531 int po_start = si[cur].postorder_start; 3532 int po_end = si[cur + 1].postorder_start; 3533 bool pushed = false; 3534 int j; 3535 3536 for (j = po_start; j < po_end; j++) { 3537 int idx = insn_postorder[j]; 3538 int callee; 3539 3540 if (bpf_is_callx(&insn[idx])) { 3541 /* find a callee that is not explored yet */ 3542 callee = -1; 3543 do { 3544 callee = next_callx_callee(env, cur, callee + 1); 3545 } while (callee >= 0 && color[callee] == 2); 3546 if (callee < 0) 3547 continue; 3548 } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { 3549 callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); 3550 if (callee < 0) { 3551 ret = -EFAULT; 3552 goto out; 3553 } 3554 } else { 3555 continue; 3556 } 3557 if (color[callee] == 2) 3558 continue; 3559 if (color[callee] == 1) { 3560 if (bpf_pseudo_func(&insn[idx])) 3561 continue; 3562 verbose(env, "recursive call from %s() to %s()\n", 3563 bpf_subprog_name(env, cur), 3564 bpf_subprog_name(env, callee)); 3565 bpf_diag_program_structure( 3566 env, idx, "recursive subprogram call", 3567 "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", 3568 "This bpf2bpf call would make the subprogram call graph recursive. " 3569 "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); 3570 ret = -EINVAL; 3571 goto out; 3572 } 3573 color[callee] = 1; 3574 dfs_stack[top++] = callee; 3575 pushed = true; 3576 break; 3577 } 3578 3579 if (!pushed) { 3580 color[cur] = 2; 3581 env->subprog_topo_order[order++] = cur; 3582 top--; 3583 } 3584 } 3585 } 3586 3587 if (env->log.level & BPF_LOG_LEVEL2) 3588 for (i = 0; i < cnt; i++) 3589 verbose(env, "topo_order[%d] = %s\n", 3590 i, bpf_subprog_name(env, env->subprog_topo_order[i])); 3591 out: 3592 kvfree(dfs_stack); 3593 kvfree(color); 3594 return ret; 3595 } 3596 3597 static void mark_stack_slots_scratched(struct bpf_verifier_env *env, 3598 int spi, int nr_slots) 3599 { 3600 int i; 3601 3602 for (i = 0; i < nr_slots; i++) 3603 mark_stack_slot_scratched(env, spi - i); 3604 } 3605 3606 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, 3607 enum bpf_reg_arg_type t) 3608 { 3609 struct bpf_reg_state *reg; 3610 3611 mark_reg_scratched(env, regno); 3612 3613 reg = ®s[regno]; 3614 if (t == SRC_OP) { 3615 /* check whether register used as source operand can be read */ 3616 if (reg->type == NOT_INIT) { 3617 verbose(env, "R%d !read_ok\n", regno); 3618 bpf_diag_unreadable_reg(env, env->insn_idx, regno); 3619 return -EACCES; 3620 } 3621 /* We don't need to worry about FP liveness because it's read-only */ 3622 if (regno == BPF_REG_FP) 3623 return 0; 3624 3625 return 0; 3626 } else { 3627 /* check whether register used as dest operand can be written to */ 3628 if (regno == BPF_REG_FP) { 3629 verbose(env, "frame pointer is read only\n"); 3630 return -EACCES; 3631 } 3632 if (t == DST_OP) 3633 mark_reg_unknown(env, regs, regno); 3634 } 3635 return 0; 3636 } 3637 3638 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 3639 enum bpf_reg_arg_type t) 3640 { 3641 struct bpf_verifier_state *vstate = env->cur_state; 3642 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3643 3644 return __check_reg_arg(env, state->regs, regno, t); 3645 } 3646 3647 static void mark_indirect_target(struct bpf_verifier_env *env, int idx) 3648 { 3649 env->insn_aux_data[idx].indirect_target = true; 3650 } 3651 3652 static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) 3653 { 3654 env->insn_aux_data[idx].non_stack_access = true; 3655 } 3656 3657 /* Layout of one packed linked register in the jump history, see linked_regs_pack() */ 3658 #define LR_FRAMENO_BITS 4 3659 #define LR_INDEX_BITS 11 3660 #define LR_FRAMENO_MASK ((1u << LR_FRAMENO_BITS) - 1) 3661 #define LR_IS_REG BIT(LR_FRAMENO_BITS) 3662 #define LR_INDEX_OFF (LR_FRAMENO_BITS + 1) 3663 #define LR_INDEX_MASK ((1u << LR_INDEX_BITS) - 1) 3664 #define LINKED_REGS_MAX BPF_LINKED_REGS_MAX 3665 3666 static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); 3667 static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS)); 3668 static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS)); 3669 static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16); 3670 3671 struct linked_reg { 3672 u8 frameno; 3673 union { 3674 u16 spi; 3675 u16 regno; 3676 }; 3677 bool is_reg; 3678 }; 3679 3680 struct linked_regs { 3681 int cnt; 3682 struct linked_reg entries[LINKED_REGS_MAX]; 3683 }; 3684 3685 static struct linked_reg *linked_regs_push(struct linked_regs *s) 3686 { 3687 if (s->cnt < LINKED_REGS_MAX) 3688 return &s->entries[s->cnt++]; 3689 3690 return NULL; 3691 } 3692 3693 /* 3694 * Pack linked registers for a jump history entry, one u16 each: 3695 * - 4 bits frameno 3696 * - 1 bit is_reg 3697 * - 11 bits register or stack slot index 3698 */ 3699 static void linked_regs_pack(const struct linked_regs *s, u16 *packed) 3700 { 3701 int i; 3702 3703 for (i = 0; i < s->cnt; ++i) { 3704 const struct linked_reg *e = &s->entries[i]; 3705 3706 packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF); 3707 } 3708 } 3709 3710 static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s) 3711 { 3712 int i; 3713 3714 s->cnt = hist->linked_regs_cnt; 3715 for (i = 0; i < s->cnt; ++i) { 3716 struct linked_reg *e = &s->entries[i]; 3717 u16 packed = hist->linked_regs[i]; 3718 3719 e->frameno = packed & LR_FRAMENO_MASK; 3720 e->is_reg = packed & LR_IS_REG; 3721 e->spi = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK; 3722 } 3723 } 3724 3725 const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) 3726 { 3727 const struct btf_type *func; 3728 struct btf *desc_btf; 3729 3730 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 3731 return NULL; 3732 3733 desc_btf = find_kfunc_desc_btf_cached(data, insn->off); 3734 if (IS_ERR(desc_btf)) 3735 return "<error>"; 3736 3737 func = btf_type_by_id(desc_btf, insn->imm); 3738 if (!func || !btf_type_is_func(func)) 3739 return "<error>"; 3740 return btf_name_by_offset(desc_btf, func->name_off); 3741 } 3742 3743 void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) 3744 { 3745 const struct bpf_insn_cbs cbs = { 3746 .cb_call = bpf_disasm_kfunc_name, 3747 .cb_print = verbose, 3748 .private_data = env, 3749 }; 3750 3751 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 3752 } 3753 3754 /* If any register R in hist->linked_regs is marked as precise in bt, 3755 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs. 3756 */ 3757 void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist) 3758 { 3759 struct linked_regs linked_regs; 3760 bool some_precise = false; 3761 int i; 3762 3763 if (!hist || !hist->linked_regs_cnt) 3764 return; 3765 3766 linked_regs_unpack(hist, &linked_regs); 3767 for (i = 0; i < linked_regs.cnt; ++i) { 3768 struct linked_reg *e = &linked_regs.entries[i]; 3769 3770 if ((e->is_reg && bt_is_frame_reg_set(bt, e->frameno, e->regno)) || 3771 (!e->is_reg && bt_is_frame_slot_set(bt, e->frameno, e->spi))) { 3772 some_precise = true; 3773 break; 3774 } 3775 } 3776 3777 if (!some_precise) 3778 return; 3779 3780 for (i = 0; i < linked_regs.cnt; ++i) { 3781 struct linked_reg *e = &linked_regs.entries[i]; 3782 3783 if (e->is_reg) 3784 bpf_bt_set_frame_reg(bt, e->frameno, e->regno); 3785 else 3786 bpf_bt_set_frame_slot(bt, e->frameno, e->spi); 3787 } 3788 } 3789 3790 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 3791 { 3792 return bpf_mark_chain_precision(env, env->cur_state, regno, NULL); 3793 } 3794 3795 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to 3796 * desired reg and stack masks across all relevant frames 3797 */ 3798 static int mark_chain_precision_batch(struct bpf_verifier_env *env, 3799 struct bpf_verifier_state *starting_state) 3800 { 3801 return bpf_mark_chain_precision(env, starting_state, -1, NULL); 3802 } 3803 3804 /* check if register is a constant scalar value */ 3805 static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) 3806 { 3807 return reg->type == SCALAR_VALUE && 3808 tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off); 3809 } 3810 3811 /* assuming is_reg_const() is true, return constant value of a register */ 3812 static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32) 3813 { 3814 return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value; 3815 } 3816 3817 static bool is_pointer_regtype(enum bpf_reg_type type) 3818 { 3819 return type != SCALAR_VALUE && type != NOT_INIT; 3820 } 3821 3822 static bool __is_pointer_value(bool allow_ptr_leaks, 3823 const struct bpf_reg_state *reg) 3824 { 3825 if (allow_ptr_leaks) 3826 return false; 3827 3828 return is_pointer_regtype(reg->type); 3829 } 3830 3831 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, 3832 struct bpf_reg_state *src_reg) 3833 { 3834 if (src_reg->type != SCALAR_VALUE) 3835 return; 3836 /* 3837 * The verifier is processing rX = rY insn and 3838 * rY->id has special linked register already. 3839 * Cleared it, since multiple rX += const are not supported. 3840 */ 3841 if (src_reg->id & BPF_ADD_CONST) 3842 clear_scalar_id(src_reg); 3843 /* 3844 * Ensure that src_reg has a valid ID that will be copied to 3845 * dst_reg and then will be used by sync_linked_regs() to 3846 * propagate min/max range. 3847 */ 3848 if (!src_reg->id && !tnum_is_const(src_reg->var_off)) 3849 src_reg->id = ++env->id_gen; 3850 } 3851 3852 static void save_register_state(struct bpf_verifier_env *env, 3853 struct bpf_func_state *state, 3854 int spi, struct bpf_reg_state *reg, 3855 int size) 3856 { 3857 struct bpf_stack_state *slot = bpf_stack_slot(state, spi); 3858 int i; 3859 3860 bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL); 3861 slot->spilled_ptr = *reg; 3862 3863 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 3864 slot->slot_type[i - 1] = STACK_SPILL; 3865 3866 /* size < 8 bytes spill */ 3867 for (; i; i--) 3868 mark_stack_slot_misc(env, &slot->slot_type[i - 1]); 3869 3870 bpf_diag_mod_end(env); 3871 } 3872 3873 static bool is_bpf_st_mem(struct bpf_insn *insn) 3874 { 3875 return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM; 3876 } 3877 3878 static int get_reg_width(struct bpf_reg_state *reg) 3879 { 3880 return fls64(reg_umax(reg)); 3881 } 3882 3883 /* See comment for mark_fastcall_pattern_for_call() */ 3884 static void check_fastcall_stack_contract(struct bpf_verifier_env *env, 3885 struct bpf_func_state *state, int insn_idx, int off) 3886 { 3887 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; 3888 struct bpf_insn_aux_data *aux = env->insn_aux_data; 3889 int i; 3890 3891 if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) 3892 return; 3893 /* access to the region [max_stack_depth .. fastcall_stack_off) 3894 * from something that is not a part of the fastcall pattern, 3895 * disable fastcall rewrites for current subprogram by setting 3896 * fastcall_stack_off to a value smaller than any possible offset. 3897 */ 3898 subprog->fastcall_stack_off = S16_MIN; 3899 /* reset fastcall aux flags within subprogram, 3900 * happens at most once per subprogram 3901 */ 3902 for (i = subprog->start; i < (subprog + 1)->start; ++i) { 3903 aux[i].fastcall_spills_num = 0; 3904 aux[i].fastcall_pattern = 0; 3905 } 3906 } 3907 3908 static void scrub_special_slot(struct bpf_func_state *state, int spi) 3909 { 3910 struct bpf_stack_state *slot = bpf_stack_slot(state, spi); 3911 int i; 3912 3913 /* regular write of data into stack destroys any spilled ptr */ 3914 slot->spilled_ptr.type = NOT_INIT; 3915 /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ 3916 if (is_stack_slot_special(slot)) 3917 for (i = 0; i < BPF_REG_SIZE; i++) 3918 scrub_spilled_slot(&slot->slot_type[i]); 3919 } 3920 3921 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 3922 * stack boundary and alignment are checked in check_mem_access() 3923 */ 3924 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 3925 /* stack frame we're writing to */ 3926 struct bpf_func_state *state, 3927 int off, int size, int value_regno, 3928 int insn_idx) 3929 { 3930 struct bpf_func_state *cur; /* state of the current function */ 3931 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 3932 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3933 struct bpf_reg_state *reg = NULL; 3934 int insn_flags = INSN_F_STACK_ACCESS; 3935 int hist_spi = spi, hist_frame = state->frameno; 3936 struct bpf_stack_state *ss = bpf_stack_slot(state, spi); 3937 3938 /* 3939 * caller checked that off % size == 0 and -env->stack_limit <= off < 0, 3940 * so it's aligned access and [off, off + size) are within stack limits 3941 */ 3942 if (!env->allow_ptr_leaks && 3943 bpf_is_spilled_reg(ss) && 3944 !bpf_is_spilled_scalar_reg(ss) && 3945 size != BPF_REG_SIZE) { 3946 const char *reason; 3947 3948 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 3949 reason = bpf_diag_fmt(env, 3950 "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. " 3951 "Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.", 3952 size, off); 3953 bpf_diag_memory( 3954 env, insn_idx, "stack spill corruption", reason, 3955 "Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it."); 3956 return -EACCES; 3957 } 3958 3959 cur = env->cur_state->frame[env->cur_state->curframe]; 3960 if (value_regno >= 0) 3961 reg = &cur->regs[value_regno]; 3962 if (!env->bypass_spec_v4) { 3963 bool sanitize = reg && is_pointer_regtype(reg->type); 3964 3965 for (i = 0; i < size; i++) { 3966 u8 type = ss->slot_type[(slot - i) % 3967 BPF_REG_SIZE]; 3968 3969 if (type != STACK_MISC && type != STACK_ZERO) { 3970 sanitize = true; 3971 break; 3972 } 3973 } 3974 3975 if (sanitize) 3976 env->insn_aux_data[insn_idx].nospec_result = true; 3977 } 3978 3979 err = destroy_if_dynptr_stack_slot(env, state, spi); 3980 if (err) 3981 return err; 3982 3983 mark_stack_slot_scratched(env, spi); 3984 if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { 3985 bool reg_value_fits; 3986 3987 reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size; 3988 /* Make sure that reg had an ID to build a relation on spill. */ 3989 if (reg_value_fits) 3990 assign_scalar_id_before_mov(env, reg); 3991 save_register_state(env, state, spi, reg, size); 3992 /* Break the relation on a narrowing spill. */ 3993 if (!reg_value_fits) 3994 clear_scalar_id(&ss->spilled_ptr); 3995 } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && 3996 env->bpf_capable) { 3997 struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; 3998 3999 memset(tmp_reg, 0, sizeof(*tmp_reg)); 4000 __mark_reg_known(tmp_reg, insn->imm); 4001 tmp_reg->type = SCALAR_VALUE; 4002 save_register_state(env, state, spi, tmp_reg, size); 4003 } else if (reg && is_pointer_regtype(reg->type)) { 4004 /* register containing pointer is being spilled into stack */ 4005 if (size != BPF_REG_SIZE) { 4006 verbose_linfo(env, insn_idx, "; "); 4007 verbose(env, "invalid size of register spill\n"); 4008 return -EACCES; 4009 } 4010 if (state != cur && reg->type == PTR_TO_STACK) { 4011 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 4012 return -EINVAL; 4013 } 4014 save_register_state(env, state, spi, reg, size); 4015 } else { 4016 u8 type = STACK_MISC; 4017 4018 if (bpf_is_spilled_reg(ss)) 4019 bpf_diag_record_scrub(env, &ss->spilled_ptr, 4020 BPF_DIAG_MOD_WRITE); 4021 scrub_special_slot(state, spi); 4022 4023 /* when we zero initialize stack slots mark them as such */ 4024 if ((reg && bpf_register_is_null(reg)) || 4025 (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { 4026 /* STACK_ZERO case happened because register spill 4027 * wasn't properly aligned at the stack slot boundary, 4028 * so it's not a register spill anymore; force 4029 * originating register to be precise to make 4030 * STACK_ZERO correct for subsequent states 4031 */ 4032 err = mark_chain_precision(env, value_regno); 4033 if (err) 4034 return err; 4035 type = STACK_ZERO; 4036 } 4037 4038 /* Mark slots affected by this stack write. */ 4039 for (i = 0; i < size; i++) 4040 ss->slot_type[(slot - i) % BPF_REG_SIZE] = type; 4041 insn_flags = 0; /* not a register spill */ 4042 } 4043 4044 if (insn_flags) 4045 return bpf_push_jmp_history(env, env->cur_state, insn_flags, 4046 hist_spi, hist_frame, NULL, 0); 4047 return 0; 4048 } 4049 4050 /* Write the stack: 'stack[ptr_reg + off] = value_regno'. 'ptr_reg' is 4051 * known to contain a variable offset. 4052 * This function checks whether the write is permitted and conservatively 4053 * tracks the effects of the write, considering that each stack slot in the 4054 * dynamic range is potentially written to. 4055 * 4056 * 'value_regno' can be -1, meaning that an unknown value is being written to 4057 * the stack. 4058 * 4059 * Spilled pointers in range are not marked as written because we don't know 4060 * what's going to be actually written. This means that read propagation for 4061 * future reads cannot be terminated by this write. 4062 * 4063 * For privileged programs, uninitialized stack slots are considered 4064 * initialized by this write (even though we don't know exactly what offsets 4065 * are going to be written to). The idea is that we don't want the verifier to 4066 * reject future reads that access slots written to through variable offsets. 4067 */ 4068 static int check_stack_write_var_off(struct bpf_verifier_env *env, 4069 /* func where register points to */ 4070 struct bpf_func_state *state, 4071 struct bpf_reg_state *ptr_reg, int off, int size, 4072 int value_regno, int insn_idx) 4073 { 4074 struct bpf_func_state *cur; /* state of the current function */ 4075 int min_off, max_off; 4076 int i, err; 4077 struct bpf_reg_state *value_reg = NULL; 4078 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 4079 bool writing_zero = false; 4080 /* set if the fact that we're writing a zero is used to let any 4081 * stack slots remain STACK_ZERO 4082 */ 4083 bool zero_used = false; 4084 4085 cur = env->cur_state->frame[env->cur_state->curframe]; 4086 min_off = reg_smin(ptr_reg) + off; 4087 max_off = reg_smax(ptr_reg) + off + size; 4088 if (value_regno >= 0) 4089 value_reg = &cur->regs[value_regno]; 4090 if ((value_reg && bpf_register_is_null(value_reg)) || 4091 (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) 4092 writing_zero = true; 4093 4094 err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1), 4095 bpf_get_spi(min_off)); 4096 if (err) 4097 return err; 4098 4099 /* Variable offset writes destroy any spilled pointers in range. */ 4100 for (i = min_off; i < max_off; i++) { 4101 struct bpf_stack_state *ss; 4102 u8 new_type, *stype; 4103 int slot, spi; 4104 4105 slot = -i - 1; 4106 spi = slot / BPF_REG_SIZE; 4107 ss = bpf_stack_slot(state, spi); 4108 stype = &ss->slot_type[slot % BPF_REG_SIZE]; 4109 mark_stack_slot_scratched(env, spi); 4110 4111 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 4112 /* Reject the write if range we may write to has not 4113 * been initialized beforehand. If we didn't reject 4114 * here, the ptr status would be erased below (even 4115 * though not all slots are actually overwritten), 4116 * possibly opening the door to leaks. 4117 * 4118 * We do however catch STACK_INVALID case below, and 4119 * only allow reading possibly uninitialized memory 4120 * later for CAP_PERFMON, as the write may not happen to 4121 * that slot. 4122 */ 4123 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 4124 insn_idx, i); 4125 return -EINVAL; 4126 } 4127 4128 /* If writing_zero and the spi slot contains a spill of value 0, 4129 * maintain the spill type. 4130 */ 4131 if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) { 4132 struct bpf_reg_state *spill_reg = &ss->spilled_ptr; 4133 4134 if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { 4135 zero_used = true; 4136 continue; 4137 } 4138 } 4139 4140 /* 4141 * Scrub slots if variable-offset stack write goes over spilled pointers. 4142 * Otherwise bpf_is_spilled_reg() may == true && spilled_ptr.type == NOT_INIT 4143 * and valid program is rejected by check_stack_read_fixed_off() 4144 * with obscure "invalid size of register fill" message. 4145 */ 4146 scrub_special_slot(state, spi); 4147 4148 /* Update the slot type. */ 4149 new_type = STACK_MISC; 4150 if (writing_zero && *stype == STACK_ZERO) { 4151 new_type = STACK_ZERO; 4152 zero_used = true; 4153 } 4154 /* If the slot is STACK_INVALID, we check whether it's OK to 4155 * pretend that it will be initialized by this write. The slot 4156 * might not actually be written to, and so if we mark it as 4157 * initialized future reads might leak uninitialized memory. 4158 * For privileged programs, we will accept such reads to slots 4159 * that may or may not be written because, if we're reject 4160 * them, the error would be too confusing. 4161 * Conservatively, treat STACK_POISON in a similar way. 4162 */ 4163 if ((*stype == STACK_INVALID || *stype == STACK_POISON) && 4164 !env->allow_uninit_stack) { 4165 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 4166 insn_idx, i); 4167 return -EINVAL; 4168 } 4169 *stype = new_type; 4170 } 4171 if (zero_used) { 4172 /* backtracking doesn't work for STACK_ZERO yet. */ 4173 err = mark_chain_precision(env, value_regno); 4174 if (err) 4175 return err; 4176 } 4177 bpf_diag_record_scrub_stack(env, state, min_off, max_off, 4178 BPF_DIAG_MOD_VAR_WRITE); 4179 return 0; 4180 } 4181 4182 /* When register 'dst_regno' is assigned some values from stack[min_off, 4183 * max_off), we set the register's type according to the types of the 4184 * respective stack slots. If all the stack values are known to be zeros, then 4185 * so is the destination reg. Otherwise, the register is considered to be 4186 * SCALAR. This function does not deal with register filling; the caller must 4187 * ensure that all spilled registers in the stack range have been marked as 4188 * read. 4189 * 4190 * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered 4191 * zero bytes. In that case, mark the contributing stack slots precise so 4192 * pruning cannot reuse a zero-spill state for a later non-zero spill state. 4193 * 4194 * Returns an error if precision backtracking fails. 4195 */ 4196 static int mark_reg_stack_read(struct bpf_verifier_env *env, 4197 /* func where src register points to */ 4198 struct bpf_func_state *ptr_state, 4199 int min_off, int max_off, int dst_regno) 4200 { 4201 struct bpf_verifier_state *vstate = env->cur_state; 4202 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4203 int i, slot, spi; 4204 u8 *stype; 4205 int zeros = 0, zero_spills = 0; 4206 4207 for (i = min_off; i < max_off; i++) { 4208 slot = -i - 1; 4209 spi = slot / BPF_REG_SIZE; 4210 mark_stack_slot_scratched(env, spi); 4211 stype = bpf_stack_slot(ptr_state, spi)->slot_type; 4212 if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { 4213 zeros++; 4214 continue; 4215 } 4216 if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && 4217 bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { 4218 zero_spills++; 4219 zeros++; 4220 continue; 4221 } 4222 break; 4223 } 4224 if (zeros == max_off - min_off) { 4225 /* Any access_size read into register is zero extended, 4226 * so the whole register == const_zero. 4227 */ 4228 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4229 if (zero_spills) { 4230 for (i = min_off; i < max_off; i++) { 4231 slot = -i - 1; 4232 spi = slot / BPF_REG_SIZE; 4233 stype = bpf_stack_slot(ptr_state, spi)->slot_type; 4234 if (stype[slot % BPF_REG_SIZE] == STACK_SPILL) 4235 bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi); 4236 } 4237 return mark_chain_precision_batch(env, env->cur_state); 4238 } 4239 } else { 4240 /* have read misc data from the stack */ 4241 mark_reg_unknown(env, state->regs, dst_regno); 4242 } 4243 4244 return 0; 4245 } 4246 4247 static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size, 4248 enum bpf_stack_slot_type type) 4249 { 4250 const char *problem, *reason, *suggestion, *kind; 4251 4252 if (type == STACK_INVALID) { 4253 reason = bpf_diag_fmt( 4254 env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " 4255 "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", 4256 size, off, i); 4257 bpf_diag_memory( 4258 env, env->insn_idx, "uninitialized stack read", reason, 4259 "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " 4260 "or load with CAP_PERFMON if uninitialized stack reads are intended."); 4261 return; 4262 } 4263 4264 switch (type) { 4265 case STACK_DYNPTR: 4266 kind = "dynptr"; 4267 suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly."; 4268 break; 4269 case STACK_ITER: 4270 kind = "iterator"; 4271 suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly."; 4272 break; 4273 case STACK_IRQ_FLAG: 4274 kind = "IRQ flag"; 4275 suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly."; 4276 break; 4277 default: 4278 return; 4279 } 4280 4281 problem = bpf_diag_fmt(env, "direct read of %s stack state", kind); 4282 reason = bpf_diag_fmt( 4283 env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. " 4284 "This state has an opaque representation that BPF programs cannot read directly.", 4285 size, off, i, kind); 4286 bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion); 4287 } 4288 4289 /* Read the stack at 'off' and put the results into the register indicated by 4290 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 4291 * spilled reg. 4292 * 4293 * 'dst_regno' can be -1, meaning that the read value is not going to a 4294 * register. 4295 * 4296 * The access is assumed to be within the current stack bounds. 4297 */ 4298 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 4299 /* func where src register points to */ 4300 struct bpf_func_state *reg_state, 4301 int off, int size, int dst_regno) 4302 { 4303 struct bpf_verifier_state *vstate = env->cur_state; 4304 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4305 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 4306 struct bpf_reg_state *reg; 4307 u8 *stype, type; 4308 int err; 4309 int insn_flags = INSN_F_STACK_ACCESS; 4310 int hist_spi = spi, hist_frame = reg_state->frameno; 4311 struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi); 4312 4313 stype = ss->slot_type; 4314 reg = &ss->spilled_ptr; 4315 4316 mark_stack_slot_scratched(env, spi); 4317 4318 /* 4319 * Refine the in-progress load record's origin to the source stack slot. 4320 */ 4321 if (dst_regno >= 0) 4322 bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); 4323 4324 if (bpf_is_spilled_reg(ss)) { 4325 u8 spill_size = bpf_spill_size(ss); 4326 4327 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 4328 if (reg->type != SCALAR_VALUE) { 4329 verbose_linfo(env, env->insn_idx, "; "); 4330 verbose(env, "invalid size of register fill\n"); 4331 return -EACCES; 4332 } 4333 4334 if (size <= spill_size && 4335 bpf_stack_narrow_access_ok(off, size, spill_size)) { 4336 if (dst_regno < 0) 4337 return 0; 4338 4339 if (env->bpf_capable && size == 4 && spill_size == 4 && 4340 get_reg_width(reg) <= 32) 4341 /* Ensure stack slot has an ID to build a relation 4342 * with the destination register on fill. 4343 */ 4344 assign_scalar_id_before_mov(env, reg); 4345 state->regs[dst_regno] = *reg; 4346 4347 /* Break the relation on a narrowing fill. 4348 * coerce_reg_to_size will adjust the boundaries. 4349 */ 4350 if (get_reg_width(reg) > size * BITS_PER_BYTE) 4351 clear_scalar_id(&state->regs[dst_regno]); 4352 } else { 4353 int spill_cnt = 0, zero_cnt = 0; 4354 4355 for (i = 0; i < size; i++) { 4356 type = stype[(slot - i) % BPF_REG_SIZE]; 4357 if (type == STACK_SPILL) { 4358 spill_cnt++; 4359 continue; 4360 } 4361 if (type == STACK_MISC) 4362 continue; 4363 if (type == STACK_ZERO) { 4364 zero_cnt++; 4365 continue; 4366 } 4367 if (type == STACK_INVALID && env->allow_uninit_stack) 4368 continue; 4369 if (type == STACK_POISON) { 4370 verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n", 4371 off, i, size); 4372 } else { 4373 verbose(env, "invalid read from stack off %d+%d size %d\n", 4374 off, i, size); 4375 bpf_diag_stack_read_invalid(env, off, i, size, type); 4376 } 4377 return -EACCES; 4378 } 4379 4380 if (dst_regno < 0) 4381 return 0; 4382 4383 if (spill_cnt == size && 4384 tnum_is_const(reg->var_off) && reg->var_off.value == 0) { 4385 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4386 /* this IS register fill, so keep insn_flags */ 4387 } else if (zero_cnt == size) { 4388 /* similarly to mark_reg_stack_read(), preserve zeroes */ 4389 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4390 insn_flags = 0; /* not restoring original register state */ 4391 } else { 4392 err = mark_reg_stack_read(env, reg_state, off, off + size, 4393 dst_regno); 4394 if (err) 4395 return err; 4396 insn_flags = 0; /* not restoring original register state */ 4397 } 4398 } 4399 } else if (dst_regno >= 0) { 4400 /* restore register state from stack */ 4401 if (env->bpf_capable) 4402 /* Ensure stack slot has an ID to build a relation 4403 * with the destination register on fill. 4404 */ 4405 assign_scalar_id_before_mov(env, reg); 4406 state->regs[dst_regno] = *reg; 4407 /* mark reg as written since spilled pointer state likely 4408 * has its liveness marks cleared by is_state_visited() 4409 * which resets stack/reg liveness for state transitions 4410 */ 4411 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 4412 /* If dst_regno==-1, the caller is asking us whether 4413 * it is acceptable to use this value as a SCALAR_VALUE 4414 * (e.g. for XADD). 4415 * We must not allow unprivileged callers to do that 4416 * with spilled pointers. 4417 */ 4418 verbose(env, "leaking pointer from stack off %d\n", 4419 off); 4420 return -EACCES; 4421 } 4422 } else { 4423 for (i = 0; i < size; i++) { 4424 type = stype[(slot - i) % BPF_REG_SIZE]; 4425 if (type == STACK_MISC) 4426 continue; 4427 if (type == STACK_ZERO) 4428 continue; 4429 if (type == STACK_INVALID && env->allow_uninit_stack) 4430 continue; 4431 if (type == STACK_POISON) { 4432 verbose(env, "reading from stack off %d+%d size %d, slot poisoned by dead code elimination\n", 4433 off, i, size); 4434 } else { 4435 verbose(env, "invalid read from stack off %d+%d size %d\n", 4436 off, i, size); 4437 bpf_diag_stack_read_invalid(env, off, i, size, type); 4438 } 4439 return -EACCES; 4440 } 4441 if (dst_regno >= 0) { 4442 err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 4443 if (err) 4444 return err; 4445 } 4446 insn_flags = 0; /* we are not restoring spilled register */ 4447 } 4448 if (insn_flags) 4449 return bpf_push_jmp_history(env, env->cur_state, insn_flags, 4450 hist_spi, hist_frame, NULL, 0); 4451 return 0; 4452 } 4453 4454 enum bpf_access_src { 4455 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 4456 ACCESS_HELPER = 2, /* the access is performed by a helper */ 4457 }; 4458 4459 static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 4460 argno_t argno, int off, int access_size, 4461 bool zero_size_allowed, 4462 enum bpf_access_type type, 4463 struct bpf_call_arg_meta *meta); 4464 4465 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 4466 { 4467 return cur_regs(env) + regno; 4468 } 4469 4470 /* Read the stack at 'reg + off' and put the result into the register 4471 * 'dst_regno'. 4472 * 'off' includes the pointer register's fixed offset(i.e. 'reg->off'), 4473 * but not its variable offset. 4474 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 4475 * 4476 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 4477 * filling registers (i.e. reads of spilled register cannot be detected when 4478 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 4479 * SCALAR_VALUE. That's why we assert that the 'reg' has a variable 4480 * offset; for a fixed offset check_stack_read_fixed_off should be used 4481 * instead. 4482 */ 4483 static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 4484 argno_t ptr_argno, int off, int size, int dst_regno) 4485 { 4486 struct bpf_func_state *ptr_state = bpf_func(env, reg); 4487 int err; 4488 int min_off, max_off; 4489 4490 /* Note that we pass a NULL meta, so raw access will not be permitted. 4491 */ 4492 err = check_stack_range_initialized(env, reg, ptr_argno, off, size, 4493 false, BPF_READ, NULL); 4494 if (err) 4495 return err; 4496 4497 min_off = reg_smin(reg) + off; 4498 max_off = reg_smax(reg) + off; 4499 err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size, 4500 dst_regno); 4501 if (err) 4502 return err; 4503 return 0; 4504 } 4505 4506 /* check_stack_read dispatches to check_stack_read_fixed_off or 4507 * check_stack_read_var_off. 4508 * 4509 * The caller must ensure that the offset falls within the allocated stack 4510 * bounds. 4511 * 4512 * 'dst_regno' is a register which will receive the value from the stack. It 4513 * can be -1, meaning that the read value is not going to a register. 4514 */ 4515 static int check_stack_read(struct bpf_verifier_env *env, 4516 struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size, 4517 int dst_regno) 4518 { 4519 struct bpf_func_state *state = bpf_func(env, reg); 4520 int err; 4521 /* Some accesses are only permitted with a static offset. */ 4522 bool var_off = !tnum_is_const(reg->var_off); 4523 4524 /* The offset is required to be static when reads don't go to a 4525 * register, in order to not leak pointers (see 4526 * check_stack_read_fixed_off). 4527 */ 4528 if (dst_regno < 0 && var_off) { 4529 const char *reason; 4530 char tn_buf[48]; 4531 4532 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4533 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 4534 tn_buf, off, size); 4535 reason = bpf_diag_fmt( 4536 env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. " 4537 "This stack access requires a constant stack offset and a precise initialized range.", 4538 tn_buf, off, size); 4539 bpf_diag_memory( 4540 env, env->insn_idx, "variable-offset stack access", reason, 4541 "Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary."); 4542 return -EACCES; 4543 } 4544 /* Variable offset is prohibited for unprivileged mode for simplicity 4545 * since it requires corresponding support in Spectre masking for stack 4546 * ALU. See also retrieve_ptr_limit(). The check in 4547 * check_stack_access_for_ptr_arithmetic() called by 4548 * adjust_ptr_min_max_vals() prevents users from creating stack pointers 4549 * with variable offsets, therefore no check is required here. Further, 4550 * just checking it here would be insufficient as speculative stack 4551 * writes could still lead to unsafe speculative behaviour. 4552 */ 4553 if (!var_off) { 4554 off += reg->var_off.value; 4555 err = check_stack_read_fixed_off(env, state, off, size, 4556 dst_regno); 4557 } else { 4558 /* Variable offset stack reads need more conservative handling 4559 * than fixed offset ones. Note that dst_regno >= 0 on this 4560 * branch. 4561 */ 4562 err = check_stack_read_var_off(env, reg, ptr_argno, off, size, 4563 dst_regno); 4564 } 4565 return err; 4566 } 4567 4568 /* 4569 * An access through a slice that may be the buffer on the stack, see bpf_reg_state::buf, is 4570 * not seen as an access to the stack. That's fine as long as nothing is tracked in the buffer. 4571 * The output of helpers and kfuncs is not tracked. Here an insn stores to [min_off, max_off) 4572 * of frame @frameno, a dynptr, an iterator or irq flags are made there, or the frame is gone. 4573 */ 4574 static void invalidate_stack_slices(struct bpf_verifier_env *env, u32 frameno, 4575 int min_off, int max_off) 4576 { 4577 struct bpf_func_state *state; 4578 struct bpf_reg_state *reg; 4579 4580 if (!env->seen_stack_slice) 4581 return; 4582 4583 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 4584 if (base_type(reg->type) != PTR_TO_MEM || !reg->buf.size || 4585 reg->buf.frameno != frameno || reg->buf.off >= max_off || 4586 reg->buf.off + (int)reg->buf.size <= min_off) 4587 continue; 4588 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_SLICE_BUF); 4589 mark_reg_invalid(env, reg); 4590 })); 4591 } 4592 4593 /* check_stack_write dispatches to check_stack_write_fixed_off or 4594 * check_stack_write_var_off. 4595 * 4596 * 'reg' is the register used as a pointer into the stack. 4597 * 'value_regno' is the register whose value we're writing to the stack. It can 4598 * be -1, meaning that we're not writing from a register. 4599 * 4600 * The caller must ensure that the offset falls within the maximum stack size. 4601 */ 4602 static int check_stack_write(struct bpf_verifier_env *env, 4603 struct bpf_reg_state *reg, int off, int size, 4604 int value_regno, int insn_idx) 4605 { 4606 struct bpf_func_state *state = bpf_func(env, reg); 4607 int min_off = reg_smin(reg) + off; 4608 int max_off = reg_smax(reg) + off + size; 4609 int err; 4610 4611 if (tnum_is_const(reg->var_off)) { 4612 off += reg->var_off.value; 4613 err = check_stack_write_fixed_off(env, state, off, size, 4614 value_regno, insn_idx); 4615 } else { 4616 /* Variable offset stack reads need more conservative handling 4617 * than fixed offset ones. 4618 */ 4619 err = check_stack_write_var_off(env, state, 4620 reg, off, size, 4621 value_regno, insn_idx); 4622 } 4623 /* After the write is modelled: the insn may store a slice to its own buffer. */ 4624 invalidate_stack_slices(env, state->frameno, min_off, max_off); 4625 return err; 4626 } 4627 4628 /* 4629 * Write a value to the outgoing stack arg area. 4630 * off is a negative offset from r11 (e.g. -8 for arg6, -16 for arg7). 4631 */ 4632 static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state, 4633 int off, struct bpf_reg_state *value_reg) 4634 { 4635 int max_stack_arg_regs = MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS; 4636 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; 4637 int spi = -off / BPF_REG_SIZE - 1; 4638 struct bpf_reg_state *arg; 4639 int err; 4640 4641 if (spi >= max_stack_arg_regs) { 4642 verbose(env, "stack arg write offset %d exceeds max %d stack args\n", 4643 off, max_stack_arg_regs); 4644 return -EINVAL; 4645 } 4646 4647 err = grow_stack_arg_slots(env, state, spi + 1); 4648 if (err) 4649 return err; 4650 4651 /* Track the max outgoing stack arg slot count. */ 4652 if (spi + 1 > subprog->max_out_stack_arg_cnt) 4653 subprog->max_out_stack_arg_cnt = spi + 1; 4654 4655 arg = &state->stack_arg_regs[spi]; 4656 bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE); 4657 4658 if (value_reg) { 4659 state->stack_arg_regs[spi] = *value_reg; 4660 } else { 4661 /* BPF_ST: store immediate, treat as scalar */ 4662 arg->type = SCALAR_VALUE; 4663 __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); 4664 } 4665 bpf_diag_mod_end(env); 4666 state->no_stack_arg_load = true; 4667 return bpf_push_jmp_history(env, env->cur_state, 4668 INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); 4669 } 4670 4671 /* 4672 * Read a value from the incoming stack arg area. 4673 * off is a positive offset from r11 (e.g. +8 for arg6, +16 for arg7). 4674 */ 4675 static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state, 4676 int off, int dst_regno) 4677 { 4678 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; 4679 struct bpf_verifier_state *vstate = env->cur_state; 4680 int spi = off / BPF_REG_SIZE - 1; 4681 struct bpf_func_state *caller, *cur; 4682 struct bpf_reg_state *arg; 4683 4684 if (state->no_stack_arg_load) { 4685 verbose(env, "r11 load must be before any r11 store or call insn\n"); 4686 return -EINVAL; 4687 } 4688 4689 if (spi + 1 > bpf_in_stack_arg_cnt(subprog)) { 4690 verbose(env, "invalid read from stack arg off %d depth %d\n", 4691 off, bpf_in_stack_arg_cnt(subprog) * BPF_REG_SIZE); 4692 return -EACCES; 4693 } 4694 4695 caller = vstate->frame[vstate->curframe - 1]; 4696 arg = bpf_get_spilled_stack_arg(spi, caller); 4697 if (!arg) { 4698 verbose(env, "invalid read from uninitialized stack arg off %d\n", off); 4699 return -EACCES; 4700 } 4701 4702 cur = vstate->frame[vstate->curframe]; 4703 bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); 4704 cur->regs[dst_regno] = *arg; 4705 bpf_diag_mod_end(env); 4706 return bpf_push_jmp_history(env, env->cur_state, 4707 INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); 4708 } 4709 4710 static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) 4711 { 4712 struct bpf_func_state *caller = cur_func(env); 4713 int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS; 4714 4715 bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi); 4716 return mark_chain_precision_batch(env, env->cur_state); 4717 } 4718 4719 static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno) 4720 { 4721 int regno = reg_from_argno(argno); 4722 4723 if (regno >= 0) 4724 return mark_chain_precision(env, regno); 4725 return mark_stack_arg_precision(env, arg_idx_from_argno(argno)); 4726 } 4727 4728 static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, 4729 int nargs, const char *callee_name, const struct btf *btf, 4730 const struct btf_param *args) 4731 { 4732 int i, spi; 4733 4734 for (i = MAX_BPF_FUNC_REG_ARGS; i < nargs; i++) { 4735 spi = i - MAX_BPF_FUNC_REG_ARGS; 4736 if (spi >= caller->out_stack_arg_cnt || 4737 caller->stack_arg_regs[spi].type == NOT_INIT) { 4738 const char *arg_name = NULL; 4739 4740 if (args && args[i].name_off) 4741 arg_name = btf_name_by_offset(btf, args[i].name_off); 4742 verbose(env, "callee expects %d args, stack arg%d is not initialized\n", 4743 nargs, spi + 1); 4744 bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi, 4745 callee_name, arg_name); 4746 return -EFAULT; 4747 } 4748 } 4749 4750 return 0; 4751 } 4752 4753 static struct bpf_reg_state *get_func_arg_reg(struct bpf_func_state *caller, 4754 struct bpf_reg_state *regs, int arg) 4755 { 4756 if (arg < MAX_BPF_FUNC_REG_ARGS) 4757 return ®s[arg + 1]; 4758 4759 return &caller->stack_arg_regs[arg - MAX_BPF_FUNC_REG_ARGS]; 4760 } 4761 4762 static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 4763 int off, int size, enum bpf_access_type type) 4764 { 4765 struct bpf_map *map = reg->map_ptr; 4766 u32 cap = bpf_map_flags_to_cap(map); 4767 4768 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 4769 verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n", 4770 map->value_size, reg_smin(reg) + off, size); 4771 return -EACCES; 4772 } 4773 4774 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 4775 verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n", 4776 map->value_size, reg_smin(reg) + off, size); 4777 return -EACCES; 4778 } 4779 4780 return 0; 4781 } 4782 4783 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 4784 static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 4785 int off, int size, u32 mem_size, 4786 bool zero_size_allowed) 4787 { 4788 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 4789 4790 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 4791 return 0; 4792 4793 switch (reg->type) { 4794 case PTR_TO_MAP_KEY: 4795 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 4796 mem_size, off, size); 4797 break; 4798 case PTR_TO_MAP_VALUE: 4799 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 4800 mem_size, off, size); 4801 break; 4802 case PTR_TO_PACKET: 4803 case PTR_TO_PACKET_META: 4804 case PTR_TO_PACKET_END: 4805 verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n", 4806 off, size, reg_arg_name(env, argno), reg->id, off, mem_size); 4807 break; 4808 case PTR_TO_CTX: 4809 verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n", 4810 mem_size, off, size); 4811 break; 4812 case PTR_TO_MEM: 4813 default: 4814 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 4815 mem_size, off, size); 4816 } 4817 4818 return -EACCES; 4819 } 4820 4821 /* check read/write into a memory region with possible variable offset */ 4822 static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 4823 int off, int size, u32 mem_size, 4824 bool zero_size_allowed) 4825 { 4826 const char *proof = ""; 4827 const char *start; 4828 s64 max_start, max_end; 4829 int err; 4830 4831 /* We may have adjusted the register pointing to memory region, so we 4832 * need to try adding each of min_value and max_value to off 4833 * to make sure our theoretical access will be safe. 4834 * 4835 * The minimum value is only important with signed 4836 * comparisons where we can't assume the floor of a 4837 * value is 0. If we are using signed variables for our 4838 * index'es we need to make sure that whatever we use 4839 * will have a set floor within our range. 4840 */ 4841 if (reg_smin(reg) < 0 && 4842 (reg_smin(reg) == S64_MIN || 4843 (off + reg_smin(reg) != (s64)(s32)(off + reg_smin(reg))) || 4844 reg_smin(reg) + off < 0)) { 4845 verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4846 reg_arg_name(env, argno)); 4847 err = -EACCES; 4848 if (bpf_diag_enabled(env)) { 4849 start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); 4850 proof = bpf_diag_fmt( 4851 env, "the minimal bound for a memory access is a negative value: %s", 4852 start); 4853 } 4854 goto report_error; 4855 } 4856 4857 err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, 4858 mem_size, zero_size_allowed); 4859 if (err) { 4860 verbose(env, "%s min value is outside of the allowed memory range\n", 4861 reg_arg_name(env, argno)); 4862 if (bpf_diag_enabled(env)) { 4863 start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); 4864 proof = bpf_diag_fmt( 4865 env, "the minimal bound for a memory access is %s and is outside of the object of size %u", 4866 start, mem_size); 4867 } 4868 goto report_error; 4869 } 4870 4871 /* If we haven't set a max value then we need to bail since we can't be 4872 * sure we won't do bad things. 4873 * If reg_umax(reg) + off could overflow, treat that as unbounded too. 4874 */ 4875 if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { 4876 verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", 4877 reg_arg_name(env, argno)); 4878 err = -EACCES; 4879 if (bpf_diag_enabled(env)) 4880 proof = bpf_diag_fmt( 4881 env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u", 4882 reg_umax(reg), BPF_MAX_VAR_OFF); 4883 goto report_error; 4884 } 4885 4886 err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, 4887 mem_size, zero_size_allowed); 4888 if (err) { 4889 verbose(env, "%s max value is outside of the allowed memory range\n", 4890 reg_arg_name(env, argno)); 4891 if (bpf_diag_enabled(env)) { 4892 max_start = (s64)reg_umax(reg) + off; 4893 max_end = max_start + size; 4894 proof = bpf_diag_fmt( 4895 env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u", 4896 max_end, max_start, size, mem_size); 4897 } 4898 goto report_error; 4899 } 4900 4901 return 0; 4902 4903 report_error: 4904 bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno), 4905 reg_arg_name(env, argno), reg_type_str(env, reg->type), proof, 4906 off, size, mem_size, reg); 4907 return err; 4908 } 4909 4910 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 4911 const struct bpf_reg_state *reg, argno_t argno, 4912 bool fixed_off_ok) 4913 { 4914 /* Access to this pointer-typed register or passing it to a helper 4915 * is only allowed in its original, unmodified form. 4916 */ 4917 4918 if (!tnum_is_const(reg->var_off)) { 4919 char tn_buf[48]; 4920 4921 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4922 verbose(env, "variable %s access var_off=%s disallowed\n", 4923 reg_type_str(env, reg->type), tn_buf); 4924 return -EACCES; 4925 } 4926 4927 if (reg_smin(reg) < 0) { 4928 verbose(env, "negative offset %s ptr %s off=%lld disallowed\n", 4929 reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); 4930 return -EACCES; 4931 } 4932 4933 if (!fixed_off_ok && reg->var_off.value != 0) { 4934 verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", 4935 reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); 4936 bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno), 4937 reg_arg_name(env, argno), reg, 4938 BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value); 4939 return -EACCES; 4940 } 4941 4942 return 0; 4943 } 4944 4945 static int check_ptr_off_reg(struct bpf_verifier_env *env, 4946 const struct bpf_reg_state *reg, int regno) 4947 { 4948 return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); 4949 } 4950 4951 static int map_kptr_match_type(struct bpf_verifier_env *env, 4952 struct btf_field *kptr_field, 4953 struct bpf_reg_state *reg, u32 regno) 4954 { 4955 const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 4956 int perm_flags; 4957 const char *reg_name = ""; 4958 4959 if (base_type(reg->type) != PTR_TO_BTF_ID) 4960 goto bad_type; 4961 4962 if (btf_is_kernel(reg->btf)) { 4963 perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU; 4964 4965 /* Only unreferenced case accepts untrusted pointers */ 4966 if (kptr_field->type == BPF_KPTR_UNREF) 4967 perm_flags |= PTR_UNTRUSTED; 4968 } else { 4969 perm_flags = PTR_MAYBE_NULL | MEM_ALLOC; 4970 if (kptr_field->type == BPF_KPTR_PERCPU) 4971 perm_flags |= MEM_PERCPU; 4972 } 4973 4974 if (type_flag(reg->type) & ~perm_flags) 4975 goto bad_type; 4976 4977 /* 4978 * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value 4979 * stored in it must carry the same flag. 4980 */ 4981 if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU)) 4982 goto bad_type; 4983 4984 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 4985 reg_name = btf_type_name(reg->btf, reg->btf_id); 4986 4987 /* For ref_ptr case, release function check should ensure we get one 4988 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 4989 * normal store of unreferenced kptr, we must ensure var_off is zero. 4990 * Since ref_ptr cannot be accessed directly by BPF insns, check for 4991 * reg->id is not needed here. 4992 */ 4993 if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true)) 4994 return -EACCES; 4995 4996 /* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and 4997 * we also need to take into account the reg->var_off. 4998 * 4999 * We want to support cases like: 5000 * 5001 * struct foo { 5002 * struct bar br; 5003 * struct baz bz; 5004 * }; 5005 * 5006 * struct foo *v; 5007 * v = func(); // PTR_TO_BTF_ID 5008 * val->foo = v; // reg->var_off is zero, btf and btf_id match type 5009 * val->bar = &v->br; // reg->var_off is still zero, but we need to retry with 5010 * // first member type of struct after comparison fails 5011 * val->baz = &v->bz; // reg->var_off is non-zero, so struct needs to be walked 5012 * // to match type 5013 * 5014 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->var_off 5015 * is zero. We must also ensure that btf_struct_ids_match does not walk 5016 * the struct to match type against first member of struct, i.e. reject 5017 * second case from above. Hence, when type is BPF_KPTR_REF, we set 5018 * strict mode to true for type match. 5019 */ 5020 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, 5021 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 5022 kptr_field->type != BPF_KPTR_UNREF, 5023 !type_is_alloc(reg->type))) 5024 goto bad_type; 5025 return 0; 5026 bad_type: 5027 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 5028 reg_type_str(env, reg->type), reg_name); 5029 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 5030 if (kptr_field->type == BPF_KPTR_UNREF) 5031 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 5032 targ_name); 5033 else 5034 verbose(env, "\n"); 5035 return -EINVAL; 5036 } 5037 5038 static bool in_sleepable(struct bpf_verifier_env *env) 5039 { 5040 return env->cur_state->in_sleepable; 5041 } 5042 5043 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() 5044 * can dereference RCU protected pointers and result is PTR_TRUSTED. 5045 */ 5046 static bool in_rcu_cs(struct bpf_verifier_env *env) 5047 { 5048 return env->cur_state->active_rcu_locks || 5049 env->cur_state->active_preempt_locks || 5050 env->cur_state->active_locks || 5051 env->cur_state->active_irq_id || 5052 !in_sleepable(env); 5053 } 5054 5055 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */ 5056 BTF_SET_START(rcu_protected_types) 5057 #ifdef CONFIG_NET 5058 BTF_ID(struct, prog_test_ref_kfunc) 5059 #endif 5060 #ifdef CONFIG_CGROUPS 5061 BTF_ID(struct, cgroup) 5062 #endif 5063 #ifdef CONFIG_BPF_JIT 5064 BTF_ID(struct, bpf_cpumask) 5065 #endif 5066 BTF_ID(struct, task_struct) 5067 #ifdef CONFIG_BPF_CRYPTO 5068 BTF_ID(struct, bpf_crypto_ctx) 5069 #endif 5070 #ifdef CONFIG_INET 5071 BTF_ID(struct, bpf_ksock) 5072 #endif 5073 BTF_SET_END(rcu_protected_types) 5074 5075 static bool rcu_protected_object(const struct btf *btf, u32 btf_id) 5076 { 5077 if (!btf_is_kernel(btf)) 5078 return true; 5079 return btf_id_set_contains(&rcu_protected_types, btf_id); 5080 } 5081 5082 static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field) 5083 { 5084 struct btf_struct_meta *meta; 5085 5086 if (btf_is_kernel(kptr_field->kptr.btf)) 5087 return NULL; 5088 5089 meta = btf_find_struct_meta(kptr_field->kptr.btf, 5090 kptr_field->kptr.btf_id); 5091 5092 return meta ? meta->record : NULL; 5093 } 5094 5095 static bool rcu_safe_kptr(const struct btf_field *field) 5096 { 5097 const struct btf_field_kptr *kptr = &field->kptr; 5098 5099 return field->type == BPF_KPTR_PERCPU || 5100 (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id)); 5101 } 5102 5103 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field) 5104 { 5105 struct btf_record *rec; 5106 u32 ret; 5107 5108 ret = PTR_MAYBE_NULL; 5109 if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { 5110 ret |= MEM_RCU; 5111 if (kptr_field->type == BPF_KPTR_PERCPU) 5112 ret |= MEM_PERCPU; 5113 else if (!btf_is_kernel(kptr_field->kptr.btf)) 5114 ret |= MEM_ALLOC; 5115 5116 rec = kptr_pointee_btf_record(kptr_field); 5117 if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE)) 5118 ret |= NON_OWN_REF; 5119 } else { 5120 ret |= PTR_UNTRUSTED; 5121 } 5122 5123 return ret; 5124 } 5125 5126 static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno, 5127 struct btf_field *field) 5128 { 5129 struct bpf_reg_state *reg; 5130 const struct btf_type *t; 5131 5132 t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id); 5133 mark_reg_known_zero(env, cur_regs(env), regno); 5134 reg = reg_state(env, regno); 5135 reg->type = PTR_TO_MEM | PTR_MAYBE_NULL; 5136 reg->mem_size = t->size; 5137 reg->id = ++env->id_gen; 5138 5139 return 0; 5140 } 5141 5142 static int check_map_kptr_access(struct bpf_verifier_env *env, 5143 int value_regno, int insn_idx, 5144 struct btf_field *kptr_field) 5145 { 5146 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 5147 int class = BPF_CLASS(insn->code); 5148 struct bpf_reg_state *val_reg; 5149 int ret; 5150 5151 /* Things we already checked for in check_map_access and caller: 5152 * - Reject cases where variable offset may touch kptr 5153 * - size of access (must be BPF_DW) 5154 * - tnum_is_const(reg->var_off) 5155 * - kptr_field->offset == off + reg->var_off.value 5156 */ 5157 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 5158 if (BPF_MODE(insn->code) != BPF_MEM) { 5159 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 5160 return -EACCES; 5161 } 5162 5163 /* We only allow loading referenced kptr, since it will be marked as 5164 * untrusted, similar to unreferenced kptr. 5165 */ 5166 if (class != BPF_LDX && 5167 (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) { 5168 verbose(env, "store to referenced kptr disallowed\n"); 5169 return -EACCES; 5170 } 5171 if (class != BPF_LDX && kptr_field->type == BPF_UPTR) { 5172 verbose(env, "store to uptr disallowed\n"); 5173 return -EACCES; 5174 } 5175 5176 if (class == BPF_LDX) { 5177 if (kptr_field->type == BPF_UPTR) 5178 return mark_uptr_ld_reg(env, value_regno, kptr_field); 5179 5180 /* We can simply mark the value_regno receiving the pointer 5181 * value from map as PTR_TO_BTF_ID, with the correct type. 5182 */ 5183 ret = mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, 5184 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 5185 btf_ld_kptr_type(env, kptr_field)); 5186 if (ret < 0) 5187 return ret; 5188 } else if (class == BPF_STX) { 5189 val_reg = reg_state(env, value_regno); 5190 if (bpf_register_is_null(val_reg)) { 5191 /* 5192 * This store is valid only because the scalar is known to be 5193 * zero. Mark it precise so another scalar cannot be pruned 5194 * against this state. 5195 */ 5196 return mark_chain_precision(env, value_regno); 5197 } 5198 if (map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 5199 return -EACCES; 5200 } else if (class == BPF_ST) { 5201 if (insn->imm) { 5202 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 5203 kptr_field->offset); 5204 return -EACCES; 5205 } 5206 } else { 5207 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 5208 return -EACCES; 5209 } 5210 return 0; 5211 } 5212 5213 /* 5214 * Return the size of the memory region accessible from a pointer to map value. 5215 * For INSN_ARRAY maps whole bpf_insn_array->ips array is accessible. 5216 */ 5217 static u32 map_mem_size(const struct bpf_map *map) 5218 { 5219 if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) 5220 return map->max_entries * sizeof(long); 5221 5222 return map->value_size; 5223 } 5224 5225 /* check read/write into a map element with possible variable offset */ 5226 static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 5227 int off, int size, bool zero_size_allowed, 5228 enum bpf_access_src src) 5229 { 5230 struct bpf_map *map = reg->map_ptr; 5231 u32 mem_size = map_mem_size(map); 5232 struct btf_record *rec; 5233 int err, i; 5234 5235 err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed); 5236 if (err) 5237 return err; 5238 5239 if (IS_ERR_OR_NULL(map->record)) 5240 return 0; 5241 rec = map->record; 5242 for (i = 0; i < rec->cnt; i++) { 5243 struct btf_field *field = &rec->fields[i]; 5244 u32 p = field->offset; 5245 5246 /* If any part of a field can be touched by load/store, reject 5247 * this program. To check that [x1, x2) overlaps with [y1, y2), 5248 * it is sufficient to check x1 < y2 && y1 < x2. 5249 */ 5250 if (reg_smin(reg) + off < p + field->size && 5251 p < reg_umax(reg) + off + size) { 5252 switch (field->type) { 5253 case BPF_KPTR_UNREF: 5254 case BPF_KPTR_REF: 5255 case BPF_KPTR_PERCPU: 5256 case BPF_UPTR: 5257 if (src != ACCESS_DIRECT) { 5258 verbose(env, "%s cannot be accessed indirectly by helper\n", 5259 btf_field_type_name(field->type)); 5260 return -EACCES; 5261 } 5262 if (!tnum_is_const(reg->var_off)) { 5263 verbose(env, "%s access cannot have variable offset\n", 5264 btf_field_type_name(field->type)); 5265 return -EACCES; 5266 } 5267 if (p != off + reg->var_off.value) { 5268 verbose(env, "%s access misaligned expected=%u off=%llu\n", 5269 btf_field_type_name(field->type), 5270 p, off + reg->var_off.value); 5271 return -EACCES; 5272 } 5273 if (size != bpf_size_to_bytes(BPF_DW)) { 5274 verbose(env, "%s access size must be BPF_DW\n", 5275 btf_field_type_name(field->type)); 5276 return -EACCES; 5277 } 5278 break; 5279 default: 5280 verbose(env, "%s cannot be accessed directly by load/store\n", 5281 btf_field_type_name(field->type)); 5282 return -EACCES; 5283 } 5284 } 5285 } 5286 return 0; 5287 } 5288 5289 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 5290 const struct bpf_call_arg_meta *meta, 5291 enum bpf_access_type t) 5292 { 5293 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 5294 5295 switch (prog_type) { 5296 /* Program types only with direct read access go here! */ 5297 case BPF_PROG_TYPE_LWT_IN: 5298 case BPF_PROG_TYPE_LWT_OUT: 5299 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 5300 case BPF_PROG_TYPE_SK_REUSEPORT: 5301 case BPF_PROG_TYPE_FLOW_DISSECTOR: 5302 case BPF_PROG_TYPE_CGROUP_SKB: 5303 if (t == BPF_WRITE) 5304 return false; 5305 fallthrough; 5306 5307 /* Program types with direct read + write access go here! */ 5308 case BPF_PROG_TYPE_SCHED_CLS: 5309 case BPF_PROG_TYPE_SCHED_ACT: 5310 case BPF_PROG_TYPE_XDP: 5311 case BPF_PROG_TYPE_LWT_XMIT: 5312 case BPF_PROG_TYPE_SK_SKB: 5313 case BPF_PROG_TYPE_SK_MSG: 5314 if (meta && !meta->btf && meta->func_id) 5315 return meta->fn->pkt_access; 5316 5317 if (t == BPF_WRITE) 5318 env->seen_direct_write = true; 5319 return true; 5320 5321 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 5322 if (t == BPF_WRITE) 5323 env->seen_direct_write = true; 5324 5325 return true; 5326 5327 default: 5328 return false; 5329 } 5330 } 5331 5332 static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, 5333 int size, bool zero_size_allowed) 5334 { 5335 int err; 5336 5337 if (reg->range < 0) { 5338 verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno)); 5339 return -EINVAL; 5340 } 5341 5342 err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed); 5343 if (err) 5344 return err; 5345 5346 /* __check_mem_access has made sure "off + size - 1" is within u16. 5347 * reg_umax(reg) can't be bigger than MAX_PACKET_OFF which is 0xffff, 5348 * otherwise find_good_pkt_pointers would have refused to set range info 5349 * that __check_mem_access would have rejected this pkt access. 5350 * Therefore, "off + reg_umax(reg) + size - 1" won't overflow u32. 5351 */ 5352 env->prog->aux->max_pkt_offset = 5353 max_t(u32, env->prog->aux->max_pkt_offset, 5354 off + reg_umax(reg) + size - 1); 5355 5356 return 0; 5357 } 5358 5359 static bool is_var_ctx_off_allowed(struct bpf_prog *prog) 5360 { 5361 return resolve_prog_type(prog) == BPF_PROG_TYPE_SYSCALL; 5362 } 5363 5364 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 5365 static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 5366 enum bpf_access_type t, struct bpf_insn_access_aux *info) 5367 { 5368 if (env->ops->is_valid_access && 5369 env->ops->is_valid_access(off, size, t, env->prog, info)) { 5370 /* A non zero info.ctx_field_size indicates that this field is a 5371 * candidate for later verifier transformation to load the whole 5372 * field and then apply a mask when accessed with a narrower 5373 * access than actual ctx access size. A zero info.ctx_field_size 5374 * will only allow for whole field access and rejects any other 5375 * type of narrower access. 5376 */ 5377 if (base_type(info->reg_type) == PTR_TO_BTF_ID) { 5378 if (info->ref_id && 5379 !find_reference_state(env->cur_state, info->ref_id)) { 5380 verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n", 5381 off); 5382 return -EACCES; 5383 } 5384 } else { 5385 env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size; 5386 } 5387 /* remember the offset of last byte accessed in ctx */ 5388 if (env->prog->aux->max_ctx_offset < off + size) 5389 env->prog->aux->max_ctx_offset = off + size; 5390 return 0; 5391 } 5392 5393 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 5394 return -EACCES; 5395 } 5396 5397 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, 5398 int off, int access_size, enum bpf_access_type t, 5399 struct bpf_insn_access_aux *info) 5400 { 5401 /* 5402 * Program types that don't rewrite ctx accesses can safely 5403 * dereference ctx pointers with fixed offsets. 5404 */ 5405 bool var_off_ok = is_var_ctx_off_allowed(env->prog); 5406 bool fixed_off_ok = !env->ops->convert_ctx_access; 5407 int err; 5408 5409 if (var_off_ok) 5410 err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false); 5411 else 5412 err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok); 5413 if (err) 5414 return err; 5415 off += reg_umax(reg); 5416 5417 err = __check_ctx_access(env, insn_idx, off, access_size, t, info); 5418 if (err) 5419 verbose_linfo(env, insn_idx, "; "); 5420 return err; 5421 } 5422 5423 static int check_flow_keys_access(struct bpf_verifier_env *env, 5424 struct bpf_reg_state *reg, argno_t argno, 5425 int off, int size) 5426 { 5427 /* Only a constant offset is allowed here; fold it into off. */ 5428 if (!tnum_is_const(reg->var_off)) { 5429 char tn_buf[48]; 5430 5431 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5432 verbose(env, "%s invalid variable offset to flow keys: off=%d, var_off=%s\n", 5433 reg_arg_name(env, argno), off, tn_buf); 5434 return -EACCES; 5435 } 5436 off += reg->var_off.value; 5437 5438 if (size < 0 || off < 0 || 5439 (u64)off + size > sizeof(struct bpf_flow_keys)) { 5440 verbose(env, "invalid access to flow keys off=%d size=%d\n", 5441 off, size); 5442 return -EACCES; 5443 } 5444 return 0; 5445 } 5446 5447 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 5448 struct bpf_reg_state *reg, argno_t argno, int off, int size, 5449 enum bpf_access_type t) 5450 { 5451 struct bpf_insn_access_aux info = {}; 5452 bool valid; 5453 5454 if (reg_smin(reg) < 0) { 5455 verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", 5456 reg_arg_name(env, argno)); 5457 return -EACCES; 5458 } 5459 5460 switch (reg->type) { 5461 case PTR_TO_SOCK_COMMON: 5462 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 5463 break; 5464 case PTR_TO_SOCKET: 5465 valid = bpf_sock_is_valid_access(off, size, t, &info); 5466 break; 5467 case PTR_TO_TCP_SOCK: 5468 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 5469 break; 5470 case PTR_TO_XDP_SOCK: 5471 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 5472 break; 5473 default: 5474 valid = false; 5475 } 5476 5477 if (valid) { 5478 env->insn_aux_data[insn_idx].ctx_field_size = 5479 info.ctx_field_size; 5480 return 0; 5481 } 5482 5483 verbose(env, "%s invalid %s access off=%d size=%d\n", 5484 reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size); 5485 5486 return -EACCES; 5487 } 5488 5489 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 5490 { 5491 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 5492 } 5493 5494 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 5495 { 5496 const struct bpf_reg_state *reg = reg_state(env, regno); 5497 5498 return reg->type == PTR_TO_CTX; 5499 } 5500 5501 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 5502 { 5503 const struct bpf_reg_state *reg = reg_state(env, regno); 5504 5505 return type_is_sk_pointer(reg->type); 5506 } 5507 5508 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 5509 { 5510 const struct bpf_reg_state *reg = reg_state(env, regno); 5511 5512 return type_is_pkt_pointer(reg->type); 5513 } 5514 5515 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 5516 { 5517 const struct bpf_reg_state *reg = reg_state(env, regno); 5518 5519 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 5520 return reg->type == PTR_TO_FLOW_KEYS; 5521 } 5522 5523 static bool is_arena_reg(struct bpf_verifier_env *env, int regno) 5524 { 5525 const struct bpf_reg_state *reg = reg_state(env, regno); 5526 5527 return reg->type == PTR_TO_ARENA; 5528 } 5529 5530 static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, 5531 struct bpf_insn *insn) 5532 { 5533 const struct bpf_reg_state *reg = reg_state(env, regno); 5534 5535 /* 5536 * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the 5537 * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load 5538 * with no exception table entry, so a fault (e.g. NULL deref) crashes 5539 * the kernel instead of being handled. Reject the source pointer types 5540 * that would have needed that protection, the remaining ones stay 5541 * allowed. 5542 */ 5543 return insn->imm == BPF_LOAD_ACQ && bpf_may_fault_on_deref(reg->type); 5544 } 5545 5546 /* Return false if @regno contains a pointer whose type isn't supported for 5547 * atomic instruction @insn. 5548 */ 5549 static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, 5550 struct bpf_insn *insn) 5551 { 5552 if (is_ctx_reg(env, regno)) 5553 return false; 5554 if (is_pkt_reg(env, regno)) 5555 return false; 5556 if (is_flow_key_reg(env, regno)) 5557 return false; 5558 if (is_sk_reg(env, regno)) 5559 return false; 5560 if (is_arena_reg(env, regno)) 5561 return bpf_jit_supports_insn(insn, true); 5562 if (is_load_acq_unsafe(env, regno, insn)) 5563 return false; 5564 return true; 5565 } 5566 5567 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 5568 #ifdef CONFIG_NET 5569 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 5570 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 5571 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 5572 #endif 5573 [CONST_PTR_TO_MAP] = btf_bpf_map_id, 5574 }; 5575 5576 static bool is_trusted_reg_type(struct bpf_verifier_env *env, 5577 const struct bpf_reg_state *reg, 5578 enum bpf_reg_type type) 5579 { 5580 /* A referenced register is always trusted. */ 5581 if (reg_is_referenced(env, reg)) 5582 return true; 5583 5584 /* Types listed in the reg2btf_ids are always trusted */ 5585 if (reg2btf_ids[base_type(type)] && 5586 !bpf_type_has_unsafe_modifiers(type)) 5587 return true; 5588 5589 /* If a register is not referenced, it is trusted if it has the 5590 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 5591 * other type modifiers may be safe, but we elect to take an opt-in 5592 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 5593 * not. 5594 * 5595 * Eventually, we should make PTR_TRUSTED the single source of truth 5596 * for whether a register is trusted. 5597 */ 5598 return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS && 5599 !bpf_type_has_unsafe_modifiers(type); 5600 } 5601 5602 static bool is_trusted_reg(struct bpf_verifier_env *env, 5603 const struct bpf_reg_state *reg) 5604 { 5605 return is_trusted_reg_type(env, reg, reg->type); 5606 } 5607 5608 static bool is_rcu_reg(const struct bpf_reg_state *reg) 5609 { 5610 return reg->type & MEM_RCU; 5611 } 5612 5613 static void clear_trusted_flags(enum bpf_type_flag *flag) 5614 { 5615 *flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU); 5616 } 5617 5618 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 5619 const struct bpf_reg_state *reg, 5620 int off, int size, bool strict) 5621 { 5622 struct tnum reg_off; 5623 int ip_align; 5624 5625 /* Byte size accesses are always allowed. */ 5626 if (!strict || size == 1) 5627 return 0; 5628 5629 /* For platforms that do not have a Kconfig enabling 5630 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 5631 * NET_IP_ALIGN is universally set to '2'. And on platforms 5632 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 5633 * to this code only in strict mode where we want to emulate 5634 * the NET_IP_ALIGN==2 checking. Therefore use an 5635 * unconditional IP align value of '2'. 5636 */ 5637 ip_align = 2; 5638 5639 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + off)); 5640 if (!tnum_is_aligned(reg_off, size)) { 5641 char tn_buf[48]; 5642 5643 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5644 verbose(env, 5645 "misaligned packet access off %d+%s+%d size %d\n", 5646 ip_align, tn_buf, off, size); 5647 return -EACCES; 5648 } 5649 5650 return 0; 5651 } 5652 5653 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 5654 const struct bpf_reg_state *reg, 5655 const char *pointer_desc, 5656 int off, int size, bool strict) 5657 { 5658 struct tnum reg_off; 5659 5660 /* Byte size accesses are always allowed. */ 5661 if (!strict || size == 1) 5662 return 0; 5663 5664 reg_off = tnum_add(reg->var_off, tnum_const(off)); 5665 if (!tnum_is_aligned(reg_off, size)) { 5666 char tn_buf[48]; 5667 5668 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5669 verbose(env, "misaligned %saccess off %s+%d size %d\n", 5670 pointer_desc, tn_buf, off, size); 5671 return -EACCES; 5672 } 5673 5674 return 0; 5675 } 5676 5677 static int check_ptr_alignment(struct bpf_verifier_env *env, 5678 const struct bpf_reg_state *reg, int off, 5679 int size, bool strict_alignment_once) 5680 { 5681 bool strict = env->strict_alignment || strict_alignment_once; 5682 const char *pointer_desc = ""; 5683 5684 switch (reg->type) { 5685 case PTR_TO_PACKET: 5686 case PTR_TO_PACKET_META: 5687 /* Special case, because of NET_IP_ALIGN. Given metadata sits 5688 * right in front, treat it the very same way. 5689 */ 5690 return check_pkt_ptr_alignment(env, reg, off, size, strict); 5691 case PTR_TO_FLOW_KEYS: 5692 pointer_desc = "flow keys "; 5693 break; 5694 case PTR_TO_MAP_KEY: 5695 pointer_desc = "key "; 5696 break; 5697 case PTR_TO_MAP_VALUE: 5698 pointer_desc = "value "; 5699 if (reg->map_ptr->map_type == BPF_MAP_TYPE_INSN_ARRAY) 5700 strict = true; 5701 break; 5702 case PTR_TO_CTX: 5703 pointer_desc = "context "; 5704 break; 5705 case PTR_TO_STACK: 5706 pointer_desc = "stack "; 5707 /* The stack spill tracking logic in check_stack_write_fixed_off() 5708 * and check_stack_read_fixed_off() relies on stack accesses being 5709 * aligned. 5710 */ 5711 strict = true; 5712 break; 5713 case PTR_TO_SOCKET: 5714 pointer_desc = "sock "; 5715 break; 5716 case PTR_TO_SOCK_COMMON: 5717 pointer_desc = "sock_common "; 5718 break; 5719 case PTR_TO_TCP_SOCK: 5720 pointer_desc = "tcp_sock "; 5721 break; 5722 case PTR_TO_XDP_SOCK: 5723 pointer_desc = "xdp_sock "; 5724 break; 5725 case PTR_TO_ARENA: 5726 return 0; 5727 default: 5728 break; 5729 } 5730 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 5731 strict); 5732 } 5733 5734 static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog) 5735 { 5736 if (!bpf_jit_supports_private_stack()) 5737 return NO_PRIV_STACK; 5738 5739 /* bpf_prog_check_recur() checks all prog types that use bpf trampoline 5740 * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked 5741 * explicitly. 5742 */ 5743 switch (prog->type) { 5744 case BPF_PROG_TYPE_KPROBE: 5745 case BPF_PROG_TYPE_TRACEPOINT: 5746 case BPF_PROG_TYPE_PERF_EVENT: 5747 case BPF_PROG_TYPE_RAW_TRACEPOINT: 5748 return PRIV_STACK_ADAPTIVE; 5749 case BPF_PROG_TYPE_TRACING: 5750 case BPF_PROG_TYPE_LSM: 5751 case BPF_PROG_TYPE_STRUCT_OPS: 5752 if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog)) 5753 return PRIV_STACK_ADAPTIVE; 5754 fallthrough; 5755 default: 5756 break; 5757 } 5758 5759 return NO_PRIV_STACK; 5760 } 5761 5762 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) 5763 { 5764 if (env->prog->jit_requested) 5765 return round_up(stack_depth, 16); 5766 5767 /* round up to 32-bytes, since this is granularity 5768 * of interpreter stack size 5769 */ 5770 return round_up(max_t(u32, stack_depth, 1), 32); 5771 } 5772 5773 /* temporary state used for call frame depth calculation */ 5774 struct bpf_subprog_call_depth_info { 5775 int ret_insn; /* caller instruction where we return to. */ 5776 int caller; /* caller subprogram idx */ 5777 int frame; /* # of consecutive static call stack frames on top of stack */ 5778 int callx_insn; /* callx instruction whose callees are being walked */ 5779 int callx_next; /* next callee of callx_insn to walk */ 5780 bool via_callx; /* the subprogram is entered via callx */ 5781 }; 5782 5783 /* starting from main bpf function walk all instructions of the function 5784 * and recursively walk all callees that given function can call. 5785 * Ignore jump and exit insns. 5786 */ 5787 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, 5788 struct bpf_subprog_call_depth_info *dinfo, 5789 bool priv_stack_supported) 5790 { 5791 struct bpf_subprog_info *subprog = env->subprog_info; 5792 struct bpf_insn *insn = env->prog->insnsi; 5793 int depth = 0, frame = 0, i, subprog_end, subprog_depth; 5794 bool tail_call_reachable = false; 5795 int total; 5796 int tmp; 5797 5798 /* no caller idx */ 5799 dinfo[idx].caller = -1; 5800 dinfo[idx].via_callx = false; 5801 5802 i = subprog[idx].start; 5803 if (!priv_stack_supported) 5804 subprog[idx].priv_stack_mode = NO_PRIV_STACK; 5805 process_func: 5806 dinfo[idx].callx_insn = -1; 5807 if (subprog[idx].has_ld_abs) { 5808 for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { 5809 if (subprog[tmp].is_cb) { 5810 verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); 5811 return -EINVAL; 5812 } 5813 } 5814 } 5815 5816 /* protect against potential stack overflow that might happen when 5817 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 5818 * depth for such case down to 256 so that the worst case scenario 5819 * would result in 8k stack size (32 which is tailcall limit * 256 = 5820 * 8k). 5821 * 5822 * To get the idea what might happen, see an example: 5823 * func1 -> sub rsp, 128 5824 * subfunc1 -> sub rsp, 256 5825 * tailcall1 -> add rsp, 256 5826 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 5827 * subfunc2 -> sub rsp, 64 5828 * subfunc22 -> sub rsp, 128 5829 * tailcall2 -> add rsp, 128 5830 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 5831 * 5832 * tailcall will unwind the current stack frame but it will not get rid 5833 * of caller's stack as shown on the example above. 5834 */ 5835 if (idx && subprog[idx].has_tail_call && depth >= 256) { 5836 verbose(env, 5837 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 5838 depth); 5839 return -EACCES; 5840 } 5841 5842 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); 5843 if (IS_ENABLED(CONFIG_X86_64) && subprog[idx].stack_arg_cnt) { 5844 /* x86-64 uses R9 for both private stack frame pointer and arg6. */ 5845 subprog[idx].priv_stack_mode = NO_PRIV_STACK; 5846 } else if (priv_stack_supported) { 5847 /* Request private stack support only if the subprog stack 5848 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to 5849 * avoid jit penalty if the stack usage is small. 5850 */ 5851 if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN && 5852 subprog_depth >= BPF_PRIV_STACK_MIN_SIZE) 5853 subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE; 5854 } 5855 5856 if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { 5857 if (subprog_depth > env->max_stack_depth) 5858 env->max_stack_depth = subprog_depth; 5859 if (subprog_depth > env->stack_limit) { 5860 verbose(env, "stack size of subprog %d is %d. Too large\n", 5861 idx, subprog_depth); 5862 return -EACCES; 5863 } 5864 } else { 5865 depth += subprog_depth; 5866 if (depth > env->max_stack_depth) 5867 env->max_stack_depth = depth; 5868 if (depth > env->stack_limit) { 5869 total = 0; 5870 for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) 5871 total++; 5872 5873 verbose(env, "combined stack size of %d calls is %d. Too large\n", 5874 total, depth); 5875 return -EACCES; 5876 } 5877 } 5878 continue_func: 5879 subprog_end = subprog[idx + 1].start; 5880 for (; i < subprog_end; i++) { 5881 int next_insn, sidx; 5882 5883 if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && 5884 bpf_is_throw_kfunc(insn + i)) { 5885 bool err = false; 5886 5887 for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { 5888 if (subprog[tmp].is_cb) { 5889 err = true; 5890 break; 5891 } 5892 } 5893 if (!err) 5894 continue; 5895 verbose(env, 5896 "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n", 5897 i, idx); 5898 return -EINVAL; 5899 } 5900 5901 if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { 5902 /* 5903 * Walk the callees and the callbacks recorded by 5904 * the main verification pass one by one, returning to 5905 * this insn after each. 5906 */ 5907 if (dinfo[idx].callx_insn != i) { 5908 dinfo[idx].callx_insn = i; 5909 dinfo[idx].callx_next = 0; 5910 } 5911 sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); 5912 if (sidx < 0) 5913 continue; 5914 dinfo[idx].callx_next = sidx + 1; 5915 dinfo[idx].ret_insn = i; 5916 next_insn = subprog[sidx].start; 5917 } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { 5918 /* find the callee */ 5919 next_insn = i + insn[i].imm + 1; 5920 sidx = bpf_find_subprog(env, next_insn); 5921 if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) 5922 return -EFAULT; 5923 if (subprog[sidx].is_async_cb) { 5924 /* async callbacks don't increase bpf prog stack size unless called directly */ 5925 if (!bpf_pseudo_call(insn + i)) 5926 continue; 5927 if (subprog[sidx].is_exception_cb) { 5928 verbose(env, "insn %d cannot call exception cb directly", i); 5929 return -EINVAL; 5930 } 5931 } 5932 /* remember insn to return to */ 5933 dinfo[idx].ret_insn = i + 1; 5934 } else { 5935 continue; 5936 } 5937 5938 /* 5939 * sort_subprogs_topo() tolerates cycles in the call graph that 5940 * go through the address of a function being taken, since it 5941 * doesn't know what it is taken for. Such cycle is a recursion 5942 * unless it's an async callback, which are skipped above. 5943 * The main verification pass limits the depth of the recursion, 5944 * but it doesn't follow calls of global functions. 5945 */ 5946 for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { 5947 if (tmp != sidx) 5948 continue; 5949 verbose(env, "recursive call from %s() to %s()\n", 5950 bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); 5951 bpf_diag_program_structure( 5952 env, i, "recursive subprogram call", 5953 "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", 5954 "This call or callback would make the subprogram call graph recursive. " 5955 "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); 5956 return -EINVAL; 5957 } 5958 5959 /* store caller info for after we return from callee */ 5960 dinfo[idx].frame = frame; 5961 5962 /* push caller idx into callee's dinfo */ 5963 dinfo[sidx].caller = idx; 5964 dinfo[sidx].via_callx = bpf_is_callx(insn + i); 5965 5966 i = next_insn; 5967 5968 idx = sidx; 5969 if (!priv_stack_supported) 5970 subprog[idx].priv_stack_mode = NO_PRIV_STACK; 5971 5972 /* sync tail_call_reachable with callee state on entry */ 5973 tail_call_reachable = subprog[idx].has_tail_call; 5974 5975 frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; 5976 if (frame >= MAX_CALL_FRAMES) { 5977 verbose(env, "the call stack of %d frames is too deep !\n", 5978 frame); 5979 return -E2BIG; 5980 } 5981 goto process_func; 5982 } 5983 /* if tail call got detected across bpf2bpf calls then mark each of the 5984 * currently present subprog frames as tail call reachable subprogs; 5985 * this info will be utilized by JIT so that we will be preserving the 5986 * tail call counter throughout bpf2bpf calls combined with tailcalls 5987 */ 5988 if (tail_call_reachable) { 5989 for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { 5990 if (subprog[tmp].is_cb) { 5991 verbose(env, "cannot tail call within callback\n"); 5992 return -EINVAL; 5993 } 5994 if (subprog[tmp].stack_arg_cnt) { 5995 verbose(env, "tail_calls are not allowed in programs with stack args\n"); 5996 return -EINVAL; 5997 } 5998 /* 5999 * JITs pass tail call counter in a register that is 6000 * not available when the callee is called via callx. 6001 */ 6002 if (dinfo[tmp].via_callx) { 6003 verbose(env, "tail_calls are not allowed in functions called via callx\n"); 6004 return -EINVAL; 6005 } 6006 subprog[tmp].tail_call_reachable = true; 6007 } 6008 } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { 6009 verbose(env, "tail_calls are not allowed in programs with stack args\n"); 6010 return -EINVAL; 6011 } 6012 6013 if (subprog[0].tail_call_reachable) 6014 env->prog->aux->tail_call_reachable = true; 6015 6016 /* end of for() loop means the last insn of the 'subprog' 6017 * was reached. Doesn't matter whether it was JA or EXIT 6018 */ 6019 if (frame == 0 && dinfo[idx].caller < 0) 6020 return 0; 6021 if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE) 6022 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); 6023 6024 /* pop caller idx from callee */ 6025 idx = dinfo[idx].caller; 6026 6027 /* retrieve caller state from its frame */ 6028 frame = dinfo[idx].frame; 6029 i = dinfo[idx].ret_insn; 6030 6031 /* reset tail_call_reachable to the parent's actual state */ 6032 tail_call_reachable = subprog[idx].tail_call_reachable; 6033 6034 goto continue_func; 6035 } 6036 6037 static int check_max_stack_depth(struct bpf_verifier_env *env) 6038 { 6039 enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN; 6040 struct bpf_subprog_call_depth_info *dinfo; 6041 struct bpf_subprog_info *si = env->subprog_info; 6042 bool priv_stack_supported; 6043 int ret; 6044 6045 dinfo = kvzalloc_objs(*dinfo, env->subprog_cnt, GFP_KERNEL_ACCOUNT); 6046 if (!dinfo) 6047 return -ENOMEM; 6048 6049 for (int i = 0; i < env->subprog_cnt; i++) { 6050 if (si[i].has_tail_call) { 6051 priv_stack_mode = NO_PRIV_STACK; 6052 break; 6053 } 6054 } 6055 6056 if (priv_stack_mode == PRIV_STACK_UNKNOWN) 6057 priv_stack_mode = bpf_enable_priv_stack(env->prog); 6058 6059 /* All async_cb subprogs use normal kernel stack. If a particular 6060 * subprog appears in both main prog and async_cb subtree, that 6061 * subprog will use normal kernel stack to avoid potential nesting. 6062 * The reverse subprog traversal ensures when main prog subtree is 6063 * checked, the subprogs appearing in async_cb subtrees are already 6064 * marked as using normal kernel stack, so stack size checking can 6065 * be done properly. 6066 */ 6067 for (int i = env->subprog_cnt - 1; i >= 0; i--) { 6068 if (!i || si[i].is_async_cb) { 6069 priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE; 6070 ret = check_max_stack_depth_subprog(env, i, dinfo, 6071 priv_stack_supported); 6072 if (ret < 0) { 6073 kvfree(dinfo); 6074 return ret; 6075 } 6076 } 6077 } 6078 6079 for (int i = 0; i < env->subprog_cnt; i++) { 6080 if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) { 6081 env->prog->aux->jits_use_priv_stack = true; 6082 break; 6083 } 6084 } 6085 6086 kvfree(dinfo); 6087 6088 return 0; 6089 } 6090 6091 static int __check_buffer_access(struct bpf_verifier_env *env, 6092 const char *buf_info, 6093 const struct bpf_reg_state *reg, 6094 argno_t argno, int off, int size, 6095 u32 *access_end) 6096 { 6097 s64 start; 6098 6099 if (!tnum_is_const(reg->var_off)) { 6100 char tn_buf[48]; 6101 6102 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6103 verbose(env, 6104 "%s invalid variable buffer offset: off=%d, var_off=%s\n", 6105 reg_arg_name(env, argno), off, tn_buf); 6106 return -EACCES; 6107 } 6108 6109 start = (s64)reg->var_off.value + off; 6110 if (start < 0) { 6111 verbose(env, 6112 "%s invalid negative %s buffer offset: off=%d, var_off=%lld\n", 6113 reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value); 6114 return -EACCES; 6115 } 6116 6117 *access_end = start + size; 6118 return 0; 6119 } 6120 6121 static int check_tp_buffer_access(struct bpf_verifier_env *env, 6122 const struct bpf_reg_state *reg, 6123 argno_t argno, int off, int size) 6124 { 6125 u32 access_end; 6126 int err; 6127 6128 err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end); 6129 if (err) 6130 return err; 6131 6132 env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access); 6133 6134 return 0; 6135 } 6136 6137 static int check_buffer_access(struct bpf_verifier_env *env, 6138 const struct bpf_reg_state *reg, 6139 argno_t argno, int off, int size, 6140 bool zero_size_allowed, 6141 u32 *max_access, bool write) 6142 { 6143 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 6144 struct bpf_prog_aux *aux = env->prog->aux; 6145 u32 access_end; 6146 int err; 6147 6148 err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end); 6149 if (err) 6150 return err; 6151 6152 *max_access = max(access_end, *max_access); 6153 if (write && size) { 6154 if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write) 6155 aux->min_rdwr_write = access_end - size; 6156 aux->max_rdwr_write = max(access_end, aux->max_rdwr_write); 6157 } 6158 6159 return 0; 6160 } 6161 6162 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 6163 static void zext_32_to_64(struct bpf_reg_state *reg) 6164 { 6165 reg_step_check_unsigned(reg, 32); 6166 reg->var_off = tnum_subreg(reg->var_off); 6167 reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); 6168 } 6169 6170 /* truncate register to smaller size (in bytes) 6171 * must be called with size < BPF_REG_SIZE 6172 */ 6173 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 6174 { 6175 u64 mask = (1ULL << (size * 8)) - 1; 6176 6177 reg_step_check_unsigned(reg, size * 8); 6178 6179 /* clear high bits in bit representation */ 6180 reg->var_off = tnum_cast(reg->var_off, size); 6181 6182 /* fix arithmetic bounds */ 6183 if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) 6184 reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); 6185 else 6186 reg_set_urange64(reg, 0, mask); 6187 6188 /* If size is smaller than 32bit register the 32bit register 6189 * values are also truncated so we push 64-bit bounds into 6190 * 32-bit bounds. Above were truncated < 32-bits already. 6191 */ 6192 if (size < 4) 6193 __mark_reg32_unbounded(reg); 6194 6195 reg_bounds_sync(reg); 6196 } 6197 6198 static void set_sext64_default_val(struct bpf_reg_state *reg, int size) 6199 { 6200 if (size == 1) { 6201 reg_set_srange64(reg, S8_MIN, S8_MAX); 6202 reg_set_srange32(reg, S8_MIN, S8_MAX); 6203 } else if (size == 2) { 6204 reg_set_srange64(reg, S16_MIN, S16_MAX); 6205 reg_set_srange32(reg, S16_MIN, S16_MAX); 6206 } else { 6207 /* size == 4 */ 6208 reg_set_srange64(reg, S32_MIN, S32_MAX); 6209 reg_set_srange32(reg, S32_MIN, S32_MAX); 6210 } 6211 reg->var_off = tnum_unknown; 6212 } 6213 6214 static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) 6215 { 6216 s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; 6217 u64 top_smax_value, top_smin_value; 6218 u64 num_bits = size * 8; 6219 s64 limit = 1LL << (num_bits - 1); 6220 6221 /* If every value fits the signed N-bit range sign extension is a noop. */ 6222 if (reg_smin(reg) >= -limit && reg_smax(reg) < limit) 6223 return; 6224 6225 /* See commen at reg_step_check_unsigned(). */ 6226 reg_step_reset(reg); 6227 6228 if (tnum_is_const(reg->var_off)) { 6229 u64_cval = reg->var_off.value; 6230 if (size == 1) 6231 reg->var_off = tnum_const((s8)u64_cval); 6232 else if (size == 2) 6233 reg->var_off = tnum_const((s16)u64_cval); 6234 else 6235 /* size == 4 */ 6236 reg->var_off = tnum_const((s32)u64_cval); 6237 6238 u64_cval = reg->var_off.value; 6239 reg->r64 = cnum64_from_urange(u64_cval, u64_cval); 6240 reg->r32 = cnum32_from_urange((u32)u64_cval, (u32)u64_cval); 6241 return; 6242 } 6243 6244 top_smax_value = ((u64)reg_smax(reg) >> num_bits) << num_bits; 6245 top_smin_value = ((u64)reg_smin(reg) >> num_bits) << num_bits; 6246 6247 if (top_smax_value != top_smin_value) 6248 goto out; 6249 6250 /* find the s64_min and s64_min after sign extension */ 6251 if (size == 1) { 6252 init_s64_max = (s8)reg_smax(reg); 6253 init_s64_min = (s8)reg_smin(reg); 6254 } else if (size == 2) { 6255 init_s64_max = (s16)reg_smax(reg); 6256 init_s64_min = (s16)reg_smin(reg); 6257 } else { 6258 init_s64_max = (s32)reg_smax(reg); 6259 init_s64_min = (s32)reg_smin(reg); 6260 } 6261 6262 s64_max = max(init_s64_max, init_s64_min); 6263 s64_min = min(init_s64_max, init_s64_min); 6264 6265 /* both of s64_max/s64_min positive or negative */ 6266 if ((s64_max >= 0) == (s64_min >= 0)) { 6267 reg_set_srange64(reg, s64_min, s64_max); 6268 reg_set_srange32(reg, s64_min, s64_max); 6269 reg->var_off = tnum_range(s64_min, s64_max); 6270 return; 6271 } 6272 6273 out: 6274 set_sext64_default_val(reg, size); 6275 } 6276 6277 static void set_sext32_default_val(struct bpf_reg_state *reg, int size) 6278 { 6279 if (size == 1) 6280 reg_set_srange32(reg, S8_MIN, S8_MAX); 6281 else 6282 /* size == 2 */ 6283 reg_set_srange32(reg, S16_MIN, S16_MAX); 6284 reg->var_off = tnum_subreg(tnum_unknown); 6285 } 6286 6287 static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) 6288 { 6289 s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val; 6290 u32 top_smax_value, top_smin_value; 6291 u32 num_bits = size * 8; 6292 6293 /* 6294 * If value fits the non-negative half of the signed N-bit range, 6295 * sign extension is a noop. 6296 */ 6297 if (reg_umax(reg) < (1ULL << (num_bits - 1))) 6298 return; 6299 6300 /* See comment at reg_step_check_unsigned(). */ 6301 reg_step_reset(reg); 6302 6303 if (tnum_is_const(reg->var_off)) { 6304 u32_val = reg->var_off.value; 6305 if (size == 1) 6306 reg->var_off = tnum_const((s8)u32_val); 6307 else 6308 reg->var_off = tnum_const((s16)u32_val); 6309 6310 u32_val = reg->var_off.value; 6311 reg_set_srange32(reg, u32_val, u32_val); 6312 return; 6313 } 6314 6315 top_smax_value = ((u32)reg_s32_max(reg) >> num_bits) << num_bits; 6316 top_smin_value = ((u32)reg_s32_min(reg) >> num_bits) << num_bits; 6317 6318 if (top_smax_value != top_smin_value) 6319 goto out; 6320 6321 /* find the s32_min and s32_min after sign extension */ 6322 if (size == 1) { 6323 init_s32_max = (s8)reg_s32_max(reg); 6324 init_s32_min = (s8)reg_s32_min(reg); 6325 } else { 6326 /* size == 2 */ 6327 init_s32_max = (s16)reg_s32_max(reg); 6328 init_s32_min = (s16)reg_s32_min(reg); 6329 } 6330 s32_max = max(init_s32_max, init_s32_min); 6331 s32_min = min(init_s32_max, init_s32_min); 6332 6333 if ((s32_min >= 0) == (s32_max >= 0)) { 6334 reg_set_srange32(reg, s32_min, s32_max); 6335 reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max)); 6336 return; 6337 } 6338 6339 out: 6340 set_sext32_default_val(reg, size); 6341 } 6342 6343 bool bpf_map_is_rdonly(const struct bpf_map *map) 6344 { 6345 /* A map is considered read-only if the following condition are true: 6346 * 6347 * 1) BPF program side cannot change any of the map content. The 6348 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 6349 * and was set at map creation time. 6350 * 2) The map value(s) have been initialized from user space by a 6351 * loader and then "frozen", such that no new map update/delete 6352 * operations from syscall side are possible for the rest of 6353 * the map's lifetime from that point onwards. 6354 * 3) Any parallel/pending map update/delete operations from syscall 6355 * side have been completed. Only after that point, it's safe to 6356 * assume that map value(s) are immutable. 6357 */ 6358 return (map->map_flags & BPF_F_RDONLY_PROG) && 6359 READ_ONCE(map->frozen) && 6360 !bpf_map_write_active(map); 6361 } 6362 6363 int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, 6364 bool is_ldsx) 6365 { 6366 void *ptr; 6367 u64 addr; 6368 int err; 6369 6370 if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 6371 return -EINVAL; 6372 err = map->ops->map_direct_value_addr(map, &addr, off); 6373 if (err) 6374 return err; 6375 ptr = (void *)(long)addr + off; 6376 6377 switch (size) { 6378 case sizeof(u8): 6379 *val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr; 6380 break; 6381 case sizeof(u16): 6382 *val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr; 6383 break; 6384 case sizeof(u32): 6385 *val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr; 6386 break; 6387 case sizeof(u64): 6388 *val = *(u64 *)ptr; 6389 break; 6390 default: 6391 return -EINVAL; 6392 } 6393 return 0; 6394 } 6395 6396 static int cmp_func_ptrs(const void *_a, const void *_b) 6397 { 6398 const struct bpf_func_ptr *a = _a, *b = _b; 6399 6400 if (a->map != b->map) 6401 return a->map < b->map ? -1 : 1; 6402 if (a->map_off != b->map_off) 6403 return a->map_off < b->map_off ? -1 : 1; 6404 return 0; 6405 } 6406 6407 /* Find the first pointer to a function at or after 'off' in the value of 'map' */ 6408 static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) 6409 { 6410 u32 l = 0, r = env->func_ptr_cnt, m; 6411 struct bpf_func_ptr *p; 6412 6413 while (l < r) { 6414 m = l + (r - l) / 2; 6415 p = &env->func_ptrs[m]; 6416 if (p->map < map || (p->map == map && p->map_off < off)) 6417 l = m + 1; 6418 else 6419 r = m; 6420 } 6421 return l; 6422 } 6423 6424 /* 6425 * Return pointers to functions that overlap with 'size' bytes at offset 'off' 6426 * of the value of 'map' and their number in 'cnt'. 6427 */ 6428 struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, 6429 const struct bpf_map *map, 6430 u64 off, u64 size, u32 *cnt) 6431 { 6432 u32 first, last; 6433 6434 *cnt = 0; 6435 if (!env->func_ptr_cnt || !size) 6436 return NULL; 6437 6438 /* a pointer that starts up to 7 bytes before 'off' overlaps too */ 6439 first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); 6440 last = func_ptr_lower_bound(env, map, off + size); 6441 if (first >= last) 6442 return NULL; 6443 6444 *cnt = last - first; 6445 return &env->func_ptrs[first]; 6446 } 6447 6448 /* Return all pointers to functions in the value of 'map' */ 6449 struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, 6450 const struct bpf_map *map, u32 *cnt) 6451 { 6452 return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); 6453 } 6454 6455 /* instructions [off, off + len) replaced the instruction at 'off' */ 6456 void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) 6457 { 6458 struct bpf_func_ptr *p; 6459 u32 i; 6460 6461 if (len <= 1) 6462 return; 6463 6464 for (i = 0; i < env->func_ptr_cnt; i++) { 6465 p = &env->func_ptrs[i]; 6466 if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) 6467 continue; 6468 p->xlated_off += len - 1; 6469 } 6470 } 6471 6472 /* 6473 * Instructions [off, off + len) are about to be removed. It's called before 6474 * the starts of subprogs are adjusted. A subprog is gone when all of its 6475 * instructions are. Otherwise, e.g. when its first instruction is a nop, 6476 * it starts where the removed instructions did. 6477 */ 6478 void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) 6479 { 6480 struct bpf_func_ptr *p; 6481 int subprog; 6482 u32 i; 6483 6484 for (i = 0; i < env->func_ptr_cnt; i++) { 6485 p = &env->func_ptrs[i]; 6486 if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) 6487 continue; 6488 if (p->xlated_off >= off + len) { 6489 p->xlated_off -= len; 6490 continue; 6491 } 6492 subprog = bpf_find_subprog(env, p->xlated_off); 6493 if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) 6494 p->xlated_off = off; 6495 else 6496 p->xlated_off = BPF_FUNC_PTR_DELETED; 6497 } 6498 } 6499 6500 #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) 6501 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) 6502 #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) 6503 #define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type) __PASTE(__type, __safe_trusted_or_null) 6504 6505 /* 6506 * Allow list few fields as RCU trusted or full trusted. 6507 * This logic doesn't allow mix tagging and will be removed once GCC supports 6508 * btf_type_tag. 6509 */ 6510 6511 /* RCU trusted: these fields are trusted in RCU CS and never NULL */ 6512 BTF_TYPE_SAFE_RCU(struct task_struct) { 6513 const cpumask_t *cpus_ptr; 6514 struct css_set __rcu *cgroups; 6515 struct task_struct __rcu *real_parent; 6516 struct task_struct *group_leader; 6517 }; 6518 6519 BTF_TYPE_SAFE_RCU(struct cgroup) { 6520 /* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */ 6521 struct kernfs_node *kn; 6522 }; 6523 6524 BTF_TYPE_SAFE_RCU(struct css_set) { 6525 struct cgroup *dfl_cgrp; 6526 }; 6527 6528 BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state) { 6529 struct cgroup *cgroup; 6530 }; 6531 6532 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */ 6533 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) { 6534 struct file __rcu *exe_file; 6535 #ifdef CONFIG_MEMCG 6536 struct task_struct __rcu *owner; 6537 #endif 6538 }; 6539 6540 /* skb->sk, req->sk are not RCU protected, but we mark them as such 6541 * because bpf prog accessible sockets are SOCK_RCU_FREE. 6542 */ 6543 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) { 6544 struct sock *sk; 6545 }; 6546 6547 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) { 6548 struct sock *sk; 6549 }; 6550 6551 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */ 6552 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) { 6553 struct seq_file *seq; 6554 }; 6555 6556 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) { 6557 struct bpf_iter_meta *meta; 6558 struct task_struct *task; 6559 }; 6560 6561 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) { 6562 struct file *file; 6563 }; 6564 6565 BTF_TYPE_SAFE_TRUSTED(struct file) { 6566 struct inode *f_inode; 6567 }; 6568 6569 /* 6570 * The pointer fields in the sched_ext ops argument containers are pinned by the 6571 * callers for the duration of the ops calls and are never NULL. 6572 */ 6573 BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { 6574 #ifdef CONFIG_EXT_GROUP_SCHED 6575 struct cgroup *cgroup; 6576 #endif 6577 }; 6578 6579 BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { 6580 struct task_struct *task; 6581 }; 6582 6583 BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { 6584 struct sched_ext_ops *ops; 6585 }; 6586 6587 BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { 6588 struct sched_ext_ops *ops; 6589 }; 6590 6591 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { 6592 struct inode *d_inode; 6593 }; 6594 6595 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { 6596 struct mm_struct *mm; 6597 }; 6598 6599 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { 6600 struct sock *sk; 6601 }; 6602 6603 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct) { 6604 struct mm_struct *vm_mm; 6605 struct file *vm_file; 6606 }; 6607 6608 static bool type_is_rcu(struct bpf_verifier_env *env, 6609 struct bpf_reg_state *reg, 6610 const char *field_name, u32 btf_id) 6611 { 6612 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct)); 6613 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup)); 6614 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set)); 6615 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup_subsys_state)); 6616 6617 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); 6618 } 6619 6620 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, 6621 struct bpf_reg_state *reg, 6622 const char *field_name, u32 btf_id) 6623 { 6624 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct)); 6625 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff)); 6626 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock)); 6627 6628 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); 6629 } 6630 6631 static bool type_is_trusted(struct bpf_verifier_env *env, 6632 struct bpf_reg_state *reg, 6633 const char *field_name, u32 btf_id) 6634 { 6635 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta)); 6636 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); 6637 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); 6638 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); 6639 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); 6640 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); 6641 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); 6642 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); 6643 6644 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); 6645 } 6646 6647 static bool type_is_trusted_or_null(struct bpf_verifier_env *env, 6648 struct bpf_reg_state *reg, 6649 const char *field_name, u32 btf_id) 6650 { 6651 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); 6652 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); 6653 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); 6654 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); 6655 6656 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, 6657 "__safe_trusted_or_null"); 6658 } 6659 6660 /* 6661 * The member holds the kernel address of arena memory, which the rewritten 6662 * load turns into an arena pointer, see arena_member_fixup(). NULL needs no 6663 * tracking, PTR_TO_ARENA covers it. 6664 */ 6665 static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, 6666 int value_regno, enum bpf_type_flag flag, const char *tname, 6667 const char *field_name) 6668 { 6669 struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; 6670 struct bpf_insn *insn; 6671 6672 /* a helper reading the member as bytes needs no rewrite */ 6673 if (value_regno < 0) 6674 return 0; 6675 if (!env->prog->aux->arena) { 6676 verbose(env, 6677 "arena pointer member %s of struct %s requires a program with an associated arena\n", 6678 field_name, tname); 6679 return -EINVAL; 6680 } 6681 /* only a plain load or an acquire load is rewritten */ 6682 insn = &env->prog->insnsi[env->insn_idx]; 6683 if (BPF_CLASS(insn->code) != BPF_LDX && 6684 !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { 6685 verbose(env, 6686 "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", 6687 field_name, tname); 6688 return -EACCES; 6689 } 6690 6691 aux->arena_member = true; 6692 if (flag & PTR_MAYBE_NULL) 6693 aux->arena_member_nullable = true; 6694 mark_reg_unknown(env, regs, value_regno); 6695 regs[value_regno].type = PTR_TO_ARENA; 6696 return 0; 6697 } 6698 6699 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 6700 struct bpf_reg_state *regs, struct bpf_reg_state *reg, 6701 argno_t argno, int off, int size, 6702 enum bpf_access_type atype, 6703 int value_regno) 6704 { 6705 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 6706 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 6707 const char *field_name = NULL; 6708 enum bpf_type_flag flag = 0; 6709 u32 btf_id = 0; 6710 s64 min_off; 6711 int ret; 6712 6713 if (!env->allow_ptr_leaks) { 6714 verbose(env, 6715 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 6716 tname); 6717 return -EPERM; 6718 } 6719 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 6720 verbose(env, 6721 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 6722 tname); 6723 return -EINVAL; 6724 } 6725 6726 if (check_add_overflow(reg_smin(reg), off, &min_off)) { 6727 verbose(env, 6728 "%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n", 6729 reg_arg_name(env, argno), tname, reg_smin(reg), off); 6730 return -EACCES; 6731 } 6732 6733 if (min_off < 0) { 6734 verbose(env, 6735 "%s is ptr_%s invalid negative access: off=%lld\n", 6736 reg_arg_name(env, argno), tname, min_off); 6737 return -EACCES; 6738 } 6739 6740 if (reg->type & MEM_USER) { 6741 verbose(env, 6742 "%s is ptr_%s access user memory\n", 6743 reg_arg_name(env, argno), tname); 6744 return -EACCES; 6745 } 6746 6747 if (reg->type & MEM_PERCPU) { 6748 verbose(env, 6749 "%s is ptr_%s access percpu memory\n", 6750 reg_arg_name(env, argno), tname); 6751 return -EACCES; 6752 } 6753 6754 if (atype != BPF_READ && bpf_may_fault_on_deref(reg->type)) { 6755 verbose(env, "only read is supported\n"); 6756 return -EACCES; 6757 } 6758 6759 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { 6760 if (!tnum_is_const(reg->var_off)) { 6761 char tn_buf[48]; 6762 6763 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6764 verbose(env, 6765 "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 6766 reg_arg_name(env, argno), tname, off, tn_buf); 6767 return -EACCES; 6768 } 6769 6770 off += reg->var_off.value; 6771 6772 if (!btf_is_kernel(reg->btf)) { 6773 verifier_bug(env, "reg->btf must be kernel btf"); 6774 return -EFAULT; 6775 } 6776 ret = env->ops->btf_struct_access(&env->log, reg, off, size); 6777 if (ret < 0) 6778 verbose(env, 6779 "%s cannot write into ptr_%s at off=%d size=%d\n", 6780 reg_arg_name(env, argno), tname, off, size); 6781 } else { 6782 /* Writes are permitted with default btf_struct_access for 6783 * program allocated objects (which always have id > 0). 6784 */ 6785 if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { 6786 verbose(env, "only read is supported\n"); 6787 return -EACCES; 6788 } 6789 6790 /* 6791 * A fault-prone allocated object may still be read through a 6792 * BPF_PROBE_MEM load after its lifetime protection ends. Writes 6793 * through such pointers were rejected above. 6794 */ 6795 if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) && 6796 !type_is_non_owning_ref(reg->type) && 6797 !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) { 6798 verifier_bug(env, "allocated object must have a referenced id"); 6799 return -EFAULT; 6800 } 6801 6802 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); 6803 } 6804 6805 if (ret < 0) 6806 return ret; 6807 6808 if (ret == PTR_TO_ARENA) 6809 return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); 6810 6811 if (ret != PTR_TO_BTF_ID) { 6812 /* just mark; */ 6813 6814 } else if (type_flag(reg->type) & PTR_UNTRUSTED) { 6815 /* If this is an untrusted pointer, all pointers formed by walking it 6816 * also inherit the untrusted flag. 6817 */ 6818 flag = PTR_UNTRUSTED; 6819 6820 } else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) { 6821 /* By default any pointer obtained from walking a trusted pointer is no 6822 * longer trusted, unless the field being accessed has explicitly been 6823 * marked as inheriting its parent's state of trust (either full or RCU). 6824 * For example: 6825 * 'cgroups' pointer is untrusted if task->cgroups dereference 6826 * happened in a sleepable program outside of bpf_rcu_read_lock() 6827 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU). 6828 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED. 6829 * 6830 * A regular RCU-protected pointer with __rcu tag can also be deemed 6831 * trusted if we are in an RCU CS. Such pointer can be NULL. 6832 */ 6833 if (type_is_trusted(env, reg, field_name, btf_id)) { 6834 flag |= PTR_TRUSTED; 6835 } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) { 6836 flag |= PTR_TRUSTED | PTR_MAYBE_NULL; 6837 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { 6838 if (type_is_rcu(env, reg, field_name, btf_id)) { 6839 /* ignore __rcu tag and mark it MEM_RCU */ 6840 flag |= MEM_RCU; 6841 } else if (flag & MEM_RCU || 6842 type_is_rcu_or_null(env, reg, field_name, btf_id)) { 6843 /* __rcu tagged pointers can be NULL */ 6844 flag |= MEM_RCU | PTR_MAYBE_NULL; 6845 6846 /* We always trust them */ 6847 if (type_is_rcu_or_null(env, reg, field_name, btf_id) && 6848 flag & PTR_UNTRUSTED) 6849 flag &= ~PTR_UNTRUSTED; 6850 } else if (flag & (MEM_PERCPU | MEM_USER)) { 6851 /* keep as-is */ 6852 } else { 6853 /* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */ 6854 clear_trusted_flags(&flag); 6855 } 6856 } else { 6857 /* 6858 * If not in RCU CS or MEM_RCU pointer can be NULL then 6859 * aggressively mark as untrusted otherwise such 6860 * pointers will be plain PTR_TO_BTF_ID without flags 6861 * and will be allowed to be passed into helpers for 6862 * compat reasons. 6863 */ 6864 flag = PTR_UNTRUSTED; 6865 } 6866 } else { 6867 /* Old compat. Deprecated */ 6868 clear_trusted_flags(&flag); 6869 } 6870 6871 /* 6872 * A __nullable member only matters where NULL would otherwise be 6873 * assumed away, which is anywhere but on a plain or an untrusted 6874 * pointer: those are probed on a dereference, which tolerates NULL, and 6875 * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A 6876 * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. 6877 */ 6878 if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) 6879 flag &= ~PTR_MAYBE_NULL; 6880 6881 if (atype == BPF_READ && value_regno >= 0) { 6882 u32 parent_id = reg_lifetime_id(env, reg); 6883 6884 ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 6885 if (ret < 0) 6886 return ret; 6887 if ((regs[value_regno].type & PTR_TRUSTED) && 6888 !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER))) 6889 regs[value_regno].parent_id = parent_id; 6890 } 6891 6892 return 0; 6893 } 6894 6895 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 6896 struct bpf_reg_state *regs, struct bpf_reg_state *reg, 6897 argno_t argno, int off, int size, 6898 enum bpf_access_type atype, 6899 int value_regno) 6900 { 6901 struct bpf_map *map = reg->map_ptr; 6902 struct bpf_reg_state map_reg; 6903 enum bpf_type_flag flag = 0; 6904 const struct btf_type *t; 6905 const char *tname; 6906 u32 btf_id; 6907 int ret; 6908 6909 if (!btf_vmlinux) { 6910 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 6911 return -ENOTSUPP; 6912 } 6913 6914 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 6915 verbose(env, "map_ptr access not supported for map type %d\n", 6916 map->map_type); 6917 return -ENOTSUPP; 6918 } 6919 6920 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 6921 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 6922 6923 if (!env->allow_ptr_leaks) { 6924 verbose(env, 6925 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 6926 tname); 6927 return -EPERM; 6928 } 6929 6930 if (off < 0) { 6931 verbose(env, "%s is %s invalid negative access: off=%d\n", 6932 reg_arg_name(env, argno), tname, off); 6933 return -EACCES; 6934 } 6935 6936 if (atype != BPF_READ) { 6937 verbose(env, "only read from %s is supported\n", tname); 6938 return -EACCES; 6939 } 6940 6941 /* Simulate access to a PTR_TO_BTF_ID */ 6942 memset(&map_reg, 0, sizeof(map_reg)); 6943 ret = mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, 6944 btf_vmlinux, *map->ops->map_btf_id, 0); 6945 if (ret < 0) 6946 return ret; 6947 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); 6948 if (ret < 0) 6949 return ret; 6950 6951 if (value_regno >= 0) { 6952 ret = mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 6953 if (ret < 0) 6954 return ret; 6955 } 6956 6957 return 0; 6958 } 6959 6960 /* 6961 * Check that the stack access at the given offset is within bounds. The 6962 * maximum valid offset is -1. 6963 * 6964 * The minimum valid offset is -env->stack_limit for writes, and 6965 * -state->allocated_stack for reads. 6966 */ 6967 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, 6968 s64 off, 6969 struct bpf_func_state *state, 6970 enum bpf_access_type t) 6971 { 6972 int min_valid_off; 6973 6974 if (t == BPF_WRITE || env->allow_uninit_stack) 6975 min_valid_off = -(int)env->stack_limit; 6976 else 6977 min_valid_off = -state->allocated_stack; 6978 6979 if (off < min_valid_off || off > -1) 6980 return -EACCES; 6981 return 0; 6982 } 6983 6984 /* Check that the stack access at 'regno + off' falls within the maximum stack 6985 * bounds. 6986 * 6987 * 'off' includes `regno->offset`, but not its dynamic part (if any). 6988 */ 6989 static int check_stack_access_within_bounds( 6990 struct bpf_verifier_env *env, struct bpf_reg_state *reg, 6991 argno_t argno, int off, int access_size, 6992 enum bpf_access_type type) 6993 { 6994 struct bpf_func_state *state = bpf_func(env, reg); 6995 s64 min_off, max_off; 6996 int err; 6997 char *err_extra; 6998 6999 if (type == BPF_READ) 7000 err_extra = " read from"; 7001 else 7002 err_extra = " write to"; 7003 7004 if (tnum_is_const(reg->var_off)) { 7005 min_off = (s64)reg->var_off.value + off; 7006 max_off = min_off + access_size; 7007 } else { 7008 if (reg_smax(reg) >= BPF_MAX_VAR_OFF || 7009 reg_smin(reg) <= -BPF_MAX_VAR_OFF) { 7010 verbose(env, "invalid unbounded variable-offset%s stack %s\n", 7011 err_extra, reg_arg_name(env, argno)); 7012 return -EACCES; 7013 } 7014 min_off = reg_smin(reg) + off; 7015 max_off = reg_smax(reg) + off + access_size; 7016 } 7017 7018 err = check_stack_slot_within_bounds(env, min_off, state, type); 7019 if (!err && max_off > 0) 7020 err = -EINVAL; /* out of stack access into non-negative offsets */ 7021 if (!err && access_size < 0) 7022 /* access_size should not be negative (or overflow an int); others checks 7023 * along the way should have prevented such an access. 7024 */ 7025 err = -EFAULT; /* invalid negative access size; integer overflow? */ 7026 7027 if (err) { 7028 if (tnum_is_const(reg->var_off)) { 7029 verbose(env, "invalid%s stack %s off=%lld size=%d\n", 7030 err_extra, reg_arg_name(env, argno), min_off, access_size); 7031 } else { 7032 char tn_buf[48]; 7033 7034 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7035 verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n", 7036 err_extra, reg_arg_name(env, argno), tn_buf, off, access_size); 7037 } 7038 return err; 7039 } 7040 7041 /* 7042 * Every stack access passes through here, including buffers passed to 7043 * helpers and kfuncs and accesses into a caller's frame, so check the 7044 * fastcall contract of the frame that owns the slots once for all of 7045 * them. Zero-sized accesses touch no stack and keep the rewrite enabled. 7046 */ 7047 if (access_size) 7048 check_fastcall_stack_contract(env, state, env->insn_idx, min_off); 7049 7050 /* Note that there is no stack access with offset zero, so the needed stack 7051 * size is -min_off, not -min_off+1. 7052 */ 7053 return bpf_grow_stack_state(env, state, -min_off /* size */); 7054 } 7055 7056 static bool get_func_retval_range(struct bpf_prog *prog, 7057 struct bpf_retval_range *range) 7058 { 7059 if (prog->type == BPF_PROG_TYPE_LSM && 7060 prog->expected_attach_type == BPF_LSM_MAC && 7061 !bpf_lsm_get_retval_range(prog, range)) { 7062 return true; 7063 } 7064 return false; 7065 } 7066 7067 static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) 7068 { 7069 struct bpf_reg_state fake_reg; 7070 7071 if (!val) 7072 return; 7073 7074 fake_reg.type = SCALAR_VALUE; 7075 __mark_reg_known(&fake_reg, val); 7076 7077 scalar_step_add(dst_reg, dst_reg, &fake_reg); 7078 scalar32_min_max_add(dst_reg, &fake_reg); 7079 scalar_min_max_add(dst_reg, &fake_reg); 7080 dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off); 7081 7082 reg_bounds_sync(dst_reg); 7083 } 7084 7085 /* there is no support for callx in the interpreter */ 7086 static int require_callx_jit(struct bpf_verifier_env *env) 7087 { 7088 if (!env->prog->jit_requested) { 7089 verbose(env, "JIT is required to use callx\n"); 7090 return -EOPNOTSUPP; 7091 } 7092 if (!bpf_jit_supports_callx()) { 7093 verbose(env, "JIT doesn't support callx\n"); 7094 return -EOPNOTSUPP; 7095 } 7096 env->prog->jit_required = true; 7097 return 0; 7098 } 7099 7100 static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, 7101 int regno, int subprog) 7102 { 7103 mark_reg_known_zero(env, regs, regno); 7104 regs[regno].type = PTR_TO_FUNC; 7105 regs[regno].subprogno = subprog; 7106 } 7107 7108 /* a read from a table of functions branches into that many states at most */ 7109 #define BPF_MAX_FUNC_PTR_TARGETS 64 7110 /* and that many pointers at most are next to what is read at variable offset */ 7111 #define BPF_MAX_FUNC_PTR_RANGE 4096 7112 7113 /* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ 7114 static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) 7115 { 7116 u64 o = min_off, next; 7117 u32 cnt = 0; 7118 7119 if (!tnum_in(offs, tnum_const(o))) { 7120 next = tnum_step(offs, o); 7121 if (next <= o) 7122 return 0; 7123 o = next; 7124 } 7125 while (o <= max_off && cnt < limit) { 7126 cnt++; 7127 next = tnum_step(offs, o); 7128 if (next <= o) 7129 break; 7130 o = next; 7131 } 7132 return cnt; 7133 } 7134 7135 /* 7136 * A read from a frozen read-only map that has pointers to functions, see 7137 * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read 7138 * is possible at, e.g. var_off is the stride of the elements of an array. 7139 * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable 7140 * and only pointers can be read, which is how an element of a table of 7141 * functions is loaded, the verification continues with each of them. Other 7142 * reads that overlap with a pointer are rejected, because their result is not 7143 * known until the program is jitted. 7144 * 7145 * Return -ENOENT if there are no pointers to functions in the bytes that are read. 7146 */ 7147 static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, 7148 int size, int value_regno) 7149 { 7150 u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; 7151 struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); 7152 struct bpf_reg_state *regs = cur_regs(env); 7153 struct bpf_map *map = reg->map_ptr; 7154 struct bpf_verifier_state *branch; 7155 int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; 7156 struct bpf_func_ptr *ptrs; 7157 u32 i, cnt, n = 0; 7158 u64 o, lo, hi; 7159 int err; 7160 7161 ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); 7162 if (!ptrs) 7163 return -ENOENT; 7164 if (cnt > BPF_MAX_FUNC_PTR_RANGE) 7165 goto overlap; 7166 7167 for (i = 0; i < cnt; i++) { 7168 /* the offsets at which the read overlaps with the pointer */ 7169 lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; 7170 lo = max(lo, min_off); 7171 hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); 7172 for (o = lo; o <= hi; o++) { 7173 if (!tnum_in(offs, tnum_const(o))) 7174 continue; 7175 if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) 7176 goto overlap; 7177 if (n == BPF_MAX_FUNC_PTR_TARGETS) { 7178 verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", 7179 map->name, BPF_MAX_FUNC_PTR_TARGETS); 7180 return -E2BIG; 7181 } 7182 subprog = bpf_find_subprog(env, ptrs[i].xlated_off); 7183 if (verifier_bug_if(subprog <= 0, env, 7184 "no function at insn %u for map '%s' offset %u", 7185 ptrs[i].xlated_off, map->name, ptrs[i].map_off)) 7186 return -EFAULT; 7187 targets[n++] = subprog; 7188 } 7189 } 7190 /* pointers are next to what is read, e.g. the data of an array of vtables */ 7191 if (!n) 7192 return -ENOENT; 7193 /* either every offset that the read is possible at is a pointer or none */ 7194 if (count_read_offsets(offs, min_off, max_off, n + 1) != n) 7195 goto overlap; 7196 7197 /* the map has the offset of the function until the program is jitted */ 7198 err = require_callx_jit(env); 7199 if (err) 7200 return err; 7201 7202 for (i = 0; i < n - 1; i++) { 7203 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, 7204 env->cur_state->speculative); 7205 if (IS_ERR(branch)) 7206 return PTR_ERR(branch); 7207 mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, 7208 targets[i]); 7209 } 7210 mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); 7211 return 0; 7212 7213 overlap: 7214 verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", 7215 size, min_off, max_off, map->name); 7216 return -EACCES; 7217 } 7218 7219 static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, 7220 int bpf_size, int value_regno, bool is_ldsx) 7221 { 7222 struct bpf_reg_state *regs = cur_regs(env); 7223 int size = bpf_size_to_bytes(bpf_size); 7224 struct bpf_map *map = reg->map_ptr; 7225 int err; 7226 7227 switch (map->map_type) { 7228 case BPF_MAP_TYPE_INSN_ARRAY: 7229 if (bpf_size != BPF_DW) { 7230 verbose(env, "Invalid read of %d bytes from insn_array\n", size); 7231 return -EACCES; 7232 } 7233 regs[value_regno] = *reg; 7234 add_scalar_to_reg(®s[value_regno], off); 7235 regs[value_regno].type = PTR_TO_INSN; 7236 return 0; 7237 case BPF_MAP_TYPE_PERCPU_ARRAY: 7238 goto reg_unknown; 7239 default: 7240 break; 7241 } 7242 7243 if (env->func_ptr_cnt) { 7244 err = check_func_ptr_read(env, reg, off, size, value_regno); 7245 if (err != -ENOENT) 7246 return err; 7247 } 7248 7249 /* If map is read-only, track its contents as scalars. */ 7250 if (tnum_is_const(reg->var_off) && 7251 bpf_map_is_rdonly(map) && 7252 map->ops->map_direct_value_addr) { 7253 int map_off = off + reg->var_off.value; 7254 u64 val = 0; 7255 7256 err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); 7257 if (err) 7258 return err; 7259 7260 regs[value_regno].type = SCALAR_VALUE; 7261 __mark_reg_known(®s[value_regno], val); 7262 return 0; 7263 } 7264 7265 reg_unknown: 7266 mark_reg_unknown(env, regs, value_regno); 7267 return 0; 7268 } 7269 7270 /* check whether memory at (regno + off) is accessible for t = (read | write) 7271 * if t==write, value_regno is a register which value is stored into memory 7272 * if t==read, value_regno is a register which will receive the value from memory 7273 * if t==write && value_regno==-1, some unknown value is stored into memory 7274 * if t==read && value_regno==-1, don't care what we read from memory 7275 */ 7276 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno, 7277 int off, int bpf_size, enum bpf_access_type t, 7278 int value_regno, bool strict_alignment_once, bool is_ldsx) 7279 { 7280 struct bpf_reg_state *regs = cur_regs(env); 7281 enum bpf_reg_type ptr_type = reg->type; 7282 int size, err = 0; 7283 7284 size = bpf_size_to_bytes(bpf_size); 7285 if (size < 0) 7286 return size; 7287 7288 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 7289 if (err) 7290 return err; 7291 7292 if (reg->type == PTR_TO_MAP_KEY) { 7293 if (t == BPF_WRITE) { 7294 verbose(env, "write to change key %s not allowed\n", 7295 reg_arg_name(env, argno)); 7296 return -EACCES; 7297 } 7298 7299 err = check_mem_region_access(env, reg, argno, off, size, 7300 reg->map_ptr->key_size, false); 7301 if (err) 7302 return err; 7303 if (value_regno >= 0) 7304 mark_reg_unknown(env, regs, value_regno); 7305 } else if (reg->type == PTR_TO_MAP_VALUE) { 7306 struct btf_field *kptr_field = NULL; 7307 7308 if (t == BPF_WRITE && value_regno >= 0 && 7309 is_pointer_value(env, value_regno)) { 7310 verbose(env, "R%d leaks addr into map\n", value_regno); 7311 return -EACCES; 7312 } 7313 err = check_map_access_type(env, reg, off, size, t); 7314 if (err) 7315 return err; 7316 err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT); 7317 if (err) 7318 return err; 7319 if (tnum_is_const(reg->var_off)) 7320 kptr_field = btf_record_find(reg->map_ptr->record, 7321 off + reg->var_off.value, BPF_KPTR | BPF_UPTR); 7322 if (kptr_field) { 7323 err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); 7324 } else if (t == BPF_READ && value_regno >= 0) { 7325 err = check_map_mem_read(env, reg, off, bpf_size, value_regno, is_ldsx); 7326 } 7327 } else if (base_type(reg->type) == PTR_TO_MEM) { 7328 bool rdonly_mem = type_is_rdonly_mem(reg->type); 7329 bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED); 7330 7331 if (type_may_be_null(reg->type)) { 7332 verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), 7333 reg_type_str(env, reg->type)); 7334 bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), 7335 reg_arg_name(env, argno), reg, 7336 BPF_DIAG_DEREF_NULLABLE_PTR, 0); 7337 return -EACCES; 7338 } 7339 7340 if (t == BPF_WRITE && rdonly_mem) { 7341 verbose(env, "%s cannot write into %s\n", 7342 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7343 return -EACCES; 7344 } 7345 7346 if (t == BPF_WRITE && value_regno >= 0 && 7347 is_pointer_value(env, value_regno)) { 7348 verbose(env, "R%d leaks addr into mem\n", value_regno); 7349 return -EACCES; 7350 } 7351 7352 if (rdonly_untrusted && !env->allow_ptr_leaks) { 7353 verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 7354 reg_type_str(env, reg->type)); 7355 bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory", 7356 "the access requires CAP_PERFMON", 7357 "Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers."); 7358 return -EPERM; 7359 } 7360 7361 /* 7362 * Accesses to untrusted PTR_TO_MEM are done through probe 7363 * instructions, hence no need to check bounds in that case. 7364 */ 7365 if (!rdonly_untrusted) 7366 err = check_mem_region_access(env, reg, argno, off, size, 7367 reg->mem_size, false); 7368 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 7369 mark_reg_unknown(env, regs, value_regno); 7370 } else if (reg->type == PTR_TO_CTX) { 7371 struct bpf_insn_access_aux info = { 7372 .reg_type = SCALAR_VALUE, 7373 .is_ldsx = is_ldsx, 7374 .log = &env->log, 7375 }; 7376 struct bpf_retval_range range; 7377 7378 if (t == BPF_WRITE && value_regno >= 0 && 7379 is_pointer_value(env, value_regno)) { 7380 verbose(env, "R%d leaks addr into ctx\n", value_regno); 7381 return -EACCES; 7382 } 7383 7384 err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info); 7385 if (!err && t == BPF_READ && value_regno >= 0) { 7386 /* ctx access returns either a scalar, or a 7387 * PTR_TO_PACKET[_META,_END]. In the latter 7388 * case, we know the offset is zero. 7389 */ 7390 if (info.reg_type == SCALAR_VALUE) { 7391 if (info.is_retval && get_func_retval_range(env->prog, &range)) { 7392 mark_reg_unknown(env, regs, value_regno); 7393 err = __mark_reg_s32_range(env, regs, value_regno, 7394 range.minval, range.maxval); 7395 if (err) 7396 return err; 7397 } else { 7398 mark_reg_unknown(env, regs, value_regno); 7399 } 7400 } else { 7401 mark_reg_known_zero(env, regs, 7402 value_regno); 7403 if (base_type(info.reg_type) == PTR_TO_BTF_ID) { 7404 regs[value_regno].btf = info.btf; 7405 regs[value_regno].btf_id = info.btf_id; 7406 regs[value_regno].id = info.ref_id; 7407 } else if (base_type(info.reg_type) == PTR_TO_MEM) { 7408 regs[value_regno].mem_size = info.mem_size; 7409 } 7410 if (type_may_be_null(info.reg_type) && !regs[value_regno].id) 7411 regs[value_regno].id = ++env->id_gen; 7412 } 7413 regs[value_regno].type = info.reg_type; 7414 } 7415 7416 } else if (reg->type == PTR_TO_STACK) { 7417 /* Basic bounds checks. */ 7418 err = check_stack_access_within_bounds(env, reg, argno, off, size, t); 7419 if (err) 7420 return err; 7421 7422 if (t == BPF_READ) 7423 err = check_stack_read(env, reg, argno, off, size, 7424 value_regno); 7425 else 7426 err = check_stack_write(env, reg, off, size, 7427 value_regno, insn_idx); 7428 } else if (reg_is_pkt_pointer(reg)) { 7429 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 7430 verbose(env, "cannot write into packet\n"); 7431 return -EACCES; 7432 } 7433 if (t == BPF_WRITE && value_regno >= 0 && 7434 is_pointer_value(env, value_regno)) { 7435 verbose(env, "R%d leaks addr into packet\n", 7436 value_regno); 7437 return -EACCES; 7438 } 7439 err = check_packet_access(env, reg, argno, off, size, false); 7440 if (!err && t == BPF_READ && value_regno >= 0) 7441 mark_reg_unknown(env, regs, value_regno); 7442 } else if (reg->type == PTR_TO_FLOW_KEYS) { 7443 if (t == BPF_WRITE && value_regno >= 0 && 7444 is_pointer_value(env, value_regno)) { 7445 verbose(env, "R%d leaks addr into flow keys\n", 7446 value_regno); 7447 return -EACCES; 7448 } 7449 7450 err = check_flow_keys_access(env, reg, argno, off, size); 7451 if (!err && t == BPF_READ && value_regno >= 0) 7452 mark_reg_unknown(env, regs, value_regno); 7453 } else if (type_is_sk_pointer(reg->type)) { 7454 if (t == BPF_WRITE) { 7455 verbose(env, "%s cannot write into %s\n", 7456 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7457 return -EACCES; 7458 } 7459 err = check_sock_access(env, insn_idx, reg, argno, off, size, t); 7460 if (!err && value_regno >= 0) 7461 mark_reg_unknown(env, regs, value_regno); 7462 } else if (reg->type == PTR_TO_TP_BUFFER) { 7463 err = check_tp_buffer_access(env, reg, argno, off, size); 7464 if (!err && t == BPF_READ && value_regno >= 0) 7465 mark_reg_unknown(env, regs, value_regno); 7466 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 7467 !type_may_be_null(reg->type)) { 7468 err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t, 7469 value_regno); 7470 } else if (reg->type == CONST_PTR_TO_MAP) { 7471 err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t, 7472 value_regno); 7473 } else if (base_type(reg->type) == PTR_TO_BUF && 7474 !type_may_be_null(reg->type)) { 7475 bool rdonly_mem = type_is_rdonly_mem(reg->type); 7476 u32 *max_access; 7477 7478 if (rdonly_mem) { 7479 if (t == BPF_WRITE) { 7480 verbose(env, "%s cannot write into %s\n", 7481 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7482 return -EACCES; 7483 } 7484 max_access = &env->prog->aux->max_rdonly_access; 7485 } else { 7486 max_access = &env->prog->aux->max_rdwr_access; 7487 } 7488 7489 err = check_buffer_access(env, reg, argno, off, size, false, 7490 max_access, t == BPF_WRITE); 7491 7492 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 7493 mark_reg_unknown(env, regs, value_regno); 7494 } else if (reg->type == PTR_TO_ARENA) { 7495 if (t == BPF_READ && value_regno >= 0) 7496 mark_reg_unknown(env, regs, value_regno); 7497 } else { 7498 enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR; 7499 7500 verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), 7501 reg_type_str(env, reg->type)); 7502 if (reg->type == SCALAR_VALUE) 7503 kind = BPF_DIAG_DEREF_SCALAR; 7504 else if (type_may_be_null(reg->type)) 7505 kind = BPF_DIAG_DEREF_NULLABLE_PTR; 7506 bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), 7507 reg_arg_name(env, argno), reg, kind, 0); 7508 return -EACCES; 7509 } 7510 7511 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 7512 regs[value_regno].type == SCALAR_VALUE) { 7513 if (!is_ldsx) { 7514 /* b/h/w load zero-extends, mark upper bits as known 0 */ 7515 coerce_reg_to_size(®s[value_regno], size); 7516 } else { 7517 /* 7518 * Sign-extension can change the register value relative 7519 * to a scalar it is linked with by id (e.g. a zero- 7520 * extending fill of the same spilled stack slot), thus 7521 * drop the shared id in that case. 7522 */ 7523 bool no_sext = reg_umax(®s[value_regno]) < 7524 (1ULL << (size * BITS_PER_BYTE - 1)); 7525 7526 coerce_reg_to_size_sx(®s[value_regno], size); 7527 if (!no_sext) 7528 clear_scalar_id(®s[value_regno]); 7529 } 7530 } 7531 7532 if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) 7533 mark_non_stack_access(env, insn_idx); 7534 7535 return err; 7536 } 7537 7538 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, 7539 bool allow_trust_mismatch); 7540 7541 static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, 7542 bool strict_alignment_once, bool is_ldsx, 7543 bool allow_trust_mismatch, const char *ctx) 7544 { 7545 struct bpf_verifier_state *vstate = env->cur_state; 7546 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7547 struct bpf_reg_state *regs = cur_regs(env); 7548 enum bpf_reg_type src_reg_type; 7549 int err; 7550 7551 /* Handle stack arg read */ 7552 if (is_stack_arg_ldx(insn)) { 7553 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 7554 if (err) 7555 return err; 7556 return check_stack_arg_read(env, state, insn->off, insn->dst_reg); 7557 } 7558 7559 /* check src operand */ 7560 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7561 if (err) 7562 return err; 7563 7564 /* check dst operand */ 7565 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 7566 if (err) 7567 return err; 7568 7569 src_reg_type = regs[insn->src_reg].type; 7570 7571 /* 7572 * check_stack_read_fixed_off() may refine the modification's origin to 7573 * the source stack slot. 7574 */ 7575 bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); 7576 err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, 7577 argno_from_reg(insn->src_reg), insn->off, 7578 BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, 7579 strict_alignment_once, is_ldsx); 7580 /* 7581 * Paths through a kernel struct pointer and through other pointers 7582 * may share the insn, see save_aux_ptr_type(). Record a load that did 7583 * not produce an arena pointer from an __arena member, the only load 7584 * through a kernel struct pointer that does, next to the one 7585 * check_arena_member_load() records, so that arena_member_fixup() can 7586 * tell when the paths disagree. 7587 */ 7588 if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && 7589 regs[insn->dst_reg].type == PTR_TO_ARENA)) 7590 env->insn_aux_data[env->insn_idx].non_arena_load = true; 7591 err = err ?: save_aux_ptr_type(env, src_reg_type, 7592 allow_trust_mismatch); 7593 err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); 7594 if (!err) 7595 bpf_diag_mod_end(env); 7596 7597 return err; 7598 } 7599 7600 static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, 7601 bool strict_alignment_once) 7602 { 7603 struct bpf_verifier_state *vstate = env->cur_state; 7604 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7605 struct bpf_reg_state *regs = cur_regs(env); 7606 enum bpf_reg_type dst_reg_type; 7607 int err; 7608 7609 /* Handle stack arg write */ 7610 if (is_stack_arg_stx(insn)) { 7611 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7612 if (err) 7613 return err; 7614 return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg); 7615 } 7616 7617 /* check src1 operand */ 7618 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7619 if (err) 7620 return err; 7621 7622 /* check src2 operand */ 7623 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7624 if (err) 7625 return err; 7626 7627 dst_reg_type = regs[insn->dst_reg].type; 7628 7629 /* Check if (dst_reg + off) is writeable. */ 7630 err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, 7631 argno_from_reg(insn->dst_reg), insn->off, 7632 BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, 7633 strict_alignment_once, false); 7634 err = err ?: save_aux_ptr_type(env, dst_reg_type, false); 7635 7636 return err; 7637 } 7638 7639 static int check_atomic_rmw(struct bpf_verifier_env *env, 7640 struct bpf_insn *insn) 7641 { 7642 struct bpf_reg_state *dst_reg; 7643 int load_reg; 7644 int err; 7645 7646 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 7647 verbose(env, "invalid atomic operand size\n"); 7648 return -EINVAL; 7649 } 7650 7651 /* check src1 operand */ 7652 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7653 if (err) 7654 return err; 7655 7656 /* check src2 operand */ 7657 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7658 if (err) 7659 return err; 7660 7661 if (insn->imm == BPF_CMPXCHG) { 7662 /* Check comparison of R0 with memory location */ 7663 const u32 aux_reg = BPF_REG_0; 7664 7665 err = check_reg_arg(env, aux_reg, SRC_OP); 7666 if (err) 7667 return err; 7668 7669 if (is_pointer_value(env, aux_reg)) { 7670 verbose(env, "R%d leaks addr into mem\n", aux_reg); 7671 return -EACCES; 7672 } 7673 } 7674 7675 if (is_pointer_value(env, insn->src_reg)) { 7676 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 7677 return -EACCES; 7678 } 7679 7680 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { 7681 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 7682 insn->dst_reg, 7683 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 7684 return -EACCES; 7685 } 7686 7687 load_reg = bpf_atomic_load_reg(insn); 7688 if (load_reg >= 0) { 7689 /* check and record load of old value */ 7690 err = check_reg_arg(env, load_reg, DST_OP); 7691 if (err) 7692 return err; 7693 } 7694 7695 dst_reg = cur_regs(env) + insn->dst_reg; 7696 7697 /* Check whether we can read the memory, with second call for fetch 7698 * case to simulate the register fill. 7699 */ 7700 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, 7701 BPF_SIZE(insn->code), BPF_READ, -1, true, false); 7702 if (!err && load_reg >= 0) { 7703 bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE); 7704 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), 7705 insn->off, BPF_SIZE(insn->code), 7706 BPF_READ, load_reg, true, false); 7707 if (!err) 7708 bpf_diag_mod_end(env); 7709 } 7710 if (err) 7711 return err; 7712 7713 err = save_aux_ptr_type(env, dst_reg->type, false); 7714 if (err) 7715 return err; 7716 /* Check whether we can write into the same memory. */ 7717 err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, 7718 BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); 7719 if (err) 7720 return err; 7721 return 0; 7722 } 7723 7724 static int check_atomic_load(struct bpf_verifier_env *env, 7725 struct bpf_insn *insn) 7726 { 7727 int err; 7728 7729 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7730 if (err) 7731 return err; 7732 7733 if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) { 7734 verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n", 7735 insn->src_reg, 7736 reg_type_str(env, reg_state(env, insn->src_reg)->type)); 7737 return -EACCES; 7738 } 7739 7740 return check_load_mem(env, insn, true, false, false, "atomic_load"); 7741 } 7742 7743 static int check_atomic_store(struct bpf_verifier_env *env, 7744 struct bpf_insn *insn) 7745 { 7746 int err; 7747 7748 err = check_store_reg(env, insn, true); 7749 if (err) 7750 return err; 7751 7752 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { 7753 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 7754 insn->dst_reg, 7755 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 7756 return -EACCES; 7757 } 7758 7759 return 0; 7760 } 7761 7762 static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn) 7763 { 7764 switch (insn->imm) { 7765 case BPF_ADD: 7766 case BPF_ADD | BPF_FETCH: 7767 case BPF_AND: 7768 case BPF_AND | BPF_FETCH: 7769 case BPF_OR: 7770 case BPF_OR | BPF_FETCH: 7771 case BPF_XOR: 7772 case BPF_XOR | BPF_FETCH: 7773 case BPF_XCHG: 7774 case BPF_CMPXCHG: 7775 return check_atomic_rmw(env, insn); 7776 case BPF_LOAD_ACQ: 7777 if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) { 7778 verbose(env, 7779 "64-bit load-acquires are only supported on 64-bit arches\n"); 7780 return -EOPNOTSUPP; 7781 } 7782 return check_atomic_load(env, insn); 7783 case BPF_STORE_REL: 7784 if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) { 7785 verbose(env, 7786 "64-bit store-releases are only supported on 64-bit arches\n"); 7787 return -EOPNOTSUPP; 7788 } 7789 return check_atomic_store(env, insn); 7790 default: 7791 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", 7792 insn->imm); 7793 return -EINVAL; 7794 } 7795 } 7796 7797 /* When register 'regno' is used to read the stack (either directly or through 7798 * a helper function) make sure that it's within stack boundary and, depending 7799 * on the access type and privileges, that all elements of the stack are 7800 * initialized. 7801 * 7802 * All registers that have been spilled on the stack in the slots within the 7803 * read offsets are marked as read. 7804 */ 7805 static int check_stack_range_initialized( 7806 struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off, 7807 int access_size, bool zero_size_allowed, 7808 enum bpf_access_type type, struct bpf_call_arg_meta *meta) 7809 { 7810 struct bpf_func_state *state = bpf_func(env, reg); 7811 int err, min_off, max_off, i, j, slot, spi; 7812 /* Some accesses can write anything into the stack, others are 7813 * read-only. 7814 */ 7815 bool clobber = type == BPF_WRITE; 7816 /* 7817 * Negative access_size signals global subprog arg check where 7818 * STACK_POISON slots are acceptable. static stack liveness 7819 * might have determined that subprog doesn't read them, 7820 * but BTF based global subprog validation isn't accurate enough. 7821 */ 7822 bool allow_poison = access_size < 0 || clobber; 7823 u32 arg_slot = arg_slot_from_argno(argno); 7824 bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && 7825 (meta->arg_raw_mem.mask & BIT(arg_slot)); 7826 bool raw_mode = uninit && env->allow_uninit_stack && 7827 !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); 7828 7829 access_size = abs(access_size); 7830 7831 if (access_size == 0 && !zero_size_allowed) { 7832 verbose(env, "invalid zero-sized read\n"); 7833 return -EACCES; 7834 } 7835 7836 err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type); 7837 if (err) 7838 return err; 7839 7840 if (tnum_is_const(reg->var_off)) { 7841 min_off = max_off = reg->var_off.value + off; 7842 } else { 7843 /* Variable offset is prohibited for unprivileged mode for 7844 * simplicity since it requires corresponding support in 7845 * Spectre masking for stack ALU. 7846 * See also retrieve_ptr_limit(). 7847 */ 7848 if (!env->bypass_spec_v1) { 7849 char tn_buf[48]; 7850 7851 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7852 verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n", 7853 reg_arg_name(env, argno), tn_buf); 7854 return -EACCES; 7855 } 7856 /* 7857 * The call may touch any byte in the possible range, but does not 7858 * definitely initialize all of it. Fall back to ordinary stack checks. 7859 */ 7860 raw_mode = false; 7861 7862 min_off = reg_smin(reg) + off; 7863 max_off = reg_smax(reg) + off; 7864 } 7865 7866 /* Unprivileged outputs retain each byte's initialization state. */ 7867 if (raw_mode) { 7868 meta->arg_raw_mem.size[arg_slot] = access_size; 7869 return 0; 7870 } 7871 7872 for (i = min_off; i < max_off + access_size; i++) { 7873 struct bpf_stack_state *ss; 7874 u8 *stype; 7875 7876 slot = -i - 1; 7877 spi = slot / BPF_REG_SIZE; 7878 if (state->allocated_stack <= slot) { 7879 verbose(env, "allocated_stack too small\n"); 7880 return -EFAULT; 7881 } 7882 7883 ss = bpf_stack_slot(state, spi); 7884 stype = &ss->slot_type[slot % BPF_REG_SIZE]; 7885 if (*stype == STACK_MISC) 7886 goto mark; 7887 if ((*stype == STACK_ZERO) || 7888 (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { 7889 if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { 7890 /* helper can write anything into the stack */ 7891 *stype = STACK_MISC; 7892 } 7893 goto mark; 7894 } 7895 7896 /* 7897 * Only the bytes marked STACK_SPILL hold the spilled register. 7898 * The rest of a narrowly spilled slot keeps its previous type 7899 * and must be initialized on its own. 7900 */ 7901 if (*stype == STACK_SPILL && 7902 (ss->spilled_ptr.type == SCALAR_VALUE || 7903 env->allow_ptr_leaks)) { 7904 if (clobber) { 7905 __mark_reg_unknown(env, &ss->spilled_ptr); 7906 for (j = 0; j < BPF_REG_SIZE; j++) 7907 scrub_spilled_slot(&ss->slot_type[j]); 7908 } 7909 goto mark; 7910 } 7911 7912 if (*stype == STACK_POISON) { 7913 if (allow_poison) { 7914 if (uninit && env->allow_uninit_stack) 7915 *stype = STACK_MISC; 7916 goto mark; 7917 } 7918 verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", 7919 reg_arg_name(env, argno), min_off, i - min_off, access_size); 7920 } else if (tnum_is_const(reg->var_off)) { 7921 verbose(env, "invalid read from stack %s off %d+%d size %d\n", 7922 reg_arg_name(env, argno), min_off, i - min_off, access_size); 7923 } else { 7924 char tn_buf[48]; 7925 7926 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7927 verbose(env, "invalid read from stack %s var_off %s+%d size %d\n", 7928 reg_arg_name(env, argno), tn_buf, i - min_off, access_size); 7929 } 7930 return -EACCES; 7931 mark: 7932 ; 7933 } 7934 return 0; 7935 } 7936 7937 static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 7938 argno_t argno, int access_size, 7939 enum bpf_access_type access_type, bool zero_size_allowed, 7940 struct bpf_call_arg_meta *meta, bool *known_memory) 7941 { 7942 struct bpf_reg_state *regs = cur_regs(env); 7943 u32 *max_access; 7944 7945 if (known_memory) 7946 *known_memory = true; 7947 7948 switch (base_type(reg->type)) { 7949 case PTR_TO_PACKET: 7950 case PTR_TO_PACKET_META: 7951 if (!may_access_direct_pkt_data(env, meta, access_type)) { 7952 verbose(env, "function access to the packet is not allowed\n"); 7953 return -EACCES; 7954 } 7955 return check_packet_access(env, reg, argno, 0, access_size, 7956 zero_size_allowed); 7957 case PTR_TO_MAP_KEY: 7958 if (access_type == BPF_WRITE) { 7959 verbose(env, "%s cannot write into %s\n", 7960 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7961 return -EACCES; 7962 } 7963 return check_mem_region_access(env, reg, argno, 0, access_size, 7964 reg->map_ptr->key_size, false); 7965 case PTR_TO_MAP_VALUE: 7966 if (check_map_access_type(env, reg, 0, access_size, access_type)) 7967 return -EACCES; 7968 return check_map_access(env, reg, argno, 0, access_size, 7969 zero_size_allowed, ACCESS_HELPER); 7970 case PTR_TO_MEM: 7971 if (type_is_rdonly_mem(reg->type)) { 7972 if (access_type == BPF_WRITE) { 7973 verbose(env, "%s cannot write into %s\n", 7974 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7975 return -EACCES; 7976 } 7977 } 7978 return check_mem_region_access(env, reg, argno, 0, 7979 access_size, reg->mem_size, 7980 zero_size_allowed); 7981 case PTR_TO_BUF: 7982 if (type_is_rdonly_mem(reg->type)) { 7983 if (access_type == BPF_WRITE) { 7984 verbose(env, "%s cannot write into %s\n", 7985 reg_arg_name(env, argno), reg_type_str(env, reg->type)); 7986 return -EACCES; 7987 } 7988 7989 max_access = &env->prog->aux->max_rdonly_access; 7990 } else { 7991 max_access = &env->prog->aux->max_rdwr_access; 7992 } 7993 return check_buffer_access(env, reg, argno, 0, 7994 access_size, zero_size_allowed, 7995 max_access, access_type == BPF_WRITE); 7996 case PTR_TO_STACK: 7997 return check_stack_range_initialized( 7998 env, reg, 7999 argno, 0, access_size, 8000 zero_size_allowed, access_type, meta); 8001 case PTR_TO_BTF_ID: 8002 return check_ptr_to_btf_access(env, regs, reg, argno, 0, 8003 access_size, access_type, -1); 8004 case PTR_TO_CTX: 8005 /* Only permit reading or writing syscall context using helper calls. */ 8006 if (is_var_ctx_off_allowed(env->prog)) { 8007 int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX, 8008 zero_size_allowed); 8009 if (err) 8010 return err; 8011 if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size) 8012 env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size; 8013 return 0; 8014 } 8015 fallthrough; 8016 default: /* scalar_value or invalid ptr */ 8017 /* Allow zero-byte read from NULL, regardless of pointer type */ 8018 if (zero_size_allowed && access_size == 0 && 8019 bpf_register_is_null(reg)) 8020 return 0; 8021 if (known_memory && base_type(reg->type) != PTR_TO_CTX) 8022 *known_memory = false; 8023 8024 verbose(env, "%s type=%s ", reg_arg_name(env, argno), 8025 reg_type_str(env, reg->type)); 8026 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 8027 return -EACCES; 8028 } 8029 } 8030 8031 enum bpf_mem_size_failure { 8032 BPF_MEM_SIZE_FAIL_NONE, 8033 BPF_MEM_SIZE_FAIL_MEMORY, 8034 BPF_MEM_SIZE_FAIL_SIZE, 8035 }; 8036 8037 /* verify arguments to helpers or kfuncs consisting of a pointer and an access 8038 * size. 8039 * 8040 * @mem_reg contains the pointer, @size_reg contains the access size. 8041 */ 8042 static int check_mem_size_reg(struct bpf_verifier_env *env, 8043 struct bpf_reg_state *mem_reg, 8044 struct bpf_reg_state *size_reg, argno_t mem_argno, 8045 argno_t size_argno, u32 access_type, 8046 bool zero_size_allowed, 8047 struct bpf_call_arg_meta *meta, 8048 enum bpf_mem_size_failure *failure) 8049 { 8050 int err = 0; 8051 8052 if (failure) 8053 *failure = BPF_MEM_SIZE_FAIL_NONE; 8054 8055 /* This is used to refine r0 return value bounds for helpers 8056 * that enforce this value as an upper bound on return values. 8057 * See do_refine_retval_range() for helpers that can refine 8058 * the return value. C type of helper is u32 so we pull register 8059 * bound from umax_value however, if negative verifier errors 8060 * out. Only upper bounds can be learned because retval is an 8061 * int type and negative retvals are allowed. 8062 */ 8063 meta->msize_max_value = reg_umax(size_reg); 8064 8065 /* 8066 * Check variable ranges byte by byte instead of using raw mode. Keep the 8067 * MEM_UNINIT annotation so invalid bytes are accepted without marking them 8068 * initialized when the caller cannot read uninitialized stack memory. 8069 */ 8070 if (!tnum_is_const(size_reg->var_off)) 8071 meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); 8072 8073 if (reg_smin(size_reg) < 0) { 8074 verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", 8075 reg_arg_name(env, size_argno)); 8076 err = -EACCES; 8077 goto size_error; 8078 } 8079 8080 if (reg_umin(size_reg) == 0 && !zero_size_allowed) { 8081 verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", 8082 reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); 8083 err = -EACCES; 8084 goto size_error; 8085 } 8086 8087 if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { 8088 verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 8089 reg_arg_name(env, size_argno)); 8090 err = -EACCES; 8091 goto size_error; 8092 } 8093 8094 if (access_type & BPF_READ) 8095 err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), 8096 BPF_READ, zero_size_allowed, meta, NULL); 8097 if (!err && access_type & BPF_WRITE) 8098 err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), 8099 BPF_WRITE, zero_size_allowed, meta, NULL); 8100 if (err && failure) 8101 *failure = BPF_MEM_SIZE_FAIL_MEMORY; 8102 8103 if (!err) 8104 err = mark_arg_precision(env, size_argno); 8105 8106 return err; 8107 8108 size_error: 8109 if (failure) 8110 *failure = BPF_MEM_SIZE_FAIL_SIZE; 8111 return err; 8112 } 8113 8114 static bool is_subprog(const struct bpf_call_arg_meta *meta) 8115 { 8116 return meta->btf && !meta->func_id; 8117 } 8118 8119 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 8120 argno_t argno, u32 mem_size, enum bpf_access_type access_type, 8121 struct bpf_call_arg_meta *meta, bool *known_memory) 8122 { 8123 int size, err = 0; 8124 8125 if (bpf_register_is_null(reg)) 8126 return mark_arg_precision(env, argno); 8127 if (known_memory) 8128 *known_memory = true; 8129 8130 if (mem_size > S32_MAX) { 8131 verbose(env, "%s memory size %u is too large\n", 8132 reg_arg_name(env, argno), mem_size); 8133 return -EACCES; 8134 } 8135 8136 /* 8137 * Only a global subprog may read poisoned stack slots: 8138 * its static stack liveness proved the callee body skips them. 8139 */ 8140 size = (is_subprog(meta) && 8141 base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; 8142 8143 if (access_type & BPF_READ) 8144 err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, 8145 known_memory); 8146 if (!err && (access_type & BPF_WRITE)) 8147 err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta, 8148 known_memory); 8149 8150 return err; 8151 } 8152 8153 static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 8154 argno_t argno, struct ret_mem_desc *ret_mem) 8155 { 8156 int regno = reg_from_argno(argno); 8157 int err; 8158 8159 if (ret_mem->found) { 8160 verifier_bug(env, "only one allocation size argument permitted"); 8161 return -EFAULT; 8162 } 8163 8164 if (!tnum_is_const(reg->var_off)) { 8165 verbose(env, "%s is not a const\n", reg_arg_name(env, argno)); 8166 return -EINVAL; 8167 } 8168 8169 if (reg->var_off.value > U32_MAX) { 8170 verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno)); 8171 return -EINVAL; 8172 } 8173 8174 if (regno >= 0) 8175 err = mark_chain_precision(env, regno); 8176 else 8177 err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); 8178 if (err) 8179 return err; 8180 8181 ret_mem->size = reg->var_off.value; 8182 ret_mem->found = true; 8183 8184 return 0; 8185 } 8186 8187 static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 8188 argno_t argno, struct bpf_call_arg_meta *meta) 8189 { 8190 int regno = reg_from_argno(argno); 8191 int err; 8192 8193 if (meta->arg_constant.found) { 8194 verifier_bug(env, "only one constant argument permitted"); 8195 return -EFAULT; 8196 } 8197 8198 if (!tnum_is_const(reg->var_off)) { 8199 verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); 8200 return -EINVAL; 8201 } 8202 8203 if (regno >= 0) 8204 err = mark_chain_precision(env, regno); 8205 else 8206 err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); 8207 if (err < 0) 8208 return err; 8209 8210 meta->arg_constant.found = true; 8211 meta->arg_constant.value = reg->var_off.value; 8212 8213 return 0; 8214 } 8215 8216 enum { 8217 PROCESS_SPIN_LOCK = (1 << 0), 8218 PROCESS_RES_LOCK = (1 << 1), 8219 PROCESS_LOCK_IRQ = (1 << 2), 8220 }; 8221 8222 /* Implementation details: 8223 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 8224 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 8225 * Two bpf_map_lookups (even with the same key) will have different reg->id. 8226 * Two separate bpf_obj_new will also have different reg->id. 8227 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 8228 * clears reg->id after value_or_null->value transition, since the verifier only 8229 * cares about the range of access to valid map value pointer and doesn't care 8230 * about actual address of the map element. 8231 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 8232 * reg->id > 0 after value_or_null->value transition. By doing so 8233 * two bpf_map_lookups will be considered two different pointers that 8234 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 8235 * returned from bpf_obj_new. 8236 * The verifier allows taking only one bpf_spin_lock at a time to avoid 8237 * dead-locks. 8238 * Since only one bpf_spin_lock is allowed the checks are simpler than 8239 * reg_is_refcounted() logic. The verifier needs to remember only 8240 * one spin_lock instead of array of acquired_refs. 8241 * env->cur_state->active_locks remembers which map value element or allocated 8242 * object got locked and clears it after bpf_spin_unlock. 8243 */ 8244 static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags) 8245 { 8246 bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; 8247 const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; 8248 struct bpf_verifier_state *cur = env->cur_state; 8249 struct bpf_reference_state *lock; 8250 bool is_const = tnum_is_const(reg->var_off); 8251 bool is_irq = flags & PROCESS_LOCK_IRQ; 8252 u64 val = reg->var_off.value; 8253 struct bpf_map *map = NULL; 8254 struct btf *btf = NULL; 8255 struct btf_record *rec; 8256 u32 spin_lock_off; 8257 int err; 8258 8259 if (!is_const) { 8260 verbose(env, 8261 "%s doesn't have constant offset. %s_lock has to be at the constant offset\n", 8262 reg_arg_name(env, argno), lock_str); 8263 return -EINVAL; 8264 } 8265 if (reg->type == PTR_TO_MAP_VALUE) { 8266 map = reg->map_ptr; 8267 if (!map->btf) { 8268 verbose(env, 8269 "map '%s' has to have BTF in order to use %s_lock\n", 8270 map->name, lock_str); 8271 return -EINVAL; 8272 } 8273 } else { 8274 btf = reg->btf; 8275 } 8276 8277 rec = reg_btf_record(reg); 8278 if (!btf_record_has_field(rec, is_res_lock ? BPF_RES_SPIN_LOCK : BPF_SPIN_LOCK)) { 8279 verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local", 8280 map ? map->name : "kptr", lock_str); 8281 return -EINVAL; 8282 } 8283 spin_lock_off = is_res_lock ? rec->res_spin_lock_off : rec->spin_lock_off; 8284 if (spin_lock_off != val) { 8285 verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n", 8286 val, lock_str, spin_lock_off); 8287 return -EINVAL; 8288 } 8289 if (is_lock) { 8290 void *ptr; 8291 int type; 8292 8293 if (map) 8294 ptr = map; 8295 else 8296 ptr = btf; 8297 8298 if (!is_res_lock && cur->active_locks) { 8299 lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL); 8300 if (lock) { 8301 verbose(env, 8302 "Locking two bpf_spin_locks are not allowed\n"); 8303 bpf_diag_lock( 8304 env, env->insn_idx, "nested spin lock", 8305 "This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.", 8306 "Unlock the current bpf_spin_lock before taking another one.", lock); 8307 return -EINVAL; 8308 } 8309 } else if (is_res_lock && cur->active_locks) { 8310 lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, 8311 reg->id, ptr); 8312 if (lock) { 8313 verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); 8314 bpf_diag_lock( 8315 env, env->insn_idx, "recursive resource spin lock", 8316 "This path already holds the same resource spin lock. Taking it again would deadlock.", 8317 "Avoid reacquiring the same resource spin lock before it is unlocked.", lock); 8318 return -EINVAL; 8319 } 8320 } 8321 8322 if (is_res_lock && is_irq) 8323 type = REF_TYPE_RES_LOCK_IRQ; 8324 else if (is_res_lock) 8325 type = REF_TYPE_RES_LOCK; 8326 else 8327 type = REF_TYPE_LOCK; 8328 err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr); 8329 if (err < 0) { 8330 verbose(env, "Failed to acquire lock state\n"); 8331 return err; 8332 } 8333 } else { 8334 void *ptr; 8335 int type; 8336 8337 if (map) 8338 ptr = map; 8339 else 8340 ptr = btf; 8341 8342 if (!cur->active_locks) { 8343 verbose(env, "%s_unlock without taking a lock\n", lock_str); 8344 bpf_diag_res( 8345 env, env->insn_idx, "unlock without lock", 8346 "This unlock operation has no matching active lock on the current path.", 8347 "Take the matching lock before this unlock, or remove the unmatched unlock path."); 8348 return -EINVAL; 8349 } 8350 8351 if (is_res_lock && is_irq) 8352 type = REF_TYPE_RES_LOCK_IRQ; 8353 else if (is_res_lock) 8354 type = REF_TYPE_RES_LOCK; 8355 else 8356 type = REF_TYPE_LOCK; 8357 8358 lock = find_lock_state(cur, type, reg->id, ptr); 8359 if (!lock) { 8360 verbose(env, "%s_unlock of different lock\n", lock_str); 8361 lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, 8362 cur->active_lock_ptr); 8363 bpf_diag_lock( 8364 env, env->insn_idx, "unlock of a different lock", 8365 "This unlock does not match any active lock with the same tracked identity on the current path.", 8366 "Unlock the same lock object that was most recently acquired.", lock); 8367 return -EINVAL; 8368 } 8369 if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { 8370 verbose(env, "%s_unlock cannot be out of order\n", lock_str); 8371 lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, 8372 cur->active_lock_ptr); 8373 bpf_diag_lock( 8374 env, env->insn_idx, "unlock out of order", 8375 "Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.", 8376 "Release nested locks in the reverse order they were acquired.", lock); 8377 return -EINVAL; 8378 } 8379 if (release_lock_state(env, type, reg->id, ptr)) { 8380 verbose(env, "%s_unlock of different lock\n", lock_str); 8381 bpf_diag_lock( 8382 env, env->insn_idx, "unlock of a different lock", 8383 "The verifier could not release a lock state matching this unlock operation.", 8384 "Pass the same lock object and lock kind that were used for the matching lock operation.", 8385 lock); 8386 return -EINVAL; 8387 } 8388 /* 8389 * Invalidate non-owning refs before RCU demotion clears their 8390 * NON_OWN_REF flag. 8391 */ 8392 invalidate_non_owning_refs(env); 8393 8394 if (!in_rcu_cs(env)) 8395 invalidate_rcu_protected_refs(env); 8396 } 8397 return 0; 8398 } 8399 8400 /* Check if @regno is a pointer to a specific field in a map value */ 8401 static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 8402 enum btf_field_type field_type, 8403 struct bpf_map_desc *map_desc) 8404 { 8405 bool is_const = tnum_is_const(reg->var_off); 8406 struct bpf_map *map = reg->map_ptr; 8407 u64 val = reg->var_off.value; 8408 const char *struct_name = btf_field_type_name(field_type); 8409 int field_off = -1; 8410 8411 if (!is_const) { 8412 verbose(env, 8413 "%s doesn't have constant offset. %s has to be at the constant offset\n", 8414 reg_arg_name(env, argno), struct_name); 8415 return -EINVAL; 8416 } 8417 if (!map->btf) { 8418 verbose(env, "map '%s' has to have BTF in order to use %s\n", map->name, 8419 struct_name); 8420 return -EINVAL; 8421 } 8422 if (!btf_record_has_field(map->record, field_type)) { 8423 verbose(env, "map '%s' has no valid %s\n", map->name, struct_name); 8424 return -EINVAL; 8425 } 8426 switch (field_type) { 8427 case BPF_TIMER: 8428 field_off = map->record->timer_off; 8429 break; 8430 case BPF_TASK_WORK: 8431 field_off = map->record->task_work_off; 8432 break; 8433 case BPF_RCU_HEAD: 8434 field_off = map->record->rcu_head_off; 8435 break; 8436 case BPF_WORKQUEUE: 8437 field_off = map->record->wq_off; 8438 break; 8439 default: 8440 verifier_bug(env, "unsupported BTF field type: %s\n", struct_name); 8441 return -EINVAL; 8442 } 8443 if (field_off != val) { 8444 verbose(env, "off %lld doesn't point to 'struct %s' that is at %d\n", 8445 val, struct_name, field_off); 8446 return -EINVAL; 8447 } 8448 if (map_desc->ptr) { 8449 verifier_bug(env, "Two map pointers in a %s helper", struct_name); 8450 return -EFAULT; 8451 } 8452 map_desc->uid = reg->map_uid; 8453 map_desc->ptr = map; 8454 return 0; 8455 } 8456 8457 static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 8458 struct bpf_map_desc *map) 8459 { 8460 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 8461 verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n"); 8462 return -EOPNOTSUPP; 8463 } 8464 return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); 8465 } 8466 8467 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 8468 struct bpf_call_arg_meta *meta) 8469 { 8470 struct bpf_reg_state *reg = reg_state(env, regno); 8471 struct btf_field *kptr_field; 8472 struct bpf_map *map_ptr; 8473 struct btf_record *rec; 8474 u32 kptr_off; 8475 8476 if (type_is_ptr_alloc_obj(reg->type)) { 8477 rec = reg_btf_record(reg); 8478 } else { /* PTR_TO_MAP_VALUE */ 8479 map_ptr = reg->map_ptr; 8480 if (!map_ptr->btf) { 8481 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 8482 map_ptr->name); 8483 return -EINVAL; 8484 } 8485 rec = map_ptr->record; 8486 meta->map.ptr = map_ptr; 8487 } 8488 8489 if (!tnum_is_const(reg->var_off)) { 8490 verbose(env, 8491 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 8492 regno); 8493 return -EINVAL; 8494 } 8495 8496 if (!btf_record_has_field(rec, BPF_KPTR)) { 8497 verbose(env, "R%d has no valid kptr\n", regno); 8498 return -EINVAL; 8499 } 8500 8501 kptr_off = reg->var_off.value; 8502 kptr_field = btf_record_find(rec, kptr_off, BPF_KPTR); 8503 if (!kptr_field) { 8504 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 8505 return -EACCES; 8506 } 8507 if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) { 8508 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 8509 return -EACCES; 8510 } 8511 meta->kptr_field = kptr_field; 8512 return 0; 8513 } 8514 8515 static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, 8516 const char *call_name, const char *reason, const char *suggestion); 8517 __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, 8518 argno_t argno, const char *call_name, 8519 const char *suggestion, const char *fmt, ...); 8520 8521 /* 8522 * Validate dynptr arguments for helper, kfunc and subprog. 8523 * 8524 * @meta carries the dynptr and referenced-object state. The dynptr is populated 8525 * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that 8526 * will be constructed) and consumed when the argument is expected to be an 8527 * initialized dynptr. The reference tracks the parent object (e.g., file or skb 8528 * in qdisc program) when constructing a dynptr. 8529 * 8530 * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK 8531 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. 8532 * 8533 * In both cases we deal with the first 8 bytes, but need to mark the next 8 8534 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of 8535 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. 8536 * 8537 * Mutability of bpf_dynptr is at two levels: the dynptr and the memory the 8538 * dynptr points to. At the first level, the verifier will make sure a 8539 * CONST_PTR_TO_DYNPTR cannot be reinitialized or destroyed. The mutability of 8540 * a dynptr's view (i.e., start and offset) is not tracked as there is not such 8541 * use case. The second level is tracked using the upper bit of bpf_dynptr->size 8542 * and checked dynamically during runtime. 8543 */ 8544 static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 8545 argno_t argno, int insn_idx, enum bpf_arg_type arg_type, 8546 struct bpf_call_arg_meta *meta) 8547 { 8548 int spi, err = 0; 8549 8550 /* MEM_UNINIT - Points to memory that is an appropriate candidate for 8551 * constructing a mutable bpf_dynptr object. 8552 * 8553 * Currently, this is only possible with PTR_TO_STACK 8554 * pointing to a region of at least 16 bytes which doesn't 8555 * contain an existing bpf_dynptr. 8556 * 8557 * OBJ_RELEASE - Points to a initialized bpf_dynptr that will be 8558 * destroyed. 8559 * 8560 * None - Points to a initialized dynptr that cannot be 8561 * reinitialized or destroyed. However, the view of the 8562 * dynptr and the memory it points to may be mutated. 8563 */ 8564 if (arg_type & MEM_UNINIT) { 8565 int i; 8566 8567 if (!is_dynptr_reg_valid_uninit(env, reg)) { 8568 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 8569 bpf_diag_res( 8570 env, insn_idx, "dynptr is already initialized", 8571 "This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", 8572 "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); 8573 return -EINVAL; 8574 } 8575 8576 /* we write BPF_DW bits (8 bytes) at a time */ 8577 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 8578 err = check_mem_access(env, insn_idx, reg, argno, 8579 i, BPF_DW, BPF_WRITE, -1, false, false); 8580 if (err) 8581 return err; 8582 } 8583 8584 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, 8585 &meta->ref_obj, &meta->dynptr); 8586 } else /* OBJ_RELEASE and None case from above */ { 8587 /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ 8588 if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { 8589 verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); 8590 bpf_diag_res( 8591 env, insn_idx, "const dynptr release", 8592 "This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.", 8593 "Release only mutable dynptrs that the program initialized or reserved."); 8594 return -EINVAL; 8595 } 8596 8597 if (!is_dynptr_reg_valid_init(env, reg)) { 8598 verbose(env, "Expected an initialized dynptr as %s\n", 8599 reg_arg_name(env, argno)); 8600 bpf_diag_res( 8601 env, insn_idx, "uninitialized dynptr use", 8602 "This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.", 8603 "Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use."); 8604 return -EINVAL; 8605 } 8606 8607 /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ 8608 if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { 8609 enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); 8610 enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg); 8611 8612 verbose(env, "Expected a dynptr of type %s as %s\n", 8613 dynptr_type_str(expected_type), reg_arg_name(env, argno)); 8614 bpf_diag_call_arg_fmt( 8615 env, insn_idx, argno, meta->func_name, 8616 "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", 8617 "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", 8618 dynptr_type_str(actual_type), dynptr_type_str(expected_type)); 8619 return -EINVAL; 8620 } 8621 8622 if (reg->type != CONST_PTR_TO_DYNPTR) { 8623 struct bpf_func_state *state = bpf_func(env, reg); 8624 8625 spi = dynptr_get_spi(env, reg); 8626 if (spi < 0) 8627 return spi; 8628 8629 mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); 8630 8631 reg = &bpf_stack_slot(state, spi)->spilled_ptr; 8632 } 8633 8634 meta->dynptr.type = reg->dynptr.type; 8635 meta->dynptr.id = reg->id; 8636 meta->dynptr.parent_id = reg->parent_id; 8637 } 8638 return err; 8639 } 8640 8641 static bool is_iter_kfunc(struct bpf_call_arg_meta *meta) 8642 { 8643 return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); 8644 } 8645 8646 static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta) 8647 { 8648 return meta->kfunc_flags & KF_ITER_NEW; 8649 } 8650 8651 static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) 8652 { 8653 return meta->kfunc_flags & KF_ITER_DESTROY; 8654 } 8655 8656 static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta) 8657 { 8658 const struct btf_type *t; 8659 8660 if (!(meta->kfunc_flags & KF_ITER_NEXT) || 8661 (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE))) 8662 return false; 8663 8664 t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL); 8665 if (!t || !btf_type_is_ptr(t)) 8666 return false; 8667 t = btf_type_skip_modifiers(meta->btf, t->type, NULL); 8668 return t && __btf_type_is_struct(t); 8669 } 8670 8671 static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, 8672 const struct btf_param *arg) 8673 { 8674 /* btf_check_iter_kfuncs() guarantees that first argument of any iter 8675 * kfunc is iter state pointer 8676 */ 8677 if (is_iter_kfunc(meta)) 8678 return arg_idx == 0; 8679 8680 /* iter passed as an argument to a generic kfunc */ 8681 return btf_param_match_suffix(meta->btf, arg, "__iter"); 8682 } 8683 8684 static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 8685 u32 arg, argno_t argno, int insn_idx, 8686 struct bpf_call_arg_meta *meta) 8687 { 8688 struct bpf_func_state *state = bpf_func(env, reg); 8689 const struct btf_type *t; 8690 int spi, err, i, nr_slots, btf_id; 8691 8692 if (reg->type != PTR_TO_STACK) { 8693 verbose(env, "%s expected pointer to an iterator on stack\n", 8694 reg_arg_name(env, argno)); 8695 bpf_diag_call_arg_fmt( 8696 env, insn_idx, argno, meta->func_name, 8697 "Pass the address of a stack iterator object for iterator new, next, and destroy calls.", 8698 "iterator state must live in verifier-tracked stack memory, but %s is %s", 8699 reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); 8700 return -EINVAL; 8701 } 8702 8703 /* For iter_{new,next,destroy} functions, btf_check_iter_kfuncs() 8704 * ensures struct convention, so we wouldn't need to do any BTF 8705 * validation here. But given iter state can be passed as a parameter 8706 * to any kfunc, if arg has "__iter" suffix, we need to be a bit more 8707 * conservative here. 8708 */ 8709 btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg); 8710 if (btf_id < 0) { 8711 verbose(env, "expected valid iter pointer as %s\n", 8712 reg_arg_name(env, argno)); 8713 bpf_diag_call_arg( 8714 env, insn_idx, argno, meta->func_name, 8715 "the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type", 8716 "Pass the exact iterator state type expected by this function."); 8717 return -EINVAL; 8718 } 8719 t = btf_type_by_id(meta->btf, btf_id); 8720 nr_slots = t->size / BPF_REG_SIZE; 8721 8722 if (is_iter_new_kfunc(meta)) { 8723 /* bpf_iter_<type>_new() expects pointer to uninit iter state */ 8724 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { 8725 verbose(env, "expected uninitialized iter_%s as %s\n", 8726 iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); 8727 bpf_diag_res( 8728 env, insn_idx, "iterator is already initialized", 8729 "Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.", 8730 "Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot."); 8731 return -EINVAL; 8732 } 8733 8734 for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { 8735 err = check_mem_access(env, insn_idx, reg, argno, 8736 i, BPF_DW, BPF_WRITE, -1, false, false); 8737 if (err) 8738 return err; 8739 } 8740 8741 err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots); 8742 if (err) 8743 return err; 8744 } else { 8745 /* iter_next() or iter_destroy(), as well as any kfunc 8746 * accepting iter argument, expect initialized iter state 8747 */ 8748 err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots); 8749 switch (err) { 8750 case 0: 8751 break; 8752 case -EINVAL: 8753 verbose(env, "expected an initialized iter_%s as %s\n", 8754 iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); 8755 bpf_diag_res( 8756 env, insn_idx, "uninitialized iterator use", 8757 "This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.", 8758 "Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use."); 8759 return err; 8760 case -EPROTO: 8761 verbose(env, "expected an RCU CS when using %s\n", meta->func_name); 8762 bpf_diag_ctx_required( 8763 env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU, 8764 "Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); 8765 return err; 8766 default: 8767 return err; 8768 } 8769 8770 spi = iter_get_spi(env, reg, nr_slots); 8771 if (spi < 0) 8772 return spi; 8773 8774 mark_stack_slots_scratched(env, spi, nr_slots); 8775 8776 /* remember meta->iter info for process_iter_next_call() */ 8777 meta->iter.spi = spi; 8778 meta->iter.frameno = reg->frameno; 8779 update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); 8780 8781 /* The iterator stays. What was borrowed from the previous element goes. */ 8782 if (iter_next_borrows_struct(meta)) { 8783 err = invalidate_reference(env, meta->ref_obj.id); 8784 if (err) 8785 return err; 8786 } 8787 8788 if (is_iter_destroy_kfunc(meta)) { 8789 err = unmark_stack_slots_iter(env, reg, nr_slots); 8790 if (err) 8791 return err; 8792 } 8793 } 8794 8795 return 0; 8796 } 8797 8798 /* Look for a previous loop entry at insn_idx: nearest parent state 8799 * stopped at insn_idx with callsites matching those in cur->frame. 8800 */ 8801 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env, 8802 struct bpf_verifier_state *cur, 8803 int insn_idx) 8804 { 8805 struct bpf_verifier_state_list *sl; 8806 struct bpf_verifier_state *st; 8807 struct list_head *pos, *head; 8808 8809 /* Explored states are pushed in stack order, most recent states come first */ 8810 head = bpf_explored_state(env, insn_idx); 8811 list_for_each(pos, head) { 8812 sl = container_of(pos, struct bpf_verifier_state_list, node); 8813 /* If st->branches != 0 state is a part of current DFS verification path, 8814 * hence cur & st for a loop. 8815 */ 8816 st = &sl->state; 8817 if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) && 8818 st->dfs_depth < cur->dfs_depth) 8819 return st; 8820 } 8821 8822 return NULL; 8823 } 8824 8825 /* 8826 * Check if scalar registers are exact for the purpose of not widening. 8827 * More lenient than regs_exact() 8828 */ 8829 static bool scalars_exact_for_widen(const struct bpf_reg_state *rold, 8830 const struct bpf_reg_state *rcur) 8831 { 8832 return !memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)); 8833 } 8834 8835 static void maybe_widen_reg(struct bpf_verifier_env *env, 8836 struct bpf_reg_state *rold, struct bpf_reg_state *rcur) 8837 { 8838 if (rold->type != SCALAR_VALUE) 8839 return; 8840 if (rold->type != rcur->type) 8841 return; 8842 if (rold->precise || rcur->precise || scalars_exact_for_widen(rold, rcur)) 8843 return; 8844 __mark_reg_unknown(env, rcur); 8845 } 8846 8847 static int widen_imprecise_scalars(struct bpf_verifier_env *env, 8848 struct bpf_verifier_state *old, 8849 struct bpf_verifier_state *cur) 8850 { 8851 struct bpf_func_state *fold, *fcur; 8852 int i, fr, num_slots; 8853 8854 for (fr = old->curframe; fr >= 0; fr--) { 8855 fold = old->frame[fr]; 8856 fcur = cur->frame[fr]; 8857 8858 for (i = 0; i < MAX_BPF_REG; i++) 8859 maybe_widen_reg(env, 8860 &fold->regs[i], 8861 &fcur->regs[i]); 8862 8863 num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur)); 8864 for (i = 0; i < num_slots; i++) { 8865 struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i); 8866 struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i); 8867 8868 if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss)) 8869 continue; 8870 8871 maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr); 8872 } 8873 } 8874 return 0; 8875 } 8876 8877 static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st, 8878 struct bpf_call_arg_meta *meta) 8879 { 8880 int iter_frameno = meta->iter.frameno; 8881 int iter_spi = meta->iter.spi; 8882 8883 return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr; 8884 } 8885 8886 /* process_iter_next_call() is called when verifier gets to iterator's next 8887 * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer 8888 * to it as just "iter_next()" in comments below. 8889 * 8890 * BPF verifier relies on a crucial contract for any iter_next() 8891 * implementation: it should *eventually* return NULL, and once that happens 8892 * it should keep returning NULL. That is, once iterator exhausts elements to 8893 * iterate, it should never reset or spuriously return new elements. 8894 * 8895 * With the assumption of such contract, process_iter_next_call() simulates 8896 * a fork in the verifier state to validate loop logic correctness and safety 8897 * without having to simulate infinite amount of iterations. 8898 * 8899 * In current state, we first assume that iter_next() returned NULL and 8900 * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such 8901 * conditions we should not form an infinite loop and should eventually reach 8902 * exit. 8903 * 8904 * Besides that, we also fork current state and enqueue it for later 8905 * verification. In a forked state we keep iterator state as ACTIVE 8906 * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We 8907 * also bump iteration depth to prevent erroneous infinite loop detection 8908 * later on (see iter_active_depths_differ() comment for details). In this 8909 * state we assume that we'll eventually loop back to another iter_next() 8910 * calls (it could be in exactly same location or in some other instruction, 8911 * it doesn't matter, we don't make any unnecessary assumptions about this, 8912 * everything revolves around iterator state in a stack slot, not which 8913 * instruction is calling iter_next()). When that happens, we either will come 8914 * to iter_next() with equivalent state and can conclude that next iteration 8915 * will proceed in exactly the same way as we just verified, so it's safe to 8916 * assume that loop converges. If not, we'll go on another iteration 8917 * simulation with a different input state, until all possible starting states 8918 * are validated or we reach maximum number of instructions limit. 8919 * 8920 * This way, we will either exhaustively discover all possible input states 8921 * that iterator loop can start with and eventually will converge, or we'll 8922 * effectively regress into bounded loop simulation logic and either reach 8923 * maximum number of instructions if loop is not provably convergent, or there 8924 * is some statically known limit on number of iterations (e.g., if there is 8925 * an explicit `if n > 100 then break;` statement somewhere in the loop). 8926 * 8927 * Iteration convergence logic in is_state_visited() relies on exact 8928 * states comparison, which ignores read and precision marks. 8929 * This is necessary because read and precision marks are not finalized 8930 * while in the loop. Exact comparison might preclude convergence for 8931 * simple programs like below: 8932 * 8933 * i = 0; 8934 * while(iter_next(&it)) 8935 * i++; 8936 * 8937 * At each iteration step i++ would produce a new distinct state and 8938 * eventually instruction processing limit would be reached. 8939 * 8940 * To avoid such behavior speculatively forget (widen) range for 8941 * imprecise scalar registers, if those registers were not precise at the 8942 * end of the previous iteration and do not match exactly. 8943 * 8944 * This is a conservative heuristic that allows to verify wide range of programs, 8945 * however it precludes verification of programs that conjure an 8946 * imprecise value on the first loop iteration and use it as precise on a second. 8947 * For example, the following safe program would fail to verify: 8948 * 8949 * struct bpf_num_iter it; 8950 * int arr[10]; 8951 * int i = 0, a = 0; 8952 * bpf_iter_num_new(&it, 0, 10); 8953 * while (bpf_iter_num_next(&it)) { 8954 * if (a == 0) { 8955 * a = 1; 8956 * i = 7; // Because i changed verifier would forget 8957 * // it's range on second loop entry. 8958 * } else { 8959 * arr[i] = 42; // This would fail to verify. 8960 * } 8961 * } 8962 * bpf_iter_num_destroy(&it); 8963 */ 8964 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, 8965 struct bpf_call_arg_meta *meta) 8966 { 8967 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; 8968 struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; 8969 struct bpf_reg_state *cur_iter, *queued_iter; 8970 8971 BTF_TYPE_EMIT(struct bpf_iter); 8972 8973 cur_iter = get_iter_from_state(cur_st, meta); 8974 8975 if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE && 8976 cur_iter->iter.state != BPF_ITER_STATE_DRAINED) { 8977 verifier_bug(env, "unexpected iterator state %d (%s)", 8978 cur_iter->iter.state, iter_state_str(cur_iter->iter.state)); 8979 return -EFAULT; 8980 } 8981 8982 if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) { 8983 /* Because iter_next() call is a checkpoint is_state_visitied() 8984 * should guarantee parent state with same call sites and insn_idx. 8985 */ 8986 if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx || 8987 !same_callsites(cur_st->parent, cur_st)) { 8988 verifier_bug(env, "bad parent state for iter next call"); 8989 return -EFAULT; 8990 } 8991 /* Note cur_st->parent in the call below, it is necessary to skip 8992 * checkpoint created for cur_st by is_state_visited() 8993 * right at this instruction. 8994 */ 8995 prev_st = find_prev_entry(env, cur_st->parent, insn_idx); 8996 /* branch out active iter state */ 8997 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); 8998 if (IS_ERR(queued_st)) 8999 return PTR_ERR(queued_st); 9000 9001 queued_iter = get_iter_from_state(queued_st, meta); 9002 queued_iter->iter.state = BPF_ITER_STATE_ACTIVE; 9003 queued_iter->iter.depth++; 9004 if (prev_st) 9005 widen_imprecise_scalars(env, prev_st, queued_st); 9006 9007 queued_fr = queued_st->frame[queued_st->curframe]; 9008 mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]); 9009 } 9010 9011 /* switch to DRAINED state, but keep the depth unchanged */ 9012 /* mark current iter state as drained and assume returned NULL */ 9013 cur_iter->iter.state = BPF_ITER_STATE_DRAINED; 9014 __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]); 9015 9016 return 0; 9017 } 9018 9019 static bool arg_type_is_mem_size(enum bpf_arg_type type) 9020 { 9021 return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; 9022 } 9023 9024 static bool arg_type_is_raw_mem(enum bpf_arg_type type) 9025 { 9026 /* 9027 * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw 9028 * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be 9029 * passed as a PTR_TO_STACK that reaches check_stack_range_initialized(). 9030 */ 9031 return (base_type(type) == ARG_PTR_TO_MEM || 9032 base_type(type) == ARG_PTR_TO_MAP_VALUE) && 9033 type & MEM_UNINIT; 9034 } 9035 9036 static bool arg_type_is_release(enum bpf_arg_type type) 9037 { 9038 return type & OBJ_RELEASE; 9039 } 9040 9041 static bool arg_type_is_dynptr(enum bpf_arg_type type) 9042 { 9043 return base_type(type) == ARG_PTR_TO_DYNPTR; 9044 } 9045 9046 /* 9047 * An argument that only ever takes a scalar, so a zero register passed to it 9048 * is a value rather than a NULL pointer. 9049 */ 9050 static bool arg_type_is_scalar(enum bpf_arg_type type) 9051 { 9052 switch (base_type(type)) { 9053 case ARG_SCALAR: 9054 case ARG_CONST_SCALAR: 9055 case ARG_MEM_SIZE: 9056 case ARG_MEM_SIZE_OR_ZERO: 9057 case ARG_CONST_MEM_SIZE: 9058 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 9059 return true; 9060 default: 9061 return false; 9062 } 9063 } 9064 9065 /* 9066 * A helper has no BTF and a nonzero function ID. A kfunc has both, while a 9067 * BPF subprogram has BTF and a zero function ID. 9068 */ 9069 static bool is_helper(const struct bpf_call_arg_meta *meta) 9070 { 9071 return !meta->btf && meta->func_id; 9072 } 9073 9074 static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) 9075 { 9076 return is_helper(meta) && meta->func_id == func_id; 9077 } 9078 9079 static bool is_kfunc(const struct bpf_call_arg_meta *meta) 9080 { 9081 return meta->btf && meta->func_id; 9082 } 9083 9084 static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) 9085 { 9086 return is_kfunc(meta) && meta->func_id == btf_id; 9087 } 9088 9089 static int resolve_map_arg_type(struct bpf_verifier_env *env, 9090 const struct bpf_call_arg_meta *meta, 9091 enum bpf_arg_type *arg_type) 9092 { 9093 if (!meta->map.ptr) { 9094 /* kernel subsystem misconfigured verifier */ 9095 verifier_bug(env, "invalid map_ptr to access map->type"); 9096 return -EFAULT; 9097 } 9098 9099 switch (meta->map.ptr->map_type) { 9100 case BPF_MAP_TYPE_SOCKMAP: 9101 case BPF_MAP_TYPE_SOCKHASH: 9102 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 9103 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 9104 } else { 9105 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 9106 return -EINVAL; 9107 } 9108 break; 9109 case BPF_MAP_TYPE_BLOOM_FILTER: 9110 if (is_helper_call(meta, BPF_FUNC_map_peek_elem)) 9111 *arg_type = ARG_PTR_TO_MAP_VALUE; 9112 break; 9113 default: 9114 break; 9115 } 9116 return 0; 9117 } 9118 9119 static int resolve_func_arg_type(struct bpf_verifier_env *env, 9120 struct bpf_reg_state *reg, u32 arg, argno_t argno, 9121 struct bpf_call_arg_meta *meta, 9122 enum bpf_arg_type *arg_type, u32 *arg_size); 9123 static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 9124 argno_t argno, enum bpf_arg_type arg_type, 9125 const struct btf *arg_btf, u32 arg_btf_id, 9126 struct bpf_call_arg_meta *meta, int insn_idx); 9127 static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, 9128 const struct btf_param *arg); 9129 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 9130 const struct btf_param *arg, 9131 const char *name); 9132 static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); 9133 static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); 9134 static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); 9135 static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta); 9136 static bool is_bpf_obj_drop_kfunc(u32 func_id); 9137 static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); 9138 static bool is_bpf_rbtree_add_kfunc(u32 func_id); 9139 static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta); 9140 static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); 9141 static int process_irq_flag(struct bpf_verifier_env *env, 9142 struct bpf_reg_state *reg, argno_t argno, 9143 struct bpf_call_arg_meta *meta); 9144 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 9145 struct bpf_reg_state *reg, 9146 argno_t argno, 9147 struct bpf_call_arg_meta *meta); 9148 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, 9149 struct bpf_reg_state *reg, 9150 argno_t argno, 9151 struct bpf_call_arg_meta *meta); 9152 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 9153 struct bpf_reg_state *reg, 9154 argno_t argno, 9155 struct bpf_call_arg_meta *meta); 9156 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, 9157 struct bpf_reg_state *reg, 9158 argno_t argno, 9159 struct bpf_call_arg_meta *meta); 9160 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env); 9161 9162 struct bpf_reg_types { 9163 const enum bpf_reg_type types[10]; 9164 u32 *btf_id; 9165 }; 9166 9167 static const struct bpf_reg_types sock_types = { 9168 .types = { 9169 PTR_TO_SOCK_COMMON, 9170 PTR_TO_SOCKET, 9171 PTR_TO_TCP_SOCK, 9172 PTR_TO_XDP_SOCK, 9173 }, 9174 }; 9175 9176 #ifdef CONFIG_NET 9177 static const struct bpf_reg_types btf_id_sock_common_types = { 9178 .types = { 9179 PTR_TO_SOCK_COMMON, 9180 PTR_TO_SOCKET, 9181 PTR_TO_TCP_SOCK, 9182 PTR_TO_XDP_SOCK, 9183 PTR_TO_BTF_ID, 9184 PTR_TO_BTF_ID | PTR_TRUSTED, 9185 }, 9186 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 9187 }; 9188 #endif 9189 9190 static const struct bpf_reg_types mem_types = { 9191 .types = { 9192 PTR_TO_STACK, 9193 PTR_TO_PACKET, 9194 PTR_TO_PACKET_META, 9195 PTR_TO_MAP_KEY, 9196 PTR_TO_MAP_VALUE, 9197 PTR_TO_MEM, 9198 PTR_TO_MEM | MEM_RINGBUF, 9199 PTR_TO_BUF, 9200 PTR_TO_BTF_ID | PTR_TRUSTED, 9201 PTR_TO_CTX, 9202 }, 9203 }; 9204 9205 static const struct bpf_reg_types map_value_or_alloc_obj_types = { 9206 .types = { 9207 PTR_TO_MAP_VALUE, 9208 PTR_TO_BTF_ID | MEM_ALLOC, 9209 } 9210 }; 9211 9212 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 9213 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 9214 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 9215 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 9216 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 9217 static const struct bpf_reg_types btf_ptr_types = { 9218 .types = { 9219 PTR_TO_BTF_ID, 9220 PTR_TO_BTF_ID | PTR_TRUSTED, 9221 PTR_TO_BTF_ID | MEM_RCU, 9222 }, 9223 }; 9224 static const struct bpf_reg_types percpu_btf_ptr_types = { 9225 .types = { 9226 PTR_TO_BTF_ID | MEM_PERCPU, 9227 PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU, 9228 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 9229 } 9230 }; 9231 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 9232 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 9233 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 9234 static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } }; 9235 static const struct bpf_reg_types arena_types = { 9236 .types = { 9237 PTR_TO_ARENA, 9238 SCALAR_VALUE, 9239 } 9240 }; 9241 static const struct bpf_reg_types ctx_out_types = { 9242 .types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED }, 9243 }; 9244 9245 static const struct bpf_reg_types alloc_obj_drop_types = { 9246 .types = { 9247 PTR_TO_BTF_ID | MEM_ALLOC, 9248 PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU, 9249 } 9250 }; 9251 9252 static const struct bpf_reg_types alloc_obj_types = { 9253 .types = { 9254 PTR_TO_BTF_ID | MEM_ALLOC, 9255 PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU, 9256 PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, 9257 PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, 9258 } 9259 }; 9260 9261 static const struct bpf_reg_types kptr_xchg_dest_types = { 9262 .types = { 9263 PTR_TO_MAP_VALUE, 9264 PTR_TO_BTF_ID | MEM_ALLOC, 9265 PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, 9266 PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, 9267 } 9268 }; 9269 static const struct bpf_reg_types dynptr_types = { 9270 .types = { 9271 PTR_TO_STACK, 9272 CONST_PTR_TO_DYNPTR, 9273 } 9274 }; 9275 9276 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 9277 [ARG_PTR_TO_MAP_KEY] = &mem_types, 9278 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 9279 [ARG_MEM_SIZE] = &scalar_types, 9280 [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, 9281 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 9282 [ARG_SCALAR] = &scalar_types, 9283 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 9284 [ARG_PTR_TO_CTX] = &context_types, 9285 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 9286 #ifdef CONFIG_NET 9287 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 9288 #endif 9289 [ARG_PTR_TO_SOCKET] = &fullsock_types, 9290 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 9291 [ARG_PTR_TO_SPIN_LOCK] = &map_value_or_alloc_obj_types, 9292 [ARG_PTR_TO_MEM] = &mem_types, 9293 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 9294 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 9295 [ARG_PTR_TO_FUNC] = &func_ptr_types, 9296 [ARG_PTR_TO_STACK] = &stack_ptr_types, 9297 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 9298 [ARG_PTR_TO_TIMER] = &map_value_types, 9299 [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, 9300 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 9301 [ARG_CONST_SCALAR] = &scalar_types, 9302 [ARG_CONST_MEM_SIZE] = &scalar_types, 9303 [ARG_PTR_TO_ALLOC_BTF_ID] = &alloc_obj_drop_types, 9304 [ARG_PTR_TO_REFCOUNTED_KPTR] = &alloc_obj_types, 9305 [ARG_PTR_TO_ITER] = &stack_ptr_types, 9306 [ARG_PTR_TO_LIST_HEAD] = &map_value_or_alloc_obj_types, 9307 [ARG_PTR_TO_LIST_NODE] = &alloc_obj_types, 9308 [ARG_PTR_TO_RB_ROOT] = &map_value_or_alloc_obj_types, 9309 [ARG_PTR_TO_RB_NODE] = &alloc_obj_types, 9310 [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, 9311 [ARG_PTR_TO_WORKQUEUE] = &map_value_types, 9312 [ARG_PTR_TO_TASK_WORK] = &map_value_types, 9313 [ARG_PTR_TO_RCU_HEAD] = &map_value_types, 9314 [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, 9315 [ARG_PTR_TO_ARENA] = &arena_types, 9316 [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, 9317 }; 9318 9319 static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, 9320 const char *call_name, const char *reason, 9321 const char *suggestion) 9322 { 9323 int arg = arg_from_argno(argno); 9324 int regno = reg_from_argno(argno); 9325 int stack_slot = -1; 9326 9327 if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5) 9328 arg = regno; 9329 if (arg > MAX_BPF_FUNC_REG_ARGS) 9330 stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1; 9331 9332 bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot, 9333 call_name && *call_name ? call_name : "call", 9334 reg_arg_name(env, argno), reason, suggestion); 9335 } 9336 9337 static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno) 9338 { 9339 return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno)); 9340 } 9341 9342 __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, 9343 argno_t argno, const char *call_name, 9344 const char *suggestion, const char *fmt, ...) 9345 { 9346 const char *reason; 9347 va_list args; 9348 9349 va_start(args, fmt); 9350 reason = bpf_diag_vfmt(env, fmt, args); 9351 va_end(args); 9352 9353 bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); 9354 } 9355 9356 static int check_func_arg_nullability(struct bpf_verifier_env *env, 9357 struct bpf_reg_state *reg, u32 arg, argno_t argno, 9358 enum bpf_arg_type arg_type, 9359 struct bpf_call_arg_meta *meta, int insn_idx) 9360 { 9361 const char *expected_type = "pointer"; 9362 9363 if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) || 9364 (!bpf_register_is_null(reg) && !type_may_be_null(reg->type))) 9365 return 0; 9366 9367 if (meta->btf) { 9368 u32 arg_btf_id; 9369 9370 arg_btf_id = btf_params(meta->func_proto)[arg].type; 9371 expected_type = bpf_diag_fmt(env, "value of type %s", 9372 bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); 9373 } 9374 9375 verbose(env, "Possibly NULL pointer passed to trusted %s\n", 9376 reg_arg_name(env, argno)); 9377 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 9378 "Add a NULL check and make the call only on the non-NULL path.", 9379 "the pointer may be NULL, but this call requires a non-NULL %s", 9380 expected_type); 9381 return -EACCES; 9382 } 9383 9384 static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 9385 u32 arg, argno_t argno, enum bpf_arg_type arg_type, 9386 struct bpf_call_arg_meta *meta, int insn_idx) 9387 { 9388 const char *expected_type = "pointer"; 9389 9390 if (!arg_type_is_release(arg_type)) 9391 return 0; 9392 9393 if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) || 9394 bpf_register_is_null(reg)) 9395 return 0; 9396 9397 verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", 9398 meta->func_name, reg_arg_name(env, argno)); 9399 9400 if (is_kfunc(meta)) { 9401 const struct btf_param *btf_arg; 9402 const struct btf_type *t; 9403 u32 ref_id; 9404 9405 btf_arg = &btf_params(meta->func_proto)[arg]; 9406 ref_id = btf_arg->type; 9407 t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); 9408 if (btf_type_is_ptr(t)) 9409 btf_type_skip_modifiers(meta->btf, t->type, &ref_id); 9410 expected_type = bpf_diag_fmt(env, "value of type %s", 9411 bpf_diag_fmt_btf_type(env, meta->btf, ref_id)); 9412 } 9413 9414 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 9415 bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.", 9416 expected_type), 9417 "release functions require a value that owns a live resource returned by a matching acquire function"); 9418 return -EINVAL; 9419 } 9420 9421 static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, 9422 const enum bpf_reg_type *types, int count) 9423 { 9424 size_t len = 0, size = 1; 9425 char *buf; 9426 int i; 9427 9428 for (i = 0; i < count; i++) 9429 size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0); 9430 9431 buf = bpf_diag_fmt_buf(env, size); 9432 if (!buf) 9433 return ""; 9434 9435 for (i = 0; i < count; i++) 9436 len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "", 9437 reg_type_str(env, types[i])); 9438 return buf; 9439 } 9440 9441 static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 9442 enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta) 9443 { 9444 enum bpf_reg_type expected, type = reg->type; 9445 const struct bpf_reg_types *compatible; 9446 const char *actual, *accepted; 9447 int i, j; 9448 9449 compatible = compatible_reg_types[base_type(arg_type)]; 9450 if (!compatible) { 9451 verifier_bug(env, "unsupported arg type %d", arg_type); 9452 return -EFAULT; 9453 } 9454 if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID && 9455 (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) 9456 goto found; 9457 9458 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 9459 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 9460 * 9461 * Same for MAYBE_NULL: 9462 * 9463 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 9464 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 9465 * 9466 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type. 9467 * 9468 * Therefore we fold these flags depending on the arg_type before comparison. 9469 */ 9470 if (arg_type & MEM_RDONLY) 9471 type &= ~MEM_RDONLY; 9472 if (arg_type & PTR_MAYBE_NULL) 9473 type &= ~PTR_MAYBE_NULL; 9474 if (base_type(arg_type) == ARG_PTR_TO_MEM) 9475 type &= ~DYNPTR_TYPE_FLAG_MASK; 9476 /* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */ 9477 if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && 9478 type_is_ptr_alloc_obj(type)) 9479 type = PTR_TO_MEM; 9480 9481 /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ 9482 if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && 9483 reg_from_argno(argno) == BPF_REG_2) { 9484 type &= ~MEM_ALLOC; 9485 type &= ~MEM_PERCPU; 9486 } 9487 9488 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 9489 expected = compatible->types[i]; 9490 if (expected == NOT_INIT) 9491 break; 9492 9493 if (type == expected) 9494 goto found; 9495 } 9496 9497 verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type)); 9498 for (j = 0; j + 1 < i; j++) 9499 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 9500 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 9501 actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); 9502 accepted = bpf_diag_expected_reg_types(env, compatible->types, i); 9503 bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, 9504 bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)), 9505 "it has type %s, but this argument accepts %s", 9506 actual, accepted); 9507 return -EACCES; 9508 9509 found: 9510 return 0; 9511 } 9512 9513 static struct btf_field * 9514 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) 9515 { 9516 struct btf_field *field; 9517 struct btf_record *rec; 9518 9519 rec = reg_btf_record(reg); 9520 if (!rec) 9521 return NULL; 9522 9523 field = btf_record_find(rec, off, fields); 9524 if (!field) 9525 return NULL; 9526 9527 return field; 9528 } 9529 9530 static int check_func_arg_reg_off(struct bpf_verifier_env *env, 9531 const struct bpf_reg_state *reg, argno_t argno, 9532 enum bpf_arg_type arg_type) 9533 { 9534 u32 type = reg->type; 9535 9536 /* When referenced register is passed to release function, its fixed 9537 * offset must be 0. 9538 * 9539 * We will check arg_type_is_release reg has id when storing 9540 * meta->release_regno. 9541 */ 9542 if (arg_type_is_release(arg_type)) { 9543 /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it 9544 * may not directly point to the object being released, but to 9545 * dynptr pointing to such object, which might be at some offset 9546 * on the stack. In that case, we simply to fallback to the 9547 * default handling. 9548 */ 9549 if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) 9550 return 0; 9551 9552 /* Doing check_ptr_off_reg check for the offset will catch this 9553 * because fixed_off_ok is false, but checking here allows us 9554 * to give the user a better error message. 9555 */ 9556 if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) { 9557 verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n", 9558 reg_arg_name(env, argno)); 9559 return -EINVAL; 9560 } 9561 } 9562 9563 switch (type) { 9564 /* Pointer types where both fixed and variable offset is explicitly allowed: */ 9565 case PTR_TO_STACK: 9566 case PTR_TO_PACKET: 9567 case PTR_TO_PACKET_META: 9568 case PTR_TO_MAP_KEY: 9569 case PTR_TO_MAP_VALUE: 9570 case PTR_TO_MEM: 9571 case PTR_TO_MEM | MEM_RDONLY: 9572 case PTR_TO_MEM | MEM_RINGBUF: 9573 case PTR_TO_BUF: 9574 case PTR_TO_BUF | MEM_RDONLY: 9575 case PTR_TO_ARENA: 9576 case SCALAR_VALUE: 9577 return 0; 9578 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 9579 * fixed offset. 9580 */ 9581 case PTR_TO_BTF_ID: 9582 case PTR_TO_BTF_ID | MEM_ALLOC: 9583 case PTR_TO_BTF_ID | PTR_TRUSTED: 9584 case PTR_TO_BTF_ID | MEM_RCU: 9585 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: 9586 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: 9587 /* When referenced PTR_TO_BTF_ID is passed to release function, 9588 * its fixed offset must be 0. bpf_refcount_acquire() returns the 9589 * pointer it was given while incrementing the refcount at the 9590 * refcount field offset, so it needs a zero offset too. In the 9591 * other cases, fixed offset can be non-zero. var_off always must 9592 * be 0 for PTR_TO_BTF_ID, hence we still need to do checks 9593 * instead of returning. 9594 */ 9595 return __check_ptr_off_reg(env, reg, argno, 9596 base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR); 9597 case PTR_TO_CTX: 9598 /* 9599 * Allow fixed and variable offsets for syscall context, but 9600 * only when the argument is passed as memory, not ctx, 9601 * otherwise we may get modified ctx in tail called programs and 9602 * global subprogs (that may act as extension prog hooks). 9603 */ 9604 if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) 9605 return 0; 9606 fallthrough; 9607 default: 9608 return __check_ptr_off_reg(env, reg, argno, false); 9609 } 9610 } 9611 9612 static int check_arg_const_str(struct bpf_verifier_env *env, 9613 struct bpf_reg_state *reg, argno_t argno) 9614 { 9615 struct bpf_map *map = reg->map_ptr; 9616 int err; 9617 int map_off; 9618 u64 map_addr; 9619 char *str_ptr; 9620 u32 cnt; 9621 9622 if (reg->type != PTR_TO_MAP_VALUE) 9623 return -EINVAL; 9624 9625 if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { 9626 verbose(env, "%s points to insn_array map which cannot be used as const string\n", 9627 reg_arg_name(env, argno)); 9628 return -EACCES; 9629 } 9630 9631 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 9632 verbose(env, "%s points to percpu_array map which cannot be used as const string\n", 9633 reg_arg_name(env, argno)); 9634 return -EACCES; 9635 } 9636 9637 if (!bpf_map_is_rdonly(map)) { 9638 verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); 9639 return -EACCES; 9640 } 9641 9642 if (!tnum_is_const(reg->var_off)) { 9643 verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno)); 9644 return -EACCES; 9645 } 9646 9647 if (!map->ops->map_direct_value_addr) { 9648 verbose(env, "no direct value access support for this map type\n"); 9649 return -EACCES; 9650 } 9651 9652 err = check_map_access(env, reg, argno, 0, 9653 map->value_size - reg->var_off.value, false, 9654 ACCESS_HELPER); 9655 if (err) 9656 return err; 9657 9658 map_off = reg->var_off.value; 9659 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 9660 if (err) { 9661 verbose(env, "direct value access on string failed\n"); 9662 return err; 9663 } 9664 9665 str_ptr = (char *)(long)(map_addr); 9666 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 9667 verbose(env, "string is not zero-terminated\n"); 9668 return -EINVAL; 9669 } 9670 /* the bytes of a pointer to a function are not known until the program is jitted */ 9671 if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { 9672 verbose(env, "string overlaps with a pointer to a function\n"); 9673 return -EACCES; 9674 } 9675 return 0; 9676 } 9677 9678 /* Returns constant key value in `value` if possible, else negative error */ 9679 static int get_constant_map_key(struct bpf_verifier_env *env, 9680 struct bpf_reg_state *key, 9681 u32 key_size, 9682 s64 *value) 9683 { 9684 struct bpf_func_state *state = bpf_func(env, key); 9685 struct bpf_stack_state *ss; 9686 struct bpf_reg_state *reg; 9687 int slot, spi, off; 9688 int spill_size = 0; 9689 int zero_size = 0; 9690 int stack_off; 9691 int i, err; 9692 u8 *stype; 9693 9694 if (!env->bpf_capable) 9695 return -EOPNOTSUPP; 9696 if (key->type != PTR_TO_STACK) 9697 return -EOPNOTSUPP; 9698 if (!tnum_is_const(key->var_off)) 9699 return -EOPNOTSUPP; 9700 9701 stack_off = key->var_off.value; 9702 slot = -stack_off - 1; 9703 spi = slot / BPF_REG_SIZE; 9704 off = slot % BPF_REG_SIZE; 9705 ss = bpf_stack_slot(state, spi); 9706 stype = ss->slot_type; 9707 9708 /* First handle precisely tracked STACK_ZERO */ 9709 for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) 9710 zero_size++; 9711 if (zero_size >= key_size) { 9712 *value = 0; 9713 return 0; 9714 } 9715 9716 /* Check that stack contains a scalar spill of expected size */ 9717 if (!bpf_is_spilled_scalar_reg(ss)) 9718 return -EOPNOTSUPP; 9719 for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) 9720 spill_size++; 9721 if (spill_size != key_size) 9722 return -EOPNOTSUPP; 9723 9724 reg = &ss->spilled_ptr; 9725 if (!tnum_is_const(reg->var_off)) 9726 /* Stack value not statically known */ 9727 return -EOPNOTSUPP; 9728 9729 /* We are relying on a constant value. So mark as precise 9730 * to prevent pruning on it. 9731 */ 9732 bpf_bt_set_frame_slot(&env->bt, key->frameno, spi); 9733 err = mark_chain_precision_batch(env, env->cur_state); 9734 if (err < 0) 9735 return err; 9736 9737 *value = reg->var_off.value; 9738 return 0; 9739 } 9740 9741 static bool can_elide_value_nullness(const struct bpf_map *map); 9742 9743 static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 9744 argno_t argno, struct bpf_call_arg_meta *meta) 9745 { 9746 /* Use map_uid (which is unique id of inner map) to reject: 9747 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 9748 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 9749 * if (inner_map1 && inner_map2) { 9750 * timer = bpf_map_lookup_elem(inner_map1); 9751 * if (timer) 9752 * // mismatch would have been allowed 9753 * bpf_timer_init(timer, inner_map2); 9754 * } 9755 * 9756 * Comparing map_ptr is enough to distinguish normal and outer maps. 9757 */ 9758 if (meta->map.ptr && 9759 (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { 9760 argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); 9761 struct btf_record *rec = meta->map.ptr->record; 9762 const char *obj_name = "workqueue"; 9763 9764 if (rec->timer_off >= 0) 9765 obj_name = "timer"; 9766 else if (rec->task_work_off >= 0) 9767 obj_name = "bpf_task_work"; 9768 else if (rec->rcu_head_off >= 0) 9769 obj_name = "bpf_rcu_head"; 9770 9771 verbose(env, "%s pointer in %s map_uid=%d ", 9772 obj_name, reg_arg_name(env, obj_argno), meta->map.uid); 9773 verbose(env, "doesn't match map pointer in %s map_uid=%d\n", 9774 reg_arg_name(env, argno), reg->map_uid); 9775 return -EINVAL; 9776 } 9777 9778 meta->map.ptr = reg->map_ptr; 9779 meta->map.uid = reg->map_uid; 9780 return 0; 9781 } 9782 9783 /* 9784 * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output 9785 * whose incoming contents are not read. 9786 */ 9787 static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type) 9788 { 9789 if (!(arg_type & MEM_WRITE)) 9790 return BPF_READ; 9791 if (arg_type & MEM_UNINIT) 9792 return BPF_WRITE; 9793 return BPF_READ | BPF_WRITE; 9794 } 9795 9796 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, 9797 struct bpf_call_arg_meta *meta, 9798 int insn_idx) 9799 { 9800 const struct btf_param *btf_arg = is_kfunc(meta) ? 9801 &btf_params(meta->func_proto)[arg] : NULL; 9802 const struct bpf_func_proto *fn = meta->fn; 9803 struct bpf_func_state *caller = cur_func(env); 9804 struct bpf_reg_state *regs = cur_regs(env); 9805 argno_t argno = argno_from_arg(slot + 1); 9806 struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); 9807 enum bpf_arg_type arg_type = fn->arg_type[arg]; 9808 int regno = reg_from_argno(argno); 9809 u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; 9810 u32 key_size; 9811 int err = 0; 9812 9813 if (arg_type == ARG_PTR_TO_PROG_AUX) { 9814 cur_aux(env)->arg_prog = regno; 9815 return 0; 9816 } 9817 9818 if (arg_type == ARG_IGNORE) 9819 return 0; 9820 9821 if (regno >= 0) { 9822 err = check_reg_arg(env, regno, SRC_OP); 9823 if (err) 9824 return err; 9825 } 9826 9827 /* Preserve the legacy helper behavior for privileged pointer leaks. */ 9828 if (arg_type == ARG_ANYTHING) { 9829 if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 9830 verbose(env, "%s leaks addr into helper function\n", 9831 reg_arg_name(env, argno)); 9832 return -EACCES; 9833 } 9834 return 0; 9835 } 9836 9837 err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); 9838 if (err) 9839 return err; 9840 9841 /* 9842 * A slice of a packet is packet data under another name. A helper which 9843 * may reallocate the packet must not go on reading from the old one. 9844 */ 9845 if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) && 9846 bpf_helper_changes_pkt_data(meta->func_id)) { 9847 verbose(env, "function access to the packet is not allowed\n"); 9848 return -EACCES; 9849 } 9850 9851 if (arg_type_is_raw_mem(arg_type)) 9852 meta->arg_raw_mem.mask |= BIT(slot); 9853 9854 if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { 9855 err = mark_arg_precision(env, argno); 9856 if (err) 9857 return err; 9858 return 0; 9859 } 9860 9861 err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx); 9862 if (err) 9863 return err; 9864 9865 err = check_reg_type(env, reg, argno, arg_type, meta); 9866 if (err) 9867 return err; 9868 9869 err = check_func_arg_reg_off(env, reg, argno, arg_type); 9870 if (err) 9871 return err; 9872 9873 err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx); 9874 if (err) 9875 return err; 9876 9877 if (reg_is_referenced(env, reg)) 9878 update_ref_obj(&meta->ref_obj, reg); 9879 9880 switch (base_type(arg_type)) { 9881 case ARG_CONST_SCALAR: 9882 err = process_const_arg(env, reg, argno, meta); 9883 if (err < 0) { 9884 if (err == -EINVAL) 9885 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 9886 "Pass a compile-time constant or a value the verifier can prove is constant at this call.", 9887 "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path", 9888 reg_arg_name(env, argno)); 9889 return err; 9890 } 9891 break; 9892 case ARG_SCALAR: 9893 break; 9894 case ARG_CONST_MAP_PTR: 9895 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 9896 err = process_map_ptr_arg(env, reg, argno, meta); 9897 if (err) 9898 return err; 9899 break; 9900 case ARG_PTR_TO_MAP_KEY: 9901 /* bpf_map_xxx(..., map_ptr, ..., key) call: 9902 * check that [key, key + map->key_size) are within 9903 * stack limits and initialized 9904 */ 9905 if (!meta->map.ptr) { 9906 /* in function declaration map_ptr must come before 9907 * map_key, so that it's verified and known before 9908 * we have to check map_key here. Otherwise it means 9909 * that kernel subsystem misconfigured verifier 9910 */ 9911 verifier_bug(env, "invalid map_ptr to access map->key"); 9912 return -EFAULT; 9913 } 9914 key_size = meta->map.ptr->key_size; 9915 err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta, 9916 NULL); 9917 if (err) 9918 return err; 9919 if (can_elide_value_nullness(meta->map.ptr)) { 9920 err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); 9921 if (err < 0) { 9922 meta->const_map_key = -1; 9923 if (err == -EOPNOTSUPP) 9924 err = 0; 9925 else 9926 return err; 9927 } 9928 } 9929 break; 9930 case ARG_PTR_TO_MAP_VALUE: 9931 if (type_may_be_null(arg_type) && bpf_register_is_null(reg)) 9932 return 0; 9933 9934 /* bpf_map_xxx(..., map_ptr, ..., value) call: 9935 * check [value, value + map->value_size) validity 9936 */ 9937 if (!meta->map.ptr) { 9938 /* kernel subsystem misconfigured verifier */ 9939 verifier_bug(env, "invalid map_ptr to access map->value"); 9940 return -EFAULT; 9941 } 9942 9943 err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, 9944 arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, 9945 false, meta, NULL); 9946 break; 9947 case ARG_PTR_TO_BTF_ID: 9948 case ARG_PTR_TO_BTF_ID_SOCK_COMMON: 9949 { 9950 const u32 *arg_btf_id = fn->arg_btf_id[arg]; 9951 const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; 9952 enum bpf_reg_type reg_type; 9953 9954 if (!meta->btf) { 9955 const struct bpf_reg_types *compatible; 9956 9957 if (base_type(reg->type) != PTR_TO_BTF_ID) 9958 break; 9959 9960 if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) 9961 return map_kptr_match_type(env, meta->kptr_field, reg, regno) ? 9962 -EACCES : 0; 9963 9964 if (!arg_btf_id) { 9965 compatible = compatible_reg_types[base_type(arg_type)]; 9966 if (!compatible->btf_id) { 9967 verifier_bug(env, "missing arg compatible BTF ID"); 9968 return -EFAULT; 9969 } 9970 arg_btf_id = compatible->btf_id; 9971 } 9972 if (arg_btf_id == BPF_PTR_POISON) { 9973 verbose(env, "verifier internal error:"); 9974 verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", 9975 reg_arg_name(env, argno)); 9976 return -EACCES; 9977 } 9978 } 9979 9980 reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL); 9981 if ((arg_type & PTR_TRUSTED) && 9982 (!is_trusted_reg_type(env, reg, reg_type) || 9983 bpf_type_has_unsafe_modifiers(reg_type))) { 9984 if (!(arg_type & MEM_RCU)) { 9985 const char *actual_type, *arg_name, *expected_type; 9986 9987 expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); 9988 verbose(env, "%s must be referenced or trusted\n", 9989 reg_arg_name(env, argno)); 9990 arg_name = reg_arg_name(env, argno); 9991 actual_type = bpf_diag_reg_type_plain(env, reg->type); 9992 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 9993 "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.", 9994 "the function requires a trusted or resource-owning pointer to %s, but %s is %s", 9995 expected_type, arg_name, actual_type); 9996 return -EINVAL; 9997 } 9998 if (!is_rcu_reg(reg)) { 9999 const char *actual_type, *arg_name, *expected_type; 10000 10001 expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); 10002 verbose(env, "%s must be a rcu pointer\n", 10003 reg_arg_name(env, argno)); 10004 arg_name = reg_arg_name(env, argno); 10005 actual_type = bpf_diag_reg_type_plain(env, reg->type); 10006 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10007 "Use this function with a pointer that is valid in an RCU read lock region.", 10008 "the function requires an RCU-protected pointer to %s, but %s is %s", 10009 expected_type, arg_name, actual_type); 10010 return -EINVAL; 10011 } 10012 } 10013 10014 err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf, 10015 *arg_btf_id, meta, insn_idx); 10016 if (err < 0) 10017 return err; 10018 break; 10019 } 10020 case ARG_PTR_TO_PERCPU_BTF_ID: 10021 if (!reg->btf_id) { 10022 verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); 10023 return -EACCES; 10024 } 10025 meta->ret_btf = reg->btf; 10026 meta->ret_btf_id = reg->btf_id; 10027 break; 10028 case ARG_PTR_TO_SPIN_LOCK: 10029 if (in_rbtree_lock_required_cb(env)) { 10030 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); 10031 return -EACCES; 10032 } 10033 if (is_helper_call(meta, BPF_FUNC_spin_lock)) { 10034 err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); 10035 if (err) 10036 return err; 10037 } else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) { 10038 err = process_spin_lock(env, reg, argno, 0); 10039 if (err) 10040 return err; 10041 } else { 10042 verifier_bug(env, "spin lock arg on unexpected helper"); 10043 return -EFAULT; 10044 } 10045 break; 10046 case ARG_PTR_TO_TIMER: 10047 err = process_timer_func(env, reg, argno, &meta->map); 10048 if (err) 10049 return err; 10050 break; 10051 case ARG_PTR_TO_CTX: 10052 if (is_bpf_cast_to_kern_ctx_kfunc(meta)) { 10053 err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 10054 if (err < 0) 10055 return -EINVAL; 10056 meta->ret_btf_id = err; 10057 } 10058 break; 10059 case ARG_PTR_TO_ARENA: 10060 break; 10061 case ARG_PTR_TO_ALLOC_BTF_ID: 10062 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { 10063 if (!is_bpf_obj_drop_kfunc(meta->func_id)) { 10064 verbose(env, "%s expected for bpf_obj_drop()\n", 10065 reg_arg_name(env, argno)); 10066 return -EINVAL; 10067 } 10068 } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { 10069 if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { 10070 verbose(env, "%s expected for bpf_percpu_obj_drop()\n", 10071 reg_arg_name(env, argno)); 10072 return -EINVAL; 10073 } 10074 } 10075 if (!reg_is_referenced(env, reg)) { 10076 verbose(env, "allocated object must be referenced\n"); 10077 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10078 "Pass the owned object pointer before it is released or transferred.", 10079 "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", 10080 reg_arg_name(env, argno)); 10081 return -EINVAL; 10082 } 10083 if (meta->btf == btf_vmlinux) { 10084 meta->arg_btf = reg->btf; 10085 meta->arg_btf_id = reg->btf_id; 10086 } 10087 break; 10088 case ARG_PTR_TO_FUNC: 10089 meta->subprogno = reg->subprogno; 10090 break; 10091 case ARG_PTR_TO_MEM: 10092 { 10093 enum bpf_access_type access_type; 10094 bool known_memory; 10095 10096 /* The access to this pointer is only checked when we hit the 10097 * next is_mem_size argument below. 10098 */ 10099 if (!(arg_type & MEM_FIXED_SIZE)) 10100 break; 10101 10102 access_type = func_arg_access_type(arg_type); 10103 10104 err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); 10105 if (err < 0) { 10106 if (known_memory) 10107 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10108 "Pass memory with at least the required number of accessible bytes and suitable read or write access.", 10109 "the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access", 10110 arg_size, 10111 bpf_diag_reg_type_plain(env, reg->type)); 10112 else 10113 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10114 "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", 10115 "the function expects %u bytes of memory, but it is %s and not verifier-known memory", 10116 arg_size, 10117 bpf_diag_reg_type_plain(env, reg->type)); 10118 return err; 10119 } 10120 /* 10121 * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting 10122 * bounds tracking information. 10123 */ 10124 if (is_subprog(meta) && meta->subprog_may_change_pkt && 10125 (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) { 10126 verbose(env, 10127 "cannot pass packet pointer %s to %s(): function may change packet data\n", 10128 reg_arg_name(env, argno), meta->func_name); 10129 return -EINVAL; 10130 } 10131 if (arg_type & MEM_ALIGNED) 10132 err = check_ptr_alignment(env, reg, 0, arg_size, true); 10133 break; 10134 } 10135 case ARG_CONST_MEM_SIZE: 10136 err = process_const_arg(env, reg, argno, meta); 10137 if (err < 0) { 10138 if (err == -EINVAL) 10139 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10140 "Pass a compile-time constant or a value the verifier can prove is constant at this call.", 10141 "the function requires this memory size to be a verifier-known constant, but %s is variable on this path", 10142 reg_arg_name(env, argno)); 10143 return err; 10144 } 10145 fallthrough; 10146 case ARG_MEM_SIZE: 10147 case ARG_MEM_SIZE_OR_ZERO: 10148 { 10149 struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); 10150 argno_t buff_argno = argno_from_arg(prev_slot + 1); 10151 enum bpf_mem_size_failure failure; 10152 const char *buff_arg, *size_arg; 10153 bool zero_size_allowed; 10154 u32 access_type; 10155 10156 if (meta->btf && bpf_register_is_null(buff_reg)) 10157 break; 10158 10159 access_type = func_arg_access_type(fn->arg_type[arg - 1]); 10160 10161 zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; 10162 10163 err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno, 10164 access_type, zero_size_allowed, meta, &failure); 10165 if (!err) 10166 break; 10167 10168 buff_arg = bpf_diag_arg_name(env, buff_argno); 10169 size_arg = bpf_diag_arg_name(env, argno); 10170 verbose(env, "%s and ", reg_arg_name(env, buff_argno)); 10171 verbose(env, "%s memory, len pair leads to invalid memory access\n", 10172 reg_arg_name(env, argno)); 10173 if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { 10174 bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name, 10175 "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", 10176 "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length", 10177 size_arg, buff_arg); 10178 } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { 10179 if (reg_smin(reg) < 0) 10180 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10181 "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", 10182 "the memory size in %s may be negative because its signed minimum is %lld", 10183 size_arg, reg_smin(reg)); 10184 else if (!zero_size_allowed && reg_umin(reg) == 0) 10185 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10186 "Ensure the memory size is non-zero before this call.", 10187 "the memory size in %s may be zero, but the function requires a non-zero size", 10188 size_arg); 10189 else 10190 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10191 "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", 10192 "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", 10193 size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ); 10194 } 10195 break; 10196 } 10197 case ARG_PTR_TO_DYNPTR: { 10198 if (is_bpf_dynptr_clone_kfunc(meta) && 10199 (arg_type & MEM_UNINIT)) { 10200 enum bpf_dynptr_type parent_type = meta->dynptr.type; 10201 10202 if (parent_type == BPF_DYNPTR_TYPE_INVALID) { 10203 verifier_bug(env, "no dynptr type for parent of clone"); 10204 return -EFAULT; 10205 } 10206 10207 arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); 10208 } 10209 10210 err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); 10211 if (err) 10212 return err; 10213 if (!arg && is_kfunc_dynptr_write(meta)) 10214 meta->dynptr_write_type = meta->dynptr.type; 10215 break; 10216 } 10217 case ARG_PTR_TO_ITER: 10218 if (is_bpf_iter_css_task_new_kfunc(meta) && 10219 !check_css_task_iter_allowlist(env)) { 10220 verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); 10221 return -EINVAL; 10222 } 10223 err = process_iter_arg(env, reg, arg, argno, insn_idx, meta); 10224 if (err < 0) 10225 return err; 10226 break; 10227 case ARG_PTR_TO_LIST_HEAD: 10228 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && 10229 !reg_is_referenced(env, reg)) { 10230 verbose(env, "allocated object must be referenced\n"); 10231 return -EINVAL; 10232 } 10233 err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); 10234 if (err < 0) 10235 return err; 10236 break; 10237 case ARG_PTR_TO_RB_ROOT: 10238 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && 10239 !reg_is_referenced(env, reg)) { 10240 verbose(env, "allocated object must be referenced\n"); 10241 return -EINVAL; 10242 } 10243 err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); 10244 if (err < 0) 10245 return err; 10246 break; 10247 case ARG_PTR_TO_LIST_NODE: 10248 if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) && 10249 type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) { 10250 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10251 verbose(env, "%s expected pointer to allocated object\n", 10252 reg_arg_name(env, argno)); 10253 return -EINVAL; 10254 } 10255 if (!reg_is_referenced(env, reg)) { 10256 verbose(env, "allocated object must be referenced\n"); 10257 return -EINVAL; 10258 } 10259 } 10260 err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); 10261 if (err < 0) 10262 return err; 10263 break; 10264 case ARG_PTR_TO_RB_NODE: 10265 if (is_bpf_rbtree_add_kfunc(meta->func_id)) { 10266 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10267 verbose(env, "%s expected pointer to allocated object\n", 10268 reg_arg_name(env, argno)); 10269 return -EINVAL; 10270 } 10271 if (!reg_is_referenced(env, reg)) { 10272 verbose(env, "allocated object must be referenced\n"); 10273 return -EINVAL; 10274 } 10275 } else { 10276 if (!type_is_non_owning_ref(reg->type) && 10277 !reg_is_referenced(env, reg)) { 10278 verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", 10279 meta->func_name); 10280 return -EINVAL; 10281 } 10282 if (in_rbtree_lock_required_cb(env)) { 10283 verbose(env, "%s not allowed in rbtree cb\n", meta->func_name); 10284 return -EINVAL; 10285 } 10286 } 10287 err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); 10288 if (err < 0) 10289 return err; 10290 break; 10291 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 10292 if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg, 10293 "rdonly_buf_size")) 10294 meta->r0_rdonly = true; 10295 err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); 10296 if (err < 0) { 10297 if (meta->btf && err == -EINVAL) 10298 bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, 10299 "Pass a verifier-known constant size for this function's buffer argument.", 10300 "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path", 10301 reg_arg_name(env, argno)); 10302 return err; 10303 } 10304 break; 10305 case ARG_PTR_TO_REFCOUNTED_KPTR: 10306 { 10307 struct btf_record *rec; 10308 10309 if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) 10310 meta->arg_owning_ref = true; 10311 10312 rec = reg_btf_record(reg); 10313 if (!rec) { 10314 verifier_bug(env, "Couldn't find btf_record"); 10315 return -EFAULT; 10316 } 10317 10318 if (rec->refcount_off < 0) { 10319 verbose(env, "%s doesn't point to a type with bpf_refcount field\n", 10320 reg_arg_name(env, argno)); 10321 return -EINVAL; 10322 } 10323 10324 meta->arg_btf = reg->btf; 10325 meta->arg_btf_id = reg->btf_id; 10326 break; 10327 } 10328 case ARG_PTR_TO_CONST_STR: 10329 { 10330 err = check_arg_const_str(env, reg, argno); 10331 if (err) 10332 return err; 10333 break; 10334 } 10335 case ARG_PTR_TO_WORKQUEUE: 10336 err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); 10337 if (err < 0) 10338 return err; 10339 break; 10340 case ARG_PTR_TO_TASK_WORK: 10341 err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); 10342 if (err < 0) 10343 return err; 10344 break; 10345 case ARG_PTR_TO_RCU_HEAD: 10346 err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map); 10347 if (err < 0) 10348 return err; 10349 break; 10350 case ARG_PTR_TO_IRQ_FLAG: 10351 err = process_irq_flag(env, reg, argno, meta); 10352 if (err < 0) 10353 return err; 10354 break; 10355 case ARG_PTR_TO_CTX_OUT: 10356 if (reg->mem_size != arg_size) { 10357 verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n", 10358 reg_arg_name(env, argno), arg_size, reg->mem_size); 10359 return -EINVAL; 10360 } 10361 break; 10362 case ARG_PTR_TO_RES_SPIN_LOCK: 10363 { 10364 int flags; 10365 10366 if (in_rbtree_lock_required_cb(env)) { 10367 verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); 10368 return -EACCES; 10369 } 10370 10371 flags = get_bpf_res_spin_lock_kfunc_flags(meta); 10372 if (!flags) 10373 return -EFAULT; 10374 err = process_spin_lock(env, reg, argno, flags); 10375 if (err < 0) 10376 return err; 10377 break; 10378 } 10379 case ARG_KPTR_XCHG_DEST: 10380 err = process_kptr_func(env, regno, meta); 10381 if (err) 10382 return err; 10383 break; 10384 } 10385 10386 return err; 10387 } 10388 10389 /* 10390 * The slots a parameter takes, from its BTF type. This has to agree with 10391 * btf_func_model_arg_slots(), which answers the same from the size the 10392 * func model recorded, or the verifier would check an argument at a slot 10393 * the JIT does not place it at. 10394 */ 10395 static u32 btf_arg_slots(const struct btf_type *t) 10396 { 10397 if (btf_type_is_int(t) || btf_type_is_struct(t)) 10398 return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; 10399 return 1; 10400 } 10401 10402 static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto) 10403 { 10404 const struct btf_param *args = btf_params(func_proto); 10405 u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; 10406 10407 for (i = 0; i < nargs; i++) 10408 slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); 10409 10410 return slots_used; 10411 } 10412 10413 /* The argument slot @slot holds an eightbyte of a by-value argument. */ 10414 static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, 10415 u32 slot, struct bpf_call_arg_meta *meta) 10416 { 10417 struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); 10418 argno_t argno = argno_from_arg(slot + 1); 10419 int regno = reg_from_argno(argno); 10420 int err; 10421 10422 if (regno >= 0) { 10423 err = check_reg_arg(env, regno, SRC_OP); 10424 if (err) 10425 return err; 10426 } 10427 10428 return check_reg_type(env, reg, argno, ARG_SCALAR, meta); 10429 } 10430 10431 static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 10432 int insn_idx) 10433 { 10434 u32 slot = 0, prev_slot = 0, proto_slots, nslots; 10435 struct bpf_func_state *caller = cur_func(env); 10436 const struct btf_param *args = NULL; 10437 u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; 10438 int err; 10439 10440 if (meta->btf) { 10441 args = btf_params(meta->func_proto); 10442 nargs = btf_type_vlen(meta->func_proto); 10443 } 10444 10445 /* 10446 * A by-value argument of more than one eightbyte takes a slot per 10447 * eightbyte, so the slots a call occupies are no longer its parameter 10448 * count. Only a proto whose parameters take a slot each can name the 10449 * argument a stack slot belongs to. 10450 */ 10451 proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs; 10452 10453 if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { 10454 err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, 10455 meta->btf, proto_slots == nargs ? args : NULL); 10456 if (err) 10457 return err; 10458 } 10459 10460 for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { 10461 const struct btf_type *t; 10462 u32 k; 10463 10464 nslots = 1; 10465 if (args) { 10466 t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); 10467 nslots = btf_arg_slots(t); 10468 } 10469 10470 if (meta->fn->arg_type[arg] == ARG_UNUSED) 10471 break; 10472 err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); 10473 if (err) 10474 return err; 10475 10476 /* 10477 * check_func_arg() took the first slot. A parameter of more 10478 * than one eightbyte is always a scalar, so every slot it 10479 * takes beyond the first holds one too. 10480 */ 10481 for (k = 1; k < nslots; k++) { 10482 err = check_arg_extra_slot(env, caller, slot + k, meta); 10483 if (err) 10484 return err; 10485 } 10486 } 10487 10488 return 0; 10489 } 10490 10491 static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 10492 int insn_idx) 10493 { 10494 struct bpf_func_state *caller = cur_func(env); 10495 u32 slot; 10496 int i, err; 10497 10498 /* 10499 * Validate every argument before initializing outputs: an input argument 10500 * may alias an output buffer. Use the normal stack-write checks to discard 10501 * stale spills and preserve the rules for special stack objects. 10502 */ 10503 for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { 10504 struct bpf_reg_state *reg; 10505 10506 if (!meta->arg_raw_mem.size[slot]) 10507 continue; 10508 reg = get_func_arg_reg(caller, cur_regs(env), slot); 10509 10510 /* 10511 * check_stack_range_initialized() checked the range, its offset is a constant. 10512 * Not check_stack_write(): unknown bytes in its buffer leave a dynptr slice valid. 10513 */ 10514 for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { 10515 err = check_stack_write_fixed_off(env, bpf_func(env, reg), 10516 reg->var_off.value + i, 1, -1, insn_idx); 10517 if (err) 10518 return err; 10519 } 10520 } 10521 10522 return 0; 10523 } 10524 10525 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 10526 { 10527 enum bpf_attach_type eatype = env->prog->expected_attach_type; 10528 enum bpf_prog_type type = resolve_prog_type(env->prog); 10529 const char *afname = env->prog->aux->attach_func_name; 10530 10531 if (func_id != BPF_FUNC_map_update_elem && 10532 func_id != BPF_FUNC_map_delete_elem) 10533 return false; 10534 10535 /* It's not possible to get access to a locked struct sock in these 10536 * contexts, so updating is safe. 10537 */ 10538 switch (type) { 10539 case BPF_PROG_TYPE_TRACING: 10540 if (eatype == BPF_TRACE_ITER && 10541 !strcmp(afname + strlen(BPF_ITER_FUNC_PREFIX), "sockmap")) 10542 return true; 10543 break; 10544 case BPF_PROG_TYPE_SOCK_OPS: 10545 /* map_update allowed only via dedicated helpers with event type checks */ 10546 if (func_id == BPF_FUNC_map_delete_elem) 10547 return true; 10548 break; 10549 case BPF_PROG_TYPE_SK_REUSEPORT: 10550 case BPF_PROG_TYPE_SK_LOOKUP: 10551 return true; 10552 default: 10553 break; 10554 } 10555 10556 verbose(env, "cannot update sockmap in this context\n"); 10557 return false; 10558 } 10559 10560 bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 10561 { 10562 return env->prog->jit_requested && 10563 bpf_jit_supports_subprog_tailcalls(); 10564 } 10565 10566 /* 10567 * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled 10568 * preemption or interrupts will not do. 10569 */ 10570 static bool has_rcu_read_lock(struct bpf_verifier_env *env) 10571 { 10572 struct bpf_verifier_state *state = env->cur_state; 10573 10574 if (state->active_rcu_locks || in_rcu_callback(env)) 10575 return true; 10576 if (in_sleepable(env)) 10577 return false; 10578 return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn; 10579 } 10580 10581 /* May the map be used outside of an RCU read side critical section? */ 10582 static bool map_needs_no_rcu(const struct bpf_map *map) 10583 { 10584 switch (map->map_type) { 10585 case BPF_MAP_TYPE_HASH: 10586 case BPF_MAP_TYPE_RHASH: 10587 case BPF_MAP_TYPE_LRU_HASH: 10588 case BPF_MAP_TYPE_ARRAY: 10589 case BPF_MAP_TYPE_PERCPU_HASH: 10590 case BPF_MAP_TYPE_PERCPU_ARRAY: 10591 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 10592 case BPF_MAP_TYPE_LPM_TRIE: 10593 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 10594 case BPF_MAP_TYPE_HASH_OF_MAPS: 10595 case BPF_MAP_TYPE_RINGBUF: 10596 case BPF_MAP_TYPE_USER_RINGBUF: 10597 case BPF_MAP_TYPE_INODE_STORAGE: 10598 case BPF_MAP_TYPE_SK_STORAGE: 10599 case BPF_MAP_TYPE_TASK_STORAGE: 10600 case BPF_MAP_TYPE_CGRP_STORAGE: 10601 case BPF_MAP_TYPE_QUEUE: 10602 case BPF_MAP_TYPE_STACK: 10603 case BPF_MAP_TYPE_ARENA: 10604 case BPF_MAP_TYPE_INSN_ARRAY: 10605 case BPF_MAP_TYPE_PROG_ARRAY: 10606 case BPF_MAP_TYPE_BLOOM_FILTER: 10607 return true; 10608 default: 10609 return false; 10610 } 10611 } 10612 10613 static int check_map_func_compatibility(struct bpf_verifier_env *env, 10614 struct bpf_map *map, int func_id) 10615 { 10616 if (!map) 10617 return 0; 10618 10619 /* 10620 * check_map_prog_compatibility() asks the program. That misses the 10621 * sleepable callback of another program, and an inner map. They may 10622 * use any map under the RCU read lock. Disabled preemption or 10623 * interrupts will not do: the maps ask for rcu_read_lock_held(). 10624 */ 10625 if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) { 10626 verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n", 10627 map->name); 10628 return -EINVAL; 10629 } 10630 10631 /* We need a two way check, first is from map perspective ... */ 10632 switch (map->map_type) { 10633 case BPF_MAP_TYPE_PROG_ARRAY: 10634 if (func_id != BPF_FUNC_tail_call) 10635 goto error; 10636 break; 10637 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 10638 if (func_id != BPF_FUNC_perf_event_read && 10639 func_id != BPF_FUNC_perf_event_output && 10640 func_id != BPF_FUNC_skb_output && 10641 func_id != BPF_FUNC_perf_event_read_value && 10642 func_id != BPF_FUNC_xdp_output) 10643 goto error; 10644 break; 10645 case BPF_MAP_TYPE_RINGBUF: 10646 if (func_id != BPF_FUNC_ringbuf_output && 10647 func_id != BPF_FUNC_ringbuf_reserve && 10648 func_id != BPF_FUNC_ringbuf_query && 10649 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 10650 func_id != BPF_FUNC_ringbuf_submit_dynptr && 10651 func_id != BPF_FUNC_ringbuf_discard_dynptr) 10652 goto error; 10653 break; 10654 case BPF_MAP_TYPE_USER_RINGBUF: 10655 if (func_id != BPF_FUNC_user_ringbuf_drain) 10656 goto error; 10657 break; 10658 case BPF_MAP_TYPE_STACK_TRACE: 10659 if (func_id != BPF_FUNC_get_stackid) 10660 goto error; 10661 break; 10662 case BPF_MAP_TYPE_CGROUP_ARRAY: 10663 if (func_id != BPF_FUNC_skb_under_cgroup && 10664 func_id != BPF_FUNC_current_task_under_cgroup) 10665 goto error; 10666 break; 10667 case BPF_MAP_TYPE_CGROUP_STORAGE: 10668 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 10669 if (func_id != BPF_FUNC_get_local_storage) 10670 goto error; 10671 break; 10672 case BPF_MAP_TYPE_DEVMAP: 10673 case BPF_MAP_TYPE_DEVMAP_HASH: 10674 if (func_id != BPF_FUNC_redirect_map && 10675 func_id != BPF_FUNC_map_lookup_elem) 10676 goto error; 10677 break; 10678 /* Restrict bpf side of cpumap and xskmap, open when use-cases 10679 * appear. 10680 */ 10681 case BPF_MAP_TYPE_CPUMAP: 10682 if (func_id != BPF_FUNC_redirect_map) 10683 goto error; 10684 break; 10685 case BPF_MAP_TYPE_XSKMAP: 10686 if (func_id != BPF_FUNC_redirect_map && 10687 func_id != BPF_FUNC_map_lookup_elem) 10688 goto error; 10689 break; 10690 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 10691 case BPF_MAP_TYPE_HASH_OF_MAPS: 10692 if (func_id != BPF_FUNC_map_lookup_elem) 10693 goto error; 10694 break; 10695 case BPF_MAP_TYPE_SOCKMAP: 10696 if (func_id != BPF_FUNC_sk_redirect_map && 10697 func_id != BPF_FUNC_sock_map_update && 10698 func_id != BPF_FUNC_msg_redirect_map && 10699 func_id != BPF_FUNC_sk_select_reuseport && 10700 func_id != BPF_FUNC_map_lookup_elem && 10701 !may_update_sockmap(env, func_id)) 10702 goto error; 10703 break; 10704 case BPF_MAP_TYPE_SOCKHASH: 10705 if (func_id != BPF_FUNC_sk_redirect_hash && 10706 func_id != BPF_FUNC_sock_hash_update && 10707 func_id != BPF_FUNC_msg_redirect_hash && 10708 func_id != BPF_FUNC_sk_select_reuseport && 10709 func_id != BPF_FUNC_map_lookup_elem && 10710 !may_update_sockmap(env, func_id)) 10711 goto error; 10712 break; 10713 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 10714 if (func_id != BPF_FUNC_sk_select_reuseport) 10715 goto error; 10716 break; 10717 case BPF_MAP_TYPE_QUEUE: 10718 case BPF_MAP_TYPE_STACK: 10719 if (func_id != BPF_FUNC_map_peek_elem && 10720 func_id != BPF_FUNC_map_pop_elem && 10721 func_id != BPF_FUNC_map_push_elem) 10722 goto error; 10723 break; 10724 case BPF_MAP_TYPE_SK_STORAGE: 10725 if (func_id != BPF_FUNC_sk_storage_get && 10726 func_id != BPF_FUNC_sk_storage_delete && 10727 func_id != BPF_FUNC_kptr_xchg) 10728 goto error; 10729 break; 10730 case BPF_MAP_TYPE_INODE_STORAGE: 10731 if (func_id != BPF_FUNC_inode_storage_get && 10732 func_id != BPF_FUNC_inode_storage_delete && 10733 func_id != BPF_FUNC_kptr_xchg) 10734 goto error; 10735 break; 10736 case BPF_MAP_TYPE_TASK_STORAGE: 10737 if (func_id != BPF_FUNC_task_storage_get && 10738 func_id != BPF_FUNC_task_storage_delete && 10739 func_id != BPF_FUNC_kptr_xchg) 10740 goto error; 10741 break; 10742 case BPF_MAP_TYPE_CGRP_STORAGE: 10743 if (func_id != BPF_FUNC_cgrp_storage_get && 10744 func_id != BPF_FUNC_cgrp_storage_delete && 10745 func_id != BPF_FUNC_kptr_xchg) 10746 goto error; 10747 break; 10748 case BPF_MAP_TYPE_BLOOM_FILTER: 10749 if (func_id != BPF_FUNC_map_peek_elem && 10750 func_id != BPF_FUNC_map_push_elem) 10751 goto error; 10752 break; 10753 case BPF_MAP_TYPE_INSN_ARRAY: 10754 goto error; 10755 default: 10756 break; 10757 } 10758 10759 /* ... and second from the function itself. */ 10760 switch (func_id) { 10761 case BPF_FUNC_tail_call: 10762 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 10763 goto error; 10764 if (env->subprog_cnt > 1 && !bpf_allow_tail_call_in_subprogs(env)) { 10765 verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n"); 10766 return -EINVAL; 10767 } 10768 break; 10769 case BPF_FUNC_perf_event_read: 10770 case BPF_FUNC_perf_event_output: 10771 case BPF_FUNC_perf_event_read_value: 10772 case BPF_FUNC_skb_output: 10773 case BPF_FUNC_xdp_output: 10774 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 10775 goto error; 10776 break; 10777 case BPF_FUNC_ringbuf_output: 10778 case BPF_FUNC_ringbuf_reserve: 10779 case BPF_FUNC_ringbuf_query: 10780 case BPF_FUNC_ringbuf_reserve_dynptr: 10781 case BPF_FUNC_ringbuf_submit_dynptr: 10782 case BPF_FUNC_ringbuf_discard_dynptr: 10783 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 10784 goto error; 10785 break; 10786 case BPF_FUNC_user_ringbuf_drain: 10787 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 10788 goto error; 10789 break; 10790 case BPF_FUNC_get_stackid: 10791 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 10792 goto error; 10793 break; 10794 case BPF_FUNC_current_task_under_cgroup: 10795 case BPF_FUNC_skb_under_cgroup: 10796 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 10797 goto error; 10798 break; 10799 case BPF_FUNC_redirect_map: 10800 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 10801 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 10802 map->map_type != BPF_MAP_TYPE_CPUMAP && 10803 map->map_type != BPF_MAP_TYPE_XSKMAP) 10804 goto error; 10805 break; 10806 case BPF_FUNC_sk_redirect_map: 10807 case BPF_FUNC_msg_redirect_map: 10808 case BPF_FUNC_sock_map_update: 10809 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 10810 goto error; 10811 break; 10812 case BPF_FUNC_sk_redirect_hash: 10813 case BPF_FUNC_msg_redirect_hash: 10814 case BPF_FUNC_sock_hash_update: 10815 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 10816 goto error; 10817 break; 10818 case BPF_FUNC_get_local_storage: 10819 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 10820 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 10821 goto error; 10822 break; 10823 case BPF_FUNC_sk_select_reuseport: 10824 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 10825 map->map_type != BPF_MAP_TYPE_SOCKMAP && 10826 map->map_type != BPF_MAP_TYPE_SOCKHASH) 10827 goto error; 10828 break; 10829 case BPF_FUNC_map_pop_elem: 10830 if (map->map_type != BPF_MAP_TYPE_QUEUE && 10831 map->map_type != BPF_MAP_TYPE_STACK) 10832 goto error; 10833 break; 10834 case BPF_FUNC_map_peek_elem: 10835 case BPF_FUNC_map_push_elem: 10836 if (map->map_type != BPF_MAP_TYPE_QUEUE && 10837 map->map_type != BPF_MAP_TYPE_STACK && 10838 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 10839 goto error; 10840 break; 10841 case BPF_FUNC_map_lookup_percpu_elem: 10842 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 10843 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 10844 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 10845 goto error; 10846 break; 10847 case BPF_FUNC_sk_storage_get: 10848 case BPF_FUNC_sk_storage_delete: 10849 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 10850 goto error; 10851 break; 10852 case BPF_FUNC_inode_storage_get: 10853 case BPF_FUNC_inode_storage_delete: 10854 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 10855 goto error; 10856 break; 10857 case BPF_FUNC_task_storage_get: 10858 case BPF_FUNC_task_storage_delete: 10859 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 10860 goto error; 10861 break; 10862 case BPF_FUNC_cgrp_storage_get: 10863 case BPF_FUNC_cgrp_storage_delete: 10864 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 10865 goto error; 10866 break; 10867 default: 10868 break; 10869 } 10870 10871 return 0; 10872 error: 10873 verbose(env, "cannot pass map_type %d into func %s#%d\n", 10874 map->map_type, func_id_name(func_id), func_id); 10875 return -EINVAL; 10876 } 10877 10878 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 10879 { 10880 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 10881 bool has_size = fn->arg_size[arg] != 0; 10882 bool is_next_size = false; 10883 10884 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 10885 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 10886 10887 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 10888 return is_next_size; 10889 10890 return has_size == is_next_size || is_next_size == is_fixed; 10891 } 10892 10893 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 10894 { 10895 /* bpf_xxx(..., buf, len) call will access 'len' 10896 * bytes from memory 'buf'. Both arg types need 10897 * to be paired, so make sure there's no buggy 10898 * helper function specification. 10899 */ 10900 if (arg_type_is_mem_size(fn->arg1_type) || 10901 check_args_pair_invalid(fn, 0) || 10902 check_args_pair_invalid(fn, 1) || 10903 check_args_pair_invalid(fn, 2) || 10904 check_args_pair_invalid(fn, 3) || 10905 check_args_pair_invalid(fn, 4)) 10906 return false; 10907 10908 return true; 10909 } 10910 10911 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 10912 { 10913 int i; 10914 10915 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 10916 if (fn->arg_type[i] == ARG_UNUSED) 10917 break; 10918 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 10919 return !!fn->arg_btf_id[i]; 10920 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 10921 return fn->arg_btf_id[i] == BPF_PTR_POISON; 10922 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 10923 /* arg_btf_id and arg_size are in a union. */ 10924 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 10925 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 10926 return false; 10927 } 10928 10929 return true; 10930 } 10931 10932 static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) 10933 { 10934 int i; 10935 10936 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 10937 enum bpf_arg_type arg_type = fn->arg_type[i]; 10938 10939 if (arg_type == ARG_UNUSED) 10940 break; 10941 if (base_type(arg_type) != ARG_PTR_TO_MEM) 10942 continue; 10943 if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) 10944 return false; 10945 } 10946 10947 return true; 10948 } 10949 10950 static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) 10951 { 10952 int i; 10953 10954 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 10955 enum bpf_arg_type arg_type = fn->arg_type[i]; 10956 10957 if (arg_type == ARG_UNUSED) 10958 break; 10959 if (arg_type_is_release(arg_type)) { 10960 if (meta->release_regno) 10961 return false; 10962 meta->release_regno = i + 1; 10963 } 10964 } 10965 10966 return true; 10967 } 10968 10969 static bool check_arg_prog_aux(struct bpf_verifier_env *env, 10970 const struct bpf_func_proto *proto) 10971 { 10972 bool seen = false; 10973 argno_t argno; 10974 u32 i; 10975 10976 for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) { 10977 if (proto->arg_type[i] == ARG_UNUSED) 10978 break; 10979 if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX) 10980 continue; 10981 10982 if (seen) { 10983 verifier_bug(env, "Only 1 prog->aux argument supported"); 10984 return false; 10985 } 10986 10987 argno = argno_from_arg(i + 1); 10988 if (reg_from_argno(argno) < 0) { 10989 verbose(env, "%s prog->aux cannot be a stack argument\n", 10990 reg_arg_name(env, argno)); 10991 return false; 10992 } 10993 10994 seen = true; 10995 } 10996 10997 return true; 10998 } 10999 11000 static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn, 11001 struct bpf_call_arg_meta *meta) 11002 { 11003 return check_arg_prog_aux(env, fn) && 11004 check_arg_pair_ok(fn) && 11005 check_mem_arg_rw_flag_ok(fn) && 11006 check_proto_release_reg(fn, meta) && 11007 check_btf_id_ok(fn) ? 0 : -EINVAL; 11008 } 11009 11010 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 11011 * are now invalid, so turn them into unknown SCALAR_VALUE. 11012 * 11013 * This also applies to dynptr slices belonging to skb and xdp dynptrs, 11014 * since these slices point to packet data. 11015 */ 11016 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 11017 { 11018 struct bpf_func_state *state; 11019 struct bpf_reg_state *reg; 11020 11021 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 11022 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) { 11023 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE); 11024 mark_reg_invalid(env, reg); 11025 } 11026 })); 11027 } 11028 11029 static bool holds_pkt_pointers(struct bpf_verifier_env *env) 11030 { 11031 struct bpf_func_state *state; 11032 struct bpf_reg_state *reg; 11033 bool ret = false; 11034 11035 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 11036 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) 11037 ret = true; 11038 })); 11039 11040 return ret; 11041 } 11042 11043 enum { 11044 AT_PKT_END = -1, 11045 BEYOND_PKT_END = -2, 11046 }; 11047 11048 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 11049 { 11050 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 11051 struct bpf_reg_state *reg = &state->regs[regn]; 11052 11053 if (reg->type != PTR_TO_PACKET) 11054 /* PTR_TO_PACKET_META is not supported yet */ 11055 return; 11056 11057 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 11058 * How far beyond pkt_end it goes is unknown. 11059 * if (!range_open) it's the case of pkt >= pkt_end 11060 * if (range_open) it's the case of pkt > pkt_end 11061 * hence this pointer is at least 1 byte bigger than pkt_end 11062 */ 11063 if (range_open) 11064 reg->range = BEYOND_PKT_END; 11065 else 11066 reg->range = AT_PKT_END; 11067 } 11068 11069 static int __release_reference_nomark(struct bpf_verifier_state *state, int id) 11070 { 11071 int i; 11072 11073 for (i = 0; i < state->acquired_refs; i++) { 11074 if (state->refs[i].type != REF_TYPE_PTR) 11075 continue; 11076 if (state->refs[i].id == id) { 11077 release_reference_state(state, i); 11078 return 0; 11079 } 11080 } 11081 return -EINVAL; 11082 } 11083 11084 static int release_reference_nomark(struct bpf_verifier_env *env, int id) 11085 { 11086 int err; 11087 11088 err = __release_reference_nomark(env->cur_state, id); 11089 if (!err) 11090 bpf_diag_record_ref_release(env, env->insn_idx, id); 11091 return err; 11092 } 11093 11094 static int idstack_push(struct bpf_idmap *idmap, u32 id) 11095 { 11096 int i; 11097 11098 if (!id) 11099 return 0; 11100 11101 for (i = 0; i < idmap->cnt; i++) 11102 if (idmap->map[i].old == id) 11103 return 0; 11104 11105 if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE)) 11106 return -EFAULT; 11107 11108 idmap->map[idmap->cnt++].old = id; 11109 return 0; 11110 } 11111 11112 static int idstack_pop(struct bpf_idmap *idmap) 11113 { 11114 if (!idmap->cnt) 11115 return 0; 11116 11117 return idmap->map[--idmap->cnt].old; 11118 } 11119 11120 /* Invalidate id and objects derived from it iteratively in a DFS manner */ 11121 static int invalidate_reference(struct bpf_verifier_env *env, int id) 11122 { 11123 u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); 11124 struct bpf_verifier_state *vstate = env->cur_state; 11125 struct bpf_idmap *idstack = &env->idmap_scratch; 11126 struct bpf_stack_state *stack; 11127 struct bpf_func_state *state; 11128 struct bpf_reg_state *reg; 11129 int i, err; 11130 11131 idstack->cnt = 0; 11132 err = idstack_push(idstack, id); 11133 if (err) 11134 return err; 11135 11136 while ((id = idstack_pop(idstack))) { 11137 /* 11138 * Child references are inaccessible after parent is released, 11139 * any child references that exist at this point are a leak. 11140 */ 11141 for (i = 0; i < vstate->acquired_refs; i++) { 11142 if (vstate->refs[i].type != REF_TYPE_PTR) 11143 continue; 11144 if (vstate->refs[i].parent_id != id) 11145 continue; 11146 verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n", 11147 vstate->refs[i].id, vstate->refs[i].insn_idx); 11148 return -EINVAL; 11149 } 11150 11151 bpf_for_each_reg_in_vstate_mask(vstate, state, reg, stack, mask, ({ 11152 if (reg->id != id && reg->parent_id != id) 11153 continue; 11154 11155 /* Free objects derived from the current object */ 11156 if (reg->parent_id == id) { 11157 err = idstack_push(idstack, reg->id); 11158 if (err) 11159 return err; 11160 } 11161 11162 /* 11163 * A dynptr occupies two stack slots that invalidate_dynptr() 11164 * clears together. Record both scrubs before invalidating it. 11165 */ 11166 if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) { 11167 struct bpf_stack_state *dyn_stack = stack; 11168 11169 if (reg->dynptr.first_slot) 11170 dyn_stack--; 11171 bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr, 11172 BPF_DIAG_MOD_REF_RELEASE); 11173 bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr, 11174 BPF_DIAG_MOD_REF_RELEASE); 11175 invalidate_dynptr(env, dyn_stack); 11176 continue; 11177 } 11178 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE); 11179 if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) 11180 mark_reg_invalid(env, reg); 11181 })); 11182 } 11183 11184 return 0; 11185 } 11186 11187 static int release_reference(struct bpf_verifier_env *env, int id) 11188 { 11189 if (find_reference_state(env->cur_state, id)) 11190 WARN_ON_ONCE(release_reference_nomark(env, id)); 11191 11192 return invalidate_reference(env, id); 11193 } 11194 11195 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) 11196 { 11197 struct bpf_func_state *unused; 11198 struct bpf_reg_state *reg; 11199 11200 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 11201 if (type_is_non_owning_ref(reg->type)) { 11202 bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF); 11203 mark_reg_invalid(env, reg); 11204 } 11205 })); 11206 } 11207 11208 static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) 11209 { 11210 struct bpf_stack_state *stack; 11211 struct bpf_func_state *state; 11212 struct bpf_reg_state *reg; 11213 u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); 11214 11215 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ 11216 if (reg->type & MEM_RCU) { 11217 bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); 11218 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF); 11219 reg->type |= PTR_UNTRUSTED; 11220 bpf_diag_mod_end(env); 11221 } 11222 })); 11223 } 11224 11225 static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) 11226 { 11227 struct bpf_func_state *state; 11228 struct bpf_reg_state *reg; 11229 int err; 11230 11231 err = release_reference_nomark(env, id); 11232 if (err) 11233 return err; 11234 11235 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 11236 if (reg->id != id) 11237 continue; 11238 if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { 11239 bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); 11240 if (!type_may_be_null(reg->type)) 11241 reg->id = 0; 11242 reg->type &= ~MEM_ALLOC; 11243 reg->type |= MEM_RCU; 11244 bpf_diag_mod_end(env); 11245 } 11246 })); 11247 11248 return err; 11249 } 11250 11251 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 11252 struct bpf_reg_state *regs) 11253 { 11254 int i; 11255 11256 bpf_diag_record_caller_saved(env, regs); 11257 11258 /* after the call registers r0 - r5 were scratched */ 11259 for (i = 0; i < CALLER_SAVED_REGS; i++) { 11260 bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); 11261 __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK); 11262 } 11263 } 11264 11265 static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env, 11266 struct bpf_func_state *state) 11267 { 11268 int i, nslots = state->out_stack_arg_cnt; 11269 11270 for (i = 0; i < nslots; i++) { 11271 bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED); 11272 bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); 11273 } 11274 } 11275 11276 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 11277 struct bpf_func_state *caller, 11278 struct bpf_func_state *callee, 11279 int insn_idx); 11280 11281 static int set_callee_state(struct bpf_verifier_env *env, 11282 struct bpf_func_state *caller, 11283 struct bpf_func_state *callee, int insn_idx); 11284 11285 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, 11286 set_callee_state_fn set_callee_state_cb, 11287 struct bpf_verifier_state *state) 11288 { 11289 struct bpf_func_state *caller, *callee; 11290 int err; 11291 11292 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 11293 verbose(env, "the call stack of %d frames is too deep\n", 11294 state->curframe + 2); 11295 return -E2BIG; 11296 } 11297 11298 if (state->frame[state->curframe + 1]) { 11299 verifier_bug(env, "Frame %d already allocated", state->curframe + 1); 11300 return -EFAULT; 11301 } 11302 11303 caller = state->frame[state->curframe]; 11304 callee = kzalloc_obj(*callee, GFP_KERNEL_ACCOUNT); 11305 if (!callee) 11306 return -ENOMEM; 11307 state->frame[state->curframe + 1] = callee; 11308 11309 /* callee cannot access r0, r6 - r9 for reading and has to write 11310 * into its own stack before reading from it. 11311 * callee can read/write into caller's stack 11312 */ 11313 init_func_state(env, callee, 11314 /* remember the callsite, it will be used by bpf_exit */ 11315 callsite, 11316 state->curframe + 1 /* frameno within this callchain */, 11317 subprog /* subprog number within this prog */); 11318 err = set_callee_state_cb(env, caller, callee, callsite); 11319 if (err) 11320 goto err_out; 11321 11322 /* only increment it after check_reg_arg() finished */ 11323 state->curframe++; 11324 11325 return 0; 11326 11327 err_out: 11328 free_func_state(callee); 11329 state->frame[state->curframe + 1] = NULL; 11330 return err; 11331 } 11332 11333 static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf, 11334 const struct btf_type *func_proto, struct bpf_func_proto *proto) 11335 { 11336 const struct btf_param *args = btf_params(func_proto); 11337 u32 arg, slot = 0; 11338 11339 memset(proto, 0, sizeof(*proto)); 11340 for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { 11341 enum bpf_arg_type arg_type = sub->args[slot].arg_type; 11342 const struct btf_type *t; 11343 11344 if (arg_type & PTR_UNTRUSTED) { 11345 /* 11346 * An __arg_untrusted argument accepts any caller value. The 11347 * callee treats it as read-only and uses probe-read instructions 11348 * to protect against invalid memory access. 11349 */ 11350 arg_type = ARG_IGNORE; 11351 } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { 11352 arg_type |= PTR_MAYBE_NULL; 11353 } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { 11354 proto->arg_size[arg] = sub->args[slot].mem_size; 11355 arg_type |= MEM_FIXED_SIZE | MEM_WRITE; 11356 } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { 11357 proto->arg_btf_id[arg] = &sub->args[slot].btf_id; 11358 } 11359 proto->arg_type[arg] = arg_type; 11360 t = btf_type_skip_modifiers(btf, args[arg].type, NULL); 11361 slot += btf_arg_slots(t); 11362 } 11363 } 11364 11365 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf) 11366 { 11367 struct bpf_subprog_info *sub = subprog_info(env, subprog); 11368 struct bpf_func_state *caller = cur_func(env); 11369 const struct btf_type *func, *func_proto; 11370 struct bpf_call_arg_meta meta; 11371 struct bpf_func_proto *fn; 11372 int ret, err; 11373 u32 i; 11374 11375 memset(&meta, 0, sizeof(meta)); 11376 meta.btf = btf; 11377 meta.subprog_may_change_pkt = sub->changes_pkt_data; 11378 meta.func_name = bpf_subprog_name(env, subprog); 11379 11380 ret = btf_prepare_func_args(env, subprog); 11381 if (ret) { 11382 if (bpf_in_stack_arg_cnt(sub) > 0) { 11383 err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, 11384 bpf_subprog_name(env, subprog), 11385 NULL, NULL); 11386 if (err) 11387 return err; 11388 } 11389 return ret; 11390 } 11391 11392 func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); 11393 func_proto = btf_type_by_id(btf, func->type); 11394 11395 fn = &env->bpf_subprog_scratch; 11396 gen_subprog_arg_proto(sub, btf, func_proto, fn); 11397 meta.fn = fn; 11398 meta.func_proto = func_proto; 11399 11400 /* 11401 * Check if all register arguments declared in BTF are initialized. 11402 * If they aren't the BTF is unreliable. 11403 */ 11404 if (!bpf_subprog_is_global(env, subprog)) { 11405 for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) 11406 if (caller->regs[BPF_REG_1 + i].type == NOT_INIT) 11407 return -EINVAL; 11408 } 11409 return check_func_args(env, &meta, env->insn_idx); 11410 } 11411 11412 /* 11413 * Check that call-site argument states match a subprog's BTF signature. 11414 * 11415 * Returns: 11416 * EFAULT - there is a verifier bug. Abort verification. 11417 * Other errors - there is a type mismatch or BTF is not available. 11418 * 0 - BTF matches with what bpf_reg_state expects. 11419 */ 11420 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog) 11421 { 11422 struct bpf_prog *prog = env->prog; 11423 struct btf *btf = prog->aux->btf; 11424 u32 btf_id; 11425 int err; 11426 11427 if (!prog->aux->func_info) 11428 return -EINVAL; 11429 11430 btf_id = prog->aux->func_info[subprog].type_id; 11431 if (!btf_id) 11432 return -EFAULT; 11433 11434 if (prog->aux->func_info_aux[subprog].unreliable) 11435 return -EINVAL; 11436 11437 err = btf_check_func_arg_match(env, subprog, btf); 11438 /* Compiler optimizations can remove arguments from static functions 11439 * or mismatched type can be passed into a global function. 11440 * In such cases mark the function as unreliable from BTF point of view. 11441 */ 11442 if (err) 11443 prog->aux->func_info_aux[subprog].unreliable = true; 11444 return err; 11445 } 11446 11447 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 11448 int insn_idx, int subprog, 11449 set_callee_state_fn set_callee_state_cb) 11450 { 11451 struct bpf_verifier_state *state = env->cur_state, *callback_state; 11452 struct bpf_func_state *caller, *callee; 11453 int err; 11454 11455 caller = state->frame[state->curframe]; 11456 err = btf_check_subprog_call(env, subprog); 11457 if (err == -EFAULT) 11458 return err; 11459 if (bpf_in_stack_arg_cnt(&env->subprog_info[subprog])) { 11460 verbose(env, "callback subprog cannot have stack args\n"); 11461 return -EINVAL; 11462 } 11463 11464 /* set_callee_state is used for direct subprog calls, but we are 11465 * interested in validating only BPF helpers that can call subprogs as 11466 * callbacks 11467 */ 11468 env->subprog_info[subprog].is_cb = true; 11469 if (bpf_pseudo_kfunc_call(insn) && 11470 !is_callback_calling_kfunc(insn->imm)) { 11471 verifier_bug(env, "kfunc %s#%d not marked as callback-calling", 11472 func_id_name(insn->imm), insn->imm); 11473 return -EFAULT; 11474 } else if (!bpf_pseudo_kfunc_call(insn) && 11475 !is_callback_calling_function(insn->imm)) { /* helper */ 11476 verifier_bug(env, "helper %s#%d not marked as callback-calling", 11477 func_id_name(insn->imm), insn->imm); 11478 return -EFAULT; 11479 } 11480 11481 if (bpf_is_async_callback_calling_insn(insn)) { 11482 struct bpf_verifier_state *async_cb; 11483 11484 /* there is no real recursion here. timer and workqueue callbacks are async */ 11485 env->subprog_info[subprog].is_async_cb = true; 11486 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 11487 insn_idx, subprog, 11488 is_async_cb_sleepable(env, insn)); 11489 if (IS_ERR(async_cb)) 11490 return PTR_ERR(async_cb); 11491 if (is_async_cb_sleepable(env, insn)) 11492 mark_sleepable_async_cb(env); 11493 callee = async_cb->frame[0]; 11494 callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; 11495 11496 /* Convert bpf_timer_set_callback() args into timer callback args */ 11497 err = set_callee_state_cb(env, caller, callee, insn_idx); 11498 if (err) 11499 return err; 11500 11501 return 0; 11502 } 11503 11504 /* 11505 * The address of the callback might be taken in another function or 11506 * read from a map, see check_func_ptr_read(). Like with callx it's 11507 * known here what is called. 11508 */ 11509 err = record_callx_edge(env, caller->subprogno, subprog); 11510 if (err) 11511 return err; 11512 11513 if (!bpf_pseudo_kfunc_call(insn) && 11514 (insn->imm == BPF_FUNC_loop || insn->imm == BPF_FUNC_for_each_map_elem)) 11515 env->subprog_info[subprog].is_loop_cb = true; 11516 else 11517 env->subprog_info[subprog].is_called = true; 11518 11519 /* for callback functions enqueue entry to callback and 11520 * proceed with next instruction within current frame. 11521 */ 11522 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); 11523 if (IS_ERR(callback_state)) 11524 return PTR_ERR(callback_state); 11525 11526 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, 11527 callback_state); 11528 if (err) 11529 return err; 11530 11531 callback_state->callback_unroll_depth++; 11532 callback_state->frame[callback_state->curframe - 1]->callback_depth++; 11533 caller->callback_depth = 0; 11534 return 0; 11535 } 11536 11537 static int process_bpf_exit_full(struct bpf_verifier_env *env, 11538 bool *do_print_state, bool exception_exit); 11539 11540 /* 11541 * Call of a static subprog. The callee is verified in the context of 11542 * the caller, hence set up a new frame and continue from the first 11543 * instruction of the callee. 11544 */ 11545 static int check_static_func_call(struct bpf_verifier_env *env, int subprog, 11546 int *insn_idx) 11547 { 11548 struct bpf_verifier_state *state = env->cur_state; 11549 struct bpf_subprog_info *caller_info; 11550 u16 callee_incoming, stack_arg_cnt; 11551 struct bpf_func_state *caller; 11552 int err; 11553 11554 caller = state->frame[state->curframe]; 11555 env->subprog_info[subprog].is_called = true; 11556 11557 /* 11558 * Track caller's total stack arg count (incoming + max outgoing). 11559 * This is needed so the JIT knows how much stack arg space to allocate. 11560 */ 11561 caller_info = &env->subprog_info[caller->subprogno]; 11562 callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]); 11563 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming; 11564 if (stack_arg_cnt > caller_info->stack_arg_cnt) 11565 caller_info->stack_arg_cnt = stack_arg_cnt; 11566 11567 /* 11568 * For regular function entry setup new frame and continue 11569 * from that frame. 11570 */ 11571 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); 11572 if (err) 11573 return err; 11574 11575 bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); 11576 clear_caller_saved_regs(env, caller->regs); 11577 11578 /* and go analyze first insn of the callee */ 11579 *insn_idx = env->subprog_info[subprog].start - 1; 11580 11581 if (env->log.level & BPF_LOG_LEVEL) { 11582 verbose(env, "caller:\n"); 11583 print_verifier_state(env, state, caller->frameno, true); 11584 verbose(env, "callee:\n"); 11585 print_verifier_state(env, state, state->curframe, true); 11586 } 11587 11588 return 0; 11589 } 11590 11591 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 11592 int *insn_idx) 11593 { 11594 struct bpf_verifier_state *state = env->cur_state; 11595 struct bpf_func_state *caller; 11596 int err, subprog, target_insn; 11597 u32 i, nregs; 11598 11599 target_insn = *insn_idx + insn->imm + 1; 11600 subprog = bpf_find_subprog(env, target_insn); 11601 if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", 11602 target_insn)) 11603 return -EFAULT; 11604 11605 caller = state->frame[state->curframe]; 11606 err = btf_check_subprog_call(env, subprog); 11607 if (err == -EFAULT) 11608 return err; 11609 if (bpf_subprog_is_global(env, subprog)) { 11610 struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); 11611 const char *sub_name = bpf_subprog_name(env, subprog); 11612 const char *operation; 11613 bool returns_void; 11614 11615 if (env->cur_state->active_locks) { 11616 verbose(env, "global function calls are not allowed while holding a lock,\n" 11617 "use static function instead\n"); 11618 operation = bpf_diag_fmt(env, "global function %s()", sub_name); 11619 bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, 11620 "Release the lock before calling the global function, or use a static function instead."); 11621 return -EINVAL; 11622 } 11623 11624 if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { 11625 const char *suggestion; 11626 11627 verbose(env, "sleepable global function %s() called in %s\n", 11628 sub_name, non_sleepable_context_description(env)); 11629 if (in_sleepable(env)) 11630 suggestion = "Move the call outside the critical section, or use a non-sleepable function."; 11631 else 11632 suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function."; 11633 operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); 11634 bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion); 11635 return -EINVAL; 11636 } 11637 11638 if (err) { 11639 verbose(env, "Caller passes invalid args into func#%d ('%s')\n", 11640 subprog, sub_name); 11641 return err; 11642 } 11643 11644 if (env->log.level & BPF_LOG_LEVEL) 11645 verbose(env, "Func#%d ('%s') is global and assumed valid.\n", 11646 subprog, sub_name); 11647 if (in_sleepable_context(env)) 11648 sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true; 11649 else if (has_rcu_read_lock(env)) 11650 sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true; 11651 else 11652 sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true; 11653 returns_void = subprog_returns_void(env, subprog); 11654 if (env->subprog_info[subprog].changes_pkt_data) 11655 clear_all_pkt_pointers(env); 11656 if (returns_void) 11657 bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); 11658 else 11659 bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); 11660 clear_caller_saved_regs(env, caller->regs); 11661 invalidate_outgoing_stack_args(env, cur_func(env)); 11662 /* 11663 * What the callers of a global function hold is not known 11664 * while it is verified. 11665 */ 11666 if (!env->subprog_info[subprog].changes_pkt_data && 11667 (state->frame[0]->subprogno || holds_pkt_pointers(env))) 11668 sub_aux->called_with_pkt_ptrs = true; 11669 11670 /* 11671 * A non-void global function returns a 64-bit SCALAR_VALUE in 11672 * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. 11673 */ 11674 if (!returns_void) { 11675 nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; 11676 mark_reg_unknown(env, caller->regs, ret_regs[0]); 11677 bpf_diag_mod_end(env); 11678 for (i = 1; i < nregs; i++) { 11679 bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, 11680 BPF_DIAG_MOD_WRITE); 11681 mark_reg_unknown(env, caller->regs, ret_regs[i]); 11682 bpf_diag_mod_end(env); 11683 } 11684 } 11685 11686 if (env->subprog_info[subprog].can_unwind) { 11687 struct bpf_verifier_state *branch; 11688 11689 if (!bpf_is_unwind_goto(insn + 1)) { 11690 verbose(env, "no unwind_goto after the call of %s()\n", sub_name); 11691 return -EINVAL; 11692 } 11693 branch = push_stack(env, *insn_idx + 1, *insn_idx, false); 11694 if (IS_ERR(branch)) 11695 return PTR_ERR(branch); 11696 branch->unwound = true; 11697 } 11698 11699 if (env->subprog_info[subprog].might_throw) { 11700 struct bpf_verifier_state *branch; 11701 11702 branch = push_stack(env, *insn_idx + 1, *insn_idx, false); 11703 if (IS_ERR(branch)) { 11704 verbose(env, "failed to push state for global subprog exception path\n"); 11705 return PTR_ERR(branch); 11706 } 11707 return process_bpf_exit_full(env, NULL, true); 11708 } 11709 11710 /* continue with next insn after call */ 11711 return 0; 11712 } 11713 11714 return check_static_func_call(env, subprog, insn_idx); 11715 } 11716 11717 /* 11718 * callx dst_reg: call a bpf subprog whose address is in dst_reg. 11719 * 11720 * The address of a subprog is either loaded into a register by ld_imm64 with 11721 * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see 11722 * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all 11723 * possible callees of callx are discovered by add_subprogs() and are reachable 11724 * in the control flow graph before the main verification pass begins. 11725 * PTR_TO_FUNC register identifies the callee, so from here on callx is verified 11726 * as a direct call of that static subprog. 11727 */ 11728 static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, 11729 int *insn_idx) 11730 { 11731 struct bpf_func_state *caller = cur_func(env); 11732 struct bpf_reg_state *reg; 11733 const char *reason; 11734 int err, subprog; 11735 11736 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11737 if (err) 11738 return err; 11739 11740 reg = reg_state(env, insn->dst_reg); 11741 if (reg->type != PTR_TO_FUNC) { 11742 verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, 11743 reg_type_str(env, reg->type)); 11744 reason = bpf_diag_fmt( 11745 env, "R%d holds %s, but callx can only call through the address of a static BPF function.", 11746 insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); 11747 bpf_diag_register_type( 11748 env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, 11749 "Load the address of a static BPF function into the register before callx."); 11750 return -EACCES; 11751 } 11752 11753 /* 11754 * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address 11755 * of a subprog can be called. 11756 */ 11757 err = check_ptr_off_reg(env, reg, insn->dst_reg); 11758 if (err) 11759 return err; 11760 11761 /* the callx is valid, but only a JIT can execute it */ 11762 err = require_callx_jit(env); 11763 if (err) 11764 return err; 11765 11766 /* PTR_TO_FUNC is a pointer to a static subprog */ 11767 subprog = reg->subprogno; 11768 err = btf_check_subprog_call(env, subprog); 11769 if (err == -EFAULT) 11770 return err; 11771 11772 err = record_callx_edge(env, caller->subprogno, subprog); 11773 if (err) 11774 return err; 11775 11776 return check_static_func_call(env, subprog, insn_idx); 11777 } 11778 11779 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 11780 struct bpf_func_state *caller, 11781 struct bpf_func_state *callee) 11782 { 11783 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 11784 * void *callback_ctx, u64 flags); 11785 * callback_fn(struct bpf_map *map, void *key, void *value, 11786 * void *callback_ctx); 11787 */ 11788 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 11789 11790 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 11791 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 11792 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 11793 callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid; 11794 11795 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 11796 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 11797 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 11798 callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; 11799 callee->regs[BPF_REG_3].id = ++env->id_gen; 11800 11801 /* pointer to stack or null */ 11802 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 11803 11804 /* unused */ 11805 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 11806 return 0; 11807 } 11808 11809 static int set_callee_state(struct bpf_verifier_env *env, 11810 struct bpf_func_state *caller, 11811 struct bpf_func_state *callee, int insn_idx) 11812 { 11813 int i; 11814 11815 /* copy r1 - r5 args that callee can access. The copy includes parent 11816 * pointers, which connects us up to the liveness chain 11817 */ 11818 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 11819 callee->regs[i] = caller->regs[i]; 11820 return 0; 11821 } 11822 11823 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 11824 struct bpf_func_state *caller, 11825 struct bpf_func_state *callee, 11826 int insn_idx) 11827 { 11828 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 11829 struct bpf_map *map; 11830 int err; 11831 11832 /* valid map_ptr and poison value does not matter */ 11833 map = insn_aux->map_ptr_state.map_ptr; 11834 if (!map->ops->map_set_for_each_callback_args || 11835 !map->ops->map_for_each_callback) { 11836 verbose(env, "callback function not allowed for map\n"); 11837 return -ENOTSUPP; 11838 } 11839 11840 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 11841 if (err) 11842 return err; 11843 11844 switch (map->map_type) { 11845 case BPF_MAP_TYPE_HASH: 11846 case BPF_MAP_TYPE_PERCPU_HASH: 11847 case BPF_MAP_TYPE_LRU_HASH: 11848 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 11849 case BPF_MAP_TYPE_RHASH: 11850 /* Hash tables run the callback under rcu_read_lock(). */ 11851 callee->in_rcu_callback_fn = true; 11852 break; 11853 default: 11854 break; 11855 } 11856 11857 callee->in_callback_fn = true; 11858 callee->callback_ret_range = retval_range(0, 1); 11859 return 0; 11860 } 11861 11862 static int set_loop_callback_state(struct bpf_verifier_env *env, 11863 struct bpf_func_state *caller, 11864 struct bpf_func_state *callee, 11865 int insn_idx) 11866 { 11867 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 11868 * u64 flags); 11869 * callback_fn(u64 index, void *callback_ctx); 11870 */ 11871 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 11872 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 11873 11874 /* unused */ 11875 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 11876 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 11877 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 11878 11879 callee->in_callback_fn = true; 11880 callee->callback_ret_range = retval_range(0, 1); 11881 return 0; 11882 } 11883 11884 static int set_timer_callback_state(struct bpf_verifier_env *env, 11885 struct bpf_func_state *caller, 11886 struct bpf_func_state *callee, 11887 int insn_idx) 11888 { 11889 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 11890 u32 map_uid = caller->regs[BPF_REG_1].map_uid; 11891 11892 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 11893 * callback_fn(struct bpf_map *map, void *key, void *value); 11894 */ 11895 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 11896 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 11897 callee->regs[BPF_REG_1].map_ptr = map_ptr; 11898 callee->regs[BPF_REG_1].map_uid = map_uid; 11899 11900 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 11901 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 11902 callee->regs[BPF_REG_2].map_ptr = map_ptr; 11903 callee->regs[BPF_REG_2].map_uid = map_uid; 11904 11905 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 11906 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 11907 callee->regs[BPF_REG_3].map_ptr = map_ptr; 11908 callee->regs[BPF_REG_3].map_uid = map_uid; 11909 callee->regs[BPF_REG_3].id = ++env->id_gen; 11910 11911 /* unused */ 11912 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 11913 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 11914 callee->in_async_callback_fn = true; 11915 callee->callback_ret_range = retval_range(0, 0); 11916 return 0; 11917 } 11918 11919 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 11920 struct bpf_func_state *caller, 11921 struct bpf_func_state *callee, 11922 int insn_idx) 11923 { 11924 /* bpf_find_vma(struct task_struct *task, u64 addr, 11925 * void *callback_fn, void *callback_ctx, u64 flags) 11926 * (callback_fn)(struct task_struct *task, 11927 * struct vm_area_struct *vma, void *callback_ctx); 11928 */ 11929 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 11930 11931 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 11932 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 11933 callee->regs[BPF_REG_2].btf = btf_vmlinux; 11934 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA]; 11935 11936 /* pointer to stack or null */ 11937 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 11938 11939 /* unused */ 11940 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 11941 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 11942 callee->in_callback_fn = true; 11943 /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ 11944 callee->in_rcu_callback_fn = true; 11945 callee->callback_ret_range = retval_range(0, 1); 11946 return 0; 11947 } 11948 11949 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 11950 struct bpf_func_state *caller, 11951 struct bpf_func_state *callee, 11952 int insn_idx) 11953 { 11954 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 11955 * callback_ctx, u64 flags); 11956 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx); 11957 */ 11958 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 11959 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); 11960 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 11961 11962 /* unused */ 11963 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 11964 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 11965 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 11966 11967 callee->in_callback_fn = true; 11968 callee->callback_ret_range = retval_range(0, 1); 11969 return 0; 11970 } 11971 11972 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, 11973 struct bpf_func_state *caller, 11974 struct bpf_func_state *callee, 11975 int insn_idx) 11976 { 11977 /* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 11978 * bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)); 11979 * 11980 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset 11981 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd 11982 * by this point, so look at 'root' 11983 */ 11984 struct btf_field *field; 11985 11986 field = reg_find_field_offset(&caller->regs[BPF_REG_1], 11987 caller->regs[BPF_REG_1].var_off.value, 11988 BPF_RB_ROOT); 11989 if (!field || !field->graph_root.value_btf_id) 11990 return -EFAULT; 11991 11992 mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root); 11993 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); 11994 mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root); 11995 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); 11996 11997 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 11998 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 11999 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 12000 callee->in_callback_fn = true; 12001 callee->callback_ret_range = retval_range(0, 1); 12002 return 0; 12003 } 12004 12005 static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, 12006 struct bpf_func_state *caller, 12007 struct bpf_func_state *callee, 12008 int insn_idx) 12009 { 12010 struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr; 12011 u32 map_uid = caller->regs[BPF_REG_3].map_uid; 12012 12013 /* 12014 * callback_fn(struct bpf_map *map, void *key, void *value); 12015 */ 12016 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 12017 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 12018 callee->regs[BPF_REG_1].map_ptr = map_ptr; 12019 callee->regs[BPF_REG_1].map_uid = map_uid; 12020 12021 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 12022 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 12023 callee->regs[BPF_REG_2].map_ptr = map_ptr; 12024 callee->regs[BPF_REG_2].map_uid = map_uid; 12025 12026 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 12027 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 12028 callee->regs[BPF_REG_3].map_ptr = map_ptr; 12029 callee->regs[BPF_REG_3].map_uid = map_uid; 12030 callee->regs[BPF_REG_3].id = ++env->id_gen; 12031 12032 /* unused */ 12033 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 12034 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 12035 callee->in_async_callback_fn = true; 12036 callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); 12037 return 0; 12038 } 12039 12040 static int set_rcu_callback_state(struct bpf_verifier_env *env, 12041 struct bpf_func_state *caller, 12042 struct bpf_func_state *callee, 12043 int insn_idx) 12044 { 12045 struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr; 12046 u32 map_uid = caller->regs[BPF_REG_2].map_uid; 12047 12048 /* 12049 * callback_fn(struct bpf_map *map, void *key, void *value); 12050 */ 12051 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 12052 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 12053 callee->regs[BPF_REG_1].map_ptr = map_ptr; 12054 callee->regs[BPF_REG_1].map_uid = map_uid; 12055 12056 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 12057 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 12058 callee->regs[BPF_REG_2].map_ptr = map_ptr; 12059 callee->regs[BPF_REG_2].map_uid = map_uid; 12060 12061 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 12062 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 12063 callee->regs[BPF_REG_3].map_ptr = map_ptr; 12064 callee->regs[BPF_REG_3].map_uid = map_uid; 12065 12066 /* unused */ 12067 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 12068 bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 12069 callee->in_async_callback_fn = true; 12070 callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); 12071 return 0; 12072 } 12073 12074 static bool is_rbtree_lock_required_kfunc(u32 btf_id); 12075 12076 static void account_processed_insn(struct bpf_verifier_env *env) 12077 { 12078 struct bpf_func_state *frame = cur_func(env); 12079 12080 env->insn_processed++; 12081 frame->insns_subtotal++; 12082 env->subprog_info[frame->subprogno].insns_self++; 12083 } 12084 12085 static void account_processed_insns(struct bpf_verifier_env *env, 12086 struct bpf_func_state *callee, 12087 struct bpf_func_state *caller) 12088 { 12089 u32 insns; 12090 12091 if (!callee) 12092 return; 12093 12094 insns = callee->insns_subtotal; 12095 12096 env->subprog_info[callee->subprogno].insns_total += insns; 12097 if (caller) 12098 caller->insns_subtotal += insns; 12099 callee->insns_subtotal = 0; 12100 } 12101 12102 static void account_current_path(struct bpf_verifier_env *env) 12103 { 12104 struct bpf_verifier_state *state = env->cur_state; 12105 int frame; 12106 12107 for (frame = state->curframe; frame >= 0; frame--) 12108 account_processed_insns(env, state->frame[frame], 12109 frame ? state->frame[frame - 1] : NULL); 12110 } 12111 12112 /* 12113 * Are we currently verifying the callback for an rbtree kfunc that must 12114 * be called with a lock held, or one of that callback's subprogs? If so, 12115 * no need to complain about an unreleased lock. 12116 */ 12117 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) 12118 { 12119 struct bpf_verifier_state *state = env->cur_state; 12120 struct bpf_insn *insn = env->prog->insnsi; 12121 struct bpf_func_state *callee; 12122 int kfunc_btf_id; 12123 u32 frame; 12124 12125 for (frame = state->curframe; frame; frame--) { 12126 callee = state->frame[frame]; 12127 if (!callee->in_callback_fn) 12128 continue; 12129 12130 kfunc_btf_id = insn[callee->callsite].imm; 12131 if (is_rbtree_lock_required_kfunc(kfunc_btf_id)) 12132 return true; 12133 } 12134 12135 return false; 12136 } 12137 12138 static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) 12139 { 12140 if (range.return_32bit) 12141 return range.minval <= reg_s32_min(reg) && reg_s32_max(reg) <= range.maxval; 12142 else 12143 return range.minval <= reg_smin(reg) && reg_smax(reg) <= range.maxval; 12144 } 12145 12146 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 12147 { 12148 struct bpf_verifier_state *state = env->cur_state, *prev_st; 12149 struct bpf_func_state *caller, *callee; 12150 struct bpf_reg_state *r0; 12151 bool in_callback_fn; 12152 u32 i, nregs; 12153 int err; 12154 12155 callee = state->frame[state->curframe]; 12156 r0 = &callee->regs[BPF_REG_0]; 12157 nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; 12158 for (i = 0; i < nregs; i++) { 12159 if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) 12160 continue; 12161 /* technically it's ok to return caller's stack pointer 12162 * (or caller's caller's pointer) back to the caller, 12163 * since these pointers are valid. Only current stack 12164 * pointer will be invalid as soon as function exits, 12165 * but let's be conservative 12166 */ 12167 verbose(env, "cannot return stack pointer to the caller\n"); 12168 return -EINVAL; 12169 } 12170 12171 caller = state->frame[state->curframe - 1]; 12172 if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) { 12173 /* the helper or the kfunc that called it doesn't expect that */ 12174 if (callee->in_callback_fn) { 12175 verbose(env, "callback cannot unwind\n"); 12176 return -EINVAL; 12177 } 12178 if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) { 12179 verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n", 12180 callee->callsite); 12181 return -EINVAL; 12182 } 12183 state->unwound = true; 12184 } 12185 if (callee->in_callback_fn) { 12186 if (r0->type != SCALAR_VALUE) { 12187 verbose(env, "R0 not a scalar value\n"); 12188 return -EACCES; 12189 } 12190 12191 /* we are going to rely on register's precise value */ 12192 err = mark_chain_precision(env, BPF_REG_0); 12193 if (err) 12194 return err; 12195 12196 /* enforce R0 return value range, and bpf_callback_t returns 64bit */ 12197 if (!retval_range_within(callee->callback_ret_range, r0)) { 12198 verbose_invalid_scalar(env, r0, callee->callback_ret_range, 12199 "At callback return", "R0"); 12200 return -EINVAL; 12201 } 12202 if (!bpf_calls_callback(env, callee->callsite)) { 12203 verifier_bug(env, "in callback at %d, callsite %d !calls_callback", 12204 *insn_idx, callee->callsite); 12205 return -EFAULT; 12206 } 12207 } else { 12208 /* 12209 * return to the caller whatever the callee had in the 12210 * return register(s) 12211 */ 12212 for (i = 0; i < nregs; i++) { 12213 u32 regno = ret_regs[i]; 12214 12215 bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], 12216 BPF_DIAG_MOD_WRITE); 12217 caller->regs[regno] = callee->regs[regno]; 12218 bpf_diag_mod_end(env); 12219 } 12220 } 12221 12222 /* Invalidate callee-local dynptrs and their slices before the frame goes away. */ 12223 err = destroy_dynptrs_in_stack_slots(env, callee, 0, 12224 callee->allocated_stack / BPF_REG_SIZE - 1); 12225 if (err) 12226 return err; 12227 invalidate_stack_slices(env, callee->frameno, INT_MIN, 0); 12228 12229 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, 12230 * there function call logic would reschedule callback visit. If iteration 12231 * converges is_state_visited() would prune that visit eventually. 12232 */ 12233 in_callback_fn = callee->in_callback_fn; 12234 if (in_callback_fn) 12235 *insn_idx = callee->callsite; 12236 else 12237 *insn_idx = callee->callsite + 1; 12238 12239 if (env->log.level & BPF_LOG_LEVEL) { 12240 verbose(env, "returning from callee:\n"); 12241 print_verifier_state(env, state, callee->frameno, true); 12242 verbose(env, "to caller at %d:\n", *insn_idx); 12243 print_verifier_state(env, state, caller->frameno, true); 12244 } 12245 account_processed_insns(env, callee, caller); 12246 /* clear everything in the callee. In case of exceptional exits using 12247 * bpf_throw, this will be done by copy_verifier_state for extra frames. */ 12248 free_func_state(callee); 12249 state->frame[state->curframe--] = NULL; 12250 invalidate_outgoing_stack_args(env, caller); 12251 12252 /* for callbacks widen imprecise scalars to make programs like below verify: 12253 * 12254 * struct ctx { int i; } 12255 * void cb(int idx, struct ctx *ctx) { ctx->i++; ... } 12256 * ... 12257 * struct ctx = { .i = 0; } 12258 * bpf_loop(100, cb, &ctx, 0); 12259 * 12260 * This is similar to what is done in process_iter_next_call() for open 12261 * coded iterators. 12262 */ 12263 prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL; 12264 if (prev_st) { 12265 err = widen_imprecise_scalars(env, prev_st, state); 12266 if (err) 12267 return err; 12268 } 12269 return 0; 12270 } 12271 12272 static int do_refine_retval_range(struct bpf_verifier_env *env, 12273 struct bpf_reg_state *regs, int ret_type, 12274 int func_id, 12275 struct bpf_call_arg_meta *meta) 12276 { 12277 struct bpf_retval_range range; 12278 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 12279 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 12280 12281 if (ret_type != RET_INTEGER) 12282 return 0; 12283 12284 switch (func_id) { 12285 case BPF_FUNC_get_stack: 12286 case BPF_FUNC_get_task_stack: 12287 case BPF_FUNC_probe_read_str: 12288 case BPF_FUNC_probe_read_kernel_str: 12289 case BPF_FUNC_probe_read_user_str: 12290 reg_set_srange64(ret_reg, -MAX_ERRNO, meta->msize_max_value); 12291 reg_set_srange32(ret_reg, -MAX_ERRNO, meta->msize_max_value); 12292 reg_bounds_sync(ret_reg); 12293 break; 12294 case BPF_FUNC_get_smp_processor_id: 12295 reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1); 12296 reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1); 12297 reg_bounds_sync(ret_reg); 12298 break; 12299 case BPF_FUNC_get_retval: 12300 /* 12301 * bpf_get_retval may see arbitrary value passed by bpf_prog_run_array_cg for 12302 * CGROUP_GETSOCKOPT type. 12303 */ 12304 if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && 12305 env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) 12306 break; 12307 12308 if (prog_type == BPF_PROG_TYPE_LSM && 12309 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 12310 if (!env->prog->aux->attach_func_proto->type) 12311 break; 12312 bpf_lsm_get_retval_range(env->prog, &range); 12313 } else { 12314 range.minval = -MAX_ERRNO; 12315 range.maxval = 0; 12316 } 12317 12318 reg_set_srange64(ret_reg, range.minval, range.maxval); 12319 reg_set_srange32(ret_reg, range.minval, range.maxval); 12320 reg_bounds_sync(ret_reg); 12321 break; 12322 } 12323 12324 return reg_bounds_sanity_check(env, ret_reg, "retval"); 12325 } 12326 12327 static int 12328 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 12329 int func_id, int insn_idx) 12330 { 12331 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 12332 struct bpf_map *map = meta->map.ptr; 12333 12334 if (func_id != BPF_FUNC_tail_call && 12335 func_id != BPF_FUNC_map_lookup_elem && 12336 func_id != BPF_FUNC_map_update_elem && 12337 func_id != BPF_FUNC_map_delete_elem && 12338 func_id != BPF_FUNC_map_push_elem && 12339 func_id != BPF_FUNC_map_pop_elem && 12340 func_id != BPF_FUNC_map_peek_elem && 12341 func_id != BPF_FUNC_for_each_map_elem && 12342 func_id != BPF_FUNC_redirect_map && 12343 func_id != BPF_FUNC_map_lookup_percpu_elem) 12344 return 0; 12345 12346 if (map == NULL) { 12347 verifier_bug(env, "expected map for helper call"); 12348 return -EFAULT; 12349 } 12350 12351 /* In case of read-only, some additional restrictions 12352 * need to be applied in order to prevent altering the 12353 * state of the map from program side. 12354 */ 12355 if ((map->map_flags & BPF_F_RDONLY_PROG) && 12356 (func_id == BPF_FUNC_map_delete_elem || 12357 func_id == BPF_FUNC_map_update_elem || 12358 func_id == BPF_FUNC_map_push_elem || 12359 func_id == BPF_FUNC_map_pop_elem)) { 12360 verbose(env, "write into map forbidden\n"); 12361 return -EACCES; 12362 } 12363 12364 if (!aux->map_ptr_state.map_ptr) 12365 bpf_map_ptr_store(aux, meta->map.ptr, 12366 !meta->map.ptr->bypass_spec_v1, false); 12367 else if (aux->map_ptr_state.map_ptr != meta->map.ptr) 12368 bpf_map_ptr_store(aux, meta->map.ptr, 12369 !meta->map.ptr->bypass_spec_v1, true); 12370 return 0; 12371 } 12372 12373 static int 12374 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 12375 int func_id, int insn_idx) 12376 { 12377 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 12378 struct bpf_reg_state *reg; 12379 struct bpf_map *map = meta->map.ptr; 12380 u64 val, max; 12381 int err; 12382 12383 if (func_id != BPF_FUNC_tail_call) 12384 return 0; 12385 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 12386 verbose(env, "expected prog array map for tail call"); 12387 return -EINVAL; 12388 } 12389 12390 reg = reg_state(env, BPF_REG_3); 12391 val = reg->var_off.value; 12392 max = map->max_entries; 12393 12394 if (!(is_reg_const(reg, false) && val < max)) { 12395 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 12396 return 0; 12397 } 12398 12399 err = mark_chain_precision(env, BPF_REG_3); 12400 if (err) 12401 return err; 12402 if (bpf_map_key_unseen(aux)) 12403 bpf_map_key_store(aux, val); 12404 else if (!bpf_map_key_poisoned(aux) && 12405 bpf_map_key_immediate(aux) != val) 12406 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 12407 return 0; 12408 } 12409 12410 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) 12411 { 12412 struct bpf_verifier_state *state = env->cur_state; 12413 enum bpf_prog_type type = resolve_prog_type(env->prog); 12414 struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); 12415 bool refs_lingering = false; 12416 int i; 12417 12418 if (!exception_exit && cur_func(env)->frameno) 12419 return 0; 12420 12421 for (i = 0; i < state->acquired_refs; i++) { 12422 if (state->refs[i].type != REF_TYPE_PTR) 12423 continue; 12424 /* Allow struct_ops programs to return a referenced kptr back to 12425 * kernel. Type checks are performed later in check_return_code. 12426 */ 12427 if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit && 12428 reg->id == state->refs[i].id) 12429 continue; 12430 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 12431 state->refs[i].id, state->refs[i].insn_idx); 12432 bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx); 12433 refs_lingering = true; 12434 } 12435 return refs_lingering ? -EINVAL : 0; 12436 } 12437 12438 static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix) 12439 { 12440 int err; 12441 12442 if (check_lock && env->cur_state->active_locks) { 12443 verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); 12444 bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK, 12445 "Release the BPF spin lock before this operation on every path."); 12446 return -EINVAL; 12447 } 12448 12449 err = check_reference_leak(env, exception_exit); 12450 if (err) { 12451 verbose(env, "%s would lead to reference leak\n", prefix); 12452 return err; 12453 } 12454 12455 if (check_lock && env->cur_state->active_irq_id) { 12456 verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); 12457 bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ, 12458 "Restore the saved IRQ state before this operation on every path."); 12459 return -EINVAL; 12460 } 12461 12462 if (check_lock && env->cur_state->active_rcu_locks) { 12463 verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); 12464 bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU, 12465 "Call bpf_rcu_read_unlock() before this operation on every path."); 12466 return -EINVAL; 12467 } 12468 12469 if (check_lock && env->cur_state->active_preempt_locks) { 12470 verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); 12471 bpf_diag_ctx_active( 12472 env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT, 12473 "Call bpf_preempt_enable() before this operation on every path."); 12474 return -EINVAL; 12475 } 12476 12477 return 0; 12478 } 12479 12480 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 12481 struct bpf_reg_state *regs) 12482 { 12483 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 12484 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 12485 struct bpf_map *fmt_map = fmt_reg->map_ptr; 12486 struct bpf_bprintf_data data = {}; 12487 int err, fmt_map_off, num_args; 12488 u64 fmt_addr; 12489 char *fmt; 12490 12491 /* data must be an array of u64 */ 12492 if (data_len_reg->var_off.value % 8) 12493 return -EINVAL; 12494 num_args = data_len_reg->var_off.value / 8; 12495 12496 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 12497 * and map_direct_value_addr is set. 12498 */ 12499 fmt_map_off = fmt_reg->var_off.value; 12500 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 12501 fmt_map_off); 12502 if (err) { 12503 verbose(env, "failed to retrieve map value address\n"); 12504 return -EFAULT; 12505 } 12506 fmt = (char *)(long)fmt_addr + fmt_map_off; 12507 12508 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 12509 * can focus on validating the format specifiers. 12510 */ 12511 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data); 12512 if (err < 0) 12513 verbose(env, "Invalid format string\n"); 12514 12515 return err; 12516 } 12517 12518 static int check_get_func_ip(struct bpf_verifier_env *env) 12519 { 12520 enum bpf_prog_type type = resolve_prog_type(env->prog); 12521 int func_id = BPF_FUNC_get_func_ip; 12522 12523 if (type == BPF_PROG_TYPE_TRACING) { 12524 if (!bpf_prog_has_trampoline(env->prog)) { 12525 verbose(env, "func %s#%d supported only for fentry/fexit/fsession/fmod_ret programs\n", 12526 func_id_name(func_id), func_id); 12527 return -ENOTSUPP; 12528 } 12529 return 0; 12530 } else if (type == BPF_PROG_TYPE_KPROBE) { 12531 return 0; 12532 } 12533 12534 verbose(env, "func %s#%d not supported for program type %d\n", 12535 func_id_name(func_id), func_id, type); 12536 return -ENOTSUPP; 12537 } 12538 12539 static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env) 12540 { 12541 return &env->insn_aux_data[env->insn_idx]; 12542 } 12543 12544 /* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */ 12545 static int loop_flag_is_zero(struct bpf_verifier_env *env) 12546 { 12547 struct bpf_reg_state *reg = reg_state(env, BPF_REG_4); 12548 int err; 12549 12550 if (!bpf_register_is_null(reg)) 12551 return 0; 12552 12553 err = mark_chain_precision(env, BPF_REG_4); 12554 if (err) 12555 return err; 12556 return 1; 12557 } 12558 12559 static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 12560 { 12561 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 12562 int flag_is_zero; 12563 12564 if (!state->initialized) { 12565 flag_is_zero = loop_flag_is_zero(env); 12566 if (flag_is_zero < 0) 12567 return flag_is_zero; 12568 state->initialized = 1; 12569 state->fit_for_inline = flag_is_zero; 12570 state->callback_subprogno = subprogno; 12571 return 0; 12572 } 12573 12574 if (!state->fit_for_inline) 12575 return 0; 12576 12577 flag_is_zero = loop_flag_is_zero(env); 12578 if (flag_is_zero < 0) 12579 return flag_is_zero; 12580 state->fit_for_inline = (flag_is_zero && 12581 state->callback_subprogno == subprogno); 12582 return 0; 12583 } 12584 12585 /* Returns whether or not the given map can potentially elide 12586 * lookup return value nullness check. This is possible if the key 12587 * is statically known. 12588 */ 12589 static bool can_elide_value_nullness(const struct bpf_map *map) 12590 { 12591 if (map->map_flags & BPF_F_INNER_MAP) 12592 return false; 12593 12594 switch (map->map_type) { 12595 case BPF_MAP_TYPE_ARRAY: 12596 case BPF_MAP_TYPE_PERCPU_ARRAY: 12597 return true; 12598 default: 12599 return false; 12600 } 12601 } 12602 12603 int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, 12604 const struct bpf_func_proto **ptr) 12605 { 12606 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) 12607 return -ERANGE; 12608 12609 if (!env->ops->get_func_proto) 12610 return -EINVAL; 12611 12612 *ptr = env->ops->get_func_proto(func_id, env->prog); 12613 return *ptr && (*ptr)->func ? 0 : -EINVAL; 12614 } 12615 12616 /* Is any frame on the current path a callback that runs under RCU? */ 12617 static bool in_rcu_callback(struct bpf_verifier_env *env) 12618 { 12619 struct bpf_verifier_state *state = env->cur_state; 12620 int i; 12621 12622 for (i = 0; i <= state->curframe; i++) 12623 if (state->frame[i]->in_rcu_callback_fn) 12624 return true; 12625 return false; 12626 } 12627 12628 /* Check if we're in a sleepable context. */ 12629 static inline bool in_sleepable_context(struct bpf_verifier_env *env) 12630 { 12631 return !in_rcu_cs(env) && !in_rcu_callback(env); 12632 } 12633 12634 static const char *non_sleepable_context_description(struct bpf_verifier_env *env) 12635 { 12636 if (in_sleepable(env)) { 12637 if (env->cur_state->active_rcu_locks) 12638 return "rcu_read_lock region"; 12639 if (env->cur_state->active_preempt_locks) 12640 return "non-preemptible region"; 12641 if (env->cur_state->active_irq_id) 12642 return "IRQ-disabled region"; 12643 if (env->cur_state->active_locks) 12644 return "lock region"; 12645 return "callback running under rcu_read_lock"; 12646 } 12647 return "non-sleepable prog"; 12648 } 12649 12650 static int release_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 12651 bool convert_rcu, bool release_dynptr) 12652 { 12653 int err = -EINVAL; 12654 12655 if (bpf_register_is_null(reg)) 12656 return 0; 12657 12658 if (release_dynptr) 12659 err = unmark_stack_slots_dynptr(env, reg); 12660 else if (convert_rcu) 12661 err = ref_convert_alloc_rcu_protected(env, reg->id); 12662 else if (reg_is_referenced(env, reg)) 12663 err = release_reference(env, reg->id); 12664 12665 return err; 12666 } 12667 12668 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 12669 int *insn_idx_p) 12670 { 12671 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 12672 bool returns_cpu_specific_alloc_ptr = false; 12673 const struct bpf_func_proto *fn = NULL; 12674 enum bpf_return_type ret_type; 12675 enum bpf_type_flag ret_flag; 12676 struct bpf_reg_state *regs; 12677 struct bpf_call_arg_meta meta; 12678 const char *operation; 12679 int insn_idx = *insn_idx_p; 12680 bool changes_data; 12681 int i, err, func_id; 12682 12683 /* find function prototype */ 12684 func_id = insn->imm; 12685 err = bpf_get_helper_proto(env, insn->imm, &fn); 12686 if (err == -ERANGE) { 12687 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); 12688 return -EINVAL; 12689 } 12690 12691 if (err) { 12692 verbose(env, "program of this type cannot use helper %s#%d\n", 12693 func_id_name(func_id), func_id); 12694 operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); 12695 bpf_diag_policy( 12696 env, insn_idx, operation, "this program type does not allow the helper", 12697 "Use a helper allowed for this program type, or move the logic to a compatible program type."); 12698 return err; 12699 } 12700 12701 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 12702 if (!env->prog->gpl_compatible && fn->gpl_only) { 12703 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 12704 operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); 12705 bpf_diag_policy( 12706 env, insn_idx, operation, 12707 "this helper is restricted to GPL-compatible programs", 12708 "Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs."); 12709 return -EINVAL; 12710 } 12711 12712 if (fn->allowed && !fn->allowed(env->prog)) { 12713 verbose(env, "helper call is not allowed in probe\n"); 12714 operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); 12715 bpf_diag_policy( 12716 env, insn_idx, operation, 12717 "the helper-specific policy callback rejected this program", 12718 "Use the helper only from an allowed attach point or program configuration."); 12719 return -EINVAL; 12720 } 12721 12722 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 12723 changes_data = bpf_helper_changes_pkt_data(func_id); 12724 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 12725 verifier_bug(env, "func %s#%d: r1 != ctx", func_id_name(func_id), func_id); 12726 return -EFAULT; 12727 } 12728 12729 memset(&meta, 0, sizeof(meta)); 12730 12731 err = check_func_proto(env, fn, &meta); 12732 if (err) { 12733 verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); 12734 return err; 12735 } 12736 12737 if (fn->might_sleep && !in_sleepable_context(env)) { 12738 const char *suggestion; 12739 12740 verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, 12741 non_sleepable_context_description(env)); 12742 if (in_sleepable(env)) 12743 suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper."; 12744 else 12745 suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper."; 12746 operation = bpf_diag_fmt(env, "sleepable helper %s#%d", 12747 func_id_name(func_id), func_id); 12748 bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); 12749 return -EINVAL; 12750 } 12751 12752 /* Track non-sleepable context for helpers. */ 12753 if (!in_sleepable_context(env)) 12754 env->insn_aux_data[insn_idx].non_sleepable = true; 12755 12756 meta.func_id = func_id; 12757 meta.func_name = func_id_name(func_id); 12758 meta.fn = fn; 12759 err = check_func_args(env, &meta, insn_idx); 12760 if (err) 12761 return err; 12762 12763 err = record_func_map(env, &meta, func_id, insn_idx); 12764 if (err) 12765 return err; 12766 12767 err = record_func_key(env, &meta, func_id, insn_idx); 12768 if (err) 12769 return err; 12770 12771 regs = cur_regs(env); 12772 12773 err = mark_raw_stack(env, &meta, insn_idx); 12774 if (err) 12775 return err; 12776 12777 if (meta.release_regno) { 12778 struct bpf_reg_state *reg = ®s[meta.release_regno]; 12779 bool convert_rcu = (func_id == BPF_FUNC_kptr_xchg) && in_rcu_cs(env) && 12780 (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU); 12781 12782 err = release_reg(env, reg, convert_rcu, !!meta.dynptr.id); 12783 if (err) 12784 return err; 12785 } 12786 12787 switch (func_id) { 12788 case BPF_FUNC_tail_call: 12789 err = check_resource_leak(env, false, true, "tail_call"); 12790 if (err) 12791 return err; 12792 break; 12793 case BPF_FUNC_get_local_storage: 12794 /* check that flags argument in get_local_storage(map, flags) is 0, 12795 * this is required because get_local_storage() can't return an error. 12796 */ 12797 if (!bpf_register_is_null(®s[BPF_REG_2])) { 12798 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 12799 return -EINVAL; 12800 } 12801 err = mark_chain_precision(env, BPF_REG_2); 12802 if (err) 12803 return err; 12804 break; 12805 case BPF_FUNC_for_each_map_elem: 12806 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12807 set_map_elem_callback_state); 12808 break; 12809 case BPF_FUNC_timer_set_callback: 12810 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12811 set_timer_callback_state); 12812 break; 12813 case BPF_FUNC_find_vma: 12814 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12815 set_find_vma_callback_state); 12816 break; 12817 case BPF_FUNC_snprintf: 12818 err = check_bpf_snprintf_call(env, regs); 12819 break; 12820 case BPF_FUNC_loop: 12821 err = update_loop_inline_state(env, meta.subprogno); 12822 if (err) 12823 return err; 12824 /* Verifier relies on R1 value to determine if bpf_loop() iteration 12825 * is finished, thus mark it precise. 12826 */ 12827 err = mark_chain_precision(env, BPF_REG_1); 12828 if (err) 12829 return err; 12830 if (cur_func(env)->callback_depth < reg_umax(®s[BPF_REG_1])) { 12831 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12832 set_loop_callback_state); 12833 } else { 12834 cur_func(env)->callback_depth = 0; 12835 if (env->log.level & BPF_LOG_LEVEL2) 12836 verbose(env, "frame%d bpf_loop iteration limit reached\n", 12837 env->cur_state->curframe); 12838 } 12839 break; 12840 case BPF_FUNC_dynptr_from_mem: 12841 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 12842 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 12843 reg_type_str(env, regs[BPF_REG_1].type)); 12844 return -EACCES; 12845 } 12846 break; 12847 case BPF_FUNC_set_retval: 12848 { 12849 struct bpf_retval_range range = { 12850 .minval = -MAX_ERRNO, 12851 .maxval = 0, 12852 .return_32bit = true 12853 }; 12854 struct bpf_reg_state *r1 = ®s[BPF_REG_1]; 12855 12856 if (r1->type != SCALAR_VALUE) { 12857 verbose(env, "R1 is not a scalar\n"); 12858 return -EINVAL; 12859 } 12860 12861 /* CGROUP_GETSOCKOPT is allowed to return arbitrary value */ 12862 if (prog_type == BPF_PROG_TYPE_CGROUP_SOCKOPT && 12863 env->prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT) 12864 break; 12865 12866 if (prog_type == BPF_PROG_TYPE_LSM && 12867 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 12868 if (!env->prog->aux->attach_func_proto->type) { 12869 /* Make sure programs that attach to void 12870 * hooks don't try to modify return value. 12871 */ 12872 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 12873 return -EINVAL; 12874 } 12875 bpf_lsm_get_retval_range(env->prog, &range); 12876 } 12877 12878 err = mark_chain_precision(env, BPF_REG_1); 12879 if (err) 12880 return err; 12881 12882 if (!retval_range_within(range, r1)) { 12883 verbose_invalid_scalar(env, r1, range, "At bpf_set_retval", "R1"); 12884 return -EINVAL; 12885 } 12886 12887 break; 12888 } 12889 case BPF_FUNC_dynptr_write: 12890 { 12891 enum bpf_dynptr_type dynptr_type = meta.dynptr.type; 12892 12893 if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) 12894 return -EFAULT; 12895 12896 if (dynptr_write_changes_pkt_data(dynptr_type)) 12897 /* this will trigger clear_all_pkt_pointers(), which will 12898 * invalidate all dynptr slices associated with the skb 12899 */ 12900 changes_data = true; 12901 12902 break; 12903 } 12904 case BPF_FUNC_per_cpu_ptr: 12905 case BPF_FUNC_this_cpu_ptr: 12906 { 12907 struct bpf_reg_state *reg = ®s[BPF_REG_1]; 12908 const struct btf_type *type; 12909 12910 if (reg->type & MEM_RCU) { 12911 type = btf_type_by_id(reg->btf, reg->btf_id); 12912 if (!type || !btf_type_is_struct(type)) { 12913 verbose(env, "Helper has invalid btf/btf_id in R1\n"); 12914 return -EFAULT; 12915 } 12916 returns_cpu_specific_alloc_ptr = true; 12917 env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; 12918 } 12919 break; 12920 } 12921 case BPF_FUNC_user_ringbuf_drain: 12922 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12923 set_user_ringbuf_callback_state); 12924 break; 12925 } 12926 12927 if (err) 12928 return err; 12929 12930 /* reset caller saved regs */ 12931 bpf_diag_record_caller_saved(env, regs); 12932 bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); 12933 for (i = 0; i < CALLER_SAVED_REGS; i++) { 12934 bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); 12935 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 12936 } 12937 invalidate_outgoing_stack_args(env, cur_func(env)); 12938 12939 /* update return register (already marked as written above) */ 12940 ret_type = fn->ret_type; 12941 ret_flag = type_flag(ret_type); 12942 12943 switch (base_type(ret_type)) { 12944 case RET_INTEGER: 12945 /* sets type to SCALAR_VALUE */ 12946 mark_reg_unknown(env, regs, BPF_REG_0); 12947 break; 12948 case RET_VOID: 12949 regs[BPF_REG_0].type = NOT_INIT; 12950 break; 12951 case RET_PTR_TO_MAP_VALUE: 12952 /* There is no offset yet applied, variable or fixed */ 12953 mark_reg_known_zero(env, regs, BPF_REG_0); 12954 /* remember map_ptr, so that check_map_access() 12955 * can check 'value_size' boundary of memory access 12956 * to map element returned from bpf_map_lookup_elem() 12957 */ 12958 if (meta.map.ptr == NULL) { 12959 verifier_bug(env, "unexpected null map_ptr"); 12960 return -EFAULT; 12961 } 12962 12963 if (func_id == BPF_FUNC_map_lookup_elem && 12964 can_elide_value_nullness(meta.map.ptr) && 12965 meta.const_map_key >= 0 && 12966 meta.const_map_key < meta.map.ptr->max_entries) 12967 ret_flag &= ~PTR_MAYBE_NULL; 12968 12969 regs[BPF_REG_0].map_ptr = meta.map.ptr; 12970 regs[BPF_REG_0].map_uid = meta.map.uid; 12971 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 12972 if (type_may_be_null(ret_flag) || 12973 btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { 12974 regs[BPF_REG_0].id = ++env->id_gen; 12975 } 12976 /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ 12977 refine_map_lookup_value(®s[BPF_REG_0]); 12978 break; 12979 case RET_PTR_TO_SOCKET: 12980 mark_reg_known_zero(env, regs, BPF_REG_0); 12981 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 12982 break; 12983 case RET_PTR_TO_SOCK_COMMON: 12984 mark_reg_known_zero(env, regs, BPF_REG_0); 12985 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 12986 break; 12987 case RET_PTR_TO_TCP_SOCK: 12988 mark_reg_known_zero(env, regs, BPF_REG_0); 12989 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 12990 break; 12991 case RET_PTR_TO_MEM: 12992 mark_reg_known_zero(env, regs, BPF_REG_0); 12993 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 12994 regs[BPF_REG_0].mem_size = meta.ret_mem.size; 12995 break; 12996 case RET_PTR_TO_MEM_OR_BTF_ID: 12997 { 12998 const struct btf_type *t; 12999 13000 mark_reg_known_zero(env, regs, BPF_REG_0); 13001 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 13002 if (!btf_type_is_struct(t)) { 13003 u32 tsize; 13004 const struct btf_type *ret; 13005 const char *tname; 13006 13007 /* resolve the type size of ksym. */ 13008 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 13009 if (IS_ERR(ret)) { 13010 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 13011 verbose(env, "unable to resolve the size of type '%s': %ld\n", 13012 tname, PTR_ERR(ret)); 13013 return -EINVAL; 13014 } 13015 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 13016 regs[BPF_REG_0].mem_size = tsize; 13017 } else { 13018 if (returns_cpu_specific_alloc_ptr) { 13019 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU; 13020 } else { 13021 /* MEM_RDONLY may be carried from ret_flag, but it 13022 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 13023 * it will confuse the check of PTR_TO_BTF_ID in 13024 * check_mem_access(). 13025 */ 13026 ret_flag &= ~MEM_RDONLY; 13027 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 13028 } 13029 13030 regs[BPF_REG_0].btf = meta.ret_btf; 13031 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 13032 } 13033 break; 13034 } 13035 case RET_PTR_TO_BTF_ID: 13036 { 13037 struct btf *ret_btf; 13038 int ret_btf_id; 13039 13040 mark_reg_known_zero(env, regs, BPF_REG_0); 13041 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 13042 if (func_id == BPF_FUNC_kptr_xchg) { 13043 ret_btf = meta.kptr_field->kptr.btf; 13044 ret_btf_id = meta.kptr_field->kptr.btf_id; 13045 if (!btf_is_kernel(ret_btf)) { 13046 regs[BPF_REG_0].type |= MEM_ALLOC; 13047 if (meta.kptr_field->type == BPF_KPTR_PERCPU) 13048 regs[BPF_REG_0].type |= MEM_PERCPU; 13049 } 13050 } else { 13051 if (fn->ret_btf_id == BPF_PTR_POISON) { 13052 verifier_bug(env, "func %s has non-overwritten BPF_PTR_POISON return type", 13053 func_id_name(func_id)); 13054 return -EFAULT; 13055 } 13056 ret_btf = btf_vmlinux; 13057 ret_btf_id = *fn->ret_btf_id; 13058 } 13059 if (ret_btf_id == 0) { 13060 verbose(env, "invalid return type %u of func %s#%d\n", 13061 base_type(ret_type), func_id_name(func_id), 13062 func_id); 13063 return -EINVAL; 13064 } 13065 regs[BPF_REG_0].btf = ret_btf; 13066 regs[BPF_REG_0].btf_id = ret_btf_id; 13067 break; 13068 } 13069 default: 13070 verbose(env, "unknown return type %u of func %s#%d\n", 13071 base_type(ret_type), func_id_name(func_id), func_id); 13072 return -EINVAL; 13073 } 13074 13075 if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) 13076 regs[BPF_REG_0].id = ++env->id_gen; 13077 13078 if (is_ptr_cast_function(func_id) && 13079 find_reference_state(env->cur_state, meta.ref_obj.id)) { 13080 struct bpf_verifier_state *branch; 13081 struct bpf_reg_state *r0; 13082 13083 err = validate_ref_obj(env, &meta.ref_obj); 13084 if (err) 13085 return err; 13086 13087 bpf_diag_mod_end(env); 13088 13089 /* 13090 * In order for a release of any of the original or cast pointers 13091 * to invalidate all other pointers, reuse the same reference id for 13092 * the cast result. 13093 * This reference id can't be used for nullness propagation, 13094 * as cast might return NULL for a non-NULL input. 13095 * Hence, explore the NULL case as a separate branch. 13096 */ 13097 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); 13098 if (IS_ERR(branch)) 13099 return PTR_ERR(branch); 13100 13101 r0 = &branch->frame[branch->curframe]->regs[BPF_REG_0]; 13102 __mark_reg_known_zero(r0); 13103 r0->type = SCALAR_VALUE; 13104 13105 bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); 13106 regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; 13107 regs[BPF_REG_0].id = meta.ref_obj.id; 13108 } else if (is_acquire_function(func_id, meta.map.ptr)) { 13109 int id = acquire_reference(env, insn_idx, 0); 13110 13111 if (id < 0) 13112 return id; 13113 13114 regs[BPF_REG_0].id = id; 13115 } 13116 13117 if (func_id == BPF_FUNC_dynptr_data) 13118 regs[BPF_REG_0].parent_id = meta.dynptr.id; 13119 13120 err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); 13121 if (err) 13122 return err; 13123 13124 bpf_diag_mod_end(env); 13125 13126 err = check_map_func_compatibility(env, meta.map.ptr, func_id); 13127 if (err) 13128 return err; 13129 13130 if ((func_id == BPF_FUNC_get_stack || 13131 func_id == BPF_FUNC_get_task_stack) && 13132 !env->prog->has_callchain_buf) { 13133 const char *err_str; 13134 13135 #ifdef CONFIG_PERF_EVENTS 13136 err = get_callchain_buffers(sysctl_perf_event_max_stack); 13137 err_str = "cannot get callchain buffer for func %s#%d\n"; 13138 #else 13139 err = -ENOTSUPP; 13140 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 13141 #endif 13142 if (err) { 13143 verbose(env, err_str, func_id_name(func_id), func_id); 13144 return err; 13145 } 13146 13147 env->prog->has_callchain_buf = true; 13148 } 13149 13150 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 13151 env->prog->call_get_stack = true; 13152 13153 if (func_id == BPF_FUNC_get_func_ip) { 13154 if (check_get_func_ip(env)) 13155 return -ENOTSUPP; 13156 env->prog->call_get_func_ip = true; 13157 } 13158 13159 if (func_id == BPF_FUNC_tail_call) { 13160 if (env->cur_state->curframe) { 13161 struct bpf_verifier_state *branch; 13162 13163 /* 13164 * A taken tail call is modeled as a return from the current 13165 * frame. A callback frame cannot be left that way because 13166 * prepare_func_exit() would apply its return contract to the 13167 * unknown R0 synthesized below. Stack-depth validation rejects 13168 * this construct anyway. 13169 */ 13170 if (cur_func(env)->in_callback_fn) { 13171 verbose(env, "cannot tail call within callback\n"); 13172 return -EINVAL; 13173 } 13174 mark_reg_scratched(env, BPF_REG_0); 13175 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); 13176 if (IS_ERR(branch)) 13177 return PTR_ERR(branch); 13178 clear_all_pkt_pointers(env); 13179 mark_reg_unknown(env, regs, BPF_REG_0); 13180 err = prepare_func_exit(env, &env->insn_idx); 13181 if (err) 13182 return err; 13183 env->insn_idx--; 13184 } else { 13185 changes_data = false; 13186 } 13187 } 13188 13189 if (changes_data) 13190 clear_all_pkt_pointers(env); 13191 return 0; 13192 } 13193 13194 /* 13195 * Mark the register(s) holding a @size byte kfunc return value as unknown 13196 * scalars and return how many registers that took. Both halves of a register 13197 * pair are treated the same way. 13198 * 13199 * R0 is written in the diagnostic modification scope the caller opened before 13200 * the caller-saved scrub, since R0 gets no scrub record of its own and that 13201 * scope is what carries its pre-call value into the history. A scope tracks a 13202 * single register and does not nest, so close it before recording the rest of 13203 * the return value. A value that fits in R0 leaves the caller's scope open for 13204 * the caller to close, so a later write to R0 still lands in it. 13205 */ 13206 static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, 13207 struct bpf_reg_state *regs, u32 size) 13208 { 13209 u32 i, nregs = ret_regs_cnt(size); 13210 13211 mark_reg_unknown(env, regs, ret_regs[0]); 13212 if (nregs == 1) 13213 return nregs; 13214 13215 bpf_diag_mod_end(env); 13216 for (i = 1; i < nregs; i++) { 13217 bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); 13218 mark_reg_unknown(env, regs, ret_regs[i]); 13219 bpf_diag_mod_end(env); 13220 } 13221 13222 return nregs; 13223 } 13224 13225 static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) 13226 { 13227 return meta->kfunc_flags & KF_ACQUIRE; 13228 } 13229 13230 static bool is_kfunc_release(struct bpf_call_arg_meta *meta) 13231 { 13232 return meta->kfunc_flags & KF_RELEASE; 13233 } 13234 13235 static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) 13236 { 13237 return meta->kfunc_flags & KF_DESTRUCTIVE; 13238 } 13239 13240 static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta) 13241 { 13242 return meta->kfunc_flags & KF_PERFMON; 13243 } 13244 13245 static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) 13246 { 13247 return meta->kfunc_flags & KF_RCU; 13248 } 13249 13250 static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) 13251 { 13252 return meta->kfunc_flags & KF_RCU_PROTECTED; 13253 } 13254 13255 static bool is_kfunc_arg_mem_size(const struct btf *btf, 13256 const struct btf_param *arg) 13257 { 13258 const struct btf_type *t; 13259 13260 t = btf_type_skip_modifiers(btf, arg->type, NULL); 13261 if (!btf_type_is_scalar(t)) 13262 return false; 13263 13264 return btf_param_match_suffix(btf, arg, "__sz"); 13265 } 13266 13267 static bool is_kfunc_arg_const_mem_size(const struct btf *btf, 13268 const struct btf_param *arg) 13269 { 13270 const struct btf_type *t; 13271 13272 t = btf_type_skip_modifiers(btf, arg->type, NULL); 13273 if (!btf_type_is_scalar(t)) 13274 return false; 13275 13276 return btf_param_match_suffix(btf, arg, "__szk"); 13277 } 13278 13279 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 13280 { 13281 return btf_param_match_suffix(btf, arg, "__k"); 13282 } 13283 13284 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 13285 { 13286 return btf_param_match_suffix(btf, arg, "__ign"); 13287 } 13288 13289 static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) 13290 { 13291 return btf_param_match_suffix(btf, arg, "__map"); 13292 } 13293 13294 static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) 13295 { 13296 return btf_param_match_suffix(btf, arg, "__const_map"); 13297 } 13298 13299 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 13300 { 13301 return btf_param_match_suffix(btf, arg, "__alloc"); 13302 } 13303 13304 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg) 13305 { 13306 return btf_param_match_suffix(btf, arg, "__uninit"); 13307 } 13308 13309 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg) 13310 { 13311 return btf_param_match_suffix(btf, arg, "__refcounted_kptr"); 13312 } 13313 13314 static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) 13315 { 13316 return btf_param_match_suffix(btf, arg, "__nullable") || 13317 btf_param_match_suffix(btf, arg, "__arena"); 13318 } 13319 13320 static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg) 13321 { 13322 return btf_param_match_suffix(btf, arg, "__nonown_allowed"); 13323 } 13324 13325 static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg) 13326 { 13327 return btf_param_match_suffix(btf, arg, "__str"); 13328 } 13329 13330 static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param *arg) 13331 { 13332 return btf_param_match_suffix(btf, arg, "__irq_flag"); 13333 } 13334 13335 static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg) 13336 { 13337 return btf_param_match_suffix(btf, arg, "__ctx_out"); 13338 } 13339 13340 static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) 13341 { 13342 return btf_param_match_suffix(btf, arg, "__arena__nullable") || 13343 btf_param_match_suffix(btf, arg, "__arena"); 13344 } 13345 13346 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 13347 const struct btf_param *arg, 13348 const char *name) 13349 { 13350 int len, target_len = strlen(name); 13351 const char *param_name; 13352 13353 param_name = btf_name_by_offset(btf, arg->name_off); 13354 if (str_is_empty(param_name)) 13355 return false; 13356 len = strlen(param_name); 13357 if (len != target_len) 13358 return false; 13359 if (strcmp(param_name, name)) 13360 return false; 13361 13362 return true; 13363 } 13364 13365 enum { 13366 KF_ARG_DYNPTR_ID, 13367 KF_ARG_LIST_HEAD_ID, 13368 KF_ARG_LIST_NODE_ID, 13369 KF_ARG_RB_ROOT_ID, 13370 KF_ARG_RB_NODE_ID, 13371 KF_ARG_WORKQUEUE_ID, 13372 KF_ARG_RES_SPIN_LOCK_ID, 13373 KF_ARG_TASK_WORK_ID, 13374 KF_ARG_PROG_AUX_ID, 13375 KF_ARG_TIMER_ID, 13376 KF_ARG_RCU_HEAD_ID 13377 }; 13378 13379 BTF_ID_LIST(kf_arg_btf_ids) 13380 BTF_ID(struct, bpf_dynptr) 13381 BTF_ID(struct, bpf_list_head) 13382 BTF_ID(struct, bpf_list_node) 13383 BTF_ID(struct, bpf_rb_root) 13384 BTF_ID(struct, bpf_rb_node) 13385 BTF_ID(struct, bpf_wq) 13386 BTF_ID(struct, bpf_res_spin_lock) 13387 BTF_ID(struct, bpf_task_work) 13388 BTF_ID(struct, bpf_prog_aux) 13389 BTF_ID(struct, bpf_timer) 13390 BTF_ID(struct, bpf_rcu_head) 13391 13392 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 13393 const struct btf_param *arg, int type) 13394 { 13395 const struct btf_type *t; 13396 u32 res_id; 13397 13398 t = btf_type_skip_modifiers(btf, arg->type, NULL); 13399 if (!t) 13400 return false; 13401 if (!btf_type_is_ptr(t)) 13402 return false; 13403 t = btf_type_skip_modifiers(btf, t->type, &res_id); 13404 if (!t) 13405 return false; 13406 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 13407 } 13408 13409 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 13410 { 13411 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 13412 } 13413 13414 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 13415 { 13416 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 13417 } 13418 13419 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 13420 { 13421 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 13422 } 13423 13424 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg) 13425 { 13426 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID); 13427 } 13428 13429 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg) 13430 { 13431 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID); 13432 } 13433 13434 static bool is_kfunc_arg_timer(const struct btf *btf, const struct btf_param *arg) 13435 { 13436 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TIMER_ID); 13437 } 13438 13439 static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg) 13440 { 13441 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID); 13442 } 13443 13444 static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param *arg) 13445 { 13446 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); 13447 } 13448 13449 static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg) 13450 { 13451 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID); 13452 } 13453 13454 static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg) 13455 { 13456 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); 13457 } 13458 13459 static bool is_rbtree_node_type(const struct btf_type *t) 13460 { 13461 return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_RB_NODE_ID]); 13462 } 13463 13464 static bool is_list_node_type(const struct btf_type *t) 13465 { 13466 return t == btf_type_by_id(btf_vmlinux, kf_arg_btf_ids[KF_ARG_LIST_NODE_ID]); 13467 } 13468 13469 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, 13470 const struct btf_param *arg) 13471 { 13472 const struct btf_type *t; 13473 13474 t = btf_type_resolve_func_ptr(btf, arg->type, NULL); 13475 if (!t) 13476 return false; 13477 13478 return true; 13479 } 13480 13481 static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param *arg) 13482 { 13483 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_PROG_AUX_ID); 13484 } 13485 13486 /* 13487 * A kfunc with KF_IMPLICIT_ARGS has two prototypes in BTF: 13488 * - the _impl prototype with full arg list (meta->func_proto) 13489 * - the BPF API prototype w/o implicit args (func->type in BTF) 13490 * To determine whether an argument is implicit, we compare its position 13491 * against the number of arguments in the prototype w/o implicit args. 13492 */ 13493 static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx) 13494 { 13495 const struct btf_type *func, *func_proto; 13496 u32 argn; 13497 13498 if (!(meta->kfunc_flags & KF_IMPLICIT_ARGS)) 13499 return false; 13500 13501 func = btf_type_by_id(meta->btf, meta->func_id); 13502 func_proto = btf_type_by_id(meta->btf, func->type); 13503 argn = btf_type_vlen(func_proto); 13504 13505 return argn <= arg_idx; 13506 } 13507 13508 #define BTF_MEMBER_MAX_DEPTH 4 13509 #define BTF_MEMBER_PATH_LEN 64 13510 13511 struct btf_member_path { 13512 const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; 13513 bool in_array[BTF_MEMBER_MAX_DEPTH]; 13514 const struct btf_type *bad_type; 13515 int depth; 13516 bool too_deep; 13517 }; 13518 13519 static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, 13520 u32 member_kinds) 13521 { 13522 if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) 13523 return true; 13524 if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) 13525 return true; 13526 return false; 13527 } 13528 13529 /* 13530 * Returns true if every member of struct @t is of a kind listed in 13531 * @member_kinds, 4 levels of nesting allowed. An array member counts as its 13532 * element type. 13533 */ 13534 static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, 13535 const struct btf_type *t, u32 member_kinds, int rec, 13536 struct btf_member_path *path) 13537 { 13538 const struct btf_type *member_type; 13539 const struct btf_member *member; 13540 bool in_array; 13541 u32 i; 13542 13543 if (!btf_type_is_struct(t)) 13544 return false; 13545 13546 for_each_member(i, t, member) { 13547 const struct btf_array *array; 13548 13549 in_array = false; 13550 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 13551 /* 13552 * Every element of an array is laid out in the value being 13553 * returned, so an array counts as its element type however many 13554 * dimensions deep that is. 13555 */ 13556 while (btf_type_is_array(member_type)) { 13557 array = btf_array(member_type); 13558 if (!array->nelems) { 13559 member_type = NULL; 13560 goto bad_member; 13561 } 13562 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 13563 in_array = true; 13564 } 13565 if (btf_type_is_struct(member_type)) { 13566 if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { 13567 verbose(env, "max struct nesting depth exceeded\n"); 13568 if (path) 13569 path->too_deep = true; 13570 return false; 13571 } 13572 if (!btf_struct_member_walk(env, btf, member_type, member_kinds, 13573 rec + 1, path)) 13574 goto bad_path; 13575 continue; 13576 } 13577 if (!btf_member_kind_allowed(btf, member_type, member_kinds)) 13578 goto bad_member; 13579 } 13580 return true; 13581 13582 bad_member: 13583 if (path) { 13584 path->depth = rec + 1; 13585 path->bad_type = member_type; 13586 } 13587 bad_path: 13588 if (path && path->depth) { 13589 path->member[rec] = member; 13590 path->in_array[rec] = in_array; 13591 } 13592 return false; 13593 } 13594 13595 bool btf_struct_is_composed_of(struct bpf_verifier_env *env, 13596 const struct btf *btf, 13597 const struct btf_type *t, u32 member_kinds) 13598 { 13599 return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); 13600 } 13601 13602 static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, 13603 const struct btf *btf, 13604 const struct btf_type *t) 13605 { 13606 return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); 13607 } 13608 13609 static int resolve_func_arg_type(struct bpf_verifier_env *env, 13610 struct bpf_reg_state *reg, u32 arg, argno_t argno, 13611 struct bpf_call_arg_meta *meta, 13612 enum bpf_arg_type *arg_type, u32 *arg_size) 13613 { 13614 const struct btf_param *args; 13615 const struct btf_type *ref_t, *resolve_ret; 13616 const struct btf *btf; 13617 const char *ref_tname; 13618 u32 ref_id; 13619 13620 if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE) 13621 return resolve_map_arg_type(env, meta, arg_type); 13622 13623 if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) 13624 return 0; 13625 13626 /* 13627 * A kfunc that releases an object, or that takes a reference to one, 13628 * needs the object itself: memory of the program cannot stand in for it. 13629 */ 13630 if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || 13631 is_kfunc_acquire(meta) || 13632 base_type(reg->type) == PTR_TO_BTF_ID || 13633 reg2btf_ids[base_type(reg->type)]) 13634 return 0; 13635 13636 args = btf_params(meta->func_proto); 13637 ref_id = *meta->fn->arg_btf_id[arg]; 13638 btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; 13639 ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); 13640 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 13641 13642 /* 13643 * A kfunc that takes a projection type (ie. xdp_md) casts it to the 13644 * underlying type. So memory of the program cannot stand in for it, 13645 * even if the projection type holds only scalars. A struct without 13646 * members (ie. iou_ctx) is a handle for an object of another type. 13647 */ 13648 if (btf_is_projection_type(ref_tname) || 13649 !btf_type_is_scalar_struct(env, btf, ref_t) || 13650 !btf_type_vlen(ref_t)) 13651 return 0; 13652 13653 resolve_ret = btf_resolve_size(btf, ref_t, arg_size); 13654 if (IS_ERR(resolve_ret)) { 13655 verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", 13656 reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, 13657 PTR_ERR(resolve_ret)); 13658 return -EINVAL; 13659 } 13660 *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE | 13661 (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT)); 13662 13663 return 0; 13664 } 13665 13666 static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, 13667 char *buf, size_t buf_sz) 13668 { 13669 size_t len = 0; 13670 int i; 13671 13672 buf[0] = '\0'; 13673 for (i = 0; i < path->depth; i++) { 13674 const char *name = btf_name_by_offset(btf, path->member[i]->name_off); 13675 13676 /* An anonymous struct or union has no name to spell. */ 13677 if (!name || !name[0]) 13678 continue; 13679 len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); 13680 if (path->in_array[i]) 13681 len += scnprintf(buf + len, buf_sz - len, "[]"); 13682 } 13683 } 13684 13685 enum special_kfunc_type { 13686 KF_bpf_obj_new_impl, 13687 KF_bpf_obj_new, 13688 KF_bpf_obj_drop_impl, 13689 KF_bpf_obj_drop, 13690 KF_bpf_refcount_acquire_impl, 13691 KF_bpf_refcount_acquire, 13692 KF_bpf_list_push_front_impl, 13693 KF_bpf_list_push_front, 13694 KF_bpf_list_push_back_impl, 13695 KF_bpf_list_push_back, 13696 KF_bpf_list_add, 13697 KF_bpf_list_pop_front, 13698 KF_bpf_list_pop_back, 13699 KF_bpf_list_del, 13700 KF_bpf_list_front, 13701 KF_bpf_list_back, 13702 KF_bpf_list_is_first, 13703 KF_bpf_list_is_last, 13704 KF_bpf_list_empty, 13705 KF_bpf_cast_to_kern_ctx, 13706 KF_bpf_rdonly_cast, 13707 KF_bpf_rcu_read_lock, 13708 KF_bpf_rcu_read_unlock, 13709 KF_bpf_rbtree_remove, 13710 KF_bpf_rbtree_add_impl, 13711 KF_bpf_rbtree_add, 13712 KF_bpf_rbtree_first, 13713 KF_bpf_rbtree_root, 13714 KF_bpf_rbtree_left, 13715 KF_bpf_rbtree_right, 13716 KF_bpf_dynptr_from_skb, 13717 KF_bpf_dynptr_from_xdp, 13718 KF_bpf_dynptr_from_skb_meta, 13719 KF_bpf_xdp_pull_data, 13720 KF_bpf_dynptr_slice, 13721 KF_bpf_dynptr_slice_rdwr, 13722 KF_bpf_dynptr_clone, 13723 KF_bpf_percpu_obj_new_impl, 13724 KF_bpf_percpu_obj_new, 13725 KF_bpf_percpu_obj_drop_impl, 13726 KF_bpf_percpu_obj_drop, 13727 KF_bpf_throw, 13728 KF_bpf_wq_set_callback, 13729 KF_bpf_preempt_disable, 13730 KF_bpf_preempt_enable, 13731 KF_bpf_iter_css_task_new, 13732 KF_bpf_session_cookie, 13733 KF_bpf_get_kmem_cache, 13734 KF_bpf_local_irq_save, 13735 KF_bpf_local_irq_restore, 13736 KF_bpf_iter_num_new, 13737 KF_bpf_iter_num_next, 13738 KF_bpf_iter_num_destroy, 13739 KF_bpf_set_dentry_xattr, 13740 KF_bpf_remove_dentry_xattr, 13741 KF_bpf_res_spin_lock, 13742 KF_bpf_res_spin_unlock, 13743 KF_bpf_res_spin_lock_irqsave, 13744 KF_bpf_res_spin_unlock_irqrestore, 13745 KF_bpf_dynptr_from_file, 13746 KF_bpf_dynptr_file_discard, 13747 KF___bpf_trap, 13748 KF_bpf_task_work_schedule_signal, 13749 KF_bpf_task_work_schedule_resume, 13750 KF_bpf_call_rcu, 13751 KF_bpf_call_rcu_tasks_trace, 13752 KF_bpf_arena_alloc_pages, 13753 KF_bpf_arena_free_pages, 13754 KF_bpf_session_is_return, 13755 }; 13756 13757 BTF_ID_LIST(special_kfunc_list) 13758 BTF_ID(func, bpf_obj_new_impl) 13759 BTF_ID(func, bpf_obj_new) 13760 BTF_ID(func, bpf_obj_drop_impl) 13761 BTF_ID(func, bpf_obj_drop) 13762 BTF_ID(func, bpf_refcount_acquire_impl) 13763 BTF_ID(func, bpf_refcount_acquire) 13764 BTF_ID(func, bpf_list_push_front_impl) 13765 BTF_ID(func, bpf_list_push_front) 13766 BTF_ID(func, bpf_list_push_back_impl) 13767 BTF_ID(func, bpf_list_push_back) 13768 BTF_ID(func, bpf_list_add) 13769 BTF_ID(func, bpf_list_pop_front) 13770 BTF_ID(func, bpf_list_pop_back) 13771 BTF_ID(func, bpf_list_del) 13772 BTF_ID(func, bpf_list_front) 13773 BTF_ID(func, bpf_list_back) 13774 BTF_ID(func, bpf_list_is_first) 13775 BTF_ID(func, bpf_list_is_last) 13776 BTF_ID(func, bpf_list_empty) 13777 BTF_ID(func, bpf_cast_to_kern_ctx) 13778 BTF_ID(func, bpf_rdonly_cast) 13779 BTF_ID(func, bpf_rcu_read_lock) 13780 BTF_ID(func, bpf_rcu_read_unlock) 13781 BTF_ID(func, bpf_rbtree_remove) 13782 BTF_ID(func, bpf_rbtree_add_impl) 13783 BTF_ID(func, bpf_rbtree_add) 13784 BTF_ID(func, bpf_rbtree_first) 13785 BTF_ID(func, bpf_rbtree_root) 13786 BTF_ID(func, bpf_rbtree_left) 13787 BTF_ID(func, bpf_rbtree_right) 13788 #ifdef CONFIG_NET 13789 BTF_ID(func, bpf_dynptr_from_skb) 13790 BTF_ID(func, bpf_dynptr_from_xdp) 13791 BTF_ID(func, bpf_dynptr_from_skb_meta) 13792 BTF_ID(func, bpf_xdp_pull_data) 13793 #else 13794 BTF_ID_UNUSED 13795 BTF_ID_UNUSED 13796 BTF_ID_UNUSED 13797 BTF_ID_UNUSED 13798 #endif 13799 BTF_ID(func, bpf_dynptr_slice) 13800 BTF_ID(func, bpf_dynptr_slice_rdwr) 13801 BTF_ID(func, bpf_dynptr_clone) 13802 BTF_ID(func, bpf_percpu_obj_new_impl) 13803 BTF_ID(func, bpf_percpu_obj_new) 13804 BTF_ID(func, bpf_percpu_obj_drop_impl) 13805 BTF_ID(func, bpf_percpu_obj_drop) 13806 BTF_ID(func, bpf_throw) 13807 BTF_ID(func, bpf_wq_set_callback) 13808 BTF_ID(func, bpf_preempt_disable) 13809 BTF_ID(func, bpf_preempt_enable) 13810 #ifdef CONFIG_CGROUPS 13811 BTF_ID(func, bpf_iter_css_task_new) 13812 #else 13813 BTF_ID_UNUSED 13814 #endif 13815 #ifdef CONFIG_BPF_EVENTS 13816 BTF_ID(func, bpf_session_cookie) 13817 #else 13818 BTF_ID_UNUSED 13819 #endif 13820 BTF_ID(func, bpf_get_kmem_cache) 13821 BTF_ID(func, bpf_local_irq_save) 13822 BTF_ID(func, bpf_local_irq_restore) 13823 BTF_ID(func, bpf_iter_num_new) 13824 BTF_ID(func, bpf_iter_num_next) 13825 BTF_ID(func, bpf_iter_num_destroy) 13826 #ifdef CONFIG_BPF_LSM 13827 BTF_ID(func, bpf_set_dentry_xattr) 13828 BTF_ID(func, bpf_remove_dentry_xattr) 13829 #else 13830 BTF_ID_UNUSED 13831 BTF_ID_UNUSED 13832 #endif 13833 BTF_ID(func, bpf_res_spin_lock) 13834 BTF_ID(func, bpf_res_spin_unlock) 13835 BTF_ID(func, bpf_res_spin_lock_irqsave) 13836 BTF_ID(func, bpf_res_spin_unlock_irqrestore) 13837 BTF_ID(func, bpf_dynptr_from_file) 13838 BTF_ID(func, bpf_dynptr_file_discard) 13839 BTF_ID(func, __bpf_trap) 13840 BTF_ID(func, bpf_task_work_schedule_signal) 13841 BTF_ID(func, bpf_task_work_schedule_resume) 13842 BTF_ID(func, bpf_call_rcu) 13843 BTF_ID(func, bpf_call_rcu_tasks_trace) 13844 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT) 13845 BTF_ID(func, bpf_arena_alloc_pages) 13846 BTF_ID(func, bpf_arena_free_pages) 13847 #else 13848 BTF_ID_UNUSED 13849 BTF_ID_UNUSED 13850 #endif 13851 #ifdef CONFIG_BPF_EVENTS 13852 BTF_ID(func, bpf_session_is_return) 13853 #else 13854 BTF_ID_UNUSED 13855 #endif 13856 13857 static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) 13858 { 13859 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]); 13860 } 13861 13862 static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta) 13863 { 13864 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]); 13865 } 13866 13867 static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta) 13868 { 13869 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]); 13870 } 13871 13872 static bool is_bpf_obj_new_kfunc(u32 func_id) 13873 { 13874 return func_id == special_kfunc_list[KF_bpf_obj_new] || 13875 func_id == special_kfunc_list[KF_bpf_obj_new_impl]; 13876 } 13877 13878 static bool is_bpf_percpu_obj_new_kfunc(u32 func_id) 13879 { 13880 return func_id == special_kfunc_list[KF_bpf_percpu_obj_new] || 13881 func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]; 13882 } 13883 13884 static bool is_bpf_obj_drop_kfunc(u32 func_id) 13885 { 13886 return func_id == special_kfunc_list[KF_bpf_obj_drop] || 13887 func_id == special_kfunc_list[KF_bpf_obj_drop_impl]; 13888 } 13889 13890 static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id) 13891 { 13892 return func_id == special_kfunc_list[KF_bpf_percpu_obj_drop] || 13893 func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]; 13894 } 13895 13896 static bool is_bpf_refcount_acquire_kfunc(u32 func_id) 13897 { 13898 return func_id == special_kfunc_list[KF_bpf_refcount_acquire] || 13899 func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 13900 } 13901 13902 static bool is_bpf_list_push_kfunc(u32 func_id) 13903 { 13904 return func_id == special_kfunc_list[KF_bpf_list_push_front] || 13905 func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 13906 func_id == special_kfunc_list[KF_bpf_list_push_back] || 13907 func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 13908 func_id == special_kfunc_list[KF_bpf_list_add]; 13909 } 13910 13911 static bool is_bpf_rbtree_add_kfunc(u32 func_id) 13912 { 13913 return func_id == special_kfunc_list[KF_bpf_rbtree_add] || 13914 func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 13915 } 13916 13917 static bool is_call_rcu_kfunc(u32 func_id) 13918 { 13919 return func_id == special_kfunc_list[KF_bpf_call_rcu] || 13920 func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace]; 13921 } 13922 13923 static bool is_task_work_add_kfunc(u32 func_id) 13924 { 13925 return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || 13926 func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume]; 13927 } 13928 13929 static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) 13930 { 13931 if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) 13932 return false; 13933 13934 return meta->kfunc_flags & KF_RET_NULL; 13935 } 13936 13937 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) 13938 { 13939 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]); 13940 } 13941 13942 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) 13943 { 13944 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]); 13945 } 13946 13947 static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) 13948 { 13949 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]); 13950 } 13951 13952 static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) 13953 { 13954 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); 13955 } 13956 13957 /* These write to what their first argument, a dynptr, points to */ 13958 BTF_SET_START(dynptr_write_kfuncs) 13959 BTF_ID(func, bpf_dynptr_copy) 13960 BTF_ID(func, bpf_dynptr_memset) 13961 #ifdef CONFIG_BPF_EVENTS 13962 BTF_ID(func, bpf_probe_read_user_dynptr) 13963 BTF_ID(func, bpf_probe_read_kernel_dynptr) 13964 BTF_ID(func, bpf_probe_read_user_str_dynptr) 13965 BTF_ID(func, bpf_probe_read_kernel_str_dynptr) 13966 BTF_ID(func, bpf_copy_from_user_dynptr) 13967 BTF_ID(func, bpf_copy_from_user_str_dynptr) 13968 BTF_ID(func, bpf_copy_from_user_task_dynptr) 13969 BTF_ID(func, bpf_copy_from_user_task_str_dynptr) 13970 #endif 13971 BTF_SET_END(dynptr_write_kfuncs) 13972 13973 static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta) 13974 { 13975 return meta->btf == btf_vmlinux && 13976 btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id); 13977 } 13978 13979 /* 13980 * A write through the dynptr of an skb may reallocate its head. Before the 13981 * arguments are checked nobody knows what kind of dynptr it is. 13982 */ 13983 bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) 13984 { 13985 return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) || 13986 is_kfunc_dynptr_write(meta); 13987 } 13988 13989 static u32 kfunc_abi_slots(const struct btf_func_model *fm) 13990 { 13991 const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); 13992 u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; 13993 u32 i, nslots, slots = 0; 13994 13995 for (i = 0; i < fm->nr_args; i++) 13996 slots += btf_func_model_arg_slots(fm, i); 13997 13998 if (!abi) 13999 return slots; 14000 14001 nslots = bpf_jit_place_args(abi, fm, pos_of_slot); 14002 for (i = 0; i < nslots; i++) 14003 if (pos_of_slot[i] + 1 > slots) 14004 slots = pos_of_slot[i] + 1; 14005 14006 return slots; 14007 } 14008 14009 static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) 14010 { 14011 const struct btf_member *member; 14012 const struct btf_type *mt; 14013 u32 align, i; 14014 14015 while (btf_type_is_array(t)) 14016 t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); 14017 14018 if (btf_type_is_int(t)) 14019 return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; 14020 if (!btf_type_is_struct(t)) 14021 return BPF_REG_SIZE; 14022 if (rec >= BTF_MEMBER_MAX_DEPTH) 14023 return 0; 14024 14025 for_each_member(i, t, member) { 14026 mt = btf_type_skip_modifiers(btf, member->type, NULL); 14027 align = __btf_func_arg_align(btf, mt, rec + 1); 14028 if (!align) 14029 return 0; 14030 if (align > BPF_REG_SIZE) 14031 return 2 * BPF_REG_SIZE; 14032 } 14033 return BPF_REG_SIZE; 14034 } 14035 14036 u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) 14037 { 14038 return __btf_func_arg_align(btf, t, 0); 14039 } 14040 14041 static int 14042 get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 14043 const struct btf_param *args, int arg, int nargs, u32 slot, 14044 struct bpf_func_proto *proto) 14045 { 14046 const struct btf_type *t, *ref_t = NULL, *resolve_ret; 14047 const u32 *ref_id_ptr = NULL; 14048 argno_t argno = argno_from_arg(slot + 1); 14049 const char *ref_tname = NULL; 14050 u32 ref_id, type_size; 14051 int arg_type; 14052 14053 proto->arg_btf_id[arg] = NULL; 14054 14055 if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) 14056 return ARG_PTR_TO_PROG_AUX; 14057 14058 if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) 14059 return ARG_IGNORE; 14060 14061 t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); 14062 14063 /* Scalar arguments are classified from their BTF suffix/name alone. */ 14064 if (btf_type_is_scalar(t)) { 14065 if (is_kfunc_arg_constant(meta->btf, &args[arg])) 14066 return ARG_CONST_SCALAR; 14067 if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) 14068 return ARG_CONST_MEM_SIZE; 14069 if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) 14070 return ARG_MEM_SIZE; 14071 if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || 14072 is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) 14073 return ARG_CONST_ALLOC_SIZE_OR_ZERO; 14074 return ARG_SCALAR; 14075 } 14076 14077 if (btf_type_is_struct(t)) { 14078 if (!t->size || t->size > 2 * BPF_REG_SIZE) { 14079 verbose(env, 14080 "%s type %s has size %u, only 1 to %d bytes " 14081 "can be passed by value\n", 14082 reg_arg_name(env, argno), btf_type_str(t), t->size, 14083 2 * BPF_REG_SIZE); 14084 return -EINVAL; 14085 } 14086 if (!btf_type_is_scalar_struct(env, meta->btf, t)) { 14087 verbose(env, "%s type %s is not composed of scalars\n", 14088 reg_arg_name(env, argno), btf_type_str(t)); 14089 return -EINVAL; 14090 } 14091 return ARG_SCALAR; 14092 } 14093 14094 if (!btf_type_is_ptr(t)) { 14095 verbose(env, "Unrecognized %s type %s\n", 14096 reg_arg_name(env, argno), btf_type_str(t)); 14097 return -EINVAL; 14098 } 14099 /* Keep a pointer to the BTF field containing the resolved referent ID. */ 14100 ref_id_ptr = &t->type; 14101 ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id); 14102 while (*ref_id_ptr != ref_id) 14103 ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type; 14104 ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); 14105 14106 /* In this function, we verify the kfunc's BTF as per the argument type, 14107 * leaving the rest of the verification with respect to the register 14108 * type to our caller. When a set of conditions hold in the BTF type of 14109 * arguments, we resolve it to a known bpf_arg_type. 14110 */ 14111 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || 14112 is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || 14113 is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) 14114 arg_type = ARG_PTR_TO_CTX; 14115 else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) 14116 arg_type = ARG_PTR_TO_CTX; 14117 else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) 14118 arg_type = ARG_PTR_TO_ALLOC_BTF_ID; 14119 else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) 14120 arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; 14121 else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) { 14122 arg_type = ARG_PTR_TO_DYNPTR; 14123 14124 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) 14125 arg_type |= DYNPTR_TYPE_SKB; 14126 else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) 14127 arg_type |= DYNPTR_TYPE_XDP; 14128 else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) 14129 arg_type |= DYNPTR_TYPE_SKB_META; 14130 else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) || 14131 is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) 14132 /* OBJ_RELEASE for the latter comes from KF_RELEASE below */ 14133 arg_type |= DYNPTR_TYPE_FILE; 14134 } else if (is_kfunc_arg_iter(meta, arg, &args[arg])) 14135 arg_type = ARG_PTR_TO_ITER; 14136 else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) 14137 arg_type = ARG_PTR_TO_LIST_HEAD; 14138 else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) 14139 arg_type = ARG_PTR_TO_LIST_NODE; 14140 else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) 14141 arg_type = ARG_PTR_TO_RB_ROOT; 14142 else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) 14143 arg_type = ARG_PTR_TO_RB_NODE; 14144 else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) 14145 arg_type = ARG_PTR_TO_CONST_STR; 14146 else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) 14147 arg_type = ARG_CONST_MAP_PTR; 14148 else if (is_kfunc_arg_map(meta->btf, &args[arg])) 14149 arg_type = ARG_PTR_TO_BTF_ID; 14150 else if (is_kfunc_arg_wq(meta->btf, &args[arg])) 14151 arg_type = ARG_PTR_TO_WORKQUEUE; 14152 else if (is_kfunc_arg_timer(meta->btf, &args[arg])) 14153 arg_type = ARG_PTR_TO_TIMER; 14154 else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) 14155 arg_type = ARG_PTR_TO_TASK_WORK; 14156 else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg])) 14157 arg_type = ARG_PTR_TO_RCU_HEAD; 14158 else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) 14159 arg_type = ARG_PTR_TO_IRQ_FLAG; 14160 else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) 14161 arg_type = ARG_PTR_TO_RES_SPIN_LOCK; 14162 else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) { 14163 if (!btf_type_is_scalar(ref_t)) { 14164 verbose(env, "%s __ctx_out argument must point to a scalar\n", 14165 reg_arg_name(env, argno)); 14166 return -EINVAL; 14167 } 14168 resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); 14169 if (IS_ERR(resolve_ret)) { 14170 verbose(env, 14171 "%s reference type('%s %s') size cannot be determined: %ld\n", 14172 reg_arg_name(env, argno), btf_type_str(ref_t), 14173 ref_tname, PTR_ERR(resolve_ret)); 14174 return -EINVAL; 14175 } 14176 proto->arg_size[arg] = type_size; 14177 arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE; 14178 } else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) 14179 arg_type = ARG_PTR_TO_FUNC; 14180 else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { 14181 if (!bpf_jit_supports_arena_args()) { 14182 verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", 14183 meta->func_name); 14184 return -ENOTSUPP; 14185 } 14186 if (!env->prog->aux->arena) { 14187 verbose(env, 14188 "%s arena pointer requires a program with an associated arena\n", 14189 reg_arg_name(env, argno)); 14190 return -EINVAL; 14191 } 14192 if (reg_from_argno(argno) < 0) { 14193 verbose(env, "%s arena pointer cannot be a stack argument\n", 14194 reg_arg_name(env, argno)); 14195 return -EINVAL; 14196 } 14197 /* 14198 * Both suffixes accept a constant zero. The function model determines 14199 * whether the JIT rebases it to the arena base or preserves NULL. 14200 * The common nullable path below records that verifier property. 14201 */ 14202 arg_type = ARG_PTR_TO_ARENA; 14203 } else if (arg + 1 < nargs && 14204 (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || 14205 is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { 14206 if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && 14207 !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { 14208 verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", 14209 reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); 14210 return -EINVAL; 14211 } 14212 arg_type = ARG_PTR_TO_MEM | MEM_WRITE; 14213 } else if (btf_type_is_struct(ref_t)) 14214 /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ 14215 arg_type = ARG_PTR_TO_BTF_ID; 14216 else { 14217 /* 14218 * Otherwise this is a fixed-size memory buffer supported by 14219 * check_helper_mem_access(): a pointer to a scalar or a struct of 14220 * scalars. The access size is derived from the pointed-to BTF type. 14221 */ 14222 if (!btf_type_is_scalar(ref_t) && 14223 !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { 14224 verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", 14225 reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); 14226 return -EINVAL; 14227 } 14228 resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); 14229 if (IS_ERR(resolve_ret)) { 14230 verbose(env, 14231 "%s reference type('%s %s') size cannot be determined: %ld\n", 14232 reg_arg_name(env, argno), btf_type_str(ref_t), 14233 ref_tname, PTR_ERR(resolve_ret)); 14234 return -EINVAL; 14235 } 14236 proto->arg_size[arg] = type_size; 14237 arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE; 14238 } 14239 14240 if (is_kfunc_arg_uninit(meta->btf, &args[arg])) 14241 arg_type |= MEM_UNINIT; 14242 14243 if (is_kfunc_arg_nullable(meta->btf, &args[arg])) 14244 arg_type |= PTR_MAYBE_NULL; 14245 14246 /* 14247 * Only the first argument of a KF_RELEASE kfunc releases anything, and 14248 * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it. 14249 */ 14250 if (is_kfunc_release(meta) && arg == 0) 14251 arg_type |= OBJ_RELEASE; 14252 14253 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { 14254 if (is_kfunc_arg_map(meta->btf, &args[arg])) 14255 proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP]; 14256 else 14257 proto->arg_btf_id[arg] = ref_id_ptr; 14258 14259 arg_type |= PTR_TRUSTED; 14260 14261 /* 14262 * A KF_RCU kfunc accepts an RCU-protected pointer where it would 14263 * otherwise demand a referenced or trusted one. Other argument kinds 14264 * have their own provenance requirements and must not inherit MEM_RCU. 14265 */ 14266 if (is_kfunc_rcu(meta)) 14267 arg_type |= MEM_RCU; 14268 } 14269 14270 return arg_type; 14271 } 14272 14273 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 14274 const struct btf_func_model *fm, struct bpf_func_proto *proto) 14275 { 14276 const struct bpf_jit_arg_abi *abi; 14277 const struct btf *btf = meta->btf; 14278 const struct btf_param *args; 14279 const struct btf_type *t; 14280 u32 i, nargs, slots_used; 14281 int arg_type; 14282 14283 args = (const struct btf_param *)(meta->func_proto + 1); 14284 nargs = btf_type_vlen(meta->func_proto); 14285 if (nargs > MAX_BPF_FUNC_ARGS) { 14286 verbose(env, "Function %s has %d > %d args\n", meta->func_name, 14287 nargs, MAX_BPF_FUNC_ARGS); 14288 return -EINVAL; 14289 } 14290 14291 for (i = 0, slots_used = 0; i < nargs; i++) { 14292 u32 nslots = btf_func_model_arg_slots(fm, i); 14293 14294 if (nslots > 1) { 14295 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 14296 if (!btf_func_arg_align(btf, t)) { 14297 verbose(env, 14298 "Function %s arg#%d type %s nests structs more than " 14299 "%d levels deep\n", 14300 meta->func_name, i, btf_type_str(t), 14301 BTF_MEMBER_MAX_DEPTH); 14302 return -EINVAL; 14303 } 14304 } 14305 slots_used += nslots; 14306 } 14307 14308 if (slots_used > MAX_BPF_FUNC_ARGS) { 14309 verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, 14310 slots_used, MAX_BPF_FUNC_ARGS); 14311 return -EINVAL; 14312 } 14313 14314 abi = bpf_jit_arg_abi(); 14315 14316 for (i = 0, slots_used = 0; i < nargs; i++) { 14317 u32 nslots = btf_func_model_arg_slots(fm, i); 14318 14319 if (!abi && nslots > 1) { 14320 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 14321 verbose(env, 14322 "Function %s arg#%d type %s cannot be passed at " 14323 "argument slot %d on this architecture\n", 14324 meta->func_name, i, btf_type_str(t), slots_used); 14325 return -EINVAL; 14326 } 14327 slots_used += nslots; 14328 14329 arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, 14330 slots_used - nslots, proto); 14331 if (arg_type < 0) 14332 return arg_type; 14333 14334 proto->arg_type[i] = arg_type; 14335 } 14336 14337 if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { 14338 verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", 14339 meta->func_name, slots_used); 14340 return -ENOTSUPP; 14341 } 14342 14343 return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; 14344 } 14345 14346 static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 14347 argno_t argno, enum bpf_arg_type arg_type, 14348 const struct btf *arg_btf, u32 arg_btf_id, 14349 struct bpf_call_arg_meta *meta, int insn_idx) 14350 { 14351 bool taking_projection, struct_same, strict_type_match = false; 14352 const struct btf_type *arg_t, *reg_t; 14353 const char *arg_tname, *reg_tname; 14354 const struct btf *reg_btf; 14355 u32 reg_btf_id; 14356 14357 if (base_type(reg->type) == PTR_TO_BTF_ID) { 14358 reg_btf = reg->btf; 14359 reg_btf_id = reg->btf_id; 14360 } else { 14361 reg_btf = btf_vmlinux; 14362 reg_btf_id = *reg2btf_ids[base_type(reg->type)]; 14363 } 14364 14365 /* 14366 * Enforce strict type matching for arguments that release a reference, 14367 * or are no-cast aliases. We do _not_ enforce strict matching by default, 14368 * as we want to enable BPF programs to pass types that are bitwise 14369 * equivalent without forcing them to explicitly cast with something 14370 * like bpf_cast_to_kern_ctx(). 14371 * 14372 * For example, say we had a type like the following: 14373 * 14374 * struct bpf_cpumask { 14375 * cpumask_t cpumask; 14376 * refcount_t usage; 14377 * }; 14378 * 14379 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed 14380 * to a struct cpumask, so it would be safe to pass a struct 14381 * bpf_cpumask * to a kfunc expecting a struct cpumask *. 14382 * 14383 * The philosophy here is similar to how we allow scalars of different 14384 * types to be passed to kfuncs as long as the size is the same. The 14385 * only difference here is that we're simply allowing 14386 * btf_struct_ids_match() to walk the struct at the 0th offset, and 14387 * resolve types. 14388 */ 14389 if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || 14390 (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, 14391 arg_btf, arg_btf_id))) 14392 strict_type_match = true; 14393 14394 arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id); 14395 arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off); 14396 reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, ®_btf_id); 14397 reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off); 14398 14399 struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id, 14400 reg->var_off.value, arg_btf, arg_btf_id, 14401 strict_type_match, !type_is_alloc(reg->type)); 14402 14403 /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot 14404 * actually use it -- it must cast to the underlying type. So we allow 14405 * caller to pass in the underlying type. Only kernel BTF has that type: 14406 * a type of the same name in the BTF of the program is program memory. 14407 * And only a pointer to the start of the object points to that type. 14408 */ 14409 taking_projection = is_kfunc(meta) && btf_is_kernel(reg_btf) && 14410 !reg->var_off.value && 14411 btf_is_projection_of(arg_tname, reg_tname); 14412 if (!taking_projection && !struct_same) { 14413 verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", 14414 meta->func_name, reg_arg_name(env, argno), 14415 btf_type_str(arg_t), arg_tname, 14416 reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname); 14417 return -EINVAL; 14418 } 14419 return 0; 14420 } 14421 14422 static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, 14423 struct bpf_call_arg_meta *meta) 14424 { 14425 int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; 14426 bool irq_save; 14427 14428 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) || 14429 is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { 14430 irq_save = true; 14431 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) 14432 kfunc_class = IRQ_LOCK_KFUNC; 14433 } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) || 14434 is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) { 14435 irq_save = false; 14436 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) 14437 kfunc_class = IRQ_LOCK_KFUNC; 14438 } else { 14439 verifier_bug(env, "unknown irq flags kfunc"); 14440 return -EFAULT; 14441 } 14442 14443 if (irq_save) { 14444 if (!is_irq_flag_reg_valid_uninit(env, reg)) { 14445 verbose(env, "expected uninitialized irq flag as %s\n", 14446 reg_arg_name(env, argno)); 14447 bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized", 14448 "Saving IRQ state requires an uninitialized stack slot for " 14449 "the IRQ flag, but this slot already contains tracked IRQ " 14450 "flag state.", 14451 "Use a fresh stack slot for this save operation, or restore " 14452 "the existing IRQ flag before reusing the slot."); 14453 return -EINVAL; 14454 } 14455 14456 err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW, 14457 BPF_WRITE, -1, false, false); 14458 if (err) 14459 return err; 14460 14461 err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class); 14462 if (err) 14463 return err; 14464 } else { 14465 err = is_irq_flag_reg_valid_init(env, reg); 14466 if (err) { 14467 verbose(env, "expected an initialized irq flag as %s\n", 14468 reg_arg_name(env, argno)); 14469 bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore", 14470 "Restoring IRQ state requires a stack slot that was " 14471 "initialized by a matching IRQ save operation on this path.", 14472 "Pass the same stack slot that was previously initialized by " 14473 "the matching IRQ save kfunc."); 14474 return err; 14475 } 14476 14477 spi = irq_flag_get_spi(env, reg); 14478 if (spi < 0) 14479 return spi; 14480 14481 mark_stack_slots_scratched(env, spi, 1); 14482 14483 err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); 14484 if (err) 14485 return err; 14486 14487 if (!in_rcu_cs(env)) 14488 invalidate_rcu_protected_refs(env); 14489 } 14490 return 0; 14491 } 14492 14493 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 14494 { 14495 struct btf_record *rec = reg_btf_record(reg); 14496 14497 if (!env->cur_state->active_locks) { 14498 verifier_bug(env, "%s w/o active lock", __func__); 14499 return -EFAULT; 14500 } 14501 14502 if (type_flag(reg->type) & NON_OWN_REF) { 14503 verifier_bug(env, "NON_OWN_REF already set"); 14504 return -EFAULT; 14505 } 14506 14507 reg->type |= NON_OWN_REF; 14508 if (rec->refcount_off >= 0) 14509 reg->type |= MEM_RCU; 14510 14511 return 0; 14512 } 14513 14514 static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) 14515 { 14516 struct bpf_func_state *unused; 14517 struct bpf_reg_state *reg; 14518 int err; 14519 14520 err = release_reference_nomark(env, id); 14521 WARN_ON_ONCE(err); 14522 14523 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 14524 if (reg->id == id) { 14525 reg->id = 0; 14526 ref_set_non_owning(env, reg); 14527 } 14528 })); 14529 14530 return; 14531 } 14532 14533 /* Implementation details: 14534 * 14535 * Each register points to some region of memory, which we define as an 14536 * allocation. Each allocation may embed a bpf_spin_lock which protects any 14537 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 14538 * allocation. The lock and the data it protects are colocated in the same 14539 * memory region. 14540 * 14541 * Hence, everytime a register holds a pointer value pointing to such 14542 * allocation, the verifier preserves a unique reg->id for it. 14543 * 14544 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 14545 * bpf_spin_lock is called. 14546 * 14547 * To enable this, lock state in the verifier captures two values: 14548 * active_lock.ptr = Register's type specific pointer 14549 * active_lock.id = A unique ID for each register pointer value 14550 * 14551 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 14552 * supported register types. 14553 * 14554 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 14555 * allocated objects is the reg->btf pointer. 14556 * 14557 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 14558 * can establish the provenance of the map value statically for each distinct 14559 * lookup into such maps. They always contain a single map value hence unique 14560 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 14561 * 14562 * So, in case of global variables, they use array maps with max_entries = 1, 14563 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 14564 * into the same map value as max_entries is 1, as described above). 14565 * 14566 * In case of inner map lookups, the inner map pointer has same map_ptr as the 14567 * outer map pointer (in verifier context), but each lookup into an inner map 14568 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 14569 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 14570 * will get different reg->id assigned to each lookup, hence different 14571 * active_lock.id. 14572 * 14573 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 14574 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 14575 * returned from bpf_obj_new. Each allocation receives a new reg->id. 14576 */ 14577 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 14578 { 14579 struct bpf_reference_state *s; 14580 void *ptr; 14581 u32 id; 14582 14583 switch ((int)reg->type) { 14584 case PTR_TO_MAP_VALUE: 14585 ptr = reg->map_ptr; 14586 break; 14587 case PTR_TO_BTF_ID | MEM_ALLOC: 14588 ptr = reg->btf; 14589 break; 14590 default: 14591 verifier_bug(env, "unknown reg type for lock check"); 14592 return -EFAULT; 14593 } 14594 id = reg->id; 14595 14596 if (!env->cur_state->active_locks) 14597 return -EINVAL; 14598 s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr); 14599 if (!s) { 14600 verbose(env, "held lock and object are not in the same allocation\n"); 14601 return -EINVAL; 14602 } 14603 return 0; 14604 } 14605 14606 static bool is_bpf_list_api_kfunc(u32 btf_id) 14607 { 14608 return is_bpf_list_push_kfunc(btf_id) || 14609 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 14610 btf_id == special_kfunc_list[KF_bpf_list_pop_back] || 14611 btf_id == special_kfunc_list[KF_bpf_list_del] || 14612 btf_id == special_kfunc_list[KF_bpf_list_front] || 14613 btf_id == special_kfunc_list[KF_bpf_list_back] || 14614 btf_id == special_kfunc_list[KF_bpf_list_is_first] || 14615 btf_id == special_kfunc_list[KF_bpf_list_is_last] || 14616 btf_id == special_kfunc_list[KF_bpf_list_empty]; 14617 } 14618 14619 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 14620 { 14621 return is_bpf_rbtree_add_kfunc(btf_id) || 14622 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 14623 btf_id == special_kfunc_list[KF_bpf_rbtree_first] || 14624 btf_id == special_kfunc_list[KF_bpf_rbtree_root] || 14625 btf_id == special_kfunc_list[KF_bpf_rbtree_left] || 14626 btf_id == special_kfunc_list[KF_bpf_rbtree_right]; 14627 } 14628 14629 static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta) 14630 { 14631 int flags = PROCESS_RES_LOCK; 14632 14633 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || 14634 is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) 14635 flags |= PROCESS_SPIN_LOCK; 14636 else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) && 14637 !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) 14638 return 0; 14639 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || 14640 is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) 14641 flags |= PROCESS_LOCK_IRQ; 14642 return flags; 14643 } 14644 14645 static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) 14646 { 14647 struct bpf_kfunc_meta kfunc; 14648 int err; 14649 14650 err = fetch_kfunc_meta(env, func_id, offset, &kfunc); 14651 if (err || !kfunc.flags) 14652 return false; 14653 14654 return *kfunc.flags & KF_SPINLOCK_SAFE; 14655 } 14656 14657 /* 14658 * True if insn calls a helper/kfunc that requires one of its arguments to 14659 * be a stack pointer with a constant offset. 14660 */ 14661 bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx) 14662 { 14663 const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 14664 const struct bpf_func_proto *fn; 14665 struct bpf_kfunc_desc *desc; 14666 u32 *flags, btf_id; 14667 int i; 14668 14669 if (bpf_helper_call(insn)) { 14670 if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) 14671 return false; 14672 } else if (bpf_pseudo_kfunc_call(insn)) { 14673 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 14674 if (!desc) 14675 return false; 14676 fn = &desc->proto; 14677 } else { 14678 return false; 14679 } 14680 14681 /* Both initialized and uninitialized stack dynptrs need a fixed offset. */ 14682 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) 14683 if (arg_type_is_dynptr(fn->arg_type[i])) 14684 return true; 14685 14686 /* vmlinux kfuncs only */ 14687 if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0) 14688 return false; 14689 btf_id = insn->imm; 14690 14691 flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog); 14692 if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY))) 14693 return true; 14694 14695 if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || 14696 btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || 14697 btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || 14698 btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] || 14699 btf_id == special_kfunc_list[KF_bpf_local_irq_save] || 14700 btf_id == special_kfunc_list[KF_bpf_local_irq_restore]) 14701 return true; 14702 14703 return false; 14704 } 14705 14706 static bool is_sync_callback_calling_kfunc(u32 btf_id) 14707 { 14708 return is_bpf_rbtree_add_kfunc(btf_id); 14709 } 14710 14711 static bool is_async_callback_calling_kfunc(u32 btf_id) 14712 { 14713 return is_bpf_wq_set_callback_kfunc(btf_id) || 14714 is_task_work_add_kfunc(btf_id) || 14715 is_call_rcu_kfunc(btf_id); 14716 } 14717 14718 bool bpf_is_throw_kfunc(struct bpf_insn *insn) 14719 { 14720 return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && 14721 insn->imm == special_kfunc_list[KF_bpf_throw]; 14722 } 14723 14724 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id) 14725 { 14726 return btf_id == special_kfunc_list[KF_bpf_wq_set_callback]; 14727 } 14728 14729 static bool is_callback_calling_kfunc(u32 btf_id) 14730 { 14731 return is_sync_callback_calling_kfunc(btf_id) || 14732 is_async_callback_calling_kfunc(btf_id); 14733 } 14734 14735 static bool is_rbtree_lock_required_kfunc(u32 btf_id) 14736 { 14737 return is_bpf_rbtree_api_kfunc(btf_id); 14738 } 14739 14740 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, 14741 enum btf_field_type head_field_type, 14742 u32 kfunc_btf_id) 14743 { 14744 bool ret; 14745 14746 switch (head_field_type) { 14747 case BPF_LIST_HEAD: 14748 ret = is_bpf_list_api_kfunc(kfunc_btf_id); 14749 break; 14750 case BPF_RB_ROOT: 14751 ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id); 14752 break; 14753 default: 14754 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", 14755 btf_field_type_name(head_field_type)); 14756 return false; 14757 } 14758 14759 if (!ret) 14760 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", 14761 btf_field_type_name(head_field_type)); 14762 return ret; 14763 } 14764 14765 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, 14766 enum btf_field_type node_field_type, 14767 u32 kfunc_btf_id) 14768 { 14769 bool ret; 14770 14771 switch (node_field_type) { 14772 case BPF_LIST_NODE: 14773 ret = is_bpf_list_push_kfunc(kfunc_btf_id) || 14774 kfunc_btf_id == special_kfunc_list[KF_bpf_list_del] || 14775 kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_first] || 14776 kfunc_btf_id == special_kfunc_list[KF_bpf_list_is_last]; 14777 break; 14778 case BPF_RB_NODE: 14779 ret = (is_bpf_rbtree_add_kfunc(kfunc_btf_id) || 14780 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 14781 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_left] || 14782 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_right]); 14783 break; 14784 default: 14785 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", 14786 btf_field_type_name(node_field_type)); 14787 return false; 14788 } 14789 14790 if (!ret) 14791 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", 14792 btf_field_type_name(node_field_type)); 14793 return ret; 14794 } 14795 14796 static int 14797 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, 14798 struct bpf_reg_state *reg, argno_t argno, 14799 struct bpf_call_arg_meta *meta, 14800 enum btf_field_type head_field_type, 14801 struct btf_field **head_field) 14802 { 14803 const char *head_type_name; 14804 struct btf_field *field; 14805 struct btf_record *rec; 14806 u32 head_off; 14807 14808 if (meta->btf != btf_vmlinux) { 14809 verifier_bug(env, "unexpected btf mismatch in kfunc call"); 14810 return -EFAULT; 14811 } 14812 14813 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) 14814 return -EFAULT; 14815 14816 head_type_name = btf_field_type_name(head_field_type); 14817 if (!tnum_is_const(reg->var_off)) { 14818 verbose(env, 14819 "%s doesn't have constant offset. %s has to be at the constant offset\n", 14820 reg_arg_name(env, argno), head_type_name); 14821 return -EINVAL; 14822 } 14823 14824 rec = reg_btf_record(reg); 14825 head_off = reg->var_off.value; 14826 field = btf_record_find(rec, head_off, head_field_type); 14827 if (!field) { 14828 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); 14829 return -EINVAL; 14830 } 14831 14832 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 14833 if (check_reg_allocation_locked(env, reg)) { 14834 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", 14835 rec->spin_lock_off, head_type_name); 14836 return -EINVAL; 14837 } 14838 14839 if (*head_field) { 14840 verifier_bug(env, "repeating %s arg", head_type_name); 14841 return -EFAULT; 14842 } 14843 *head_field = field; 14844 return 0; 14845 } 14846 14847 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 14848 struct bpf_reg_state *reg, argno_t argno, 14849 struct bpf_call_arg_meta *meta) 14850 { 14851 return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, 14852 &meta->arg_list_head.field); 14853 } 14854 14855 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, 14856 struct bpf_reg_state *reg, argno_t argno, 14857 struct bpf_call_arg_meta *meta) 14858 { 14859 return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, 14860 &meta->arg_rbtree_root.field); 14861 } 14862 14863 static int 14864 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, 14865 struct bpf_reg_state *reg, argno_t argno, 14866 struct bpf_call_arg_meta *meta, 14867 enum btf_field_type head_field_type, 14868 enum btf_field_type node_field_type, 14869 struct btf_field **node_field) 14870 { 14871 const char *node_type_name; 14872 const struct btf_type *et, *t; 14873 struct btf_field *field; 14874 u32 node_off; 14875 14876 if (meta->btf != btf_vmlinux) { 14877 verifier_bug(env, "unexpected btf mismatch in kfunc call"); 14878 return -EFAULT; 14879 } 14880 14881 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) 14882 return -EFAULT; 14883 14884 node_type_name = btf_field_type_name(node_field_type); 14885 if (!tnum_is_const(reg->var_off)) { 14886 verbose(env, 14887 "%s doesn't have constant offset. %s has to be at the constant offset\n", 14888 reg_arg_name(env, argno), node_type_name); 14889 return -EINVAL; 14890 } 14891 14892 node_off = reg->var_off.value; 14893 field = reg_find_field_offset(reg, node_off, node_field_type); 14894 if (!field) { 14895 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); 14896 return -EINVAL; 14897 } 14898 14899 field = *node_field; 14900 14901 et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); 14902 t = btf_type_by_id(reg->btf, reg->btf_id); 14903 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, 14904 field->graph_root.value_btf_id, true, 14905 !type_is_alloc(reg->type))) { 14906 verbose(env, "operation on %s expects arg#1 %s at offset=%d " 14907 "in struct %s, but arg is at offset=%d in struct %s\n", 14908 btf_field_type_name(head_field_type), 14909 btf_field_type_name(node_field_type), 14910 field->graph_root.node_offset, 14911 btf_name_by_offset(field->graph_root.btf, et->name_off), 14912 node_off, btf_name_by_offset(reg->btf, t->name_off)); 14913 return -EINVAL; 14914 } 14915 meta->arg_btf = reg->btf; 14916 meta->arg_btf_id = reg->btf_id; 14917 14918 if (node_off != field->graph_root.node_offset) { 14919 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", 14920 node_off, btf_field_type_name(node_field_type), 14921 field->graph_root.node_offset, 14922 btf_name_by_offset(field->graph_root.btf, et->name_off)); 14923 return -EINVAL; 14924 } 14925 14926 return 0; 14927 } 14928 14929 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 14930 struct bpf_reg_state *reg, argno_t argno, 14931 struct bpf_call_arg_meta *meta) 14932 { 14933 return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, 14934 BPF_LIST_HEAD, BPF_LIST_NODE, 14935 &meta->arg_list_head.field); 14936 } 14937 14938 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, 14939 struct bpf_reg_state *reg, argno_t argno, 14940 struct bpf_call_arg_meta *meta) 14941 { 14942 return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, 14943 BPF_RB_ROOT, BPF_RB_NODE, 14944 &meta->arg_rbtree_root.field); 14945 } 14946 14947 /* 14948 * css_task iter allowlist is needed to avoid dead locking on css_set_lock. 14949 * LSM hooks and iters (both sleepable and non-sleepable) are safe. 14950 * Any sleepable progs are also safe since bpf_check_attach_target() enforce 14951 * them can only be attached to some specific hook points. 14952 */ 14953 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) 14954 { 14955 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 14956 14957 switch (prog_type) { 14958 case BPF_PROG_TYPE_LSM: 14959 return true; 14960 case BPF_PROG_TYPE_TRACING: 14961 if (env->prog->expected_attach_type == BPF_TRACE_ITER) 14962 return true; 14963 fallthrough; 14964 default: 14965 return in_sleepable(env); 14966 } 14967 } 14968 14969 int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, 14970 s32 func_id, 14971 s16 offset, 14972 struct bpf_call_arg_meta *meta) 14973 { 14974 struct bpf_kfunc_meta kfunc; 14975 int err; 14976 14977 memset(meta, 0, sizeof(*meta)); 14978 14979 err = fetch_kfunc_meta(env, func_id, offset, &kfunc); 14980 if (err) 14981 return err; 14982 14983 meta->btf = kfunc.btf; 14984 meta->func_id = kfunc.id; 14985 meta->func_proto = kfunc.proto; 14986 meta->func_name = kfunc.name; 14987 14988 if (!kfunc.flags || !btf_kfunc_is_allowed(kfunc.btf, kfunc.id, env->prog)) 14989 return -EACCES; 14990 14991 meta->kfunc_flags = *kfunc.flags; 14992 14993 /* Only support release referenced argument passed by register */ 14994 if (is_kfunc_release(meta)) 14995 meta->release_regno = BPF_REG_1; 14996 14997 return 0; 14998 } 14999 15000 /* 15001 * Describe a helper's stack access through argument @arg (0-based, R1-R5). 15002 * must_write means the verifier destroys prior state throughout size bytes. 15003 * In other words, must_write follows the verifier's model of the 15004 * function's behaviour, which is safe to use for is_state_visited() pruning. 15005 */ 15006 struct arg_access_info 15007 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, 15008 int arg, int insn_idx) 15009 { 15010 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 15011 struct arg_access_info info = { 15012 .size = U32_MAX, 15013 .may_read = true, 15014 .may_write = true, 15015 }; 15016 const struct bpf_func_proto *fn; 15017 enum bpf_access_type access_type; 15018 enum bpf_arg_type at; 15019 bool exact_size = true; 15020 u64 size = U32_MAX; 15021 15022 if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) 15023 return info; 15024 15025 at = fn->arg_type[arg]; 15026 access_type = func_arg_access_type(at); 15027 15028 switch (base_type(at)) { 15029 case ARG_PTR_TO_MAP_KEY: 15030 case ARG_PTR_TO_MAP_VALUE: { 15031 bool is_key = base_type(at) == ARG_PTR_TO_MAP_KEY; 15032 u64 val; 15033 int i, map_reg; 15034 15035 for (i = 0; i < arg; i++) { 15036 if (base_type(fn->arg_type[i]) == ARG_CONST_MAP_PTR) 15037 break; 15038 } 15039 if (i >= arg) 15040 goto scan_all_maps; 15041 15042 map_reg = BPF_REG_1 + i; 15043 15044 if (!(aux->const_reg_map_mask & BIT(map_reg))) 15045 goto scan_all_maps; 15046 15047 i = aux->const_reg_vals[map_reg]; 15048 if (i < env->used_map_cnt) { 15049 /* Bloom-filter peek reads the value buffer. */ 15050 if (!is_key && insn->imm == BPF_FUNC_map_peek_elem && 15051 env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER) 15052 access_type = BPF_READ; 15053 size = is_key ? env->used_maps[i]->key_size 15054 : env->used_maps[i]->value_size; 15055 goto out; 15056 } 15057 scan_all_maps: 15058 /* 15059 * Map pointer is not known at this call site (e.g. different 15060 * maps on merged paths). Conservatively return the largest 15061 * key_size or value_size across all maps used by the program. 15062 * This is only an upper bound, so it cannot establish a definite 15063 * write. Map-dependent argument types can also turn an output 15064 * into an input, so conservatively retain a read dependency. 15065 */ 15066 exact_size = false; 15067 access_type |= BPF_READ; 15068 val = 0; 15069 for (i = 0; i < env->used_map_cnt; i++) { 15070 struct bpf_map *map = env->used_maps[i]; 15071 u32 sz = is_key ? map->key_size : map->value_size; 15072 15073 if (sz > val) 15074 val = sz; 15075 if (map->inner_map_meta) { 15076 sz = is_key ? map->inner_map_meta->key_size 15077 : map->inner_map_meta->value_size; 15078 if (sz > val) 15079 val = sz; 15080 } 15081 } 15082 if (!val) 15083 return info; 15084 size = val; 15085 goto out; 15086 } 15087 case ARG_PTR_TO_MEM: 15088 if (at & MEM_FIXED_SIZE) { 15089 size = fn->arg_size[arg]; 15090 goto out; 15091 } 15092 if (arg + 1 < ARRAY_SIZE(fn->arg_type) && 15093 arg_type_is_mem_size(fn->arg_type[arg + 1])) { 15094 int size_reg = BPF_REG_1 + arg + 1; 15095 15096 if (aux->const_reg_mask & BIT(size_reg)) { 15097 size = aux->const_reg_vals[size_reg]; 15098 goto out; 15099 } 15100 } 15101 /* Preserve access directions even when the extent is unknown. */ 15102 goto out; 15103 case ARG_PTR_TO_DYNPTR: 15104 size = BPF_DYNPTR_SIZE; 15105 break; 15106 case ARG_PTR_TO_STACK: 15107 /* 15108 * Only used by bpf_calls_callback() helpers. The helper itself 15109 * doesn't access stack. The callback subprog does and it's 15110 * analyzed separately. 15111 */ 15112 return (struct arg_access_info) {}; 15113 default: 15114 return info; 15115 } 15116 out: 15117 info.size = min_t(u64, size, U32_MAX); 15118 info.may_read = !!(access_type & BPF_READ); 15119 info.may_write = !!(access_type & BPF_WRITE); 15120 info.must_write = info.may_write && exact_size && info.size != U32_MAX; 15121 /* Generic unprivileged outputs retain prior initialization state. */ 15122 if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) { 15123 info.may_read = true; 15124 info.must_write = false; 15125 } 15126 return info; 15127 } 15128 15129 /* 15130 * Describe a kfunc's stack access through argument slot @arg (0-based). 15131 * As for helpers, must_write describes destruction of prior verifier state, 15132 * rather than a guarantee that the callee writes every byte at runtime. 15133 */ 15134 struct arg_access_info 15135 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, 15136 int arg, int insn_idx) 15137 { 15138 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 15139 struct arg_access_info info = { 15140 .size = U32_MAX, 15141 .may_read = true, 15142 .may_write = true, 15143 }; 15144 struct bpf_call_arg_meta meta; 15145 const struct btf_param *args; 15146 const struct btf_type *t, *ref_t; 15147 const struct btf *btf; 15148 u32 i, slot, nargs, type_size; 15149 u64 size; 15150 15151 if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) 15152 return info; 15153 15154 btf = meta.btf; 15155 args = btf_params(meta.func_proto); 15156 nargs = btf_type_vlen(meta.func_proto); 15157 15158 /* 15159 * @arg is an argument slot and a 16-byte parameter takes two of them, 15160 * so walk the parameters to find the one that starts at this slot. A 15161 * slot holding the upper eightbyte of such a parameter belongs to no 15162 * pointer, and neither does a slot past the last parameter. 15163 */ 15164 for (i = 0, slot = 0; i < nargs && slot < arg; i++) 15165 slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); 15166 if (i >= nargs || slot != arg) 15167 return (struct arg_access_info) {}; 15168 15169 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 15170 if (!btf_type_is_ptr(t)) 15171 return (struct arg_access_info) {}; 15172 15173 /* dynptr: fixed 16-byte on-stack representation */ 15174 if (is_kfunc_arg_dynptr(btf, &args[i])) { 15175 size = BPF_DYNPTR_SIZE; 15176 goto out; 15177 } 15178 15179 /* ptr + __sz/__szk pair: the size follows the pointer */ 15180 if (i + 1 < nargs && 15181 (btf_param_match_suffix(btf, &args[i + 1], "__sz") || 15182 btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { 15183 int size_reg = BPF_REG_1 + arg + 1; 15184 15185 if (size_reg <= MAX_BPF_FUNC_REG_ARGS && 15186 (aux->const_reg_mask & BIT(size_reg))) { 15187 size = aux->const_reg_vals[size_reg]; 15188 goto out; 15189 } 15190 /* Unknown size: the read may extend anywhere up to the frame top. */ 15191 return info; 15192 } 15193 15194 /* fixed-size pointed-to type: resolve via BTF */ 15195 ref_t = btf_type_skip_modifiers(btf, t->type, NULL); 15196 if (!IS_ERR(btf_resolve_size(btf, ref_t, &type_size))) { 15197 size = type_size; 15198 goto out; 15199 } 15200 15201 return info; 15202 out: 15203 info.size = min_t(u64, size, U32_MAX); 15204 /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ 15205 if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) 15206 info.may_read = false; 15207 if (is_kfunc_arg_uninit(btf, &args[i]) && 15208 (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) 15209 info.may_read = false; 15210 info.must_write = !info.may_read && info.size != U32_MAX; 15211 return info; 15212 } 15213 15214 /* 15215 * Called before dead code elimination, some subprograms may be genuinely unreachable. 15216 * Suppress diagnostics so callers can use a conservative result on error. 15217 * The main pass would report errors for reachable subprograms. 15218 */ 15219 static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) 15220 { 15221 struct bpf_prog *prog = env->prog; 15222 u32 log_level = env->log.level; 15223 int err; 15224 15225 if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id) 15226 return false; 15227 15228 env->log.level = 0; 15229 err = btf_prepare_func_args(env, subprog); 15230 env->log.level = log_level; 15231 return !err; 15232 } 15233 15234 /* Prepare argument metadata for liveness; the main program is handled separately. */ 15235 static void prepare_subprog_args(struct bpf_verifier_env *env) 15236 { 15237 int i; 15238 15239 for (i = 1; i < env->subprog_cnt; i++) { 15240 if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i)) 15241 /* Keep all argument registers live if the prototype is unavailable. */ 15242 env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS; 15243 } 15244 } 15245 15246 /* 15247 * Derive stack effects for a global subprog's argument slot @arg (0-based) 15248 * from its declared type, matching the main pass's argument checks. 15249 */ 15250 struct arg_access_info 15251 bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, 15252 int arg, int insn_idx) 15253 { 15254 struct arg_access_info info = { 15255 .size = U32_MAX, 15256 .may_read = true, 15257 .may_write = true, 15258 }; 15259 struct bpf_subprog_info *sub; 15260 int callee; 15261 15262 callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); 15263 if (callee < 0) 15264 return info; 15265 sub = subprog_info(env, callee); 15266 if (!try_btf_prepare_func_args(env, callee)) 15267 return info; 15268 15269 switch (base_type(sub->args[arg].arg_type)) { 15270 case ARG_UNUSED: /* Not an argument slot. */ 15271 case ARG_SCALAR: /* A scalar, the callee can't dereference it. */ 15272 return (struct arg_access_info) {}; 15273 case ARG_PTR_TO_BTF_ID: 15274 /* 15275 * Trusted arguments cannot be stack pointers. 15276 * Untrusted arguments use ARG_IGNORE: 15277 * probe reads need no caller stack state, writes are prohibited. 15278 */ 15279 return (struct arg_access_info) {}; 15280 case ARG_PTR_TO_MEM: 15281 /* mem_size is zero for untrusted pointers of unknown size. */ 15282 if (sub->args[arg].mem_size) { 15283 info.size = sub->args[arg].mem_size; 15284 info.must_write = true; 15285 } 15286 return info; 15287 case ARG_PTR_TO_DYNPTR: 15288 /* The callee may adjust the view, but cannot reinitialize the dynptr. */ 15289 info.size = BPF_DYNPTR_SIZE; 15290 return info; 15291 default: 15292 return info; 15293 } 15294 } 15295 15296 /* check special kfuncs and return: 15297 * 1 - not fall-through to 'else' branch, continue verification 15298 * 0 - fall-through to 'else' branch 15299 * < 0 - not fall-through to 'else' branch, return error 15300 */ 15301 static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 15302 struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux, 15303 const struct btf_type *ptr_type, struct btf *desc_btf) 15304 { 15305 const struct btf_type *ret_t; 15306 int err = 0; 15307 15308 if (meta->btf != btf_vmlinux) 15309 return 0; 15310 15311 if (is_bpf_obj_new_kfunc(meta->func_id) || is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 15312 struct btf_struct_meta *struct_meta; 15313 struct btf *ret_btf; 15314 u32 ret_btf_id; 15315 15316 if (is_bpf_obj_new_kfunc(meta->func_id) && !bpf_global_ma_set) 15317 return -ENOMEM; 15318 15319 if (((u64)(u32)meta->arg_constant.value) != meta->arg_constant.value) { 15320 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 15321 return -EINVAL; 15322 } 15323 15324 ret_btf = env->prog->aux->btf; 15325 ret_btf_id = meta->arg_constant.value; 15326 15327 /* This may be NULL due to user not supplying a BTF */ 15328 if (!ret_btf) { 15329 verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); 15330 return -EINVAL; 15331 } 15332 15333 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 15334 if (!ret_t || !__btf_type_is_struct(ret_t)) { 15335 verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); 15336 return -EINVAL; 15337 } 15338 15339 if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 15340 if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) { 15341 verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", 15342 ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE); 15343 return -EINVAL; 15344 } 15345 15346 if (!bpf_global_percpu_ma_set) { 15347 mutex_lock(&bpf_percpu_ma_lock); 15348 if (!bpf_global_percpu_ma_set) { 15349 /* Charge memory allocated with bpf_global_percpu_ma to 15350 * root memcg. The obj_cgroup for root memcg is NULL. 15351 */ 15352 err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL); 15353 if (!err) 15354 bpf_global_percpu_ma_set = true; 15355 } 15356 mutex_unlock(&bpf_percpu_ma_lock); 15357 if (err) 15358 return err; 15359 } 15360 15361 mutex_lock(&bpf_percpu_ma_lock); 15362 err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size); 15363 mutex_unlock(&bpf_percpu_ma_lock); 15364 if (err) 15365 return err; 15366 } 15367 15368 struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); 15369 if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { 15370 if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { 15371 verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); 15372 return -EINVAL; 15373 } 15374 15375 if (struct_meta) { 15376 verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); 15377 return -EINVAL; 15378 } 15379 } 15380 15381 mark_reg_known_zero(env, regs, BPF_REG_0); 15382 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 15383 regs[BPF_REG_0].btf = ret_btf; 15384 regs[BPF_REG_0].btf_id = ret_btf_id; 15385 if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) 15386 regs[BPF_REG_0].type |= MEM_PERCPU; 15387 15388 insn_aux->obj_new_size = ret_t->size; 15389 insn_aux->kptr_struct_meta = struct_meta; 15390 } else if (is_bpf_refcount_acquire_kfunc(meta->func_id)) { 15391 mark_reg_known_zero(env, regs, BPF_REG_0); 15392 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 15393 regs[BPF_REG_0].btf = meta->arg_btf; 15394 regs[BPF_REG_0].btf_id = meta->arg_btf_id; 15395 15396 insn_aux->kptr_struct_meta = 15397 btf_find_struct_meta(meta->arg_btf, 15398 meta->arg_btf_id); 15399 } else if (is_list_node_type(ptr_type)) { 15400 struct btf_field *field = meta->arg_list_head.field; 15401 15402 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 15403 } else if (is_rbtree_node_type(ptr_type)) { 15404 struct btf_field *field = meta->arg_rbtree_root.field; 15405 15406 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 15407 } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { 15408 mark_reg_known_zero(env, regs, BPF_REG_0); 15409 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 15410 regs[BPF_REG_0].btf = desc_btf; 15411 regs[BPF_REG_0].btf_id = meta->ret_btf_id; 15412 } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) { 15413 ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); 15414 if (!ret_t) { 15415 verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", 15416 meta->arg_constant.value); 15417 return -EINVAL; 15418 } else if (btf_type_is_struct(ret_t)) { 15419 mark_reg_known_zero(env, regs, BPF_REG_0); 15420 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 15421 regs[BPF_REG_0].btf = desc_btf; 15422 regs[BPF_REG_0].btf_id = meta->arg_constant.value; 15423 } else if (btf_type_is_void(ret_t)) { 15424 mark_reg_known_zero(env, regs, BPF_REG_0); 15425 regs[BPF_REG_0].type = PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; 15426 regs[BPF_REG_0].mem_size = 0; 15427 } else { 15428 verbose(env, 15429 "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); 15430 return -EINVAL; 15431 } 15432 } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) || 15433 is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) { 15434 enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); 15435 15436 mark_reg_known_zero(env, regs, BPF_REG_0); 15437 15438 if (!meta->arg_constant.found) { 15439 verifier_bug(env, "bpf_dynptr_slice(_rdwr) no constant size"); 15440 return -EFAULT; 15441 } 15442 15443 regs[BPF_REG_0].mem_size = meta->arg_constant.value; 15444 15445 /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ 15446 regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; 15447 15448 if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) { 15449 regs[BPF_REG_0].type |= MEM_RDONLY; 15450 } else { 15451 /* this will set env->seen_direct_write to true */ 15452 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { 15453 verbose(env, "the prog does not allow writes to packet data\n"); 15454 return -EINVAL; 15455 } 15456 } 15457 15458 if (!meta->dynptr.id) { 15459 verifier_bug(env, "no dynptr id"); 15460 return -EFAULT; 15461 } 15462 regs[BPF_REG_0].parent_id = meta->dynptr.id; 15463 regs[BPF_REG_0].buf = meta->slice_buf; 15464 if (meta->slice_buf.size) 15465 env->seen_stack_slice = true; 15466 } else { 15467 return 0; 15468 } 15469 15470 return 1; 15471 } 15472 15473 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); 15474 15475 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 15476 int *insn_idx_p) 15477 { 15478 bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable; 15479 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 15480 struct bpf_reg_state *regs = cur_regs(env); 15481 const char *func_name, *ptr_type_name; 15482 const struct btf_type *t, *ptr_type; 15483 struct bpf_call_arg_meta meta; 15484 struct bpf_insn_aux_data *insn_aux; 15485 const char *operation; 15486 int err, insn_idx = *insn_idx_p; 15487 u32 i, proto_slots, ptr_type_id, ret_nregs = 1; 15488 struct bpf_kfunc_desc *desc; 15489 struct btf *desc_btf; 15490 int id; 15491 15492 /* skip for now, but return error when we find this in fixup_kfunc_call */ 15493 if (!insn->imm) 15494 return 0; 15495 15496 err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); 15497 if (err == -EACCES && meta.func_name) { 15498 verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); 15499 operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name); 15500 bpf_diag_policy( 15501 env, insn_idx, operation, "this program cannot call the kfunc", 15502 "Use a kfunc allowed for this program type and attach point, or change the program context."); 15503 } 15504 if (err) 15505 return err; 15506 desc_btf = meta.btf; 15507 func_name = meta.func_name; 15508 insn_aux = &env->insn_aux_data[insn_idx]; 15509 15510 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 15511 if (!desc) { 15512 verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); 15513 return -EFAULT; 15514 } 15515 meta.fn = &desc->proto; 15516 15517 insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); 15518 15519 if (!insn->off && 15520 (insn->imm == special_kfunc_list[KF_bpf_res_spin_lock] || 15521 insn->imm == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { 15522 struct bpf_verifier_state *branch; 15523 struct bpf_reg_state *regs; 15524 15525 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); 15526 if (IS_ERR(branch)) { 15527 verbose(env, "failed to push state for failed lock acquisition\n"); 15528 return PTR_ERR(branch); 15529 } 15530 15531 regs = branch->frame[branch->curframe]->regs; 15532 15533 /* Clear r0-r5 registers in forked state */ 15534 for (i = 0; i < CALLER_SAVED_REGS; i++) 15535 bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); 15536 15537 mark_reg_unknown(env, regs, BPF_REG_0); 15538 err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1); 15539 if (err) { 15540 verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); 15541 return err; 15542 } 15543 } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) { 15544 verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n"); 15545 return -EFAULT; 15546 } 15547 15548 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 15549 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 15550 operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name); 15551 bpf_diag_policy( 15552 env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT", 15553 "Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs."); 15554 return -EACCES; 15555 } 15556 15557 if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) { 15558 verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 15559 func_name); 15560 operation = bpf_diag_fmt(env, "kfunc %s", func_name); 15561 bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON", 15562 "Load the program with CAP_PERFMON, or avoid the kfunc."); 15563 return -EPERM; 15564 } 15565 15566 sleepable = bpf_is_kfunc_sleepable(&meta); 15567 if (sleepable && !in_sleepable_context(env)) { 15568 const char *suggestion; 15569 15570 if (in_sleepable(env)) { 15571 verbose(env, "kernel func %s is sleepable within %s\n", 15572 func_name, non_sleepable_context_description(env)); 15573 suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."; 15574 } else { 15575 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", 15576 func_name); 15577 suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."; 15578 } 15579 operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); 15580 bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); 15581 return -EACCES; 15582 } 15583 15584 /* Track non-sleepable context for kfuncs, same as for helpers. */ 15585 if (!in_sleepable_context(env)) 15586 insn_aux->non_sleepable = true; 15587 15588 /* Check the arguments */ 15589 err = check_func_args(env, &meta, insn_idx); 15590 if (err < 0) 15591 return err; 15592 15593 err = mark_raw_stack(env, &meta, insn_idx); 15594 if (err) 15595 return err; 15596 15597 if ((is_bpf_obj_drop_kfunc(meta.func_id) || 15598 is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || 15599 /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ 15600 (prog_type == BPF_PROG_TYPE_TRACING && env->prog->expected_attach_type != BPF_TRACE_ITER 15601 && !env->prog->sleepable))) { 15602 struct btf_struct_meta *struct_meta; 15603 15604 struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); 15605 if (struct_meta && btf_record_has_nmi_unsafe_fields(struct_meta->record)) { 15606 verbose(env, "%s cannot be used in tracing programs on types with NMI unsafe fields\n", 15607 func_name); 15608 return -EINVAL; 15609 } 15610 } 15611 15612 if (is_bpf_rbtree_add_kfunc(meta.func_id)) { 15613 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 15614 set_rbtree_add_callback_state); 15615 if (err) { 15616 verbose(env, "kfunc %s#%d failed callback verification\n", 15617 func_name, meta.func_id); 15618 return err; 15619 } 15620 } 15621 15622 if (is_bpf_wq_set_callback_kfunc(meta.func_id)) { 15623 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 15624 set_timer_callback_state); 15625 if (err) { 15626 verbose(env, "kfunc %s#%d failed callback verification\n", 15627 func_name, meta.func_id); 15628 return err; 15629 } 15630 } 15631 15632 if (is_task_work_add_kfunc(meta.func_id)) { 15633 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 15634 set_task_work_schedule_callback_state); 15635 if (err) { 15636 verbose(env, "kfunc %s#%d failed callback verification\n", 15637 func_name, meta.func_id); 15638 return err; 15639 } 15640 } 15641 15642 if (is_call_rcu_kfunc(meta.func_id)) { 15643 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 15644 set_rcu_callback_state); 15645 if (err) { 15646 verbose(env, "kfunc %s#%d failed callback verification\n", 15647 func_name, meta.func_id); 15648 return err; 15649 } 15650 } 15651 15652 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 15653 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 15654 15655 preempt_disable = is_kfunc_bpf_preempt_disable(&meta); 15656 preempt_enable = is_kfunc_bpf_preempt_enable(&meta); 15657 15658 if (rcu_lock) { 15659 env->cur_state->active_rcu_locks++; 15660 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true, 15661 env->cur_state->active_rcu_locks); 15662 } else if (rcu_unlock) { 15663 if (env->cur_state->active_rcu_locks == 0) { 15664 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 15665 bpf_diag_ctx_underflow( 15666 env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, 15667 "Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region."); 15668 return -EINVAL; 15669 } 15670 env->cur_state->active_rcu_locks--; 15671 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false, 15672 env->cur_state->active_rcu_locks); 15673 if (!in_rcu_cs(env)) 15674 invalidate_rcu_protected_refs(env); 15675 } else if (preempt_disable) { 15676 env->cur_state->active_preempt_locks++; 15677 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true, 15678 env->cur_state->active_preempt_locks); 15679 } else if (preempt_enable) { 15680 if (env->cur_state->active_preempt_locks == 0) { 15681 verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); 15682 bpf_diag_ctx_underflow( 15683 env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT, 15684 "Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption."); 15685 return -EINVAL; 15686 } 15687 env->cur_state->active_preempt_locks--; 15688 bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false, 15689 env->cur_state->active_preempt_locks); 15690 if (!in_rcu_cs(env)) 15691 invalidate_rcu_protected_refs(env); 15692 } 15693 15694 if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { 15695 verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); 15696 return -EACCES; 15697 } 15698 15699 if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { 15700 verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); 15701 bpf_diag_ctx_required( 15702 env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, 15703 "Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); 15704 return -EACCES; 15705 } 15706 15707 /* In case of release function, we get register number of refcounted 15708 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 15709 */ 15710 if (meta.release_regno) { 15711 err = release_reg(env, ®s[meta.release_regno], false, !!meta.dynptr.id); 15712 if (err) 15713 return err; 15714 } 15715 15716 if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) { 15717 id = regs[BPF_REG_2].id; 15718 insn_aux->insert_off = regs[BPF_REG_2].var_off.value; 15719 insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); 15720 ref_convert_owning_non_owning(env, id); 15721 } 15722 15723 if ((is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice]) || 15724 is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) && 15725 dynptr_slice_may_be_buffer(meta.dynptr.type)) { 15726 struct bpf_reg_state *buf = ®s[BPF_REG_3]; 15727 15728 if (buf->type == PTR_TO_STACK) { 15729 meta.slice_buf.frameno = buf->frameno; 15730 meta.slice_buf.off = reg_smin(buf); 15731 meta.slice_buf.size = reg_smax(buf) - reg_smin(buf) + 15732 meta.arg_constant.value; 15733 } else if (base_type(buf->type) == PTR_TO_MEM) { 15734 /* a slice is the buffer of the next one */ 15735 meta.slice_buf = buf->buf; 15736 } 15737 } 15738 15739 if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { 15740 if (!bpf_jit_supports_exceptions()) { 15741 verbose(env, "JIT does not support calling kfunc %s#%d\n", 15742 func_name, meta.func_id); 15743 return -ENOTSUPP; 15744 } 15745 env->seen_exception = true; 15746 15747 /* In the case of the default callback, the cookie value passed 15748 * to bpf_throw becomes the return value of the program. 15749 */ 15750 if (!env->exception_callback_subprog) { 15751 err = check_return_code(env, BPF_REG_1, "R1"); 15752 if (err < 0) 15753 return err; 15754 } 15755 } 15756 15757 bpf_diag_record_caller_saved(env, regs); 15758 bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); 15759 for (i = 0; i < CALLER_SAVED_REGS; i++) { 15760 u32 regno = caller_saved[i]; 15761 15762 bpf_mark_reg_not_init(env, ®s[regno]); 15763 } 15764 invalidate_outgoing_stack_args(env, cur_func(env)); 15765 15766 /* Check return type */ 15767 t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL); 15768 15769 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 15770 if (meta.btf != btf_vmlinux || 15771 (!is_bpf_obj_new_kfunc(meta.func_id) && 15772 !is_bpf_percpu_obj_new_kfunc(meta.func_id) && 15773 !is_bpf_refcount_acquire_kfunc(meta.func_id))) { 15774 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 15775 return -EINVAL; 15776 } 15777 } 15778 15779 if (btf_type_is_scalar(t)) { 15780 ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); 15781 if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || 15782 meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) 15783 __mark_reg_const_zero(env, ®s[BPF_REG_0]); 15784 } else if (btf_type_is_struct(t)) { 15785 struct btf_member_path path = {}; 15786 const char *member_note = ""; 15787 15788 /* 15789 * The returned struct comes back as raw register bits modeled 15790 * as an unknown scalar, so a pointer member would be laundered 15791 * into a scalar and escape provenance and reference tracking. 15792 * Only scalars and arena pointers are allowed; see 15793 * btf_validate_return_type() in btf.c for why an arena pointer 15794 * is safe here. 15795 */ 15796 if (!btf_struct_member_walk(env, desc_btf, t, 15797 BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { 15798 verbose(env, 15799 "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", 15800 func_name, btf_type_str(t), 15801 btf_name_by_offset(desc_btf, t->name_off)); 15802 if (path.too_deep) { 15803 member_note = bpf_diag_fmt( 15804 env, " It nests structs more than %d levels deep.", 15805 BTF_MEMBER_MAX_DEPTH); 15806 } else if (path.depth) { 15807 char bad_name[BTF_MEMBER_PATH_LEN]; 15808 15809 btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); 15810 if (!path.bad_type) { 15811 verbose(env, "member '%s' is a zero-length array\n", 15812 bad_name); 15813 member_note = bpf_diag_fmt( 15814 env, " Its member '%s' is a zero-length array.", 15815 bad_name); 15816 } else { 15817 verbose(env, "member '%s' has type %s\n", bad_name, 15818 btf_type_str(path.bad_type)); 15819 member_note = bpf_diag_fmt( 15820 env, 15821 " Its member '%s' is %s, not a scalar or an arena pointer.", 15822 bad_name, btf_type_str(path.bad_type)); 15823 } 15824 } 15825 bpf_diag_program_structure( 15826 env, insn_idx, "unsupported kernel function return type", 15827 "Call a kernel function that returns only scalars or arena pointers by value.", 15828 "%s() returns %s %s by value.%s " 15829 "Only kfuncs returning scalar values or arena pointers, or " 15830 "structures composed of scalar values or arena pointers are " 15831 "supported.", 15832 func_name, btf_type_str(t), 15833 btf_name_by_offset(desc_btf, t->name_off), member_note); 15834 return -EINVAL; 15835 } 15836 ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); 15837 } else if (btf_type_is_ptr(t)) { 15838 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 15839 err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); 15840 if (err) { 15841 if (err < 0) 15842 return err; 15843 } else if (btf_type_is_void(ptr_type)) { 15844 /* kfunc returning 'void *' is equivalent to returning scalar */ 15845 mark_reg_unknown(env, regs, BPF_REG_0); 15846 } else if (!__btf_type_is_struct(ptr_type)) { 15847 if (!meta.ret_mem.found) { 15848 __u32 sz; 15849 15850 if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { 15851 meta.ret_mem.found = true; 15852 meta.ret_mem.size = sz; 15853 meta.r0_rdonly = true; 15854 } 15855 15856 if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) 15857 meta.r0_rdonly = false; 15858 } 15859 if (!meta.ret_mem.found) { 15860 ptr_type_name = btf_name_by_offset(desc_btf, 15861 ptr_type->name_off); 15862 verbose(env, 15863 "kernel function %s returns pointer type %s %s is not supported\n", 15864 func_name, 15865 btf_type_str(ptr_type), 15866 ptr_type_name); 15867 return -EINVAL; 15868 } 15869 15870 mark_reg_known_zero(env, regs, BPF_REG_0); 15871 regs[BPF_REG_0].type = PTR_TO_MEM; 15872 regs[BPF_REG_0].mem_size = meta.ret_mem.size; 15873 15874 if (meta.r0_rdonly) 15875 regs[BPF_REG_0].type |= MEM_RDONLY; 15876 15877 /* Ensures we don't access the memory after a release_reference() */ 15878 if (meta.ref_obj.id) { 15879 err = validate_ref_obj(env, &meta.ref_obj); 15880 if (err) 15881 return err; 15882 regs[BPF_REG_0].parent_id = meta.ref_obj.id; 15883 } 15884 15885 if (is_kfunc_rcu_protected(&meta)) 15886 regs[BPF_REG_0].type |= MEM_RCU; 15887 } else { 15888 enum bpf_reg_type type = PTR_TO_BTF_ID; 15889 15890 if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache]) 15891 type |= PTR_UNTRUSTED; 15892 else if (is_kfunc_rcu_protected(&meta) || 15893 (bpf_is_iter_next_kfunc(&meta) && 15894 (get_iter_from_state(env->cur_state, &meta) 15895 ->type & MEM_RCU))) { 15896 /* 15897 * If the iterator's constructor (the _new 15898 * function e.g., bpf_iter_task_new) has been 15899 * annotated with BPF kfunc flag 15900 * KF_RCU_PROTECTED and was called within a RCU 15901 * read-side critical section, also propagate 15902 * the MEM_RCU flag to the pointer returned from 15903 * the iterator's next function (e.g., 15904 * bpf_iter_task_next). 15905 */ 15906 type |= MEM_RCU; 15907 } else { 15908 /* 15909 * Any PTR_TO_BTF_ID that is returned from a BPF 15910 * kfunc should by default be treated as 15911 * implicitly trusted. 15912 */ 15913 type |= PTR_TRUSTED; 15914 } 15915 15916 mark_reg_known_zero(env, regs, BPF_REG_0); 15917 regs[BPF_REG_0].btf = desc_btf; 15918 regs[BPF_REG_0].type = type; 15919 regs[BPF_REG_0].btf_id = ptr_type_id; 15920 if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU)) 15921 regs[BPF_REG_0].parent_id = meta.ref_obj.id; 15922 } 15923 15924 if (is_kfunc_ret_null(&meta)) { 15925 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 15926 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 15927 regs[BPF_REG_0].id = ++env->id_gen; 15928 } 15929 if (is_kfunc_acquire(&meta)) { 15930 id = acquire_reference(env, insn_idx, 0); 15931 if (id < 0) 15932 return id; 15933 regs[BPF_REG_0].id = id; 15934 } else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) { 15935 ref_set_non_owning(env, ®s[BPF_REG_0]); 15936 } 15937 15938 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 15939 regs[BPF_REG_0].id = ++env->id_gen; 15940 } else if (btf_type_is_void(t)) { 15941 if (meta.btf == btf_vmlinux) { 15942 if (is_bpf_obj_drop_kfunc(meta.func_id) || 15943 is_bpf_percpu_obj_drop_kfunc(meta.func_id)) { 15944 insn_aux->kptr_struct_meta = 15945 btf_find_struct_meta(meta.arg_btf, 15946 meta.arg_btf_id); 15947 } 15948 } 15949 } 15950 15951 if (is_kfunc_dynptr_write(&meta) ? 15952 dynptr_write_changes_pkt_data(meta.dynptr_write_type) : 15953 bpf_is_kfunc_pkt_changing(&meta)) 15954 clear_all_pkt_pointers(env); 15955 15956 /* 15957 * These kfuncs write to the head of the skb in place if what they write 15958 * to is linear. Like a direct write that takes bpf_unclone_prologue(). 15959 */ 15960 if (is_kfunc_dynptr_write(&meta) && 15961 dynptr_write_changes_pkt_data(meta.dynptr_write_type)) 15962 may_access_direct_pkt_data(env, NULL, BPF_WRITE); 15963 15964 proto_slots = kfunc_abi_slots(&desc->func_model); 15965 if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { 15966 struct bpf_func_state *caller = cur_func(env); 15967 struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; 15968 u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; 15969 u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; 15970 15971 if (stack_arg_cnt > caller_info->stack_arg_cnt) 15972 caller_info->stack_arg_cnt = stack_arg_cnt; 15973 } 15974 15975 /* 15976 * Record R0 before process_iter_next_call() snapshots the alternate 15977 * iterator path's diagnostic position. 15978 */ 15979 if (ret_nregs == 1) 15980 bpf_diag_mod_end(env); 15981 15982 if (bpf_is_iter_next_kfunc(&meta)) { 15983 err = process_iter_next_call(env, insn_idx, &meta); 15984 if (err) 15985 return err; 15986 } 15987 15988 if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) 15989 env->prog->call_session_cookie = true; 15990 15991 if (bpf_is_throw_kfunc(insn)) 15992 return process_bpf_exit_full(env, NULL, true); 15993 15994 return 0; 15995 } 15996 15997 static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env, 15998 const struct bpf_reg_state *reg, 15999 enum bpf_reg_type type) 16000 { 16001 bool known = tnum_is_const(reg->var_off); 16002 s64 val = reg->var_off.value; 16003 s64 smin = reg_smin(reg); 16004 16005 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 16006 verbose(env, "math between %s pointer and %lld is not allowed\n", 16007 reg_type_str(env, type), val); 16008 return false; 16009 } 16010 16011 if (smin == S64_MIN) { 16012 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 16013 reg_type_str(env, type)); 16014 return false; 16015 } 16016 16017 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 16018 verbose(env, "value %lld makes %s pointer be out of bounds\n", 16019 smin, reg_type_str(env, type)); 16020 return false; 16021 } 16022 16023 return true; 16024 } 16025 16026 static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env, 16027 const struct bpf_reg_state *reg, 16028 enum bpf_reg_type type) 16029 { 16030 bool known = tnum_is_const(reg->var_off); 16031 s64 val = reg->var_off.value; 16032 s64 smin = reg_smin(reg); 16033 16034 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 16035 verbose(env, "%s pointer offset %lld is not allowed\n", 16036 reg_type_str(env, type), val); 16037 return false; 16038 } 16039 16040 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 16041 verbose(env, "%s pointer offset %lld is not allowed\n", 16042 reg_type_str(env, type), smin); 16043 return false; 16044 } 16045 16046 return true; 16047 } 16048 16049 enum { 16050 REASON_BOUNDS = -1, 16051 REASON_TYPE = -2, 16052 REASON_PATHS = -3, 16053 REASON_LIMIT = -4, 16054 REASON_STACK = -5, 16055 }; 16056 16057 static int retrieve_ptr_limit(const struct bpf_verifier_env *env, 16058 const struct bpf_reg_state *ptr_reg, 16059 u32 *alu_limit, bool mask_to_left) 16060 { 16061 u32 max = 0, ptr_limit = 0; 16062 16063 switch (ptr_reg->type) { 16064 case PTR_TO_STACK: 16065 /* Offset 0 is out-of-bounds, but acceptable start for the 16066 * left direction, see BPF_REG_FP. Also, unknown scalar 16067 * offset where we would need to deal with min/max bounds is 16068 * currently prohibited for unprivileged. 16069 */ 16070 max = env->stack_limit + mask_to_left; 16071 ptr_limit = -ptr_reg->var_off.value; 16072 break; 16073 case PTR_TO_MAP_VALUE: 16074 max = ptr_reg->map_ptr->value_size; 16075 ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg); 16076 break; 16077 default: 16078 return REASON_TYPE; 16079 } 16080 16081 if (ptr_limit >= max) 16082 return REASON_LIMIT; 16083 *alu_limit = ptr_limit; 16084 return 0; 16085 } 16086 16087 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 16088 const struct bpf_insn *insn) 16089 { 16090 return env->bypass_spec_v1 || 16091 BPF_SRC(insn->code) == BPF_K || 16092 cur_aux(env)->nospec; 16093 } 16094 16095 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 16096 u32 alu_state, u32 alu_limit) 16097 { 16098 /* If we arrived here from different branches with different 16099 * state or limits to sanitize, then this won't work. 16100 */ 16101 if (aux->alu_state && 16102 (aux->alu_state != alu_state || 16103 aux->alu_limit != alu_limit)) 16104 return REASON_PATHS; 16105 16106 /* Corresponding fixup done in do_misc_fixups(). */ 16107 aux->alu_state = alu_state; 16108 aux->alu_limit = alu_limit; 16109 return 0; 16110 } 16111 16112 static int sanitize_val_alu(struct bpf_verifier_env *env, 16113 struct bpf_insn *insn) 16114 { 16115 struct bpf_insn_aux_data *aux = cur_aux(env); 16116 16117 if (can_skip_alu_sanitation(env, insn)) 16118 return 0; 16119 16120 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 16121 } 16122 16123 static bool sanitize_needed(u8 opcode) 16124 { 16125 return opcode == BPF_ADD || opcode == BPF_SUB; 16126 } 16127 16128 struct bpf_sanitize_info { 16129 struct bpf_insn_aux_data aux; 16130 bool mask_to_left; 16131 }; 16132 16133 static int sanitize_speculative_path(struct bpf_verifier_env *env, 16134 const struct bpf_insn *insn, 16135 u32 next_idx, u32 curr_idx) 16136 { 16137 struct bpf_verifier_state *branch; 16138 struct bpf_reg_state *regs; 16139 16140 branch = push_stack(env, next_idx, curr_idx, true); 16141 if (!IS_ERR(branch) && insn) { 16142 regs = branch->frame[branch->curframe]->regs; 16143 if (BPF_SRC(insn->code) == BPF_K) { 16144 mark_reg_unknown(env, regs, insn->dst_reg); 16145 } else if (BPF_SRC(insn->code) == BPF_X) { 16146 mark_reg_unknown(env, regs, insn->dst_reg); 16147 mark_reg_unknown(env, regs, insn->src_reg); 16148 } 16149 } 16150 return PTR_ERR_OR_ZERO(branch); 16151 } 16152 16153 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 16154 struct bpf_insn *insn, 16155 const struct bpf_reg_state *ptr_reg, 16156 const struct bpf_reg_state *off_reg, 16157 struct bpf_reg_state *dst_reg, 16158 struct bpf_sanitize_info *info, 16159 const bool commit_window) 16160 { 16161 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 16162 struct bpf_verifier_state *vstate = env->cur_state; 16163 bool off_is_imm = tnum_is_const(off_reg->var_off); 16164 bool off_is_neg = reg_smin(off_reg) < 0; 16165 bool ptr_is_dst_reg = ptr_reg == dst_reg; 16166 u8 opcode = BPF_OP(insn->code); 16167 u32 alu_state, alu_limit; 16168 struct bpf_reg_state tmp; 16169 int err; 16170 16171 if (can_skip_alu_sanitation(env, insn)) 16172 return 0; 16173 16174 /* We already marked aux for masking from non-speculative 16175 * paths, thus we got here in the first place. We only care 16176 * to explore bad access from here. 16177 */ 16178 if (vstate->speculative) 16179 goto do_sim; 16180 16181 if (!commit_window) { 16182 if (!tnum_is_const(off_reg->var_off) && 16183 (reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0)) 16184 return REASON_BOUNDS; 16185 16186 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 16187 (opcode == BPF_SUB && !off_is_neg); 16188 } 16189 16190 err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left); 16191 if (err < 0) 16192 return err; 16193 16194 if (commit_window) { 16195 /* In commit phase we narrow the masking window based on 16196 * the observed pointer move after the simulated operation. 16197 */ 16198 alu_state = info->aux.alu_state; 16199 alu_limit = abs(info->aux.alu_limit - alu_limit); 16200 } else { 16201 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 16202 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 16203 alu_state |= ptr_is_dst_reg ? 16204 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 16205 16206 /* Limit pruning on unknown scalars to enable deep search for 16207 * potential masking differences from other program paths. 16208 */ 16209 if (!off_is_imm) 16210 env->explore_alu_limits = true; 16211 } 16212 16213 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 16214 if (err < 0) 16215 return err; 16216 do_sim: 16217 /* If we're in commit phase, we're done here given we already 16218 * pushed the truncated dst_reg into the speculative verification 16219 * stack. 16220 * 16221 * Also, when register is a known constant, we rewrite register-based 16222 * operation to immediate-based, and thus do not need masking (and as 16223 * a consequence, do not need to simulate the zero-truncation either). 16224 */ 16225 if (commit_window || off_is_imm) 16226 return 0; 16227 16228 /* Simulate and find potential out-of-bounds access under 16229 * speculative execution from truncation as a result of 16230 * masking when off was not within expected range. If off 16231 * sits in dst, then we temporarily need to move ptr there 16232 * to simulate dst (== 0) +/-= ptr. Needed, for example, 16233 * for cases where we use K-based arithmetic in one direction 16234 * and truncated reg-based in the other in order to explore 16235 * bad access. 16236 */ 16237 if (!ptr_is_dst_reg) { 16238 tmp = *dst_reg; 16239 *dst_reg = *ptr_reg; 16240 } 16241 err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx); 16242 if (err < 0) 16243 return REASON_STACK; 16244 if (!ptr_is_dst_reg) 16245 *dst_reg = tmp; 16246 return 0; 16247 } 16248 16249 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 16250 { 16251 struct bpf_verifier_state *vstate = env->cur_state; 16252 16253 /* If we simulate paths under speculation, we don't update the 16254 * insn as 'seen' such that when we verify unreachable paths in 16255 * the non-speculative domain, sanitize_dead_code() can still 16256 * rewrite/sanitize them. 16257 */ 16258 if (!vstate->speculative) 16259 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 16260 } 16261 16262 static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason) 16263 { 16264 static const char *err = "pointer arithmetic with it prohibited for !root"; 16265 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 16266 u32 dst = insn->dst_reg, src = insn->src_reg; 16267 struct bpf_reg_state *regs = cur_regs(env); 16268 16269 switch (reason) { 16270 case REASON_BOUNDS: 16271 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 16272 regs[src].type == SCALAR_VALUE ? src : dst, err); 16273 break; 16274 case REASON_TYPE: 16275 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 16276 regs[src].type == SCALAR_VALUE ? dst : src, err); 16277 break; 16278 case REASON_PATHS: 16279 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 16280 dst, op, err); 16281 break; 16282 case REASON_LIMIT: 16283 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 16284 dst, op, err); 16285 break; 16286 case REASON_STACK: 16287 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 16288 dst, err); 16289 return -ENOMEM; 16290 default: 16291 verifier_bug(env, "unknown reason (%d)", reason); 16292 break; 16293 } 16294 16295 return -EACCES; 16296 } 16297 16298 /* check that stack access falls within stack limits and that 'reg' doesn't 16299 * have a variable offset. 16300 * 16301 * Variable offset is prohibited for unprivileged mode for simplicity since it 16302 * requires corresponding support in Spectre masking for stack ALU. See also 16303 * retrieve_ptr_limit(). 16304 */ 16305 static int check_stack_access_for_ptr_arithmetic( 16306 struct bpf_verifier_env *env, 16307 int regno, 16308 const struct bpf_reg_state *reg, 16309 int off) 16310 { 16311 if (!tnum_is_const(reg->var_off)) { 16312 char tn_buf[48]; 16313 16314 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 16315 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 16316 regno, tn_buf, off); 16317 return -EACCES; 16318 } 16319 16320 if (off >= 0 || off < -(int)env->stack_limit) { 16321 verbose(env, "R%d stack pointer arithmetic goes out of range, " 16322 "prohibited for !root; off=%d\n", regno, off); 16323 return -EACCES; 16324 } 16325 16326 return 0; 16327 } 16328 16329 static int sanitize_check_bounds(struct bpf_verifier_env *env, 16330 const struct bpf_insn *insn, 16331 struct bpf_reg_state *dst_reg) 16332 { 16333 u32 dst = insn->dst_reg; 16334 16335 /* For unprivileged we require that resulting offset must be in bounds 16336 * in order to be able to sanitize access later on. 16337 */ 16338 if (env->bypass_spec_v1) 16339 return 0; 16340 16341 switch (dst_reg->type) { 16342 case PTR_TO_STACK: 16343 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 16344 dst_reg->var_off.value)) 16345 return -EACCES; 16346 break; 16347 case PTR_TO_MAP_VALUE: 16348 if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) { 16349 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 16350 "prohibited for !root\n", dst); 16351 return -EACCES; 16352 } 16353 break; 16354 default: 16355 return -EOPNOTSUPP; 16356 } 16357 16358 return 0; 16359 } 16360 16361 static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) 16362 { 16363 if (tnum_is_const(src_reg->var_off)) 16364 scalar_step_scale(dst_reg, src_reg->var_off.value); 16365 else 16366 reg_step_reset(dst_reg); 16367 } 16368 16369 static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) 16370 { 16371 u64 amount = src_reg->var_off.value; 16372 16373 if (tnum_is_const(src_reg->var_off) && amount < 64) 16374 scalar_step_scale(dst_reg, 1ULL << amount); 16375 else 16376 reg_step_reset(dst_reg); 16377 } 16378 16379 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 16380 * Caller should also handle BPF_MOV case separately. 16381 * If we return -EACCES, caller may want to try again treating pointer as a 16382 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 16383 */ 16384 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, 16385 u32 ptr_regno, const struct bpf_reg_state *ptr_reg, 16386 const struct bpf_reg_state *off_reg) 16387 { 16388 struct bpf_verifier_state *vstate = env->cur_state; 16389 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 16390 struct bpf_reg_state *regs = state->regs, *dst_reg; 16391 bool known = tnum_is_const(off_reg->var_off); 16392 s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg); 16393 u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg); 16394 struct bpf_sanitize_info info = {}; 16395 u8 opcode = BPF_OP(insn->code); 16396 u32 dst = insn->dst_reg; 16397 const char *reason; 16398 int ret, bounds_ret; 16399 bool to_scalar; 16400 16401 dst_reg = ®s[dst]; 16402 /* 16403 * Only += and -= keep a pointer. Any other op makes a number of it, 16404 * which is fine when the program may see values of pointers anyway. 16405 */ 16406 to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; 16407 16408 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 16409 smin_val > smax_val || umin_val > umax_val) { 16410 /* Taint dst register if offset had invalid bounds derived from 16411 * e.g. dead branches. 16412 */ 16413 __mark_reg_unknown(env, dst_reg); 16414 return 0; 16415 } 16416 16417 if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { 16418 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 16419 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 16420 __mark_reg_unknown(env, dst_reg); 16421 return 0; 16422 } 16423 16424 verbose(env, 16425 "R%d 32-bit pointer arithmetic prohibited\n", 16426 dst); 16427 reason = bpf_diag_fmt( 16428 env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.", 16429 ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); 16430 bpf_diag_register_type( 16431 env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason, 16432 "Use a 64-bit ALU instruction with an allowed, bounded scalar offset."); 16433 return -EACCES; 16434 } 16435 16436 if (ptr_reg->type & PTR_MAYBE_NULL) { 16437 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 16438 dst, reg_type_str(env, ptr_reg->type)); 16439 reason = bpf_diag_fmt( 16440 env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.", 16441 ptr_regno, reg_type_str(env, ptr_reg->type)); 16442 bpf_diag_register_type( 16443 env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason, 16444 "Make sure that a NULL check precedes any arithmetic performed on the pointer."); 16445 return -EACCES; 16446 } 16447 16448 switch (base_type(ptr_reg->type)) { 16449 case PTR_TO_CTX: 16450 case PTR_TO_MAP_VALUE: 16451 case PTR_TO_MAP_KEY: 16452 case PTR_TO_STACK: 16453 case PTR_TO_PACKET_META: 16454 case PTR_TO_PACKET: 16455 case PTR_TO_TP_BUFFER: 16456 case PTR_TO_BTF_ID: 16457 case PTR_TO_MEM: 16458 case PTR_TO_BUF: 16459 case PTR_TO_FUNC: 16460 case CONST_PTR_TO_DYNPTR: 16461 break; 16462 case PTR_TO_FLOW_KEYS: 16463 if (known) 16464 break; 16465 fallthrough; 16466 case CONST_PTR_TO_MAP: 16467 /* smin_val represents the known value */ 16468 if (known && smin_val == 0 && opcode == BPF_ADD) 16469 break; 16470 fallthrough; 16471 default: 16472 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 16473 dst, reg_type_str(env, ptr_reg->type)); 16474 reason = bpf_diag_fmt( 16475 env, "R%d holds %s. This pointer kind does not allow offset arithmetic.", 16476 ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); 16477 bpf_diag_register_type( 16478 env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason, 16479 "Do not change this pointer's offset; use it only in operations accepted for its kind."); 16480 return -EACCES; 16481 } 16482 16483 if (to_scalar) { 16484 __mark_reg_unknown(env, dst_reg); 16485 return 0; 16486 } 16487 16488 /* For 'scalar += pointer', dst_reg inherits the complete pointer 16489 * register state. Individual fields may be adjusted later by pointer 16490 * arithmetic. Callers guarantee that below does not overwrite off_reg. 16491 */ 16492 if (dst_reg != ptr_reg) 16493 *dst_reg = *ptr_reg; 16494 16495 /* 16496 * Accesses to untrusted PTR_TO_MEM are done through probe 16497 * instructions, hence no need to track offsets. 16498 */ 16499 if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) 16500 return 0; 16501 16502 if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) { 16503 reason = bpf_diag_fmt( 16504 env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].", 16505 ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF); 16506 bpf_diag_register_type( 16507 env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, 16508 "Clamp or bounds-check the scalar offset before applying it to the pointer."); 16509 return -EINVAL; 16510 } 16511 if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) { 16512 reason = bpf_diag_fmt( 16513 env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.", 16514 ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, 16515 bpf_diag_reg_type_plain(env, ptr_reg->type)); 16516 bpf_diag_register_type( 16517 env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, 16518 "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic."); 16519 return -EINVAL; 16520 } 16521 16522 if (sanitize_needed(opcode)) { 16523 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 16524 &info, false); 16525 if (ret < 0) 16526 return sanitize_err(env, insn, ret); 16527 } 16528 16529 /* 16530 * Pointer types do not carry 32-bit bounds at the moment. Blank r32 16531 * only after sanitize_ptr_alu() may have snapshotted dst_reg into a 16532 * speculative path: otherwise reg_bounds_sanity_check() might hit some 16533 * constraints violations. 16534 */ 16535 __mark_reg32_unbounded(dst_reg); 16536 16537 switch (opcode) { 16538 case BPF_ADD: 16539 /* 16540 * dst_reg gets the pointer type and since some positive 16541 * integer value was added to the pointer, give it a new 'id' 16542 * if it's a PTR_TO_PACKET. 16543 * this creates a new 'base' pointer, off_reg (variable) gets 16544 * added into the variable offset, and we copy the fixed offset 16545 * from ptr_reg. 16546 */ 16547 scalar_step_add(dst_reg, ptr_reg, off_reg); 16548 dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); 16549 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 16550 dst_reg->raw = ptr_reg->raw; 16551 if (reg_is_pkt_pointer(ptr_reg)) { 16552 if (!known) 16553 dst_reg->id = ++env->id_gen; 16554 /* 16555 * Clear range for unknown addends since we can't know 16556 * where the pkt pointer ended up. Also clear AT_PKT_END / 16557 * BEYOND_PKT_END from prior comparison as any pointer 16558 * arithmetic invalidates them. 16559 */ 16560 if (!known || dst_reg->range < 0) 16561 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 16562 } 16563 break; 16564 case BPF_SUB: 16565 if (dst_reg != ptr_reg) { 16566 /* scalar -= pointer. Creates an unknown scalar */ 16567 verbose(env, "R%d tried to subtract pointer from scalar\n", 16568 dst); 16569 reason = bpf_diag_fmt( 16570 env, "This operation subtracts pointer register R%d from scalar register R%d. " 16571 "The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.", 16572 ptr_regno, dst); 16573 bpf_diag_register_type( 16574 env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason, 16575 "Keep the pointer as the base; only add or subtract bounded scalars when permitted."); 16576 return -EACCES; 16577 } 16578 /* We don't allow subtraction from FP, because (according to 16579 * test_verifier.c test "invalid fp arithmetic", JITs might not 16580 * be able to deal with it. 16581 */ 16582 if (ptr_reg->type == PTR_TO_STACK) { 16583 verbose(env, "R%d subtraction from stack pointer prohibited\n", 16584 dst); 16585 reason = bpf_diag_fmt( 16586 env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.", 16587 ptr_regno); 16588 bpf_diag_register_type( 16589 env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason, 16590 "Use addition from R10 to form stack addresses within the tracked stack frame."); 16591 return -EACCES; 16592 } 16593 dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); 16594 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 16595 dst_reg->raw = ptr_reg->raw; 16596 if (reg_is_pkt_pointer(ptr_reg)) { 16597 if (!known) 16598 dst_reg->id = ++env->id_gen; 16599 /* 16600 * Clear range if the subtrahend may be negative since 16601 * pkt pointer could move past its bounds. A positive 16602 * subtrahend moves it backwards keeping positive range 16603 * intact. Also clear AT_PKT_END / BEYOND_PKT_END from 16604 * prior comparison as arithmetic invalidates them. 16605 */ 16606 if ((!known && smin_val < 0) || dst_reg->range < 0) 16607 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 16608 } 16609 reg_step_reset(dst_reg); 16610 break; 16611 case BPF_AND: 16612 case BPF_OR: 16613 case BPF_XOR: 16614 /* bitwise ops on pointers are troublesome, prohibit. */ 16615 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 16616 dst, bpf_alu_string[opcode >> 4]); 16617 reason = bpf_diag_fmt( 16618 env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.", 16619 ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), 16620 bpf_alu_string[opcode >> 4]); 16621 bpf_diag_register_type( 16622 env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason, 16623 "Do bitwise operations on scalar values, not on pointer-valued registers."); 16624 return -EACCES; 16625 default: 16626 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 16627 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 16628 dst, bpf_alu_string[opcode >> 4]); 16629 reason = bpf_diag_fmt( 16630 env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.", 16631 ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), 16632 bpf_alu_string[opcode >> 4]); 16633 bpf_diag_register_type( 16634 env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason, 16635 "Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value."); 16636 return -EACCES; 16637 } 16638 16639 if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) { 16640 reason = bpf_diag_fmt( 16641 env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.", 16642 dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, 16643 bpf_diag_reg_type_plain(env, ptr_reg->type)); 16644 bpf_diag_register_type( 16645 env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, 16646 "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); 16647 return -EINVAL; 16648 } 16649 /* 16650 * A packet pointer that keeps its id or range is checked against a 16651 * range set from the checked pointer's umax, so var_off must not tighten 16652 * its umax. r32 must still match var_off for reg_bounds_sanity_check(). 16653 */ 16654 if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0)) 16655 __update_reg32_bounds(dst_reg); 16656 else 16657 reg_bounds_sync(dst_reg); 16658 bounds_ret = sanitize_check_bounds(env, insn, dst_reg); 16659 if (bounds_ret == -EACCES) 16660 return bounds_ret; 16661 if (sanitize_needed(opcode)) { 16662 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 16663 &info, true); 16664 if (verifier_bug_if(!can_skip_alu_sanitation(env, insn) 16665 && !env->cur_state->speculative 16666 && bounds_ret 16667 && !ret, 16668 env, "Pointer type unsupported by sanitize_check_bounds() not rejected by retrieve_ptr_limit() as required")) { 16669 return -EFAULT; 16670 } 16671 if (ret < 0) 16672 return sanitize_err(env, insn, ret); 16673 } 16674 16675 return 0; 16676 } 16677 16678 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 16679 struct bpf_reg_state *src_reg) 16680 { 16681 dst_reg->r32 = cnum32_add(dst_reg->r32, src_reg->r32); 16682 } 16683 16684 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 16685 struct bpf_reg_state *src_reg) 16686 { 16687 dst_reg->r64 = cnum64_add(dst_reg->r64, src_reg->r64); 16688 } 16689 16690 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 16691 struct bpf_reg_state *src_reg) 16692 { 16693 dst_reg->r32 = cnum32_add(dst_reg->r32, cnum32_negate(src_reg->r32)); 16694 } 16695 16696 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 16697 struct bpf_reg_state *src_reg) 16698 { 16699 dst_reg->r64 = cnum64_add(dst_reg->r64, cnum64_negate(src_reg->r64)); 16700 } 16701 16702 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 16703 struct bpf_reg_state *src_reg) 16704 { 16705 s32 smin = reg_s32_min(dst_reg); 16706 s32 smax = reg_s32_max(dst_reg); 16707 u32 umin = reg_u32_min(dst_reg); 16708 u32 umax = reg_u32_max(dst_reg); 16709 s32 tmp_prod[4]; 16710 16711 if (check_mul_overflow(umax, reg_u32_max(src_reg), &umax) || 16712 check_mul_overflow(umin, reg_u32_min(src_reg), &umin)) { 16713 /* Overflow possible, we know nothing */ 16714 umin = 0; 16715 umax = U32_MAX; 16716 } 16717 if (check_mul_overflow(smin, reg_s32_min(src_reg), &tmp_prod[0]) || 16718 check_mul_overflow(smin, reg_s32_max(src_reg), &tmp_prod[1]) || 16719 check_mul_overflow(smax, reg_s32_min(src_reg), &tmp_prod[2]) || 16720 check_mul_overflow(smax, reg_s32_max(src_reg), &tmp_prod[3])) { 16721 /* Overflow possible, we know nothing */ 16722 smin = S32_MIN; 16723 smax = S32_MAX; 16724 } else { 16725 smin = min_array(tmp_prod, 4); 16726 smax = max_array(tmp_prod, 4); 16727 } 16728 16729 dst_reg->r32 = cnum32_intersect(cnum32_from_urange(umin, umax), 16730 cnum32_from_srange(smin, smax)); 16731 } 16732 16733 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 16734 struct bpf_reg_state *src_reg) 16735 { 16736 s64 smin = reg_smin(dst_reg); 16737 s64 smax = reg_smax(dst_reg); 16738 u64 umin = reg_umin(dst_reg); 16739 u64 umax = reg_umax(dst_reg); 16740 s64 tmp_prod[4]; 16741 16742 if (check_mul_overflow(umax, reg_umax(src_reg), &umax) || 16743 check_mul_overflow(umin, reg_umin(src_reg), &umin)) { 16744 /* Overflow possible, we know nothing */ 16745 umin = 0; 16746 umax = U64_MAX; 16747 } 16748 if (check_mul_overflow(smin, reg_smin(src_reg), &tmp_prod[0]) || 16749 check_mul_overflow(smin, reg_smax(src_reg), &tmp_prod[1]) || 16750 check_mul_overflow(smax, reg_smin(src_reg), &tmp_prod[2]) || 16751 check_mul_overflow(smax, reg_smax(src_reg), &tmp_prod[3])) { 16752 /* Overflow possible, we know nothing */ 16753 smin = S64_MIN; 16754 smax = S64_MAX; 16755 } else { 16756 smin = min_array(tmp_prod, 4); 16757 smax = max_array(tmp_prod, 4); 16758 } 16759 16760 dst_reg->r64 = cnum64_intersect(cnum64_from_urange(umin, umax), 16761 cnum64_from_srange(smin, smax)); 16762 } 16763 16764 static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg, 16765 struct bpf_reg_state *src_reg) 16766 { 16767 u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */ 16768 16769 reg_set_urange32(dst_reg, reg_u32_min(dst_reg) / src_val, 16770 reg_u32_max(dst_reg) / src_val); 16771 16772 /* Reset other ranges/tnum to unbounded/unknown. */ 16773 reset_reg64_and_tnum(dst_reg); 16774 } 16775 16776 static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg, 16777 struct bpf_reg_state *src_reg) 16778 { 16779 u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */ 16780 16781 reg_set_urange64(dst_reg, div64_u64(reg_umin(dst_reg), src_val), 16782 div64_u64(reg_umax(dst_reg), src_val)); 16783 16784 /* Reset other ranges/tnum to unbounded/unknown. */ 16785 reset_reg32_and_tnum(dst_reg); 16786 } 16787 16788 static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg, 16789 struct bpf_reg_state *src_reg) 16790 { 16791 s32 smin = reg_s32_min(dst_reg); 16792 s32 smax = reg_s32_max(dst_reg); 16793 s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */ 16794 s32 res1, res2; 16795 16796 /* BPF div specification: S32_MIN / -1 = S32_MIN */ 16797 if (smin == S32_MIN && src_val == -1) { 16798 /* 16799 * If the dividend range contains more than just S32_MIN, 16800 * we cannot precisely track the result, so it becomes unbounded. 16801 * e.g., [S32_MIN, S32_MIN+10]/(-1), 16802 * = {S32_MIN} U [-(S32_MIN+10), -(S32_MIN+1)] 16803 * = {S32_MIN} U [S32_MAX-9, S32_MAX] = [S32_MIN, S32_MAX] 16804 * Otherwise (if dividend is exactly S32_MIN), result remains S32_MIN. 16805 */ 16806 if (smax != S32_MIN) { 16807 smin = S32_MIN; 16808 smax = S32_MAX; 16809 } 16810 goto reset; 16811 } 16812 16813 res1 = smin / src_val; 16814 res2 = smax / src_val; 16815 smin = min(res1, res2); 16816 smax = max(res1, res2); 16817 16818 reset: 16819 reg_set_srange32(dst_reg, smin, smax); 16820 /* Reset other ranges/tnum to unbounded/unknown. */ 16821 reset_reg64_and_tnum(dst_reg); 16822 } 16823 16824 static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg, 16825 struct bpf_reg_state *src_reg) 16826 { 16827 s64 smin = reg_smin(dst_reg); 16828 s64 smax = reg_smax(dst_reg); 16829 s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */ 16830 s64 res1, res2; 16831 16832 /* BPF div specification: S64_MIN / -1 = S64_MIN */ 16833 if (smin == S64_MIN && src_val == -1) { 16834 /* 16835 * If the dividend range contains more than just S64_MIN, 16836 * we cannot precisely track the result, so it becomes unbounded. 16837 * e.g., [S64_MIN, S64_MIN+10]/(-1), 16838 * = {S64_MIN} U [-(S64_MIN+10), -(S64_MIN+1)] 16839 * = {S64_MIN} U [S64_MAX-9, S64_MAX] = [S64_MIN, S64_MAX] 16840 * Otherwise (if dividend is exactly S64_MIN), result remains S64_MIN. 16841 */ 16842 if (smax != S64_MIN) { 16843 smin = S64_MIN; 16844 smax = S64_MAX; 16845 } 16846 goto reset; 16847 } 16848 16849 res1 = div64_s64(smin, src_val); 16850 res2 = div64_s64(smax, src_val); 16851 smin = min(res1, res2); 16852 smax = max(res1, res2); 16853 16854 reset: 16855 reg_set_srange64(dst_reg, smin, smax); 16856 /* Reset other ranges/tnum to unbounded/unknown. */ 16857 reset_reg32_and_tnum(dst_reg); 16858 } 16859 16860 static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg, 16861 struct bpf_reg_state *src_reg) 16862 { 16863 u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */ 16864 u32 res_max = src_val - 1; 16865 16866 /* 16867 * If dst_umax <= res_max, the result remains unchanged. 16868 * e.g., [2, 5] % 10 = [2, 5]. 16869 */ 16870 if (reg_u32_max(dst_reg) <= res_max) 16871 return; 16872 16873 reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max)); 16874 16875 /* Reset other ranges/tnum to unbounded/unknown. */ 16876 reset_reg64_and_tnum(dst_reg); 16877 } 16878 16879 static void scalar_min_max_umod(struct bpf_reg_state *dst_reg, 16880 struct bpf_reg_state *src_reg) 16881 { 16882 u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */ 16883 u64 res_max = src_val - 1; 16884 16885 /* 16886 * If dst_umax <= res_max, the result remains unchanged. 16887 * e.g., [2, 5] % 10 = [2, 5]. 16888 */ 16889 if (reg_umax(dst_reg) <= res_max) 16890 return; 16891 16892 reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max)); 16893 16894 /* Reset other ranges/tnum to unbounded/unknown. */ 16895 reset_reg32_and_tnum(dst_reg); 16896 } 16897 16898 static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg, 16899 struct bpf_reg_state *src_reg) 16900 { 16901 s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */ 16902 16903 /* 16904 * Safe absolute value calculation: 16905 * If src_val == S32_MIN (-2147483648), src_abs becomes 2147483648. 16906 * Here use unsigned integer to avoid overflow. 16907 */ 16908 u32 src_abs = (src_val > 0) ? (u32)src_val : -(u32)src_val; 16909 16910 /* 16911 * Calculate the maximum possible absolute value of the result. 16912 * Even if src_abs is 2147483648 (S32_MIN), subtracting 1 gives 16913 * 2147483647 (S32_MAX), which fits perfectly in s32. 16914 */ 16915 s32 res_max_abs = src_abs - 1; 16916 16917 /* 16918 * If the dividend is already within the result range, 16919 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. 16920 */ 16921 if (reg_s32_min(dst_reg) >= -res_max_abs && reg_s32_max(dst_reg) <= res_max_abs) 16922 return; 16923 16924 /* General case: result has the same sign as the dividend. */ 16925 if (reg_s32_min(dst_reg) >= 0) { 16926 reg_set_srange32(dst_reg, 0, min(reg_s32_max(dst_reg), res_max_abs)); 16927 } else if (reg_s32_max(dst_reg) <= 0) { 16928 reg_set_srange32(dst_reg, max(reg_s32_min(dst_reg), -res_max_abs), 0); 16929 } else { 16930 reg_set_srange32(dst_reg, -res_max_abs, res_max_abs); 16931 } 16932 16933 /* Reset other ranges/tnum to unbounded/unknown. */ 16934 reset_reg64_and_tnum(dst_reg); 16935 } 16936 16937 static void scalar_min_max_smod(struct bpf_reg_state *dst_reg, 16938 struct bpf_reg_state *src_reg) 16939 { 16940 s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */ 16941 16942 /* 16943 * Safe absolute value calculation: 16944 * If src_val == S64_MIN (-2^63), src_abs becomes 2^63. 16945 * Here use unsigned integer to avoid overflow. 16946 */ 16947 u64 src_abs = (src_val > 0) ? (u64)src_val : -(u64)src_val; 16948 16949 /* 16950 * Calculate the maximum possible absolute value of the result. 16951 * Even if src_abs is 2^63 (S64_MIN), subtracting 1 gives 16952 * 2^63 - 1 (S64_MAX), which fits perfectly in s64. 16953 */ 16954 s64 res_max_abs = src_abs - 1; 16955 16956 /* 16957 * If the dividend is already within the result range, 16958 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5]. 16959 */ 16960 if (reg_smin(dst_reg) >= -res_max_abs && reg_smax(dst_reg) <= res_max_abs) 16961 return; 16962 16963 /* General case: result has the same sign as the dividend. */ 16964 if (reg_smin(dst_reg) >= 0) { 16965 reg_set_srange64(dst_reg, 0, min(reg_smax(dst_reg), res_max_abs)); 16966 } else if (reg_smax(dst_reg) <= 0) { 16967 reg_set_srange64(dst_reg, max(reg_smin(dst_reg), -res_max_abs), 0); 16968 } else { 16969 reg_set_srange64(dst_reg, -res_max_abs, res_max_abs); 16970 } 16971 16972 /* Reset other ranges/tnum to unbounded/unknown. */ 16973 reset_reg32_and_tnum(dst_reg); 16974 } 16975 16976 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 16977 struct bpf_reg_state *src_reg) 16978 { 16979 bool src_known = tnum_subreg_is_const(src_reg->var_off); 16980 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 16981 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 16982 u32 umax_val = reg_u32_max(src_reg); 16983 16984 if (src_known && dst_known) { 16985 __mark_reg32_known(dst_reg, var32_off.value); 16986 return; 16987 } 16988 16989 /* We get our minimum from the var_off, since that's inherently 16990 * bitwise. Our maximum is the minimum of the operands' maxima. 16991 */ 16992 reg_set_urange32(dst_reg, 16993 var32_off.value, 16994 min(reg_u32_max(dst_reg), umax_val)); 16995 } 16996 16997 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 16998 struct bpf_reg_state *src_reg) 16999 { 17000 bool src_known = tnum_is_const(src_reg->var_off); 17001 bool dst_known = tnum_is_const(dst_reg->var_off); 17002 u64 umax_val = reg_umax(src_reg); 17003 17004 if (src_known && dst_known) { 17005 __mark_reg_known(dst_reg, dst_reg->var_off.value); 17006 return; 17007 } 17008 17009 /* We get our minimum from the var_off, since that's inherently 17010 * bitwise. Our maximum is the minimum of the operands' maxima. 17011 */ 17012 reg_set_urange64(dst_reg, 17013 dst_reg->var_off.value, 17014 min(reg_umax(dst_reg), umax_val)); 17015 17016 /* We may learn something more from the var_off */ 17017 __update_reg_bounds(dst_reg); 17018 } 17019 17020 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 17021 struct bpf_reg_state *src_reg) 17022 { 17023 bool src_known = tnum_subreg_is_const(src_reg->var_off); 17024 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 17025 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 17026 u32 umin_val = reg_u32_min(src_reg); 17027 17028 if (src_known && dst_known) { 17029 __mark_reg32_known(dst_reg, var32_off.value); 17030 return; 17031 } 17032 17033 /* We get our maximum from the var_off, and our minimum is the 17034 * maximum of the operands' minima 17035 */ 17036 reg_set_urange32(dst_reg, 17037 max(reg_u32_min(dst_reg), umin_val), 17038 var32_off.value | var32_off.mask); 17039 } 17040 17041 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 17042 struct bpf_reg_state *src_reg) 17043 { 17044 bool src_known = tnum_is_const(src_reg->var_off); 17045 bool dst_known = tnum_is_const(dst_reg->var_off); 17046 u64 umin_val = reg_umin(src_reg); 17047 17048 if (src_known && dst_known) { 17049 __mark_reg_known(dst_reg, dst_reg->var_off.value); 17050 return; 17051 } 17052 17053 /* We get our maximum from the var_off, and our minimum is the 17054 * maximum of the operands' minima 17055 */ 17056 reg_set_urange64(dst_reg, 17057 max(reg_umin(dst_reg), umin_val), 17058 dst_reg->var_off.value | dst_reg->var_off.mask); 17059 17060 /* We may learn something more from the var_off */ 17061 __update_reg_bounds(dst_reg); 17062 } 17063 17064 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 17065 struct bpf_reg_state *src_reg) 17066 { 17067 bool src_known = tnum_subreg_is_const(src_reg->var_off); 17068 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 17069 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 17070 17071 if (src_known && dst_known) { 17072 __mark_reg32_known(dst_reg, var32_off.value); 17073 return; 17074 } 17075 17076 /* We get both minimum and maximum from the var32_off. */ 17077 reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask); 17078 } 17079 17080 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 17081 struct bpf_reg_state *src_reg) 17082 { 17083 bool src_known = tnum_is_const(src_reg->var_off); 17084 bool dst_known = tnum_is_const(dst_reg->var_off); 17085 17086 if (src_known && dst_known) { 17087 /* dst_reg->var_off.value has been updated earlier */ 17088 __mark_reg_known(dst_reg, dst_reg->var_off.value); 17089 return; 17090 } 17091 17092 /* We get both minimum and maximum from the var_off. */ 17093 reg_set_urange64(dst_reg, 17094 dst_reg->var_off.value, 17095 dst_reg->var_off.value | dst_reg->var_off.mask); 17096 } 17097 17098 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 17099 u64 umin_val, u64 umax_val) 17100 { 17101 /* If we might shift our top bit out, then we know nothing */ 17102 if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val)) 17103 reg_set_urange32(dst_reg, 0, U32_MAX); 17104 else 17105 /* We lose all sign bit information (except what we can pick 17106 * up from var_off) 17107 */ 17108 reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val, 17109 reg_u32_max(dst_reg) << umax_val); 17110 } 17111 17112 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 17113 struct bpf_reg_state *src_reg) 17114 { 17115 u32 umax_val = reg_u32_max(src_reg); 17116 u32 umin_val = reg_u32_min(src_reg); 17117 /* u32 alu operation will zext upper bits */ 17118 struct tnum subreg = tnum_subreg(dst_reg->var_off); 17119 17120 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 17121 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 17122 /* Not required but being careful mark reg64 bounds as unknown so 17123 * that we are forced to pick them up from tnum and zext later and 17124 * if some path skips this step we are still safe. 17125 */ 17126 __mark_reg64_unbounded(dst_reg); 17127 __update_reg32_bounds(dst_reg); 17128 } 17129 17130 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 17131 u64 umin_val, u64 umax_val) 17132 { 17133 struct cnum64 u, s; 17134 17135 /* Special case <<32 because it is a common compiler pattern to sign 17136 * extend subreg by doing <<32 s>>32. smin/smax assignments are correct 17137 * because s32 bounds don't flip sign when shifting to the left by 17138 * 32bits. 17139 */ 17140 if (umin_val == 32 && umax_val == 32) 17141 s = cnum64_from_srange((s64)reg_s32_min(dst_reg) << 32, 17142 (s64)reg_s32_max(dst_reg) << 32); 17143 else 17144 s = CNUM64_UNBOUNDED; 17145 17146 /* If we might shift our top bit out, then we know nothing */ 17147 if (reg_umax(dst_reg) > 1ULL << (63 - umax_val)) 17148 u = CNUM64_UNBOUNDED; 17149 else 17150 u = cnum64_from_urange(reg_umin(dst_reg) << umin_val, 17151 reg_umax(dst_reg) << umax_val); 17152 17153 dst_reg->r64 = cnum64_intersect(u, s); 17154 } 17155 17156 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 17157 struct bpf_reg_state *src_reg) 17158 { 17159 u64 umax_val = reg_umax(src_reg); 17160 u64 umin_val = reg_umin(src_reg); 17161 17162 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 17163 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 17164 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 17165 17166 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 17167 /* We may learn something more from the var_off */ 17168 __update_reg_bounds(dst_reg); 17169 } 17170 17171 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 17172 struct bpf_reg_state *src_reg) 17173 { 17174 struct tnum subreg = tnum_subreg(dst_reg->var_off); 17175 u32 umax_val = reg_u32_max(src_reg); 17176 u32 umin_val = reg_u32_min(src_reg); 17177 17178 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 17179 * be negative, then either: 17180 * 1) src_reg might be zero, so the sign bit of the result is 17181 * unknown, so we lose our signed bounds 17182 * 2) it's known negative, thus the unsigned bounds capture the 17183 * signed bounds 17184 * 3) the signed bounds cross zero, so they tell us nothing 17185 * about the result 17186 * If the value in dst_reg is known nonnegative, then again the 17187 * unsigned bounds capture the signed bounds. 17188 * Thus, in all cases it suffices to blow away our signed bounds 17189 * and rely on inferring new ones from the unsigned bounds and 17190 * var_off of the result. 17191 */ 17192 17193 dst_reg->var_off = tnum_rshift(subreg, umin_val); 17194 reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val, 17195 reg_u32_max(dst_reg) >> umin_val); 17196 17197 __mark_reg64_unbounded(dst_reg); 17198 __update_reg32_bounds(dst_reg); 17199 } 17200 17201 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 17202 struct bpf_reg_state *src_reg) 17203 { 17204 u64 umax_val = reg_umax(src_reg); 17205 u64 umin_val = reg_umin(src_reg); 17206 17207 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 17208 * be negative, then either: 17209 * 1) src_reg might be zero, so the sign bit of the result is 17210 * unknown, so we lose our signed bounds 17211 * 2) it's known negative, thus the unsigned bounds capture the 17212 * signed bounds 17213 * 3) the signed bounds cross zero, so they tell us nothing 17214 * about the result 17215 * If the value in dst_reg is known nonnegative, then again the 17216 * unsigned bounds capture the signed bounds. 17217 * Thus, in all cases it suffices to blow away our signed bounds 17218 * and rely on inferring new ones from the unsigned bounds and 17219 * var_off of the result. 17220 */ 17221 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 17222 reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val, 17223 reg_umax(dst_reg) >> umin_val); 17224 17225 /* Its not easy to operate on alu32 bounds here because it depends 17226 * on bits being shifted in. Take easy way out and mark unbounded 17227 * so we can recalculate later from tnum. 17228 */ 17229 __mark_reg32_unbounded(dst_reg); 17230 __update_reg_bounds(dst_reg); 17231 } 17232 17233 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 17234 struct bpf_reg_state *src_reg) 17235 { 17236 u64 umin_val = reg_u32_min(src_reg); 17237 17238 /* Upon reaching here, src_known is true and 17239 * umax_val is equal to umin_val. 17240 * Blow away the dst_reg umin_value/umax_value and rely on 17241 * dst_reg var_off to refine the result. 17242 */ 17243 reg_set_srange32(dst_reg, 17244 (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val), 17245 (u32)(((s32)reg_s32_max(dst_reg)) >> umin_val)); 17246 17247 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 17248 17249 __mark_reg64_unbounded(dst_reg); 17250 __update_reg32_bounds(dst_reg); 17251 } 17252 17253 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 17254 struct bpf_reg_state *src_reg) 17255 { 17256 u64 umin_val = reg_umin(src_reg); 17257 17258 /* Upon reaching here, src_known is true and umax_val is equal 17259 * to umin_val. 17260 */ 17261 reg_set_srange64(dst_reg, reg_smin(dst_reg) >> umin_val, 17262 reg_smax(dst_reg) >> umin_val); 17263 17264 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 17265 17266 /* Its not easy to operate on alu32 bounds here because it depends 17267 * on bits being shifted in from upper 32-bits. Take easy way out 17268 * and mark unbounded so we can recalculate later from tnum. 17269 */ 17270 __mark_reg32_unbounded(dst_reg); 17271 __update_reg_bounds(dst_reg); 17272 } 17273 17274 static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *insn) 17275 { 17276 /* 17277 * Byte swap operation - update var_off using tnum_bswap. 17278 * Three cases: 17279 * 1. bswap(16|32|64): opcode=0xd7 (BPF_END | BPF_ALU64 | BPF_TO_LE) 17280 * unconditional swap 17281 * 2. to_le(16|32|64): opcode=0xd4 (BPF_END | BPF_ALU | BPF_TO_LE) 17282 * swap on big-endian, truncation or no-op on little-endian 17283 * 3. to_be(16|32|64): opcode=0xdc (BPF_END | BPF_ALU | BPF_TO_BE) 17284 * swap on little-endian, truncation or no-op on big-endian 17285 */ 17286 17287 bool alu64 = BPF_CLASS(insn->code) == BPF_ALU64; 17288 bool to_le = BPF_SRC(insn->code) == BPF_TO_LE; 17289 bool is_big_endian; 17290 #ifdef CONFIG_CPU_BIG_ENDIAN 17291 is_big_endian = true; 17292 #else 17293 is_big_endian = false; 17294 #endif 17295 /* Apply bswap if alu64 or switch between big-endian and little-endian machines */ 17296 bool need_bswap = alu64 || (to_le == is_big_endian); 17297 17298 /* 17299 * If the register is mutated, manually reset its scalar ID to break 17300 * any existing ties and avoid incorrect bounds propagation. 17301 */ 17302 if (need_bswap || insn->imm == 16 || insn->imm == 32) 17303 clear_scalar_id(dst_reg); 17304 17305 if (need_bswap) { 17306 if (insn->imm == 16) 17307 dst_reg->var_off = tnum_bswap16(dst_reg->var_off); 17308 else if (insn->imm == 32) 17309 dst_reg->var_off = tnum_bswap32(dst_reg->var_off); 17310 else if (insn->imm == 64) 17311 dst_reg->var_off = tnum_bswap64(dst_reg->var_off); 17312 /* 17313 * Byteswap scrambles the range, so we must reset bounds. 17314 * Bounds will be re-derived from the new tnum later. 17315 */ 17316 __mark_reg_unbounded(dst_reg); 17317 reg_step_reset(dst_reg); 17318 } 17319 /* For bswap16/32, truncate dst register to match the swapped size */ 17320 if (insn->imm == 16 || insn->imm == 32) 17321 coerce_reg_to_size(dst_reg, insn->imm / 8); 17322 } 17323 17324 static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, 17325 const struct bpf_reg_state *src_reg) 17326 { 17327 bool src_is_const = false; 17328 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 17329 17330 if (insn_bitness == 32) { 17331 if (tnum_subreg_is_const(src_reg->var_off) 17332 && reg_s32_min(src_reg) == reg_s32_max(src_reg) 17333 && reg_u32_min(src_reg) == reg_u32_max(src_reg)) 17334 src_is_const = true; 17335 } else { 17336 if (tnum_is_const(src_reg->var_off) 17337 && reg_smin(src_reg) == reg_smax(src_reg) 17338 && reg_umin(src_reg) == reg_umax(src_reg)) 17339 src_is_const = true; 17340 } 17341 17342 switch (BPF_OP(insn->code)) { 17343 case BPF_ADD: 17344 case BPF_SUB: 17345 case BPF_NEG: 17346 case BPF_AND: 17347 case BPF_XOR: 17348 case BPF_OR: 17349 case BPF_MUL: 17350 case BPF_END: 17351 return true; 17352 17353 /* 17354 * Division and modulo operators range is only safe to compute when the 17355 * divisor is a constant. 17356 */ 17357 case BPF_DIV: 17358 case BPF_MOD: 17359 return src_is_const; 17360 17361 /* Shift operators range is only computable if shift dimension operand 17362 * is a constant. Shifts greater than 31 or 63 are undefined. This 17363 * includes shifts by a negative number. 17364 */ 17365 case BPF_LSH: 17366 case BPF_RSH: 17367 case BPF_ARSH: 17368 return (src_is_const && reg_umax(src_reg) < insn_bitness); 17369 default: 17370 return false; 17371 } 17372 } 17373 17374 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, 17375 struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg); 17376 17377 static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, 17378 struct bpf_reg_state *dst_reg, const struct bpf_reg_state *src_reg) 17379 { 17380 struct bpf_verifier_state *branch; 17381 struct bpf_reg_state *reg; 17382 bool alu32; 17383 17384 if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) 17385 alu32 = false; 17386 else if (reg_s32_min(dst_reg) == -1 && reg_s32_max(dst_reg) == 0) 17387 alu32 = true; 17388 else 17389 return 0; 17390 17391 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); 17392 if (IS_ERR(branch)) 17393 return PTR_ERR(branch); 17394 17395 reg = &branch->frame[branch->curframe]->regs[insn->dst_reg]; 17396 if (alu32) { 17397 __mark_reg32_known(reg, 0); 17398 __mark_reg32_known(dst_reg, -1ull); 17399 } else { 17400 __mark_reg_known(reg, 0); 17401 __mark_reg_known(dst_reg, -1ull); 17402 } 17403 /* 17404 * The insn is done with in the branch too, and what gets to the next insn is a state 17405 * like any other. At this insn the branch is a part of the state that is looked at: 17406 * it must not be compared with it. 17407 */ 17408 clear_scalar_id(reg); 17409 return adjust_scalar_min_max_vals(env, insn, reg, *src_reg); 17410 } 17411 17412 /* WARNING: This function does calculations on 64-bit values, but the actual 17413 * execution may occur on 32-bit values. Therefore, things like bitshifts 17414 * need extra checks in the 32-bit case. 17415 */ 17416 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 17417 struct bpf_insn *insn, 17418 struct bpf_reg_state *dst_reg, 17419 struct bpf_reg_state src_reg) 17420 { 17421 u8 opcode = BPF_OP(insn->code); 17422 s16 off = insn->off; 17423 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 17424 int ret; 17425 17426 if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) { 17427 __mark_reg_unknown(env, dst_reg); 17428 return 0; 17429 } 17430 17431 if (sanitize_needed(opcode)) { 17432 ret = sanitize_val_alu(env, insn); 17433 if (ret < 0) 17434 return sanitize_err(env, insn, ret); 17435 } 17436 17437 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 17438 * There are two classes of instructions: The first class we track both 17439 * alu32 and alu64 sign/unsigned bounds independently this provides the 17440 * greatest amount of precision when alu operations are mixed with jmp32 17441 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 17442 * and BPF_OR. This is possible because these ops have fairly easy to 17443 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 17444 * See alu32 verifier tests for examples. The second class of 17445 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 17446 * with regards to tracking sign/unsigned bounds because the bits may 17447 * cross subreg boundaries in the alu64 case. When this happens we mark 17448 * the reg unbounded in the subreg bound space and use the resulting 17449 * tnum to calculate an approximation of the sign/unsigned bounds. 17450 */ 17451 switch (opcode) { 17452 case BPF_ADD: 17453 scalar_step_add(dst_reg, dst_reg, &src_reg); 17454 scalar32_min_max_add(dst_reg, &src_reg); 17455 scalar_min_max_add(dst_reg, &src_reg); 17456 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 17457 break; 17458 case BPF_SUB: 17459 scalar32_min_max_sub(dst_reg, &src_reg); 17460 scalar_min_max_sub(dst_reg, &src_reg); 17461 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 17462 reg_step_reset(dst_reg); 17463 break; 17464 case BPF_NEG: 17465 env->fake_reg[0] = *dst_reg; 17466 __mark_reg_known(dst_reg, 0); 17467 scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); 17468 scalar_min_max_sub(dst_reg, &env->fake_reg[0]); 17469 dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); 17470 reg_step_reset(dst_reg); 17471 break; 17472 case BPF_MUL: 17473 scalar_step_mul(dst_reg, &src_reg); 17474 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 17475 scalar32_min_max_mul(dst_reg, &src_reg); 17476 scalar_min_max_mul(dst_reg, &src_reg); 17477 break; 17478 case BPF_DIV: 17479 /* BPF div specification: x / 0 = 0 */ 17480 if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) { 17481 ___mark_reg_known(dst_reg, 0); 17482 break; 17483 } 17484 if (alu32) 17485 if (off == 1) 17486 scalar32_min_max_sdiv(dst_reg, &src_reg); 17487 else 17488 scalar32_min_max_udiv(dst_reg, &src_reg); 17489 else 17490 if (off == 1) 17491 scalar_min_max_sdiv(dst_reg, &src_reg); 17492 else 17493 scalar_min_max_udiv(dst_reg, &src_reg); 17494 reg_step_reset(dst_reg); 17495 break; 17496 case BPF_MOD: 17497 /* BPF mod specification: x % 0 = x */ 17498 if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) 17499 break; 17500 if (alu32) 17501 if (off == 1) 17502 scalar32_min_max_smod(dst_reg, &src_reg); 17503 else 17504 scalar32_min_max_umod(dst_reg, &src_reg); 17505 else 17506 if (off == 1) 17507 scalar_min_max_smod(dst_reg, &src_reg); 17508 else 17509 scalar_min_max_umod(dst_reg, &src_reg); 17510 reg_step_reset(dst_reg); 17511 break; 17512 case BPF_AND: 17513 if (tnum_is_const(src_reg.var_off)) { 17514 ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); 17515 if (ret) 17516 return ret; 17517 } 17518 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 17519 scalar32_min_max_and(dst_reg, &src_reg); 17520 scalar_min_max_and(dst_reg, &src_reg); 17521 reg_step_reset(dst_reg); 17522 break; 17523 case BPF_OR: 17524 if (tnum_is_const(src_reg.var_off)) { 17525 ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); 17526 if (ret) 17527 return ret; 17528 } 17529 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 17530 scalar32_min_max_or(dst_reg, &src_reg); 17531 scalar_min_max_or(dst_reg, &src_reg); 17532 reg_step_reset(dst_reg); 17533 break; 17534 case BPF_XOR: 17535 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 17536 scalar32_min_max_xor(dst_reg, &src_reg); 17537 scalar_min_max_xor(dst_reg, &src_reg); 17538 reg_step_reset(dst_reg); 17539 break; 17540 case BPF_LSH: 17541 scalar_step_lsh(dst_reg, &src_reg); 17542 if (alu32) 17543 scalar32_min_max_lsh(dst_reg, &src_reg); 17544 else 17545 scalar_min_max_lsh(dst_reg, &src_reg); 17546 break; 17547 case BPF_RSH: 17548 if (alu32) 17549 scalar32_min_max_rsh(dst_reg, &src_reg); 17550 else 17551 scalar_min_max_rsh(dst_reg, &src_reg); 17552 reg_step_reset(dst_reg); 17553 break; 17554 case BPF_ARSH: 17555 if (alu32) 17556 scalar32_min_max_arsh(dst_reg, &src_reg); 17557 else 17558 scalar_min_max_arsh(dst_reg, &src_reg); 17559 reg_step_reset(dst_reg); 17560 break; 17561 case BPF_END: 17562 scalar_byte_swap(dst_reg, insn); 17563 reg_step_reset(dst_reg); 17564 break; 17565 default: 17566 break; 17567 } 17568 17569 /* 17570 * ALU32 ops are zero extended into 64bit register. 17571 * 17572 * BPF_END is already handled inside the helper (truncation), 17573 * so skip zext here to avoid unexpected zero extension. 17574 * e.g., le64: opcode=(BPF_END|BPF_ALU|BPF_TO_LE), imm=0x40 17575 * This is a 64bit byte swap operation with alu32==true, 17576 * but we should not zero extend the result. 17577 */ 17578 if (alu32 && opcode != BPF_END) 17579 zext_32_to_64(dst_reg); 17580 reg_bounds_sync(dst_reg); 17581 return 0; 17582 } 17583 17584 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 17585 * and var_off. 17586 */ 17587 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 17588 struct bpf_insn *insn) 17589 { 17590 struct bpf_verifier_state *vstate = env->cur_state; 17591 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 17592 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 17593 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 17594 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 17595 struct bpf_insn_aux_data *aux = cur_aux(env); 17596 u8 opcode = BPF_OP(insn->code); 17597 int err; 17598 17599 dst_reg = ®s[insn->dst_reg]; 17600 if (BPF_SRC(insn->code) == BPF_X) 17601 src_reg = ®s[insn->src_reg]; 17602 else 17603 src_reg = NULL; 17604 17605 /* Case where at least one operand is an arena. */ 17606 if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { 17607 17608 if (dst_reg->type != PTR_TO_ARENA) 17609 *dst_reg = *src_reg; 17610 17611 if (BPF_CLASS(insn->code) == BPF_ALU64) { 17612 /* 17613 * Only arena pointers set needs_zext, but doing so 17614 * modifies the instruction at fixup time to an ALU32 17615 * and makes it unsuitable for 64-bit scalar args. We 17616 * prevent zext from being set if the instruction has 17617 * been previously called with non-arena registers. 17618 */ 17619 if (aux->prevent_zext) { 17620 verbose(env, "same insn cannot be used with and without arena pointer\n"); 17621 return -EINVAL; 17622 } 17623 17624 /* 17625 * 32-bit operations zero upper bits automatically. 17626 * 64-bit operations need to be converted to 32. 17627 */ 17628 aux->needs_zext = true; 17629 aux->zext_dst = true; 17630 } 17631 17632 /* Any arithmetic operations are allowed on arena pointers */ 17633 return 0; 17634 } 17635 17636 /* Prevent the instruction from being used with arena pointers (see above). */ 17637 if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) { 17638 if (aux->needs_zext) { 17639 verbose(env, "same insn cannot be used with and without arena pointer\n"); 17640 return -EINVAL; 17641 } 17642 17643 aux->prevent_zext = true; 17644 } 17645 17646 if (dst_reg->type != SCALAR_VALUE) 17647 ptr_reg = dst_reg; 17648 17649 if (BPF_SRC(insn->code) == BPF_X) { 17650 if (src_reg->type != SCALAR_VALUE) { 17651 if (dst_reg->type != SCALAR_VALUE) { 17652 /* Combining two pointers by any ALU op yields 17653 * an arbitrary scalar. Disallow all math except 17654 * pointer subtraction 17655 */ 17656 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 17657 mark_reg_unknown(env, regs, insn->dst_reg); 17658 return 0; 17659 } 17660 verbose(env, "R%d pointer %s pointer prohibited\n", 17661 insn->dst_reg, 17662 bpf_alu_string[opcode >> 4]); 17663 return -EACCES; 17664 } else { 17665 /* scalar += pointer 17666 * This is legal, but we have to reverse our 17667 * src/dest handling in computing the range 17668 */ 17669 err = mark_chain_precision(env, insn->dst_reg); 17670 if (err) 17671 return err; 17672 off_reg = *dst_reg; 17673 return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg, 17674 &off_reg); 17675 } 17676 } else if (ptr_reg) { 17677 /* pointer += scalar */ 17678 err = mark_chain_precision(env, insn->src_reg); 17679 if (err) 17680 return err; 17681 return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg); 17682 } else if (dst_reg->precise) { 17683 /* if dst_reg is precise, src_reg should be precise as well */ 17684 err = mark_chain_precision(env, insn->src_reg); 17685 if (err) 17686 return err; 17687 } 17688 } else { 17689 /* Pretend the src is a reg with a known value, since we only 17690 * need to be able to read from this state. 17691 */ 17692 off_reg.type = SCALAR_VALUE; 17693 __mark_reg_known(&off_reg, insn->imm); 17694 src_reg = &off_reg; 17695 if (ptr_reg) /* pointer += K */ 17696 return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg); 17697 } 17698 17699 /* Got here implies adding two SCALAR_VALUEs */ 17700 if (WARN_ON_ONCE(ptr_reg)) { 17701 print_verifier_state(env, vstate, vstate->curframe, true); 17702 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 17703 return -EFAULT; 17704 } 17705 if (WARN_ON(!src_reg)) { 17706 print_verifier_state(env, vstate, vstate->curframe, true); 17707 verbose(env, "verifier internal error: no src_reg\n"); 17708 return -EFAULT; 17709 } 17710 /* 17711 * For alu32 linked register tracking, we need to check dst_reg's 17712 * umax_value before the ALU operation. After adjust_scalar_min_max_vals(), 17713 * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX. 17714 */ 17715 u64 dst_umax = reg_umax(dst_reg); 17716 17717 err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 17718 if (err) 17719 return err; 17720 /* 17721 * Compilers can generate the code 17722 * r1 = r2 17723 * r1 += 0x1 17724 * if r2 < 1000 goto ... 17725 * use r1 in memory access 17726 * So remember constant delta between r2 and r1 and update r1 after 17727 * 'if' condition. 17728 */ 17729 if (env->bpf_capable && 17730 (BPF_OP(insn->code) == BPF_ADD || BPF_OP(insn->code) == BPF_SUB) && 17731 dst_reg->id && is_reg_const(src_reg, alu32) && 17732 !(BPF_SRC(insn->code) == BPF_X && insn->src_reg == insn->dst_reg)) { 17733 u64 val = reg_const_value(src_reg, alu32); 17734 s32 off; 17735 17736 if (!alu32 && ((s64)val < S32_MIN || (s64)val > S32_MAX)) 17737 goto clear_id; 17738 17739 if (alu32 && (dst_umax > U32_MAX)) 17740 goto clear_id; 17741 17742 off = (s32)val; 17743 17744 if (BPF_OP(insn->code) == BPF_SUB) { 17745 /* Negating S32_MIN would overflow */ 17746 if (off == S32_MIN) 17747 goto clear_id; 17748 off = -off; 17749 } 17750 17751 if (dst_reg->id & BPF_ADD_CONST) { 17752 /* 17753 * If the register already went through rX += val 17754 * we cannot accumulate another val into rx->off. 17755 */ 17756 clear_id: 17757 clear_scalar_id(dst_reg); 17758 } else { 17759 if (alu32) 17760 dst_reg->id |= BPF_ADD_CONST32; 17761 else 17762 dst_reg->id |= BPF_ADD_CONST64; 17763 dst_reg->delta = off; 17764 } 17765 } else { 17766 /* 17767 * Make sure ID is cleared otherwise dst_reg min/max could be 17768 * incorrectly propagated into other registers by sync_linked_regs() 17769 */ 17770 clear_scalar_id(dst_reg); 17771 } 17772 return 0; 17773 } 17774 17775 /* 17776 * Many checks done by this function are quite conservative. 17777 * This is because main verification pass does not maintain 17778 * enough information to track object identities for some 17779 * of the interesting types, e.g. PTR_TO_MEM. 17780 */ 17781 bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, 17782 const struct bpf_reg_state *reg_b) 17783 { 17784 if (reg_a == reg_b) 17785 return true; 17786 /* Require matching flags and base types. */ 17787 if (reg_a->type != reg_b->type) 17788 return false; 17789 /* NULL can't be compared to some base+offset pointer. */ 17790 if (type_may_be_null(reg_a->type)) 17791 return false; 17792 17793 switch (base_type(reg_a->type)) { 17794 case PTR_TO_STACK: 17795 return reg_a->frameno == reg_b->frameno; 17796 case PTR_TO_MAP_VALUE: 17797 if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid) 17798 return false; 17799 if (reg_a->id && reg_a->id == reg_b->id) 17800 return true; 17801 /* 17802 * A plain single-element array has one stable value address, 17803 * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal(). 17804 */ 17805 if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY && 17806 reg_a->map_ptr->max_entries == 1 && 17807 !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP)) 17808 return true; 17809 /* 17810 * The rules above can be simplified / relaxed if: 17811 * - fresh IDs would always be assigned for map-value lookups; 17812 * - direct map value loads would always have and ID of zero. 17813 */ 17814 return false; 17815 case PTR_TO_MEM: 17816 case PTR_TO_BUF: 17817 return reg_a->id && reg_a->id == reg_b->id; 17818 17819 default: 17820 return false; 17821 } 17822 } 17823 17824 /* 17825 * 'range - smax' bytes from a packet pointer are in the packet. reg_bounds_sync() can make smax 17826 * smaller than @smax: with a step of 8, or with 3 zero bits in var_off, 58 becomes 56. Make the 17827 * range smaller by the same number, so that the number of bytes does not grow. umax is not 17828 * sufficient: it is U64_MAX when the offset can be negative. 17829 */ 17830 void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax) 17831 { 17832 if (type_is_pkt_pointer(reg->type) && reg->range > 0) 17833 reg->range = max_t(s64, reg->range - (smax - reg_smax(reg)), 0); 17834 } 17835 17836 int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 17837 struct cnum64 r64, struct cnum32 r32, u16 base, u16 step) 17838 { 17839 reg->r64 = r64; 17840 reg->r32 = r32; 17841 reg->step = step; 17842 reg->base = base; 17843 reg->var_off = tnum_unknown; 17844 reg_bounds_sync(reg); /* this should infer the tnum alignment */ 17845 bpf_trim_pkt_range(reg, cnum64_smax(r64)); 17846 return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); 17847 } 17848 17849 /* 17850 * Checks if pointer and packet metadata matches between two registers. 17851 * For use with bpf_reg_union(). 17852 */ 17853 bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b) 17854 { 17855 if (a->type != b->type) 17856 return false; 17857 if (a->type == SCALAR_VALUE) 17858 return true; 17859 return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) && 17860 a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid; 17861 } 17862 17863 /* 17864 * Extends acc's scalar range to include src's scalar range. 17865 * Caller must ensure that other kinds of metadata is compatible. 17866 * Caller must clear acc's scalar ID. 17867 */ 17868 int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, 17869 const struct bpf_reg_state *src) 17870 { 17871 u16 base, step; 17872 17873 if (!bpf_reg_union_compatible(acc, src)) { 17874 verifier_bug(env, "union of incompatible registers"); 17875 return -EFAULT; 17876 } 17877 acc->r64 = cnum64_union(acc->r64, src->r64); 17878 acc->r32 = cnum32_union(acc->r32, src->r32); 17879 acc->var_off = tnum_union(acc->var_off, src->var_off); 17880 17881 /* Retain a common congruence if the bases agree modulo the gcd. */ 17882 step = gcd(acc->step, src->step); 17883 base = acc->base % step; 17884 if (base != src->base % step) { 17885 reg_step_reset(acc); 17886 } else { 17887 acc->base = base; 17888 acc->step = step; 17889 } 17890 reg_bounds_sync(acc); 17891 return reg_bounds_sanity_check(env, acc, "bpf_reg_union"); 17892 } 17893 17894 /* check validity of 32-bit and 64-bit arithmetic operations */ 17895 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 17896 { 17897 struct bpf_reg_state *regs = cur_regs(env); 17898 u8 opcode = BPF_OP(insn->code); 17899 int err; 17900 17901 bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); 17902 17903 if (opcode == BPF_END || opcode == BPF_NEG) { 17904 /* check src operand */ 17905 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 17906 if (err) 17907 return err; 17908 17909 if (is_pointer_value(env, insn->dst_reg)) { 17910 verbose(env, "R%d pointer arithmetic prohibited\n", 17911 insn->dst_reg); 17912 return -EACCES; 17913 } 17914 17915 /* check dest operand */ 17916 if (regs[insn->dst_reg].type == SCALAR_VALUE) { 17917 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 17918 err = err ?: adjust_scalar_min_max_vals(env, insn, 17919 ®s[insn->dst_reg], 17920 regs[insn->dst_reg]); 17921 } else { 17922 err = check_reg_arg(env, insn->dst_reg, DST_OP); 17923 } 17924 if (err) 17925 return err; 17926 17927 } else if (opcode == BPF_MOV) { 17928 17929 if (BPF_SRC(insn->code) == BPF_X) { 17930 if (insn->off == BPF_ADDR_SPACE_CAST) { 17931 if (!env->prog->aux->arena) { 17932 verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n"); 17933 return -EINVAL; 17934 } 17935 } 17936 17937 /* check src operand */ 17938 err = check_reg_arg(env, insn->src_reg, SRC_OP); 17939 if (err) 17940 return err; 17941 } 17942 17943 /* check dest operand, mark as required later */ 17944 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 17945 if (err) 17946 return err; 17947 17948 if (BPF_SRC(insn->code) == BPF_X) { 17949 struct bpf_reg_state *src_reg = regs + insn->src_reg; 17950 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 17951 17952 if (BPF_CLASS(insn->code) == BPF_ALU64) { 17953 if (insn->imm) { 17954 /* off == BPF_ADDR_SPACE_CAST */ 17955 mark_reg_unknown(env, regs, insn->dst_reg); 17956 if (insn->imm == 1) /* cast from as(1) to as(0) */ 17957 dst_reg->type = PTR_TO_ARENA; 17958 } else if (insn->off == 0) { 17959 /* case: R1 = R2 17960 * copy register state to dest reg 17961 */ 17962 assign_scalar_id_before_mov(env, src_reg); 17963 *dst_reg = *src_reg; 17964 } else { 17965 /* case: R1 = (s8, s16 s32)R2 */ 17966 if (is_pointer_value(env, insn->src_reg)) { 17967 verbose(env, 17968 "R%d sign-extension part of pointer\n", 17969 insn->src_reg); 17970 return -EACCES; 17971 } else if (src_reg->type == SCALAR_VALUE) { 17972 bool no_sext; 17973 17974 no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); 17975 if (no_sext) 17976 assign_scalar_id_before_mov(env, src_reg); 17977 *dst_reg = *src_reg; 17978 if (!no_sext) 17979 clear_scalar_id(dst_reg); 17980 coerce_reg_to_size_sx(dst_reg, insn->off >> 3); 17981 } else { 17982 mark_reg_unknown(env, regs, insn->dst_reg); 17983 } 17984 } 17985 } else { 17986 /* R1 = (u32) R2 */ 17987 if (is_pointer_value(env, insn->src_reg)) { 17988 verbose(env, 17989 "R%d partial copy of pointer\n", 17990 insn->src_reg); 17991 return -EACCES; 17992 } else if (src_reg->type == SCALAR_VALUE) { 17993 if (insn->off == 0) { 17994 bool is_src_reg_u32 = get_reg_width(src_reg) <= 32; 17995 17996 if (is_src_reg_u32) 17997 assign_scalar_id_before_mov(env, src_reg); 17998 *dst_reg = *src_reg; 17999 /* Make sure ID is cleared if src_reg is not in u32 18000 * range otherwise dst_reg min/max could be incorrectly 18001 * propagated into src_reg by sync_linked_regs() 18002 */ 18003 if (!is_src_reg_u32) 18004 clear_scalar_id(dst_reg); 18005 } else { 18006 /* case: W1 = (s8, s16)W2 */ 18007 bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); 18008 18009 if (no_sext) 18010 assign_scalar_id_before_mov(env, src_reg); 18011 *dst_reg = *src_reg; 18012 if (!no_sext) 18013 clear_scalar_id(dst_reg); 18014 coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); 18015 } 18016 } else { 18017 mark_reg_unknown(env, regs, 18018 insn->dst_reg); 18019 } 18020 zext_32_to_64(dst_reg); 18021 reg_bounds_sync(dst_reg); 18022 } 18023 } else { 18024 /* case: R = imm 18025 * remember the value we stored into this reg 18026 */ 18027 /* clear any state __mark_reg_known doesn't set */ 18028 mark_reg_unknown(env, regs, insn->dst_reg); 18029 regs[insn->dst_reg].type = SCALAR_VALUE; 18030 if (BPF_CLASS(insn->code) == BPF_ALU64) { 18031 __mark_reg_known(regs + insn->dst_reg, 18032 insn->imm); 18033 } else { 18034 __mark_reg_known(regs + insn->dst_reg, 18035 (u32)insn->imm); 18036 } 18037 } 18038 18039 } else { /* all other ALU ops: and, sub, xor, add, ... */ 18040 18041 if (BPF_SRC(insn->code) == BPF_X) { 18042 /* check src1 operand */ 18043 err = check_reg_arg(env, insn->src_reg, SRC_OP); 18044 if (err) 18045 return err; 18046 } 18047 18048 /* check src2 operand */ 18049 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 18050 if (err) 18051 return err; 18052 18053 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 18054 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 18055 verbose(env, "div by zero\n"); 18056 return -EINVAL; 18057 } 18058 18059 if ((opcode == BPF_LSH || opcode == BPF_RSH || 18060 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 18061 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 18062 18063 if (insn->imm < 0 || insn->imm >= size) { 18064 verbose(env, "invalid shift %d\n", insn->imm); 18065 return -EINVAL; 18066 } 18067 } 18068 18069 /* check dest operand */ 18070 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 18071 err = err ?: adjust_reg_min_max_vals(env, insn); 18072 if (err) 18073 return err; 18074 } 18075 18076 err = reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); 18077 if (err) 18078 return err; 18079 18080 bpf_diag_mod_end(env); 18081 return 0; 18082 } 18083 18084 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 18085 struct bpf_reg_state *dst_reg, 18086 enum bpf_reg_type type, 18087 bool range_right_open) 18088 { 18089 struct bpf_func_state *state; 18090 struct bpf_reg_state *reg; 18091 int new_range; 18092 18093 if (reg_umax(dst_reg) == 0 && range_right_open) 18094 /* This doesn't give us any range */ 18095 return; 18096 18097 if (reg_umax(dst_reg) > MAX_PACKET_OFF) 18098 /* Risk of overflow. For instance, ptr + (1<<63) may be less 18099 * than pkt_end, but that's because it's also less than pkt. 18100 */ 18101 return; 18102 18103 new_range = reg_umax(dst_reg); 18104 if (range_right_open) 18105 new_range++; 18106 18107 /* Examples for register markings: 18108 * 18109 * pkt_data in dst register: 18110 * 18111 * r2 = r3; 18112 * r2 += 8; 18113 * if (r2 > pkt_end) goto <handle exception> 18114 * <access okay> 18115 * 18116 * r2 = r3; 18117 * r2 += 8; 18118 * if (r2 < pkt_end) goto <access okay> 18119 * <handle exception> 18120 * 18121 * Where: 18122 * r2 == dst_reg, pkt_end == src_reg 18123 * r2=pkt(id=n,off=8,r=0) 18124 * r3=pkt(id=n,off=0,r=0) 18125 * 18126 * pkt_data in src register: 18127 * 18128 * r2 = r3; 18129 * r2 += 8; 18130 * if (pkt_end >= r2) goto <access okay> 18131 * <handle exception> 18132 * 18133 * r2 = r3; 18134 * r2 += 8; 18135 * if (pkt_end <= r2) goto <handle exception> 18136 * <access okay> 18137 * 18138 * Where: 18139 * pkt_end == dst_reg, r2 == src_reg 18140 * r2=pkt(id=n,off=8,r=0) 18141 * r3=pkt(id=n,off=0,r=0) 18142 * 18143 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 18144 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 18145 * and [r3, r3 + 8-1) respectively is safe to access depending on 18146 * the check. 18147 */ 18148 18149 /* If our ids match, then we must have the same max_value. And we 18150 * don't care about the other reg's fixed offset, since if it's too big 18151 * the range won't allow anything. 18152 * reg_umax(dst_reg) is known < MAX_PACKET_OFF, therefore it fits in a u16. 18153 */ 18154 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 18155 if (reg->type == type && reg->id == dst_reg->id) 18156 /* keep the maximum range already checked */ 18157 reg->range = max(reg->range, new_range); 18158 })); 18159 } 18160 18161 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, 18162 u8 opcode, bool is_jmp32); 18163 /* 18164 * Learn more information about live branches by simulating refinement on both branches. 18165 * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means 18166 * that branch is dead. 18167 */ 18168 static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32) 18169 { 18170 /* Fallthrough (FALSE) branch */ 18171 regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32); 18172 reg_bounds_sync(&env->false_reg1); 18173 reg_bounds_sync(&env->false_reg2); 18174 /* 18175 * If there is a range bounds violation in *any* of the abstract values in either 18176 * reg_states in the FALSE branch (i.e. reg1, reg2), the FALSE branch must be dead. Only 18177 * TRUE branch will be taken. 18178 */ 18179 if (range_bounds_violation(&env->false_reg1) || range_bounds_violation(&env->false_reg2)) 18180 return 1; 18181 18182 /* Jump (TRUE) branch */ 18183 regs_refine_cond_op(&env->true_reg1, &env->true_reg2, opcode, is_jmp32); 18184 reg_bounds_sync(&env->true_reg1); 18185 reg_bounds_sync(&env->true_reg2); 18186 /* 18187 * If there is a range bounds violation in *any* of the abstract values in either 18188 * reg_states in the TRUE branch (i.e. true_reg1, true_reg2), the TRUE branch must be dead. 18189 * Only FALSE branch will be taken. 18190 */ 18191 if (range_bounds_violation(&env->true_reg1) || range_bounds_violation(&env->true_reg2)) 18192 return 0; 18193 18194 /* Both branches are possible, we can't determine which one will be taken. */ 18195 return -1; 18196 } 18197 18198 /* 18199 * <reg1> <op> <reg2>, currently assuming reg2 is a constant 18200 */ 18201 static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1, 18202 struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32) 18203 { 18204 struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off; 18205 struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off; 18206 u64 umin1 = is_jmp32 ? (u64)reg_u32_min(reg1) : reg_umin(reg1); 18207 u64 umax1 = is_jmp32 ? (u64)reg_u32_max(reg1) : reg_umax(reg1); 18208 s64 smin1 = is_jmp32 ? (s64)reg_s32_min(reg1) : reg_smin(reg1); 18209 s64 smax1 = is_jmp32 ? (s64)reg_s32_max(reg1) : reg_smax(reg1); 18210 u64 umin2 = is_jmp32 ? (u64)reg_u32_min(reg2) : reg_umin(reg2); 18211 u64 umax2 = is_jmp32 ? (u64)reg_u32_max(reg2) : reg_umax(reg2); 18212 s64 smin2 = is_jmp32 ? (s64)reg_s32_min(reg2) : reg_smin(reg2); 18213 s64 smax2 = is_jmp32 ? (s64)reg_s32_max(reg2) : reg_smax(reg2); 18214 18215 if (reg1 == reg2) { 18216 switch (opcode) { 18217 case BPF_JGE: 18218 case BPF_JLE: 18219 case BPF_JSGE: 18220 case BPF_JSLE: 18221 case BPF_JEQ: 18222 return 1; 18223 case BPF_JGT: 18224 case BPF_JLT: 18225 case BPF_JSGT: 18226 case BPF_JSLT: 18227 case BPF_JNE: 18228 return 0; 18229 case BPF_JSET: 18230 if (tnum_is_const(t1)) 18231 return t1.value != 0; 18232 else 18233 return (smin1 <= 0 && smax1 >= 0) ? -1 : 1; 18234 default: 18235 return -1; 18236 } 18237 } 18238 18239 switch (opcode) { 18240 case BPF_JEQ: 18241 /* constants, umin/umax and smin/smax checks would be 18242 * redundant in this case because they all should match 18243 */ 18244 if (tnum_is_const(t1) && tnum_is_const(t2)) 18245 return t1.value == t2.value; 18246 if (!tnum_overlap(t1, t2)) 18247 return 0; 18248 /* non-overlapping ranges */ 18249 if (umin1 > umax2 || umax1 < umin2) 18250 return 0; 18251 if (smin1 > smax2 || smax1 < smin2) 18252 return 0; 18253 if (!is_jmp32) { 18254 /* if 64-bit ranges are inconclusive, see if we can 18255 * utilize 32-bit subrange knowledge to eliminate 18256 * branches that can't be taken a priori 18257 */ 18258 if (reg_u32_min(reg1) > reg_u32_max(reg2) || 18259 reg_u32_max(reg1) < reg_u32_min(reg2)) 18260 return 0; 18261 if (reg_s32_min(reg1) > reg_s32_max(reg2) || 18262 reg_s32_max(reg1) < reg_s32_min(reg2)) 18263 return 0; 18264 } 18265 break; 18266 case BPF_JNE: 18267 /* constants, umin/umax and smin/smax checks would be 18268 * redundant in this case because they all should match 18269 */ 18270 if (tnum_is_const(t1) && tnum_is_const(t2)) 18271 return t1.value != t2.value; 18272 if (!tnum_overlap(t1, t2)) 18273 return 1; 18274 /* non-overlapping ranges */ 18275 if (umin1 > umax2 || umax1 < umin2) 18276 return 1; 18277 if (smin1 > smax2 || smax1 < smin2) 18278 return 1; 18279 if (!is_jmp32) { 18280 /* if 64-bit ranges are inconclusive, see if we can 18281 * utilize 32-bit subrange knowledge to eliminate 18282 * branches that can't be taken a priori 18283 */ 18284 if (reg_u32_min(reg1) > reg_u32_max(reg2) || 18285 reg_u32_max(reg1) < reg_u32_min(reg2)) 18286 return 1; 18287 if (reg_s32_min(reg1) > reg_s32_max(reg2) || 18288 reg_s32_max(reg1) < reg_s32_min(reg2)) 18289 return 1; 18290 } 18291 break; 18292 case BPF_JSET: 18293 if (!is_reg_const(reg2, is_jmp32)) { 18294 swap(reg1, reg2); 18295 swap(t1, t2); 18296 } 18297 if (!is_reg_const(reg2, is_jmp32)) 18298 return -1; 18299 if ((~t1.mask & t1.value) & t2.value) 18300 return 1; 18301 if (!((t1.mask | t1.value) & t2.value)) 18302 return 0; 18303 break; 18304 case BPF_JGT: 18305 if (umin1 > umax2) 18306 return 1; 18307 else if (umax1 <= umin2) 18308 return 0; 18309 break; 18310 case BPF_JSGT: 18311 if (smin1 > smax2) 18312 return 1; 18313 else if (smax1 <= smin2) 18314 return 0; 18315 break; 18316 case BPF_JLT: 18317 if (umax1 < umin2) 18318 return 1; 18319 else if (umin1 >= umax2) 18320 return 0; 18321 break; 18322 case BPF_JSLT: 18323 if (smax1 < smin2) 18324 return 1; 18325 else if (smin1 >= smax2) 18326 return 0; 18327 break; 18328 case BPF_JGE: 18329 if (umin1 >= umax2) 18330 return 1; 18331 else if (umax1 < umin2) 18332 return 0; 18333 break; 18334 case BPF_JSGE: 18335 if (smin1 >= smax2) 18336 return 1; 18337 else if (smax1 < smin2) 18338 return 0; 18339 break; 18340 case BPF_JLE: 18341 if (umax1 <= umin2) 18342 return 1; 18343 else if (umin1 > umax2) 18344 return 0; 18345 break; 18346 case BPF_JSLE: 18347 if (smax1 <= smin2) 18348 return 1; 18349 else if (smin1 > smax2) 18350 return 0; 18351 break; 18352 } 18353 18354 return simulate_both_branches_taken(env, opcode, is_jmp32); 18355 } 18356 18357 int bpf_flip_opcode(u32 opcode) 18358 { 18359 /* How can we transform "a <op> b" into "b <op> a"? */ 18360 static const u8 opcode_flip[16] = { 18361 /* these stay the same */ 18362 [BPF_JEQ >> 4] = BPF_JEQ, 18363 [BPF_JNE >> 4] = BPF_JNE, 18364 [BPF_JSET >> 4] = BPF_JSET, 18365 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 18366 [BPF_JGE >> 4] = BPF_JLE, 18367 [BPF_JGT >> 4] = BPF_JLT, 18368 [BPF_JLE >> 4] = BPF_JGE, 18369 [BPF_JLT >> 4] = BPF_JGT, 18370 [BPF_JSGE >> 4] = BPF_JSLE, 18371 [BPF_JSGT >> 4] = BPF_JSLT, 18372 [BPF_JSLE >> 4] = BPF_JSGE, 18373 [BPF_JSLT >> 4] = BPF_JSGT 18374 }; 18375 return opcode_flip[opcode >> 4]; 18376 } 18377 18378 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 18379 struct bpf_reg_state *src_reg, 18380 u8 opcode) 18381 { 18382 struct bpf_reg_state *pkt; 18383 18384 if (src_reg->type == PTR_TO_PACKET_END) { 18385 pkt = dst_reg; 18386 } else if (dst_reg->type == PTR_TO_PACKET_END) { 18387 pkt = src_reg; 18388 opcode = bpf_flip_opcode(opcode); 18389 } else { 18390 return -1; 18391 } 18392 18393 if (pkt->range >= 0) 18394 return -1; 18395 18396 switch (opcode) { 18397 case BPF_JLE: 18398 /* pkt <= pkt_end */ 18399 fallthrough; 18400 case BPF_JGT: 18401 /* pkt > pkt_end */ 18402 if (pkt->range == BEYOND_PKT_END) 18403 /* pkt has at last one extra byte beyond pkt_end */ 18404 return opcode == BPF_JGT; 18405 break; 18406 case BPF_JLT: 18407 /* pkt < pkt_end */ 18408 fallthrough; 18409 case BPF_JGE: 18410 /* pkt >= pkt_end */ 18411 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 18412 return opcode == BPF_JGE; 18413 break; 18414 } 18415 return -1; 18416 } 18417 18418 /* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;" 18419 * and return: 18420 * 1 - branch will be taken and "goto target" will be executed 18421 * 0 - branch will not be taken and fall-through to next insn 18422 * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value 18423 * range [0,10] 18424 */ 18425 static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *reg1, 18426 struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32) 18427 { 18428 if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32) 18429 return is_pkt_ptr_branch_taken(reg1, reg2, opcode); 18430 18431 if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { 18432 u64 val; 18433 18434 /* 18435 * The low 32 bits of a valid pointer may well be zero, hence 18436 * nothing below applies to a 32-bit comparison. 18437 */ 18438 if (is_jmp32) 18439 return -1; 18440 18441 /* arrange that reg2 is a scalar, and reg1 is a pointer */ 18442 if (!is_reg_const(reg2, is_jmp32)) { 18443 opcode = bpf_flip_opcode(opcode); 18444 swap(reg1, reg2); 18445 } 18446 /* and ensure that reg2 is a constant */ 18447 if (!is_reg_const(reg2, is_jmp32)) 18448 return -1; 18449 18450 if (!reg_not_null(env, reg1)) 18451 return -1; 18452 18453 /* If pointer is valid tests against zero will fail so we can 18454 * use this to direct branch taken. 18455 */ 18456 val = reg_const_value(reg2, is_jmp32); 18457 if (val != 0) 18458 return -1; 18459 18460 switch (opcode) { 18461 case BPF_JEQ: 18462 return 0; 18463 case BPF_JNE: 18464 return 1; 18465 default: 18466 return -1; 18467 } 18468 } 18469 18470 /* now deal with two scalars, but not necessarily constants */ 18471 return is_scalar_branch_taken(env, reg1, reg2, opcode, is_jmp32); 18472 } 18473 18474 /* Opcode that corresponds to a *false* branch condition. 18475 * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 18476 */ 18477 u8 bpf_rev_opcode(u8 opcode) 18478 { 18479 switch (opcode) { 18480 case BPF_JEQ: return BPF_JNE; 18481 case BPF_JNE: return BPF_JEQ; 18482 /* JSET doesn't have it's reverse opcode in BPF, so add 18483 * BPF_X flag to denote the reverse of that operation 18484 */ 18485 case BPF_JSET: return BPF_JSET | BPF_X; 18486 case BPF_JSET | BPF_X: return BPF_JSET; 18487 case BPF_JGE: return BPF_JLT; 18488 case BPF_JGT: return BPF_JLE; 18489 case BPF_JLE: return BPF_JGT; 18490 case BPF_JLT: return BPF_JGE; 18491 case BPF_JSGE: return BPF_JSLT; 18492 case BPF_JSGT: return BPF_JSLE; 18493 case BPF_JSLE: return BPF_JSGT; 18494 case BPF_JSLT: return BPF_JSGE; 18495 default: return 0; 18496 } 18497 } 18498 18499 /* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */ 18500 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, 18501 u8 opcode, bool is_jmp32) 18502 { 18503 struct tnum t; 18504 u64 val; 18505 18506 /* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */ 18507 switch (opcode) { 18508 case BPF_JGE: 18509 case BPF_JGT: 18510 case BPF_JSGE: 18511 case BPF_JSGT: 18512 opcode = bpf_flip_opcode(opcode); 18513 swap(reg1, reg2); 18514 break; 18515 default: 18516 break; 18517 } 18518 18519 switch (opcode) { 18520 case BPF_JEQ: 18521 if (is_jmp32) { 18522 reg1->r32 = cnum32_intersect(reg1->r32, reg2->r32); 18523 reg2->r32 = reg1->r32; 18524 18525 t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); 18526 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 18527 reg2->var_off = tnum_with_subreg(reg2->var_off, t); 18528 } else { 18529 reg1->r64 = cnum64_intersect(reg1->r64, reg2->r64); 18530 reg2->r64 = reg1->r64; 18531 18532 reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); 18533 reg2->var_off = reg1->var_off; 18534 } 18535 break; 18536 case BPF_JNE: 18537 if (!is_reg_const(reg2, is_jmp32)) 18538 swap(reg1, reg2); 18539 if (!is_reg_const(reg2, is_jmp32)) 18540 break; 18541 18542 /* try to recompute the bound of reg1 if reg2 is a const and 18543 * is exactly the edge of reg1. 18544 */ 18545 val = reg_const_value(reg2, is_jmp32); 18546 if (is_jmp32) { 18547 /* Complement of the range [val, val] as cnum32. */ 18548 cnum32_intersect_with(®1->r32, (struct cnum32){ val + 1, U32_MAX - 1 }); 18549 } else { 18550 /* Complement of the range [val, val] as cnum64. */ 18551 cnum64_intersect_with(®1->r64, (struct cnum64){ val + 1, U64_MAX - 1 }); 18552 } 18553 break; 18554 case BPF_JSET: 18555 if (!is_reg_const(reg2, is_jmp32)) 18556 swap(reg1, reg2); 18557 if (!is_reg_const(reg2, is_jmp32)) 18558 break; 18559 val = reg_const_value(reg2, is_jmp32); 18560 /* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X) 18561 * requires single bit to learn something useful. E.g., if we 18562 * know that `r1 & 0x3` is true, then which bits (0, 1, or both) 18563 * are actually set? We can learn something definite only if 18564 * it's a single-bit value to begin with. 18565 * 18566 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have 18567 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor 18568 * bit 1 is set, which we can readily use in adjustments. 18569 */ 18570 if (hweight64(val) != 1) 18571 break; 18572 if (is_jmp32) { 18573 t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); 18574 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 18575 } else { 18576 reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); 18577 } 18578 break; 18579 case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */ 18580 if (!is_reg_const(reg2, is_jmp32)) 18581 swap(reg1, reg2); 18582 if (!is_reg_const(reg2, is_jmp32)) 18583 break; 18584 val = reg_const_value(reg2, is_jmp32); 18585 /* Forget the ranges before narrowing tnums, to avoid invariant 18586 * violations if we're on a dead branch. 18587 */ 18588 __mark_reg_unbounded(reg1); 18589 reg_step_reset(reg1); 18590 if (is_jmp32) { 18591 t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); 18592 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 18593 } else { 18594 reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val)); 18595 } 18596 break; 18597 case BPF_JLE: 18598 if (is_jmp32) { 18599 cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2)); 18600 cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1), U32_MAX); 18601 } else { 18602 cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2)); 18603 cnum64_intersect_with_urange(®2->r64, reg_umin(reg1), U64_MAX); 18604 } 18605 break; 18606 case BPF_JLT: 18607 if (is_jmp32) { 18608 cnum32_intersect_with_urange(®1->r32, 0, reg_u32_max(reg2) - 1); 18609 cnum32_intersect_with_urange(®2->r32, reg_u32_min(reg1) + 1, U32_MAX); 18610 } else { 18611 cnum64_intersect_with_urange(®1->r64, 0, reg_umax(reg2) - 1); 18612 cnum64_intersect_with_urange(®2->r64, reg_umin(reg1) + 1, U64_MAX); 18613 } 18614 break; 18615 case BPF_JSLE: 18616 if (is_jmp32) { 18617 cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2)); 18618 cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1), S32_MAX); 18619 } else { 18620 cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2)); 18621 cnum64_intersect_with_srange(®2->r64, reg_smin(reg1), S64_MAX); 18622 } 18623 break; 18624 case BPF_JSLT: 18625 if (is_jmp32) { 18626 cnum32_intersect_with_srange(®1->r32, S32_MIN, reg_s32_max(reg2) - 1); 18627 cnum32_intersect_with_srange(®2->r32, reg_s32_min(reg1) + 1, S32_MAX); 18628 } else { 18629 cnum64_intersect_with_srange(®1->r64, S64_MIN, reg_smax(reg2) - 1); 18630 cnum64_intersect_with_srange(®2->r64, reg_smin(reg1) + 1, S64_MAX); 18631 } 18632 break; 18633 default: 18634 return; 18635 } 18636 } 18637 18638 /* Check for invariant violations on the registers for both branches of a condition */ 18639 static int regs_bounds_sanity_check_branches(struct bpf_verifier_env *env) 18640 { 18641 int err; 18642 18643 err = reg_bounds_sanity_check(env, &env->true_reg1, "true_reg1"); 18644 err = err ?: reg_bounds_sanity_check(env, &env->true_reg2, "true_reg2"); 18645 err = err ?: reg_bounds_sanity_check(env, &env->false_reg1, "false_reg1"); 18646 err = err ?: reg_bounds_sanity_check(env, &env->false_reg2, "false_reg2"); 18647 return err; 18648 } 18649 18650 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 18651 struct bpf_reg_state *reg, u32 id, 18652 bool is_null) 18653 { 18654 if (type_may_be_null(reg->type) && reg->id == id && 18655 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 18656 /* Old offset should have been known-zero, because we don't 18657 * allow pointer arithmetic on pointers that might be NULL. 18658 * If we see this happening, don't convert the register. 18659 * 18660 * But in some cases, some helpers that return local kptrs 18661 * advance offset for the returned pointer. In those cases, 18662 * it is fine to expect to see reg->var_off. 18663 */ 18664 if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && 18665 WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0))) 18666 return; 18667 if (is_null) { 18668 /* We don't need id from this point 18669 * onwards anymore, thus we should better reset it, 18670 * so that state pruning has chances to take effect. 18671 */ 18672 __mark_reg_known_zero(reg); 18673 reg->type = SCALAR_VALUE; 18674 18675 return; 18676 } 18677 18678 mark_ptr_not_null_reg(reg); 18679 18680 /* 18681 * reg->id is preserved for object relationship tracking 18682 * and spin_lock lock state tracking 18683 */ 18684 } 18685 } 18686 18687 /* The logic is similar to find_good_pkt_pointers(), both could eventually 18688 * be folded together at some point. 18689 */ 18690 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 18691 bool is_null) 18692 { 18693 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 18694 struct bpf_reg_state *regs = state->regs, *reg; 18695 u32 id = regs[regno].id; 18696 18697 if (is_null && find_reference_state(vstate, id)) 18698 /* regs[regno] is in the " == NULL" branch. 18699 * No one could have freed the reference state before 18700 * doing the NULL check. 18701 */ 18702 WARN_ON_ONCE(__release_reference_nomark(vstate, id)); 18703 18704 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 18705 mark_ptr_or_null_reg(state, reg, id, is_null); 18706 })); 18707 } 18708 18709 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 18710 struct bpf_reg_state *dst_reg, 18711 struct bpf_reg_state *src_reg, 18712 struct bpf_verifier_state *this_branch, 18713 struct bpf_verifier_state *other_branch) 18714 { 18715 if (BPF_SRC(insn->code) != BPF_X) 18716 return false; 18717 18718 /* Pointers are always 64-bit. */ 18719 if (BPF_CLASS(insn->code) == BPF_JMP32) 18720 return false; 18721 18722 switch (BPF_OP(insn->code)) { 18723 case BPF_JGT: 18724 if ((dst_reg->type == PTR_TO_PACKET && 18725 src_reg->type == PTR_TO_PACKET_END) || 18726 (dst_reg->type == PTR_TO_PACKET_META && 18727 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 18728 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 18729 find_good_pkt_pointers(this_branch, dst_reg, 18730 dst_reg->type, false); 18731 mark_pkt_end(other_branch, insn->dst_reg, true); 18732 } else if ((dst_reg->type == PTR_TO_PACKET_END && 18733 src_reg->type == PTR_TO_PACKET) || 18734 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 18735 src_reg->type == PTR_TO_PACKET_META)) { 18736 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 18737 find_good_pkt_pointers(other_branch, src_reg, 18738 src_reg->type, true); 18739 mark_pkt_end(this_branch, insn->src_reg, false); 18740 } else { 18741 return false; 18742 } 18743 break; 18744 case BPF_JLT: 18745 if ((dst_reg->type == PTR_TO_PACKET && 18746 src_reg->type == PTR_TO_PACKET_END) || 18747 (dst_reg->type == PTR_TO_PACKET_META && 18748 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 18749 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 18750 find_good_pkt_pointers(other_branch, dst_reg, 18751 dst_reg->type, true); 18752 mark_pkt_end(this_branch, insn->dst_reg, false); 18753 } else if ((dst_reg->type == PTR_TO_PACKET_END && 18754 src_reg->type == PTR_TO_PACKET) || 18755 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 18756 src_reg->type == PTR_TO_PACKET_META)) { 18757 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 18758 find_good_pkt_pointers(this_branch, src_reg, 18759 src_reg->type, false); 18760 mark_pkt_end(other_branch, insn->src_reg, true); 18761 } else { 18762 return false; 18763 } 18764 break; 18765 case BPF_JGE: 18766 if ((dst_reg->type == PTR_TO_PACKET && 18767 src_reg->type == PTR_TO_PACKET_END) || 18768 (dst_reg->type == PTR_TO_PACKET_META && 18769 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 18770 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 18771 find_good_pkt_pointers(this_branch, dst_reg, 18772 dst_reg->type, true); 18773 mark_pkt_end(other_branch, insn->dst_reg, false); 18774 } else if ((dst_reg->type == PTR_TO_PACKET_END && 18775 src_reg->type == PTR_TO_PACKET) || 18776 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 18777 src_reg->type == PTR_TO_PACKET_META)) { 18778 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 18779 find_good_pkt_pointers(other_branch, src_reg, 18780 src_reg->type, false); 18781 mark_pkt_end(this_branch, insn->src_reg, true); 18782 } else { 18783 return false; 18784 } 18785 break; 18786 case BPF_JLE: 18787 if ((dst_reg->type == PTR_TO_PACKET && 18788 src_reg->type == PTR_TO_PACKET_END) || 18789 (dst_reg->type == PTR_TO_PACKET_META && 18790 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 18791 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 18792 find_good_pkt_pointers(other_branch, dst_reg, 18793 dst_reg->type, false); 18794 mark_pkt_end(this_branch, insn->dst_reg, true); 18795 } else if ((dst_reg->type == PTR_TO_PACKET_END && 18796 src_reg->type == PTR_TO_PACKET) || 18797 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 18798 src_reg->type == PTR_TO_PACKET_META)) { 18799 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 18800 find_good_pkt_pointers(this_branch, src_reg, 18801 src_reg->type, true); 18802 mark_pkt_end(other_branch, insn->src_reg, false); 18803 } else { 18804 return false; 18805 } 18806 break; 18807 default: 18808 return false; 18809 } 18810 18811 return true; 18812 } 18813 18814 static void __collect_linked_regs(struct linked_regs *reg_set, struct bpf_reg_state *reg, 18815 u32 id, u32 frameno, u32 spi_or_reg, bool is_reg) 18816 { 18817 struct linked_reg *e; 18818 18819 if (reg->type != SCALAR_VALUE || (reg->id & ~BPF_ADD_CONST) != id) 18820 return; 18821 18822 e = linked_regs_push(reg_set); 18823 if (e) { 18824 e->frameno = frameno; 18825 e->is_reg = is_reg; 18826 e->regno = spi_or_reg; 18827 } else { 18828 clear_scalar_id(reg); 18829 } 18830 } 18831 18832 /* For all R being scalar registers or spilled scalar registers 18833 * in verifier state, save R in linked_regs if R->id == id. 18834 * If there are too many Rs sharing same id, reset id for leftover Rs. 18835 */ 18836 static void collect_linked_regs(struct bpf_verifier_env *env, 18837 struct bpf_verifier_state *vstate, 18838 u32 id, 18839 struct linked_regs *linked_regs) 18840 { 18841 struct bpf_insn_aux_data *aux = env->insn_aux_data; 18842 struct bpf_func_state *func; 18843 struct bpf_reg_state *reg; 18844 u16 live_regs; 18845 int i, j; 18846 18847 id = id & ~BPF_ADD_CONST; 18848 for (i = vstate->curframe; i >= 0; i--) { 18849 live_regs = aux[bpf_frame_insn_idx(vstate, i)].live_regs_before; 18850 func = vstate->frame[i]; 18851 for (j = 0; j < BPF_REG_FP; j++) { 18852 if (!(live_regs & BIT(j))) 18853 continue; 18854 reg = &func->regs[j]; 18855 __collect_linked_regs(linked_regs, reg, id, i, j, true); 18856 } 18857 for (j = 0; j < bpf_stack_nr_slots(func); j++) { 18858 struct bpf_stack_state *ss = bpf_stack_slot(func, j); 18859 18860 if (!bpf_is_spilled_reg(ss)) 18861 continue; 18862 reg = &ss->spilled_ptr; 18863 __collect_linked_regs(linked_regs, reg, id, i, j, false); 18864 } 18865 } 18866 } 18867 18868 /* For all R in linked_regs, copy known_reg range into R 18869 * if R->id == known_reg->id. 18870 */ 18871 static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *vstate, 18872 struct bpf_reg_state *known_reg, struct linked_regs *linked_regs) 18873 { 18874 struct bpf_reg_state fake_reg; 18875 struct bpf_reg_state *reg; 18876 struct linked_reg *e; 18877 int i; 18878 18879 for (i = 0; i < linked_regs->cnt; ++i) { 18880 e = &linked_regs->entries[i]; 18881 reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] 18882 : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; 18883 if (reg->type != SCALAR_VALUE || reg == known_reg) 18884 continue; 18885 if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) 18886 continue; 18887 /* 18888 * Skip mixed 32/64-bit links: the delta relationship doesn't 18889 * hold across different ALU widths. 18890 */ 18891 if (((reg->id ^ known_reg->id) & BPF_ADD_CONST) == BPF_ADD_CONST) 18892 continue; 18893 if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || 18894 reg->delta == known_reg->delta) { 18895 *reg = *known_reg; 18896 } else { 18897 s32 saved_off = reg->delta; 18898 u32 saved_id = reg->id; 18899 18900 fake_reg.type = SCALAR_VALUE; 18901 __mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta); 18902 18903 /* reg = known_reg; reg += delta */ 18904 *reg = *known_reg; 18905 /* 18906 * Must preserve off and id, otherwise another sync_linked_regs() 18907 * will be incorrect. 18908 */ 18909 reg->delta = saved_off; 18910 reg->id = saved_id; 18911 18912 scalar_step_add(reg, reg, &fake_reg); 18913 scalar32_min_max_add(reg, &fake_reg); 18914 scalar_min_max_add(reg, &fake_reg); 18915 reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); 18916 if ((reg->id | known_reg->id) & BPF_ADD_CONST32) 18917 zext_32_to_64(reg); 18918 reg_bounds_sync(reg); 18919 } 18920 if (e->is_reg) 18921 mark_reg_scratched(env, e->regno); 18922 else 18923 mark_stack_slot_scratched(env, e->spi); 18924 } 18925 } 18926 18927 static int check_cond_jmp_op(struct bpf_verifier_env *env, 18928 struct bpf_insn *insn, int *insn_idx) 18929 { 18930 struct bpf_verifier_state *this_branch = env->cur_state; 18931 struct bpf_verifier_state *other_branch; 18932 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 18933 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 18934 struct bpf_reg_state *eq_branch_regs; 18935 struct linked_regs linked_regs = {}; 18936 u8 opcode = BPF_OP(insn->code); 18937 int insn_flags = 0; 18938 bool is_jmp32; 18939 int pred = -1; 18940 int err; 18941 18942 /* Only conditional jumps are expected to reach here. */ 18943 if (opcode == BPF_JA || opcode > BPF_JCOND) { 18944 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 18945 return -EINVAL; 18946 } 18947 18948 if (bpf_is_unwind_goto(insn)) { 18949 if (!env->cur_state->unwound) 18950 return 0; 18951 env->cur_state->unwound = false; 18952 /* 18953 * unwind_exit changed its return address, so the return is 18954 * mispredicted and what follows unwind_goto runs speculatively. 18955 */ 18956 if (!env->bypass_spec_v1) { 18957 err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx); 18958 if (err) 18959 return err; 18960 } 18961 *insn_idx += insn->off; 18962 return 0; 18963 } 18964 18965 if (opcode == BPF_JCOND) { 18966 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; 18967 int idx = *insn_idx; 18968 18969 prev_st = find_prev_entry(env, cur_st->parent, idx); 18970 18971 /* branch out 'fallthrough' insn as a new state to explore */ 18972 queued_st = push_stack(env, idx + 1, idx, false); 18973 if (IS_ERR(queued_st)) 18974 return PTR_ERR(queued_st); 18975 18976 queued_st->may_goto_depth++; 18977 if (prev_st) 18978 widen_imprecise_scalars(env, prev_st, queued_st); 18979 *insn_idx += insn->off; 18980 return 0; 18981 } 18982 18983 /* check src2 operand */ 18984 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 18985 if (err) 18986 return err; 18987 18988 dst_reg = ®s[insn->dst_reg]; 18989 if (BPF_SRC(insn->code) == BPF_X) { 18990 /* check src1 operand */ 18991 err = check_reg_arg(env, insn->src_reg, SRC_OP); 18992 if (err) 18993 return err; 18994 18995 src_reg = ®s[insn->src_reg]; 18996 if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) && 18997 is_pointer_value(env, insn->src_reg)) { 18998 verbose(env, "R%d pointer comparison prohibited\n", 18999 insn->src_reg); 19000 return -EACCES; 19001 } 19002 19003 if (src_reg->type == PTR_TO_STACK) 19004 insn_flags |= INSN_F_SRC_REG_STACK; 19005 if (dst_reg->type == PTR_TO_STACK) 19006 insn_flags |= INSN_F_DST_REG_STACK; 19007 } else { 19008 src_reg = &env->fake_reg[0]; 19009 memset(src_reg, 0, sizeof(*src_reg)); 19010 src_reg->type = SCALAR_VALUE; 19011 __mark_reg_known(src_reg, insn->imm); 19012 19013 if (dst_reg->type == PTR_TO_STACK) 19014 insn_flags |= INSN_F_DST_REG_STACK; 19015 } 19016 19017 if (insn_flags) { 19018 err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0); 19019 if (err) 19020 return err; 19021 } 19022 19023 /* 19024 * Collect the linked registers before env->{true,false}_reg{1,2} setup, 19025 * otherwise ids dropped by collect_linked_regs() would be resurrected 19026 * when env->{true,false}_reg{1,2} are copied back. 19027 */ 19028 if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id) 19029 collect_linked_regs(env, this_branch, src_reg->id, &linked_regs); 19030 if (dst_reg->type == SCALAR_VALUE && dst_reg->id) 19031 collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs); 19032 19033 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 19034 env->false_reg1 = *dst_reg; 19035 env->false_reg2 = *src_reg; 19036 env->true_reg1 = *dst_reg; 19037 env->true_reg2 = *src_reg; 19038 pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32); 19039 if (pred >= 0) { 19040 /* If we get here with a dst_reg pointer type it is because 19041 * above is_branch_taken() special cased the 0 comparison. 19042 */ 19043 if (!__is_pointer_value(false, dst_reg)) 19044 err = mark_chain_precision(env, insn->dst_reg); 19045 if (BPF_SRC(insn->code) == BPF_X && !err && 19046 !__is_pointer_value(false, src_reg)) 19047 err = mark_chain_precision(env, insn->src_reg); 19048 if (err) 19049 return err; 19050 } 19051 19052 if (pred == 1) { 19053 /* Only follow the goto, ignore fall-through. If needed, push 19054 * the fall-through branch for simulation under speculative 19055 * execution. 19056 */ 19057 if (!env->bypass_spec_v1) { 19058 err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx); 19059 if (err < 0) 19060 return err; 19061 } 19062 if (env->log.level & BPF_LOG_LEVEL) 19063 print_insn_state(env, this_branch, this_branch->curframe); 19064 *insn_idx += insn->off; 19065 return 0; 19066 } else if (pred == 0) { 19067 /* Only follow the fall-through branch, since that's where the 19068 * program will go. If needed, push the goto branch for 19069 * simulation under speculative execution. 19070 */ 19071 if (!env->bypass_spec_v1) { 19072 err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1, 19073 *insn_idx); 19074 if (err < 0) 19075 return err; 19076 } 19077 if (env->log.level & BPF_LOG_LEVEL) 19078 print_insn_state(env, this_branch, this_branch->curframe); 19079 return 0; 19080 } 19081 19082 /* Push scalar registers sharing same ID to jump history, 19083 * do this before creating 'other_branch', so that both 19084 * 'this_branch' and 'other_branch' share this history 19085 * if parent state is created. 19086 */ 19087 if (linked_regs.cnt > 1) { 19088 u16 packed[LINKED_REGS_MAX]; 19089 19090 linked_regs_pack(&linked_regs, packed); 19091 err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt); 19092 if (err) 19093 return err; 19094 } 19095 19096 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false); 19097 if (IS_ERR(other_branch)) 19098 return PTR_ERR(other_branch); 19099 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 19100 19101 err = regs_bounds_sanity_check_branches(env); 19102 if (err) 19103 return err; 19104 19105 *dst_reg = env->false_reg1; 19106 *src_reg = env->false_reg2; 19107 other_branch_regs[insn->dst_reg] = env->true_reg1; 19108 if (BPF_SRC(insn->code) == BPF_X) 19109 other_branch_regs[insn->src_reg] = env->true_reg2; 19110 19111 if (BPF_SRC(insn->code) == BPF_X && 19112 src_reg->type == SCALAR_VALUE && src_reg->id && 19113 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 19114 sync_linked_regs(env, this_branch, src_reg, &linked_regs); 19115 sync_linked_regs(env, other_branch, &other_branch_regs[insn->src_reg], 19116 &linked_regs); 19117 } 19118 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 19119 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 19120 sync_linked_regs(env, this_branch, dst_reg, &linked_regs); 19121 sync_linked_regs(env, other_branch, &other_branch_regs[insn->dst_reg], 19122 &linked_regs); 19123 } 19124 19125 /* if one pointer register is compared to another pointer 19126 * register check if PTR_MAYBE_NULL could be lifted. 19127 * E.g. register A - maybe null 19128 * register B - not null 19129 * for JNE A, B, ... - A is not null in the false branch; 19130 * for JEQ A, B, ... - A is not null in the true branch. 19131 * 19132 * Since PTR_TO_BTF_ID points to a kernel struct that does 19133 * not need to be null checked by the BPF program, i.e., 19134 * could be null even without PTR_MAYBE_NULL marking, so 19135 * only propagate nullness when neither reg is that type. 19136 */ 19137 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 19138 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 19139 base_type(src_reg->type) != PTR_TO_BTF_ID && 19140 base_type(dst_reg->type) != PTR_TO_BTF_ID) { 19141 eq_branch_regs = NULL; 19142 switch (opcode) { 19143 case BPF_JEQ: 19144 eq_branch_regs = other_branch_regs; 19145 break; 19146 case BPF_JNE: 19147 eq_branch_regs = regs; 19148 break; 19149 default: 19150 /* do nothing */ 19151 break; 19152 } 19153 if (eq_branch_regs) { 19154 /* src == dst && dst != NULL => src != NULL */ 19155 if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type)) 19156 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 19157 /* src == dst && src != NULL => dst != NULL */ 19158 if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type)) 19159 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 19160 } 19161 } 19162 19163 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 19164 * Also does the same detection for a register whose the value is 19165 * known to be 0. 19166 * NOTE: these optimizations below are related with pointer comparison 19167 * which will never be JMP32. 19168 */ 19169 if (!is_jmp32 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 19170 type_may_be_null(dst_reg->type) && 19171 ((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) || 19172 (BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) { 19173 /* 19174 * For BPF_X the zero is a property of this execution path, 19175 * hence src_reg has to be precise. 19176 */ 19177 if (BPF_SRC(insn->code) == BPF_X) { 19178 err = mark_chain_precision(env, insn->src_reg); 19179 if (err) 19180 return err; 19181 } 19182 /* Mark all identical registers in each branch as either 19183 * safe or unknown depending R == 0 or R != 0 conditional. 19184 */ 19185 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 19186 opcode == BPF_JNE); 19187 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 19188 opcode == BPF_JEQ); 19189 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 19190 this_branch, other_branch) && 19191 is_pointer_value(env, insn->dst_reg)) { 19192 verbose(env, "R%d pointer comparison prohibited\n", 19193 insn->dst_reg); 19194 return -EACCES; 19195 } 19196 if (env->log.level & BPF_LOG_LEVEL) 19197 print_insn_state(env, this_branch, this_branch->curframe); 19198 return 0; 19199 } 19200 19201 /* verify BPF_LD_IMM64 instruction */ 19202 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 19203 { 19204 struct bpf_insn_aux_data *aux = cur_aux(env); 19205 struct bpf_reg_state *regs = cur_regs(env); 19206 struct bpf_reg_state *dst_reg; 19207 struct bpf_map *map; 19208 int err; 19209 19210 if (BPF_SIZE(insn->code) != BPF_DW) { 19211 verbose(env, "invalid BPF_LD_IMM insn\n"); 19212 return -EINVAL; 19213 } 19214 19215 err = check_reg_arg(env, insn->dst_reg, DST_OP); 19216 if (err) 19217 return err; 19218 19219 dst_reg = ®s[insn->dst_reg]; 19220 bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE); 19221 if (insn->src_reg == 0) { 19222 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 19223 19224 dst_reg->type = SCALAR_VALUE; 19225 __mark_reg_known(®s[insn->dst_reg], imm); 19226 bpf_diag_mod_end(env); 19227 return 0; 19228 } 19229 19230 /* All special src_reg cases are listed below. From this point onwards 19231 * we either succeed and assign a corresponding dst_reg->type after 19232 * zeroing the offset, or fail and reject the program. 19233 */ 19234 mark_reg_known_zero(env, regs, insn->dst_reg); 19235 19236 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 19237 dst_reg->type = aux->btf_var.reg_type; 19238 switch (base_type(dst_reg->type)) { 19239 case PTR_TO_MEM: 19240 dst_reg->mem_size = aux->btf_var.mem_size; 19241 break; 19242 case PTR_TO_BTF_ID: 19243 dst_reg->btf = aux->btf_var.btf; 19244 dst_reg->btf_id = aux->btf_var.btf_id; 19245 break; 19246 default: 19247 verifier_bug(env, "pseudo btf id: unexpected dst reg type"); 19248 return -EFAULT; 19249 } 19250 bpf_diag_mod_end(env); 19251 return 0; 19252 } 19253 19254 if (insn->src_reg == BPF_PSEUDO_FUNC) { 19255 struct bpf_prog_aux *aux = env->prog->aux; 19256 u32 subprogno = bpf_find_subprog(env, 19257 env->insn_idx + insn->imm + 1); 19258 19259 if (!aux->func_info) { 19260 verbose(env, "missing btf func_info\n"); 19261 return -EINVAL; 19262 } 19263 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 19264 verbose(env, "callback function not static\n"); 19265 return -EINVAL; 19266 } 19267 /* 19268 * When env->subprog_cnt == 1 this instruction won't be rewritten 19269 * to hold a real function address. Assume that no usable program 19270 * combines e.g. main and timer callback and just reject here. 19271 */ 19272 if (subprogno == 0) { 19273 verbose(env, "callback function cannot be the main program\n"); 19274 return -EINVAL; 19275 } 19276 19277 dst_reg->type = PTR_TO_FUNC; 19278 dst_reg->subprogno = subprogno; 19279 bpf_diag_mod_end(env); 19280 return 0; 19281 } 19282 19283 map = env->used_maps[aux->map_index]; 19284 19285 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 19286 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 19287 if (map->map_type == BPF_MAP_TYPE_ARENA) { 19288 __mark_reg_unknown(env, dst_reg); 19289 dst_reg->map_ptr = map; 19290 bpf_diag_mod_end(env); 19291 return 0; 19292 } 19293 __mark_reg_known(dst_reg, aux->map_off); 19294 dst_reg->type = PTR_TO_MAP_VALUE; 19295 dst_reg->map_ptr = map; 19296 WARN_ON_ONCE(map->map_type != BPF_MAP_TYPE_INSN_ARRAY && 19297 map->max_entries != 1); 19298 /* We want reg->id to be same (0) as map_value is not distinct */ 19299 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 19300 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 19301 dst_reg->type = CONST_PTR_TO_MAP; 19302 dst_reg->map_ptr = map; 19303 } else { 19304 verifier_bug(env, "unexpected src reg value for ldimm64"); 19305 return -EFAULT; 19306 } 19307 19308 bpf_diag_mod_end(env); 19309 return 0; 19310 } 19311 19312 static bool may_access_skb(enum bpf_prog_type type) 19313 { 19314 switch (type) { 19315 case BPF_PROG_TYPE_SOCKET_FILTER: 19316 case BPF_PROG_TYPE_SCHED_CLS: 19317 case BPF_PROG_TYPE_SCHED_ACT: 19318 return true; 19319 default: 19320 return false; 19321 } 19322 } 19323 19324 /* verify safety of LD_ABS|LD_IND instructions: 19325 * - they can only appear in the programs where ctx == skb 19326 * - since they are wrappers of function calls, they scratch R1-R5 registers, 19327 * preserve R6-R9, and store return value into R0 19328 * 19329 * Implicit input: 19330 * ctx == skb == R6 == CTX 19331 * 19332 * Explicit input: 19333 * SRC == any register 19334 * IMM == 32-bit immediate 19335 * 19336 * Output: 19337 * R0 - 8/16/32-bit skb data converted to cpu endianness 19338 */ 19339 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 19340 { 19341 struct bpf_verifier_state *state = env->cur_state; 19342 struct bpf_reg_state *regs = cur_regs(env); 19343 static const int ctx_reg = BPF_REG_6; 19344 u8 mode = BPF_MODE(insn->code); 19345 int i, err; 19346 19347 if (!may_access_skb(resolve_prog_type(env->prog))) { 19348 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 19349 return -EINVAL; 19350 } 19351 19352 for (i = state->curframe; i; i--) { 19353 if (state->frame[i]->in_callback_fn) { 19354 verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); 19355 return -EINVAL; 19356 } 19357 } 19358 19359 if (!env->ops->gen_ld_abs) { 19360 verifier_bug(env, "gen_ld_abs is null"); 19361 return -EFAULT; 19362 } 19363 19364 /* check whether implicit source operand (register R6) is readable */ 19365 err = check_reg_arg(env, ctx_reg, SRC_OP); 19366 if (err) 19367 return err; 19368 19369 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 19370 * gen_ld_abs() may terminate the program at runtime, leading to 19371 * reference leak. 19372 */ 19373 err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]"); 19374 if (err) 19375 return err; 19376 19377 if (regs[ctx_reg].type != PTR_TO_CTX) { 19378 verbose(env, 19379 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 19380 return -EINVAL; 19381 } 19382 19383 if (mode == BPF_IND) { 19384 /* check explicit source operand */ 19385 err = check_reg_arg(env, insn->src_reg, SRC_OP); 19386 if (err) 19387 return err; 19388 } 19389 19390 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 19391 if (err < 0) 19392 return err; 19393 19394 /* reset caller saved regs to unreadable */ 19395 bpf_diag_record_caller_saved(env, regs); 19396 bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); 19397 for (i = 0; i < CALLER_SAVED_REGS; i++) { 19398 bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); 19399 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 19400 } 19401 19402 /* mark destination R0 register as readable, since it contains 19403 * the value fetched from the packet. 19404 * Already marked as written above. 19405 */ 19406 mark_reg_unknown(env, regs, BPF_REG_0); 19407 bpf_diag_mod_end(env); 19408 /* 19409 * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 19410 * which must be explored by the verifier when in a subprog. 19411 */ 19412 if (env->cur_state->curframe) { 19413 struct bpf_verifier_state *branch; 19414 19415 mark_reg_scratched(env, BPF_REG_0); 19416 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); 19417 if (IS_ERR(branch)) 19418 return PTR_ERR(branch); 19419 mark_reg_known_zero(env, regs, BPF_REG_0); 19420 err = prepare_func_exit(env, &env->insn_idx); 19421 if (err) 19422 return err; 19423 env->insn_idx--; 19424 } 19425 return 0; 19426 } 19427 19428 static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range) 19429 { 19430 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 19431 19432 /* Default return value range. */ 19433 *range = retval_range(0, 1); 19434 19435 switch (prog_type) { 19436 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 19437 switch (env->prog->expected_attach_type) { 19438 case BPF_CGROUP_UDP4_RECVMSG: 19439 case BPF_CGROUP_UDP6_RECVMSG: 19440 case BPF_CGROUP_UNIX_RECVMSG: 19441 case BPF_CGROUP_INET4_GETPEERNAME: 19442 case BPF_CGROUP_INET6_GETPEERNAME: 19443 case BPF_CGROUP_UNIX_GETPEERNAME: 19444 case BPF_CGROUP_INET4_GETSOCKNAME: 19445 case BPF_CGROUP_INET6_GETSOCKNAME: 19446 case BPF_CGROUP_UNIX_GETSOCKNAME: 19447 *range = retval_range(1, 1); 19448 break; 19449 case BPF_CGROUP_INET4_BIND: 19450 case BPF_CGROUP_INET6_BIND: 19451 *range = retval_range(0, 3); 19452 break; 19453 default: 19454 break; 19455 } 19456 break; 19457 case BPF_PROG_TYPE_CGROUP_SKB: 19458 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) 19459 *range = retval_range(0, 3); 19460 break; 19461 case BPF_PROG_TYPE_CGROUP_SOCK: 19462 case BPF_PROG_TYPE_SOCK_OPS: 19463 case BPF_PROG_TYPE_CGROUP_DEVICE: 19464 case BPF_PROG_TYPE_CGROUP_SYSCTL: 19465 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 19466 break; 19467 case BPF_PROG_TYPE_RAW_TRACEPOINT: 19468 if (!env->prog->aux->attach_btf_id) 19469 return false; 19470 *range = retval_range(0, 0); 19471 break; 19472 case BPF_PROG_TYPE_TRACING: 19473 switch (env->prog->expected_attach_type) { 19474 case BPF_TRACE_FENTRY: 19475 case BPF_TRACE_FEXIT: 19476 case BPF_TRACE_FSESSION: 19477 case BPF_TRACE_FENTRY_MULTI: 19478 case BPF_TRACE_FEXIT_MULTI: 19479 case BPF_TRACE_FSESSION_MULTI: 19480 *range = retval_range(0, 0); 19481 break; 19482 case BPF_TRACE_RAW_TP: 19483 case BPF_MODIFY_RETURN: 19484 return false; 19485 case BPF_TRACE_ITER: 19486 default: 19487 break; 19488 } 19489 break; 19490 case BPF_PROG_TYPE_KPROBE: 19491 switch (env->prog->expected_attach_type) { 19492 case BPF_TRACE_KPROBE_SESSION: 19493 case BPF_TRACE_UPROBE_SESSION: 19494 break; 19495 default: 19496 return false; 19497 } 19498 break; 19499 case BPF_PROG_TYPE_SK_LOOKUP: 19500 *range = retval_range(SK_DROP, SK_PASS); 19501 break; 19502 19503 case BPF_PROG_TYPE_LSM: 19504 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 19505 /* no range found, any return value is allowed */ 19506 if (!get_func_retval_range(env->prog, range)) 19507 return false; 19508 /* no restricted range, any return value is allowed */ 19509 if (range->minval == S32_MIN && range->maxval == S32_MAX) 19510 return false; 19511 range->return_32bit = true; 19512 } else if (!env->prog->aux->attach_func_proto->type) { 19513 /* Make sure programs that attach to void 19514 * hooks don't try to modify return value. 19515 */ 19516 *range = retval_range(1, 1); 19517 } 19518 break; 19519 19520 case BPF_PROG_TYPE_NETFILTER: 19521 *range = retval_range(NF_DROP, NF_ACCEPT); 19522 break; 19523 case BPF_PROG_TYPE_STRUCT_OPS: 19524 *range = retval_range(0, 0); 19525 break; 19526 case BPF_PROG_TYPE_EXT: 19527 /* freplace program can return anything as its return value 19528 * depends on the to-be-replaced kernel func or bpf program. 19529 */ 19530 default: 19531 return false; 19532 } 19533 19534 /* Continue calculating. */ 19535 19536 return true; 19537 } 19538 19539 static bool program_returns_void(struct bpf_verifier_env *env) 19540 { 19541 const struct bpf_prog *prog = env->prog; 19542 enum bpf_prog_type prog_type = prog->type; 19543 19544 switch (prog_type) { 19545 case BPF_PROG_TYPE_LSM: 19546 /* See return_retval_range, for BPF_LSM_CGROUP can be 0 or 0-1 depending on hook. */ 19547 if (prog->expected_attach_type != BPF_LSM_CGROUP && 19548 !prog->aux->attach_func_proto->type) 19549 return true; 19550 break; 19551 case BPF_PROG_TYPE_STRUCT_OPS: 19552 if (!prog->aux->attach_func_proto->type) 19553 return true; 19554 break; 19555 case BPF_PROG_TYPE_EXT: 19556 /* 19557 * If the actual program is an extension, let it 19558 * return void - attaching will succeed only if the 19559 * program being replaced also returns void, and since 19560 * it has passed verification its actual type doesn't matter. 19561 */ 19562 if (subprog_returns_void(env, 0)) 19563 return true; 19564 break; 19565 default: 19566 break; 19567 } 19568 return false; 19569 } 19570 19571 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name) 19572 { 19573 const char *exit_ctx = "At program exit"; 19574 struct tnum enforce_attach_type_range = tnum_unknown; 19575 const struct bpf_prog *prog = env->prog; 19576 struct bpf_reg_state *reg = reg_state(env, regno); 19577 struct bpf_retval_range range = retval_range(0, 1); 19578 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 19579 struct bpf_func_state *frame = env->cur_state->frame[0]; 19580 const struct btf_type *reg_type, *ret_type = NULL; 19581 int err; 19582 19583 /* LSM and struct_ops func-ptr's return type could be "void" */ 19584 if (!frame->in_async_callback_fn && program_returns_void(env)) 19585 return 0; 19586 19587 if (prog_type == BPF_PROG_TYPE_STRUCT_OPS) { 19588 /* Allow a struct_ops program to return a referenced kptr if it 19589 * matches the operator's return type and is in its unmodified 19590 * form. A scalar zero (i.e., a null pointer) is also allowed. 19591 */ 19592 reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL; 19593 ret_type = btf_type_resolve_ptr(prog->aux->attach_btf, 19594 prog->aux->attach_func_proto->type, 19595 NULL); 19596 if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg)) 19597 return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false); 19598 } 19599 19600 /* eBPF calling convention is such that R0 is used 19601 * to return the value from eBPF program. 19602 * Make sure that it's readable at this time 19603 * of bpf_exit, which means that program wrote 19604 * something into it earlier 19605 */ 19606 err = check_reg_arg(env, regno, SRC_OP); 19607 if (err) 19608 return err; 19609 19610 if (is_pointer_value(env, regno)) { 19611 verbose(env, "R%d leaks addr as return value\n", regno); 19612 return -EACCES; 19613 } 19614 19615 if (frame->in_async_callback_fn) { 19616 exit_ctx = "At async callback return"; 19617 range = frame->callback_ret_range; 19618 goto enforce_retval; 19619 } 19620 19621 if (prog_type == BPF_PROG_TYPE_STRUCT_OPS && !ret_type) 19622 return 0; 19623 19624 if (prog_type == BPF_PROG_TYPE_CGROUP_SKB && (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS)) 19625 enforce_attach_type_range = tnum_range(2, 3); 19626 19627 if (!return_retval_range(env, &range)) 19628 return 0; 19629 19630 enforce_retval: 19631 if (reg->type != SCALAR_VALUE) { 19632 verbose(env, "%s the register R%d is not a known value (%s)\n", 19633 exit_ctx, regno, reg_type_str(env, reg->type)); 19634 return -EINVAL; 19635 } 19636 19637 err = mark_chain_precision(env, regno); 19638 if (err) 19639 return err; 19640 19641 if (!retval_range_within(range, reg)) { 19642 verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); 19643 if (prog->expected_attach_type == BPF_LSM_CGROUP && 19644 prog_type == BPF_PROG_TYPE_LSM && 19645 !prog->aux->attach_func_proto->type) 19646 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 19647 return -EINVAL; 19648 } 19649 19650 if (!tnum_is_unknown(enforce_attach_type_range) && 19651 tnum_in(enforce_attach_type_range, reg->var_off)) 19652 env->prog->enforce_expected_attach_type = 1; 19653 return 0; 19654 } 19655 19656 static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) 19657 { 19658 struct bpf_reg_state *reg; 19659 int err; 19660 19661 err = check_reg_arg(env, regno, SRC_OP); 19662 if (err) 19663 return err; 19664 19665 /* Pointers to arena are safe to pass between subprograms. */ 19666 if (is_arena_reg(env, regno)) 19667 return 0; 19668 19669 if (is_pointer_value(env, regno)) { 19670 verbose(env, "R%d leaks addr as return value\n", regno); 19671 return -EACCES; 19672 } 19673 19674 reg = reg_state(env, regno); 19675 if (reg->type != SCALAR_VALUE) { 19676 verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", 19677 regno, reg_type_str(env, reg->type)); 19678 return -EINVAL; 19679 } 19680 19681 return 0; 19682 } 19683 19684 static int check_global_subprog_return_code(struct bpf_verifier_env *env) 19685 { 19686 struct bpf_func_state *cur_frame = cur_func(env); 19687 u32 subprog = cur_frame->subprogno; 19688 u32 i, nregs; 19689 int err; 19690 19691 if (subprog_returns_void(env, subprog)) 19692 return 0; 19693 19694 nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; 19695 for (i = 0; i < nregs; i++) { 19696 err = check_global_ret_scalar_reg(env, ret_regs[i]); 19697 if (err) 19698 return err; 19699 } 19700 19701 return 0; 19702 } 19703 19704 /* Bitmask with 1s for all caller saved registers */ 19705 #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) 19706 19707 /* True if do_misc_fixups() replaces calls to helper number 'imm', 19708 * replacement patch is presumed to follow bpf_fastcall contract 19709 * (see mark_fastcall_pattern_for_call() below). 19710 */ 19711 bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) 19712 { 19713 switch (imm) { 19714 #ifdef CONFIG_X86_64 19715 case BPF_FUNC_get_smp_processor_id: 19716 #ifdef CONFIG_SMP 19717 case BPF_FUNC_get_current_task_btf: 19718 case BPF_FUNC_get_current_task: 19719 #endif 19720 return env->prog->jit_requested && bpf_jit_supports_percpu_insn(); 19721 #endif 19722 default: 19723 return false; 19724 } 19725 } 19726 19727 /* If @call is a kfunc or helper call, fills @cs and returns true, 19728 * otherwise returns false. 19729 */ 19730 bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, 19731 struct bpf_call_summary *cs) 19732 { 19733 struct bpf_call_arg_meta meta; 19734 const struct bpf_func_proto *fn; 19735 int i; 19736 19737 if (bpf_helper_call(call)) { 19738 if (bpf_get_helper_proto(env, call->imm, &fn) < 0) 19739 /* error would be reported later */ 19740 return false; 19741 cs->fastcall = fn->allow_fastcall && 19742 (bpf_verifier_inlines_helper_call(env, call->imm) || 19743 bpf_jit_inlines_helper_call(call->imm)); 19744 cs->is_void = fn->ret_type == RET_VOID; 19745 cs->arg_slot_cnt = 0; 19746 for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { 19747 if (fn->arg_type[i] == ARG_UNUSED) 19748 break; 19749 cs->arg_slot_cnt++; 19750 } 19751 return true; 19752 } 19753 19754 if (bpf_pseudo_kfunc_call(call)) { 19755 int err; 19756 19757 err = bpf_fetch_kfunc_arg_meta(env, call->imm, call->off, &meta); 19758 if (err < 0) 19759 /* error would be reported later */ 19760 return false; 19761 cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto); 19762 cs->fastcall = meta.kfunc_flags & KF_FASTCALL; 19763 cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); 19764 return true; 19765 } 19766 19767 return false; 19768 } 19769 19770 /* LLVM define a bpf_fastcall function attribute. 19771 * This attribute means that function scratches only some of 19772 * the caller saved registers defined by ABI. 19773 * For BPF the set of such registers could be defined as follows: 19774 * - R0 is scratched only if function is non-void; 19775 * - R1-R5 are scratched only if corresponding parameter type is defined 19776 * in the function prototype. 19777 * 19778 * The contract between kernel and clang allows to simultaneously use 19779 * such functions and maintain backwards compatibility with old 19780 * kernels that don't understand bpf_fastcall calls: 19781 * 19782 * - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5 19783 * registers are not scratched by the call; 19784 * 19785 * - as a post-processing step, clang visits each bpf_fastcall call and adds 19786 * spill/fill for every live r0-r5; 19787 * 19788 * - stack offsets used for the spill/fill are allocated as lowest 19789 * stack offsets in whole function and are not used for any other 19790 * purposes; 19791 * 19792 * - when kernel loads a program, it looks for such patterns 19793 * (bpf_fastcall function surrounded by spills/fills) and checks if 19794 * spill/fill stack offsets are used exclusively in fastcall patterns; 19795 * 19796 * - if so, and if verifier or current JIT inlines the call to the 19797 * bpf_fastcall function (e.g. a helper call), kernel removes unnecessary 19798 * spill/fill pairs; 19799 * 19800 * - when old kernel loads a program, presence of spill/fill pairs 19801 * keeps BPF program valid, albeit slightly less efficient. 19802 * 19803 * For example: 19804 * 19805 * r1 = 1; 19806 * r2 = 2; 19807 * *(u64 *)(r10 - 8) = r1; r1 = 1; 19808 * *(u64 *)(r10 - 16) = r2; r2 = 2; 19809 * call %[to_be_inlined] --> call %[to_be_inlined] 19810 * r2 = *(u64 *)(r10 - 16); r0 = r1; 19811 * r1 = *(u64 *)(r10 - 8); r0 += r2; 19812 * r0 = r1; exit; 19813 * r0 += r2; 19814 * exit; 19815 * 19816 * The purpose of mark_fastcall_pattern_for_call is to: 19817 * - look for such patterns; 19818 * - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern; 19819 * - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction; 19820 * - update env->subprog_info[*]->fastcall_stack_off to find an offset 19821 * at which bpf_fastcall spill/fill stack slots start; 19822 * - update env->subprog_info[*]->keep_fastcall_stack. 19823 * 19824 * The .fastcall_pattern and .fastcall_stack_off are used by 19825 * check_fastcall_stack_contract() to check if every stack access to 19826 * fastcall spill/fill stack slot originates from spill/fill 19827 * instructions, members of fastcall patterns. 19828 * 19829 * If such condition holds true for a subprogram, fastcall patterns could 19830 * be rewritten by remove_fastcall_spills_fills(). 19831 * Otherwise bpf_fastcall patterns are not changed in the subprogram 19832 * (code, presumably, generated by an older clang version). 19833 * 19834 * For example, it is *not* safe to remove spill/fill below: 19835 * 19836 * r1 = 1; 19837 * *(u64 *)(r10 - 8) = r1; r1 = 1; 19838 * call %[to_be_inlined] --> call %[to_be_inlined] 19839 * r1 = *(u64 *)(r10 - 8); r0 = *(u64 *)(r10 - 8); <---- wrong !!! 19840 * r0 = *(u64 *)(r10 - 8); r0 += r1; 19841 * r0 += r1; exit; 19842 * exit; 19843 * 19844 * Both uses of the marks assume that a pattern is entered at its first 19845 * spill and thus executes as a unit, hence a pattern is not grown past 19846 * an instruction targeted by a jump. 19847 */ 19848 static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, 19849 struct bpf_subprog_info *subprog, 19850 int insn_idx, s16 lowest_off) 19851 { 19852 struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx; 19853 struct bpf_insn *call = &env->prog->insnsi[insn_idx]; 19854 u32 clobbered_regs_mask; 19855 struct bpf_call_summary cs; 19856 u32 expected_regs_mask; 19857 s16 off; 19858 int i; 19859 19860 if (!bpf_get_call_summary(env, call, &cs)) 19861 return; 19862 19863 /* A bitmask specifying which caller saved registers are clobbered 19864 * by a call to a helper/kfunc *as if* this helper/kfunc follows 19865 * bpf_fastcall contract: 19866 * - includes R0 if function is non-void; 19867 * - includes R1-R5 if corresponding parameter has is described 19868 * in the function prototype. 19869 */ 19870 clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); 19871 /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ 19872 expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; 19873 19874 /* match pairs of form: 19875 * 19876 * *(u64 *)(r10 - Y) = rX (where Y % 8 == 0) 19877 * ... 19878 * call %[to_be_inlined] 19879 * ... 19880 * rX = *(u64 *)(r10 - Y) 19881 */ 19882 for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) { 19883 if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) 19884 break; 19885 /* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */ 19886 if (bpf_is_jump_target(env, insn_idx - i + 1) || 19887 bpf_is_jump_target(env, insn_idx + i)) 19888 break; 19889 stx = &insns[insn_idx - i]; 19890 ldx = &insns[insn_idx + i]; 19891 /* must be a stack spill/fill pair */ 19892 if (stx->code != (BPF_STX | BPF_MEM | BPF_DW) || 19893 ldx->code != (BPF_LDX | BPF_MEM | BPF_DW) || 19894 stx->dst_reg != BPF_REG_10 || 19895 ldx->src_reg != BPF_REG_10) 19896 break; 19897 /* must be a spill/fill for the same reg */ 19898 if (stx->src_reg != ldx->dst_reg) 19899 break; 19900 /* must be one of the previously unseen registers */ 19901 if ((BIT(stx->src_reg) & expected_regs_mask) == 0) 19902 break; 19903 /* must be a spill/fill for the same expected offset, 19904 * no need to check offset alignment, BPF_DW stack access 19905 * is always 8-byte aligned. 19906 */ 19907 if (stx->off != off || ldx->off != off) 19908 break; 19909 expected_regs_mask &= ~BIT(stx->src_reg); 19910 env->insn_aux_data[insn_idx - i].fastcall_pattern = 1; 19911 env->insn_aux_data[insn_idx + i].fastcall_pattern = 1; 19912 } 19913 if (i == 1) 19914 return; 19915 19916 /* Conditionally set 'fastcall_spills_num' to allow forward 19917 * compatibility when more helper functions are marked as 19918 * bpf_fastcall at compile time than current kernel supports, e.g: 19919 * 19920 * 1: *(u64 *)(r10 - 8) = r1 19921 * 2: call A ;; assume A is bpf_fastcall for current kernel 19922 * 3: r1 = *(u64 *)(r10 - 8) 19923 * 4: *(u64 *)(r10 - 8) = r1 19924 * 5: call B ;; assume B is not bpf_fastcall for current kernel 19925 * 6: r1 = *(u64 *)(r10 - 8) 19926 * 19927 * There is no need to block bpf_fastcall rewrite for such program. 19928 * Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy, 19929 * don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills() 19930 * does not remove spill/fill pair {4,6}. 19931 */ 19932 if (cs.fastcall) 19933 env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1; 19934 else 19935 subprog->keep_fastcall_stack = 1; 19936 subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off); 19937 } 19938 19939 static int mark_fastcall_patterns(struct bpf_verifier_env *env) 19940 { 19941 struct bpf_subprog_info *subprog = env->subprog_info; 19942 struct bpf_insn *insn; 19943 s16 lowest_off; 19944 int s, i; 19945 19946 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { 19947 /* find lowest stack spill offset used in this subprog */ 19948 lowest_off = 0; 19949 for (i = subprog->start; i < (subprog + 1)->start; ++i) { 19950 insn = env->prog->insnsi + i; 19951 if (insn->code != (BPF_STX | BPF_MEM | BPF_DW) || 19952 insn->dst_reg != BPF_REG_10) 19953 continue; 19954 lowest_off = min(lowest_off, insn->off); 19955 } 19956 /* use this offset to find fastcall patterns */ 19957 for (i = subprog->start; i < (subprog + 1)->start; ++i) { 19958 insn = env->prog->insnsi + i; 19959 if (insn->code != (BPF_JMP | BPF_CALL)) 19960 continue; 19961 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); 19962 } 19963 } 19964 return 0; 19965 } 19966 19967 static void adjust_btf_func(struct bpf_verifier_env *env) 19968 { 19969 struct bpf_prog_aux *aux = env->prog->aux; 19970 int i; 19971 19972 if (!aux->func_info) 19973 return; 19974 19975 /* func_info is not available for hidden subprogs */ 19976 for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) 19977 aux->func_info[i].insn_off = env->subprog_info[i].start; 19978 } 19979 19980 /* Find id in idset and increment its count, or add new entry */ 19981 static void idset_cnt_inc(struct bpf_idset *idset, u32 id) 19982 { 19983 u32 i; 19984 19985 for (i = 0; i < idset->num_ids; i++) { 19986 if (idset->entries[i].id == id) { 19987 idset->entries[i].cnt++; 19988 return; 19989 } 19990 } 19991 /* New id */ 19992 if (idset->num_ids < BPF_ID_MAP_SIZE) { 19993 idset->entries[idset->num_ids].id = id; 19994 idset->entries[idset->num_ids].cnt = 1; 19995 idset->num_ids++; 19996 } 19997 } 19998 19999 /* Find id in idset and return its count, or 0 if not found */ 20000 static u32 idset_cnt_get(struct bpf_idset *idset, u32 id) 20001 { 20002 u32 i; 20003 20004 for (i = 0; i < idset->num_ids; i++) { 20005 if (idset->entries[i].id == id) 20006 return idset->entries[i].cnt; 20007 } 20008 return 0; 20009 } 20010 20011 /* 20012 * Clear singular scalar ids in a state. 20013 * A register with a non-zero id is called singular if no other register shares 20014 * the same base id. Such registers can be treated as independent (id=0). 20015 */ 20016 void bpf_clear_singular_ids(struct bpf_verifier_env *env, 20017 struct bpf_verifier_state *st) 20018 { 20019 struct bpf_idset *idset = &env->idset_scratch; 20020 struct bpf_func_state *func; 20021 struct bpf_reg_state *reg; 20022 20023 idset->num_ids = 0; 20024 20025 bpf_for_each_reg_in_vstate(st, func, reg, ({ 20026 if (reg->type != SCALAR_VALUE) 20027 continue; 20028 if (!reg->id) 20029 continue; 20030 idset_cnt_inc(idset, reg->id & ~BPF_ADD_CONST); 20031 })); 20032 20033 bpf_for_each_reg_in_vstate(st, func, reg, ({ 20034 if (reg->type != SCALAR_VALUE) 20035 continue; 20036 if (!reg->id) 20037 continue; 20038 if (idset_cnt_get(idset, reg->id & ~BPF_ADD_CONST) == 1) 20039 clear_scalar_id(reg); 20040 })); 20041 } 20042 20043 /* Return true if it's OK to have the same insn return a different type. */ 20044 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 20045 { 20046 switch (base_type(type)) { 20047 case PTR_TO_CTX: 20048 case PTR_TO_SOCKET: 20049 case PTR_TO_SOCK_COMMON: 20050 case PTR_TO_TCP_SOCK: 20051 case PTR_TO_XDP_SOCK: 20052 case PTR_TO_BTF_ID: 20053 case PTR_TO_ARENA: 20054 return false; 20055 case PTR_TO_MEM: 20056 return !bpf_may_fault_on_deref(type); 20057 default: 20058 return true; 20059 } 20060 } 20061 20062 /* If an instruction was previously used with particular pointer types, then we 20063 * need to be careful to avoid cases such as the below, where it may be ok 20064 * for one branch accessing the pointer, but not ok for the other branch: 20065 * 20066 * R1 = sock_ptr 20067 * goto X; 20068 * ... 20069 * R1 = some_other_valid_ptr; 20070 * goto X; 20071 * ... 20072 * R2 = *(u32 *)(R1 + 0); 20073 */ 20074 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 20075 { 20076 return src != prev && (!reg_type_mismatch_ok(src) || 20077 !reg_type_mismatch_ok(prev)); 20078 } 20079 20080 static bool is_ptr_to_mem(enum bpf_reg_type type) 20081 { 20082 return base_type(type) == PTR_TO_MEM; 20083 } 20084 20085 static enum bpf_reg_type merge_ptr_types(enum bpf_reg_type type_a, 20086 enum bpf_reg_type type_b) 20087 { 20088 bool to_mem = is_ptr_to_mem(type_a) || is_ptr_to_mem(type_b); 20089 enum bpf_reg_type type_merged = to_mem ? PTR_TO_MEM : PTR_TO_BTF_ID; 20090 20091 if (bpf_may_fault_on_deref(type_a) || bpf_may_fault_on_deref(type_b)) 20092 type_merged |= to_mem ? MEM_RDONLY | PTR_UNTRUSTED : 20093 PTR_UNTRUSTED; 20094 else 20095 type_merged |= ((type_a | type_b) & MEM_RDONLY); 20096 return type_merged; 20097 } 20098 20099 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, 20100 bool allow_trust_mismatch) 20101 { 20102 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; 20103 20104 if (*prev_type == NOT_INIT) { 20105 /* Saw a valid insn 20106 * dst_reg = *(u32 *)(src_reg + off) 20107 * save type to validate intersecting paths 20108 */ 20109 *prev_type = type; 20110 } else if (reg_type_mismatch(type, *prev_type)) { 20111 /* Abuser program is trying to use the same insn 20112 * dst_reg = *(u32*) (src_reg + off) 20113 * with different pointer types: 20114 * src_reg == ctx in one branch and 20115 * src_reg == stack|map in some other branch. 20116 * Reject it. 20117 */ 20118 if (allow_trust_mismatch && 20119 bpf_is_ptr_to_mem_or_btf_id(type) && 20120 bpf_is_ptr_to_mem_or_btf_id(*prev_type)) { 20121 /* 20122 * Have to support a use case when one path through 20123 * the program yields a TRUSTED pointer while another 20124 * is UNTRUSTED. Merge them into a type which keeps 20125 * the BPF_PROBE_MEM/BPF_PROBE_MEMSX rewrite when 20126 * either side needs it. 20127 */ 20128 *prev_type = merge_ptr_types(type, *prev_type); 20129 } else { 20130 verbose(env, "same insn cannot be used with different pointers\n"); 20131 return -EINVAL; 20132 } 20133 } 20134 20135 return 0; 20136 } 20137 20138 enum { 20139 PROCESS_BPF_EXIT = 1, 20140 INSN_IDX_UPDATED = 2, 20141 }; 20142 20143 static int process_bpf_exit_full(struct bpf_verifier_env *env, 20144 bool *do_print_state, 20145 bool exception_exit) 20146 { 20147 struct bpf_func_state *cur_frame = cur_func(env); 20148 20149 /* We must do check_reference_leak here before 20150 * prepare_func_exit to handle the case when 20151 * state->curframe > 0, it may be a callback function, 20152 * for which reference_state must match caller reference 20153 * state when it exits. 20154 */ 20155 int err = check_resource_leak(env, exception_exit, 20156 exception_exit || !env->cur_state->curframe, 20157 exception_exit ? "bpf_throw" : 20158 "BPF_EXIT instruction in main prog"); 20159 if (err) 20160 return err; 20161 20162 /* The side effect of the prepare_func_exit which is 20163 * being skipped is that it frees bpf_func_state. 20164 * Typically, process_bpf_exit will only be hit with 20165 * outermost exit. copy_verifier_state in pop_stack will 20166 * handle freeing of any extra bpf_func_state left over 20167 * from not processing all nested function exits. We 20168 * also skip return code checks as they are not needed 20169 * for exceptional exits. 20170 */ 20171 if (exception_exit) 20172 return PROCESS_BPF_EXIT; 20173 20174 if (env->cur_state->curframe) { 20175 /* exit from nested function */ 20176 err = prepare_func_exit(env, &env->insn_idx); 20177 if (err) 20178 return err; 20179 *do_print_state = true; 20180 return INSN_IDX_UPDATED; 20181 } 20182 20183 /* 20184 * Return from a regular global subprogram differs from return 20185 * from the main program or async/exception callback. 20186 * Main program exit implies return code restrictions 20187 * that depend on program type. 20188 * Exit from exception callback is equivalent to main program exit. 20189 * Exit from async callback implies return code restrictions 20190 * that depend on async scheduling mechanism. 20191 */ 20192 if (cur_frame->subprogno && 20193 !cur_frame->in_async_callback_fn && 20194 !cur_frame->in_exception_callback_fn) 20195 err = check_global_subprog_return_code(env); 20196 else 20197 err = check_return_code(env, BPF_REG_0, "R0"); 20198 if (err) 20199 return err; 20200 return PROCESS_BPF_EXIT; 20201 } 20202 20203 static int indirect_jump_min_max_index(struct bpf_verifier_env *env, 20204 int regno, 20205 struct bpf_map *map, 20206 u32 *pmin_index, u32 *pmax_index) 20207 { 20208 struct bpf_reg_state *reg = reg_state(env, regno); 20209 u64 min_index = reg_umin(reg); 20210 u64 max_index = reg_umax(reg); 20211 const u32 size = 8; 20212 20213 if (min_index > (u64) U32_MAX * size) { 20214 verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg)); 20215 return -ERANGE; 20216 } 20217 if (max_index > (u64) U32_MAX * size) { 20218 verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg)); 20219 return -ERANGE; 20220 } 20221 20222 min_index /= size; 20223 max_index /= size; 20224 20225 if (max_index >= map->max_entries) { 20226 verbose(env, "R%u points to outside of jump table: [%llu,%llu] max_entries %u\n", 20227 regno, min_index, max_index, map->max_entries); 20228 return -EINVAL; 20229 } 20230 20231 *pmin_index = min_index; 20232 *pmax_index = max_index; 20233 return 0; 20234 } 20235 20236 static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target, 20237 u32 subprog_start, u32 subprog_end) 20238 { 20239 verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n", 20240 env->insn_idx, target, subprog_start, subprog_end); 20241 bpf_diag_program_structure( 20242 env, env->insn_idx, "indirect jump leaves subprogram", 20243 "Keep every reachable jump-table target inside the subprogram of the indirect jump.", 20244 "Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).", 20245 env->insn_idx, target, subprog_start, subprog_end); 20246 return -EINVAL; 20247 } 20248 20249 /* gotox *dst_reg */ 20250 static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn) 20251 { 20252 struct bpf_verifier_state *other_branch; 20253 struct bpf_subprog_info *subprog; 20254 u32 subprog_start, subprog_end; 20255 struct bpf_reg_state *dst_reg; 20256 struct bpf_map *map; 20257 u32 min_index, max_index; 20258 int err = 0; 20259 int n; 20260 int i; 20261 20262 dst_reg = reg_state(env, insn->dst_reg); 20263 if (dst_reg->type != PTR_TO_INSN) { 20264 verbose(env, "R%d has type %s, expected PTR_TO_INSN\n", 20265 insn->dst_reg, reg_type_str(env, dst_reg->type)); 20266 return -EINVAL; 20267 } 20268 20269 map = dst_reg->map_ptr; 20270 if (verifier_bug_if(!map, env, "R%d has an empty map pointer", insn->dst_reg)) 20271 return -EFAULT; 20272 20273 if (verifier_bug_if(map->map_type != BPF_MAP_TYPE_INSN_ARRAY, env, 20274 "R%d has incorrect map type %d", insn->dst_reg, map->map_type)) 20275 return -EFAULT; 20276 20277 err = indirect_jump_min_max_index(env, insn->dst_reg, map, &min_index, &max_index); 20278 if (err) 20279 return err; 20280 20281 /* Ensure that the buffer is large enough */ 20282 if (!env->gotox_tmp_buf || env->gotox_tmp_buf->cnt < max_index - min_index + 1) { 20283 env->gotox_tmp_buf = bpf_iarray_realloc(env->gotox_tmp_buf, 20284 max_index - min_index + 1); 20285 if (!env->gotox_tmp_buf) 20286 return -ENOMEM; 20287 } 20288 20289 n = bpf_copy_insn_array_uniq(map, min_index, max_index, env->gotox_tmp_buf->items); 20290 if (n < 0) 20291 return n; 20292 if (n == 0) { 20293 verbose(env, "register R%d doesn't point to any offset in map id=%d\n", 20294 insn->dst_reg, map->id); 20295 return -EINVAL; 20296 } 20297 20298 subprog = bpf_find_containing_subprog(env, env->insn_idx); 20299 if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx)) 20300 return -EFAULT; 20301 subprog_start = subprog->start; 20302 subprog_end = (subprog + 1)->start; 20303 20304 /* 20305 * Maps are confined to a subprog, see compute_subprog_jts(), and the 20306 * targets are sorted and free of duplicates, see sort_insn_array_uniq(), 20307 * so either all of them belong to this subprog or none of them does. 20308 */ 20309 if (env->gotox_tmp_buf->items[0] < subprog_start) 20310 return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0], 20311 subprog_start, subprog_end); 20312 if (env->gotox_tmp_buf->items[n - 1] >= subprog_end) 20313 return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1], 20314 subprog_start, subprog_end); 20315 20316 for (i = 0; i < n - 1; i++) { 20317 mark_indirect_target(env, env->gotox_tmp_buf->items[i]); 20318 other_branch = push_stack(env, env->gotox_tmp_buf->items[i], 20319 env->insn_idx, env->cur_state->speculative); 20320 if (IS_ERR(other_branch)) 20321 return PTR_ERR(other_branch); 20322 } 20323 env->insn_idx = env->gotox_tmp_buf->items[n-1]; 20324 mark_indirect_target(env, env->insn_idx); 20325 return INSN_IDX_UPDATED; 20326 } 20327 20328 static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) 20329 { 20330 int err; 20331 struct bpf_insn *insn = &env->prog->insnsi[env->insn_idx]; 20332 u8 class = BPF_CLASS(insn->code); 20333 20334 switch (class) { 20335 case BPF_ALU: 20336 case BPF_ALU64: 20337 return check_alu_op(env, insn); 20338 20339 case BPF_LDX: 20340 return check_load_mem(env, insn, false, 20341 BPF_MODE(insn->code) == BPF_MEMSX, 20342 true, "ldx"); 20343 20344 case BPF_STX: 20345 if (BPF_MODE(insn->code) == BPF_ATOMIC) 20346 return check_atomic(env, insn); 20347 return check_store_reg(env, insn, false); 20348 20349 case BPF_ST: { 20350 /* Handle stack arg write (store immediate) */ 20351 if (is_stack_arg_st(insn)) { 20352 struct bpf_verifier_state *vstate = env->cur_state; 20353 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 20354 20355 return check_stack_arg_write(env, state, insn->off, NULL); 20356 } 20357 20358 enum bpf_reg_type dst_reg_type; 20359 20360 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 20361 if (err) 20362 return err; 20363 20364 dst_reg_type = cur_regs(env)[insn->dst_reg].type; 20365 20366 err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, 20367 argno_from_reg(insn->dst_reg), 20368 insn->off, BPF_SIZE(insn->code), 20369 BPF_WRITE, -1, false, false); 20370 if (err) 20371 return err; 20372 20373 return save_aux_ptr_type(env, dst_reg_type, false); 20374 } 20375 case BPF_JMP: 20376 case BPF_JMP32: { 20377 u8 opcode = BPF_OP(insn->code); 20378 20379 env->jmps_processed++; 20380 if (opcode == BPF_CALL) { 20381 if (env->cur_state->active_locks) { 20382 /* similar to static subprog calls callx is allowed under a lock */ 20383 if (!bpf_is_callx(insn) && 20384 ((insn->src_reg == BPF_REG_0 && 20385 insn->imm != BPF_FUNC_spin_unlock && 20386 insn->imm != BPF_FUNC_kptr_xchg) || 20387 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 20388 !kfunc_spin_allowed(env, insn->imm, insn->off)))) { 20389 verbose(env, 20390 "function calls are not allowed while holding a lock\n"); 20391 bpf_diag_ctx_active( 20392 env, env->insn_idx, 20393 "function call", BPF_DIAG_CONTEXT_LOCK, 20394 "Release the BPF spin lock before making this call, or move the call outside the locked region."); 20395 return -EINVAL; 20396 } 20397 } 20398 mark_reg_scratched(env, BPF_REG_0); 20399 if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) 20400 cur_func(env)->no_stack_arg_load = true; 20401 if (bpf_is_callx(insn)) 20402 return check_func_callx(env, insn, &env->insn_idx); 20403 if (insn->src_reg == BPF_PSEUDO_CALL) 20404 return check_func_call(env, insn, &env->insn_idx); 20405 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 20406 return check_kfunc_call(env, insn, &env->insn_idx); 20407 return check_helper_call(env, insn, &env->insn_idx); 20408 } else if (opcode == BPF_JA) { 20409 if (BPF_SRC(insn->code) == BPF_X) 20410 return check_indirect_jump(env, insn); 20411 20412 if (class == BPF_JMP) 20413 env->insn_idx += insn->off + 1; 20414 else 20415 env->insn_idx += insn->imm + 1; 20416 return INSN_IDX_UPDATED; 20417 } else if (opcode == BPF_EXIT) { 20418 return process_bpf_exit_full(env, do_print_state, false); 20419 } 20420 return check_cond_jmp_op(env, insn, &env->insn_idx); 20421 } 20422 case BPF_LD: { 20423 u8 mode = BPF_MODE(insn->code); 20424 20425 if (mode == BPF_ABS || mode == BPF_IND) 20426 return check_ld_abs(env, insn); 20427 20428 if (mode == BPF_IMM) { 20429 err = check_ld_imm(env, insn); 20430 if (err) 20431 return err; 20432 20433 env->insn_idx++; 20434 sanitize_mark_insn_seen(env); 20435 } 20436 return 0; 20437 } 20438 } 20439 /* all class values are handled above. silence compiler warning */ 20440 return -EFAULT; 20441 } 20442 20443 /* 20444 * Push the loop entered at insn_idx onto the stack. Usually this adds a single 20445 * entry on top of its enclosing loop. However, an edge may enter an inner loop 20446 * directly, bypassing the header(s) of its enclosing loop(s), e.g.: 20447 * 20448 * 1: for (...): // enclosing loop, header at 1 20449 * 2: for (...): // inner loop, header at 2 20450 * ... 20451 * 3: if ...: 20452 * goto 2b; // enters loop 2 without going through header 1 20453 * 20454 * Such a bypass makes the enclosing loop irreducible, so its header is missing 20455 * from the stack and both headers (1) and (2) need to be pushed onto stack. 20456 * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds; 20457 * the bypassed ancestors are irreducible and pushed as non-terminating. 20458 * 20459 * Assumes loop_stack_pop() has already truncated the stack to the common ancestor 20460 * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack. 20461 */ 20462 static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed) 20463 { 20464 struct bpf_insn_aux_data *aux = env->insn_aux_data; 20465 struct bpf_func_state *frame = cur_func(env); 20466 struct loop_stack_entry *loop_stack = frame->loop_stack; 20467 u32 missing_headers[LOOP_STACK_SIZE]; 20468 u32 cnt = frame->loop_stack_cnt; 20469 u32 num_missing = 0; 20470 int h; 20471 20472 *pushed = false; 20473 20474 for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) { 20475 if (cnt && loop_stack[cnt - 1].loop_id == h) 20476 break; 20477 if (num_missing == LOOP_STACK_SIZE) 20478 goto e2big; 20479 missing_headers[num_missing] = h; 20480 } 20481 20482 if (num_missing == 0) 20483 return 0; 20484 if (cnt + num_missing > LOOP_STACK_SIZE) 20485 goto e2big; 20486 frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing, 20487 sizeof(*frame->loop_stack)); 20488 if (!frame->loop_stack) 20489 return -ENOMEM; 20490 loop_stack = frame->loop_stack; 20491 20492 for (; num_missing; num_missing--, cnt++) { 20493 h = missing_headers[num_missing - 1]; 20494 loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h }; 20495 if (env->log.level & BPF_LOG_LEVEL2) 20496 verbose(env, "entering loop %d\n", h); 20497 *pushed = true; 20498 } 20499 frame->loop_stack_cnt = cnt; 20500 return 0; 20501 20502 e2big: 20503 verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx); 20504 return -E2BIG; 20505 } 20506 20507 /* 20508 * An exit from a loop can cross several nested loops, e.g.: 20509 * 20510 * 1: for (...): 20511 * 2: for (...): 20512 * 3: for (...): 20513 * if ...: // before goto the loop stack is [1, 2, 3 <top>] 20514 * goto 1b; // after goto it should become [1] 20515 */ 20516 static int loop_stack_pop(struct bpf_verifier_env *env) 20517 { 20518 struct bpf_insn_aux_data *aux = env->insn_aux_data; 20519 struct bpf_func_state *frame = cur_func(env); 20520 int new_cnt = 0; 20521 int h, i; 20522 20523 /* 20524 * Walk the loop nest of insn_idx outwards (innermost first) and stop at 20525 * the first loop that is present on the stack - that loop is the common 20526 * ancestor and becomes the new top. If no enclosing loop is on the 20527 * stack, drop all loops. 20528 */ 20529 for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) { 20530 for (i = frame->loop_stack_cnt - 1; i >= 0; i--) { 20531 if (frame->loop_stack[i].loop_id == h) { 20532 new_cnt = i + 1; 20533 goto pop; 20534 } 20535 } 20536 } 20537 20538 pop: 20539 if (env->log.level & BPF_LOG_LEVEL2) 20540 for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--) 20541 verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id); 20542 frame->loop_stack_cnt = new_cnt; 20543 return 0; 20544 } 20545 20546 static int maybe_clamp_scev_regs(struct bpf_verifier_env *env) 20547 { 20548 struct bpf_func_state *frame = cur_func(env); 20549 struct loop_stack_entry *entry; 20550 20551 if (frame->loop_stack_cnt == 0) { 20552 verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx); 20553 return -EFAULT; 20554 } 20555 20556 entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; 20557 if (!entry->terminates) 20558 return 0; 20559 20560 return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state, 20561 entry->max_header_count); 20562 } 20563 20564 static int handle_loop_entry_exit(struct bpf_verifier_env *env) 20565 { 20566 struct bpf_verifier_state *cur = env->cur_state; 20567 struct bpf_func_state *frame = cur_func(env); 20568 struct loop_stack_entry *entry; 20569 struct bpf_loop_iters iters; 20570 bool pushed, terminates; 20571 int err; 20572 20573 err = loop_stack_pop(env); 20574 if (err) 20575 return err; 20576 20577 err = loop_stack_push(env, &pushed); 20578 if (err) 20579 return err; 20580 20581 if (frame->loop_stack_cnt == 0) 20582 return 0; 20583 20584 entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; 20585 if (env->insn_idx != entry->loop_id) 20586 return 0; 20587 20588 /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */ 20589 if (!pushed) 20590 return maybe_clamp_scev_regs(env); 20591 20592 err = bpf_compute_loop_iters(env, env->cur_state, &iters); 20593 if (err < 0) 20594 return err; 20595 20596 terminates = err == 1; 20597 if (!terminates) 20598 return 0; 20599 20600 /* Create a loop_entry checkpoint before widening */ 20601 err = bpf_split_cur_state(env); 20602 if (err) 20603 return err; 20604 20605 entry->min_header_count = iters.min_header_count; 20606 entry->max_header_count = iters.max_header_count; 20607 entry->entry_state = cur->parent; 20608 entry->terminates = true; 20609 err = bpf_widen_scev_regs(env, cur, cur->parent, &iters); 20610 if (err < 0) 20611 return err; 20612 return 0; 20613 } 20614 20615 static int do_check(struct bpf_verifier_env *env) 20616 { 20617 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 20618 struct bpf_verifier_state *state = env->cur_state; 20619 struct bpf_insn *insns = env->prog->insnsi; 20620 int insn_cnt = env->prog->len; 20621 bool do_print_state = false; 20622 int prev_insn_idx = -1; 20623 20624 for (;;) { 20625 struct bpf_insn *insn; 20626 struct bpf_insn_aux_data *insn_aux; 20627 int err; 20628 20629 if (signal_pending(current)) 20630 return -EAGAIN; 20631 20632 cond_resched_tasks_rcu_qs(); 20633 20634 /* reset current history entry on each new instruction */ 20635 env->cur_hist_ent = NULL; 20636 20637 env->prev_insn_idx = prev_insn_idx; 20638 if (env->insn_idx >= insn_cnt) { 20639 verbose(env, "invalid insn idx %d insn_cnt %d\n", 20640 env->insn_idx, insn_cnt); 20641 return -EFAULT; 20642 } 20643 20644 insn = &insns[env->insn_idx]; 20645 insn_aux = &env->insn_aux_data[env->insn_idx]; 20646 20647 account_processed_insn(env); 20648 20649 if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 20650 verbose(env, 20651 "BPF program is too large. Processed %d insn\n", 20652 env->insn_processed); 20653 return -E2BIG; 20654 } 20655 20656 state->last_insn_idx = env->prev_insn_idx; 20657 state->insn_idx = env->insn_idx; 20658 /* 20659 * Record the incoming edge so active and queued paths use the same 20660 * branch-recording path. A zero-offset conditional has identical 20661 * successors, so its outcome cannot be reconstructed from the edge. 20662 */ 20663 if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) { 20664 struct bpf_insn *prev_insn = &insns[prev_insn_idx]; 20665 int fallthrough_idx = prev_insn_idx + 1; 20666 int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1; 20667 u8 class = BPF_CLASS(prev_insn->code); 20668 u8 opcode = BPF_OP(prev_insn->code); 20669 20670 if ((class == BPF_JMP || class == BPF_JMP32) && 20671 opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT && 20672 opcode <= BPF_JCOND && branch_idx != fallthrough_idx) { 20673 if (env->insn_idx == branch_idx) 20674 bpf_diag_record_branch(env, prev_insn_idx, true); 20675 else if (env->insn_idx == fallthrough_idx) 20676 bpf_diag_record_branch(env, prev_insn_idx, false); 20677 } 20678 } 20679 20680 /* 20681 * Possibly widen the registers before creating a checkpoint 20682 * in bpf_is_state_visited(). The next loop iteration will 20683 * have a chance to hit this checkpoint and converge. 20684 */ 20685 err = handle_loop_entry_exit(env); 20686 if (err) 20687 return err; 20688 20689 if (bpf_is_prune_point(env, env->insn_idx)) { 20690 err = bpf_is_state_visited(env, env->insn_idx); 20691 if (err < 0) 20692 return err; 20693 if (err == 1) { 20694 /* found equivalent state, can prune the search */ 20695 if (env->log.level & BPF_LOG_LEVEL) { 20696 if (do_print_state) 20697 verbose(env, "\nfrom %d to %d%s: safe\n", 20698 env->prev_insn_idx, env->insn_idx, 20699 env->cur_state->speculative ? 20700 " (speculative execution)" : ""); 20701 else 20702 verbose(env, "%d: safe\n", env->insn_idx); 20703 } 20704 goto process_bpf_exit; 20705 } 20706 } 20707 20708 if (bpf_is_jmp_point(env, env->insn_idx)) { 20709 err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0); 20710 if (err) 20711 return err; 20712 } 20713 20714 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 20715 verbose(env, "\nfrom %d to %d%s:", 20716 env->prev_insn_idx, env->insn_idx, 20717 env->cur_state->speculative ? 20718 " (speculative execution)" : ""); 20719 print_verifier_state(env, state, state->curframe, true); 20720 do_print_state = false; 20721 } 20722 20723 if (bpf_loop_at_index(env, env->insn_idx) >= 0 && 20724 cur_func(env)->loop_stack_cnt == 0) { 20725 verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n", 20726 env->insn_idx, bpf_loop_at_index(env, env->insn_idx)); 20727 return -EFAULT; 20728 } 20729 20730 if (env->log.level & BPF_LOG_LEVEL) { 20731 if (verifier_state_scratched(env)) 20732 print_insn_state(env, state, state->curframe); 20733 20734 verbose_linfo(env, env->insn_idx, "; "); 20735 env->prev_log_pos = env->log.end_pos; 20736 verbose(env, "%d: ", env->insn_idx); 20737 bpf_verbose_insn(env, insn); 20738 verbose(env, "\n"); 20739 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; 20740 env->prev_log_pos = env->log.end_pos; 20741 } 20742 20743 if (bpf_prog_is_offloaded(env->prog->aux)) { 20744 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 20745 env->prev_insn_idx); 20746 if (err) 20747 return err; 20748 } 20749 20750 sanitize_mark_insn_seen(env); 20751 prev_insn_idx = env->insn_idx; 20752 20753 /* Sanity check: precomputed constants must match verifier state */ 20754 if (!state->speculative && insn_aux->const_reg_mask) { 20755 struct bpf_reg_state *regs = cur_regs(env); 20756 u16 mask = insn_aux->const_reg_mask; 20757 20758 for (int r = 0; r < ARRAY_SIZE(insn_aux->const_reg_vals); r++) { 20759 u32 cval = insn_aux->const_reg_vals[r]; 20760 20761 if (!(mask & BIT(r))) 20762 continue; 20763 if (regs[r].type != SCALAR_VALUE) 20764 continue; 20765 if (!tnum_is_const(regs[r].var_off)) 20766 continue; 20767 if (verifier_bug_if((u32)regs[r].var_off.value != cval, 20768 env, "const R%d: %u != %llu", 20769 r, cval, regs[r].var_off.value)) 20770 return -EFAULT; 20771 } 20772 } 20773 20774 /* Reduce verification complexity by stopping speculative path 20775 * verification when a nospec is encountered. 20776 */ 20777 if (state->speculative && insn_aux->nospec) 20778 goto process_bpf_exit; 20779 20780 err = do_check_insn(env, &do_print_state); 20781 if (error_recoverable_with_nospec(err) && state->speculative) { 20782 /* Prevent this speculative path from ever reaching the 20783 * insn that would have been unsafe to execute. 20784 */ 20785 insn_aux->nospec = true; 20786 /* If it was an ADD/SUB insn, potentially remove any 20787 * markings for alu sanitization. 20788 */ 20789 insn_aux->alu_state = 0; 20790 goto process_bpf_exit; 20791 } else if (err < 0) { 20792 return err; 20793 } else if (err == PROCESS_BPF_EXIT) { 20794 goto process_bpf_exit; 20795 } else if (err == INSN_IDX_UPDATED) { 20796 } else if (err == 0) { 20797 env->insn_idx++; 20798 } 20799 20800 if (state->speculative && insn_aux->nospec_result) { 20801 /* If we are on a path that performed a jump-op, this 20802 * may skip a nospec patched-in after the jump. This can 20803 * currently never happen because nospec_result is only 20804 * used for the write-ops 20805 * `*(size*)(dst_reg+off)=src_reg|imm32` and helper 20806 * calls. These must never skip the following insn 20807 * (i.e., bpf_insn_successors()'s opcode_info.can_jump 20808 * is false). Still, add a warning to document this in 20809 * case nospec_result is used elsewhere in the future. 20810 * 20811 * All non-branch instructions have a single 20812 * fall-through edge. For these, nospec_result should 20813 * already work. 20814 */ 20815 if (verifier_bug_if((BPF_CLASS(insn->code) == BPF_JMP || 20816 BPF_CLASS(insn->code) == BPF_JMP32) && 20817 BPF_OP(insn->code) != BPF_CALL, env, 20818 "speculation barrier after jump instruction may not have the desired effect")) 20819 return -EFAULT; 20820 process_bpf_exit: 20821 account_current_path(env); 20822 mark_verifier_state_scratched(env); 20823 err = bpf_update_branch_counts(env, env->cur_state); 20824 if (err) 20825 return err; 20826 err = pop_stack(env, &prev_insn_idx, &env->insn_idx, 20827 pop_log); 20828 if (err < 0) { 20829 if (err != -ENOENT) 20830 return err; 20831 break; 20832 } else { 20833 do_print_state = true; 20834 continue; 20835 } 20836 } 20837 } 20838 20839 return 0; 20840 } 20841 20842 static int find_btf_percpu_datasec(struct btf *btf) 20843 { 20844 const struct btf_type *t; 20845 const char *tname; 20846 int i, n; 20847 20848 /* 20849 * Both vmlinux and module each have their own ".data..percpu" 20850 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 20851 * types to look at only module's own BTF types. 20852 */ 20853 n = btf_nr_types(btf); 20854 for (i = btf_named_start_id(btf, true); i < n; i++) { 20855 t = btf_type_by_id(btf, i); 20856 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 20857 continue; 20858 20859 tname = btf_name_by_offset(btf, t->name_off); 20860 if (!strcmp(tname, ".data..percpu")) 20861 return i; 20862 } 20863 20864 return -ENOENT; 20865 } 20866 20867 /* 20868 * Add btf to the env->used_btfs array. If needed, refcount the 20869 * corresponding kernel module. To simplify caller's logic 20870 * in case of error or if btf was added before the function 20871 * decreases the btf refcount. 20872 */ 20873 static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) 20874 { 20875 struct btf_mod_pair *btf_mod; 20876 int ret = 0; 20877 int i; 20878 20879 /* check whether we recorded this BTF (and maybe module) already */ 20880 for (i = 0; i < env->used_btf_cnt; i++) 20881 if (env->used_btfs[i].btf == btf) 20882 goto ret_put; 20883 20884 if (env->signature) { 20885 verbose(env, "signed program cannot bind any BTF\n"); 20886 ret = -EACCES; 20887 goto ret_put; 20888 } 20889 if (env->used_btf_cnt >= MAX_USED_BTFS) { 20890 verbose(env, "The total number of btfs per program has reached the limit of %u\n", 20891 MAX_USED_BTFS); 20892 ret = -E2BIG; 20893 goto ret_put; 20894 } 20895 20896 btf_mod = &env->used_btfs[env->used_btf_cnt]; 20897 btf_mod->btf = btf; 20898 btf_mod->module = NULL; 20899 20900 /* if we reference variables from kernel module, bump its refcount */ 20901 if (btf_is_module(btf)) { 20902 btf_mod->module = btf_try_get_module(btf); 20903 if (!btf_mod->module) { 20904 ret = -ENXIO; 20905 goto ret_put; 20906 } 20907 } 20908 20909 env->used_btf_cnt++; 20910 return 0; 20911 20912 ret_put: 20913 /* Either error or this BTF was already added */ 20914 btf_put(btf); 20915 return ret; 20916 } 20917 20918 /* replace pseudo btf_id with kernel symbol address */ 20919 static int __check_pseudo_btf_id(struct bpf_verifier_env *env, 20920 struct bpf_insn *insn, 20921 struct bpf_insn_aux_data *aux, 20922 struct btf *btf) 20923 { 20924 const struct btf_var_secinfo *vsi; 20925 const struct btf_type *datasec; 20926 const struct btf_type *t; 20927 const char *sym_name; 20928 bool percpu = false; 20929 u32 type, id = insn->imm; 20930 s32 datasec_id; 20931 u64 addr; 20932 int i; 20933 20934 t = btf_type_by_id(btf, id); 20935 if (!t) { 20936 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 20937 return -ENOENT; 20938 } 20939 20940 if (!btf_type_is_var(t) && !btf_type_is_func(t)) { 20941 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); 20942 return -EINVAL; 20943 } 20944 20945 sym_name = btf_name_by_offset(btf, t->name_off); 20946 addr = kallsyms_lookup_name(sym_name); 20947 if (!addr) { 20948 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 20949 sym_name); 20950 return -ENOENT; 20951 } 20952 insn[0].imm = (u32)addr; 20953 insn[1].imm = addr >> 32; 20954 20955 if (btf_type_is_func(t)) { 20956 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 20957 aux->btf_var.mem_size = 0; 20958 return 0; 20959 } 20960 20961 datasec_id = find_btf_percpu_datasec(btf); 20962 if (datasec_id > 0) { 20963 datasec = btf_type_by_id(btf, datasec_id); 20964 for_each_vsi(i, datasec, vsi) { 20965 if (vsi->type == id) { 20966 percpu = true; 20967 break; 20968 } 20969 } 20970 } 20971 20972 type = t->type; 20973 t = btf_type_skip_modifiers(btf, type, NULL); 20974 if (percpu) { 20975 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 20976 aux->btf_var.btf = btf; 20977 aux->btf_var.btf_id = type; 20978 } else if (!btf_type_is_struct(t)) { 20979 const struct btf_type *ret; 20980 const char *tname; 20981 u32 tsize; 20982 20983 /* resolve the type size of ksym. */ 20984 ret = btf_resolve_size(btf, t, &tsize); 20985 if (IS_ERR(ret)) { 20986 tname = btf_name_by_offset(btf, t->name_off); 20987 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 20988 tname, PTR_ERR(ret)); 20989 return -EINVAL; 20990 } 20991 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 20992 aux->btf_var.mem_size = tsize; 20993 } else { 20994 aux->btf_var.reg_type = PTR_TO_BTF_ID; 20995 aux->btf_var.btf = btf; 20996 aux->btf_var.btf_id = type; 20997 } 20998 20999 return 0; 21000 } 21001 21002 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 21003 struct bpf_insn *insn, 21004 struct bpf_insn_aux_data *aux) 21005 { 21006 struct btf *btf; 21007 int btf_fd; 21008 int err; 21009 21010 btf_fd = insn[1].imm; 21011 if (btf_fd) { 21012 btf = btf_get_by_fd(btf_fd); 21013 if (IS_ERR(btf)) { 21014 verbose(env, "invalid module BTF object FD specified.\n"); 21015 return -EINVAL; 21016 } 21017 } else { 21018 if (!btf_vmlinux) { 21019 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 21020 return -EINVAL; 21021 } 21022 btf_get(btf_vmlinux); 21023 btf = btf_vmlinux; 21024 } 21025 21026 err = __check_pseudo_btf_id(env, insn, aux, btf); 21027 if (err) { 21028 btf_put(btf); 21029 return err; 21030 } 21031 21032 return __add_used_btf(env, btf); 21033 } 21034 21035 static bool is_tracing_prog_type(enum bpf_prog_type type) 21036 { 21037 switch (type) { 21038 case BPF_PROG_TYPE_KPROBE: 21039 case BPF_PROG_TYPE_TRACEPOINT: 21040 case BPF_PROG_TYPE_PERF_EVENT: 21041 case BPF_PROG_TYPE_RAW_TRACEPOINT: 21042 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 21043 return true; 21044 default: 21045 return false; 21046 } 21047 } 21048 21049 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 21050 { 21051 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 21052 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 21053 } 21054 21055 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 21056 struct bpf_map *map, 21057 struct bpf_prog *prog) 21058 21059 { 21060 enum bpf_prog_type prog_type = resolve_prog_type(prog); 21061 21062 if (btf_record_has_field(map->record, BPF_LIST_HEAD) || 21063 btf_record_has_field(map->record, BPF_RB_ROOT)) { 21064 if (is_tracing_prog_type(prog_type)) { 21065 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); 21066 return -EINVAL; 21067 } 21068 } 21069 21070 if (btf_record_has_field(map->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { 21071 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 21072 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 21073 return -EINVAL; 21074 } 21075 } 21076 21077 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 21078 if (is_tracing_prog_type(prog_type)) { 21079 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 21080 return -EINVAL; 21081 } 21082 } 21083 21084 if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && 21085 !bpf_offload_prog_map_match(prog, map)) { 21086 verbose(env, "offload device mismatch between prog and map\n"); 21087 return -EINVAL; 21088 } 21089 21090 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 21091 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 21092 return -EINVAL; 21093 } 21094 21095 if (prog->sleepable && !map_needs_no_rcu(map)) { 21096 verbose(env, 21097 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 21098 return -EINVAL; 21099 } 21100 21101 if (bpf_map_is_cgroup_storage(map) && 21102 bpf_cgroup_storage_assign(env->prog->aux, map)) { 21103 verbose(env, "only one cgroup storage of each type is allowed\n"); 21104 return -EBUSY; 21105 } 21106 21107 if (map->map_type == BPF_MAP_TYPE_ARENA) { 21108 if (env->prog->aux->arena) { 21109 verbose(env, "Only one arena per program\n"); 21110 return -EBUSY; 21111 } 21112 if (!env->allow_ptr_leaks || !env->bpf_capable) { 21113 verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); 21114 return -EPERM; 21115 } 21116 if (!env->prog->jit_requested) { 21117 verbose(env, "JIT is required to use arena\n"); 21118 return -EOPNOTSUPP; 21119 } 21120 if (!bpf_jit_supports_arena()) { 21121 verbose(env, "JIT doesn't support arena\n"); 21122 return -EOPNOTSUPP; 21123 } 21124 env->prog->aux->arena = (void *)map; 21125 env->prog->jit_required = true; 21126 if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { 21127 verbose(env, "arena's user address must be set via map_extra or mmap()\n"); 21128 return -EINVAL; 21129 } 21130 } 21131 21132 return 0; 21133 } 21134 21135 /* 21136 * Keep track of whether the map is used by one program only. Such program may 21137 * store the addresses of its functions into the map when it's frozen, see 21138 * resolve_func_ptrs(), since nothing else relies on what the map has. After 21139 * that the map is not available to other programs. 21140 */ 21141 static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) 21142 { 21143 unsigned long me = (unsigned long)env->prog->aux, old; 21144 21145 for (;;) { 21146 old = READ_ONCE(map->user); 21147 if (old == me || old == BPF_MAP_USER_MANY) 21148 return 0; 21149 if (old & BPF_MAP_USER_PATCHED) { 21150 verbose(env, "map '%s' has addresses of functions of another program\n", 21151 map->name); 21152 return -EBUSY; 21153 } 21154 if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) 21155 return 0; 21156 } 21157 } 21158 21159 /* 21160 * An exclusive map is bound to the digest of its program. The digest covers 21161 * the insns as they are verified, after the CO-RE relocations the kernel 21162 * applies, so it exists only once check_and_resolve_insns() has computed it, 21163 * and the maps are checked against it there, whether an insn referred to 21164 * them or a continuous fd_array bound them before. 21165 */ 21166 static int check_excl_maps(struct bpf_verifier_env *env) 21167 { 21168 int i; 21169 21170 for (i = 0; i < env->used_map_cnt; i++) { 21171 struct bpf_map *map = env->used_maps[i]; 21172 21173 if (map->excl_prog_sha && 21174 memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { 21175 verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); 21176 return -EACCES; 21177 } 21178 } 21179 return 0; 21180 } 21181 21182 /* 21183 * The program may use every map it referred to, see check_excl_maps(). Only now 21184 * does it leave a trace on them: it claims each as its user, see 21185 * bpf_map_claim(), and takes over its insn arrays. A program rejected before 21186 * this point, one that is not the owner of an exclusive map in particular, 21187 * leaves the maps as it found them. Nothing consults a claim before 21188 * resolve_func_ptrs() and the fixup passes, which run later. 21189 */ 21190 static int claim_used_maps(struct bpf_verifier_env *env) 21191 { 21192 int i, err; 21193 21194 for (i = 0; i < env->used_map_cnt; i++) { 21195 struct bpf_map *map = env->used_maps[i]; 21196 21197 err = bpf_map_claim(env, map); 21198 if (err) 21199 return err; 21200 21201 if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) 21202 continue; 21203 err = bpf_insn_array_init(map, env->prog); 21204 if (err) { 21205 verbose(env, "Failed to properly initialize insn array\n"); 21206 return err; 21207 } 21208 env->insn_array_maps[env->insn_array_map_cnt++] = map; 21209 env->prog->jit_required = true; 21210 } 21211 return 0; 21212 } 21213 21214 static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) 21215 { 21216 int i, err; 21217 21218 /* check whether we recorded this map already */ 21219 for (i = 0; i < env->used_map_cnt; i++) 21220 if (env->used_maps[i] == map) 21221 return i; 21222 21223 if (env->signature && 21224 env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) { 21225 verbose(env, "signed program cannot bind map '%s' not covered by the signature\n", 21226 map->name); 21227 return -EACCES; 21228 } 21229 if (env->used_map_cnt >= MAX_USED_MAPS) { 21230 verbose(env, "The total number of maps per program has reached the limit of %u\n", 21231 MAX_USED_MAPS); 21232 return -E2BIG; 21233 } 21234 21235 err = check_map_prog_compatibility(env, map, env->prog); 21236 if (err) 21237 return err; 21238 21239 if (bpf_prog_runs_sleepable(env->prog)) 21240 atomic64_inc(&map->sleepable_refcnt); 21241 21242 /* hold the map. If the program is rejected by verifier, 21243 * the map will be released by release_maps() or it 21244 * will be used by the valid program until it's unloaded 21245 * and all maps are released in bpf_free_used_maps() 21246 */ 21247 bpf_map_inc(map); 21248 21249 env->used_maps[env->used_map_cnt++] = map; 21250 21251 return env->used_map_cnt - 1; 21252 } 21253 21254 /* Add map behind fd to used maps list, if it's not already there, and return 21255 * its index. 21256 * Returns <0 on error, or >= 0 index, on success. 21257 */ 21258 static int add_used_map(struct bpf_verifier_env *env, int fd) 21259 { 21260 struct bpf_map *map; 21261 CLASS(fd, f)(fd); 21262 21263 map = __bpf_map_get(f); 21264 if (IS_ERR(map)) { 21265 verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); 21266 return PTR_ERR(map); 21267 } 21268 21269 return __add_used_map(env, map); 21270 } 21271 21272 static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx) 21273 { 21274 struct bpf_map *map; 21275 21276 if (idx >= env->fd_array_cnt) { 21277 verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n", 21278 idx, env->fd_array_cnt); 21279 return -EINVAL; 21280 } 21281 map = fd_slot_map(env->fd_array[idx]); 21282 if (!map) { 21283 verbose(env, "fd_idx %u is not a map\n", idx); 21284 return -EINVAL; 21285 } 21286 return __add_used_map(env, map); 21287 } 21288 21289 static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx) 21290 { 21291 int fd; 21292 21293 if (copy_from_bpfptr_offset(&fd, env->fd_array_raw, 21294 (size_t)idx * sizeof(fd), sizeof(fd))) 21295 return -EFAULT; 21296 return add_used_map(env, fd); 21297 } 21298 21299 static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx) 21300 { 21301 if (env->fd_array) 21302 return fd_array_get_map_idx_continuous(env, idx); 21303 if (env->signature) { 21304 verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n"); 21305 return -EACCES; 21306 } 21307 if (!bpfptr_is_null(env->fd_array_raw)) 21308 return fd_array_get_map_idx_sparse(env, idx); 21309 21310 verbose(env, "fd_idx without fd_array is invalid\n"); 21311 return -EPROTO; 21312 } 21313 21314 static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) 21315 { 21316 u8 class = BPF_CLASS(insn->code); 21317 u8 opcode = BPF_OP(insn->code); 21318 21319 switch (opcode) { 21320 case BPF_NEG: 21321 if (BPF_SRC(insn->code) != BPF_K || insn->src_reg != BPF_REG_0 || 21322 insn->off != 0 || insn->imm != 0) { 21323 verbose(env, "BPF_NEG uses reserved fields\n"); 21324 return -EINVAL; 21325 } 21326 return 0; 21327 case BPF_END: 21328 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 21329 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 21330 (class == BPF_ALU64 && BPF_SRC(insn->code) != BPF_TO_LE)) { 21331 verbose(env, "BPF_END uses reserved fields\n"); 21332 return -EINVAL; 21333 } 21334 return 0; 21335 case BPF_MOV: 21336 if (BPF_SRC(insn->code) == BPF_X) { 21337 if (class == BPF_ALU) { 21338 if ((insn->off != 0 && insn->off != 8 && insn->off != 16) || 21339 insn->imm) { 21340 verbose(env, "BPF_MOV uses reserved fields\n"); 21341 return -EINVAL; 21342 } 21343 } else if (insn->off == BPF_ADDR_SPACE_CAST) { 21344 if (insn->imm != 1 && insn->imm != 1u << 16) { 21345 verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); 21346 return -EINVAL; 21347 } 21348 } else if ((insn->off != 0 && insn->off != 8 && 21349 insn->off != 16 && insn->off != 32) || insn->imm) { 21350 verbose(env, "BPF_MOV uses reserved fields\n"); 21351 return -EINVAL; 21352 } 21353 } else if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 21354 verbose(env, "BPF_MOV uses reserved fields\n"); 21355 return -EINVAL; 21356 } 21357 return 0; 21358 case BPF_ADD: 21359 case BPF_SUB: 21360 case BPF_AND: 21361 case BPF_OR: 21362 case BPF_XOR: 21363 case BPF_LSH: 21364 case BPF_RSH: 21365 case BPF_ARSH: 21366 case BPF_MUL: 21367 case BPF_DIV: 21368 case BPF_MOD: 21369 if (BPF_SRC(insn->code) == BPF_X) { 21370 if (insn->imm != 0 || (insn->off != 0 && insn->off != 1) || 21371 (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 21372 verbose(env, "BPF_ALU uses reserved fields\n"); 21373 return -EINVAL; 21374 } 21375 } else if (insn->src_reg != BPF_REG_0 || 21376 (insn->off != 0 && insn->off != 1) || 21377 (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 21378 verbose(env, "BPF_ALU uses reserved fields\n"); 21379 return -EINVAL; 21380 } 21381 return 0; 21382 default: 21383 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 21384 return -EINVAL; 21385 } 21386 } 21387 21388 static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) 21389 { 21390 u8 class = BPF_CLASS(insn->code); 21391 u8 opcode = BPF_OP(insn->code); 21392 21393 switch (opcode) { 21394 case BPF_CALL: 21395 if (bpf_is_callx(insn)) { 21396 /* callx dst_reg */ 21397 if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { 21398 verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); 21399 return -EINVAL; 21400 } 21401 return 0; 21402 } 21403 if (BPF_SRC(insn->code) != BPF_K || 21404 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || 21405 (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && 21406 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 21407 insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { 21408 verbose(env, "BPF_CALL uses reserved fields\n"); 21409 return -EINVAL; 21410 } 21411 return 0; 21412 case BPF_JA: 21413 if (BPF_SRC(insn->code) == BPF_X) { 21414 if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { 21415 verbose(env, "BPF_JA|BPF_X uses reserved fields\n"); 21416 return -EINVAL; 21417 } 21418 } else if (insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || 21419 (class == BPF_JMP && insn->imm != 0) || 21420 (class == BPF_JMP32 && insn->off != 0)) { 21421 verbose(env, "BPF_JA uses reserved fields\n"); 21422 return -EINVAL; 21423 } 21424 return 0; 21425 case BPF_EXIT: 21426 if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 || 21427 insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 || 21428 class == BPF_JMP32) { 21429 verbose(env, "BPF_EXIT uses reserved fields\n"); 21430 return -EINVAL; 21431 } 21432 return 0; 21433 case BPF_JCOND: 21434 if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO || 21435 insn->dst_reg || insn->imm) { 21436 verbose(env, "invalid may_goto imm %d\n", insn->imm); 21437 return -EINVAL; 21438 } 21439 return 0; 21440 default: 21441 if (BPF_SRC(insn->code) == BPF_X) { 21442 if (insn->imm != 0) { 21443 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 21444 return -EINVAL; 21445 } 21446 } else if (insn->src_reg != BPF_REG_0) { 21447 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 21448 return -EINVAL; 21449 } 21450 return 0; 21451 } 21452 } 21453 21454 static int check_insn_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) 21455 { 21456 switch (BPF_CLASS(insn->code)) { 21457 case BPF_ALU: 21458 case BPF_ALU64: 21459 return check_alu_fields(env, insn); 21460 case BPF_LDX: 21461 if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) || 21462 insn->imm != 0) { 21463 verbose(env, "BPF_LDX uses reserved fields\n"); 21464 return -EINVAL; 21465 } 21466 return 0; 21467 case BPF_STX: 21468 if (BPF_MODE(insn->code) == BPF_ATOMIC) 21469 return 0; 21470 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 21471 verbose(env, "BPF_STX uses reserved fields\n"); 21472 return -EINVAL; 21473 } 21474 return 0; 21475 case BPF_ST: 21476 if (BPF_MODE(insn->code) != BPF_MEM || insn->src_reg != BPF_REG_0) { 21477 verbose(env, "BPF_ST uses reserved fields\n"); 21478 return -EINVAL; 21479 } 21480 return 0; 21481 case BPF_JMP: 21482 case BPF_JMP32: 21483 return check_jmp_fields(env, insn); 21484 case BPF_LD: { 21485 u8 mode = BPF_MODE(insn->code); 21486 21487 if (mode == BPF_ABS || mode == BPF_IND) { 21488 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 21489 BPF_SIZE(insn->code) == BPF_DW || 21490 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 21491 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 21492 return -EINVAL; 21493 } 21494 } else if (mode != BPF_IMM) { 21495 verbose(env, "invalid BPF_LD mode\n"); 21496 return -EINVAL; 21497 } 21498 return 0; 21499 } 21500 default: 21501 verbose(env, "unknown insn class %d\n", BPF_CLASS(insn->code)); 21502 return -EINVAL; 21503 } 21504 } 21505 21506 /* 21507 * Check that insns are sane and rewrite pseudo imm in ld_imm64 instructions: 21508 * 21509 * 1. if it accesses map FD, replace it with actual map pointer. 21510 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 21511 * 21512 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 21513 */ 21514 static int check_and_resolve_insns(struct bpf_verifier_env *env) 21515 { 21516 struct bpf_insn *insn = env->prog->insnsi; 21517 int insn_cnt = env->prog->len; 21518 int i, err; 21519 21520 err = bpf_prog_calc_tag(env->prog); 21521 if (err) 21522 return err; 21523 21524 for (i = 0; i < insn_cnt; i++, insn++) { 21525 if (insn->dst_reg >= MAX_BPF_REG && 21526 !is_stack_arg_st(insn) && !is_stack_arg_stx(insn)) { 21527 verbose(env, "R%d is invalid\n", insn->dst_reg); 21528 return -EINVAL; 21529 } 21530 if (insn->src_reg >= MAX_BPF_REG && !is_stack_arg_ldx(insn)) { 21531 verbose(env, "R%d is invalid\n", insn->src_reg); 21532 return -EINVAL; 21533 } 21534 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 21535 struct bpf_insn_aux_data *aux; 21536 struct bpf_map *map; 21537 int map_idx; 21538 u64 addr; 21539 21540 if (i == insn_cnt - 1 || insn[1].code != 0 || 21541 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 21542 insn[1].off != 0) { 21543 verbose(env, "invalid bpf_ld_imm64 insn\n"); 21544 return -EINVAL; 21545 } 21546 21547 if (insn[0].off != 0) { 21548 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 21549 return -EINVAL; 21550 } 21551 21552 if (insn[0].src_reg == 0) 21553 /* valid generic load 64-bit imm */ 21554 goto next_insn; 21555 21556 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 21557 aux = &env->insn_aux_data[i]; 21558 err = check_pseudo_btf_id(env, insn, aux); 21559 if (err) 21560 return err; 21561 goto next_insn; 21562 } 21563 21564 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 21565 aux = &env->insn_aux_data[i]; 21566 aux->ptr_type = PTR_TO_FUNC; 21567 goto next_insn; 21568 } 21569 21570 /* In final convert_pseudo_ld_imm64() step, this is 21571 * converted into regular 64-bit imm load insn. 21572 */ 21573 switch (insn[0].src_reg) { 21574 case BPF_PSEUDO_MAP_VALUE: 21575 case BPF_PSEUDO_MAP_IDX_VALUE: 21576 break; 21577 case BPF_PSEUDO_MAP_FD: 21578 case BPF_PSEUDO_MAP_IDX: 21579 if (insn[1].imm == 0) 21580 break; 21581 fallthrough; 21582 default: 21583 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 21584 return -EINVAL; 21585 } 21586 21587 switch (insn[0].src_reg) { 21588 case BPF_PSEUDO_MAP_IDX_VALUE: 21589 case BPF_PSEUDO_MAP_IDX: 21590 map_idx = fd_array_get_map_idx(env, insn[0].imm); 21591 break; 21592 default: 21593 if (env->signature) { 21594 verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n"); 21595 return -EINVAL; 21596 } 21597 map_idx = add_used_map(env, insn[0].imm); 21598 break; 21599 } 21600 21601 if (map_idx < 0) 21602 return map_idx; 21603 map = env->used_maps[map_idx]; 21604 21605 aux = &env->insn_aux_data[i]; 21606 aux->map_index = map_idx; 21607 21608 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 21609 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 21610 addr = (unsigned long)map; 21611 } else { 21612 u32 off = insn[1].imm; 21613 21614 if (!map->ops->map_direct_value_addr) { 21615 verbose(env, "no direct value access support for this map type\n"); 21616 return -EINVAL; 21617 } 21618 21619 err = map->ops->map_direct_value_addr(map, &addr, off); 21620 if (err) { 21621 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 21622 map->value_size, off); 21623 return err; 21624 } 21625 21626 aux->map_off = off; 21627 addr += off; 21628 } 21629 21630 insn[0].imm = (u32)addr; 21631 insn[1].imm = addr >> 32; 21632 21633 next_insn: 21634 insn++; 21635 i++; 21636 continue; 21637 } 21638 21639 /* Basic sanity check before we invest more work here. */ 21640 if (!bpf_opcode_in_insntable(insn->code)) { 21641 verbose(env, "unknown opcode %02x\n", insn->code); 21642 return -EINVAL; 21643 } 21644 21645 err = check_insn_fields(env, insn); 21646 if (err) 21647 return err; 21648 } 21649 21650 /* now all pseudo BPF_LD_IMM64 instructions load valid 21651 * 'struct bpf_map *' into a register instead of user map_fd. 21652 * These pointers will be used later by verifier to validate map access. 21653 */ 21654 err = check_excl_maps(env); 21655 if (err) 21656 return err; 21657 return claim_used_maps(env); 21658 } 21659 21660 /* that many pointers to functions are recognized in the maps of a program at most */ 21661 #define BPF_MAX_FUNC_PTRS (1 << 20) 21662 21663 static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, 21664 u32 xlated_off) 21665 { 21666 struct bpf_func_ptr *ptrs; 21667 21668 if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { 21669 verbose(env, "too many pointers to functions in maps, the limit is %d\n", 21670 BPF_MAX_FUNC_PTRS); 21671 return -E2BIG; 21672 } 21673 21674 /* grow by doubling, the array is sorted and searched later */ 21675 if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { 21676 ptrs = kvrealloc(env->func_ptrs, 21677 array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), 21678 GFP_KERNEL_ACCOUNT); 21679 if (!ptrs) 21680 return -ENOMEM; 21681 env->func_ptrs = ptrs; 21682 } 21683 env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ 21684 .map = map, 21685 .map_off = map_off, 21686 .orig_off = xlated_off, 21687 .xlated_off = xlated_off, 21688 }; 21689 return 0; 21690 } 21691 21692 /* 21693 * Compilers put pointers to functions into read-only data: tables of functions, 21694 * structures of operations, vtables, where they are mixed with other data. 21695 * The loader stores such data in a frozen read-only array map and resolves 21696 * a pointer to a static function to the offset in bytes of its first 21697 * instruction in the program: the address of the function in the program. 21698 * 21699 * Find 64-bit values that look like that in the maps of a program that uses 21700 * callx. It's a guess. When the value is not a pointer, the program either 21701 * fails to load, because it does with a pointer what can be done with 21702 * a number only, or it sees the address of a function instead of the number. 21703 * It's known before the control flow graph of the program is built and the main 21704 * verification pass begins which functions may be called via callx. 21705 * 21706 * When the program is jitted the offsets are replaced with the addresses of 21707 * the functions in the map itself, see jit_subprogs(). Hence the program has to 21708 * be the only user of the map, see bpf_map_claim(): nothing else may rely on 21709 * what the map had. 21710 * 21711 * The program reads the addresses of its functions from there like any other 21712 * data, so it has to be allowed to leak pointers. 21713 */ 21714 static int resolve_func_ptrs(struct bpf_verifier_env *env) 21715 { 21716 int insn_cnt = env->prog->len; 21717 int i, err, subprog; 21718 struct bpf_map *map; 21719 u64 addr, val; 21720 u32 off; 21721 21722 if (!env->has_callx || !env->allow_ptr_leaks) 21723 return 0; 21724 21725 for (i = 0; i < env->used_map_cnt; i++) { 21726 map = env->used_maps[i]; 21727 /* coincidences in maps that are shared with other programs don't matter */ 21728 if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) 21729 continue; 21730 if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || 21731 !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || 21732 !IS_ERR_OR_NULL(map->record)) 21733 continue; 21734 if (map->ops->map_direct_value_addr(map, &addr, 0)) 21735 continue; 21736 21737 for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { 21738 if (!(off % SZ_64K)) 21739 cond_resched(); 21740 val = *(u64 *)(unsigned long)(addr + off); 21741 if (!val || val % sizeof(struct bpf_insn) || 21742 val / sizeof(struct bpf_insn) >= insn_cnt) 21743 continue; 21744 subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); 21745 if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) 21746 continue; 21747 err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); 21748 if (err) 21749 return err; 21750 } 21751 } 21752 if (env->func_ptr_cnt) 21753 sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), 21754 cmp_func_ptrs, NULL); 21755 return 0; 21756 } 21757 21758 /* drop refcnt of maps used by the rejected program */ 21759 static void release_maps(struct bpf_verifier_env *env) 21760 { 21761 __bpf_free_used_maps(env->prog->aux, env->used_maps, 21762 env->used_map_cnt); 21763 } 21764 21765 /* drop refcnt of maps used by the rejected program */ 21766 static void release_btfs(struct bpf_verifier_env *env) 21767 { 21768 __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt); 21769 } 21770 21771 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 21772 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 21773 { 21774 struct bpf_insn *insn = env->prog->insnsi; 21775 int insn_cnt = env->prog->len; 21776 int i; 21777 21778 for (i = 0; i < insn_cnt; i++, insn++) { 21779 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 21780 continue; 21781 if (insn->src_reg == BPF_PSEUDO_FUNC) 21782 continue; 21783 insn->src_reg = 0; 21784 } 21785 } 21786 21787 static void release_insn_arrays(struct bpf_verifier_env *env) 21788 { 21789 int i; 21790 21791 for (i = 0; i < env->insn_array_map_cnt; i++) 21792 bpf_insn_array_release(env->insn_array_maps[i]); 21793 } 21794 21795 /* The verifier does more data flow analysis than llvm and will not 21796 * explore branches that are dead at run time. Malicious programs can 21797 * have dead code too. Therefore replace all dead at-run-time code 21798 * with 'ja -1'. 21799 * 21800 * Just nops are not optimal, e.g. if they would sit at the end of the 21801 * program and through another bug we would manage to jump there, then 21802 * we'd execute beyond program memory otherwise. Returning exception 21803 * code also wouldn't work since we can have subprogs where the dead 21804 * code could be located. 21805 */ 21806 static void sanitize_dead_code(struct bpf_verifier_env *env) 21807 { 21808 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 21809 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 21810 struct bpf_insn *insn = env->prog->insnsi; 21811 const int insn_cnt = env->prog->len; 21812 int i; 21813 21814 for (i = 0; i < insn_cnt; i++) { 21815 if (aux_data[i].seen) 21816 continue; 21817 memcpy(insn + i, &trap, sizeof(trap)); 21818 aux_data[i].zext_dst = false; 21819 } 21820 } 21821 21822 static void free_states(struct bpf_verifier_env *env) 21823 { 21824 struct bpf_verifier_state_list *sl; 21825 struct list_head *head, *pos, *tmp; 21826 struct bpf_scc_info *info; 21827 int i, j; 21828 21829 bpf_free_verifier_state(env->cur_state, true); 21830 env->cur_state = NULL; 21831 while (!pop_stack(env, NULL, NULL, false)); 21832 21833 list_for_each_safe(pos, tmp, &env->free_list) { 21834 sl = container_of(pos, struct bpf_verifier_state_list, node); 21835 bpf_free_verifier_state(&sl->state, false); 21836 kfree(sl); 21837 } 21838 INIT_LIST_HEAD(&env->free_list); 21839 21840 for (i = 0; i < env->scc_cnt; ++i) { 21841 info = env->scc_info[i]; 21842 if (!info) 21843 continue; 21844 for (j = 0; j < info->num_visits; j++) 21845 bpf_free_backedges(&info->visits[j]); 21846 kvfree(info); 21847 env->scc_info[i] = NULL; 21848 } 21849 21850 if (!env->explored_states) 21851 return; 21852 21853 for (i = 0; i < state_htab_size(env); i++) { 21854 head = &env->explored_states[i]; 21855 21856 list_for_each_safe(pos, tmp, head) { 21857 sl = container_of(pos, struct bpf_verifier_state_list, node); 21858 bpf_free_verifier_state(&sl->state, false); 21859 kfree(sl); 21860 } 21861 INIT_LIST_HEAD(&env->explored_states[i]); 21862 } 21863 } 21864 21865 static int do_check_common(struct bpf_verifier_env *env, int subprog, 21866 enum bpf_func_context context) 21867 { 21868 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 21869 struct bpf_subprog_info *sub = subprog_info(env, subprog); 21870 struct bpf_prog_aux *aux = env->prog->aux; 21871 struct bpf_verifier_state *state; 21872 struct bpf_reg_state *regs; 21873 u32 old_insns_total = sub->insns_total; 21874 u32 insn_processed = env->insn_processed; 21875 int ret, i; 21876 21877 env->prev_linfo = NULL; 21878 env->pass_cnt++; 21879 21880 state = kzalloc_obj(struct bpf_verifier_state, GFP_KERNEL_ACCOUNT); 21881 if (!state) 21882 return -ENOMEM; 21883 state->curframe = 0; 21884 state->speculative = false; 21885 state->branches = 1; 21886 state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE; 21887 env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK; 21888 state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); 21889 if (!state->frame[0]) { 21890 kfree(state); 21891 return -ENOMEM; 21892 } 21893 env->cur_state = state; 21894 init_func_state(env, state->frame[0], 21895 BPF_MAIN_FUNC /* callsite */, 21896 0 /* frameno */, 21897 subprog); 21898 state->first_insn_idx = env->subprog_info[subprog].start; 21899 state->last_insn_idx = -1; 21900 21901 regs = state->frame[state->curframe]->regs; 21902 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 21903 const char *sub_name = bpf_subprog_name(env, subprog); 21904 struct bpf_subprog_arg_info *arg; 21905 struct bpf_reg_state *reg; 21906 21907 if (env->log.level & BPF_LOG_LEVEL) 21908 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); 21909 ret = btf_prepare_func_args(env, subprog); 21910 if (ret) 21911 goto out; 21912 21913 if (subprog_is_exc_cb(env, subprog)) { 21914 state->frame[0]->in_exception_callback_fn = true; 21915 21916 /* 21917 * Global functions are scalar or void, make sure 21918 * we return a scalar. 21919 */ 21920 if (subprog_returns_void(env, subprog)) { 21921 verbose(env, "exception cb cannot return void\n"); 21922 ret = -EINVAL; 21923 goto out; 21924 } 21925 21926 /* Also ensure the callback only has a single scalar argument. */ 21927 if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { 21928 verbose(env, "exception cb only supports single integer argument\n"); 21929 ret = -EINVAL; 21930 goto out; 21931 } 21932 } 21933 for (i = BPF_REG_1; 21934 i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { 21935 arg = &sub->args[i - BPF_REG_1]; 21936 reg = ®s[i]; 21937 21938 if (arg->arg_type == ARG_PTR_TO_CTX) { 21939 reg->type = PTR_TO_CTX; 21940 mark_reg_known_zero(env, regs, i); 21941 } else if (arg->arg_type == ARG_SCALAR) { 21942 reg->type = SCALAR_VALUE; 21943 mark_reg_unknown(env, regs, i); 21944 } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { 21945 /* 21946 * The callers may hand in a dynptr of any type. 21947 * Take it for that of an skb, so that a write 21948 * through it and its slices are treated with care. 21949 */ 21950 __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0); 21951 } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { 21952 reg->type = PTR_TO_MEM; 21953 reg->type |= arg->arg_type & 21954 (PTR_MAYBE_NULL | PTR_UNTRUSTED | MEM_RDONLY); 21955 mark_reg_known_zero(env, regs, i); 21956 reg->mem_size = arg->mem_size; 21957 if (arg->arg_type & PTR_MAYBE_NULL) 21958 reg->id = ++env->id_gen; 21959 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { 21960 reg->type = PTR_TO_BTF_ID; 21961 if (arg->arg_type & PTR_MAYBE_NULL) 21962 reg->type |= PTR_MAYBE_NULL; 21963 if (arg->arg_type & PTR_UNTRUSTED) 21964 reg->type |= PTR_UNTRUSTED; 21965 if (arg->arg_type & PTR_TRUSTED) 21966 reg->type |= PTR_TRUSTED; 21967 mark_reg_known_zero(env, regs, i); 21968 reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */ 21969 reg->btf_id = arg->btf_id; 21970 reg->id = ++env->id_gen; 21971 } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { 21972 /* caller can pass either PTR_TO_ARENA or SCALAR */ 21973 mark_reg_unknown(env, regs, i); 21974 } else { 21975 verifier_bug(env, "unhandled arg#%d type %d", 21976 i - BPF_REG_1 + 1, arg->arg_type); 21977 ret = -EFAULT; 21978 goto out; 21979 } 21980 } 21981 if (env->prog->type == BPF_PROG_TYPE_EXT && 21982 sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { 21983 verbose(env, "freplace programs with >%d args not supported yet\n", 21984 MAX_BPF_FUNC_REG_ARGS); 21985 ret = -EINVAL; 21986 goto out; 21987 } 21988 } else { 21989 /* if main BPF program has associated BTF info, validate that 21990 * it's matching expected signature, and otherwise mark BTF 21991 * info for main program as unreliable 21992 */ 21993 if (env->prog->aux->func_info_aux) { 21994 ret = btf_prepare_func_args(env, 0); 21995 if (ret || sub->arg_slot_cnt != 1 || 21996 sub->args[0].arg_type != ARG_PTR_TO_CTX) { 21997 env->prog->aux->func_info_aux[0].unreliable = true; 21998 sub->arg_slot_cnt = 1; 21999 sub->stack_arg_cnt = 0; 22000 } 22001 } 22002 22003 /* 1st arg to a function */ 22004 regs[BPF_REG_1].type = PTR_TO_CTX; 22005 mark_reg_known_zero(env, regs, BPF_REG_1); 22006 } 22007 22008 /* Acquire references for struct_ops program arguments tagged with "__ref" */ 22009 if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { 22010 for (i = 0; i < aux->ctx_arg_info_size; i++) { 22011 ret = aux->ctx_arg_info[i].refcounted ? acquire_reference(env, 0, 0) : 0; 22012 if (ret < 0) 22013 goto out; 22014 22015 aux->ctx_arg_info[i].ref_id = ret; 22016 } 22017 } 22018 22019 ret = do_check(env); 22020 out: 22021 account_current_path(env); 22022 if (!ret) { 22023 if (pop_log) 22024 bpf_vlog_reset(&env->log, 0); 22025 bpf_diag_event_log_restore(env, 0); 22026 } 22027 free_states(env); 22028 22029 /* 22030 * The override is needed to account for async subprograms, which 22031 * are verified with their own set of stack frames and thus are 22032 * not accounted as callees by account_current_path(). 22033 * Accumulate their total counts as total counts of the main or 22034 * global subprog hosting the async call. 22035 * Start from the saved total of earlier contexts: adding to the current 22036 * total would count this pass's synchronous paths twice. 22037 */ 22038 sub->insns_total = old_insns_total + (env->insn_processed - insn_processed); 22039 return ret; 22040 } 22041 22042 /* Lazily verify all global functions based on their BTF, if they are called 22043 * from main BPF program or any of subprograms transitively. 22044 * BPF global subprogs called from dead code are not validated. 22045 * All callable global functions must pass verification. 22046 * Otherwise the whole program is rejected. 22047 * Consider: 22048 * int bar(int); 22049 * int foo(int f) 22050 * { 22051 * return bar(f); 22052 * } 22053 * int bar(int b) 22054 * { 22055 * ... 22056 * } 22057 * foo() will be verified first for R1=any_scalar_value. During verification it 22058 * will be assumed that bar() already verified successfully and call to bar() 22059 * from foo() will be checked for type match only. Later bar() will be verified 22060 * independently to check that it's safe for R1=any_scalar_value. 22061 */ 22062 static int do_check_subprogs(struct bpf_verifier_env *env) 22063 { 22064 struct bpf_prog_aux *aux = env->prog->aux; 22065 struct bpf_func_info_aux *sub_aux; 22066 int context, i, ret, new_cnt; 22067 22068 if (!aux->func_info) 22069 return 0; 22070 22071 /* 22072 * Callbacks cannot throw, so the exception callback always runs in the 22073 * main program's context. It is presumed to be always called. 22074 */ 22075 if (env->exception_callback_subprog) { 22076 sub_aux = subprog_aux(env, env->exception_callback_subprog); 22077 sub_aux->called[env->prog->sleepable] = true; 22078 } 22079 22080 again: 22081 new_cnt = 0; 22082 for (i = 1; i < env->subprog_cnt; i++) { 22083 if (!bpf_subprog_is_global(env, i)) 22084 continue; 22085 22086 sub_aux = subprog_aux(env, i); 22087 for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) { 22088 if (!sub_aux->called[context] || sub_aux->verified[context]) 22089 continue; 22090 22091 env->insn_idx = env->subprog_info[i].start; 22092 WARN_ON_ONCE(env->insn_idx == 0); 22093 ret = do_check_common(env, i, context); 22094 if (ret) 22095 return ret; 22096 if (env->log.level & BPF_LOG_LEVEL) 22097 verbose(env, "Func#%d ('%s') is safe for any args " 22098 "that match its prototype\n", 22099 i, bpf_subprog_name(env, i)); 22100 22101 sub_aux->verified[context] = true; 22102 new_cnt++; 22103 } 22104 } 22105 22106 /* 22107 * We can't loop forever as each pass verifies at least one new context, 22108 * and there are only three contexts per global subprog. 22109 */ 22110 if (new_cnt) 22111 goto again; 22112 22113 return 0; 22114 } 22115 22116 static int do_check_main(struct bpf_verifier_env *env) 22117 { 22118 int ret; 22119 22120 env->insn_idx = 0; 22121 ret = do_check_common(env, 0, env->prog->sleepable); 22122 if (!ret) 22123 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 22124 return ret; 22125 } 22126 22127 static void print_verification_stats(struct bpf_verifier_env *env) 22128 { 22129 /* Skip over hidden subprogs which are not verified. */ 22130 int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 22131 22132 if (env->log.level & BPF_LOG_STATS) { 22133 verbose(env, "verification time %lld usec\n", 22134 div_u64(env->verification_time, 1000)); 22135 verbose(env, "stack depth max %d\n", env->max_stack_depth); 22136 for (i = 0; i < subprog_cnt; i++) { 22137 const char *name = env->subprog_info[i].name; 22138 const char *kind; 22139 22140 if (!name || !name[0]) 22141 name = "<unknown>"; 22142 kind = i == 0 ? "main" : 22143 bpf_subprog_is_global(env, i) ? "global" : "static"; 22144 verbose(env, "subprog %d (%s) %s insns_self %d insns_total %d stack %d\n", 22145 i, name, kind, env->subprog_info[i].insns_self, 22146 env->subprog_info[i].insns_total, 22147 env->subprog_info[i].stack_depth); 22148 } 22149 } 22150 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 22151 "total_states %d peak_states %d mark_read %d\n", 22152 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 22153 env->max_states_per_insn, env->total_states, 22154 env->peak_states, env->longest_mark_read_walk); 22155 } 22156 22157 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog, 22158 const struct bpf_ctx_arg_aux *info, u32 cnt) 22159 { 22160 prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT); 22161 prog->aux->ctx_arg_info_size = cnt; 22162 22163 return prog->aux->ctx_arg_info ? 0 : -ENOMEM; 22164 } 22165 22166 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 22167 { 22168 const struct btf_type *t, *func_proto; 22169 const struct bpf_struct_ops_desc *st_ops_desc; 22170 const struct bpf_struct_ops_arg_info *arg_info; 22171 const struct bpf_struct_ops *st_ops; 22172 const struct btf_member *member; 22173 struct bpf_prog *prog = env->prog; 22174 bool has_refcounted_arg = false; 22175 u32 btf_id, member_idx, member_off; 22176 struct btf *btf; 22177 const char *mname; 22178 int i, err; 22179 22180 if (!prog->gpl_compatible) { 22181 verbose(env, "struct ops programs must have a GPL compatible license\n"); 22182 return -EINVAL; 22183 } 22184 22185 if (!prog->aux->attach_btf_id) 22186 return -ENOTSUPP; 22187 22188 btf = prog->aux->attach_btf; 22189 if (btf_is_module(btf)) { 22190 /* Make sure st_ops is valid through the lifetime of env */ 22191 env->attach_btf_mod = btf_try_get_module(btf); 22192 if (!env->attach_btf_mod) { 22193 verbose(env, "struct_ops module %s is not found\n", 22194 btf_get_name(btf)); 22195 return -ENOTSUPP; 22196 } 22197 } 22198 22199 btf_id = prog->aux->attach_btf_id; 22200 st_ops_desc = bpf_struct_ops_find(btf, btf_id); 22201 if (!st_ops_desc) { 22202 verbose(env, "attach_btf_id %u is not a supported struct\n", 22203 btf_id); 22204 return -ENOTSUPP; 22205 } 22206 st_ops = st_ops_desc->st_ops; 22207 22208 t = st_ops_desc->type; 22209 member_idx = prog->expected_attach_type; 22210 if (member_idx >= btf_type_vlen(t)) { 22211 verbose(env, "attach to invalid member idx %u of struct %s\n", 22212 member_idx, st_ops->name); 22213 return -EINVAL; 22214 } 22215 22216 member = &btf_type_member(t)[member_idx]; 22217 mname = btf_name_by_offset(btf, member->name_off); 22218 func_proto = btf_type_resolve_func_ptr(btf, member->type, 22219 NULL); 22220 if (!func_proto) { 22221 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 22222 mname, member_idx, st_ops->name); 22223 return -EINVAL; 22224 } 22225 22226 member_off = __btf_member_bit_offset(t, member) / 8; 22227 err = bpf_struct_ops_supported(st_ops, member_off); 22228 if (err) { 22229 verbose(env, "attach to unsupported member %s of struct %s\n", 22230 mname, st_ops->name); 22231 return err; 22232 } 22233 22234 if (st_ops->check_member) { 22235 err = st_ops->check_member(t, member, prog); 22236 22237 if (err) { 22238 verbose(env, "attach to unsupported member %s of struct %s\n", 22239 mname, st_ops->name); 22240 return err; 22241 } 22242 } 22243 22244 if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) { 22245 verbose(env, "Private stack not supported by jit\n"); 22246 return -EACCES; 22247 } 22248 22249 arg_info = &st_ops_desc->arg_info[member_idx]; 22250 for (i = 0; i < arg_info->cnt; i++) { 22251 const struct bpf_ctx_arg_aux *info = &arg_info->info[i]; 22252 22253 if (info->refcounted) 22254 has_refcounted_arg = true; 22255 if (base_type(info->reg_type) == PTR_TO_ARENA) { 22256 if (!bpf_jit_supports_arena_args()) { 22257 verbose(env, "JIT does not support arena arguments\n"); 22258 return -ENOTSUPP; 22259 } 22260 if (!prog->aux->arena) { 22261 verbose(env, 22262 "arena argument of %s requires a program with an associated arena\n", 22263 mname); 22264 return -EINVAL; 22265 } 22266 } 22267 } 22268 22269 /* Tail call is not allowed for programs with refcounted arguments since we 22270 * cannot guarantee that valid refcounted kptrs will be passed to the callee. 22271 */ 22272 for (i = 0; i < env->subprog_cnt; i++) { 22273 if (has_refcounted_arg && env->subprog_info[i].has_tail_call) { 22274 verbose(env, "program with __ref argument cannot tail call\n"); 22275 return -EINVAL; 22276 } 22277 } 22278 22279 prog->aux->st_ops = st_ops; 22280 prog->aux->attach_st_ops_member_off = member_off; 22281 22282 prog->aux->attach_func_proto = func_proto; 22283 prog->aux->attach_func_name = mname; 22284 env->ops = st_ops->verifier_ops; 22285 22286 return bpf_prog_ctx_arg_info_init(prog, arg_info->info, arg_info->cnt); 22287 } 22288 #define SECURITY_PREFIX "security_" 22289 22290 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 22291 22292 /* list of non-sleepable functions that are otherwise on 22293 * ALLOW_ERROR_INJECTION list 22294 */ 22295 BTF_SET_START(btf_non_sleepable_error_inject) 22296 /* Three functions below can be called from sleepable and non-sleepable context. 22297 * Assume non-sleepable from bpf safety point of view. 22298 */ 22299 BTF_ID(func, __filemap_add_folio) 22300 #ifdef CONFIG_FAIL_PAGE_ALLOC 22301 BTF_ID(func, should_fail_alloc_page) 22302 #endif 22303 #ifdef CONFIG_FAILSLAB 22304 BTF_ID(func, should_failslab) 22305 #endif 22306 BTF_SET_END(btf_non_sleepable_error_inject) 22307 22308 static int check_non_sleepable_error_inject(u32 btf_id) 22309 { 22310 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 22311 } 22312 22313 static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name) 22314 { 22315 /* fentry/fexit/fmod_ret progs can be sleepable if they are 22316 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 22317 */ 22318 if (!check_non_sleepable_error_inject(btf_id) && 22319 within_error_injection_list(addr)) 22320 return 0; 22321 22322 return -EINVAL; 22323 } 22324 22325 static int check_attach_modify_return(unsigned long addr, const char *func_name) 22326 { 22327 if (within_error_injection_list(addr) || 22328 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 22329 return 0; 22330 22331 return -EINVAL; 22332 } 22333 22334 #else 22335 22336 /* Unfortunately, the arch-specific prefixes are hard-coded in arch syscall code 22337 * so we need to hard-code them, too. Ftrace has arch_syscall_match_sym_name() 22338 * but that just compares two concrete function names. 22339 */ 22340 static bool has_arch_syscall_prefix(const char *func_name) 22341 { 22342 #if defined(__x86_64__) 22343 return !strncmp(func_name, "__x64_", 6); 22344 #elif defined(__i386__) 22345 return !strncmp(func_name, "__ia32_", 7); 22346 #elif defined(__s390x__) 22347 return !strncmp(func_name, "__s390x_", 8); 22348 #elif defined(__aarch64__) 22349 return !strncmp(func_name, "__arm64_", 8); 22350 #elif defined(__riscv) 22351 return !strncmp(func_name, "__riscv_", 8); 22352 #elif defined(__powerpc__) || defined(__powerpc64__) 22353 return !strncmp(func_name, "sys_", 4); 22354 #elif defined(__loongarch__) 22355 return !strncmp(func_name, "sys_", 4); 22356 #else 22357 return false; 22358 #endif 22359 } 22360 22361 /* Without error injection, allow sleepable and fmod_ret progs on syscalls. */ 22362 22363 static int check_attach_sleepable(u32 btf_id, unsigned long addr, const char *func_name) 22364 { 22365 if (has_arch_syscall_prefix(func_name)) 22366 return 0; 22367 22368 return -EINVAL; 22369 } 22370 22371 static int check_attach_modify_return(unsigned long addr, const char *func_name) 22372 { 22373 if (has_arch_syscall_prefix(func_name) || 22374 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 22375 return 0; 22376 22377 return -EINVAL; 22378 } 22379 22380 #endif /* CONFIG_FUNCTION_ERROR_INJECTION */ 22381 22382 static bool is_tracing_multi_id(const struct bpf_prog *prog, u32 btf_id) 22383 { 22384 return is_tracing_multi(prog->expected_attach_type) && bpf_multi_func_btf_id[0] == btf_id; 22385 } 22386 22387 static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct bpf_prog *prog, 22388 const struct btf *btf) 22389 { 22390 const struct btf_type *t; 22391 const char *tname; 22392 22393 if (!btf_is_kernel(btf)) 22394 return -EINVAL; 22395 22396 switch (prog->type) { 22397 case BPF_PROG_TYPE_TRACING: 22398 t = btf_type_by_id(btf, btf_id); 22399 if (!t) 22400 return -EINVAL; 22401 tname = btf_name_by_offset(btf, t->name_off); 22402 if (!tname) 22403 return -EINVAL; 22404 22405 /* 22406 * *.multi sleepable programs will pass initial sleepable check, 22407 * the actual attached btf ids are checked later during the link 22408 * attachment. 22409 */ 22410 if (is_tracing_multi_id(prog, btf_id)) 22411 return 0; 22412 if (!check_attach_sleepable(btf_id, addr, tname)) 22413 return 0; 22414 /* 22415 * fentry/fexit/fmod_ret progs can also be sleepable if they are 22416 * in the fmodret id set with the KF_SLEEPABLE flag. 22417 */ 22418 else { 22419 u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, prog); 22420 22421 if (flags && (*flags & KF_SLEEPABLE)) 22422 return 0; 22423 } 22424 break; 22425 case BPF_PROG_TYPE_LSM: 22426 /* 22427 * LSM progs check that they are attached to bpf_lsm_*() funcs. 22428 * Only some of them are sleepable. 22429 */ 22430 if (bpf_lsm_is_sleepable_hook(btf_id)) 22431 return 0; 22432 break; 22433 default: 22434 break; 22435 } 22436 return -EINVAL; 22437 } 22438 22439 /* 22440 * Resolve the prototype describing a trace target's real ABI. A 22441 * KF_IMPLICIT_ARGS kfunc has its injected args stripped from the public 22442 * prototype, so use the _impl prototype; other targets use their own. 22443 */ 22444 static const struct btf_type * 22445 btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id) 22446 { 22447 const struct btf_type *func; 22448 struct module *mod = NULL; 22449 const char *name; 22450 int implicit; 22451 22452 func = btf_type_by_id(btf, func_id); 22453 if (!func || !btf_type_is_func(func)) 22454 return NULL; 22455 name = btf_name_by_offset(btf, func->name_off); 22456 22457 /* 22458 * btf_kfunc_check_flag() reads kfunc_set_tab, which for a module is 22459 * stable only once it is live; hold a module ref across the read to 22460 * exclude a concurrent module load. 22461 */ 22462 if (btf_is_module(btf)) { 22463 mod = btf_try_get_module(btf); 22464 if (!mod) 22465 return NULL; 22466 } 22467 implicit = btf_kfunc_check_flag(btf, func_id, KF_IMPLICIT_ARGS); 22468 module_put(mod); 22469 22470 if (implicit == -EINVAL) { 22471 bpf_log(log, "kfunc %s has inconsistent KF_IMPLICIT_ARGS\n", name); 22472 return NULL; 22473 } 22474 if (implicit > 0) 22475 return find_kfunc_impl_proto(log, btf, name); 22476 22477 return btf_type_by_id(btf, func->type); 22478 } 22479 22480 static bool attach_uses_trampoline_retval(enum bpf_attach_type type) 22481 { 22482 switch (type) { 22483 case BPF_MODIFY_RETURN: 22484 case BPF_TRACE_FEXIT: 22485 case BPF_TRACE_FEXIT_MULTI: 22486 case BPF_TRACE_FSESSION: 22487 case BPF_TRACE_FSESSION_MULTI: 22488 return true; 22489 default: 22490 return false; 22491 } 22492 } 22493 22494 /* 22495 * BPF code passes argument slot N where the kernel convention passes its Nth 22496 * u64 argument, so describe a BPF function to its trampoline as one u64 per 22497 * slot, and every arch places its arguments right. 22498 */ 22499 static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, 22500 struct btf_func_model *m) 22501 { 22502 u32 i, nslots = 0; 22503 22504 for (i = 0; i < m->nr_args; i++) 22505 nslots += btf_func_model_arg_slots(m, i); 22506 if (nslots > MAX_BPF_FUNC_ARGS) { 22507 bpf_log(log, "The function %s has %u argument slots. Too many.\n", 22508 tname, nslots); 22509 return -EINVAL; 22510 } 22511 22512 for (i = 0; i < nslots; i++) { 22513 m->arg_size[i] = sizeof(u64); 22514 m->arg_flags[i] = 0; 22515 } 22516 m->nr_args = nslots; 22517 return 0; 22518 } 22519 22520 /* 22521 * The verifier lets a static function use stack arguments counted from its 22522 * BTF, even if that BTF turns out unreliable. A trampoline on the function is 22523 * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the 22524 * stack arguments, so such a function cannot be attached to. 22525 */ 22526 static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, 22527 const struct btf_type *func_proto, const char *tname) 22528 { 22529 struct btf_func_model m; 22530 u32 i, nslots = 0; 22531 22532 if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { 22533 /* at least a slot for each argument */ 22534 nslots = btf_type_vlen(func_proto); 22535 } else { 22536 for (i = 0; i < m.nr_args; i++) 22537 nslots += btf_func_model_arg_slots(&m, i); 22538 } 22539 if (nslots <= MAX_BPF_FUNC_REG_ARGS) 22540 return 0; 22541 22542 bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", 22543 tname); 22544 return -EINVAL; 22545 } 22546 22547 /* Functions may share a type id. The first of them is attached to. */ 22548 static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id) 22549 { 22550 const struct bpf_prog_aux *aux = tgt_prog->aux; 22551 int i; 22552 22553 for (i = 0; i < aux->func_info_cnt; i++) 22554 if (aux->func_info[i].type_id == btf_id) 22555 return i; 22556 22557 return -1; 22558 } 22559 22560 /* 22561 * The prologue of an extension gives a cloned skb a new head, unless the 22562 * prologue of the target program has done that. The callers of the function 22563 * that it replaces must not hold on to the old head. 22564 */ 22565 int bpf_check_ext_prologue(struct bpf_verifier_log *log, 22566 const struct bpf_prog *tgt_prog, u32 btf_id) 22567 { 22568 const struct bpf_prog_aux *aux = tgt_prog->aux; 22569 int subprog = find_tgt_subprog(tgt_prog, btf_id); 22570 22571 if (aux->has_prologue) 22572 return 0; 22573 22574 if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs) 22575 return 0; 22576 22577 bpf_log(log, 22578 "Extension writing packet data cannot replace a function whose callers keep packet pointers\n"); 22579 return -EINVAL; 22580 } 22581 22582 /* 22583 * An extension is verified as a program that runs under the RCU read lock. Does 22584 * the function that it is to replace? 22585 */ 22586 static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) 22587 { 22588 const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog]; 22589 22590 if (!subprog) 22591 return !prog->sleepable; 22592 22593 return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] && 22594 !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; 22595 } 22596 22597 /* 22598 * The callers of a global function are checked against the arguments that its 22599 * BTF describes, and an extension is verified with those of its own BTF. It 22600 * must not expect more than the callers of the function provide. 22601 * 22602 * While an extension is loaded, bpf_check_attach_target() runs before its 22603 * arguments are known. check_attach_btf_id() does the check when they are. 22604 */ 22605 static int check_ext_args(struct bpf_verifier_log *log, const struct bpf_prog *prog, 22606 const struct bpf_prog *tgt_prog, int subprog) 22607 { 22608 const struct bpf_subprog_arg_info *tgt = tgt_prog->aux->func_info_aux[subprog].args; 22609 const struct bpf_subprog_arg_info *ext = prog->aux->func_info_aux[0].args; 22610 int i; 22611 22612 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS && ext[i].arg_type != ARG_UNUSED; i++) { 22613 /* An extension may accept NULL where the callers never pass it. */ 22614 if ((ext[i].arg_type == tgt[i].arg_type || 22615 ext[i].arg_type == (tgt[i].arg_type | PTR_MAYBE_NULL)) && 22616 (base_type(ext[i].arg_type) == ARG_PTR_TO_MEM ? 22617 ext[i].mem_size <= tgt[i].mem_size : ext[i].btf_id == tgt[i].btf_id)) 22618 continue; 22619 22620 bpf_log(log, "Extension program's argument in R%d is incompatible with original\n", 22621 i + 1); 22622 return -EINVAL; 22623 } 22624 22625 return 0; 22626 } 22627 22628 int bpf_check_attach_target(struct bpf_verifier_log *log, 22629 const struct bpf_prog *prog, 22630 const struct bpf_prog *tgt_prog, 22631 u32 btf_id, 22632 struct bpf_attach_target_info *tgt_info) 22633 { 22634 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 22635 bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING; 22636 char trace_symbol[KSYM_SYMBOL_LEN]; 22637 const char prefix[] = "btf_trace_"; 22638 struct bpf_raw_event_map *btp; 22639 int ret = 0, subprog = -1; 22640 const struct btf_type *t; 22641 bool conservative = true; 22642 const char *tname, *fname; 22643 struct btf *btf; 22644 long addr = 0; 22645 struct module *mod = NULL; 22646 22647 if (!btf_id) { 22648 bpf_log(log, "Tracing programs must provide btf_id\n"); 22649 return -EINVAL; 22650 } 22651 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 22652 if (!btf) { 22653 bpf_log(log, 22654 "Tracing program can only be attached to another program annotated with BTF\n"); 22655 return -EINVAL; 22656 } 22657 t = btf_type_by_id(btf, btf_id); 22658 if (!t) { 22659 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 22660 return -EINVAL; 22661 } 22662 tname = btf_name_by_offset(btf, t->name_off); 22663 if (!tname) { 22664 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 22665 return -EINVAL; 22666 } 22667 if (tgt_prog) { 22668 struct bpf_prog_aux *aux = tgt_prog->aux; 22669 bool tgt_changes_pkt_data; 22670 bool tgt_might_sleep; 22671 bool tgt_can_unwind; 22672 22673 if (bpf_prog_is_dev_bound(prog->aux) && 22674 !bpf_prog_dev_bound_match(prog, tgt_prog)) { 22675 bpf_log(log, "Target program bound device mismatch"); 22676 return -EINVAL; 22677 } 22678 22679 subprog = find_tgt_subprog(tgt_prog, btf_id); 22680 if (subprog == -1) { 22681 bpf_log(log, "Subprog %s doesn't exist\n", tname); 22682 return -EINVAL; 22683 } 22684 /* 22685 * A struct_ops indirect trampoline converts arena arguments 22686 * before invoking its program. A tracing or extension program 22687 * attached to the main program would see the converted offset as a 22688 * regular BTF pointer. 22689 */ 22690 if (subprog == 0 && bpf_prog_has_arena_ctx_arg(tgt_prog)) { 22691 bpf_log(log, "Cannot attach to a target with arena context arguments\n"); 22692 return -EOPNOTSUPP; 22693 } 22694 if (aux->func && aux->func[subprog]->aux->exception_cb) { 22695 bpf_log(log, 22696 "%s programs cannot attach to exception callback\n", 22697 prog_extension ? "Extension" : "Tracing"); 22698 return -EINVAL; 22699 } 22700 /* unwind_exit of the main prog is exit, unless it's an extension */ 22701 tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) && 22702 (aux->func ? aux->func[subprog]->aux->can_unwind 22703 : aux->can_unwind); 22704 /* unwind_exit would return to the trampoline that called the target */ 22705 if (tgt_can_unwind && !prog_extension && 22706 prog->expected_attach_type != BPF_TRACE_FENTRY) { 22707 bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname); 22708 return -EINVAL; 22709 } 22710 conservative = aux->func_info_aux[subprog].unreliable; 22711 if (prog_extension) { 22712 if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL || 22713 conservative) { 22714 bpf_log(log, 22715 "Cannot replace static functions\n"); 22716 return -EINVAL; 22717 } 22718 if (!prog->jit_requested) { 22719 bpf_log(log, 22720 "Extension programs should be JITed\n"); 22721 return -EINVAL; 22722 } 22723 /* 22724 * How much of the buffer of the tracepoint is used 22725 * is checked for the program that is attached to it. 22726 */ 22727 if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { 22728 bpf_log(log, 22729 "Cannot extend a writable raw tracepoint program\n"); 22730 return -EINVAL; 22731 } 22732 if (!func_has_rcu_read_lock(tgt_prog, subprog)) { 22733 bpf_log(log, 22734 "Cannot extend a function that runs without RCU read lock\n"); 22735 return -EINVAL; 22736 } 22737 tgt_changes_pkt_data = aux->func 22738 ? aux->func[subprog]->aux->changes_pkt_data 22739 : aux->changes_pkt_data; 22740 if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) { 22741 bpf_log(log, 22742 "Extension program changes packet data, while original does not\n"); 22743 return -EINVAL; 22744 } 22745 if (prog->aux->has_prologue && 22746 bpf_check_ext_prologue(log, tgt_prog, btf_id)) 22747 return -EINVAL; 22748 22749 tgt_might_sleep = aux->func 22750 ? aux->func[subprog]->aux->might_sleep 22751 : aux->might_sleep; 22752 if (prog->aux->might_sleep && !tgt_might_sleep) { 22753 bpf_log(log, 22754 "Extension program may sleep, while original does not\n"); 22755 return -EINVAL; 22756 } 22757 /* There is no unwind_goto after the calls of the original */ 22758 if (prog->aux->can_unwind && !tgt_can_unwind) { 22759 bpf_log(log, 22760 "Extension program can unwind, while original does not\n"); 22761 return -EINVAL; 22762 } 22763 } 22764 if (!tgt_prog->jited) { 22765 bpf_log(log, "Can attach to only JITed progs\n"); 22766 return -EINVAL; 22767 } 22768 if (prog_tracing) { 22769 if (aux->attach_tracing_prog) { 22770 /* 22771 * Target program is an fentry/fexit which is already attached 22772 * to another tracing program. More levels of nesting 22773 * attachment are not allowed. 22774 */ 22775 bpf_log(log, "Cannot nest tracing program attach more than once\n"); 22776 return -EINVAL; 22777 } 22778 } else if (tgt_prog->type == prog->type) { 22779 /* 22780 * To avoid potential call chain cycles, prevent attaching of a 22781 * program extension to another extension. It's ok to attach 22782 * fentry/fexit to extension program. 22783 */ 22784 bpf_log(log, "Cannot recursively attach\n"); 22785 return -EINVAL; 22786 } 22787 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 22788 prog_extension && 22789 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 22790 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT || 22791 tgt_prog->expected_attach_type == BPF_TRACE_FENTRY_MULTI || 22792 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || 22793 tgt_prog->expected_attach_type == BPF_TRACE_FSESSION || 22794 tgt_prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { 22795 /* Program extensions can extend all program types 22796 * except fentry/fexit. The reason is the following. 22797 * The fentry/fexit programs are used for performance 22798 * analysis, stats and can be attached to any program 22799 * type. When extension program is replacing XDP function 22800 * it is necessary to allow performance analysis of all 22801 * functions. Both original XDP program and its program 22802 * extension. Hence attaching fentry/fexit to 22803 * BPF_PROG_TYPE_EXT is allowed. If extending of 22804 * fentry/fexit was allowed it would be possible to create 22805 * long call chain fentry->extension->fentry->extension 22806 * beyond reasonable stack size. Hence extending fentry 22807 * is not allowed. 22808 */ 22809 bpf_log(log, "Cannot extend fentry/fexit/fsession\n"); 22810 return -EINVAL; 22811 } 22812 } else { 22813 if (prog_extension) { 22814 bpf_log(log, "Cannot replace kernel functions\n"); 22815 return -EINVAL; 22816 } 22817 } 22818 22819 switch (prog->expected_attach_type) { 22820 case BPF_TRACE_RAW_TP: 22821 if (tgt_prog) { 22822 bpf_log(log, 22823 "Only FENTRY/FEXIT/FSESSION progs are attachable to another BPF prog\n"); 22824 return -EINVAL; 22825 } 22826 if (!btf_type_is_typedef(t)) { 22827 bpf_log(log, "attach_btf_id %u is not a typedef\n", 22828 btf_id); 22829 return -EINVAL; 22830 } 22831 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 22832 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 22833 btf_id, tname); 22834 return -EINVAL; 22835 } 22836 tname += sizeof(prefix) - 1; 22837 22838 /* The func_proto of "btf_trace_##tname" is generated from typedef without argument 22839 * names. Thus using bpf_raw_event_map to get argument names. 22840 */ 22841 btp = bpf_get_raw_tracepoint(tname); 22842 if (!btp) 22843 return -EINVAL; 22844 if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) { 22845 bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n", 22846 tname); 22847 bpf_put_raw_tracepoint(btp); 22848 return -EINVAL; 22849 } 22850 fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL, 22851 trace_symbol); 22852 bpf_put_raw_tracepoint(btp); 22853 22854 if (fname) 22855 ret = btf_find_by_name_kind(btf, fname, BTF_KIND_FUNC); 22856 22857 if (!fname || ret < 0) { 22858 bpf_log(log, "Cannot find btf of tracepoint template, fall back to %s%s.\n", 22859 prefix, tname); 22860 t = btf_type_by_id(btf, t->type); 22861 if (!btf_type_is_ptr(t)) 22862 /* should never happen in valid vmlinux build */ 22863 return -EINVAL; 22864 } else { 22865 t = btf_type_by_id(btf, ret); 22866 if (!btf_type_is_func(t)) 22867 /* should never happen in valid vmlinux build */ 22868 return -EINVAL; 22869 } 22870 22871 t = btf_type_by_id(btf, t->type); 22872 if (!btf_type_is_func_proto(t)) 22873 /* should never happen in valid vmlinux build */ 22874 return -EINVAL; 22875 22876 break; 22877 case BPF_TRACE_ITER: 22878 if (!btf_type_is_func(t)) { 22879 bpf_log(log, "attach_btf_id %u is not a function\n", 22880 btf_id); 22881 return -EINVAL; 22882 } 22883 t = btf_type_by_id(btf, t->type); 22884 if (!btf_type_is_func_proto(t)) 22885 return -EINVAL; 22886 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 22887 if (ret) 22888 return ret; 22889 break; 22890 default: 22891 if (!prog_extension) 22892 return -EINVAL; 22893 fallthrough; 22894 case BPF_MODIFY_RETURN: 22895 case BPF_LSM_MAC: 22896 case BPF_LSM_CGROUP: 22897 case BPF_TRACE_FENTRY: 22898 case BPF_TRACE_FEXIT: 22899 case BPF_TRACE_FSESSION: 22900 case BPF_TRACE_FSESSION_MULTI: 22901 case BPF_TRACE_FENTRY_MULTI: 22902 case BPF_TRACE_FEXIT_MULTI: 22903 if ((prog->expected_attach_type == BPF_TRACE_FSESSION || 22904 prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && 22905 !bpf_jit_supports_fsession()) { 22906 bpf_log(log, "JIT does not support fsession\n"); 22907 return -EOPNOTSUPP; 22908 } 22909 if (!btf_type_is_func(t)) { 22910 bpf_log(log, "attach_btf_id %u is not a function\n", 22911 btf_id); 22912 return -EINVAL; 22913 } 22914 if (prog_extension && 22915 (btf_check_type_match(log, prog, btf, t) || 22916 check_ext_args(log, prog, tgt_prog, subprog))) 22917 return -EINVAL; 22918 t = btf_attach_func_proto(log, btf, btf_id); 22919 if (!t || !btf_type_is_func_proto(t)) 22920 return -EINVAL; 22921 22922 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 22923 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 22924 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 22925 return -EINVAL; 22926 22927 if (tgt_prog && conservative) { 22928 if (subprog > 0) { 22929 ret = check_unreliable_func_args(log, btf, t, tname); 22930 if (ret) 22931 return ret; 22932 } 22933 t = NULL; 22934 } 22935 22936 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 22937 if (ret < 0) 22938 return ret; 22939 if (tgt_prog) { 22940 ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); 22941 if (ret < 0) 22942 return ret; 22943 } 22944 22945 if (tgt_info->fmodel.ret_size > 8 && 22946 attach_uses_trampoline_retval(prog->expected_attach_type)) { 22947 bpf_log(log, 22948 "Attach to function %s with a >8 byte return value is not supported for this attach type\n", 22949 tname); 22950 return -EOPNOTSUPP; 22951 } 22952 22953 /* 22954 * *.multi programs don't need an address during program 22955 * verification, we just take the module ref if needed. 22956 */ 22957 if (is_tracing_multi_id(prog, btf_id)) { 22958 if (btf_is_module(btf)) { 22959 mod = btf_try_get_module(btf); 22960 if (!mod) 22961 return -ENOENT; 22962 } 22963 addr = 0; 22964 } else if (tgt_prog) { 22965 if (subprog == 0) 22966 addr = (long) tgt_prog->bpf_func; 22967 else 22968 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 22969 } else { 22970 if (btf_is_module(btf)) { 22971 mod = btf_try_get_module(btf); 22972 if (mod) 22973 addr = find_kallsyms_symbol_value(mod, tname); 22974 else 22975 addr = 0; 22976 } else { 22977 addr = kallsyms_lookup_name(tname); 22978 } 22979 if (!addr) { 22980 module_put(mod); 22981 bpf_log(log, 22982 "The address of function %s cannot be found\n", 22983 tname); 22984 return -ENOENT; 22985 } 22986 } 22987 22988 if (prog->sleepable) { 22989 ret = btf_id_allow_sleepable(btf_id, addr, prog, btf); 22990 if (ret) { 22991 module_put(mod); 22992 bpf_log(log, "%s is not sleepable\n", tname); 22993 return ret; 22994 } 22995 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 22996 if (tgt_prog) { 22997 module_put(mod); 22998 bpf_log(log, "can't modify return codes of BPF programs\n"); 22999 return -EINVAL; 23000 } 23001 ret = -EINVAL; 23002 if (btf_kfunc_is_modify_return(btf, btf_id, prog) || 23003 !check_attach_modify_return(addr, tname)) 23004 ret = 0; 23005 if (ret) { 23006 module_put(mod); 23007 bpf_log(log, "%s() is not modifiable\n", tname); 23008 return ret; 23009 } 23010 } 23011 23012 break; 23013 } 23014 tgt_info->tgt_addr = addr; 23015 tgt_info->tgt_name = tname; 23016 tgt_info->tgt_type = t; 23017 tgt_info->tgt_mod = mod; 23018 return 0; 23019 } 23020 23021 BTF_SET_START(btf_id_deny) 23022 BTF_ID_UNUSED 23023 #ifdef CONFIG_SMP 23024 BTF_ID(func, ___migrate_enable) 23025 BTF_ID(func, migrate_disable) 23026 BTF_ID(func, migrate_enable) 23027 #endif 23028 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 23029 BTF_ID(func, rcu_read_unlock_strict) 23030 #endif 23031 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE) 23032 BTF_ID(func, preempt_count_add) 23033 BTF_ID(func, preempt_count_sub) 23034 #endif 23035 #ifdef CONFIG_PREEMPT_RCU 23036 BTF_ID(func, __rcu_read_lock) 23037 BTF_ID(func, __rcu_read_unlock) 23038 #endif 23039 BTF_SET_END(btf_id_deny) 23040 23041 /* fexit and fmod_ret can't be used to attach to __noreturn functions. 23042 * Currently, we must manually list all __noreturn functions here. Once a more 23043 * robust solution is implemented, this workaround can be removed. 23044 */ 23045 BTF_SET_START(noreturn_deny) 23046 #ifdef CONFIG_IA32_EMULATION 23047 BTF_ID(func, __ia32_sys_exit) 23048 BTF_ID(func, __ia32_sys_exit_group) 23049 #endif 23050 #ifdef CONFIG_KUNIT 23051 BTF_ID(func, __kunit_abort) 23052 BTF_ID(func, kunit_try_catch_throw) 23053 #endif 23054 #ifdef CONFIG_MODULES 23055 BTF_ID(func, __module_put_and_kthread_exit) 23056 #endif 23057 #ifdef CONFIG_X86_64 23058 BTF_ID(func, __x64_sys_exit) 23059 BTF_ID(func, __x64_sys_exit_group) 23060 #endif 23061 BTF_ID(func, do_exit) 23062 BTF_ID(func, do_group_exit) 23063 BTF_ID(func, kthread_complete_and_exit) 23064 BTF_ID(func, make_task_dead) 23065 BTF_SET_END(noreturn_deny) 23066 23067 static bool can_be_sleepable(struct bpf_prog *prog) 23068 { 23069 if (prog->type == BPF_PROG_TYPE_TRACING) { 23070 switch (prog->expected_attach_type) { 23071 case BPF_TRACE_FENTRY: 23072 case BPF_TRACE_FEXIT: 23073 case BPF_MODIFY_RETURN: 23074 case BPF_TRACE_ITER: 23075 case BPF_TRACE_FSESSION: 23076 case BPF_TRACE_RAW_TP: 23077 case BPF_TRACE_FENTRY_MULTI: 23078 case BPF_TRACE_FEXIT_MULTI: 23079 case BPF_TRACE_FSESSION_MULTI: 23080 return true; 23081 default: 23082 return false; 23083 } 23084 } 23085 if (prog->type == BPF_PROG_TYPE_LSM) 23086 return prog->expected_attach_type != BPF_LSM_CGROUP; 23087 23088 return prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || 23089 prog->type == BPF_PROG_TYPE_STRUCT_OPS || 23090 prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || 23091 prog->type == BPF_PROG_TYPE_TRACEPOINT; 23092 } 23093 23094 static int check_attach_btf_id(struct bpf_verifier_env *env) 23095 { 23096 struct bpf_prog *prog = env->prog; 23097 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 23098 struct bpf_attach_target_info tgt_info = {}; 23099 u32 btf_id = prog->aux->attach_btf_id; 23100 struct bpf_trampoline *tr; 23101 int ret; 23102 u64 key; 23103 23104 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 23105 if (prog->sleepable) 23106 /* attach_btf_id checked to be zero already */ 23107 return 0; 23108 verbose(env, "Syscall programs can only be sleepable\n"); 23109 return -EINVAL; 23110 } 23111 23112 if (prog->sleepable && !can_be_sleepable(prog)) { 23113 verbose(env, "Program of this type cannot be sleepable\n"); 23114 return -EINVAL; 23115 } 23116 23117 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 23118 return check_struct_ops_btf_id(env); 23119 23120 if (prog->type != BPF_PROG_TYPE_TRACING && 23121 prog->type != BPF_PROG_TYPE_LSM && 23122 prog->type != BPF_PROG_TYPE_EXT) 23123 return 0; 23124 23125 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 23126 if (ret) 23127 return ret; 23128 23129 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 23130 /* to make freplace equivalent to their targets, they need to 23131 * inherit env->ops and expected_attach_type for the rest of the 23132 * verification 23133 */ 23134 env->ops = bpf_verifier_ops[tgt_prog->type]; 23135 prog->expected_attach_type = tgt_prog->expected_attach_type; 23136 23137 /* only now it is known what the arguments of prog stand for */ 23138 ret = btf_prepare_func_args(env, 0); 23139 if (ret) 23140 return ret; 23141 ret = check_ext_args(&env->log, prog, tgt_prog, 23142 find_tgt_subprog(tgt_prog, btf_id)); 23143 if (ret) 23144 return ret; 23145 } 23146 23147 /* store info about the attachment target that will be used later */ 23148 prog->aux->attach_func_proto = tgt_info.tgt_type; 23149 prog->aux->attach_func_name = tgt_info.tgt_name; 23150 prog->aux->mod = tgt_info.tgt_mod; 23151 23152 if (tgt_prog) { 23153 prog->aux->saved_dst_prog_type = tgt_prog->type; 23154 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 23155 } 23156 23157 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 23158 prog->aux->attach_btf_trace = true; 23159 return 0; 23160 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 23161 return bpf_iter_prog_supported(prog); 23162 } 23163 23164 if (prog->type == BPF_PROG_TYPE_LSM) { 23165 ret = bpf_lsm_verify_prog(&env->log, prog); 23166 if (ret < 0) 23167 return ret; 23168 } else if (prog->type == BPF_PROG_TYPE_TRACING && 23169 btf_id_set_contains(&btf_id_deny, btf_id)) { 23170 verbose(env, "Attaching tracing programs to function '%s' is rejected.\n", 23171 tgt_info.tgt_name); 23172 return -EINVAL; 23173 } else if ((prog->expected_attach_type == BPF_TRACE_FEXIT || 23174 prog->expected_attach_type == BPF_TRACE_FSESSION || 23175 prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI || 23176 prog->expected_attach_type == BPF_MODIFY_RETURN) && 23177 btf_id_set_contains(&noreturn_deny, btf_id)) { 23178 verbose(env, "Attaching fexit/fsession/fmod_ret to __noreturn function '%s' is rejected.\n", 23179 tgt_info.tgt_name); 23180 return -EINVAL; 23181 } 23182 23183 /* 23184 * We don't get trampoline for tracing_multi programs at this point, 23185 * it's done when tracing_multi link is created. 23186 */ 23187 if (prog->type == BPF_PROG_TYPE_TRACING && 23188 is_tracing_multi(prog->expected_attach_type)) 23189 return 0; 23190 23191 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 23192 tr = bpf_trampoline_get(key, &tgt_info); 23193 if (!tr) 23194 return -ENOMEM; 23195 23196 if (tgt_prog && tgt_prog->aux->tail_call_reachable) 23197 bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); 23198 23199 prog->aux->dst_trampoline = tr; 23200 return 0; 23201 } 23202 23203 int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id, 23204 struct bpf_attach_target_info *tgt_info) 23205 { 23206 const struct btf_type *t; 23207 unsigned long addr; 23208 const char *tname; 23209 int err; 23210 23211 if (!btf_id || !btf) 23212 return -EINVAL; 23213 23214 /* Check noreturn attachment. */ 23215 if ((prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI || 23216 prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) && 23217 btf_id_set_contains(&noreturn_deny, btf_id)) 23218 return -EINVAL; 23219 /* Check denied attachment. */ 23220 if (btf_id_set_contains(&btf_id_deny, btf_id)) 23221 return -EINVAL; 23222 23223 /* Check and get function target data. */ 23224 t = btf_type_by_id(btf, btf_id); 23225 if (!t) 23226 return -EINVAL; 23227 tname = btf_name_by_offset(btf, t->name_off); 23228 if (!tname) 23229 return -EINVAL; 23230 t = btf_attach_func_proto(NULL, btf, btf_id); 23231 if (!t || !btf_type_is_func_proto(t)) 23232 return -EINVAL; 23233 err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); 23234 if (err < 0) 23235 return err; 23236 if (tgt_info->fmodel.ret_size > 8 && 23237 attach_uses_trampoline_retval(prog->expected_attach_type)) 23238 return -EOPNOTSUPP; 23239 if (btf_is_module(btf)) { 23240 /* The bpf program already holds reference to module. */ 23241 if (WARN_ON_ONCE(!prog->aux->mod)) 23242 return -EINVAL; 23243 addr = find_kallsyms_symbol_value(prog->aux->mod, tname); 23244 } else { 23245 addr = kallsyms_lookup_name(tname); 23246 } 23247 if (!addr || !ftrace_location(addr)) 23248 return -ENOENT; 23249 23250 /* Check sleepable program attachment. */ 23251 if (prog->sleepable) { 23252 err = btf_id_allow_sleepable(btf_id, addr, prog, btf); 23253 if (err) 23254 return err; 23255 } 23256 tgt_info->tgt_addr = addr; 23257 return 0; 23258 } 23259 23260 struct btf *bpf_get_btf_vmlinux(void) 23261 { 23262 /* Pairs with the smp_store_release() on the parse path below. */ 23263 struct btf *btf = smp_load_acquire(&btf_vmlinux); 23264 23265 if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 23266 mutex_lock(&btf_vmlinux_lock); 23267 btf = btf_vmlinux; 23268 if (!btf) { 23269 btf = btf_parse_vmlinux(); 23270 /* 23271 * Order the parsed BTF contents and the globals the 23272 * parse populated (e.g. bpf_ctx_convert.t) before 23273 * the pointer publication. Pairs with the acquire 23274 * on the lockless fast path above. 23275 */ 23276 smp_store_release(&btf_vmlinux, btf); 23277 } 23278 mutex_unlock(&btf_vmlinux_lock); 23279 } 23280 return btf; 23281 } 23282 23283 /* 23284 * The add_fd_from_fd_array() is executed only if fd_array_cnt is non-zero. In 23285 * this case expect that every file descriptor in the array is either a map or 23286 * a BTF. Everything else is considered to be trash. 23287 */ 23288 static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd) 23289 { 23290 struct bpf_map *map; 23291 struct btf *btf; 23292 CLASS(fd, f)(fd); 23293 int err; 23294 23295 map = __bpf_map_get(f); 23296 if (!IS_ERR(map)) { 23297 err = __add_used_map(env, map); 23298 if (err < 0) 23299 return err; 23300 fd_slot_set_map(&env->fd_array[idx], map); 23301 return 0; 23302 } 23303 23304 btf = __btf_get_by_fd(f); 23305 if (!IS_ERR(btf)) { 23306 btf_get(btf); 23307 err = __add_used_btf(env, btf); 23308 if (err < 0) 23309 return err; 23310 fd_slot_set_btf(&env->fd_array[idx], btf); 23311 return 0; 23312 } 23313 23314 verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); 23315 return PTR_ERR(map); 23316 } 23317 23318 /* 23319 * A continuous fd_array is resolved into an in-memory cache with one slot 23320 * per entry. The bound here is deliberately generous and not derived from 23321 * the per-program object limits: Duplicate entries /are/ permitted, and 23322 * the number of distinct maps and BTFs a program can bind is enforced when 23323 * each entry is resolved by __add_used_map() and __add_used_btf(). 23324 */ 23325 #define MAX_FD_ARRAY_CNT 4096 23326 23327 static int process_fd_array_continuous(struct bpf_verifier_env *env, 23328 bpfptr_t fd_array, u32 cnt) 23329 { 23330 int fd, ret; 23331 u32 i; 23332 23333 if (cnt > MAX_FD_ARRAY_CNT) { 23334 verbose(env, "fd_array has too many entries (%u, max %u)\n", 23335 cnt, MAX_FD_ARRAY_CNT); 23336 return -E2BIG; 23337 } 23338 23339 env->fd_array = kvzalloc_objs(*env->fd_array, cnt, GFP_KERNEL_ACCOUNT); 23340 if (!env->fd_array) 23341 return -ENOMEM; 23342 env->fd_array_cnt = cnt; 23343 for (i = 0; i < cnt; i++) { 23344 if (copy_from_bpfptr_offset(&fd, fd_array, 23345 (size_t)i * sizeof(fd), sizeof(fd))) 23346 return -EFAULT; 23347 ret = add_fd_from_fd_array(env, i, fd); 23348 if (ret) 23349 return ret; 23350 } 23351 return 0; 23352 } 23353 23354 static int process_fd_array(struct bpf_verifier_env *env, 23355 union bpf_attr *attr, bpfptr_t uattr) 23356 { 23357 bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 23358 23359 if (bpfptr_is_null(fd_array)) { 23360 if (attr->fd_array_cnt) { 23361 verbose(env, "fd_array_cnt %u without fd_array is invalid\n", 23362 attr->fd_array_cnt); 23363 return -EINVAL; 23364 } 23365 return 0; 23366 } 23367 /* 23368 * New API: the caller passes fd_array_cnt and a continuous array that 23369 * is resolved and bound up front. Legacy API (no fd_array_cnt): keep 23370 * the caller's array and resolve entries on the spot at each reference. 23371 */ 23372 if (attr->fd_array_cnt) 23373 return process_fd_array_continuous(env, fd_array, 23374 attr->fd_array_cnt); 23375 env->fd_array_raw = fd_array; 23376 return 0; 23377 } 23378 23379 /* replace a generic kfunc with a specialized version if necessary */ 23380 static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc, int insn_idx) 23381 { 23382 struct bpf_prog *prog = env->prog; 23383 bool seen_direct_write; 23384 void *xdp_kfunc; 23385 bool is_rdonly; 23386 u32 func_id = desc->func_id; 23387 u16 offset = desc->offset; 23388 unsigned long addr = desc->addr; 23389 23390 if (offset) /* return if module BTF is used */ 23391 return 0; 23392 23393 if (bpf_dev_bound_kfunc_id(func_id)) { 23394 xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); 23395 if (xdp_kfunc) 23396 addr = (unsigned long)xdp_kfunc; 23397 /* fallback to default kfunc when not supported by netdev */ 23398 } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { 23399 seen_direct_write = env->seen_direct_write; 23400 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); 23401 23402 if (is_rdonly) 23403 addr = (unsigned long)bpf_dynptr_from_skb_rdonly; 23404 23405 /* restore env->seen_direct_write to its original value, since 23406 * may_access_direct_pkt_data mutates it 23407 */ 23408 env->seen_direct_write = seen_direct_write; 23409 } else if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr]) { 23410 if (bpf_lsm_has_d_inode_locked(prog)) 23411 addr = (unsigned long)bpf_set_dentry_xattr_locked; 23412 } else if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr]) { 23413 if (bpf_lsm_has_d_inode_locked(prog)) 23414 addr = (unsigned long)bpf_remove_dentry_xattr_locked; 23415 } else if (func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { 23416 if (!env->insn_aux_data[insn_idx].non_sleepable) 23417 addr = (unsigned long)bpf_dynptr_from_file_sleepable; 23418 } else if (func_id == special_kfunc_list[KF_bpf_arena_alloc_pages]) { 23419 if (env->insn_aux_data[insn_idx].non_sleepable) 23420 addr = (unsigned long)bpf_arena_alloc_pages_non_sleepable; 23421 } else if (func_id == special_kfunc_list[KF_bpf_arena_free_pages]) { 23422 if (env->insn_aux_data[insn_idx].non_sleepable) 23423 addr = (unsigned long)bpf_arena_free_pages_non_sleepable; 23424 } 23425 desc->addr = addr; 23426 return 0; 23427 } 23428 23429 static int add_kfunc_desc_target(struct bpf_verifier_env *env, 23430 const struct bpf_kfunc_desc *target_desc, 23431 u16 base_desc_idx, u16 *desc_idx) 23432 { 23433 struct bpf_kfunc_desc desc = *target_desc; 23434 struct bpf_kfunc_desc_tab *new_tab; 23435 struct bpf_kfunc_desc_tab *tab; 23436 struct bpf_prog_aux *prog_aux; 23437 u32 i; 23438 23439 prog_aux = env->prog->aux; 23440 tab = prog_aux->kfunc_tab; 23441 23442 if (base_desc_idx >= tab->nr_base_descs) { 23443 verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", 23444 base_desc_idx, tab->nr_base_descs); 23445 return -EFAULT; 23446 } 23447 if (tab->descs[base_desc_idx].func_id != desc.func_id) { 23448 verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", 23449 base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); 23450 return -EFAULT; 23451 } 23452 if (tab->descs[base_desc_idx].offset != desc.offset) { 23453 verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", 23454 base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); 23455 return -EFAULT; 23456 } 23457 23458 if (tab->descs[base_desc_idx].addr == desc.addr) { 23459 *desc_idx = base_desc_idx; 23460 return 0; 23461 } 23462 23463 for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { 23464 if (tab->descs[i].func_id == desc.func_id && 23465 tab->descs[i].offset == desc.offset && 23466 tab->descs[i].addr == desc.addr) { 23467 *desc_idx = i; 23468 return 0; 23469 } 23470 } 23471 23472 if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { 23473 verbose(env, "too many different kernel function call targets\n"); 23474 return -E2BIG; 23475 } 23476 23477 new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), 23478 GFP_KERNEL_ACCOUNT); 23479 if (!new_tab) 23480 return -ENOMEM; 23481 tab = new_tab; 23482 prog_aux->kfunc_tab = tab; 23483 23484 *desc_idx = tab->nr_descs; 23485 tab->descs[tab->nr_descs++] = desc; 23486 return 0; 23487 } 23488 23489 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, 23490 u16 struct_meta_reg, 23491 u16 node_offset_reg, 23492 struct bpf_insn *insn, 23493 struct bpf_insn *insn_buf, 23494 int *cnt) 23495 { 23496 struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta; 23497 struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) }; 23498 23499 insn_buf[0] = addr[0]; 23500 insn_buf[1] = addr[1]; 23501 insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off); 23502 insn_buf[3] = *insn; 23503 *cnt = 4; 23504 } 23505 23506 int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 23507 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 23508 { 23509 struct bpf_kfunc_desc desc_copy; 23510 struct bpf_kfunc_desc *desc; 23511 unsigned long call_imm; 23512 u16 base_desc_idx, desc_idx; 23513 bool near_call; 23514 int err; 23515 23516 if (!insn->imm) { 23517 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 23518 return -EINVAL; 23519 } 23520 23521 *cnt = 0; 23522 23523 /* insn->imm has the btf func_id. Replace it with an offset relative to 23524 * __bpf_call_base, unless the JIT needs to call functions that are 23525 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()). 23526 */ 23527 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 23528 if (!desc) { 23529 verifier_bug(env, "kernel function descriptor not found for func_id %u", 23530 insn->imm); 23531 return -EFAULT; 23532 } 23533 base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; 23534 23535 near_call = !bpf_jit_supports_far_kfunc_call(); 23536 desc_copy = *desc; 23537 desc = &desc_copy; 23538 23539 err = specialize_kfunc(env, desc, insn_idx); 23540 if (err) 23541 return err; 23542 23543 if (near_call) { 23544 call_imm = BPF_CALL_IMM(desc->addr); 23545 if ((unsigned long)(s32)call_imm != call_imm) { 23546 verbose(env, "address of kernel func_id %u is out of range\n", 23547 desc->func_id); 23548 return -EINVAL; 23549 } 23550 insn->imm = call_imm; 23551 } 23552 23553 err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); 23554 if (err) 23555 return err; 23556 insn->off = desc_idx; 23557 23558 if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { 23559 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23560 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 23561 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 23562 23563 if (is_bpf_percpu_obj_new_kfunc(desc->func_id) && kptr_struct_meta) { 23564 verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", 23565 insn_idx); 23566 return -EFAULT; 23567 } 23568 23569 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 23570 insn_buf[1] = addr[0]; 23571 insn_buf[2] = addr[1]; 23572 insn_buf[3] = *insn; 23573 *cnt = 4; 23574 } else if (is_bpf_obj_drop_kfunc(desc->func_id) || 23575 is_bpf_percpu_obj_drop_kfunc(desc->func_id) || 23576 is_bpf_refcount_acquire_kfunc(desc->func_id)) { 23577 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23578 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 23579 23580 if (is_bpf_percpu_obj_drop_kfunc(desc->func_id) && kptr_struct_meta) { 23581 verifier_bug(env, "NULL kptr_struct_meta expected at insn_idx %d", 23582 insn_idx); 23583 return -EFAULT; 23584 } 23585 23586 if (is_bpf_refcount_acquire_kfunc(desc->func_id) && !kptr_struct_meta) { 23587 verifier_bug(env, "kptr_struct_meta expected at insn_idx %d", 23588 insn_idx); 23589 return -EFAULT; 23590 } 23591 23592 insn_buf[0] = addr[0]; 23593 insn_buf[1] = addr[1]; 23594 insn_buf[2] = *insn; 23595 *cnt = 3; 23596 } else if (is_bpf_list_push_kfunc(desc->func_id) || 23597 is_bpf_rbtree_add_kfunc(desc->func_id)) { 23598 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 23599 int struct_meta_reg = BPF_REG_3; 23600 int node_offset_reg = BPF_REG_4; 23601 23602 /* list_add/rbtree_add have an extra arg (prev/less), 23603 * so args-to-fixup are in diff regs. 23604 */ 23605 if (desc->func_id == special_kfunc_list[KF_bpf_list_add] || 23606 is_bpf_rbtree_add_kfunc(desc->func_id)) { 23607 struct_meta_reg = BPF_REG_4; 23608 node_offset_reg = BPF_REG_5; 23609 } 23610 23611 if (!kptr_struct_meta) { 23612 verifier_bug(env, "kptr_struct_meta expected at insn_idx %d", 23613 insn_idx); 23614 return -EFAULT; 23615 } 23616 23617 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, 23618 node_offset_reg, insn, insn_buf, cnt); 23619 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 23620 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 23621 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 23622 *cnt = 1; 23623 } else if (desc->func_id == special_kfunc_list[KF_bpf_session_is_return] && 23624 (env->prog->expected_attach_type == BPF_TRACE_FSESSION || 23625 env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { 23626 23627 /* 23628 * inline the bpf_session_is_return() for fsession: 23629 * bool bpf_session_is_return(void *ctx) 23630 * { 23631 * return (((u64 *)ctx)[-1] >> BPF_TRAMP_IS_RETURN_SHIFT) & 1; 23632 * } 23633 */ 23634 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 23635 insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_IS_RETURN_SHIFT); 23636 insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1); 23637 *cnt = 3; 23638 } else if (desc->func_id == special_kfunc_list[KF_bpf_session_cookie] && 23639 (env->prog->expected_attach_type == BPF_TRACE_FSESSION || 23640 env->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)) { 23641 /* 23642 * inline bpf_session_cookie() for fsession: 23643 * __u64 *bpf_session_cookie(void *ctx) 23644 * { 23645 * u64 off = (((u64 *)ctx)[-1] >> BPF_TRAMP_COOKIE_INDEX_SHIFT) & 0xFF; 23646 * return &((u64 *)ctx)[-off]; 23647 * } 23648 */ 23649 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 23650 insn_buf[1] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, BPF_TRAMP_COOKIE_INDEX_SHIFT); 23651 insn_buf[2] = BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFF); 23652 insn_buf[3] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 23653 insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); 23654 insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); 23655 *cnt = 6; 23656 } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { 23657 /* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */ 23658 int i = 0; 23659 23660 /* if (start > end) goto einval; */ 23661 insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8); 23662 /* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */ 23663 insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3); 23664 insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2); 23665 insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8); 23666 /* s->cur = start - 1; s->end = end; return 0; */ 23667 insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1); 23668 insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0); 23669 insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4); 23670 insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); 23671 insn_buf[i++] = BPF_JMP_A(5); 23672 /* einval: s->cur = s->end = 0; return -EINVAL; */ 23673 insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); 23674 insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 23675 insn_buf[i++] = BPF_JMP_A(2); 23676 /* e2big: s->cur = s->end = 0; return -E2BIG; */ 23677 insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); 23678 insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); 23679 *cnt = i; 23680 } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { 23681 /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ 23682 int i = 0; 23683 23684 /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ 23685 insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); 23686 insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); 23687 insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); 23688 insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); 23689 /* s->cur = r0; return &s->cur; */ 23690 insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); 23691 insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 23692 insn_buf[i++] = BPF_JMP_A(2); 23693 /* done: s->cur = s->end = 0; return NULL; */ 23694 insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); 23695 insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); 23696 *cnt = i; 23697 } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) { 23698 /* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */ 23699 insn_buf[0] = BPF_JMP_A(0); 23700 *cnt = 1; 23701 } 23702 23703 if (env->insn_aux_data[insn_idx].arg_prog) { 23704 u32 regno = env->insn_aux_data[insn_idx].arg_prog; 23705 struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) }; 23706 int idx = *cnt; 23707 23708 insn_buf[idx++] = ld_addrs[0]; 23709 insn_buf[idx++] = ld_addrs[1]; 23710 insn_buf[idx++] = *insn; 23711 *cnt = idx; 23712 } 23713 return 0; 23714 } 23715 23716 /* 23717 * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably 23718 * above the largest signature the kernel can verify, and far below anything 23719 * that would make rejecting a load expensive. Deliberately a fixed number so 23720 * that what the syscall accepts does not depend on PAGE_SIZE. 23721 */ 23722 #define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) 23723 23724 static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) 23725 { 23726 switch (keyring_id) { 23727 case 0: 23728 return BPF_SIG_KEYRING_BUILTIN; 23729 case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: 23730 return BPF_SIG_KEYRING_SECONDARY; 23731 case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: 23732 return BPF_SIG_KEYRING_PLATFORM; 23733 case KEY_SPEC_BPF_KEYRING: 23734 return BPF_SIG_KEYRING_BPF; 23735 default: 23736 return BPF_SIG_KEYRING_USER; 23737 } 23738 } 23739 23740 /* 23741 * Verify the PKCS#7 signature of a loaded program. Called from bpf_check() 23742 * once the program's metadata maps have been resolved into used_maps, so 23743 * the exact maps folded into the signature are the ones the program binds. 23744 * 23745 * The signature covers the instructions followed by the frozen contents of 23746 * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the 23747 * verdict and keyring info are recorded on prog->aux. 23748 */ 23749 static int bpf_prog_verify_signature(struct bpf_verifier_env *env, 23750 union bpf_attr *attr, bool is_kernel) 23751 { 23752 bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); 23753 struct bpf_dynptr_kern sig_ptr, data_ptr; 23754 struct bpf_prog *prog = env->prog; 23755 struct bpf_map **maps = env->used_maps; 23756 struct bpf_key *key = NULL; 23757 void *sig, *data = NULL; 23758 u32 map_cnt = env->used_map_cnt; 23759 u32 i, off, insns_sz; 23760 u64 data_sz; 23761 int err = 0; 23762 23763 /* 23764 * The signature covers the insns as submitted. CO-RE relocations would 23765 * have the kernel rewrite them afterwards, with values the loader picks 23766 * through the BTF it supplies, see bpf_check_core_relo(). 23767 */ 23768 if (attr->core_relo_cnt) { 23769 verbose(env, "signed program cannot carry CO-RE relocations\n"); 23770 return -EACCES; 23771 } 23772 if (!attr->signature_size || 23773 attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) 23774 return -EINVAL; 23775 23776 if (system_keyring_id_check(attr->keyring_id) == 0) { 23777 key = bpf_lookup_system_key(attr->keyring_id); 23778 } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { 23779 key = bpf_lookup_keyring(); 23780 } else if (bpf_keyring_enforced()) { 23781 verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); 23782 return -EPERM; 23783 } else { 23784 key = bpf_lookup_user_key(attr->keyring_id, 0); 23785 } 23786 if (!key) { 23787 if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { 23788 verbose(env, "the bpf keyring is empty or has not been restricted\n"); 23789 return -ENOKEY; 23790 } 23791 verbose(env, "cannot resolve signing keyring with keyring_id %d\n", 23792 attr->keyring_id); 23793 return -EINVAL; 23794 } 23795 23796 sig = kvmemdup_bpfptr(usig, attr->signature_size); 23797 if (IS_ERR(sig)) { 23798 bpf_key_put(key); 23799 return PTR_ERR(sig); 23800 } 23801 23802 insns_sz = prog->len * sizeof(struct bpf_insn); 23803 data_sz = insns_sz; 23804 for (i = 0; i < map_cnt; i++) { 23805 struct bpf_map *map = maps[i]; 23806 23807 if (map->map_type != BPF_MAP_TYPE_ARRAY || 23808 !map->ops->map_direct_value_addr) { 23809 verbose(env, "signed program metadata map '%s' must be an array\n", 23810 map->name); 23811 err = -EINVAL; 23812 goto out; 23813 } 23814 if (!READ_ONCE(map->frozen)) { 23815 verbose(env, "signed program metadata map '%s' must be frozen\n", 23816 map->name); 23817 err = -EPERM; 23818 goto out; 23819 } 23820 if (bpf_map_write_active(map)) { 23821 verbose(env, "signed program metadata map '%s' has active writers\n", 23822 map->name); 23823 err = -EBUSY; 23824 goto out; 23825 } 23826 if (!map->excl_prog_sha) { 23827 verbose(env, "signed program metadata map '%s' must be exclusive\n", 23828 map->name); 23829 err = -EPERM; 23830 goto out; 23831 } 23832 data_sz += map->value_size; 23833 } 23834 if (bpf_dynptr_check_size(data_sz)) { 23835 verbose(env, "signed payload too large: %llu bytes\n", data_sz); 23836 err = -E2BIG; 23837 goto out; 23838 } 23839 data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); 23840 if (!data) { 23841 err = -ENOMEM; 23842 goto out; 23843 } 23844 memcpy(data, prog->insnsi, insns_sz); 23845 off = insns_sz; 23846 for (i = 0; i < map_cnt; i++) { 23847 struct bpf_map *map = maps[i]; 23848 u64 addr; 23849 23850 err = map->ops->map_direct_value_addr(map, &addr, 0); 23851 if (err) { 23852 verbose(env, "failed to read signed metadata map '%s': %d\n", 23853 map->name, err); 23854 goto out; 23855 } 23856 memcpy(data + off, (void *)(unsigned long)addr, 23857 map->value_size); 23858 off += map->value_size; 23859 } 23860 23861 bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz); 23862 bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, 23863 attr->signature_size); 23864 23865 err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr, 23866 (struct bpf_dynptr *)&sig_ptr, key); 23867 if (err) { 23868 verbose(env, "signature verification failed: %d\n", err); 23869 } else { 23870 verbose(env, "signature verification passed\n"); 23871 prog->aux->sig.keyring_serial = bpf_key_serial(key); 23872 prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); 23873 prog->aux->sig.verdict = BPF_SIG_VERIFIED; 23874 } 23875 out: 23876 kvfree(data); 23877 bpf_key_put(key); 23878 kvfree(sig); 23879 return err; 23880 } 23881 23882 /* 23883 * bpf_clone_redirect() transmits from inside the program, so a tc egress or 23884 * lwt_xmit program that redirects to its own device runs again on top of its 23885 * own frame, and again from there, until the datapath's recursion limit drops 23886 * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack 23887 * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program 23888 * calling it keeps the smaller budget. The redirect helpers that transmit 23889 * after the program has returned leave no frame behind. 23890 */ 23891 static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) 23892 { 23893 const struct bpf_insn *insn = prog->insnsi; 23894 int i; 23895 23896 for (i = 0; i < prog->len; i++, insn++) 23897 if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect) 23898 return true; 23899 return false; 23900 } 23901 23902 /* Various log level 2 information about the program */ 23903 static void log_program(struct bpf_verifier_env *env) 23904 { 23905 struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; 23906 struct bpf_insn *insns = env->prog->insnsi; 23907 u32 insn_cnt = env->prog->len; 23908 u64 pos, insn_pos; 23909 u32 i, j; 23910 23911 verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n"); 23912 for (i = 0; i < insn_cnt; ++i) { 23913 verbose_linfo(env, i, " ; "); 23914 if (env->insn_aux_data[i].scc) 23915 verbose(env, "%3d ", env->insn_aux_data[i].scc); 23916 else 23917 verbose(env, " "); 23918 if (env->insn_aux_data[i].loop_header >= 0) 23919 verbose(env, "%3d ", env->insn_aux_data[i].loop_header); 23920 else 23921 verbose(env, " "); 23922 verbose(env, "%3d ", env->idoms[i]); 23923 verbose(env, "%3d: ", i); 23924 for (j = BPF_REG_0; j < BPF_REG_10; ++j) 23925 if (insn_aux[i].live_regs_before & BIT(j)) 23926 verbose(env, "%d", j); 23927 else 23928 verbose(env, "."); 23929 verbose(env, " "); 23930 pos = env->log.end_pos; 23931 bpf_verbose_insn(env, &insns[i]); 23932 insn_pos = env->log.end_pos; 23933 if (insn_aux[i].zext_dst) 23934 verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); 23935 verbose(env, "\n"); 23936 if (bpf_is_ldimm64(&insns[i])) 23937 i++; 23938 } 23939 } 23940 23941 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, 23942 struct bpf_log_attr *attr_log) 23943 { 23944 u64 start_time = ktime_get_ns(); 23945 struct bpf_verifier_env *env; 23946 int i, len, ret = -EINVAL, err; 23947 bool is_priv; 23948 23949 BTF_TYPE_EMIT(enum bpf_features); 23950 23951 /* no program is valid */ 23952 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 23953 return -EINVAL; 23954 23955 /* 'struct bpf_verifier_env' can be global, but since it's not small, 23956 * allocate/free it every time bpf_check() is called 23957 */ 23958 env = kvzalloc_obj(struct bpf_verifier_env, GFP_KERNEL_ACCOUNT); 23959 if (!env) 23960 return -ENOMEM; 23961 23962 env->bt.env = env; 23963 env->prog = *prog; 23964 env->ops = bpf_verifier_ops[env->prog->type]; 23965 env->stack_limit = bpf_prog_stack_limit(env->prog); 23966 if (bpf_prog_reenters_datapath(env->prog)) 23967 env->stack_limit = MAX_BPF_STACK; 23968 23969 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); 23970 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); 23971 env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); 23972 env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); 23973 env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); 23974 env->signature = attr->signature; 23975 23976 /* user could have requested verbose verifier output 23977 * and supplied buffer to store the verification trace 23978 */ 23979 ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); 23980 if (ret) 23981 goto err_free_env; 23982 ret = bpf_diag_init(env); 23983 if (ret) 23984 goto err_prep; 23985 if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) { 23986 verbose(env, "invalid bpf_ld_imm64 insn\n"); 23987 ret = -EINVAL; 23988 goto err_prep; 23989 } 23990 23991 ret = process_fd_array(env, attr, uattr); 23992 if (ret) 23993 goto err_prep; 23994 23995 if (env->signature) { 23996 ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel); 23997 if (ret) 23998 goto err_prep; 23999 } 24000 24001 ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token, 24002 uattr.is_kernel); 24003 if (ret) 24004 goto err_prep; 24005 24006 bpf_get_btf_vmlinux(); 24007 24008 /* Serialize verification of unprivileged programs. */ 24009 if (!is_priv) 24010 mutex_lock(&bpf_verifier_lock); 24011 24012 len = env->insn_aux_data_len = env->prog->len; 24013 env->insn_aux_data = 24014 __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), 24015 GFP_KERNEL_ACCOUNT | __GFP_ZERO); 24016 ret = -ENOMEM; 24017 if (!env->insn_aux_data) 24018 goto skip_full_check; 24019 for (i = 0; i < len; i++) 24020 env->insn_aux_data[i].orig_idx = i; 24021 env->succ = bpf_iarray_realloc(NULL, 2); 24022 if (!env->succ) 24023 goto skip_full_check; 24024 24025 mark_verifier_state_clean(env); 24026 24027 if (IS_ERR(btf_vmlinux)) { 24028 /* Either gcc or pahole or kernel are broken. */ 24029 verbose(env, "in-kernel BTF is malformed\n"); 24030 ret = PTR_ERR(btf_vmlinux); 24031 goto skip_full_check; 24032 } 24033 24034 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 24035 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 24036 env->strict_alignment = true; 24037 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 24038 env->strict_alignment = false; 24039 24040 if (is_priv) 24041 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 24042 env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; 24043 24044 env->explored_states = kvzalloc_objs(struct list_head, 24045 state_htab_size(env), 24046 GFP_KERNEL_ACCOUNT); 24047 ret = -ENOMEM; 24048 if (!env->explored_states) 24049 goto skip_full_check; 24050 24051 for (i = 0; i < state_htab_size(env); i++) 24052 INIT_LIST_HEAD(&env->explored_states[i]); 24053 INIT_LIST_HEAD(&env->free_list); 24054 24055 /* Prepare BTF and func_info needed to discover all subprograms. */ 24056 ret = bpf_prepare_btf_info(env, attr, uattr); 24057 if (ret < 0) 24058 goto skip_full_check; 24059 24060 /* Apply CO-RE before validating the program's instruction layout. */ 24061 ret = bpf_check_core_relo(env, attr, uattr); 24062 if (ret < 0) 24063 goto skip_full_check; 24064 24065 /* Discover all subprograms and collect the properties needed by BTF validation. */ 24066 ret = add_subprogs(env); 24067 if (ret < 0) 24068 goto skip_full_check; 24069 24070 find_subprog_properties(env); 24071 24072 /* Validate BTF before reporting subprogram layout errors. */ 24073 ret = bpf_check_btf_info(env, attr, uattr); 24074 if (ret < 0) 24075 goto skip_full_check; 24076 24077 ret = check_subprogs(env); 24078 if (ret < 0) 24079 goto skip_full_check; 24080 24081 /* Validate instructions and resolve the program's referenced resources. */ 24082 ret = check_and_resolve_insns(env); 24083 if (ret < 0) 24084 goto skip_full_check; 24085 24086 /* Find pointers to functions in the read-only maps of the program. */ 24087 ret = resolve_func_ptrs(env); 24088 if (ret < 0) 24089 goto skip_full_check; 24090 24091 /* Build kfunc prototypes after resolving program resources. */ 24092 ret = add_kfuncs(env); 24093 if (ret < 0) 24094 goto skip_full_check; 24095 24096 if (bpf_prog_is_offloaded(env->prog->aux)) { 24097 ret = bpf_prog_offload_verifier_prep(env->prog); 24098 if (ret) 24099 goto skip_full_check; 24100 } 24101 24102 ret = bpf_check_cfg(env); 24103 if (ret < 0) 24104 goto skip_full_check; 24105 24106 ret = check_unwind_insns(env); 24107 if (ret < 0) 24108 goto skip_full_check; 24109 24110 ret = bpf_compute_postorder(env); 24111 if (ret < 0) 24112 goto skip_full_check; 24113 24114 ret = bpf_stack_liveness_init(env); 24115 if (ret) 24116 goto skip_full_check; 24117 24118 ret = check_attach_btf_id(env); 24119 if (ret) 24120 goto skip_full_check; 24121 24122 ret = bpf_compute_const_regs(env); 24123 if (ret < 0) 24124 goto skip_full_check; 24125 24126 ret = bpf_prune_dead_branches(env); 24127 if (ret < 0) 24128 goto skip_full_check; 24129 24130 ret = sort_subprogs_topo(env); 24131 if (ret < 0) 24132 goto skip_full_check; 24133 24134 ret = bpf_compute_scc(env); 24135 if (ret < 0) 24136 goto skip_full_check; 24137 24138 /* must precede the first bpf_ret_reg_pair() user below */ 24139 ret = bpf_compute_subprog_ret_regs(env); 24140 if (ret < 0) 24141 goto skip_full_check; 24142 24143 prepare_subprog_args(env); 24144 24145 ret = bpf_compute_idoms(env); 24146 if (ret < 0) 24147 goto skip_full_check; 24148 24149 ret = bpf_compute_loops(env); 24150 if (ret < 0) 24151 goto skip_full_check; 24152 24153 ret = bpf_compute_live_registers(env); 24154 if (ret < 0) 24155 goto skip_full_check; 24156 24157 if (env->log.level & BPF_LOG_LEVEL2) 24158 log_program(env); 24159 24160 ret = bpf_init_scev(env); 24161 if (ret < 0) 24162 goto skip_full_check; 24163 24164 ret = bpf_compute_scev(env); 24165 if (ret < 0) 24166 goto skip_full_check; 24167 24168 bpf_free_subprog_jts(env); 24169 24170 ret = mark_fastcall_patterns(env); 24171 if (ret < 0) 24172 goto skip_full_check; 24173 24174 ret = do_check_main(env); 24175 ret = ret ?: do_check_subprogs(env); 24176 24177 /* reject recursion through the callx edges found by the main pass */ 24178 if (ret == 0 && env->callx_edges) 24179 ret = sort_subprogs_topo(env); 24180 24181 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) 24182 ret = bpf_prog_offload_finalize(env); 24183 24184 skip_full_check: 24185 kvfree(env->explored_states); 24186 24187 /* might decrease stack depth, keep it before passes that 24188 * allocate additional slots. 24189 */ 24190 if (ret == 0) 24191 ret = bpf_remove_fastcall_spills_fills(env); 24192 24193 if (ret == 0) 24194 ret = check_max_stack_depth(env); 24195 24196 /* instruction rewrites happen after this point */ 24197 if (ret == 0) 24198 ret = bpf_optimize_bpf_loop(env); 24199 24200 if (is_priv) { 24201 if (ret == 0) 24202 bpf_opt_hard_wire_dead_code_branches(env); 24203 if (ret == 0) 24204 ret = bpf_opt_remove_dead_code(env); 24205 if (ret == 0) 24206 ret = bpf_opt_remove_nops(env); 24207 } else { 24208 if (ret == 0) 24209 sanitize_dead_code(env); 24210 } 24211 24212 if (ret == 0) 24213 /* program is valid, convert *(u32*)(ctx + off) accesses */ 24214 ret = bpf_convert_ctx_accesses(env); 24215 24216 if (ret == 0) 24217 ret = bpf_do_misc_fixups(env); 24218 24219 /* do 32-bit optimization after insn patching has done so those patched 24220 * insns could be handled correctly. 24221 */ 24222 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { 24223 ret = bpf_opt_subreg_zext_lo32_rnd_hi32(env, attr); 24224 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 24225 : false; 24226 } 24227 24228 if (ret == 0) 24229 ret = bpf_fixup_call_args(env); 24230 24231 env->verification_time = ktime_get_ns() - start_time; 24232 print_verification_stats(env); 24233 env->prog->aux->verified_insns = env->insn_processed; 24234 24235 /* preserve original error even if log finalization is successful */ 24236 err = bpf_log_attr_finalize(attr_log, &env->log); 24237 if (err) 24238 ret = err; 24239 24240 if (ret) 24241 goto err_release_maps; 24242 24243 if (env->used_map_cnt) { 24244 /* if program passed verifier, update used_maps in bpf_prog_info */ 24245 env->prog->aux->used_maps = kmalloc_objs(env->used_maps[0], 24246 env->used_map_cnt, 24247 GFP_KERNEL_ACCOUNT); 24248 24249 if (!env->prog->aux->used_maps) { 24250 ret = -ENOMEM; 24251 goto err_release_maps; 24252 } 24253 24254 memcpy(env->prog->aux->used_maps, env->used_maps, 24255 sizeof(env->used_maps[0]) * env->used_map_cnt); 24256 env->prog->aux->used_map_cnt = env->used_map_cnt; 24257 } 24258 if (env->used_btf_cnt) { 24259 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 24260 env->prog->aux->used_btfs = kmalloc_objs(env->used_btfs[0], 24261 env->used_btf_cnt, 24262 GFP_KERNEL_ACCOUNT); 24263 if (!env->prog->aux->used_btfs) { 24264 ret = -ENOMEM; 24265 goto err_release_maps; 24266 } 24267 24268 memcpy(env->prog->aux->used_btfs, env->used_btfs, 24269 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 24270 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 24271 } 24272 if (env->used_map_cnt || env->used_btf_cnt) { 24273 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 24274 * bpf_ld_imm64 instructions 24275 */ 24276 convert_pseudo_ld_imm64(env); 24277 } 24278 24279 adjust_btf_func(env); 24280 24281 /* extension progs temporarily inherit the attach_type of their targets 24282 for verification purposes, so set it back to zero before returning 24283 */ 24284 if (env->prog->type == BPF_PROG_TYPE_EXT) 24285 env->prog->expected_attach_type = 0; 24286 24287 env->prog = __bpf_prog_select_runtime(env, env->prog, &ret); 24288 24289 err_release_maps: 24290 if (ret) 24291 release_insn_arrays(env); 24292 if (!env->prog->aux->used_maps) 24293 /* if we didn't copy map pointers into bpf_prog_info, release 24294 * them now. Otherwise free_used_maps() will release them. 24295 */ 24296 release_maps(env); 24297 if (!env->prog->aux->used_btfs) 24298 release_btfs(env); 24299 24300 *prog = env->prog; 24301 24302 module_put(env->attach_btf_mod); 24303 if (!is_priv) 24304 mutex_unlock(&bpf_verifier_lock); 24305 goto err_free_env; 24306 err_prep: 24307 err = bpf_log_attr_finalize(attr_log, &env->log); 24308 if (err) 24309 ret = err; 24310 release_insn_arrays(env); 24311 release_maps(env); 24312 release_btfs(env); 24313 err_free_env: 24314 bpf_free_subprog_jts(env); 24315 if (env->insn_aux_data) 24316 bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); 24317 vfree(env->insn_aux_data); 24318 kvfree(env->fd_array); 24319 bpf_free_scev(env); 24320 bpf_stack_liveness_free(env); 24321 kvfree(env->cfg.postorder_nums); 24322 kvfree(env->cfg.insn_postorder); 24323 kvfree(env->scc_info); 24324 kvfree(env->succ); 24325 kvfree(env->gotox_tmp_buf); 24326 kvfree(env->callx_edges); 24327 kvfree(env->func_ptrs); 24328 bpf_diag_free(env); 24329 kvfree(env->idoms); 24330 kvfree(env); 24331 return ret; 24332 } 24333