1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 #include <linux/poison.h> 27 #include <linux/module.h> 28 #include <linux/cpumask.h> 29 #include <linux/bpf_mem_alloc.h> 30 #include <net/xdp.h> 31 32 #include "disasm.h" 33 34 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 35 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 36 [_id] = & _name ## _verifier_ops, 37 #define BPF_MAP_TYPE(_id, _ops) 38 #define BPF_LINK_TYPE(_id, _name) 39 #include <linux/bpf_types.h> 40 #undef BPF_PROG_TYPE 41 #undef BPF_MAP_TYPE 42 #undef BPF_LINK_TYPE 43 }; 44 45 struct bpf_mem_alloc bpf_global_percpu_ma; 46 static bool bpf_global_percpu_ma_set; 47 48 /* bpf_check() is a static code analyzer that walks eBPF program 49 * instruction by instruction and updates register/stack state. 50 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 51 * 52 * The first pass is depth-first-search to check that the program is a DAG. 53 * It rejects the following programs: 54 * - larger than BPF_MAXINSNS insns 55 * - if loop is present (detected via back-edge) 56 * - unreachable insns exist (shouldn't be a forest. program = one function) 57 * - out of bounds or malformed jumps 58 * The second pass is all possible path descent from the 1st insn. 59 * Since it's analyzing all paths through the program, the length of the 60 * analysis is limited to 64k insn, which may be hit even if total number of 61 * insn is less then 4K, but there are too many branches that change stack/regs. 62 * Number of 'branches to be analyzed' is limited to 1k 63 * 64 * On entry to each instruction, each register has a type, and the instruction 65 * changes the types of the registers depending on instruction semantics. 66 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 67 * copied to R1. 68 * 69 * All registers are 64-bit. 70 * R0 - return register 71 * R1-R5 argument passing registers 72 * R6-R9 callee saved registers 73 * R10 - frame pointer read-only 74 * 75 * At the start of BPF program the register R1 contains a pointer to bpf_context 76 * and has type PTR_TO_CTX. 77 * 78 * Verifier tracks arithmetic operations on pointers in case: 79 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 80 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 81 * 1st insn copies R10 (which has FRAME_PTR) type into R1 82 * and 2nd arithmetic instruction is pattern matched to recognize 83 * that it wants to construct a pointer to some element within stack. 84 * So after 2nd insn, the register R1 has type PTR_TO_STACK 85 * (and -20 constant is saved for further stack bounds checking). 86 * Meaning that this reg is a pointer to stack plus known immediate constant. 87 * 88 * Most of the time the registers have SCALAR_VALUE type, which 89 * means the register has some value, but it's not a valid pointer. 90 * (like pointer plus pointer becomes SCALAR_VALUE type) 91 * 92 * When verifier sees load or store instructions the type of base register 93 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 94 * four pointer types recognized by check_mem_access() function. 95 * 96 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 97 * and the range of [ptr, ptr + map's value_size) is accessible. 98 * 99 * registers used to pass values to function calls are checked against 100 * function argument constraints. 101 * 102 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 103 * It means that the register type passed to this function must be 104 * PTR_TO_STACK and it will be used inside the function as 105 * 'pointer to map element key' 106 * 107 * For example the argument constraints for bpf_map_lookup_elem(): 108 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 109 * .arg1_type = ARG_CONST_MAP_PTR, 110 * .arg2_type = ARG_PTR_TO_MAP_KEY, 111 * 112 * ret_type says that this function returns 'pointer to map elem value or null' 113 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 114 * 2nd argument should be a pointer to stack, which will be used inside 115 * the helper function as a pointer to map element key. 116 * 117 * On the kernel side the helper function looks like: 118 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 119 * { 120 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 121 * void *key = (void *) (unsigned long) r2; 122 * void *value; 123 * 124 * here kernel can access 'key' and 'map' pointers safely, knowing that 125 * [key, key + map->key_size) bytes are valid and were initialized on 126 * the stack of eBPF program. 127 * } 128 * 129 * Corresponding eBPF program may look like: 130 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 131 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 132 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 133 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 134 * here verifier looks at prototype of map_lookup_elem() and sees: 135 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 136 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 137 * 138 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 139 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 140 * and were initialized prior to this call. 141 * If it's ok, then verifier allows this BPF_CALL insn and looks at 142 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 143 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 144 * returns either pointer to map value or NULL. 145 * 146 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 147 * insn, the register holding that pointer in the true branch changes state to 148 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 149 * branch. See check_cond_jmp_op(). 150 * 151 * After the call R0 is set to return type of the function and registers R1-R5 152 * are set to NOT_INIT to indicate that they are no longer readable. 153 * 154 * The following reference types represent a potential reference to a kernel 155 * resource which, after first being allocated, must be checked and freed by 156 * the BPF program: 157 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 158 * 159 * When the verifier sees a helper call return a reference type, it allocates a 160 * pointer id for the reference and stores it in the current function state. 161 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 162 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 163 * passes through a NULL-check conditional. For the branch wherein the state is 164 * changed to CONST_IMM, the verifier releases the reference. 165 * 166 * For each helper function that allocates a reference, such as 167 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 168 * bpf_sk_release(). When a reference type passes into the release function, 169 * the verifier also releases the reference. If any unchecked or unreleased 170 * reference remains at the end of the program, the verifier rejects it. 171 */ 172 173 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 174 struct bpf_verifier_stack_elem { 175 /* verifier state is 'st' 176 * before processing instruction 'insn_idx' 177 * and after processing instruction 'prev_insn_idx' 178 */ 179 struct bpf_verifier_state st; 180 int insn_idx; 181 int prev_insn_idx; 182 struct bpf_verifier_stack_elem *next; 183 /* length of verifier log at the time this state was pushed on stack */ 184 u32 log_pos; 185 }; 186 187 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 188 #define BPF_COMPLEXITY_LIMIT_STATES 64 189 190 #define BPF_MAP_KEY_POISON (1ULL << 63) 191 #define BPF_MAP_KEY_SEEN (1ULL << 62) 192 193 #define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512 194 195 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 196 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 197 static void invalidate_non_owning_refs(struct bpf_verifier_env *env); 198 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); 199 static int ref_set_non_owning(struct bpf_verifier_env *env, 200 struct bpf_reg_state *reg); 201 static void specialize_kfunc(struct bpf_verifier_env *env, 202 u32 func_id, u16 offset, unsigned long *addr); 203 static bool is_trusted_reg(const struct bpf_reg_state *reg); 204 205 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 206 { 207 return aux->map_ptr_state.poison; 208 } 209 210 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 211 { 212 return aux->map_ptr_state.unpriv; 213 } 214 215 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 216 struct bpf_map *map, 217 bool unpriv, bool poison) 218 { 219 unpriv |= bpf_map_ptr_unpriv(aux); 220 aux->map_ptr_state.unpriv = unpriv; 221 aux->map_ptr_state.poison = poison; 222 aux->map_ptr_state.map_ptr = map; 223 } 224 225 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 226 { 227 return aux->map_key_state & BPF_MAP_KEY_POISON; 228 } 229 230 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 231 { 232 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 233 } 234 235 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 236 { 237 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 238 } 239 240 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 241 { 242 bool poisoned = bpf_map_key_poisoned(aux); 243 244 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 245 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 246 } 247 248 static bool bpf_helper_call(const struct bpf_insn *insn) 249 { 250 return insn->code == (BPF_JMP | BPF_CALL) && 251 insn->src_reg == 0; 252 } 253 254 static bool bpf_pseudo_call(const struct bpf_insn *insn) 255 { 256 return insn->code == (BPF_JMP | BPF_CALL) && 257 insn->src_reg == BPF_PSEUDO_CALL; 258 } 259 260 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 261 { 262 return insn->code == (BPF_JMP | BPF_CALL) && 263 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 264 } 265 266 struct bpf_call_arg_meta { 267 struct bpf_map *map_ptr; 268 bool raw_mode; 269 bool pkt_access; 270 u8 release_regno; 271 int regno; 272 int access_size; 273 int mem_size; 274 u64 msize_max_value; 275 int ref_obj_id; 276 int dynptr_id; 277 int map_uid; 278 int func_id; 279 struct btf *btf; 280 u32 btf_id; 281 struct btf *ret_btf; 282 u32 ret_btf_id; 283 u32 subprogno; 284 struct btf_field *kptr_field; 285 }; 286 287 struct bpf_kfunc_call_arg_meta { 288 /* In parameters */ 289 struct btf *btf; 290 u32 func_id; 291 u32 kfunc_flags; 292 const struct btf_type *func_proto; 293 const char *func_name; 294 /* Out parameters */ 295 u32 ref_obj_id; 296 u8 release_regno; 297 bool r0_rdonly; 298 u32 ret_btf_id; 299 u64 r0_size; 300 u32 subprogno; 301 struct { 302 u64 value; 303 bool found; 304 } arg_constant; 305 306 /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, 307 * generally to pass info about user-defined local kptr types to later 308 * verification logic 309 * bpf_obj_drop/bpf_percpu_obj_drop 310 * Record the local kptr type to be drop'd 311 * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) 312 * Record the local kptr type to be refcount_incr'd and use 313 * arg_owning_ref to determine whether refcount_acquire should be 314 * fallible 315 */ 316 struct btf *arg_btf; 317 u32 arg_btf_id; 318 bool arg_owning_ref; 319 320 struct { 321 struct btf_field *field; 322 } arg_list_head; 323 struct { 324 struct btf_field *field; 325 } arg_rbtree_root; 326 struct { 327 enum bpf_dynptr_type type; 328 u32 id; 329 u32 ref_obj_id; 330 } initialized_dynptr; 331 struct { 332 u8 spi; 333 u8 frameno; 334 } iter; 335 struct { 336 struct bpf_map *ptr; 337 int uid; 338 } map; 339 u64 mem_size; 340 }; 341 342 struct btf *btf_vmlinux; 343 344 static const char *btf_type_name(const struct btf *btf, u32 id) 345 { 346 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 347 } 348 349 static DEFINE_MUTEX(bpf_verifier_lock); 350 static DEFINE_MUTEX(bpf_percpu_ma_lock); 351 352 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 353 { 354 struct bpf_verifier_env *env = private_data; 355 va_list args; 356 357 if (!bpf_verifier_log_needed(&env->log)) 358 return; 359 360 va_start(args, fmt); 361 bpf_verifier_vlog(&env->log, fmt, args); 362 va_end(args); 363 } 364 365 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 366 struct bpf_reg_state *reg, 367 struct bpf_retval_range range, const char *ctx, 368 const char *reg_name) 369 { 370 bool unknown = true; 371 372 verbose(env, "%s the register %s has", ctx, reg_name); 373 if (reg->smin_value > S64_MIN) { 374 verbose(env, " smin=%lld", reg->smin_value); 375 unknown = false; 376 } 377 if (reg->smax_value < S64_MAX) { 378 verbose(env, " smax=%lld", reg->smax_value); 379 unknown = false; 380 } 381 if (unknown) 382 verbose(env, " unknown scalar value"); 383 verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); 384 } 385 386 static bool type_may_be_null(u32 type) 387 { 388 return type & PTR_MAYBE_NULL; 389 } 390 391 static bool reg_not_null(const struct bpf_reg_state *reg) 392 { 393 enum bpf_reg_type type; 394 395 type = reg->type; 396 if (type_may_be_null(type)) 397 return false; 398 399 type = base_type(type); 400 return type == PTR_TO_SOCKET || 401 type == PTR_TO_TCP_SOCK || 402 type == PTR_TO_MAP_VALUE || 403 type == PTR_TO_MAP_KEY || 404 type == PTR_TO_SOCK_COMMON || 405 (type == PTR_TO_BTF_ID && is_trusted_reg(reg)) || 406 type == PTR_TO_MEM; 407 } 408 409 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 410 { 411 struct btf_record *rec = NULL; 412 struct btf_struct_meta *meta; 413 414 if (reg->type == PTR_TO_MAP_VALUE) { 415 rec = reg->map_ptr->record; 416 } else if (type_is_ptr_alloc_obj(reg->type)) { 417 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 418 if (meta) 419 rec = meta->record; 420 } 421 return rec; 422 } 423 424 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) 425 { 426 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; 427 428 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; 429 } 430 431 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) 432 { 433 struct bpf_func_info *info; 434 435 if (!env->prog->aux->func_info) 436 return ""; 437 438 info = &env->prog->aux->func_info[subprog]; 439 return btf_type_name(env->prog->aux->btf, info->type_id); 440 } 441 442 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) 443 { 444 struct bpf_subprog_info *info = subprog_info(env, subprog); 445 446 info->is_cb = true; 447 info->is_async_cb = true; 448 info->is_exception_cb = true; 449 } 450 451 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) 452 { 453 return subprog_info(env, subprog)->is_exception_cb; 454 } 455 456 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 457 { 458 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); 459 } 460 461 static bool type_is_rdonly_mem(u32 type) 462 { 463 return type & MEM_RDONLY; 464 } 465 466 static bool is_acquire_function(enum bpf_func_id func_id, 467 const struct bpf_map *map) 468 { 469 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 470 471 if (func_id == BPF_FUNC_sk_lookup_tcp || 472 func_id == BPF_FUNC_sk_lookup_udp || 473 func_id == BPF_FUNC_skc_lookup_tcp || 474 func_id == BPF_FUNC_ringbuf_reserve || 475 func_id == BPF_FUNC_kptr_xchg) 476 return true; 477 478 if (func_id == BPF_FUNC_map_lookup_elem && 479 (map_type == BPF_MAP_TYPE_SOCKMAP || 480 map_type == BPF_MAP_TYPE_SOCKHASH)) 481 return true; 482 483 return false; 484 } 485 486 static bool is_ptr_cast_function(enum bpf_func_id func_id) 487 { 488 return func_id == BPF_FUNC_tcp_sock || 489 func_id == BPF_FUNC_sk_fullsock || 490 func_id == BPF_FUNC_skc_to_tcp_sock || 491 func_id == BPF_FUNC_skc_to_tcp6_sock || 492 func_id == BPF_FUNC_skc_to_udp6_sock || 493 func_id == BPF_FUNC_skc_to_mptcp_sock || 494 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 495 func_id == BPF_FUNC_skc_to_tcp_request_sock; 496 } 497 498 static bool is_dynptr_ref_function(enum bpf_func_id func_id) 499 { 500 return func_id == BPF_FUNC_dynptr_data; 501 } 502 503 static bool is_sync_callback_calling_kfunc(u32 btf_id); 504 static bool is_async_callback_calling_kfunc(u32 btf_id); 505 static bool is_callback_calling_kfunc(u32 btf_id); 506 static bool is_bpf_throw_kfunc(struct bpf_insn *insn); 507 508 static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id); 509 510 static bool is_sync_callback_calling_function(enum bpf_func_id func_id) 511 { 512 return func_id == BPF_FUNC_for_each_map_elem || 513 func_id == BPF_FUNC_find_vma || 514 func_id == BPF_FUNC_loop || 515 func_id == BPF_FUNC_user_ringbuf_drain; 516 } 517 518 static bool is_async_callback_calling_function(enum bpf_func_id func_id) 519 { 520 return func_id == BPF_FUNC_timer_set_callback; 521 } 522 523 static bool is_callback_calling_function(enum bpf_func_id func_id) 524 { 525 return is_sync_callback_calling_function(func_id) || 526 is_async_callback_calling_function(func_id); 527 } 528 529 static bool is_sync_callback_calling_insn(struct bpf_insn *insn) 530 { 531 return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) || 532 (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm)); 533 } 534 535 static bool is_async_callback_calling_insn(struct bpf_insn *insn) 536 { 537 return (bpf_helper_call(insn) && is_async_callback_calling_function(insn->imm)) || 538 (bpf_pseudo_kfunc_call(insn) && is_async_callback_calling_kfunc(insn->imm)); 539 } 540 541 static bool is_may_goto_insn(struct bpf_insn *insn) 542 { 543 return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; 544 } 545 546 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx) 547 { 548 return is_may_goto_insn(&env->prog->insnsi[insn_idx]); 549 } 550 551 static bool is_storage_get_function(enum bpf_func_id func_id) 552 { 553 return func_id == BPF_FUNC_sk_storage_get || 554 func_id == BPF_FUNC_inode_storage_get || 555 func_id == BPF_FUNC_task_storage_get || 556 func_id == BPF_FUNC_cgrp_storage_get; 557 } 558 559 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, 560 const struct bpf_map *map) 561 { 562 int ref_obj_uses = 0; 563 564 if (is_ptr_cast_function(func_id)) 565 ref_obj_uses++; 566 if (is_acquire_function(func_id, map)) 567 ref_obj_uses++; 568 if (is_dynptr_ref_function(func_id)) 569 ref_obj_uses++; 570 571 return ref_obj_uses > 1; 572 } 573 574 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 575 { 576 return BPF_CLASS(insn->code) == BPF_STX && 577 BPF_MODE(insn->code) == BPF_ATOMIC && 578 insn->imm == BPF_CMPXCHG; 579 } 580 581 static int __get_spi(s32 off) 582 { 583 return (-off - 1) / BPF_REG_SIZE; 584 } 585 586 static struct bpf_func_state *func(struct bpf_verifier_env *env, 587 const struct bpf_reg_state *reg) 588 { 589 struct bpf_verifier_state *cur = env->cur_state; 590 591 return cur->frame[reg->frameno]; 592 } 593 594 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 595 { 596 int allocated_slots = state->allocated_stack / BPF_REG_SIZE; 597 598 /* We need to check that slots between [spi - nr_slots + 1, spi] are 599 * within [0, allocated_stack). 600 * 601 * Please note that the spi grows downwards. For example, a dynptr 602 * takes the size of two stack slots; the first slot will be at 603 * spi and the second slot will be at spi - 1. 604 */ 605 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 606 } 607 608 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 609 const char *obj_kind, int nr_slots) 610 { 611 int off, spi; 612 613 if (!tnum_is_const(reg->var_off)) { 614 verbose(env, "%s has to be at a constant offset\n", obj_kind); 615 return -EINVAL; 616 } 617 618 off = reg->off + reg->var_off.value; 619 if (off % BPF_REG_SIZE) { 620 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 621 return -EINVAL; 622 } 623 624 spi = __get_spi(off); 625 if (spi + 1 < nr_slots) { 626 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 627 return -EINVAL; 628 } 629 630 if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots)) 631 return -ERANGE; 632 return spi; 633 } 634 635 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 636 { 637 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); 638 } 639 640 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) 641 { 642 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); 643 } 644 645 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 646 { 647 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 648 case DYNPTR_TYPE_LOCAL: 649 return BPF_DYNPTR_TYPE_LOCAL; 650 case DYNPTR_TYPE_RINGBUF: 651 return BPF_DYNPTR_TYPE_RINGBUF; 652 case DYNPTR_TYPE_SKB: 653 return BPF_DYNPTR_TYPE_SKB; 654 case DYNPTR_TYPE_XDP: 655 return BPF_DYNPTR_TYPE_XDP; 656 default: 657 return BPF_DYNPTR_TYPE_INVALID; 658 } 659 } 660 661 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type) 662 { 663 switch (type) { 664 case BPF_DYNPTR_TYPE_LOCAL: 665 return DYNPTR_TYPE_LOCAL; 666 case BPF_DYNPTR_TYPE_RINGBUF: 667 return DYNPTR_TYPE_RINGBUF; 668 case BPF_DYNPTR_TYPE_SKB: 669 return DYNPTR_TYPE_SKB; 670 case BPF_DYNPTR_TYPE_XDP: 671 return DYNPTR_TYPE_XDP; 672 default: 673 return 0; 674 } 675 } 676 677 static bool dynptr_type_refcounted(enum bpf_dynptr_type type) 678 { 679 return type == BPF_DYNPTR_TYPE_RINGBUF; 680 } 681 682 static void __mark_dynptr_reg(struct bpf_reg_state *reg, 683 enum bpf_dynptr_type type, 684 bool first_slot, int dynptr_id); 685 686 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 687 struct bpf_reg_state *reg); 688 689 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, 690 struct bpf_reg_state *sreg1, 691 struct bpf_reg_state *sreg2, 692 enum bpf_dynptr_type type) 693 { 694 int id = ++env->id_gen; 695 696 __mark_dynptr_reg(sreg1, type, true, id); 697 __mark_dynptr_reg(sreg2, type, false, id); 698 } 699 700 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, 701 struct bpf_reg_state *reg, 702 enum bpf_dynptr_type type) 703 { 704 __mark_dynptr_reg(reg, type, true, ++env->id_gen); 705 } 706 707 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 708 struct bpf_func_state *state, int spi); 709 710 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 711 enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id) 712 { 713 struct bpf_func_state *state = func(env, reg); 714 enum bpf_dynptr_type type; 715 int spi, i, err; 716 717 spi = dynptr_get_spi(env, reg); 718 if (spi < 0) 719 return spi; 720 721 /* We cannot assume both spi and spi - 1 belong to the same dynptr, 722 * hence we need to call destroy_if_dynptr_stack_slot twice for both, 723 * to ensure that for the following example: 724 * [d1][d1][d2][d2] 725 * spi 3 2 1 0 726 * So marking spi = 2 should lead to destruction of both d1 and d2. In 727 * case they do belong to same dynptr, second call won't see slot_type 728 * as STACK_DYNPTR and will simply skip destruction. 729 */ 730 err = destroy_if_dynptr_stack_slot(env, state, spi); 731 if (err) 732 return err; 733 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); 734 if (err) 735 return err; 736 737 for (i = 0; i < BPF_REG_SIZE; i++) { 738 state->stack[spi].slot_type[i] = STACK_DYNPTR; 739 state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; 740 } 741 742 type = arg_to_dynptr_type(arg_type); 743 if (type == BPF_DYNPTR_TYPE_INVALID) 744 return -EINVAL; 745 746 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, 747 &state->stack[spi - 1].spilled_ptr, type); 748 749 if (dynptr_type_refcounted(type)) { 750 /* The id is used to track proper releasing */ 751 int id; 752 753 if (clone_ref_obj_id) 754 id = clone_ref_obj_id; 755 else 756 id = acquire_reference_state(env, insn_idx); 757 758 if (id < 0) 759 return id; 760 761 state->stack[spi].spilled_ptr.ref_obj_id = id; 762 state->stack[spi - 1].spilled_ptr.ref_obj_id = id; 763 } 764 765 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 766 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 767 768 return 0; 769 } 770 771 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) 772 { 773 int i; 774 775 for (i = 0; i < BPF_REG_SIZE; i++) { 776 state->stack[spi].slot_type[i] = STACK_INVALID; 777 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 778 } 779 780 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 781 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 782 783 /* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot? 784 * 785 * While we don't allow reading STACK_INVALID, it is still possible to 786 * do <8 byte writes marking some but not all slots as STACK_MISC. Then, 787 * helpers or insns can do partial read of that part without failing, 788 * but check_stack_range_initialized, check_stack_read_var_off, and 789 * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of 790 * the slot conservatively. Hence we need to prevent those liveness 791 * marking walks. 792 * 793 * This was not a problem before because STACK_INVALID is only set by 794 * default (where the default reg state has its reg->parent as NULL), or 795 * in clean_live_states after REG_LIVE_DONE (at which point 796 * mark_reg_read won't walk reg->parent chain), but not randomly during 797 * verifier state exploration (like we did above). Hence, for our case 798 * parentage chain will still be live (i.e. reg->parent may be 799 * non-NULL), while earlier reg->parent was NULL, so we need 800 * REG_LIVE_WRITTEN to screen off read marker propagation when it is 801 * done later on reads or by mark_dynptr_read as well to unnecessary 802 * mark registers in verifier state. 803 */ 804 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 805 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 806 } 807 808 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 809 { 810 struct bpf_func_state *state = func(env, reg); 811 int spi, ref_obj_id, i; 812 813 spi = dynptr_get_spi(env, reg); 814 if (spi < 0) 815 return spi; 816 817 if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 818 invalidate_dynptr(env, state, spi); 819 return 0; 820 } 821 822 ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id; 823 824 /* If the dynptr has a ref_obj_id, then we need to invalidate 825 * two things: 826 * 827 * 1) Any dynptrs with a matching ref_obj_id (clones) 828 * 2) Any slices derived from this dynptr. 829 */ 830 831 /* Invalidate any slices associated with this dynptr */ 832 WARN_ON_ONCE(release_reference(env, ref_obj_id)); 833 834 /* Invalidate any dynptr clones */ 835 for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) { 836 if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id) 837 continue; 838 839 /* it should always be the case that if the ref obj id 840 * matches then the stack slot also belongs to a 841 * dynptr 842 */ 843 if (state->stack[i].slot_type[0] != STACK_DYNPTR) { 844 verbose(env, "verifier internal error: misconfigured ref_obj_id\n"); 845 return -EFAULT; 846 } 847 if (state->stack[i].spilled_ptr.dynptr.first_slot) 848 invalidate_dynptr(env, state, i); 849 } 850 851 return 0; 852 } 853 854 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 855 struct bpf_reg_state *reg); 856 857 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 858 { 859 if (!env->allow_ptr_leaks) 860 __mark_reg_not_init(env, reg); 861 else 862 __mark_reg_unknown(env, reg); 863 } 864 865 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 866 struct bpf_func_state *state, int spi) 867 { 868 struct bpf_func_state *fstate; 869 struct bpf_reg_state *dreg; 870 int i, dynptr_id; 871 872 /* We always ensure that STACK_DYNPTR is never set partially, 873 * hence just checking for slot_type[0] is enough. This is 874 * different for STACK_SPILL, where it may be only set for 875 * 1 byte, so code has to use is_spilled_reg. 876 */ 877 if (state->stack[spi].slot_type[0] != STACK_DYNPTR) 878 return 0; 879 880 /* Reposition spi to first slot */ 881 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 882 spi = spi + 1; 883 884 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 885 verbose(env, "cannot overwrite referenced dynptr\n"); 886 return -EINVAL; 887 } 888 889 mark_stack_slot_scratched(env, spi); 890 mark_stack_slot_scratched(env, spi - 1); 891 892 /* Writing partially to one dynptr stack slot destroys both. */ 893 for (i = 0; i < BPF_REG_SIZE; i++) { 894 state->stack[spi].slot_type[i] = STACK_INVALID; 895 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 896 } 897 898 dynptr_id = state->stack[spi].spilled_ptr.id; 899 /* Invalidate any slices associated with this dynptr */ 900 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ 901 /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */ 902 if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM) 903 continue; 904 if (dreg->dynptr_id == dynptr_id) 905 mark_reg_invalid(env, dreg); 906 })); 907 908 /* Do not release reference state, we are destroying dynptr on stack, 909 * not using some helper to release it. Just reset register. 910 */ 911 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 912 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 913 914 /* Same reason as unmark_stack_slots_dynptr above */ 915 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 916 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 917 918 return 0; 919 } 920 921 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 922 { 923 int spi; 924 925 if (reg->type == CONST_PTR_TO_DYNPTR) 926 return false; 927 928 spi = dynptr_get_spi(env, reg); 929 930 /* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an 931 * error because this just means the stack state hasn't been updated yet. 932 * We will do check_mem_access to check and update stack bounds later. 933 */ 934 if (spi < 0 && spi != -ERANGE) 935 return false; 936 937 /* We don't need to check if the stack slots are marked by previous 938 * dynptr initializations because we allow overwriting existing unreferenced 939 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls 940 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are 941 * touching are completely destructed before we reinitialize them for a new 942 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early 943 * instead of delaying it until the end where the user will get "Unreleased 944 * reference" error. 945 */ 946 return true; 947 } 948 949 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 950 { 951 struct bpf_func_state *state = func(env, reg); 952 int i, spi; 953 954 /* This already represents first slot of initialized bpf_dynptr. 955 * 956 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to 957 * check_func_arg_reg_off's logic, so we don't need to check its 958 * offset and alignment. 959 */ 960 if (reg->type == CONST_PTR_TO_DYNPTR) 961 return true; 962 963 spi = dynptr_get_spi(env, reg); 964 if (spi < 0) 965 return false; 966 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 967 return false; 968 969 for (i = 0; i < BPF_REG_SIZE; i++) { 970 if (state->stack[spi].slot_type[i] != STACK_DYNPTR || 971 state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) 972 return false; 973 } 974 975 return true; 976 } 977 978 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 979 enum bpf_arg_type arg_type) 980 { 981 struct bpf_func_state *state = func(env, reg); 982 enum bpf_dynptr_type dynptr_type; 983 int spi; 984 985 /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ 986 if (arg_type == ARG_PTR_TO_DYNPTR) 987 return true; 988 989 dynptr_type = arg_to_dynptr_type(arg_type); 990 if (reg->type == CONST_PTR_TO_DYNPTR) { 991 return reg->dynptr.type == dynptr_type; 992 } else { 993 spi = dynptr_get_spi(env, reg); 994 if (spi < 0) 995 return false; 996 return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; 997 } 998 } 999 1000 static void __mark_reg_known_zero(struct bpf_reg_state *reg); 1001 1002 static bool in_rcu_cs(struct bpf_verifier_env *env); 1003 1004 static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta); 1005 1006 static int mark_stack_slots_iter(struct bpf_verifier_env *env, 1007 struct bpf_kfunc_call_arg_meta *meta, 1008 struct bpf_reg_state *reg, int insn_idx, 1009 struct btf *btf, u32 btf_id, int nr_slots) 1010 { 1011 struct bpf_func_state *state = func(env, reg); 1012 int spi, i, j, id; 1013 1014 spi = iter_get_spi(env, reg, nr_slots); 1015 if (spi < 0) 1016 return spi; 1017 1018 id = acquire_reference_state(env, insn_idx); 1019 if (id < 0) 1020 return id; 1021 1022 for (i = 0; i < nr_slots; i++) { 1023 struct bpf_stack_state *slot = &state->stack[spi - i]; 1024 struct bpf_reg_state *st = &slot->spilled_ptr; 1025 1026 __mark_reg_known_zero(st); 1027 st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ 1028 if (is_kfunc_rcu_protected(meta)) { 1029 if (in_rcu_cs(env)) 1030 st->type |= MEM_RCU; 1031 else 1032 st->type |= PTR_UNTRUSTED; 1033 } 1034 st->live |= REG_LIVE_WRITTEN; 1035 st->ref_obj_id = i == 0 ? id : 0; 1036 st->iter.btf = btf; 1037 st->iter.btf_id = btf_id; 1038 st->iter.state = BPF_ITER_STATE_ACTIVE; 1039 st->iter.depth = 0; 1040 1041 for (j = 0; j < BPF_REG_SIZE; j++) 1042 slot->slot_type[j] = STACK_ITER; 1043 1044 mark_stack_slot_scratched(env, spi - i); 1045 } 1046 1047 return 0; 1048 } 1049 1050 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, 1051 struct bpf_reg_state *reg, int nr_slots) 1052 { 1053 struct bpf_func_state *state = func(env, reg); 1054 int spi, i, j; 1055 1056 spi = iter_get_spi(env, reg, nr_slots); 1057 if (spi < 0) 1058 return spi; 1059 1060 for (i = 0; i < nr_slots; i++) { 1061 struct bpf_stack_state *slot = &state->stack[spi - i]; 1062 struct bpf_reg_state *st = &slot->spilled_ptr; 1063 1064 if (i == 0) 1065 WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); 1066 1067 __mark_reg_not_init(env, st); 1068 1069 /* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */ 1070 st->live |= REG_LIVE_WRITTEN; 1071 1072 for (j = 0; j < BPF_REG_SIZE; j++) 1073 slot->slot_type[j] = STACK_INVALID; 1074 1075 mark_stack_slot_scratched(env, spi - i); 1076 } 1077 1078 return 0; 1079 } 1080 1081 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, 1082 struct bpf_reg_state *reg, int nr_slots) 1083 { 1084 struct bpf_func_state *state = func(env, reg); 1085 int spi, i, j; 1086 1087 /* For -ERANGE (i.e. spi not falling into allocated stack slots), we 1088 * will do check_mem_access to check and update stack bounds later, so 1089 * return true for that case. 1090 */ 1091 spi = iter_get_spi(env, reg, nr_slots); 1092 if (spi == -ERANGE) 1093 return true; 1094 if (spi < 0) 1095 return false; 1096 1097 for (i = 0; i < nr_slots; i++) { 1098 struct bpf_stack_state *slot = &state->stack[spi - i]; 1099 1100 for (j = 0; j < BPF_REG_SIZE; j++) 1101 if (slot->slot_type[j] == STACK_ITER) 1102 return false; 1103 } 1104 1105 return true; 1106 } 1107 1108 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1109 struct btf *btf, u32 btf_id, int nr_slots) 1110 { 1111 struct bpf_func_state *state = func(env, reg); 1112 int spi, i, j; 1113 1114 spi = iter_get_spi(env, reg, nr_slots); 1115 if (spi < 0) 1116 return -EINVAL; 1117 1118 for (i = 0; i < nr_slots; i++) { 1119 struct bpf_stack_state *slot = &state->stack[spi - i]; 1120 struct bpf_reg_state *st = &slot->spilled_ptr; 1121 1122 if (st->type & PTR_UNTRUSTED) 1123 return -EPROTO; 1124 /* only main (first) slot has ref_obj_id set */ 1125 if (i == 0 && !st->ref_obj_id) 1126 return -EINVAL; 1127 if (i != 0 && st->ref_obj_id) 1128 return -EINVAL; 1129 if (st->iter.btf != btf || st->iter.btf_id != btf_id) 1130 return -EINVAL; 1131 1132 for (j = 0; j < BPF_REG_SIZE; j++) 1133 if (slot->slot_type[j] != STACK_ITER) 1134 return -EINVAL; 1135 } 1136 1137 return 0; 1138 } 1139 1140 /* Check if given stack slot is "special": 1141 * - spilled register state (STACK_SPILL); 1142 * - dynptr state (STACK_DYNPTR); 1143 * - iter state (STACK_ITER). 1144 */ 1145 static bool is_stack_slot_special(const struct bpf_stack_state *stack) 1146 { 1147 enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1]; 1148 1149 switch (type) { 1150 case STACK_SPILL: 1151 case STACK_DYNPTR: 1152 case STACK_ITER: 1153 return true; 1154 case STACK_INVALID: 1155 case STACK_MISC: 1156 case STACK_ZERO: 1157 return false; 1158 default: 1159 WARN_ONCE(1, "unknown stack slot type %d\n", type); 1160 return true; 1161 } 1162 } 1163 1164 /* The reg state of a pointer or a bounded scalar was saved when 1165 * it was spilled to the stack. 1166 */ 1167 static bool is_spilled_reg(const struct bpf_stack_state *stack) 1168 { 1169 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 1170 } 1171 1172 static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack) 1173 { 1174 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL && 1175 stack->spilled_ptr.type == SCALAR_VALUE; 1176 } 1177 1178 static bool is_spilled_scalar_reg64(const struct bpf_stack_state *stack) 1179 { 1180 return stack->slot_type[0] == STACK_SPILL && 1181 stack->spilled_ptr.type == SCALAR_VALUE; 1182 } 1183 1184 /* Mark stack slot as STACK_MISC, unless it is already STACK_INVALID, in which 1185 * case they are equivalent, or it's STACK_ZERO, in which case we preserve 1186 * more precise STACK_ZERO. 1187 * Note, in uprivileged mode leaving STACK_INVALID is wrong, so we take 1188 * env->allow_ptr_leaks into account and force STACK_MISC, if necessary. 1189 */ 1190 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype) 1191 { 1192 if (*stype == STACK_ZERO) 1193 return; 1194 if (env->allow_ptr_leaks && *stype == STACK_INVALID) 1195 return; 1196 *stype = STACK_MISC; 1197 } 1198 1199 static void scrub_spilled_slot(u8 *stype) 1200 { 1201 if (*stype != STACK_INVALID) 1202 *stype = STACK_MISC; 1203 } 1204 1205 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1206 * small to hold src. This is different from krealloc since we don't want to preserve 1207 * the contents of dst. 1208 * 1209 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1210 * not be allocated. 1211 */ 1212 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1213 { 1214 size_t alloc_bytes; 1215 void *orig = dst; 1216 size_t bytes; 1217 1218 if (ZERO_OR_NULL_PTR(src)) 1219 goto out; 1220 1221 if (unlikely(check_mul_overflow(n, size, &bytes))) 1222 return NULL; 1223 1224 alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes)); 1225 dst = krealloc(orig, alloc_bytes, flags); 1226 if (!dst) { 1227 kfree(orig); 1228 return NULL; 1229 } 1230 1231 memcpy(dst, src, bytes); 1232 out: 1233 return dst ? dst : ZERO_SIZE_PTR; 1234 } 1235 1236 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1237 * small to hold new_n items. new items are zeroed out if the array grows. 1238 * 1239 * Contrary to krealloc_array, does not free arr if new_n is zero. 1240 */ 1241 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1242 { 1243 size_t alloc_size; 1244 void *new_arr; 1245 1246 if (!new_n || old_n == new_n) 1247 goto out; 1248 1249 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1250 new_arr = krealloc(arr, alloc_size, GFP_KERNEL); 1251 if (!new_arr) { 1252 kfree(arr); 1253 return NULL; 1254 } 1255 arr = new_arr; 1256 1257 if (new_n > old_n) 1258 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1259 1260 out: 1261 return arr ? arr : ZERO_SIZE_PTR; 1262 } 1263 1264 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1265 { 1266 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1267 sizeof(struct bpf_reference_state), GFP_KERNEL); 1268 if (!dst->refs) 1269 return -ENOMEM; 1270 1271 dst->acquired_refs = src->acquired_refs; 1272 return 0; 1273 } 1274 1275 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1276 { 1277 size_t n = src->allocated_stack / BPF_REG_SIZE; 1278 1279 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1280 GFP_KERNEL); 1281 if (!dst->stack) 1282 return -ENOMEM; 1283 1284 dst->allocated_stack = src->allocated_stack; 1285 return 0; 1286 } 1287 1288 static int resize_reference_state(struct bpf_func_state *state, size_t n) 1289 { 1290 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1291 sizeof(struct bpf_reference_state)); 1292 if (!state->refs) 1293 return -ENOMEM; 1294 1295 state->acquired_refs = n; 1296 return 0; 1297 } 1298 1299 /* Possibly update state->allocated_stack to be at least size bytes. Also 1300 * possibly update the function's high-water mark in its bpf_subprog_info. 1301 */ 1302 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) 1303 { 1304 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; 1305 1306 /* The stack size is always a multiple of BPF_REG_SIZE. */ 1307 size = round_up(size, BPF_REG_SIZE); 1308 n = size / BPF_REG_SIZE; 1309 1310 if (old_n >= n) 1311 return 0; 1312 1313 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1314 if (!state->stack) 1315 return -ENOMEM; 1316 1317 state->allocated_stack = size; 1318 1319 /* update known max for given subprogram */ 1320 if (env->subprog_info[state->subprogno].stack_depth < size) 1321 env->subprog_info[state->subprogno].stack_depth = size; 1322 1323 return 0; 1324 } 1325 1326 /* Acquire a pointer id from the env and update the state->refs to include 1327 * this new pointer reference. 1328 * On success, returns a valid pointer id to associate with the register 1329 * On failure, returns a negative errno. 1330 */ 1331 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1332 { 1333 struct bpf_func_state *state = cur_func(env); 1334 int new_ofs = state->acquired_refs; 1335 int id, err; 1336 1337 err = resize_reference_state(state, state->acquired_refs + 1); 1338 if (err) 1339 return err; 1340 id = ++env->id_gen; 1341 state->refs[new_ofs].id = id; 1342 state->refs[new_ofs].insn_idx = insn_idx; 1343 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; 1344 1345 return id; 1346 } 1347 1348 /* release function corresponding to acquire_reference_state(). Idempotent. */ 1349 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1350 { 1351 int i, last_idx; 1352 1353 last_idx = state->acquired_refs - 1; 1354 for (i = 0; i < state->acquired_refs; i++) { 1355 if (state->refs[i].id == ptr_id) { 1356 /* Cannot release caller references in callbacks */ 1357 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 1358 return -EINVAL; 1359 if (last_idx && i != last_idx) 1360 memcpy(&state->refs[i], &state->refs[last_idx], 1361 sizeof(*state->refs)); 1362 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1363 state->acquired_refs--; 1364 return 0; 1365 } 1366 } 1367 return -EINVAL; 1368 } 1369 1370 static void free_func_state(struct bpf_func_state *state) 1371 { 1372 if (!state) 1373 return; 1374 kfree(state->refs); 1375 kfree(state->stack); 1376 kfree(state); 1377 } 1378 1379 static void clear_jmp_history(struct bpf_verifier_state *state) 1380 { 1381 kfree(state->jmp_history); 1382 state->jmp_history = NULL; 1383 state->jmp_history_cnt = 0; 1384 } 1385 1386 static void free_verifier_state(struct bpf_verifier_state *state, 1387 bool free_self) 1388 { 1389 int i; 1390 1391 for (i = 0; i <= state->curframe; i++) { 1392 free_func_state(state->frame[i]); 1393 state->frame[i] = NULL; 1394 } 1395 clear_jmp_history(state); 1396 if (free_self) 1397 kfree(state); 1398 } 1399 1400 /* copy verifier state from src to dst growing dst stack space 1401 * when necessary to accommodate larger src stack 1402 */ 1403 static int copy_func_state(struct bpf_func_state *dst, 1404 const struct bpf_func_state *src) 1405 { 1406 int err; 1407 1408 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1409 err = copy_reference_state(dst, src); 1410 if (err) 1411 return err; 1412 return copy_stack_state(dst, src); 1413 } 1414 1415 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1416 const struct bpf_verifier_state *src) 1417 { 1418 struct bpf_func_state *dst; 1419 int i, err; 1420 1421 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1422 src->jmp_history_cnt, sizeof(*dst_state->jmp_history), 1423 GFP_USER); 1424 if (!dst_state->jmp_history) 1425 return -ENOMEM; 1426 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1427 1428 /* if dst has more stack frames then src frame, free them, this is also 1429 * necessary in case of exceptional exits using bpf_throw. 1430 */ 1431 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1432 free_func_state(dst_state->frame[i]); 1433 dst_state->frame[i] = NULL; 1434 } 1435 dst_state->speculative = src->speculative; 1436 dst_state->active_rcu_lock = src->active_rcu_lock; 1437 dst_state->active_preempt_lock = src->active_preempt_lock; 1438 dst_state->in_sleepable = src->in_sleepable; 1439 dst_state->curframe = src->curframe; 1440 dst_state->active_lock.ptr = src->active_lock.ptr; 1441 dst_state->active_lock.id = src->active_lock.id; 1442 dst_state->branches = src->branches; 1443 dst_state->parent = src->parent; 1444 dst_state->first_insn_idx = src->first_insn_idx; 1445 dst_state->last_insn_idx = src->last_insn_idx; 1446 dst_state->dfs_depth = src->dfs_depth; 1447 dst_state->callback_unroll_depth = src->callback_unroll_depth; 1448 dst_state->used_as_loop_entry = src->used_as_loop_entry; 1449 dst_state->may_goto_depth = src->may_goto_depth; 1450 for (i = 0; i <= src->curframe; i++) { 1451 dst = dst_state->frame[i]; 1452 if (!dst) { 1453 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1454 if (!dst) 1455 return -ENOMEM; 1456 dst_state->frame[i] = dst; 1457 } 1458 err = copy_func_state(dst, src->frame[i]); 1459 if (err) 1460 return err; 1461 } 1462 return 0; 1463 } 1464 1465 static u32 state_htab_size(struct bpf_verifier_env *env) 1466 { 1467 return env->prog->len; 1468 } 1469 1470 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx) 1471 { 1472 struct bpf_verifier_state *cur = env->cur_state; 1473 struct bpf_func_state *state = cur->frame[cur->curframe]; 1474 1475 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 1476 } 1477 1478 static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b) 1479 { 1480 int fr; 1481 1482 if (a->curframe != b->curframe) 1483 return false; 1484 1485 for (fr = a->curframe; fr >= 0; fr--) 1486 if (a->frame[fr]->callsite != b->frame[fr]->callsite) 1487 return false; 1488 1489 return true; 1490 } 1491 1492 /* Open coded iterators allow back-edges in the state graph in order to 1493 * check unbounded loops that iterators. 1494 * 1495 * In is_state_visited() it is necessary to know if explored states are 1496 * part of some loops in order to decide whether non-exact states 1497 * comparison could be used: 1498 * - non-exact states comparison establishes sub-state relation and uses 1499 * read and precision marks to do so, these marks are propagated from 1500 * children states and thus are not guaranteed to be final in a loop; 1501 * - exact states comparison just checks if current and explored states 1502 * are identical (and thus form a back-edge). 1503 * 1504 * Paper "A New Algorithm for Identifying Loops in Decompilation" 1505 * by Tao Wei, Jian Mao, Wei Zou and Yu Chen [1] presents a convenient 1506 * algorithm for loop structure detection and gives an overview of 1507 * relevant terminology. It also has helpful illustrations. 1508 * 1509 * [1] https://api.semanticscholar.org/CorpusID:15784067 1510 * 1511 * We use a similar algorithm but because loop nested structure is 1512 * irrelevant for verifier ours is significantly simpler and resembles 1513 * strongly connected components algorithm from Sedgewick's textbook. 1514 * 1515 * Define topmost loop entry as a first node of the loop traversed in a 1516 * depth first search starting from initial state. The goal of the loop 1517 * tracking algorithm is to associate topmost loop entries with states 1518 * derived from these entries. 1519 * 1520 * For each step in the DFS states traversal algorithm needs to identify 1521 * the following situations: 1522 * 1523 * initial initial initial 1524 * | | | 1525 * V V V 1526 * ... ... .---------> hdr 1527 * | | | | 1528 * V V | V 1529 * cur .-> succ | .------... 1530 * | | | | | | 1531 * V | V | V V 1532 * succ '-- cur | ... ... 1533 * | | | 1534 * | V V 1535 * | succ <- cur 1536 * | | 1537 * | V 1538 * | ... 1539 * | | 1540 * '----' 1541 * 1542 * (A) successor state of cur (B) successor state of cur or it's entry 1543 * not yet traversed are in current DFS path, thus cur and succ 1544 * are members of the same outermost loop 1545 * 1546 * initial initial 1547 * | | 1548 * V V 1549 * ... ... 1550 * | | 1551 * V V 1552 * .------... .------... 1553 * | | | | 1554 * V V V V 1555 * .-> hdr ... ... ... 1556 * | | | | | 1557 * | V V V V 1558 * | succ <- cur succ <- cur 1559 * | | | 1560 * | V V 1561 * | ... ... 1562 * | | | 1563 * '----' exit 1564 * 1565 * (C) successor state of cur is a part of some loop but this loop 1566 * does not include cur or successor state is not in a loop at all. 1567 * 1568 * Algorithm could be described as the following python code: 1569 * 1570 * traversed = set() # Set of traversed nodes 1571 * entries = {} # Mapping from node to loop entry 1572 * depths = {} # Depth level assigned to graph node 1573 * path = set() # Current DFS path 1574 * 1575 * # Find outermost loop entry known for n 1576 * def get_loop_entry(n): 1577 * h = entries.get(n, None) 1578 * while h in entries and entries[h] != h: 1579 * h = entries[h] 1580 * return h 1581 * 1582 * # Update n's loop entry if h's outermost entry comes 1583 * # before n's outermost entry in current DFS path. 1584 * def update_loop_entry(n, h): 1585 * n1 = get_loop_entry(n) or n 1586 * h1 = get_loop_entry(h) or h 1587 * if h1 in path and depths[h1] <= depths[n1]: 1588 * entries[n] = h1 1589 * 1590 * def dfs(n, depth): 1591 * traversed.add(n) 1592 * path.add(n) 1593 * depths[n] = depth 1594 * for succ in G.successors(n): 1595 * if succ not in traversed: 1596 * # Case A: explore succ and update cur's loop entry 1597 * # only if succ's entry is in current DFS path. 1598 * dfs(succ, depth + 1) 1599 * h = get_loop_entry(succ) 1600 * update_loop_entry(n, h) 1601 * else: 1602 * # Case B or C depending on `h1 in path` check in update_loop_entry(). 1603 * update_loop_entry(n, succ) 1604 * path.remove(n) 1605 * 1606 * To adapt this algorithm for use with verifier: 1607 * - use st->branch == 0 as a signal that DFS of succ had been finished 1608 * and cur's loop entry has to be updated (case A), handle this in 1609 * update_branch_counts(); 1610 * - use st->branch > 0 as a signal that st is in the current DFS path; 1611 * - handle cases B and C in is_state_visited(); 1612 * - update topmost loop entry for intermediate states in get_loop_entry(). 1613 */ 1614 static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_state *st) 1615 { 1616 struct bpf_verifier_state *topmost = st->loop_entry, *old; 1617 1618 while (topmost && topmost->loop_entry && topmost != topmost->loop_entry) 1619 topmost = topmost->loop_entry; 1620 /* Update loop entries for intermediate states to avoid this 1621 * traversal in future get_loop_entry() calls. 1622 */ 1623 while (st && st->loop_entry != topmost) { 1624 old = st->loop_entry; 1625 st->loop_entry = topmost; 1626 st = old; 1627 } 1628 return topmost; 1629 } 1630 1631 static void update_loop_entry(struct bpf_verifier_state *cur, struct bpf_verifier_state *hdr) 1632 { 1633 struct bpf_verifier_state *cur1, *hdr1; 1634 1635 cur1 = get_loop_entry(cur) ?: cur; 1636 hdr1 = get_loop_entry(hdr) ?: hdr; 1637 /* The head1->branches check decides between cases B and C in 1638 * comment for get_loop_entry(). If hdr1->branches == 0 then 1639 * head's topmost loop entry is not in current DFS path, 1640 * hence 'cur' and 'hdr' are not in the same loop and there is 1641 * no need to update cur->loop_entry. 1642 */ 1643 if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) { 1644 cur->loop_entry = hdr; 1645 hdr->used_as_loop_entry = true; 1646 } 1647 } 1648 1649 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1650 { 1651 while (st) { 1652 u32 br = --st->branches; 1653 1654 /* br == 0 signals that DFS exploration for 'st' is finished, 1655 * thus it is necessary to update parent's loop entry if it 1656 * turned out that st is a part of some loop. 1657 * This is a part of 'case A' in get_loop_entry() comment. 1658 */ 1659 if (br == 0 && st->parent && st->loop_entry) 1660 update_loop_entry(st->parent, st->loop_entry); 1661 1662 /* WARN_ON(br > 1) technically makes sense here, 1663 * but see comment in push_stack(), hence: 1664 */ 1665 WARN_ONCE((int)br < 0, 1666 "BUG update_branch_counts:branches_to_explore=%d\n", 1667 br); 1668 if (br) 1669 break; 1670 st = st->parent; 1671 } 1672 } 1673 1674 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1675 int *insn_idx, bool pop_log) 1676 { 1677 struct bpf_verifier_state *cur = env->cur_state; 1678 struct bpf_verifier_stack_elem *elem, *head = env->head; 1679 int err; 1680 1681 if (env->head == NULL) 1682 return -ENOENT; 1683 1684 if (cur) { 1685 err = copy_verifier_state(cur, &head->st); 1686 if (err) 1687 return err; 1688 } 1689 if (pop_log) 1690 bpf_vlog_reset(&env->log, head->log_pos); 1691 if (insn_idx) 1692 *insn_idx = head->insn_idx; 1693 if (prev_insn_idx) 1694 *prev_insn_idx = head->prev_insn_idx; 1695 elem = head->next; 1696 free_verifier_state(&head->st, false); 1697 kfree(head); 1698 env->head = elem; 1699 env->stack_size--; 1700 return 0; 1701 } 1702 1703 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1704 int insn_idx, int prev_insn_idx, 1705 bool speculative) 1706 { 1707 struct bpf_verifier_state *cur = env->cur_state; 1708 struct bpf_verifier_stack_elem *elem; 1709 int err; 1710 1711 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1712 if (!elem) 1713 goto err; 1714 1715 elem->insn_idx = insn_idx; 1716 elem->prev_insn_idx = prev_insn_idx; 1717 elem->next = env->head; 1718 elem->log_pos = env->log.end_pos; 1719 env->head = elem; 1720 env->stack_size++; 1721 err = copy_verifier_state(&elem->st, cur); 1722 if (err) 1723 goto err; 1724 elem->st.speculative |= speculative; 1725 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1726 verbose(env, "The sequence of %d jumps is too complex.\n", 1727 env->stack_size); 1728 goto err; 1729 } 1730 if (elem->st.parent) { 1731 ++elem->st.parent->branches; 1732 /* WARN_ON(branches > 2) technically makes sense here, 1733 * but 1734 * 1. speculative states will bump 'branches' for non-branch 1735 * instructions 1736 * 2. is_state_visited() heuristics may decide not to create 1737 * a new state for a sequence of branches and all such current 1738 * and cloned states will be pointing to a single parent state 1739 * which might have large 'branches' count. 1740 */ 1741 } 1742 return &elem->st; 1743 err: 1744 free_verifier_state(env->cur_state, true); 1745 env->cur_state = NULL; 1746 /* pop all elements and return */ 1747 while (!pop_stack(env, NULL, NULL, false)); 1748 return NULL; 1749 } 1750 1751 #define CALLER_SAVED_REGS 6 1752 static const int caller_saved[CALLER_SAVED_REGS] = { 1753 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1754 }; 1755 1756 /* This helper doesn't clear reg->id */ 1757 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1758 { 1759 reg->var_off = tnum_const(imm); 1760 reg->smin_value = (s64)imm; 1761 reg->smax_value = (s64)imm; 1762 reg->umin_value = imm; 1763 reg->umax_value = imm; 1764 1765 reg->s32_min_value = (s32)imm; 1766 reg->s32_max_value = (s32)imm; 1767 reg->u32_min_value = (u32)imm; 1768 reg->u32_max_value = (u32)imm; 1769 } 1770 1771 /* Mark the unknown part of a register (variable offset or scalar value) as 1772 * known to have the value @imm. 1773 */ 1774 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1775 { 1776 /* Clear off and union(map_ptr, range) */ 1777 memset(((u8 *)reg) + sizeof(reg->type), 0, 1778 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1779 reg->id = 0; 1780 reg->ref_obj_id = 0; 1781 ___mark_reg_known(reg, imm); 1782 } 1783 1784 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1785 { 1786 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1787 reg->s32_min_value = (s32)imm; 1788 reg->s32_max_value = (s32)imm; 1789 reg->u32_min_value = (u32)imm; 1790 reg->u32_max_value = (u32)imm; 1791 } 1792 1793 /* Mark the 'variable offset' part of a register as zero. This should be 1794 * used only on registers holding a pointer type. 1795 */ 1796 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1797 { 1798 __mark_reg_known(reg, 0); 1799 } 1800 1801 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1802 { 1803 __mark_reg_known(reg, 0); 1804 reg->type = SCALAR_VALUE; 1805 /* all scalars are assumed imprecise initially (unless unprivileged, 1806 * in which case everything is forced to be precise) 1807 */ 1808 reg->precise = !env->bpf_capable; 1809 } 1810 1811 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1812 struct bpf_reg_state *regs, u32 regno) 1813 { 1814 if (WARN_ON(regno >= MAX_BPF_REG)) { 1815 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1816 /* Something bad happened, let's kill all regs */ 1817 for (regno = 0; regno < MAX_BPF_REG; regno++) 1818 __mark_reg_not_init(env, regs + regno); 1819 return; 1820 } 1821 __mark_reg_known_zero(regs + regno); 1822 } 1823 1824 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, 1825 bool first_slot, int dynptr_id) 1826 { 1827 /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for 1828 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply 1829 * set it unconditionally as it is ignored for STACK_DYNPTR anyway. 1830 */ 1831 __mark_reg_known_zero(reg); 1832 reg->type = CONST_PTR_TO_DYNPTR; 1833 /* Give each dynptr a unique id to uniquely associate slices to it. */ 1834 reg->id = dynptr_id; 1835 reg->dynptr.type = type; 1836 reg->dynptr.first_slot = first_slot; 1837 } 1838 1839 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1840 { 1841 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1842 const struct bpf_map *map = reg->map_ptr; 1843 1844 if (map->inner_map_meta) { 1845 reg->type = CONST_PTR_TO_MAP; 1846 reg->map_ptr = map->inner_map_meta; 1847 /* transfer reg's id which is unique for every map_lookup_elem 1848 * as UID of the inner map. 1849 */ 1850 if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER)) 1851 reg->map_uid = reg->id; 1852 if (btf_record_has_field(map->inner_map_meta->record, BPF_WORKQUEUE)) 1853 reg->map_uid = reg->id; 1854 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1855 reg->type = PTR_TO_XDP_SOCK; 1856 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1857 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1858 reg->type = PTR_TO_SOCKET; 1859 } else { 1860 reg->type = PTR_TO_MAP_VALUE; 1861 } 1862 return; 1863 } 1864 1865 reg->type &= ~PTR_MAYBE_NULL; 1866 } 1867 1868 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno, 1869 struct btf_field_graph_root *ds_head) 1870 { 1871 __mark_reg_known_zero(®s[regno]); 1872 regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC; 1873 regs[regno].btf = ds_head->btf; 1874 regs[regno].btf_id = ds_head->value_btf_id; 1875 regs[regno].off = ds_head->node_offset; 1876 } 1877 1878 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1879 { 1880 return type_is_pkt_pointer(reg->type); 1881 } 1882 1883 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1884 { 1885 return reg_is_pkt_pointer(reg) || 1886 reg->type == PTR_TO_PACKET_END; 1887 } 1888 1889 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg) 1890 { 1891 return base_type(reg->type) == PTR_TO_MEM && 1892 (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP); 1893 } 1894 1895 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1896 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1897 enum bpf_reg_type which) 1898 { 1899 /* The register can already have a range from prior markings. 1900 * This is fine as long as it hasn't been advanced from its 1901 * origin. 1902 */ 1903 return reg->type == which && 1904 reg->id == 0 && 1905 reg->off == 0 && 1906 tnum_equals_const(reg->var_off, 0); 1907 } 1908 1909 /* Reset the min/max bounds of a register */ 1910 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1911 { 1912 reg->smin_value = S64_MIN; 1913 reg->smax_value = S64_MAX; 1914 reg->umin_value = 0; 1915 reg->umax_value = U64_MAX; 1916 1917 reg->s32_min_value = S32_MIN; 1918 reg->s32_max_value = S32_MAX; 1919 reg->u32_min_value = 0; 1920 reg->u32_max_value = U32_MAX; 1921 } 1922 1923 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1924 { 1925 reg->smin_value = S64_MIN; 1926 reg->smax_value = S64_MAX; 1927 reg->umin_value = 0; 1928 reg->umax_value = U64_MAX; 1929 } 1930 1931 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1932 { 1933 reg->s32_min_value = S32_MIN; 1934 reg->s32_max_value = S32_MAX; 1935 reg->u32_min_value = 0; 1936 reg->u32_max_value = U32_MAX; 1937 } 1938 1939 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1940 { 1941 struct tnum var32_off = tnum_subreg(reg->var_off); 1942 1943 /* min signed is max(sign bit) | min(other bits) */ 1944 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1945 var32_off.value | (var32_off.mask & S32_MIN)); 1946 /* max signed is min(sign bit) | max(other bits) */ 1947 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1948 var32_off.value | (var32_off.mask & S32_MAX)); 1949 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1950 reg->u32_max_value = min(reg->u32_max_value, 1951 (u32)(var32_off.value | var32_off.mask)); 1952 } 1953 1954 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1955 { 1956 /* min signed is max(sign bit) | min(other bits) */ 1957 reg->smin_value = max_t(s64, reg->smin_value, 1958 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1959 /* max signed is min(sign bit) | max(other bits) */ 1960 reg->smax_value = min_t(s64, reg->smax_value, 1961 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1962 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1963 reg->umax_value = min(reg->umax_value, 1964 reg->var_off.value | reg->var_off.mask); 1965 } 1966 1967 static void __update_reg_bounds(struct bpf_reg_state *reg) 1968 { 1969 __update_reg32_bounds(reg); 1970 __update_reg64_bounds(reg); 1971 } 1972 1973 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1974 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1975 { 1976 /* If upper 32 bits of u64/s64 range don't change, we can use lower 32 1977 * bits to improve our u32/s32 boundaries. 1978 * 1979 * E.g., the case where we have upper 32 bits as zero ([10, 20] in 1980 * u64) is pretty trivial, it's obvious that in u32 we'll also have 1981 * [10, 20] range. But this property holds for any 64-bit range as 1982 * long as upper 32 bits in that entire range of values stay the same. 1983 * 1984 * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311] 1985 * in decimal) has the same upper 32 bits throughout all the values in 1986 * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15]) 1987 * range. 1988 * 1989 * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32, 1990 * following the rules outlined below about u64/s64 correspondence 1991 * (which equally applies to u32 vs s32 correspondence). In general it 1992 * depends on actual hexadecimal values of 32-bit range. They can form 1993 * only valid u32, or only valid s32 ranges in some cases. 1994 * 1995 * So we use all these insights to derive bounds for subregisters here. 1996 */ 1997 if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) { 1998 /* u64 to u32 casting preserves validity of low 32 bits as 1999 * a range, if upper 32 bits are the same 2000 */ 2001 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value); 2002 reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value); 2003 2004 if ((s32)reg->umin_value <= (s32)reg->umax_value) { 2005 reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value); 2006 reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value); 2007 } 2008 } 2009 if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) { 2010 /* low 32 bits should form a proper u32 range */ 2011 if ((u32)reg->smin_value <= (u32)reg->smax_value) { 2012 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value); 2013 reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value); 2014 } 2015 /* low 32 bits should form a proper s32 range */ 2016 if ((s32)reg->smin_value <= (s32)reg->smax_value) { 2017 reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value); 2018 reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value); 2019 } 2020 } 2021 /* Special case where upper bits form a small sequence of two 2022 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to 2023 * 0x00000000 is also valid), while lower bits form a proper s32 range 2024 * going from negative numbers to positive numbers. E.g., let's say we 2025 * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]). 2026 * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff, 2027 * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits, 2028 * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]). 2029 * Note that it doesn't have to be 0xffffffff going to 0x00000000 in 2030 * upper 32 bits. As a random example, s64 range 2031 * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range 2032 * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister. 2033 */ 2034 if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) && 2035 (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) { 2036 reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value); 2037 reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value); 2038 } 2039 if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) && 2040 (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) { 2041 reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value); 2042 reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value); 2043 } 2044 /* if u32 range forms a valid s32 range (due to matching sign bit), 2045 * try to learn from that 2046 */ 2047 if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) { 2048 reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value); 2049 reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value); 2050 } 2051 /* If we cannot cross the sign boundary, then signed and unsigned bounds 2052 * are the same, so combine. This works even in the negative case, e.g. 2053 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 2054 */ 2055 if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) { 2056 reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value); 2057 reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value); 2058 } 2059 } 2060 2061 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 2062 { 2063 /* If u64 range forms a valid s64 range (due to matching sign bit), 2064 * try to learn from that. Let's do a bit of ASCII art to see when 2065 * this is happening. Let's take u64 range first: 2066 * 2067 * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX 2068 * |-------------------------------|--------------------------------| 2069 * 2070 * Valid u64 range is formed when umin and umax are anywhere in the 2071 * range [0, U64_MAX], and umin <= umax. u64 case is simple and 2072 * straightforward. Let's see how s64 range maps onto the same range 2073 * of values, annotated below the line for comparison: 2074 * 2075 * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX 2076 * |-------------------------------|--------------------------------| 2077 * 0 S64_MAX S64_MIN -1 2078 * 2079 * So s64 values basically start in the middle and they are logically 2080 * contiguous to the right of it, wrapping around from -1 to 0, and 2081 * then finishing as S64_MAX (0x7fffffffffffffff) right before 2082 * S64_MIN. We can try drawing the continuity of u64 vs s64 values 2083 * more visually as mapped to sign-agnostic range of hex values. 2084 * 2085 * u64 start u64 end 2086 * _______________________________________________________________ 2087 * / \ 2088 * 0 0x7fffffffffffffff 0x8000000000000000 U64_MAX 2089 * |-------------------------------|--------------------------------| 2090 * 0 S64_MAX S64_MIN -1 2091 * / \ 2092 * >------------------------------ -------------------------------> 2093 * s64 continues... s64 end s64 start s64 "midpoint" 2094 * 2095 * What this means is that, in general, we can't always derive 2096 * something new about u64 from any random s64 range, and vice versa. 2097 * 2098 * But we can do that in two particular cases. One is when entire 2099 * u64/s64 range is *entirely* contained within left half of the above 2100 * diagram or when it is *entirely* contained in the right half. I.e.: 2101 * 2102 * |-------------------------------|--------------------------------| 2103 * ^ ^ ^ ^ 2104 * A B C D 2105 * 2106 * [A, B] and [C, D] are contained entirely in their respective halves 2107 * and form valid contiguous ranges as both u64 and s64 values. [A, B] 2108 * will be non-negative both as u64 and s64 (and in fact it will be 2109 * identical ranges no matter the signedness). [C, D] treated as s64 2110 * will be a range of negative values, while in u64 it will be 2111 * non-negative range of values larger than 0x8000000000000000. 2112 * 2113 * Now, any other range here can't be represented in both u64 and s64 2114 * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid 2115 * contiguous u64 ranges, but they are discontinuous in s64. [B, C] 2116 * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX], 2117 * for example. Similarly, valid s64 range [D, A] (going from negative 2118 * to positive values), would be two separate [D, U64_MAX] and [0, A] 2119 * ranges as u64. Currently reg_state can't represent two segments per 2120 * numeric domain, so in such situations we can only derive maximal 2121 * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64). 2122 * 2123 * So we use these facts to derive umin/umax from smin/smax and vice 2124 * versa only if they stay within the same "half". This is equivalent 2125 * to checking sign bit: lower half will have sign bit as zero, upper 2126 * half have sign bit 1. Below in code we simplify this by just 2127 * casting umin/umax as smin/smax and checking if they form valid 2128 * range, and vice versa. Those are equivalent checks. 2129 */ 2130 if ((s64)reg->umin_value <= (s64)reg->umax_value) { 2131 reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value); 2132 reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value); 2133 } 2134 /* If we cannot cross the sign boundary, then signed and unsigned bounds 2135 * are the same, so combine. This works even in the negative case, e.g. 2136 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 2137 */ 2138 if ((u64)reg->smin_value <= (u64)reg->smax_value) { 2139 reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value); 2140 reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value); 2141 } 2142 } 2143 2144 static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg) 2145 { 2146 /* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit 2147 * values on both sides of 64-bit range in hope to have tighter range. 2148 * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from 2149 * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff]. 2150 * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound 2151 * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of 2152 * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a 2153 * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff]. 2154 * We just need to make sure that derived bounds we are intersecting 2155 * with are well-formed ranges in respective s64 or u64 domain, just 2156 * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments. 2157 */ 2158 __u64 new_umin, new_umax; 2159 __s64 new_smin, new_smax; 2160 2161 /* u32 -> u64 tightening, it's always well-formed */ 2162 new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value; 2163 new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value; 2164 reg->umin_value = max_t(u64, reg->umin_value, new_umin); 2165 reg->umax_value = min_t(u64, reg->umax_value, new_umax); 2166 /* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */ 2167 new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value; 2168 new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value; 2169 reg->smin_value = max_t(s64, reg->smin_value, new_smin); 2170 reg->smax_value = min_t(s64, reg->smax_value, new_smax); 2171 2172 /* if s32 can be treated as valid u32 range, we can use it as well */ 2173 if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) { 2174 /* s32 -> u64 tightening */ 2175 new_umin = (reg->umin_value & ~0xffffffffULL) | (u32)reg->s32_min_value; 2176 new_umax = (reg->umax_value & ~0xffffffffULL) | (u32)reg->s32_max_value; 2177 reg->umin_value = max_t(u64, reg->umin_value, new_umin); 2178 reg->umax_value = min_t(u64, reg->umax_value, new_umax); 2179 /* s32 -> s64 tightening */ 2180 new_smin = (reg->smin_value & ~0xffffffffULL) | (u32)reg->s32_min_value; 2181 new_smax = (reg->smax_value & ~0xffffffffULL) | (u32)reg->s32_max_value; 2182 reg->smin_value = max_t(s64, reg->smin_value, new_smin); 2183 reg->smax_value = min_t(s64, reg->smax_value, new_smax); 2184 } 2185 } 2186 2187 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 2188 { 2189 __reg32_deduce_bounds(reg); 2190 __reg64_deduce_bounds(reg); 2191 __reg_deduce_mixed_bounds(reg); 2192 } 2193 2194 /* Attempts to improve var_off based on unsigned min/max information */ 2195 static void __reg_bound_offset(struct bpf_reg_state *reg) 2196 { 2197 struct tnum var64_off = tnum_intersect(reg->var_off, 2198 tnum_range(reg->umin_value, 2199 reg->umax_value)); 2200 struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), 2201 tnum_range(reg->u32_min_value, 2202 reg->u32_max_value)); 2203 2204 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 2205 } 2206 2207 static void reg_bounds_sync(struct bpf_reg_state *reg) 2208 { 2209 /* We might have learned new bounds from the var_off. */ 2210 __update_reg_bounds(reg); 2211 /* We might have learned something about the sign bit. */ 2212 __reg_deduce_bounds(reg); 2213 __reg_deduce_bounds(reg); 2214 /* We might have learned some bits from the bounds. */ 2215 __reg_bound_offset(reg); 2216 /* Intersecting with the old var_off might have improved our bounds 2217 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 2218 * then new var_off is (0; 0x7f...fc) which improves our umax. 2219 */ 2220 __update_reg_bounds(reg); 2221 } 2222 2223 static int reg_bounds_sanity_check(struct bpf_verifier_env *env, 2224 struct bpf_reg_state *reg, const char *ctx) 2225 { 2226 const char *msg; 2227 2228 if (reg->umin_value > reg->umax_value || 2229 reg->smin_value > reg->smax_value || 2230 reg->u32_min_value > reg->u32_max_value || 2231 reg->s32_min_value > reg->s32_max_value) { 2232 msg = "range bounds violation"; 2233 goto out; 2234 } 2235 2236 if (tnum_is_const(reg->var_off)) { 2237 u64 uval = reg->var_off.value; 2238 s64 sval = (s64)uval; 2239 2240 if (reg->umin_value != uval || reg->umax_value != uval || 2241 reg->smin_value != sval || reg->smax_value != sval) { 2242 msg = "const tnum out of sync with range bounds"; 2243 goto out; 2244 } 2245 } 2246 2247 if (tnum_subreg_is_const(reg->var_off)) { 2248 u32 uval32 = tnum_subreg(reg->var_off).value; 2249 s32 sval32 = (s32)uval32; 2250 2251 if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 || 2252 reg->s32_min_value != sval32 || reg->s32_max_value != sval32) { 2253 msg = "const subreg tnum out of sync with range bounds"; 2254 goto out; 2255 } 2256 } 2257 2258 return 0; 2259 out: 2260 verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " 2261 "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n", 2262 ctx, msg, reg->umin_value, reg->umax_value, 2263 reg->smin_value, reg->smax_value, 2264 reg->u32_min_value, reg->u32_max_value, 2265 reg->s32_min_value, reg->s32_max_value, 2266 reg->var_off.value, reg->var_off.mask); 2267 if (env->test_reg_invariants) 2268 return -EFAULT; 2269 __mark_reg_unbounded(reg); 2270 return 0; 2271 } 2272 2273 static bool __reg32_bound_s64(s32 a) 2274 { 2275 return a >= 0 && a <= S32_MAX; 2276 } 2277 2278 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 2279 { 2280 reg->umin_value = reg->u32_min_value; 2281 reg->umax_value = reg->u32_max_value; 2282 2283 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 2284 * be positive otherwise set to worse case bounds and refine later 2285 * from tnum. 2286 */ 2287 if (__reg32_bound_s64(reg->s32_min_value) && 2288 __reg32_bound_s64(reg->s32_max_value)) { 2289 reg->smin_value = reg->s32_min_value; 2290 reg->smax_value = reg->s32_max_value; 2291 } else { 2292 reg->smin_value = 0; 2293 reg->smax_value = U32_MAX; 2294 } 2295 } 2296 2297 /* Mark a register as having a completely unknown (scalar) value. */ 2298 static void __mark_reg_unknown_imprecise(struct bpf_reg_state *reg) 2299 { 2300 /* 2301 * Clear type, off, and union(map_ptr, range) and 2302 * padding between 'type' and union 2303 */ 2304 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 2305 reg->type = SCALAR_VALUE; 2306 reg->id = 0; 2307 reg->ref_obj_id = 0; 2308 reg->var_off = tnum_unknown; 2309 reg->frameno = 0; 2310 reg->precise = false; 2311 __mark_reg_unbounded(reg); 2312 } 2313 2314 /* Mark a register as having a completely unknown (scalar) value, 2315 * initialize .precise as true when not bpf capable. 2316 */ 2317 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 2318 struct bpf_reg_state *reg) 2319 { 2320 __mark_reg_unknown_imprecise(reg); 2321 reg->precise = !env->bpf_capable; 2322 } 2323 2324 static void mark_reg_unknown(struct bpf_verifier_env *env, 2325 struct bpf_reg_state *regs, u32 regno) 2326 { 2327 if (WARN_ON(regno >= MAX_BPF_REG)) { 2328 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 2329 /* Something bad happened, let's kill all regs except FP */ 2330 for (regno = 0; regno < BPF_REG_FP; regno++) 2331 __mark_reg_not_init(env, regs + regno); 2332 return; 2333 } 2334 __mark_reg_unknown(env, regs + regno); 2335 } 2336 2337 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 2338 struct bpf_reg_state *reg) 2339 { 2340 __mark_reg_unknown(env, reg); 2341 reg->type = NOT_INIT; 2342 } 2343 2344 static void mark_reg_not_init(struct bpf_verifier_env *env, 2345 struct bpf_reg_state *regs, u32 regno) 2346 { 2347 if (WARN_ON(regno >= MAX_BPF_REG)) { 2348 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 2349 /* Something bad happened, let's kill all regs except FP */ 2350 for (regno = 0; regno < BPF_REG_FP; regno++) 2351 __mark_reg_not_init(env, regs + regno); 2352 return; 2353 } 2354 __mark_reg_not_init(env, regs + regno); 2355 } 2356 2357 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 2358 struct bpf_reg_state *regs, u32 regno, 2359 enum bpf_reg_type reg_type, 2360 struct btf *btf, u32 btf_id, 2361 enum bpf_type_flag flag) 2362 { 2363 if (reg_type == SCALAR_VALUE) { 2364 mark_reg_unknown(env, regs, regno); 2365 return; 2366 } 2367 mark_reg_known_zero(env, regs, regno); 2368 regs[regno].type = PTR_TO_BTF_ID | flag; 2369 regs[regno].btf = btf; 2370 regs[regno].btf_id = btf_id; 2371 if (type_may_be_null(flag)) 2372 regs[regno].id = ++env->id_gen; 2373 } 2374 2375 #define DEF_NOT_SUBREG (0) 2376 static void init_reg_state(struct bpf_verifier_env *env, 2377 struct bpf_func_state *state) 2378 { 2379 struct bpf_reg_state *regs = state->regs; 2380 int i; 2381 2382 for (i = 0; i < MAX_BPF_REG; i++) { 2383 mark_reg_not_init(env, regs, i); 2384 regs[i].live = REG_LIVE_NONE; 2385 regs[i].parent = NULL; 2386 regs[i].subreg_def = DEF_NOT_SUBREG; 2387 } 2388 2389 /* frame pointer */ 2390 regs[BPF_REG_FP].type = PTR_TO_STACK; 2391 mark_reg_known_zero(env, regs, BPF_REG_FP); 2392 regs[BPF_REG_FP].frameno = state->frameno; 2393 } 2394 2395 static struct bpf_retval_range retval_range(s32 minval, s32 maxval) 2396 { 2397 return (struct bpf_retval_range){ minval, maxval }; 2398 } 2399 2400 #define BPF_MAIN_FUNC (-1) 2401 static void init_func_state(struct bpf_verifier_env *env, 2402 struct bpf_func_state *state, 2403 int callsite, int frameno, int subprogno) 2404 { 2405 state->callsite = callsite; 2406 state->frameno = frameno; 2407 state->subprogno = subprogno; 2408 state->callback_ret_range = retval_range(0, 0); 2409 init_reg_state(env, state); 2410 mark_verifier_state_scratched(env); 2411 } 2412 2413 /* Similar to push_stack(), but for async callbacks */ 2414 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 2415 int insn_idx, int prev_insn_idx, 2416 int subprog, bool is_sleepable) 2417 { 2418 struct bpf_verifier_stack_elem *elem; 2419 struct bpf_func_state *frame; 2420 2421 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 2422 if (!elem) 2423 goto err; 2424 2425 elem->insn_idx = insn_idx; 2426 elem->prev_insn_idx = prev_insn_idx; 2427 elem->next = env->head; 2428 elem->log_pos = env->log.end_pos; 2429 env->head = elem; 2430 env->stack_size++; 2431 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 2432 verbose(env, 2433 "The sequence of %d jumps is too complex for async cb.\n", 2434 env->stack_size); 2435 goto err; 2436 } 2437 /* Unlike push_stack() do not copy_verifier_state(). 2438 * The caller state doesn't matter. 2439 * This is async callback. It starts in a fresh stack. 2440 * Initialize it similar to do_check_common(). 2441 */ 2442 elem->st.branches = 1; 2443 elem->st.in_sleepable = is_sleepable; 2444 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 2445 if (!frame) 2446 goto err; 2447 init_func_state(env, frame, 2448 BPF_MAIN_FUNC /* callsite */, 2449 0 /* frameno within this callchain */, 2450 subprog /* subprog number within this prog */); 2451 elem->st.frame[0] = frame; 2452 return &elem->st; 2453 err: 2454 free_verifier_state(env->cur_state, true); 2455 env->cur_state = NULL; 2456 /* pop all elements and return */ 2457 while (!pop_stack(env, NULL, NULL, false)); 2458 return NULL; 2459 } 2460 2461 2462 enum reg_arg_type { 2463 SRC_OP, /* register is used as source operand */ 2464 DST_OP, /* register is used as destination operand */ 2465 DST_OP_NO_MARK /* same as above, check only, don't mark */ 2466 }; 2467 2468 static int cmp_subprogs(const void *a, const void *b) 2469 { 2470 return ((struct bpf_subprog_info *)a)->start - 2471 ((struct bpf_subprog_info *)b)->start; 2472 } 2473 2474 static int find_subprog(struct bpf_verifier_env *env, int off) 2475 { 2476 struct bpf_subprog_info *p; 2477 2478 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 2479 sizeof(env->subprog_info[0]), cmp_subprogs); 2480 if (!p) 2481 return -ENOENT; 2482 return p - env->subprog_info; 2483 2484 } 2485 2486 static int add_subprog(struct bpf_verifier_env *env, int off) 2487 { 2488 int insn_cnt = env->prog->len; 2489 int ret; 2490 2491 if (off >= insn_cnt || off < 0) { 2492 verbose(env, "call to invalid destination\n"); 2493 return -EINVAL; 2494 } 2495 ret = find_subprog(env, off); 2496 if (ret >= 0) 2497 return ret; 2498 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 2499 verbose(env, "too many subprograms\n"); 2500 return -E2BIG; 2501 } 2502 /* determine subprog starts. The end is one before the next starts */ 2503 env->subprog_info[env->subprog_cnt++].start = off; 2504 sort(env->subprog_info, env->subprog_cnt, 2505 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 2506 return env->subprog_cnt - 1; 2507 } 2508 2509 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) 2510 { 2511 struct bpf_prog_aux *aux = env->prog->aux; 2512 struct btf *btf = aux->btf; 2513 const struct btf_type *t; 2514 u32 main_btf_id, id; 2515 const char *name; 2516 int ret, i; 2517 2518 /* Non-zero func_info_cnt implies valid btf */ 2519 if (!aux->func_info_cnt) 2520 return 0; 2521 main_btf_id = aux->func_info[0].type_id; 2522 2523 t = btf_type_by_id(btf, main_btf_id); 2524 if (!t) { 2525 verbose(env, "invalid btf id for main subprog in func_info\n"); 2526 return -EINVAL; 2527 } 2528 2529 name = btf_find_decl_tag_value(btf, t, -1, "exception_callback:"); 2530 if (IS_ERR(name)) { 2531 ret = PTR_ERR(name); 2532 /* If there is no tag present, there is no exception callback */ 2533 if (ret == -ENOENT) 2534 ret = 0; 2535 else if (ret == -EEXIST) 2536 verbose(env, "multiple exception callback tags for main subprog\n"); 2537 return ret; 2538 } 2539 2540 ret = btf_find_by_name_kind(btf, name, BTF_KIND_FUNC); 2541 if (ret < 0) { 2542 verbose(env, "exception callback '%s' could not be found in BTF\n", name); 2543 return ret; 2544 } 2545 id = ret; 2546 t = btf_type_by_id(btf, id); 2547 if (btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 2548 verbose(env, "exception callback '%s' must have global linkage\n", name); 2549 return -EINVAL; 2550 } 2551 ret = 0; 2552 for (i = 0; i < aux->func_info_cnt; i++) { 2553 if (aux->func_info[i].type_id != id) 2554 continue; 2555 ret = aux->func_info[i].insn_off; 2556 /* Further func_info and subprog checks will also happen 2557 * later, so assume this is the right insn_off for now. 2558 */ 2559 if (!ret) { 2560 verbose(env, "invalid exception callback insn_off in func_info: 0\n"); 2561 ret = -EINVAL; 2562 } 2563 } 2564 if (!ret) { 2565 verbose(env, "exception callback type id not found in func_info\n"); 2566 ret = -EINVAL; 2567 } 2568 return ret; 2569 } 2570 2571 #define MAX_KFUNC_DESCS 256 2572 #define MAX_KFUNC_BTFS 256 2573 2574 struct bpf_kfunc_desc { 2575 struct btf_func_model func_model; 2576 u32 func_id; 2577 s32 imm; 2578 u16 offset; 2579 unsigned long addr; 2580 }; 2581 2582 struct bpf_kfunc_btf { 2583 struct btf *btf; 2584 struct module *module; 2585 u16 offset; 2586 }; 2587 2588 struct bpf_kfunc_desc_tab { 2589 /* Sorted by func_id (BTF ID) and offset (fd_array offset) during 2590 * verification. JITs do lookups by bpf_insn, where func_id may not be 2591 * available, therefore at the end of verification do_misc_fixups() 2592 * sorts this by imm and offset. 2593 */ 2594 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 2595 u32 nr_descs; 2596 }; 2597 2598 struct bpf_kfunc_btf_tab { 2599 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 2600 u32 nr_descs; 2601 }; 2602 2603 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 2604 { 2605 const struct bpf_kfunc_desc *d0 = a; 2606 const struct bpf_kfunc_desc *d1 = b; 2607 2608 /* func_id is not greater than BTF_MAX_TYPE */ 2609 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 2610 } 2611 2612 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 2613 { 2614 const struct bpf_kfunc_btf *d0 = a; 2615 const struct bpf_kfunc_btf *d1 = b; 2616 2617 return d0->offset - d1->offset; 2618 } 2619 2620 static const struct bpf_kfunc_desc * 2621 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 2622 { 2623 struct bpf_kfunc_desc desc = { 2624 .func_id = func_id, 2625 .offset = offset, 2626 }; 2627 struct bpf_kfunc_desc_tab *tab; 2628 2629 tab = prog->aux->kfunc_tab; 2630 return bsearch(&desc, tab->descs, tab->nr_descs, 2631 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 2632 } 2633 2634 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, 2635 u16 btf_fd_idx, u8 **func_addr) 2636 { 2637 const struct bpf_kfunc_desc *desc; 2638 2639 desc = find_kfunc_desc(prog, func_id, btf_fd_idx); 2640 if (!desc) 2641 return -EFAULT; 2642 2643 *func_addr = (u8 *)desc->addr; 2644 return 0; 2645 } 2646 2647 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 2648 s16 offset) 2649 { 2650 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 2651 struct bpf_kfunc_btf_tab *tab; 2652 struct bpf_kfunc_btf *b; 2653 struct module *mod; 2654 struct btf *btf; 2655 int btf_fd; 2656 2657 tab = env->prog->aux->kfunc_btf_tab; 2658 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 2659 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 2660 if (!b) { 2661 if (tab->nr_descs == MAX_KFUNC_BTFS) { 2662 verbose(env, "too many different module BTFs\n"); 2663 return ERR_PTR(-E2BIG); 2664 } 2665 2666 if (bpfptr_is_null(env->fd_array)) { 2667 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 2668 return ERR_PTR(-EPROTO); 2669 } 2670 2671 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 2672 offset * sizeof(btf_fd), 2673 sizeof(btf_fd))) 2674 return ERR_PTR(-EFAULT); 2675 2676 btf = btf_get_by_fd(btf_fd); 2677 if (IS_ERR(btf)) { 2678 verbose(env, "invalid module BTF fd specified\n"); 2679 return btf; 2680 } 2681 2682 if (!btf_is_module(btf)) { 2683 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 2684 btf_put(btf); 2685 return ERR_PTR(-EINVAL); 2686 } 2687 2688 mod = btf_try_get_module(btf); 2689 if (!mod) { 2690 btf_put(btf); 2691 return ERR_PTR(-ENXIO); 2692 } 2693 2694 b = &tab->descs[tab->nr_descs++]; 2695 b->btf = btf; 2696 b->module = mod; 2697 b->offset = offset; 2698 2699 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2700 kfunc_btf_cmp_by_off, NULL); 2701 } 2702 return b->btf; 2703 } 2704 2705 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2706 { 2707 if (!tab) 2708 return; 2709 2710 while (tab->nr_descs--) { 2711 module_put(tab->descs[tab->nr_descs].module); 2712 btf_put(tab->descs[tab->nr_descs].btf); 2713 } 2714 kfree(tab); 2715 } 2716 2717 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2718 { 2719 if (offset) { 2720 if (offset < 0) { 2721 /* In the future, this can be allowed to increase limit 2722 * of fd index into fd_array, interpreted as u16. 2723 */ 2724 verbose(env, "negative offset disallowed for kernel module function call\n"); 2725 return ERR_PTR(-EINVAL); 2726 } 2727 2728 return __find_kfunc_desc_btf(env, offset); 2729 } 2730 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2731 } 2732 2733 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 2734 { 2735 const struct btf_type *func, *func_proto; 2736 struct bpf_kfunc_btf_tab *btf_tab; 2737 struct bpf_kfunc_desc_tab *tab; 2738 struct bpf_prog_aux *prog_aux; 2739 struct bpf_kfunc_desc *desc; 2740 const char *func_name; 2741 struct btf *desc_btf; 2742 unsigned long call_imm; 2743 unsigned long addr; 2744 int err; 2745 2746 prog_aux = env->prog->aux; 2747 tab = prog_aux->kfunc_tab; 2748 btf_tab = prog_aux->kfunc_btf_tab; 2749 if (!tab) { 2750 if (!btf_vmlinux) { 2751 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 2752 return -ENOTSUPP; 2753 } 2754 2755 if (!env->prog->jit_requested) { 2756 verbose(env, "JIT is required for calling kernel function\n"); 2757 return -ENOTSUPP; 2758 } 2759 2760 if (!bpf_jit_supports_kfunc_call()) { 2761 verbose(env, "JIT does not support calling kernel function\n"); 2762 return -ENOTSUPP; 2763 } 2764 2765 if (!env->prog->gpl_compatible) { 2766 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 2767 return -EINVAL; 2768 } 2769 2770 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 2771 if (!tab) 2772 return -ENOMEM; 2773 prog_aux->kfunc_tab = tab; 2774 } 2775 2776 /* func_id == 0 is always invalid, but instead of returning an error, be 2777 * conservative and wait until the code elimination pass before returning 2778 * error, so that invalid calls that get pruned out can be in BPF programs 2779 * loaded from userspace. It is also required that offset be untouched 2780 * for such calls. 2781 */ 2782 if (!func_id && !offset) 2783 return 0; 2784 2785 if (!btf_tab && offset) { 2786 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 2787 if (!btf_tab) 2788 return -ENOMEM; 2789 prog_aux->kfunc_btf_tab = btf_tab; 2790 } 2791 2792 desc_btf = find_kfunc_desc_btf(env, offset); 2793 if (IS_ERR(desc_btf)) { 2794 verbose(env, "failed to find BTF for kernel function\n"); 2795 return PTR_ERR(desc_btf); 2796 } 2797 2798 if (find_kfunc_desc(env->prog, func_id, offset)) 2799 return 0; 2800 2801 if (tab->nr_descs == MAX_KFUNC_DESCS) { 2802 verbose(env, "too many different kernel function calls\n"); 2803 return -E2BIG; 2804 } 2805 2806 func = btf_type_by_id(desc_btf, func_id); 2807 if (!func || !btf_type_is_func(func)) { 2808 verbose(env, "kernel btf_id %u is not a function\n", 2809 func_id); 2810 return -EINVAL; 2811 } 2812 func_proto = btf_type_by_id(desc_btf, func->type); 2813 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 2814 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 2815 func_id); 2816 return -EINVAL; 2817 } 2818 2819 func_name = btf_name_by_offset(desc_btf, func->name_off); 2820 addr = kallsyms_lookup_name(func_name); 2821 if (!addr) { 2822 verbose(env, "cannot find address for kernel function %s\n", 2823 func_name); 2824 return -EINVAL; 2825 } 2826 specialize_kfunc(env, func_id, offset, &addr); 2827 2828 if (bpf_jit_supports_far_kfunc_call()) { 2829 call_imm = func_id; 2830 } else { 2831 call_imm = BPF_CALL_IMM(addr); 2832 /* Check whether the relative offset overflows desc->imm */ 2833 if ((unsigned long)(s32)call_imm != call_imm) { 2834 verbose(env, "address of kernel function %s is out of range\n", 2835 func_name); 2836 return -EINVAL; 2837 } 2838 } 2839 2840 if (bpf_dev_bound_kfunc_id(func_id)) { 2841 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); 2842 if (err) 2843 return err; 2844 } 2845 2846 desc = &tab->descs[tab->nr_descs++]; 2847 desc->func_id = func_id; 2848 desc->imm = call_imm; 2849 desc->offset = offset; 2850 desc->addr = addr; 2851 err = btf_distill_func_proto(&env->log, desc_btf, 2852 func_proto, func_name, 2853 &desc->func_model); 2854 if (!err) 2855 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2856 kfunc_desc_cmp_by_id_off, NULL); 2857 return err; 2858 } 2859 2860 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) 2861 { 2862 const struct bpf_kfunc_desc *d0 = a; 2863 const struct bpf_kfunc_desc *d1 = b; 2864 2865 if (d0->imm != d1->imm) 2866 return d0->imm < d1->imm ? -1 : 1; 2867 if (d0->offset != d1->offset) 2868 return d0->offset < d1->offset ? -1 : 1; 2869 return 0; 2870 } 2871 2872 static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog) 2873 { 2874 struct bpf_kfunc_desc_tab *tab; 2875 2876 tab = prog->aux->kfunc_tab; 2877 if (!tab) 2878 return; 2879 2880 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2881 kfunc_desc_cmp_by_imm_off, NULL); 2882 } 2883 2884 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2885 { 2886 return !!prog->aux->kfunc_tab; 2887 } 2888 2889 const struct btf_func_model * 2890 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2891 const struct bpf_insn *insn) 2892 { 2893 const struct bpf_kfunc_desc desc = { 2894 .imm = insn->imm, 2895 .offset = insn->off, 2896 }; 2897 const struct bpf_kfunc_desc *res; 2898 struct bpf_kfunc_desc_tab *tab; 2899 2900 tab = prog->aux->kfunc_tab; 2901 res = bsearch(&desc, tab->descs, tab->nr_descs, 2902 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); 2903 2904 return res ? &res->func_model : NULL; 2905 } 2906 2907 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2908 { 2909 struct bpf_subprog_info *subprog = env->subprog_info; 2910 int i, ret, insn_cnt = env->prog->len, ex_cb_insn; 2911 struct bpf_insn *insn = env->prog->insnsi; 2912 2913 /* Add entry function. */ 2914 ret = add_subprog(env, 0); 2915 if (ret) 2916 return ret; 2917 2918 for (i = 0; i < insn_cnt; i++, insn++) { 2919 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2920 !bpf_pseudo_kfunc_call(insn)) 2921 continue; 2922 2923 if (!env->bpf_capable) { 2924 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2925 return -EPERM; 2926 } 2927 2928 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2929 ret = add_subprog(env, i + insn->imm + 1); 2930 else 2931 ret = add_kfunc_call(env, insn->imm, insn->off); 2932 2933 if (ret < 0) 2934 return ret; 2935 } 2936 2937 ret = bpf_find_exception_callback_insn_off(env); 2938 if (ret < 0) 2939 return ret; 2940 ex_cb_insn = ret; 2941 2942 /* If ex_cb_insn > 0, this means that the main program has a subprog 2943 * marked using BTF decl tag to serve as the exception callback. 2944 */ 2945 if (ex_cb_insn) { 2946 ret = add_subprog(env, ex_cb_insn); 2947 if (ret < 0) 2948 return ret; 2949 for (i = 1; i < env->subprog_cnt; i++) { 2950 if (env->subprog_info[i].start != ex_cb_insn) 2951 continue; 2952 env->exception_callback_subprog = i; 2953 mark_subprog_exc_cb(env, i); 2954 break; 2955 } 2956 } 2957 2958 /* Add a fake 'exit' subprog which could simplify subprog iteration 2959 * logic. 'subprog_cnt' should not be increased. 2960 */ 2961 subprog[env->subprog_cnt].start = insn_cnt; 2962 2963 if (env->log.level & BPF_LOG_LEVEL2) 2964 for (i = 0; i < env->subprog_cnt; i++) 2965 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2966 2967 return 0; 2968 } 2969 2970 static int check_subprogs(struct bpf_verifier_env *env) 2971 { 2972 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2973 struct bpf_subprog_info *subprog = env->subprog_info; 2974 struct bpf_insn *insn = env->prog->insnsi; 2975 int insn_cnt = env->prog->len; 2976 2977 /* now check that all jumps are within the same subprog */ 2978 subprog_start = subprog[cur_subprog].start; 2979 subprog_end = subprog[cur_subprog + 1].start; 2980 for (i = 0; i < insn_cnt; i++) { 2981 u8 code = insn[i].code; 2982 2983 if (code == (BPF_JMP | BPF_CALL) && 2984 insn[i].src_reg == 0 && 2985 insn[i].imm == BPF_FUNC_tail_call) { 2986 subprog[cur_subprog].has_tail_call = true; 2987 subprog[cur_subprog].tail_call_reachable = true; 2988 } 2989 if (BPF_CLASS(code) == BPF_LD && 2990 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2991 subprog[cur_subprog].has_ld_abs = true; 2992 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2993 goto next; 2994 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2995 goto next; 2996 if (code == (BPF_JMP32 | BPF_JA)) 2997 off = i + insn[i].imm + 1; 2998 else 2999 off = i + insn[i].off + 1; 3000 if (off < subprog_start || off >= subprog_end) { 3001 verbose(env, "jump out of range from insn %d to %d\n", i, off); 3002 return -EINVAL; 3003 } 3004 next: 3005 if (i == subprog_end - 1) { 3006 /* to avoid fall-through from one subprog into another 3007 * the last insn of the subprog should be either exit 3008 * or unconditional jump back or bpf_throw call 3009 */ 3010 if (code != (BPF_JMP | BPF_EXIT) && 3011 code != (BPF_JMP32 | BPF_JA) && 3012 code != (BPF_JMP | BPF_JA)) { 3013 verbose(env, "last insn is not an exit or jmp\n"); 3014 return -EINVAL; 3015 } 3016 subprog_start = subprog_end; 3017 cur_subprog++; 3018 if (cur_subprog < env->subprog_cnt) 3019 subprog_end = subprog[cur_subprog + 1].start; 3020 } 3021 } 3022 return 0; 3023 } 3024 3025 /* Parentage chain of this register (or stack slot) should take care of all 3026 * issues like callee-saved registers, stack slot allocation time, etc. 3027 */ 3028 static int mark_reg_read(struct bpf_verifier_env *env, 3029 const struct bpf_reg_state *state, 3030 struct bpf_reg_state *parent, u8 flag) 3031 { 3032 bool writes = parent == state->parent; /* Observe write marks */ 3033 int cnt = 0; 3034 3035 while (parent) { 3036 /* if read wasn't screened by an earlier write ... */ 3037 if (writes && state->live & REG_LIVE_WRITTEN) 3038 break; 3039 if (parent->live & REG_LIVE_DONE) { 3040 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 3041 reg_type_str(env, parent->type), 3042 parent->var_off.value, parent->off); 3043 return -EFAULT; 3044 } 3045 /* The first condition is more likely to be true than the 3046 * second, checked it first. 3047 */ 3048 if ((parent->live & REG_LIVE_READ) == flag || 3049 parent->live & REG_LIVE_READ64) 3050 /* The parentage chain never changes and 3051 * this parent was already marked as LIVE_READ. 3052 * There is no need to keep walking the chain again and 3053 * keep re-marking all parents as LIVE_READ. 3054 * This case happens when the same register is read 3055 * multiple times without writes into it in-between. 3056 * Also, if parent has the stronger REG_LIVE_READ64 set, 3057 * then no need to set the weak REG_LIVE_READ32. 3058 */ 3059 break; 3060 /* ... then we depend on parent's value */ 3061 parent->live |= flag; 3062 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 3063 if (flag == REG_LIVE_READ64) 3064 parent->live &= ~REG_LIVE_READ32; 3065 state = parent; 3066 parent = state->parent; 3067 writes = true; 3068 cnt++; 3069 } 3070 3071 if (env->longest_mark_read_walk < cnt) 3072 env->longest_mark_read_walk = cnt; 3073 return 0; 3074 } 3075 3076 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 3077 { 3078 struct bpf_func_state *state = func(env, reg); 3079 int spi, ret; 3080 3081 /* For CONST_PTR_TO_DYNPTR, it must have already been done by 3082 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in 3083 * check_kfunc_call. 3084 */ 3085 if (reg->type == CONST_PTR_TO_DYNPTR) 3086 return 0; 3087 spi = dynptr_get_spi(env, reg); 3088 if (spi < 0) 3089 return spi; 3090 /* Caller ensures dynptr is valid and initialized, which means spi is in 3091 * bounds and spi is the first dynptr slot. Simply mark stack slot as 3092 * read. 3093 */ 3094 ret = mark_reg_read(env, &state->stack[spi].spilled_ptr, 3095 state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64); 3096 if (ret) 3097 return ret; 3098 return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr, 3099 state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64); 3100 } 3101 3102 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 3103 int spi, int nr_slots) 3104 { 3105 struct bpf_func_state *state = func(env, reg); 3106 int err, i; 3107 3108 for (i = 0; i < nr_slots; i++) { 3109 struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr; 3110 3111 err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); 3112 if (err) 3113 return err; 3114 3115 mark_stack_slot_scratched(env, spi - i); 3116 } 3117 3118 return 0; 3119 } 3120 3121 /* This function is supposed to be used by the following 32-bit optimization 3122 * code only. It returns TRUE if the source or destination register operates 3123 * on 64-bit, otherwise return FALSE. 3124 */ 3125 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 3126 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 3127 { 3128 u8 code, class, op; 3129 3130 code = insn->code; 3131 class = BPF_CLASS(code); 3132 op = BPF_OP(code); 3133 if (class == BPF_JMP) { 3134 /* BPF_EXIT for "main" will reach here. Return TRUE 3135 * conservatively. 3136 */ 3137 if (op == BPF_EXIT) 3138 return true; 3139 if (op == BPF_CALL) { 3140 /* BPF to BPF call will reach here because of marking 3141 * caller saved clobber with DST_OP_NO_MARK for which we 3142 * don't care the register def because they are anyway 3143 * marked as NOT_INIT already. 3144 */ 3145 if (insn->src_reg == BPF_PSEUDO_CALL) 3146 return false; 3147 /* Helper call will reach here because of arg type 3148 * check, conservatively return TRUE. 3149 */ 3150 if (t == SRC_OP) 3151 return true; 3152 3153 return false; 3154 } 3155 } 3156 3157 if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) 3158 return false; 3159 3160 if (class == BPF_ALU64 || class == BPF_JMP || 3161 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 3162 return true; 3163 3164 if (class == BPF_ALU || class == BPF_JMP32) 3165 return false; 3166 3167 if (class == BPF_LDX) { 3168 if (t != SRC_OP) 3169 return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; 3170 /* LDX source must be ptr. */ 3171 return true; 3172 } 3173 3174 if (class == BPF_STX) { 3175 /* BPF_STX (including atomic variants) has multiple source 3176 * operands, one of which is a ptr. Check whether the caller is 3177 * asking about it. 3178 */ 3179 if (t == SRC_OP && reg->type != SCALAR_VALUE) 3180 return true; 3181 return BPF_SIZE(code) == BPF_DW; 3182 } 3183 3184 if (class == BPF_LD) { 3185 u8 mode = BPF_MODE(code); 3186 3187 /* LD_IMM64 */ 3188 if (mode == BPF_IMM) 3189 return true; 3190 3191 /* Both LD_IND and LD_ABS return 32-bit data. */ 3192 if (t != SRC_OP) 3193 return false; 3194 3195 /* Implicit ctx ptr. */ 3196 if (regno == BPF_REG_6) 3197 return true; 3198 3199 /* Explicit source could be any width. */ 3200 return true; 3201 } 3202 3203 if (class == BPF_ST) 3204 /* The only source register for BPF_ST is a ptr. */ 3205 return true; 3206 3207 /* Conservatively return true at default. */ 3208 return true; 3209 } 3210 3211 /* Return the regno defined by the insn, or -1. */ 3212 static int insn_def_regno(const struct bpf_insn *insn) 3213 { 3214 switch (BPF_CLASS(insn->code)) { 3215 case BPF_JMP: 3216 case BPF_JMP32: 3217 case BPF_ST: 3218 return -1; 3219 case BPF_STX: 3220 if ((BPF_MODE(insn->code) == BPF_ATOMIC || 3221 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 3222 (insn->imm & BPF_FETCH)) { 3223 if (insn->imm == BPF_CMPXCHG) 3224 return BPF_REG_0; 3225 else 3226 return insn->src_reg; 3227 } else { 3228 return -1; 3229 } 3230 default: 3231 return insn->dst_reg; 3232 } 3233 } 3234 3235 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 3236 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 3237 { 3238 int dst_reg = insn_def_regno(insn); 3239 3240 if (dst_reg == -1) 3241 return false; 3242 3243 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 3244 } 3245 3246 static void mark_insn_zext(struct bpf_verifier_env *env, 3247 struct bpf_reg_state *reg) 3248 { 3249 s32 def_idx = reg->subreg_def; 3250 3251 if (def_idx == DEF_NOT_SUBREG) 3252 return; 3253 3254 env->insn_aux_data[def_idx - 1].zext_dst = true; 3255 /* The dst will be zero extended, so won't be sub-register anymore. */ 3256 reg->subreg_def = DEF_NOT_SUBREG; 3257 } 3258 3259 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, 3260 enum reg_arg_type t) 3261 { 3262 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 3263 struct bpf_reg_state *reg; 3264 bool rw64; 3265 3266 if (regno >= MAX_BPF_REG) { 3267 verbose(env, "R%d is invalid\n", regno); 3268 return -EINVAL; 3269 } 3270 3271 mark_reg_scratched(env, regno); 3272 3273 reg = ®s[regno]; 3274 rw64 = is_reg64(env, insn, regno, reg, t); 3275 if (t == SRC_OP) { 3276 /* check whether register used as source operand can be read */ 3277 if (reg->type == NOT_INIT) { 3278 verbose(env, "R%d !read_ok\n", regno); 3279 return -EACCES; 3280 } 3281 /* We don't need to worry about FP liveness because it's read-only */ 3282 if (regno == BPF_REG_FP) 3283 return 0; 3284 3285 if (rw64) 3286 mark_insn_zext(env, reg); 3287 3288 return mark_reg_read(env, reg, reg->parent, 3289 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 3290 } else { 3291 /* check whether register used as dest operand can be written to */ 3292 if (regno == BPF_REG_FP) { 3293 verbose(env, "frame pointer is read only\n"); 3294 return -EACCES; 3295 } 3296 reg->live |= REG_LIVE_WRITTEN; 3297 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 3298 if (t == DST_OP) 3299 mark_reg_unknown(env, regs, regno); 3300 } 3301 return 0; 3302 } 3303 3304 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 3305 enum reg_arg_type t) 3306 { 3307 struct bpf_verifier_state *vstate = env->cur_state; 3308 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3309 3310 return __check_reg_arg(env, state->regs, regno, t); 3311 } 3312 3313 static int insn_stack_access_flags(int frameno, int spi) 3314 { 3315 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno; 3316 } 3317 3318 static int insn_stack_access_spi(int insn_flags) 3319 { 3320 return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK; 3321 } 3322 3323 static int insn_stack_access_frameno(int insn_flags) 3324 { 3325 return insn_flags & INSN_F_FRAMENO_MASK; 3326 } 3327 3328 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) 3329 { 3330 env->insn_aux_data[idx].jmp_point = true; 3331 } 3332 3333 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) 3334 { 3335 return env->insn_aux_data[insn_idx].jmp_point; 3336 } 3337 3338 /* for any branch, call, exit record the history of jmps in the given state */ 3339 static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, 3340 int insn_flags) 3341 { 3342 u32 cnt = cur->jmp_history_cnt; 3343 struct bpf_jmp_history_entry *p; 3344 size_t alloc_size; 3345 3346 /* combine instruction flags if we already recorded this instruction */ 3347 if (env->cur_hist_ent) { 3348 /* atomic instructions push insn_flags twice, for READ and 3349 * WRITE sides, but they should agree on stack slot 3350 */ 3351 WARN_ONCE((env->cur_hist_ent->flags & insn_flags) && 3352 (env->cur_hist_ent->flags & insn_flags) != insn_flags, 3353 "verifier insn history bug: insn_idx %d cur flags %x new flags %x\n", 3354 env->insn_idx, env->cur_hist_ent->flags, insn_flags); 3355 env->cur_hist_ent->flags |= insn_flags; 3356 return 0; 3357 } 3358 3359 cnt++; 3360 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); 3361 p = krealloc(cur->jmp_history, alloc_size, GFP_USER); 3362 if (!p) 3363 return -ENOMEM; 3364 cur->jmp_history = p; 3365 3366 p = &cur->jmp_history[cnt - 1]; 3367 p->idx = env->insn_idx; 3368 p->prev_idx = env->prev_insn_idx; 3369 p->flags = insn_flags; 3370 cur->jmp_history_cnt = cnt; 3371 env->cur_hist_ent = p; 3372 3373 return 0; 3374 } 3375 3376 static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st, 3377 u32 hist_end, int insn_idx) 3378 { 3379 if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx) 3380 return &st->jmp_history[hist_end - 1]; 3381 return NULL; 3382 } 3383 3384 /* Backtrack one insn at a time. If idx is not at the top of recorded 3385 * history then previous instruction came from straight line execution. 3386 * Return -ENOENT if we exhausted all instructions within given state. 3387 * 3388 * It's legal to have a bit of a looping with the same starting and ending 3389 * insn index within the same state, e.g.: 3->4->5->3, so just because current 3390 * instruction index is the same as state's first_idx doesn't mean we are 3391 * done. If there is still some jump history left, we should keep going. We 3392 * need to take into account that we might have a jump history between given 3393 * state's parent and itself, due to checkpointing. In this case, we'll have 3394 * history entry recording a jump from last instruction of parent state and 3395 * first instruction of given state. 3396 */ 3397 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 3398 u32 *history) 3399 { 3400 u32 cnt = *history; 3401 3402 if (i == st->first_insn_idx) { 3403 if (cnt == 0) 3404 return -ENOENT; 3405 if (cnt == 1 && st->jmp_history[0].idx == i) 3406 return -ENOENT; 3407 } 3408 3409 if (cnt && st->jmp_history[cnt - 1].idx == i) { 3410 i = st->jmp_history[cnt - 1].prev_idx; 3411 (*history)--; 3412 } else { 3413 i--; 3414 } 3415 return i; 3416 } 3417 3418 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 3419 { 3420 const struct btf_type *func; 3421 struct btf *desc_btf; 3422 3423 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 3424 return NULL; 3425 3426 desc_btf = find_kfunc_desc_btf(data, insn->off); 3427 if (IS_ERR(desc_btf)) 3428 return "<error>"; 3429 3430 func = btf_type_by_id(desc_btf, insn->imm); 3431 return btf_name_by_offset(desc_btf, func->name_off); 3432 } 3433 3434 static inline void bt_init(struct backtrack_state *bt, u32 frame) 3435 { 3436 bt->frame = frame; 3437 } 3438 3439 static inline void bt_reset(struct backtrack_state *bt) 3440 { 3441 struct bpf_verifier_env *env = bt->env; 3442 3443 memset(bt, 0, sizeof(*bt)); 3444 bt->env = env; 3445 } 3446 3447 static inline u32 bt_empty(struct backtrack_state *bt) 3448 { 3449 u64 mask = 0; 3450 int i; 3451 3452 for (i = 0; i <= bt->frame; i++) 3453 mask |= bt->reg_masks[i] | bt->stack_masks[i]; 3454 3455 return mask == 0; 3456 } 3457 3458 static inline int bt_subprog_enter(struct backtrack_state *bt) 3459 { 3460 if (bt->frame == MAX_CALL_FRAMES - 1) { 3461 verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame); 3462 WARN_ONCE(1, "verifier backtracking bug"); 3463 return -EFAULT; 3464 } 3465 bt->frame++; 3466 return 0; 3467 } 3468 3469 static inline int bt_subprog_exit(struct backtrack_state *bt) 3470 { 3471 if (bt->frame == 0) { 3472 verbose(bt->env, "BUG subprog exit from frame 0\n"); 3473 WARN_ONCE(1, "verifier backtracking bug"); 3474 return -EFAULT; 3475 } 3476 bt->frame--; 3477 return 0; 3478 } 3479 3480 static inline void bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) 3481 { 3482 bt->reg_masks[frame] |= 1 << reg; 3483 } 3484 3485 static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) 3486 { 3487 bt->reg_masks[frame] &= ~(1 << reg); 3488 } 3489 3490 static inline void bt_set_reg(struct backtrack_state *bt, u32 reg) 3491 { 3492 bt_set_frame_reg(bt, bt->frame, reg); 3493 } 3494 3495 static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) 3496 { 3497 bt_clear_frame_reg(bt, bt->frame, reg); 3498 } 3499 3500 static inline void bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) 3501 { 3502 bt->stack_masks[frame] |= 1ull << slot; 3503 } 3504 3505 static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) 3506 { 3507 bt->stack_masks[frame] &= ~(1ull << slot); 3508 } 3509 3510 static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) 3511 { 3512 return bt->reg_masks[frame]; 3513 } 3514 3515 static inline u32 bt_reg_mask(struct backtrack_state *bt) 3516 { 3517 return bt->reg_masks[bt->frame]; 3518 } 3519 3520 static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) 3521 { 3522 return bt->stack_masks[frame]; 3523 } 3524 3525 static inline u64 bt_stack_mask(struct backtrack_state *bt) 3526 { 3527 return bt->stack_masks[bt->frame]; 3528 } 3529 3530 static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) 3531 { 3532 return bt->reg_masks[bt->frame] & (1 << reg); 3533 } 3534 3535 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) 3536 { 3537 return bt->stack_masks[frame] & (1ull << slot); 3538 } 3539 3540 /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ 3541 static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) 3542 { 3543 DECLARE_BITMAP(mask, 64); 3544 bool first = true; 3545 int i, n; 3546 3547 buf[0] = '\0'; 3548 3549 bitmap_from_u64(mask, reg_mask); 3550 for_each_set_bit(i, mask, 32) { 3551 n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i); 3552 first = false; 3553 buf += n; 3554 buf_sz -= n; 3555 if (buf_sz < 0) 3556 break; 3557 } 3558 } 3559 /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ 3560 static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) 3561 { 3562 DECLARE_BITMAP(mask, 64); 3563 bool first = true; 3564 int i, n; 3565 3566 buf[0] = '\0'; 3567 3568 bitmap_from_u64(mask, stack_mask); 3569 for_each_set_bit(i, mask, 64) { 3570 n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); 3571 first = false; 3572 buf += n; 3573 buf_sz -= n; 3574 if (buf_sz < 0) 3575 break; 3576 } 3577 } 3578 3579 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); 3580 3581 /* For given verifier state backtrack_insn() is called from the last insn to 3582 * the first insn. Its purpose is to compute a bitmask of registers and 3583 * stack slots that needs precision in the parent verifier state. 3584 * 3585 * @idx is an index of the instruction we are currently processing; 3586 * @subseq_idx is an index of the subsequent instruction that: 3587 * - *would be* executed next, if jump history is viewed in forward order; 3588 * - *was* processed previously during backtracking. 3589 */ 3590 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, 3591 struct bpf_jmp_history_entry *hist, struct backtrack_state *bt) 3592 { 3593 const struct bpf_insn_cbs cbs = { 3594 .cb_call = disasm_kfunc_name, 3595 .cb_print = verbose, 3596 .private_data = env, 3597 }; 3598 struct bpf_insn *insn = env->prog->insnsi + idx; 3599 u8 class = BPF_CLASS(insn->code); 3600 u8 opcode = BPF_OP(insn->code); 3601 u8 mode = BPF_MODE(insn->code); 3602 u32 dreg = insn->dst_reg; 3603 u32 sreg = insn->src_reg; 3604 u32 spi, i, fr; 3605 3606 if (insn->code == 0) 3607 return 0; 3608 if (env->log.level & BPF_LOG_LEVEL2) { 3609 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); 3610 verbose(env, "mark_precise: frame%d: regs=%s ", 3611 bt->frame, env->tmp_str_buf); 3612 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); 3613 verbose(env, "stack=%s before ", env->tmp_str_buf); 3614 verbose(env, "%d: ", idx); 3615 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 3616 } 3617 3618 if (class == BPF_ALU || class == BPF_ALU64) { 3619 if (!bt_is_reg_set(bt, dreg)) 3620 return 0; 3621 if (opcode == BPF_END || opcode == BPF_NEG) { 3622 /* sreg is reserved and unused 3623 * dreg still need precision before this insn 3624 */ 3625 return 0; 3626 } else if (opcode == BPF_MOV) { 3627 if (BPF_SRC(insn->code) == BPF_X) { 3628 /* dreg = sreg or dreg = (s8, s16, s32)sreg 3629 * dreg needs precision after this insn 3630 * sreg needs precision before this insn 3631 */ 3632 bt_clear_reg(bt, dreg); 3633 if (sreg != BPF_REG_FP) 3634 bt_set_reg(bt, sreg); 3635 } else { 3636 /* dreg = K 3637 * dreg needs precision after this insn. 3638 * Corresponding register is already marked 3639 * as precise=true in this verifier state. 3640 * No further markings in parent are necessary 3641 */ 3642 bt_clear_reg(bt, dreg); 3643 } 3644 } else { 3645 if (BPF_SRC(insn->code) == BPF_X) { 3646 /* dreg += sreg 3647 * both dreg and sreg need precision 3648 * before this insn 3649 */ 3650 if (sreg != BPF_REG_FP) 3651 bt_set_reg(bt, sreg); 3652 } /* else dreg += K 3653 * dreg still needs precision before this insn 3654 */ 3655 } 3656 } else if (class == BPF_LDX) { 3657 if (!bt_is_reg_set(bt, dreg)) 3658 return 0; 3659 bt_clear_reg(bt, dreg); 3660 3661 /* scalars can only be spilled into stack w/o losing precision. 3662 * Load from any other memory can be zero extended. 3663 * The desire to keep that precision is already indicated 3664 * by 'precise' mark in corresponding register of this state. 3665 * No further tracking necessary. 3666 */ 3667 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) 3668 return 0; 3669 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 3670 * that [fp - off] slot contains scalar that needs to be 3671 * tracked with precision 3672 */ 3673 spi = insn_stack_access_spi(hist->flags); 3674 fr = insn_stack_access_frameno(hist->flags); 3675 bt_set_frame_slot(bt, fr, spi); 3676 } else if (class == BPF_STX || class == BPF_ST) { 3677 if (bt_is_reg_set(bt, dreg)) 3678 /* stx & st shouldn't be using _scalar_ dst_reg 3679 * to access memory. It means backtracking 3680 * encountered a case of pointer subtraction. 3681 */ 3682 return -ENOTSUPP; 3683 /* scalars can only be spilled into stack */ 3684 if (!hist || !(hist->flags & INSN_F_STACK_ACCESS)) 3685 return 0; 3686 spi = insn_stack_access_spi(hist->flags); 3687 fr = insn_stack_access_frameno(hist->flags); 3688 if (!bt_is_frame_slot_set(bt, fr, spi)) 3689 return 0; 3690 bt_clear_frame_slot(bt, fr, spi); 3691 if (class == BPF_STX) 3692 bt_set_reg(bt, sreg); 3693 } else if (class == BPF_JMP || class == BPF_JMP32) { 3694 if (bpf_pseudo_call(insn)) { 3695 int subprog_insn_idx, subprog; 3696 3697 subprog_insn_idx = idx + insn->imm + 1; 3698 subprog = find_subprog(env, subprog_insn_idx); 3699 if (subprog < 0) 3700 return -EFAULT; 3701 3702 if (subprog_is_global(env, subprog)) { 3703 /* check that jump history doesn't have any 3704 * extra instructions from subprog; the next 3705 * instruction after call to global subprog 3706 * should be literally next instruction in 3707 * caller program 3708 */ 3709 WARN_ONCE(idx + 1 != subseq_idx, "verifier backtracking bug"); 3710 /* r1-r5 are invalidated after subprog call, 3711 * so for global func call it shouldn't be set 3712 * anymore 3713 */ 3714 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { 3715 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); 3716 WARN_ONCE(1, "verifier backtracking bug"); 3717 return -EFAULT; 3718 } 3719 /* global subprog always sets R0 */ 3720 bt_clear_reg(bt, BPF_REG_0); 3721 return 0; 3722 } else { 3723 /* static subprog call instruction, which 3724 * means that we are exiting current subprog, 3725 * so only r1-r5 could be still requested as 3726 * precise, r0 and r6-r10 or any stack slot in 3727 * the current frame should be zero by now 3728 */ 3729 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { 3730 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); 3731 WARN_ONCE(1, "verifier backtracking bug"); 3732 return -EFAULT; 3733 } 3734 /* we are now tracking register spills correctly, 3735 * so any instance of leftover slots is a bug 3736 */ 3737 if (bt_stack_mask(bt) != 0) { 3738 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); 3739 WARN_ONCE(1, "verifier backtracking bug (subprog leftover stack slots)"); 3740 return -EFAULT; 3741 } 3742 /* propagate r1-r5 to the caller */ 3743 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 3744 if (bt_is_reg_set(bt, i)) { 3745 bt_clear_reg(bt, i); 3746 bt_set_frame_reg(bt, bt->frame - 1, i); 3747 } 3748 } 3749 if (bt_subprog_exit(bt)) 3750 return -EFAULT; 3751 return 0; 3752 } 3753 } else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) { 3754 /* exit from callback subprog to callback-calling helper or 3755 * kfunc call. Use idx/subseq_idx check to discern it from 3756 * straight line code backtracking. 3757 * Unlike the subprog call handling above, we shouldn't 3758 * propagate precision of r1-r5 (if any requested), as they are 3759 * not actually arguments passed directly to callback subprogs 3760 */ 3761 if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) { 3762 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); 3763 WARN_ONCE(1, "verifier backtracking bug"); 3764 return -EFAULT; 3765 } 3766 if (bt_stack_mask(bt) != 0) { 3767 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); 3768 WARN_ONCE(1, "verifier backtracking bug (callback leftover stack slots)"); 3769 return -EFAULT; 3770 } 3771 /* clear r1-r5 in callback subprog's mask */ 3772 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 3773 bt_clear_reg(bt, i); 3774 if (bt_subprog_exit(bt)) 3775 return -EFAULT; 3776 return 0; 3777 } else if (opcode == BPF_CALL) { 3778 /* kfunc with imm==0 is invalid and fixup_kfunc_call will 3779 * catch this error later. Make backtracking conservative 3780 * with ENOTSUPP. 3781 */ 3782 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) 3783 return -ENOTSUPP; 3784 /* regular helper call sets R0 */ 3785 bt_clear_reg(bt, BPF_REG_0); 3786 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { 3787 /* if backtracing was looking for registers R1-R5 3788 * they should have been found already. 3789 */ 3790 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); 3791 WARN_ONCE(1, "verifier backtracking bug"); 3792 return -EFAULT; 3793 } 3794 } else if (opcode == BPF_EXIT) { 3795 bool r0_precise; 3796 3797 /* Backtracking to a nested function call, 'idx' is a part of 3798 * the inner frame 'subseq_idx' is a part of the outer frame. 3799 * In case of a regular function call, instructions giving 3800 * precision to registers R1-R5 should have been found already. 3801 * In case of a callback, it is ok to have R1-R5 marked for 3802 * backtracking, as these registers are set by the function 3803 * invoking callback. 3804 */ 3805 if (subseq_idx >= 0 && calls_callback(env, subseq_idx)) 3806 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 3807 bt_clear_reg(bt, i); 3808 if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { 3809 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); 3810 WARN_ONCE(1, "verifier backtracking bug"); 3811 return -EFAULT; 3812 } 3813 3814 /* BPF_EXIT in subprog or callback always returns 3815 * right after the call instruction, so by checking 3816 * whether the instruction at subseq_idx-1 is subprog 3817 * call or not we can distinguish actual exit from 3818 * *subprog* from exit from *callback*. In the former 3819 * case, we need to propagate r0 precision, if 3820 * necessary. In the former we never do that. 3821 */ 3822 r0_precise = subseq_idx - 1 >= 0 && 3823 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && 3824 bt_is_reg_set(bt, BPF_REG_0); 3825 3826 bt_clear_reg(bt, BPF_REG_0); 3827 if (bt_subprog_enter(bt)) 3828 return -EFAULT; 3829 3830 if (r0_precise) 3831 bt_set_reg(bt, BPF_REG_0); 3832 /* r6-r9 and stack slots will stay set in caller frame 3833 * bitmasks until we return back from callee(s) 3834 */ 3835 return 0; 3836 } else if (BPF_SRC(insn->code) == BPF_X) { 3837 if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg)) 3838 return 0; 3839 /* dreg <cond> sreg 3840 * Both dreg and sreg need precision before 3841 * this insn. If only sreg was marked precise 3842 * before it would be equally necessary to 3843 * propagate it to dreg. 3844 */ 3845 bt_set_reg(bt, dreg); 3846 bt_set_reg(bt, sreg); 3847 /* else dreg <cond> K 3848 * Only dreg still needs precision before 3849 * this insn, so for the K-based conditional 3850 * there is nothing new to be marked. 3851 */ 3852 } 3853 } else if (class == BPF_LD) { 3854 if (!bt_is_reg_set(bt, dreg)) 3855 return 0; 3856 bt_clear_reg(bt, dreg); 3857 /* It's ld_imm64 or ld_abs or ld_ind. 3858 * For ld_imm64 no further tracking of precision 3859 * into parent is necessary 3860 */ 3861 if (mode == BPF_IND || mode == BPF_ABS) 3862 /* to be analyzed */ 3863 return -ENOTSUPP; 3864 } 3865 return 0; 3866 } 3867 3868 /* the scalar precision tracking algorithm: 3869 * . at the start all registers have precise=false. 3870 * . scalar ranges are tracked as normal through alu and jmp insns. 3871 * . once precise value of the scalar register is used in: 3872 * . ptr + scalar alu 3873 * . if (scalar cond K|scalar) 3874 * . helper_call(.., scalar, ...) where ARG_CONST is expected 3875 * backtrack through the verifier states and mark all registers and 3876 * stack slots with spilled constants that these scalar regisers 3877 * should be precise. 3878 * . during state pruning two registers (or spilled stack slots) 3879 * are equivalent if both are not precise. 3880 * 3881 * Note the verifier cannot simply walk register parentage chain, 3882 * since many different registers and stack slots could have been 3883 * used to compute single precise scalar. 3884 * 3885 * The approach of starting with precise=true for all registers and then 3886 * backtrack to mark a register as not precise when the verifier detects 3887 * that program doesn't care about specific value (e.g., when helper 3888 * takes register as ARG_ANYTHING parameter) is not safe. 3889 * 3890 * It's ok to walk single parentage chain of the verifier states. 3891 * It's possible that this backtracking will go all the way till 1st insn. 3892 * All other branches will be explored for needing precision later. 3893 * 3894 * The backtracking needs to deal with cases like: 3895 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 3896 * r9 -= r8 3897 * r5 = r9 3898 * if r5 > 0x79f goto pc+7 3899 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 3900 * r5 += 1 3901 * ... 3902 * call bpf_perf_event_output#25 3903 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 3904 * 3905 * and this case: 3906 * r6 = 1 3907 * call foo // uses callee's r6 inside to compute r0 3908 * r0 += r6 3909 * if r0 == 0 goto 3910 * 3911 * to track above reg_mask/stack_mask needs to be independent for each frame. 3912 * 3913 * Also if parent's curframe > frame where backtracking started, 3914 * the verifier need to mark registers in both frames, otherwise callees 3915 * may incorrectly prune callers. This is similar to 3916 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 3917 * 3918 * For now backtracking falls back into conservative marking. 3919 */ 3920 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 3921 struct bpf_verifier_state *st) 3922 { 3923 struct bpf_func_state *func; 3924 struct bpf_reg_state *reg; 3925 int i, j; 3926 3927 if (env->log.level & BPF_LOG_LEVEL2) { 3928 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", 3929 st->curframe); 3930 } 3931 3932 /* big hammer: mark all scalars precise in this path. 3933 * pop_stack may still get !precise scalars. 3934 * We also skip current state and go straight to first parent state, 3935 * because precision markings in current non-checkpointed state are 3936 * not needed. See why in the comment in __mark_chain_precision below. 3937 */ 3938 for (st = st->parent; st; st = st->parent) { 3939 for (i = 0; i <= st->curframe; i++) { 3940 func = st->frame[i]; 3941 for (j = 0; j < BPF_REG_FP; j++) { 3942 reg = &func->regs[j]; 3943 if (reg->type != SCALAR_VALUE || reg->precise) 3944 continue; 3945 reg->precise = true; 3946 if (env->log.level & BPF_LOG_LEVEL2) { 3947 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", 3948 i, j); 3949 } 3950 } 3951 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3952 if (!is_spilled_reg(&func->stack[j])) 3953 continue; 3954 reg = &func->stack[j].spilled_ptr; 3955 if (reg->type != SCALAR_VALUE || reg->precise) 3956 continue; 3957 reg->precise = true; 3958 if (env->log.level & BPF_LOG_LEVEL2) { 3959 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", 3960 i, -(j + 1) * 8); 3961 } 3962 } 3963 } 3964 } 3965 } 3966 3967 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 3968 { 3969 struct bpf_func_state *func; 3970 struct bpf_reg_state *reg; 3971 int i, j; 3972 3973 for (i = 0; i <= st->curframe; i++) { 3974 func = st->frame[i]; 3975 for (j = 0; j < BPF_REG_FP; j++) { 3976 reg = &func->regs[j]; 3977 if (reg->type != SCALAR_VALUE) 3978 continue; 3979 reg->precise = false; 3980 } 3981 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3982 if (!is_spilled_reg(&func->stack[j])) 3983 continue; 3984 reg = &func->stack[j].spilled_ptr; 3985 if (reg->type != SCALAR_VALUE) 3986 continue; 3987 reg->precise = false; 3988 } 3989 } 3990 } 3991 3992 static bool idset_contains(struct bpf_idset *s, u32 id) 3993 { 3994 u32 i; 3995 3996 for (i = 0; i < s->count; ++i) 3997 if (s->ids[i] == (id & ~BPF_ADD_CONST)) 3998 return true; 3999 4000 return false; 4001 } 4002 4003 static int idset_push(struct bpf_idset *s, u32 id) 4004 { 4005 if (WARN_ON_ONCE(s->count >= ARRAY_SIZE(s->ids))) 4006 return -EFAULT; 4007 s->ids[s->count++] = id & ~BPF_ADD_CONST; 4008 return 0; 4009 } 4010 4011 static void idset_reset(struct bpf_idset *s) 4012 { 4013 s->count = 0; 4014 } 4015 4016 /* Collect a set of IDs for all registers currently marked as precise in env->bt. 4017 * Mark all registers with these IDs as precise. 4018 */ 4019 static int mark_precise_scalar_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 4020 { 4021 struct bpf_idset *precise_ids = &env->idset_scratch; 4022 struct backtrack_state *bt = &env->bt; 4023 struct bpf_func_state *func; 4024 struct bpf_reg_state *reg; 4025 DECLARE_BITMAP(mask, 64); 4026 int i, fr; 4027 4028 idset_reset(precise_ids); 4029 4030 for (fr = bt->frame; fr >= 0; fr--) { 4031 func = st->frame[fr]; 4032 4033 bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); 4034 for_each_set_bit(i, mask, 32) { 4035 reg = &func->regs[i]; 4036 if (!reg->id || reg->type != SCALAR_VALUE) 4037 continue; 4038 if (idset_push(precise_ids, reg->id)) 4039 return -EFAULT; 4040 } 4041 4042 bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); 4043 for_each_set_bit(i, mask, 64) { 4044 if (i >= func->allocated_stack / BPF_REG_SIZE) 4045 break; 4046 if (!is_spilled_scalar_reg(&func->stack[i])) 4047 continue; 4048 reg = &func->stack[i].spilled_ptr; 4049 if (!reg->id) 4050 continue; 4051 if (idset_push(precise_ids, reg->id)) 4052 return -EFAULT; 4053 } 4054 } 4055 4056 for (fr = 0; fr <= st->curframe; ++fr) { 4057 func = st->frame[fr]; 4058 4059 for (i = BPF_REG_0; i < BPF_REG_10; ++i) { 4060 reg = &func->regs[i]; 4061 if (!reg->id) 4062 continue; 4063 if (!idset_contains(precise_ids, reg->id)) 4064 continue; 4065 bt_set_frame_reg(bt, fr, i); 4066 } 4067 for (i = 0; i < func->allocated_stack / BPF_REG_SIZE; ++i) { 4068 if (!is_spilled_scalar_reg(&func->stack[i])) 4069 continue; 4070 reg = &func->stack[i].spilled_ptr; 4071 if (!reg->id) 4072 continue; 4073 if (!idset_contains(precise_ids, reg->id)) 4074 continue; 4075 bt_set_frame_slot(bt, fr, i); 4076 } 4077 } 4078 4079 return 0; 4080 } 4081 4082 /* 4083 * __mark_chain_precision() backtracks BPF program instruction sequence and 4084 * chain of verifier states making sure that register *regno* (if regno >= 0) 4085 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked 4086 * SCALARS, as well as any other registers and slots that contribute to 4087 * a tracked state of given registers/stack slots, depending on specific BPF 4088 * assembly instructions (see backtrack_insns() for exact instruction handling 4089 * logic). This backtracking relies on recorded jmp_history and is able to 4090 * traverse entire chain of parent states. This process ends only when all the 4091 * necessary registers/slots and their transitive dependencies are marked as 4092 * precise. 4093 * 4094 * One important and subtle aspect is that precise marks *do not matter* in 4095 * the currently verified state (current state). It is important to understand 4096 * why this is the case. 4097 * 4098 * First, note that current state is the state that is not yet "checkpointed", 4099 * i.e., it is not yet put into env->explored_states, and it has no children 4100 * states as well. It's ephemeral, and can end up either a) being discarded if 4101 * compatible explored state is found at some point or BPF_EXIT instruction is 4102 * reached or b) checkpointed and put into env->explored_states, branching out 4103 * into one or more children states. 4104 * 4105 * In the former case, precise markings in current state are completely 4106 * ignored by state comparison code (see regsafe() for details). Only 4107 * checkpointed ("old") state precise markings are important, and if old 4108 * state's register/slot is precise, regsafe() assumes current state's 4109 * register/slot as precise and checks value ranges exactly and precisely. If 4110 * states turn out to be compatible, current state's necessary precise 4111 * markings and any required parent states' precise markings are enforced 4112 * after the fact with propagate_precision() logic, after the fact. But it's 4113 * important to realize that in this case, even after marking current state 4114 * registers/slots as precise, we immediately discard current state. So what 4115 * actually matters is any of the precise markings propagated into current 4116 * state's parent states, which are always checkpointed (due to b) case above). 4117 * As such, for scenario a) it doesn't matter if current state has precise 4118 * markings set or not. 4119 * 4120 * Now, for the scenario b), checkpointing and forking into child(ren) 4121 * state(s). Note that before current state gets to checkpointing step, any 4122 * processed instruction always assumes precise SCALAR register/slot 4123 * knowledge: if precise value or range is useful to prune jump branch, BPF 4124 * verifier takes this opportunity enthusiastically. Similarly, when 4125 * register's value is used to calculate offset or memory address, exact 4126 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to 4127 * what we mentioned above about state comparison ignoring precise markings 4128 * during state comparison, BPF verifier ignores and also assumes precise 4129 * markings *at will* during instruction verification process. But as verifier 4130 * assumes precision, it also propagates any precision dependencies across 4131 * parent states, which are not yet finalized, so can be further restricted 4132 * based on new knowledge gained from restrictions enforced by their children 4133 * states. This is so that once those parent states are finalized, i.e., when 4134 * they have no more active children state, state comparison logic in 4135 * is_state_visited() would enforce strict and precise SCALAR ranges, if 4136 * required for correctness. 4137 * 4138 * To build a bit more intuition, note also that once a state is checkpointed, 4139 * the path we took to get to that state is not important. This is crucial 4140 * property for state pruning. When state is checkpointed and finalized at 4141 * some instruction index, it can be correctly and safely used to "short 4142 * circuit" any *compatible* state that reaches exactly the same instruction 4143 * index. I.e., if we jumped to that instruction from a completely different 4144 * code path than original finalized state was derived from, it doesn't 4145 * matter, current state can be discarded because from that instruction 4146 * forward having a compatible state will ensure we will safely reach the 4147 * exit. States describe preconditions for further exploration, but completely 4148 * forget the history of how we got here. 4149 * 4150 * This also means that even if we needed precise SCALAR range to get to 4151 * finalized state, but from that point forward *that same* SCALAR register is 4152 * never used in a precise context (i.e., it's precise value is not needed for 4153 * correctness), it's correct and safe to mark such register as "imprecise" 4154 * (i.e., precise marking set to false). This is what we rely on when we do 4155 * not set precise marking in current state. If no child state requires 4156 * precision for any given SCALAR register, it's safe to dictate that it can 4157 * be imprecise. If any child state does require this register to be precise, 4158 * we'll mark it precise later retroactively during precise markings 4159 * propagation from child state to parent states. 4160 * 4161 * Skipping precise marking setting in current state is a mild version of 4162 * relying on the above observation. But we can utilize this property even 4163 * more aggressively by proactively forgetting any precise marking in the 4164 * current state (which we inherited from the parent state), right before we 4165 * checkpoint it and branch off into new child state. This is done by 4166 * mark_all_scalars_imprecise() to hopefully get more permissive and generic 4167 * finalized states which help in short circuiting more future states. 4168 */ 4169 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) 4170 { 4171 struct backtrack_state *bt = &env->bt; 4172 struct bpf_verifier_state *st = env->cur_state; 4173 int first_idx = st->first_insn_idx; 4174 int last_idx = env->insn_idx; 4175 int subseq_idx = -1; 4176 struct bpf_func_state *func; 4177 struct bpf_reg_state *reg; 4178 bool skip_first = true; 4179 int i, fr, err; 4180 4181 if (!env->bpf_capable) 4182 return 0; 4183 4184 /* set frame number from which we are starting to backtrack */ 4185 bt_init(bt, env->cur_state->curframe); 4186 4187 /* Do sanity checks against current state of register and/or stack 4188 * slot, but don't set precise flag in current state, as precision 4189 * tracking in the current state is unnecessary. 4190 */ 4191 func = st->frame[bt->frame]; 4192 if (regno >= 0) { 4193 reg = &func->regs[regno]; 4194 if (reg->type != SCALAR_VALUE) { 4195 WARN_ONCE(1, "backtracing misuse"); 4196 return -EFAULT; 4197 } 4198 bt_set_reg(bt, regno); 4199 } 4200 4201 if (bt_empty(bt)) 4202 return 0; 4203 4204 for (;;) { 4205 DECLARE_BITMAP(mask, 64); 4206 u32 history = st->jmp_history_cnt; 4207 struct bpf_jmp_history_entry *hist; 4208 4209 if (env->log.level & BPF_LOG_LEVEL2) { 4210 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", 4211 bt->frame, last_idx, first_idx, subseq_idx); 4212 } 4213 4214 /* If some register with scalar ID is marked as precise, 4215 * make sure that all registers sharing this ID are also precise. 4216 * This is needed to estimate effect of find_equal_scalars(). 4217 * Do this at the last instruction of each state, 4218 * bpf_reg_state::id fields are valid for these instructions. 4219 * 4220 * Allows to track precision in situation like below: 4221 * 4222 * r2 = unknown value 4223 * ... 4224 * --- state #0 --- 4225 * ... 4226 * r1 = r2 // r1 and r2 now share the same ID 4227 * ... 4228 * --- state #1 {r1.id = A, r2.id = A} --- 4229 * ... 4230 * if (r2 > 10) goto exit; // find_equal_scalars() assigns range to r1 4231 * ... 4232 * --- state #2 {r1.id = A, r2.id = A} --- 4233 * r3 = r10 4234 * r3 += r1 // need to mark both r1 and r2 4235 */ 4236 if (mark_precise_scalar_ids(env, st)) 4237 return -EFAULT; 4238 4239 if (last_idx < 0) { 4240 /* we are at the entry into subprog, which 4241 * is expected for global funcs, but only if 4242 * requested precise registers are R1-R5 4243 * (which are global func's input arguments) 4244 */ 4245 if (st->curframe == 0 && 4246 st->frame[0]->subprogno > 0 && 4247 st->frame[0]->callsite == BPF_MAIN_FUNC && 4248 bt_stack_mask(bt) == 0 && 4249 (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { 4250 bitmap_from_u64(mask, bt_reg_mask(bt)); 4251 for_each_set_bit(i, mask, 32) { 4252 reg = &st->frame[0]->regs[i]; 4253 bt_clear_reg(bt, i); 4254 if (reg->type == SCALAR_VALUE) 4255 reg->precise = true; 4256 } 4257 return 0; 4258 } 4259 4260 verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n", 4261 st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); 4262 WARN_ONCE(1, "verifier backtracking bug"); 4263 return -EFAULT; 4264 } 4265 4266 for (i = last_idx;;) { 4267 if (skip_first) { 4268 err = 0; 4269 skip_first = false; 4270 } else { 4271 hist = get_jmp_hist_entry(st, history, i); 4272 err = backtrack_insn(env, i, subseq_idx, hist, bt); 4273 } 4274 if (err == -ENOTSUPP) { 4275 mark_all_scalars_precise(env, env->cur_state); 4276 bt_reset(bt); 4277 return 0; 4278 } else if (err) { 4279 return err; 4280 } 4281 if (bt_empty(bt)) 4282 /* Found assignment(s) into tracked register in this state. 4283 * Since this state is already marked, just return. 4284 * Nothing to be tracked further in the parent state. 4285 */ 4286 return 0; 4287 subseq_idx = i; 4288 i = get_prev_insn_idx(st, i, &history); 4289 if (i == -ENOENT) 4290 break; 4291 if (i >= env->prog->len) { 4292 /* This can happen if backtracking reached insn 0 4293 * and there are still reg_mask or stack_mask 4294 * to backtrack. 4295 * It means the backtracking missed the spot where 4296 * particular register was initialized with a constant. 4297 */ 4298 verbose(env, "BUG backtracking idx %d\n", i); 4299 WARN_ONCE(1, "verifier backtracking bug"); 4300 return -EFAULT; 4301 } 4302 } 4303 st = st->parent; 4304 if (!st) 4305 break; 4306 4307 for (fr = bt->frame; fr >= 0; fr--) { 4308 func = st->frame[fr]; 4309 bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr)); 4310 for_each_set_bit(i, mask, 32) { 4311 reg = &func->regs[i]; 4312 if (reg->type != SCALAR_VALUE) { 4313 bt_clear_frame_reg(bt, fr, i); 4314 continue; 4315 } 4316 if (reg->precise) 4317 bt_clear_frame_reg(bt, fr, i); 4318 else 4319 reg->precise = true; 4320 } 4321 4322 bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); 4323 for_each_set_bit(i, mask, 64) { 4324 if (i >= func->allocated_stack / BPF_REG_SIZE) { 4325 verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n", 4326 i, func->allocated_stack / BPF_REG_SIZE); 4327 WARN_ONCE(1, "verifier backtracking bug (stack slot out of bounds)"); 4328 return -EFAULT; 4329 } 4330 4331 if (!is_spilled_scalar_reg(&func->stack[i])) { 4332 bt_clear_frame_slot(bt, fr, i); 4333 continue; 4334 } 4335 reg = &func->stack[i].spilled_ptr; 4336 if (reg->precise) 4337 bt_clear_frame_slot(bt, fr, i); 4338 else 4339 reg->precise = true; 4340 } 4341 if (env->log.level & BPF_LOG_LEVEL2) { 4342 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, 4343 bt_frame_reg_mask(bt, fr)); 4344 verbose(env, "mark_precise: frame%d: parent state regs=%s ", 4345 fr, env->tmp_str_buf); 4346 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, 4347 bt_frame_stack_mask(bt, fr)); 4348 verbose(env, "stack=%s: ", env->tmp_str_buf); 4349 print_verifier_state(env, func, true); 4350 } 4351 } 4352 4353 if (bt_empty(bt)) 4354 return 0; 4355 4356 subseq_idx = first_idx; 4357 last_idx = st->last_insn_idx; 4358 first_idx = st->first_insn_idx; 4359 } 4360 4361 /* if we still have requested precise regs or slots, we missed 4362 * something (e.g., stack access through non-r10 register), so 4363 * fallback to marking all precise 4364 */ 4365 if (!bt_empty(bt)) { 4366 mark_all_scalars_precise(env, env->cur_state); 4367 bt_reset(bt); 4368 } 4369 4370 return 0; 4371 } 4372 4373 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 4374 { 4375 return __mark_chain_precision(env, regno); 4376 } 4377 4378 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to 4379 * desired reg and stack masks across all relevant frames 4380 */ 4381 static int mark_chain_precision_batch(struct bpf_verifier_env *env) 4382 { 4383 return __mark_chain_precision(env, -1); 4384 } 4385 4386 static bool is_spillable_regtype(enum bpf_reg_type type) 4387 { 4388 switch (base_type(type)) { 4389 case PTR_TO_MAP_VALUE: 4390 case PTR_TO_STACK: 4391 case PTR_TO_CTX: 4392 case PTR_TO_PACKET: 4393 case PTR_TO_PACKET_META: 4394 case PTR_TO_PACKET_END: 4395 case PTR_TO_FLOW_KEYS: 4396 case CONST_PTR_TO_MAP: 4397 case PTR_TO_SOCKET: 4398 case PTR_TO_SOCK_COMMON: 4399 case PTR_TO_TCP_SOCK: 4400 case PTR_TO_XDP_SOCK: 4401 case PTR_TO_BTF_ID: 4402 case PTR_TO_BUF: 4403 case PTR_TO_MEM: 4404 case PTR_TO_FUNC: 4405 case PTR_TO_MAP_KEY: 4406 case PTR_TO_ARENA: 4407 return true; 4408 default: 4409 return false; 4410 } 4411 } 4412 4413 /* Does this register contain a constant zero? */ 4414 static bool register_is_null(struct bpf_reg_state *reg) 4415 { 4416 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 4417 } 4418 4419 /* check if register is a constant scalar value */ 4420 static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) 4421 { 4422 return reg->type == SCALAR_VALUE && 4423 tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off); 4424 } 4425 4426 /* assuming is_reg_const() is true, return constant value of a register */ 4427 static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32) 4428 { 4429 return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value; 4430 } 4431 4432 static bool __is_pointer_value(bool allow_ptr_leaks, 4433 const struct bpf_reg_state *reg) 4434 { 4435 if (allow_ptr_leaks) 4436 return false; 4437 4438 return reg->type != SCALAR_VALUE; 4439 } 4440 4441 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, 4442 struct bpf_reg_state *src_reg) 4443 { 4444 if (src_reg->type != SCALAR_VALUE) 4445 return; 4446 4447 if (src_reg->id & BPF_ADD_CONST) { 4448 /* 4449 * The verifier is processing rX = rY insn and 4450 * rY->id has special linked register already. 4451 * Cleared it, since multiple rX += const are not supported. 4452 */ 4453 src_reg->id = 0; 4454 src_reg->off = 0; 4455 } 4456 4457 if (!src_reg->id && !tnum_is_const(src_reg->var_off)) 4458 /* Ensure that src_reg has a valid ID that will be copied to 4459 * dst_reg and then will be used by find_equal_scalars() to 4460 * propagate min/max range. 4461 */ 4462 src_reg->id = ++env->id_gen; 4463 } 4464 4465 /* Copy src state preserving dst->parent and dst->live fields */ 4466 static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src) 4467 { 4468 struct bpf_reg_state *parent = dst->parent; 4469 enum bpf_reg_liveness live = dst->live; 4470 4471 *dst = *src; 4472 dst->parent = parent; 4473 dst->live = live; 4474 } 4475 4476 static void save_register_state(struct bpf_verifier_env *env, 4477 struct bpf_func_state *state, 4478 int spi, struct bpf_reg_state *reg, 4479 int size) 4480 { 4481 int i; 4482 4483 copy_register_state(&state->stack[spi].spilled_ptr, reg); 4484 if (size == BPF_REG_SIZE) 4485 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 4486 4487 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 4488 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 4489 4490 /* size < 8 bytes spill */ 4491 for (; i; i--) 4492 mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); 4493 } 4494 4495 static bool is_bpf_st_mem(struct bpf_insn *insn) 4496 { 4497 return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM; 4498 } 4499 4500 static int get_reg_width(struct bpf_reg_state *reg) 4501 { 4502 return fls64(reg->umax_value); 4503 } 4504 4505 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 4506 * stack boundary and alignment are checked in check_mem_access() 4507 */ 4508 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 4509 /* stack frame we're writing to */ 4510 struct bpf_func_state *state, 4511 int off, int size, int value_regno, 4512 int insn_idx) 4513 { 4514 struct bpf_func_state *cur; /* state of the current function */ 4515 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 4516 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 4517 struct bpf_reg_state *reg = NULL; 4518 int insn_flags = insn_stack_access_flags(state->frameno, spi); 4519 4520 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 4521 * so it's aligned access and [off, off + size) are within stack limits 4522 */ 4523 if (!env->allow_ptr_leaks && 4524 is_spilled_reg(&state->stack[spi]) && 4525 size != BPF_REG_SIZE) { 4526 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 4527 return -EACCES; 4528 } 4529 4530 cur = env->cur_state->frame[env->cur_state->curframe]; 4531 if (value_regno >= 0) 4532 reg = &cur->regs[value_regno]; 4533 if (!env->bypass_spec_v4) { 4534 bool sanitize = reg && is_spillable_regtype(reg->type); 4535 4536 for (i = 0; i < size; i++) { 4537 u8 type = state->stack[spi].slot_type[i]; 4538 4539 if (type != STACK_MISC && type != STACK_ZERO) { 4540 sanitize = true; 4541 break; 4542 } 4543 } 4544 4545 if (sanitize) 4546 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 4547 } 4548 4549 err = destroy_if_dynptr_stack_slot(env, state, spi); 4550 if (err) 4551 return err; 4552 4553 mark_stack_slot_scratched(env, spi); 4554 if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { 4555 bool reg_value_fits; 4556 4557 reg_value_fits = get_reg_width(reg) <= BITS_PER_BYTE * size; 4558 /* Make sure that reg had an ID to build a relation on spill. */ 4559 if (reg_value_fits) 4560 assign_scalar_id_before_mov(env, reg); 4561 save_register_state(env, state, spi, reg, size); 4562 /* Break the relation on a narrowing spill. */ 4563 if (!reg_value_fits) 4564 state->stack[spi].spilled_ptr.id = 0; 4565 } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && 4566 env->bpf_capable) { 4567 struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; 4568 4569 memset(tmp_reg, 0, sizeof(*tmp_reg)); 4570 __mark_reg_known(tmp_reg, insn->imm); 4571 tmp_reg->type = SCALAR_VALUE; 4572 save_register_state(env, state, spi, tmp_reg, size); 4573 } else if (reg && is_spillable_regtype(reg->type)) { 4574 /* register containing pointer is being spilled into stack */ 4575 if (size != BPF_REG_SIZE) { 4576 verbose_linfo(env, insn_idx, "; "); 4577 verbose(env, "invalid size of register spill\n"); 4578 return -EACCES; 4579 } 4580 if (state != cur && reg->type == PTR_TO_STACK) { 4581 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 4582 return -EINVAL; 4583 } 4584 save_register_state(env, state, spi, reg, size); 4585 } else { 4586 u8 type = STACK_MISC; 4587 4588 /* regular write of data into stack destroys any spilled ptr */ 4589 state->stack[spi].spilled_ptr.type = NOT_INIT; 4590 /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ 4591 if (is_stack_slot_special(&state->stack[spi])) 4592 for (i = 0; i < BPF_REG_SIZE; i++) 4593 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 4594 4595 /* only mark the slot as written if all 8 bytes were written 4596 * otherwise read propagation may incorrectly stop too soon 4597 * when stack slots are partially written. 4598 * This heuristic means that read propagation will be 4599 * conservative, since it will add reg_live_read marks 4600 * to stack slots all the way to first state when programs 4601 * writes+reads less than 8 bytes 4602 */ 4603 if (size == BPF_REG_SIZE) 4604 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 4605 4606 /* when we zero initialize stack slots mark them as such */ 4607 if ((reg && register_is_null(reg)) || 4608 (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { 4609 /* STACK_ZERO case happened because register spill 4610 * wasn't properly aligned at the stack slot boundary, 4611 * so it's not a register spill anymore; force 4612 * originating register to be precise to make 4613 * STACK_ZERO correct for subsequent states 4614 */ 4615 err = mark_chain_precision(env, value_regno); 4616 if (err) 4617 return err; 4618 type = STACK_ZERO; 4619 } 4620 4621 /* Mark slots affected by this stack write. */ 4622 for (i = 0; i < size; i++) 4623 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; 4624 insn_flags = 0; /* not a register spill */ 4625 } 4626 4627 if (insn_flags) 4628 return push_jmp_history(env, env->cur_state, insn_flags); 4629 return 0; 4630 } 4631 4632 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 4633 * known to contain a variable offset. 4634 * This function checks whether the write is permitted and conservatively 4635 * tracks the effects of the write, considering that each stack slot in the 4636 * dynamic range is potentially written to. 4637 * 4638 * 'off' includes 'regno->off'. 4639 * 'value_regno' can be -1, meaning that an unknown value is being written to 4640 * the stack. 4641 * 4642 * Spilled pointers in range are not marked as written because we don't know 4643 * what's going to be actually written. This means that read propagation for 4644 * future reads cannot be terminated by this write. 4645 * 4646 * For privileged programs, uninitialized stack slots are considered 4647 * initialized by this write (even though we don't know exactly what offsets 4648 * are going to be written to). The idea is that we don't want the verifier to 4649 * reject future reads that access slots written to through variable offsets. 4650 */ 4651 static int check_stack_write_var_off(struct bpf_verifier_env *env, 4652 /* func where register points to */ 4653 struct bpf_func_state *state, 4654 int ptr_regno, int off, int size, 4655 int value_regno, int insn_idx) 4656 { 4657 struct bpf_func_state *cur; /* state of the current function */ 4658 int min_off, max_off; 4659 int i, err; 4660 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 4661 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 4662 bool writing_zero = false; 4663 /* set if the fact that we're writing a zero is used to let any 4664 * stack slots remain STACK_ZERO 4665 */ 4666 bool zero_used = false; 4667 4668 cur = env->cur_state->frame[env->cur_state->curframe]; 4669 ptr_reg = &cur->regs[ptr_regno]; 4670 min_off = ptr_reg->smin_value + off; 4671 max_off = ptr_reg->smax_value + off + size; 4672 if (value_regno >= 0) 4673 value_reg = &cur->regs[value_regno]; 4674 if ((value_reg && register_is_null(value_reg)) || 4675 (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) 4676 writing_zero = true; 4677 4678 for (i = min_off; i < max_off; i++) { 4679 int spi; 4680 4681 spi = __get_spi(i); 4682 err = destroy_if_dynptr_stack_slot(env, state, spi); 4683 if (err) 4684 return err; 4685 } 4686 4687 /* Variable offset writes destroy any spilled pointers in range. */ 4688 for (i = min_off; i < max_off; i++) { 4689 u8 new_type, *stype; 4690 int slot, spi; 4691 4692 slot = -i - 1; 4693 spi = slot / BPF_REG_SIZE; 4694 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 4695 mark_stack_slot_scratched(env, spi); 4696 4697 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 4698 /* Reject the write if range we may write to has not 4699 * been initialized beforehand. If we didn't reject 4700 * here, the ptr status would be erased below (even 4701 * though not all slots are actually overwritten), 4702 * possibly opening the door to leaks. 4703 * 4704 * We do however catch STACK_INVALID case below, and 4705 * only allow reading possibly uninitialized memory 4706 * later for CAP_PERFMON, as the write may not happen to 4707 * that slot. 4708 */ 4709 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 4710 insn_idx, i); 4711 return -EINVAL; 4712 } 4713 4714 /* If writing_zero and the spi slot contains a spill of value 0, 4715 * maintain the spill type. 4716 */ 4717 if (writing_zero && *stype == STACK_SPILL && 4718 is_spilled_scalar_reg(&state->stack[spi])) { 4719 struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; 4720 4721 if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { 4722 zero_used = true; 4723 continue; 4724 } 4725 } 4726 4727 /* Erase all other spilled pointers. */ 4728 state->stack[spi].spilled_ptr.type = NOT_INIT; 4729 4730 /* Update the slot type. */ 4731 new_type = STACK_MISC; 4732 if (writing_zero && *stype == STACK_ZERO) { 4733 new_type = STACK_ZERO; 4734 zero_used = true; 4735 } 4736 /* If the slot is STACK_INVALID, we check whether it's OK to 4737 * pretend that it will be initialized by this write. The slot 4738 * might not actually be written to, and so if we mark it as 4739 * initialized future reads might leak uninitialized memory. 4740 * For privileged programs, we will accept such reads to slots 4741 * that may or may not be written because, if we're reject 4742 * them, the error would be too confusing. 4743 */ 4744 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 4745 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 4746 insn_idx, i); 4747 return -EINVAL; 4748 } 4749 *stype = new_type; 4750 } 4751 if (zero_used) { 4752 /* backtracking doesn't work for STACK_ZERO yet. */ 4753 err = mark_chain_precision(env, value_regno); 4754 if (err) 4755 return err; 4756 } 4757 return 0; 4758 } 4759 4760 /* When register 'dst_regno' is assigned some values from stack[min_off, 4761 * max_off), we set the register's type according to the types of the 4762 * respective stack slots. If all the stack values are known to be zeros, then 4763 * so is the destination reg. Otherwise, the register is considered to be 4764 * SCALAR. This function does not deal with register filling; the caller must 4765 * ensure that all spilled registers in the stack range have been marked as 4766 * read. 4767 */ 4768 static void mark_reg_stack_read(struct bpf_verifier_env *env, 4769 /* func where src register points to */ 4770 struct bpf_func_state *ptr_state, 4771 int min_off, int max_off, int dst_regno) 4772 { 4773 struct bpf_verifier_state *vstate = env->cur_state; 4774 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4775 int i, slot, spi; 4776 u8 *stype; 4777 int zeros = 0; 4778 4779 for (i = min_off; i < max_off; i++) { 4780 slot = -i - 1; 4781 spi = slot / BPF_REG_SIZE; 4782 mark_stack_slot_scratched(env, spi); 4783 stype = ptr_state->stack[spi].slot_type; 4784 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 4785 break; 4786 zeros++; 4787 } 4788 if (zeros == max_off - min_off) { 4789 /* Any access_size read into register is zero extended, 4790 * so the whole register == const_zero. 4791 */ 4792 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4793 } else { 4794 /* have read misc data from the stack */ 4795 mark_reg_unknown(env, state->regs, dst_regno); 4796 } 4797 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4798 } 4799 4800 /* Read the stack at 'off' and put the results into the register indicated by 4801 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 4802 * spilled reg. 4803 * 4804 * 'dst_regno' can be -1, meaning that the read value is not going to a 4805 * register. 4806 * 4807 * The access is assumed to be within the current stack bounds. 4808 */ 4809 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 4810 /* func where src register points to */ 4811 struct bpf_func_state *reg_state, 4812 int off, int size, int dst_regno) 4813 { 4814 struct bpf_verifier_state *vstate = env->cur_state; 4815 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4816 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 4817 struct bpf_reg_state *reg; 4818 u8 *stype, type; 4819 int insn_flags = insn_stack_access_flags(reg_state->frameno, spi); 4820 4821 stype = reg_state->stack[spi].slot_type; 4822 reg = ®_state->stack[spi].spilled_ptr; 4823 4824 mark_stack_slot_scratched(env, spi); 4825 4826 if (is_spilled_reg(®_state->stack[spi])) { 4827 u8 spill_size = 1; 4828 4829 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 4830 spill_size++; 4831 4832 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 4833 if (reg->type != SCALAR_VALUE) { 4834 verbose_linfo(env, env->insn_idx, "; "); 4835 verbose(env, "invalid size of register fill\n"); 4836 return -EACCES; 4837 } 4838 4839 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4840 if (dst_regno < 0) 4841 return 0; 4842 4843 if (size <= spill_size && 4844 bpf_stack_narrow_access_ok(off, size, spill_size)) { 4845 /* The earlier check_reg_arg() has decided the 4846 * subreg_def for this insn. Save it first. 4847 */ 4848 s32 subreg_def = state->regs[dst_regno].subreg_def; 4849 4850 copy_register_state(&state->regs[dst_regno], reg); 4851 state->regs[dst_regno].subreg_def = subreg_def; 4852 4853 /* Break the relation on a narrowing fill. 4854 * coerce_reg_to_size will adjust the boundaries. 4855 */ 4856 if (get_reg_width(reg) > size * BITS_PER_BYTE) 4857 state->regs[dst_regno].id = 0; 4858 } else { 4859 int spill_cnt = 0, zero_cnt = 0; 4860 4861 for (i = 0; i < size; i++) { 4862 type = stype[(slot - i) % BPF_REG_SIZE]; 4863 if (type == STACK_SPILL) { 4864 spill_cnt++; 4865 continue; 4866 } 4867 if (type == STACK_MISC) 4868 continue; 4869 if (type == STACK_ZERO) { 4870 zero_cnt++; 4871 continue; 4872 } 4873 if (type == STACK_INVALID && env->allow_uninit_stack) 4874 continue; 4875 verbose(env, "invalid read from stack off %d+%d size %d\n", 4876 off, i, size); 4877 return -EACCES; 4878 } 4879 4880 if (spill_cnt == size && 4881 tnum_is_const(reg->var_off) && reg->var_off.value == 0) { 4882 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4883 /* this IS register fill, so keep insn_flags */ 4884 } else if (zero_cnt == size) { 4885 /* similarly to mark_reg_stack_read(), preserve zeroes */ 4886 __mark_reg_const_zero(env, &state->regs[dst_regno]); 4887 insn_flags = 0; /* not restoring original register state */ 4888 } else { 4889 mark_reg_unknown(env, state->regs, dst_regno); 4890 insn_flags = 0; /* not restoring original register state */ 4891 } 4892 } 4893 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4894 } else if (dst_regno >= 0) { 4895 /* restore register state from stack */ 4896 copy_register_state(&state->regs[dst_regno], reg); 4897 /* mark reg as written since spilled pointer state likely 4898 * has its liveness marks cleared by is_state_visited() 4899 * which resets stack/reg liveness for state transitions 4900 */ 4901 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4902 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 4903 /* If dst_regno==-1, the caller is asking us whether 4904 * it is acceptable to use this value as a SCALAR_VALUE 4905 * (e.g. for XADD). 4906 * We must not allow unprivileged callers to do that 4907 * with spilled pointers. 4908 */ 4909 verbose(env, "leaking pointer from stack off %d\n", 4910 off); 4911 return -EACCES; 4912 } 4913 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4914 } else { 4915 for (i = 0; i < size; i++) { 4916 type = stype[(slot - i) % BPF_REG_SIZE]; 4917 if (type == STACK_MISC) 4918 continue; 4919 if (type == STACK_ZERO) 4920 continue; 4921 if (type == STACK_INVALID && env->allow_uninit_stack) 4922 continue; 4923 verbose(env, "invalid read from stack off %d+%d size %d\n", 4924 off, i, size); 4925 return -EACCES; 4926 } 4927 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4928 if (dst_regno >= 0) 4929 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 4930 insn_flags = 0; /* we are not restoring spilled register */ 4931 } 4932 if (insn_flags) 4933 return push_jmp_history(env, env->cur_state, insn_flags); 4934 return 0; 4935 } 4936 4937 enum bpf_access_src { 4938 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 4939 ACCESS_HELPER = 2, /* the access is performed by a helper */ 4940 }; 4941 4942 static int check_stack_range_initialized(struct bpf_verifier_env *env, 4943 int regno, int off, int access_size, 4944 bool zero_size_allowed, 4945 enum bpf_access_src type, 4946 struct bpf_call_arg_meta *meta); 4947 4948 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 4949 { 4950 return cur_regs(env) + regno; 4951 } 4952 4953 /* Read the stack at 'ptr_regno + off' and put the result into the register 4954 * 'dst_regno'. 4955 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 4956 * but not its variable offset. 4957 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 4958 * 4959 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 4960 * filling registers (i.e. reads of spilled register cannot be detected when 4961 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 4962 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 4963 * offset; for a fixed offset check_stack_read_fixed_off should be used 4964 * instead. 4965 */ 4966 static int check_stack_read_var_off(struct bpf_verifier_env *env, 4967 int ptr_regno, int off, int size, int dst_regno) 4968 { 4969 /* The state of the source register. */ 4970 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 4971 struct bpf_func_state *ptr_state = func(env, reg); 4972 int err; 4973 int min_off, max_off; 4974 4975 /* Note that we pass a NULL meta, so raw access will not be permitted. 4976 */ 4977 err = check_stack_range_initialized(env, ptr_regno, off, size, 4978 false, ACCESS_DIRECT, NULL); 4979 if (err) 4980 return err; 4981 4982 min_off = reg->smin_value + off; 4983 max_off = reg->smax_value + off; 4984 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 4985 return 0; 4986 } 4987 4988 /* check_stack_read dispatches to check_stack_read_fixed_off or 4989 * check_stack_read_var_off. 4990 * 4991 * The caller must ensure that the offset falls within the allocated stack 4992 * bounds. 4993 * 4994 * 'dst_regno' is a register which will receive the value from the stack. It 4995 * can be -1, meaning that the read value is not going to a register. 4996 */ 4997 static int check_stack_read(struct bpf_verifier_env *env, 4998 int ptr_regno, int off, int size, 4999 int dst_regno) 5000 { 5001 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 5002 struct bpf_func_state *state = func(env, reg); 5003 int err; 5004 /* Some accesses are only permitted with a static offset. */ 5005 bool var_off = !tnum_is_const(reg->var_off); 5006 5007 /* The offset is required to be static when reads don't go to a 5008 * register, in order to not leak pointers (see 5009 * check_stack_read_fixed_off). 5010 */ 5011 if (dst_regno < 0 && var_off) { 5012 char tn_buf[48]; 5013 5014 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5015 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 5016 tn_buf, off, size); 5017 return -EACCES; 5018 } 5019 /* Variable offset is prohibited for unprivileged mode for simplicity 5020 * since it requires corresponding support in Spectre masking for stack 5021 * ALU. See also retrieve_ptr_limit(). The check in 5022 * check_stack_access_for_ptr_arithmetic() called by 5023 * adjust_ptr_min_max_vals() prevents users from creating stack pointers 5024 * with variable offsets, therefore no check is required here. Further, 5025 * just checking it here would be insufficient as speculative stack 5026 * writes could still lead to unsafe speculative behaviour. 5027 */ 5028 if (!var_off) { 5029 off += reg->var_off.value; 5030 err = check_stack_read_fixed_off(env, state, off, size, 5031 dst_regno); 5032 } else { 5033 /* Variable offset stack reads need more conservative handling 5034 * than fixed offset ones. Note that dst_regno >= 0 on this 5035 * branch. 5036 */ 5037 err = check_stack_read_var_off(env, ptr_regno, off, size, 5038 dst_regno); 5039 } 5040 return err; 5041 } 5042 5043 5044 /* check_stack_write dispatches to check_stack_write_fixed_off or 5045 * check_stack_write_var_off. 5046 * 5047 * 'ptr_regno' is the register used as a pointer into the stack. 5048 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 5049 * 'value_regno' is the register whose value we're writing to the stack. It can 5050 * be -1, meaning that we're not writing from a register. 5051 * 5052 * The caller must ensure that the offset falls within the maximum stack size. 5053 */ 5054 static int check_stack_write(struct bpf_verifier_env *env, 5055 int ptr_regno, int off, int size, 5056 int value_regno, int insn_idx) 5057 { 5058 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 5059 struct bpf_func_state *state = func(env, reg); 5060 int err; 5061 5062 if (tnum_is_const(reg->var_off)) { 5063 off += reg->var_off.value; 5064 err = check_stack_write_fixed_off(env, state, off, size, 5065 value_regno, insn_idx); 5066 } else { 5067 /* Variable offset stack reads need more conservative handling 5068 * than fixed offset ones. 5069 */ 5070 err = check_stack_write_var_off(env, state, 5071 ptr_regno, off, size, 5072 value_regno, insn_idx); 5073 } 5074 return err; 5075 } 5076 5077 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 5078 int off, int size, enum bpf_access_type type) 5079 { 5080 struct bpf_reg_state *regs = cur_regs(env); 5081 struct bpf_map *map = regs[regno].map_ptr; 5082 u32 cap = bpf_map_flags_to_cap(map); 5083 5084 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 5085 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 5086 map->value_size, off, size); 5087 return -EACCES; 5088 } 5089 5090 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 5091 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 5092 map->value_size, off, size); 5093 return -EACCES; 5094 } 5095 5096 return 0; 5097 } 5098 5099 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 5100 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 5101 int off, int size, u32 mem_size, 5102 bool zero_size_allowed) 5103 { 5104 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 5105 struct bpf_reg_state *reg; 5106 5107 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 5108 return 0; 5109 5110 reg = &cur_regs(env)[regno]; 5111 switch (reg->type) { 5112 case PTR_TO_MAP_KEY: 5113 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 5114 mem_size, off, size); 5115 break; 5116 case PTR_TO_MAP_VALUE: 5117 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 5118 mem_size, off, size); 5119 break; 5120 case PTR_TO_PACKET: 5121 case PTR_TO_PACKET_META: 5122 case PTR_TO_PACKET_END: 5123 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 5124 off, size, regno, reg->id, off, mem_size); 5125 break; 5126 case PTR_TO_MEM: 5127 default: 5128 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 5129 mem_size, off, size); 5130 } 5131 5132 return -EACCES; 5133 } 5134 5135 /* check read/write into a memory region with possible variable offset */ 5136 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 5137 int off, int size, u32 mem_size, 5138 bool zero_size_allowed) 5139 { 5140 struct bpf_verifier_state *vstate = env->cur_state; 5141 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 5142 struct bpf_reg_state *reg = &state->regs[regno]; 5143 int err; 5144 5145 /* We may have adjusted the register pointing to memory region, so we 5146 * need to try adding each of min_value and max_value to off 5147 * to make sure our theoretical access will be safe. 5148 * 5149 * The minimum value is only important with signed 5150 * comparisons where we can't assume the floor of a 5151 * value is 0. If we are using signed variables for our 5152 * index'es we need to make sure that whatever we use 5153 * will have a set floor within our range. 5154 */ 5155 if (reg->smin_value < 0 && 5156 (reg->smin_value == S64_MIN || 5157 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 5158 reg->smin_value + off < 0)) { 5159 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 5160 regno); 5161 return -EACCES; 5162 } 5163 err = __check_mem_access(env, regno, reg->smin_value + off, size, 5164 mem_size, zero_size_allowed); 5165 if (err) { 5166 verbose(env, "R%d min value is outside of the allowed memory range\n", 5167 regno); 5168 return err; 5169 } 5170 5171 /* If we haven't set a max value then we need to bail since we can't be 5172 * sure we won't do bad things. 5173 * If reg->umax_value + off could overflow, treat that as unbounded too. 5174 */ 5175 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 5176 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 5177 regno); 5178 return -EACCES; 5179 } 5180 err = __check_mem_access(env, regno, reg->umax_value + off, size, 5181 mem_size, zero_size_allowed); 5182 if (err) { 5183 verbose(env, "R%d max value is outside of the allowed memory range\n", 5184 regno); 5185 return err; 5186 } 5187 5188 return 0; 5189 } 5190 5191 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 5192 const struct bpf_reg_state *reg, int regno, 5193 bool fixed_off_ok) 5194 { 5195 /* Access to this pointer-typed register or passing it to a helper 5196 * is only allowed in its original, unmodified form. 5197 */ 5198 5199 if (reg->off < 0) { 5200 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 5201 reg_type_str(env, reg->type), regno, reg->off); 5202 return -EACCES; 5203 } 5204 5205 if (!fixed_off_ok && reg->off) { 5206 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 5207 reg_type_str(env, reg->type), regno, reg->off); 5208 return -EACCES; 5209 } 5210 5211 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 5212 char tn_buf[48]; 5213 5214 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5215 verbose(env, "variable %s access var_off=%s disallowed\n", 5216 reg_type_str(env, reg->type), tn_buf); 5217 return -EACCES; 5218 } 5219 5220 return 0; 5221 } 5222 5223 static int check_ptr_off_reg(struct bpf_verifier_env *env, 5224 const struct bpf_reg_state *reg, int regno) 5225 { 5226 return __check_ptr_off_reg(env, reg, regno, false); 5227 } 5228 5229 static int map_kptr_match_type(struct bpf_verifier_env *env, 5230 struct btf_field *kptr_field, 5231 struct bpf_reg_state *reg, u32 regno) 5232 { 5233 const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 5234 int perm_flags; 5235 const char *reg_name = ""; 5236 5237 if (btf_is_kernel(reg->btf)) { 5238 perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU; 5239 5240 /* Only unreferenced case accepts untrusted pointers */ 5241 if (kptr_field->type == BPF_KPTR_UNREF) 5242 perm_flags |= PTR_UNTRUSTED; 5243 } else { 5244 perm_flags = PTR_MAYBE_NULL | MEM_ALLOC; 5245 if (kptr_field->type == BPF_KPTR_PERCPU) 5246 perm_flags |= MEM_PERCPU; 5247 } 5248 5249 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 5250 goto bad_type; 5251 5252 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 5253 reg_name = btf_type_name(reg->btf, reg->btf_id); 5254 5255 /* For ref_ptr case, release function check should ensure we get one 5256 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 5257 * normal store of unreferenced kptr, we must ensure var_off is zero. 5258 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 5259 * reg->off and reg->ref_obj_id are not needed here. 5260 */ 5261 if (__check_ptr_off_reg(env, reg, regno, true)) 5262 return -EACCES; 5263 5264 /* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and 5265 * we also need to take into account the reg->off. 5266 * 5267 * We want to support cases like: 5268 * 5269 * struct foo { 5270 * struct bar br; 5271 * struct baz bz; 5272 * }; 5273 * 5274 * struct foo *v; 5275 * v = func(); // PTR_TO_BTF_ID 5276 * val->foo = v; // reg->off is zero, btf and btf_id match type 5277 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 5278 * // first member type of struct after comparison fails 5279 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 5280 * // to match type 5281 * 5282 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 5283 * is zero. We must also ensure that btf_struct_ids_match does not walk 5284 * the struct to match type against first member of struct, i.e. reject 5285 * second case from above. Hence, when type is BPF_KPTR_REF, we set 5286 * strict mode to true for type match. 5287 */ 5288 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 5289 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 5290 kptr_field->type != BPF_KPTR_UNREF)) 5291 goto bad_type; 5292 return 0; 5293 bad_type: 5294 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 5295 reg_type_str(env, reg->type), reg_name); 5296 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 5297 if (kptr_field->type == BPF_KPTR_UNREF) 5298 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 5299 targ_name); 5300 else 5301 verbose(env, "\n"); 5302 return -EINVAL; 5303 } 5304 5305 static bool in_sleepable(struct bpf_verifier_env *env) 5306 { 5307 return env->prog->sleepable || 5308 (env->cur_state && env->cur_state->in_sleepable); 5309 } 5310 5311 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() 5312 * can dereference RCU protected pointers and result is PTR_TRUSTED. 5313 */ 5314 static bool in_rcu_cs(struct bpf_verifier_env *env) 5315 { 5316 return env->cur_state->active_rcu_lock || 5317 env->cur_state->active_lock.ptr || 5318 !in_sleepable(env); 5319 } 5320 5321 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */ 5322 BTF_SET_START(rcu_protected_types) 5323 BTF_ID(struct, prog_test_ref_kfunc) 5324 #ifdef CONFIG_CGROUPS 5325 BTF_ID(struct, cgroup) 5326 #endif 5327 #ifdef CONFIG_BPF_JIT 5328 BTF_ID(struct, bpf_cpumask) 5329 #endif 5330 BTF_ID(struct, task_struct) 5331 BTF_ID(struct, bpf_crypto_ctx) 5332 BTF_SET_END(rcu_protected_types) 5333 5334 static bool rcu_protected_object(const struct btf *btf, u32 btf_id) 5335 { 5336 if (!btf_is_kernel(btf)) 5337 return true; 5338 return btf_id_set_contains(&rcu_protected_types, btf_id); 5339 } 5340 5341 static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field) 5342 { 5343 struct btf_struct_meta *meta; 5344 5345 if (btf_is_kernel(kptr_field->kptr.btf)) 5346 return NULL; 5347 5348 meta = btf_find_struct_meta(kptr_field->kptr.btf, 5349 kptr_field->kptr.btf_id); 5350 5351 return meta ? meta->record : NULL; 5352 } 5353 5354 static bool rcu_safe_kptr(const struct btf_field *field) 5355 { 5356 const struct btf_field_kptr *kptr = &field->kptr; 5357 5358 return field->type == BPF_KPTR_PERCPU || 5359 (field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id)); 5360 } 5361 5362 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field) 5363 { 5364 struct btf_record *rec; 5365 u32 ret; 5366 5367 ret = PTR_MAYBE_NULL; 5368 if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { 5369 ret |= MEM_RCU; 5370 if (kptr_field->type == BPF_KPTR_PERCPU) 5371 ret |= MEM_PERCPU; 5372 else if (!btf_is_kernel(kptr_field->kptr.btf)) 5373 ret |= MEM_ALLOC; 5374 5375 rec = kptr_pointee_btf_record(kptr_field); 5376 if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE)) 5377 ret |= NON_OWN_REF; 5378 } else { 5379 ret |= PTR_UNTRUSTED; 5380 } 5381 5382 return ret; 5383 } 5384 5385 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 5386 int value_regno, int insn_idx, 5387 struct btf_field *kptr_field) 5388 { 5389 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 5390 int class = BPF_CLASS(insn->code); 5391 struct bpf_reg_state *val_reg; 5392 5393 /* Things we already checked for in check_map_access and caller: 5394 * - Reject cases where variable offset may touch kptr 5395 * - size of access (must be BPF_DW) 5396 * - tnum_is_const(reg->var_off) 5397 * - kptr_field->offset == off + reg->var_off.value 5398 */ 5399 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 5400 if (BPF_MODE(insn->code) != BPF_MEM) { 5401 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 5402 return -EACCES; 5403 } 5404 5405 /* We only allow loading referenced kptr, since it will be marked as 5406 * untrusted, similar to unreferenced kptr. 5407 */ 5408 if (class != BPF_LDX && 5409 (kptr_field->type == BPF_KPTR_REF || kptr_field->type == BPF_KPTR_PERCPU)) { 5410 verbose(env, "store to referenced kptr disallowed\n"); 5411 return -EACCES; 5412 } 5413 5414 if (class == BPF_LDX) { 5415 val_reg = reg_state(env, value_regno); 5416 /* We can simply mark the value_regno receiving the pointer 5417 * value from map as PTR_TO_BTF_ID, with the correct type. 5418 */ 5419 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 5420 kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field)); 5421 } else if (class == BPF_STX) { 5422 val_reg = reg_state(env, value_regno); 5423 if (!register_is_null(val_reg) && 5424 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 5425 return -EACCES; 5426 } else if (class == BPF_ST) { 5427 if (insn->imm) { 5428 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 5429 kptr_field->offset); 5430 return -EACCES; 5431 } 5432 } else { 5433 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 5434 return -EACCES; 5435 } 5436 return 0; 5437 } 5438 5439 /* check read/write into a map element with possible variable offset */ 5440 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 5441 int off, int size, bool zero_size_allowed, 5442 enum bpf_access_src src) 5443 { 5444 struct bpf_verifier_state *vstate = env->cur_state; 5445 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 5446 struct bpf_reg_state *reg = &state->regs[regno]; 5447 struct bpf_map *map = reg->map_ptr; 5448 struct btf_record *rec; 5449 int err, i; 5450 5451 err = check_mem_region_access(env, regno, off, size, map->value_size, 5452 zero_size_allowed); 5453 if (err) 5454 return err; 5455 5456 if (IS_ERR_OR_NULL(map->record)) 5457 return 0; 5458 rec = map->record; 5459 for (i = 0; i < rec->cnt; i++) { 5460 struct btf_field *field = &rec->fields[i]; 5461 u32 p = field->offset; 5462 5463 /* If any part of a field can be touched by load/store, reject 5464 * this program. To check that [x1, x2) overlaps with [y1, y2), 5465 * it is sufficient to check x1 < y2 && y1 < x2. 5466 */ 5467 if (reg->smin_value + off < p + field->size && 5468 p < reg->umax_value + off + size) { 5469 switch (field->type) { 5470 case BPF_KPTR_UNREF: 5471 case BPF_KPTR_REF: 5472 case BPF_KPTR_PERCPU: 5473 if (src != ACCESS_DIRECT) { 5474 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 5475 return -EACCES; 5476 } 5477 if (!tnum_is_const(reg->var_off)) { 5478 verbose(env, "kptr access cannot have variable offset\n"); 5479 return -EACCES; 5480 } 5481 if (p != off + reg->var_off.value) { 5482 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 5483 p, off + reg->var_off.value); 5484 return -EACCES; 5485 } 5486 if (size != bpf_size_to_bytes(BPF_DW)) { 5487 verbose(env, "kptr access size must be BPF_DW\n"); 5488 return -EACCES; 5489 } 5490 break; 5491 default: 5492 verbose(env, "%s cannot be accessed directly by load/store\n", 5493 btf_field_type_name(field->type)); 5494 return -EACCES; 5495 } 5496 } 5497 } 5498 return 0; 5499 } 5500 5501 #define MAX_PACKET_OFF 0xffff 5502 5503 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 5504 const struct bpf_call_arg_meta *meta, 5505 enum bpf_access_type t) 5506 { 5507 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 5508 5509 switch (prog_type) { 5510 /* Program types only with direct read access go here! */ 5511 case BPF_PROG_TYPE_LWT_IN: 5512 case BPF_PROG_TYPE_LWT_OUT: 5513 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 5514 case BPF_PROG_TYPE_SK_REUSEPORT: 5515 case BPF_PROG_TYPE_FLOW_DISSECTOR: 5516 case BPF_PROG_TYPE_CGROUP_SKB: 5517 if (t == BPF_WRITE) 5518 return false; 5519 fallthrough; 5520 5521 /* Program types with direct read + write access go here! */ 5522 case BPF_PROG_TYPE_SCHED_CLS: 5523 case BPF_PROG_TYPE_SCHED_ACT: 5524 case BPF_PROG_TYPE_XDP: 5525 case BPF_PROG_TYPE_LWT_XMIT: 5526 case BPF_PROG_TYPE_SK_SKB: 5527 case BPF_PROG_TYPE_SK_MSG: 5528 if (meta) 5529 return meta->pkt_access; 5530 5531 env->seen_direct_write = true; 5532 return true; 5533 5534 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 5535 if (t == BPF_WRITE) 5536 env->seen_direct_write = true; 5537 5538 return true; 5539 5540 default: 5541 return false; 5542 } 5543 } 5544 5545 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 5546 int size, bool zero_size_allowed) 5547 { 5548 struct bpf_reg_state *regs = cur_regs(env); 5549 struct bpf_reg_state *reg = ®s[regno]; 5550 int err; 5551 5552 /* We may have added a variable offset to the packet pointer; but any 5553 * reg->range we have comes after that. We are only checking the fixed 5554 * offset. 5555 */ 5556 5557 /* We don't allow negative numbers, because we aren't tracking enough 5558 * detail to prove they're safe. 5559 */ 5560 if (reg->smin_value < 0) { 5561 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 5562 regno); 5563 return -EACCES; 5564 } 5565 5566 err = reg->range < 0 ? -EINVAL : 5567 __check_mem_access(env, regno, off, size, reg->range, 5568 zero_size_allowed); 5569 if (err) { 5570 verbose(env, "R%d offset is outside of the packet\n", regno); 5571 return err; 5572 } 5573 5574 /* __check_mem_access has made sure "off + size - 1" is within u16. 5575 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 5576 * otherwise find_good_pkt_pointers would have refused to set range info 5577 * that __check_mem_access would have rejected this pkt access. 5578 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 5579 */ 5580 env->prog->aux->max_pkt_offset = 5581 max_t(u32, env->prog->aux->max_pkt_offset, 5582 off + reg->umax_value + size - 1); 5583 5584 return err; 5585 } 5586 5587 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 5588 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 5589 enum bpf_access_type t, enum bpf_reg_type *reg_type, 5590 struct btf **btf, u32 *btf_id) 5591 { 5592 struct bpf_insn_access_aux info = { 5593 .reg_type = *reg_type, 5594 .log = &env->log, 5595 }; 5596 5597 if (env->ops->is_valid_access && 5598 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 5599 /* A non zero info.ctx_field_size indicates that this field is a 5600 * candidate for later verifier transformation to load the whole 5601 * field and then apply a mask when accessed with a narrower 5602 * access than actual ctx access size. A zero info.ctx_field_size 5603 * will only allow for whole field access and rejects any other 5604 * type of narrower access. 5605 */ 5606 *reg_type = info.reg_type; 5607 5608 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 5609 *btf = info.btf; 5610 *btf_id = info.btf_id; 5611 } else { 5612 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 5613 } 5614 /* remember the offset of last byte accessed in ctx */ 5615 if (env->prog->aux->max_ctx_offset < off + size) 5616 env->prog->aux->max_ctx_offset = off + size; 5617 return 0; 5618 } 5619 5620 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 5621 return -EACCES; 5622 } 5623 5624 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 5625 int size) 5626 { 5627 if (size < 0 || off < 0 || 5628 (u64)off + size > sizeof(struct bpf_flow_keys)) { 5629 verbose(env, "invalid access to flow keys off=%d size=%d\n", 5630 off, size); 5631 return -EACCES; 5632 } 5633 return 0; 5634 } 5635 5636 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 5637 u32 regno, int off, int size, 5638 enum bpf_access_type t) 5639 { 5640 struct bpf_reg_state *regs = cur_regs(env); 5641 struct bpf_reg_state *reg = ®s[regno]; 5642 struct bpf_insn_access_aux info = {}; 5643 bool valid; 5644 5645 if (reg->smin_value < 0) { 5646 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 5647 regno); 5648 return -EACCES; 5649 } 5650 5651 switch (reg->type) { 5652 case PTR_TO_SOCK_COMMON: 5653 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 5654 break; 5655 case PTR_TO_SOCKET: 5656 valid = bpf_sock_is_valid_access(off, size, t, &info); 5657 break; 5658 case PTR_TO_TCP_SOCK: 5659 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 5660 break; 5661 case PTR_TO_XDP_SOCK: 5662 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 5663 break; 5664 default: 5665 valid = false; 5666 } 5667 5668 5669 if (valid) { 5670 env->insn_aux_data[insn_idx].ctx_field_size = 5671 info.ctx_field_size; 5672 return 0; 5673 } 5674 5675 verbose(env, "R%d invalid %s access off=%d size=%d\n", 5676 regno, reg_type_str(env, reg->type), off, size); 5677 5678 return -EACCES; 5679 } 5680 5681 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 5682 { 5683 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 5684 } 5685 5686 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 5687 { 5688 const struct bpf_reg_state *reg = reg_state(env, regno); 5689 5690 return reg->type == PTR_TO_CTX; 5691 } 5692 5693 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 5694 { 5695 const struct bpf_reg_state *reg = reg_state(env, regno); 5696 5697 return type_is_sk_pointer(reg->type); 5698 } 5699 5700 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 5701 { 5702 const struct bpf_reg_state *reg = reg_state(env, regno); 5703 5704 return type_is_pkt_pointer(reg->type); 5705 } 5706 5707 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 5708 { 5709 const struct bpf_reg_state *reg = reg_state(env, regno); 5710 5711 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 5712 return reg->type == PTR_TO_FLOW_KEYS; 5713 } 5714 5715 static bool is_arena_reg(struct bpf_verifier_env *env, int regno) 5716 { 5717 const struct bpf_reg_state *reg = reg_state(env, regno); 5718 5719 return reg->type == PTR_TO_ARENA; 5720 } 5721 5722 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 5723 #ifdef CONFIG_NET 5724 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 5725 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 5726 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 5727 #endif 5728 [CONST_PTR_TO_MAP] = btf_bpf_map_id, 5729 }; 5730 5731 static bool is_trusted_reg(const struct bpf_reg_state *reg) 5732 { 5733 /* A referenced register is always trusted. */ 5734 if (reg->ref_obj_id) 5735 return true; 5736 5737 /* Types listed in the reg2btf_ids are always trusted */ 5738 if (reg2btf_ids[base_type(reg->type)] && 5739 !bpf_type_has_unsafe_modifiers(reg->type)) 5740 return true; 5741 5742 /* If a register is not referenced, it is trusted if it has the 5743 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 5744 * other type modifiers may be safe, but we elect to take an opt-in 5745 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 5746 * not. 5747 * 5748 * Eventually, we should make PTR_TRUSTED the single source of truth 5749 * for whether a register is trusted. 5750 */ 5751 return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && 5752 !bpf_type_has_unsafe_modifiers(reg->type); 5753 } 5754 5755 static bool is_rcu_reg(const struct bpf_reg_state *reg) 5756 { 5757 return reg->type & MEM_RCU; 5758 } 5759 5760 static void clear_trusted_flags(enum bpf_type_flag *flag) 5761 { 5762 *flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU); 5763 } 5764 5765 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 5766 const struct bpf_reg_state *reg, 5767 int off, int size, bool strict) 5768 { 5769 struct tnum reg_off; 5770 int ip_align; 5771 5772 /* Byte size accesses are always allowed. */ 5773 if (!strict || size == 1) 5774 return 0; 5775 5776 /* For platforms that do not have a Kconfig enabling 5777 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 5778 * NET_IP_ALIGN is universally set to '2'. And on platforms 5779 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 5780 * to this code only in strict mode where we want to emulate 5781 * the NET_IP_ALIGN==2 checking. Therefore use an 5782 * unconditional IP align value of '2'. 5783 */ 5784 ip_align = 2; 5785 5786 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 5787 if (!tnum_is_aligned(reg_off, size)) { 5788 char tn_buf[48]; 5789 5790 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5791 verbose(env, 5792 "misaligned packet access off %d+%s+%d+%d size %d\n", 5793 ip_align, tn_buf, reg->off, off, size); 5794 return -EACCES; 5795 } 5796 5797 return 0; 5798 } 5799 5800 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 5801 const struct bpf_reg_state *reg, 5802 const char *pointer_desc, 5803 int off, int size, bool strict) 5804 { 5805 struct tnum reg_off; 5806 5807 /* Byte size accesses are always allowed. */ 5808 if (!strict || size == 1) 5809 return 0; 5810 5811 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 5812 if (!tnum_is_aligned(reg_off, size)) { 5813 char tn_buf[48]; 5814 5815 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5816 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 5817 pointer_desc, tn_buf, reg->off, off, size); 5818 return -EACCES; 5819 } 5820 5821 return 0; 5822 } 5823 5824 static int check_ptr_alignment(struct bpf_verifier_env *env, 5825 const struct bpf_reg_state *reg, int off, 5826 int size, bool strict_alignment_once) 5827 { 5828 bool strict = env->strict_alignment || strict_alignment_once; 5829 const char *pointer_desc = ""; 5830 5831 switch (reg->type) { 5832 case PTR_TO_PACKET: 5833 case PTR_TO_PACKET_META: 5834 /* Special case, because of NET_IP_ALIGN. Given metadata sits 5835 * right in front, treat it the very same way. 5836 */ 5837 return check_pkt_ptr_alignment(env, reg, off, size, strict); 5838 case PTR_TO_FLOW_KEYS: 5839 pointer_desc = "flow keys "; 5840 break; 5841 case PTR_TO_MAP_KEY: 5842 pointer_desc = "key "; 5843 break; 5844 case PTR_TO_MAP_VALUE: 5845 pointer_desc = "value "; 5846 break; 5847 case PTR_TO_CTX: 5848 pointer_desc = "context "; 5849 break; 5850 case PTR_TO_STACK: 5851 pointer_desc = "stack "; 5852 /* The stack spill tracking logic in check_stack_write_fixed_off() 5853 * and check_stack_read_fixed_off() relies on stack accesses being 5854 * aligned. 5855 */ 5856 strict = true; 5857 break; 5858 case PTR_TO_SOCKET: 5859 pointer_desc = "sock "; 5860 break; 5861 case PTR_TO_SOCK_COMMON: 5862 pointer_desc = "sock_common "; 5863 break; 5864 case PTR_TO_TCP_SOCK: 5865 pointer_desc = "tcp_sock "; 5866 break; 5867 case PTR_TO_XDP_SOCK: 5868 pointer_desc = "xdp_sock "; 5869 break; 5870 case PTR_TO_ARENA: 5871 return 0; 5872 default: 5873 break; 5874 } 5875 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 5876 strict); 5877 } 5878 5879 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) 5880 { 5881 if (env->prog->jit_requested) 5882 return round_up(stack_depth, 16); 5883 5884 /* round up to 32-bytes, since this is granularity 5885 * of interpreter stack size 5886 */ 5887 return round_up(max_t(u32, stack_depth, 1), 32); 5888 } 5889 5890 /* starting from main bpf function walk all instructions of the function 5891 * and recursively walk all callees that given function can call. 5892 * Ignore jump and exit insns. 5893 * Since recursion is prevented by check_cfg() this algorithm 5894 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 5895 */ 5896 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx) 5897 { 5898 struct bpf_subprog_info *subprog = env->subprog_info; 5899 struct bpf_insn *insn = env->prog->insnsi; 5900 int depth = 0, frame = 0, i, subprog_end; 5901 bool tail_call_reachable = false; 5902 int ret_insn[MAX_CALL_FRAMES]; 5903 int ret_prog[MAX_CALL_FRAMES]; 5904 int j; 5905 5906 i = subprog[idx].start; 5907 process_func: 5908 /* protect against potential stack overflow that might happen when 5909 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 5910 * depth for such case down to 256 so that the worst case scenario 5911 * would result in 8k stack size (32 which is tailcall limit * 256 = 5912 * 8k). 5913 * 5914 * To get the idea what might happen, see an example: 5915 * func1 -> sub rsp, 128 5916 * subfunc1 -> sub rsp, 256 5917 * tailcall1 -> add rsp, 256 5918 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 5919 * subfunc2 -> sub rsp, 64 5920 * subfunc22 -> sub rsp, 128 5921 * tailcall2 -> add rsp, 128 5922 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 5923 * 5924 * tailcall will unwind the current stack frame but it will not get rid 5925 * of caller's stack as shown on the example above. 5926 */ 5927 if (idx && subprog[idx].has_tail_call && depth >= 256) { 5928 verbose(env, 5929 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 5930 depth); 5931 return -EACCES; 5932 } 5933 depth += round_up_stack_depth(env, subprog[idx].stack_depth); 5934 if (depth > MAX_BPF_STACK) { 5935 verbose(env, "combined stack size of %d calls is %d. Too large\n", 5936 frame + 1, depth); 5937 return -EACCES; 5938 } 5939 continue_func: 5940 subprog_end = subprog[idx + 1].start; 5941 for (; i < subprog_end; i++) { 5942 int next_insn, sidx; 5943 5944 if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { 5945 bool err = false; 5946 5947 if (!is_bpf_throw_kfunc(insn + i)) 5948 continue; 5949 if (subprog[idx].is_cb) 5950 err = true; 5951 for (int c = 0; c < frame && !err; c++) { 5952 if (subprog[ret_prog[c]].is_cb) { 5953 err = true; 5954 break; 5955 } 5956 } 5957 if (!err) 5958 continue; 5959 verbose(env, 5960 "bpf_throw kfunc (insn %d) cannot be called from callback subprog %d\n", 5961 i, idx); 5962 return -EINVAL; 5963 } 5964 5965 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 5966 continue; 5967 /* remember insn and function to return to */ 5968 ret_insn[frame] = i + 1; 5969 ret_prog[frame] = idx; 5970 5971 /* find the callee */ 5972 next_insn = i + insn[i].imm + 1; 5973 sidx = find_subprog(env, next_insn); 5974 if (sidx < 0) { 5975 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 5976 next_insn); 5977 return -EFAULT; 5978 } 5979 if (subprog[sidx].is_async_cb) { 5980 if (subprog[sidx].has_tail_call) { 5981 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 5982 return -EFAULT; 5983 } 5984 /* async callbacks don't increase bpf prog stack size unless called directly */ 5985 if (!bpf_pseudo_call(insn + i)) 5986 continue; 5987 if (subprog[sidx].is_exception_cb) { 5988 verbose(env, "insn %d cannot call exception cb directly\n", i); 5989 return -EINVAL; 5990 } 5991 } 5992 i = next_insn; 5993 idx = sidx; 5994 5995 if (subprog[idx].has_tail_call) 5996 tail_call_reachable = true; 5997 5998 frame++; 5999 if (frame >= MAX_CALL_FRAMES) { 6000 verbose(env, "the call stack of %d frames is too deep !\n", 6001 frame); 6002 return -E2BIG; 6003 } 6004 goto process_func; 6005 } 6006 /* if tail call got detected across bpf2bpf calls then mark each of the 6007 * currently present subprog frames as tail call reachable subprogs; 6008 * this info will be utilized by JIT so that we will be preserving the 6009 * tail call counter throughout bpf2bpf calls combined with tailcalls 6010 */ 6011 if (tail_call_reachable) 6012 for (j = 0; j < frame; j++) { 6013 if (subprog[ret_prog[j]].is_exception_cb) { 6014 verbose(env, "cannot tail call within exception cb\n"); 6015 return -EINVAL; 6016 } 6017 subprog[ret_prog[j]].tail_call_reachable = true; 6018 } 6019 if (subprog[0].tail_call_reachable) 6020 env->prog->aux->tail_call_reachable = true; 6021 6022 /* end of for() loop means the last insn of the 'subprog' 6023 * was reached. Doesn't matter whether it was JA or EXIT 6024 */ 6025 if (frame == 0) 6026 return 0; 6027 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); 6028 frame--; 6029 i = ret_insn[frame]; 6030 idx = ret_prog[frame]; 6031 goto continue_func; 6032 } 6033 6034 static int check_max_stack_depth(struct bpf_verifier_env *env) 6035 { 6036 struct bpf_subprog_info *si = env->subprog_info; 6037 int ret; 6038 6039 for (int i = 0; i < env->subprog_cnt; i++) { 6040 if (!i || si[i].is_async_cb) { 6041 ret = check_max_stack_depth_subprog(env, i); 6042 if (ret < 0) 6043 return ret; 6044 } 6045 continue; 6046 } 6047 return 0; 6048 } 6049 6050 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 6051 static int get_callee_stack_depth(struct bpf_verifier_env *env, 6052 const struct bpf_insn *insn, int idx) 6053 { 6054 int start = idx + insn->imm + 1, subprog; 6055 6056 subprog = find_subprog(env, start); 6057 if (subprog < 0) { 6058 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 6059 start); 6060 return -EFAULT; 6061 } 6062 return env->subprog_info[subprog].stack_depth; 6063 } 6064 #endif 6065 6066 static int __check_buffer_access(struct bpf_verifier_env *env, 6067 const char *buf_info, 6068 const struct bpf_reg_state *reg, 6069 int regno, int off, int size) 6070 { 6071 if (off < 0) { 6072 verbose(env, 6073 "R%d invalid %s buffer access: off=%d, size=%d\n", 6074 regno, buf_info, off, size); 6075 return -EACCES; 6076 } 6077 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 6078 char tn_buf[48]; 6079 6080 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6081 verbose(env, 6082 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 6083 regno, off, tn_buf); 6084 return -EACCES; 6085 } 6086 6087 return 0; 6088 } 6089 6090 static int check_tp_buffer_access(struct bpf_verifier_env *env, 6091 const struct bpf_reg_state *reg, 6092 int regno, int off, int size) 6093 { 6094 int err; 6095 6096 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 6097 if (err) 6098 return err; 6099 6100 if (off + size > env->prog->aux->max_tp_access) 6101 env->prog->aux->max_tp_access = off + size; 6102 6103 return 0; 6104 } 6105 6106 static int check_buffer_access(struct bpf_verifier_env *env, 6107 const struct bpf_reg_state *reg, 6108 int regno, int off, int size, 6109 bool zero_size_allowed, 6110 u32 *max_access) 6111 { 6112 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 6113 int err; 6114 6115 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 6116 if (err) 6117 return err; 6118 6119 if (off + size > *max_access) 6120 *max_access = off + size; 6121 6122 return 0; 6123 } 6124 6125 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 6126 static void zext_32_to_64(struct bpf_reg_state *reg) 6127 { 6128 reg->var_off = tnum_subreg(reg->var_off); 6129 __reg_assign_32_into_64(reg); 6130 } 6131 6132 /* truncate register to smaller size (in bytes) 6133 * must be called with size < BPF_REG_SIZE 6134 */ 6135 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 6136 { 6137 u64 mask; 6138 6139 /* clear high bits in bit representation */ 6140 reg->var_off = tnum_cast(reg->var_off, size); 6141 6142 /* fix arithmetic bounds */ 6143 mask = ((u64)1 << (size * 8)) - 1; 6144 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 6145 reg->umin_value &= mask; 6146 reg->umax_value &= mask; 6147 } else { 6148 reg->umin_value = 0; 6149 reg->umax_value = mask; 6150 } 6151 reg->smin_value = reg->umin_value; 6152 reg->smax_value = reg->umax_value; 6153 6154 /* If size is smaller than 32bit register the 32bit register 6155 * values are also truncated so we push 64-bit bounds into 6156 * 32-bit bounds. Above were truncated < 32-bits already. 6157 */ 6158 if (size < 4) 6159 __mark_reg32_unbounded(reg); 6160 6161 reg_bounds_sync(reg); 6162 } 6163 6164 static void set_sext64_default_val(struct bpf_reg_state *reg, int size) 6165 { 6166 if (size == 1) { 6167 reg->smin_value = reg->s32_min_value = S8_MIN; 6168 reg->smax_value = reg->s32_max_value = S8_MAX; 6169 } else if (size == 2) { 6170 reg->smin_value = reg->s32_min_value = S16_MIN; 6171 reg->smax_value = reg->s32_max_value = S16_MAX; 6172 } else { 6173 /* size == 4 */ 6174 reg->smin_value = reg->s32_min_value = S32_MIN; 6175 reg->smax_value = reg->s32_max_value = S32_MAX; 6176 } 6177 reg->umin_value = reg->u32_min_value = 0; 6178 reg->umax_value = U64_MAX; 6179 reg->u32_max_value = U32_MAX; 6180 reg->var_off = tnum_unknown; 6181 } 6182 6183 static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) 6184 { 6185 s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; 6186 u64 top_smax_value, top_smin_value; 6187 u64 num_bits = size * 8; 6188 6189 if (tnum_is_const(reg->var_off)) { 6190 u64_cval = reg->var_off.value; 6191 if (size == 1) 6192 reg->var_off = tnum_const((s8)u64_cval); 6193 else if (size == 2) 6194 reg->var_off = tnum_const((s16)u64_cval); 6195 else 6196 /* size == 4 */ 6197 reg->var_off = tnum_const((s32)u64_cval); 6198 6199 u64_cval = reg->var_off.value; 6200 reg->smax_value = reg->smin_value = u64_cval; 6201 reg->umax_value = reg->umin_value = u64_cval; 6202 reg->s32_max_value = reg->s32_min_value = u64_cval; 6203 reg->u32_max_value = reg->u32_min_value = u64_cval; 6204 return; 6205 } 6206 6207 top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits; 6208 top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits; 6209 6210 if (top_smax_value != top_smin_value) 6211 goto out; 6212 6213 /* find the s64_min and s64_min after sign extension */ 6214 if (size == 1) { 6215 init_s64_max = (s8)reg->smax_value; 6216 init_s64_min = (s8)reg->smin_value; 6217 } else if (size == 2) { 6218 init_s64_max = (s16)reg->smax_value; 6219 init_s64_min = (s16)reg->smin_value; 6220 } else { 6221 init_s64_max = (s32)reg->smax_value; 6222 init_s64_min = (s32)reg->smin_value; 6223 } 6224 6225 s64_max = max(init_s64_max, init_s64_min); 6226 s64_min = min(init_s64_max, init_s64_min); 6227 6228 /* both of s64_max/s64_min positive or negative */ 6229 if ((s64_max >= 0) == (s64_min >= 0)) { 6230 reg->smin_value = reg->s32_min_value = s64_min; 6231 reg->smax_value = reg->s32_max_value = s64_max; 6232 reg->umin_value = reg->u32_min_value = s64_min; 6233 reg->umax_value = reg->u32_max_value = s64_max; 6234 reg->var_off = tnum_range(s64_min, s64_max); 6235 return; 6236 } 6237 6238 out: 6239 set_sext64_default_val(reg, size); 6240 } 6241 6242 static void set_sext32_default_val(struct bpf_reg_state *reg, int size) 6243 { 6244 if (size == 1) { 6245 reg->s32_min_value = S8_MIN; 6246 reg->s32_max_value = S8_MAX; 6247 } else { 6248 /* size == 2 */ 6249 reg->s32_min_value = S16_MIN; 6250 reg->s32_max_value = S16_MAX; 6251 } 6252 reg->u32_min_value = 0; 6253 reg->u32_max_value = U32_MAX; 6254 reg->var_off = tnum_subreg(tnum_unknown); 6255 } 6256 6257 static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) 6258 { 6259 s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val; 6260 u32 top_smax_value, top_smin_value; 6261 u32 num_bits = size * 8; 6262 6263 if (tnum_is_const(reg->var_off)) { 6264 u32_val = reg->var_off.value; 6265 if (size == 1) 6266 reg->var_off = tnum_const((s8)u32_val); 6267 else 6268 reg->var_off = tnum_const((s16)u32_val); 6269 6270 u32_val = reg->var_off.value; 6271 reg->s32_min_value = reg->s32_max_value = u32_val; 6272 reg->u32_min_value = reg->u32_max_value = u32_val; 6273 return; 6274 } 6275 6276 top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits; 6277 top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits; 6278 6279 if (top_smax_value != top_smin_value) 6280 goto out; 6281 6282 /* find the s32_min and s32_min after sign extension */ 6283 if (size == 1) { 6284 init_s32_max = (s8)reg->s32_max_value; 6285 init_s32_min = (s8)reg->s32_min_value; 6286 } else { 6287 /* size == 2 */ 6288 init_s32_max = (s16)reg->s32_max_value; 6289 init_s32_min = (s16)reg->s32_min_value; 6290 } 6291 s32_max = max(init_s32_max, init_s32_min); 6292 s32_min = min(init_s32_max, init_s32_min); 6293 6294 if ((s32_min >= 0) == (s32_max >= 0)) { 6295 reg->s32_min_value = s32_min; 6296 reg->s32_max_value = s32_max; 6297 reg->u32_min_value = (u32)s32_min; 6298 reg->u32_max_value = (u32)s32_max; 6299 reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max)); 6300 return; 6301 } 6302 6303 out: 6304 set_sext32_default_val(reg, size); 6305 } 6306 6307 static bool bpf_map_is_rdonly(const struct bpf_map *map) 6308 { 6309 /* A map is considered read-only if the following condition are true: 6310 * 6311 * 1) BPF program side cannot change any of the map content. The 6312 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 6313 * and was set at map creation time. 6314 * 2) The map value(s) have been initialized from user space by a 6315 * loader and then "frozen", such that no new map update/delete 6316 * operations from syscall side are possible for the rest of 6317 * the map's lifetime from that point onwards. 6318 * 3) Any parallel/pending map update/delete operations from syscall 6319 * side have been completed. Only after that point, it's safe to 6320 * assume that map value(s) are immutable. 6321 */ 6322 return (map->map_flags & BPF_F_RDONLY_PROG) && 6323 READ_ONCE(map->frozen) && 6324 !bpf_map_write_active(map); 6325 } 6326 6327 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, 6328 bool is_ldsx) 6329 { 6330 void *ptr; 6331 u64 addr; 6332 int err; 6333 6334 err = map->ops->map_direct_value_addr(map, &addr, off); 6335 if (err) 6336 return err; 6337 ptr = (void *)(long)addr + off; 6338 6339 switch (size) { 6340 case sizeof(u8): 6341 *val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr; 6342 break; 6343 case sizeof(u16): 6344 *val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr; 6345 break; 6346 case sizeof(u32): 6347 *val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr; 6348 break; 6349 case sizeof(u64): 6350 *val = *(u64 *)ptr; 6351 break; 6352 default: 6353 return -EINVAL; 6354 } 6355 return 0; 6356 } 6357 6358 #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) 6359 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) 6360 #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) 6361 #define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type) __PASTE(__type, __safe_trusted_or_null) 6362 6363 /* 6364 * Allow list few fields as RCU trusted or full trusted. 6365 * This logic doesn't allow mix tagging and will be removed once GCC supports 6366 * btf_type_tag. 6367 */ 6368 6369 /* RCU trusted: these fields are trusted in RCU CS and never NULL */ 6370 BTF_TYPE_SAFE_RCU(struct task_struct) { 6371 const cpumask_t *cpus_ptr; 6372 struct css_set __rcu *cgroups; 6373 struct task_struct __rcu *real_parent; 6374 struct task_struct *group_leader; 6375 }; 6376 6377 BTF_TYPE_SAFE_RCU(struct cgroup) { 6378 /* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */ 6379 struct kernfs_node *kn; 6380 }; 6381 6382 BTF_TYPE_SAFE_RCU(struct css_set) { 6383 struct cgroup *dfl_cgrp; 6384 }; 6385 6386 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */ 6387 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) { 6388 struct file __rcu *exe_file; 6389 }; 6390 6391 /* skb->sk, req->sk are not RCU protected, but we mark them as such 6392 * because bpf prog accessible sockets are SOCK_RCU_FREE. 6393 */ 6394 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) { 6395 struct sock *sk; 6396 }; 6397 6398 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) { 6399 struct sock *sk; 6400 }; 6401 6402 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */ 6403 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) { 6404 struct seq_file *seq; 6405 }; 6406 6407 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) { 6408 struct bpf_iter_meta *meta; 6409 struct task_struct *task; 6410 }; 6411 6412 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) { 6413 struct file *file; 6414 }; 6415 6416 BTF_TYPE_SAFE_TRUSTED(struct file) { 6417 struct inode *f_inode; 6418 }; 6419 6420 BTF_TYPE_SAFE_TRUSTED(struct dentry) { 6421 /* no negative dentry-s in places where bpf can see it */ 6422 struct inode *d_inode; 6423 }; 6424 6425 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { 6426 struct sock *sk; 6427 }; 6428 6429 static bool type_is_rcu(struct bpf_verifier_env *env, 6430 struct bpf_reg_state *reg, 6431 const char *field_name, u32 btf_id) 6432 { 6433 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct)); 6434 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup)); 6435 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set)); 6436 6437 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); 6438 } 6439 6440 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, 6441 struct bpf_reg_state *reg, 6442 const char *field_name, u32 btf_id) 6443 { 6444 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct)); 6445 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff)); 6446 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock)); 6447 6448 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); 6449 } 6450 6451 static bool type_is_trusted(struct bpf_verifier_env *env, 6452 struct bpf_reg_state *reg, 6453 const char *field_name, u32 btf_id) 6454 { 6455 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta)); 6456 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); 6457 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); 6458 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); 6459 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct dentry)); 6460 6461 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); 6462 } 6463 6464 static bool type_is_trusted_or_null(struct bpf_verifier_env *env, 6465 struct bpf_reg_state *reg, 6466 const char *field_name, u32 btf_id) 6467 { 6468 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); 6469 6470 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, 6471 "__safe_trusted_or_null"); 6472 } 6473 6474 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 6475 struct bpf_reg_state *regs, 6476 int regno, int off, int size, 6477 enum bpf_access_type atype, 6478 int value_regno) 6479 { 6480 struct bpf_reg_state *reg = regs + regno; 6481 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 6482 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 6483 const char *field_name = NULL; 6484 enum bpf_type_flag flag = 0; 6485 u32 btf_id = 0; 6486 int ret; 6487 6488 if (!env->allow_ptr_leaks) { 6489 verbose(env, 6490 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 6491 tname); 6492 return -EPERM; 6493 } 6494 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 6495 verbose(env, 6496 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 6497 tname); 6498 return -EINVAL; 6499 } 6500 if (off < 0) { 6501 verbose(env, 6502 "R%d is ptr_%s invalid negative access: off=%d\n", 6503 regno, tname, off); 6504 return -EACCES; 6505 } 6506 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 6507 char tn_buf[48]; 6508 6509 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6510 verbose(env, 6511 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 6512 regno, tname, off, tn_buf); 6513 return -EACCES; 6514 } 6515 6516 if (reg->type & MEM_USER) { 6517 verbose(env, 6518 "R%d is ptr_%s access user memory: off=%d\n", 6519 regno, tname, off); 6520 return -EACCES; 6521 } 6522 6523 if (reg->type & MEM_PERCPU) { 6524 verbose(env, 6525 "R%d is ptr_%s access percpu memory: off=%d\n", 6526 regno, tname, off); 6527 return -EACCES; 6528 } 6529 6530 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { 6531 if (!btf_is_kernel(reg->btf)) { 6532 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); 6533 return -EFAULT; 6534 } 6535 ret = env->ops->btf_struct_access(&env->log, reg, off, size); 6536 } else { 6537 /* Writes are permitted with default btf_struct_access for 6538 * program allocated objects (which always have ref_obj_id > 0), 6539 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. 6540 */ 6541 if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { 6542 verbose(env, "only read is supported\n"); 6543 return -EACCES; 6544 } 6545 6546 if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && 6547 !(reg->type & MEM_RCU) && !reg->ref_obj_id) { 6548 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 6549 return -EFAULT; 6550 } 6551 6552 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); 6553 } 6554 6555 if (ret < 0) 6556 return ret; 6557 6558 if (ret != PTR_TO_BTF_ID) { 6559 /* just mark; */ 6560 6561 } else if (type_flag(reg->type) & PTR_UNTRUSTED) { 6562 /* If this is an untrusted pointer, all pointers formed by walking it 6563 * also inherit the untrusted flag. 6564 */ 6565 flag = PTR_UNTRUSTED; 6566 6567 } else if (is_trusted_reg(reg) || is_rcu_reg(reg)) { 6568 /* By default any pointer obtained from walking a trusted pointer is no 6569 * longer trusted, unless the field being accessed has explicitly been 6570 * marked as inheriting its parent's state of trust (either full or RCU). 6571 * For example: 6572 * 'cgroups' pointer is untrusted if task->cgroups dereference 6573 * happened in a sleepable program outside of bpf_rcu_read_lock() 6574 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU). 6575 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED. 6576 * 6577 * A regular RCU-protected pointer with __rcu tag can also be deemed 6578 * trusted if we are in an RCU CS. Such pointer can be NULL. 6579 */ 6580 if (type_is_trusted(env, reg, field_name, btf_id)) { 6581 flag |= PTR_TRUSTED; 6582 } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) { 6583 flag |= PTR_TRUSTED | PTR_MAYBE_NULL; 6584 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { 6585 if (type_is_rcu(env, reg, field_name, btf_id)) { 6586 /* ignore __rcu tag and mark it MEM_RCU */ 6587 flag |= MEM_RCU; 6588 } else if (flag & MEM_RCU || 6589 type_is_rcu_or_null(env, reg, field_name, btf_id)) { 6590 /* __rcu tagged pointers can be NULL */ 6591 flag |= MEM_RCU | PTR_MAYBE_NULL; 6592 6593 /* We always trust them */ 6594 if (type_is_rcu_or_null(env, reg, field_name, btf_id) && 6595 flag & PTR_UNTRUSTED) 6596 flag &= ~PTR_UNTRUSTED; 6597 } else if (flag & (MEM_PERCPU | MEM_USER)) { 6598 /* keep as-is */ 6599 } else { 6600 /* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */ 6601 clear_trusted_flags(&flag); 6602 } 6603 } else { 6604 /* 6605 * If not in RCU CS or MEM_RCU pointer can be NULL then 6606 * aggressively mark as untrusted otherwise such 6607 * pointers will be plain PTR_TO_BTF_ID without flags 6608 * and will be allowed to be passed into helpers for 6609 * compat reasons. 6610 */ 6611 flag = PTR_UNTRUSTED; 6612 } 6613 } else { 6614 /* Old compat. Deprecated */ 6615 clear_trusted_flags(&flag); 6616 } 6617 6618 if (atype == BPF_READ && value_regno >= 0) 6619 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 6620 6621 return 0; 6622 } 6623 6624 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 6625 struct bpf_reg_state *regs, 6626 int regno, int off, int size, 6627 enum bpf_access_type atype, 6628 int value_regno) 6629 { 6630 struct bpf_reg_state *reg = regs + regno; 6631 struct bpf_map *map = reg->map_ptr; 6632 struct bpf_reg_state map_reg; 6633 enum bpf_type_flag flag = 0; 6634 const struct btf_type *t; 6635 const char *tname; 6636 u32 btf_id; 6637 int ret; 6638 6639 if (!btf_vmlinux) { 6640 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 6641 return -ENOTSUPP; 6642 } 6643 6644 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 6645 verbose(env, "map_ptr access not supported for map type %d\n", 6646 map->map_type); 6647 return -ENOTSUPP; 6648 } 6649 6650 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 6651 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 6652 6653 if (!env->allow_ptr_leaks) { 6654 verbose(env, 6655 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 6656 tname); 6657 return -EPERM; 6658 } 6659 6660 if (off < 0) { 6661 verbose(env, "R%d is %s invalid negative access: off=%d\n", 6662 regno, tname, off); 6663 return -EACCES; 6664 } 6665 6666 if (atype != BPF_READ) { 6667 verbose(env, "only read from %s is supported\n", tname); 6668 return -EACCES; 6669 } 6670 6671 /* Simulate access to a PTR_TO_BTF_ID */ 6672 memset(&map_reg, 0, sizeof(map_reg)); 6673 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); 6674 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); 6675 if (ret < 0) 6676 return ret; 6677 6678 if (value_regno >= 0) 6679 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 6680 6681 return 0; 6682 } 6683 6684 /* Check that the stack access at the given offset is within bounds. The 6685 * maximum valid offset is -1. 6686 * 6687 * The minimum valid offset is -MAX_BPF_STACK for writes, and 6688 * -state->allocated_stack for reads. 6689 */ 6690 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, 6691 s64 off, 6692 struct bpf_func_state *state, 6693 enum bpf_access_type t) 6694 { 6695 int min_valid_off; 6696 6697 if (t == BPF_WRITE || env->allow_uninit_stack) 6698 min_valid_off = -MAX_BPF_STACK; 6699 else 6700 min_valid_off = -state->allocated_stack; 6701 6702 if (off < min_valid_off || off > -1) 6703 return -EACCES; 6704 return 0; 6705 } 6706 6707 /* Check that the stack access at 'regno + off' falls within the maximum stack 6708 * bounds. 6709 * 6710 * 'off' includes `regno->offset`, but not its dynamic part (if any). 6711 */ 6712 static int check_stack_access_within_bounds( 6713 struct bpf_verifier_env *env, 6714 int regno, int off, int access_size, 6715 enum bpf_access_src src, enum bpf_access_type type) 6716 { 6717 struct bpf_reg_state *regs = cur_regs(env); 6718 struct bpf_reg_state *reg = regs + regno; 6719 struct bpf_func_state *state = func(env, reg); 6720 s64 min_off, max_off; 6721 int err; 6722 char *err_extra; 6723 6724 if (src == ACCESS_HELPER) 6725 /* We don't know if helpers are reading or writing (or both). */ 6726 err_extra = " indirect access to"; 6727 else if (type == BPF_READ) 6728 err_extra = " read from"; 6729 else 6730 err_extra = " write to"; 6731 6732 if (tnum_is_const(reg->var_off)) { 6733 min_off = (s64)reg->var_off.value + off; 6734 max_off = min_off + access_size; 6735 } else { 6736 if (reg->smax_value >= BPF_MAX_VAR_OFF || 6737 reg->smin_value <= -BPF_MAX_VAR_OFF) { 6738 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 6739 err_extra, regno); 6740 return -EACCES; 6741 } 6742 min_off = reg->smin_value + off; 6743 max_off = reg->smax_value + off + access_size; 6744 } 6745 6746 err = check_stack_slot_within_bounds(env, min_off, state, type); 6747 if (!err && max_off > 0) 6748 err = -EINVAL; /* out of stack access into non-negative offsets */ 6749 if (!err && access_size < 0) 6750 /* access_size should not be negative (or overflow an int); others checks 6751 * along the way should have prevented such an access. 6752 */ 6753 err = -EFAULT; /* invalid negative access size; integer overflow? */ 6754 6755 if (err) { 6756 if (tnum_is_const(reg->var_off)) { 6757 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 6758 err_extra, regno, off, access_size); 6759 } else { 6760 char tn_buf[48]; 6761 6762 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6763 verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n", 6764 err_extra, regno, tn_buf, off, access_size); 6765 } 6766 return err; 6767 } 6768 6769 /* Note that there is no stack access with offset zero, so the needed stack 6770 * size is -min_off, not -min_off+1. 6771 */ 6772 return grow_stack_state(env, state, -min_off /* size */); 6773 } 6774 6775 /* check whether memory at (regno + off) is accessible for t = (read | write) 6776 * if t==write, value_regno is a register which value is stored into memory 6777 * if t==read, value_regno is a register which will receive the value from memory 6778 * if t==write && value_regno==-1, some unknown value is stored into memory 6779 * if t==read && value_regno==-1, don't care what we read from memory 6780 */ 6781 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 6782 int off, int bpf_size, enum bpf_access_type t, 6783 int value_regno, bool strict_alignment_once, bool is_ldsx) 6784 { 6785 struct bpf_reg_state *regs = cur_regs(env); 6786 struct bpf_reg_state *reg = regs + regno; 6787 int size, err = 0; 6788 6789 size = bpf_size_to_bytes(bpf_size); 6790 if (size < 0) 6791 return size; 6792 6793 /* alignment checks will add in reg->off themselves */ 6794 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 6795 if (err) 6796 return err; 6797 6798 /* for access checks, reg->off is just part of off */ 6799 off += reg->off; 6800 6801 if (reg->type == PTR_TO_MAP_KEY) { 6802 if (t == BPF_WRITE) { 6803 verbose(env, "write to change key R%d not allowed\n", regno); 6804 return -EACCES; 6805 } 6806 6807 err = check_mem_region_access(env, regno, off, size, 6808 reg->map_ptr->key_size, false); 6809 if (err) 6810 return err; 6811 if (value_regno >= 0) 6812 mark_reg_unknown(env, regs, value_regno); 6813 } else if (reg->type == PTR_TO_MAP_VALUE) { 6814 struct btf_field *kptr_field = NULL; 6815 6816 if (t == BPF_WRITE && value_regno >= 0 && 6817 is_pointer_value(env, value_regno)) { 6818 verbose(env, "R%d leaks addr into map\n", value_regno); 6819 return -EACCES; 6820 } 6821 err = check_map_access_type(env, regno, off, size, t); 6822 if (err) 6823 return err; 6824 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 6825 if (err) 6826 return err; 6827 if (tnum_is_const(reg->var_off)) 6828 kptr_field = btf_record_find(reg->map_ptr->record, 6829 off + reg->var_off.value, BPF_KPTR); 6830 if (kptr_field) { 6831 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 6832 } else if (t == BPF_READ && value_regno >= 0) { 6833 struct bpf_map *map = reg->map_ptr; 6834 6835 /* if map is read-only, track its contents as scalars */ 6836 if (tnum_is_const(reg->var_off) && 6837 bpf_map_is_rdonly(map) && 6838 map->ops->map_direct_value_addr) { 6839 int map_off = off + reg->var_off.value; 6840 u64 val = 0; 6841 6842 err = bpf_map_direct_read(map, map_off, size, 6843 &val, is_ldsx); 6844 if (err) 6845 return err; 6846 6847 regs[value_regno].type = SCALAR_VALUE; 6848 __mark_reg_known(®s[value_regno], val); 6849 } else { 6850 mark_reg_unknown(env, regs, value_regno); 6851 } 6852 } 6853 } else if (base_type(reg->type) == PTR_TO_MEM) { 6854 bool rdonly_mem = type_is_rdonly_mem(reg->type); 6855 6856 if (type_may_be_null(reg->type)) { 6857 verbose(env, "R%d invalid mem access '%s'\n", regno, 6858 reg_type_str(env, reg->type)); 6859 return -EACCES; 6860 } 6861 6862 if (t == BPF_WRITE && rdonly_mem) { 6863 verbose(env, "R%d cannot write into %s\n", 6864 regno, reg_type_str(env, reg->type)); 6865 return -EACCES; 6866 } 6867 6868 if (t == BPF_WRITE && value_regno >= 0 && 6869 is_pointer_value(env, value_regno)) { 6870 verbose(env, "R%d leaks addr into mem\n", value_regno); 6871 return -EACCES; 6872 } 6873 6874 err = check_mem_region_access(env, regno, off, size, 6875 reg->mem_size, false); 6876 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 6877 mark_reg_unknown(env, regs, value_regno); 6878 } else if (reg->type == PTR_TO_CTX) { 6879 enum bpf_reg_type reg_type = SCALAR_VALUE; 6880 struct btf *btf = NULL; 6881 u32 btf_id = 0; 6882 6883 if (t == BPF_WRITE && value_regno >= 0 && 6884 is_pointer_value(env, value_regno)) { 6885 verbose(env, "R%d leaks addr into ctx\n", value_regno); 6886 return -EACCES; 6887 } 6888 6889 err = check_ptr_off_reg(env, reg, regno); 6890 if (err < 0) 6891 return err; 6892 6893 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 6894 &btf_id); 6895 if (err) 6896 verbose_linfo(env, insn_idx, "; "); 6897 if (!err && t == BPF_READ && value_regno >= 0) { 6898 /* ctx access returns either a scalar, or a 6899 * PTR_TO_PACKET[_META,_END]. In the latter 6900 * case, we know the offset is zero. 6901 */ 6902 if (reg_type == SCALAR_VALUE) { 6903 mark_reg_unknown(env, regs, value_regno); 6904 } else { 6905 mark_reg_known_zero(env, regs, 6906 value_regno); 6907 if (type_may_be_null(reg_type)) 6908 regs[value_regno].id = ++env->id_gen; 6909 /* A load of ctx field could have different 6910 * actual load size with the one encoded in the 6911 * insn. When the dst is PTR, it is for sure not 6912 * a sub-register. 6913 */ 6914 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 6915 if (base_type(reg_type) == PTR_TO_BTF_ID) { 6916 regs[value_regno].btf = btf; 6917 regs[value_regno].btf_id = btf_id; 6918 } 6919 } 6920 regs[value_regno].type = reg_type; 6921 } 6922 6923 } else if (reg->type == PTR_TO_STACK) { 6924 /* Basic bounds checks. */ 6925 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 6926 if (err) 6927 return err; 6928 6929 if (t == BPF_READ) 6930 err = check_stack_read(env, regno, off, size, 6931 value_regno); 6932 else 6933 err = check_stack_write(env, regno, off, size, 6934 value_regno, insn_idx); 6935 } else if (reg_is_pkt_pointer(reg)) { 6936 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 6937 verbose(env, "cannot write into packet\n"); 6938 return -EACCES; 6939 } 6940 if (t == BPF_WRITE && value_regno >= 0 && 6941 is_pointer_value(env, value_regno)) { 6942 verbose(env, "R%d leaks addr into packet\n", 6943 value_regno); 6944 return -EACCES; 6945 } 6946 err = check_packet_access(env, regno, off, size, false); 6947 if (!err && t == BPF_READ && value_regno >= 0) 6948 mark_reg_unknown(env, regs, value_regno); 6949 } else if (reg->type == PTR_TO_FLOW_KEYS) { 6950 if (t == BPF_WRITE && value_regno >= 0 && 6951 is_pointer_value(env, value_regno)) { 6952 verbose(env, "R%d leaks addr into flow keys\n", 6953 value_regno); 6954 return -EACCES; 6955 } 6956 6957 err = check_flow_keys_access(env, off, size); 6958 if (!err && t == BPF_READ && value_regno >= 0) 6959 mark_reg_unknown(env, regs, value_regno); 6960 } else if (type_is_sk_pointer(reg->type)) { 6961 if (t == BPF_WRITE) { 6962 verbose(env, "R%d cannot write into %s\n", 6963 regno, reg_type_str(env, reg->type)); 6964 return -EACCES; 6965 } 6966 err = check_sock_access(env, insn_idx, regno, off, size, t); 6967 if (!err && value_regno >= 0) 6968 mark_reg_unknown(env, regs, value_regno); 6969 } else if (reg->type == PTR_TO_TP_BUFFER) { 6970 err = check_tp_buffer_access(env, reg, regno, off, size); 6971 if (!err && t == BPF_READ && value_regno >= 0) 6972 mark_reg_unknown(env, regs, value_regno); 6973 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 6974 !type_may_be_null(reg->type)) { 6975 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 6976 value_regno); 6977 } else if (reg->type == CONST_PTR_TO_MAP) { 6978 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 6979 value_regno); 6980 } else if (base_type(reg->type) == PTR_TO_BUF) { 6981 bool rdonly_mem = type_is_rdonly_mem(reg->type); 6982 u32 *max_access; 6983 6984 if (rdonly_mem) { 6985 if (t == BPF_WRITE) { 6986 verbose(env, "R%d cannot write into %s\n", 6987 regno, reg_type_str(env, reg->type)); 6988 return -EACCES; 6989 } 6990 max_access = &env->prog->aux->max_rdonly_access; 6991 } else { 6992 max_access = &env->prog->aux->max_rdwr_access; 6993 } 6994 6995 err = check_buffer_access(env, reg, regno, off, size, false, 6996 max_access); 6997 6998 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 6999 mark_reg_unknown(env, regs, value_regno); 7000 } else if (reg->type == PTR_TO_ARENA) { 7001 if (t == BPF_READ && value_regno >= 0) 7002 mark_reg_unknown(env, regs, value_regno); 7003 } else { 7004 verbose(env, "R%d invalid mem access '%s'\n", regno, 7005 reg_type_str(env, reg->type)); 7006 return -EACCES; 7007 } 7008 7009 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 7010 regs[value_regno].type == SCALAR_VALUE) { 7011 if (!is_ldsx) 7012 /* b/h/w load zero-extends, mark upper bits as known 0 */ 7013 coerce_reg_to_size(®s[value_regno], size); 7014 else 7015 coerce_reg_to_size_sx(®s[value_regno], size); 7016 } 7017 return err; 7018 } 7019 7020 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, 7021 bool allow_trust_mismatch); 7022 7023 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 7024 { 7025 int load_reg; 7026 int err; 7027 7028 switch (insn->imm) { 7029 case BPF_ADD: 7030 case BPF_ADD | BPF_FETCH: 7031 case BPF_AND: 7032 case BPF_AND | BPF_FETCH: 7033 case BPF_OR: 7034 case BPF_OR | BPF_FETCH: 7035 case BPF_XOR: 7036 case BPF_XOR | BPF_FETCH: 7037 case BPF_XCHG: 7038 case BPF_CMPXCHG: 7039 break; 7040 default: 7041 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 7042 return -EINVAL; 7043 } 7044 7045 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 7046 verbose(env, "invalid atomic operand size\n"); 7047 return -EINVAL; 7048 } 7049 7050 /* check src1 operand */ 7051 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7052 if (err) 7053 return err; 7054 7055 /* check src2 operand */ 7056 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7057 if (err) 7058 return err; 7059 7060 if (insn->imm == BPF_CMPXCHG) { 7061 /* Check comparison of R0 with memory location */ 7062 const u32 aux_reg = BPF_REG_0; 7063 7064 err = check_reg_arg(env, aux_reg, SRC_OP); 7065 if (err) 7066 return err; 7067 7068 if (is_pointer_value(env, aux_reg)) { 7069 verbose(env, "R%d leaks addr into mem\n", aux_reg); 7070 return -EACCES; 7071 } 7072 } 7073 7074 if (is_pointer_value(env, insn->src_reg)) { 7075 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 7076 return -EACCES; 7077 } 7078 7079 if (is_ctx_reg(env, insn->dst_reg) || 7080 is_pkt_reg(env, insn->dst_reg) || 7081 is_flow_key_reg(env, insn->dst_reg) || 7082 is_sk_reg(env, insn->dst_reg) || 7083 (is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) { 7084 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 7085 insn->dst_reg, 7086 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 7087 return -EACCES; 7088 } 7089 7090 if (insn->imm & BPF_FETCH) { 7091 if (insn->imm == BPF_CMPXCHG) 7092 load_reg = BPF_REG_0; 7093 else 7094 load_reg = insn->src_reg; 7095 7096 /* check and record load of old value */ 7097 err = check_reg_arg(env, load_reg, DST_OP); 7098 if (err) 7099 return err; 7100 } else { 7101 /* This instruction accesses a memory location but doesn't 7102 * actually load it into a register. 7103 */ 7104 load_reg = -1; 7105 } 7106 7107 /* Check whether we can read the memory, with second call for fetch 7108 * case to simulate the register fill. 7109 */ 7110 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 7111 BPF_SIZE(insn->code), BPF_READ, -1, true, false); 7112 if (!err && load_reg >= 0) 7113 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 7114 BPF_SIZE(insn->code), BPF_READ, load_reg, 7115 true, false); 7116 if (err) 7117 return err; 7118 7119 if (is_arena_reg(env, insn->dst_reg)) { 7120 err = save_aux_ptr_type(env, PTR_TO_ARENA, false); 7121 if (err) 7122 return err; 7123 } 7124 /* Check whether we can write into the same memory. */ 7125 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 7126 BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); 7127 if (err) 7128 return err; 7129 return 0; 7130 } 7131 7132 /* When register 'regno' is used to read the stack (either directly or through 7133 * a helper function) make sure that it's within stack boundary and, depending 7134 * on the access type and privileges, that all elements of the stack are 7135 * initialized. 7136 * 7137 * 'off' includes 'regno->off', but not its dynamic part (if any). 7138 * 7139 * All registers that have been spilled on the stack in the slots within the 7140 * read offsets are marked as read. 7141 */ 7142 static int check_stack_range_initialized( 7143 struct bpf_verifier_env *env, int regno, int off, 7144 int access_size, bool zero_size_allowed, 7145 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 7146 { 7147 struct bpf_reg_state *reg = reg_state(env, regno); 7148 struct bpf_func_state *state = func(env, reg); 7149 int err, min_off, max_off, i, j, slot, spi; 7150 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 7151 enum bpf_access_type bounds_check_type; 7152 /* Some accesses can write anything into the stack, others are 7153 * read-only. 7154 */ 7155 bool clobber = false; 7156 7157 if (access_size == 0 && !zero_size_allowed) { 7158 verbose(env, "invalid zero-sized read\n"); 7159 return -EACCES; 7160 } 7161 7162 if (type == ACCESS_HELPER) { 7163 /* The bounds checks for writes are more permissive than for 7164 * reads. However, if raw_mode is not set, we'll do extra 7165 * checks below. 7166 */ 7167 bounds_check_type = BPF_WRITE; 7168 clobber = true; 7169 } else { 7170 bounds_check_type = BPF_READ; 7171 } 7172 err = check_stack_access_within_bounds(env, regno, off, access_size, 7173 type, bounds_check_type); 7174 if (err) 7175 return err; 7176 7177 7178 if (tnum_is_const(reg->var_off)) { 7179 min_off = max_off = reg->var_off.value + off; 7180 } else { 7181 /* Variable offset is prohibited for unprivileged mode for 7182 * simplicity since it requires corresponding support in 7183 * Spectre masking for stack ALU. 7184 * See also retrieve_ptr_limit(). 7185 */ 7186 if (!env->bypass_spec_v1) { 7187 char tn_buf[48]; 7188 7189 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7190 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 7191 regno, err_extra, tn_buf); 7192 return -EACCES; 7193 } 7194 /* Only initialized buffer on stack is allowed to be accessed 7195 * with variable offset. With uninitialized buffer it's hard to 7196 * guarantee that whole memory is marked as initialized on 7197 * helper return since specific bounds are unknown what may 7198 * cause uninitialized stack leaking. 7199 */ 7200 if (meta && meta->raw_mode) 7201 meta = NULL; 7202 7203 min_off = reg->smin_value + off; 7204 max_off = reg->smax_value + off; 7205 } 7206 7207 if (meta && meta->raw_mode) { 7208 /* Ensure we won't be overwriting dynptrs when simulating byte 7209 * by byte access in check_helper_call using meta.access_size. 7210 * This would be a problem if we have a helper in the future 7211 * which takes: 7212 * 7213 * helper(uninit_mem, len, dynptr) 7214 * 7215 * Now, uninint_mem may overlap with dynptr pointer. Hence, it 7216 * may end up writing to dynptr itself when touching memory from 7217 * arg 1. This can be relaxed on a case by case basis for known 7218 * safe cases, but reject due to the possibilitiy of aliasing by 7219 * default. 7220 */ 7221 for (i = min_off; i < max_off + access_size; i++) { 7222 int stack_off = -i - 1; 7223 7224 spi = __get_spi(i); 7225 /* raw_mode may write past allocated_stack */ 7226 if (state->allocated_stack <= stack_off) 7227 continue; 7228 if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { 7229 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); 7230 return -EACCES; 7231 } 7232 } 7233 meta->access_size = access_size; 7234 meta->regno = regno; 7235 return 0; 7236 } 7237 7238 for (i = min_off; i < max_off + access_size; i++) { 7239 u8 *stype; 7240 7241 slot = -i - 1; 7242 spi = slot / BPF_REG_SIZE; 7243 if (state->allocated_stack <= slot) { 7244 verbose(env, "verifier bug: allocated_stack too small"); 7245 return -EFAULT; 7246 } 7247 7248 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 7249 if (*stype == STACK_MISC) 7250 goto mark; 7251 if ((*stype == STACK_ZERO) || 7252 (*stype == STACK_INVALID && env->allow_uninit_stack)) { 7253 if (clobber) { 7254 /* helper can write anything into the stack */ 7255 *stype = STACK_MISC; 7256 } 7257 goto mark; 7258 } 7259 7260 if (is_spilled_reg(&state->stack[spi]) && 7261 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 7262 env->allow_ptr_leaks)) { 7263 if (clobber) { 7264 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 7265 for (j = 0; j < BPF_REG_SIZE; j++) 7266 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 7267 } 7268 goto mark; 7269 } 7270 7271 if (tnum_is_const(reg->var_off)) { 7272 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 7273 err_extra, regno, min_off, i - min_off, access_size); 7274 } else { 7275 char tn_buf[48]; 7276 7277 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7278 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 7279 err_extra, regno, tn_buf, i - min_off, access_size); 7280 } 7281 return -EACCES; 7282 mark: 7283 /* reading any byte out of 8-byte 'spill_slot' will cause 7284 * the whole slot to be marked as 'read' 7285 */ 7286 mark_reg_read(env, &state->stack[spi].spilled_ptr, 7287 state->stack[spi].spilled_ptr.parent, 7288 REG_LIVE_READ64); 7289 /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not 7290 * be sure that whether stack slot is written to or not. Hence, 7291 * we must still conservatively propagate reads upwards even if 7292 * helper may write to the entire memory range. 7293 */ 7294 } 7295 return 0; 7296 } 7297 7298 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 7299 int access_size, bool zero_size_allowed, 7300 struct bpf_call_arg_meta *meta) 7301 { 7302 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7303 u32 *max_access; 7304 7305 switch (base_type(reg->type)) { 7306 case PTR_TO_PACKET: 7307 case PTR_TO_PACKET_META: 7308 return check_packet_access(env, regno, reg->off, access_size, 7309 zero_size_allowed); 7310 case PTR_TO_MAP_KEY: 7311 if (meta && meta->raw_mode) { 7312 verbose(env, "R%d cannot write into %s\n", regno, 7313 reg_type_str(env, reg->type)); 7314 return -EACCES; 7315 } 7316 return check_mem_region_access(env, regno, reg->off, access_size, 7317 reg->map_ptr->key_size, false); 7318 case PTR_TO_MAP_VALUE: 7319 if (check_map_access_type(env, regno, reg->off, access_size, 7320 meta && meta->raw_mode ? BPF_WRITE : 7321 BPF_READ)) 7322 return -EACCES; 7323 return check_map_access(env, regno, reg->off, access_size, 7324 zero_size_allowed, ACCESS_HELPER); 7325 case PTR_TO_MEM: 7326 if (type_is_rdonly_mem(reg->type)) { 7327 if (meta && meta->raw_mode) { 7328 verbose(env, "R%d cannot write into %s\n", regno, 7329 reg_type_str(env, reg->type)); 7330 return -EACCES; 7331 } 7332 } 7333 return check_mem_region_access(env, regno, reg->off, 7334 access_size, reg->mem_size, 7335 zero_size_allowed); 7336 case PTR_TO_BUF: 7337 if (type_is_rdonly_mem(reg->type)) { 7338 if (meta && meta->raw_mode) { 7339 verbose(env, "R%d cannot write into %s\n", regno, 7340 reg_type_str(env, reg->type)); 7341 return -EACCES; 7342 } 7343 7344 max_access = &env->prog->aux->max_rdonly_access; 7345 } else { 7346 max_access = &env->prog->aux->max_rdwr_access; 7347 } 7348 return check_buffer_access(env, reg, regno, reg->off, 7349 access_size, zero_size_allowed, 7350 max_access); 7351 case PTR_TO_STACK: 7352 return check_stack_range_initialized( 7353 env, 7354 regno, reg->off, access_size, 7355 zero_size_allowed, ACCESS_HELPER, meta); 7356 case PTR_TO_BTF_ID: 7357 return check_ptr_to_btf_access(env, regs, regno, reg->off, 7358 access_size, BPF_READ, -1); 7359 case PTR_TO_CTX: 7360 /* in case the function doesn't know how to access the context, 7361 * (because we are in a program of type SYSCALL for example), we 7362 * can not statically check its size. 7363 * Dynamically check it now. 7364 */ 7365 if (!env->ops->convert_ctx_access) { 7366 enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; 7367 int offset = access_size - 1; 7368 7369 /* Allow zero-byte read from PTR_TO_CTX */ 7370 if (access_size == 0) 7371 return zero_size_allowed ? 0 : -EACCES; 7372 7373 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, 7374 atype, -1, false, false); 7375 } 7376 7377 fallthrough; 7378 default: /* scalar_value or invalid ptr */ 7379 /* Allow zero-byte read from NULL, regardless of pointer type */ 7380 if (zero_size_allowed && access_size == 0 && 7381 register_is_null(reg)) 7382 return 0; 7383 7384 verbose(env, "R%d type=%s ", regno, 7385 reg_type_str(env, reg->type)); 7386 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 7387 return -EACCES; 7388 } 7389 } 7390 7391 /* verify arguments to helpers or kfuncs consisting of a pointer and an access 7392 * size. 7393 * 7394 * @regno is the register containing the access size. regno-1 is the register 7395 * containing the pointer. 7396 */ 7397 static int check_mem_size_reg(struct bpf_verifier_env *env, 7398 struct bpf_reg_state *reg, u32 regno, 7399 bool zero_size_allowed, 7400 struct bpf_call_arg_meta *meta) 7401 { 7402 int err; 7403 7404 /* This is used to refine r0 return value bounds for helpers 7405 * that enforce this value as an upper bound on return values. 7406 * See do_refine_retval_range() for helpers that can refine 7407 * the return value. C type of helper is u32 so we pull register 7408 * bound from umax_value however, if negative verifier errors 7409 * out. Only upper bounds can be learned because retval is an 7410 * int type and negative retvals are allowed. 7411 */ 7412 meta->msize_max_value = reg->umax_value; 7413 7414 /* The register is SCALAR_VALUE; the access check 7415 * happens using its boundaries. 7416 */ 7417 if (!tnum_is_const(reg->var_off)) 7418 /* For unprivileged variable accesses, disable raw 7419 * mode so that the program is required to 7420 * initialize all the memory that the helper could 7421 * just partially fill up. 7422 */ 7423 meta = NULL; 7424 7425 if (reg->smin_value < 0) { 7426 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 7427 regno); 7428 return -EACCES; 7429 } 7430 7431 if (reg->umin_value == 0 && !zero_size_allowed) { 7432 verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n", 7433 regno, reg->umin_value, reg->umax_value); 7434 return -EACCES; 7435 } 7436 7437 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 7438 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 7439 regno); 7440 return -EACCES; 7441 } 7442 err = check_helper_mem_access(env, regno - 1, 7443 reg->umax_value, 7444 zero_size_allowed, meta); 7445 if (!err) 7446 err = mark_chain_precision(env, regno); 7447 return err; 7448 } 7449 7450 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 7451 u32 regno, u32 mem_size) 7452 { 7453 bool may_be_null = type_may_be_null(reg->type); 7454 struct bpf_reg_state saved_reg; 7455 struct bpf_call_arg_meta meta; 7456 int err; 7457 7458 if (register_is_null(reg)) 7459 return 0; 7460 7461 memset(&meta, 0, sizeof(meta)); 7462 /* Assuming that the register contains a value check if the memory 7463 * access is safe. Temporarily save and restore the register's state as 7464 * the conversion shouldn't be visible to a caller. 7465 */ 7466 if (may_be_null) { 7467 saved_reg = *reg; 7468 mark_ptr_not_null_reg(reg); 7469 } 7470 7471 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 7472 /* Check access for BPF_WRITE */ 7473 meta.raw_mode = true; 7474 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 7475 7476 if (may_be_null) 7477 *reg = saved_reg; 7478 7479 return err; 7480 } 7481 7482 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 7483 u32 regno) 7484 { 7485 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 7486 bool may_be_null = type_may_be_null(mem_reg->type); 7487 struct bpf_reg_state saved_reg; 7488 struct bpf_call_arg_meta meta; 7489 int err; 7490 7491 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 7492 7493 memset(&meta, 0, sizeof(meta)); 7494 7495 if (may_be_null) { 7496 saved_reg = *mem_reg; 7497 mark_ptr_not_null_reg(mem_reg); 7498 } 7499 7500 err = check_mem_size_reg(env, reg, regno, true, &meta); 7501 /* Check access for BPF_WRITE */ 7502 meta.raw_mode = true; 7503 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 7504 7505 if (may_be_null) 7506 *mem_reg = saved_reg; 7507 return err; 7508 } 7509 7510 /* Implementation details: 7511 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 7512 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 7513 * Two bpf_map_lookups (even with the same key) will have different reg->id. 7514 * Two separate bpf_obj_new will also have different reg->id. 7515 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 7516 * clears reg->id after value_or_null->value transition, since the verifier only 7517 * cares about the range of access to valid map value pointer and doesn't care 7518 * about actual address of the map element. 7519 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 7520 * reg->id > 0 after value_or_null->value transition. By doing so 7521 * two bpf_map_lookups will be considered two different pointers that 7522 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 7523 * returned from bpf_obj_new. 7524 * The verifier allows taking only one bpf_spin_lock at a time to avoid 7525 * dead-locks. 7526 * Since only one bpf_spin_lock is allowed the checks are simpler than 7527 * reg_is_refcounted() logic. The verifier needs to remember only 7528 * one spin_lock instead of array of acquired_refs. 7529 * cur_state->active_lock remembers which map value element or allocated 7530 * object got locked and clears it after bpf_spin_unlock. 7531 */ 7532 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 7533 bool is_lock) 7534 { 7535 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7536 struct bpf_verifier_state *cur = env->cur_state; 7537 bool is_const = tnum_is_const(reg->var_off); 7538 u64 val = reg->var_off.value; 7539 struct bpf_map *map = NULL; 7540 struct btf *btf = NULL; 7541 struct btf_record *rec; 7542 7543 if (!is_const) { 7544 verbose(env, 7545 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 7546 regno); 7547 return -EINVAL; 7548 } 7549 if (reg->type == PTR_TO_MAP_VALUE) { 7550 map = reg->map_ptr; 7551 if (!map->btf) { 7552 verbose(env, 7553 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 7554 map->name); 7555 return -EINVAL; 7556 } 7557 } else { 7558 btf = reg->btf; 7559 } 7560 7561 rec = reg_btf_record(reg); 7562 if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) { 7563 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", 7564 map ? map->name : "kptr"); 7565 return -EINVAL; 7566 } 7567 if (rec->spin_lock_off != val + reg->off) { 7568 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", 7569 val + reg->off, rec->spin_lock_off); 7570 return -EINVAL; 7571 } 7572 if (is_lock) { 7573 if (cur->active_lock.ptr) { 7574 verbose(env, 7575 "Locking two bpf_spin_locks are not allowed\n"); 7576 return -EINVAL; 7577 } 7578 if (map) 7579 cur->active_lock.ptr = map; 7580 else 7581 cur->active_lock.ptr = btf; 7582 cur->active_lock.id = reg->id; 7583 } else { 7584 void *ptr; 7585 7586 if (map) 7587 ptr = map; 7588 else 7589 ptr = btf; 7590 7591 if (!cur->active_lock.ptr) { 7592 verbose(env, "bpf_spin_unlock without taking a lock\n"); 7593 return -EINVAL; 7594 } 7595 if (cur->active_lock.ptr != ptr || 7596 cur->active_lock.id != reg->id) { 7597 verbose(env, "bpf_spin_unlock of different lock\n"); 7598 return -EINVAL; 7599 } 7600 7601 invalidate_non_owning_refs(env); 7602 7603 cur->active_lock.ptr = NULL; 7604 cur->active_lock.id = 0; 7605 } 7606 return 0; 7607 } 7608 7609 static int process_timer_func(struct bpf_verifier_env *env, int regno, 7610 struct bpf_call_arg_meta *meta) 7611 { 7612 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7613 bool is_const = tnum_is_const(reg->var_off); 7614 struct bpf_map *map = reg->map_ptr; 7615 u64 val = reg->var_off.value; 7616 7617 if (!is_const) { 7618 verbose(env, 7619 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 7620 regno); 7621 return -EINVAL; 7622 } 7623 if (!map->btf) { 7624 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 7625 map->name); 7626 return -EINVAL; 7627 } 7628 if (!btf_record_has_field(map->record, BPF_TIMER)) { 7629 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); 7630 return -EINVAL; 7631 } 7632 if (map->record->timer_off != val + reg->off) { 7633 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 7634 val + reg->off, map->record->timer_off); 7635 return -EINVAL; 7636 } 7637 if (meta->map_ptr) { 7638 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 7639 return -EFAULT; 7640 } 7641 meta->map_uid = reg->map_uid; 7642 meta->map_ptr = map; 7643 return 0; 7644 } 7645 7646 static int process_wq_func(struct bpf_verifier_env *env, int regno, 7647 struct bpf_kfunc_call_arg_meta *meta) 7648 { 7649 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7650 struct bpf_map *map = reg->map_ptr; 7651 u64 val = reg->var_off.value; 7652 7653 if (map->record->wq_off != val + reg->off) { 7654 verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n", 7655 val + reg->off, map->record->wq_off); 7656 return -EINVAL; 7657 } 7658 meta->map.uid = reg->map_uid; 7659 meta->map.ptr = map; 7660 return 0; 7661 } 7662 7663 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 7664 struct bpf_call_arg_meta *meta) 7665 { 7666 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7667 struct bpf_map *map_ptr = reg->map_ptr; 7668 struct btf_field *kptr_field; 7669 u32 kptr_off; 7670 7671 if (!tnum_is_const(reg->var_off)) { 7672 verbose(env, 7673 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 7674 regno); 7675 return -EINVAL; 7676 } 7677 if (!map_ptr->btf) { 7678 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 7679 map_ptr->name); 7680 return -EINVAL; 7681 } 7682 if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 7683 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 7684 return -EINVAL; 7685 } 7686 7687 meta->map_ptr = map_ptr; 7688 kptr_off = reg->off + reg->var_off.value; 7689 kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 7690 if (!kptr_field) { 7691 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 7692 return -EACCES; 7693 } 7694 if (kptr_field->type != BPF_KPTR_REF && kptr_field->type != BPF_KPTR_PERCPU) { 7695 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 7696 return -EACCES; 7697 } 7698 meta->kptr_field = kptr_field; 7699 return 0; 7700 } 7701 7702 /* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK 7703 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. 7704 * 7705 * In both cases we deal with the first 8 bytes, but need to mark the next 8 7706 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of 7707 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. 7708 * 7709 * Mutability of bpf_dynptr is at two levels, one is at the level of struct 7710 * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct 7711 * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can 7712 * mutate the view of the dynptr and also possibly destroy it. In the latter 7713 * case, it cannot mutate the bpf_dynptr itself but it can still mutate the 7714 * memory that dynptr points to. 7715 * 7716 * The verifier will keep track both levels of mutation (bpf_dynptr's in 7717 * reg->type and the memory's in reg->dynptr.type), but there is no support for 7718 * readonly dynptr view yet, hence only the first case is tracked and checked. 7719 * 7720 * This is consistent with how C applies the const modifier to a struct object, 7721 * where the pointer itself inside bpf_dynptr becomes const but not what it 7722 * points to. 7723 * 7724 * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument 7725 * type, and declare it as 'const struct bpf_dynptr *' in their prototype. 7726 */ 7727 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, 7728 enum bpf_arg_type arg_type, int clone_ref_obj_id) 7729 { 7730 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7731 int err; 7732 7733 if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { 7734 verbose(env, 7735 "arg#%d expected pointer to stack or const struct bpf_dynptr\n", 7736 regno); 7737 return -EINVAL; 7738 } 7739 7740 /* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an 7741 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*): 7742 */ 7743 if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) { 7744 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); 7745 return -EFAULT; 7746 } 7747 7748 /* MEM_UNINIT - Points to memory that is an appropriate candidate for 7749 * constructing a mutable bpf_dynptr object. 7750 * 7751 * Currently, this is only possible with PTR_TO_STACK 7752 * pointing to a region of at least 16 bytes which doesn't 7753 * contain an existing bpf_dynptr. 7754 * 7755 * MEM_RDONLY - Points to a initialized bpf_dynptr that will not be 7756 * mutated or destroyed. However, the memory it points to 7757 * may be mutated. 7758 * 7759 * None - Points to a initialized dynptr that can be mutated and 7760 * destroyed, including mutation of the memory it points 7761 * to. 7762 */ 7763 if (arg_type & MEM_UNINIT) { 7764 int i; 7765 7766 if (!is_dynptr_reg_valid_uninit(env, reg)) { 7767 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 7768 return -EINVAL; 7769 } 7770 7771 /* we write BPF_DW bits (8 bytes) at a time */ 7772 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 7773 err = check_mem_access(env, insn_idx, regno, 7774 i, BPF_DW, BPF_WRITE, -1, false, false); 7775 if (err) 7776 return err; 7777 } 7778 7779 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); 7780 } else /* MEM_RDONLY and None case from above */ { 7781 /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ 7782 if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) { 7783 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); 7784 return -EINVAL; 7785 } 7786 7787 if (!is_dynptr_reg_valid_init(env, reg)) { 7788 verbose(env, 7789 "Expected an initialized dynptr as arg #%d\n", 7790 regno); 7791 return -EINVAL; 7792 } 7793 7794 /* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */ 7795 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { 7796 verbose(env, 7797 "Expected a dynptr of type %s as arg #%d\n", 7798 dynptr_type_str(arg_to_dynptr_type(arg_type)), regno); 7799 return -EINVAL; 7800 } 7801 7802 err = mark_dynptr_read(env, reg); 7803 } 7804 return err; 7805 } 7806 7807 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) 7808 { 7809 struct bpf_func_state *state = func(env, reg); 7810 7811 return state->stack[spi].spilled_ptr.ref_obj_id; 7812 } 7813 7814 static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) 7815 { 7816 return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); 7817 } 7818 7819 static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) 7820 { 7821 return meta->kfunc_flags & KF_ITER_NEW; 7822 } 7823 7824 static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta) 7825 { 7826 return meta->kfunc_flags & KF_ITER_NEXT; 7827 } 7828 7829 static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) 7830 { 7831 return meta->kfunc_flags & KF_ITER_DESTROY; 7832 } 7833 7834 static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg) 7835 { 7836 /* btf_check_iter_kfuncs() guarantees that first argument of any iter 7837 * kfunc is iter state pointer 7838 */ 7839 return arg == 0 && is_iter_kfunc(meta); 7840 } 7841 7842 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, 7843 struct bpf_kfunc_call_arg_meta *meta) 7844 { 7845 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7846 const struct btf_type *t; 7847 const struct btf_param *arg; 7848 int spi, err, i, nr_slots; 7849 u32 btf_id; 7850 7851 /* btf_check_iter_kfuncs() ensures we don't need to validate anything here */ 7852 arg = &btf_params(meta->func_proto)[0]; 7853 t = btf_type_skip_modifiers(meta->btf, arg->type, NULL); /* PTR */ 7854 t = btf_type_skip_modifiers(meta->btf, t->type, &btf_id); /* STRUCT */ 7855 nr_slots = t->size / BPF_REG_SIZE; 7856 7857 if (is_iter_new_kfunc(meta)) { 7858 /* bpf_iter_<type>_new() expects pointer to uninit iter state */ 7859 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { 7860 verbose(env, "expected uninitialized iter_%s as arg #%d\n", 7861 iter_type_str(meta->btf, btf_id), regno); 7862 return -EINVAL; 7863 } 7864 7865 for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { 7866 err = check_mem_access(env, insn_idx, regno, 7867 i, BPF_DW, BPF_WRITE, -1, false, false); 7868 if (err) 7869 return err; 7870 } 7871 7872 err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots); 7873 if (err) 7874 return err; 7875 } else { 7876 /* iter_next() or iter_destroy() expect initialized iter state*/ 7877 err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots); 7878 switch (err) { 7879 case 0: 7880 break; 7881 case -EINVAL: 7882 verbose(env, "expected an initialized iter_%s as arg #%d\n", 7883 iter_type_str(meta->btf, btf_id), regno); 7884 return err; 7885 case -EPROTO: 7886 verbose(env, "expected an RCU CS when using %s\n", meta->func_name); 7887 return err; 7888 default: 7889 return err; 7890 } 7891 7892 spi = iter_get_spi(env, reg, nr_slots); 7893 if (spi < 0) 7894 return spi; 7895 7896 err = mark_iter_read(env, reg, spi, nr_slots); 7897 if (err) 7898 return err; 7899 7900 /* remember meta->iter info for process_iter_next_call() */ 7901 meta->iter.spi = spi; 7902 meta->iter.frameno = reg->frameno; 7903 meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); 7904 7905 if (is_iter_destroy_kfunc(meta)) { 7906 err = unmark_stack_slots_iter(env, reg, nr_slots); 7907 if (err) 7908 return err; 7909 } 7910 } 7911 7912 return 0; 7913 } 7914 7915 /* Look for a previous loop entry at insn_idx: nearest parent state 7916 * stopped at insn_idx with callsites matching those in cur->frame. 7917 */ 7918 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env, 7919 struct bpf_verifier_state *cur, 7920 int insn_idx) 7921 { 7922 struct bpf_verifier_state_list *sl; 7923 struct bpf_verifier_state *st; 7924 7925 /* Explored states are pushed in stack order, most recent states come first */ 7926 sl = *explored_state(env, insn_idx); 7927 for (; sl; sl = sl->next) { 7928 /* If st->branches != 0 state is a part of current DFS verification path, 7929 * hence cur & st for a loop. 7930 */ 7931 st = &sl->state; 7932 if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) && 7933 st->dfs_depth < cur->dfs_depth) 7934 return st; 7935 } 7936 7937 return NULL; 7938 } 7939 7940 static void reset_idmap_scratch(struct bpf_verifier_env *env); 7941 static bool regs_exact(const struct bpf_reg_state *rold, 7942 const struct bpf_reg_state *rcur, 7943 struct bpf_idmap *idmap); 7944 7945 static void maybe_widen_reg(struct bpf_verifier_env *env, 7946 struct bpf_reg_state *rold, struct bpf_reg_state *rcur, 7947 struct bpf_idmap *idmap) 7948 { 7949 if (rold->type != SCALAR_VALUE) 7950 return; 7951 if (rold->type != rcur->type) 7952 return; 7953 if (rold->precise || rcur->precise || regs_exact(rold, rcur, idmap)) 7954 return; 7955 __mark_reg_unknown(env, rcur); 7956 } 7957 7958 static int widen_imprecise_scalars(struct bpf_verifier_env *env, 7959 struct bpf_verifier_state *old, 7960 struct bpf_verifier_state *cur) 7961 { 7962 struct bpf_func_state *fold, *fcur; 7963 int i, fr; 7964 7965 reset_idmap_scratch(env); 7966 for (fr = old->curframe; fr >= 0; fr--) { 7967 fold = old->frame[fr]; 7968 fcur = cur->frame[fr]; 7969 7970 for (i = 0; i < MAX_BPF_REG; i++) 7971 maybe_widen_reg(env, 7972 &fold->regs[i], 7973 &fcur->regs[i], 7974 &env->idmap_scratch); 7975 7976 for (i = 0; i < fold->allocated_stack / BPF_REG_SIZE; i++) { 7977 if (!is_spilled_reg(&fold->stack[i]) || 7978 !is_spilled_reg(&fcur->stack[i])) 7979 continue; 7980 7981 maybe_widen_reg(env, 7982 &fold->stack[i].spilled_ptr, 7983 &fcur->stack[i].spilled_ptr, 7984 &env->idmap_scratch); 7985 } 7986 } 7987 return 0; 7988 } 7989 7990 /* process_iter_next_call() is called when verifier gets to iterator's next 7991 * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer 7992 * to it as just "iter_next()" in comments below. 7993 * 7994 * BPF verifier relies on a crucial contract for any iter_next() 7995 * implementation: it should *eventually* return NULL, and once that happens 7996 * it should keep returning NULL. That is, once iterator exhausts elements to 7997 * iterate, it should never reset or spuriously return new elements. 7998 * 7999 * With the assumption of such contract, process_iter_next_call() simulates 8000 * a fork in the verifier state to validate loop logic correctness and safety 8001 * without having to simulate infinite amount of iterations. 8002 * 8003 * In current state, we first assume that iter_next() returned NULL and 8004 * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such 8005 * conditions we should not form an infinite loop and should eventually reach 8006 * exit. 8007 * 8008 * Besides that, we also fork current state and enqueue it for later 8009 * verification. In a forked state we keep iterator state as ACTIVE 8010 * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We 8011 * also bump iteration depth to prevent erroneous infinite loop detection 8012 * later on (see iter_active_depths_differ() comment for details). In this 8013 * state we assume that we'll eventually loop back to another iter_next() 8014 * calls (it could be in exactly same location or in some other instruction, 8015 * it doesn't matter, we don't make any unnecessary assumptions about this, 8016 * everything revolves around iterator state in a stack slot, not which 8017 * instruction is calling iter_next()). When that happens, we either will come 8018 * to iter_next() with equivalent state and can conclude that next iteration 8019 * will proceed in exactly the same way as we just verified, so it's safe to 8020 * assume that loop converges. If not, we'll go on another iteration 8021 * simulation with a different input state, until all possible starting states 8022 * are validated or we reach maximum number of instructions limit. 8023 * 8024 * This way, we will either exhaustively discover all possible input states 8025 * that iterator loop can start with and eventually will converge, or we'll 8026 * effectively regress into bounded loop simulation logic and either reach 8027 * maximum number of instructions if loop is not provably convergent, or there 8028 * is some statically known limit on number of iterations (e.g., if there is 8029 * an explicit `if n > 100 then break;` statement somewhere in the loop). 8030 * 8031 * Iteration convergence logic in is_state_visited() relies on exact 8032 * states comparison, which ignores read and precision marks. 8033 * This is necessary because read and precision marks are not finalized 8034 * while in the loop. Exact comparison might preclude convergence for 8035 * simple programs like below: 8036 * 8037 * i = 0; 8038 * while(iter_next(&it)) 8039 * i++; 8040 * 8041 * At each iteration step i++ would produce a new distinct state and 8042 * eventually instruction processing limit would be reached. 8043 * 8044 * To avoid such behavior speculatively forget (widen) range for 8045 * imprecise scalar registers, if those registers were not precise at the 8046 * end of the previous iteration and do not match exactly. 8047 * 8048 * This is a conservative heuristic that allows to verify wide range of programs, 8049 * however it precludes verification of programs that conjure an 8050 * imprecise value on the first loop iteration and use it as precise on a second. 8051 * For example, the following safe program would fail to verify: 8052 * 8053 * struct bpf_num_iter it; 8054 * int arr[10]; 8055 * int i = 0, a = 0; 8056 * bpf_iter_num_new(&it, 0, 10); 8057 * while (bpf_iter_num_next(&it)) { 8058 * if (a == 0) { 8059 * a = 1; 8060 * i = 7; // Because i changed verifier would forget 8061 * // it's range on second loop entry. 8062 * } else { 8063 * arr[i] = 42; // This would fail to verify. 8064 * } 8065 * } 8066 * bpf_iter_num_destroy(&it); 8067 */ 8068 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, 8069 struct bpf_kfunc_call_arg_meta *meta) 8070 { 8071 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; 8072 struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; 8073 struct bpf_reg_state *cur_iter, *queued_iter; 8074 int iter_frameno = meta->iter.frameno; 8075 int iter_spi = meta->iter.spi; 8076 8077 BTF_TYPE_EMIT(struct bpf_iter); 8078 8079 cur_iter = &env->cur_state->frame[iter_frameno]->stack[iter_spi].spilled_ptr; 8080 8081 if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE && 8082 cur_iter->iter.state != BPF_ITER_STATE_DRAINED) { 8083 verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n", 8084 cur_iter->iter.state, iter_state_str(cur_iter->iter.state)); 8085 return -EFAULT; 8086 } 8087 8088 if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) { 8089 /* Because iter_next() call is a checkpoint is_state_visitied() 8090 * should guarantee parent state with same call sites and insn_idx. 8091 */ 8092 if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx || 8093 !same_callsites(cur_st->parent, cur_st)) { 8094 verbose(env, "bug: bad parent state for iter next call"); 8095 return -EFAULT; 8096 } 8097 /* Note cur_st->parent in the call below, it is necessary to skip 8098 * checkpoint created for cur_st by is_state_visited() 8099 * right at this instruction. 8100 */ 8101 prev_st = find_prev_entry(env, cur_st->parent, insn_idx); 8102 /* branch out active iter state */ 8103 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); 8104 if (!queued_st) 8105 return -ENOMEM; 8106 8107 queued_iter = &queued_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; 8108 queued_iter->iter.state = BPF_ITER_STATE_ACTIVE; 8109 queued_iter->iter.depth++; 8110 if (prev_st) 8111 widen_imprecise_scalars(env, prev_st, queued_st); 8112 8113 queued_fr = queued_st->frame[queued_st->curframe]; 8114 mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]); 8115 } 8116 8117 /* switch to DRAINED state, but keep the depth unchanged */ 8118 /* mark current iter state as drained and assume returned NULL */ 8119 cur_iter->iter.state = BPF_ITER_STATE_DRAINED; 8120 __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]); 8121 8122 return 0; 8123 } 8124 8125 static bool arg_type_is_mem_size(enum bpf_arg_type type) 8126 { 8127 return type == ARG_CONST_SIZE || 8128 type == ARG_CONST_SIZE_OR_ZERO; 8129 } 8130 8131 static bool arg_type_is_release(enum bpf_arg_type type) 8132 { 8133 return type & OBJ_RELEASE; 8134 } 8135 8136 static bool arg_type_is_dynptr(enum bpf_arg_type type) 8137 { 8138 return base_type(type) == ARG_PTR_TO_DYNPTR; 8139 } 8140 8141 static int int_ptr_type_to_size(enum bpf_arg_type type) 8142 { 8143 if (type == ARG_PTR_TO_INT) 8144 return sizeof(u32); 8145 else if (type == ARG_PTR_TO_LONG) 8146 return sizeof(u64); 8147 8148 return -EINVAL; 8149 } 8150 8151 static int resolve_map_arg_type(struct bpf_verifier_env *env, 8152 const struct bpf_call_arg_meta *meta, 8153 enum bpf_arg_type *arg_type) 8154 { 8155 if (!meta->map_ptr) { 8156 /* kernel subsystem misconfigured verifier */ 8157 verbose(env, "invalid map_ptr to access map->type\n"); 8158 return -EACCES; 8159 } 8160 8161 switch (meta->map_ptr->map_type) { 8162 case BPF_MAP_TYPE_SOCKMAP: 8163 case BPF_MAP_TYPE_SOCKHASH: 8164 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 8165 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 8166 } else { 8167 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 8168 return -EINVAL; 8169 } 8170 break; 8171 case BPF_MAP_TYPE_BLOOM_FILTER: 8172 if (meta->func_id == BPF_FUNC_map_peek_elem) 8173 *arg_type = ARG_PTR_TO_MAP_VALUE; 8174 break; 8175 default: 8176 break; 8177 } 8178 return 0; 8179 } 8180 8181 struct bpf_reg_types { 8182 const enum bpf_reg_type types[10]; 8183 u32 *btf_id; 8184 }; 8185 8186 static const struct bpf_reg_types sock_types = { 8187 .types = { 8188 PTR_TO_SOCK_COMMON, 8189 PTR_TO_SOCKET, 8190 PTR_TO_TCP_SOCK, 8191 PTR_TO_XDP_SOCK, 8192 }, 8193 }; 8194 8195 #ifdef CONFIG_NET 8196 static const struct bpf_reg_types btf_id_sock_common_types = { 8197 .types = { 8198 PTR_TO_SOCK_COMMON, 8199 PTR_TO_SOCKET, 8200 PTR_TO_TCP_SOCK, 8201 PTR_TO_XDP_SOCK, 8202 PTR_TO_BTF_ID, 8203 PTR_TO_BTF_ID | PTR_TRUSTED, 8204 }, 8205 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 8206 }; 8207 #endif 8208 8209 static const struct bpf_reg_types mem_types = { 8210 .types = { 8211 PTR_TO_STACK, 8212 PTR_TO_PACKET, 8213 PTR_TO_PACKET_META, 8214 PTR_TO_MAP_KEY, 8215 PTR_TO_MAP_VALUE, 8216 PTR_TO_MEM, 8217 PTR_TO_MEM | MEM_RINGBUF, 8218 PTR_TO_BUF, 8219 PTR_TO_BTF_ID | PTR_TRUSTED, 8220 }, 8221 }; 8222 8223 static const struct bpf_reg_types int_ptr_types = { 8224 .types = { 8225 PTR_TO_STACK, 8226 PTR_TO_PACKET, 8227 PTR_TO_PACKET_META, 8228 PTR_TO_MAP_KEY, 8229 PTR_TO_MAP_VALUE, 8230 }, 8231 }; 8232 8233 static const struct bpf_reg_types spin_lock_types = { 8234 .types = { 8235 PTR_TO_MAP_VALUE, 8236 PTR_TO_BTF_ID | MEM_ALLOC, 8237 } 8238 }; 8239 8240 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 8241 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 8242 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 8243 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 8244 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 8245 static const struct bpf_reg_types btf_ptr_types = { 8246 .types = { 8247 PTR_TO_BTF_ID, 8248 PTR_TO_BTF_ID | PTR_TRUSTED, 8249 PTR_TO_BTF_ID | MEM_RCU, 8250 }, 8251 }; 8252 static const struct bpf_reg_types percpu_btf_ptr_types = { 8253 .types = { 8254 PTR_TO_BTF_ID | MEM_PERCPU, 8255 PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU, 8256 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 8257 } 8258 }; 8259 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 8260 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 8261 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 8262 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 8263 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 8264 static const struct bpf_reg_types dynptr_types = { 8265 .types = { 8266 PTR_TO_STACK, 8267 CONST_PTR_TO_DYNPTR, 8268 } 8269 }; 8270 8271 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 8272 [ARG_PTR_TO_MAP_KEY] = &mem_types, 8273 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 8274 [ARG_CONST_SIZE] = &scalar_types, 8275 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 8276 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 8277 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 8278 [ARG_PTR_TO_CTX] = &context_types, 8279 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 8280 #ifdef CONFIG_NET 8281 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 8282 #endif 8283 [ARG_PTR_TO_SOCKET] = &fullsock_types, 8284 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 8285 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 8286 [ARG_PTR_TO_MEM] = &mem_types, 8287 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 8288 [ARG_PTR_TO_INT] = &int_ptr_types, 8289 [ARG_PTR_TO_LONG] = &int_ptr_types, 8290 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 8291 [ARG_PTR_TO_FUNC] = &func_ptr_types, 8292 [ARG_PTR_TO_STACK] = &stack_ptr_types, 8293 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 8294 [ARG_PTR_TO_TIMER] = &timer_types, 8295 [ARG_PTR_TO_KPTR] = &kptr_types, 8296 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 8297 }; 8298 8299 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 8300 enum bpf_arg_type arg_type, 8301 const u32 *arg_btf_id, 8302 struct bpf_call_arg_meta *meta) 8303 { 8304 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 8305 enum bpf_reg_type expected, type = reg->type; 8306 const struct bpf_reg_types *compatible; 8307 int i, j; 8308 8309 compatible = compatible_reg_types[base_type(arg_type)]; 8310 if (!compatible) { 8311 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 8312 return -EFAULT; 8313 } 8314 8315 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 8316 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 8317 * 8318 * Same for MAYBE_NULL: 8319 * 8320 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 8321 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 8322 * 8323 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type. 8324 * 8325 * Therefore we fold these flags depending on the arg_type before comparison. 8326 */ 8327 if (arg_type & MEM_RDONLY) 8328 type &= ~MEM_RDONLY; 8329 if (arg_type & PTR_MAYBE_NULL) 8330 type &= ~PTR_MAYBE_NULL; 8331 if (base_type(arg_type) == ARG_PTR_TO_MEM) 8332 type &= ~DYNPTR_TYPE_FLAG_MASK; 8333 8334 if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type)) { 8335 type &= ~MEM_ALLOC; 8336 type &= ~MEM_PERCPU; 8337 } 8338 8339 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 8340 expected = compatible->types[i]; 8341 if (expected == NOT_INIT) 8342 break; 8343 8344 if (type == expected) 8345 goto found; 8346 } 8347 8348 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 8349 for (j = 0; j + 1 < i; j++) 8350 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 8351 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 8352 return -EACCES; 8353 8354 found: 8355 if (base_type(reg->type) != PTR_TO_BTF_ID) 8356 return 0; 8357 8358 if (compatible == &mem_types) { 8359 if (!(arg_type & MEM_RDONLY)) { 8360 verbose(env, 8361 "%s() may write into memory pointed by R%d type=%s\n", 8362 func_id_name(meta->func_id), 8363 regno, reg_type_str(env, reg->type)); 8364 return -EACCES; 8365 } 8366 return 0; 8367 } 8368 8369 switch ((int)reg->type) { 8370 case PTR_TO_BTF_ID: 8371 case PTR_TO_BTF_ID | PTR_TRUSTED: 8372 case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: 8373 case PTR_TO_BTF_ID | MEM_RCU: 8374 case PTR_TO_BTF_ID | PTR_MAYBE_NULL: 8375 case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: 8376 { 8377 /* For bpf_sk_release, it needs to match against first member 8378 * 'struct sock_common', hence make an exception for it. This 8379 * allows bpf_sk_release to work for multiple socket types. 8380 */ 8381 bool strict_type_match = arg_type_is_release(arg_type) && 8382 meta->func_id != BPF_FUNC_sk_release; 8383 8384 if (type_may_be_null(reg->type) && 8385 (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { 8386 verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); 8387 return -EACCES; 8388 } 8389 8390 if (!arg_btf_id) { 8391 if (!compatible->btf_id) { 8392 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 8393 return -EFAULT; 8394 } 8395 arg_btf_id = compatible->btf_id; 8396 } 8397 8398 if (meta->func_id == BPF_FUNC_kptr_xchg) { 8399 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 8400 return -EACCES; 8401 } else { 8402 if (arg_btf_id == BPF_PTR_POISON) { 8403 verbose(env, "verifier internal error:"); 8404 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", 8405 regno); 8406 return -EACCES; 8407 } 8408 8409 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 8410 btf_vmlinux, *arg_btf_id, 8411 strict_type_match)) { 8412 verbose(env, "R%d is of type %s but %s is expected\n", 8413 regno, btf_type_name(reg->btf, reg->btf_id), 8414 btf_type_name(btf_vmlinux, *arg_btf_id)); 8415 return -EACCES; 8416 } 8417 } 8418 break; 8419 } 8420 case PTR_TO_BTF_ID | MEM_ALLOC: 8421 case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: 8422 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && 8423 meta->func_id != BPF_FUNC_kptr_xchg) { 8424 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); 8425 return -EFAULT; 8426 } 8427 if (meta->func_id == BPF_FUNC_kptr_xchg) { 8428 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 8429 return -EACCES; 8430 } 8431 break; 8432 case PTR_TO_BTF_ID | MEM_PERCPU: 8433 case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: 8434 case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: 8435 /* Handled by helper specific checks */ 8436 break; 8437 default: 8438 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); 8439 return -EFAULT; 8440 } 8441 return 0; 8442 } 8443 8444 static struct btf_field * 8445 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) 8446 { 8447 struct btf_field *field; 8448 struct btf_record *rec; 8449 8450 rec = reg_btf_record(reg); 8451 if (!rec) 8452 return NULL; 8453 8454 field = btf_record_find(rec, off, fields); 8455 if (!field) 8456 return NULL; 8457 8458 return field; 8459 } 8460 8461 static int check_func_arg_reg_off(struct bpf_verifier_env *env, 8462 const struct bpf_reg_state *reg, int regno, 8463 enum bpf_arg_type arg_type) 8464 { 8465 u32 type = reg->type; 8466 8467 /* When referenced register is passed to release function, its fixed 8468 * offset must be 0. 8469 * 8470 * We will check arg_type_is_release reg has ref_obj_id when storing 8471 * meta->release_regno. 8472 */ 8473 if (arg_type_is_release(arg_type)) { 8474 /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it 8475 * may not directly point to the object being released, but to 8476 * dynptr pointing to such object, which might be at some offset 8477 * on the stack. In that case, we simply to fallback to the 8478 * default handling. 8479 */ 8480 if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) 8481 return 0; 8482 8483 /* Doing check_ptr_off_reg check for the offset will catch this 8484 * because fixed_off_ok is false, but checking here allows us 8485 * to give the user a better error message. 8486 */ 8487 if (reg->off) { 8488 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", 8489 regno); 8490 return -EINVAL; 8491 } 8492 return __check_ptr_off_reg(env, reg, regno, false); 8493 } 8494 8495 switch (type) { 8496 /* Pointer types where both fixed and variable offset is explicitly allowed: */ 8497 case PTR_TO_STACK: 8498 case PTR_TO_PACKET: 8499 case PTR_TO_PACKET_META: 8500 case PTR_TO_MAP_KEY: 8501 case PTR_TO_MAP_VALUE: 8502 case PTR_TO_MEM: 8503 case PTR_TO_MEM | MEM_RDONLY: 8504 case PTR_TO_MEM | MEM_RINGBUF: 8505 case PTR_TO_BUF: 8506 case PTR_TO_BUF | MEM_RDONLY: 8507 case PTR_TO_ARENA: 8508 case SCALAR_VALUE: 8509 return 0; 8510 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 8511 * fixed offset. 8512 */ 8513 case PTR_TO_BTF_ID: 8514 case PTR_TO_BTF_ID | MEM_ALLOC: 8515 case PTR_TO_BTF_ID | PTR_TRUSTED: 8516 case PTR_TO_BTF_ID | MEM_RCU: 8517 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: 8518 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: 8519 /* When referenced PTR_TO_BTF_ID is passed to release function, 8520 * its fixed offset must be 0. In the other cases, fixed offset 8521 * can be non-zero. This was already checked above. So pass 8522 * fixed_off_ok as true to allow fixed offset for all other 8523 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we 8524 * still need to do checks instead of returning. 8525 */ 8526 return __check_ptr_off_reg(env, reg, regno, true); 8527 default: 8528 return __check_ptr_off_reg(env, reg, regno, false); 8529 } 8530 } 8531 8532 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, 8533 const struct bpf_func_proto *fn, 8534 struct bpf_reg_state *regs) 8535 { 8536 struct bpf_reg_state *state = NULL; 8537 int i; 8538 8539 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) 8540 if (arg_type_is_dynptr(fn->arg_type[i])) { 8541 if (state) { 8542 verbose(env, "verifier internal error: multiple dynptr args\n"); 8543 return NULL; 8544 } 8545 state = ®s[BPF_REG_1 + i]; 8546 } 8547 8548 if (!state) 8549 verbose(env, "verifier internal error: no dynptr arg found\n"); 8550 8551 return state; 8552 } 8553 8554 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 8555 { 8556 struct bpf_func_state *state = func(env, reg); 8557 int spi; 8558 8559 if (reg->type == CONST_PTR_TO_DYNPTR) 8560 return reg->id; 8561 spi = dynptr_get_spi(env, reg); 8562 if (spi < 0) 8563 return spi; 8564 return state->stack[spi].spilled_ptr.id; 8565 } 8566 8567 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 8568 { 8569 struct bpf_func_state *state = func(env, reg); 8570 int spi; 8571 8572 if (reg->type == CONST_PTR_TO_DYNPTR) 8573 return reg->ref_obj_id; 8574 spi = dynptr_get_spi(env, reg); 8575 if (spi < 0) 8576 return spi; 8577 return state->stack[spi].spilled_ptr.ref_obj_id; 8578 } 8579 8580 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, 8581 struct bpf_reg_state *reg) 8582 { 8583 struct bpf_func_state *state = func(env, reg); 8584 int spi; 8585 8586 if (reg->type == CONST_PTR_TO_DYNPTR) 8587 return reg->dynptr.type; 8588 8589 spi = __get_spi(reg->off); 8590 if (spi < 0) { 8591 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); 8592 return BPF_DYNPTR_TYPE_INVALID; 8593 } 8594 8595 return state->stack[spi].spilled_ptr.dynptr.type; 8596 } 8597 8598 static int check_reg_const_str(struct bpf_verifier_env *env, 8599 struct bpf_reg_state *reg, u32 regno) 8600 { 8601 struct bpf_map *map = reg->map_ptr; 8602 int err; 8603 int map_off; 8604 u64 map_addr; 8605 char *str_ptr; 8606 8607 if (reg->type != PTR_TO_MAP_VALUE) 8608 return -EINVAL; 8609 8610 if (!bpf_map_is_rdonly(map)) { 8611 verbose(env, "R%d does not point to a readonly map'\n", regno); 8612 return -EACCES; 8613 } 8614 8615 if (!tnum_is_const(reg->var_off)) { 8616 verbose(env, "R%d is not a constant address'\n", regno); 8617 return -EACCES; 8618 } 8619 8620 if (!map->ops->map_direct_value_addr) { 8621 verbose(env, "no direct value access support for this map type\n"); 8622 return -EACCES; 8623 } 8624 8625 err = check_map_access(env, regno, reg->off, 8626 map->value_size - reg->off, false, 8627 ACCESS_HELPER); 8628 if (err) 8629 return err; 8630 8631 map_off = reg->off + reg->var_off.value; 8632 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 8633 if (err) { 8634 verbose(env, "direct value access on string failed\n"); 8635 return err; 8636 } 8637 8638 str_ptr = (char *)(long)(map_addr); 8639 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 8640 verbose(env, "string is not zero-terminated\n"); 8641 return -EINVAL; 8642 } 8643 return 0; 8644 } 8645 8646 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 8647 struct bpf_call_arg_meta *meta, 8648 const struct bpf_func_proto *fn, 8649 int insn_idx) 8650 { 8651 u32 regno = BPF_REG_1 + arg; 8652 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 8653 enum bpf_arg_type arg_type = fn->arg_type[arg]; 8654 enum bpf_reg_type type = reg->type; 8655 u32 *arg_btf_id = NULL; 8656 int err = 0; 8657 8658 if (arg_type == ARG_DONTCARE) 8659 return 0; 8660 8661 err = check_reg_arg(env, regno, SRC_OP); 8662 if (err) 8663 return err; 8664 8665 if (arg_type == ARG_ANYTHING) { 8666 if (is_pointer_value(env, regno)) { 8667 verbose(env, "R%d leaks addr into helper function\n", 8668 regno); 8669 return -EACCES; 8670 } 8671 return 0; 8672 } 8673 8674 if (type_is_pkt_pointer(type) && 8675 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 8676 verbose(env, "helper access to the packet is not allowed\n"); 8677 return -EACCES; 8678 } 8679 8680 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { 8681 err = resolve_map_arg_type(env, meta, &arg_type); 8682 if (err) 8683 return err; 8684 } 8685 8686 if (register_is_null(reg) && type_may_be_null(arg_type)) 8687 /* A NULL register has a SCALAR_VALUE type, so skip 8688 * type checking. 8689 */ 8690 goto skip_type_check; 8691 8692 /* arg_btf_id and arg_size are in a union. */ 8693 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || 8694 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 8695 arg_btf_id = fn->arg_btf_id[arg]; 8696 8697 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 8698 if (err) 8699 return err; 8700 8701 err = check_func_arg_reg_off(env, reg, regno, arg_type); 8702 if (err) 8703 return err; 8704 8705 skip_type_check: 8706 if (arg_type_is_release(arg_type)) { 8707 if (arg_type_is_dynptr(arg_type)) { 8708 struct bpf_func_state *state = func(env, reg); 8709 int spi; 8710 8711 /* Only dynptr created on stack can be released, thus 8712 * the get_spi and stack state checks for spilled_ptr 8713 * should only be done before process_dynptr_func for 8714 * PTR_TO_STACK. 8715 */ 8716 if (reg->type == PTR_TO_STACK) { 8717 spi = dynptr_get_spi(env, reg); 8718 if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) { 8719 verbose(env, "arg %d is an unacquired reference\n", regno); 8720 return -EINVAL; 8721 } 8722 } else { 8723 verbose(env, "cannot release unowned const bpf_dynptr\n"); 8724 return -EINVAL; 8725 } 8726 } else if (!reg->ref_obj_id && !register_is_null(reg)) { 8727 verbose(env, "R%d must be referenced when passed to release function\n", 8728 regno); 8729 return -EINVAL; 8730 } 8731 if (meta->release_regno) { 8732 verbose(env, "verifier internal error: more than one release argument\n"); 8733 return -EFAULT; 8734 } 8735 meta->release_regno = regno; 8736 } 8737 8738 if (reg->ref_obj_id) { 8739 if (meta->ref_obj_id) { 8740 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 8741 regno, reg->ref_obj_id, 8742 meta->ref_obj_id); 8743 return -EFAULT; 8744 } 8745 meta->ref_obj_id = reg->ref_obj_id; 8746 } 8747 8748 switch (base_type(arg_type)) { 8749 case ARG_CONST_MAP_PTR: 8750 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 8751 if (meta->map_ptr) { 8752 /* Use map_uid (which is unique id of inner map) to reject: 8753 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 8754 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 8755 * if (inner_map1 && inner_map2) { 8756 * timer = bpf_map_lookup_elem(inner_map1); 8757 * if (timer) 8758 * // mismatch would have been allowed 8759 * bpf_timer_init(timer, inner_map2); 8760 * } 8761 * 8762 * Comparing map_ptr is enough to distinguish normal and outer maps. 8763 */ 8764 if (meta->map_ptr != reg->map_ptr || 8765 meta->map_uid != reg->map_uid) { 8766 verbose(env, 8767 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 8768 meta->map_uid, reg->map_uid); 8769 return -EINVAL; 8770 } 8771 } 8772 meta->map_ptr = reg->map_ptr; 8773 meta->map_uid = reg->map_uid; 8774 break; 8775 case ARG_PTR_TO_MAP_KEY: 8776 /* bpf_map_xxx(..., map_ptr, ..., key) call: 8777 * check that [key, key + map->key_size) are within 8778 * stack limits and initialized 8779 */ 8780 if (!meta->map_ptr) { 8781 /* in function declaration map_ptr must come before 8782 * map_key, so that it's verified and known before 8783 * we have to check map_key here. Otherwise it means 8784 * that kernel subsystem misconfigured verifier 8785 */ 8786 verbose(env, "invalid map_ptr to access map->key\n"); 8787 return -EACCES; 8788 } 8789 err = check_helper_mem_access(env, regno, 8790 meta->map_ptr->key_size, false, 8791 NULL); 8792 break; 8793 case ARG_PTR_TO_MAP_VALUE: 8794 if (type_may_be_null(arg_type) && register_is_null(reg)) 8795 return 0; 8796 8797 /* bpf_map_xxx(..., map_ptr, ..., value) call: 8798 * check [value, value + map->value_size) validity 8799 */ 8800 if (!meta->map_ptr) { 8801 /* kernel subsystem misconfigured verifier */ 8802 verbose(env, "invalid map_ptr to access map->value\n"); 8803 return -EACCES; 8804 } 8805 meta->raw_mode = arg_type & MEM_UNINIT; 8806 err = check_helper_mem_access(env, regno, 8807 meta->map_ptr->value_size, false, 8808 meta); 8809 break; 8810 case ARG_PTR_TO_PERCPU_BTF_ID: 8811 if (!reg->btf_id) { 8812 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 8813 return -EACCES; 8814 } 8815 meta->ret_btf = reg->btf; 8816 meta->ret_btf_id = reg->btf_id; 8817 break; 8818 case ARG_PTR_TO_SPIN_LOCK: 8819 if (in_rbtree_lock_required_cb(env)) { 8820 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); 8821 return -EACCES; 8822 } 8823 if (meta->func_id == BPF_FUNC_spin_lock) { 8824 err = process_spin_lock(env, regno, true); 8825 if (err) 8826 return err; 8827 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 8828 err = process_spin_lock(env, regno, false); 8829 if (err) 8830 return err; 8831 } else { 8832 verbose(env, "verifier internal error\n"); 8833 return -EFAULT; 8834 } 8835 break; 8836 case ARG_PTR_TO_TIMER: 8837 err = process_timer_func(env, regno, meta); 8838 if (err) 8839 return err; 8840 break; 8841 case ARG_PTR_TO_FUNC: 8842 meta->subprogno = reg->subprogno; 8843 break; 8844 case ARG_PTR_TO_MEM: 8845 /* The access to this pointer is only checked when we hit the 8846 * next is_mem_size argument below. 8847 */ 8848 meta->raw_mode = arg_type & MEM_UNINIT; 8849 if (arg_type & MEM_FIXED_SIZE) { 8850 err = check_helper_mem_access(env, regno, 8851 fn->arg_size[arg], false, 8852 meta); 8853 } 8854 break; 8855 case ARG_CONST_SIZE: 8856 err = check_mem_size_reg(env, reg, regno, false, meta); 8857 break; 8858 case ARG_CONST_SIZE_OR_ZERO: 8859 err = check_mem_size_reg(env, reg, regno, true, meta); 8860 break; 8861 case ARG_PTR_TO_DYNPTR: 8862 err = process_dynptr_func(env, regno, insn_idx, arg_type, 0); 8863 if (err) 8864 return err; 8865 break; 8866 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 8867 if (!tnum_is_const(reg->var_off)) { 8868 verbose(env, "R%d is not a known constant'\n", 8869 regno); 8870 return -EACCES; 8871 } 8872 meta->mem_size = reg->var_off.value; 8873 err = mark_chain_precision(env, regno); 8874 if (err) 8875 return err; 8876 break; 8877 case ARG_PTR_TO_INT: 8878 case ARG_PTR_TO_LONG: 8879 { 8880 int size = int_ptr_type_to_size(arg_type); 8881 8882 err = check_helper_mem_access(env, regno, size, false, meta); 8883 if (err) 8884 return err; 8885 err = check_ptr_alignment(env, reg, 0, size, true); 8886 break; 8887 } 8888 case ARG_PTR_TO_CONST_STR: 8889 { 8890 err = check_reg_const_str(env, reg, regno); 8891 if (err) 8892 return err; 8893 break; 8894 } 8895 case ARG_PTR_TO_KPTR: 8896 err = process_kptr_func(env, regno, meta); 8897 if (err) 8898 return err; 8899 break; 8900 } 8901 8902 return err; 8903 } 8904 8905 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 8906 { 8907 enum bpf_attach_type eatype = env->prog->expected_attach_type; 8908 enum bpf_prog_type type = resolve_prog_type(env->prog); 8909 8910 if (func_id != BPF_FUNC_map_update_elem && 8911 func_id != BPF_FUNC_map_delete_elem) 8912 return false; 8913 8914 /* It's not possible to get access to a locked struct sock in these 8915 * contexts, so updating is safe. 8916 */ 8917 switch (type) { 8918 case BPF_PROG_TYPE_TRACING: 8919 if (eatype == BPF_TRACE_ITER) 8920 return true; 8921 break; 8922 case BPF_PROG_TYPE_SOCK_OPS: 8923 /* map_update allowed only via dedicated helpers with event type checks */ 8924 if (func_id == BPF_FUNC_map_delete_elem) 8925 return true; 8926 break; 8927 case BPF_PROG_TYPE_SOCKET_FILTER: 8928 case BPF_PROG_TYPE_SCHED_CLS: 8929 case BPF_PROG_TYPE_SCHED_ACT: 8930 case BPF_PROG_TYPE_XDP: 8931 case BPF_PROG_TYPE_SK_REUSEPORT: 8932 case BPF_PROG_TYPE_FLOW_DISSECTOR: 8933 case BPF_PROG_TYPE_SK_LOOKUP: 8934 return true; 8935 default: 8936 break; 8937 } 8938 8939 verbose(env, "cannot update sockmap in this context\n"); 8940 return false; 8941 } 8942 8943 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 8944 { 8945 return env->prog->jit_requested && 8946 bpf_jit_supports_subprog_tailcalls(); 8947 } 8948 8949 static int check_map_func_compatibility(struct bpf_verifier_env *env, 8950 struct bpf_map *map, int func_id) 8951 { 8952 if (!map) 8953 return 0; 8954 8955 /* We need a two way check, first is from map perspective ... */ 8956 switch (map->map_type) { 8957 case BPF_MAP_TYPE_PROG_ARRAY: 8958 if (func_id != BPF_FUNC_tail_call) 8959 goto error; 8960 break; 8961 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 8962 if (func_id != BPF_FUNC_perf_event_read && 8963 func_id != BPF_FUNC_perf_event_output && 8964 func_id != BPF_FUNC_skb_output && 8965 func_id != BPF_FUNC_perf_event_read_value && 8966 func_id != BPF_FUNC_xdp_output) 8967 goto error; 8968 break; 8969 case BPF_MAP_TYPE_RINGBUF: 8970 if (func_id != BPF_FUNC_ringbuf_output && 8971 func_id != BPF_FUNC_ringbuf_reserve && 8972 func_id != BPF_FUNC_ringbuf_query && 8973 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 8974 func_id != BPF_FUNC_ringbuf_submit_dynptr && 8975 func_id != BPF_FUNC_ringbuf_discard_dynptr) 8976 goto error; 8977 break; 8978 case BPF_MAP_TYPE_USER_RINGBUF: 8979 if (func_id != BPF_FUNC_user_ringbuf_drain) 8980 goto error; 8981 break; 8982 case BPF_MAP_TYPE_STACK_TRACE: 8983 if (func_id != BPF_FUNC_get_stackid) 8984 goto error; 8985 break; 8986 case BPF_MAP_TYPE_CGROUP_ARRAY: 8987 if (func_id != BPF_FUNC_skb_under_cgroup && 8988 func_id != BPF_FUNC_current_task_under_cgroup) 8989 goto error; 8990 break; 8991 case BPF_MAP_TYPE_CGROUP_STORAGE: 8992 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 8993 if (func_id != BPF_FUNC_get_local_storage) 8994 goto error; 8995 break; 8996 case BPF_MAP_TYPE_DEVMAP: 8997 case BPF_MAP_TYPE_DEVMAP_HASH: 8998 if (func_id != BPF_FUNC_redirect_map && 8999 func_id != BPF_FUNC_map_lookup_elem) 9000 goto error; 9001 break; 9002 /* Restrict bpf side of cpumap and xskmap, open when use-cases 9003 * appear. 9004 */ 9005 case BPF_MAP_TYPE_CPUMAP: 9006 if (func_id != BPF_FUNC_redirect_map) 9007 goto error; 9008 break; 9009 case BPF_MAP_TYPE_XSKMAP: 9010 if (func_id != BPF_FUNC_redirect_map && 9011 func_id != BPF_FUNC_map_lookup_elem) 9012 goto error; 9013 break; 9014 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 9015 case BPF_MAP_TYPE_HASH_OF_MAPS: 9016 if (func_id != BPF_FUNC_map_lookup_elem) 9017 goto error; 9018 break; 9019 case BPF_MAP_TYPE_SOCKMAP: 9020 if (func_id != BPF_FUNC_sk_redirect_map && 9021 func_id != BPF_FUNC_sock_map_update && 9022 func_id != BPF_FUNC_msg_redirect_map && 9023 func_id != BPF_FUNC_sk_select_reuseport && 9024 func_id != BPF_FUNC_map_lookup_elem && 9025 !may_update_sockmap(env, func_id)) 9026 goto error; 9027 break; 9028 case BPF_MAP_TYPE_SOCKHASH: 9029 if (func_id != BPF_FUNC_sk_redirect_hash && 9030 func_id != BPF_FUNC_sock_hash_update && 9031 func_id != BPF_FUNC_msg_redirect_hash && 9032 func_id != BPF_FUNC_sk_select_reuseport && 9033 func_id != BPF_FUNC_map_lookup_elem && 9034 !may_update_sockmap(env, func_id)) 9035 goto error; 9036 break; 9037 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 9038 if (func_id != BPF_FUNC_sk_select_reuseport) 9039 goto error; 9040 break; 9041 case BPF_MAP_TYPE_QUEUE: 9042 case BPF_MAP_TYPE_STACK: 9043 if (func_id != BPF_FUNC_map_peek_elem && 9044 func_id != BPF_FUNC_map_pop_elem && 9045 func_id != BPF_FUNC_map_push_elem) 9046 goto error; 9047 break; 9048 case BPF_MAP_TYPE_SK_STORAGE: 9049 if (func_id != BPF_FUNC_sk_storage_get && 9050 func_id != BPF_FUNC_sk_storage_delete && 9051 func_id != BPF_FUNC_kptr_xchg) 9052 goto error; 9053 break; 9054 case BPF_MAP_TYPE_INODE_STORAGE: 9055 if (func_id != BPF_FUNC_inode_storage_get && 9056 func_id != BPF_FUNC_inode_storage_delete && 9057 func_id != BPF_FUNC_kptr_xchg) 9058 goto error; 9059 break; 9060 case BPF_MAP_TYPE_TASK_STORAGE: 9061 if (func_id != BPF_FUNC_task_storage_get && 9062 func_id != BPF_FUNC_task_storage_delete && 9063 func_id != BPF_FUNC_kptr_xchg) 9064 goto error; 9065 break; 9066 case BPF_MAP_TYPE_CGRP_STORAGE: 9067 if (func_id != BPF_FUNC_cgrp_storage_get && 9068 func_id != BPF_FUNC_cgrp_storage_delete && 9069 func_id != BPF_FUNC_kptr_xchg) 9070 goto error; 9071 break; 9072 case BPF_MAP_TYPE_BLOOM_FILTER: 9073 if (func_id != BPF_FUNC_map_peek_elem && 9074 func_id != BPF_FUNC_map_push_elem) 9075 goto error; 9076 break; 9077 default: 9078 break; 9079 } 9080 9081 /* ... and second from the function itself. */ 9082 switch (func_id) { 9083 case BPF_FUNC_tail_call: 9084 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 9085 goto error; 9086 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 9087 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 9088 return -EINVAL; 9089 } 9090 break; 9091 case BPF_FUNC_perf_event_read: 9092 case BPF_FUNC_perf_event_output: 9093 case BPF_FUNC_perf_event_read_value: 9094 case BPF_FUNC_skb_output: 9095 case BPF_FUNC_xdp_output: 9096 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 9097 goto error; 9098 break; 9099 case BPF_FUNC_ringbuf_output: 9100 case BPF_FUNC_ringbuf_reserve: 9101 case BPF_FUNC_ringbuf_query: 9102 case BPF_FUNC_ringbuf_reserve_dynptr: 9103 case BPF_FUNC_ringbuf_submit_dynptr: 9104 case BPF_FUNC_ringbuf_discard_dynptr: 9105 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 9106 goto error; 9107 break; 9108 case BPF_FUNC_user_ringbuf_drain: 9109 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 9110 goto error; 9111 break; 9112 case BPF_FUNC_get_stackid: 9113 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 9114 goto error; 9115 break; 9116 case BPF_FUNC_current_task_under_cgroup: 9117 case BPF_FUNC_skb_under_cgroup: 9118 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 9119 goto error; 9120 break; 9121 case BPF_FUNC_redirect_map: 9122 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 9123 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 9124 map->map_type != BPF_MAP_TYPE_CPUMAP && 9125 map->map_type != BPF_MAP_TYPE_XSKMAP) 9126 goto error; 9127 break; 9128 case BPF_FUNC_sk_redirect_map: 9129 case BPF_FUNC_msg_redirect_map: 9130 case BPF_FUNC_sock_map_update: 9131 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 9132 goto error; 9133 break; 9134 case BPF_FUNC_sk_redirect_hash: 9135 case BPF_FUNC_msg_redirect_hash: 9136 case BPF_FUNC_sock_hash_update: 9137 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 9138 goto error; 9139 break; 9140 case BPF_FUNC_get_local_storage: 9141 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 9142 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 9143 goto error; 9144 break; 9145 case BPF_FUNC_sk_select_reuseport: 9146 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 9147 map->map_type != BPF_MAP_TYPE_SOCKMAP && 9148 map->map_type != BPF_MAP_TYPE_SOCKHASH) 9149 goto error; 9150 break; 9151 case BPF_FUNC_map_pop_elem: 9152 if (map->map_type != BPF_MAP_TYPE_QUEUE && 9153 map->map_type != BPF_MAP_TYPE_STACK) 9154 goto error; 9155 break; 9156 case BPF_FUNC_map_peek_elem: 9157 case BPF_FUNC_map_push_elem: 9158 if (map->map_type != BPF_MAP_TYPE_QUEUE && 9159 map->map_type != BPF_MAP_TYPE_STACK && 9160 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 9161 goto error; 9162 break; 9163 case BPF_FUNC_map_lookup_percpu_elem: 9164 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 9165 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 9166 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 9167 goto error; 9168 break; 9169 case BPF_FUNC_sk_storage_get: 9170 case BPF_FUNC_sk_storage_delete: 9171 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 9172 goto error; 9173 break; 9174 case BPF_FUNC_inode_storage_get: 9175 case BPF_FUNC_inode_storage_delete: 9176 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 9177 goto error; 9178 break; 9179 case BPF_FUNC_task_storage_get: 9180 case BPF_FUNC_task_storage_delete: 9181 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 9182 goto error; 9183 break; 9184 case BPF_FUNC_cgrp_storage_get: 9185 case BPF_FUNC_cgrp_storage_delete: 9186 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 9187 goto error; 9188 break; 9189 default: 9190 break; 9191 } 9192 9193 return 0; 9194 error: 9195 verbose(env, "cannot pass map_type %d into func %s#%d\n", 9196 map->map_type, func_id_name(func_id), func_id); 9197 return -EINVAL; 9198 } 9199 9200 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 9201 { 9202 int count = 0; 9203 9204 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 9205 count++; 9206 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 9207 count++; 9208 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 9209 count++; 9210 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 9211 count++; 9212 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 9213 count++; 9214 9215 /* We only support one arg being in raw mode at the moment, 9216 * which is sufficient for the helper functions we have 9217 * right now. 9218 */ 9219 return count <= 1; 9220 } 9221 9222 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 9223 { 9224 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 9225 bool has_size = fn->arg_size[arg] != 0; 9226 bool is_next_size = false; 9227 9228 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 9229 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 9230 9231 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 9232 return is_next_size; 9233 9234 return has_size == is_next_size || is_next_size == is_fixed; 9235 } 9236 9237 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 9238 { 9239 /* bpf_xxx(..., buf, len) call will access 'len' 9240 * bytes from memory 'buf'. Both arg types need 9241 * to be paired, so make sure there's no buggy 9242 * helper function specification. 9243 */ 9244 if (arg_type_is_mem_size(fn->arg1_type) || 9245 check_args_pair_invalid(fn, 0) || 9246 check_args_pair_invalid(fn, 1) || 9247 check_args_pair_invalid(fn, 2) || 9248 check_args_pair_invalid(fn, 3) || 9249 check_args_pair_invalid(fn, 4)) 9250 return false; 9251 9252 return true; 9253 } 9254 9255 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 9256 { 9257 int i; 9258 9259 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 9260 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 9261 return !!fn->arg_btf_id[i]; 9262 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 9263 return fn->arg_btf_id[i] == BPF_PTR_POISON; 9264 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 9265 /* arg_btf_id and arg_size are in a union. */ 9266 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 9267 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 9268 return false; 9269 } 9270 9271 return true; 9272 } 9273 9274 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 9275 { 9276 return check_raw_mode_ok(fn) && 9277 check_arg_pair_ok(fn) && 9278 check_btf_id_ok(fn) ? 0 : -EINVAL; 9279 } 9280 9281 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 9282 * are now invalid, so turn them into unknown SCALAR_VALUE. 9283 * 9284 * This also applies to dynptr slices belonging to skb and xdp dynptrs, 9285 * since these slices point to packet data. 9286 */ 9287 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 9288 { 9289 struct bpf_func_state *state; 9290 struct bpf_reg_state *reg; 9291 9292 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 9293 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) 9294 mark_reg_invalid(env, reg); 9295 })); 9296 } 9297 9298 enum { 9299 AT_PKT_END = -1, 9300 BEYOND_PKT_END = -2, 9301 }; 9302 9303 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 9304 { 9305 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 9306 struct bpf_reg_state *reg = &state->regs[regn]; 9307 9308 if (reg->type != PTR_TO_PACKET) 9309 /* PTR_TO_PACKET_META is not supported yet */ 9310 return; 9311 9312 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 9313 * How far beyond pkt_end it goes is unknown. 9314 * if (!range_open) it's the case of pkt >= pkt_end 9315 * if (range_open) it's the case of pkt > pkt_end 9316 * hence this pointer is at least 1 byte bigger than pkt_end 9317 */ 9318 if (range_open) 9319 reg->range = BEYOND_PKT_END; 9320 else 9321 reg->range = AT_PKT_END; 9322 } 9323 9324 /* The pointer with the specified id has released its reference to kernel 9325 * resources. Identify all copies of the same pointer and clear the reference. 9326 */ 9327 static int release_reference(struct bpf_verifier_env *env, 9328 int ref_obj_id) 9329 { 9330 struct bpf_func_state *state; 9331 struct bpf_reg_state *reg; 9332 int err; 9333 9334 err = release_reference_state(cur_func(env), ref_obj_id); 9335 if (err) 9336 return err; 9337 9338 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 9339 if (reg->ref_obj_id == ref_obj_id) 9340 mark_reg_invalid(env, reg); 9341 })); 9342 9343 return 0; 9344 } 9345 9346 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) 9347 { 9348 struct bpf_func_state *unused; 9349 struct bpf_reg_state *reg; 9350 9351 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 9352 if (type_is_non_owning_ref(reg->type)) 9353 mark_reg_invalid(env, reg); 9354 })); 9355 } 9356 9357 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 9358 struct bpf_reg_state *regs) 9359 { 9360 int i; 9361 9362 /* after the call registers r0 - r5 were scratched */ 9363 for (i = 0; i < CALLER_SAVED_REGS; i++) { 9364 mark_reg_not_init(env, regs, caller_saved[i]); 9365 __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK); 9366 } 9367 } 9368 9369 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 9370 struct bpf_func_state *caller, 9371 struct bpf_func_state *callee, 9372 int insn_idx); 9373 9374 static int set_callee_state(struct bpf_verifier_env *env, 9375 struct bpf_func_state *caller, 9376 struct bpf_func_state *callee, int insn_idx); 9377 9378 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, 9379 set_callee_state_fn set_callee_state_cb, 9380 struct bpf_verifier_state *state) 9381 { 9382 struct bpf_func_state *caller, *callee; 9383 int err; 9384 9385 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 9386 verbose(env, "the call stack of %d frames is too deep\n", 9387 state->curframe + 2); 9388 return -E2BIG; 9389 } 9390 9391 if (state->frame[state->curframe + 1]) { 9392 verbose(env, "verifier bug. Frame %d already allocated\n", 9393 state->curframe + 1); 9394 return -EFAULT; 9395 } 9396 9397 caller = state->frame[state->curframe]; 9398 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 9399 if (!callee) 9400 return -ENOMEM; 9401 state->frame[state->curframe + 1] = callee; 9402 9403 /* callee cannot access r0, r6 - r9 for reading and has to write 9404 * into its own stack before reading from it. 9405 * callee can read/write into caller's stack 9406 */ 9407 init_func_state(env, callee, 9408 /* remember the callsite, it will be used by bpf_exit */ 9409 callsite, 9410 state->curframe + 1 /* frameno within this callchain */, 9411 subprog /* subprog number within this prog */); 9412 /* Transfer references to the callee */ 9413 err = copy_reference_state(callee, caller); 9414 err = err ?: set_callee_state_cb(env, caller, callee, callsite); 9415 if (err) 9416 goto err_out; 9417 9418 /* only increment it after check_reg_arg() finished */ 9419 state->curframe++; 9420 9421 return 0; 9422 9423 err_out: 9424 free_func_state(callee); 9425 state->frame[state->curframe + 1] = NULL; 9426 return err; 9427 } 9428 9429 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, 9430 const struct btf *btf, 9431 struct bpf_reg_state *regs) 9432 { 9433 struct bpf_subprog_info *sub = subprog_info(env, subprog); 9434 struct bpf_verifier_log *log = &env->log; 9435 u32 i; 9436 int ret; 9437 9438 ret = btf_prepare_func_args(env, subprog); 9439 if (ret) 9440 return ret; 9441 9442 /* check that BTF function arguments match actual types that the 9443 * verifier sees. 9444 */ 9445 for (i = 0; i < sub->arg_cnt; i++) { 9446 u32 regno = i + 1; 9447 struct bpf_reg_state *reg = ®s[regno]; 9448 struct bpf_subprog_arg_info *arg = &sub->args[i]; 9449 9450 if (arg->arg_type == ARG_ANYTHING) { 9451 if (reg->type != SCALAR_VALUE) { 9452 bpf_log(log, "R%d is not a scalar\n", regno); 9453 return -EINVAL; 9454 } 9455 } else if (arg->arg_type == ARG_PTR_TO_CTX) { 9456 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); 9457 if (ret < 0) 9458 return ret; 9459 /* If function expects ctx type in BTF check that caller 9460 * is passing PTR_TO_CTX. 9461 */ 9462 if (reg->type != PTR_TO_CTX) { 9463 bpf_log(log, "arg#%d expects pointer to ctx\n", i); 9464 return -EINVAL; 9465 } 9466 } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { 9467 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); 9468 if (ret < 0) 9469 return ret; 9470 if (check_mem_reg(env, reg, regno, arg->mem_size)) 9471 return -EINVAL; 9472 if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) { 9473 bpf_log(log, "arg#%d is expected to be non-NULL\n", i); 9474 return -EINVAL; 9475 } 9476 } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { 9477 /* 9478 * Can pass any value and the kernel won't crash, but 9479 * only PTR_TO_ARENA or SCALAR make sense. Everything 9480 * else is a bug in the bpf program. Point it out to 9481 * the user at the verification time instead of 9482 * run-time debug nightmare. 9483 */ 9484 if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { 9485 bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno); 9486 return -EINVAL; 9487 } 9488 } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) { 9489 ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR); 9490 if (ret) 9491 return ret; 9492 9493 ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); 9494 if (ret) 9495 return ret; 9496 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { 9497 struct bpf_call_arg_meta meta; 9498 int err; 9499 9500 if (register_is_null(reg) && type_may_be_null(arg->arg_type)) 9501 continue; 9502 9503 memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ 9504 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); 9505 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); 9506 if (err) 9507 return err; 9508 } else { 9509 bpf_log(log, "verifier bug: unrecognized arg#%d type %d\n", 9510 i, arg->arg_type); 9511 return -EFAULT; 9512 } 9513 } 9514 9515 return 0; 9516 } 9517 9518 /* Compare BTF of a function call with given bpf_reg_state. 9519 * Returns: 9520 * EFAULT - there is a verifier bug. Abort verification. 9521 * EINVAL - there is a type mismatch or BTF is not available. 9522 * 0 - BTF matches with what bpf_reg_state expects. 9523 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. 9524 */ 9525 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 9526 struct bpf_reg_state *regs) 9527 { 9528 struct bpf_prog *prog = env->prog; 9529 struct btf *btf = prog->aux->btf; 9530 u32 btf_id; 9531 int err; 9532 9533 if (!prog->aux->func_info) 9534 return -EINVAL; 9535 9536 btf_id = prog->aux->func_info[subprog].type_id; 9537 if (!btf_id) 9538 return -EFAULT; 9539 9540 if (prog->aux->func_info_aux[subprog].unreliable) 9541 return -EINVAL; 9542 9543 err = btf_check_func_arg_match(env, subprog, btf, regs); 9544 /* Compiler optimizations can remove arguments from static functions 9545 * or mismatched type can be passed into a global function. 9546 * In such cases mark the function as unreliable from BTF point of view. 9547 */ 9548 if (err) 9549 prog->aux->func_info_aux[subprog].unreliable = true; 9550 return err; 9551 } 9552 9553 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 9554 int insn_idx, int subprog, 9555 set_callee_state_fn set_callee_state_cb) 9556 { 9557 struct bpf_verifier_state *state = env->cur_state, *callback_state; 9558 struct bpf_func_state *caller, *callee; 9559 int err; 9560 9561 caller = state->frame[state->curframe]; 9562 err = btf_check_subprog_call(env, subprog, caller->regs); 9563 if (err == -EFAULT) 9564 return err; 9565 9566 /* set_callee_state is used for direct subprog calls, but we are 9567 * interested in validating only BPF helpers that can call subprogs as 9568 * callbacks 9569 */ 9570 env->subprog_info[subprog].is_cb = true; 9571 if (bpf_pseudo_kfunc_call(insn) && 9572 !is_callback_calling_kfunc(insn->imm)) { 9573 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", 9574 func_id_name(insn->imm), insn->imm); 9575 return -EFAULT; 9576 } else if (!bpf_pseudo_kfunc_call(insn) && 9577 !is_callback_calling_function(insn->imm)) { /* helper */ 9578 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", 9579 func_id_name(insn->imm), insn->imm); 9580 return -EFAULT; 9581 } 9582 9583 if (is_async_callback_calling_insn(insn)) { 9584 struct bpf_verifier_state *async_cb; 9585 9586 /* there is no real recursion here. timer and workqueue callbacks are async */ 9587 env->subprog_info[subprog].is_async_cb = true; 9588 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 9589 insn_idx, subprog, 9590 is_bpf_wq_set_callback_impl_kfunc(insn->imm)); 9591 if (!async_cb) 9592 return -EFAULT; 9593 callee = async_cb->frame[0]; 9594 callee->async_entry_cnt = caller->async_entry_cnt + 1; 9595 9596 /* Convert bpf_timer_set_callback() args into timer callback args */ 9597 err = set_callee_state_cb(env, caller, callee, insn_idx); 9598 if (err) 9599 return err; 9600 9601 return 0; 9602 } 9603 9604 /* for callback functions enqueue entry to callback and 9605 * proceed with next instruction within current frame. 9606 */ 9607 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); 9608 if (!callback_state) 9609 return -ENOMEM; 9610 9611 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, 9612 callback_state); 9613 if (err) 9614 return err; 9615 9616 callback_state->callback_unroll_depth++; 9617 callback_state->frame[callback_state->curframe - 1]->callback_depth++; 9618 caller->callback_depth = 0; 9619 return 0; 9620 } 9621 9622 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 9623 int *insn_idx) 9624 { 9625 struct bpf_verifier_state *state = env->cur_state; 9626 struct bpf_func_state *caller; 9627 int err, subprog, target_insn; 9628 9629 target_insn = *insn_idx + insn->imm + 1; 9630 subprog = find_subprog(env, target_insn); 9631 if (subprog < 0) { 9632 verbose(env, "verifier bug. No program starts at insn %d\n", target_insn); 9633 return -EFAULT; 9634 } 9635 9636 caller = state->frame[state->curframe]; 9637 err = btf_check_subprog_call(env, subprog, caller->regs); 9638 if (err == -EFAULT) 9639 return err; 9640 if (subprog_is_global(env, subprog)) { 9641 const char *sub_name = subprog_name(env, subprog); 9642 9643 /* Only global subprogs cannot be called with a lock held. */ 9644 if (env->cur_state->active_lock.ptr) { 9645 verbose(env, "global function calls are not allowed while holding a lock,\n" 9646 "use static function instead\n"); 9647 return -EINVAL; 9648 } 9649 9650 /* Only global subprogs cannot be called with preemption disabled. */ 9651 if (env->cur_state->active_preempt_lock) { 9652 verbose(env, "global function calls are not allowed with preemption disabled,\n" 9653 "use static function instead\n"); 9654 return -EINVAL; 9655 } 9656 9657 if (err) { 9658 verbose(env, "Caller passes invalid args into func#%d ('%s')\n", 9659 subprog, sub_name); 9660 return err; 9661 } 9662 9663 verbose(env, "Func#%d ('%s') is global and assumed valid.\n", 9664 subprog, sub_name); 9665 /* mark global subprog for verifying after main prog */ 9666 subprog_aux(env, subprog)->called = true; 9667 clear_caller_saved_regs(env, caller->regs); 9668 9669 /* All global functions return a 64-bit SCALAR_VALUE */ 9670 mark_reg_unknown(env, caller->regs, BPF_REG_0); 9671 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 9672 9673 /* continue with next insn after call */ 9674 return 0; 9675 } 9676 9677 /* for regular function entry setup new frame and continue 9678 * from that frame. 9679 */ 9680 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); 9681 if (err) 9682 return err; 9683 9684 clear_caller_saved_regs(env, caller->regs); 9685 9686 /* and go analyze first insn of the callee */ 9687 *insn_idx = env->subprog_info[subprog].start - 1; 9688 9689 if (env->log.level & BPF_LOG_LEVEL) { 9690 verbose(env, "caller:\n"); 9691 print_verifier_state(env, caller, true); 9692 verbose(env, "callee:\n"); 9693 print_verifier_state(env, state->frame[state->curframe], true); 9694 } 9695 9696 return 0; 9697 } 9698 9699 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 9700 struct bpf_func_state *caller, 9701 struct bpf_func_state *callee) 9702 { 9703 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 9704 * void *callback_ctx, u64 flags); 9705 * callback_fn(struct bpf_map *map, void *key, void *value, 9706 * void *callback_ctx); 9707 */ 9708 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 9709 9710 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 9711 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 9712 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 9713 9714 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 9715 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 9716 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 9717 9718 /* pointer to stack or null */ 9719 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 9720 9721 /* unused */ 9722 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9723 return 0; 9724 } 9725 9726 static int set_callee_state(struct bpf_verifier_env *env, 9727 struct bpf_func_state *caller, 9728 struct bpf_func_state *callee, int insn_idx) 9729 { 9730 int i; 9731 9732 /* copy r1 - r5 args that callee can access. The copy includes parent 9733 * pointers, which connects us up to the liveness chain 9734 */ 9735 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 9736 callee->regs[i] = caller->regs[i]; 9737 return 0; 9738 } 9739 9740 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 9741 struct bpf_func_state *caller, 9742 struct bpf_func_state *callee, 9743 int insn_idx) 9744 { 9745 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 9746 struct bpf_map *map; 9747 int err; 9748 9749 /* valid map_ptr and poison value does not matter */ 9750 map = insn_aux->map_ptr_state.map_ptr; 9751 if (!map->ops->map_set_for_each_callback_args || 9752 !map->ops->map_for_each_callback) { 9753 verbose(env, "callback function not allowed for map\n"); 9754 return -ENOTSUPP; 9755 } 9756 9757 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 9758 if (err) 9759 return err; 9760 9761 callee->in_callback_fn = true; 9762 callee->callback_ret_range = retval_range(0, 1); 9763 return 0; 9764 } 9765 9766 static int set_loop_callback_state(struct bpf_verifier_env *env, 9767 struct bpf_func_state *caller, 9768 struct bpf_func_state *callee, 9769 int insn_idx) 9770 { 9771 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 9772 * u64 flags); 9773 * callback_fn(u32 index, void *callback_ctx); 9774 */ 9775 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 9776 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 9777 9778 /* unused */ 9779 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 9780 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 9781 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9782 9783 callee->in_callback_fn = true; 9784 callee->callback_ret_range = retval_range(0, 1); 9785 return 0; 9786 } 9787 9788 static int set_timer_callback_state(struct bpf_verifier_env *env, 9789 struct bpf_func_state *caller, 9790 struct bpf_func_state *callee, 9791 int insn_idx) 9792 { 9793 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 9794 9795 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 9796 * callback_fn(struct bpf_map *map, void *key, void *value); 9797 */ 9798 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 9799 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 9800 callee->regs[BPF_REG_1].map_ptr = map_ptr; 9801 9802 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 9803 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 9804 callee->regs[BPF_REG_2].map_ptr = map_ptr; 9805 9806 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 9807 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 9808 callee->regs[BPF_REG_3].map_ptr = map_ptr; 9809 9810 /* unused */ 9811 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 9812 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9813 callee->in_async_callback_fn = true; 9814 callee->callback_ret_range = retval_range(0, 1); 9815 return 0; 9816 } 9817 9818 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 9819 struct bpf_func_state *caller, 9820 struct bpf_func_state *callee, 9821 int insn_idx) 9822 { 9823 /* bpf_find_vma(struct task_struct *task, u64 addr, 9824 * void *callback_fn, void *callback_ctx, u64 flags) 9825 * (callback_fn)(struct task_struct *task, 9826 * struct vm_area_struct *vma, void *callback_ctx); 9827 */ 9828 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 9829 9830 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 9831 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 9832 callee->regs[BPF_REG_2].btf = btf_vmlinux; 9833 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA]; 9834 9835 /* pointer to stack or null */ 9836 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 9837 9838 /* unused */ 9839 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 9840 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9841 callee->in_callback_fn = true; 9842 callee->callback_ret_range = retval_range(0, 1); 9843 return 0; 9844 } 9845 9846 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 9847 struct bpf_func_state *caller, 9848 struct bpf_func_state *callee, 9849 int insn_idx) 9850 { 9851 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 9852 * callback_ctx, u64 flags); 9853 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx); 9854 */ 9855 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 9856 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); 9857 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 9858 9859 /* unused */ 9860 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 9861 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 9862 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9863 9864 callee->in_callback_fn = true; 9865 callee->callback_ret_range = retval_range(0, 1); 9866 return 0; 9867 } 9868 9869 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, 9870 struct bpf_func_state *caller, 9871 struct bpf_func_state *callee, 9872 int insn_idx) 9873 { 9874 /* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 9875 * bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)); 9876 * 9877 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset 9878 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd 9879 * by this point, so look at 'root' 9880 */ 9881 struct btf_field *field; 9882 9883 field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off, 9884 BPF_RB_ROOT); 9885 if (!field || !field->graph_root.value_btf_id) 9886 return -EFAULT; 9887 9888 mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root); 9889 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); 9890 mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root); 9891 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); 9892 9893 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 9894 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 9895 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 9896 callee->in_callback_fn = true; 9897 callee->callback_ret_range = retval_range(0, 1); 9898 return 0; 9899 } 9900 9901 static bool is_rbtree_lock_required_kfunc(u32 btf_id); 9902 9903 /* Are we currently verifying the callback for a rbtree helper that must 9904 * be called with lock held? If so, no need to complain about unreleased 9905 * lock 9906 */ 9907 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) 9908 { 9909 struct bpf_verifier_state *state = env->cur_state; 9910 struct bpf_insn *insn = env->prog->insnsi; 9911 struct bpf_func_state *callee; 9912 int kfunc_btf_id; 9913 9914 if (!state->curframe) 9915 return false; 9916 9917 callee = state->frame[state->curframe]; 9918 9919 if (!callee->in_callback_fn) 9920 return false; 9921 9922 kfunc_btf_id = insn[callee->callsite].imm; 9923 return is_rbtree_lock_required_kfunc(kfunc_btf_id); 9924 } 9925 9926 static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg) 9927 { 9928 return range.minval <= reg->smin_value && reg->smax_value <= range.maxval; 9929 } 9930 9931 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 9932 { 9933 struct bpf_verifier_state *state = env->cur_state, *prev_st; 9934 struct bpf_func_state *caller, *callee; 9935 struct bpf_reg_state *r0; 9936 bool in_callback_fn; 9937 int err; 9938 9939 callee = state->frame[state->curframe]; 9940 r0 = &callee->regs[BPF_REG_0]; 9941 if (r0->type == PTR_TO_STACK) { 9942 /* technically it's ok to return caller's stack pointer 9943 * (or caller's caller's pointer) back to the caller, 9944 * since these pointers are valid. Only current stack 9945 * pointer will be invalid as soon as function exits, 9946 * but let's be conservative 9947 */ 9948 verbose(env, "cannot return stack pointer to the caller\n"); 9949 return -EINVAL; 9950 } 9951 9952 caller = state->frame[state->curframe - 1]; 9953 if (callee->in_callback_fn) { 9954 if (r0->type != SCALAR_VALUE) { 9955 verbose(env, "R0 not a scalar value\n"); 9956 return -EACCES; 9957 } 9958 9959 /* we are going to rely on register's precise value */ 9960 err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64); 9961 err = err ?: mark_chain_precision(env, BPF_REG_0); 9962 if (err) 9963 return err; 9964 9965 /* enforce R0 return value range */ 9966 if (!retval_range_within(callee->callback_ret_range, r0)) { 9967 verbose_invalid_scalar(env, r0, callee->callback_ret_range, 9968 "At callback return", "R0"); 9969 return -EINVAL; 9970 } 9971 if (!calls_callback(env, callee->callsite)) { 9972 verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n", 9973 *insn_idx, callee->callsite); 9974 return -EFAULT; 9975 } 9976 } else { 9977 /* return to the caller whatever r0 had in the callee */ 9978 caller->regs[BPF_REG_0] = *r0; 9979 } 9980 9981 /* callback_fn frame should have released its own additions to parent's 9982 * reference state at this point, or check_reference_leak would 9983 * complain, hence it must be the same as the caller. There is no need 9984 * to copy it back. 9985 */ 9986 if (!callee->in_callback_fn) { 9987 /* Transfer references to the caller */ 9988 err = copy_reference_state(caller, callee); 9989 if (err) 9990 return err; 9991 } 9992 9993 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, 9994 * there function call logic would reschedule callback visit. If iteration 9995 * converges is_state_visited() would prune that visit eventually. 9996 */ 9997 in_callback_fn = callee->in_callback_fn; 9998 if (in_callback_fn) 9999 *insn_idx = callee->callsite; 10000 else 10001 *insn_idx = callee->callsite + 1; 10002 10003 if (env->log.level & BPF_LOG_LEVEL) { 10004 verbose(env, "returning from callee:\n"); 10005 print_verifier_state(env, callee, true); 10006 verbose(env, "to caller at %d:\n", *insn_idx); 10007 print_verifier_state(env, caller, true); 10008 } 10009 /* clear everything in the callee. In case of exceptional exits using 10010 * bpf_throw, this will be done by copy_verifier_state for extra frames. */ 10011 free_func_state(callee); 10012 state->frame[state->curframe--] = NULL; 10013 10014 /* for callbacks widen imprecise scalars to make programs like below verify: 10015 * 10016 * struct ctx { int i; } 10017 * void cb(int idx, struct ctx *ctx) { ctx->i++; ... } 10018 * ... 10019 * struct ctx = { .i = 0; } 10020 * bpf_loop(100, cb, &ctx, 0); 10021 * 10022 * This is similar to what is done in process_iter_next_call() for open 10023 * coded iterators. 10024 */ 10025 prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL; 10026 if (prev_st) { 10027 err = widen_imprecise_scalars(env, prev_st, state); 10028 if (err) 10029 return err; 10030 } 10031 return 0; 10032 } 10033 10034 static int do_refine_retval_range(struct bpf_verifier_env *env, 10035 struct bpf_reg_state *regs, int ret_type, 10036 int func_id, 10037 struct bpf_call_arg_meta *meta) 10038 { 10039 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 10040 10041 if (ret_type != RET_INTEGER) 10042 return 0; 10043 10044 switch (func_id) { 10045 case BPF_FUNC_get_stack: 10046 case BPF_FUNC_get_task_stack: 10047 case BPF_FUNC_probe_read_str: 10048 case BPF_FUNC_probe_read_kernel_str: 10049 case BPF_FUNC_probe_read_user_str: 10050 ret_reg->smax_value = meta->msize_max_value; 10051 ret_reg->s32_max_value = meta->msize_max_value; 10052 ret_reg->smin_value = -MAX_ERRNO; 10053 ret_reg->s32_min_value = -MAX_ERRNO; 10054 reg_bounds_sync(ret_reg); 10055 break; 10056 case BPF_FUNC_get_smp_processor_id: 10057 ret_reg->umax_value = nr_cpu_ids - 1; 10058 ret_reg->u32_max_value = nr_cpu_ids - 1; 10059 ret_reg->smax_value = nr_cpu_ids - 1; 10060 ret_reg->s32_max_value = nr_cpu_ids - 1; 10061 ret_reg->umin_value = 0; 10062 ret_reg->u32_min_value = 0; 10063 ret_reg->smin_value = 0; 10064 ret_reg->s32_min_value = 0; 10065 reg_bounds_sync(ret_reg); 10066 break; 10067 } 10068 10069 return reg_bounds_sanity_check(env, ret_reg, "retval"); 10070 } 10071 10072 static int 10073 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 10074 int func_id, int insn_idx) 10075 { 10076 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 10077 struct bpf_map *map = meta->map_ptr; 10078 10079 if (func_id != BPF_FUNC_tail_call && 10080 func_id != BPF_FUNC_map_lookup_elem && 10081 func_id != BPF_FUNC_map_update_elem && 10082 func_id != BPF_FUNC_map_delete_elem && 10083 func_id != BPF_FUNC_map_push_elem && 10084 func_id != BPF_FUNC_map_pop_elem && 10085 func_id != BPF_FUNC_map_peek_elem && 10086 func_id != BPF_FUNC_for_each_map_elem && 10087 func_id != BPF_FUNC_redirect_map && 10088 func_id != BPF_FUNC_map_lookup_percpu_elem) 10089 return 0; 10090 10091 if (map == NULL) { 10092 verbose(env, "kernel subsystem misconfigured verifier\n"); 10093 return -EINVAL; 10094 } 10095 10096 /* In case of read-only, some additional restrictions 10097 * need to be applied in order to prevent altering the 10098 * state of the map from program side. 10099 */ 10100 if ((map->map_flags & BPF_F_RDONLY_PROG) && 10101 (func_id == BPF_FUNC_map_delete_elem || 10102 func_id == BPF_FUNC_map_update_elem || 10103 func_id == BPF_FUNC_map_push_elem || 10104 func_id == BPF_FUNC_map_pop_elem)) { 10105 verbose(env, "write into map forbidden\n"); 10106 return -EACCES; 10107 } 10108 10109 if (!aux->map_ptr_state.map_ptr) 10110 bpf_map_ptr_store(aux, meta->map_ptr, 10111 !meta->map_ptr->bypass_spec_v1, false); 10112 else if (aux->map_ptr_state.map_ptr != meta->map_ptr) 10113 bpf_map_ptr_store(aux, meta->map_ptr, 10114 !meta->map_ptr->bypass_spec_v1, true); 10115 return 0; 10116 } 10117 10118 static int 10119 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 10120 int func_id, int insn_idx) 10121 { 10122 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 10123 struct bpf_reg_state *regs = cur_regs(env), *reg; 10124 struct bpf_map *map = meta->map_ptr; 10125 u64 val, max; 10126 int err; 10127 10128 if (func_id != BPF_FUNC_tail_call) 10129 return 0; 10130 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 10131 verbose(env, "kernel subsystem misconfigured verifier\n"); 10132 return -EINVAL; 10133 } 10134 10135 reg = ®s[BPF_REG_3]; 10136 val = reg->var_off.value; 10137 max = map->max_entries; 10138 10139 if (!(is_reg_const(reg, false) && val < max)) { 10140 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 10141 return 0; 10142 } 10143 10144 err = mark_chain_precision(env, BPF_REG_3); 10145 if (err) 10146 return err; 10147 if (bpf_map_key_unseen(aux)) 10148 bpf_map_key_store(aux, val); 10149 else if (!bpf_map_key_poisoned(aux) && 10150 bpf_map_key_immediate(aux) != val) 10151 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 10152 return 0; 10153 } 10154 10155 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) 10156 { 10157 struct bpf_func_state *state = cur_func(env); 10158 bool refs_lingering = false; 10159 int i; 10160 10161 if (!exception_exit && state->frameno && !state->in_callback_fn) 10162 return 0; 10163 10164 for (i = 0; i < state->acquired_refs; i++) { 10165 if (!exception_exit && state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 10166 continue; 10167 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 10168 state->refs[i].id, state->refs[i].insn_idx); 10169 refs_lingering = true; 10170 } 10171 return refs_lingering ? -EINVAL : 0; 10172 } 10173 10174 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 10175 struct bpf_reg_state *regs) 10176 { 10177 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 10178 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 10179 struct bpf_map *fmt_map = fmt_reg->map_ptr; 10180 struct bpf_bprintf_data data = {}; 10181 int err, fmt_map_off, num_args; 10182 u64 fmt_addr; 10183 char *fmt; 10184 10185 /* data must be an array of u64 */ 10186 if (data_len_reg->var_off.value % 8) 10187 return -EINVAL; 10188 num_args = data_len_reg->var_off.value / 8; 10189 10190 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 10191 * and map_direct_value_addr is set. 10192 */ 10193 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 10194 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 10195 fmt_map_off); 10196 if (err) { 10197 verbose(env, "verifier bug\n"); 10198 return -EFAULT; 10199 } 10200 fmt = (char *)(long)fmt_addr + fmt_map_off; 10201 10202 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 10203 * can focus on validating the format specifiers. 10204 */ 10205 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data); 10206 if (err < 0) 10207 verbose(env, "Invalid format string\n"); 10208 10209 return err; 10210 } 10211 10212 static int check_get_func_ip(struct bpf_verifier_env *env) 10213 { 10214 enum bpf_prog_type type = resolve_prog_type(env->prog); 10215 int func_id = BPF_FUNC_get_func_ip; 10216 10217 if (type == BPF_PROG_TYPE_TRACING) { 10218 if (!bpf_prog_has_trampoline(env->prog)) { 10219 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 10220 func_id_name(func_id), func_id); 10221 return -ENOTSUPP; 10222 } 10223 return 0; 10224 } else if (type == BPF_PROG_TYPE_KPROBE) { 10225 return 0; 10226 } 10227 10228 verbose(env, "func %s#%d not supported for program type %d\n", 10229 func_id_name(func_id), func_id, type); 10230 return -ENOTSUPP; 10231 } 10232 10233 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 10234 { 10235 return &env->insn_aux_data[env->insn_idx]; 10236 } 10237 10238 static bool loop_flag_is_zero(struct bpf_verifier_env *env) 10239 { 10240 struct bpf_reg_state *regs = cur_regs(env); 10241 struct bpf_reg_state *reg = ®s[BPF_REG_4]; 10242 bool reg_is_null = register_is_null(reg); 10243 10244 if (reg_is_null) 10245 mark_chain_precision(env, BPF_REG_4); 10246 10247 return reg_is_null; 10248 } 10249 10250 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 10251 { 10252 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 10253 10254 if (!state->initialized) { 10255 state->initialized = 1; 10256 state->fit_for_inline = loop_flag_is_zero(env); 10257 state->callback_subprogno = subprogno; 10258 return; 10259 } 10260 10261 if (!state->fit_for_inline) 10262 return; 10263 10264 state->fit_for_inline = (loop_flag_is_zero(env) && 10265 state->callback_subprogno == subprogno); 10266 } 10267 10268 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 10269 int *insn_idx_p) 10270 { 10271 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 10272 bool returns_cpu_specific_alloc_ptr = false; 10273 const struct bpf_func_proto *fn = NULL; 10274 enum bpf_return_type ret_type; 10275 enum bpf_type_flag ret_flag; 10276 struct bpf_reg_state *regs; 10277 struct bpf_call_arg_meta meta; 10278 int insn_idx = *insn_idx_p; 10279 bool changes_data; 10280 int i, err, func_id; 10281 10282 /* find function prototype */ 10283 func_id = insn->imm; 10284 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 10285 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 10286 func_id); 10287 return -EINVAL; 10288 } 10289 10290 if (env->ops->get_func_proto) 10291 fn = env->ops->get_func_proto(func_id, env->prog); 10292 if (!fn) { 10293 verbose(env, "program of this type cannot use helper %s#%d\n", 10294 func_id_name(func_id), func_id); 10295 return -EINVAL; 10296 } 10297 10298 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 10299 if (!env->prog->gpl_compatible && fn->gpl_only) { 10300 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 10301 return -EINVAL; 10302 } 10303 10304 if (fn->allowed && !fn->allowed(env->prog)) { 10305 verbose(env, "helper call is not allowed in probe\n"); 10306 return -EINVAL; 10307 } 10308 10309 if (!in_sleepable(env) && fn->might_sleep) { 10310 verbose(env, "helper call might sleep in a non-sleepable prog\n"); 10311 return -EINVAL; 10312 } 10313 10314 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 10315 changes_data = bpf_helper_changes_pkt_data(fn->func); 10316 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 10317 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 10318 func_id_name(func_id), func_id); 10319 return -EINVAL; 10320 } 10321 10322 memset(&meta, 0, sizeof(meta)); 10323 meta.pkt_access = fn->pkt_access; 10324 10325 err = check_func_proto(fn, func_id); 10326 if (err) { 10327 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 10328 func_id_name(func_id), func_id); 10329 return err; 10330 } 10331 10332 if (env->cur_state->active_rcu_lock) { 10333 if (fn->might_sleep) { 10334 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", 10335 func_id_name(func_id), func_id); 10336 return -EINVAL; 10337 } 10338 10339 if (in_sleepable(env) && is_storage_get_function(func_id)) 10340 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 10341 } 10342 10343 if (env->cur_state->active_preempt_lock) { 10344 if (fn->might_sleep) { 10345 verbose(env, "sleepable helper %s#%d in non-preemptible region\n", 10346 func_id_name(func_id), func_id); 10347 return -EINVAL; 10348 } 10349 10350 if (in_sleepable(env) && is_storage_get_function(func_id)) 10351 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 10352 } 10353 10354 meta.func_id = func_id; 10355 /* check args */ 10356 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 10357 err = check_func_arg(env, i, &meta, fn, insn_idx); 10358 if (err) 10359 return err; 10360 } 10361 10362 err = record_func_map(env, &meta, func_id, insn_idx); 10363 if (err) 10364 return err; 10365 10366 err = record_func_key(env, &meta, func_id, insn_idx); 10367 if (err) 10368 return err; 10369 10370 /* Mark slots with STACK_MISC in case of raw mode, stack offset 10371 * is inferred from register state. 10372 */ 10373 for (i = 0; i < meta.access_size; i++) { 10374 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 10375 BPF_WRITE, -1, false, false); 10376 if (err) 10377 return err; 10378 } 10379 10380 regs = cur_regs(env); 10381 10382 if (meta.release_regno) { 10383 err = -EINVAL; 10384 /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot 10385 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr 10386 * is safe to do directly. 10387 */ 10388 if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { 10389 if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) { 10390 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); 10391 return -EFAULT; 10392 } 10393 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); 10394 } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) { 10395 u32 ref_obj_id = meta.ref_obj_id; 10396 bool in_rcu = in_rcu_cs(env); 10397 struct bpf_func_state *state; 10398 struct bpf_reg_state *reg; 10399 10400 err = release_reference_state(cur_func(env), ref_obj_id); 10401 if (!err) { 10402 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 10403 if (reg->ref_obj_id == ref_obj_id) { 10404 if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { 10405 reg->ref_obj_id = 0; 10406 reg->type &= ~MEM_ALLOC; 10407 reg->type |= MEM_RCU; 10408 } else { 10409 mark_reg_invalid(env, reg); 10410 } 10411 } 10412 })); 10413 } 10414 } else if (meta.ref_obj_id) { 10415 err = release_reference(env, meta.ref_obj_id); 10416 } else if (register_is_null(®s[meta.release_regno])) { 10417 /* meta.ref_obj_id can only be 0 if register that is meant to be 10418 * released is NULL, which must be > R0. 10419 */ 10420 err = 0; 10421 } 10422 if (err) { 10423 verbose(env, "func %s#%d reference has not been acquired before\n", 10424 func_id_name(func_id), func_id); 10425 return err; 10426 } 10427 } 10428 10429 switch (func_id) { 10430 case BPF_FUNC_tail_call: 10431 err = check_reference_leak(env, false); 10432 if (err) { 10433 verbose(env, "tail_call would lead to reference leak\n"); 10434 return err; 10435 } 10436 break; 10437 case BPF_FUNC_get_local_storage: 10438 /* check that flags argument in get_local_storage(map, flags) is 0, 10439 * this is required because get_local_storage() can't return an error. 10440 */ 10441 if (!register_is_null(®s[BPF_REG_2])) { 10442 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 10443 return -EINVAL; 10444 } 10445 break; 10446 case BPF_FUNC_for_each_map_elem: 10447 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 10448 set_map_elem_callback_state); 10449 break; 10450 case BPF_FUNC_timer_set_callback: 10451 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 10452 set_timer_callback_state); 10453 break; 10454 case BPF_FUNC_find_vma: 10455 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 10456 set_find_vma_callback_state); 10457 break; 10458 case BPF_FUNC_snprintf: 10459 err = check_bpf_snprintf_call(env, regs); 10460 break; 10461 case BPF_FUNC_loop: 10462 update_loop_inline_state(env, meta.subprogno); 10463 /* Verifier relies on R1 value to determine if bpf_loop() iteration 10464 * is finished, thus mark it precise. 10465 */ 10466 err = mark_chain_precision(env, BPF_REG_1); 10467 if (err) 10468 return err; 10469 if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) { 10470 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 10471 set_loop_callback_state); 10472 } else { 10473 cur_func(env)->callback_depth = 0; 10474 if (env->log.level & BPF_LOG_LEVEL2) 10475 verbose(env, "frame%d bpf_loop iteration limit reached\n", 10476 env->cur_state->curframe); 10477 } 10478 break; 10479 case BPF_FUNC_dynptr_from_mem: 10480 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 10481 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 10482 reg_type_str(env, regs[BPF_REG_1].type)); 10483 return -EACCES; 10484 } 10485 break; 10486 case BPF_FUNC_set_retval: 10487 if (prog_type == BPF_PROG_TYPE_LSM && 10488 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 10489 if (!env->prog->aux->attach_func_proto->type) { 10490 /* Make sure programs that attach to void 10491 * hooks don't try to modify return value. 10492 */ 10493 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 10494 return -EINVAL; 10495 } 10496 } 10497 break; 10498 case BPF_FUNC_dynptr_data: 10499 { 10500 struct bpf_reg_state *reg; 10501 int id, ref_obj_id; 10502 10503 reg = get_dynptr_arg_reg(env, fn, regs); 10504 if (!reg) 10505 return -EFAULT; 10506 10507 10508 if (meta.dynptr_id) { 10509 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); 10510 return -EFAULT; 10511 } 10512 if (meta.ref_obj_id) { 10513 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); 10514 return -EFAULT; 10515 } 10516 10517 id = dynptr_id(env, reg); 10518 if (id < 0) { 10519 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 10520 return id; 10521 } 10522 10523 ref_obj_id = dynptr_ref_obj_id(env, reg); 10524 if (ref_obj_id < 0) { 10525 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); 10526 return ref_obj_id; 10527 } 10528 10529 meta.dynptr_id = id; 10530 meta.ref_obj_id = ref_obj_id; 10531 10532 break; 10533 } 10534 case BPF_FUNC_dynptr_write: 10535 { 10536 enum bpf_dynptr_type dynptr_type; 10537 struct bpf_reg_state *reg; 10538 10539 reg = get_dynptr_arg_reg(env, fn, regs); 10540 if (!reg) 10541 return -EFAULT; 10542 10543 dynptr_type = dynptr_get_type(env, reg); 10544 if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) 10545 return -EFAULT; 10546 10547 if (dynptr_type == BPF_DYNPTR_TYPE_SKB) 10548 /* this will trigger clear_all_pkt_pointers(), which will 10549 * invalidate all dynptr slices associated with the skb 10550 */ 10551 changes_data = true; 10552 10553 break; 10554 } 10555 case BPF_FUNC_per_cpu_ptr: 10556 case BPF_FUNC_this_cpu_ptr: 10557 { 10558 struct bpf_reg_state *reg = ®s[BPF_REG_1]; 10559 const struct btf_type *type; 10560 10561 if (reg->type & MEM_RCU) { 10562 type = btf_type_by_id(reg->btf, reg->btf_id); 10563 if (!type || !btf_type_is_struct(type)) { 10564 verbose(env, "Helper has invalid btf/btf_id in R1\n"); 10565 return -EFAULT; 10566 } 10567 returns_cpu_specific_alloc_ptr = true; 10568 env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; 10569 } 10570 break; 10571 } 10572 case BPF_FUNC_user_ringbuf_drain: 10573 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 10574 set_user_ringbuf_callback_state); 10575 break; 10576 } 10577 10578 if (err) 10579 return err; 10580 10581 /* reset caller saved regs */ 10582 for (i = 0; i < CALLER_SAVED_REGS; i++) { 10583 mark_reg_not_init(env, regs, caller_saved[i]); 10584 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 10585 } 10586 10587 /* helper call returns 64-bit value. */ 10588 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 10589 10590 /* update return register (already marked as written above) */ 10591 ret_type = fn->ret_type; 10592 ret_flag = type_flag(ret_type); 10593 10594 switch (base_type(ret_type)) { 10595 case RET_INTEGER: 10596 /* sets type to SCALAR_VALUE */ 10597 mark_reg_unknown(env, regs, BPF_REG_0); 10598 break; 10599 case RET_VOID: 10600 regs[BPF_REG_0].type = NOT_INIT; 10601 break; 10602 case RET_PTR_TO_MAP_VALUE: 10603 /* There is no offset yet applied, variable or fixed */ 10604 mark_reg_known_zero(env, regs, BPF_REG_0); 10605 /* remember map_ptr, so that check_map_access() 10606 * can check 'value_size' boundary of memory access 10607 * to map element returned from bpf_map_lookup_elem() 10608 */ 10609 if (meta.map_ptr == NULL) { 10610 verbose(env, 10611 "kernel subsystem misconfigured verifier\n"); 10612 return -EINVAL; 10613 } 10614 regs[BPF_REG_0].map_ptr = meta.map_ptr; 10615 regs[BPF_REG_0].map_uid = meta.map_uid; 10616 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 10617 if (!type_may_be_null(ret_type) && 10618 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 10619 regs[BPF_REG_0].id = ++env->id_gen; 10620 } 10621 break; 10622 case RET_PTR_TO_SOCKET: 10623 mark_reg_known_zero(env, regs, BPF_REG_0); 10624 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 10625 break; 10626 case RET_PTR_TO_SOCK_COMMON: 10627 mark_reg_known_zero(env, regs, BPF_REG_0); 10628 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 10629 break; 10630 case RET_PTR_TO_TCP_SOCK: 10631 mark_reg_known_zero(env, regs, BPF_REG_0); 10632 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 10633 break; 10634 case RET_PTR_TO_MEM: 10635 mark_reg_known_zero(env, regs, BPF_REG_0); 10636 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 10637 regs[BPF_REG_0].mem_size = meta.mem_size; 10638 break; 10639 case RET_PTR_TO_MEM_OR_BTF_ID: 10640 { 10641 const struct btf_type *t; 10642 10643 mark_reg_known_zero(env, regs, BPF_REG_0); 10644 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 10645 if (!btf_type_is_struct(t)) { 10646 u32 tsize; 10647 const struct btf_type *ret; 10648 const char *tname; 10649 10650 /* resolve the type size of ksym. */ 10651 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 10652 if (IS_ERR(ret)) { 10653 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 10654 verbose(env, "unable to resolve the size of type '%s': %ld\n", 10655 tname, PTR_ERR(ret)); 10656 return -EINVAL; 10657 } 10658 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 10659 regs[BPF_REG_0].mem_size = tsize; 10660 } else { 10661 if (returns_cpu_specific_alloc_ptr) { 10662 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU; 10663 } else { 10664 /* MEM_RDONLY may be carried from ret_flag, but it 10665 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 10666 * it will confuse the check of PTR_TO_BTF_ID in 10667 * check_mem_access(). 10668 */ 10669 ret_flag &= ~MEM_RDONLY; 10670 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 10671 } 10672 10673 regs[BPF_REG_0].btf = meta.ret_btf; 10674 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 10675 } 10676 break; 10677 } 10678 case RET_PTR_TO_BTF_ID: 10679 { 10680 struct btf *ret_btf; 10681 int ret_btf_id; 10682 10683 mark_reg_known_zero(env, regs, BPF_REG_0); 10684 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 10685 if (func_id == BPF_FUNC_kptr_xchg) { 10686 ret_btf = meta.kptr_field->kptr.btf; 10687 ret_btf_id = meta.kptr_field->kptr.btf_id; 10688 if (!btf_is_kernel(ret_btf)) { 10689 regs[BPF_REG_0].type |= MEM_ALLOC; 10690 if (meta.kptr_field->type == BPF_KPTR_PERCPU) 10691 regs[BPF_REG_0].type |= MEM_PERCPU; 10692 } 10693 } else { 10694 if (fn->ret_btf_id == BPF_PTR_POISON) { 10695 verbose(env, "verifier internal error:"); 10696 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", 10697 func_id_name(func_id)); 10698 return -EINVAL; 10699 } 10700 ret_btf = btf_vmlinux; 10701 ret_btf_id = *fn->ret_btf_id; 10702 } 10703 if (ret_btf_id == 0) { 10704 verbose(env, "invalid return type %u of func %s#%d\n", 10705 base_type(ret_type), func_id_name(func_id), 10706 func_id); 10707 return -EINVAL; 10708 } 10709 regs[BPF_REG_0].btf = ret_btf; 10710 regs[BPF_REG_0].btf_id = ret_btf_id; 10711 break; 10712 } 10713 default: 10714 verbose(env, "unknown return type %u of func %s#%d\n", 10715 base_type(ret_type), func_id_name(func_id), func_id); 10716 return -EINVAL; 10717 } 10718 10719 if (type_may_be_null(regs[BPF_REG_0].type)) 10720 regs[BPF_REG_0].id = ++env->id_gen; 10721 10722 if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { 10723 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", 10724 func_id_name(func_id), func_id); 10725 return -EFAULT; 10726 } 10727 10728 if (is_dynptr_ref_function(func_id)) 10729 regs[BPF_REG_0].dynptr_id = meta.dynptr_id; 10730 10731 if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { 10732 /* For release_reference() */ 10733 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 10734 } else if (is_acquire_function(func_id, meta.map_ptr)) { 10735 int id = acquire_reference_state(env, insn_idx); 10736 10737 if (id < 0) 10738 return id; 10739 /* For mark_ptr_or_null_reg() */ 10740 regs[BPF_REG_0].id = id; 10741 /* For release_reference() */ 10742 regs[BPF_REG_0].ref_obj_id = id; 10743 } 10744 10745 err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); 10746 if (err) 10747 return err; 10748 10749 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 10750 if (err) 10751 return err; 10752 10753 if ((func_id == BPF_FUNC_get_stack || 10754 func_id == BPF_FUNC_get_task_stack) && 10755 !env->prog->has_callchain_buf) { 10756 const char *err_str; 10757 10758 #ifdef CONFIG_PERF_EVENTS 10759 err = get_callchain_buffers(sysctl_perf_event_max_stack); 10760 err_str = "cannot get callchain buffer for func %s#%d\n"; 10761 #else 10762 err = -ENOTSUPP; 10763 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 10764 #endif 10765 if (err) { 10766 verbose(env, err_str, func_id_name(func_id), func_id); 10767 return err; 10768 } 10769 10770 env->prog->has_callchain_buf = true; 10771 } 10772 10773 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 10774 env->prog->call_get_stack = true; 10775 10776 if (func_id == BPF_FUNC_get_func_ip) { 10777 if (check_get_func_ip(env)) 10778 return -ENOTSUPP; 10779 env->prog->call_get_func_ip = true; 10780 } 10781 10782 if (changes_data) 10783 clear_all_pkt_pointers(env); 10784 return 0; 10785 } 10786 10787 /* mark_btf_func_reg_size() is used when the reg size is determined by 10788 * the BTF func_proto's return value size and argument. 10789 */ 10790 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 10791 size_t reg_size) 10792 { 10793 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 10794 10795 if (regno == BPF_REG_0) { 10796 /* Function return value */ 10797 reg->live |= REG_LIVE_WRITTEN; 10798 reg->subreg_def = reg_size == sizeof(u64) ? 10799 DEF_NOT_SUBREG : env->insn_idx + 1; 10800 } else { 10801 /* Function argument */ 10802 if (reg_size == sizeof(u64)) { 10803 mark_insn_zext(env, reg); 10804 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 10805 } else { 10806 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 10807 } 10808 } 10809 } 10810 10811 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) 10812 { 10813 return meta->kfunc_flags & KF_ACQUIRE; 10814 } 10815 10816 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) 10817 { 10818 return meta->kfunc_flags & KF_RELEASE; 10819 } 10820 10821 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta) 10822 { 10823 return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta); 10824 } 10825 10826 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) 10827 { 10828 return meta->kfunc_flags & KF_SLEEPABLE; 10829 } 10830 10831 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) 10832 { 10833 return meta->kfunc_flags & KF_DESTRUCTIVE; 10834 } 10835 10836 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) 10837 { 10838 return meta->kfunc_flags & KF_RCU; 10839 } 10840 10841 static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta) 10842 { 10843 return meta->kfunc_flags & KF_RCU_PROTECTED; 10844 } 10845 10846 static bool is_kfunc_arg_mem_size(const struct btf *btf, 10847 const struct btf_param *arg, 10848 const struct bpf_reg_state *reg) 10849 { 10850 const struct btf_type *t; 10851 10852 t = btf_type_skip_modifiers(btf, arg->type, NULL); 10853 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 10854 return false; 10855 10856 return btf_param_match_suffix(btf, arg, "__sz"); 10857 } 10858 10859 static bool is_kfunc_arg_const_mem_size(const struct btf *btf, 10860 const struct btf_param *arg, 10861 const struct bpf_reg_state *reg) 10862 { 10863 const struct btf_type *t; 10864 10865 t = btf_type_skip_modifiers(btf, arg->type, NULL); 10866 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 10867 return false; 10868 10869 return btf_param_match_suffix(btf, arg, "__szk"); 10870 } 10871 10872 static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg) 10873 { 10874 return btf_param_match_suffix(btf, arg, "__opt"); 10875 } 10876 10877 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 10878 { 10879 return btf_param_match_suffix(btf, arg, "__k"); 10880 } 10881 10882 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 10883 { 10884 return btf_param_match_suffix(btf, arg, "__ign"); 10885 } 10886 10887 static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) 10888 { 10889 return btf_param_match_suffix(btf, arg, "__map"); 10890 } 10891 10892 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 10893 { 10894 return btf_param_match_suffix(btf, arg, "__alloc"); 10895 } 10896 10897 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg) 10898 { 10899 return btf_param_match_suffix(btf, arg, "__uninit"); 10900 } 10901 10902 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg) 10903 { 10904 return btf_param_match_suffix(btf, arg, "__refcounted_kptr"); 10905 } 10906 10907 static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) 10908 { 10909 return btf_param_match_suffix(btf, arg, "__nullable"); 10910 } 10911 10912 static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg) 10913 { 10914 return btf_param_match_suffix(btf, arg, "__str"); 10915 } 10916 10917 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 10918 const struct btf_param *arg, 10919 const char *name) 10920 { 10921 int len, target_len = strlen(name); 10922 const char *param_name; 10923 10924 param_name = btf_name_by_offset(btf, arg->name_off); 10925 if (str_is_empty(param_name)) 10926 return false; 10927 len = strlen(param_name); 10928 if (len != target_len) 10929 return false; 10930 if (strcmp(param_name, name)) 10931 return false; 10932 10933 return true; 10934 } 10935 10936 enum { 10937 KF_ARG_DYNPTR_ID, 10938 KF_ARG_LIST_HEAD_ID, 10939 KF_ARG_LIST_NODE_ID, 10940 KF_ARG_RB_ROOT_ID, 10941 KF_ARG_RB_NODE_ID, 10942 KF_ARG_WORKQUEUE_ID, 10943 }; 10944 10945 BTF_ID_LIST(kf_arg_btf_ids) 10946 BTF_ID(struct, bpf_dynptr) 10947 BTF_ID(struct, bpf_list_head) 10948 BTF_ID(struct, bpf_list_node) 10949 BTF_ID(struct, bpf_rb_root) 10950 BTF_ID(struct, bpf_rb_node) 10951 BTF_ID(struct, bpf_wq) 10952 10953 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 10954 const struct btf_param *arg, int type) 10955 { 10956 const struct btf_type *t; 10957 u32 res_id; 10958 10959 t = btf_type_skip_modifiers(btf, arg->type, NULL); 10960 if (!t) 10961 return false; 10962 if (!btf_type_is_ptr(t)) 10963 return false; 10964 t = btf_type_skip_modifiers(btf, t->type, &res_id); 10965 if (!t) 10966 return false; 10967 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 10968 } 10969 10970 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 10971 { 10972 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 10973 } 10974 10975 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 10976 { 10977 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 10978 } 10979 10980 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 10981 { 10982 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 10983 } 10984 10985 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg) 10986 { 10987 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID); 10988 } 10989 10990 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg) 10991 { 10992 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID); 10993 } 10994 10995 static bool is_kfunc_arg_wq(const struct btf *btf, const struct btf_param *arg) 10996 { 10997 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_WORKQUEUE_ID); 10998 } 10999 11000 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, 11001 const struct btf_param *arg) 11002 { 11003 const struct btf_type *t; 11004 11005 t = btf_type_resolve_func_ptr(btf, arg->type, NULL); 11006 if (!t) 11007 return false; 11008 11009 return true; 11010 } 11011 11012 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 11013 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 11014 const struct btf *btf, 11015 const struct btf_type *t, int rec) 11016 { 11017 const struct btf_type *member_type; 11018 const struct btf_member *member; 11019 u32 i; 11020 11021 if (!btf_type_is_struct(t)) 11022 return false; 11023 11024 for_each_member(i, t, member) { 11025 const struct btf_array *array; 11026 11027 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 11028 if (btf_type_is_struct(member_type)) { 11029 if (rec >= 3) { 11030 verbose(env, "max struct nesting depth exceeded\n"); 11031 return false; 11032 } 11033 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) 11034 return false; 11035 continue; 11036 } 11037 if (btf_type_is_array(member_type)) { 11038 array = btf_array(member_type); 11039 if (!array->nelems) 11040 return false; 11041 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 11042 if (!btf_type_is_scalar(member_type)) 11043 return false; 11044 continue; 11045 } 11046 if (!btf_type_is_scalar(member_type)) 11047 return false; 11048 } 11049 return true; 11050 } 11051 11052 enum kfunc_ptr_arg_type { 11053 KF_ARG_PTR_TO_CTX, 11054 KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ 11055 KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ 11056 KF_ARG_PTR_TO_DYNPTR, 11057 KF_ARG_PTR_TO_ITER, 11058 KF_ARG_PTR_TO_LIST_HEAD, 11059 KF_ARG_PTR_TO_LIST_NODE, 11060 KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ 11061 KF_ARG_PTR_TO_MEM, 11062 KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ 11063 KF_ARG_PTR_TO_CALLBACK, 11064 KF_ARG_PTR_TO_RB_ROOT, 11065 KF_ARG_PTR_TO_RB_NODE, 11066 KF_ARG_PTR_TO_NULL, 11067 KF_ARG_PTR_TO_CONST_STR, 11068 KF_ARG_PTR_TO_MAP, 11069 KF_ARG_PTR_TO_WORKQUEUE, 11070 }; 11071 11072 enum special_kfunc_type { 11073 KF_bpf_obj_new_impl, 11074 KF_bpf_obj_drop_impl, 11075 KF_bpf_refcount_acquire_impl, 11076 KF_bpf_list_push_front_impl, 11077 KF_bpf_list_push_back_impl, 11078 KF_bpf_list_pop_front, 11079 KF_bpf_list_pop_back, 11080 KF_bpf_cast_to_kern_ctx, 11081 KF_bpf_rdonly_cast, 11082 KF_bpf_rcu_read_lock, 11083 KF_bpf_rcu_read_unlock, 11084 KF_bpf_rbtree_remove, 11085 KF_bpf_rbtree_add_impl, 11086 KF_bpf_rbtree_first, 11087 KF_bpf_dynptr_from_skb, 11088 KF_bpf_dynptr_from_xdp, 11089 KF_bpf_dynptr_slice, 11090 KF_bpf_dynptr_slice_rdwr, 11091 KF_bpf_dynptr_clone, 11092 KF_bpf_percpu_obj_new_impl, 11093 KF_bpf_percpu_obj_drop_impl, 11094 KF_bpf_throw, 11095 KF_bpf_wq_set_callback_impl, 11096 KF_bpf_preempt_disable, 11097 KF_bpf_preempt_enable, 11098 KF_bpf_iter_css_task_new, 11099 KF_bpf_session_cookie, 11100 }; 11101 11102 BTF_SET_START(special_kfunc_set) 11103 BTF_ID(func, bpf_obj_new_impl) 11104 BTF_ID(func, bpf_obj_drop_impl) 11105 BTF_ID(func, bpf_refcount_acquire_impl) 11106 BTF_ID(func, bpf_list_push_front_impl) 11107 BTF_ID(func, bpf_list_push_back_impl) 11108 BTF_ID(func, bpf_list_pop_front) 11109 BTF_ID(func, bpf_list_pop_back) 11110 BTF_ID(func, bpf_cast_to_kern_ctx) 11111 BTF_ID(func, bpf_rdonly_cast) 11112 BTF_ID(func, bpf_rbtree_remove) 11113 BTF_ID(func, bpf_rbtree_add_impl) 11114 BTF_ID(func, bpf_rbtree_first) 11115 BTF_ID(func, bpf_dynptr_from_skb) 11116 BTF_ID(func, bpf_dynptr_from_xdp) 11117 BTF_ID(func, bpf_dynptr_slice) 11118 BTF_ID(func, bpf_dynptr_slice_rdwr) 11119 BTF_ID(func, bpf_dynptr_clone) 11120 BTF_ID(func, bpf_percpu_obj_new_impl) 11121 BTF_ID(func, bpf_percpu_obj_drop_impl) 11122 BTF_ID(func, bpf_throw) 11123 BTF_ID(func, bpf_wq_set_callback_impl) 11124 #ifdef CONFIG_CGROUPS 11125 BTF_ID(func, bpf_iter_css_task_new) 11126 #endif 11127 BTF_SET_END(special_kfunc_set) 11128 11129 BTF_ID_LIST(special_kfunc_list) 11130 BTF_ID(func, bpf_obj_new_impl) 11131 BTF_ID(func, bpf_obj_drop_impl) 11132 BTF_ID(func, bpf_refcount_acquire_impl) 11133 BTF_ID(func, bpf_list_push_front_impl) 11134 BTF_ID(func, bpf_list_push_back_impl) 11135 BTF_ID(func, bpf_list_pop_front) 11136 BTF_ID(func, bpf_list_pop_back) 11137 BTF_ID(func, bpf_cast_to_kern_ctx) 11138 BTF_ID(func, bpf_rdonly_cast) 11139 BTF_ID(func, bpf_rcu_read_lock) 11140 BTF_ID(func, bpf_rcu_read_unlock) 11141 BTF_ID(func, bpf_rbtree_remove) 11142 BTF_ID(func, bpf_rbtree_add_impl) 11143 BTF_ID(func, bpf_rbtree_first) 11144 BTF_ID(func, bpf_dynptr_from_skb) 11145 BTF_ID(func, bpf_dynptr_from_xdp) 11146 BTF_ID(func, bpf_dynptr_slice) 11147 BTF_ID(func, bpf_dynptr_slice_rdwr) 11148 BTF_ID(func, bpf_dynptr_clone) 11149 BTF_ID(func, bpf_percpu_obj_new_impl) 11150 BTF_ID(func, bpf_percpu_obj_drop_impl) 11151 BTF_ID(func, bpf_throw) 11152 BTF_ID(func, bpf_wq_set_callback_impl) 11153 BTF_ID(func, bpf_preempt_disable) 11154 BTF_ID(func, bpf_preempt_enable) 11155 #ifdef CONFIG_CGROUPS 11156 BTF_ID(func, bpf_iter_css_task_new) 11157 #else 11158 BTF_ID_UNUSED 11159 #endif 11160 #ifdef CONFIG_BPF_EVENTS 11161 BTF_ID(func, bpf_session_cookie) 11162 #else 11163 BTF_ID_UNUSED 11164 #endif 11165 11166 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 11167 { 11168 if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] && 11169 meta->arg_owning_ref) { 11170 return false; 11171 } 11172 11173 return meta->kfunc_flags & KF_RET_NULL; 11174 } 11175 11176 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) 11177 { 11178 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; 11179 } 11180 11181 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) 11182 { 11183 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; 11184 } 11185 11186 static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta) 11187 { 11188 return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; 11189 } 11190 11191 static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) 11192 { 11193 return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; 11194 } 11195 11196 static enum kfunc_ptr_arg_type 11197 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, 11198 struct bpf_kfunc_call_arg_meta *meta, 11199 const struct btf_type *t, const struct btf_type *ref_t, 11200 const char *ref_tname, const struct btf_param *args, 11201 int argno, int nargs) 11202 { 11203 u32 regno = argno + 1; 11204 struct bpf_reg_state *regs = cur_regs(env); 11205 struct bpf_reg_state *reg = ®s[regno]; 11206 bool arg_mem_size = false; 11207 11208 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) 11209 return KF_ARG_PTR_TO_CTX; 11210 11211 /* In this function, we verify the kfunc's BTF as per the argument type, 11212 * leaving the rest of the verification with respect to the register 11213 * type to our caller. When a set of conditions hold in the BTF type of 11214 * arguments, we resolve it to a known kfunc_ptr_arg_type. 11215 */ 11216 if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) 11217 return KF_ARG_PTR_TO_CTX; 11218 11219 if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && register_is_null(reg)) 11220 return KF_ARG_PTR_TO_NULL; 11221 11222 if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) 11223 return KF_ARG_PTR_TO_ALLOC_BTF_ID; 11224 11225 if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno])) 11226 return KF_ARG_PTR_TO_REFCOUNTED_KPTR; 11227 11228 if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) 11229 return KF_ARG_PTR_TO_DYNPTR; 11230 11231 if (is_kfunc_arg_iter(meta, argno)) 11232 return KF_ARG_PTR_TO_ITER; 11233 11234 if (is_kfunc_arg_list_head(meta->btf, &args[argno])) 11235 return KF_ARG_PTR_TO_LIST_HEAD; 11236 11237 if (is_kfunc_arg_list_node(meta->btf, &args[argno])) 11238 return KF_ARG_PTR_TO_LIST_NODE; 11239 11240 if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno])) 11241 return KF_ARG_PTR_TO_RB_ROOT; 11242 11243 if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno])) 11244 return KF_ARG_PTR_TO_RB_NODE; 11245 11246 if (is_kfunc_arg_const_str(meta->btf, &args[argno])) 11247 return KF_ARG_PTR_TO_CONST_STR; 11248 11249 if (is_kfunc_arg_map(meta->btf, &args[argno])) 11250 return KF_ARG_PTR_TO_MAP; 11251 11252 if (is_kfunc_arg_wq(meta->btf, &args[argno])) 11253 return KF_ARG_PTR_TO_WORKQUEUE; 11254 11255 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 11256 if (!btf_type_is_struct(ref_t)) { 11257 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", 11258 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 11259 return -EINVAL; 11260 } 11261 return KF_ARG_PTR_TO_BTF_ID; 11262 } 11263 11264 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) 11265 return KF_ARG_PTR_TO_CALLBACK; 11266 11267 if (argno + 1 < nargs && 11268 (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]) || 11269 is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]))) 11270 arg_mem_size = true; 11271 11272 /* This is the catch all argument type of register types supported by 11273 * check_helper_mem_access. However, we only allow when argument type is 11274 * pointer to scalar, or struct composed (recursively) of scalars. When 11275 * arg_mem_size is true, the pointer can be void *. 11276 */ 11277 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && 11278 (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { 11279 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", 11280 argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); 11281 return -EINVAL; 11282 } 11283 return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; 11284 } 11285 11286 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, 11287 struct bpf_reg_state *reg, 11288 const struct btf_type *ref_t, 11289 const char *ref_tname, u32 ref_id, 11290 struct bpf_kfunc_call_arg_meta *meta, 11291 int argno) 11292 { 11293 const struct btf_type *reg_ref_t; 11294 bool strict_type_match = false; 11295 const struct btf *reg_btf; 11296 const char *reg_ref_tname; 11297 bool taking_projection; 11298 bool struct_same; 11299 u32 reg_ref_id; 11300 11301 if (base_type(reg->type) == PTR_TO_BTF_ID) { 11302 reg_btf = reg->btf; 11303 reg_ref_id = reg->btf_id; 11304 } else { 11305 reg_btf = btf_vmlinux; 11306 reg_ref_id = *reg2btf_ids[base_type(reg->type)]; 11307 } 11308 11309 /* Enforce strict type matching for calls to kfuncs that are acquiring 11310 * or releasing a reference, or are no-cast aliases. We do _not_ 11311 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default, 11312 * as we want to enable BPF programs to pass types that are bitwise 11313 * equivalent without forcing them to explicitly cast with something 11314 * like bpf_cast_to_kern_ctx(). 11315 * 11316 * For example, say we had a type like the following: 11317 * 11318 * struct bpf_cpumask { 11319 * cpumask_t cpumask; 11320 * refcount_t usage; 11321 * }; 11322 * 11323 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed 11324 * to a struct cpumask, so it would be safe to pass a struct 11325 * bpf_cpumask * to a kfunc expecting a struct cpumask *. 11326 * 11327 * The philosophy here is similar to how we allow scalars of different 11328 * types to be passed to kfuncs as long as the size is the same. The 11329 * only difference here is that we're simply allowing 11330 * btf_struct_ids_match() to walk the struct at the 0th offset, and 11331 * resolve types. 11332 */ 11333 if (is_kfunc_acquire(meta) || 11334 (is_kfunc_release(meta) && reg->ref_obj_id) || 11335 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) 11336 strict_type_match = true; 11337 11338 WARN_ON_ONCE(is_kfunc_trusted_args(meta) && reg->off); 11339 11340 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); 11341 reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); 11342 struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match); 11343 /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot 11344 * actually use it -- it must cast to the underlying type. So we allow 11345 * caller to pass in the underlying type. 11346 */ 11347 taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); 11348 if (!taking_projection && !struct_same) { 11349 verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", 11350 meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, 11351 btf_type_str(reg_ref_t), reg_ref_tname); 11352 return -EINVAL; 11353 } 11354 return 0; 11355 } 11356 11357 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 11358 { 11359 struct bpf_verifier_state *state = env->cur_state; 11360 struct btf_record *rec = reg_btf_record(reg); 11361 11362 if (!state->active_lock.ptr) { 11363 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); 11364 return -EFAULT; 11365 } 11366 11367 if (type_flag(reg->type) & NON_OWN_REF) { 11368 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); 11369 return -EFAULT; 11370 } 11371 11372 reg->type |= NON_OWN_REF; 11373 if (rec->refcount_off >= 0) 11374 reg->type |= MEM_RCU; 11375 11376 return 0; 11377 } 11378 11379 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) 11380 { 11381 struct bpf_func_state *state, *unused; 11382 struct bpf_reg_state *reg; 11383 int i; 11384 11385 state = cur_func(env); 11386 11387 if (!ref_obj_id) { 11388 verbose(env, "verifier internal error: ref_obj_id is zero for " 11389 "owning -> non-owning conversion\n"); 11390 return -EFAULT; 11391 } 11392 11393 for (i = 0; i < state->acquired_refs; i++) { 11394 if (state->refs[i].id != ref_obj_id) 11395 continue; 11396 11397 /* Clear ref_obj_id here so release_reference doesn't clobber 11398 * the whole reg 11399 */ 11400 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 11401 if (reg->ref_obj_id == ref_obj_id) { 11402 reg->ref_obj_id = 0; 11403 ref_set_non_owning(env, reg); 11404 } 11405 })); 11406 return 0; 11407 } 11408 11409 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); 11410 return -EFAULT; 11411 } 11412 11413 /* Implementation details: 11414 * 11415 * Each register points to some region of memory, which we define as an 11416 * allocation. Each allocation may embed a bpf_spin_lock which protects any 11417 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 11418 * allocation. The lock and the data it protects are colocated in the same 11419 * memory region. 11420 * 11421 * Hence, everytime a register holds a pointer value pointing to such 11422 * allocation, the verifier preserves a unique reg->id for it. 11423 * 11424 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 11425 * bpf_spin_lock is called. 11426 * 11427 * To enable this, lock state in the verifier captures two values: 11428 * active_lock.ptr = Register's type specific pointer 11429 * active_lock.id = A unique ID for each register pointer value 11430 * 11431 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 11432 * supported register types. 11433 * 11434 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 11435 * allocated objects is the reg->btf pointer. 11436 * 11437 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 11438 * can establish the provenance of the map value statically for each distinct 11439 * lookup into such maps. They always contain a single map value hence unique 11440 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 11441 * 11442 * So, in case of global variables, they use array maps with max_entries = 1, 11443 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 11444 * into the same map value as max_entries is 1, as described above). 11445 * 11446 * In case of inner map lookups, the inner map pointer has same map_ptr as the 11447 * outer map pointer (in verifier context), but each lookup into an inner map 11448 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 11449 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 11450 * will get different reg->id assigned to each lookup, hence different 11451 * active_lock.id. 11452 * 11453 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 11454 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 11455 * returned from bpf_obj_new. Each allocation receives a new reg->id. 11456 */ 11457 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 11458 { 11459 void *ptr; 11460 u32 id; 11461 11462 switch ((int)reg->type) { 11463 case PTR_TO_MAP_VALUE: 11464 ptr = reg->map_ptr; 11465 break; 11466 case PTR_TO_BTF_ID | MEM_ALLOC: 11467 ptr = reg->btf; 11468 break; 11469 default: 11470 verbose(env, "verifier internal error: unknown reg type for lock check\n"); 11471 return -EFAULT; 11472 } 11473 id = reg->id; 11474 11475 if (!env->cur_state->active_lock.ptr) 11476 return -EINVAL; 11477 if (env->cur_state->active_lock.ptr != ptr || 11478 env->cur_state->active_lock.id != id) { 11479 verbose(env, "held lock and object are not in the same allocation\n"); 11480 return -EINVAL; 11481 } 11482 return 0; 11483 } 11484 11485 static bool is_bpf_list_api_kfunc(u32 btf_id) 11486 { 11487 return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 11488 btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 11489 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 11490 btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 11491 } 11492 11493 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 11494 { 11495 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 11496 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 11497 btf_id == special_kfunc_list[KF_bpf_rbtree_first]; 11498 } 11499 11500 static bool is_bpf_graph_api_kfunc(u32 btf_id) 11501 { 11502 return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) || 11503 btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 11504 } 11505 11506 static bool is_sync_callback_calling_kfunc(u32 btf_id) 11507 { 11508 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 11509 } 11510 11511 static bool is_async_callback_calling_kfunc(u32 btf_id) 11512 { 11513 return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl]; 11514 } 11515 11516 static bool is_bpf_throw_kfunc(struct bpf_insn *insn) 11517 { 11518 return bpf_pseudo_kfunc_call(insn) && insn->off == 0 && 11519 insn->imm == special_kfunc_list[KF_bpf_throw]; 11520 } 11521 11522 static bool is_bpf_wq_set_callback_impl_kfunc(u32 btf_id) 11523 { 11524 return btf_id == special_kfunc_list[KF_bpf_wq_set_callback_impl]; 11525 } 11526 11527 static bool is_callback_calling_kfunc(u32 btf_id) 11528 { 11529 return is_sync_callback_calling_kfunc(btf_id) || 11530 is_async_callback_calling_kfunc(btf_id); 11531 } 11532 11533 static bool is_rbtree_lock_required_kfunc(u32 btf_id) 11534 { 11535 return is_bpf_rbtree_api_kfunc(btf_id); 11536 } 11537 11538 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, 11539 enum btf_field_type head_field_type, 11540 u32 kfunc_btf_id) 11541 { 11542 bool ret; 11543 11544 switch (head_field_type) { 11545 case BPF_LIST_HEAD: 11546 ret = is_bpf_list_api_kfunc(kfunc_btf_id); 11547 break; 11548 case BPF_RB_ROOT: 11549 ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id); 11550 break; 11551 default: 11552 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", 11553 btf_field_type_name(head_field_type)); 11554 return false; 11555 } 11556 11557 if (!ret) 11558 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", 11559 btf_field_type_name(head_field_type)); 11560 return ret; 11561 } 11562 11563 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, 11564 enum btf_field_type node_field_type, 11565 u32 kfunc_btf_id) 11566 { 11567 bool ret; 11568 11569 switch (node_field_type) { 11570 case BPF_LIST_NODE: 11571 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 11572 kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]); 11573 break; 11574 case BPF_RB_NODE: 11575 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 11576 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]); 11577 break; 11578 default: 11579 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", 11580 btf_field_type_name(node_field_type)); 11581 return false; 11582 } 11583 11584 if (!ret) 11585 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", 11586 btf_field_type_name(node_field_type)); 11587 return ret; 11588 } 11589 11590 static int 11591 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, 11592 struct bpf_reg_state *reg, u32 regno, 11593 struct bpf_kfunc_call_arg_meta *meta, 11594 enum btf_field_type head_field_type, 11595 struct btf_field **head_field) 11596 { 11597 const char *head_type_name; 11598 struct btf_field *field; 11599 struct btf_record *rec; 11600 u32 head_off; 11601 11602 if (meta->btf != btf_vmlinux) { 11603 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 11604 return -EFAULT; 11605 } 11606 11607 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) 11608 return -EFAULT; 11609 11610 head_type_name = btf_field_type_name(head_field_type); 11611 if (!tnum_is_const(reg->var_off)) { 11612 verbose(env, 11613 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 11614 regno, head_type_name); 11615 return -EINVAL; 11616 } 11617 11618 rec = reg_btf_record(reg); 11619 head_off = reg->off + reg->var_off.value; 11620 field = btf_record_find(rec, head_off, head_field_type); 11621 if (!field) { 11622 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); 11623 return -EINVAL; 11624 } 11625 11626 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 11627 if (check_reg_allocation_locked(env, reg)) { 11628 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", 11629 rec->spin_lock_off, head_type_name); 11630 return -EINVAL; 11631 } 11632 11633 if (*head_field) { 11634 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); 11635 return -EFAULT; 11636 } 11637 *head_field = field; 11638 return 0; 11639 } 11640 11641 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 11642 struct bpf_reg_state *reg, u32 regno, 11643 struct bpf_kfunc_call_arg_meta *meta) 11644 { 11645 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, 11646 &meta->arg_list_head.field); 11647 } 11648 11649 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, 11650 struct bpf_reg_state *reg, u32 regno, 11651 struct bpf_kfunc_call_arg_meta *meta) 11652 { 11653 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, 11654 &meta->arg_rbtree_root.field); 11655 } 11656 11657 static int 11658 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, 11659 struct bpf_reg_state *reg, u32 regno, 11660 struct bpf_kfunc_call_arg_meta *meta, 11661 enum btf_field_type head_field_type, 11662 enum btf_field_type node_field_type, 11663 struct btf_field **node_field) 11664 { 11665 const char *node_type_name; 11666 const struct btf_type *et, *t; 11667 struct btf_field *field; 11668 u32 node_off; 11669 11670 if (meta->btf != btf_vmlinux) { 11671 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 11672 return -EFAULT; 11673 } 11674 11675 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) 11676 return -EFAULT; 11677 11678 node_type_name = btf_field_type_name(node_field_type); 11679 if (!tnum_is_const(reg->var_off)) { 11680 verbose(env, 11681 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 11682 regno, node_type_name); 11683 return -EINVAL; 11684 } 11685 11686 node_off = reg->off + reg->var_off.value; 11687 field = reg_find_field_offset(reg, node_off, node_field_type); 11688 if (!field) { 11689 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); 11690 return -EINVAL; 11691 } 11692 11693 field = *node_field; 11694 11695 et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); 11696 t = btf_type_by_id(reg->btf, reg->btf_id); 11697 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, 11698 field->graph_root.value_btf_id, true)) { 11699 verbose(env, "operation on %s expects arg#1 %s at offset=%d " 11700 "in struct %s, but arg is at offset=%d in struct %s\n", 11701 btf_field_type_name(head_field_type), 11702 btf_field_type_name(node_field_type), 11703 field->graph_root.node_offset, 11704 btf_name_by_offset(field->graph_root.btf, et->name_off), 11705 node_off, btf_name_by_offset(reg->btf, t->name_off)); 11706 return -EINVAL; 11707 } 11708 meta->arg_btf = reg->btf; 11709 meta->arg_btf_id = reg->btf_id; 11710 11711 if (node_off != field->graph_root.node_offset) { 11712 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", 11713 node_off, btf_field_type_name(node_field_type), 11714 field->graph_root.node_offset, 11715 btf_name_by_offset(field->graph_root.btf, et->name_off)); 11716 return -EINVAL; 11717 } 11718 11719 return 0; 11720 } 11721 11722 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 11723 struct bpf_reg_state *reg, u32 regno, 11724 struct bpf_kfunc_call_arg_meta *meta) 11725 { 11726 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 11727 BPF_LIST_HEAD, BPF_LIST_NODE, 11728 &meta->arg_list_head.field); 11729 } 11730 11731 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, 11732 struct bpf_reg_state *reg, u32 regno, 11733 struct bpf_kfunc_call_arg_meta *meta) 11734 { 11735 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 11736 BPF_RB_ROOT, BPF_RB_NODE, 11737 &meta->arg_rbtree_root.field); 11738 } 11739 11740 /* 11741 * css_task iter allowlist is needed to avoid dead locking on css_set_lock. 11742 * LSM hooks and iters (both sleepable and non-sleepable) are safe. 11743 * Any sleepable progs are also safe since bpf_check_attach_target() enforce 11744 * them can only be attached to some specific hook points. 11745 */ 11746 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) 11747 { 11748 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 11749 11750 switch (prog_type) { 11751 case BPF_PROG_TYPE_LSM: 11752 return true; 11753 case BPF_PROG_TYPE_TRACING: 11754 if (env->prog->expected_attach_type == BPF_TRACE_ITER) 11755 return true; 11756 fallthrough; 11757 default: 11758 return in_sleepable(env); 11759 } 11760 } 11761 11762 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, 11763 int insn_idx) 11764 { 11765 const char *func_name = meta->func_name, *ref_tname; 11766 const struct btf *btf = meta->btf; 11767 const struct btf_param *args; 11768 struct btf_record *rec; 11769 u32 i, nargs; 11770 int ret; 11771 11772 args = (const struct btf_param *)(meta->func_proto + 1); 11773 nargs = btf_type_vlen(meta->func_proto); 11774 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 11775 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, 11776 MAX_BPF_FUNC_REG_ARGS); 11777 return -EINVAL; 11778 } 11779 11780 /* Check that BTF function arguments match actual types that the 11781 * verifier sees. 11782 */ 11783 for (i = 0; i < nargs; i++) { 11784 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; 11785 const struct btf_type *t, *ref_t, *resolve_ret; 11786 enum bpf_arg_type arg_type = ARG_DONTCARE; 11787 u32 regno = i + 1, ref_id, type_size; 11788 bool is_ret_buf_sz = false; 11789 int kf_arg_type; 11790 11791 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 11792 11793 if (is_kfunc_arg_ignore(btf, &args[i])) 11794 continue; 11795 11796 if (btf_type_is_scalar(t)) { 11797 if (reg->type != SCALAR_VALUE) { 11798 verbose(env, "R%d is not a scalar\n", regno); 11799 return -EINVAL; 11800 } 11801 11802 if (is_kfunc_arg_constant(meta->btf, &args[i])) { 11803 if (meta->arg_constant.found) { 11804 verbose(env, "verifier internal error: only one constant argument permitted\n"); 11805 return -EFAULT; 11806 } 11807 if (!tnum_is_const(reg->var_off)) { 11808 verbose(env, "R%d must be a known constant\n", regno); 11809 return -EINVAL; 11810 } 11811 ret = mark_chain_precision(env, regno); 11812 if (ret < 0) 11813 return ret; 11814 meta->arg_constant.found = true; 11815 meta->arg_constant.value = reg->var_off.value; 11816 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { 11817 meta->r0_rdonly = true; 11818 is_ret_buf_sz = true; 11819 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { 11820 is_ret_buf_sz = true; 11821 } 11822 11823 if (is_ret_buf_sz) { 11824 if (meta->r0_size) { 11825 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); 11826 return -EINVAL; 11827 } 11828 11829 if (!tnum_is_const(reg->var_off)) { 11830 verbose(env, "R%d is not a const\n", regno); 11831 return -EINVAL; 11832 } 11833 11834 meta->r0_size = reg->var_off.value; 11835 ret = mark_chain_precision(env, regno); 11836 if (ret) 11837 return ret; 11838 } 11839 continue; 11840 } 11841 11842 if (!btf_type_is_ptr(t)) { 11843 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); 11844 return -EINVAL; 11845 } 11846 11847 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) && 11848 (register_is_null(reg) || type_may_be_null(reg->type)) && 11849 !is_kfunc_arg_nullable(meta->btf, &args[i])) { 11850 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); 11851 return -EACCES; 11852 } 11853 11854 if (reg->ref_obj_id) { 11855 if (is_kfunc_release(meta) && meta->ref_obj_id) { 11856 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 11857 regno, reg->ref_obj_id, 11858 meta->ref_obj_id); 11859 return -EFAULT; 11860 } 11861 meta->ref_obj_id = reg->ref_obj_id; 11862 if (is_kfunc_release(meta)) 11863 meta->release_regno = regno; 11864 } 11865 11866 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 11867 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 11868 11869 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); 11870 if (kf_arg_type < 0) 11871 return kf_arg_type; 11872 11873 switch (kf_arg_type) { 11874 case KF_ARG_PTR_TO_NULL: 11875 continue; 11876 case KF_ARG_PTR_TO_MAP: 11877 if (!reg->map_ptr) { 11878 verbose(env, "pointer in R%d isn't map pointer\n", regno); 11879 return -EINVAL; 11880 } 11881 if (meta->map.ptr && reg->map_ptr->record->wq_off >= 0) { 11882 /* Use map_uid (which is unique id of inner map) to reject: 11883 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 11884 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 11885 * if (inner_map1 && inner_map2) { 11886 * wq = bpf_map_lookup_elem(inner_map1); 11887 * if (wq) 11888 * // mismatch would have been allowed 11889 * bpf_wq_init(wq, inner_map2); 11890 * } 11891 * 11892 * Comparing map_ptr is enough to distinguish normal and outer maps. 11893 */ 11894 if (meta->map.ptr != reg->map_ptr || 11895 meta->map.uid != reg->map_uid) { 11896 verbose(env, 11897 "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 11898 meta->map.uid, reg->map_uid); 11899 return -EINVAL; 11900 } 11901 } 11902 meta->map.ptr = reg->map_ptr; 11903 meta->map.uid = reg->map_uid; 11904 fallthrough; 11905 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 11906 case KF_ARG_PTR_TO_BTF_ID: 11907 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 11908 break; 11909 11910 if (!is_trusted_reg(reg)) { 11911 if (!is_kfunc_rcu(meta)) { 11912 verbose(env, "R%d must be referenced or trusted\n", regno); 11913 return -EINVAL; 11914 } 11915 if (!is_rcu_reg(reg)) { 11916 verbose(env, "R%d must be a rcu pointer\n", regno); 11917 return -EINVAL; 11918 } 11919 } 11920 11921 fallthrough; 11922 case KF_ARG_PTR_TO_CTX: 11923 /* Trusted arguments have the same offset checks as release arguments */ 11924 arg_type |= OBJ_RELEASE; 11925 break; 11926 case KF_ARG_PTR_TO_DYNPTR: 11927 case KF_ARG_PTR_TO_ITER: 11928 case KF_ARG_PTR_TO_LIST_HEAD: 11929 case KF_ARG_PTR_TO_LIST_NODE: 11930 case KF_ARG_PTR_TO_RB_ROOT: 11931 case KF_ARG_PTR_TO_RB_NODE: 11932 case KF_ARG_PTR_TO_MEM: 11933 case KF_ARG_PTR_TO_MEM_SIZE: 11934 case KF_ARG_PTR_TO_CALLBACK: 11935 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: 11936 case KF_ARG_PTR_TO_CONST_STR: 11937 case KF_ARG_PTR_TO_WORKQUEUE: 11938 /* Trusted by default */ 11939 break; 11940 default: 11941 WARN_ON_ONCE(1); 11942 return -EFAULT; 11943 } 11944 11945 if (is_kfunc_release(meta) && reg->ref_obj_id) 11946 arg_type |= OBJ_RELEASE; 11947 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 11948 if (ret < 0) 11949 return ret; 11950 11951 switch (kf_arg_type) { 11952 case KF_ARG_PTR_TO_CTX: 11953 if (reg->type != PTR_TO_CTX) { 11954 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); 11955 return -EINVAL; 11956 } 11957 11958 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 11959 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 11960 if (ret < 0) 11961 return -EINVAL; 11962 meta->ret_btf_id = ret; 11963 } 11964 break; 11965 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 11966 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { 11967 if (meta->func_id != special_kfunc_list[KF_bpf_obj_drop_impl]) { 11968 verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i); 11969 return -EINVAL; 11970 } 11971 } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { 11972 if (meta->func_id != special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 11973 verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i); 11974 return -EINVAL; 11975 } 11976 } else { 11977 verbose(env, "arg#%d expected pointer to allocated object\n", i); 11978 return -EINVAL; 11979 } 11980 if (!reg->ref_obj_id) { 11981 verbose(env, "allocated object must be referenced\n"); 11982 return -EINVAL; 11983 } 11984 if (meta->btf == btf_vmlinux) { 11985 meta->arg_btf = reg->btf; 11986 meta->arg_btf_id = reg->btf_id; 11987 } 11988 break; 11989 case KF_ARG_PTR_TO_DYNPTR: 11990 { 11991 enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; 11992 int clone_ref_obj_id = 0; 11993 11994 if (reg->type == CONST_PTR_TO_DYNPTR) 11995 dynptr_arg_type |= MEM_RDONLY; 11996 11997 if (is_kfunc_arg_uninit(btf, &args[i])) 11998 dynptr_arg_type |= MEM_UNINIT; 11999 12000 if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { 12001 dynptr_arg_type |= DYNPTR_TYPE_SKB; 12002 } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { 12003 dynptr_arg_type |= DYNPTR_TYPE_XDP; 12004 } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && 12005 (dynptr_arg_type & MEM_UNINIT)) { 12006 enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type; 12007 12008 if (parent_type == BPF_DYNPTR_TYPE_INVALID) { 12009 verbose(env, "verifier internal error: no dynptr type for parent of clone\n"); 12010 return -EFAULT; 12011 } 12012 12013 dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); 12014 clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id; 12015 if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) { 12016 verbose(env, "verifier internal error: missing ref obj id for parent of clone\n"); 12017 return -EFAULT; 12018 } 12019 } 12020 12021 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); 12022 if (ret < 0) 12023 return ret; 12024 12025 if (!(dynptr_arg_type & MEM_UNINIT)) { 12026 int id = dynptr_id(env, reg); 12027 12028 if (id < 0) { 12029 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 12030 return id; 12031 } 12032 meta->initialized_dynptr.id = id; 12033 meta->initialized_dynptr.type = dynptr_get_type(env, reg); 12034 meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg); 12035 } 12036 12037 break; 12038 } 12039 case KF_ARG_PTR_TO_ITER: 12040 if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { 12041 if (!check_css_task_iter_allowlist(env)) { 12042 verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); 12043 return -EINVAL; 12044 } 12045 } 12046 ret = process_iter_arg(env, regno, insn_idx, meta); 12047 if (ret < 0) 12048 return ret; 12049 break; 12050 case KF_ARG_PTR_TO_LIST_HEAD: 12051 if (reg->type != PTR_TO_MAP_VALUE && 12052 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 12053 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 12054 return -EINVAL; 12055 } 12056 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 12057 verbose(env, "allocated object must be referenced\n"); 12058 return -EINVAL; 12059 } 12060 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); 12061 if (ret < 0) 12062 return ret; 12063 break; 12064 case KF_ARG_PTR_TO_RB_ROOT: 12065 if (reg->type != PTR_TO_MAP_VALUE && 12066 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 12067 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 12068 return -EINVAL; 12069 } 12070 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 12071 verbose(env, "allocated object must be referenced\n"); 12072 return -EINVAL; 12073 } 12074 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); 12075 if (ret < 0) 12076 return ret; 12077 break; 12078 case KF_ARG_PTR_TO_LIST_NODE: 12079 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 12080 verbose(env, "arg#%d expected pointer to allocated object\n", i); 12081 return -EINVAL; 12082 } 12083 if (!reg->ref_obj_id) { 12084 verbose(env, "allocated object must be referenced\n"); 12085 return -EINVAL; 12086 } 12087 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); 12088 if (ret < 0) 12089 return ret; 12090 break; 12091 case KF_ARG_PTR_TO_RB_NODE: 12092 if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) { 12093 if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) { 12094 verbose(env, "rbtree_remove node input must be non-owning ref\n"); 12095 return -EINVAL; 12096 } 12097 if (in_rbtree_lock_required_cb(env)) { 12098 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); 12099 return -EINVAL; 12100 } 12101 } else { 12102 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 12103 verbose(env, "arg#%d expected pointer to allocated object\n", i); 12104 return -EINVAL; 12105 } 12106 if (!reg->ref_obj_id) { 12107 verbose(env, "allocated object must be referenced\n"); 12108 return -EINVAL; 12109 } 12110 } 12111 12112 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); 12113 if (ret < 0) 12114 return ret; 12115 break; 12116 case KF_ARG_PTR_TO_MAP: 12117 /* If argument has '__map' suffix expect 'struct bpf_map *' */ 12118 ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; 12119 ref_t = btf_type_by_id(btf_vmlinux, ref_id); 12120 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 12121 fallthrough; 12122 case KF_ARG_PTR_TO_BTF_ID: 12123 /* Only base_type is checked, further checks are done here */ 12124 if ((base_type(reg->type) != PTR_TO_BTF_ID || 12125 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && 12126 !reg2btf_ids[base_type(reg->type)]) { 12127 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); 12128 verbose(env, "expected %s or socket\n", 12129 reg_type_str(env, base_type(reg->type) | 12130 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 12131 return -EINVAL; 12132 } 12133 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 12134 if (ret < 0) 12135 return ret; 12136 break; 12137 case KF_ARG_PTR_TO_MEM: 12138 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 12139 if (IS_ERR(resolve_ret)) { 12140 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 12141 i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); 12142 return -EINVAL; 12143 } 12144 ret = check_mem_reg(env, reg, regno, type_size); 12145 if (ret < 0) 12146 return ret; 12147 break; 12148 case KF_ARG_PTR_TO_MEM_SIZE: 12149 { 12150 struct bpf_reg_state *buff_reg = ®s[regno]; 12151 const struct btf_param *buff_arg = &args[i]; 12152 struct bpf_reg_state *size_reg = ®s[regno + 1]; 12153 const struct btf_param *size_arg = &args[i + 1]; 12154 12155 if (!register_is_null(buff_reg) || !is_kfunc_arg_optional(meta->btf, buff_arg)) { 12156 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); 12157 if (ret < 0) { 12158 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); 12159 return ret; 12160 } 12161 } 12162 12163 if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { 12164 if (meta->arg_constant.found) { 12165 verbose(env, "verifier internal error: only one constant argument permitted\n"); 12166 return -EFAULT; 12167 } 12168 if (!tnum_is_const(size_reg->var_off)) { 12169 verbose(env, "R%d must be a known constant\n", regno + 1); 12170 return -EINVAL; 12171 } 12172 meta->arg_constant.found = true; 12173 meta->arg_constant.value = size_reg->var_off.value; 12174 } 12175 12176 /* Skip next '__sz' or '__szk' argument */ 12177 i++; 12178 break; 12179 } 12180 case KF_ARG_PTR_TO_CALLBACK: 12181 if (reg->type != PTR_TO_FUNC) { 12182 verbose(env, "arg%d expected pointer to func\n", i); 12183 return -EINVAL; 12184 } 12185 meta->subprogno = reg->subprogno; 12186 break; 12187 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: 12188 if (!type_is_ptr_alloc_obj(reg->type)) { 12189 verbose(env, "arg#%d is neither owning or non-owning ref\n", i); 12190 return -EINVAL; 12191 } 12192 if (!type_is_non_owning_ref(reg->type)) 12193 meta->arg_owning_ref = true; 12194 12195 rec = reg_btf_record(reg); 12196 if (!rec) { 12197 verbose(env, "verifier internal error: Couldn't find btf_record\n"); 12198 return -EFAULT; 12199 } 12200 12201 if (rec->refcount_off < 0) { 12202 verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); 12203 return -EINVAL; 12204 } 12205 12206 meta->arg_btf = reg->btf; 12207 meta->arg_btf_id = reg->btf_id; 12208 break; 12209 case KF_ARG_PTR_TO_CONST_STR: 12210 if (reg->type != PTR_TO_MAP_VALUE) { 12211 verbose(env, "arg#%d doesn't point to a const string\n", i); 12212 return -EINVAL; 12213 } 12214 ret = check_reg_const_str(env, reg, regno); 12215 if (ret) 12216 return ret; 12217 break; 12218 case KF_ARG_PTR_TO_WORKQUEUE: 12219 if (reg->type != PTR_TO_MAP_VALUE) { 12220 verbose(env, "arg#%d doesn't point to a map value\n", i); 12221 return -EINVAL; 12222 } 12223 ret = process_wq_func(env, regno, meta); 12224 if (ret < 0) 12225 return ret; 12226 break; 12227 } 12228 } 12229 12230 if (is_kfunc_release(meta) && !meta->release_regno) { 12231 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", 12232 func_name); 12233 return -EINVAL; 12234 } 12235 12236 return 0; 12237 } 12238 12239 static int fetch_kfunc_meta(struct bpf_verifier_env *env, 12240 struct bpf_insn *insn, 12241 struct bpf_kfunc_call_arg_meta *meta, 12242 const char **kfunc_name) 12243 { 12244 const struct btf_type *func, *func_proto; 12245 u32 func_id, *kfunc_flags; 12246 const char *func_name; 12247 struct btf *desc_btf; 12248 12249 if (kfunc_name) 12250 *kfunc_name = NULL; 12251 12252 if (!insn->imm) 12253 return -EINVAL; 12254 12255 desc_btf = find_kfunc_desc_btf(env, insn->off); 12256 if (IS_ERR(desc_btf)) 12257 return PTR_ERR(desc_btf); 12258 12259 func_id = insn->imm; 12260 func = btf_type_by_id(desc_btf, func_id); 12261 func_name = btf_name_by_offset(desc_btf, func->name_off); 12262 if (kfunc_name) 12263 *kfunc_name = func_name; 12264 func_proto = btf_type_by_id(desc_btf, func->type); 12265 12266 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog); 12267 if (!kfunc_flags) { 12268 return -EACCES; 12269 } 12270 12271 memset(meta, 0, sizeof(*meta)); 12272 meta->btf = desc_btf; 12273 meta->func_id = func_id; 12274 meta->kfunc_flags = *kfunc_flags; 12275 meta->func_proto = func_proto; 12276 meta->func_name = func_name; 12277 12278 return 0; 12279 } 12280 12281 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name); 12282 12283 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 12284 int *insn_idx_p) 12285 { 12286 bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable; 12287 u32 i, nargs, ptr_type_id, release_ref_obj_id; 12288 struct bpf_reg_state *regs = cur_regs(env); 12289 const char *func_name, *ptr_type_name; 12290 const struct btf_type *t, *ptr_type; 12291 struct bpf_kfunc_call_arg_meta meta; 12292 struct bpf_insn_aux_data *insn_aux; 12293 int err, insn_idx = *insn_idx_p; 12294 const struct btf_param *args; 12295 const struct btf_type *ret_t; 12296 struct btf *desc_btf; 12297 12298 /* skip for now, but return error when we find this in fixup_kfunc_call */ 12299 if (!insn->imm) 12300 return 0; 12301 12302 err = fetch_kfunc_meta(env, insn, &meta, &func_name); 12303 if (err == -EACCES && func_name) 12304 verbose(env, "calling kernel function %s is not allowed\n", func_name); 12305 if (err) 12306 return err; 12307 desc_btf = meta.btf; 12308 insn_aux = &env->insn_aux_data[insn_idx]; 12309 12310 insn_aux->is_iter_next = is_iter_next_kfunc(&meta); 12311 12312 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 12313 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 12314 return -EACCES; 12315 } 12316 12317 sleepable = is_kfunc_sleepable(&meta); 12318 if (sleepable && !in_sleepable(env)) { 12319 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); 12320 return -EACCES; 12321 } 12322 12323 /* Check the arguments */ 12324 err = check_kfunc_args(env, &meta, insn_idx); 12325 if (err < 0) 12326 return err; 12327 12328 if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 12329 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12330 set_rbtree_add_callback_state); 12331 if (err) { 12332 verbose(env, "kfunc %s#%d failed callback verification\n", 12333 func_name, meta.func_id); 12334 return err; 12335 } 12336 } 12337 12338 if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) { 12339 meta.r0_size = sizeof(u64); 12340 meta.r0_rdonly = false; 12341 } 12342 12343 if (is_bpf_wq_set_callback_impl_kfunc(meta.func_id)) { 12344 err = push_callback_call(env, insn, insn_idx, meta.subprogno, 12345 set_timer_callback_state); 12346 if (err) { 12347 verbose(env, "kfunc %s#%d failed callback verification\n", 12348 func_name, meta.func_id); 12349 return err; 12350 } 12351 } 12352 12353 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 12354 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 12355 12356 preempt_disable = is_kfunc_bpf_preempt_disable(&meta); 12357 preempt_enable = is_kfunc_bpf_preempt_enable(&meta); 12358 12359 if (env->cur_state->active_rcu_lock) { 12360 struct bpf_func_state *state; 12361 struct bpf_reg_state *reg; 12362 u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER); 12363 12364 if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { 12365 verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); 12366 return -EACCES; 12367 } 12368 12369 if (rcu_lock) { 12370 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); 12371 return -EINVAL; 12372 } else if (rcu_unlock) { 12373 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({ 12374 if (reg->type & MEM_RCU) { 12375 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); 12376 reg->type |= PTR_UNTRUSTED; 12377 } 12378 })); 12379 env->cur_state->active_rcu_lock = false; 12380 } else if (sleepable) { 12381 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); 12382 return -EACCES; 12383 } 12384 } else if (rcu_lock) { 12385 env->cur_state->active_rcu_lock = true; 12386 } else if (rcu_unlock) { 12387 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 12388 return -EINVAL; 12389 } 12390 12391 if (env->cur_state->active_preempt_lock) { 12392 if (preempt_disable) { 12393 env->cur_state->active_preempt_lock++; 12394 } else if (preempt_enable) { 12395 env->cur_state->active_preempt_lock--; 12396 } else if (sleepable) { 12397 verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name); 12398 return -EACCES; 12399 } 12400 } else if (preempt_disable) { 12401 env->cur_state->active_preempt_lock++; 12402 } else if (preempt_enable) { 12403 verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); 12404 return -EINVAL; 12405 } 12406 12407 /* In case of release function, we get register number of refcounted 12408 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 12409 */ 12410 if (meta.release_regno) { 12411 err = release_reference(env, regs[meta.release_regno].ref_obj_id); 12412 if (err) { 12413 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 12414 func_name, meta.func_id); 12415 return err; 12416 } 12417 } 12418 12419 if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 12420 meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 12421 meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 12422 release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; 12423 insn_aux->insert_off = regs[BPF_REG_2].off; 12424 insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id); 12425 err = ref_convert_owning_non_owning(env, release_ref_obj_id); 12426 if (err) { 12427 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", 12428 func_name, meta.func_id); 12429 return err; 12430 } 12431 12432 err = release_reference(env, release_ref_obj_id); 12433 if (err) { 12434 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 12435 func_name, meta.func_id); 12436 return err; 12437 } 12438 } 12439 12440 if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { 12441 if (!bpf_jit_supports_exceptions()) { 12442 verbose(env, "JIT does not support calling kfunc %s#%d\n", 12443 func_name, meta.func_id); 12444 return -ENOTSUPP; 12445 } 12446 env->seen_exception = true; 12447 12448 /* In the case of the default callback, the cookie value passed 12449 * to bpf_throw becomes the return value of the program. 12450 */ 12451 if (!env->exception_callback_subprog) { 12452 err = check_return_code(env, BPF_REG_1, "R1"); 12453 if (err < 0) 12454 return err; 12455 } 12456 } 12457 12458 for (i = 0; i < CALLER_SAVED_REGS; i++) 12459 mark_reg_not_init(env, regs, caller_saved[i]); 12460 12461 /* Check return type */ 12462 t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL); 12463 12464 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 12465 /* Only exception is bpf_obj_new_impl */ 12466 if (meta.btf != btf_vmlinux || 12467 (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] && 12468 meta.func_id != special_kfunc_list[KF_bpf_percpu_obj_new_impl] && 12469 meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) { 12470 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 12471 return -EINVAL; 12472 } 12473 } 12474 12475 if (btf_type_is_scalar(t)) { 12476 mark_reg_unknown(env, regs, BPF_REG_0); 12477 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 12478 } else if (btf_type_is_ptr(t)) { 12479 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 12480 12481 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 12482 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 12483 meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 12484 struct btf_struct_meta *struct_meta; 12485 struct btf *ret_btf; 12486 u32 ret_btf_id; 12487 12488 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl] && !bpf_global_ma_set) 12489 return -ENOMEM; 12490 12491 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { 12492 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 12493 return -EINVAL; 12494 } 12495 12496 ret_btf = env->prog->aux->btf; 12497 ret_btf_id = meta.arg_constant.value; 12498 12499 /* This may be NULL due to user not supplying a BTF */ 12500 if (!ret_btf) { 12501 verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); 12502 return -EINVAL; 12503 } 12504 12505 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 12506 if (!ret_t || !__btf_type_is_struct(ret_t)) { 12507 verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); 12508 return -EINVAL; 12509 } 12510 12511 if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 12512 if (ret_t->size > BPF_GLOBAL_PERCPU_MA_MAX_SIZE) { 12513 verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", 12514 ret_t->size, BPF_GLOBAL_PERCPU_MA_MAX_SIZE); 12515 return -EINVAL; 12516 } 12517 12518 if (!bpf_global_percpu_ma_set) { 12519 mutex_lock(&bpf_percpu_ma_lock); 12520 if (!bpf_global_percpu_ma_set) { 12521 /* Charge memory allocated with bpf_global_percpu_ma to 12522 * root memcg. The obj_cgroup for root memcg is NULL. 12523 */ 12524 err = bpf_mem_alloc_percpu_init(&bpf_global_percpu_ma, NULL); 12525 if (!err) 12526 bpf_global_percpu_ma_set = true; 12527 } 12528 mutex_unlock(&bpf_percpu_ma_lock); 12529 if (err) 12530 return err; 12531 } 12532 12533 mutex_lock(&bpf_percpu_ma_lock); 12534 err = bpf_mem_alloc_percpu_unit_init(&bpf_global_percpu_ma, ret_t->size); 12535 mutex_unlock(&bpf_percpu_ma_lock); 12536 if (err) 12537 return err; 12538 } 12539 12540 struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); 12541 if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 12542 if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { 12543 verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); 12544 return -EINVAL; 12545 } 12546 12547 if (struct_meta) { 12548 verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); 12549 return -EINVAL; 12550 } 12551 } 12552 12553 mark_reg_known_zero(env, regs, BPF_REG_0); 12554 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 12555 regs[BPF_REG_0].btf = ret_btf; 12556 regs[BPF_REG_0].btf_id = ret_btf_id; 12557 if (meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) 12558 regs[BPF_REG_0].type |= MEM_PERCPU; 12559 12560 insn_aux->obj_new_size = ret_t->size; 12561 insn_aux->kptr_struct_meta = struct_meta; 12562 } else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 12563 mark_reg_known_zero(env, regs, BPF_REG_0); 12564 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 12565 regs[BPF_REG_0].btf = meta.arg_btf; 12566 regs[BPF_REG_0].btf_id = meta.arg_btf_id; 12567 12568 insn_aux->kptr_struct_meta = 12569 btf_find_struct_meta(meta.arg_btf, 12570 meta.arg_btf_id); 12571 } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 12572 meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 12573 struct btf_field *field = meta.arg_list_head.field; 12574 12575 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 12576 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] || 12577 meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 12578 struct btf_field *field = meta.arg_rbtree_root.field; 12579 12580 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 12581 } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 12582 mark_reg_known_zero(env, regs, BPF_REG_0); 12583 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 12584 regs[BPF_REG_0].btf = desc_btf; 12585 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 12586 } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 12587 ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value); 12588 if (!ret_t || !btf_type_is_struct(ret_t)) { 12589 verbose(env, 12590 "kfunc bpf_rdonly_cast type ID argument must be of a struct\n"); 12591 return -EINVAL; 12592 } 12593 12594 mark_reg_known_zero(env, regs, BPF_REG_0); 12595 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 12596 regs[BPF_REG_0].btf = desc_btf; 12597 regs[BPF_REG_0].btf_id = meta.arg_constant.value; 12598 } else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] || 12599 meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { 12600 enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type); 12601 12602 mark_reg_known_zero(env, regs, BPF_REG_0); 12603 12604 if (!meta.arg_constant.found) { 12605 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); 12606 return -EFAULT; 12607 } 12608 12609 regs[BPF_REG_0].mem_size = meta.arg_constant.value; 12610 12611 /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ 12612 regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; 12613 12614 if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { 12615 regs[BPF_REG_0].type |= MEM_RDONLY; 12616 } else { 12617 /* this will set env->seen_direct_write to true */ 12618 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { 12619 verbose(env, "the prog does not allow writes to packet data\n"); 12620 return -EINVAL; 12621 } 12622 } 12623 12624 if (!meta.initialized_dynptr.id) { 12625 verbose(env, "verifier internal error: no dynptr id\n"); 12626 return -EFAULT; 12627 } 12628 regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id; 12629 12630 /* we don't need to set BPF_REG_0's ref obj id 12631 * because packet slices are not refcounted (see 12632 * dynptr_type_refcounted) 12633 */ 12634 } else { 12635 verbose(env, "kernel function %s unhandled dynamic return type\n", 12636 meta.func_name); 12637 return -EFAULT; 12638 } 12639 } else if (btf_type_is_void(ptr_type)) { 12640 /* kfunc returning 'void *' is equivalent to returning scalar */ 12641 mark_reg_unknown(env, regs, BPF_REG_0); 12642 } else if (!__btf_type_is_struct(ptr_type)) { 12643 if (!meta.r0_size) { 12644 __u32 sz; 12645 12646 if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { 12647 meta.r0_size = sz; 12648 meta.r0_rdonly = true; 12649 } 12650 } 12651 if (!meta.r0_size) { 12652 ptr_type_name = btf_name_by_offset(desc_btf, 12653 ptr_type->name_off); 12654 verbose(env, 12655 "kernel function %s returns pointer type %s %s is not supported\n", 12656 func_name, 12657 btf_type_str(ptr_type), 12658 ptr_type_name); 12659 return -EINVAL; 12660 } 12661 12662 mark_reg_known_zero(env, regs, BPF_REG_0); 12663 regs[BPF_REG_0].type = PTR_TO_MEM; 12664 regs[BPF_REG_0].mem_size = meta.r0_size; 12665 12666 if (meta.r0_rdonly) 12667 regs[BPF_REG_0].type |= MEM_RDONLY; 12668 12669 /* Ensures we don't access the memory after a release_reference() */ 12670 if (meta.ref_obj_id) 12671 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 12672 } else { 12673 mark_reg_known_zero(env, regs, BPF_REG_0); 12674 regs[BPF_REG_0].btf = desc_btf; 12675 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 12676 regs[BPF_REG_0].btf_id = ptr_type_id; 12677 } 12678 12679 if (is_kfunc_ret_null(&meta)) { 12680 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 12681 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 12682 regs[BPF_REG_0].id = ++env->id_gen; 12683 } 12684 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 12685 if (is_kfunc_acquire(&meta)) { 12686 int id = acquire_reference_state(env, insn_idx); 12687 12688 if (id < 0) 12689 return id; 12690 if (is_kfunc_ret_null(&meta)) 12691 regs[BPF_REG_0].id = id; 12692 regs[BPF_REG_0].ref_obj_id = id; 12693 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 12694 ref_set_non_owning(env, ®s[BPF_REG_0]); 12695 } 12696 12697 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 12698 regs[BPF_REG_0].id = ++env->id_gen; 12699 } else if (btf_type_is_void(t)) { 12700 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 12701 if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 12702 meta.func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl]) { 12703 insn_aux->kptr_struct_meta = 12704 btf_find_struct_meta(meta.arg_btf, 12705 meta.arg_btf_id); 12706 } 12707 } 12708 } 12709 12710 nargs = btf_type_vlen(meta.func_proto); 12711 args = (const struct btf_param *)(meta.func_proto + 1); 12712 for (i = 0; i < nargs; i++) { 12713 u32 regno = i + 1; 12714 12715 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 12716 if (btf_type_is_ptr(t)) 12717 mark_btf_func_reg_size(env, regno, sizeof(void *)); 12718 else 12719 /* scalar. ensured by btf_check_kfunc_arg_match() */ 12720 mark_btf_func_reg_size(env, regno, t->size); 12721 } 12722 12723 if (is_iter_next_kfunc(&meta)) { 12724 err = process_iter_next_call(env, insn_idx, &meta); 12725 if (err) 12726 return err; 12727 } 12728 12729 return 0; 12730 } 12731 12732 static bool signed_add_overflows(s64 a, s64 b) 12733 { 12734 /* Do the add in u64, where overflow is well-defined */ 12735 s64 res = (s64)((u64)a + (u64)b); 12736 12737 if (b < 0) 12738 return res > a; 12739 return res < a; 12740 } 12741 12742 static bool signed_add32_overflows(s32 a, s32 b) 12743 { 12744 /* Do the add in u32, where overflow is well-defined */ 12745 s32 res = (s32)((u32)a + (u32)b); 12746 12747 if (b < 0) 12748 return res > a; 12749 return res < a; 12750 } 12751 12752 static bool signed_add16_overflows(s16 a, s16 b) 12753 { 12754 /* Do the add in u16, where overflow is well-defined */ 12755 s16 res = (s16)((u16)a + (u16)b); 12756 12757 if (b < 0) 12758 return res > a; 12759 return res < a; 12760 } 12761 12762 static bool signed_sub_overflows(s64 a, s64 b) 12763 { 12764 /* Do the sub in u64, where overflow is well-defined */ 12765 s64 res = (s64)((u64)a - (u64)b); 12766 12767 if (b < 0) 12768 return res < a; 12769 return res > a; 12770 } 12771 12772 static bool signed_sub32_overflows(s32 a, s32 b) 12773 { 12774 /* Do the sub in u32, where overflow is well-defined */ 12775 s32 res = (s32)((u32)a - (u32)b); 12776 12777 if (b < 0) 12778 return res < a; 12779 return res > a; 12780 } 12781 12782 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 12783 const struct bpf_reg_state *reg, 12784 enum bpf_reg_type type) 12785 { 12786 bool known = tnum_is_const(reg->var_off); 12787 s64 val = reg->var_off.value; 12788 s64 smin = reg->smin_value; 12789 12790 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 12791 verbose(env, "math between %s pointer and %lld is not allowed\n", 12792 reg_type_str(env, type), val); 12793 return false; 12794 } 12795 12796 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 12797 verbose(env, "%s pointer offset %d is not allowed\n", 12798 reg_type_str(env, type), reg->off); 12799 return false; 12800 } 12801 12802 if (smin == S64_MIN) { 12803 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 12804 reg_type_str(env, type)); 12805 return false; 12806 } 12807 12808 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 12809 verbose(env, "value %lld makes %s pointer be out of bounds\n", 12810 smin, reg_type_str(env, type)); 12811 return false; 12812 } 12813 12814 return true; 12815 } 12816 12817 enum { 12818 REASON_BOUNDS = -1, 12819 REASON_TYPE = -2, 12820 REASON_PATHS = -3, 12821 REASON_LIMIT = -4, 12822 REASON_STACK = -5, 12823 }; 12824 12825 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 12826 u32 *alu_limit, bool mask_to_left) 12827 { 12828 u32 max = 0, ptr_limit = 0; 12829 12830 switch (ptr_reg->type) { 12831 case PTR_TO_STACK: 12832 /* Offset 0 is out-of-bounds, but acceptable start for the 12833 * left direction, see BPF_REG_FP. Also, unknown scalar 12834 * offset where we would need to deal with min/max bounds is 12835 * currently prohibited for unprivileged. 12836 */ 12837 max = MAX_BPF_STACK + mask_to_left; 12838 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 12839 break; 12840 case PTR_TO_MAP_VALUE: 12841 max = ptr_reg->map_ptr->value_size; 12842 ptr_limit = (mask_to_left ? 12843 ptr_reg->smin_value : 12844 ptr_reg->umax_value) + ptr_reg->off; 12845 break; 12846 default: 12847 return REASON_TYPE; 12848 } 12849 12850 if (ptr_limit >= max) 12851 return REASON_LIMIT; 12852 *alu_limit = ptr_limit; 12853 return 0; 12854 } 12855 12856 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 12857 const struct bpf_insn *insn) 12858 { 12859 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 12860 } 12861 12862 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 12863 u32 alu_state, u32 alu_limit) 12864 { 12865 /* If we arrived here from different branches with different 12866 * state or limits to sanitize, then this won't work. 12867 */ 12868 if (aux->alu_state && 12869 (aux->alu_state != alu_state || 12870 aux->alu_limit != alu_limit)) 12871 return REASON_PATHS; 12872 12873 /* Corresponding fixup done in do_misc_fixups(). */ 12874 aux->alu_state = alu_state; 12875 aux->alu_limit = alu_limit; 12876 return 0; 12877 } 12878 12879 static int sanitize_val_alu(struct bpf_verifier_env *env, 12880 struct bpf_insn *insn) 12881 { 12882 struct bpf_insn_aux_data *aux = cur_aux(env); 12883 12884 if (can_skip_alu_sanitation(env, insn)) 12885 return 0; 12886 12887 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 12888 } 12889 12890 static bool sanitize_needed(u8 opcode) 12891 { 12892 return opcode == BPF_ADD || opcode == BPF_SUB; 12893 } 12894 12895 struct bpf_sanitize_info { 12896 struct bpf_insn_aux_data aux; 12897 bool mask_to_left; 12898 }; 12899 12900 static struct bpf_verifier_state * 12901 sanitize_speculative_path(struct bpf_verifier_env *env, 12902 const struct bpf_insn *insn, 12903 u32 next_idx, u32 curr_idx) 12904 { 12905 struct bpf_verifier_state *branch; 12906 struct bpf_reg_state *regs; 12907 12908 branch = push_stack(env, next_idx, curr_idx, true); 12909 if (branch && insn) { 12910 regs = branch->frame[branch->curframe]->regs; 12911 if (BPF_SRC(insn->code) == BPF_K) { 12912 mark_reg_unknown(env, regs, insn->dst_reg); 12913 } else if (BPF_SRC(insn->code) == BPF_X) { 12914 mark_reg_unknown(env, regs, insn->dst_reg); 12915 mark_reg_unknown(env, regs, insn->src_reg); 12916 } 12917 } 12918 return branch; 12919 } 12920 12921 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 12922 struct bpf_insn *insn, 12923 const struct bpf_reg_state *ptr_reg, 12924 const struct bpf_reg_state *off_reg, 12925 struct bpf_reg_state *dst_reg, 12926 struct bpf_sanitize_info *info, 12927 const bool commit_window) 12928 { 12929 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 12930 struct bpf_verifier_state *vstate = env->cur_state; 12931 bool off_is_imm = tnum_is_const(off_reg->var_off); 12932 bool off_is_neg = off_reg->smin_value < 0; 12933 bool ptr_is_dst_reg = ptr_reg == dst_reg; 12934 u8 opcode = BPF_OP(insn->code); 12935 u32 alu_state, alu_limit; 12936 struct bpf_reg_state tmp; 12937 bool ret; 12938 int err; 12939 12940 if (can_skip_alu_sanitation(env, insn)) 12941 return 0; 12942 12943 /* We already marked aux for masking from non-speculative 12944 * paths, thus we got here in the first place. We only care 12945 * to explore bad access from here. 12946 */ 12947 if (vstate->speculative) 12948 goto do_sim; 12949 12950 if (!commit_window) { 12951 if (!tnum_is_const(off_reg->var_off) && 12952 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 12953 return REASON_BOUNDS; 12954 12955 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 12956 (opcode == BPF_SUB && !off_is_neg); 12957 } 12958 12959 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 12960 if (err < 0) 12961 return err; 12962 12963 if (commit_window) { 12964 /* In commit phase we narrow the masking window based on 12965 * the observed pointer move after the simulated operation. 12966 */ 12967 alu_state = info->aux.alu_state; 12968 alu_limit = abs(info->aux.alu_limit - alu_limit); 12969 } else { 12970 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 12971 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 12972 alu_state |= ptr_is_dst_reg ? 12973 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 12974 12975 /* Limit pruning on unknown scalars to enable deep search for 12976 * potential masking differences from other program paths. 12977 */ 12978 if (!off_is_imm) 12979 env->explore_alu_limits = true; 12980 } 12981 12982 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 12983 if (err < 0) 12984 return err; 12985 do_sim: 12986 /* If we're in commit phase, we're done here given we already 12987 * pushed the truncated dst_reg into the speculative verification 12988 * stack. 12989 * 12990 * Also, when register is a known constant, we rewrite register-based 12991 * operation to immediate-based, and thus do not need masking (and as 12992 * a consequence, do not need to simulate the zero-truncation either). 12993 */ 12994 if (commit_window || off_is_imm) 12995 return 0; 12996 12997 /* Simulate and find potential out-of-bounds access under 12998 * speculative execution from truncation as a result of 12999 * masking when off was not within expected range. If off 13000 * sits in dst, then we temporarily need to move ptr there 13001 * to simulate dst (== 0) +/-= ptr. Needed, for example, 13002 * for cases where we use K-based arithmetic in one direction 13003 * and truncated reg-based in the other in order to explore 13004 * bad access. 13005 */ 13006 if (!ptr_is_dst_reg) { 13007 tmp = *dst_reg; 13008 copy_register_state(dst_reg, ptr_reg); 13009 } 13010 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 13011 env->insn_idx); 13012 if (!ptr_is_dst_reg && ret) 13013 *dst_reg = tmp; 13014 return !ret ? REASON_STACK : 0; 13015 } 13016 13017 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 13018 { 13019 struct bpf_verifier_state *vstate = env->cur_state; 13020 13021 /* If we simulate paths under speculation, we don't update the 13022 * insn as 'seen' such that when we verify unreachable paths in 13023 * the non-speculative domain, sanitize_dead_code() can still 13024 * rewrite/sanitize them. 13025 */ 13026 if (!vstate->speculative) 13027 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 13028 } 13029 13030 static int sanitize_err(struct bpf_verifier_env *env, 13031 const struct bpf_insn *insn, int reason, 13032 const struct bpf_reg_state *off_reg, 13033 const struct bpf_reg_state *dst_reg) 13034 { 13035 static const char *err = "pointer arithmetic with it prohibited for !root"; 13036 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 13037 u32 dst = insn->dst_reg, src = insn->src_reg; 13038 13039 switch (reason) { 13040 case REASON_BOUNDS: 13041 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 13042 off_reg == dst_reg ? dst : src, err); 13043 break; 13044 case REASON_TYPE: 13045 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 13046 off_reg == dst_reg ? src : dst, err); 13047 break; 13048 case REASON_PATHS: 13049 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 13050 dst, op, err); 13051 break; 13052 case REASON_LIMIT: 13053 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 13054 dst, op, err); 13055 break; 13056 case REASON_STACK: 13057 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 13058 dst, err); 13059 break; 13060 default: 13061 verbose(env, "verifier internal error: unknown reason (%d)\n", 13062 reason); 13063 break; 13064 } 13065 13066 return -EACCES; 13067 } 13068 13069 /* check that stack access falls within stack limits and that 'reg' doesn't 13070 * have a variable offset. 13071 * 13072 * Variable offset is prohibited for unprivileged mode for simplicity since it 13073 * requires corresponding support in Spectre masking for stack ALU. See also 13074 * retrieve_ptr_limit(). 13075 * 13076 * 13077 * 'off' includes 'reg->off'. 13078 */ 13079 static int check_stack_access_for_ptr_arithmetic( 13080 struct bpf_verifier_env *env, 13081 int regno, 13082 const struct bpf_reg_state *reg, 13083 int off) 13084 { 13085 if (!tnum_is_const(reg->var_off)) { 13086 char tn_buf[48]; 13087 13088 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 13089 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 13090 regno, tn_buf, off); 13091 return -EACCES; 13092 } 13093 13094 if (off >= 0 || off < -MAX_BPF_STACK) { 13095 verbose(env, "R%d stack pointer arithmetic goes out of range, " 13096 "prohibited for !root; off=%d\n", regno, off); 13097 return -EACCES; 13098 } 13099 13100 return 0; 13101 } 13102 13103 static int sanitize_check_bounds(struct bpf_verifier_env *env, 13104 const struct bpf_insn *insn, 13105 const struct bpf_reg_state *dst_reg) 13106 { 13107 u32 dst = insn->dst_reg; 13108 13109 /* For unprivileged we require that resulting offset must be in bounds 13110 * in order to be able to sanitize access later on. 13111 */ 13112 if (env->bypass_spec_v1) 13113 return 0; 13114 13115 switch (dst_reg->type) { 13116 case PTR_TO_STACK: 13117 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 13118 dst_reg->off + dst_reg->var_off.value)) 13119 return -EACCES; 13120 break; 13121 case PTR_TO_MAP_VALUE: 13122 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 13123 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 13124 "prohibited for !root\n", dst); 13125 return -EACCES; 13126 } 13127 break; 13128 default: 13129 break; 13130 } 13131 13132 return 0; 13133 } 13134 13135 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 13136 * Caller should also handle BPF_MOV case separately. 13137 * If we return -EACCES, caller may want to try again treating pointer as a 13138 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 13139 */ 13140 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 13141 struct bpf_insn *insn, 13142 const struct bpf_reg_state *ptr_reg, 13143 const struct bpf_reg_state *off_reg) 13144 { 13145 struct bpf_verifier_state *vstate = env->cur_state; 13146 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 13147 struct bpf_reg_state *regs = state->regs, *dst_reg; 13148 bool known = tnum_is_const(off_reg->var_off); 13149 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 13150 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 13151 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 13152 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 13153 struct bpf_sanitize_info info = {}; 13154 u8 opcode = BPF_OP(insn->code); 13155 u32 dst = insn->dst_reg; 13156 int ret; 13157 13158 dst_reg = ®s[dst]; 13159 13160 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 13161 smin_val > smax_val || umin_val > umax_val) { 13162 /* Taint dst register if offset had invalid bounds derived from 13163 * e.g. dead branches. 13164 */ 13165 __mark_reg_unknown(env, dst_reg); 13166 return 0; 13167 } 13168 13169 if (BPF_CLASS(insn->code) != BPF_ALU64) { 13170 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 13171 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 13172 __mark_reg_unknown(env, dst_reg); 13173 return 0; 13174 } 13175 13176 verbose(env, 13177 "R%d 32-bit pointer arithmetic prohibited\n", 13178 dst); 13179 return -EACCES; 13180 } 13181 13182 if (ptr_reg->type & PTR_MAYBE_NULL) { 13183 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 13184 dst, reg_type_str(env, ptr_reg->type)); 13185 return -EACCES; 13186 } 13187 13188 switch (base_type(ptr_reg->type)) { 13189 case PTR_TO_CTX: 13190 case PTR_TO_MAP_VALUE: 13191 case PTR_TO_MAP_KEY: 13192 case PTR_TO_STACK: 13193 case PTR_TO_PACKET_META: 13194 case PTR_TO_PACKET: 13195 case PTR_TO_TP_BUFFER: 13196 case PTR_TO_BTF_ID: 13197 case PTR_TO_MEM: 13198 case PTR_TO_BUF: 13199 case PTR_TO_FUNC: 13200 case CONST_PTR_TO_DYNPTR: 13201 break; 13202 case PTR_TO_FLOW_KEYS: 13203 if (known) 13204 break; 13205 fallthrough; 13206 case CONST_PTR_TO_MAP: 13207 /* smin_val represents the known value */ 13208 if (known && smin_val == 0 && opcode == BPF_ADD) 13209 break; 13210 fallthrough; 13211 default: 13212 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 13213 dst, reg_type_str(env, ptr_reg->type)); 13214 return -EACCES; 13215 } 13216 13217 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 13218 * The id may be overwritten later if we create a new variable offset. 13219 */ 13220 dst_reg->type = ptr_reg->type; 13221 dst_reg->id = ptr_reg->id; 13222 13223 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 13224 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 13225 return -EINVAL; 13226 13227 /* pointer types do not carry 32-bit bounds at the moment. */ 13228 __mark_reg32_unbounded(dst_reg); 13229 13230 if (sanitize_needed(opcode)) { 13231 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 13232 &info, false); 13233 if (ret < 0) 13234 return sanitize_err(env, insn, ret, off_reg, dst_reg); 13235 } 13236 13237 switch (opcode) { 13238 case BPF_ADD: 13239 /* We can take a fixed offset as long as it doesn't overflow 13240 * the s32 'off' field 13241 */ 13242 if (known && (ptr_reg->off + smin_val == 13243 (s64)(s32)(ptr_reg->off + smin_val))) { 13244 /* pointer += K. Accumulate it into fixed offset */ 13245 dst_reg->smin_value = smin_ptr; 13246 dst_reg->smax_value = smax_ptr; 13247 dst_reg->umin_value = umin_ptr; 13248 dst_reg->umax_value = umax_ptr; 13249 dst_reg->var_off = ptr_reg->var_off; 13250 dst_reg->off = ptr_reg->off + smin_val; 13251 dst_reg->raw = ptr_reg->raw; 13252 break; 13253 } 13254 /* A new variable offset is created. Note that off_reg->off 13255 * == 0, since it's a scalar. 13256 * dst_reg gets the pointer type and since some positive 13257 * integer value was added to the pointer, give it a new 'id' 13258 * if it's a PTR_TO_PACKET. 13259 * this creates a new 'base' pointer, off_reg (variable) gets 13260 * added into the variable offset, and we copy the fixed offset 13261 * from ptr_reg. 13262 */ 13263 if (signed_add_overflows(smin_ptr, smin_val) || 13264 signed_add_overflows(smax_ptr, smax_val)) { 13265 dst_reg->smin_value = S64_MIN; 13266 dst_reg->smax_value = S64_MAX; 13267 } else { 13268 dst_reg->smin_value = smin_ptr + smin_val; 13269 dst_reg->smax_value = smax_ptr + smax_val; 13270 } 13271 if (umin_ptr + umin_val < umin_ptr || 13272 umax_ptr + umax_val < umax_ptr) { 13273 dst_reg->umin_value = 0; 13274 dst_reg->umax_value = U64_MAX; 13275 } else { 13276 dst_reg->umin_value = umin_ptr + umin_val; 13277 dst_reg->umax_value = umax_ptr + umax_val; 13278 } 13279 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 13280 dst_reg->off = ptr_reg->off; 13281 dst_reg->raw = ptr_reg->raw; 13282 if (reg_is_pkt_pointer(ptr_reg)) { 13283 dst_reg->id = ++env->id_gen; 13284 /* something was added to pkt_ptr, set range to zero */ 13285 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 13286 } 13287 break; 13288 case BPF_SUB: 13289 if (dst_reg == off_reg) { 13290 /* scalar -= pointer. Creates an unknown scalar */ 13291 verbose(env, "R%d tried to subtract pointer from scalar\n", 13292 dst); 13293 return -EACCES; 13294 } 13295 /* We don't allow subtraction from FP, because (according to 13296 * test_verifier.c test "invalid fp arithmetic", JITs might not 13297 * be able to deal with it. 13298 */ 13299 if (ptr_reg->type == PTR_TO_STACK) { 13300 verbose(env, "R%d subtraction from stack pointer prohibited\n", 13301 dst); 13302 return -EACCES; 13303 } 13304 if (known && (ptr_reg->off - smin_val == 13305 (s64)(s32)(ptr_reg->off - smin_val))) { 13306 /* pointer -= K. Subtract it from fixed offset */ 13307 dst_reg->smin_value = smin_ptr; 13308 dst_reg->smax_value = smax_ptr; 13309 dst_reg->umin_value = umin_ptr; 13310 dst_reg->umax_value = umax_ptr; 13311 dst_reg->var_off = ptr_reg->var_off; 13312 dst_reg->id = ptr_reg->id; 13313 dst_reg->off = ptr_reg->off - smin_val; 13314 dst_reg->raw = ptr_reg->raw; 13315 break; 13316 } 13317 /* A new variable offset is created. If the subtrahend is known 13318 * nonnegative, then any reg->range we had before is still good. 13319 */ 13320 if (signed_sub_overflows(smin_ptr, smax_val) || 13321 signed_sub_overflows(smax_ptr, smin_val)) { 13322 /* Overflow possible, we know nothing */ 13323 dst_reg->smin_value = S64_MIN; 13324 dst_reg->smax_value = S64_MAX; 13325 } else { 13326 dst_reg->smin_value = smin_ptr - smax_val; 13327 dst_reg->smax_value = smax_ptr - smin_val; 13328 } 13329 if (umin_ptr < umax_val) { 13330 /* Overflow possible, we know nothing */ 13331 dst_reg->umin_value = 0; 13332 dst_reg->umax_value = U64_MAX; 13333 } else { 13334 /* Cannot overflow (as long as bounds are consistent) */ 13335 dst_reg->umin_value = umin_ptr - umax_val; 13336 dst_reg->umax_value = umax_ptr - umin_val; 13337 } 13338 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 13339 dst_reg->off = ptr_reg->off; 13340 dst_reg->raw = ptr_reg->raw; 13341 if (reg_is_pkt_pointer(ptr_reg)) { 13342 dst_reg->id = ++env->id_gen; 13343 /* something was added to pkt_ptr, set range to zero */ 13344 if (smin_val < 0) 13345 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 13346 } 13347 break; 13348 case BPF_AND: 13349 case BPF_OR: 13350 case BPF_XOR: 13351 /* bitwise ops on pointers are troublesome, prohibit. */ 13352 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 13353 dst, bpf_alu_string[opcode >> 4]); 13354 return -EACCES; 13355 default: 13356 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 13357 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 13358 dst, bpf_alu_string[opcode >> 4]); 13359 return -EACCES; 13360 } 13361 13362 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 13363 return -EINVAL; 13364 reg_bounds_sync(dst_reg); 13365 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 13366 return -EACCES; 13367 if (sanitize_needed(opcode)) { 13368 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 13369 &info, true); 13370 if (ret < 0) 13371 return sanitize_err(env, insn, ret, off_reg, dst_reg); 13372 } 13373 13374 return 0; 13375 } 13376 13377 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 13378 struct bpf_reg_state *src_reg) 13379 { 13380 s32 smin_val = src_reg->s32_min_value; 13381 s32 smax_val = src_reg->s32_max_value; 13382 u32 umin_val = src_reg->u32_min_value; 13383 u32 umax_val = src_reg->u32_max_value; 13384 13385 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 13386 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 13387 dst_reg->s32_min_value = S32_MIN; 13388 dst_reg->s32_max_value = S32_MAX; 13389 } else { 13390 dst_reg->s32_min_value += smin_val; 13391 dst_reg->s32_max_value += smax_val; 13392 } 13393 if (dst_reg->u32_min_value + umin_val < umin_val || 13394 dst_reg->u32_max_value + umax_val < umax_val) { 13395 dst_reg->u32_min_value = 0; 13396 dst_reg->u32_max_value = U32_MAX; 13397 } else { 13398 dst_reg->u32_min_value += umin_val; 13399 dst_reg->u32_max_value += umax_val; 13400 } 13401 } 13402 13403 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 13404 struct bpf_reg_state *src_reg) 13405 { 13406 s64 smin_val = src_reg->smin_value; 13407 s64 smax_val = src_reg->smax_value; 13408 u64 umin_val = src_reg->umin_value; 13409 u64 umax_val = src_reg->umax_value; 13410 13411 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 13412 signed_add_overflows(dst_reg->smax_value, smax_val)) { 13413 dst_reg->smin_value = S64_MIN; 13414 dst_reg->smax_value = S64_MAX; 13415 } else { 13416 dst_reg->smin_value += smin_val; 13417 dst_reg->smax_value += smax_val; 13418 } 13419 if (dst_reg->umin_value + umin_val < umin_val || 13420 dst_reg->umax_value + umax_val < umax_val) { 13421 dst_reg->umin_value = 0; 13422 dst_reg->umax_value = U64_MAX; 13423 } else { 13424 dst_reg->umin_value += umin_val; 13425 dst_reg->umax_value += umax_val; 13426 } 13427 } 13428 13429 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 13430 struct bpf_reg_state *src_reg) 13431 { 13432 s32 smin_val = src_reg->s32_min_value; 13433 s32 smax_val = src_reg->s32_max_value; 13434 u32 umin_val = src_reg->u32_min_value; 13435 u32 umax_val = src_reg->u32_max_value; 13436 13437 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 13438 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 13439 /* Overflow possible, we know nothing */ 13440 dst_reg->s32_min_value = S32_MIN; 13441 dst_reg->s32_max_value = S32_MAX; 13442 } else { 13443 dst_reg->s32_min_value -= smax_val; 13444 dst_reg->s32_max_value -= smin_val; 13445 } 13446 if (dst_reg->u32_min_value < umax_val) { 13447 /* Overflow possible, we know nothing */ 13448 dst_reg->u32_min_value = 0; 13449 dst_reg->u32_max_value = U32_MAX; 13450 } else { 13451 /* Cannot overflow (as long as bounds are consistent) */ 13452 dst_reg->u32_min_value -= umax_val; 13453 dst_reg->u32_max_value -= umin_val; 13454 } 13455 } 13456 13457 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 13458 struct bpf_reg_state *src_reg) 13459 { 13460 s64 smin_val = src_reg->smin_value; 13461 s64 smax_val = src_reg->smax_value; 13462 u64 umin_val = src_reg->umin_value; 13463 u64 umax_val = src_reg->umax_value; 13464 13465 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 13466 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 13467 /* Overflow possible, we know nothing */ 13468 dst_reg->smin_value = S64_MIN; 13469 dst_reg->smax_value = S64_MAX; 13470 } else { 13471 dst_reg->smin_value -= smax_val; 13472 dst_reg->smax_value -= smin_val; 13473 } 13474 if (dst_reg->umin_value < umax_val) { 13475 /* Overflow possible, we know nothing */ 13476 dst_reg->umin_value = 0; 13477 dst_reg->umax_value = U64_MAX; 13478 } else { 13479 /* Cannot overflow (as long as bounds are consistent) */ 13480 dst_reg->umin_value -= umax_val; 13481 dst_reg->umax_value -= umin_val; 13482 } 13483 } 13484 13485 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 13486 struct bpf_reg_state *src_reg) 13487 { 13488 s32 smin_val = src_reg->s32_min_value; 13489 u32 umin_val = src_reg->u32_min_value; 13490 u32 umax_val = src_reg->u32_max_value; 13491 13492 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 13493 /* Ain't nobody got time to multiply that sign */ 13494 __mark_reg32_unbounded(dst_reg); 13495 return; 13496 } 13497 /* Both values are positive, so we can work with unsigned and 13498 * copy the result to signed (unless it exceeds S32_MAX). 13499 */ 13500 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 13501 /* Potential overflow, we know nothing */ 13502 __mark_reg32_unbounded(dst_reg); 13503 return; 13504 } 13505 dst_reg->u32_min_value *= umin_val; 13506 dst_reg->u32_max_value *= umax_val; 13507 if (dst_reg->u32_max_value > S32_MAX) { 13508 /* Overflow possible, we know nothing */ 13509 dst_reg->s32_min_value = S32_MIN; 13510 dst_reg->s32_max_value = S32_MAX; 13511 } else { 13512 dst_reg->s32_min_value = dst_reg->u32_min_value; 13513 dst_reg->s32_max_value = dst_reg->u32_max_value; 13514 } 13515 } 13516 13517 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 13518 struct bpf_reg_state *src_reg) 13519 { 13520 s64 smin_val = src_reg->smin_value; 13521 u64 umin_val = src_reg->umin_value; 13522 u64 umax_val = src_reg->umax_value; 13523 13524 if (smin_val < 0 || dst_reg->smin_value < 0) { 13525 /* Ain't nobody got time to multiply that sign */ 13526 __mark_reg64_unbounded(dst_reg); 13527 return; 13528 } 13529 /* Both values are positive, so we can work with unsigned and 13530 * copy the result to signed (unless it exceeds S64_MAX). 13531 */ 13532 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 13533 /* Potential overflow, we know nothing */ 13534 __mark_reg64_unbounded(dst_reg); 13535 return; 13536 } 13537 dst_reg->umin_value *= umin_val; 13538 dst_reg->umax_value *= umax_val; 13539 if (dst_reg->umax_value > S64_MAX) { 13540 /* Overflow possible, we know nothing */ 13541 dst_reg->smin_value = S64_MIN; 13542 dst_reg->smax_value = S64_MAX; 13543 } else { 13544 dst_reg->smin_value = dst_reg->umin_value; 13545 dst_reg->smax_value = dst_reg->umax_value; 13546 } 13547 } 13548 13549 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 13550 struct bpf_reg_state *src_reg) 13551 { 13552 bool src_known = tnum_subreg_is_const(src_reg->var_off); 13553 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 13554 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 13555 u32 umax_val = src_reg->u32_max_value; 13556 13557 if (src_known && dst_known) { 13558 __mark_reg32_known(dst_reg, var32_off.value); 13559 return; 13560 } 13561 13562 /* We get our minimum from the var_off, since that's inherently 13563 * bitwise. Our maximum is the minimum of the operands' maxima. 13564 */ 13565 dst_reg->u32_min_value = var32_off.value; 13566 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 13567 13568 /* Safe to set s32 bounds by casting u32 result into s32 when u32 13569 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. 13570 */ 13571 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { 13572 dst_reg->s32_min_value = dst_reg->u32_min_value; 13573 dst_reg->s32_max_value = dst_reg->u32_max_value; 13574 } else { 13575 dst_reg->s32_min_value = S32_MIN; 13576 dst_reg->s32_max_value = S32_MAX; 13577 } 13578 } 13579 13580 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 13581 struct bpf_reg_state *src_reg) 13582 { 13583 bool src_known = tnum_is_const(src_reg->var_off); 13584 bool dst_known = tnum_is_const(dst_reg->var_off); 13585 u64 umax_val = src_reg->umax_value; 13586 13587 if (src_known && dst_known) { 13588 __mark_reg_known(dst_reg, dst_reg->var_off.value); 13589 return; 13590 } 13591 13592 /* We get our minimum from the var_off, since that's inherently 13593 * bitwise. Our maximum is the minimum of the operands' maxima. 13594 */ 13595 dst_reg->umin_value = dst_reg->var_off.value; 13596 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 13597 13598 /* Safe to set s64 bounds by casting u64 result into s64 when u64 13599 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. 13600 */ 13601 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { 13602 dst_reg->smin_value = dst_reg->umin_value; 13603 dst_reg->smax_value = dst_reg->umax_value; 13604 } else { 13605 dst_reg->smin_value = S64_MIN; 13606 dst_reg->smax_value = S64_MAX; 13607 } 13608 /* We may learn something more from the var_off */ 13609 __update_reg_bounds(dst_reg); 13610 } 13611 13612 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 13613 struct bpf_reg_state *src_reg) 13614 { 13615 bool src_known = tnum_subreg_is_const(src_reg->var_off); 13616 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 13617 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 13618 u32 umin_val = src_reg->u32_min_value; 13619 13620 if (src_known && dst_known) { 13621 __mark_reg32_known(dst_reg, var32_off.value); 13622 return; 13623 } 13624 13625 /* We get our maximum from the var_off, and our minimum is the 13626 * maximum of the operands' minima 13627 */ 13628 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 13629 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 13630 13631 /* Safe to set s32 bounds by casting u32 result into s32 when u32 13632 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. 13633 */ 13634 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { 13635 dst_reg->s32_min_value = dst_reg->u32_min_value; 13636 dst_reg->s32_max_value = dst_reg->u32_max_value; 13637 } else { 13638 dst_reg->s32_min_value = S32_MIN; 13639 dst_reg->s32_max_value = S32_MAX; 13640 } 13641 } 13642 13643 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 13644 struct bpf_reg_state *src_reg) 13645 { 13646 bool src_known = tnum_is_const(src_reg->var_off); 13647 bool dst_known = tnum_is_const(dst_reg->var_off); 13648 u64 umin_val = src_reg->umin_value; 13649 13650 if (src_known && dst_known) { 13651 __mark_reg_known(dst_reg, dst_reg->var_off.value); 13652 return; 13653 } 13654 13655 /* We get our maximum from the var_off, and our minimum is the 13656 * maximum of the operands' minima 13657 */ 13658 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 13659 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 13660 13661 /* Safe to set s64 bounds by casting u64 result into s64 when u64 13662 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. 13663 */ 13664 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { 13665 dst_reg->smin_value = dst_reg->umin_value; 13666 dst_reg->smax_value = dst_reg->umax_value; 13667 } else { 13668 dst_reg->smin_value = S64_MIN; 13669 dst_reg->smax_value = S64_MAX; 13670 } 13671 /* We may learn something more from the var_off */ 13672 __update_reg_bounds(dst_reg); 13673 } 13674 13675 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 13676 struct bpf_reg_state *src_reg) 13677 { 13678 bool src_known = tnum_subreg_is_const(src_reg->var_off); 13679 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 13680 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 13681 13682 if (src_known && dst_known) { 13683 __mark_reg32_known(dst_reg, var32_off.value); 13684 return; 13685 } 13686 13687 /* We get both minimum and maximum from the var32_off. */ 13688 dst_reg->u32_min_value = var32_off.value; 13689 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 13690 13691 /* Safe to set s32 bounds by casting u32 result into s32 when u32 13692 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded. 13693 */ 13694 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { 13695 dst_reg->s32_min_value = dst_reg->u32_min_value; 13696 dst_reg->s32_max_value = dst_reg->u32_max_value; 13697 } else { 13698 dst_reg->s32_min_value = S32_MIN; 13699 dst_reg->s32_max_value = S32_MAX; 13700 } 13701 } 13702 13703 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 13704 struct bpf_reg_state *src_reg) 13705 { 13706 bool src_known = tnum_is_const(src_reg->var_off); 13707 bool dst_known = tnum_is_const(dst_reg->var_off); 13708 13709 if (src_known && dst_known) { 13710 /* dst_reg->var_off.value has been updated earlier */ 13711 __mark_reg_known(dst_reg, dst_reg->var_off.value); 13712 return; 13713 } 13714 13715 /* We get both minimum and maximum from the var_off. */ 13716 dst_reg->umin_value = dst_reg->var_off.value; 13717 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 13718 13719 /* Safe to set s64 bounds by casting u64 result into s64 when u64 13720 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded. 13721 */ 13722 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { 13723 dst_reg->smin_value = dst_reg->umin_value; 13724 dst_reg->smax_value = dst_reg->umax_value; 13725 } else { 13726 dst_reg->smin_value = S64_MIN; 13727 dst_reg->smax_value = S64_MAX; 13728 } 13729 13730 __update_reg_bounds(dst_reg); 13731 } 13732 13733 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 13734 u64 umin_val, u64 umax_val) 13735 { 13736 /* We lose all sign bit information (except what we can pick 13737 * up from var_off) 13738 */ 13739 dst_reg->s32_min_value = S32_MIN; 13740 dst_reg->s32_max_value = S32_MAX; 13741 /* If we might shift our top bit out, then we know nothing */ 13742 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 13743 dst_reg->u32_min_value = 0; 13744 dst_reg->u32_max_value = U32_MAX; 13745 } else { 13746 dst_reg->u32_min_value <<= umin_val; 13747 dst_reg->u32_max_value <<= umax_val; 13748 } 13749 } 13750 13751 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 13752 struct bpf_reg_state *src_reg) 13753 { 13754 u32 umax_val = src_reg->u32_max_value; 13755 u32 umin_val = src_reg->u32_min_value; 13756 /* u32 alu operation will zext upper bits */ 13757 struct tnum subreg = tnum_subreg(dst_reg->var_off); 13758 13759 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 13760 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 13761 /* Not required but being careful mark reg64 bounds as unknown so 13762 * that we are forced to pick them up from tnum and zext later and 13763 * if some path skips this step we are still safe. 13764 */ 13765 __mark_reg64_unbounded(dst_reg); 13766 __update_reg32_bounds(dst_reg); 13767 } 13768 13769 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 13770 u64 umin_val, u64 umax_val) 13771 { 13772 /* Special case <<32 because it is a common compiler pattern to sign 13773 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 13774 * positive we know this shift will also be positive so we can track 13775 * bounds correctly. Otherwise we lose all sign bit information except 13776 * what we can pick up from var_off. Perhaps we can generalize this 13777 * later to shifts of any length. 13778 */ 13779 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 13780 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 13781 else 13782 dst_reg->smax_value = S64_MAX; 13783 13784 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 13785 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 13786 else 13787 dst_reg->smin_value = S64_MIN; 13788 13789 /* If we might shift our top bit out, then we know nothing */ 13790 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 13791 dst_reg->umin_value = 0; 13792 dst_reg->umax_value = U64_MAX; 13793 } else { 13794 dst_reg->umin_value <<= umin_val; 13795 dst_reg->umax_value <<= umax_val; 13796 } 13797 } 13798 13799 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 13800 struct bpf_reg_state *src_reg) 13801 { 13802 u64 umax_val = src_reg->umax_value; 13803 u64 umin_val = src_reg->umin_value; 13804 13805 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 13806 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 13807 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 13808 13809 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 13810 /* We may learn something more from the var_off */ 13811 __update_reg_bounds(dst_reg); 13812 } 13813 13814 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 13815 struct bpf_reg_state *src_reg) 13816 { 13817 struct tnum subreg = tnum_subreg(dst_reg->var_off); 13818 u32 umax_val = src_reg->u32_max_value; 13819 u32 umin_val = src_reg->u32_min_value; 13820 13821 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 13822 * be negative, then either: 13823 * 1) src_reg might be zero, so the sign bit of the result is 13824 * unknown, so we lose our signed bounds 13825 * 2) it's known negative, thus the unsigned bounds capture the 13826 * signed bounds 13827 * 3) the signed bounds cross zero, so they tell us nothing 13828 * about the result 13829 * If the value in dst_reg is known nonnegative, then again the 13830 * unsigned bounds capture the signed bounds. 13831 * Thus, in all cases it suffices to blow away our signed bounds 13832 * and rely on inferring new ones from the unsigned bounds and 13833 * var_off of the result. 13834 */ 13835 dst_reg->s32_min_value = S32_MIN; 13836 dst_reg->s32_max_value = S32_MAX; 13837 13838 dst_reg->var_off = tnum_rshift(subreg, umin_val); 13839 dst_reg->u32_min_value >>= umax_val; 13840 dst_reg->u32_max_value >>= umin_val; 13841 13842 __mark_reg64_unbounded(dst_reg); 13843 __update_reg32_bounds(dst_reg); 13844 } 13845 13846 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 13847 struct bpf_reg_state *src_reg) 13848 { 13849 u64 umax_val = src_reg->umax_value; 13850 u64 umin_val = src_reg->umin_value; 13851 13852 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 13853 * be negative, then either: 13854 * 1) src_reg might be zero, so the sign bit of the result is 13855 * unknown, so we lose our signed bounds 13856 * 2) it's known negative, thus the unsigned bounds capture the 13857 * signed bounds 13858 * 3) the signed bounds cross zero, so they tell us nothing 13859 * about the result 13860 * If the value in dst_reg is known nonnegative, then again the 13861 * unsigned bounds capture the signed bounds. 13862 * Thus, in all cases it suffices to blow away our signed bounds 13863 * and rely on inferring new ones from the unsigned bounds and 13864 * var_off of the result. 13865 */ 13866 dst_reg->smin_value = S64_MIN; 13867 dst_reg->smax_value = S64_MAX; 13868 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 13869 dst_reg->umin_value >>= umax_val; 13870 dst_reg->umax_value >>= umin_val; 13871 13872 /* Its not easy to operate on alu32 bounds here because it depends 13873 * on bits being shifted in. Take easy way out and mark unbounded 13874 * so we can recalculate later from tnum. 13875 */ 13876 __mark_reg32_unbounded(dst_reg); 13877 __update_reg_bounds(dst_reg); 13878 } 13879 13880 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 13881 struct bpf_reg_state *src_reg) 13882 { 13883 u64 umin_val = src_reg->u32_min_value; 13884 13885 /* Upon reaching here, src_known is true and 13886 * umax_val is equal to umin_val. 13887 */ 13888 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 13889 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 13890 13891 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 13892 13893 /* blow away the dst_reg umin_value/umax_value and rely on 13894 * dst_reg var_off to refine the result. 13895 */ 13896 dst_reg->u32_min_value = 0; 13897 dst_reg->u32_max_value = U32_MAX; 13898 13899 __mark_reg64_unbounded(dst_reg); 13900 __update_reg32_bounds(dst_reg); 13901 } 13902 13903 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 13904 struct bpf_reg_state *src_reg) 13905 { 13906 u64 umin_val = src_reg->umin_value; 13907 13908 /* Upon reaching here, src_known is true and umax_val is equal 13909 * to umin_val. 13910 */ 13911 dst_reg->smin_value >>= umin_val; 13912 dst_reg->smax_value >>= umin_val; 13913 13914 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 13915 13916 /* blow away the dst_reg umin_value/umax_value and rely on 13917 * dst_reg var_off to refine the result. 13918 */ 13919 dst_reg->umin_value = 0; 13920 dst_reg->umax_value = U64_MAX; 13921 13922 /* Its not easy to operate on alu32 bounds here because it depends 13923 * on bits being shifted in from upper 32-bits. Take easy way out 13924 * and mark unbounded so we can recalculate later from tnum. 13925 */ 13926 __mark_reg32_unbounded(dst_reg); 13927 __update_reg_bounds(dst_reg); 13928 } 13929 13930 static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, 13931 const struct bpf_reg_state *src_reg) 13932 { 13933 bool src_is_const = false; 13934 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 13935 13936 if (insn_bitness == 32) { 13937 if (tnum_subreg_is_const(src_reg->var_off) 13938 && src_reg->s32_min_value == src_reg->s32_max_value 13939 && src_reg->u32_min_value == src_reg->u32_max_value) 13940 src_is_const = true; 13941 } else { 13942 if (tnum_is_const(src_reg->var_off) 13943 && src_reg->smin_value == src_reg->smax_value 13944 && src_reg->umin_value == src_reg->umax_value) 13945 src_is_const = true; 13946 } 13947 13948 switch (BPF_OP(insn->code)) { 13949 case BPF_ADD: 13950 case BPF_SUB: 13951 case BPF_AND: 13952 case BPF_XOR: 13953 case BPF_OR: 13954 case BPF_MUL: 13955 return true; 13956 13957 /* Shift operators range is only computable if shift dimension operand 13958 * is a constant. Shifts greater than 31 or 63 are undefined. This 13959 * includes shifts by a negative number. 13960 */ 13961 case BPF_LSH: 13962 case BPF_RSH: 13963 case BPF_ARSH: 13964 return (src_is_const && src_reg->umax_value < insn_bitness); 13965 default: 13966 return false; 13967 } 13968 } 13969 13970 /* WARNING: This function does calculations on 64-bit values, but the actual 13971 * execution may occur on 32-bit values. Therefore, things like bitshifts 13972 * need extra checks in the 32-bit case. 13973 */ 13974 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 13975 struct bpf_insn *insn, 13976 struct bpf_reg_state *dst_reg, 13977 struct bpf_reg_state src_reg) 13978 { 13979 u8 opcode = BPF_OP(insn->code); 13980 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 13981 int ret; 13982 13983 if (!is_safe_to_compute_dst_reg_range(insn, &src_reg)) { 13984 __mark_reg_unknown(env, dst_reg); 13985 return 0; 13986 } 13987 13988 if (sanitize_needed(opcode)) { 13989 ret = sanitize_val_alu(env, insn); 13990 if (ret < 0) 13991 return sanitize_err(env, insn, ret, NULL, NULL); 13992 } 13993 13994 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 13995 * There are two classes of instructions: The first class we track both 13996 * alu32 and alu64 sign/unsigned bounds independently this provides the 13997 * greatest amount of precision when alu operations are mixed with jmp32 13998 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 13999 * and BPF_OR. This is possible because these ops have fairly easy to 14000 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 14001 * See alu32 verifier tests for examples. The second class of 14002 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 14003 * with regards to tracking sign/unsigned bounds because the bits may 14004 * cross subreg boundaries in the alu64 case. When this happens we mark 14005 * the reg unbounded in the subreg bound space and use the resulting 14006 * tnum to calculate an approximation of the sign/unsigned bounds. 14007 */ 14008 switch (opcode) { 14009 case BPF_ADD: 14010 scalar32_min_max_add(dst_reg, &src_reg); 14011 scalar_min_max_add(dst_reg, &src_reg); 14012 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 14013 break; 14014 case BPF_SUB: 14015 scalar32_min_max_sub(dst_reg, &src_reg); 14016 scalar_min_max_sub(dst_reg, &src_reg); 14017 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 14018 break; 14019 case BPF_MUL: 14020 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 14021 scalar32_min_max_mul(dst_reg, &src_reg); 14022 scalar_min_max_mul(dst_reg, &src_reg); 14023 break; 14024 case BPF_AND: 14025 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 14026 scalar32_min_max_and(dst_reg, &src_reg); 14027 scalar_min_max_and(dst_reg, &src_reg); 14028 break; 14029 case BPF_OR: 14030 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 14031 scalar32_min_max_or(dst_reg, &src_reg); 14032 scalar_min_max_or(dst_reg, &src_reg); 14033 break; 14034 case BPF_XOR: 14035 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 14036 scalar32_min_max_xor(dst_reg, &src_reg); 14037 scalar_min_max_xor(dst_reg, &src_reg); 14038 break; 14039 case BPF_LSH: 14040 if (alu32) 14041 scalar32_min_max_lsh(dst_reg, &src_reg); 14042 else 14043 scalar_min_max_lsh(dst_reg, &src_reg); 14044 break; 14045 case BPF_RSH: 14046 if (alu32) 14047 scalar32_min_max_rsh(dst_reg, &src_reg); 14048 else 14049 scalar_min_max_rsh(dst_reg, &src_reg); 14050 break; 14051 case BPF_ARSH: 14052 if (alu32) 14053 scalar32_min_max_arsh(dst_reg, &src_reg); 14054 else 14055 scalar_min_max_arsh(dst_reg, &src_reg); 14056 break; 14057 default: 14058 break; 14059 } 14060 14061 /* ALU32 ops are zero extended into 64bit register */ 14062 if (alu32) 14063 zext_32_to_64(dst_reg); 14064 reg_bounds_sync(dst_reg); 14065 return 0; 14066 } 14067 14068 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 14069 * and var_off. 14070 */ 14071 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 14072 struct bpf_insn *insn) 14073 { 14074 struct bpf_verifier_state *vstate = env->cur_state; 14075 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 14076 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 14077 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 14078 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 14079 u8 opcode = BPF_OP(insn->code); 14080 int err; 14081 14082 dst_reg = ®s[insn->dst_reg]; 14083 src_reg = NULL; 14084 14085 if (dst_reg->type == PTR_TO_ARENA) { 14086 struct bpf_insn_aux_data *aux = cur_aux(env); 14087 14088 if (BPF_CLASS(insn->code) == BPF_ALU64) 14089 /* 14090 * 32-bit operations zero upper bits automatically. 14091 * 64-bit operations need to be converted to 32. 14092 */ 14093 aux->needs_zext = true; 14094 14095 /* Any arithmetic operations are allowed on arena pointers */ 14096 return 0; 14097 } 14098 14099 if (dst_reg->type != SCALAR_VALUE) 14100 ptr_reg = dst_reg; 14101 14102 if (BPF_SRC(insn->code) == BPF_X) { 14103 src_reg = ®s[insn->src_reg]; 14104 if (src_reg->type != SCALAR_VALUE) { 14105 if (dst_reg->type != SCALAR_VALUE) { 14106 /* Combining two pointers by any ALU op yields 14107 * an arbitrary scalar. Disallow all math except 14108 * pointer subtraction 14109 */ 14110 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 14111 mark_reg_unknown(env, regs, insn->dst_reg); 14112 return 0; 14113 } 14114 verbose(env, "R%d pointer %s pointer prohibited\n", 14115 insn->dst_reg, 14116 bpf_alu_string[opcode >> 4]); 14117 return -EACCES; 14118 } else { 14119 /* scalar += pointer 14120 * This is legal, but we have to reverse our 14121 * src/dest handling in computing the range 14122 */ 14123 err = mark_chain_precision(env, insn->dst_reg); 14124 if (err) 14125 return err; 14126 return adjust_ptr_min_max_vals(env, insn, 14127 src_reg, dst_reg); 14128 } 14129 } else if (ptr_reg) { 14130 /* pointer += scalar */ 14131 err = mark_chain_precision(env, insn->src_reg); 14132 if (err) 14133 return err; 14134 return adjust_ptr_min_max_vals(env, insn, 14135 dst_reg, src_reg); 14136 } else if (dst_reg->precise) { 14137 /* if dst_reg is precise, src_reg should be precise as well */ 14138 err = mark_chain_precision(env, insn->src_reg); 14139 if (err) 14140 return err; 14141 } 14142 } else { 14143 /* Pretend the src is a reg with a known value, since we only 14144 * need to be able to read from this state. 14145 */ 14146 off_reg.type = SCALAR_VALUE; 14147 __mark_reg_known(&off_reg, insn->imm); 14148 src_reg = &off_reg; 14149 if (ptr_reg) /* pointer += K */ 14150 return adjust_ptr_min_max_vals(env, insn, 14151 ptr_reg, src_reg); 14152 } 14153 14154 /* Got here implies adding two SCALAR_VALUEs */ 14155 if (WARN_ON_ONCE(ptr_reg)) { 14156 print_verifier_state(env, state, true); 14157 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 14158 return -EINVAL; 14159 } 14160 if (WARN_ON(!src_reg)) { 14161 print_verifier_state(env, state, true); 14162 verbose(env, "verifier internal error: no src_reg\n"); 14163 return -EINVAL; 14164 } 14165 err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 14166 if (err) 14167 return err; 14168 /* 14169 * Compilers can generate the code 14170 * r1 = r2 14171 * r1 += 0x1 14172 * if r2 < 1000 goto ... 14173 * use r1 in memory access 14174 * So remember constant delta between r2 and r1 and update r1 after 14175 * 'if' condition. 14176 */ 14177 if (env->bpf_capable && BPF_OP(insn->code) == BPF_ADD && 14178 dst_reg->id && is_reg_const(src_reg, alu32)) { 14179 u64 val = reg_const_value(src_reg, alu32); 14180 14181 if ((dst_reg->id & BPF_ADD_CONST) || 14182 /* prevent overflow in find_equal_scalars() later */ 14183 val > (u32)S32_MAX) { 14184 /* 14185 * If the register already went through rX += val 14186 * we cannot accumulate another val into rx->off. 14187 */ 14188 dst_reg->off = 0; 14189 dst_reg->id = 0; 14190 } else { 14191 dst_reg->id |= BPF_ADD_CONST; 14192 dst_reg->off = val; 14193 } 14194 } else { 14195 /* 14196 * Make sure ID is cleared otherwise dst_reg min/max could be 14197 * incorrectly propagated into other registers by find_equal_scalars() 14198 */ 14199 dst_reg->id = 0; 14200 } 14201 return 0; 14202 } 14203 14204 /* check validity of 32-bit and 64-bit arithmetic operations */ 14205 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 14206 { 14207 struct bpf_reg_state *regs = cur_regs(env); 14208 u8 opcode = BPF_OP(insn->code); 14209 int err; 14210 14211 if (opcode == BPF_END || opcode == BPF_NEG) { 14212 if (opcode == BPF_NEG) { 14213 if (BPF_SRC(insn->code) != BPF_K || 14214 insn->src_reg != BPF_REG_0 || 14215 insn->off != 0 || insn->imm != 0) { 14216 verbose(env, "BPF_NEG uses reserved fields\n"); 14217 return -EINVAL; 14218 } 14219 } else { 14220 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 14221 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 14222 (BPF_CLASS(insn->code) == BPF_ALU64 && 14223 BPF_SRC(insn->code) != BPF_TO_LE)) { 14224 verbose(env, "BPF_END uses reserved fields\n"); 14225 return -EINVAL; 14226 } 14227 } 14228 14229 /* check src operand */ 14230 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 14231 if (err) 14232 return err; 14233 14234 if (is_pointer_value(env, insn->dst_reg)) { 14235 verbose(env, "R%d pointer arithmetic prohibited\n", 14236 insn->dst_reg); 14237 return -EACCES; 14238 } 14239 14240 /* check dest operand */ 14241 err = check_reg_arg(env, insn->dst_reg, DST_OP); 14242 if (err) 14243 return err; 14244 14245 } else if (opcode == BPF_MOV) { 14246 14247 if (BPF_SRC(insn->code) == BPF_X) { 14248 if (BPF_CLASS(insn->code) == BPF_ALU) { 14249 if ((insn->off != 0 && insn->off != 8 && insn->off != 16) || 14250 insn->imm) { 14251 verbose(env, "BPF_MOV uses reserved fields\n"); 14252 return -EINVAL; 14253 } 14254 } else if (insn->off == BPF_ADDR_SPACE_CAST) { 14255 if (insn->imm != 1 && insn->imm != 1u << 16) { 14256 verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); 14257 return -EINVAL; 14258 } 14259 if (!env->prog->aux->arena) { 14260 verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n"); 14261 return -EINVAL; 14262 } 14263 } else { 14264 if ((insn->off != 0 && insn->off != 8 && insn->off != 16 && 14265 insn->off != 32) || insn->imm) { 14266 verbose(env, "BPF_MOV uses reserved fields\n"); 14267 return -EINVAL; 14268 } 14269 } 14270 14271 /* check src operand */ 14272 err = check_reg_arg(env, insn->src_reg, SRC_OP); 14273 if (err) 14274 return err; 14275 } else { 14276 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 14277 verbose(env, "BPF_MOV uses reserved fields\n"); 14278 return -EINVAL; 14279 } 14280 } 14281 14282 /* check dest operand, mark as required later */ 14283 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 14284 if (err) 14285 return err; 14286 14287 if (BPF_SRC(insn->code) == BPF_X) { 14288 struct bpf_reg_state *src_reg = regs + insn->src_reg; 14289 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 14290 14291 if (BPF_CLASS(insn->code) == BPF_ALU64) { 14292 if (insn->imm) { 14293 /* off == BPF_ADDR_SPACE_CAST */ 14294 mark_reg_unknown(env, regs, insn->dst_reg); 14295 if (insn->imm == 1) { /* cast from as(1) to as(0) */ 14296 dst_reg->type = PTR_TO_ARENA; 14297 /* PTR_TO_ARENA is 32-bit */ 14298 dst_reg->subreg_def = env->insn_idx + 1; 14299 } 14300 } else if (insn->off == 0) { 14301 /* case: R1 = R2 14302 * copy register state to dest reg 14303 */ 14304 assign_scalar_id_before_mov(env, src_reg); 14305 copy_register_state(dst_reg, src_reg); 14306 dst_reg->live |= REG_LIVE_WRITTEN; 14307 dst_reg->subreg_def = DEF_NOT_SUBREG; 14308 } else { 14309 /* case: R1 = (s8, s16 s32)R2 */ 14310 if (is_pointer_value(env, insn->src_reg)) { 14311 verbose(env, 14312 "R%d sign-extension part of pointer\n", 14313 insn->src_reg); 14314 return -EACCES; 14315 } else if (src_reg->type == SCALAR_VALUE) { 14316 bool no_sext; 14317 14318 no_sext = src_reg->umax_value < (1ULL << (insn->off - 1)); 14319 if (no_sext) 14320 assign_scalar_id_before_mov(env, src_reg); 14321 copy_register_state(dst_reg, src_reg); 14322 if (!no_sext) 14323 dst_reg->id = 0; 14324 coerce_reg_to_size_sx(dst_reg, insn->off >> 3); 14325 dst_reg->live |= REG_LIVE_WRITTEN; 14326 dst_reg->subreg_def = DEF_NOT_SUBREG; 14327 } else { 14328 mark_reg_unknown(env, regs, insn->dst_reg); 14329 } 14330 } 14331 } else { 14332 /* R1 = (u32) R2 */ 14333 if (is_pointer_value(env, insn->src_reg)) { 14334 verbose(env, 14335 "R%d partial copy of pointer\n", 14336 insn->src_reg); 14337 return -EACCES; 14338 } else if (src_reg->type == SCALAR_VALUE) { 14339 if (insn->off == 0) { 14340 bool is_src_reg_u32 = get_reg_width(src_reg) <= 32; 14341 14342 if (is_src_reg_u32) 14343 assign_scalar_id_before_mov(env, src_reg); 14344 copy_register_state(dst_reg, src_reg); 14345 /* Make sure ID is cleared if src_reg is not in u32 14346 * range otherwise dst_reg min/max could be incorrectly 14347 * propagated into src_reg by find_equal_scalars() 14348 */ 14349 if (!is_src_reg_u32) 14350 dst_reg->id = 0; 14351 dst_reg->live |= REG_LIVE_WRITTEN; 14352 dst_reg->subreg_def = env->insn_idx + 1; 14353 } else { 14354 /* case: W1 = (s8, s16)W2 */ 14355 bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1)); 14356 14357 if (no_sext) 14358 assign_scalar_id_before_mov(env, src_reg); 14359 copy_register_state(dst_reg, src_reg); 14360 if (!no_sext) 14361 dst_reg->id = 0; 14362 dst_reg->live |= REG_LIVE_WRITTEN; 14363 dst_reg->subreg_def = env->insn_idx + 1; 14364 coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); 14365 } 14366 } else { 14367 mark_reg_unknown(env, regs, 14368 insn->dst_reg); 14369 } 14370 zext_32_to_64(dst_reg); 14371 reg_bounds_sync(dst_reg); 14372 } 14373 } else { 14374 /* case: R = imm 14375 * remember the value we stored into this reg 14376 */ 14377 /* clear any state __mark_reg_known doesn't set */ 14378 mark_reg_unknown(env, regs, insn->dst_reg); 14379 regs[insn->dst_reg].type = SCALAR_VALUE; 14380 if (BPF_CLASS(insn->code) == BPF_ALU64) { 14381 __mark_reg_known(regs + insn->dst_reg, 14382 insn->imm); 14383 } else { 14384 __mark_reg_known(regs + insn->dst_reg, 14385 (u32)insn->imm); 14386 } 14387 } 14388 14389 } else if (opcode > BPF_END) { 14390 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 14391 return -EINVAL; 14392 14393 } else { /* all other ALU ops: and, sub, xor, add, ... */ 14394 14395 if (BPF_SRC(insn->code) == BPF_X) { 14396 if (insn->imm != 0 || insn->off > 1 || 14397 (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 14398 verbose(env, "BPF_ALU uses reserved fields\n"); 14399 return -EINVAL; 14400 } 14401 /* check src1 operand */ 14402 err = check_reg_arg(env, insn->src_reg, SRC_OP); 14403 if (err) 14404 return err; 14405 } else { 14406 if (insn->src_reg != BPF_REG_0 || insn->off > 1 || 14407 (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) { 14408 verbose(env, "BPF_ALU uses reserved fields\n"); 14409 return -EINVAL; 14410 } 14411 } 14412 14413 /* check src2 operand */ 14414 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 14415 if (err) 14416 return err; 14417 14418 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 14419 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 14420 verbose(env, "div by zero\n"); 14421 return -EINVAL; 14422 } 14423 14424 if ((opcode == BPF_LSH || opcode == BPF_RSH || 14425 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 14426 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 14427 14428 if (insn->imm < 0 || insn->imm >= size) { 14429 verbose(env, "invalid shift %d\n", insn->imm); 14430 return -EINVAL; 14431 } 14432 } 14433 14434 /* check dest operand */ 14435 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 14436 err = err ?: adjust_reg_min_max_vals(env, insn); 14437 if (err) 14438 return err; 14439 } 14440 14441 return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); 14442 } 14443 14444 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 14445 struct bpf_reg_state *dst_reg, 14446 enum bpf_reg_type type, 14447 bool range_right_open) 14448 { 14449 struct bpf_func_state *state; 14450 struct bpf_reg_state *reg; 14451 int new_range; 14452 14453 if (dst_reg->off < 0 || 14454 (dst_reg->off == 0 && range_right_open)) 14455 /* This doesn't give us any range */ 14456 return; 14457 14458 if (dst_reg->umax_value > MAX_PACKET_OFF || 14459 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 14460 /* Risk of overflow. For instance, ptr + (1<<63) may be less 14461 * than pkt_end, but that's because it's also less than pkt. 14462 */ 14463 return; 14464 14465 new_range = dst_reg->off; 14466 if (range_right_open) 14467 new_range++; 14468 14469 /* Examples for register markings: 14470 * 14471 * pkt_data in dst register: 14472 * 14473 * r2 = r3; 14474 * r2 += 8; 14475 * if (r2 > pkt_end) goto <handle exception> 14476 * <access okay> 14477 * 14478 * r2 = r3; 14479 * r2 += 8; 14480 * if (r2 < pkt_end) goto <access okay> 14481 * <handle exception> 14482 * 14483 * Where: 14484 * r2 == dst_reg, pkt_end == src_reg 14485 * r2=pkt(id=n,off=8,r=0) 14486 * r3=pkt(id=n,off=0,r=0) 14487 * 14488 * pkt_data in src register: 14489 * 14490 * r2 = r3; 14491 * r2 += 8; 14492 * if (pkt_end >= r2) goto <access okay> 14493 * <handle exception> 14494 * 14495 * r2 = r3; 14496 * r2 += 8; 14497 * if (pkt_end <= r2) goto <handle exception> 14498 * <access okay> 14499 * 14500 * Where: 14501 * pkt_end == dst_reg, r2 == src_reg 14502 * r2=pkt(id=n,off=8,r=0) 14503 * r3=pkt(id=n,off=0,r=0) 14504 * 14505 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 14506 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 14507 * and [r3, r3 + 8-1) respectively is safe to access depending on 14508 * the check. 14509 */ 14510 14511 /* If our ids match, then we must have the same max_value. And we 14512 * don't care about the other reg's fixed offset, since if it's too big 14513 * the range won't allow anything. 14514 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 14515 */ 14516 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 14517 if (reg->type == type && reg->id == dst_reg->id) 14518 /* keep the maximum range already checked */ 14519 reg->range = max(reg->range, new_range); 14520 })); 14521 } 14522 14523 /* 14524 * <reg1> <op> <reg2>, currently assuming reg2 is a constant 14525 */ 14526 static int is_scalar_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, 14527 u8 opcode, bool is_jmp32) 14528 { 14529 struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off; 14530 struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off; 14531 u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value; 14532 u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value; 14533 s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value; 14534 s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value; 14535 u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value; 14536 u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value; 14537 s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value; 14538 s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value; 14539 14540 switch (opcode) { 14541 case BPF_JEQ: 14542 /* constants, umin/umax and smin/smax checks would be 14543 * redundant in this case because they all should match 14544 */ 14545 if (tnum_is_const(t1) && tnum_is_const(t2)) 14546 return t1.value == t2.value; 14547 /* non-overlapping ranges */ 14548 if (umin1 > umax2 || umax1 < umin2) 14549 return 0; 14550 if (smin1 > smax2 || smax1 < smin2) 14551 return 0; 14552 if (!is_jmp32) { 14553 /* if 64-bit ranges are inconclusive, see if we can 14554 * utilize 32-bit subrange knowledge to eliminate 14555 * branches that can't be taken a priori 14556 */ 14557 if (reg1->u32_min_value > reg2->u32_max_value || 14558 reg1->u32_max_value < reg2->u32_min_value) 14559 return 0; 14560 if (reg1->s32_min_value > reg2->s32_max_value || 14561 reg1->s32_max_value < reg2->s32_min_value) 14562 return 0; 14563 } 14564 break; 14565 case BPF_JNE: 14566 /* constants, umin/umax and smin/smax checks would be 14567 * redundant in this case because they all should match 14568 */ 14569 if (tnum_is_const(t1) && tnum_is_const(t2)) 14570 return t1.value != t2.value; 14571 /* non-overlapping ranges */ 14572 if (umin1 > umax2 || umax1 < umin2) 14573 return 1; 14574 if (smin1 > smax2 || smax1 < smin2) 14575 return 1; 14576 if (!is_jmp32) { 14577 /* if 64-bit ranges are inconclusive, see if we can 14578 * utilize 32-bit subrange knowledge to eliminate 14579 * branches that can't be taken a priori 14580 */ 14581 if (reg1->u32_min_value > reg2->u32_max_value || 14582 reg1->u32_max_value < reg2->u32_min_value) 14583 return 1; 14584 if (reg1->s32_min_value > reg2->s32_max_value || 14585 reg1->s32_max_value < reg2->s32_min_value) 14586 return 1; 14587 } 14588 break; 14589 case BPF_JSET: 14590 if (!is_reg_const(reg2, is_jmp32)) { 14591 swap(reg1, reg2); 14592 swap(t1, t2); 14593 } 14594 if (!is_reg_const(reg2, is_jmp32)) 14595 return -1; 14596 if ((~t1.mask & t1.value) & t2.value) 14597 return 1; 14598 if (!((t1.mask | t1.value) & t2.value)) 14599 return 0; 14600 break; 14601 case BPF_JGT: 14602 if (umin1 > umax2) 14603 return 1; 14604 else if (umax1 <= umin2) 14605 return 0; 14606 break; 14607 case BPF_JSGT: 14608 if (smin1 > smax2) 14609 return 1; 14610 else if (smax1 <= smin2) 14611 return 0; 14612 break; 14613 case BPF_JLT: 14614 if (umax1 < umin2) 14615 return 1; 14616 else if (umin1 >= umax2) 14617 return 0; 14618 break; 14619 case BPF_JSLT: 14620 if (smax1 < smin2) 14621 return 1; 14622 else if (smin1 >= smax2) 14623 return 0; 14624 break; 14625 case BPF_JGE: 14626 if (umin1 >= umax2) 14627 return 1; 14628 else if (umax1 < umin2) 14629 return 0; 14630 break; 14631 case BPF_JSGE: 14632 if (smin1 >= smax2) 14633 return 1; 14634 else if (smax1 < smin2) 14635 return 0; 14636 break; 14637 case BPF_JLE: 14638 if (umax1 <= umin2) 14639 return 1; 14640 else if (umin1 > umax2) 14641 return 0; 14642 break; 14643 case BPF_JSLE: 14644 if (smax1 <= smin2) 14645 return 1; 14646 else if (smin1 > smax2) 14647 return 0; 14648 break; 14649 } 14650 14651 return -1; 14652 } 14653 14654 static int flip_opcode(u32 opcode) 14655 { 14656 /* How can we transform "a <op> b" into "b <op> a"? */ 14657 static const u8 opcode_flip[16] = { 14658 /* these stay the same */ 14659 [BPF_JEQ >> 4] = BPF_JEQ, 14660 [BPF_JNE >> 4] = BPF_JNE, 14661 [BPF_JSET >> 4] = BPF_JSET, 14662 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 14663 [BPF_JGE >> 4] = BPF_JLE, 14664 [BPF_JGT >> 4] = BPF_JLT, 14665 [BPF_JLE >> 4] = BPF_JGE, 14666 [BPF_JLT >> 4] = BPF_JGT, 14667 [BPF_JSGE >> 4] = BPF_JSLE, 14668 [BPF_JSGT >> 4] = BPF_JSLT, 14669 [BPF_JSLE >> 4] = BPF_JSGE, 14670 [BPF_JSLT >> 4] = BPF_JSGT 14671 }; 14672 return opcode_flip[opcode >> 4]; 14673 } 14674 14675 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 14676 struct bpf_reg_state *src_reg, 14677 u8 opcode) 14678 { 14679 struct bpf_reg_state *pkt; 14680 14681 if (src_reg->type == PTR_TO_PACKET_END) { 14682 pkt = dst_reg; 14683 } else if (dst_reg->type == PTR_TO_PACKET_END) { 14684 pkt = src_reg; 14685 opcode = flip_opcode(opcode); 14686 } else { 14687 return -1; 14688 } 14689 14690 if (pkt->range >= 0) 14691 return -1; 14692 14693 switch (opcode) { 14694 case BPF_JLE: 14695 /* pkt <= pkt_end */ 14696 fallthrough; 14697 case BPF_JGT: 14698 /* pkt > pkt_end */ 14699 if (pkt->range == BEYOND_PKT_END) 14700 /* pkt has at last one extra byte beyond pkt_end */ 14701 return opcode == BPF_JGT; 14702 break; 14703 case BPF_JLT: 14704 /* pkt < pkt_end */ 14705 fallthrough; 14706 case BPF_JGE: 14707 /* pkt >= pkt_end */ 14708 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 14709 return opcode == BPF_JGE; 14710 break; 14711 } 14712 return -1; 14713 } 14714 14715 /* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;" 14716 * and return: 14717 * 1 - branch will be taken and "goto target" will be executed 14718 * 0 - branch will not be taken and fall-through to next insn 14719 * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value 14720 * range [0,10] 14721 */ 14722 static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, 14723 u8 opcode, bool is_jmp32) 14724 { 14725 if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32) 14726 return is_pkt_ptr_branch_taken(reg1, reg2, opcode); 14727 14728 if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { 14729 u64 val; 14730 14731 /* arrange that reg2 is a scalar, and reg1 is a pointer */ 14732 if (!is_reg_const(reg2, is_jmp32)) { 14733 opcode = flip_opcode(opcode); 14734 swap(reg1, reg2); 14735 } 14736 /* and ensure that reg2 is a constant */ 14737 if (!is_reg_const(reg2, is_jmp32)) 14738 return -1; 14739 14740 if (!reg_not_null(reg1)) 14741 return -1; 14742 14743 /* If pointer is valid tests against zero will fail so we can 14744 * use this to direct branch taken. 14745 */ 14746 val = reg_const_value(reg2, is_jmp32); 14747 if (val != 0) 14748 return -1; 14749 14750 switch (opcode) { 14751 case BPF_JEQ: 14752 return 0; 14753 case BPF_JNE: 14754 return 1; 14755 default: 14756 return -1; 14757 } 14758 } 14759 14760 /* now deal with two scalars, but not necessarily constants */ 14761 return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32); 14762 } 14763 14764 /* Opcode that corresponds to a *false* branch condition. 14765 * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 14766 */ 14767 static u8 rev_opcode(u8 opcode) 14768 { 14769 switch (opcode) { 14770 case BPF_JEQ: return BPF_JNE; 14771 case BPF_JNE: return BPF_JEQ; 14772 /* JSET doesn't have it's reverse opcode in BPF, so add 14773 * BPF_X flag to denote the reverse of that operation 14774 */ 14775 case BPF_JSET: return BPF_JSET | BPF_X; 14776 case BPF_JSET | BPF_X: return BPF_JSET; 14777 case BPF_JGE: return BPF_JLT; 14778 case BPF_JGT: return BPF_JLE; 14779 case BPF_JLE: return BPF_JGT; 14780 case BPF_JLT: return BPF_JGE; 14781 case BPF_JSGE: return BPF_JSLT; 14782 case BPF_JSGT: return BPF_JSLE; 14783 case BPF_JSLE: return BPF_JSGT; 14784 case BPF_JSLT: return BPF_JSGE; 14785 default: return 0; 14786 } 14787 } 14788 14789 /* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */ 14790 static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, 14791 u8 opcode, bool is_jmp32) 14792 { 14793 struct tnum t; 14794 u64 val; 14795 14796 /* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */ 14797 switch (opcode) { 14798 case BPF_JGE: 14799 case BPF_JGT: 14800 case BPF_JSGE: 14801 case BPF_JSGT: 14802 opcode = flip_opcode(opcode); 14803 swap(reg1, reg2); 14804 break; 14805 default: 14806 break; 14807 } 14808 14809 switch (opcode) { 14810 case BPF_JEQ: 14811 if (is_jmp32) { 14812 reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); 14813 reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); 14814 reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); 14815 reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); 14816 reg2->u32_min_value = reg1->u32_min_value; 14817 reg2->u32_max_value = reg1->u32_max_value; 14818 reg2->s32_min_value = reg1->s32_min_value; 14819 reg2->s32_max_value = reg1->s32_max_value; 14820 14821 t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off)); 14822 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 14823 reg2->var_off = tnum_with_subreg(reg2->var_off, t); 14824 } else { 14825 reg1->umin_value = max(reg1->umin_value, reg2->umin_value); 14826 reg1->umax_value = min(reg1->umax_value, reg2->umax_value); 14827 reg1->smin_value = max(reg1->smin_value, reg2->smin_value); 14828 reg1->smax_value = min(reg1->smax_value, reg2->smax_value); 14829 reg2->umin_value = reg1->umin_value; 14830 reg2->umax_value = reg1->umax_value; 14831 reg2->smin_value = reg1->smin_value; 14832 reg2->smax_value = reg1->smax_value; 14833 14834 reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off); 14835 reg2->var_off = reg1->var_off; 14836 } 14837 break; 14838 case BPF_JNE: 14839 if (!is_reg_const(reg2, is_jmp32)) 14840 swap(reg1, reg2); 14841 if (!is_reg_const(reg2, is_jmp32)) 14842 break; 14843 14844 /* try to recompute the bound of reg1 if reg2 is a const and 14845 * is exactly the edge of reg1. 14846 */ 14847 val = reg_const_value(reg2, is_jmp32); 14848 if (is_jmp32) { 14849 /* u32_min_value is not equal to 0xffffffff at this point, 14850 * because otherwise u32_max_value is 0xffffffff as well, 14851 * in such a case both reg1 and reg2 would be constants, 14852 * jump would be predicted and reg_set_min_max() won't 14853 * be called. 14854 * 14855 * Same reasoning works for all {u,s}{min,max}{32,64} cases 14856 * below. 14857 */ 14858 if (reg1->u32_min_value == (u32)val) 14859 reg1->u32_min_value++; 14860 if (reg1->u32_max_value == (u32)val) 14861 reg1->u32_max_value--; 14862 if (reg1->s32_min_value == (s32)val) 14863 reg1->s32_min_value++; 14864 if (reg1->s32_max_value == (s32)val) 14865 reg1->s32_max_value--; 14866 } else { 14867 if (reg1->umin_value == (u64)val) 14868 reg1->umin_value++; 14869 if (reg1->umax_value == (u64)val) 14870 reg1->umax_value--; 14871 if (reg1->smin_value == (s64)val) 14872 reg1->smin_value++; 14873 if (reg1->smax_value == (s64)val) 14874 reg1->smax_value--; 14875 } 14876 break; 14877 case BPF_JSET: 14878 if (!is_reg_const(reg2, is_jmp32)) 14879 swap(reg1, reg2); 14880 if (!is_reg_const(reg2, is_jmp32)) 14881 break; 14882 val = reg_const_value(reg2, is_jmp32); 14883 /* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X) 14884 * requires single bit to learn something useful. E.g., if we 14885 * know that `r1 & 0x3` is true, then which bits (0, 1, or both) 14886 * are actually set? We can learn something definite only if 14887 * it's a single-bit value to begin with. 14888 * 14889 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have 14890 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor 14891 * bit 1 is set, which we can readily use in adjustments. 14892 */ 14893 if (!is_power_of_2(val)) 14894 break; 14895 if (is_jmp32) { 14896 t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); 14897 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 14898 } else { 14899 reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); 14900 } 14901 break; 14902 case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ 14903 if (!is_reg_const(reg2, is_jmp32)) 14904 swap(reg1, reg2); 14905 if (!is_reg_const(reg2, is_jmp32)) 14906 break; 14907 val = reg_const_value(reg2, is_jmp32); 14908 if (is_jmp32) { 14909 t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); 14910 reg1->var_off = tnum_with_subreg(reg1->var_off, t); 14911 } else { 14912 reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val)); 14913 } 14914 break; 14915 case BPF_JLE: 14916 if (is_jmp32) { 14917 reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value); 14918 reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value); 14919 } else { 14920 reg1->umax_value = min(reg1->umax_value, reg2->umax_value); 14921 reg2->umin_value = max(reg1->umin_value, reg2->umin_value); 14922 } 14923 break; 14924 case BPF_JLT: 14925 if (is_jmp32) { 14926 reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1); 14927 reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value); 14928 } else { 14929 reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1); 14930 reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value); 14931 } 14932 break; 14933 case BPF_JSLE: 14934 if (is_jmp32) { 14935 reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value); 14936 reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value); 14937 } else { 14938 reg1->smax_value = min(reg1->smax_value, reg2->smax_value); 14939 reg2->smin_value = max(reg1->smin_value, reg2->smin_value); 14940 } 14941 break; 14942 case BPF_JSLT: 14943 if (is_jmp32) { 14944 reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1); 14945 reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value); 14946 } else { 14947 reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1); 14948 reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value); 14949 } 14950 break; 14951 default: 14952 return; 14953 } 14954 } 14955 14956 /* Adjusts the register min/max values in the case that the dst_reg and 14957 * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K 14958 * check, in which case we have a fake SCALAR_VALUE representing insn->imm). 14959 * Technically we can do similar adjustments for pointers to the same object, 14960 * but we don't support that right now. 14961 */ 14962 static int reg_set_min_max(struct bpf_verifier_env *env, 14963 struct bpf_reg_state *true_reg1, 14964 struct bpf_reg_state *true_reg2, 14965 struct bpf_reg_state *false_reg1, 14966 struct bpf_reg_state *false_reg2, 14967 u8 opcode, bool is_jmp32) 14968 { 14969 int err; 14970 14971 /* If either register is a pointer, we can't learn anything about its 14972 * variable offset from the compare (unless they were a pointer into 14973 * the same object, but we don't bother with that). 14974 */ 14975 if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE) 14976 return 0; 14977 14978 /* fallthrough (FALSE) branch */ 14979 regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32); 14980 reg_bounds_sync(false_reg1); 14981 reg_bounds_sync(false_reg2); 14982 14983 /* jump (TRUE) branch */ 14984 regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32); 14985 reg_bounds_sync(true_reg1); 14986 reg_bounds_sync(true_reg2); 14987 14988 err = reg_bounds_sanity_check(env, true_reg1, "true_reg1"); 14989 err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2"); 14990 err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1"); 14991 err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2"); 14992 return err; 14993 } 14994 14995 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 14996 struct bpf_reg_state *reg, u32 id, 14997 bool is_null) 14998 { 14999 if (type_may_be_null(reg->type) && reg->id == id && 15000 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 15001 /* Old offset (both fixed and variable parts) should have been 15002 * known-zero, because we don't allow pointer arithmetic on 15003 * pointers that might be NULL. If we see this happening, don't 15004 * convert the register. 15005 * 15006 * But in some cases, some helpers that return local kptrs 15007 * advance offset for the returned pointer. In those cases, it 15008 * is fine to expect to see reg->off. 15009 */ 15010 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) 15011 return; 15012 if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && 15013 WARN_ON_ONCE(reg->off)) 15014 return; 15015 15016 if (is_null) { 15017 reg->type = SCALAR_VALUE; 15018 /* We don't need id and ref_obj_id from this point 15019 * onwards anymore, thus we should better reset it, 15020 * so that state pruning has chances to take effect. 15021 */ 15022 reg->id = 0; 15023 reg->ref_obj_id = 0; 15024 15025 return; 15026 } 15027 15028 mark_ptr_not_null_reg(reg); 15029 15030 if (!reg_may_point_to_spin_lock(reg)) { 15031 /* For not-NULL ptr, reg->ref_obj_id will be reset 15032 * in release_reference(). 15033 * 15034 * reg->id is still used by spin_lock ptr. Other 15035 * than spin_lock ptr type, reg->id can be reset. 15036 */ 15037 reg->id = 0; 15038 } 15039 } 15040 } 15041 15042 /* The logic is similar to find_good_pkt_pointers(), both could eventually 15043 * be folded together at some point. 15044 */ 15045 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 15046 bool is_null) 15047 { 15048 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 15049 struct bpf_reg_state *regs = state->regs, *reg; 15050 u32 ref_obj_id = regs[regno].ref_obj_id; 15051 u32 id = regs[regno].id; 15052 15053 if (ref_obj_id && ref_obj_id == id && is_null) 15054 /* regs[regno] is in the " == NULL" branch. 15055 * No one could have freed the reference state before 15056 * doing the NULL check. 15057 */ 15058 WARN_ON_ONCE(release_reference_state(state, id)); 15059 15060 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 15061 mark_ptr_or_null_reg(state, reg, id, is_null); 15062 })); 15063 } 15064 15065 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 15066 struct bpf_reg_state *dst_reg, 15067 struct bpf_reg_state *src_reg, 15068 struct bpf_verifier_state *this_branch, 15069 struct bpf_verifier_state *other_branch) 15070 { 15071 if (BPF_SRC(insn->code) != BPF_X) 15072 return false; 15073 15074 /* Pointers are always 64-bit. */ 15075 if (BPF_CLASS(insn->code) == BPF_JMP32) 15076 return false; 15077 15078 switch (BPF_OP(insn->code)) { 15079 case BPF_JGT: 15080 if ((dst_reg->type == PTR_TO_PACKET && 15081 src_reg->type == PTR_TO_PACKET_END) || 15082 (dst_reg->type == PTR_TO_PACKET_META && 15083 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 15084 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 15085 find_good_pkt_pointers(this_branch, dst_reg, 15086 dst_reg->type, false); 15087 mark_pkt_end(other_branch, insn->dst_reg, true); 15088 } else if ((dst_reg->type == PTR_TO_PACKET_END && 15089 src_reg->type == PTR_TO_PACKET) || 15090 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 15091 src_reg->type == PTR_TO_PACKET_META)) { 15092 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 15093 find_good_pkt_pointers(other_branch, src_reg, 15094 src_reg->type, true); 15095 mark_pkt_end(this_branch, insn->src_reg, false); 15096 } else { 15097 return false; 15098 } 15099 break; 15100 case BPF_JLT: 15101 if ((dst_reg->type == PTR_TO_PACKET && 15102 src_reg->type == PTR_TO_PACKET_END) || 15103 (dst_reg->type == PTR_TO_PACKET_META && 15104 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 15105 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 15106 find_good_pkt_pointers(other_branch, dst_reg, 15107 dst_reg->type, true); 15108 mark_pkt_end(this_branch, insn->dst_reg, false); 15109 } else if ((dst_reg->type == PTR_TO_PACKET_END && 15110 src_reg->type == PTR_TO_PACKET) || 15111 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 15112 src_reg->type == PTR_TO_PACKET_META)) { 15113 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 15114 find_good_pkt_pointers(this_branch, src_reg, 15115 src_reg->type, false); 15116 mark_pkt_end(other_branch, insn->src_reg, true); 15117 } else { 15118 return false; 15119 } 15120 break; 15121 case BPF_JGE: 15122 if ((dst_reg->type == PTR_TO_PACKET && 15123 src_reg->type == PTR_TO_PACKET_END) || 15124 (dst_reg->type == PTR_TO_PACKET_META && 15125 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 15126 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 15127 find_good_pkt_pointers(this_branch, dst_reg, 15128 dst_reg->type, true); 15129 mark_pkt_end(other_branch, insn->dst_reg, false); 15130 } else if ((dst_reg->type == PTR_TO_PACKET_END && 15131 src_reg->type == PTR_TO_PACKET) || 15132 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 15133 src_reg->type == PTR_TO_PACKET_META)) { 15134 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 15135 find_good_pkt_pointers(other_branch, src_reg, 15136 src_reg->type, false); 15137 mark_pkt_end(this_branch, insn->src_reg, true); 15138 } else { 15139 return false; 15140 } 15141 break; 15142 case BPF_JLE: 15143 if ((dst_reg->type == PTR_TO_PACKET && 15144 src_reg->type == PTR_TO_PACKET_END) || 15145 (dst_reg->type == PTR_TO_PACKET_META && 15146 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 15147 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 15148 find_good_pkt_pointers(other_branch, dst_reg, 15149 dst_reg->type, false); 15150 mark_pkt_end(this_branch, insn->dst_reg, true); 15151 } else if ((dst_reg->type == PTR_TO_PACKET_END && 15152 src_reg->type == PTR_TO_PACKET) || 15153 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 15154 src_reg->type == PTR_TO_PACKET_META)) { 15155 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 15156 find_good_pkt_pointers(this_branch, src_reg, 15157 src_reg->type, true); 15158 mark_pkt_end(other_branch, insn->src_reg, false); 15159 } else { 15160 return false; 15161 } 15162 break; 15163 default: 15164 return false; 15165 } 15166 15167 return true; 15168 } 15169 15170 static void find_equal_scalars(struct bpf_verifier_state *vstate, 15171 struct bpf_reg_state *known_reg) 15172 { 15173 struct bpf_reg_state fake_reg; 15174 struct bpf_func_state *state; 15175 struct bpf_reg_state *reg; 15176 15177 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 15178 if (reg->type != SCALAR_VALUE || reg == known_reg) 15179 continue; 15180 if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) 15181 continue; 15182 if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || 15183 reg->off == known_reg->off) { 15184 copy_register_state(reg, known_reg); 15185 } else { 15186 s32 saved_off = reg->off; 15187 15188 fake_reg.type = SCALAR_VALUE; 15189 __mark_reg_known(&fake_reg, (s32)reg->off - (s32)known_reg->off); 15190 15191 /* reg = known_reg; reg += delta */ 15192 copy_register_state(reg, known_reg); 15193 /* 15194 * Must preserve off, id and add_const flag, 15195 * otherwise another find_equal_scalars() will be incorrect. 15196 */ 15197 reg->off = saved_off; 15198 15199 scalar32_min_max_add(reg, &fake_reg); 15200 scalar_min_max_add(reg, &fake_reg); 15201 reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); 15202 } 15203 })); 15204 } 15205 15206 static int check_cond_jmp_op(struct bpf_verifier_env *env, 15207 struct bpf_insn *insn, int *insn_idx) 15208 { 15209 struct bpf_verifier_state *this_branch = env->cur_state; 15210 struct bpf_verifier_state *other_branch; 15211 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 15212 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 15213 struct bpf_reg_state *eq_branch_regs; 15214 u8 opcode = BPF_OP(insn->code); 15215 bool is_jmp32; 15216 int pred = -1; 15217 int err; 15218 15219 /* Only conditional jumps are expected to reach here. */ 15220 if (opcode == BPF_JA || opcode > BPF_JCOND) { 15221 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 15222 return -EINVAL; 15223 } 15224 15225 if (opcode == BPF_JCOND) { 15226 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; 15227 int idx = *insn_idx; 15228 15229 if (insn->code != (BPF_JMP | BPF_JCOND) || 15230 insn->src_reg != BPF_MAY_GOTO || 15231 insn->dst_reg || insn->imm || insn->off == 0) { 15232 verbose(env, "invalid may_goto off %d imm %d\n", 15233 insn->off, insn->imm); 15234 return -EINVAL; 15235 } 15236 prev_st = find_prev_entry(env, cur_st->parent, idx); 15237 15238 /* branch out 'fallthrough' insn as a new state to explore */ 15239 queued_st = push_stack(env, idx + 1, idx, false); 15240 if (!queued_st) 15241 return -ENOMEM; 15242 15243 queued_st->may_goto_depth++; 15244 if (prev_st) 15245 widen_imprecise_scalars(env, prev_st, queued_st); 15246 *insn_idx += insn->off; 15247 return 0; 15248 } 15249 15250 /* check src2 operand */ 15251 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 15252 if (err) 15253 return err; 15254 15255 dst_reg = ®s[insn->dst_reg]; 15256 if (BPF_SRC(insn->code) == BPF_X) { 15257 if (insn->imm != 0) { 15258 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 15259 return -EINVAL; 15260 } 15261 15262 /* check src1 operand */ 15263 err = check_reg_arg(env, insn->src_reg, SRC_OP); 15264 if (err) 15265 return err; 15266 15267 src_reg = ®s[insn->src_reg]; 15268 if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) && 15269 is_pointer_value(env, insn->src_reg)) { 15270 verbose(env, "R%d pointer comparison prohibited\n", 15271 insn->src_reg); 15272 return -EACCES; 15273 } 15274 } else { 15275 if (insn->src_reg != BPF_REG_0) { 15276 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 15277 return -EINVAL; 15278 } 15279 src_reg = &env->fake_reg[0]; 15280 memset(src_reg, 0, sizeof(*src_reg)); 15281 src_reg->type = SCALAR_VALUE; 15282 __mark_reg_known(src_reg, insn->imm); 15283 } 15284 15285 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 15286 pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32); 15287 if (pred >= 0) { 15288 /* If we get here with a dst_reg pointer type it is because 15289 * above is_branch_taken() special cased the 0 comparison. 15290 */ 15291 if (!__is_pointer_value(false, dst_reg)) 15292 err = mark_chain_precision(env, insn->dst_reg); 15293 if (BPF_SRC(insn->code) == BPF_X && !err && 15294 !__is_pointer_value(false, src_reg)) 15295 err = mark_chain_precision(env, insn->src_reg); 15296 if (err) 15297 return err; 15298 } 15299 15300 if (pred == 1) { 15301 /* Only follow the goto, ignore fall-through. If needed, push 15302 * the fall-through branch for simulation under speculative 15303 * execution. 15304 */ 15305 if (!env->bypass_spec_v1 && 15306 !sanitize_speculative_path(env, insn, *insn_idx + 1, 15307 *insn_idx)) 15308 return -EFAULT; 15309 if (env->log.level & BPF_LOG_LEVEL) 15310 print_insn_state(env, this_branch->frame[this_branch->curframe]); 15311 *insn_idx += insn->off; 15312 return 0; 15313 } else if (pred == 0) { 15314 /* Only follow the fall-through branch, since that's where the 15315 * program will go. If needed, push the goto branch for 15316 * simulation under speculative execution. 15317 */ 15318 if (!env->bypass_spec_v1 && 15319 !sanitize_speculative_path(env, insn, 15320 *insn_idx + insn->off + 1, 15321 *insn_idx)) 15322 return -EFAULT; 15323 if (env->log.level & BPF_LOG_LEVEL) 15324 print_insn_state(env, this_branch->frame[this_branch->curframe]); 15325 return 0; 15326 } 15327 15328 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 15329 false); 15330 if (!other_branch) 15331 return -EFAULT; 15332 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 15333 15334 if (BPF_SRC(insn->code) == BPF_X) { 15335 err = reg_set_min_max(env, 15336 &other_branch_regs[insn->dst_reg], 15337 &other_branch_regs[insn->src_reg], 15338 dst_reg, src_reg, opcode, is_jmp32); 15339 } else /* BPF_SRC(insn->code) == BPF_K */ { 15340 /* reg_set_min_max() can mangle the fake_reg. Make a copy 15341 * so that these are two different memory locations. The 15342 * src_reg is not used beyond here in context of K. 15343 */ 15344 memcpy(&env->fake_reg[1], &env->fake_reg[0], 15345 sizeof(env->fake_reg[0])); 15346 err = reg_set_min_max(env, 15347 &other_branch_regs[insn->dst_reg], 15348 &env->fake_reg[0], 15349 dst_reg, &env->fake_reg[1], 15350 opcode, is_jmp32); 15351 } 15352 if (err) 15353 return err; 15354 15355 if (BPF_SRC(insn->code) == BPF_X && 15356 src_reg->type == SCALAR_VALUE && src_reg->id && 15357 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 15358 find_equal_scalars(this_branch, src_reg); 15359 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 15360 } 15361 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 15362 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 15363 find_equal_scalars(this_branch, dst_reg); 15364 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 15365 } 15366 15367 /* if one pointer register is compared to another pointer 15368 * register check if PTR_MAYBE_NULL could be lifted. 15369 * E.g. register A - maybe null 15370 * register B - not null 15371 * for JNE A, B, ... - A is not null in the false branch; 15372 * for JEQ A, B, ... - A is not null in the true branch. 15373 * 15374 * Since PTR_TO_BTF_ID points to a kernel struct that does 15375 * not need to be null checked by the BPF program, i.e., 15376 * could be null even without PTR_MAYBE_NULL marking, so 15377 * only propagate nullness when neither reg is that type. 15378 */ 15379 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 15380 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 15381 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && 15382 base_type(src_reg->type) != PTR_TO_BTF_ID && 15383 base_type(dst_reg->type) != PTR_TO_BTF_ID) { 15384 eq_branch_regs = NULL; 15385 switch (opcode) { 15386 case BPF_JEQ: 15387 eq_branch_regs = other_branch_regs; 15388 break; 15389 case BPF_JNE: 15390 eq_branch_regs = regs; 15391 break; 15392 default: 15393 /* do nothing */ 15394 break; 15395 } 15396 if (eq_branch_regs) { 15397 if (type_may_be_null(src_reg->type)) 15398 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 15399 else 15400 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 15401 } 15402 } 15403 15404 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 15405 * NOTE: these optimizations below are related with pointer comparison 15406 * which will never be JMP32. 15407 */ 15408 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 15409 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 15410 type_may_be_null(dst_reg->type)) { 15411 /* Mark all identical registers in each branch as either 15412 * safe or unknown depending R == 0 or R != 0 conditional. 15413 */ 15414 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 15415 opcode == BPF_JNE); 15416 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 15417 opcode == BPF_JEQ); 15418 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 15419 this_branch, other_branch) && 15420 is_pointer_value(env, insn->dst_reg)) { 15421 verbose(env, "R%d pointer comparison prohibited\n", 15422 insn->dst_reg); 15423 return -EACCES; 15424 } 15425 if (env->log.level & BPF_LOG_LEVEL) 15426 print_insn_state(env, this_branch->frame[this_branch->curframe]); 15427 return 0; 15428 } 15429 15430 /* verify BPF_LD_IMM64 instruction */ 15431 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 15432 { 15433 struct bpf_insn_aux_data *aux = cur_aux(env); 15434 struct bpf_reg_state *regs = cur_regs(env); 15435 struct bpf_reg_state *dst_reg; 15436 struct bpf_map *map; 15437 int err; 15438 15439 if (BPF_SIZE(insn->code) != BPF_DW) { 15440 verbose(env, "invalid BPF_LD_IMM insn\n"); 15441 return -EINVAL; 15442 } 15443 if (insn->off != 0) { 15444 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 15445 return -EINVAL; 15446 } 15447 15448 err = check_reg_arg(env, insn->dst_reg, DST_OP); 15449 if (err) 15450 return err; 15451 15452 dst_reg = ®s[insn->dst_reg]; 15453 if (insn->src_reg == 0) { 15454 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 15455 15456 dst_reg->type = SCALAR_VALUE; 15457 __mark_reg_known(®s[insn->dst_reg], imm); 15458 return 0; 15459 } 15460 15461 /* All special src_reg cases are listed below. From this point onwards 15462 * we either succeed and assign a corresponding dst_reg->type after 15463 * zeroing the offset, or fail and reject the program. 15464 */ 15465 mark_reg_known_zero(env, regs, insn->dst_reg); 15466 15467 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 15468 dst_reg->type = aux->btf_var.reg_type; 15469 switch (base_type(dst_reg->type)) { 15470 case PTR_TO_MEM: 15471 dst_reg->mem_size = aux->btf_var.mem_size; 15472 break; 15473 case PTR_TO_BTF_ID: 15474 dst_reg->btf = aux->btf_var.btf; 15475 dst_reg->btf_id = aux->btf_var.btf_id; 15476 break; 15477 default: 15478 verbose(env, "bpf verifier is misconfigured\n"); 15479 return -EFAULT; 15480 } 15481 return 0; 15482 } 15483 15484 if (insn->src_reg == BPF_PSEUDO_FUNC) { 15485 struct bpf_prog_aux *aux = env->prog->aux; 15486 u32 subprogno = find_subprog(env, 15487 env->insn_idx + insn->imm + 1); 15488 15489 if (!aux->func_info) { 15490 verbose(env, "missing btf func_info\n"); 15491 return -EINVAL; 15492 } 15493 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 15494 verbose(env, "callback function not static\n"); 15495 return -EINVAL; 15496 } 15497 15498 dst_reg->type = PTR_TO_FUNC; 15499 dst_reg->subprogno = subprogno; 15500 return 0; 15501 } 15502 15503 map = env->used_maps[aux->map_index]; 15504 dst_reg->map_ptr = map; 15505 15506 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 15507 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 15508 if (map->map_type == BPF_MAP_TYPE_ARENA) { 15509 __mark_reg_unknown(env, dst_reg); 15510 return 0; 15511 } 15512 dst_reg->type = PTR_TO_MAP_VALUE; 15513 dst_reg->off = aux->map_off; 15514 WARN_ON_ONCE(map->max_entries != 1); 15515 /* We want reg->id to be same (0) as map_value is not distinct */ 15516 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 15517 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 15518 dst_reg->type = CONST_PTR_TO_MAP; 15519 } else { 15520 verbose(env, "bpf verifier is misconfigured\n"); 15521 return -EINVAL; 15522 } 15523 15524 return 0; 15525 } 15526 15527 static bool may_access_skb(enum bpf_prog_type type) 15528 { 15529 switch (type) { 15530 case BPF_PROG_TYPE_SOCKET_FILTER: 15531 case BPF_PROG_TYPE_SCHED_CLS: 15532 case BPF_PROG_TYPE_SCHED_ACT: 15533 return true; 15534 default: 15535 return false; 15536 } 15537 } 15538 15539 /* verify safety of LD_ABS|LD_IND instructions: 15540 * - they can only appear in the programs where ctx == skb 15541 * - since they are wrappers of function calls, they scratch R1-R5 registers, 15542 * preserve R6-R9, and store return value into R0 15543 * 15544 * Implicit input: 15545 * ctx == skb == R6 == CTX 15546 * 15547 * Explicit input: 15548 * SRC == any register 15549 * IMM == 32-bit immediate 15550 * 15551 * Output: 15552 * R0 - 8/16/32-bit skb data converted to cpu endianness 15553 */ 15554 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 15555 { 15556 struct bpf_reg_state *regs = cur_regs(env); 15557 static const int ctx_reg = BPF_REG_6; 15558 u8 mode = BPF_MODE(insn->code); 15559 int i, err; 15560 15561 if (!may_access_skb(resolve_prog_type(env->prog))) { 15562 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 15563 return -EINVAL; 15564 } 15565 15566 if (!env->ops->gen_ld_abs) { 15567 verbose(env, "bpf verifier is misconfigured\n"); 15568 return -EINVAL; 15569 } 15570 15571 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 15572 BPF_SIZE(insn->code) == BPF_DW || 15573 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 15574 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 15575 return -EINVAL; 15576 } 15577 15578 /* check whether implicit source operand (register R6) is readable */ 15579 err = check_reg_arg(env, ctx_reg, SRC_OP); 15580 if (err) 15581 return err; 15582 15583 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 15584 * gen_ld_abs() may terminate the program at runtime, leading to 15585 * reference leak. 15586 */ 15587 err = check_reference_leak(env, false); 15588 if (err) { 15589 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 15590 return err; 15591 } 15592 15593 if (env->cur_state->active_lock.ptr) { 15594 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 15595 return -EINVAL; 15596 } 15597 15598 if (env->cur_state->active_rcu_lock) { 15599 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); 15600 return -EINVAL; 15601 } 15602 15603 if (env->cur_state->active_preempt_lock) { 15604 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_preempt_disable-ed region\n"); 15605 return -EINVAL; 15606 } 15607 15608 if (regs[ctx_reg].type != PTR_TO_CTX) { 15609 verbose(env, 15610 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 15611 return -EINVAL; 15612 } 15613 15614 if (mode == BPF_IND) { 15615 /* check explicit source operand */ 15616 err = check_reg_arg(env, insn->src_reg, SRC_OP); 15617 if (err) 15618 return err; 15619 } 15620 15621 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 15622 if (err < 0) 15623 return err; 15624 15625 /* reset caller saved regs to unreadable */ 15626 for (i = 0; i < CALLER_SAVED_REGS; i++) { 15627 mark_reg_not_init(env, regs, caller_saved[i]); 15628 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 15629 } 15630 15631 /* mark destination R0 register as readable, since it contains 15632 * the value fetched from the packet. 15633 * Already marked as written above. 15634 */ 15635 mark_reg_unknown(env, regs, BPF_REG_0); 15636 /* ld_abs load up to 32-bit skb data. */ 15637 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 15638 return 0; 15639 } 15640 15641 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name) 15642 { 15643 const char *exit_ctx = "At program exit"; 15644 struct tnum enforce_attach_type_range = tnum_unknown; 15645 const struct bpf_prog *prog = env->prog; 15646 struct bpf_reg_state *reg; 15647 struct bpf_retval_range range = retval_range(0, 1); 15648 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 15649 int err; 15650 struct bpf_func_state *frame = env->cur_state->frame[0]; 15651 const bool is_subprog = frame->subprogno; 15652 15653 /* LSM and struct_ops func-ptr's return type could be "void" */ 15654 if (!is_subprog || frame->in_exception_callback_fn) { 15655 switch (prog_type) { 15656 case BPF_PROG_TYPE_LSM: 15657 if (prog->expected_attach_type == BPF_LSM_CGROUP) 15658 /* See below, can be 0 or 0-1 depending on hook. */ 15659 break; 15660 fallthrough; 15661 case BPF_PROG_TYPE_STRUCT_OPS: 15662 if (!prog->aux->attach_func_proto->type) 15663 return 0; 15664 break; 15665 default: 15666 break; 15667 } 15668 } 15669 15670 /* eBPF calling convention is such that R0 is used 15671 * to return the value from eBPF program. 15672 * Make sure that it's readable at this time 15673 * of bpf_exit, which means that program wrote 15674 * something into it earlier 15675 */ 15676 err = check_reg_arg(env, regno, SRC_OP); 15677 if (err) 15678 return err; 15679 15680 if (is_pointer_value(env, regno)) { 15681 verbose(env, "R%d leaks addr as return value\n", regno); 15682 return -EACCES; 15683 } 15684 15685 reg = cur_regs(env) + regno; 15686 15687 if (frame->in_async_callback_fn) { 15688 /* enforce return zero from async callbacks like timer */ 15689 exit_ctx = "At async callback return"; 15690 range = retval_range(0, 0); 15691 goto enforce_retval; 15692 } 15693 15694 if (is_subprog && !frame->in_exception_callback_fn) { 15695 if (reg->type != SCALAR_VALUE) { 15696 verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", 15697 regno, reg_type_str(env, reg->type)); 15698 return -EINVAL; 15699 } 15700 return 0; 15701 } 15702 15703 switch (prog_type) { 15704 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 15705 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 15706 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 15707 env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG || 15708 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 15709 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 15710 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME || 15711 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 15712 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME || 15713 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME) 15714 range = retval_range(1, 1); 15715 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 15716 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 15717 range = retval_range(0, 3); 15718 break; 15719 case BPF_PROG_TYPE_CGROUP_SKB: 15720 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 15721 range = retval_range(0, 3); 15722 enforce_attach_type_range = tnum_range(2, 3); 15723 } 15724 break; 15725 case BPF_PROG_TYPE_CGROUP_SOCK: 15726 case BPF_PROG_TYPE_SOCK_OPS: 15727 case BPF_PROG_TYPE_CGROUP_DEVICE: 15728 case BPF_PROG_TYPE_CGROUP_SYSCTL: 15729 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 15730 break; 15731 case BPF_PROG_TYPE_RAW_TRACEPOINT: 15732 if (!env->prog->aux->attach_btf_id) 15733 return 0; 15734 range = retval_range(0, 0); 15735 break; 15736 case BPF_PROG_TYPE_TRACING: 15737 switch (env->prog->expected_attach_type) { 15738 case BPF_TRACE_FENTRY: 15739 case BPF_TRACE_FEXIT: 15740 range = retval_range(0, 0); 15741 break; 15742 case BPF_TRACE_RAW_TP: 15743 case BPF_MODIFY_RETURN: 15744 return 0; 15745 case BPF_TRACE_ITER: 15746 break; 15747 default: 15748 return -ENOTSUPP; 15749 } 15750 break; 15751 case BPF_PROG_TYPE_SK_LOOKUP: 15752 range = retval_range(SK_DROP, SK_PASS); 15753 break; 15754 15755 case BPF_PROG_TYPE_LSM: 15756 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 15757 /* Regular BPF_PROG_TYPE_LSM programs can return 15758 * any value. 15759 */ 15760 return 0; 15761 } 15762 if (!env->prog->aux->attach_func_proto->type) { 15763 /* Make sure programs that attach to void 15764 * hooks don't try to modify return value. 15765 */ 15766 range = retval_range(1, 1); 15767 } 15768 break; 15769 15770 case BPF_PROG_TYPE_NETFILTER: 15771 range = retval_range(NF_DROP, NF_ACCEPT); 15772 break; 15773 case BPF_PROG_TYPE_EXT: 15774 /* freplace program can return anything as its return value 15775 * depends on the to-be-replaced kernel func or bpf program. 15776 */ 15777 default: 15778 return 0; 15779 } 15780 15781 enforce_retval: 15782 if (reg->type != SCALAR_VALUE) { 15783 verbose(env, "%s the register R%d is not a known value (%s)\n", 15784 exit_ctx, regno, reg_type_str(env, reg->type)); 15785 return -EINVAL; 15786 } 15787 15788 err = mark_chain_precision(env, regno); 15789 if (err) 15790 return err; 15791 15792 if (!retval_range_within(range, reg)) { 15793 verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); 15794 if (!is_subprog && 15795 prog->expected_attach_type == BPF_LSM_CGROUP && 15796 prog_type == BPF_PROG_TYPE_LSM && 15797 !prog->aux->attach_func_proto->type) 15798 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 15799 return -EINVAL; 15800 } 15801 15802 if (!tnum_is_unknown(enforce_attach_type_range) && 15803 tnum_in(enforce_attach_type_range, reg->var_off)) 15804 env->prog->enforce_expected_attach_type = 1; 15805 return 0; 15806 } 15807 15808 /* non-recursive DFS pseudo code 15809 * 1 procedure DFS-iterative(G,v): 15810 * 2 label v as discovered 15811 * 3 let S be a stack 15812 * 4 S.push(v) 15813 * 5 while S is not empty 15814 * 6 t <- S.peek() 15815 * 7 if t is what we're looking for: 15816 * 8 return t 15817 * 9 for all edges e in G.adjacentEdges(t) do 15818 * 10 if edge e is already labelled 15819 * 11 continue with the next edge 15820 * 12 w <- G.adjacentVertex(t,e) 15821 * 13 if vertex w is not discovered and not explored 15822 * 14 label e as tree-edge 15823 * 15 label w as discovered 15824 * 16 S.push(w) 15825 * 17 continue at 5 15826 * 18 else if vertex w is discovered 15827 * 19 label e as back-edge 15828 * 20 else 15829 * 21 // vertex w is explored 15830 * 22 label e as forward- or cross-edge 15831 * 23 label t as explored 15832 * 24 S.pop() 15833 * 15834 * convention: 15835 * 0x10 - discovered 15836 * 0x11 - discovered and fall-through edge labelled 15837 * 0x12 - discovered and fall-through and branch edges labelled 15838 * 0x20 - explored 15839 */ 15840 15841 enum { 15842 DISCOVERED = 0x10, 15843 EXPLORED = 0x20, 15844 FALLTHROUGH = 1, 15845 BRANCH = 2, 15846 }; 15847 15848 static void mark_prune_point(struct bpf_verifier_env *env, int idx) 15849 { 15850 env->insn_aux_data[idx].prune_point = true; 15851 } 15852 15853 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) 15854 { 15855 return env->insn_aux_data[insn_idx].prune_point; 15856 } 15857 15858 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) 15859 { 15860 env->insn_aux_data[idx].force_checkpoint = true; 15861 } 15862 15863 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) 15864 { 15865 return env->insn_aux_data[insn_idx].force_checkpoint; 15866 } 15867 15868 static void mark_calls_callback(struct bpf_verifier_env *env, int idx) 15869 { 15870 env->insn_aux_data[idx].calls_callback = true; 15871 } 15872 15873 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx) 15874 { 15875 return env->insn_aux_data[insn_idx].calls_callback; 15876 } 15877 15878 enum { 15879 DONE_EXPLORING = 0, 15880 KEEP_EXPLORING = 1, 15881 }; 15882 15883 /* t, w, e - match pseudo-code above: 15884 * t - index of current instruction 15885 * w - next instruction 15886 * e - edge 15887 */ 15888 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) 15889 { 15890 int *insn_stack = env->cfg.insn_stack; 15891 int *insn_state = env->cfg.insn_state; 15892 15893 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 15894 return DONE_EXPLORING; 15895 15896 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 15897 return DONE_EXPLORING; 15898 15899 if (w < 0 || w >= env->prog->len) { 15900 verbose_linfo(env, t, "%d: ", t); 15901 verbose(env, "jump out of range from insn %d to %d\n", t, w); 15902 return -EINVAL; 15903 } 15904 15905 if (e == BRANCH) { 15906 /* mark branch target for state pruning */ 15907 mark_prune_point(env, w); 15908 mark_jmp_point(env, w); 15909 } 15910 15911 if (insn_state[w] == 0) { 15912 /* tree-edge */ 15913 insn_state[t] = DISCOVERED | e; 15914 insn_state[w] = DISCOVERED; 15915 if (env->cfg.cur_stack >= env->prog->len) 15916 return -E2BIG; 15917 insn_stack[env->cfg.cur_stack++] = w; 15918 return KEEP_EXPLORING; 15919 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 15920 if (env->bpf_capable) 15921 return DONE_EXPLORING; 15922 verbose_linfo(env, t, "%d: ", t); 15923 verbose_linfo(env, w, "%d: ", w); 15924 verbose(env, "back-edge from insn %d to %d\n", t, w); 15925 return -EINVAL; 15926 } else if (insn_state[w] == EXPLORED) { 15927 /* forward- or cross-edge */ 15928 insn_state[t] = DISCOVERED | e; 15929 } else { 15930 verbose(env, "insn state internal bug\n"); 15931 return -EFAULT; 15932 } 15933 return DONE_EXPLORING; 15934 } 15935 15936 static int visit_func_call_insn(int t, struct bpf_insn *insns, 15937 struct bpf_verifier_env *env, 15938 bool visit_callee) 15939 { 15940 int ret, insn_sz; 15941 15942 insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1; 15943 ret = push_insn(t, t + insn_sz, FALLTHROUGH, env); 15944 if (ret) 15945 return ret; 15946 15947 mark_prune_point(env, t + insn_sz); 15948 /* when we exit from subprog, we need to record non-linear history */ 15949 mark_jmp_point(env, t + insn_sz); 15950 15951 if (visit_callee) { 15952 mark_prune_point(env, t); 15953 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env); 15954 } 15955 return ret; 15956 } 15957 15958 /* Visits the instruction at index t and returns one of the following: 15959 * < 0 - an error occurred 15960 * DONE_EXPLORING - the instruction was fully explored 15961 * KEEP_EXPLORING - there is still work to be done before it is fully explored 15962 */ 15963 static int visit_insn(int t, struct bpf_verifier_env *env) 15964 { 15965 struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; 15966 int ret, off, insn_sz; 15967 15968 if (bpf_pseudo_func(insn)) 15969 return visit_func_call_insn(t, insns, env, true); 15970 15971 /* All non-branch instructions have a single fall-through edge. */ 15972 if (BPF_CLASS(insn->code) != BPF_JMP && 15973 BPF_CLASS(insn->code) != BPF_JMP32) { 15974 insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; 15975 return push_insn(t, t + insn_sz, FALLTHROUGH, env); 15976 } 15977 15978 switch (BPF_OP(insn->code)) { 15979 case BPF_EXIT: 15980 return DONE_EXPLORING; 15981 15982 case BPF_CALL: 15983 if (is_async_callback_calling_insn(insn)) 15984 /* Mark this call insn as a prune point to trigger 15985 * is_state_visited() check before call itself is 15986 * processed by __check_func_call(). Otherwise new 15987 * async state will be pushed for further exploration. 15988 */ 15989 mark_prune_point(env, t); 15990 /* For functions that invoke callbacks it is not known how many times 15991 * callback would be called. Verifier models callback calling functions 15992 * by repeatedly visiting callback bodies and returning to origin call 15993 * instruction. 15994 * In order to stop such iteration verifier needs to identify when a 15995 * state identical some state from a previous iteration is reached. 15996 * Check below forces creation of checkpoint before callback calling 15997 * instruction to allow search for such identical states. 15998 */ 15999 if (is_sync_callback_calling_insn(insn)) { 16000 mark_calls_callback(env, t); 16001 mark_force_checkpoint(env, t); 16002 mark_prune_point(env, t); 16003 mark_jmp_point(env, t); 16004 } 16005 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 16006 struct bpf_kfunc_call_arg_meta meta; 16007 16008 ret = fetch_kfunc_meta(env, insn, &meta, NULL); 16009 if (ret == 0 && is_iter_next_kfunc(&meta)) { 16010 mark_prune_point(env, t); 16011 /* Checking and saving state checkpoints at iter_next() call 16012 * is crucial for fast convergence of open-coded iterator loop 16013 * logic, so we need to force it. If we don't do that, 16014 * is_state_visited() might skip saving a checkpoint, causing 16015 * unnecessarily long sequence of not checkpointed 16016 * instructions and jumps, leading to exhaustion of jump 16017 * history buffer, and potentially other undesired outcomes. 16018 * It is expected that with correct open-coded iterators 16019 * convergence will happen quickly, so we don't run a risk of 16020 * exhausting memory. 16021 */ 16022 mark_force_checkpoint(env, t); 16023 } 16024 } 16025 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); 16026 16027 case BPF_JA: 16028 if (BPF_SRC(insn->code) != BPF_K) 16029 return -EINVAL; 16030 16031 if (BPF_CLASS(insn->code) == BPF_JMP) 16032 off = insn->off; 16033 else 16034 off = insn->imm; 16035 16036 /* unconditional jump with single edge */ 16037 ret = push_insn(t, t + off + 1, FALLTHROUGH, env); 16038 if (ret) 16039 return ret; 16040 16041 mark_prune_point(env, t + off + 1); 16042 mark_jmp_point(env, t + off + 1); 16043 16044 return ret; 16045 16046 default: 16047 /* conditional jump with two edges */ 16048 mark_prune_point(env, t); 16049 if (is_may_goto_insn(insn)) 16050 mark_force_checkpoint(env, t); 16051 16052 ret = push_insn(t, t + 1, FALLTHROUGH, env); 16053 if (ret) 16054 return ret; 16055 16056 return push_insn(t, t + insn->off + 1, BRANCH, env); 16057 } 16058 } 16059 16060 /* non-recursive depth-first-search to detect loops in BPF program 16061 * loop == back-edge in directed graph 16062 */ 16063 static int check_cfg(struct bpf_verifier_env *env) 16064 { 16065 int insn_cnt = env->prog->len; 16066 int *insn_stack, *insn_state; 16067 int ex_insn_beg, i, ret = 0; 16068 bool ex_done = false; 16069 16070 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 16071 if (!insn_state) 16072 return -ENOMEM; 16073 16074 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 16075 if (!insn_stack) { 16076 kvfree(insn_state); 16077 return -ENOMEM; 16078 } 16079 16080 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 16081 insn_stack[0] = 0; /* 0 is the first instruction */ 16082 env->cfg.cur_stack = 1; 16083 16084 walk_cfg: 16085 while (env->cfg.cur_stack > 0) { 16086 int t = insn_stack[env->cfg.cur_stack - 1]; 16087 16088 ret = visit_insn(t, env); 16089 switch (ret) { 16090 case DONE_EXPLORING: 16091 insn_state[t] = EXPLORED; 16092 env->cfg.cur_stack--; 16093 break; 16094 case KEEP_EXPLORING: 16095 break; 16096 default: 16097 if (ret > 0) { 16098 verbose(env, "visit_insn internal bug\n"); 16099 ret = -EFAULT; 16100 } 16101 goto err_free; 16102 } 16103 } 16104 16105 if (env->cfg.cur_stack < 0) { 16106 verbose(env, "pop stack internal bug\n"); 16107 ret = -EFAULT; 16108 goto err_free; 16109 } 16110 16111 if (env->exception_callback_subprog && !ex_done) { 16112 ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start; 16113 16114 insn_state[ex_insn_beg] = DISCOVERED; 16115 insn_stack[0] = ex_insn_beg; 16116 env->cfg.cur_stack = 1; 16117 ex_done = true; 16118 goto walk_cfg; 16119 } 16120 16121 for (i = 0; i < insn_cnt; i++) { 16122 struct bpf_insn *insn = &env->prog->insnsi[i]; 16123 16124 if (insn_state[i] != EXPLORED) { 16125 verbose(env, "unreachable insn %d\n", i); 16126 ret = -EINVAL; 16127 goto err_free; 16128 } 16129 if (bpf_is_ldimm64(insn)) { 16130 if (insn_state[i + 1] != 0) { 16131 verbose(env, "jump into the middle of ldimm64 insn %d\n", i); 16132 ret = -EINVAL; 16133 goto err_free; 16134 } 16135 i++; /* skip second half of ldimm64 */ 16136 } 16137 } 16138 ret = 0; /* cfg looks good */ 16139 16140 err_free: 16141 kvfree(insn_state); 16142 kvfree(insn_stack); 16143 env->cfg.insn_state = env->cfg.insn_stack = NULL; 16144 return ret; 16145 } 16146 16147 static int check_abnormal_return(struct bpf_verifier_env *env) 16148 { 16149 int i; 16150 16151 for (i = 1; i < env->subprog_cnt; i++) { 16152 if (env->subprog_info[i].has_ld_abs) { 16153 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 16154 return -EINVAL; 16155 } 16156 if (env->subprog_info[i].has_tail_call) { 16157 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 16158 return -EINVAL; 16159 } 16160 } 16161 return 0; 16162 } 16163 16164 /* The minimum supported BTF func info size */ 16165 #define MIN_BPF_FUNCINFO_SIZE 8 16166 #define MAX_FUNCINFO_REC_SIZE 252 16167 16168 static int check_btf_func_early(struct bpf_verifier_env *env, 16169 const union bpf_attr *attr, 16170 bpfptr_t uattr) 16171 { 16172 u32 krec_size = sizeof(struct bpf_func_info); 16173 const struct btf_type *type, *func_proto; 16174 u32 i, nfuncs, urec_size, min_size; 16175 struct bpf_func_info *krecord; 16176 struct bpf_prog *prog; 16177 const struct btf *btf; 16178 u32 prev_offset = 0; 16179 bpfptr_t urecord; 16180 int ret = -ENOMEM; 16181 16182 nfuncs = attr->func_info_cnt; 16183 if (!nfuncs) { 16184 if (check_abnormal_return(env)) 16185 return -EINVAL; 16186 return 0; 16187 } 16188 16189 urec_size = attr->func_info_rec_size; 16190 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 16191 urec_size > MAX_FUNCINFO_REC_SIZE || 16192 urec_size % sizeof(u32)) { 16193 verbose(env, "invalid func info rec size %u\n", urec_size); 16194 return -EINVAL; 16195 } 16196 16197 prog = env->prog; 16198 btf = prog->aux->btf; 16199 16200 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 16201 min_size = min_t(u32, krec_size, urec_size); 16202 16203 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 16204 if (!krecord) 16205 return -ENOMEM; 16206 16207 for (i = 0; i < nfuncs; i++) { 16208 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 16209 if (ret) { 16210 if (ret == -E2BIG) { 16211 verbose(env, "nonzero tailing record in func info"); 16212 /* set the size kernel expects so loader can zero 16213 * out the rest of the record. 16214 */ 16215 if (copy_to_bpfptr_offset(uattr, 16216 offsetof(union bpf_attr, func_info_rec_size), 16217 &min_size, sizeof(min_size))) 16218 ret = -EFAULT; 16219 } 16220 goto err_free; 16221 } 16222 16223 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 16224 ret = -EFAULT; 16225 goto err_free; 16226 } 16227 16228 /* check insn_off */ 16229 ret = -EINVAL; 16230 if (i == 0) { 16231 if (krecord[i].insn_off) { 16232 verbose(env, 16233 "nonzero insn_off %u for the first func info record", 16234 krecord[i].insn_off); 16235 goto err_free; 16236 } 16237 } else if (krecord[i].insn_off <= prev_offset) { 16238 verbose(env, 16239 "same or smaller insn offset (%u) than previous func info record (%u)", 16240 krecord[i].insn_off, prev_offset); 16241 goto err_free; 16242 } 16243 16244 /* check type_id */ 16245 type = btf_type_by_id(btf, krecord[i].type_id); 16246 if (!type || !btf_type_is_func(type)) { 16247 verbose(env, "invalid type id %d in func info", 16248 krecord[i].type_id); 16249 goto err_free; 16250 } 16251 16252 func_proto = btf_type_by_id(btf, type->type); 16253 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 16254 /* btf_func_check() already verified it during BTF load */ 16255 goto err_free; 16256 16257 prev_offset = krecord[i].insn_off; 16258 bpfptr_add(&urecord, urec_size); 16259 } 16260 16261 prog->aux->func_info = krecord; 16262 prog->aux->func_info_cnt = nfuncs; 16263 return 0; 16264 16265 err_free: 16266 kvfree(krecord); 16267 return ret; 16268 } 16269 16270 static int check_btf_func(struct bpf_verifier_env *env, 16271 const union bpf_attr *attr, 16272 bpfptr_t uattr) 16273 { 16274 const struct btf_type *type, *func_proto, *ret_type; 16275 u32 i, nfuncs, urec_size; 16276 struct bpf_func_info *krecord; 16277 struct bpf_func_info_aux *info_aux = NULL; 16278 struct bpf_prog *prog; 16279 const struct btf *btf; 16280 bpfptr_t urecord; 16281 bool scalar_return; 16282 int ret = -ENOMEM; 16283 16284 nfuncs = attr->func_info_cnt; 16285 if (!nfuncs) { 16286 if (check_abnormal_return(env)) 16287 return -EINVAL; 16288 return 0; 16289 } 16290 if (nfuncs != env->subprog_cnt) { 16291 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 16292 return -EINVAL; 16293 } 16294 16295 urec_size = attr->func_info_rec_size; 16296 16297 prog = env->prog; 16298 btf = prog->aux->btf; 16299 16300 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 16301 16302 krecord = prog->aux->func_info; 16303 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 16304 if (!info_aux) 16305 return -ENOMEM; 16306 16307 for (i = 0; i < nfuncs; i++) { 16308 /* check insn_off */ 16309 ret = -EINVAL; 16310 16311 if (env->subprog_info[i].start != krecord[i].insn_off) { 16312 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 16313 goto err_free; 16314 } 16315 16316 /* Already checked type_id */ 16317 type = btf_type_by_id(btf, krecord[i].type_id); 16318 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 16319 /* Already checked func_proto */ 16320 func_proto = btf_type_by_id(btf, type->type); 16321 16322 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 16323 scalar_return = 16324 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); 16325 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 16326 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 16327 goto err_free; 16328 } 16329 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 16330 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 16331 goto err_free; 16332 } 16333 16334 bpfptr_add(&urecord, urec_size); 16335 } 16336 16337 prog->aux->func_info_aux = info_aux; 16338 return 0; 16339 16340 err_free: 16341 kfree(info_aux); 16342 return ret; 16343 } 16344 16345 static void adjust_btf_func(struct bpf_verifier_env *env) 16346 { 16347 struct bpf_prog_aux *aux = env->prog->aux; 16348 int i; 16349 16350 if (!aux->func_info) 16351 return; 16352 16353 /* func_info is not available for hidden subprogs */ 16354 for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) 16355 aux->func_info[i].insn_off = env->subprog_info[i].start; 16356 } 16357 16358 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 16359 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 16360 16361 static int check_btf_line(struct bpf_verifier_env *env, 16362 const union bpf_attr *attr, 16363 bpfptr_t uattr) 16364 { 16365 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 16366 struct bpf_subprog_info *sub; 16367 struct bpf_line_info *linfo; 16368 struct bpf_prog *prog; 16369 const struct btf *btf; 16370 bpfptr_t ulinfo; 16371 int err; 16372 16373 nr_linfo = attr->line_info_cnt; 16374 if (!nr_linfo) 16375 return 0; 16376 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 16377 return -EINVAL; 16378 16379 rec_size = attr->line_info_rec_size; 16380 if (rec_size < MIN_BPF_LINEINFO_SIZE || 16381 rec_size > MAX_LINEINFO_REC_SIZE || 16382 rec_size & (sizeof(u32) - 1)) 16383 return -EINVAL; 16384 16385 /* Need to zero it in case the userspace may 16386 * pass in a smaller bpf_line_info object. 16387 */ 16388 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 16389 GFP_KERNEL | __GFP_NOWARN); 16390 if (!linfo) 16391 return -ENOMEM; 16392 16393 prog = env->prog; 16394 btf = prog->aux->btf; 16395 16396 s = 0; 16397 sub = env->subprog_info; 16398 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 16399 expected_size = sizeof(struct bpf_line_info); 16400 ncopy = min_t(u32, expected_size, rec_size); 16401 for (i = 0; i < nr_linfo; i++) { 16402 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 16403 if (err) { 16404 if (err == -E2BIG) { 16405 verbose(env, "nonzero tailing record in line_info"); 16406 if (copy_to_bpfptr_offset(uattr, 16407 offsetof(union bpf_attr, line_info_rec_size), 16408 &expected_size, sizeof(expected_size))) 16409 err = -EFAULT; 16410 } 16411 goto err_free; 16412 } 16413 16414 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 16415 err = -EFAULT; 16416 goto err_free; 16417 } 16418 16419 /* 16420 * Check insn_off to ensure 16421 * 1) strictly increasing AND 16422 * 2) bounded by prog->len 16423 * 16424 * The linfo[0].insn_off == 0 check logically falls into 16425 * the later "missing bpf_line_info for func..." case 16426 * because the first linfo[0].insn_off must be the 16427 * first sub also and the first sub must have 16428 * subprog_info[0].start == 0. 16429 */ 16430 if ((i && linfo[i].insn_off <= prev_offset) || 16431 linfo[i].insn_off >= prog->len) { 16432 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 16433 i, linfo[i].insn_off, prev_offset, 16434 prog->len); 16435 err = -EINVAL; 16436 goto err_free; 16437 } 16438 16439 if (!prog->insnsi[linfo[i].insn_off].code) { 16440 verbose(env, 16441 "Invalid insn code at line_info[%u].insn_off\n", 16442 i); 16443 err = -EINVAL; 16444 goto err_free; 16445 } 16446 16447 if (!btf_name_by_offset(btf, linfo[i].line_off) || 16448 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 16449 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 16450 err = -EINVAL; 16451 goto err_free; 16452 } 16453 16454 if (s != env->subprog_cnt) { 16455 if (linfo[i].insn_off == sub[s].start) { 16456 sub[s].linfo_idx = i; 16457 s++; 16458 } else if (sub[s].start < linfo[i].insn_off) { 16459 verbose(env, "missing bpf_line_info for func#%u\n", s); 16460 err = -EINVAL; 16461 goto err_free; 16462 } 16463 } 16464 16465 prev_offset = linfo[i].insn_off; 16466 bpfptr_add(&ulinfo, rec_size); 16467 } 16468 16469 if (s != env->subprog_cnt) { 16470 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 16471 env->subprog_cnt - s, s); 16472 err = -EINVAL; 16473 goto err_free; 16474 } 16475 16476 prog->aux->linfo = linfo; 16477 prog->aux->nr_linfo = nr_linfo; 16478 16479 return 0; 16480 16481 err_free: 16482 kvfree(linfo); 16483 return err; 16484 } 16485 16486 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 16487 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 16488 16489 static int check_core_relo(struct bpf_verifier_env *env, 16490 const union bpf_attr *attr, 16491 bpfptr_t uattr) 16492 { 16493 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 16494 struct bpf_core_relo core_relo = {}; 16495 struct bpf_prog *prog = env->prog; 16496 const struct btf *btf = prog->aux->btf; 16497 struct bpf_core_ctx ctx = { 16498 .log = &env->log, 16499 .btf = btf, 16500 }; 16501 bpfptr_t u_core_relo; 16502 int err; 16503 16504 nr_core_relo = attr->core_relo_cnt; 16505 if (!nr_core_relo) 16506 return 0; 16507 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 16508 return -EINVAL; 16509 16510 rec_size = attr->core_relo_rec_size; 16511 if (rec_size < MIN_CORE_RELO_SIZE || 16512 rec_size > MAX_CORE_RELO_SIZE || 16513 rec_size % sizeof(u32)) 16514 return -EINVAL; 16515 16516 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 16517 expected_size = sizeof(struct bpf_core_relo); 16518 ncopy = min_t(u32, expected_size, rec_size); 16519 16520 /* Unlike func_info and line_info, copy and apply each CO-RE 16521 * relocation record one at a time. 16522 */ 16523 for (i = 0; i < nr_core_relo; i++) { 16524 /* future proofing when sizeof(bpf_core_relo) changes */ 16525 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 16526 if (err) { 16527 if (err == -E2BIG) { 16528 verbose(env, "nonzero tailing record in core_relo"); 16529 if (copy_to_bpfptr_offset(uattr, 16530 offsetof(union bpf_attr, core_relo_rec_size), 16531 &expected_size, sizeof(expected_size))) 16532 err = -EFAULT; 16533 } 16534 break; 16535 } 16536 16537 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 16538 err = -EFAULT; 16539 break; 16540 } 16541 16542 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 16543 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 16544 i, core_relo.insn_off, prog->len); 16545 err = -EINVAL; 16546 break; 16547 } 16548 16549 err = bpf_core_apply(&ctx, &core_relo, i, 16550 &prog->insnsi[core_relo.insn_off / 8]); 16551 if (err) 16552 break; 16553 bpfptr_add(&u_core_relo, rec_size); 16554 } 16555 return err; 16556 } 16557 16558 static int check_btf_info_early(struct bpf_verifier_env *env, 16559 const union bpf_attr *attr, 16560 bpfptr_t uattr) 16561 { 16562 struct btf *btf; 16563 int err; 16564 16565 if (!attr->func_info_cnt && !attr->line_info_cnt) { 16566 if (check_abnormal_return(env)) 16567 return -EINVAL; 16568 return 0; 16569 } 16570 16571 btf = btf_get_by_fd(attr->prog_btf_fd); 16572 if (IS_ERR(btf)) 16573 return PTR_ERR(btf); 16574 if (btf_is_kernel(btf)) { 16575 btf_put(btf); 16576 return -EACCES; 16577 } 16578 env->prog->aux->btf = btf; 16579 16580 err = check_btf_func_early(env, attr, uattr); 16581 if (err) 16582 return err; 16583 return 0; 16584 } 16585 16586 static int check_btf_info(struct bpf_verifier_env *env, 16587 const union bpf_attr *attr, 16588 bpfptr_t uattr) 16589 { 16590 int err; 16591 16592 if (!attr->func_info_cnt && !attr->line_info_cnt) { 16593 if (check_abnormal_return(env)) 16594 return -EINVAL; 16595 return 0; 16596 } 16597 16598 err = check_btf_func(env, attr, uattr); 16599 if (err) 16600 return err; 16601 16602 err = check_btf_line(env, attr, uattr); 16603 if (err) 16604 return err; 16605 16606 err = check_core_relo(env, attr, uattr); 16607 if (err) 16608 return err; 16609 16610 return 0; 16611 } 16612 16613 /* check %cur's range satisfies %old's */ 16614 static bool range_within(const struct bpf_reg_state *old, 16615 const struct bpf_reg_state *cur) 16616 { 16617 return old->umin_value <= cur->umin_value && 16618 old->umax_value >= cur->umax_value && 16619 old->smin_value <= cur->smin_value && 16620 old->smax_value >= cur->smax_value && 16621 old->u32_min_value <= cur->u32_min_value && 16622 old->u32_max_value >= cur->u32_max_value && 16623 old->s32_min_value <= cur->s32_min_value && 16624 old->s32_max_value >= cur->s32_max_value; 16625 } 16626 16627 /* If in the old state two registers had the same id, then they need to have 16628 * the same id in the new state as well. But that id could be different from 16629 * the old state, so we need to track the mapping from old to new ids. 16630 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 16631 * regs with old id 5 must also have new id 9 for the new state to be safe. But 16632 * regs with a different old id could still have new id 9, we don't care about 16633 * that. 16634 * So we look through our idmap to see if this old id has been seen before. If 16635 * so, we require the new id to match; otherwise, we add the id pair to the map. 16636 */ 16637 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) 16638 { 16639 struct bpf_id_pair *map = idmap->map; 16640 unsigned int i; 16641 16642 /* either both IDs should be set or both should be zero */ 16643 if (!!old_id != !!cur_id) 16644 return false; 16645 16646 if (old_id == 0) /* cur_id == 0 as well */ 16647 return true; 16648 16649 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 16650 if (!map[i].old) { 16651 /* Reached an empty slot; haven't seen this id before */ 16652 map[i].old = old_id; 16653 map[i].cur = cur_id; 16654 return true; 16655 } 16656 if (map[i].old == old_id) 16657 return map[i].cur == cur_id; 16658 if (map[i].cur == cur_id) 16659 return false; 16660 } 16661 /* We ran out of idmap slots, which should be impossible */ 16662 WARN_ON_ONCE(1); 16663 return false; 16664 } 16665 16666 /* Similar to check_ids(), but allocate a unique temporary ID 16667 * for 'old_id' or 'cur_id' of zero. 16668 * This makes pairs like '0 vs unique ID', 'unique ID vs 0' valid. 16669 */ 16670 static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) 16671 { 16672 old_id = old_id ? old_id : ++idmap->tmp_id_gen; 16673 cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen; 16674 16675 return check_ids(old_id, cur_id, idmap); 16676 } 16677 16678 static void clean_func_state(struct bpf_verifier_env *env, 16679 struct bpf_func_state *st) 16680 { 16681 enum bpf_reg_liveness live; 16682 int i, j; 16683 16684 for (i = 0; i < BPF_REG_FP; i++) { 16685 live = st->regs[i].live; 16686 /* liveness must not touch this register anymore */ 16687 st->regs[i].live |= REG_LIVE_DONE; 16688 if (!(live & REG_LIVE_READ)) 16689 /* since the register is unused, clear its state 16690 * to make further comparison simpler 16691 */ 16692 __mark_reg_not_init(env, &st->regs[i]); 16693 } 16694 16695 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 16696 live = st->stack[i].spilled_ptr.live; 16697 /* liveness must not touch this stack slot anymore */ 16698 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 16699 if (!(live & REG_LIVE_READ)) { 16700 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 16701 for (j = 0; j < BPF_REG_SIZE; j++) 16702 st->stack[i].slot_type[j] = STACK_INVALID; 16703 } 16704 } 16705 } 16706 16707 static void clean_verifier_state(struct bpf_verifier_env *env, 16708 struct bpf_verifier_state *st) 16709 { 16710 int i; 16711 16712 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 16713 /* all regs in this state in all frames were already marked */ 16714 return; 16715 16716 for (i = 0; i <= st->curframe; i++) 16717 clean_func_state(env, st->frame[i]); 16718 } 16719 16720 /* the parentage chains form a tree. 16721 * the verifier states are added to state lists at given insn and 16722 * pushed into state stack for future exploration. 16723 * when the verifier reaches bpf_exit insn some of the verifer states 16724 * stored in the state lists have their final liveness state already, 16725 * but a lot of states will get revised from liveness point of view when 16726 * the verifier explores other branches. 16727 * Example: 16728 * 1: r0 = 1 16729 * 2: if r1 == 100 goto pc+1 16730 * 3: r0 = 2 16731 * 4: exit 16732 * when the verifier reaches exit insn the register r0 in the state list of 16733 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 16734 * of insn 2 and goes exploring further. At the insn 4 it will walk the 16735 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 16736 * 16737 * Since the verifier pushes the branch states as it sees them while exploring 16738 * the program the condition of walking the branch instruction for the second 16739 * time means that all states below this branch were already explored and 16740 * their final liveness marks are already propagated. 16741 * Hence when the verifier completes the search of state list in is_state_visited() 16742 * we can call this clean_live_states() function to mark all liveness states 16743 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 16744 * will not be used. 16745 * This function also clears the registers and stack for states that !READ 16746 * to simplify state merging. 16747 * 16748 * Important note here that walking the same branch instruction in the callee 16749 * doesn't meant that the states are DONE. The verifier has to compare 16750 * the callsites 16751 */ 16752 static void clean_live_states(struct bpf_verifier_env *env, int insn, 16753 struct bpf_verifier_state *cur) 16754 { 16755 struct bpf_verifier_state_list *sl; 16756 16757 sl = *explored_state(env, insn); 16758 while (sl) { 16759 if (sl->state.branches) 16760 goto next; 16761 if (sl->state.insn_idx != insn || 16762 !same_callsites(&sl->state, cur)) 16763 goto next; 16764 clean_verifier_state(env, &sl->state); 16765 next: 16766 sl = sl->next; 16767 } 16768 } 16769 16770 static bool regs_exact(const struct bpf_reg_state *rold, 16771 const struct bpf_reg_state *rcur, 16772 struct bpf_idmap *idmap) 16773 { 16774 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 16775 check_ids(rold->id, rcur->id, idmap) && 16776 check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); 16777 } 16778 16779 enum exact_level { 16780 NOT_EXACT, 16781 EXACT, 16782 RANGE_WITHIN 16783 }; 16784 16785 /* Returns true if (rold safe implies rcur safe) */ 16786 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 16787 struct bpf_reg_state *rcur, struct bpf_idmap *idmap, 16788 enum exact_level exact) 16789 { 16790 if (exact == EXACT) 16791 return regs_exact(rold, rcur, idmap); 16792 16793 if (!(rold->live & REG_LIVE_READ) && exact == NOT_EXACT) 16794 /* explored state didn't use this */ 16795 return true; 16796 if (rold->type == NOT_INIT) { 16797 if (exact == NOT_EXACT || rcur->type == NOT_INIT) 16798 /* explored state can't have used this */ 16799 return true; 16800 } 16801 16802 /* Enforce that register types have to match exactly, including their 16803 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general 16804 * rule. 16805 * 16806 * One can make a point that using a pointer register as unbounded 16807 * SCALAR would be technically acceptable, but this could lead to 16808 * pointer leaks because scalars are allowed to leak while pointers 16809 * are not. We could make this safe in special cases if root is 16810 * calling us, but it's probably not worth the hassle. 16811 * 16812 * Also, register types that are *not* MAYBE_NULL could technically be 16813 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE 16814 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point 16815 * to the same map). 16816 * However, if the old MAYBE_NULL register then got NULL checked, 16817 * doing so could have affected others with the same id, and we can't 16818 * check for that because we lost the id when we converted to 16819 * a non-MAYBE_NULL variant. 16820 * So, as a general rule we don't allow mixing MAYBE_NULL and 16821 * non-MAYBE_NULL registers as well. 16822 */ 16823 if (rold->type != rcur->type) 16824 return false; 16825 16826 switch (base_type(rold->type)) { 16827 case SCALAR_VALUE: 16828 if (env->explore_alu_limits) { 16829 /* explore_alu_limits disables tnum_in() and range_within() 16830 * logic and requires everything to be strict 16831 */ 16832 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 16833 check_scalar_ids(rold->id, rcur->id, idmap); 16834 } 16835 if (!rold->precise && exact == NOT_EXACT) 16836 return true; 16837 if ((rold->id & BPF_ADD_CONST) != (rcur->id & BPF_ADD_CONST)) 16838 return false; 16839 if ((rold->id & BPF_ADD_CONST) && (rold->off != rcur->off)) 16840 return false; 16841 /* Why check_ids() for scalar registers? 16842 * 16843 * Consider the following BPF code: 16844 * 1: r6 = ... unbound scalar, ID=a ... 16845 * 2: r7 = ... unbound scalar, ID=b ... 16846 * 3: if (r6 > r7) goto +1 16847 * 4: r6 = r7 16848 * 5: if (r6 > X) goto ... 16849 * 6: ... memory operation using r7 ... 16850 * 16851 * First verification path is [1-6]: 16852 * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7; 16853 * - at (5) r6 would be marked <= X, find_equal_scalars() would also mark 16854 * r7 <= X, because r6 and r7 share same id. 16855 * Next verification path is [1-4, 6]. 16856 * 16857 * Instruction (6) would be reached in two states: 16858 * I. r6{.id=b}, r7{.id=b} via path 1-6; 16859 * II. r6{.id=a}, r7{.id=b} via path 1-4, 6. 16860 * 16861 * Use check_ids() to distinguish these states. 16862 * --- 16863 * Also verify that new value satisfies old value range knowledge. 16864 */ 16865 return range_within(rold, rcur) && 16866 tnum_in(rold->var_off, rcur->var_off) && 16867 check_scalar_ids(rold->id, rcur->id, idmap); 16868 case PTR_TO_MAP_KEY: 16869 case PTR_TO_MAP_VALUE: 16870 case PTR_TO_MEM: 16871 case PTR_TO_BUF: 16872 case PTR_TO_TP_BUFFER: 16873 /* If the new min/max/var_off satisfy the old ones and 16874 * everything else matches, we are OK. 16875 */ 16876 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 && 16877 range_within(rold, rcur) && 16878 tnum_in(rold->var_off, rcur->var_off) && 16879 check_ids(rold->id, rcur->id, idmap) && 16880 check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); 16881 case PTR_TO_PACKET_META: 16882 case PTR_TO_PACKET: 16883 /* We must have at least as much range as the old ptr 16884 * did, so that any accesses which were safe before are 16885 * still safe. This is true even if old range < old off, 16886 * since someone could have accessed through (ptr - k), or 16887 * even done ptr -= k in a register, to get a safe access. 16888 */ 16889 if (rold->range > rcur->range) 16890 return false; 16891 /* If the offsets don't match, we can't trust our alignment; 16892 * nor can we be sure that we won't fall out of range. 16893 */ 16894 if (rold->off != rcur->off) 16895 return false; 16896 /* id relations must be preserved */ 16897 if (!check_ids(rold->id, rcur->id, idmap)) 16898 return false; 16899 /* new val must satisfy old val knowledge */ 16900 return range_within(rold, rcur) && 16901 tnum_in(rold->var_off, rcur->var_off); 16902 case PTR_TO_STACK: 16903 /* two stack pointers are equal only if they're pointing to 16904 * the same stack frame, since fp-8 in foo != fp-8 in bar 16905 */ 16906 return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; 16907 case PTR_TO_ARENA: 16908 return true; 16909 default: 16910 return regs_exact(rold, rcur, idmap); 16911 } 16912 } 16913 16914 static struct bpf_reg_state unbound_reg; 16915 16916 static __init int unbound_reg_init(void) 16917 { 16918 __mark_reg_unknown_imprecise(&unbound_reg); 16919 unbound_reg.live |= REG_LIVE_READ; 16920 return 0; 16921 } 16922 late_initcall(unbound_reg_init); 16923 16924 static bool is_stack_all_misc(struct bpf_verifier_env *env, 16925 struct bpf_stack_state *stack) 16926 { 16927 u32 i; 16928 16929 for (i = 0; i < ARRAY_SIZE(stack->slot_type); ++i) { 16930 if ((stack->slot_type[i] == STACK_MISC) || 16931 (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack)) 16932 continue; 16933 return false; 16934 } 16935 16936 return true; 16937 } 16938 16939 static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env, 16940 struct bpf_stack_state *stack) 16941 { 16942 if (is_spilled_scalar_reg64(stack)) 16943 return &stack->spilled_ptr; 16944 16945 if (is_stack_all_misc(env, stack)) 16946 return &unbound_reg; 16947 16948 return NULL; 16949 } 16950 16951 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 16952 struct bpf_func_state *cur, struct bpf_idmap *idmap, 16953 enum exact_level exact) 16954 { 16955 int i, spi; 16956 16957 /* walk slots of the explored stack and ignore any additional 16958 * slots in the current stack, since explored(safe) state 16959 * didn't use them 16960 */ 16961 for (i = 0; i < old->allocated_stack; i++) { 16962 struct bpf_reg_state *old_reg, *cur_reg; 16963 16964 spi = i / BPF_REG_SIZE; 16965 16966 if (exact != NOT_EXACT && 16967 old->stack[spi].slot_type[i % BPF_REG_SIZE] != 16968 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 16969 return false; 16970 16971 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ) 16972 && exact == NOT_EXACT) { 16973 i += BPF_REG_SIZE - 1; 16974 /* explored state didn't use this */ 16975 continue; 16976 } 16977 16978 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 16979 continue; 16980 16981 if (env->allow_uninit_stack && 16982 old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) 16983 continue; 16984 16985 /* explored stack has more populated slots than current stack 16986 * and these slots were used 16987 */ 16988 if (i >= cur->allocated_stack) 16989 return false; 16990 16991 /* 64-bit scalar spill vs all slots MISC and vice versa. 16992 * Load from all slots MISC produces unbound scalar. 16993 * Construct a fake register for such stack and call 16994 * regsafe() to ensure scalar ids are compared. 16995 */ 16996 old_reg = scalar_reg_for_stack(env, &old->stack[spi]); 16997 cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]); 16998 if (old_reg && cur_reg) { 16999 if (!regsafe(env, old_reg, cur_reg, idmap, exact)) 17000 return false; 17001 i += BPF_REG_SIZE - 1; 17002 continue; 17003 } 17004 17005 /* if old state was safe with misc data in the stack 17006 * it will be safe with zero-initialized stack. 17007 * The opposite is not true 17008 */ 17009 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 17010 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 17011 continue; 17012 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 17013 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 17014 /* Ex: old explored (safe) state has STACK_SPILL in 17015 * this stack slot, but current has STACK_MISC -> 17016 * this verifier states are not equivalent, 17017 * return false to continue verification of this path 17018 */ 17019 return false; 17020 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 17021 continue; 17022 /* Both old and cur are having same slot_type */ 17023 switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { 17024 case STACK_SPILL: 17025 /* when explored and current stack slot are both storing 17026 * spilled registers, check that stored pointers types 17027 * are the same as well. 17028 * Ex: explored safe path could have stored 17029 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 17030 * but current path has stored: 17031 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 17032 * such verifier states are not equivalent. 17033 * return false to continue verification of this path 17034 */ 17035 if (!regsafe(env, &old->stack[spi].spilled_ptr, 17036 &cur->stack[spi].spilled_ptr, idmap, exact)) 17037 return false; 17038 break; 17039 case STACK_DYNPTR: 17040 old_reg = &old->stack[spi].spilled_ptr; 17041 cur_reg = &cur->stack[spi].spilled_ptr; 17042 if (old_reg->dynptr.type != cur_reg->dynptr.type || 17043 old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || 17044 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 17045 return false; 17046 break; 17047 case STACK_ITER: 17048 old_reg = &old->stack[spi].spilled_ptr; 17049 cur_reg = &cur->stack[spi].spilled_ptr; 17050 /* iter.depth is not compared between states as it 17051 * doesn't matter for correctness and would otherwise 17052 * prevent convergence; we maintain it only to prevent 17053 * infinite loop check triggering, see 17054 * iter_active_depths_differ() 17055 */ 17056 if (old_reg->iter.btf != cur_reg->iter.btf || 17057 old_reg->iter.btf_id != cur_reg->iter.btf_id || 17058 old_reg->iter.state != cur_reg->iter.state || 17059 /* ignore {old_reg,cur_reg}->iter.depth, see above */ 17060 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 17061 return false; 17062 break; 17063 case STACK_MISC: 17064 case STACK_ZERO: 17065 case STACK_INVALID: 17066 continue; 17067 /* Ensure that new unhandled slot types return false by default */ 17068 default: 17069 return false; 17070 } 17071 } 17072 return true; 17073 } 17074 17075 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, 17076 struct bpf_idmap *idmap) 17077 { 17078 int i; 17079 17080 if (old->acquired_refs != cur->acquired_refs) 17081 return false; 17082 17083 for (i = 0; i < old->acquired_refs; i++) { 17084 if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap)) 17085 return false; 17086 } 17087 17088 return true; 17089 } 17090 17091 /* compare two verifier states 17092 * 17093 * all states stored in state_list are known to be valid, since 17094 * verifier reached 'bpf_exit' instruction through them 17095 * 17096 * this function is called when verifier exploring different branches of 17097 * execution popped from the state stack. If it sees an old state that has 17098 * more strict register state and more strict stack state then this execution 17099 * branch doesn't need to be explored further, since verifier already 17100 * concluded that more strict state leads to valid finish. 17101 * 17102 * Therefore two states are equivalent if register state is more conservative 17103 * and explored stack state is more conservative than the current one. 17104 * Example: 17105 * explored current 17106 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 17107 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 17108 * 17109 * In other words if current stack state (one being explored) has more 17110 * valid slots than old one that already passed validation, it means 17111 * the verifier can stop exploring and conclude that current state is valid too 17112 * 17113 * Similarly with registers. If explored state has register type as invalid 17114 * whereas register type in current state is meaningful, it means that 17115 * the current state will reach 'bpf_exit' instruction safely 17116 */ 17117 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 17118 struct bpf_func_state *cur, enum exact_level exact) 17119 { 17120 int i; 17121 17122 if (old->callback_depth > cur->callback_depth) 17123 return false; 17124 17125 for (i = 0; i < MAX_BPF_REG; i++) 17126 if (!regsafe(env, &old->regs[i], &cur->regs[i], 17127 &env->idmap_scratch, exact)) 17128 return false; 17129 17130 if (!stacksafe(env, old, cur, &env->idmap_scratch, exact)) 17131 return false; 17132 17133 if (!refsafe(old, cur, &env->idmap_scratch)) 17134 return false; 17135 17136 return true; 17137 } 17138 17139 static void reset_idmap_scratch(struct bpf_verifier_env *env) 17140 { 17141 env->idmap_scratch.tmp_id_gen = env->id_gen; 17142 memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map)); 17143 } 17144 17145 static bool states_equal(struct bpf_verifier_env *env, 17146 struct bpf_verifier_state *old, 17147 struct bpf_verifier_state *cur, 17148 enum exact_level exact) 17149 { 17150 int i; 17151 17152 if (old->curframe != cur->curframe) 17153 return false; 17154 17155 reset_idmap_scratch(env); 17156 17157 /* Verification state from speculative execution simulation 17158 * must never prune a non-speculative execution one. 17159 */ 17160 if (old->speculative && !cur->speculative) 17161 return false; 17162 17163 if (old->active_lock.ptr != cur->active_lock.ptr) 17164 return false; 17165 17166 /* Old and cur active_lock's have to be either both present 17167 * or both absent. 17168 */ 17169 if (!!old->active_lock.id != !!cur->active_lock.id) 17170 return false; 17171 17172 if (old->active_lock.id && 17173 !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch)) 17174 return false; 17175 17176 if (old->active_rcu_lock != cur->active_rcu_lock) 17177 return false; 17178 17179 if (old->active_preempt_lock != cur->active_preempt_lock) 17180 return false; 17181 17182 if (old->in_sleepable != cur->in_sleepable) 17183 return false; 17184 17185 /* for states to be equal callsites have to be the same 17186 * and all frame states need to be equivalent 17187 */ 17188 for (i = 0; i <= old->curframe; i++) { 17189 if (old->frame[i]->callsite != cur->frame[i]->callsite) 17190 return false; 17191 if (!func_states_equal(env, old->frame[i], cur->frame[i], exact)) 17192 return false; 17193 } 17194 return true; 17195 } 17196 17197 /* Return 0 if no propagation happened. Return negative error code if error 17198 * happened. Otherwise, return the propagated bit. 17199 */ 17200 static int propagate_liveness_reg(struct bpf_verifier_env *env, 17201 struct bpf_reg_state *reg, 17202 struct bpf_reg_state *parent_reg) 17203 { 17204 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 17205 u8 flag = reg->live & REG_LIVE_READ; 17206 int err; 17207 17208 /* When comes here, read flags of PARENT_REG or REG could be any of 17209 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 17210 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 17211 */ 17212 if (parent_flag == REG_LIVE_READ64 || 17213 /* Or if there is no read flag from REG. */ 17214 !flag || 17215 /* Or if the read flag from REG is the same as PARENT_REG. */ 17216 parent_flag == flag) 17217 return 0; 17218 17219 err = mark_reg_read(env, reg, parent_reg, flag); 17220 if (err) 17221 return err; 17222 17223 return flag; 17224 } 17225 17226 /* A write screens off any subsequent reads; but write marks come from the 17227 * straight-line code between a state and its parent. When we arrive at an 17228 * equivalent state (jump target or such) we didn't arrive by the straight-line 17229 * code, so read marks in the state must propagate to the parent regardless 17230 * of the state's write marks. That's what 'parent == state->parent' comparison 17231 * in mark_reg_read() is for. 17232 */ 17233 static int propagate_liveness(struct bpf_verifier_env *env, 17234 const struct bpf_verifier_state *vstate, 17235 struct bpf_verifier_state *vparent) 17236 { 17237 struct bpf_reg_state *state_reg, *parent_reg; 17238 struct bpf_func_state *state, *parent; 17239 int i, frame, err = 0; 17240 17241 if (vparent->curframe != vstate->curframe) { 17242 WARN(1, "propagate_live: parent frame %d current frame %d\n", 17243 vparent->curframe, vstate->curframe); 17244 return -EFAULT; 17245 } 17246 /* Propagate read liveness of registers... */ 17247 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 17248 for (frame = 0; frame <= vstate->curframe; frame++) { 17249 parent = vparent->frame[frame]; 17250 state = vstate->frame[frame]; 17251 parent_reg = parent->regs; 17252 state_reg = state->regs; 17253 /* We don't need to worry about FP liveness, it's read-only */ 17254 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 17255 err = propagate_liveness_reg(env, &state_reg[i], 17256 &parent_reg[i]); 17257 if (err < 0) 17258 return err; 17259 if (err == REG_LIVE_READ64) 17260 mark_insn_zext(env, &parent_reg[i]); 17261 } 17262 17263 /* Propagate stack slots. */ 17264 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 17265 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 17266 parent_reg = &parent->stack[i].spilled_ptr; 17267 state_reg = &state->stack[i].spilled_ptr; 17268 err = propagate_liveness_reg(env, state_reg, 17269 parent_reg); 17270 if (err < 0) 17271 return err; 17272 } 17273 } 17274 return 0; 17275 } 17276 17277 /* find precise scalars in the previous equivalent state and 17278 * propagate them into the current state 17279 */ 17280 static int propagate_precision(struct bpf_verifier_env *env, 17281 const struct bpf_verifier_state *old) 17282 { 17283 struct bpf_reg_state *state_reg; 17284 struct bpf_func_state *state; 17285 int i, err = 0, fr; 17286 bool first; 17287 17288 for (fr = old->curframe; fr >= 0; fr--) { 17289 state = old->frame[fr]; 17290 state_reg = state->regs; 17291 first = true; 17292 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 17293 if (state_reg->type != SCALAR_VALUE || 17294 !state_reg->precise || 17295 !(state_reg->live & REG_LIVE_READ)) 17296 continue; 17297 if (env->log.level & BPF_LOG_LEVEL2) { 17298 if (first) 17299 verbose(env, "frame %d: propagating r%d", fr, i); 17300 else 17301 verbose(env, ",r%d", i); 17302 } 17303 bt_set_frame_reg(&env->bt, fr, i); 17304 first = false; 17305 } 17306 17307 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 17308 if (!is_spilled_reg(&state->stack[i])) 17309 continue; 17310 state_reg = &state->stack[i].spilled_ptr; 17311 if (state_reg->type != SCALAR_VALUE || 17312 !state_reg->precise || 17313 !(state_reg->live & REG_LIVE_READ)) 17314 continue; 17315 if (env->log.level & BPF_LOG_LEVEL2) { 17316 if (first) 17317 verbose(env, "frame %d: propagating fp%d", 17318 fr, (-i - 1) * BPF_REG_SIZE); 17319 else 17320 verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE); 17321 } 17322 bt_set_frame_slot(&env->bt, fr, i); 17323 first = false; 17324 } 17325 if (!first) 17326 verbose(env, "\n"); 17327 } 17328 17329 err = mark_chain_precision_batch(env); 17330 if (err < 0) 17331 return err; 17332 17333 return 0; 17334 } 17335 17336 static bool states_maybe_looping(struct bpf_verifier_state *old, 17337 struct bpf_verifier_state *cur) 17338 { 17339 struct bpf_func_state *fold, *fcur; 17340 int i, fr = cur->curframe; 17341 17342 if (old->curframe != fr) 17343 return false; 17344 17345 fold = old->frame[fr]; 17346 fcur = cur->frame[fr]; 17347 for (i = 0; i < MAX_BPF_REG; i++) 17348 if (memcmp(&fold->regs[i], &fcur->regs[i], 17349 offsetof(struct bpf_reg_state, parent))) 17350 return false; 17351 return true; 17352 } 17353 17354 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx) 17355 { 17356 return env->insn_aux_data[insn_idx].is_iter_next; 17357 } 17358 17359 /* is_state_visited() handles iter_next() (see process_iter_next_call() for 17360 * terminology) calls specially: as opposed to bounded BPF loops, it *expects* 17361 * states to match, which otherwise would look like an infinite loop. So while 17362 * iter_next() calls are taken care of, we still need to be careful and 17363 * prevent erroneous and too eager declaration of "ininite loop", when 17364 * iterators are involved. 17365 * 17366 * Here's a situation in pseudo-BPF assembly form: 17367 * 17368 * 0: again: ; set up iter_next() call args 17369 * 1: r1 = &it ; <CHECKPOINT HERE> 17370 * 2: call bpf_iter_num_next ; this is iter_next() call 17371 * 3: if r0 == 0 goto done 17372 * 4: ... something useful here ... 17373 * 5: goto again ; another iteration 17374 * 6: done: 17375 * 7: r1 = &it 17376 * 8: call bpf_iter_num_destroy ; clean up iter state 17377 * 9: exit 17378 * 17379 * This is a typical loop. Let's assume that we have a prune point at 1:, 17380 * before we get to `call bpf_iter_num_next` (e.g., because of that `goto 17381 * again`, assuming other heuristics don't get in a way). 17382 * 17383 * When we first time come to 1:, let's say we have some state X. We proceed 17384 * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit. 17385 * Now we come back to validate that forked ACTIVE state. We proceed through 17386 * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we 17387 * are converging. But the problem is that we don't know that yet, as this 17388 * convergence has to happen at iter_next() call site only. So if nothing is 17389 * done, at 1: verifier will use bounded loop logic and declare infinite 17390 * looping (and would be *technically* correct, if not for iterator's 17391 * "eventual sticky NULL" contract, see process_iter_next_call()). But we 17392 * don't want that. So what we do in process_iter_next_call() when we go on 17393 * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's 17394 * a different iteration. So when we suspect an infinite loop, we additionally 17395 * check if any of the *ACTIVE* iterator states depths differ. If yes, we 17396 * pretend we are not looping and wait for next iter_next() call. 17397 * 17398 * This only applies to ACTIVE state. In DRAINED state we don't expect to 17399 * loop, because that would actually mean infinite loop, as DRAINED state is 17400 * "sticky", and so we'll keep returning into the same instruction with the 17401 * same state (at least in one of possible code paths). 17402 * 17403 * This approach allows to keep infinite loop heuristic even in the face of 17404 * active iterator. E.g., C snippet below is and will be detected as 17405 * inifintely looping: 17406 * 17407 * struct bpf_iter_num it; 17408 * int *p, x; 17409 * 17410 * bpf_iter_num_new(&it, 0, 10); 17411 * while ((p = bpf_iter_num_next(&t))) { 17412 * x = p; 17413 * while (x--) {} // <<-- infinite loop here 17414 * } 17415 * 17416 */ 17417 static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur) 17418 { 17419 struct bpf_reg_state *slot, *cur_slot; 17420 struct bpf_func_state *state; 17421 int i, fr; 17422 17423 for (fr = old->curframe; fr >= 0; fr--) { 17424 state = old->frame[fr]; 17425 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 17426 if (state->stack[i].slot_type[0] != STACK_ITER) 17427 continue; 17428 17429 slot = &state->stack[i].spilled_ptr; 17430 if (slot->iter.state != BPF_ITER_STATE_ACTIVE) 17431 continue; 17432 17433 cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; 17434 if (cur_slot->iter.depth != slot->iter.depth) 17435 return true; 17436 } 17437 } 17438 return false; 17439 } 17440 17441 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 17442 { 17443 struct bpf_verifier_state_list *new_sl; 17444 struct bpf_verifier_state_list *sl, **pprev; 17445 struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry; 17446 int i, j, n, err, states_cnt = 0; 17447 bool force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx); 17448 bool add_new_state = force_new_state; 17449 bool force_exact; 17450 17451 /* bpf progs typically have pruning point every 4 instructions 17452 * http://vger.kernel.org/bpfconf2019.html#session-1 17453 * Do not add new state for future pruning if the verifier hasn't seen 17454 * at least 2 jumps and at least 8 instructions. 17455 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 17456 * In tests that amounts to up to 50% reduction into total verifier 17457 * memory consumption and 20% verifier time speedup. 17458 */ 17459 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 17460 env->insn_processed - env->prev_insn_processed >= 8) 17461 add_new_state = true; 17462 17463 pprev = explored_state(env, insn_idx); 17464 sl = *pprev; 17465 17466 clean_live_states(env, insn_idx, cur); 17467 17468 while (sl) { 17469 states_cnt++; 17470 if (sl->state.insn_idx != insn_idx) 17471 goto next; 17472 17473 if (sl->state.branches) { 17474 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 17475 17476 if (frame->in_async_callback_fn && 17477 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 17478 /* Different async_entry_cnt means that the verifier is 17479 * processing another entry into async callback. 17480 * Seeing the same state is not an indication of infinite 17481 * loop or infinite recursion. 17482 * But finding the same state doesn't mean that it's safe 17483 * to stop processing the current state. The previous state 17484 * hasn't yet reached bpf_exit, since state.branches > 0. 17485 * Checking in_async_callback_fn alone is not enough either. 17486 * Since the verifier still needs to catch infinite loops 17487 * inside async callbacks. 17488 */ 17489 goto skip_inf_loop_check; 17490 } 17491 /* BPF open-coded iterators loop detection is special. 17492 * states_maybe_looping() logic is too simplistic in detecting 17493 * states that *might* be equivalent, because it doesn't know 17494 * about ID remapping, so don't even perform it. 17495 * See process_iter_next_call() and iter_active_depths_differ() 17496 * for overview of the logic. When current and one of parent 17497 * states are detected as equivalent, it's a good thing: we prove 17498 * convergence and can stop simulating further iterations. 17499 * It's safe to assume that iterator loop will finish, taking into 17500 * account iter_next() contract of eventually returning 17501 * sticky NULL result. 17502 * 17503 * Note, that states have to be compared exactly in this case because 17504 * read and precision marks might not be finalized inside the loop. 17505 * E.g. as in the program below: 17506 * 17507 * 1. r7 = -16 17508 * 2. r6 = bpf_get_prandom_u32() 17509 * 3. while (bpf_iter_num_next(&fp[-8])) { 17510 * 4. if (r6 != 42) { 17511 * 5. r7 = -32 17512 * 6. r6 = bpf_get_prandom_u32() 17513 * 7. continue 17514 * 8. } 17515 * 9. r0 = r10 17516 * 10. r0 += r7 17517 * 11. r8 = *(u64 *)(r0 + 0) 17518 * 12. r6 = bpf_get_prandom_u32() 17519 * 13. } 17520 * 17521 * Here verifier would first visit path 1-3, create a checkpoint at 3 17522 * with r7=-16, continue to 4-7,3. Existing checkpoint at 3 does 17523 * not have read or precision mark for r7 yet, thus inexact states 17524 * comparison would discard current state with r7=-32 17525 * => unsafe memory access at 11 would not be caught. 17526 */ 17527 if (is_iter_next_insn(env, insn_idx)) { 17528 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { 17529 struct bpf_func_state *cur_frame; 17530 struct bpf_reg_state *iter_state, *iter_reg; 17531 int spi; 17532 17533 cur_frame = cur->frame[cur->curframe]; 17534 /* btf_check_iter_kfuncs() enforces that 17535 * iter state pointer is always the first arg 17536 */ 17537 iter_reg = &cur_frame->regs[BPF_REG_1]; 17538 /* current state is valid due to states_equal(), 17539 * so we can assume valid iter and reg state, 17540 * no need for extra (re-)validations 17541 */ 17542 spi = __get_spi(iter_reg->off + iter_reg->var_off.value); 17543 iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr; 17544 if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { 17545 update_loop_entry(cur, &sl->state); 17546 goto hit; 17547 } 17548 } 17549 goto skip_inf_loop_check; 17550 } 17551 if (is_may_goto_insn_at(env, insn_idx)) { 17552 if (sl->state.may_goto_depth != cur->may_goto_depth && 17553 states_equal(env, &sl->state, cur, RANGE_WITHIN)) { 17554 update_loop_entry(cur, &sl->state); 17555 goto hit; 17556 } 17557 } 17558 if (calls_callback(env, insn_idx)) { 17559 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) 17560 goto hit; 17561 goto skip_inf_loop_check; 17562 } 17563 /* attempt to detect infinite loop to avoid unnecessary doomed work */ 17564 if (states_maybe_looping(&sl->state, cur) && 17565 states_equal(env, &sl->state, cur, EXACT) && 17566 !iter_active_depths_differ(&sl->state, cur) && 17567 sl->state.may_goto_depth == cur->may_goto_depth && 17568 sl->state.callback_unroll_depth == cur->callback_unroll_depth) { 17569 verbose_linfo(env, insn_idx, "; "); 17570 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 17571 verbose(env, "cur state:"); 17572 print_verifier_state(env, cur->frame[cur->curframe], true); 17573 verbose(env, "old state:"); 17574 print_verifier_state(env, sl->state.frame[cur->curframe], true); 17575 return -EINVAL; 17576 } 17577 /* if the verifier is processing a loop, avoid adding new state 17578 * too often, since different loop iterations have distinct 17579 * states and may not help future pruning. 17580 * This threshold shouldn't be too low to make sure that 17581 * a loop with large bound will be rejected quickly. 17582 * The most abusive loop will be: 17583 * r1 += 1 17584 * if r1 < 1000000 goto pc-2 17585 * 1M insn_procssed limit / 100 == 10k peak states. 17586 * This threshold shouldn't be too high either, since states 17587 * at the end of the loop are likely to be useful in pruning. 17588 */ 17589 skip_inf_loop_check: 17590 if (!force_new_state && 17591 env->jmps_processed - env->prev_jmps_processed < 20 && 17592 env->insn_processed - env->prev_insn_processed < 100) 17593 add_new_state = false; 17594 goto miss; 17595 } 17596 /* If sl->state is a part of a loop and this loop's entry is a part of 17597 * current verification path then states have to be compared exactly. 17598 * 'force_exact' is needed to catch the following case: 17599 * 17600 * initial Here state 'succ' was processed first, 17601 * | it was eventually tracked to produce a 17602 * V state identical to 'hdr'. 17603 * .---------> hdr All branches from 'succ' had been explored 17604 * | | and thus 'succ' has its .branches == 0. 17605 * | V 17606 * | .------... Suppose states 'cur' and 'succ' correspond 17607 * | | | to the same instruction + callsites. 17608 * | V V In such case it is necessary to check 17609 * | ... ... if 'succ' and 'cur' are states_equal(). 17610 * | | | If 'succ' and 'cur' are a part of the 17611 * | V V same loop exact flag has to be set. 17612 * | succ <- cur To check if that is the case, verify 17613 * | | if loop entry of 'succ' is in current 17614 * | V DFS path. 17615 * | ... 17616 * | | 17617 * '----' 17618 * 17619 * Additional details are in the comment before get_loop_entry(). 17620 */ 17621 loop_entry = get_loop_entry(&sl->state); 17622 force_exact = loop_entry && loop_entry->branches > 0; 17623 if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) { 17624 if (force_exact) 17625 update_loop_entry(cur, loop_entry); 17626 hit: 17627 sl->hit_cnt++; 17628 /* reached equivalent register/stack state, 17629 * prune the search. 17630 * Registers read by the continuation are read by us. 17631 * If we have any write marks in env->cur_state, they 17632 * will prevent corresponding reads in the continuation 17633 * from reaching our parent (an explored_state). Our 17634 * own state will get the read marks recorded, but 17635 * they'll be immediately forgotten as we're pruning 17636 * this state and will pop a new one. 17637 */ 17638 err = propagate_liveness(env, &sl->state, cur); 17639 17640 /* if previous state reached the exit with precision and 17641 * current state is equivalent to it (except precision marks) 17642 * the precision needs to be propagated back in 17643 * the current state. 17644 */ 17645 if (is_jmp_point(env, env->insn_idx)) 17646 err = err ? : push_jmp_history(env, cur, 0); 17647 err = err ? : propagate_precision(env, &sl->state); 17648 if (err) 17649 return err; 17650 return 1; 17651 } 17652 miss: 17653 /* when new state is not going to be added do not increase miss count. 17654 * Otherwise several loop iterations will remove the state 17655 * recorded earlier. The goal of these heuristics is to have 17656 * states from some iterations of the loop (some in the beginning 17657 * and some at the end) to help pruning. 17658 */ 17659 if (add_new_state) 17660 sl->miss_cnt++; 17661 /* heuristic to determine whether this state is beneficial 17662 * to keep checking from state equivalence point of view. 17663 * Higher numbers increase max_states_per_insn and verification time, 17664 * but do not meaningfully decrease insn_processed. 17665 * 'n' controls how many times state could miss before eviction. 17666 * Use bigger 'n' for checkpoints because evicting checkpoint states 17667 * too early would hinder iterator convergence. 17668 */ 17669 n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; 17670 if (sl->miss_cnt > sl->hit_cnt * n + n) { 17671 /* the state is unlikely to be useful. Remove it to 17672 * speed up verification 17673 */ 17674 *pprev = sl->next; 17675 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE && 17676 !sl->state.used_as_loop_entry) { 17677 u32 br = sl->state.branches; 17678 17679 WARN_ONCE(br, 17680 "BUG live_done but branches_to_explore %d\n", 17681 br); 17682 free_verifier_state(&sl->state, false); 17683 kfree(sl); 17684 env->peak_states--; 17685 } else { 17686 /* cannot free this state, since parentage chain may 17687 * walk it later. Add it for free_list instead to 17688 * be freed at the end of verification 17689 */ 17690 sl->next = env->free_list; 17691 env->free_list = sl; 17692 } 17693 sl = *pprev; 17694 continue; 17695 } 17696 next: 17697 pprev = &sl->next; 17698 sl = *pprev; 17699 } 17700 17701 if (env->max_states_per_insn < states_cnt) 17702 env->max_states_per_insn = states_cnt; 17703 17704 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 17705 return 0; 17706 17707 if (!add_new_state) 17708 return 0; 17709 17710 /* There were no equivalent states, remember the current one. 17711 * Technically the current state is not proven to be safe yet, 17712 * but it will either reach outer most bpf_exit (which means it's safe) 17713 * or it will be rejected. When there are no loops the verifier won't be 17714 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 17715 * again on the way to bpf_exit. 17716 * When looping the sl->state.branches will be > 0 and this state 17717 * will not be considered for equivalence until branches == 0. 17718 */ 17719 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 17720 if (!new_sl) 17721 return -ENOMEM; 17722 env->total_states++; 17723 env->peak_states++; 17724 env->prev_jmps_processed = env->jmps_processed; 17725 env->prev_insn_processed = env->insn_processed; 17726 17727 /* forget precise markings we inherited, see __mark_chain_precision */ 17728 if (env->bpf_capable) 17729 mark_all_scalars_imprecise(env, cur); 17730 17731 /* add new state to the head of linked list */ 17732 new = &new_sl->state; 17733 err = copy_verifier_state(new, cur); 17734 if (err) { 17735 free_verifier_state(new, false); 17736 kfree(new_sl); 17737 return err; 17738 } 17739 new->insn_idx = insn_idx; 17740 WARN_ONCE(new->branches != 1, 17741 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 17742 17743 cur->parent = new; 17744 cur->first_insn_idx = insn_idx; 17745 cur->dfs_depth = new->dfs_depth + 1; 17746 clear_jmp_history(cur); 17747 new_sl->next = *explored_state(env, insn_idx); 17748 *explored_state(env, insn_idx) = new_sl; 17749 /* connect new state to parentage chain. Current frame needs all 17750 * registers connected. Only r6 - r9 of the callers are alive (pushed 17751 * to the stack implicitly by JITs) so in callers' frames connect just 17752 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 17753 * the state of the call instruction (with WRITTEN set), and r0 comes 17754 * from callee with its full parentage chain, anyway. 17755 */ 17756 /* clear write marks in current state: the writes we did are not writes 17757 * our child did, so they don't screen off its reads from us. 17758 * (There are no read marks in current state, because reads always mark 17759 * their parent and current state never has children yet. Only 17760 * explored_states can get read marks.) 17761 */ 17762 for (j = 0; j <= cur->curframe; j++) { 17763 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 17764 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 17765 for (i = 0; i < BPF_REG_FP; i++) 17766 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 17767 } 17768 17769 /* all stack frames are accessible from callee, clear them all */ 17770 for (j = 0; j <= cur->curframe; j++) { 17771 struct bpf_func_state *frame = cur->frame[j]; 17772 struct bpf_func_state *newframe = new->frame[j]; 17773 17774 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 17775 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 17776 frame->stack[i].spilled_ptr.parent = 17777 &newframe->stack[i].spilled_ptr; 17778 } 17779 } 17780 return 0; 17781 } 17782 17783 /* Return true if it's OK to have the same insn return a different type. */ 17784 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 17785 { 17786 switch (base_type(type)) { 17787 case PTR_TO_CTX: 17788 case PTR_TO_SOCKET: 17789 case PTR_TO_SOCK_COMMON: 17790 case PTR_TO_TCP_SOCK: 17791 case PTR_TO_XDP_SOCK: 17792 case PTR_TO_BTF_ID: 17793 case PTR_TO_ARENA: 17794 return false; 17795 default: 17796 return true; 17797 } 17798 } 17799 17800 /* If an instruction was previously used with particular pointer types, then we 17801 * need to be careful to avoid cases such as the below, where it may be ok 17802 * for one branch accessing the pointer, but not ok for the other branch: 17803 * 17804 * R1 = sock_ptr 17805 * goto X; 17806 * ... 17807 * R1 = some_other_valid_ptr; 17808 * goto X; 17809 * ... 17810 * R2 = *(u32 *)(R1 + 0); 17811 */ 17812 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 17813 { 17814 return src != prev && (!reg_type_mismatch_ok(src) || 17815 !reg_type_mismatch_ok(prev)); 17816 } 17817 17818 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, 17819 bool allow_trust_mismatch) 17820 { 17821 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; 17822 17823 if (*prev_type == NOT_INIT) { 17824 /* Saw a valid insn 17825 * dst_reg = *(u32 *)(src_reg + off) 17826 * save type to validate intersecting paths 17827 */ 17828 *prev_type = type; 17829 } else if (reg_type_mismatch(type, *prev_type)) { 17830 /* Abuser program is trying to use the same insn 17831 * dst_reg = *(u32*) (src_reg + off) 17832 * with different pointer types: 17833 * src_reg == ctx in one branch and 17834 * src_reg == stack|map in some other branch. 17835 * Reject it. 17836 */ 17837 if (allow_trust_mismatch && 17838 base_type(type) == PTR_TO_BTF_ID && 17839 base_type(*prev_type) == PTR_TO_BTF_ID) { 17840 /* 17841 * Have to support a use case when one path through 17842 * the program yields TRUSTED pointer while another 17843 * is UNTRUSTED. Fallback to UNTRUSTED to generate 17844 * BPF_PROBE_MEM/BPF_PROBE_MEMSX. 17845 */ 17846 *prev_type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 17847 } else { 17848 verbose(env, "same insn cannot be used with different pointers\n"); 17849 return -EINVAL; 17850 } 17851 } 17852 17853 return 0; 17854 } 17855 17856 static int do_check(struct bpf_verifier_env *env) 17857 { 17858 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 17859 struct bpf_verifier_state *state = env->cur_state; 17860 struct bpf_insn *insns = env->prog->insnsi; 17861 struct bpf_reg_state *regs; 17862 int insn_cnt = env->prog->len; 17863 bool do_print_state = false; 17864 int prev_insn_idx = -1; 17865 17866 for (;;) { 17867 bool exception_exit = false; 17868 struct bpf_insn *insn; 17869 u8 class; 17870 int err; 17871 17872 /* reset current history entry on each new instruction */ 17873 env->cur_hist_ent = NULL; 17874 17875 env->prev_insn_idx = prev_insn_idx; 17876 if (env->insn_idx >= insn_cnt) { 17877 verbose(env, "invalid insn idx %d insn_cnt %d\n", 17878 env->insn_idx, insn_cnt); 17879 return -EFAULT; 17880 } 17881 17882 insn = &insns[env->insn_idx]; 17883 class = BPF_CLASS(insn->code); 17884 17885 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 17886 verbose(env, 17887 "BPF program is too large. Processed %d insn\n", 17888 env->insn_processed); 17889 return -E2BIG; 17890 } 17891 17892 state->last_insn_idx = env->prev_insn_idx; 17893 17894 if (is_prune_point(env, env->insn_idx)) { 17895 err = is_state_visited(env, env->insn_idx); 17896 if (err < 0) 17897 return err; 17898 if (err == 1) { 17899 /* found equivalent state, can prune the search */ 17900 if (env->log.level & BPF_LOG_LEVEL) { 17901 if (do_print_state) 17902 verbose(env, "\nfrom %d to %d%s: safe\n", 17903 env->prev_insn_idx, env->insn_idx, 17904 env->cur_state->speculative ? 17905 " (speculative execution)" : ""); 17906 else 17907 verbose(env, "%d: safe\n", env->insn_idx); 17908 } 17909 goto process_bpf_exit; 17910 } 17911 } 17912 17913 if (is_jmp_point(env, env->insn_idx)) { 17914 err = push_jmp_history(env, state, 0); 17915 if (err) 17916 return err; 17917 } 17918 17919 if (signal_pending(current)) 17920 return -EAGAIN; 17921 17922 if (need_resched()) 17923 cond_resched(); 17924 17925 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 17926 verbose(env, "\nfrom %d to %d%s:", 17927 env->prev_insn_idx, env->insn_idx, 17928 env->cur_state->speculative ? 17929 " (speculative execution)" : ""); 17930 print_verifier_state(env, state->frame[state->curframe], true); 17931 do_print_state = false; 17932 } 17933 17934 if (env->log.level & BPF_LOG_LEVEL) { 17935 const struct bpf_insn_cbs cbs = { 17936 .cb_call = disasm_kfunc_name, 17937 .cb_print = verbose, 17938 .private_data = env, 17939 }; 17940 17941 if (verifier_state_scratched(env)) 17942 print_insn_state(env, state->frame[state->curframe]); 17943 17944 verbose_linfo(env, env->insn_idx, "; "); 17945 env->prev_log_pos = env->log.end_pos; 17946 verbose(env, "%d: ", env->insn_idx); 17947 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 17948 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; 17949 env->prev_log_pos = env->log.end_pos; 17950 } 17951 17952 if (bpf_prog_is_offloaded(env->prog->aux)) { 17953 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 17954 env->prev_insn_idx); 17955 if (err) 17956 return err; 17957 } 17958 17959 regs = cur_regs(env); 17960 sanitize_mark_insn_seen(env); 17961 prev_insn_idx = env->insn_idx; 17962 17963 if (class == BPF_ALU || class == BPF_ALU64) { 17964 err = check_alu_op(env, insn); 17965 if (err) 17966 return err; 17967 17968 } else if (class == BPF_LDX) { 17969 enum bpf_reg_type src_reg_type; 17970 17971 /* check for reserved fields is already done */ 17972 17973 /* check src operand */ 17974 err = check_reg_arg(env, insn->src_reg, SRC_OP); 17975 if (err) 17976 return err; 17977 17978 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 17979 if (err) 17980 return err; 17981 17982 src_reg_type = regs[insn->src_reg].type; 17983 17984 /* check that memory (src_reg + off) is readable, 17985 * the state of dst_reg will be updated by this func 17986 */ 17987 err = check_mem_access(env, env->insn_idx, insn->src_reg, 17988 insn->off, BPF_SIZE(insn->code), 17989 BPF_READ, insn->dst_reg, false, 17990 BPF_MODE(insn->code) == BPF_MEMSX); 17991 err = err ?: save_aux_ptr_type(env, src_reg_type, true); 17992 err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], "ldx"); 17993 if (err) 17994 return err; 17995 } else if (class == BPF_STX) { 17996 enum bpf_reg_type dst_reg_type; 17997 17998 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 17999 err = check_atomic(env, env->insn_idx, insn); 18000 if (err) 18001 return err; 18002 env->insn_idx++; 18003 continue; 18004 } 18005 18006 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 18007 verbose(env, "BPF_STX uses reserved fields\n"); 18008 return -EINVAL; 18009 } 18010 18011 /* check src1 operand */ 18012 err = check_reg_arg(env, insn->src_reg, SRC_OP); 18013 if (err) 18014 return err; 18015 /* check src2 operand */ 18016 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 18017 if (err) 18018 return err; 18019 18020 dst_reg_type = regs[insn->dst_reg].type; 18021 18022 /* check that memory (dst_reg + off) is writeable */ 18023 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 18024 insn->off, BPF_SIZE(insn->code), 18025 BPF_WRITE, insn->src_reg, false, false); 18026 if (err) 18027 return err; 18028 18029 err = save_aux_ptr_type(env, dst_reg_type, false); 18030 if (err) 18031 return err; 18032 } else if (class == BPF_ST) { 18033 enum bpf_reg_type dst_reg_type; 18034 18035 if (BPF_MODE(insn->code) != BPF_MEM || 18036 insn->src_reg != BPF_REG_0) { 18037 verbose(env, "BPF_ST uses reserved fields\n"); 18038 return -EINVAL; 18039 } 18040 /* check src operand */ 18041 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 18042 if (err) 18043 return err; 18044 18045 dst_reg_type = regs[insn->dst_reg].type; 18046 18047 /* check that memory (dst_reg + off) is writeable */ 18048 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 18049 insn->off, BPF_SIZE(insn->code), 18050 BPF_WRITE, -1, false, false); 18051 if (err) 18052 return err; 18053 18054 err = save_aux_ptr_type(env, dst_reg_type, false); 18055 if (err) 18056 return err; 18057 } else if (class == BPF_JMP || class == BPF_JMP32) { 18058 u8 opcode = BPF_OP(insn->code); 18059 18060 env->jmps_processed++; 18061 if (opcode == BPF_CALL) { 18062 if (BPF_SRC(insn->code) != BPF_K || 18063 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 18064 && insn->off != 0) || 18065 (insn->src_reg != BPF_REG_0 && 18066 insn->src_reg != BPF_PSEUDO_CALL && 18067 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 18068 insn->dst_reg != BPF_REG_0 || 18069 class == BPF_JMP32) { 18070 verbose(env, "BPF_CALL uses reserved fields\n"); 18071 return -EINVAL; 18072 } 18073 18074 if (env->cur_state->active_lock.ptr) { 18075 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 18076 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 18077 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) { 18078 verbose(env, "function calls are not allowed while holding a lock\n"); 18079 return -EINVAL; 18080 } 18081 } 18082 if (insn->src_reg == BPF_PSEUDO_CALL) { 18083 err = check_func_call(env, insn, &env->insn_idx); 18084 } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 18085 err = check_kfunc_call(env, insn, &env->insn_idx); 18086 if (!err && is_bpf_throw_kfunc(insn)) { 18087 exception_exit = true; 18088 goto process_bpf_exit_full; 18089 } 18090 } else { 18091 err = check_helper_call(env, insn, &env->insn_idx); 18092 } 18093 if (err) 18094 return err; 18095 18096 mark_reg_scratched(env, BPF_REG_0); 18097 } else if (opcode == BPF_JA) { 18098 if (BPF_SRC(insn->code) != BPF_K || 18099 insn->src_reg != BPF_REG_0 || 18100 insn->dst_reg != BPF_REG_0 || 18101 (class == BPF_JMP && insn->imm != 0) || 18102 (class == BPF_JMP32 && insn->off != 0)) { 18103 verbose(env, "BPF_JA uses reserved fields\n"); 18104 return -EINVAL; 18105 } 18106 18107 if (class == BPF_JMP) 18108 env->insn_idx += insn->off + 1; 18109 else 18110 env->insn_idx += insn->imm + 1; 18111 continue; 18112 18113 } else if (opcode == BPF_EXIT) { 18114 if (BPF_SRC(insn->code) != BPF_K || 18115 insn->imm != 0 || 18116 insn->src_reg != BPF_REG_0 || 18117 insn->dst_reg != BPF_REG_0 || 18118 class == BPF_JMP32) { 18119 verbose(env, "BPF_EXIT uses reserved fields\n"); 18120 return -EINVAL; 18121 } 18122 process_bpf_exit_full: 18123 if (env->cur_state->active_lock.ptr && !env->cur_state->curframe) { 18124 verbose(env, "bpf_spin_unlock is missing\n"); 18125 return -EINVAL; 18126 } 18127 18128 if (env->cur_state->active_rcu_lock && !env->cur_state->curframe) { 18129 verbose(env, "bpf_rcu_read_unlock is missing\n"); 18130 return -EINVAL; 18131 } 18132 18133 if (env->cur_state->active_preempt_lock && !env->cur_state->curframe) { 18134 verbose(env, "%d bpf_preempt_enable%s missing\n", 18135 env->cur_state->active_preempt_lock, 18136 env->cur_state->active_preempt_lock == 1 ? " is" : "(s) are"); 18137 return -EINVAL; 18138 } 18139 18140 /* We must do check_reference_leak here before 18141 * prepare_func_exit to handle the case when 18142 * state->curframe > 0, it may be a callback 18143 * function, for which reference_state must 18144 * match caller reference state when it exits. 18145 */ 18146 err = check_reference_leak(env, exception_exit); 18147 if (err) 18148 return err; 18149 18150 /* The side effect of the prepare_func_exit 18151 * which is being skipped is that it frees 18152 * bpf_func_state. Typically, process_bpf_exit 18153 * will only be hit with outermost exit. 18154 * copy_verifier_state in pop_stack will handle 18155 * freeing of any extra bpf_func_state left over 18156 * from not processing all nested function 18157 * exits. We also skip return code checks as 18158 * they are not needed for exceptional exits. 18159 */ 18160 if (exception_exit) 18161 goto process_bpf_exit; 18162 18163 if (state->curframe) { 18164 /* exit from nested function */ 18165 err = prepare_func_exit(env, &env->insn_idx); 18166 if (err) 18167 return err; 18168 do_print_state = true; 18169 continue; 18170 } 18171 18172 err = check_return_code(env, BPF_REG_0, "R0"); 18173 if (err) 18174 return err; 18175 process_bpf_exit: 18176 mark_verifier_state_scratched(env); 18177 update_branch_counts(env, env->cur_state); 18178 err = pop_stack(env, &prev_insn_idx, 18179 &env->insn_idx, pop_log); 18180 if (err < 0) { 18181 if (err != -ENOENT) 18182 return err; 18183 break; 18184 } else { 18185 do_print_state = true; 18186 continue; 18187 } 18188 } else { 18189 err = check_cond_jmp_op(env, insn, &env->insn_idx); 18190 if (err) 18191 return err; 18192 } 18193 } else if (class == BPF_LD) { 18194 u8 mode = BPF_MODE(insn->code); 18195 18196 if (mode == BPF_ABS || mode == BPF_IND) { 18197 err = check_ld_abs(env, insn); 18198 if (err) 18199 return err; 18200 18201 } else if (mode == BPF_IMM) { 18202 err = check_ld_imm(env, insn); 18203 if (err) 18204 return err; 18205 18206 env->insn_idx++; 18207 sanitize_mark_insn_seen(env); 18208 } else { 18209 verbose(env, "invalid BPF_LD mode\n"); 18210 return -EINVAL; 18211 } 18212 } else { 18213 verbose(env, "unknown insn class %d\n", class); 18214 return -EINVAL; 18215 } 18216 18217 env->insn_idx++; 18218 } 18219 18220 return 0; 18221 } 18222 18223 static int find_btf_percpu_datasec(struct btf *btf) 18224 { 18225 const struct btf_type *t; 18226 const char *tname; 18227 int i, n; 18228 18229 /* 18230 * Both vmlinux and module each have their own ".data..percpu" 18231 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 18232 * types to look at only module's own BTF types. 18233 */ 18234 n = btf_nr_types(btf); 18235 if (btf_is_module(btf)) 18236 i = btf_nr_types(btf_vmlinux); 18237 else 18238 i = 1; 18239 18240 for(; i < n; i++) { 18241 t = btf_type_by_id(btf, i); 18242 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 18243 continue; 18244 18245 tname = btf_name_by_offset(btf, t->name_off); 18246 if (!strcmp(tname, ".data..percpu")) 18247 return i; 18248 } 18249 18250 return -ENOENT; 18251 } 18252 18253 /* replace pseudo btf_id with kernel symbol address */ 18254 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 18255 struct bpf_insn *insn, 18256 struct bpf_insn_aux_data *aux) 18257 { 18258 const struct btf_var_secinfo *vsi; 18259 const struct btf_type *datasec; 18260 struct btf_mod_pair *btf_mod; 18261 const struct btf_type *t; 18262 const char *sym_name; 18263 bool percpu = false; 18264 u32 type, id = insn->imm; 18265 struct btf *btf; 18266 s32 datasec_id; 18267 u64 addr; 18268 int i, btf_fd, err; 18269 18270 btf_fd = insn[1].imm; 18271 if (btf_fd) { 18272 btf = btf_get_by_fd(btf_fd); 18273 if (IS_ERR(btf)) { 18274 verbose(env, "invalid module BTF object FD specified.\n"); 18275 return -EINVAL; 18276 } 18277 } else { 18278 if (!btf_vmlinux) { 18279 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 18280 return -EINVAL; 18281 } 18282 btf = btf_vmlinux; 18283 btf_get(btf); 18284 } 18285 18286 t = btf_type_by_id(btf, id); 18287 if (!t) { 18288 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 18289 err = -ENOENT; 18290 goto err_put; 18291 } 18292 18293 if (!btf_type_is_var(t) && !btf_type_is_func(t)) { 18294 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); 18295 err = -EINVAL; 18296 goto err_put; 18297 } 18298 18299 sym_name = btf_name_by_offset(btf, t->name_off); 18300 addr = kallsyms_lookup_name(sym_name); 18301 if (!addr) { 18302 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 18303 sym_name); 18304 err = -ENOENT; 18305 goto err_put; 18306 } 18307 insn[0].imm = (u32)addr; 18308 insn[1].imm = addr >> 32; 18309 18310 if (btf_type_is_func(t)) { 18311 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 18312 aux->btf_var.mem_size = 0; 18313 goto check_btf; 18314 } 18315 18316 datasec_id = find_btf_percpu_datasec(btf); 18317 if (datasec_id > 0) { 18318 datasec = btf_type_by_id(btf, datasec_id); 18319 for_each_vsi(i, datasec, vsi) { 18320 if (vsi->type == id) { 18321 percpu = true; 18322 break; 18323 } 18324 } 18325 } 18326 18327 type = t->type; 18328 t = btf_type_skip_modifiers(btf, type, NULL); 18329 if (percpu) { 18330 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 18331 aux->btf_var.btf = btf; 18332 aux->btf_var.btf_id = type; 18333 } else if (!btf_type_is_struct(t)) { 18334 const struct btf_type *ret; 18335 const char *tname; 18336 u32 tsize; 18337 18338 /* resolve the type size of ksym. */ 18339 ret = btf_resolve_size(btf, t, &tsize); 18340 if (IS_ERR(ret)) { 18341 tname = btf_name_by_offset(btf, t->name_off); 18342 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 18343 tname, PTR_ERR(ret)); 18344 err = -EINVAL; 18345 goto err_put; 18346 } 18347 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 18348 aux->btf_var.mem_size = tsize; 18349 } else { 18350 aux->btf_var.reg_type = PTR_TO_BTF_ID; 18351 aux->btf_var.btf = btf; 18352 aux->btf_var.btf_id = type; 18353 } 18354 check_btf: 18355 /* check whether we recorded this BTF (and maybe module) already */ 18356 for (i = 0; i < env->used_btf_cnt; i++) { 18357 if (env->used_btfs[i].btf == btf) { 18358 btf_put(btf); 18359 return 0; 18360 } 18361 } 18362 18363 if (env->used_btf_cnt >= MAX_USED_BTFS) { 18364 err = -E2BIG; 18365 goto err_put; 18366 } 18367 18368 btf_mod = &env->used_btfs[env->used_btf_cnt]; 18369 btf_mod->btf = btf; 18370 btf_mod->module = NULL; 18371 18372 /* if we reference variables from kernel module, bump its refcount */ 18373 if (btf_is_module(btf)) { 18374 btf_mod->module = btf_try_get_module(btf); 18375 if (!btf_mod->module) { 18376 err = -ENXIO; 18377 goto err_put; 18378 } 18379 } 18380 18381 env->used_btf_cnt++; 18382 18383 return 0; 18384 err_put: 18385 btf_put(btf); 18386 return err; 18387 } 18388 18389 static bool is_tracing_prog_type(enum bpf_prog_type type) 18390 { 18391 switch (type) { 18392 case BPF_PROG_TYPE_KPROBE: 18393 case BPF_PROG_TYPE_TRACEPOINT: 18394 case BPF_PROG_TYPE_PERF_EVENT: 18395 case BPF_PROG_TYPE_RAW_TRACEPOINT: 18396 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 18397 return true; 18398 default: 18399 return false; 18400 } 18401 } 18402 18403 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 18404 struct bpf_map *map, 18405 struct bpf_prog *prog) 18406 18407 { 18408 enum bpf_prog_type prog_type = resolve_prog_type(prog); 18409 18410 if (btf_record_has_field(map->record, BPF_LIST_HEAD) || 18411 btf_record_has_field(map->record, BPF_RB_ROOT)) { 18412 if (is_tracing_prog_type(prog_type)) { 18413 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); 18414 return -EINVAL; 18415 } 18416 } 18417 18418 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 18419 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 18420 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 18421 return -EINVAL; 18422 } 18423 18424 if (is_tracing_prog_type(prog_type)) { 18425 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 18426 return -EINVAL; 18427 } 18428 } 18429 18430 if (btf_record_has_field(map->record, BPF_TIMER)) { 18431 if (is_tracing_prog_type(prog_type)) { 18432 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 18433 return -EINVAL; 18434 } 18435 } 18436 18437 if (btf_record_has_field(map->record, BPF_WORKQUEUE)) { 18438 if (is_tracing_prog_type(prog_type)) { 18439 verbose(env, "tracing progs cannot use bpf_wq yet\n"); 18440 return -EINVAL; 18441 } 18442 } 18443 18444 if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && 18445 !bpf_offload_prog_map_match(prog, map)) { 18446 verbose(env, "offload device mismatch between prog and map\n"); 18447 return -EINVAL; 18448 } 18449 18450 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 18451 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 18452 return -EINVAL; 18453 } 18454 18455 if (prog->sleepable) 18456 switch (map->map_type) { 18457 case BPF_MAP_TYPE_HASH: 18458 case BPF_MAP_TYPE_LRU_HASH: 18459 case BPF_MAP_TYPE_ARRAY: 18460 case BPF_MAP_TYPE_PERCPU_HASH: 18461 case BPF_MAP_TYPE_PERCPU_ARRAY: 18462 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 18463 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 18464 case BPF_MAP_TYPE_HASH_OF_MAPS: 18465 case BPF_MAP_TYPE_RINGBUF: 18466 case BPF_MAP_TYPE_USER_RINGBUF: 18467 case BPF_MAP_TYPE_INODE_STORAGE: 18468 case BPF_MAP_TYPE_SK_STORAGE: 18469 case BPF_MAP_TYPE_TASK_STORAGE: 18470 case BPF_MAP_TYPE_CGRP_STORAGE: 18471 case BPF_MAP_TYPE_QUEUE: 18472 case BPF_MAP_TYPE_STACK: 18473 case BPF_MAP_TYPE_ARENA: 18474 break; 18475 default: 18476 verbose(env, 18477 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 18478 return -EINVAL; 18479 } 18480 18481 return 0; 18482 } 18483 18484 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 18485 { 18486 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 18487 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 18488 } 18489 18490 /* find and rewrite pseudo imm in ld_imm64 instructions: 18491 * 18492 * 1. if it accesses map FD, replace it with actual map pointer. 18493 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 18494 * 18495 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 18496 */ 18497 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 18498 { 18499 struct bpf_insn *insn = env->prog->insnsi; 18500 int insn_cnt = env->prog->len; 18501 int i, j, err; 18502 18503 err = bpf_prog_calc_tag(env->prog); 18504 if (err) 18505 return err; 18506 18507 for (i = 0; i < insn_cnt; i++, insn++) { 18508 if (BPF_CLASS(insn->code) == BPF_LDX && 18509 ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) || 18510 insn->imm != 0)) { 18511 verbose(env, "BPF_LDX uses reserved fields\n"); 18512 return -EINVAL; 18513 } 18514 18515 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 18516 struct bpf_insn_aux_data *aux; 18517 struct bpf_map *map; 18518 struct fd f; 18519 u64 addr; 18520 u32 fd; 18521 18522 if (i == insn_cnt - 1 || insn[1].code != 0 || 18523 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 18524 insn[1].off != 0) { 18525 verbose(env, "invalid bpf_ld_imm64 insn\n"); 18526 return -EINVAL; 18527 } 18528 18529 if (insn[0].src_reg == 0) 18530 /* valid generic load 64-bit imm */ 18531 goto next_insn; 18532 18533 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 18534 aux = &env->insn_aux_data[i]; 18535 err = check_pseudo_btf_id(env, insn, aux); 18536 if (err) 18537 return err; 18538 goto next_insn; 18539 } 18540 18541 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 18542 aux = &env->insn_aux_data[i]; 18543 aux->ptr_type = PTR_TO_FUNC; 18544 goto next_insn; 18545 } 18546 18547 /* In final convert_pseudo_ld_imm64() step, this is 18548 * converted into regular 64-bit imm load insn. 18549 */ 18550 switch (insn[0].src_reg) { 18551 case BPF_PSEUDO_MAP_VALUE: 18552 case BPF_PSEUDO_MAP_IDX_VALUE: 18553 break; 18554 case BPF_PSEUDO_MAP_FD: 18555 case BPF_PSEUDO_MAP_IDX: 18556 if (insn[1].imm == 0) 18557 break; 18558 fallthrough; 18559 default: 18560 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 18561 return -EINVAL; 18562 } 18563 18564 switch (insn[0].src_reg) { 18565 case BPF_PSEUDO_MAP_IDX_VALUE: 18566 case BPF_PSEUDO_MAP_IDX: 18567 if (bpfptr_is_null(env->fd_array)) { 18568 verbose(env, "fd_idx without fd_array is invalid\n"); 18569 return -EPROTO; 18570 } 18571 if (copy_from_bpfptr_offset(&fd, env->fd_array, 18572 insn[0].imm * sizeof(fd), 18573 sizeof(fd))) 18574 return -EFAULT; 18575 break; 18576 default: 18577 fd = insn[0].imm; 18578 break; 18579 } 18580 18581 f = fdget(fd); 18582 map = __bpf_map_get(f); 18583 if (IS_ERR(map)) { 18584 verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); 18585 return PTR_ERR(map); 18586 } 18587 18588 err = check_map_prog_compatibility(env, map, env->prog); 18589 if (err) { 18590 fdput(f); 18591 return err; 18592 } 18593 18594 aux = &env->insn_aux_data[i]; 18595 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 18596 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 18597 addr = (unsigned long)map; 18598 } else { 18599 u32 off = insn[1].imm; 18600 18601 if (off >= BPF_MAX_VAR_OFF) { 18602 verbose(env, "direct value offset of %u is not allowed\n", off); 18603 fdput(f); 18604 return -EINVAL; 18605 } 18606 18607 if (!map->ops->map_direct_value_addr) { 18608 verbose(env, "no direct value access support for this map type\n"); 18609 fdput(f); 18610 return -EINVAL; 18611 } 18612 18613 err = map->ops->map_direct_value_addr(map, &addr, off); 18614 if (err) { 18615 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 18616 map->value_size, off); 18617 fdput(f); 18618 return err; 18619 } 18620 18621 aux->map_off = off; 18622 addr += off; 18623 } 18624 18625 insn[0].imm = (u32)addr; 18626 insn[1].imm = addr >> 32; 18627 18628 /* check whether we recorded this map already */ 18629 for (j = 0; j < env->used_map_cnt; j++) { 18630 if (env->used_maps[j] == map) { 18631 aux->map_index = j; 18632 fdput(f); 18633 goto next_insn; 18634 } 18635 } 18636 18637 if (env->used_map_cnt >= MAX_USED_MAPS) { 18638 verbose(env, "The total number of maps per program has reached the limit of %u\n", 18639 MAX_USED_MAPS); 18640 fdput(f); 18641 return -E2BIG; 18642 } 18643 18644 if (env->prog->sleepable) 18645 atomic64_inc(&map->sleepable_refcnt); 18646 /* hold the map. If the program is rejected by verifier, 18647 * the map will be released by release_maps() or it 18648 * will be used by the valid program until it's unloaded 18649 * and all maps are released in bpf_free_used_maps() 18650 */ 18651 bpf_map_inc(map); 18652 18653 aux->map_index = env->used_map_cnt; 18654 env->used_maps[env->used_map_cnt++] = map; 18655 18656 if (bpf_map_is_cgroup_storage(map) && 18657 bpf_cgroup_storage_assign(env->prog->aux, map)) { 18658 verbose(env, "only one cgroup storage of each type is allowed\n"); 18659 fdput(f); 18660 return -EBUSY; 18661 } 18662 if (map->map_type == BPF_MAP_TYPE_ARENA) { 18663 if (env->prog->aux->arena) { 18664 verbose(env, "Only one arena per program\n"); 18665 fdput(f); 18666 return -EBUSY; 18667 } 18668 if (!env->allow_ptr_leaks || !env->bpf_capable) { 18669 verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); 18670 fdput(f); 18671 return -EPERM; 18672 } 18673 if (!env->prog->jit_requested) { 18674 verbose(env, "JIT is required to use arena\n"); 18675 fdput(f); 18676 return -EOPNOTSUPP; 18677 } 18678 if (!bpf_jit_supports_arena()) { 18679 verbose(env, "JIT doesn't support arena\n"); 18680 fdput(f); 18681 return -EOPNOTSUPP; 18682 } 18683 env->prog->aux->arena = (void *)map; 18684 if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { 18685 verbose(env, "arena's user address must be set via map_extra or mmap()\n"); 18686 fdput(f); 18687 return -EINVAL; 18688 } 18689 } 18690 18691 fdput(f); 18692 next_insn: 18693 insn++; 18694 i++; 18695 continue; 18696 } 18697 18698 /* Basic sanity check before we invest more work here. */ 18699 if (!bpf_opcode_in_insntable(insn->code)) { 18700 verbose(env, "unknown opcode %02x\n", insn->code); 18701 return -EINVAL; 18702 } 18703 } 18704 18705 /* now all pseudo BPF_LD_IMM64 instructions load valid 18706 * 'struct bpf_map *' into a register instead of user map_fd. 18707 * These pointers will be used later by verifier to validate map access. 18708 */ 18709 return 0; 18710 } 18711 18712 /* drop refcnt of maps used by the rejected program */ 18713 static void release_maps(struct bpf_verifier_env *env) 18714 { 18715 __bpf_free_used_maps(env->prog->aux, env->used_maps, 18716 env->used_map_cnt); 18717 } 18718 18719 /* drop refcnt of maps used by the rejected program */ 18720 static void release_btfs(struct bpf_verifier_env *env) 18721 { 18722 __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt); 18723 } 18724 18725 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 18726 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 18727 { 18728 struct bpf_insn *insn = env->prog->insnsi; 18729 int insn_cnt = env->prog->len; 18730 int i; 18731 18732 for (i = 0; i < insn_cnt; i++, insn++) { 18733 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 18734 continue; 18735 if (insn->src_reg == BPF_PSEUDO_FUNC) 18736 continue; 18737 insn->src_reg = 0; 18738 } 18739 } 18740 18741 /* single env->prog->insni[off] instruction was replaced with the range 18742 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 18743 * [0, off) and [off, end) to new locations, so the patched range stays zero 18744 */ 18745 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 18746 struct bpf_insn_aux_data *new_data, 18747 struct bpf_prog *new_prog, u32 off, u32 cnt) 18748 { 18749 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 18750 struct bpf_insn *insn = new_prog->insnsi; 18751 u32 old_seen = old_data[off].seen; 18752 u32 prog_len; 18753 int i; 18754 18755 /* aux info at OFF always needs adjustment, no matter fast path 18756 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 18757 * original insn at old prog. 18758 */ 18759 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 18760 18761 if (cnt == 1) 18762 return; 18763 prog_len = new_prog->len; 18764 18765 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 18766 memcpy(new_data + off + cnt - 1, old_data + off, 18767 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 18768 for (i = off; i < off + cnt - 1; i++) { 18769 /* Expand insni[off]'s seen count to the patched range. */ 18770 new_data[i].seen = old_seen; 18771 new_data[i].zext_dst = insn_has_def32(env, insn + i); 18772 } 18773 env->insn_aux_data = new_data; 18774 vfree(old_data); 18775 } 18776 18777 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 18778 { 18779 int i; 18780 18781 if (len == 1) 18782 return; 18783 /* NOTE: fake 'exit' subprog should be updated as well. */ 18784 for (i = 0; i <= env->subprog_cnt; i++) { 18785 if (env->subprog_info[i].start <= off) 18786 continue; 18787 env->subprog_info[i].start += len - 1; 18788 } 18789 } 18790 18791 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 18792 { 18793 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 18794 int i, sz = prog->aux->size_poke_tab; 18795 struct bpf_jit_poke_descriptor *desc; 18796 18797 for (i = 0; i < sz; i++) { 18798 desc = &tab[i]; 18799 if (desc->insn_idx <= off) 18800 continue; 18801 desc->insn_idx += len - 1; 18802 } 18803 } 18804 18805 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 18806 const struct bpf_insn *patch, u32 len) 18807 { 18808 struct bpf_prog *new_prog; 18809 struct bpf_insn_aux_data *new_data = NULL; 18810 18811 if (len > 1) { 18812 new_data = vzalloc(array_size(env->prog->len + len - 1, 18813 sizeof(struct bpf_insn_aux_data))); 18814 if (!new_data) 18815 return NULL; 18816 } 18817 18818 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 18819 if (IS_ERR(new_prog)) { 18820 if (PTR_ERR(new_prog) == -ERANGE) 18821 verbose(env, 18822 "insn %d cannot be patched due to 16-bit range\n", 18823 env->insn_aux_data[off].orig_idx); 18824 vfree(new_data); 18825 return NULL; 18826 } 18827 adjust_insn_aux_data(env, new_data, new_prog, off, len); 18828 adjust_subprog_starts(env, off, len); 18829 adjust_poke_descs(new_prog, off, len); 18830 return new_prog; 18831 } 18832 18833 /* 18834 * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the 18835 * jump offset by 'delta'. 18836 */ 18837 static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta) 18838 { 18839 struct bpf_insn *insn = prog->insnsi; 18840 u32 insn_cnt = prog->len, i; 18841 18842 for (i = 0; i < insn_cnt; i++, insn++) { 18843 u8 code = insn->code; 18844 18845 if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || 18846 BPF_OP(code) == BPF_CALL || BPF_OP(code) == BPF_EXIT) 18847 continue; 18848 18849 if (insn->code == (BPF_JMP32 | BPF_JA)) { 18850 if (i + 1 + insn->imm != tgt_idx) 18851 continue; 18852 if (signed_add32_overflows(insn->imm, delta)) 18853 return -ERANGE; 18854 insn->imm += delta; 18855 } else { 18856 if (i + 1 + insn->off != tgt_idx) 18857 continue; 18858 if (signed_add16_overflows(insn->imm, delta)) 18859 return -ERANGE; 18860 insn->off += delta; 18861 } 18862 } 18863 return 0; 18864 } 18865 18866 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 18867 u32 off, u32 cnt) 18868 { 18869 int i, j; 18870 18871 /* find first prog starting at or after off (first to remove) */ 18872 for (i = 0; i < env->subprog_cnt; i++) 18873 if (env->subprog_info[i].start >= off) 18874 break; 18875 /* find first prog starting at or after off + cnt (first to stay) */ 18876 for (j = i; j < env->subprog_cnt; j++) 18877 if (env->subprog_info[j].start >= off + cnt) 18878 break; 18879 /* if j doesn't start exactly at off + cnt, we are just removing 18880 * the front of previous prog 18881 */ 18882 if (env->subprog_info[j].start != off + cnt) 18883 j--; 18884 18885 if (j > i) { 18886 struct bpf_prog_aux *aux = env->prog->aux; 18887 int move; 18888 18889 /* move fake 'exit' subprog as well */ 18890 move = env->subprog_cnt + 1 - j; 18891 18892 memmove(env->subprog_info + i, 18893 env->subprog_info + j, 18894 sizeof(*env->subprog_info) * move); 18895 env->subprog_cnt -= j - i; 18896 18897 /* remove func_info */ 18898 if (aux->func_info) { 18899 move = aux->func_info_cnt - j; 18900 18901 memmove(aux->func_info + i, 18902 aux->func_info + j, 18903 sizeof(*aux->func_info) * move); 18904 aux->func_info_cnt -= j - i; 18905 /* func_info->insn_off is set after all code rewrites, 18906 * in adjust_btf_func() - no need to adjust 18907 */ 18908 } 18909 } else { 18910 /* convert i from "first prog to remove" to "first to adjust" */ 18911 if (env->subprog_info[i].start == off) 18912 i++; 18913 } 18914 18915 /* update fake 'exit' subprog as well */ 18916 for (; i <= env->subprog_cnt; i++) 18917 env->subprog_info[i].start -= cnt; 18918 18919 return 0; 18920 } 18921 18922 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 18923 u32 cnt) 18924 { 18925 struct bpf_prog *prog = env->prog; 18926 u32 i, l_off, l_cnt, nr_linfo; 18927 struct bpf_line_info *linfo; 18928 18929 nr_linfo = prog->aux->nr_linfo; 18930 if (!nr_linfo) 18931 return 0; 18932 18933 linfo = prog->aux->linfo; 18934 18935 /* find first line info to remove, count lines to be removed */ 18936 for (i = 0; i < nr_linfo; i++) 18937 if (linfo[i].insn_off >= off) 18938 break; 18939 18940 l_off = i; 18941 l_cnt = 0; 18942 for (; i < nr_linfo; i++) 18943 if (linfo[i].insn_off < off + cnt) 18944 l_cnt++; 18945 else 18946 break; 18947 18948 /* First live insn doesn't match first live linfo, it needs to "inherit" 18949 * last removed linfo. prog is already modified, so prog->len == off 18950 * means no live instructions after (tail of the program was removed). 18951 */ 18952 if (prog->len != off && l_cnt && 18953 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 18954 l_cnt--; 18955 linfo[--i].insn_off = off + cnt; 18956 } 18957 18958 /* remove the line info which refer to the removed instructions */ 18959 if (l_cnt) { 18960 memmove(linfo + l_off, linfo + i, 18961 sizeof(*linfo) * (nr_linfo - i)); 18962 18963 prog->aux->nr_linfo -= l_cnt; 18964 nr_linfo = prog->aux->nr_linfo; 18965 } 18966 18967 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 18968 for (i = l_off; i < nr_linfo; i++) 18969 linfo[i].insn_off -= cnt; 18970 18971 /* fix up all subprogs (incl. 'exit') which start >= off */ 18972 for (i = 0; i <= env->subprog_cnt; i++) 18973 if (env->subprog_info[i].linfo_idx > l_off) { 18974 /* program may have started in the removed region but 18975 * may not be fully removed 18976 */ 18977 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 18978 env->subprog_info[i].linfo_idx -= l_cnt; 18979 else 18980 env->subprog_info[i].linfo_idx = l_off; 18981 } 18982 18983 return 0; 18984 } 18985 18986 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 18987 { 18988 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 18989 unsigned int orig_prog_len = env->prog->len; 18990 int err; 18991 18992 if (bpf_prog_is_offloaded(env->prog->aux)) 18993 bpf_prog_offload_remove_insns(env, off, cnt); 18994 18995 err = bpf_remove_insns(env->prog, off, cnt); 18996 if (err) 18997 return err; 18998 18999 err = adjust_subprog_starts_after_remove(env, off, cnt); 19000 if (err) 19001 return err; 19002 19003 err = bpf_adj_linfo_after_remove(env, off, cnt); 19004 if (err) 19005 return err; 19006 19007 memmove(aux_data + off, aux_data + off + cnt, 19008 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 19009 19010 return 0; 19011 } 19012 19013 /* The verifier does more data flow analysis than llvm and will not 19014 * explore branches that are dead at run time. Malicious programs can 19015 * have dead code too. Therefore replace all dead at-run-time code 19016 * with 'ja -1'. 19017 * 19018 * Just nops are not optimal, e.g. if they would sit at the end of the 19019 * program and through another bug we would manage to jump there, then 19020 * we'd execute beyond program memory otherwise. Returning exception 19021 * code also wouldn't work since we can have subprogs where the dead 19022 * code could be located. 19023 */ 19024 static void sanitize_dead_code(struct bpf_verifier_env *env) 19025 { 19026 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 19027 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 19028 struct bpf_insn *insn = env->prog->insnsi; 19029 const int insn_cnt = env->prog->len; 19030 int i; 19031 19032 for (i = 0; i < insn_cnt; i++) { 19033 if (aux_data[i].seen) 19034 continue; 19035 memcpy(insn + i, &trap, sizeof(trap)); 19036 aux_data[i].zext_dst = false; 19037 } 19038 } 19039 19040 static bool insn_is_cond_jump(u8 code) 19041 { 19042 u8 op; 19043 19044 op = BPF_OP(code); 19045 if (BPF_CLASS(code) == BPF_JMP32) 19046 return op != BPF_JA; 19047 19048 if (BPF_CLASS(code) != BPF_JMP) 19049 return false; 19050 19051 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 19052 } 19053 19054 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 19055 { 19056 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 19057 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 19058 struct bpf_insn *insn = env->prog->insnsi; 19059 const int insn_cnt = env->prog->len; 19060 int i; 19061 19062 for (i = 0; i < insn_cnt; i++, insn++) { 19063 if (!insn_is_cond_jump(insn->code)) 19064 continue; 19065 19066 if (!aux_data[i + 1].seen) 19067 ja.off = insn->off; 19068 else if (!aux_data[i + 1 + insn->off].seen) 19069 ja.off = 0; 19070 else 19071 continue; 19072 19073 if (bpf_prog_is_offloaded(env->prog->aux)) 19074 bpf_prog_offload_replace_insn(env, i, &ja); 19075 19076 memcpy(insn, &ja, sizeof(ja)); 19077 } 19078 } 19079 19080 static int opt_remove_dead_code(struct bpf_verifier_env *env) 19081 { 19082 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 19083 int insn_cnt = env->prog->len; 19084 int i, err; 19085 19086 for (i = 0; i < insn_cnt; i++) { 19087 int j; 19088 19089 j = 0; 19090 while (i + j < insn_cnt && !aux_data[i + j].seen) 19091 j++; 19092 if (!j) 19093 continue; 19094 19095 err = verifier_remove_insns(env, i, j); 19096 if (err) 19097 return err; 19098 insn_cnt = env->prog->len; 19099 } 19100 19101 return 0; 19102 } 19103 19104 static int opt_remove_nops(struct bpf_verifier_env *env) 19105 { 19106 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 19107 struct bpf_insn *insn = env->prog->insnsi; 19108 int insn_cnt = env->prog->len; 19109 int i, err; 19110 19111 for (i = 0; i < insn_cnt; i++) { 19112 if (memcmp(&insn[i], &ja, sizeof(ja))) 19113 continue; 19114 19115 err = verifier_remove_insns(env, i, 1); 19116 if (err) 19117 return err; 19118 insn_cnt--; 19119 i--; 19120 } 19121 19122 return 0; 19123 } 19124 19125 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 19126 const union bpf_attr *attr) 19127 { 19128 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 19129 struct bpf_insn_aux_data *aux = env->insn_aux_data; 19130 int i, patch_len, delta = 0, len = env->prog->len; 19131 struct bpf_insn *insns = env->prog->insnsi; 19132 struct bpf_prog *new_prog; 19133 bool rnd_hi32; 19134 19135 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 19136 zext_patch[1] = BPF_ZEXT_REG(0); 19137 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 19138 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 19139 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 19140 for (i = 0; i < len; i++) { 19141 int adj_idx = i + delta; 19142 struct bpf_insn insn; 19143 int load_reg; 19144 19145 insn = insns[adj_idx]; 19146 load_reg = insn_def_regno(&insn); 19147 if (!aux[adj_idx].zext_dst) { 19148 u8 code, class; 19149 u32 imm_rnd; 19150 19151 if (!rnd_hi32) 19152 continue; 19153 19154 code = insn.code; 19155 class = BPF_CLASS(code); 19156 if (load_reg == -1) 19157 continue; 19158 19159 /* NOTE: arg "reg" (the fourth one) is only used for 19160 * BPF_STX + SRC_OP, so it is safe to pass NULL 19161 * here. 19162 */ 19163 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 19164 if (class == BPF_LD && 19165 BPF_MODE(code) == BPF_IMM) 19166 i++; 19167 continue; 19168 } 19169 19170 /* ctx load could be transformed into wider load. */ 19171 if (class == BPF_LDX && 19172 aux[adj_idx].ptr_type == PTR_TO_CTX) 19173 continue; 19174 19175 imm_rnd = get_random_u32(); 19176 rnd_hi32_patch[0] = insn; 19177 rnd_hi32_patch[1].imm = imm_rnd; 19178 rnd_hi32_patch[3].dst_reg = load_reg; 19179 patch = rnd_hi32_patch; 19180 patch_len = 4; 19181 goto apply_patch_buffer; 19182 } 19183 19184 /* Add in an zero-extend instruction if a) the JIT has requested 19185 * it or b) it's a CMPXCHG. 19186 * 19187 * The latter is because: BPF_CMPXCHG always loads a value into 19188 * R0, therefore always zero-extends. However some archs' 19189 * equivalent instruction only does this load when the 19190 * comparison is successful. This detail of CMPXCHG is 19191 * orthogonal to the general zero-extension behaviour of the 19192 * CPU, so it's treated independently of bpf_jit_needs_zext. 19193 */ 19194 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 19195 continue; 19196 19197 /* Zero-extension is done by the caller. */ 19198 if (bpf_pseudo_kfunc_call(&insn)) 19199 continue; 19200 19201 if (WARN_ON(load_reg == -1)) { 19202 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 19203 return -EFAULT; 19204 } 19205 19206 zext_patch[0] = insn; 19207 zext_patch[1].dst_reg = load_reg; 19208 zext_patch[1].src_reg = load_reg; 19209 patch = zext_patch; 19210 patch_len = 2; 19211 apply_patch_buffer: 19212 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 19213 if (!new_prog) 19214 return -ENOMEM; 19215 env->prog = new_prog; 19216 insns = new_prog->insnsi; 19217 aux = env->insn_aux_data; 19218 delta += patch_len - 1; 19219 } 19220 19221 return 0; 19222 } 19223 19224 /* convert load instructions that access fields of a context type into a 19225 * sequence of instructions that access fields of the underlying structure: 19226 * struct __sk_buff -> struct sk_buff 19227 * struct bpf_sock_ops -> struct sock 19228 */ 19229 static int convert_ctx_accesses(struct bpf_verifier_env *env) 19230 { 19231 const struct bpf_verifier_ops *ops = env->ops; 19232 int i, cnt, size, ctx_field_size, delta = 0; 19233 const int insn_cnt = env->prog->len; 19234 struct bpf_insn insn_buf[16], *insn; 19235 u32 target_size, size_default, off; 19236 struct bpf_prog *new_prog; 19237 enum bpf_access_type type; 19238 bool is_narrower_load; 19239 19240 if (ops->gen_prologue || env->seen_direct_write) { 19241 if (!ops->gen_prologue) { 19242 verbose(env, "bpf verifier is misconfigured\n"); 19243 return -EINVAL; 19244 } 19245 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 19246 env->prog); 19247 if (cnt >= ARRAY_SIZE(insn_buf)) { 19248 verbose(env, "bpf verifier is misconfigured\n"); 19249 return -EINVAL; 19250 } else if (cnt) { 19251 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 19252 if (!new_prog) 19253 return -ENOMEM; 19254 19255 env->prog = new_prog; 19256 delta += cnt - 1; 19257 } 19258 } 19259 19260 if (bpf_prog_is_offloaded(env->prog->aux)) 19261 return 0; 19262 19263 insn = env->prog->insnsi + delta; 19264 19265 for (i = 0; i < insn_cnt; i++, insn++) { 19266 bpf_convert_ctx_access_t convert_ctx_access; 19267 u8 mode; 19268 19269 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 19270 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 19271 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 19272 insn->code == (BPF_LDX | BPF_MEM | BPF_DW) || 19273 insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) || 19274 insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) || 19275 insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) { 19276 type = BPF_READ; 19277 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 19278 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 19279 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 19280 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 19281 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 19282 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 19283 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 19284 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 19285 type = BPF_WRITE; 19286 } else if ((insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) || 19287 insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) && 19288 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { 19289 insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); 19290 env->prog->aux->num_exentries++; 19291 continue; 19292 } else { 19293 continue; 19294 } 19295 19296 if (type == BPF_WRITE && 19297 env->insn_aux_data[i + delta].sanitize_stack_spill) { 19298 struct bpf_insn patch[] = { 19299 *insn, 19300 BPF_ST_NOSPEC(), 19301 }; 19302 19303 cnt = ARRAY_SIZE(patch); 19304 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 19305 if (!new_prog) 19306 return -ENOMEM; 19307 19308 delta += cnt - 1; 19309 env->prog = new_prog; 19310 insn = new_prog->insnsi + i + delta; 19311 continue; 19312 } 19313 19314 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 19315 case PTR_TO_CTX: 19316 if (!ops->convert_ctx_access) 19317 continue; 19318 convert_ctx_access = ops->convert_ctx_access; 19319 break; 19320 case PTR_TO_SOCKET: 19321 case PTR_TO_SOCK_COMMON: 19322 convert_ctx_access = bpf_sock_convert_ctx_access; 19323 break; 19324 case PTR_TO_TCP_SOCK: 19325 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 19326 break; 19327 case PTR_TO_XDP_SOCK: 19328 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 19329 break; 19330 case PTR_TO_BTF_ID: 19331 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 19332 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 19333 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot 19334 * be said once it is marked PTR_UNTRUSTED, hence we must handle 19335 * any faults for loads into such types. BPF_WRITE is disallowed 19336 * for this case. 19337 */ 19338 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 19339 if (type == BPF_READ) { 19340 if (BPF_MODE(insn->code) == BPF_MEM) 19341 insn->code = BPF_LDX | BPF_PROBE_MEM | 19342 BPF_SIZE((insn)->code); 19343 else 19344 insn->code = BPF_LDX | BPF_PROBE_MEMSX | 19345 BPF_SIZE((insn)->code); 19346 env->prog->aux->num_exentries++; 19347 } 19348 continue; 19349 case PTR_TO_ARENA: 19350 if (BPF_MODE(insn->code) == BPF_MEMSX) { 19351 verbose(env, "sign extending loads from arena are not supported yet\n"); 19352 return -EOPNOTSUPP; 19353 } 19354 insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code); 19355 env->prog->aux->num_exentries++; 19356 continue; 19357 default: 19358 continue; 19359 } 19360 19361 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 19362 size = BPF_LDST_BYTES(insn); 19363 mode = BPF_MODE(insn->code); 19364 19365 /* If the read access is a narrower load of the field, 19366 * convert to a 4/8-byte load, to minimum program type specific 19367 * convert_ctx_access changes. If conversion is successful, 19368 * we will apply proper mask to the result. 19369 */ 19370 is_narrower_load = size < ctx_field_size; 19371 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 19372 off = insn->off; 19373 if (is_narrower_load) { 19374 u8 size_code; 19375 19376 if (type == BPF_WRITE) { 19377 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 19378 return -EINVAL; 19379 } 19380 19381 size_code = BPF_H; 19382 if (ctx_field_size == 4) 19383 size_code = BPF_W; 19384 else if (ctx_field_size == 8) 19385 size_code = BPF_DW; 19386 19387 insn->off = off & ~(size_default - 1); 19388 insn->code = BPF_LDX | BPF_MEM | size_code; 19389 } 19390 19391 target_size = 0; 19392 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 19393 &target_size); 19394 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 19395 (ctx_field_size && !target_size)) { 19396 verbose(env, "bpf verifier is misconfigured\n"); 19397 return -EINVAL; 19398 } 19399 19400 if (is_narrower_load && size < target_size) { 19401 u8 shift = bpf_ctx_narrow_access_offset( 19402 off, size, size_default) * 8; 19403 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 19404 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 19405 return -EINVAL; 19406 } 19407 if (ctx_field_size <= 4) { 19408 if (shift) 19409 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 19410 insn->dst_reg, 19411 shift); 19412 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 19413 (1 << size * 8) - 1); 19414 } else { 19415 if (shift) 19416 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 19417 insn->dst_reg, 19418 shift); 19419 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 19420 (1ULL << size * 8) - 1); 19421 } 19422 } 19423 if (mode == BPF_MEMSX) 19424 insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, 19425 insn->dst_reg, insn->dst_reg, 19426 size * 8, 0); 19427 19428 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 19429 if (!new_prog) 19430 return -ENOMEM; 19431 19432 delta += cnt - 1; 19433 19434 /* keep walking new program and skip insns we just inserted */ 19435 env->prog = new_prog; 19436 insn = new_prog->insnsi + i + delta; 19437 } 19438 19439 return 0; 19440 } 19441 19442 static int jit_subprogs(struct bpf_verifier_env *env) 19443 { 19444 struct bpf_prog *prog = env->prog, **func, *tmp; 19445 int i, j, subprog_start, subprog_end = 0, len, subprog; 19446 struct bpf_map *map_ptr; 19447 struct bpf_insn *insn; 19448 void *old_bpf_func; 19449 int err, num_exentries; 19450 19451 if (env->subprog_cnt <= 1) 19452 return 0; 19453 19454 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 19455 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 19456 continue; 19457 19458 /* Upon error here we cannot fall back to interpreter but 19459 * need a hard reject of the program. Thus -EFAULT is 19460 * propagated in any case. 19461 */ 19462 subprog = find_subprog(env, i + insn->imm + 1); 19463 if (subprog < 0) { 19464 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 19465 i + insn->imm + 1); 19466 return -EFAULT; 19467 } 19468 /* temporarily remember subprog id inside insn instead of 19469 * aux_data, since next loop will split up all insns into funcs 19470 */ 19471 insn->off = subprog; 19472 /* remember original imm in case JIT fails and fallback 19473 * to interpreter will be needed 19474 */ 19475 env->insn_aux_data[i].call_imm = insn->imm; 19476 /* point imm to __bpf_call_base+1 from JITs point of view */ 19477 insn->imm = 1; 19478 if (bpf_pseudo_func(insn)) { 19479 #if defined(MODULES_VADDR) 19480 u64 addr = MODULES_VADDR; 19481 #else 19482 u64 addr = VMALLOC_START; 19483 #endif 19484 /* jit (e.g. x86_64) may emit fewer instructions 19485 * if it learns a u32 imm is the same as a u64 imm. 19486 * Set close enough to possible prog address. 19487 */ 19488 insn[0].imm = (u32)addr; 19489 insn[1].imm = addr >> 32; 19490 } 19491 } 19492 19493 err = bpf_prog_alloc_jited_linfo(prog); 19494 if (err) 19495 goto out_undo_insn; 19496 19497 err = -ENOMEM; 19498 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 19499 if (!func) 19500 goto out_undo_insn; 19501 19502 for (i = 0; i < env->subprog_cnt; i++) { 19503 subprog_start = subprog_end; 19504 subprog_end = env->subprog_info[i + 1].start; 19505 19506 len = subprog_end - subprog_start; 19507 /* bpf_prog_run() doesn't call subprogs directly, 19508 * hence main prog stats include the runtime of subprogs. 19509 * subprogs don't have IDs and not reachable via prog_get_next_id 19510 * func[i]->stats will never be accessed and stays NULL 19511 */ 19512 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 19513 if (!func[i]) 19514 goto out_free; 19515 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 19516 len * sizeof(struct bpf_insn)); 19517 func[i]->type = prog->type; 19518 func[i]->len = len; 19519 if (bpf_prog_calc_tag(func[i])) 19520 goto out_free; 19521 func[i]->is_func = 1; 19522 func[i]->sleepable = prog->sleepable; 19523 func[i]->aux->func_idx = i; 19524 /* Below members will be freed only at prog->aux */ 19525 func[i]->aux->btf = prog->aux->btf; 19526 func[i]->aux->func_info = prog->aux->func_info; 19527 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 19528 func[i]->aux->poke_tab = prog->aux->poke_tab; 19529 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 19530 19531 for (j = 0; j < prog->aux->size_poke_tab; j++) { 19532 struct bpf_jit_poke_descriptor *poke; 19533 19534 poke = &prog->aux->poke_tab[j]; 19535 if (poke->insn_idx < subprog_end && 19536 poke->insn_idx >= subprog_start) 19537 poke->aux = func[i]->aux; 19538 } 19539 19540 func[i]->aux->name[0] = 'F'; 19541 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 19542 func[i]->jit_requested = 1; 19543 func[i]->blinding_requested = prog->blinding_requested; 19544 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 19545 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 19546 func[i]->aux->linfo = prog->aux->linfo; 19547 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 19548 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 19549 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 19550 func[i]->aux->arena = prog->aux->arena; 19551 num_exentries = 0; 19552 insn = func[i]->insnsi; 19553 for (j = 0; j < func[i]->len; j++, insn++) { 19554 if (BPF_CLASS(insn->code) == BPF_LDX && 19555 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 19556 BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 19557 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) 19558 num_exentries++; 19559 if ((BPF_CLASS(insn->code) == BPF_STX || 19560 BPF_CLASS(insn->code) == BPF_ST) && 19561 BPF_MODE(insn->code) == BPF_PROBE_MEM32) 19562 num_exentries++; 19563 if (BPF_CLASS(insn->code) == BPF_STX && 19564 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 19565 num_exentries++; 19566 } 19567 func[i]->aux->num_exentries = num_exentries; 19568 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 19569 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; 19570 if (!i) 19571 func[i]->aux->exception_boundary = env->seen_exception; 19572 func[i] = bpf_int_jit_compile(func[i]); 19573 if (!func[i]->jited) { 19574 err = -ENOTSUPP; 19575 goto out_free; 19576 } 19577 cond_resched(); 19578 } 19579 19580 /* at this point all bpf functions were successfully JITed 19581 * now populate all bpf_calls with correct addresses and 19582 * run last pass of JIT 19583 */ 19584 for (i = 0; i < env->subprog_cnt; i++) { 19585 insn = func[i]->insnsi; 19586 for (j = 0; j < func[i]->len; j++, insn++) { 19587 if (bpf_pseudo_func(insn)) { 19588 subprog = insn->off; 19589 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 19590 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 19591 continue; 19592 } 19593 if (!bpf_pseudo_call(insn)) 19594 continue; 19595 subprog = insn->off; 19596 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 19597 } 19598 19599 /* we use the aux data to keep a list of the start addresses 19600 * of the JITed images for each function in the program 19601 * 19602 * for some architectures, such as powerpc64, the imm field 19603 * might not be large enough to hold the offset of the start 19604 * address of the callee's JITed image from __bpf_call_base 19605 * 19606 * in such cases, we can lookup the start address of a callee 19607 * by using its subprog id, available from the off field of 19608 * the call instruction, as an index for this list 19609 */ 19610 func[i]->aux->func = func; 19611 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 19612 func[i]->aux->real_func_cnt = env->subprog_cnt; 19613 } 19614 for (i = 0; i < env->subprog_cnt; i++) { 19615 old_bpf_func = func[i]->bpf_func; 19616 tmp = bpf_int_jit_compile(func[i]); 19617 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 19618 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 19619 err = -ENOTSUPP; 19620 goto out_free; 19621 } 19622 cond_resched(); 19623 } 19624 19625 /* finally lock prog and jit images for all functions and 19626 * populate kallsysm. Begin at the first subprogram, since 19627 * bpf_prog_load will add the kallsyms for the main program. 19628 */ 19629 for (i = 1; i < env->subprog_cnt; i++) { 19630 err = bpf_prog_lock_ro(func[i]); 19631 if (err) 19632 goto out_free; 19633 } 19634 19635 for (i = 1; i < env->subprog_cnt; i++) 19636 bpf_prog_kallsyms_add(func[i]); 19637 19638 /* Last step: make now unused interpreter insns from main 19639 * prog consistent for later dump requests, so they can 19640 * later look the same as if they were interpreted only. 19641 */ 19642 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 19643 if (bpf_pseudo_func(insn)) { 19644 insn[0].imm = env->insn_aux_data[i].call_imm; 19645 insn[1].imm = insn->off; 19646 insn->off = 0; 19647 continue; 19648 } 19649 if (!bpf_pseudo_call(insn)) 19650 continue; 19651 insn->off = env->insn_aux_data[i].call_imm; 19652 subprog = find_subprog(env, i + insn->off + 1); 19653 insn->imm = subprog; 19654 } 19655 19656 prog->jited = 1; 19657 prog->bpf_func = func[0]->bpf_func; 19658 prog->jited_len = func[0]->jited_len; 19659 prog->aux->extable = func[0]->aux->extable; 19660 prog->aux->num_exentries = func[0]->aux->num_exentries; 19661 prog->aux->func = func; 19662 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; 19663 prog->aux->real_func_cnt = env->subprog_cnt; 19664 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; 19665 prog->aux->exception_boundary = func[0]->aux->exception_boundary; 19666 bpf_prog_jit_attempt_done(prog); 19667 return 0; 19668 out_free: 19669 /* We failed JIT'ing, so at this point we need to unregister poke 19670 * descriptors from subprogs, so that kernel is not attempting to 19671 * patch it anymore as we're freeing the subprog JIT memory. 19672 */ 19673 for (i = 0; i < prog->aux->size_poke_tab; i++) { 19674 map_ptr = prog->aux->poke_tab[i].tail_call.map; 19675 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 19676 } 19677 /* At this point we're guaranteed that poke descriptors are not 19678 * live anymore. We can just unlink its descriptor table as it's 19679 * released with the main prog. 19680 */ 19681 for (i = 0; i < env->subprog_cnt; i++) { 19682 if (!func[i]) 19683 continue; 19684 func[i]->aux->poke_tab = NULL; 19685 bpf_jit_free(func[i]); 19686 } 19687 kfree(func); 19688 out_undo_insn: 19689 /* cleanup main prog to be interpreted */ 19690 prog->jit_requested = 0; 19691 prog->blinding_requested = 0; 19692 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 19693 if (!bpf_pseudo_call(insn)) 19694 continue; 19695 insn->off = 0; 19696 insn->imm = env->insn_aux_data[i].call_imm; 19697 } 19698 bpf_prog_jit_attempt_done(prog); 19699 return err; 19700 } 19701 19702 static int fixup_call_args(struct bpf_verifier_env *env) 19703 { 19704 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 19705 struct bpf_prog *prog = env->prog; 19706 struct bpf_insn *insn = prog->insnsi; 19707 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 19708 int i, depth; 19709 #endif 19710 int err = 0; 19711 19712 if (env->prog->jit_requested && 19713 !bpf_prog_is_offloaded(env->prog->aux)) { 19714 err = jit_subprogs(env); 19715 if (err == 0) 19716 return 0; 19717 if (err == -EFAULT) 19718 return err; 19719 } 19720 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 19721 if (has_kfunc_call) { 19722 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 19723 return -EINVAL; 19724 } 19725 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 19726 /* When JIT fails the progs with bpf2bpf calls and tail_calls 19727 * have to be rejected, since interpreter doesn't support them yet. 19728 */ 19729 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 19730 return -EINVAL; 19731 } 19732 for (i = 0; i < prog->len; i++, insn++) { 19733 if (bpf_pseudo_func(insn)) { 19734 /* When JIT fails the progs with callback calls 19735 * have to be rejected, since interpreter doesn't support them yet. 19736 */ 19737 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 19738 return -EINVAL; 19739 } 19740 19741 if (!bpf_pseudo_call(insn)) 19742 continue; 19743 depth = get_callee_stack_depth(env, insn, i); 19744 if (depth < 0) 19745 return depth; 19746 bpf_patch_call_args(insn, depth); 19747 } 19748 err = 0; 19749 #endif 19750 return err; 19751 } 19752 19753 /* replace a generic kfunc with a specialized version if necessary */ 19754 static void specialize_kfunc(struct bpf_verifier_env *env, 19755 u32 func_id, u16 offset, unsigned long *addr) 19756 { 19757 struct bpf_prog *prog = env->prog; 19758 bool seen_direct_write; 19759 void *xdp_kfunc; 19760 bool is_rdonly; 19761 19762 if (bpf_dev_bound_kfunc_id(func_id)) { 19763 xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); 19764 if (xdp_kfunc) { 19765 *addr = (unsigned long)xdp_kfunc; 19766 return; 19767 } 19768 /* fallback to default kfunc when not supported by netdev */ 19769 } 19770 19771 if (offset) 19772 return; 19773 19774 if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { 19775 seen_direct_write = env->seen_direct_write; 19776 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); 19777 19778 if (is_rdonly) 19779 *addr = (unsigned long)bpf_dynptr_from_skb_rdonly; 19780 19781 /* restore env->seen_direct_write to its original value, since 19782 * may_access_direct_pkt_data mutates it 19783 */ 19784 env->seen_direct_write = seen_direct_write; 19785 } 19786 } 19787 19788 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, 19789 u16 struct_meta_reg, 19790 u16 node_offset_reg, 19791 struct bpf_insn *insn, 19792 struct bpf_insn *insn_buf, 19793 int *cnt) 19794 { 19795 struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta; 19796 struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) }; 19797 19798 insn_buf[0] = addr[0]; 19799 insn_buf[1] = addr[1]; 19800 insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off); 19801 insn_buf[3] = *insn; 19802 *cnt = 4; 19803 } 19804 19805 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 19806 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 19807 { 19808 const struct bpf_kfunc_desc *desc; 19809 19810 if (!insn->imm) { 19811 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 19812 return -EINVAL; 19813 } 19814 19815 *cnt = 0; 19816 19817 /* insn->imm has the btf func_id. Replace it with an offset relative to 19818 * __bpf_call_base, unless the JIT needs to call functions that are 19819 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()). 19820 */ 19821 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 19822 if (!desc) { 19823 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 19824 insn->imm); 19825 return -EFAULT; 19826 } 19827 19828 if (!bpf_jit_supports_far_kfunc_call()) 19829 insn->imm = BPF_CALL_IMM(desc->addr); 19830 if (insn->off) 19831 return 0; 19832 if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl] || 19833 desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl]) { 19834 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 19835 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 19836 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 19837 19838 if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_new_impl] && kptr_struct_meta) { 19839 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", 19840 insn_idx); 19841 return -EFAULT; 19842 } 19843 19844 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 19845 insn_buf[1] = addr[0]; 19846 insn_buf[2] = addr[1]; 19847 insn_buf[3] = *insn; 19848 *cnt = 4; 19849 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 19850 desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] || 19851 desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 19852 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 19853 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 19854 19855 if (desc->func_id == special_kfunc_list[KF_bpf_percpu_obj_drop_impl] && kptr_struct_meta) { 19856 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", 19857 insn_idx); 19858 return -EFAULT; 19859 } 19860 19861 if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] && 19862 !kptr_struct_meta) { 19863 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", 19864 insn_idx); 19865 return -EFAULT; 19866 } 19867 19868 insn_buf[0] = addr[0]; 19869 insn_buf[1] = addr[1]; 19870 insn_buf[2] = *insn; 19871 *cnt = 3; 19872 } else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 19873 desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 19874 desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 19875 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 19876 int struct_meta_reg = BPF_REG_3; 19877 int node_offset_reg = BPF_REG_4; 19878 19879 /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */ 19880 if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 19881 struct_meta_reg = BPF_REG_4; 19882 node_offset_reg = BPF_REG_5; 19883 } 19884 19885 if (!kptr_struct_meta) { 19886 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", 19887 insn_idx); 19888 return -EFAULT; 19889 } 19890 19891 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, 19892 node_offset_reg, insn, insn_buf, cnt); 19893 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 19894 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 19895 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 19896 *cnt = 1; 19897 } else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) { 19898 struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) }; 19899 19900 insn_buf[0] = ld_addrs[0]; 19901 insn_buf[1] = ld_addrs[1]; 19902 insn_buf[2] = *insn; 19903 *cnt = 3; 19904 } 19905 return 0; 19906 } 19907 19908 /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ 19909 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) 19910 { 19911 struct bpf_subprog_info *info = env->subprog_info; 19912 int cnt = env->subprog_cnt; 19913 struct bpf_prog *prog; 19914 19915 /* We only reserve one slot for hidden subprogs in subprog_info. */ 19916 if (env->hidden_subprog_cnt) { 19917 verbose(env, "verifier internal error: only one hidden subprog supported\n"); 19918 return -EFAULT; 19919 } 19920 /* We're not patching any existing instruction, just appending the new 19921 * ones for the hidden subprog. Hence all of the adjustment operations 19922 * in bpf_patch_insn_data are no-ops. 19923 */ 19924 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); 19925 if (!prog) 19926 return -ENOMEM; 19927 env->prog = prog; 19928 info[cnt + 1].start = info[cnt].start; 19929 info[cnt].start = prog->len - len + 1; 19930 env->subprog_cnt++; 19931 env->hidden_subprog_cnt++; 19932 return 0; 19933 } 19934 19935 /* Do various post-verification rewrites in a single program pass. 19936 * These rewrites simplify JIT and interpreter implementations. 19937 */ 19938 static int do_misc_fixups(struct bpf_verifier_env *env) 19939 { 19940 struct bpf_prog *prog = env->prog; 19941 enum bpf_attach_type eatype = prog->expected_attach_type; 19942 enum bpf_prog_type prog_type = resolve_prog_type(prog); 19943 struct bpf_insn *insn = prog->insnsi; 19944 const struct bpf_func_proto *fn; 19945 const int insn_cnt = prog->len; 19946 const struct bpf_map_ops *ops; 19947 struct bpf_insn_aux_data *aux; 19948 struct bpf_insn insn_buf[16]; 19949 struct bpf_prog *new_prog; 19950 struct bpf_map *map_ptr; 19951 int i, ret, cnt, delta = 0, cur_subprog = 0; 19952 struct bpf_subprog_info *subprogs = env->subprog_info; 19953 u16 stack_depth = subprogs[cur_subprog].stack_depth; 19954 u16 stack_depth_extra = 0; 19955 19956 if (env->seen_exception && !env->exception_callback_subprog) { 19957 struct bpf_insn patch[] = { 19958 env->prog->insnsi[insn_cnt - 1], 19959 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), 19960 BPF_EXIT_INSN(), 19961 }; 19962 19963 ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch)); 19964 if (ret < 0) 19965 return ret; 19966 prog = env->prog; 19967 insn = prog->insnsi; 19968 19969 env->exception_callback_subprog = env->subprog_cnt - 1; 19970 /* Don't update insn_cnt, as add_hidden_subprog always appends insns */ 19971 mark_subprog_exc_cb(env, env->exception_callback_subprog); 19972 } 19973 19974 for (i = 0; i < insn_cnt;) { 19975 if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { 19976 if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || 19977 (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { 19978 /* convert to 32-bit mov that clears upper 32-bit */ 19979 insn->code = BPF_ALU | BPF_MOV | BPF_X; 19980 /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ 19981 insn->off = 0; 19982 insn->imm = 0; 19983 } /* cast from as(0) to as(1) should be handled by JIT */ 19984 goto next_insn; 19985 } 19986 19987 if (env->insn_aux_data[i + delta].needs_zext) 19988 /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ 19989 insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); 19990 19991 /* Make divide-by-zero exceptions impossible. */ 19992 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 19993 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 19994 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 19995 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 19996 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 19997 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 19998 struct bpf_insn *patchlet; 19999 struct bpf_insn chk_and_div[] = { 20000 /* [R,W]x div 0 -> 0 */ 20001 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 20002 BPF_JNE | BPF_K, insn->src_reg, 20003 0, 2, 0), 20004 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 20005 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 20006 *insn, 20007 }; 20008 struct bpf_insn chk_and_mod[] = { 20009 /* [R,W]x mod 0 -> [R,W]x */ 20010 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 20011 BPF_JEQ | BPF_K, insn->src_reg, 20012 0, 1 + (is64 ? 0 : 1), 0), 20013 *insn, 20014 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 20015 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 20016 }; 20017 20018 patchlet = isdiv ? chk_and_div : chk_and_mod; 20019 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 20020 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 20021 20022 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 20023 if (!new_prog) 20024 return -ENOMEM; 20025 20026 delta += cnt - 1; 20027 env->prog = prog = new_prog; 20028 insn = new_prog->insnsi + i + delta; 20029 goto next_insn; 20030 } 20031 20032 /* Make it impossible to de-reference a userspace address */ 20033 if (BPF_CLASS(insn->code) == BPF_LDX && 20034 (BPF_MODE(insn->code) == BPF_PROBE_MEM || 20035 BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) { 20036 struct bpf_insn *patch = &insn_buf[0]; 20037 u64 uaddress_limit = bpf_arch_uaddress_limit(); 20038 20039 if (!uaddress_limit) 20040 goto next_insn; 20041 20042 *patch++ = BPF_MOV64_REG(BPF_REG_AX, insn->src_reg); 20043 if (insn->off) 20044 *patch++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, insn->off); 20045 *patch++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_AX, 32); 20046 *patch++ = BPF_JMP_IMM(BPF_JLE, BPF_REG_AX, uaddress_limit >> 32, 2); 20047 *patch++ = *insn; 20048 *patch++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 20049 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); 20050 20051 cnt = patch - insn_buf; 20052 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20053 if (!new_prog) 20054 return -ENOMEM; 20055 20056 delta += cnt - 1; 20057 env->prog = prog = new_prog; 20058 insn = new_prog->insnsi + i + delta; 20059 goto next_insn; 20060 } 20061 20062 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 20063 if (BPF_CLASS(insn->code) == BPF_LD && 20064 (BPF_MODE(insn->code) == BPF_ABS || 20065 BPF_MODE(insn->code) == BPF_IND)) { 20066 cnt = env->ops->gen_ld_abs(insn, insn_buf); 20067 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 20068 verbose(env, "bpf verifier is misconfigured\n"); 20069 return -EINVAL; 20070 } 20071 20072 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20073 if (!new_prog) 20074 return -ENOMEM; 20075 20076 delta += cnt - 1; 20077 env->prog = prog = new_prog; 20078 insn = new_prog->insnsi + i + delta; 20079 goto next_insn; 20080 } 20081 20082 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 20083 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 20084 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 20085 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 20086 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 20087 struct bpf_insn *patch = &insn_buf[0]; 20088 bool issrc, isneg, isimm; 20089 u32 off_reg; 20090 20091 aux = &env->insn_aux_data[i + delta]; 20092 if (!aux->alu_state || 20093 aux->alu_state == BPF_ALU_NON_POINTER) 20094 goto next_insn; 20095 20096 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 20097 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 20098 BPF_ALU_SANITIZE_SRC; 20099 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 20100 20101 off_reg = issrc ? insn->src_reg : insn->dst_reg; 20102 if (isimm) { 20103 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 20104 } else { 20105 if (isneg) 20106 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 20107 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 20108 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 20109 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 20110 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 20111 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 20112 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 20113 } 20114 if (!issrc) 20115 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 20116 insn->src_reg = BPF_REG_AX; 20117 if (isneg) 20118 insn->code = insn->code == code_add ? 20119 code_sub : code_add; 20120 *patch++ = *insn; 20121 if (issrc && isneg && !isimm) 20122 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 20123 cnt = patch - insn_buf; 20124 20125 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20126 if (!new_prog) 20127 return -ENOMEM; 20128 20129 delta += cnt - 1; 20130 env->prog = prog = new_prog; 20131 insn = new_prog->insnsi + i + delta; 20132 goto next_insn; 20133 } 20134 20135 if (is_may_goto_insn(insn)) { 20136 int stack_off = -stack_depth - 8; 20137 20138 stack_depth_extra = 8; 20139 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); 20140 if (insn->off >= 0) 20141 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); 20142 else 20143 insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); 20144 insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); 20145 insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); 20146 cnt = 4; 20147 20148 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20149 if (!new_prog) 20150 return -ENOMEM; 20151 20152 delta += cnt - 1; 20153 env->prog = prog = new_prog; 20154 insn = new_prog->insnsi + i + delta; 20155 goto next_insn; 20156 } 20157 20158 if (insn->code != (BPF_JMP | BPF_CALL)) 20159 goto next_insn; 20160 if (insn->src_reg == BPF_PSEUDO_CALL) 20161 goto next_insn; 20162 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 20163 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 20164 if (ret) 20165 return ret; 20166 if (cnt == 0) 20167 goto next_insn; 20168 20169 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20170 if (!new_prog) 20171 return -ENOMEM; 20172 20173 delta += cnt - 1; 20174 env->prog = prog = new_prog; 20175 insn = new_prog->insnsi + i + delta; 20176 goto next_insn; 20177 } 20178 20179 /* Skip inlining the helper call if the JIT does it. */ 20180 if (bpf_jit_inlines_helper_call(insn->imm)) 20181 goto next_insn; 20182 20183 if (insn->imm == BPF_FUNC_get_route_realm) 20184 prog->dst_needed = 1; 20185 if (insn->imm == BPF_FUNC_get_prandom_u32) 20186 bpf_user_rnd_init_once(); 20187 if (insn->imm == BPF_FUNC_override_return) 20188 prog->kprobe_override = 1; 20189 if (insn->imm == BPF_FUNC_tail_call) { 20190 /* If we tail call into other programs, we 20191 * cannot make any assumptions since they can 20192 * be replaced dynamically during runtime in 20193 * the program array. 20194 */ 20195 prog->cb_access = 1; 20196 if (!allow_tail_call_in_subprogs(env)) 20197 prog->aux->stack_depth = MAX_BPF_STACK; 20198 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 20199 20200 /* mark bpf_tail_call as different opcode to avoid 20201 * conditional branch in the interpreter for every normal 20202 * call and to prevent accidental JITing by JIT compiler 20203 * that doesn't support bpf_tail_call yet 20204 */ 20205 insn->imm = 0; 20206 insn->code = BPF_JMP | BPF_TAIL_CALL; 20207 20208 aux = &env->insn_aux_data[i + delta]; 20209 if (env->bpf_capable && !prog->blinding_requested && 20210 prog->jit_requested && 20211 !bpf_map_key_poisoned(aux) && 20212 !bpf_map_ptr_poisoned(aux) && 20213 !bpf_map_ptr_unpriv(aux)) { 20214 struct bpf_jit_poke_descriptor desc = { 20215 .reason = BPF_POKE_REASON_TAIL_CALL, 20216 .tail_call.map = aux->map_ptr_state.map_ptr, 20217 .tail_call.key = bpf_map_key_immediate(aux), 20218 .insn_idx = i + delta, 20219 }; 20220 20221 ret = bpf_jit_add_poke_descriptor(prog, &desc); 20222 if (ret < 0) { 20223 verbose(env, "adding tail call poke descriptor failed\n"); 20224 return ret; 20225 } 20226 20227 insn->imm = ret + 1; 20228 goto next_insn; 20229 } 20230 20231 if (!bpf_map_ptr_unpriv(aux)) 20232 goto next_insn; 20233 20234 /* instead of changing every JIT dealing with tail_call 20235 * emit two extra insns: 20236 * if (index >= max_entries) goto out; 20237 * index &= array->index_mask; 20238 * to avoid out-of-bounds cpu speculation 20239 */ 20240 if (bpf_map_ptr_poisoned(aux)) { 20241 verbose(env, "tail_call abusing map_ptr\n"); 20242 return -EINVAL; 20243 } 20244 20245 map_ptr = aux->map_ptr_state.map_ptr; 20246 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 20247 map_ptr->max_entries, 2); 20248 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 20249 container_of(map_ptr, 20250 struct bpf_array, 20251 map)->index_mask); 20252 insn_buf[2] = *insn; 20253 cnt = 3; 20254 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20255 if (!new_prog) 20256 return -ENOMEM; 20257 20258 delta += cnt - 1; 20259 env->prog = prog = new_prog; 20260 insn = new_prog->insnsi + i + delta; 20261 goto next_insn; 20262 } 20263 20264 if (insn->imm == BPF_FUNC_timer_set_callback) { 20265 /* The verifier will process callback_fn as many times as necessary 20266 * with different maps and the register states prepared by 20267 * set_timer_callback_state will be accurate. 20268 * 20269 * The following use case is valid: 20270 * map1 is shared by prog1, prog2, prog3. 20271 * prog1 calls bpf_timer_init for some map1 elements 20272 * prog2 calls bpf_timer_set_callback for some map1 elements. 20273 * Those that were not bpf_timer_init-ed will return -EINVAL. 20274 * prog3 calls bpf_timer_start for some map1 elements. 20275 * Those that were not both bpf_timer_init-ed and 20276 * bpf_timer_set_callback-ed will return -EINVAL. 20277 */ 20278 struct bpf_insn ld_addrs[2] = { 20279 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 20280 }; 20281 20282 insn_buf[0] = ld_addrs[0]; 20283 insn_buf[1] = ld_addrs[1]; 20284 insn_buf[2] = *insn; 20285 cnt = 3; 20286 20287 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20288 if (!new_prog) 20289 return -ENOMEM; 20290 20291 delta += cnt - 1; 20292 env->prog = prog = new_prog; 20293 insn = new_prog->insnsi + i + delta; 20294 goto patch_call_imm; 20295 } 20296 20297 if (is_storage_get_function(insn->imm)) { 20298 if (!in_sleepable(env) || 20299 env->insn_aux_data[i + delta].storage_get_func_atomic) 20300 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 20301 else 20302 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 20303 insn_buf[1] = *insn; 20304 cnt = 2; 20305 20306 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20307 if (!new_prog) 20308 return -ENOMEM; 20309 20310 delta += cnt - 1; 20311 env->prog = prog = new_prog; 20312 insn = new_prog->insnsi + i + delta; 20313 goto patch_call_imm; 20314 } 20315 20316 /* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ 20317 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { 20318 /* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data, 20319 * bpf_mem_alloc() returns a ptr to the percpu data ptr. 20320 */ 20321 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0); 20322 insn_buf[1] = *insn; 20323 cnt = 2; 20324 20325 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20326 if (!new_prog) 20327 return -ENOMEM; 20328 20329 delta += cnt - 1; 20330 env->prog = prog = new_prog; 20331 insn = new_prog->insnsi + i + delta; 20332 goto patch_call_imm; 20333 } 20334 20335 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 20336 * and other inlining handlers are currently limited to 64 bit 20337 * only. 20338 */ 20339 if (prog->jit_requested && BITS_PER_LONG == 64 && 20340 (insn->imm == BPF_FUNC_map_lookup_elem || 20341 insn->imm == BPF_FUNC_map_update_elem || 20342 insn->imm == BPF_FUNC_map_delete_elem || 20343 insn->imm == BPF_FUNC_map_push_elem || 20344 insn->imm == BPF_FUNC_map_pop_elem || 20345 insn->imm == BPF_FUNC_map_peek_elem || 20346 insn->imm == BPF_FUNC_redirect_map || 20347 insn->imm == BPF_FUNC_for_each_map_elem || 20348 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 20349 aux = &env->insn_aux_data[i + delta]; 20350 if (bpf_map_ptr_poisoned(aux)) 20351 goto patch_call_imm; 20352 20353 map_ptr = aux->map_ptr_state.map_ptr; 20354 ops = map_ptr->ops; 20355 if (insn->imm == BPF_FUNC_map_lookup_elem && 20356 ops->map_gen_lookup) { 20357 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 20358 if (cnt == -EOPNOTSUPP) 20359 goto patch_map_ops_generic; 20360 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 20361 verbose(env, "bpf verifier is misconfigured\n"); 20362 return -EINVAL; 20363 } 20364 20365 new_prog = bpf_patch_insn_data(env, i + delta, 20366 insn_buf, cnt); 20367 if (!new_prog) 20368 return -ENOMEM; 20369 20370 delta += cnt - 1; 20371 env->prog = prog = new_prog; 20372 insn = new_prog->insnsi + i + delta; 20373 goto next_insn; 20374 } 20375 20376 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 20377 (void *(*)(struct bpf_map *map, void *key))NULL)); 20378 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 20379 (long (*)(struct bpf_map *map, void *key))NULL)); 20380 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 20381 (long (*)(struct bpf_map *map, void *key, void *value, 20382 u64 flags))NULL)); 20383 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 20384 (long (*)(struct bpf_map *map, void *value, 20385 u64 flags))NULL)); 20386 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 20387 (long (*)(struct bpf_map *map, void *value))NULL)); 20388 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 20389 (long (*)(struct bpf_map *map, void *value))NULL)); 20390 BUILD_BUG_ON(!__same_type(ops->map_redirect, 20391 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 20392 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 20393 (long (*)(struct bpf_map *map, 20394 bpf_callback_t callback_fn, 20395 void *callback_ctx, 20396 u64 flags))NULL)); 20397 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 20398 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 20399 20400 patch_map_ops_generic: 20401 switch (insn->imm) { 20402 case BPF_FUNC_map_lookup_elem: 20403 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 20404 goto next_insn; 20405 case BPF_FUNC_map_update_elem: 20406 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 20407 goto next_insn; 20408 case BPF_FUNC_map_delete_elem: 20409 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 20410 goto next_insn; 20411 case BPF_FUNC_map_push_elem: 20412 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 20413 goto next_insn; 20414 case BPF_FUNC_map_pop_elem: 20415 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 20416 goto next_insn; 20417 case BPF_FUNC_map_peek_elem: 20418 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 20419 goto next_insn; 20420 case BPF_FUNC_redirect_map: 20421 insn->imm = BPF_CALL_IMM(ops->map_redirect); 20422 goto next_insn; 20423 case BPF_FUNC_for_each_map_elem: 20424 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 20425 goto next_insn; 20426 case BPF_FUNC_map_lookup_percpu_elem: 20427 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 20428 goto next_insn; 20429 } 20430 20431 goto patch_call_imm; 20432 } 20433 20434 /* Implement bpf_jiffies64 inline. */ 20435 if (prog->jit_requested && BITS_PER_LONG == 64 && 20436 insn->imm == BPF_FUNC_jiffies64) { 20437 struct bpf_insn ld_jiffies_addr[2] = { 20438 BPF_LD_IMM64(BPF_REG_0, 20439 (unsigned long)&jiffies), 20440 }; 20441 20442 insn_buf[0] = ld_jiffies_addr[0]; 20443 insn_buf[1] = ld_jiffies_addr[1]; 20444 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 20445 BPF_REG_0, 0); 20446 cnt = 3; 20447 20448 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 20449 cnt); 20450 if (!new_prog) 20451 return -ENOMEM; 20452 20453 delta += cnt - 1; 20454 env->prog = prog = new_prog; 20455 insn = new_prog->insnsi + i + delta; 20456 goto next_insn; 20457 } 20458 20459 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML) 20460 /* Implement bpf_get_smp_processor_id() inline. */ 20461 if (insn->imm == BPF_FUNC_get_smp_processor_id && 20462 prog->jit_requested && bpf_jit_supports_percpu_insn()) { 20463 /* BPF_FUNC_get_smp_processor_id inlining is an 20464 * optimization, so if pcpu_hot.cpu_number is ever 20465 * changed in some incompatible and hard to support 20466 * way, it's fine to back out this inlining logic 20467 */ 20468 insn_buf[0] = BPF_MOV32_IMM(BPF_REG_0, (u32)(unsigned long)&pcpu_hot.cpu_number); 20469 insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); 20470 insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); 20471 cnt = 3; 20472 20473 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20474 if (!new_prog) 20475 return -ENOMEM; 20476 20477 delta += cnt - 1; 20478 env->prog = prog = new_prog; 20479 insn = new_prog->insnsi + i + delta; 20480 goto next_insn; 20481 } 20482 #endif 20483 /* Implement bpf_get_func_arg inline. */ 20484 if (prog_type == BPF_PROG_TYPE_TRACING && 20485 insn->imm == BPF_FUNC_get_func_arg) { 20486 /* Load nr_args from ctx - 8 */ 20487 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 20488 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 20489 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 20490 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 20491 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 20492 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 20493 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 20494 insn_buf[7] = BPF_JMP_A(1); 20495 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 20496 cnt = 9; 20497 20498 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20499 if (!new_prog) 20500 return -ENOMEM; 20501 20502 delta += cnt - 1; 20503 env->prog = prog = new_prog; 20504 insn = new_prog->insnsi + i + delta; 20505 goto next_insn; 20506 } 20507 20508 /* Implement bpf_get_func_ret inline. */ 20509 if (prog_type == BPF_PROG_TYPE_TRACING && 20510 insn->imm == BPF_FUNC_get_func_ret) { 20511 if (eatype == BPF_TRACE_FEXIT || 20512 eatype == BPF_MODIFY_RETURN) { 20513 /* Load nr_args from ctx - 8 */ 20514 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 20515 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 20516 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 20517 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 20518 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 20519 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 20520 cnt = 6; 20521 } else { 20522 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 20523 cnt = 1; 20524 } 20525 20526 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20527 if (!new_prog) 20528 return -ENOMEM; 20529 20530 delta += cnt - 1; 20531 env->prog = prog = new_prog; 20532 insn = new_prog->insnsi + i + delta; 20533 goto next_insn; 20534 } 20535 20536 /* Implement get_func_arg_cnt inline. */ 20537 if (prog_type == BPF_PROG_TYPE_TRACING && 20538 insn->imm == BPF_FUNC_get_func_arg_cnt) { 20539 /* Load nr_args from ctx - 8 */ 20540 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 20541 20542 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 20543 if (!new_prog) 20544 return -ENOMEM; 20545 20546 env->prog = prog = new_prog; 20547 insn = new_prog->insnsi + i + delta; 20548 goto next_insn; 20549 } 20550 20551 /* Implement bpf_get_func_ip inline. */ 20552 if (prog_type == BPF_PROG_TYPE_TRACING && 20553 insn->imm == BPF_FUNC_get_func_ip) { 20554 /* Load IP address from ctx - 16 */ 20555 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 20556 20557 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 20558 if (!new_prog) 20559 return -ENOMEM; 20560 20561 env->prog = prog = new_prog; 20562 insn = new_prog->insnsi + i + delta; 20563 goto next_insn; 20564 } 20565 20566 /* Implement bpf_get_branch_snapshot inline. */ 20567 if (IS_ENABLED(CONFIG_PERF_EVENTS) && 20568 prog->jit_requested && BITS_PER_LONG == 64 && 20569 insn->imm == BPF_FUNC_get_branch_snapshot) { 20570 /* We are dealing with the following func protos: 20571 * u64 bpf_get_branch_snapshot(void *buf, u32 size, u64 flags); 20572 * int perf_snapshot_branch_stack(struct perf_branch_entry *entries, u32 cnt); 20573 */ 20574 const u32 br_entry_size = sizeof(struct perf_branch_entry); 20575 20576 /* struct perf_branch_entry is part of UAPI and is 20577 * used as an array element, so extremely unlikely to 20578 * ever grow or shrink 20579 */ 20580 BUILD_BUG_ON(br_entry_size != 24); 20581 20582 /* if (unlikely(flags)) return -EINVAL */ 20583 insn_buf[0] = BPF_JMP_IMM(BPF_JNE, BPF_REG_3, 0, 7); 20584 20585 /* Transform size (bytes) into number of entries (cnt = size / 24). 20586 * But to avoid expensive division instruction, we implement 20587 * divide-by-3 through multiplication, followed by further 20588 * division by 8 through 3-bit right shift. 20589 * Refer to book "Hacker's Delight, 2nd ed." by Henry S. Warren, Jr., 20590 * p. 227, chapter "Unsigned Division by 3" for details and proofs. 20591 * 20592 * N / 3 <=> M * N / 2^33, where M = (2^33 + 1) / 3 = 0xaaaaaaab. 20593 */ 20594 insn_buf[1] = BPF_MOV32_IMM(BPF_REG_0, 0xaaaaaaab); 20595 insn_buf[2] = BPF_ALU64_REG(BPF_MUL, BPF_REG_2, BPF_REG_0); 20596 insn_buf[3] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_2, 36); 20597 20598 /* call perf_snapshot_branch_stack implementation */ 20599 insn_buf[4] = BPF_EMIT_CALL(static_call_query(perf_snapshot_branch_stack)); 20600 /* if (entry_cnt == 0) return -ENOENT */ 20601 insn_buf[5] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 4); 20602 /* return entry_cnt * sizeof(struct perf_branch_entry) */ 20603 insn_buf[6] = BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, br_entry_size); 20604 insn_buf[7] = BPF_JMP_A(3); 20605 /* return -EINVAL; */ 20606 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 20607 insn_buf[9] = BPF_JMP_A(1); 20608 /* return -ENOENT; */ 20609 insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); 20610 cnt = 11; 20611 20612 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20613 if (!new_prog) 20614 return -ENOMEM; 20615 20616 delta += cnt - 1; 20617 env->prog = prog = new_prog; 20618 insn = new_prog->insnsi + i + delta; 20619 continue; 20620 } 20621 20622 /* Implement bpf_kptr_xchg inline */ 20623 if (prog->jit_requested && BITS_PER_LONG == 64 && 20624 insn->imm == BPF_FUNC_kptr_xchg && 20625 bpf_jit_supports_ptr_xchg()) { 20626 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2); 20627 insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); 20628 cnt = 2; 20629 20630 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 20631 if (!new_prog) 20632 return -ENOMEM; 20633 20634 delta += cnt - 1; 20635 env->prog = prog = new_prog; 20636 insn = new_prog->insnsi + i + delta; 20637 goto next_insn; 20638 } 20639 patch_call_imm: 20640 fn = env->ops->get_func_proto(insn->imm, env->prog); 20641 /* all functions that have prototype and verifier allowed 20642 * programs to call them, must be real in-kernel functions 20643 */ 20644 if (!fn->func) { 20645 verbose(env, 20646 "kernel subsystem misconfigured func %s#%d\n", 20647 func_id_name(insn->imm), insn->imm); 20648 return -EFAULT; 20649 } 20650 insn->imm = fn->func - __bpf_call_base; 20651 next_insn: 20652 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 20653 subprogs[cur_subprog].stack_depth += stack_depth_extra; 20654 subprogs[cur_subprog].stack_extra = stack_depth_extra; 20655 cur_subprog++; 20656 stack_depth = subprogs[cur_subprog].stack_depth; 20657 stack_depth_extra = 0; 20658 } 20659 i++; 20660 insn++; 20661 } 20662 20663 env->prog->aux->stack_depth = subprogs[0].stack_depth; 20664 for (i = 0; i < env->subprog_cnt; i++) { 20665 int subprog_start = subprogs[i].start; 20666 int stack_slots = subprogs[i].stack_extra / 8; 20667 20668 if (!stack_slots) 20669 continue; 20670 if (stack_slots > 1) { 20671 verbose(env, "verifier bug: stack_slots supports may_goto only\n"); 20672 return -EFAULT; 20673 } 20674 20675 /* Add ST insn to subprog prologue to init extra stack */ 20676 insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, 20677 -subprogs[i].stack_depth, BPF_MAX_LOOPS); 20678 /* Copy first actual insn to preserve it */ 20679 insn_buf[1] = env->prog->insnsi[subprog_start]; 20680 20681 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2); 20682 if (!new_prog) 20683 return -ENOMEM; 20684 env->prog = prog = new_prog; 20685 /* 20686 * If may_goto is a first insn of a prog there could be a jmp 20687 * insn that points to it, hence adjust all such jmps to point 20688 * to insn after BPF_ST that inits may_goto count. 20689 * Adjustment will succeed because bpf_patch_insn_data() didn't fail. 20690 */ 20691 WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1)); 20692 } 20693 20694 /* Since poke tab is now finalized, publish aux to tracker. */ 20695 for (i = 0; i < prog->aux->size_poke_tab; i++) { 20696 map_ptr = prog->aux->poke_tab[i].tail_call.map; 20697 if (!map_ptr->ops->map_poke_track || 20698 !map_ptr->ops->map_poke_untrack || 20699 !map_ptr->ops->map_poke_run) { 20700 verbose(env, "bpf verifier is misconfigured\n"); 20701 return -EINVAL; 20702 } 20703 20704 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 20705 if (ret < 0) { 20706 verbose(env, "tracking tail call prog failed\n"); 20707 return ret; 20708 } 20709 } 20710 20711 sort_kfunc_descs_by_imm_off(env->prog); 20712 20713 return 0; 20714 } 20715 20716 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 20717 int position, 20718 s32 stack_base, 20719 u32 callback_subprogno, 20720 u32 *cnt) 20721 { 20722 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 20723 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 20724 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 20725 int reg_loop_max = BPF_REG_6; 20726 int reg_loop_cnt = BPF_REG_7; 20727 int reg_loop_ctx = BPF_REG_8; 20728 20729 struct bpf_prog *new_prog; 20730 u32 callback_start; 20731 u32 call_insn_offset; 20732 s32 callback_offset; 20733 20734 /* This represents an inlined version of bpf_iter.c:bpf_loop, 20735 * be careful to modify this code in sync. 20736 */ 20737 struct bpf_insn insn_buf[] = { 20738 /* Return error and jump to the end of the patch if 20739 * expected number of iterations is too big. 20740 */ 20741 BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), 20742 BPF_MOV32_IMM(BPF_REG_0, -E2BIG), 20743 BPF_JMP_IMM(BPF_JA, 0, 0, 16), 20744 /* spill R6, R7, R8 to use these as loop vars */ 20745 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), 20746 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), 20747 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), 20748 /* initialize loop vars */ 20749 BPF_MOV64_REG(reg_loop_max, BPF_REG_1), 20750 BPF_MOV32_IMM(reg_loop_cnt, 0), 20751 BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), 20752 /* loop header, 20753 * if reg_loop_cnt >= reg_loop_max skip the loop body 20754 */ 20755 BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), 20756 /* callback call, 20757 * correct callback offset would be set after patching 20758 */ 20759 BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), 20760 BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), 20761 BPF_CALL_REL(0), 20762 /* increment loop counter */ 20763 BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), 20764 /* jump to loop header if callback returned 0 */ 20765 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), 20766 /* return value of bpf_loop, 20767 * set R0 to the number of iterations 20768 */ 20769 BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), 20770 /* restore original values of R6, R7, R8 */ 20771 BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), 20772 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), 20773 BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), 20774 }; 20775 20776 *cnt = ARRAY_SIZE(insn_buf); 20777 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); 20778 if (!new_prog) 20779 return new_prog; 20780 20781 /* callback start is known only after patching */ 20782 callback_start = env->subprog_info[callback_subprogno].start; 20783 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 20784 call_insn_offset = position + 12; 20785 callback_offset = callback_start - call_insn_offset - 1; 20786 new_prog->insnsi[call_insn_offset].imm = callback_offset; 20787 20788 return new_prog; 20789 } 20790 20791 static bool is_bpf_loop_call(struct bpf_insn *insn) 20792 { 20793 return insn->code == (BPF_JMP | BPF_CALL) && 20794 insn->src_reg == 0 && 20795 insn->imm == BPF_FUNC_loop; 20796 } 20797 20798 /* For all sub-programs in the program (including main) check 20799 * insn_aux_data to see if there are bpf_loop calls that require 20800 * inlining. If such calls are found the calls are replaced with a 20801 * sequence of instructions produced by `inline_bpf_loop` function and 20802 * subprog stack_depth is increased by the size of 3 registers. 20803 * This stack space is used to spill values of the R6, R7, R8. These 20804 * registers are used to store the loop bound, counter and context 20805 * variables. 20806 */ 20807 static int optimize_bpf_loop(struct bpf_verifier_env *env) 20808 { 20809 struct bpf_subprog_info *subprogs = env->subprog_info; 20810 int i, cur_subprog = 0, cnt, delta = 0; 20811 struct bpf_insn *insn = env->prog->insnsi; 20812 int insn_cnt = env->prog->len; 20813 u16 stack_depth = subprogs[cur_subprog].stack_depth; 20814 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 20815 u16 stack_depth_extra = 0; 20816 20817 for (i = 0; i < insn_cnt; i++, insn++) { 20818 struct bpf_loop_inline_state *inline_state = 20819 &env->insn_aux_data[i + delta].loop_inline_state; 20820 20821 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 20822 struct bpf_prog *new_prog; 20823 20824 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 20825 new_prog = inline_bpf_loop(env, 20826 i + delta, 20827 -(stack_depth + stack_depth_extra), 20828 inline_state->callback_subprogno, 20829 &cnt); 20830 if (!new_prog) 20831 return -ENOMEM; 20832 20833 delta += cnt - 1; 20834 env->prog = new_prog; 20835 insn = new_prog->insnsi + i + delta; 20836 } 20837 20838 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 20839 subprogs[cur_subprog].stack_depth += stack_depth_extra; 20840 cur_subprog++; 20841 stack_depth = subprogs[cur_subprog].stack_depth; 20842 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 20843 stack_depth_extra = 0; 20844 } 20845 } 20846 20847 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 20848 20849 return 0; 20850 } 20851 20852 static void free_states(struct bpf_verifier_env *env) 20853 { 20854 struct bpf_verifier_state_list *sl, *sln; 20855 int i; 20856 20857 sl = env->free_list; 20858 while (sl) { 20859 sln = sl->next; 20860 free_verifier_state(&sl->state, false); 20861 kfree(sl); 20862 sl = sln; 20863 } 20864 env->free_list = NULL; 20865 20866 if (!env->explored_states) 20867 return; 20868 20869 for (i = 0; i < state_htab_size(env); i++) { 20870 sl = env->explored_states[i]; 20871 20872 while (sl) { 20873 sln = sl->next; 20874 free_verifier_state(&sl->state, false); 20875 kfree(sl); 20876 sl = sln; 20877 } 20878 env->explored_states[i] = NULL; 20879 } 20880 } 20881 20882 static int do_check_common(struct bpf_verifier_env *env, int subprog) 20883 { 20884 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 20885 struct bpf_subprog_info *sub = subprog_info(env, subprog); 20886 struct bpf_verifier_state *state; 20887 struct bpf_reg_state *regs; 20888 int ret, i; 20889 20890 env->prev_linfo = NULL; 20891 env->pass_cnt++; 20892 20893 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 20894 if (!state) 20895 return -ENOMEM; 20896 state->curframe = 0; 20897 state->speculative = false; 20898 state->branches = 1; 20899 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 20900 if (!state->frame[0]) { 20901 kfree(state); 20902 return -ENOMEM; 20903 } 20904 env->cur_state = state; 20905 init_func_state(env, state->frame[0], 20906 BPF_MAIN_FUNC /* callsite */, 20907 0 /* frameno */, 20908 subprog); 20909 state->first_insn_idx = env->subprog_info[subprog].start; 20910 state->last_insn_idx = -1; 20911 20912 regs = state->frame[state->curframe]->regs; 20913 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 20914 const char *sub_name = subprog_name(env, subprog); 20915 struct bpf_subprog_arg_info *arg; 20916 struct bpf_reg_state *reg; 20917 20918 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); 20919 ret = btf_prepare_func_args(env, subprog); 20920 if (ret) 20921 goto out; 20922 20923 if (subprog_is_exc_cb(env, subprog)) { 20924 state->frame[0]->in_exception_callback_fn = true; 20925 /* We have already ensured that the callback returns an integer, just 20926 * like all global subprogs. We need to determine it only has a single 20927 * scalar argument. 20928 */ 20929 if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { 20930 verbose(env, "exception cb only supports single integer argument\n"); 20931 ret = -EINVAL; 20932 goto out; 20933 } 20934 } 20935 for (i = BPF_REG_1; i <= sub->arg_cnt; i++) { 20936 arg = &sub->args[i - BPF_REG_1]; 20937 reg = ®s[i]; 20938 20939 if (arg->arg_type == ARG_PTR_TO_CTX) { 20940 reg->type = PTR_TO_CTX; 20941 mark_reg_known_zero(env, regs, i); 20942 } else if (arg->arg_type == ARG_ANYTHING) { 20943 reg->type = SCALAR_VALUE; 20944 mark_reg_unknown(env, regs, i); 20945 } else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) { 20946 /* assume unspecial LOCAL dynptr type */ 20947 __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen); 20948 } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { 20949 reg->type = PTR_TO_MEM; 20950 if (arg->arg_type & PTR_MAYBE_NULL) 20951 reg->type |= PTR_MAYBE_NULL; 20952 mark_reg_known_zero(env, regs, i); 20953 reg->mem_size = arg->mem_size; 20954 reg->id = ++env->id_gen; 20955 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { 20956 reg->type = PTR_TO_BTF_ID; 20957 if (arg->arg_type & PTR_MAYBE_NULL) 20958 reg->type |= PTR_MAYBE_NULL; 20959 if (arg->arg_type & PTR_UNTRUSTED) 20960 reg->type |= PTR_UNTRUSTED; 20961 if (arg->arg_type & PTR_TRUSTED) 20962 reg->type |= PTR_TRUSTED; 20963 mark_reg_known_zero(env, regs, i); 20964 reg->btf = bpf_get_btf_vmlinux(); /* can't fail at this point */ 20965 reg->btf_id = arg->btf_id; 20966 reg->id = ++env->id_gen; 20967 } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { 20968 /* caller can pass either PTR_TO_ARENA or SCALAR */ 20969 mark_reg_unknown(env, regs, i); 20970 } else { 20971 WARN_ONCE(1, "BUG: unhandled arg#%d type %d\n", 20972 i - BPF_REG_1, arg->arg_type); 20973 ret = -EFAULT; 20974 goto out; 20975 } 20976 } 20977 } else { 20978 /* if main BPF program has associated BTF info, validate that 20979 * it's matching expected signature, and otherwise mark BTF 20980 * info for main program as unreliable 20981 */ 20982 if (env->prog->aux->func_info_aux) { 20983 ret = btf_prepare_func_args(env, 0); 20984 if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) 20985 env->prog->aux->func_info_aux[0].unreliable = true; 20986 } 20987 20988 /* 1st arg to a function */ 20989 regs[BPF_REG_1].type = PTR_TO_CTX; 20990 mark_reg_known_zero(env, regs, BPF_REG_1); 20991 } 20992 20993 ret = do_check(env); 20994 out: 20995 /* check for NULL is necessary, since cur_state can be freed inside 20996 * do_check() under memory pressure. 20997 */ 20998 if (env->cur_state) { 20999 free_verifier_state(env->cur_state, true); 21000 env->cur_state = NULL; 21001 } 21002 while (!pop_stack(env, NULL, NULL, false)); 21003 if (!ret && pop_log) 21004 bpf_vlog_reset(&env->log, 0); 21005 free_states(env); 21006 return ret; 21007 } 21008 21009 /* Lazily verify all global functions based on their BTF, if they are called 21010 * from main BPF program or any of subprograms transitively. 21011 * BPF global subprogs called from dead code are not validated. 21012 * All callable global functions must pass verification. 21013 * Otherwise the whole program is rejected. 21014 * Consider: 21015 * int bar(int); 21016 * int foo(int f) 21017 * { 21018 * return bar(f); 21019 * } 21020 * int bar(int b) 21021 * { 21022 * ... 21023 * } 21024 * foo() will be verified first for R1=any_scalar_value. During verification it 21025 * will be assumed that bar() already verified successfully and call to bar() 21026 * from foo() will be checked for type match only. Later bar() will be verified 21027 * independently to check that it's safe for R1=any_scalar_value. 21028 */ 21029 static int do_check_subprogs(struct bpf_verifier_env *env) 21030 { 21031 struct bpf_prog_aux *aux = env->prog->aux; 21032 struct bpf_func_info_aux *sub_aux; 21033 int i, ret, new_cnt; 21034 21035 if (!aux->func_info) 21036 return 0; 21037 21038 /* exception callback is presumed to be always called */ 21039 if (env->exception_callback_subprog) 21040 subprog_aux(env, env->exception_callback_subprog)->called = true; 21041 21042 again: 21043 new_cnt = 0; 21044 for (i = 1; i < env->subprog_cnt; i++) { 21045 if (!subprog_is_global(env, i)) 21046 continue; 21047 21048 sub_aux = subprog_aux(env, i); 21049 if (!sub_aux->called || sub_aux->verified) 21050 continue; 21051 21052 env->insn_idx = env->subprog_info[i].start; 21053 WARN_ON_ONCE(env->insn_idx == 0); 21054 ret = do_check_common(env, i); 21055 if (ret) { 21056 return ret; 21057 } else if (env->log.level & BPF_LOG_LEVEL) { 21058 verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", 21059 i, subprog_name(env, i)); 21060 } 21061 21062 /* We verified new global subprog, it might have called some 21063 * more global subprogs that we haven't verified yet, so we 21064 * need to do another pass over subprogs to verify those. 21065 */ 21066 sub_aux->verified = true; 21067 new_cnt++; 21068 } 21069 21070 /* We can't loop forever as we verify at least one global subprog on 21071 * each pass. 21072 */ 21073 if (new_cnt) 21074 goto again; 21075 21076 return 0; 21077 } 21078 21079 static int do_check_main(struct bpf_verifier_env *env) 21080 { 21081 int ret; 21082 21083 env->insn_idx = 0; 21084 ret = do_check_common(env, 0); 21085 if (!ret) 21086 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 21087 return ret; 21088 } 21089 21090 21091 static void print_verification_stats(struct bpf_verifier_env *env) 21092 { 21093 int i; 21094 21095 if (env->log.level & BPF_LOG_STATS) { 21096 verbose(env, "verification time %lld usec\n", 21097 div_u64(env->verification_time, 1000)); 21098 verbose(env, "stack depth "); 21099 for (i = 0; i < env->subprog_cnt; i++) { 21100 u32 depth = env->subprog_info[i].stack_depth; 21101 21102 verbose(env, "%d", depth); 21103 if (i + 1 < env->subprog_cnt) 21104 verbose(env, "+"); 21105 } 21106 verbose(env, "\n"); 21107 } 21108 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 21109 "total_states %d peak_states %d mark_read %d\n", 21110 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 21111 env->max_states_per_insn, env->total_states, 21112 env->peak_states, env->longest_mark_read_walk); 21113 } 21114 21115 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 21116 { 21117 const struct btf_type *t, *func_proto; 21118 const struct bpf_struct_ops_desc *st_ops_desc; 21119 const struct bpf_struct_ops *st_ops; 21120 const struct btf_member *member; 21121 struct bpf_prog *prog = env->prog; 21122 u32 btf_id, member_idx; 21123 struct btf *btf; 21124 const char *mname; 21125 21126 if (!prog->gpl_compatible) { 21127 verbose(env, "struct ops programs must have a GPL compatible license\n"); 21128 return -EINVAL; 21129 } 21130 21131 if (!prog->aux->attach_btf_id) 21132 return -ENOTSUPP; 21133 21134 btf = prog->aux->attach_btf; 21135 if (btf_is_module(btf)) { 21136 /* Make sure st_ops is valid through the lifetime of env */ 21137 env->attach_btf_mod = btf_try_get_module(btf); 21138 if (!env->attach_btf_mod) { 21139 verbose(env, "struct_ops module %s is not found\n", 21140 btf_get_name(btf)); 21141 return -ENOTSUPP; 21142 } 21143 } 21144 21145 btf_id = prog->aux->attach_btf_id; 21146 st_ops_desc = bpf_struct_ops_find(btf, btf_id); 21147 if (!st_ops_desc) { 21148 verbose(env, "attach_btf_id %u is not a supported struct\n", 21149 btf_id); 21150 return -ENOTSUPP; 21151 } 21152 st_ops = st_ops_desc->st_ops; 21153 21154 t = st_ops_desc->type; 21155 member_idx = prog->expected_attach_type; 21156 if (member_idx >= btf_type_vlen(t)) { 21157 verbose(env, "attach to invalid member idx %u of struct %s\n", 21158 member_idx, st_ops->name); 21159 return -EINVAL; 21160 } 21161 21162 member = &btf_type_member(t)[member_idx]; 21163 mname = btf_name_by_offset(btf, member->name_off); 21164 func_proto = btf_type_resolve_func_ptr(btf, member->type, 21165 NULL); 21166 if (!func_proto) { 21167 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 21168 mname, member_idx, st_ops->name); 21169 return -EINVAL; 21170 } 21171 21172 if (st_ops->check_member) { 21173 int err = st_ops->check_member(t, member, prog); 21174 21175 if (err) { 21176 verbose(env, "attach to unsupported member %s of struct %s\n", 21177 mname, st_ops->name); 21178 return err; 21179 } 21180 } 21181 21182 /* btf_ctx_access() used this to provide argument type info */ 21183 prog->aux->ctx_arg_info = 21184 st_ops_desc->arg_info[member_idx].info; 21185 prog->aux->ctx_arg_info_size = 21186 st_ops_desc->arg_info[member_idx].cnt; 21187 21188 prog->aux->attach_func_proto = func_proto; 21189 prog->aux->attach_func_name = mname; 21190 env->ops = st_ops->verifier_ops; 21191 21192 return 0; 21193 } 21194 #define SECURITY_PREFIX "security_" 21195 21196 static int check_attach_modify_return(unsigned long addr, const char *func_name) 21197 { 21198 if (within_error_injection_list(addr) || 21199 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 21200 return 0; 21201 21202 return -EINVAL; 21203 } 21204 21205 /* list of non-sleepable functions that are otherwise on 21206 * ALLOW_ERROR_INJECTION list 21207 */ 21208 BTF_SET_START(btf_non_sleepable_error_inject) 21209 /* Three functions below can be called from sleepable and non-sleepable context. 21210 * Assume non-sleepable from bpf safety point of view. 21211 */ 21212 BTF_ID(func, __filemap_add_folio) 21213 BTF_ID(func, should_fail_alloc_page) 21214 BTF_ID(func, should_failslab) 21215 BTF_SET_END(btf_non_sleepable_error_inject) 21216 21217 static int check_non_sleepable_error_inject(u32 btf_id) 21218 { 21219 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 21220 } 21221 21222 int bpf_check_attach_target(struct bpf_verifier_log *log, 21223 const struct bpf_prog *prog, 21224 const struct bpf_prog *tgt_prog, 21225 u32 btf_id, 21226 struct bpf_attach_target_info *tgt_info) 21227 { 21228 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 21229 bool prog_tracing = prog->type == BPF_PROG_TYPE_TRACING; 21230 const char prefix[] = "btf_trace_"; 21231 int ret = 0, subprog = -1, i; 21232 const struct btf_type *t; 21233 bool conservative = true; 21234 const char *tname; 21235 struct btf *btf; 21236 long addr = 0; 21237 struct module *mod = NULL; 21238 21239 if (!btf_id) { 21240 bpf_log(log, "Tracing programs must provide btf_id\n"); 21241 return -EINVAL; 21242 } 21243 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 21244 if (!btf) { 21245 bpf_log(log, 21246 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 21247 return -EINVAL; 21248 } 21249 t = btf_type_by_id(btf, btf_id); 21250 if (!t) { 21251 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 21252 return -EINVAL; 21253 } 21254 tname = btf_name_by_offset(btf, t->name_off); 21255 if (!tname) { 21256 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 21257 return -EINVAL; 21258 } 21259 if (tgt_prog) { 21260 struct bpf_prog_aux *aux = tgt_prog->aux; 21261 21262 if (bpf_prog_is_dev_bound(prog->aux) && 21263 !bpf_prog_dev_bound_match(prog, tgt_prog)) { 21264 bpf_log(log, "Target program bound device mismatch"); 21265 return -EINVAL; 21266 } 21267 21268 for (i = 0; i < aux->func_info_cnt; i++) 21269 if (aux->func_info[i].type_id == btf_id) { 21270 subprog = i; 21271 break; 21272 } 21273 if (subprog == -1) { 21274 bpf_log(log, "Subprog %s doesn't exist\n", tname); 21275 return -EINVAL; 21276 } 21277 if (aux->func && aux->func[subprog]->aux->exception_cb) { 21278 bpf_log(log, 21279 "%s programs cannot attach to exception callback\n", 21280 prog_extension ? "Extension" : "FENTRY/FEXIT"); 21281 return -EINVAL; 21282 } 21283 conservative = aux->func_info_aux[subprog].unreliable; 21284 if (prog_extension) { 21285 if (conservative) { 21286 bpf_log(log, 21287 "Cannot replace static functions\n"); 21288 return -EINVAL; 21289 } 21290 if (!prog->jit_requested) { 21291 bpf_log(log, 21292 "Extension programs should be JITed\n"); 21293 return -EINVAL; 21294 } 21295 } 21296 if (!tgt_prog->jited) { 21297 bpf_log(log, "Can attach to only JITed progs\n"); 21298 return -EINVAL; 21299 } 21300 if (prog_tracing) { 21301 if (aux->attach_tracing_prog) { 21302 /* 21303 * Target program is an fentry/fexit which is already attached 21304 * to another tracing program. More levels of nesting 21305 * attachment are not allowed. 21306 */ 21307 bpf_log(log, "Cannot nest tracing program attach more than once\n"); 21308 return -EINVAL; 21309 } 21310 } else if (tgt_prog->type == prog->type) { 21311 /* 21312 * To avoid potential call chain cycles, prevent attaching of a 21313 * program extension to another extension. It's ok to attach 21314 * fentry/fexit to extension program. 21315 */ 21316 bpf_log(log, "Cannot recursively attach\n"); 21317 return -EINVAL; 21318 } 21319 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 21320 prog_extension && 21321 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 21322 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 21323 /* Program extensions can extend all program types 21324 * except fentry/fexit. The reason is the following. 21325 * The fentry/fexit programs are used for performance 21326 * analysis, stats and can be attached to any program 21327 * type. When extension program is replacing XDP function 21328 * it is necessary to allow performance analysis of all 21329 * functions. Both original XDP program and its program 21330 * extension. Hence attaching fentry/fexit to 21331 * BPF_PROG_TYPE_EXT is allowed. If extending of 21332 * fentry/fexit was allowed it would be possible to create 21333 * long call chain fentry->extension->fentry->extension 21334 * beyond reasonable stack size. Hence extending fentry 21335 * is not allowed. 21336 */ 21337 bpf_log(log, "Cannot extend fentry/fexit\n"); 21338 return -EINVAL; 21339 } 21340 } else { 21341 if (prog_extension) { 21342 bpf_log(log, "Cannot replace kernel functions\n"); 21343 return -EINVAL; 21344 } 21345 } 21346 21347 switch (prog->expected_attach_type) { 21348 case BPF_TRACE_RAW_TP: 21349 if (tgt_prog) { 21350 bpf_log(log, 21351 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 21352 return -EINVAL; 21353 } 21354 if (!btf_type_is_typedef(t)) { 21355 bpf_log(log, "attach_btf_id %u is not a typedef\n", 21356 btf_id); 21357 return -EINVAL; 21358 } 21359 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 21360 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 21361 btf_id, tname); 21362 return -EINVAL; 21363 } 21364 tname += sizeof(prefix) - 1; 21365 t = btf_type_by_id(btf, t->type); 21366 if (!btf_type_is_ptr(t)) 21367 /* should never happen in valid vmlinux build */ 21368 return -EINVAL; 21369 t = btf_type_by_id(btf, t->type); 21370 if (!btf_type_is_func_proto(t)) 21371 /* should never happen in valid vmlinux build */ 21372 return -EINVAL; 21373 21374 break; 21375 case BPF_TRACE_ITER: 21376 if (!btf_type_is_func(t)) { 21377 bpf_log(log, "attach_btf_id %u is not a function\n", 21378 btf_id); 21379 return -EINVAL; 21380 } 21381 t = btf_type_by_id(btf, t->type); 21382 if (!btf_type_is_func_proto(t)) 21383 return -EINVAL; 21384 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 21385 if (ret) 21386 return ret; 21387 break; 21388 default: 21389 if (!prog_extension) 21390 return -EINVAL; 21391 fallthrough; 21392 case BPF_MODIFY_RETURN: 21393 case BPF_LSM_MAC: 21394 case BPF_LSM_CGROUP: 21395 case BPF_TRACE_FENTRY: 21396 case BPF_TRACE_FEXIT: 21397 if (!btf_type_is_func(t)) { 21398 bpf_log(log, "attach_btf_id %u is not a function\n", 21399 btf_id); 21400 return -EINVAL; 21401 } 21402 if (prog_extension && 21403 btf_check_type_match(log, prog, btf, t)) 21404 return -EINVAL; 21405 t = btf_type_by_id(btf, t->type); 21406 if (!btf_type_is_func_proto(t)) 21407 return -EINVAL; 21408 21409 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 21410 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 21411 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 21412 return -EINVAL; 21413 21414 if (tgt_prog && conservative) 21415 t = NULL; 21416 21417 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 21418 if (ret < 0) 21419 return ret; 21420 21421 if (tgt_prog) { 21422 if (subprog == 0) 21423 addr = (long) tgt_prog->bpf_func; 21424 else 21425 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 21426 } else { 21427 if (btf_is_module(btf)) { 21428 mod = btf_try_get_module(btf); 21429 if (mod) 21430 addr = find_kallsyms_symbol_value(mod, tname); 21431 else 21432 addr = 0; 21433 } else { 21434 addr = kallsyms_lookup_name(tname); 21435 } 21436 if (!addr) { 21437 module_put(mod); 21438 bpf_log(log, 21439 "The address of function %s cannot be found\n", 21440 tname); 21441 return -ENOENT; 21442 } 21443 } 21444 21445 if (prog->sleepable) { 21446 ret = -EINVAL; 21447 switch (prog->type) { 21448 case BPF_PROG_TYPE_TRACING: 21449 21450 /* fentry/fexit/fmod_ret progs can be sleepable if they are 21451 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 21452 */ 21453 if (!check_non_sleepable_error_inject(btf_id) && 21454 within_error_injection_list(addr)) 21455 ret = 0; 21456 /* fentry/fexit/fmod_ret progs can also be sleepable if they are 21457 * in the fmodret id set with the KF_SLEEPABLE flag. 21458 */ 21459 else { 21460 u32 *flags = btf_kfunc_is_modify_return(btf, btf_id, 21461 prog); 21462 21463 if (flags && (*flags & KF_SLEEPABLE)) 21464 ret = 0; 21465 } 21466 break; 21467 case BPF_PROG_TYPE_LSM: 21468 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 21469 * Only some of them are sleepable. 21470 */ 21471 if (bpf_lsm_is_sleepable_hook(btf_id)) 21472 ret = 0; 21473 break; 21474 default: 21475 break; 21476 } 21477 if (ret) { 21478 module_put(mod); 21479 bpf_log(log, "%s is not sleepable\n", tname); 21480 return ret; 21481 } 21482 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 21483 if (tgt_prog) { 21484 module_put(mod); 21485 bpf_log(log, "can't modify return codes of BPF programs\n"); 21486 return -EINVAL; 21487 } 21488 ret = -EINVAL; 21489 if (btf_kfunc_is_modify_return(btf, btf_id, prog) || 21490 !check_attach_modify_return(addr, tname)) 21491 ret = 0; 21492 if (ret) { 21493 module_put(mod); 21494 bpf_log(log, "%s() is not modifiable\n", tname); 21495 return ret; 21496 } 21497 } 21498 21499 break; 21500 } 21501 tgt_info->tgt_addr = addr; 21502 tgt_info->tgt_name = tname; 21503 tgt_info->tgt_type = t; 21504 tgt_info->tgt_mod = mod; 21505 return 0; 21506 } 21507 21508 BTF_SET_START(btf_id_deny) 21509 BTF_ID_UNUSED 21510 #ifdef CONFIG_SMP 21511 BTF_ID(func, migrate_disable) 21512 BTF_ID(func, migrate_enable) 21513 #endif 21514 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 21515 BTF_ID(func, rcu_read_unlock_strict) 21516 #endif 21517 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE) 21518 BTF_ID(func, preempt_count_add) 21519 BTF_ID(func, preempt_count_sub) 21520 #endif 21521 #ifdef CONFIG_PREEMPT_RCU 21522 BTF_ID(func, __rcu_read_lock) 21523 BTF_ID(func, __rcu_read_unlock) 21524 #endif 21525 BTF_SET_END(btf_id_deny) 21526 21527 static bool can_be_sleepable(struct bpf_prog *prog) 21528 { 21529 if (prog->type == BPF_PROG_TYPE_TRACING) { 21530 switch (prog->expected_attach_type) { 21531 case BPF_TRACE_FENTRY: 21532 case BPF_TRACE_FEXIT: 21533 case BPF_MODIFY_RETURN: 21534 case BPF_TRACE_ITER: 21535 return true; 21536 default: 21537 return false; 21538 } 21539 } 21540 return prog->type == BPF_PROG_TYPE_LSM || 21541 prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || 21542 prog->type == BPF_PROG_TYPE_STRUCT_OPS; 21543 } 21544 21545 static int check_attach_btf_id(struct bpf_verifier_env *env) 21546 { 21547 struct bpf_prog *prog = env->prog; 21548 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 21549 struct bpf_attach_target_info tgt_info = {}; 21550 u32 btf_id = prog->aux->attach_btf_id; 21551 struct bpf_trampoline *tr; 21552 int ret; 21553 u64 key; 21554 21555 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 21556 if (prog->sleepable) 21557 /* attach_btf_id checked to be zero already */ 21558 return 0; 21559 verbose(env, "Syscall programs can only be sleepable\n"); 21560 return -EINVAL; 21561 } 21562 21563 if (prog->sleepable && !can_be_sleepable(prog)) { 21564 verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); 21565 return -EINVAL; 21566 } 21567 21568 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 21569 return check_struct_ops_btf_id(env); 21570 21571 if (prog->type != BPF_PROG_TYPE_TRACING && 21572 prog->type != BPF_PROG_TYPE_LSM && 21573 prog->type != BPF_PROG_TYPE_EXT) 21574 return 0; 21575 21576 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 21577 if (ret) 21578 return ret; 21579 21580 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 21581 /* to make freplace equivalent to their targets, they need to 21582 * inherit env->ops and expected_attach_type for the rest of the 21583 * verification 21584 */ 21585 env->ops = bpf_verifier_ops[tgt_prog->type]; 21586 prog->expected_attach_type = tgt_prog->expected_attach_type; 21587 } 21588 21589 /* store info about the attachment target that will be used later */ 21590 prog->aux->attach_func_proto = tgt_info.tgt_type; 21591 prog->aux->attach_func_name = tgt_info.tgt_name; 21592 prog->aux->mod = tgt_info.tgt_mod; 21593 21594 if (tgt_prog) { 21595 prog->aux->saved_dst_prog_type = tgt_prog->type; 21596 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 21597 } 21598 21599 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 21600 prog->aux->attach_btf_trace = true; 21601 return 0; 21602 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 21603 if (!bpf_iter_prog_supported(prog)) 21604 return -EINVAL; 21605 return 0; 21606 } 21607 21608 if (prog->type == BPF_PROG_TYPE_LSM) { 21609 ret = bpf_lsm_verify_prog(&env->log, prog); 21610 if (ret < 0) 21611 return ret; 21612 } else if (prog->type == BPF_PROG_TYPE_TRACING && 21613 btf_id_set_contains(&btf_id_deny, btf_id)) { 21614 return -EINVAL; 21615 } 21616 21617 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 21618 tr = bpf_trampoline_get(key, &tgt_info); 21619 if (!tr) 21620 return -ENOMEM; 21621 21622 if (tgt_prog && tgt_prog->aux->tail_call_reachable) 21623 tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; 21624 21625 prog->aux->dst_trampoline = tr; 21626 return 0; 21627 } 21628 21629 struct btf *bpf_get_btf_vmlinux(void) 21630 { 21631 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 21632 mutex_lock(&bpf_verifier_lock); 21633 if (!btf_vmlinux) 21634 btf_vmlinux = btf_parse_vmlinux(); 21635 mutex_unlock(&bpf_verifier_lock); 21636 } 21637 return btf_vmlinux; 21638 } 21639 21640 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) 21641 { 21642 u64 start_time = ktime_get_ns(); 21643 struct bpf_verifier_env *env; 21644 int i, len, ret = -EINVAL, err; 21645 u32 log_true_size; 21646 bool is_priv; 21647 21648 /* no program is valid */ 21649 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 21650 return -EINVAL; 21651 21652 /* 'struct bpf_verifier_env' can be global, but since it's not small, 21653 * allocate/free it every time bpf_check() is called 21654 */ 21655 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 21656 if (!env) 21657 return -ENOMEM; 21658 21659 env->bt.env = env; 21660 21661 len = (*prog)->len; 21662 env->insn_aux_data = 21663 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 21664 ret = -ENOMEM; 21665 if (!env->insn_aux_data) 21666 goto err_free_env; 21667 for (i = 0; i < len; i++) 21668 env->insn_aux_data[i].orig_idx = i; 21669 env->prog = *prog; 21670 env->ops = bpf_verifier_ops[env->prog->type]; 21671 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 21672 21673 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); 21674 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); 21675 env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); 21676 env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); 21677 env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); 21678 21679 bpf_get_btf_vmlinux(); 21680 21681 /* grab the mutex to protect few globals used by verifier */ 21682 if (!is_priv) 21683 mutex_lock(&bpf_verifier_lock); 21684 21685 /* user could have requested verbose verifier output 21686 * and supplied buffer to store the verification trace 21687 */ 21688 ret = bpf_vlog_init(&env->log, attr->log_level, 21689 (char __user *) (unsigned long) attr->log_buf, 21690 attr->log_size); 21691 if (ret) 21692 goto err_unlock; 21693 21694 mark_verifier_state_clean(env); 21695 21696 if (IS_ERR(btf_vmlinux)) { 21697 /* Either gcc or pahole or kernel are broken. */ 21698 verbose(env, "in-kernel BTF is malformed\n"); 21699 ret = PTR_ERR(btf_vmlinux); 21700 goto skip_full_check; 21701 } 21702 21703 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 21704 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 21705 env->strict_alignment = true; 21706 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 21707 env->strict_alignment = false; 21708 21709 if (is_priv) 21710 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 21711 env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; 21712 21713 env->explored_states = kvcalloc(state_htab_size(env), 21714 sizeof(struct bpf_verifier_state_list *), 21715 GFP_USER); 21716 ret = -ENOMEM; 21717 if (!env->explored_states) 21718 goto skip_full_check; 21719 21720 ret = check_btf_info_early(env, attr, uattr); 21721 if (ret < 0) 21722 goto skip_full_check; 21723 21724 ret = add_subprog_and_kfunc(env); 21725 if (ret < 0) 21726 goto skip_full_check; 21727 21728 ret = check_subprogs(env); 21729 if (ret < 0) 21730 goto skip_full_check; 21731 21732 ret = check_btf_info(env, attr, uattr); 21733 if (ret < 0) 21734 goto skip_full_check; 21735 21736 ret = check_attach_btf_id(env); 21737 if (ret) 21738 goto skip_full_check; 21739 21740 ret = resolve_pseudo_ldimm64(env); 21741 if (ret < 0) 21742 goto skip_full_check; 21743 21744 if (bpf_prog_is_offloaded(env->prog->aux)) { 21745 ret = bpf_prog_offload_verifier_prep(env->prog); 21746 if (ret) 21747 goto skip_full_check; 21748 } 21749 21750 ret = check_cfg(env); 21751 if (ret < 0) 21752 goto skip_full_check; 21753 21754 ret = do_check_main(env); 21755 ret = ret ?: do_check_subprogs(env); 21756 21757 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) 21758 ret = bpf_prog_offload_finalize(env); 21759 21760 skip_full_check: 21761 kvfree(env->explored_states); 21762 21763 if (ret == 0) 21764 ret = check_max_stack_depth(env); 21765 21766 /* instruction rewrites happen after this point */ 21767 if (ret == 0) 21768 ret = optimize_bpf_loop(env); 21769 21770 if (is_priv) { 21771 if (ret == 0) 21772 opt_hard_wire_dead_code_branches(env); 21773 if (ret == 0) 21774 ret = opt_remove_dead_code(env); 21775 if (ret == 0) 21776 ret = opt_remove_nops(env); 21777 } else { 21778 if (ret == 0) 21779 sanitize_dead_code(env); 21780 } 21781 21782 if (ret == 0) 21783 /* program is valid, convert *(u32*)(ctx + off) accesses */ 21784 ret = convert_ctx_accesses(env); 21785 21786 if (ret == 0) 21787 ret = do_misc_fixups(env); 21788 21789 /* do 32-bit optimization after insn patching has done so those patched 21790 * insns could be handled correctly. 21791 */ 21792 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { 21793 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 21794 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 21795 : false; 21796 } 21797 21798 if (ret == 0) 21799 ret = fixup_call_args(env); 21800 21801 env->verification_time = ktime_get_ns() - start_time; 21802 print_verification_stats(env); 21803 env->prog->aux->verified_insns = env->insn_processed; 21804 21805 /* preserve original error even if log finalization is successful */ 21806 err = bpf_vlog_finalize(&env->log, &log_true_size); 21807 if (err) 21808 ret = err; 21809 21810 if (uattr_size >= offsetofend(union bpf_attr, log_true_size) && 21811 copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size), 21812 &log_true_size, sizeof(log_true_size))) { 21813 ret = -EFAULT; 21814 goto err_release_maps; 21815 } 21816 21817 if (ret) 21818 goto err_release_maps; 21819 21820 if (env->used_map_cnt) { 21821 /* if program passed verifier, update used_maps in bpf_prog_info */ 21822 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 21823 sizeof(env->used_maps[0]), 21824 GFP_KERNEL); 21825 21826 if (!env->prog->aux->used_maps) { 21827 ret = -ENOMEM; 21828 goto err_release_maps; 21829 } 21830 21831 memcpy(env->prog->aux->used_maps, env->used_maps, 21832 sizeof(env->used_maps[0]) * env->used_map_cnt); 21833 env->prog->aux->used_map_cnt = env->used_map_cnt; 21834 } 21835 if (env->used_btf_cnt) { 21836 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 21837 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 21838 sizeof(env->used_btfs[0]), 21839 GFP_KERNEL); 21840 if (!env->prog->aux->used_btfs) { 21841 ret = -ENOMEM; 21842 goto err_release_maps; 21843 } 21844 21845 memcpy(env->prog->aux->used_btfs, env->used_btfs, 21846 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 21847 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 21848 } 21849 if (env->used_map_cnt || env->used_btf_cnt) { 21850 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 21851 * bpf_ld_imm64 instructions 21852 */ 21853 convert_pseudo_ld_imm64(env); 21854 } 21855 21856 adjust_btf_func(env); 21857 21858 err_release_maps: 21859 if (!env->prog->aux->used_maps) 21860 /* if we didn't copy map pointers into bpf_prog_info, release 21861 * them now. Otherwise free_used_maps() will release them. 21862 */ 21863 release_maps(env); 21864 if (!env->prog->aux->used_btfs) 21865 release_btfs(env); 21866 21867 /* extension progs temporarily inherit the attach_type of their targets 21868 for verification purposes, so set it back to zero before returning 21869 */ 21870 if (env->prog->type == BPF_PROG_TYPE_EXT) 21871 env->prog->expected_attach_type = 0; 21872 21873 *prog = env->prog; 21874 21875 module_put(env->attach_btf_mod); 21876 err_unlock: 21877 if (!is_priv) 21878 mutex_unlock(&bpf_verifier_lock); 21879 vfree(env->insn_aux_data); 21880 err_free_env: 21881 kfree(env); 21882 return ret; 21883 } 21884