1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/bpf.h> 11 #include <linux/btf.h> 12 #include <linux/bpf_verifier.h> 13 #include <linux/filter.h> 14 #include <net/netlink.h> 15 #include <linux/file.h> 16 #include <linux/vmalloc.h> 17 #include <linux/stringify.h> 18 #include <linux/bsearch.h> 19 #include <linux/sort.h> 20 #include <linux/perf_event.h> 21 #include <linux/ctype.h> 22 #include <linux/error-injection.h> 23 #include <linux/bpf_lsm.h> 24 #include <linux/btf_ids.h> 25 26 #include "disasm.h" 27 28 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 29 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 30 [_id] = & _name ## _verifier_ops, 31 #define BPF_MAP_TYPE(_id, _ops) 32 #define BPF_LINK_TYPE(_id, _name) 33 #include <linux/bpf_types.h> 34 #undef BPF_PROG_TYPE 35 #undef BPF_MAP_TYPE 36 #undef BPF_LINK_TYPE 37 }; 38 39 /* bpf_check() is a static code analyzer that walks eBPF program 40 * instruction by instruction and updates register/stack state. 41 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 42 * 43 * The first pass is depth-first-search to check that the program is a DAG. 44 * It rejects the following programs: 45 * - larger than BPF_MAXINSNS insns 46 * - if loop is present (detected via back-edge) 47 * - unreachable insns exist (shouldn't be a forest. program = one function) 48 * - out of bounds or malformed jumps 49 * The second pass is all possible path descent from the 1st insn. 50 * Since it's analyzing all pathes through the program, the length of the 51 * analysis is limited to 64k insn, which may be hit even if total number of 52 * insn is less then 4K, but there are too many branches that change stack/regs. 53 * Number of 'branches to be analyzed' is limited to 1k 54 * 55 * On entry to each instruction, each register has a type, and the instruction 56 * changes the types of the registers depending on instruction semantics. 57 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 58 * copied to R1. 59 * 60 * All registers are 64-bit. 61 * R0 - return register 62 * R1-R5 argument passing registers 63 * R6-R9 callee saved registers 64 * R10 - frame pointer read-only 65 * 66 * At the start of BPF program the register R1 contains a pointer to bpf_context 67 * and has type PTR_TO_CTX. 68 * 69 * Verifier tracks arithmetic operations on pointers in case: 70 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 71 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 72 * 1st insn copies R10 (which has FRAME_PTR) type into R1 73 * and 2nd arithmetic instruction is pattern matched to recognize 74 * that it wants to construct a pointer to some element within stack. 75 * So after 2nd insn, the register R1 has type PTR_TO_STACK 76 * (and -20 constant is saved for further stack bounds checking). 77 * Meaning that this reg is a pointer to stack plus known immediate constant. 78 * 79 * Most of the time the registers have SCALAR_VALUE type, which 80 * means the register has some value, but it's not a valid pointer. 81 * (like pointer plus pointer becomes SCALAR_VALUE type) 82 * 83 * When verifier sees load or store instructions the type of base register 84 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 85 * four pointer types recognized by check_mem_access() function. 86 * 87 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 88 * and the range of [ptr, ptr + map's value_size) is accessible. 89 * 90 * registers used to pass values to function calls are checked against 91 * function argument constraints. 92 * 93 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 94 * It means that the register type passed to this function must be 95 * PTR_TO_STACK and it will be used inside the function as 96 * 'pointer to map element key' 97 * 98 * For example the argument constraints for bpf_map_lookup_elem(): 99 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 100 * .arg1_type = ARG_CONST_MAP_PTR, 101 * .arg2_type = ARG_PTR_TO_MAP_KEY, 102 * 103 * ret_type says that this function returns 'pointer to map elem value or null' 104 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 105 * 2nd argument should be a pointer to stack, which will be used inside 106 * the helper function as a pointer to map element key. 107 * 108 * On the kernel side the helper function looks like: 109 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 110 * { 111 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 112 * void *key = (void *) (unsigned long) r2; 113 * void *value; 114 * 115 * here kernel can access 'key' and 'map' pointers safely, knowing that 116 * [key, key + map->key_size) bytes are valid and were initialized on 117 * the stack of eBPF program. 118 * } 119 * 120 * Corresponding eBPF program may look like: 121 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 122 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 123 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 124 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 125 * here verifier looks at prototype of map_lookup_elem() and sees: 126 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 127 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 128 * 129 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 130 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 131 * and were initialized prior to this call. 132 * If it's ok, then verifier allows this BPF_CALL insn and looks at 133 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 134 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 135 * returns ether pointer to map value or NULL. 136 * 137 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 138 * insn, the register holding that pointer in the true branch changes state to 139 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 140 * branch. See check_cond_jmp_op(). 141 * 142 * After the call R0 is set to return type of the function and registers R1-R5 143 * are set to NOT_INIT to indicate that they are no longer readable. 144 * 145 * The following reference types represent a potential reference to a kernel 146 * resource which, after first being allocated, must be checked and freed by 147 * the BPF program: 148 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 149 * 150 * When the verifier sees a helper call return a reference type, it allocates a 151 * pointer id for the reference and stores it in the current function state. 152 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 153 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 154 * passes through a NULL-check conditional. For the branch wherein the state is 155 * changed to CONST_IMM, the verifier releases the reference. 156 * 157 * For each helper function that allocates a reference, such as 158 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 159 * bpf_sk_release(). When a reference type passes into the release function, 160 * the verifier also releases the reference. If any unchecked or unreleased 161 * reference remains at the end of the program, the verifier rejects it. 162 */ 163 164 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 165 struct bpf_verifier_stack_elem { 166 /* verifer state is 'st' 167 * before processing instruction 'insn_idx' 168 * and after processing instruction 'prev_insn_idx' 169 */ 170 struct bpf_verifier_state st; 171 int insn_idx; 172 int prev_insn_idx; 173 struct bpf_verifier_stack_elem *next; 174 /* length of verifier log at the time this state was pushed on stack */ 175 u32 log_pos; 176 }; 177 178 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 179 #define BPF_COMPLEXITY_LIMIT_STATES 64 180 181 #define BPF_MAP_KEY_POISON (1ULL << 63) 182 #define BPF_MAP_KEY_SEEN (1ULL << 62) 183 184 #define BPF_MAP_PTR_UNPRIV 1UL 185 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 186 POISON_POINTER_DELTA)) 187 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 188 189 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 190 { 191 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 192 } 193 194 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 195 { 196 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 197 } 198 199 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 200 const struct bpf_map *map, bool unpriv) 201 { 202 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 203 unpriv |= bpf_map_ptr_unpriv(aux); 204 aux->map_ptr_state = (unsigned long)map | 205 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 206 } 207 208 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 209 { 210 return aux->map_key_state & BPF_MAP_KEY_POISON; 211 } 212 213 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 214 { 215 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 216 } 217 218 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 219 { 220 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 221 } 222 223 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 224 { 225 bool poisoned = bpf_map_key_poisoned(aux); 226 227 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 229 } 230 231 struct bpf_call_arg_meta { 232 struct bpf_map *map_ptr; 233 bool raw_mode; 234 bool pkt_access; 235 int regno; 236 int access_size; 237 int mem_size; 238 u64 msize_max_value; 239 int ref_obj_id; 240 int func_id; 241 struct btf *btf; 242 u32 btf_id; 243 struct btf *ret_btf; 244 u32 ret_btf_id; 245 }; 246 247 struct btf *btf_vmlinux; 248 249 static DEFINE_MUTEX(bpf_verifier_lock); 250 251 static const struct bpf_line_info * 252 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 253 { 254 const struct bpf_line_info *linfo; 255 const struct bpf_prog *prog; 256 u32 i, nr_linfo; 257 258 prog = env->prog; 259 nr_linfo = prog->aux->nr_linfo; 260 261 if (!nr_linfo || insn_off >= prog->len) 262 return NULL; 263 264 linfo = prog->aux->linfo; 265 for (i = 1; i < nr_linfo; i++) 266 if (insn_off < linfo[i].insn_off) 267 break; 268 269 return &linfo[i - 1]; 270 } 271 272 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 273 va_list args) 274 { 275 unsigned int n; 276 277 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 278 279 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 280 "verifier log line truncated - local buffer too short\n"); 281 282 n = min(log->len_total - log->len_used - 1, n); 283 log->kbuf[n] = '\0'; 284 285 if (log->level == BPF_LOG_KERNEL) { 286 pr_err("BPF:%s\n", log->kbuf); 287 return; 288 } 289 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 290 log->len_used += n; 291 else 292 log->ubuf = NULL; 293 } 294 295 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 296 { 297 char zero = 0; 298 299 if (!bpf_verifier_log_needed(log)) 300 return; 301 302 log->len_used = new_pos; 303 if (put_user(zero, log->ubuf + new_pos)) 304 log->ubuf = NULL; 305 } 306 307 /* log_level controls verbosity level of eBPF verifier. 308 * bpf_verifier_log_write() is used to dump the verification trace to the log, 309 * so the user can figure out what's wrong with the program 310 */ 311 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 312 const char *fmt, ...) 313 { 314 va_list args; 315 316 if (!bpf_verifier_log_needed(&env->log)) 317 return; 318 319 va_start(args, fmt); 320 bpf_verifier_vlog(&env->log, fmt, args); 321 va_end(args); 322 } 323 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 324 325 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 326 { 327 struct bpf_verifier_env *env = private_data; 328 va_list args; 329 330 if (!bpf_verifier_log_needed(&env->log)) 331 return; 332 333 va_start(args, fmt); 334 bpf_verifier_vlog(&env->log, fmt, args); 335 va_end(args); 336 } 337 338 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 339 const char *fmt, ...) 340 { 341 va_list args; 342 343 if (!bpf_verifier_log_needed(log)) 344 return; 345 346 va_start(args, fmt); 347 bpf_verifier_vlog(log, fmt, args); 348 va_end(args); 349 } 350 351 static const char *ltrim(const char *s) 352 { 353 while (isspace(*s)) 354 s++; 355 356 return s; 357 } 358 359 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 360 u32 insn_off, 361 const char *prefix_fmt, ...) 362 { 363 const struct bpf_line_info *linfo; 364 365 if (!bpf_verifier_log_needed(&env->log)) 366 return; 367 368 linfo = find_linfo(env, insn_off); 369 if (!linfo || linfo == env->prev_linfo) 370 return; 371 372 if (prefix_fmt) { 373 va_list args; 374 375 va_start(args, prefix_fmt); 376 bpf_verifier_vlog(&env->log, prefix_fmt, args); 377 va_end(args); 378 } 379 380 verbose(env, "%s\n", 381 ltrim(btf_name_by_offset(env->prog->aux->btf, 382 linfo->line_off))); 383 384 env->prev_linfo = linfo; 385 } 386 387 static bool type_is_pkt_pointer(enum bpf_reg_type type) 388 { 389 return type == PTR_TO_PACKET || 390 type == PTR_TO_PACKET_META; 391 } 392 393 static bool type_is_sk_pointer(enum bpf_reg_type type) 394 { 395 return type == PTR_TO_SOCKET || 396 type == PTR_TO_SOCK_COMMON || 397 type == PTR_TO_TCP_SOCK || 398 type == PTR_TO_XDP_SOCK; 399 } 400 401 static bool reg_type_not_null(enum bpf_reg_type type) 402 { 403 return type == PTR_TO_SOCKET || 404 type == PTR_TO_TCP_SOCK || 405 type == PTR_TO_MAP_VALUE || 406 type == PTR_TO_SOCK_COMMON; 407 } 408 409 static bool reg_type_may_be_null(enum bpf_reg_type type) 410 { 411 return type == PTR_TO_MAP_VALUE_OR_NULL || 412 type == PTR_TO_SOCKET_OR_NULL || 413 type == PTR_TO_SOCK_COMMON_OR_NULL || 414 type == PTR_TO_TCP_SOCK_OR_NULL || 415 type == PTR_TO_BTF_ID_OR_NULL || 416 type == PTR_TO_MEM_OR_NULL || 417 type == PTR_TO_RDONLY_BUF_OR_NULL || 418 type == PTR_TO_RDWR_BUF_OR_NULL; 419 } 420 421 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 422 { 423 return reg->type == PTR_TO_MAP_VALUE && 424 map_value_has_spin_lock(reg->map_ptr); 425 } 426 427 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) 428 { 429 return type == PTR_TO_SOCKET || 430 type == PTR_TO_SOCKET_OR_NULL || 431 type == PTR_TO_TCP_SOCK || 432 type == PTR_TO_TCP_SOCK_OR_NULL || 433 type == PTR_TO_MEM || 434 type == PTR_TO_MEM_OR_NULL; 435 } 436 437 static bool arg_type_may_be_refcounted(enum bpf_arg_type type) 438 { 439 return type == ARG_PTR_TO_SOCK_COMMON; 440 } 441 442 static bool arg_type_may_be_null(enum bpf_arg_type type) 443 { 444 return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || 445 type == ARG_PTR_TO_MEM_OR_NULL || 446 type == ARG_PTR_TO_CTX_OR_NULL || 447 type == ARG_PTR_TO_SOCKET_OR_NULL || 448 type == ARG_PTR_TO_ALLOC_MEM_OR_NULL; 449 } 450 451 /* Determine whether the function releases some resources allocated by another 452 * function call. The first reference type argument will be assumed to be 453 * released by release_reference(). 454 */ 455 static bool is_release_function(enum bpf_func_id func_id) 456 { 457 return func_id == BPF_FUNC_sk_release || 458 func_id == BPF_FUNC_ringbuf_submit || 459 func_id == BPF_FUNC_ringbuf_discard; 460 } 461 462 static bool may_be_acquire_function(enum bpf_func_id func_id) 463 { 464 return func_id == BPF_FUNC_sk_lookup_tcp || 465 func_id == BPF_FUNC_sk_lookup_udp || 466 func_id == BPF_FUNC_skc_lookup_tcp || 467 func_id == BPF_FUNC_map_lookup_elem || 468 func_id == BPF_FUNC_ringbuf_reserve; 469 } 470 471 static bool is_acquire_function(enum bpf_func_id func_id, 472 const struct bpf_map *map) 473 { 474 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 475 476 if (func_id == BPF_FUNC_sk_lookup_tcp || 477 func_id == BPF_FUNC_sk_lookup_udp || 478 func_id == BPF_FUNC_skc_lookup_tcp || 479 func_id == BPF_FUNC_ringbuf_reserve) 480 return true; 481 482 if (func_id == BPF_FUNC_map_lookup_elem && 483 (map_type == BPF_MAP_TYPE_SOCKMAP || 484 map_type == BPF_MAP_TYPE_SOCKHASH)) 485 return true; 486 487 return false; 488 } 489 490 static bool is_ptr_cast_function(enum bpf_func_id func_id) 491 { 492 return func_id == BPF_FUNC_tcp_sock || 493 func_id == BPF_FUNC_sk_fullsock || 494 func_id == BPF_FUNC_skc_to_tcp_sock || 495 func_id == BPF_FUNC_skc_to_tcp6_sock || 496 func_id == BPF_FUNC_skc_to_udp6_sock || 497 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 498 func_id == BPF_FUNC_skc_to_tcp_request_sock; 499 } 500 501 /* string representation of 'enum bpf_reg_type' */ 502 static const char * const reg_type_str[] = { 503 [NOT_INIT] = "?", 504 [SCALAR_VALUE] = "inv", 505 [PTR_TO_CTX] = "ctx", 506 [CONST_PTR_TO_MAP] = "map_ptr", 507 [PTR_TO_MAP_VALUE] = "map_value", 508 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", 509 [PTR_TO_STACK] = "fp", 510 [PTR_TO_PACKET] = "pkt", 511 [PTR_TO_PACKET_META] = "pkt_meta", 512 [PTR_TO_PACKET_END] = "pkt_end", 513 [PTR_TO_FLOW_KEYS] = "flow_keys", 514 [PTR_TO_SOCKET] = "sock", 515 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", 516 [PTR_TO_SOCK_COMMON] = "sock_common", 517 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", 518 [PTR_TO_TCP_SOCK] = "tcp_sock", 519 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", 520 [PTR_TO_TP_BUFFER] = "tp_buffer", 521 [PTR_TO_XDP_SOCK] = "xdp_sock", 522 [PTR_TO_BTF_ID] = "ptr_", 523 [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", 524 [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", 525 [PTR_TO_MEM] = "mem", 526 [PTR_TO_MEM_OR_NULL] = "mem_or_null", 527 [PTR_TO_RDONLY_BUF] = "rdonly_buf", 528 [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", 529 [PTR_TO_RDWR_BUF] = "rdwr_buf", 530 [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", 531 }; 532 533 static char slot_type_char[] = { 534 [STACK_INVALID] = '?', 535 [STACK_SPILL] = 'r', 536 [STACK_MISC] = 'm', 537 [STACK_ZERO] = '0', 538 }; 539 540 static void print_liveness(struct bpf_verifier_env *env, 541 enum bpf_reg_liveness live) 542 { 543 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 544 verbose(env, "_"); 545 if (live & REG_LIVE_READ) 546 verbose(env, "r"); 547 if (live & REG_LIVE_WRITTEN) 548 verbose(env, "w"); 549 if (live & REG_LIVE_DONE) 550 verbose(env, "D"); 551 } 552 553 static struct bpf_func_state *func(struct bpf_verifier_env *env, 554 const struct bpf_reg_state *reg) 555 { 556 struct bpf_verifier_state *cur = env->cur_state; 557 558 return cur->frame[reg->frameno]; 559 } 560 561 static const char *kernel_type_name(const struct btf* btf, u32 id) 562 { 563 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 564 } 565 566 static void print_verifier_state(struct bpf_verifier_env *env, 567 const struct bpf_func_state *state) 568 { 569 const struct bpf_reg_state *reg; 570 enum bpf_reg_type t; 571 int i; 572 573 if (state->frameno) 574 verbose(env, " frame%d:", state->frameno); 575 for (i = 0; i < MAX_BPF_REG; i++) { 576 reg = &state->regs[i]; 577 t = reg->type; 578 if (t == NOT_INIT) 579 continue; 580 verbose(env, " R%d", i); 581 print_liveness(env, reg->live); 582 verbose(env, "=%s", reg_type_str[t]); 583 if (t == SCALAR_VALUE && reg->precise) 584 verbose(env, "P"); 585 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 586 tnum_is_const(reg->var_off)) { 587 /* reg->off should be 0 for SCALAR_VALUE */ 588 verbose(env, "%lld", reg->var_off.value + reg->off); 589 } else { 590 if (t == PTR_TO_BTF_ID || 591 t == PTR_TO_BTF_ID_OR_NULL || 592 t == PTR_TO_PERCPU_BTF_ID) 593 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 594 verbose(env, "(id=%d", reg->id); 595 if (reg_type_may_be_refcounted_or_null(t)) 596 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); 597 if (t != SCALAR_VALUE) 598 verbose(env, ",off=%d", reg->off); 599 if (type_is_pkt_pointer(t)) 600 verbose(env, ",r=%d", reg->range); 601 else if (t == CONST_PTR_TO_MAP || 602 t == PTR_TO_MAP_VALUE || 603 t == PTR_TO_MAP_VALUE_OR_NULL) 604 verbose(env, ",ks=%d,vs=%d", 605 reg->map_ptr->key_size, 606 reg->map_ptr->value_size); 607 if (tnum_is_const(reg->var_off)) { 608 /* Typically an immediate SCALAR_VALUE, but 609 * could be a pointer whose offset is too big 610 * for reg->off 611 */ 612 verbose(env, ",imm=%llx", reg->var_off.value); 613 } else { 614 if (reg->smin_value != reg->umin_value && 615 reg->smin_value != S64_MIN) 616 verbose(env, ",smin_value=%lld", 617 (long long)reg->smin_value); 618 if (reg->smax_value != reg->umax_value && 619 reg->smax_value != S64_MAX) 620 verbose(env, ",smax_value=%lld", 621 (long long)reg->smax_value); 622 if (reg->umin_value != 0) 623 verbose(env, ",umin_value=%llu", 624 (unsigned long long)reg->umin_value); 625 if (reg->umax_value != U64_MAX) 626 verbose(env, ",umax_value=%llu", 627 (unsigned long long)reg->umax_value); 628 if (!tnum_is_unknown(reg->var_off)) { 629 char tn_buf[48]; 630 631 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 632 verbose(env, ",var_off=%s", tn_buf); 633 } 634 if (reg->s32_min_value != reg->smin_value && 635 reg->s32_min_value != S32_MIN) 636 verbose(env, ",s32_min_value=%d", 637 (int)(reg->s32_min_value)); 638 if (reg->s32_max_value != reg->smax_value && 639 reg->s32_max_value != S32_MAX) 640 verbose(env, ",s32_max_value=%d", 641 (int)(reg->s32_max_value)); 642 if (reg->u32_min_value != reg->umin_value && 643 reg->u32_min_value != U32_MIN) 644 verbose(env, ",u32_min_value=%d", 645 (int)(reg->u32_min_value)); 646 if (reg->u32_max_value != reg->umax_value && 647 reg->u32_max_value != U32_MAX) 648 verbose(env, ",u32_max_value=%d", 649 (int)(reg->u32_max_value)); 650 } 651 verbose(env, ")"); 652 } 653 } 654 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 655 char types_buf[BPF_REG_SIZE + 1]; 656 bool valid = false; 657 int j; 658 659 for (j = 0; j < BPF_REG_SIZE; j++) { 660 if (state->stack[i].slot_type[j] != STACK_INVALID) 661 valid = true; 662 types_buf[j] = slot_type_char[ 663 state->stack[i].slot_type[j]]; 664 } 665 types_buf[BPF_REG_SIZE] = 0; 666 if (!valid) 667 continue; 668 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 669 print_liveness(env, state->stack[i].spilled_ptr.live); 670 if (state->stack[i].slot_type[0] == STACK_SPILL) { 671 reg = &state->stack[i].spilled_ptr; 672 t = reg->type; 673 verbose(env, "=%s", reg_type_str[t]); 674 if (t == SCALAR_VALUE && reg->precise) 675 verbose(env, "P"); 676 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 677 verbose(env, "%lld", reg->var_off.value + reg->off); 678 } else { 679 verbose(env, "=%s", types_buf); 680 } 681 } 682 if (state->acquired_refs && state->refs[0].id) { 683 verbose(env, " refs=%d", state->refs[0].id); 684 for (i = 1; i < state->acquired_refs; i++) 685 if (state->refs[i].id) 686 verbose(env, ",%d", state->refs[i].id); 687 } 688 verbose(env, "\n"); 689 } 690 691 #define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ 692 static int copy_##NAME##_state(struct bpf_func_state *dst, \ 693 const struct bpf_func_state *src) \ 694 { \ 695 if (!src->FIELD) \ 696 return 0; \ 697 if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ 698 /* internal bug, make state invalid to reject the program */ \ 699 memset(dst, 0, sizeof(*dst)); \ 700 return -EFAULT; \ 701 } \ 702 memcpy(dst->FIELD, src->FIELD, \ 703 sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ 704 return 0; \ 705 } 706 /* copy_reference_state() */ 707 COPY_STATE_FN(reference, acquired_refs, refs, 1) 708 /* copy_stack_state() */ 709 COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) 710 #undef COPY_STATE_FN 711 712 #define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ 713 static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ 714 bool copy_old) \ 715 { \ 716 u32 old_size = state->COUNT; \ 717 struct bpf_##NAME##_state *new_##FIELD; \ 718 int slot = size / SIZE; \ 719 \ 720 if (size <= old_size || !size) { \ 721 if (copy_old) \ 722 return 0; \ 723 state->COUNT = slot * SIZE; \ 724 if (!size && old_size) { \ 725 kfree(state->FIELD); \ 726 state->FIELD = NULL; \ 727 } \ 728 return 0; \ 729 } \ 730 new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ 731 GFP_KERNEL); \ 732 if (!new_##FIELD) \ 733 return -ENOMEM; \ 734 if (copy_old) { \ 735 if (state->FIELD) \ 736 memcpy(new_##FIELD, state->FIELD, \ 737 sizeof(*new_##FIELD) * (old_size / SIZE)); \ 738 memset(new_##FIELD + old_size / SIZE, 0, \ 739 sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ 740 } \ 741 state->COUNT = slot * SIZE; \ 742 kfree(state->FIELD); \ 743 state->FIELD = new_##FIELD; \ 744 return 0; \ 745 } 746 /* realloc_reference_state() */ 747 REALLOC_STATE_FN(reference, acquired_refs, refs, 1) 748 /* realloc_stack_state() */ 749 REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) 750 #undef REALLOC_STATE_FN 751 752 /* do_check() starts with zero-sized stack in struct bpf_verifier_state to 753 * make it consume minimal amount of memory. check_stack_write() access from 754 * the program calls into realloc_func_state() to grow the stack size. 755 * Note there is a non-zero 'parent' pointer inside bpf_verifier_state 756 * which realloc_stack_state() copies over. It points to previous 757 * bpf_verifier_state which is never reallocated. 758 */ 759 static int realloc_func_state(struct bpf_func_state *state, int stack_size, 760 int refs_size, bool copy_old) 761 { 762 int err = realloc_reference_state(state, refs_size, copy_old); 763 if (err) 764 return err; 765 return realloc_stack_state(state, stack_size, copy_old); 766 } 767 768 /* Acquire a pointer id from the env and update the state->refs to include 769 * this new pointer reference. 770 * On success, returns a valid pointer id to associate with the register 771 * On failure, returns a negative errno. 772 */ 773 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 774 { 775 struct bpf_func_state *state = cur_func(env); 776 int new_ofs = state->acquired_refs; 777 int id, err; 778 779 err = realloc_reference_state(state, state->acquired_refs + 1, true); 780 if (err) 781 return err; 782 id = ++env->id_gen; 783 state->refs[new_ofs].id = id; 784 state->refs[new_ofs].insn_idx = insn_idx; 785 786 return id; 787 } 788 789 /* release function corresponding to acquire_reference_state(). Idempotent. */ 790 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 791 { 792 int i, last_idx; 793 794 last_idx = state->acquired_refs - 1; 795 for (i = 0; i < state->acquired_refs; i++) { 796 if (state->refs[i].id == ptr_id) { 797 if (last_idx && i != last_idx) 798 memcpy(&state->refs[i], &state->refs[last_idx], 799 sizeof(*state->refs)); 800 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 801 state->acquired_refs--; 802 return 0; 803 } 804 } 805 return -EINVAL; 806 } 807 808 static int transfer_reference_state(struct bpf_func_state *dst, 809 struct bpf_func_state *src) 810 { 811 int err = realloc_reference_state(dst, src->acquired_refs, false); 812 if (err) 813 return err; 814 err = copy_reference_state(dst, src); 815 if (err) 816 return err; 817 return 0; 818 } 819 820 static void free_func_state(struct bpf_func_state *state) 821 { 822 if (!state) 823 return; 824 kfree(state->refs); 825 kfree(state->stack); 826 kfree(state); 827 } 828 829 static void clear_jmp_history(struct bpf_verifier_state *state) 830 { 831 kfree(state->jmp_history); 832 state->jmp_history = NULL; 833 state->jmp_history_cnt = 0; 834 } 835 836 static void free_verifier_state(struct bpf_verifier_state *state, 837 bool free_self) 838 { 839 int i; 840 841 for (i = 0; i <= state->curframe; i++) { 842 free_func_state(state->frame[i]); 843 state->frame[i] = NULL; 844 } 845 clear_jmp_history(state); 846 if (free_self) 847 kfree(state); 848 } 849 850 /* copy verifier state from src to dst growing dst stack space 851 * when necessary to accommodate larger src stack 852 */ 853 static int copy_func_state(struct bpf_func_state *dst, 854 const struct bpf_func_state *src) 855 { 856 int err; 857 858 err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, 859 false); 860 if (err) 861 return err; 862 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 863 err = copy_reference_state(dst, src); 864 if (err) 865 return err; 866 return copy_stack_state(dst, src); 867 } 868 869 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 870 const struct bpf_verifier_state *src) 871 { 872 struct bpf_func_state *dst; 873 u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt; 874 int i, err; 875 876 if (dst_state->jmp_history_cnt < src->jmp_history_cnt) { 877 kfree(dst_state->jmp_history); 878 dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER); 879 if (!dst_state->jmp_history) 880 return -ENOMEM; 881 } 882 memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz); 883 dst_state->jmp_history_cnt = src->jmp_history_cnt; 884 885 /* if dst has more stack frames then src frame, free them */ 886 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 887 free_func_state(dst_state->frame[i]); 888 dst_state->frame[i] = NULL; 889 } 890 dst_state->speculative = src->speculative; 891 dst_state->curframe = src->curframe; 892 dst_state->active_spin_lock = src->active_spin_lock; 893 dst_state->branches = src->branches; 894 dst_state->parent = src->parent; 895 dst_state->first_insn_idx = src->first_insn_idx; 896 dst_state->last_insn_idx = src->last_insn_idx; 897 for (i = 0; i <= src->curframe; i++) { 898 dst = dst_state->frame[i]; 899 if (!dst) { 900 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 901 if (!dst) 902 return -ENOMEM; 903 dst_state->frame[i] = dst; 904 } 905 err = copy_func_state(dst, src->frame[i]); 906 if (err) 907 return err; 908 } 909 return 0; 910 } 911 912 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 913 { 914 while (st) { 915 u32 br = --st->branches; 916 917 /* WARN_ON(br > 1) technically makes sense here, 918 * but see comment in push_stack(), hence: 919 */ 920 WARN_ONCE((int)br < 0, 921 "BUG update_branch_counts:branches_to_explore=%d\n", 922 br); 923 if (br) 924 break; 925 st = st->parent; 926 } 927 } 928 929 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 930 int *insn_idx, bool pop_log) 931 { 932 struct bpf_verifier_state *cur = env->cur_state; 933 struct bpf_verifier_stack_elem *elem, *head = env->head; 934 int err; 935 936 if (env->head == NULL) 937 return -ENOENT; 938 939 if (cur) { 940 err = copy_verifier_state(cur, &head->st); 941 if (err) 942 return err; 943 } 944 if (pop_log) 945 bpf_vlog_reset(&env->log, head->log_pos); 946 if (insn_idx) 947 *insn_idx = head->insn_idx; 948 if (prev_insn_idx) 949 *prev_insn_idx = head->prev_insn_idx; 950 elem = head->next; 951 free_verifier_state(&head->st, false); 952 kfree(head); 953 env->head = elem; 954 env->stack_size--; 955 return 0; 956 } 957 958 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 959 int insn_idx, int prev_insn_idx, 960 bool speculative) 961 { 962 struct bpf_verifier_state *cur = env->cur_state; 963 struct bpf_verifier_stack_elem *elem; 964 int err; 965 966 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 967 if (!elem) 968 goto err; 969 970 elem->insn_idx = insn_idx; 971 elem->prev_insn_idx = prev_insn_idx; 972 elem->next = env->head; 973 elem->log_pos = env->log.len_used; 974 env->head = elem; 975 env->stack_size++; 976 err = copy_verifier_state(&elem->st, cur); 977 if (err) 978 goto err; 979 elem->st.speculative |= speculative; 980 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 981 verbose(env, "The sequence of %d jumps is too complex.\n", 982 env->stack_size); 983 goto err; 984 } 985 if (elem->st.parent) { 986 ++elem->st.parent->branches; 987 /* WARN_ON(branches > 2) technically makes sense here, 988 * but 989 * 1. speculative states will bump 'branches' for non-branch 990 * instructions 991 * 2. is_state_visited() heuristics may decide not to create 992 * a new state for a sequence of branches and all such current 993 * and cloned states will be pointing to a single parent state 994 * which might have large 'branches' count. 995 */ 996 } 997 return &elem->st; 998 err: 999 free_verifier_state(env->cur_state, true); 1000 env->cur_state = NULL; 1001 /* pop all elements and return */ 1002 while (!pop_stack(env, NULL, NULL, false)); 1003 return NULL; 1004 } 1005 1006 #define CALLER_SAVED_REGS 6 1007 static const int caller_saved[CALLER_SAVED_REGS] = { 1008 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1009 }; 1010 1011 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1012 struct bpf_reg_state *reg); 1013 1014 /* This helper doesn't clear reg->id */ 1015 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1016 { 1017 reg->var_off = tnum_const(imm); 1018 reg->smin_value = (s64)imm; 1019 reg->smax_value = (s64)imm; 1020 reg->umin_value = imm; 1021 reg->umax_value = imm; 1022 1023 reg->s32_min_value = (s32)imm; 1024 reg->s32_max_value = (s32)imm; 1025 reg->u32_min_value = (u32)imm; 1026 reg->u32_max_value = (u32)imm; 1027 } 1028 1029 /* Mark the unknown part of a register (variable offset or scalar value) as 1030 * known to have the value @imm. 1031 */ 1032 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1033 { 1034 /* Clear id, off, and union(map_ptr, range) */ 1035 memset(((u8 *)reg) + sizeof(reg->type), 0, 1036 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1037 ___mark_reg_known(reg, imm); 1038 } 1039 1040 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1041 { 1042 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1043 reg->s32_min_value = (s32)imm; 1044 reg->s32_max_value = (s32)imm; 1045 reg->u32_min_value = (u32)imm; 1046 reg->u32_max_value = (u32)imm; 1047 } 1048 1049 /* Mark the 'variable offset' part of a register as zero. This should be 1050 * used only on registers holding a pointer type. 1051 */ 1052 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1053 { 1054 __mark_reg_known(reg, 0); 1055 } 1056 1057 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1058 { 1059 __mark_reg_known(reg, 0); 1060 reg->type = SCALAR_VALUE; 1061 } 1062 1063 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1064 struct bpf_reg_state *regs, u32 regno) 1065 { 1066 if (WARN_ON(regno >= MAX_BPF_REG)) { 1067 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1068 /* Something bad happened, let's kill all regs */ 1069 for (regno = 0; regno < MAX_BPF_REG; regno++) 1070 __mark_reg_not_init(env, regs + regno); 1071 return; 1072 } 1073 __mark_reg_known_zero(regs + regno); 1074 } 1075 1076 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1077 { 1078 return type_is_pkt_pointer(reg->type); 1079 } 1080 1081 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1082 { 1083 return reg_is_pkt_pointer(reg) || 1084 reg->type == PTR_TO_PACKET_END; 1085 } 1086 1087 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1088 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1089 enum bpf_reg_type which) 1090 { 1091 /* The register can already have a range from prior markings. 1092 * This is fine as long as it hasn't been advanced from its 1093 * origin. 1094 */ 1095 return reg->type == which && 1096 reg->id == 0 && 1097 reg->off == 0 && 1098 tnum_equals_const(reg->var_off, 0); 1099 } 1100 1101 /* Reset the min/max bounds of a register */ 1102 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1103 { 1104 reg->smin_value = S64_MIN; 1105 reg->smax_value = S64_MAX; 1106 reg->umin_value = 0; 1107 reg->umax_value = U64_MAX; 1108 1109 reg->s32_min_value = S32_MIN; 1110 reg->s32_max_value = S32_MAX; 1111 reg->u32_min_value = 0; 1112 reg->u32_max_value = U32_MAX; 1113 } 1114 1115 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1116 { 1117 reg->smin_value = S64_MIN; 1118 reg->smax_value = S64_MAX; 1119 reg->umin_value = 0; 1120 reg->umax_value = U64_MAX; 1121 } 1122 1123 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1124 { 1125 reg->s32_min_value = S32_MIN; 1126 reg->s32_max_value = S32_MAX; 1127 reg->u32_min_value = 0; 1128 reg->u32_max_value = U32_MAX; 1129 } 1130 1131 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1132 { 1133 struct tnum var32_off = tnum_subreg(reg->var_off); 1134 1135 /* min signed is max(sign bit) | min(other bits) */ 1136 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1137 var32_off.value | (var32_off.mask & S32_MIN)); 1138 /* max signed is min(sign bit) | max(other bits) */ 1139 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1140 var32_off.value | (var32_off.mask & S32_MAX)); 1141 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1142 reg->u32_max_value = min(reg->u32_max_value, 1143 (u32)(var32_off.value | var32_off.mask)); 1144 } 1145 1146 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1147 { 1148 /* min signed is max(sign bit) | min(other bits) */ 1149 reg->smin_value = max_t(s64, reg->smin_value, 1150 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1151 /* max signed is min(sign bit) | max(other bits) */ 1152 reg->smax_value = min_t(s64, reg->smax_value, 1153 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1154 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1155 reg->umax_value = min(reg->umax_value, 1156 reg->var_off.value | reg->var_off.mask); 1157 } 1158 1159 static void __update_reg_bounds(struct bpf_reg_state *reg) 1160 { 1161 __update_reg32_bounds(reg); 1162 __update_reg64_bounds(reg); 1163 } 1164 1165 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1166 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1167 { 1168 /* Learn sign from signed bounds. 1169 * If we cannot cross the sign boundary, then signed and unsigned bounds 1170 * are the same, so combine. This works even in the negative case, e.g. 1171 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1172 */ 1173 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1174 reg->s32_min_value = reg->u32_min_value = 1175 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1176 reg->s32_max_value = reg->u32_max_value = 1177 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1178 return; 1179 } 1180 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1181 * boundary, so we must be careful. 1182 */ 1183 if ((s32)reg->u32_max_value >= 0) { 1184 /* Positive. We can't learn anything from the smin, but smax 1185 * is positive, hence safe. 1186 */ 1187 reg->s32_min_value = reg->u32_min_value; 1188 reg->s32_max_value = reg->u32_max_value = 1189 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1190 } else if ((s32)reg->u32_min_value < 0) { 1191 /* Negative. We can't learn anything from the smax, but smin 1192 * is negative, hence safe. 1193 */ 1194 reg->s32_min_value = reg->u32_min_value = 1195 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1196 reg->s32_max_value = reg->u32_max_value; 1197 } 1198 } 1199 1200 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1201 { 1202 /* Learn sign from signed bounds. 1203 * If we cannot cross the sign boundary, then signed and unsigned bounds 1204 * are the same, so combine. This works even in the negative case, e.g. 1205 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1206 */ 1207 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1208 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1209 reg->umin_value); 1210 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1211 reg->umax_value); 1212 return; 1213 } 1214 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1215 * boundary, so we must be careful. 1216 */ 1217 if ((s64)reg->umax_value >= 0) { 1218 /* Positive. We can't learn anything from the smin, but smax 1219 * is positive, hence safe. 1220 */ 1221 reg->smin_value = reg->umin_value; 1222 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1223 reg->umax_value); 1224 } else if ((s64)reg->umin_value < 0) { 1225 /* Negative. We can't learn anything from the smax, but smin 1226 * is negative, hence safe. 1227 */ 1228 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1229 reg->umin_value); 1230 reg->smax_value = reg->umax_value; 1231 } 1232 } 1233 1234 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1235 { 1236 __reg32_deduce_bounds(reg); 1237 __reg64_deduce_bounds(reg); 1238 } 1239 1240 /* Attempts to improve var_off based on unsigned min/max information */ 1241 static void __reg_bound_offset(struct bpf_reg_state *reg) 1242 { 1243 struct tnum var64_off = tnum_intersect(reg->var_off, 1244 tnum_range(reg->umin_value, 1245 reg->umax_value)); 1246 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1247 tnum_range(reg->u32_min_value, 1248 reg->u32_max_value)); 1249 1250 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1251 } 1252 1253 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1254 { 1255 reg->umin_value = reg->u32_min_value; 1256 reg->umax_value = reg->u32_max_value; 1257 /* Attempt to pull 32-bit signed bounds into 64-bit bounds 1258 * but must be positive otherwise set to worse case bounds 1259 * and refine later from tnum. 1260 */ 1261 if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0) 1262 reg->smax_value = reg->s32_max_value; 1263 else 1264 reg->smax_value = U32_MAX; 1265 if (reg->s32_min_value >= 0) 1266 reg->smin_value = reg->s32_min_value; 1267 else 1268 reg->smin_value = 0; 1269 } 1270 1271 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1272 { 1273 /* special case when 64-bit register has upper 32-bit register 1274 * zeroed. Typically happens after zext or <<32, >>32 sequence 1275 * allowing us to use 32-bit bounds directly, 1276 */ 1277 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1278 __reg_assign_32_into_64(reg); 1279 } else { 1280 /* Otherwise the best we can do is push lower 32bit known and 1281 * unknown bits into register (var_off set from jmp logic) 1282 * then learn as much as possible from the 64-bit tnum 1283 * known and unknown bits. The previous smin/smax bounds are 1284 * invalid here because of jmp32 compare so mark them unknown 1285 * so they do not impact tnum bounds calculation. 1286 */ 1287 __mark_reg64_unbounded(reg); 1288 __update_reg_bounds(reg); 1289 } 1290 1291 /* Intersecting with the old var_off might have improved our bounds 1292 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1293 * then new var_off is (0; 0x7f...fc) which improves our umax. 1294 */ 1295 __reg_deduce_bounds(reg); 1296 __reg_bound_offset(reg); 1297 __update_reg_bounds(reg); 1298 } 1299 1300 static bool __reg64_bound_s32(s64 a) 1301 { 1302 return a > S32_MIN && a < S32_MAX; 1303 } 1304 1305 static bool __reg64_bound_u32(u64 a) 1306 { 1307 if (a > U32_MIN && a < U32_MAX) 1308 return true; 1309 return false; 1310 } 1311 1312 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1313 { 1314 __mark_reg32_unbounded(reg); 1315 1316 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1317 reg->s32_min_value = (s32)reg->smin_value; 1318 reg->s32_max_value = (s32)reg->smax_value; 1319 } 1320 if (__reg64_bound_u32(reg->umin_value)) 1321 reg->u32_min_value = (u32)reg->umin_value; 1322 if (__reg64_bound_u32(reg->umax_value)) 1323 reg->u32_max_value = (u32)reg->umax_value; 1324 1325 /* Intersecting with the old var_off might have improved our bounds 1326 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1327 * then new var_off is (0; 0x7f...fc) which improves our umax. 1328 */ 1329 __reg_deduce_bounds(reg); 1330 __reg_bound_offset(reg); 1331 __update_reg_bounds(reg); 1332 } 1333 1334 /* Mark a register as having a completely unknown (scalar) value. */ 1335 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1336 struct bpf_reg_state *reg) 1337 { 1338 /* 1339 * Clear type, id, off, and union(map_ptr, range) and 1340 * padding between 'type' and union 1341 */ 1342 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1343 reg->type = SCALAR_VALUE; 1344 reg->var_off = tnum_unknown; 1345 reg->frameno = 0; 1346 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; 1347 __mark_reg_unbounded(reg); 1348 } 1349 1350 static void mark_reg_unknown(struct bpf_verifier_env *env, 1351 struct bpf_reg_state *regs, u32 regno) 1352 { 1353 if (WARN_ON(regno >= MAX_BPF_REG)) { 1354 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1355 /* Something bad happened, let's kill all regs except FP */ 1356 for (regno = 0; regno < BPF_REG_FP; regno++) 1357 __mark_reg_not_init(env, regs + regno); 1358 return; 1359 } 1360 __mark_reg_unknown(env, regs + regno); 1361 } 1362 1363 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1364 struct bpf_reg_state *reg) 1365 { 1366 __mark_reg_unknown(env, reg); 1367 reg->type = NOT_INIT; 1368 } 1369 1370 static void mark_reg_not_init(struct bpf_verifier_env *env, 1371 struct bpf_reg_state *regs, u32 regno) 1372 { 1373 if (WARN_ON(regno >= MAX_BPF_REG)) { 1374 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1375 /* Something bad happened, let's kill all regs except FP */ 1376 for (regno = 0; regno < BPF_REG_FP; regno++) 1377 __mark_reg_not_init(env, regs + regno); 1378 return; 1379 } 1380 __mark_reg_not_init(env, regs + regno); 1381 } 1382 1383 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1384 struct bpf_reg_state *regs, u32 regno, 1385 enum bpf_reg_type reg_type, 1386 struct btf *btf, u32 btf_id) 1387 { 1388 if (reg_type == SCALAR_VALUE) { 1389 mark_reg_unknown(env, regs, regno); 1390 return; 1391 } 1392 mark_reg_known_zero(env, regs, regno); 1393 regs[regno].type = PTR_TO_BTF_ID; 1394 regs[regno].btf = btf; 1395 regs[regno].btf_id = btf_id; 1396 } 1397 1398 #define DEF_NOT_SUBREG (0) 1399 static void init_reg_state(struct bpf_verifier_env *env, 1400 struct bpf_func_state *state) 1401 { 1402 struct bpf_reg_state *regs = state->regs; 1403 int i; 1404 1405 for (i = 0; i < MAX_BPF_REG; i++) { 1406 mark_reg_not_init(env, regs, i); 1407 regs[i].live = REG_LIVE_NONE; 1408 regs[i].parent = NULL; 1409 regs[i].subreg_def = DEF_NOT_SUBREG; 1410 } 1411 1412 /* frame pointer */ 1413 regs[BPF_REG_FP].type = PTR_TO_STACK; 1414 mark_reg_known_zero(env, regs, BPF_REG_FP); 1415 regs[BPF_REG_FP].frameno = state->frameno; 1416 } 1417 1418 #define BPF_MAIN_FUNC (-1) 1419 static void init_func_state(struct bpf_verifier_env *env, 1420 struct bpf_func_state *state, 1421 int callsite, int frameno, int subprogno) 1422 { 1423 state->callsite = callsite; 1424 state->frameno = frameno; 1425 state->subprogno = subprogno; 1426 init_reg_state(env, state); 1427 } 1428 1429 enum reg_arg_type { 1430 SRC_OP, /* register is used as source operand */ 1431 DST_OP, /* register is used as destination operand */ 1432 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1433 }; 1434 1435 static int cmp_subprogs(const void *a, const void *b) 1436 { 1437 return ((struct bpf_subprog_info *)a)->start - 1438 ((struct bpf_subprog_info *)b)->start; 1439 } 1440 1441 static int find_subprog(struct bpf_verifier_env *env, int off) 1442 { 1443 struct bpf_subprog_info *p; 1444 1445 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1446 sizeof(env->subprog_info[0]), cmp_subprogs); 1447 if (!p) 1448 return -ENOENT; 1449 return p - env->subprog_info; 1450 1451 } 1452 1453 static int add_subprog(struct bpf_verifier_env *env, int off) 1454 { 1455 int insn_cnt = env->prog->len; 1456 int ret; 1457 1458 if (off >= insn_cnt || off < 0) { 1459 verbose(env, "call to invalid destination\n"); 1460 return -EINVAL; 1461 } 1462 ret = find_subprog(env, off); 1463 if (ret >= 0) 1464 return 0; 1465 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1466 verbose(env, "too many subprograms\n"); 1467 return -E2BIG; 1468 } 1469 env->subprog_info[env->subprog_cnt++].start = off; 1470 sort(env->subprog_info, env->subprog_cnt, 1471 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1472 return 0; 1473 } 1474 1475 static int check_subprogs(struct bpf_verifier_env *env) 1476 { 1477 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; 1478 struct bpf_subprog_info *subprog = env->subprog_info; 1479 struct bpf_insn *insn = env->prog->insnsi; 1480 int insn_cnt = env->prog->len; 1481 1482 /* Add entry function. */ 1483 ret = add_subprog(env, 0); 1484 if (ret < 0) 1485 return ret; 1486 1487 /* determine subprog starts. The end is one before the next starts */ 1488 for (i = 0; i < insn_cnt; i++) { 1489 if (insn[i].code != (BPF_JMP | BPF_CALL)) 1490 continue; 1491 if (insn[i].src_reg != BPF_PSEUDO_CALL) 1492 continue; 1493 if (!env->bpf_capable) { 1494 verbose(env, 1495 "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 1496 return -EPERM; 1497 } 1498 ret = add_subprog(env, i + insn[i].imm + 1); 1499 if (ret < 0) 1500 return ret; 1501 } 1502 1503 /* Add a fake 'exit' subprog which could simplify subprog iteration 1504 * logic. 'subprog_cnt' should not be increased. 1505 */ 1506 subprog[env->subprog_cnt].start = insn_cnt; 1507 1508 if (env->log.level & BPF_LOG_LEVEL2) 1509 for (i = 0; i < env->subprog_cnt; i++) 1510 verbose(env, "func#%d @%d\n", i, subprog[i].start); 1511 1512 /* now check that all jumps are within the same subprog */ 1513 subprog_start = subprog[cur_subprog].start; 1514 subprog_end = subprog[cur_subprog + 1].start; 1515 for (i = 0; i < insn_cnt; i++) { 1516 u8 code = insn[i].code; 1517 1518 if (code == (BPF_JMP | BPF_CALL) && 1519 insn[i].imm == BPF_FUNC_tail_call && 1520 insn[i].src_reg != BPF_PSEUDO_CALL) 1521 subprog[cur_subprog].has_tail_call = true; 1522 if (BPF_CLASS(code) == BPF_LD && 1523 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 1524 subprog[cur_subprog].has_ld_abs = true; 1525 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 1526 goto next; 1527 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 1528 goto next; 1529 off = i + insn[i].off + 1; 1530 if (off < subprog_start || off >= subprog_end) { 1531 verbose(env, "jump out of range from insn %d to %d\n", i, off); 1532 return -EINVAL; 1533 } 1534 next: 1535 if (i == subprog_end - 1) { 1536 /* to avoid fall-through from one subprog into another 1537 * the last insn of the subprog should be either exit 1538 * or unconditional jump back 1539 */ 1540 if (code != (BPF_JMP | BPF_EXIT) && 1541 code != (BPF_JMP | BPF_JA)) { 1542 verbose(env, "last insn is not an exit or jmp\n"); 1543 return -EINVAL; 1544 } 1545 subprog_start = subprog_end; 1546 cur_subprog++; 1547 if (cur_subprog < env->subprog_cnt) 1548 subprog_end = subprog[cur_subprog + 1].start; 1549 } 1550 } 1551 return 0; 1552 } 1553 1554 /* Parentage chain of this register (or stack slot) should take care of all 1555 * issues like callee-saved registers, stack slot allocation time, etc. 1556 */ 1557 static int mark_reg_read(struct bpf_verifier_env *env, 1558 const struct bpf_reg_state *state, 1559 struct bpf_reg_state *parent, u8 flag) 1560 { 1561 bool writes = parent == state->parent; /* Observe write marks */ 1562 int cnt = 0; 1563 1564 while (parent) { 1565 /* if read wasn't screened by an earlier write ... */ 1566 if (writes && state->live & REG_LIVE_WRITTEN) 1567 break; 1568 if (parent->live & REG_LIVE_DONE) { 1569 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 1570 reg_type_str[parent->type], 1571 parent->var_off.value, parent->off); 1572 return -EFAULT; 1573 } 1574 /* The first condition is more likely to be true than the 1575 * second, checked it first. 1576 */ 1577 if ((parent->live & REG_LIVE_READ) == flag || 1578 parent->live & REG_LIVE_READ64) 1579 /* The parentage chain never changes and 1580 * this parent was already marked as LIVE_READ. 1581 * There is no need to keep walking the chain again and 1582 * keep re-marking all parents as LIVE_READ. 1583 * This case happens when the same register is read 1584 * multiple times without writes into it in-between. 1585 * Also, if parent has the stronger REG_LIVE_READ64 set, 1586 * then no need to set the weak REG_LIVE_READ32. 1587 */ 1588 break; 1589 /* ... then we depend on parent's value */ 1590 parent->live |= flag; 1591 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 1592 if (flag == REG_LIVE_READ64) 1593 parent->live &= ~REG_LIVE_READ32; 1594 state = parent; 1595 parent = state->parent; 1596 writes = true; 1597 cnt++; 1598 } 1599 1600 if (env->longest_mark_read_walk < cnt) 1601 env->longest_mark_read_walk = cnt; 1602 return 0; 1603 } 1604 1605 /* This function is supposed to be used by the following 32-bit optimization 1606 * code only. It returns TRUE if the source or destination register operates 1607 * on 64-bit, otherwise return FALSE. 1608 */ 1609 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 1610 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 1611 { 1612 u8 code, class, op; 1613 1614 code = insn->code; 1615 class = BPF_CLASS(code); 1616 op = BPF_OP(code); 1617 if (class == BPF_JMP) { 1618 /* BPF_EXIT for "main" will reach here. Return TRUE 1619 * conservatively. 1620 */ 1621 if (op == BPF_EXIT) 1622 return true; 1623 if (op == BPF_CALL) { 1624 /* BPF to BPF call will reach here because of marking 1625 * caller saved clobber with DST_OP_NO_MARK for which we 1626 * don't care the register def because they are anyway 1627 * marked as NOT_INIT already. 1628 */ 1629 if (insn->src_reg == BPF_PSEUDO_CALL) 1630 return false; 1631 /* Helper call will reach here because of arg type 1632 * check, conservatively return TRUE. 1633 */ 1634 if (t == SRC_OP) 1635 return true; 1636 1637 return false; 1638 } 1639 } 1640 1641 if (class == BPF_ALU64 || class == BPF_JMP || 1642 /* BPF_END always use BPF_ALU class. */ 1643 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 1644 return true; 1645 1646 if (class == BPF_ALU || class == BPF_JMP32) 1647 return false; 1648 1649 if (class == BPF_LDX) { 1650 if (t != SRC_OP) 1651 return BPF_SIZE(code) == BPF_DW; 1652 /* LDX source must be ptr. */ 1653 return true; 1654 } 1655 1656 if (class == BPF_STX) { 1657 if (reg->type != SCALAR_VALUE) 1658 return true; 1659 return BPF_SIZE(code) == BPF_DW; 1660 } 1661 1662 if (class == BPF_LD) { 1663 u8 mode = BPF_MODE(code); 1664 1665 /* LD_IMM64 */ 1666 if (mode == BPF_IMM) 1667 return true; 1668 1669 /* Both LD_IND and LD_ABS return 32-bit data. */ 1670 if (t != SRC_OP) 1671 return false; 1672 1673 /* Implicit ctx ptr. */ 1674 if (regno == BPF_REG_6) 1675 return true; 1676 1677 /* Explicit source could be any width. */ 1678 return true; 1679 } 1680 1681 if (class == BPF_ST) 1682 /* The only source register for BPF_ST is a ptr. */ 1683 return true; 1684 1685 /* Conservatively return true at default. */ 1686 return true; 1687 } 1688 1689 /* Return TRUE if INSN doesn't have explicit value define. */ 1690 static bool insn_no_def(struct bpf_insn *insn) 1691 { 1692 u8 class = BPF_CLASS(insn->code); 1693 1694 return (class == BPF_JMP || class == BPF_JMP32 || 1695 class == BPF_STX || class == BPF_ST); 1696 } 1697 1698 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 1699 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 1700 { 1701 if (insn_no_def(insn)) 1702 return false; 1703 1704 return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP); 1705 } 1706 1707 static void mark_insn_zext(struct bpf_verifier_env *env, 1708 struct bpf_reg_state *reg) 1709 { 1710 s32 def_idx = reg->subreg_def; 1711 1712 if (def_idx == DEF_NOT_SUBREG) 1713 return; 1714 1715 env->insn_aux_data[def_idx - 1].zext_dst = true; 1716 /* The dst will be zero extended, so won't be sub-register anymore. */ 1717 reg->subreg_def = DEF_NOT_SUBREG; 1718 } 1719 1720 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 1721 enum reg_arg_type t) 1722 { 1723 struct bpf_verifier_state *vstate = env->cur_state; 1724 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 1725 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 1726 struct bpf_reg_state *reg, *regs = state->regs; 1727 bool rw64; 1728 1729 if (regno >= MAX_BPF_REG) { 1730 verbose(env, "R%d is invalid\n", regno); 1731 return -EINVAL; 1732 } 1733 1734 reg = ®s[regno]; 1735 rw64 = is_reg64(env, insn, regno, reg, t); 1736 if (t == SRC_OP) { 1737 /* check whether register used as source operand can be read */ 1738 if (reg->type == NOT_INIT) { 1739 verbose(env, "R%d !read_ok\n", regno); 1740 return -EACCES; 1741 } 1742 /* We don't need to worry about FP liveness because it's read-only */ 1743 if (regno == BPF_REG_FP) 1744 return 0; 1745 1746 if (rw64) 1747 mark_insn_zext(env, reg); 1748 1749 return mark_reg_read(env, reg, reg->parent, 1750 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 1751 } else { 1752 /* check whether register used as dest operand can be written to */ 1753 if (regno == BPF_REG_FP) { 1754 verbose(env, "frame pointer is read only\n"); 1755 return -EACCES; 1756 } 1757 reg->live |= REG_LIVE_WRITTEN; 1758 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 1759 if (t == DST_OP) 1760 mark_reg_unknown(env, regs, regno); 1761 } 1762 return 0; 1763 } 1764 1765 /* for any branch, call, exit record the history of jmps in the given state */ 1766 static int push_jmp_history(struct bpf_verifier_env *env, 1767 struct bpf_verifier_state *cur) 1768 { 1769 u32 cnt = cur->jmp_history_cnt; 1770 struct bpf_idx_pair *p; 1771 1772 cnt++; 1773 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); 1774 if (!p) 1775 return -ENOMEM; 1776 p[cnt - 1].idx = env->insn_idx; 1777 p[cnt - 1].prev_idx = env->prev_insn_idx; 1778 cur->jmp_history = p; 1779 cur->jmp_history_cnt = cnt; 1780 return 0; 1781 } 1782 1783 /* Backtrack one insn at a time. If idx is not at the top of recorded 1784 * history then previous instruction came from straight line execution. 1785 */ 1786 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 1787 u32 *history) 1788 { 1789 u32 cnt = *history; 1790 1791 if (cnt && st->jmp_history[cnt - 1].idx == i) { 1792 i = st->jmp_history[cnt - 1].prev_idx; 1793 (*history)--; 1794 } else { 1795 i--; 1796 } 1797 return i; 1798 } 1799 1800 /* For given verifier state backtrack_insn() is called from the last insn to 1801 * the first insn. Its purpose is to compute a bitmask of registers and 1802 * stack slots that needs precision in the parent verifier state. 1803 */ 1804 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 1805 u32 *reg_mask, u64 *stack_mask) 1806 { 1807 const struct bpf_insn_cbs cbs = { 1808 .cb_print = verbose, 1809 .private_data = env, 1810 }; 1811 struct bpf_insn *insn = env->prog->insnsi + idx; 1812 u8 class = BPF_CLASS(insn->code); 1813 u8 opcode = BPF_OP(insn->code); 1814 u8 mode = BPF_MODE(insn->code); 1815 u32 dreg = 1u << insn->dst_reg; 1816 u32 sreg = 1u << insn->src_reg; 1817 u32 spi; 1818 1819 if (insn->code == 0) 1820 return 0; 1821 if (env->log.level & BPF_LOG_LEVEL) { 1822 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 1823 verbose(env, "%d: ", idx); 1824 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 1825 } 1826 1827 if (class == BPF_ALU || class == BPF_ALU64) { 1828 if (!(*reg_mask & dreg)) 1829 return 0; 1830 if (opcode == BPF_MOV) { 1831 if (BPF_SRC(insn->code) == BPF_X) { 1832 /* dreg = sreg 1833 * dreg needs precision after this insn 1834 * sreg needs precision before this insn 1835 */ 1836 *reg_mask &= ~dreg; 1837 *reg_mask |= sreg; 1838 } else { 1839 /* dreg = K 1840 * dreg needs precision after this insn. 1841 * Corresponding register is already marked 1842 * as precise=true in this verifier state. 1843 * No further markings in parent are necessary 1844 */ 1845 *reg_mask &= ~dreg; 1846 } 1847 } else { 1848 if (BPF_SRC(insn->code) == BPF_X) { 1849 /* dreg += sreg 1850 * both dreg and sreg need precision 1851 * before this insn 1852 */ 1853 *reg_mask |= sreg; 1854 } /* else dreg += K 1855 * dreg still needs precision before this insn 1856 */ 1857 } 1858 } else if (class == BPF_LDX) { 1859 if (!(*reg_mask & dreg)) 1860 return 0; 1861 *reg_mask &= ~dreg; 1862 1863 /* scalars can only be spilled into stack w/o losing precision. 1864 * Load from any other memory can be zero extended. 1865 * The desire to keep that precision is already indicated 1866 * by 'precise' mark in corresponding register of this state. 1867 * No further tracking necessary. 1868 */ 1869 if (insn->src_reg != BPF_REG_FP) 1870 return 0; 1871 if (BPF_SIZE(insn->code) != BPF_DW) 1872 return 0; 1873 1874 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 1875 * that [fp - off] slot contains scalar that needs to be 1876 * tracked with precision 1877 */ 1878 spi = (-insn->off - 1) / BPF_REG_SIZE; 1879 if (spi >= 64) { 1880 verbose(env, "BUG spi %d\n", spi); 1881 WARN_ONCE(1, "verifier backtracking bug"); 1882 return -EFAULT; 1883 } 1884 *stack_mask |= 1ull << spi; 1885 } else if (class == BPF_STX || class == BPF_ST) { 1886 if (*reg_mask & dreg) 1887 /* stx & st shouldn't be using _scalar_ dst_reg 1888 * to access memory. It means backtracking 1889 * encountered a case of pointer subtraction. 1890 */ 1891 return -ENOTSUPP; 1892 /* scalars can only be spilled into stack */ 1893 if (insn->dst_reg != BPF_REG_FP) 1894 return 0; 1895 if (BPF_SIZE(insn->code) != BPF_DW) 1896 return 0; 1897 spi = (-insn->off - 1) / BPF_REG_SIZE; 1898 if (spi >= 64) { 1899 verbose(env, "BUG spi %d\n", spi); 1900 WARN_ONCE(1, "verifier backtracking bug"); 1901 return -EFAULT; 1902 } 1903 if (!(*stack_mask & (1ull << spi))) 1904 return 0; 1905 *stack_mask &= ~(1ull << spi); 1906 if (class == BPF_STX) 1907 *reg_mask |= sreg; 1908 } else if (class == BPF_JMP || class == BPF_JMP32) { 1909 if (opcode == BPF_CALL) { 1910 if (insn->src_reg == BPF_PSEUDO_CALL) 1911 return -ENOTSUPP; 1912 /* regular helper call sets R0 */ 1913 *reg_mask &= ~1; 1914 if (*reg_mask & 0x3f) { 1915 /* if backtracing was looking for registers R1-R5 1916 * they should have been found already. 1917 */ 1918 verbose(env, "BUG regs %x\n", *reg_mask); 1919 WARN_ONCE(1, "verifier backtracking bug"); 1920 return -EFAULT; 1921 } 1922 } else if (opcode == BPF_EXIT) { 1923 return -ENOTSUPP; 1924 } 1925 } else if (class == BPF_LD) { 1926 if (!(*reg_mask & dreg)) 1927 return 0; 1928 *reg_mask &= ~dreg; 1929 /* It's ld_imm64 or ld_abs or ld_ind. 1930 * For ld_imm64 no further tracking of precision 1931 * into parent is necessary 1932 */ 1933 if (mode == BPF_IND || mode == BPF_ABS) 1934 /* to be analyzed */ 1935 return -ENOTSUPP; 1936 } 1937 return 0; 1938 } 1939 1940 /* the scalar precision tracking algorithm: 1941 * . at the start all registers have precise=false. 1942 * . scalar ranges are tracked as normal through alu and jmp insns. 1943 * . once precise value of the scalar register is used in: 1944 * . ptr + scalar alu 1945 * . if (scalar cond K|scalar) 1946 * . helper_call(.., scalar, ...) where ARG_CONST is expected 1947 * backtrack through the verifier states and mark all registers and 1948 * stack slots with spilled constants that these scalar regisers 1949 * should be precise. 1950 * . during state pruning two registers (or spilled stack slots) 1951 * are equivalent if both are not precise. 1952 * 1953 * Note the verifier cannot simply walk register parentage chain, 1954 * since many different registers and stack slots could have been 1955 * used to compute single precise scalar. 1956 * 1957 * The approach of starting with precise=true for all registers and then 1958 * backtrack to mark a register as not precise when the verifier detects 1959 * that program doesn't care about specific value (e.g., when helper 1960 * takes register as ARG_ANYTHING parameter) is not safe. 1961 * 1962 * It's ok to walk single parentage chain of the verifier states. 1963 * It's possible that this backtracking will go all the way till 1st insn. 1964 * All other branches will be explored for needing precision later. 1965 * 1966 * The backtracking needs to deal with cases like: 1967 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 1968 * r9 -= r8 1969 * r5 = r9 1970 * if r5 > 0x79f goto pc+7 1971 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 1972 * r5 += 1 1973 * ... 1974 * call bpf_perf_event_output#25 1975 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 1976 * 1977 * and this case: 1978 * r6 = 1 1979 * call foo // uses callee's r6 inside to compute r0 1980 * r0 += r6 1981 * if r0 == 0 goto 1982 * 1983 * to track above reg_mask/stack_mask needs to be independent for each frame. 1984 * 1985 * Also if parent's curframe > frame where backtracking started, 1986 * the verifier need to mark registers in both frames, otherwise callees 1987 * may incorrectly prune callers. This is similar to 1988 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 1989 * 1990 * For now backtracking falls back into conservative marking. 1991 */ 1992 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 1993 struct bpf_verifier_state *st) 1994 { 1995 struct bpf_func_state *func; 1996 struct bpf_reg_state *reg; 1997 int i, j; 1998 1999 /* big hammer: mark all scalars precise in this path. 2000 * pop_stack may still get !precise scalars. 2001 */ 2002 for (; st; st = st->parent) 2003 for (i = 0; i <= st->curframe; i++) { 2004 func = st->frame[i]; 2005 for (j = 0; j < BPF_REG_FP; j++) { 2006 reg = &func->regs[j]; 2007 if (reg->type != SCALAR_VALUE) 2008 continue; 2009 reg->precise = true; 2010 } 2011 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2012 if (func->stack[j].slot_type[0] != STACK_SPILL) 2013 continue; 2014 reg = &func->stack[j].spilled_ptr; 2015 if (reg->type != SCALAR_VALUE) 2016 continue; 2017 reg->precise = true; 2018 } 2019 } 2020 } 2021 2022 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, 2023 int spi) 2024 { 2025 struct bpf_verifier_state *st = env->cur_state; 2026 int first_idx = st->first_insn_idx; 2027 int last_idx = env->insn_idx; 2028 struct bpf_func_state *func; 2029 struct bpf_reg_state *reg; 2030 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2031 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2032 bool skip_first = true; 2033 bool new_marks = false; 2034 int i, err; 2035 2036 if (!env->bpf_capable) 2037 return 0; 2038 2039 func = st->frame[st->curframe]; 2040 if (regno >= 0) { 2041 reg = &func->regs[regno]; 2042 if (reg->type != SCALAR_VALUE) { 2043 WARN_ONCE(1, "backtracing misuse"); 2044 return -EFAULT; 2045 } 2046 if (!reg->precise) 2047 new_marks = true; 2048 else 2049 reg_mask = 0; 2050 reg->precise = true; 2051 } 2052 2053 while (spi >= 0) { 2054 if (func->stack[spi].slot_type[0] != STACK_SPILL) { 2055 stack_mask = 0; 2056 break; 2057 } 2058 reg = &func->stack[spi].spilled_ptr; 2059 if (reg->type != SCALAR_VALUE) { 2060 stack_mask = 0; 2061 break; 2062 } 2063 if (!reg->precise) 2064 new_marks = true; 2065 else 2066 stack_mask = 0; 2067 reg->precise = true; 2068 break; 2069 } 2070 2071 if (!new_marks) 2072 return 0; 2073 if (!reg_mask && !stack_mask) 2074 return 0; 2075 for (;;) { 2076 DECLARE_BITMAP(mask, 64); 2077 u32 history = st->jmp_history_cnt; 2078 2079 if (env->log.level & BPF_LOG_LEVEL) 2080 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2081 for (i = last_idx;;) { 2082 if (skip_first) { 2083 err = 0; 2084 skip_first = false; 2085 } else { 2086 err = backtrack_insn(env, i, ®_mask, &stack_mask); 2087 } 2088 if (err == -ENOTSUPP) { 2089 mark_all_scalars_precise(env, st); 2090 return 0; 2091 } else if (err) { 2092 return err; 2093 } 2094 if (!reg_mask && !stack_mask) 2095 /* Found assignment(s) into tracked register in this state. 2096 * Since this state is already marked, just return. 2097 * Nothing to be tracked further in the parent state. 2098 */ 2099 return 0; 2100 if (i == first_idx) 2101 break; 2102 i = get_prev_insn_idx(st, i, &history); 2103 if (i >= env->prog->len) { 2104 /* This can happen if backtracking reached insn 0 2105 * and there are still reg_mask or stack_mask 2106 * to backtrack. 2107 * It means the backtracking missed the spot where 2108 * particular register was initialized with a constant. 2109 */ 2110 verbose(env, "BUG backtracking idx %d\n", i); 2111 WARN_ONCE(1, "verifier backtracking bug"); 2112 return -EFAULT; 2113 } 2114 } 2115 st = st->parent; 2116 if (!st) 2117 break; 2118 2119 new_marks = false; 2120 func = st->frame[st->curframe]; 2121 bitmap_from_u64(mask, reg_mask); 2122 for_each_set_bit(i, mask, 32) { 2123 reg = &func->regs[i]; 2124 if (reg->type != SCALAR_VALUE) { 2125 reg_mask &= ~(1u << i); 2126 continue; 2127 } 2128 if (!reg->precise) 2129 new_marks = true; 2130 reg->precise = true; 2131 } 2132 2133 bitmap_from_u64(mask, stack_mask); 2134 for_each_set_bit(i, mask, 64) { 2135 if (i >= func->allocated_stack / BPF_REG_SIZE) { 2136 /* the sequence of instructions: 2137 * 2: (bf) r3 = r10 2138 * 3: (7b) *(u64 *)(r3 -8) = r0 2139 * 4: (79) r4 = *(u64 *)(r10 -8) 2140 * doesn't contain jmps. It's backtracked 2141 * as a single block. 2142 * During backtracking insn 3 is not recognized as 2143 * stack access, so at the end of backtracking 2144 * stack slot fp-8 is still marked in stack_mask. 2145 * However the parent state may not have accessed 2146 * fp-8 and it's "unallocated" stack space. 2147 * In such case fallback to conservative. 2148 */ 2149 mark_all_scalars_precise(env, st); 2150 return 0; 2151 } 2152 2153 if (func->stack[i].slot_type[0] != STACK_SPILL) { 2154 stack_mask &= ~(1ull << i); 2155 continue; 2156 } 2157 reg = &func->stack[i].spilled_ptr; 2158 if (reg->type != SCALAR_VALUE) { 2159 stack_mask &= ~(1ull << i); 2160 continue; 2161 } 2162 if (!reg->precise) 2163 new_marks = true; 2164 reg->precise = true; 2165 } 2166 if (env->log.level & BPF_LOG_LEVEL) { 2167 print_verifier_state(env, func); 2168 verbose(env, "parent %s regs=%x stack=%llx marks\n", 2169 new_marks ? "didn't have" : "already had", 2170 reg_mask, stack_mask); 2171 } 2172 2173 if (!reg_mask && !stack_mask) 2174 break; 2175 if (!new_marks) 2176 break; 2177 2178 last_idx = st->last_insn_idx; 2179 first_idx = st->first_insn_idx; 2180 } 2181 return 0; 2182 } 2183 2184 static int mark_chain_precision(struct bpf_verifier_env *env, int regno) 2185 { 2186 return __mark_chain_precision(env, regno, -1); 2187 } 2188 2189 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) 2190 { 2191 return __mark_chain_precision(env, -1, spi); 2192 } 2193 2194 static bool is_spillable_regtype(enum bpf_reg_type type) 2195 { 2196 switch (type) { 2197 case PTR_TO_MAP_VALUE: 2198 case PTR_TO_MAP_VALUE_OR_NULL: 2199 case PTR_TO_STACK: 2200 case PTR_TO_CTX: 2201 case PTR_TO_PACKET: 2202 case PTR_TO_PACKET_META: 2203 case PTR_TO_PACKET_END: 2204 case PTR_TO_FLOW_KEYS: 2205 case CONST_PTR_TO_MAP: 2206 case PTR_TO_SOCKET: 2207 case PTR_TO_SOCKET_OR_NULL: 2208 case PTR_TO_SOCK_COMMON: 2209 case PTR_TO_SOCK_COMMON_OR_NULL: 2210 case PTR_TO_TCP_SOCK: 2211 case PTR_TO_TCP_SOCK_OR_NULL: 2212 case PTR_TO_XDP_SOCK: 2213 case PTR_TO_BTF_ID: 2214 case PTR_TO_BTF_ID_OR_NULL: 2215 case PTR_TO_RDONLY_BUF: 2216 case PTR_TO_RDONLY_BUF_OR_NULL: 2217 case PTR_TO_RDWR_BUF: 2218 case PTR_TO_RDWR_BUF_OR_NULL: 2219 case PTR_TO_PERCPU_BTF_ID: 2220 case PTR_TO_MEM: 2221 case PTR_TO_MEM_OR_NULL: 2222 return true; 2223 default: 2224 return false; 2225 } 2226 } 2227 2228 /* Does this register contain a constant zero? */ 2229 static bool register_is_null(struct bpf_reg_state *reg) 2230 { 2231 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 2232 } 2233 2234 static bool register_is_const(struct bpf_reg_state *reg) 2235 { 2236 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 2237 } 2238 2239 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 2240 { 2241 return tnum_is_unknown(reg->var_off) && 2242 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 2243 reg->umin_value == 0 && reg->umax_value == U64_MAX && 2244 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 2245 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 2246 } 2247 2248 static bool register_is_bounded(struct bpf_reg_state *reg) 2249 { 2250 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 2251 } 2252 2253 static bool __is_pointer_value(bool allow_ptr_leaks, 2254 const struct bpf_reg_state *reg) 2255 { 2256 if (allow_ptr_leaks) 2257 return false; 2258 2259 return reg->type != SCALAR_VALUE; 2260 } 2261 2262 static void save_register_state(struct bpf_func_state *state, 2263 int spi, struct bpf_reg_state *reg) 2264 { 2265 int i; 2266 2267 state->stack[spi].spilled_ptr = *reg; 2268 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2269 2270 for (i = 0; i < BPF_REG_SIZE; i++) 2271 state->stack[spi].slot_type[i] = STACK_SPILL; 2272 } 2273 2274 /* check_stack_read/write functions track spill/fill of registers, 2275 * stack boundary and alignment are checked in check_mem_access() 2276 */ 2277 static int check_stack_write(struct bpf_verifier_env *env, 2278 struct bpf_func_state *state, /* func where register points to */ 2279 int off, int size, int value_regno, int insn_idx) 2280 { 2281 struct bpf_func_state *cur; /* state of the current function */ 2282 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 2283 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 2284 struct bpf_reg_state *reg = NULL; 2285 2286 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), 2287 state->acquired_refs, true); 2288 if (err) 2289 return err; 2290 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 2291 * so it's aligned access and [off, off + size) are within stack limits 2292 */ 2293 if (!env->allow_ptr_leaks && 2294 state->stack[spi].slot_type[0] == STACK_SPILL && 2295 size != BPF_REG_SIZE) { 2296 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 2297 return -EACCES; 2298 } 2299 2300 cur = env->cur_state->frame[env->cur_state->curframe]; 2301 if (value_regno >= 0) 2302 reg = &cur->regs[value_regno]; 2303 2304 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && 2305 !register_is_null(reg) && env->bpf_capable) { 2306 if (dst_reg != BPF_REG_FP) { 2307 /* The backtracking logic can only recognize explicit 2308 * stack slot address like [fp - 8]. Other spill of 2309 * scalar via different register has to be conervative. 2310 * Backtrack from here and mark all registers as precise 2311 * that contributed into 'reg' being a constant. 2312 */ 2313 err = mark_chain_precision(env, value_regno); 2314 if (err) 2315 return err; 2316 } 2317 save_register_state(state, spi, reg); 2318 } else if (reg && is_spillable_regtype(reg->type)) { 2319 /* register containing pointer is being spilled into stack */ 2320 if (size != BPF_REG_SIZE) { 2321 verbose_linfo(env, insn_idx, "; "); 2322 verbose(env, "invalid size of register spill\n"); 2323 return -EACCES; 2324 } 2325 2326 if (state != cur && reg->type == PTR_TO_STACK) { 2327 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 2328 return -EINVAL; 2329 } 2330 2331 if (!env->bypass_spec_v4) { 2332 bool sanitize = false; 2333 2334 if (state->stack[spi].slot_type[0] == STACK_SPILL && 2335 register_is_const(&state->stack[spi].spilled_ptr)) 2336 sanitize = true; 2337 for (i = 0; i < BPF_REG_SIZE; i++) 2338 if (state->stack[spi].slot_type[i] == STACK_MISC) { 2339 sanitize = true; 2340 break; 2341 } 2342 if (sanitize) { 2343 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; 2344 int soff = (-spi - 1) * BPF_REG_SIZE; 2345 2346 /* detected reuse of integer stack slot with a pointer 2347 * which means either llvm is reusing stack slot or 2348 * an attacker is trying to exploit CVE-2018-3639 2349 * (speculative store bypass) 2350 * Have to sanitize that slot with preemptive 2351 * store of zero. 2352 */ 2353 if (*poff && *poff != soff) { 2354 /* disallow programs where single insn stores 2355 * into two different stack slots, since verifier 2356 * cannot sanitize them 2357 */ 2358 verbose(env, 2359 "insn %d cannot access two stack slots fp%d and fp%d", 2360 insn_idx, *poff, soff); 2361 return -EINVAL; 2362 } 2363 *poff = soff; 2364 } 2365 } 2366 save_register_state(state, spi, reg); 2367 } else { 2368 u8 type = STACK_MISC; 2369 2370 /* regular write of data into stack destroys any spilled ptr */ 2371 state->stack[spi].spilled_ptr.type = NOT_INIT; 2372 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 2373 if (state->stack[spi].slot_type[0] == STACK_SPILL) 2374 for (i = 0; i < BPF_REG_SIZE; i++) 2375 state->stack[spi].slot_type[i] = STACK_MISC; 2376 2377 /* only mark the slot as written if all 8 bytes were written 2378 * otherwise read propagation may incorrectly stop too soon 2379 * when stack slots are partially written. 2380 * This heuristic means that read propagation will be 2381 * conservative, since it will add reg_live_read marks 2382 * to stack slots all the way to first state when programs 2383 * writes+reads less than 8 bytes 2384 */ 2385 if (size == BPF_REG_SIZE) 2386 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2387 2388 /* when we zero initialize stack slots mark them as such */ 2389 if (reg && register_is_null(reg)) { 2390 /* backtracking doesn't work for STACK_ZERO yet. */ 2391 err = mark_chain_precision(env, value_regno); 2392 if (err) 2393 return err; 2394 type = STACK_ZERO; 2395 } 2396 2397 /* Mark slots affected by this stack write. */ 2398 for (i = 0; i < size; i++) 2399 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 2400 type; 2401 } 2402 return 0; 2403 } 2404 2405 static int check_stack_read(struct bpf_verifier_env *env, 2406 struct bpf_func_state *reg_state /* func where register points to */, 2407 int off, int size, int value_regno) 2408 { 2409 struct bpf_verifier_state *vstate = env->cur_state; 2410 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2411 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 2412 struct bpf_reg_state *reg; 2413 u8 *stype; 2414 2415 if (reg_state->allocated_stack <= slot) { 2416 verbose(env, "invalid read from stack off %d+0 size %d\n", 2417 off, size); 2418 return -EACCES; 2419 } 2420 stype = reg_state->stack[spi].slot_type; 2421 reg = ®_state->stack[spi].spilled_ptr; 2422 2423 if (stype[0] == STACK_SPILL) { 2424 if (size != BPF_REG_SIZE) { 2425 if (reg->type != SCALAR_VALUE) { 2426 verbose_linfo(env, env->insn_idx, "; "); 2427 verbose(env, "invalid size of register fill\n"); 2428 return -EACCES; 2429 } 2430 if (value_regno >= 0) { 2431 mark_reg_unknown(env, state->regs, value_regno); 2432 state->regs[value_regno].live |= REG_LIVE_WRITTEN; 2433 } 2434 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2435 return 0; 2436 } 2437 for (i = 1; i < BPF_REG_SIZE; i++) { 2438 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { 2439 verbose(env, "corrupted spill memory\n"); 2440 return -EACCES; 2441 } 2442 } 2443 2444 if (value_regno >= 0) { 2445 /* restore register state from stack */ 2446 state->regs[value_regno] = *reg; 2447 /* mark reg as written since spilled pointer state likely 2448 * has its liveness marks cleared by is_state_visited() 2449 * which resets stack/reg liveness for state transitions 2450 */ 2451 state->regs[value_regno].live |= REG_LIVE_WRITTEN; 2452 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 2453 /* If value_regno==-1, the caller is asking us whether 2454 * it is acceptable to use this value as a SCALAR_VALUE 2455 * (e.g. for XADD). 2456 * We must not allow unprivileged callers to do that 2457 * with spilled pointers. 2458 */ 2459 verbose(env, "leaking pointer from stack off %d\n", 2460 off); 2461 return -EACCES; 2462 } 2463 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2464 } else { 2465 int zeros = 0; 2466 2467 for (i = 0; i < size; i++) { 2468 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) 2469 continue; 2470 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { 2471 zeros++; 2472 continue; 2473 } 2474 verbose(env, "invalid read from stack off %d+%d size %d\n", 2475 off, i, size); 2476 return -EACCES; 2477 } 2478 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 2479 if (value_regno >= 0) { 2480 if (zeros == size) { 2481 /* any size read into register is zero extended, 2482 * so the whole register == const_zero 2483 */ 2484 __mark_reg_const_zero(&state->regs[value_regno]); 2485 /* backtracking doesn't support STACK_ZERO yet, 2486 * so mark it precise here, so that later 2487 * backtracking can stop here. 2488 * Backtracking may not need this if this register 2489 * doesn't participate in pointer adjustment. 2490 * Forward propagation of precise flag is not 2491 * necessary either. This mark is only to stop 2492 * backtracking. Any register that contributed 2493 * to const 0 was marked precise before spill. 2494 */ 2495 state->regs[value_regno].precise = true; 2496 } else { 2497 /* have read misc data from the stack */ 2498 mark_reg_unknown(env, state->regs, value_regno); 2499 } 2500 state->regs[value_regno].live |= REG_LIVE_WRITTEN; 2501 } 2502 } 2503 return 0; 2504 } 2505 2506 static int check_stack_access(struct bpf_verifier_env *env, 2507 const struct bpf_reg_state *reg, 2508 int off, int size) 2509 { 2510 /* Stack accesses must be at a fixed offset, so that we 2511 * can determine what type of data were returned. See 2512 * check_stack_read(). 2513 */ 2514 if (!tnum_is_const(reg->var_off)) { 2515 char tn_buf[48]; 2516 2517 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2518 verbose(env, "variable stack access var_off=%s off=%d size=%d\n", 2519 tn_buf, off, size); 2520 return -EACCES; 2521 } 2522 2523 if (off >= 0 || off < -MAX_BPF_STACK) { 2524 verbose(env, "invalid stack off=%d size=%d\n", off, size); 2525 return -EACCES; 2526 } 2527 2528 return 0; 2529 } 2530 2531 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 2532 int off, int size, enum bpf_access_type type) 2533 { 2534 struct bpf_reg_state *regs = cur_regs(env); 2535 struct bpf_map *map = regs[regno].map_ptr; 2536 u32 cap = bpf_map_flags_to_cap(map); 2537 2538 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 2539 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 2540 map->value_size, off, size); 2541 return -EACCES; 2542 } 2543 2544 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 2545 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 2546 map->value_size, off, size); 2547 return -EACCES; 2548 } 2549 2550 return 0; 2551 } 2552 2553 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 2554 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 2555 int off, int size, u32 mem_size, 2556 bool zero_size_allowed) 2557 { 2558 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 2559 struct bpf_reg_state *reg; 2560 2561 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 2562 return 0; 2563 2564 reg = &cur_regs(env)[regno]; 2565 switch (reg->type) { 2566 case PTR_TO_MAP_VALUE: 2567 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 2568 mem_size, off, size); 2569 break; 2570 case PTR_TO_PACKET: 2571 case PTR_TO_PACKET_META: 2572 case PTR_TO_PACKET_END: 2573 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 2574 off, size, regno, reg->id, off, mem_size); 2575 break; 2576 case PTR_TO_MEM: 2577 default: 2578 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 2579 mem_size, off, size); 2580 } 2581 2582 return -EACCES; 2583 } 2584 2585 /* check read/write into a memory region with possible variable offset */ 2586 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 2587 int off, int size, u32 mem_size, 2588 bool zero_size_allowed) 2589 { 2590 struct bpf_verifier_state *vstate = env->cur_state; 2591 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2592 struct bpf_reg_state *reg = &state->regs[regno]; 2593 int err; 2594 2595 /* We may have adjusted the register pointing to memory region, so we 2596 * need to try adding each of min_value and max_value to off 2597 * to make sure our theoretical access will be safe. 2598 */ 2599 if (env->log.level & BPF_LOG_LEVEL) 2600 print_verifier_state(env, state); 2601 2602 /* The minimum value is only important with signed 2603 * comparisons where we can't assume the floor of a 2604 * value is 0. If we are using signed variables for our 2605 * index'es we need to make sure that whatever we use 2606 * will have a set floor within our range. 2607 */ 2608 if (reg->smin_value < 0 && 2609 (reg->smin_value == S64_MIN || 2610 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 2611 reg->smin_value + off < 0)) { 2612 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 2613 regno); 2614 return -EACCES; 2615 } 2616 err = __check_mem_access(env, regno, reg->smin_value + off, size, 2617 mem_size, zero_size_allowed); 2618 if (err) { 2619 verbose(env, "R%d min value is outside of the allowed memory range\n", 2620 regno); 2621 return err; 2622 } 2623 2624 /* If we haven't set a max value then we need to bail since we can't be 2625 * sure we won't do bad things. 2626 * If reg->umax_value + off could overflow, treat that as unbounded too. 2627 */ 2628 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 2629 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 2630 regno); 2631 return -EACCES; 2632 } 2633 err = __check_mem_access(env, regno, reg->umax_value + off, size, 2634 mem_size, zero_size_allowed); 2635 if (err) { 2636 verbose(env, "R%d max value is outside of the allowed memory range\n", 2637 regno); 2638 return err; 2639 } 2640 2641 return 0; 2642 } 2643 2644 /* check read/write into a map element with possible variable offset */ 2645 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 2646 int off, int size, bool zero_size_allowed) 2647 { 2648 struct bpf_verifier_state *vstate = env->cur_state; 2649 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2650 struct bpf_reg_state *reg = &state->regs[regno]; 2651 struct bpf_map *map = reg->map_ptr; 2652 int err; 2653 2654 err = check_mem_region_access(env, regno, off, size, map->value_size, 2655 zero_size_allowed); 2656 if (err) 2657 return err; 2658 2659 if (map_value_has_spin_lock(map)) { 2660 u32 lock = map->spin_lock_off; 2661 2662 /* if any part of struct bpf_spin_lock can be touched by 2663 * load/store reject this program. 2664 * To check that [x1, x2) overlaps with [y1, y2) 2665 * it is sufficient to check x1 < y2 && y1 < x2. 2666 */ 2667 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && 2668 lock < reg->umax_value + off + size) { 2669 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); 2670 return -EACCES; 2671 } 2672 } 2673 return err; 2674 } 2675 2676 #define MAX_PACKET_OFF 0xffff 2677 2678 static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) 2679 { 2680 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; 2681 } 2682 2683 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 2684 const struct bpf_call_arg_meta *meta, 2685 enum bpf_access_type t) 2686 { 2687 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 2688 2689 switch (prog_type) { 2690 /* Program types only with direct read access go here! */ 2691 case BPF_PROG_TYPE_LWT_IN: 2692 case BPF_PROG_TYPE_LWT_OUT: 2693 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 2694 case BPF_PROG_TYPE_SK_REUSEPORT: 2695 case BPF_PROG_TYPE_FLOW_DISSECTOR: 2696 case BPF_PROG_TYPE_CGROUP_SKB: 2697 if (t == BPF_WRITE) 2698 return false; 2699 fallthrough; 2700 2701 /* Program types with direct read + write access go here! */ 2702 case BPF_PROG_TYPE_SCHED_CLS: 2703 case BPF_PROG_TYPE_SCHED_ACT: 2704 case BPF_PROG_TYPE_XDP: 2705 case BPF_PROG_TYPE_LWT_XMIT: 2706 case BPF_PROG_TYPE_SK_SKB: 2707 case BPF_PROG_TYPE_SK_MSG: 2708 if (meta) 2709 return meta->pkt_access; 2710 2711 env->seen_direct_write = true; 2712 return true; 2713 2714 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 2715 if (t == BPF_WRITE) 2716 env->seen_direct_write = true; 2717 2718 return true; 2719 2720 default: 2721 return false; 2722 } 2723 } 2724 2725 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 2726 int size, bool zero_size_allowed) 2727 { 2728 struct bpf_reg_state *regs = cur_regs(env); 2729 struct bpf_reg_state *reg = ®s[regno]; 2730 int err; 2731 2732 /* We may have added a variable offset to the packet pointer; but any 2733 * reg->range we have comes after that. We are only checking the fixed 2734 * offset. 2735 */ 2736 2737 /* We don't allow negative numbers, because we aren't tracking enough 2738 * detail to prove they're safe. 2739 */ 2740 if (reg->smin_value < 0) { 2741 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 2742 regno); 2743 return -EACCES; 2744 } 2745 2746 err = reg->range < 0 ? -EINVAL : 2747 __check_mem_access(env, regno, off, size, reg->range, 2748 zero_size_allowed); 2749 if (err) { 2750 verbose(env, "R%d offset is outside of the packet\n", regno); 2751 return err; 2752 } 2753 2754 /* __check_mem_access has made sure "off + size - 1" is within u16. 2755 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 2756 * otherwise find_good_pkt_pointers would have refused to set range info 2757 * that __check_mem_access would have rejected this pkt access. 2758 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 2759 */ 2760 env->prog->aux->max_pkt_offset = 2761 max_t(u32, env->prog->aux->max_pkt_offset, 2762 off + reg->umax_value + size - 1); 2763 2764 return err; 2765 } 2766 2767 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 2768 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 2769 enum bpf_access_type t, enum bpf_reg_type *reg_type, 2770 struct btf **btf, u32 *btf_id) 2771 { 2772 struct bpf_insn_access_aux info = { 2773 .reg_type = *reg_type, 2774 .log = &env->log, 2775 }; 2776 2777 if (env->ops->is_valid_access && 2778 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 2779 /* A non zero info.ctx_field_size indicates that this field is a 2780 * candidate for later verifier transformation to load the whole 2781 * field and then apply a mask when accessed with a narrower 2782 * access than actual ctx access size. A zero info.ctx_field_size 2783 * will only allow for whole field access and rejects any other 2784 * type of narrower access. 2785 */ 2786 *reg_type = info.reg_type; 2787 2788 if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { 2789 *btf = info.btf; 2790 *btf_id = info.btf_id; 2791 } else { 2792 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 2793 } 2794 /* remember the offset of last byte accessed in ctx */ 2795 if (env->prog->aux->max_ctx_offset < off + size) 2796 env->prog->aux->max_ctx_offset = off + size; 2797 return 0; 2798 } 2799 2800 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 2801 return -EACCES; 2802 } 2803 2804 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 2805 int size) 2806 { 2807 if (size < 0 || off < 0 || 2808 (u64)off + size > sizeof(struct bpf_flow_keys)) { 2809 verbose(env, "invalid access to flow keys off=%d size=%d\n", 2810 off, size); 2811 return -EACCES; 2812 } 2813 return 0; 2814 } 2815 2816 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 2817 u32 regno, int off, int size, 2818 enum bpf_access_type t) 2819 { 2820 struct bpf_reg_state *regs = cur_regs(env); 2821 struct bpf_reg_state *reg = ®s[regno]; 2822 struct bpf_insn_access_aux info = {}; 2823 bool valid; 2824 2825 if (reg->smin_value < 0) { 2826 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 2827 regno); 2828 return -EACCES; 2829 } 2830 2831 switch (reg->type) { 2832 case PTR_TO_SOCK_COMMON: 2833 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 2834 break; 2835 case PTR_TO_SOCKET: 2836 valid = bpf_sock_is_valid_access(off, size, t, &info); 2837 break; 2838 case PTR_TO_TCP_SOCK: 2839 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 2840 break; 2841 case PTR_TO_XDP_SOCK: 2842 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 2843 break; 2844 default: 2845 valid = false; 2846 } 2847 2848 2849 if (valid) { 2850 env->insn_aux_data[insn_idx].ctx_field_size = 2851 info.ctx_field_size; 2852 return 0; 2853 } 2854 2855 verbose(env, "R%d invalid %s access off=%d size=%d\n", 2856 regno, reg_type_str[reg->type], off, size); 2857 2858 return -EACCES; 2859 } 2860 2861 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 2862 { 2863 return cur_regs(env) + regno; 2864 } 2865 2866 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 2867 { 2868 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 2869 } 2870 2871 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 2872 { 2873 const struct bpf_reg_state *reg = reg_state(env, regno); 2874 2875 return reg->type == PTR_TO_CTX; 2876 } 2877 2878 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 2879 { 2880 const struct bpf_reg_state *reg = reg_state(env, regno); 2881 2882 return type_is_sk_pointer(reg->type); 2883 } 2884 2885 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 2886 { 2887 const struct bpf_reg_state *reg = reg_state(env, regno); 2888 2889 return type_is_pkt_pointer(reg->type); 2890 } 2891 2892 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 2893 { 2894 const struct bpf_reg_state *reg = reg_state(env, regno); 2895 2896 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 2897 return reg->type == PTR_TO_FLOW_KEYS; 2898 } 2899 2900 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 2901 const struct bpf_reg_state *reg, 2902 int off, int size, bool strict) 2903 { 2904 struct tnum reg_off; 2905 int ip_align; 2906 2907 /* Byte size accesses are always allowed. */ 2908 if (!strict || size == 1) 2909 return 0; 2910 2911 /* For platforms that do not have a Kconfig enabling 2912 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 2913 * NET_IP_ALIGN is universally set to '2'. And on platforms 2914 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 2915 * to this code only in strict mode where we want to emulate 2916 * the NET_IP_ALIGN==2 checking. Therefore use an 2917 * unconditional IP align value of '2'. 2918 */ 2919 ip_align = 2; 2920 2921 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 2922 if (!tnum_is_aligned(reg_off, size)) { 2923 char tn_buf[48]; 2924 2925 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2926 verbose(env, 2927 "misaligned packet access off %d+%s+%d+%d size %d\n", 2928 ip_align, tn_buf, reg->off, off, size); 2929 return -EACCES; 2930 } 2931 2932 return 0; 2933 } 2934 2935 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 2936 const struct bpf_reg_state *reg, 2937 const char *pointer_desc, 2938 int off, int size, bool strict) 2939 { 2940 struct tnum reg_off; 2941 2942 /* Byte size accesses are always allowed. */ 2943 if (!strict || size == 1) 2944 return 0; 2945 2946 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 2947 if (!tnum_is_aligned(reg_off, size)) { 2948 char tn_buf[48]; 2949 2950 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2951 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 2952 pointer_desc, tn_buf, reg->off, off, size); 2953 return -EACCES; 2954 } 2955 2956 return 0; 2957 } 2958 2959 static int check_ptr_alignment(struct bpf_verifier_env *env, 2960 const struct bpf_reg_state *reg, int off, 2961 int size, bool strict_alignment_once) 2962 { 2963 bool strict = env->strict_alignment || strict_alignment_once; 2964 const char *pointer_desc = ""; 2965 2966 switch (reg->type) { 2967 case PTR_TO_PACKET: 2968 case PTR_TO_PACKET_META: 2969 /* Special case, because of NET_IP_ALIGN. Given metadata sits 2970 * right in front, treat it the very same way. 2971 */ 2972 return check_pkt_ptr_alignment(env, reg, off, size, strict); 2973 case PTR_TO_FLOW_KEYS: 2974 pointer_desc = "flow keys "; 2975 break; 2976 case PTR_TO_MAP_VALUE: 2977 pointer_desc = "value "; 2978 break; 2979 case PTR_TO_CTX: 2980 pointer_desc = "context "; 2981 break; 2982 case PTR_TO_STACK: 2983 pointer_desc = "stack "; 2984 /* The stack spill tracking logic in check_stack_write() 2985 * and check_stack_read() relies on stack accesses being 2986 * aligned. 2987 */ 2988 strict = true; 2989 break; 2990 case PTR_TO_SOCKET: 2991 pointer_desc = "sock "; 2992 break; 2993 case PTR_TO_SOCK_COMMON: 2994 pointer_desc = "sock_common "; 2995 break; 2996 case PTR_TO_TCP_SOCK: 2997 pointer_desc = "tcp_sock "; 2998 break; 2999 case PTR_TO_XDP_SOCK: 3000 pointer_desc = "xdp_sock "; 3001 break; 3002 default: 3003 break; 3004 } 3005 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 3006 strict); 3007 } 3008 3009 static int update_stack_depth(struct bpf_verifier_env *env, 3010 const struct bpf_func_state *func, 3011 int off) 3012 { 3013 u16 stack = env->subprog_info[func->subprogno].stack_depth; 3014 3015 if (stack >= -off) 3016 return 0; 3017 3018 /* update known max for given subprogram */ 3019 env->subprog_info[func->subprogno].stack_depth = -off; 3020 return 0; 3021 } 3022 3023 /* starting from main bpf function walk all instructions of the function 3024 * and recursively walk all callees that given function can call. 3025 * Ignore jump and exit insns. 3026 * Since recursion is prevented by check_cfg() this algorithm 3027 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 3028 */ 3029 static int check_max_stack_depth(struct bpf_verifier_env *env) 3030 { 3031 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 3032 struct bpf_subprog_info *subprog = env->subprog_info; 3033 struct bpf_insn *insn = env->prog->insnsi; 3034 bool tail_call_reachable = false; 3035 int ret_insn[MAX_CALL_FRAMES]; 3036 int ret_prog[MAX_CALL_FRAMES]; 3037 int j; 3038 3039 process_func: 3040 /* protect against potential stack overflow that might happen when 3041 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 3042 * depth for such case down to 256 so that the worst case scenario 3043 * would result in 8k stack size (32 which is tailcall limit * 256 = 3044 * 8k). 3045 * 3046 * To get the idea what might happen, see an example: 3047 * func1 -> sub rsp, 128 3048 * subfunc1 -> sub rsp, 256 3049 * tailcall1 -> add rsp, 256 3050 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 3051 * subfunc2 -> sub rsp, 64 3052 * subfunc22 -> sub rsp, 128 3053 * tailcall2 -> add rsp, 128 3054 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 3055 * 3056 * tailcall will unwind the current stack frame but it will not get rid 3057 * of caller's stack as shown on the example above. 3058 */ 3059 if (idx && subprog[idx].has_tail_call && depth >= 256) { 3060 verbose(env, 3061 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 3062 depth); 3063 return -EACCES; 3064 } 3065 /* round up to 32-bytes, since this is granularity 3066 * of interpreter stack size 3067 */ 3068 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3069 if (depth > MAX_BPF_STACK) { 3070 verbose(env, "combined stack size of %d calls is %d. Too large\n", 3071 frame + 1, depth); 3072 return -EACCES; 3073 } 3074 continue_func: 3075 subprog_end = subprog[idx + 1].start; 3076 for (; i < subprog_end; i++) { 3077 if (insn[i].code != (BPF_JMP | BPF_CALL)) 3078 continue; 3079 if (insn[i].src_reg != BPF_PSEUDO_CALL) 3080 continue; 3081 /* remember insn and function to return to */ 3082 ret_insn[frame] = i + 1; 3083 ret_prog[frame] = idx; 3084 3085 /* find the callee */ 3086 i = i + insn[i].imm + 1; 3087 idx = find_subprog(env, i); 3088 if (idx < 0) { 3089 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3090 i); 3091 return -EFAULT; 3092 } 3093 3094 if (subprog[idx].has_tail_call) 3095 tail_call_reachable = true; 3096 3097 frame++; 3098 if (frame >= MAX_CALL_FRAMES) { 3099 verbose(env, "the call stack of %d frames is too deep !\n", 3100 frame); 3101 return -E2BIG; 3102 } 3103 goto process_func; 3104 } 3105 /* if tail call got detected across bpf2bpf calls then mark each of the 3106 * currently present subprog frames as tail call reachable subprogs; 3107 * this info will be utilized by JIT so that we will be preserving the 3108 * tail call counter throughout bpf2bpf calls combined with tailcalls 3109 */ 3110 if (tail_call_reachable) 3111 for (j = 0; j < frame; j++) 3112 subprog[ret_prog[j]].tail_call_reachable = true; 3113 3114 /* end of for() loop means the last insn of the 'subprog' 3115 * was reached. Doesn't matter whether it was JA or EXIT 3116 */ 3117 if (frame == 0) 3118 return 0; 3119 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 3120 frame--; 3121 i = ret_insn[frame]; 3122 idx = ret_prog[frame]; 3123 goto continue_func; 3124 } 3125 3126 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 3127 static int get_callee_stack_depth(struct bpf_verifier_env *env, 3128 const struct bpf_insn *insn, int idx) 3129 { 3130 int start = idx + insn->imm + 1, subprog; 3131 3132 subprog = find_subprog(env, start); 3133 if (subprog < 0) { 3134 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 3135 start); 3136 return -EFAULT; 3137 } 3138 return env->subprog_info[subprog].stack_depth; 3139 } 3140 #endif 3141 3142 int check_ctx_reg(struct bpf_verifier_env *env, 3143 const struct bpf_reg_state *reg, int regno) 3144 { 3145 /* Access to ctx or passing it to a helper is only allowed in 3146 * its original, unmodified form. 3147 */ 3148 3149 if (reg->off) { 3150 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", 3151 regno, reg->off); 3152 return -EACCES; 3153 } 3154 3155 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3156 char tn_buf[48]; 3157 3158 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3159 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); 3160 return -EACCES; 3161 } 3162 3163 return 0; 3164 } 3165 3166 static int __check_buffer_access(struct bpf_verifier_env *env, 3167 const char *buf_info, 3168 const struct bpf_reg_state *reg, 3169 int regno, int off, int size) 3170 { 3171 if (off < 0) { 3172 verbose(env, 3173 "R%d invalid %s buffer access: off=%d, size=%d\n", 3174 regno, buf_info, off, size); 3175 return -EACCES; 3176 } 3177 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3178 char tn_buf[48]; 3179 3180 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3181 verbose(env, 3182 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 3183 regno, off, tn_buf); 3184 return -EACCES; 3185 } 3186 3187 return 0; 3188 } 3189 3190 static int check_tp_buffer_access(struct bpf_verifier_env *env, 3191 const struct bpf_reg_state *reg, 3192 int regno, int off, int size) 3193 { 3194 int err; 3195 3196 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 3197 if (err) 3198 return err; 3199 3200 if (off + size > env->prog->aux->max_tp_access) 3201 env->prog->aux->max_tp_access = off + size; 3202 3203 return 0; 3204 } 3205 3206 static int check_buffer_access(struct bpf_verifier_env *env, 3207 const struct bpf_reg_state *reg, 3208 int regno, int off, int size, 3209 bool zero_size_allowed, 3210 const char *buf_info, 3211 u32 *max_access) 3212 { 3213 int err; 3214 3215 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 3216 if (err) 3217 return err; 3218 3219 if (off + size > *max_access) 3220 *max_access = off + size; 3221 3222 return 0; 3223 } 3224 3225 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 3226 static void zext_32_to_64(struct bpf_reg_state *reg) 3227 { 3228 reg->var_off = tnum_subreg(reg->var_off); 3229 __reg_assign_32_into_64(reg); 3230 } 3231 3232 /* truncate register to smaller size (in bytes) 3233 * must be called with size < BPF_REG_SIZE 3234 */ 3235 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 3236 { 3237 u64 mask; 3238 3239 /* clear high bits in bit representation */ 3240 reg->var_off = tnum_cast(reg->var_off, size); 3241 3242 /* fix arithmetic bounds */ 3243 mask = ((u64)1 << (size * 8)) - 1; 3244 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 3245 reg->umin_value &= mask; 3246 reg->umax_value &= mask; 3247 } else { 3248 reg->umin_value = 0; 3249 reg->umax_value = mask; 3250 } 3251 reg->smin_value = reg->umin_value; 3252 reg->smax_value = reg->umax_value; 3253 3254 /* If size is smaller than 32bit register the 32bit register 3255 * values are also truncated so we push 64-bit bounds into 3256 * 32-bit bounds. Above were truncated < 32-bits already. 3257 */ 3258 if (size >= 4) 3259 return; 3260 __reg_combine_64_into_32(reg); 3261 } 3262 3263 static bool bpf_map_is_rdonly(const struct bpf_map *map) 3264 { 3265 return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen; 3266 } 3267 3268 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 3269 { 3270 void *ptr; 3271 u64 addr; 3272 int err; 3273 3274 err = map->ops->map_direct_value_addr(map, &addr, off); 3275 if (err) 3276 return err; 3277 ptr = (void *)(long)addr + off; 3278 3279 switch (size) { 3280 case sizeof(u8): 3281 *val = (u64)*(u8 *)ptr; 3282 break; 3283 case sizeof(u16): 3284 *val = (u64)*(u16 *)ptr; 3285 break; 3286 case sizeof(u32): 3287 *val = (u64)*(u32 *)ptr; 3288 break; 3289 case sizeof(u64): 3290 *val = *(u64 *)ptr; 3291 break; 3292 default: 3293 return -EINVAL; 3294 } 3295 return 0; 3296 } 3297 3298 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 3299 struct bpf_reg_state *regs, 3300 int regno, int off, int size, 3301 enum bpf_access_type atype, 3302 int value_regno) 3303 { 3304 struct bpf_reg_state *reg = regs + regno; 3305 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 3306 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 3307 u32 btf_id; 3308 int ret; 3309 3310 if (off < 0) { 3311 verbose(env, 3312 "R%d is ptr_%s invalid negative access: off=%d\n", 3313 regno, tname, off); 3314 return -EACCES; 3315 } 3316 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3317 char tn_buf[48]; 3318 3319 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3320 verbose(env, 3321 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 3322 regno, tname, off, tn_buf); 3323 return -EACCES; 3324 } 3325 3326 if (env->ops->btf_struct_access) { 3327 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, 3328 off, size, atype, &btf_id); 3329 } else { 3330 if (atype != BPF_READ) { 3331 verbose(env, "only read is supported\n"); 3332 return -EACCES; 3333 } 3334 3335 ret = btf_struct_access(&env->log, reg->btf, t, off, size, 3336 atype, &btf_id); 3337 } 3338 3339 if (ret < 0) 3340 return ret; 3341 3342 if (atype == BPF_READ && value_regno >= 0) 3343 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id); 3344 3345 return 0; 3346 } 3347 3348 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 3349 struct bpf_reg_state *regs, 3350 int regno, int off, int size, 3351 enum bpf_access_type atype, 3352 int value_regno) 3353 { 3354 struct bpf_reg_state *reg = regs + regno; 3355 struct bpf_map *map = reg->map_ptr; 3356 const struct btf_type *t; 3357 const char *tname; 3358 u32 btf_id; 3359 int ret; 3360 3361 if (!btf_vmlinux) { 3362 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 3363 return -ENOTSUPP; 3364 } 3365 3366 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 3367 verbose(env, "map_ptr access not supported for map type %d\n", 3368 map->map_type); 3369 return -ENOTSUPP; 3370 } 3371 3372 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 3373 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 3374 3375 if (!env->allow_ptr_to_map_access) { 3376 verbose(env, 3377 "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 3378 tname); 3379 return -EPERM; 3380 } 3381 3382 if (off < 0) { 3383 verbose(env, "R%d is %s invalid negative access: off=%d\n", 3384 regno, tname, off); 3385 return -EACCES; 3386 } 3387 3388 if (atype != BPF_READ) { 3389 verbose(env, "only read from %s is supported\n", tname); 3390 return -EACCES; 3391 } 3392 3393 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id); 3394 if (ret < 0) 3395 return ret; 3396 3397 if (value_regno >= 0) 3398 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id); 3399 3400 return 0; 3401 } 3402 3403 3404 /* check whether memory at (regno + off) is accessible for t = (read | write) 3405 * if t==write, value_regno is a register which value is stored into memory 3406 * if t==read, value_regno is a register which will receive the value from memory 3407 * if t==write && value_regno==-1, some unknown value is stored into memory 3408 * if t==read && value_regno==-1, don't care what we read from memory 3409 */ 3410 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 3411 int off, int bpf_size, enum bpf_access_type t, 3412 int value_regno, bool strict_alignment_once) 3413 { 3414 struct bpf_reg_state *regs = cur_regs(env); 3415 struct bpf_reg_state *reg = regs + regno; 3416 struct bpf_func_state *state; 3417 int size, err = 0; 3418 3419 size = bpf_size_to_bytes(bpf_size); 3420 if (size < 0) 3421 return size; 3422 3423 /* alignment checks will add in reg->off themselves */ 3424 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 3425 if (err) 3426 return err; 3427 3428 /* for access checks, reg->off is just part of off */ 3429 off += reg->off; 3430 3431 if (reg->type == PTR_TO_MAP_VALUE) { 3432 if (t == BPF_WRITE && value_regno >= 0 && 3433 is_pointer_value(env, value_regno)) { 3434 verbose(env, "R%d leaks addr into map\n", value_regno); 3435 return -EACCES; 3436 } 3437 err = check_map_access_type(env, regno, off, size, t); 3438 if (err) 3439 return err; 3440 err = check_map_access(env, regno, off, size, false); 3441 if (!err && t == BPF_READ && value_regno >= 0) { 3442 struct bpf_map *map = reg->map_ptr; 3443 3444 /* if map is read-only, track its contents as scalars */ 3445 if (tnum_is_const(reg->var_off) && 3446 bpf_map_is_rdonly(map) && 3447 map->ops->map_direct_value_addr) { 3448 int map_off = off + reg->var_off.value; 3449 u64 val = 0; 3450 3451 err = bpf_map_direct_read(map, map_off, size, 3452 &val); 3453 if (err) 3454 return err; 3455 3456 regs[value_regno].type = SCALAR_VALUE; 3457 __mark_reg_known(®s[value_regno], val); 3458 } else { 3459 mark_reg_unknown(env, regs, value_regno); 3460 } 3461 } 3462 } else if (reg->type == PTR_TO_MEM) { 3463 if (t == BPF_WRITE && value_regno >= 0 && 3464 is_pointer_value(env, value_regno)) { 3465 verbose(env, "R%d leaks addr into mem\n", value_regno); 3466 return -EACCES; 3467 } 3468 err = check_mem_region_access(env, regno, off, size, 3469 reg->mem_size, false); 3470 if (!err && t == BPF_READ && value_regno >= 0) 3471 mark_reg_unknown(env, regs, value_regno); 3472 } else if (reg->type == PTR_TO_CTX) { 3473 enum bpf_reg_type reg_type = SCALAR_VALUE; 3474 struct btf *btf = NULL; 3475 u32 btf_id = 0; 3476 3477 if (t == BPF_WRITE && value_regno >= 0 && 3478 is_pointer_value(env, value_regno)) { 3479 verbose(env, "R%d leaks addr into ctx\n", value_regno); 3480 return -EACCES; 3481 } 3482 3483 err = check_ctx_reg(env, reg, regno); 3484 if (err < 0) 3485 return err; 3486 3487 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id); 3488 if (err) 3489 verbose_linfo(env, insn_idx, "; "); 3490 if (!err && t == BPF_READ && value_regno >= 0) { 3491 /* ctx access returns either a scalar, or a 3492 * PTR_TO_PACKET[_META,_END]. In the latter 3493 * case, we know the offset is zero. 3494 */ 3495 if (reg_type == SCALAR_VALUE) { 3496 mark_reg_unknown(env, regs, value_regno); 3497 } else { 3498 mark_reg_known_zero(env, regs, 3499 value_regno); 3500 if (reg_type_may_be_null(reg_type)) 3501 regs[value_regno].id = ++env->id_gen; 3502 /* A load of ctx field could have different 3503 * actual load size with the one encoded in the 3504 * insn. When the dst is PTR, it is for sure not 3505 * a sub-register. 3506 */ 3507 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 3508 if (reg_type == PTR_TO_BTF_ID || 3509 reg_type == PTR_TO_BTF_ID_OR_NULL) { 3510 regs[value_regno].btf = btf; 3511 regs[value_regno].btf_id = btf_id; 3512 } 3513 } 3514 regs[value_regno].type = reg_type; 3515 } 3516 3517 } else if (reg->type == PTR_TO_STACK) { 3518 off += reg->var_off.value; 3519 err = check_stack_access(env, reg, off, size); 3520 if (err) 3521 return err; 3522 3523 state = func(env, reg); 3524 err = update_stack_depth(env, state, off); 3525 if (err) 3526 return err; 3527 3528 if (t == BPF_WRITE) 3529 err = check_stack_write(env, state, off, size, 3530 value_regno, insn_idx); 3531 else 3532 err = check_stack_read(env, state, off, size, 3533 value_regno); 3534 } else if (reg_is_pkt_pointer(reg)) { 3535 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 3536 verbose(env, "cannot write into packet\n"); 3537 return -EACCES; 3538 } 3539 if (t == BPF_WRITE && value_regno >= 0 && 3540 is_pointer_value(env, value_regno)) { 3541 verbose(env, "R%d leaks addr into packet\n", 3542 value_regno); 3543 return -EACCES; 3544 } 3545 err = check_packet_access(env, regno, off, size, false); 3546 if (!err && t == BPF_READ && value_regno >= 0) 3547 mark_reg_unknown(env, regs, value_regno); 3548 } else if (reg->type == PTR_TO_FLOW_KEYS) { 3549 if (t == BPF_WRITE && value_regno >= 0 && 3550 is_pointer_value(env, value_regno)) { 3551 verbose(env, "R%d leaks addr into flow keys\n", 3552 value_regno); 3553 return -EACCES; 3554 } 3555 3556 err = check_flow_keys_access(env, off, size); 3557 if (!err && t == BPF_READ && value_regno >= 0) 3558 mark_reg_unknown(env, regs, value_regno); 3559 } else if (type_is_sk_pointer(reg->type)) { 3560 if (t == BPF_WRITE) { 3561 verbose(env, "R%d cannot write into %s\n", 3562 regno, reg_type_str[reg->type]); 3563 return -EACCES; 3564 } 3565 err = check_sock_access(env, insn_idx, regno, off, size, t); 3566 if (!err && value_regno >= 0) 3567 mark_reg_unknown(env, regs, value_regno); 3568 } else if (reg->type == PTR_TO_TP_BUFFER) { 3569 err = check_tp_buffer_access(env, reg, regno, off, size); 3570 if (!err && t == BPF_READ && value_regno >= 0) 3571 mark_reg_unknown(env, regs, value_regno); 3572 } else if (reg->type == PTR_TO_BTF_ID) { 3573 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 3574 value_regno); 3575 } else if (reg->type == CONST_PTR_TO_MAP) { 3576 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 3577 value_regno); 3578 } else if (reg->type == PTR_TO_RDONLY_BUF) { 3579 if (t == BPF_WRITE) { 3580 verbose(env, "R%d cannot write into %s\n", 3581 regno, reg_type_str[reg->type]); 3582 return -EACCES; 3583 } 3584 err = check_buffer_access(env, reg, regno, off, size, false, 3585 "rdonly", 3586 &env->prog->aux->max_rdonly_access); 3587 if (!err && value_regno >= 0) 3588 mark_reg_unknown(env, regs, value_regno); 3589 } else if (reg->type == PTR_TO_RDWR_BUF) { 3590 err = check_buffer_access(env, reg, regno, off, size, false, 3591 "rdwr", 3592 &env->prog->aux->max_rdwr_access); 3593 if (!err && t == BPF_READ && value_regno >= 0) 3594 mark_reg_unknown(env, regs, value_regno); 3595 } else { 3596 verbose(env, "R%d invalid mem access '%s'\n", regno, 3597 reg_type_str[reg->type]); 3598 return -EACCES; 3599 } 3600 3601 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 3602 regs[value_regno].type == SCALAR_VALUE) { 3603 /* b/h/w load zero-extends, mark upper bits as known 0 */ 3604 coerce_reg_to_size(®s[value_regno], size); 3605 } 3606 return err; 3607 } 3608 3609 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 3610 { 3611 int err; 3612 3613 if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || 3614 insn->imm != 0) { 3615 verbose(env, "BPF_XADD uses reserved fields\n"); 3616 return -EINVAL; 3617 } 3618 3619 /* check src1 operand */ 3620 err = check_reg_arg(env, insn->src_reg, SRC_OP); 3621 if (err) 3622 return err; 3623 3624 /* check src2 operand */ 3625 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 3626 if (err) 3627 return err; 3628 3629 if (is_pointer_value(env, insn->src_reg)) { 3630 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 3631 return -EACCES; 3632 } 3633 3634 if (is_ctx_reg(env, insn->dst_reg) || 3635 is_pkt_reg(env, insn->dst_reg) || 3636 is_flow_key_reg(env, insn->dst_reg) || 3637 is_sk_reg(env, insn->dst_reg)) { 3638 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", 3639 insn->dst_reg, 3640 reg_type_str[reg_state(env, insn->dst_reg)->type]); 3641 return -EACCES; 3642 } 3643 3644 /* check whether atomic_add can read the memory */ 3645 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 3646 BPF_SIZE(insn->code), BPF_READ, -1, true); 3647 if (err) 3648 return err; 3649 3650 /* check whether atomic_add can write into the same memory */ 3651 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 3652 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 3653 } 3654 3655 static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, 3656 int off, int access_size, 3657 bool zero_size_allowed) 3658 { 3659 struct bpf_reg_state *reg = reg_state(env, regno); 3660 3661 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || 3662 access_size < 0 || (access_size == 0 && !zero_size_allowed)) { 3663 if (tnum_is_const(reg->var_off)) { 3664 verbose(env, "invalid stack type R%d off=%d access_size=%d\n", 3665 regno, off, access_size); 3666 } else { 3667 char tn_buf[48]; 3668 3669 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3670 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", 3671 regno, tn_buf, access_size); 3672 } 3673 return -EACCES; 3674 } 3675 return 0; 3676 } 3677 3678 /* when register 'regno' is passed into function that will read 'access_size' 3679 * bytes from that pointer, make sure that it's within stack boundary 3680 * and all elements of stack are initialized. 3681 * Unlike most pointer bounds-checking functions, this one doesn't take an 3682 * 'off' argument, so it has to add in reg->off itself. 3683 */ 3684 static int check_stack_boundary(struct bpf_verifier_env *env, int regno, 3685 int access_size, bool zero_size_allowed, 3686 struct bpf_call_arg_meta *meta) 3687 { 3688 struct bpf_reg_state *reg = reg_state(env, regno); 3689 struct bpf_func_state *state = func(env, reg); 3690 int err, min_off, max_off, i, j, slot, spi; 3691 3692 if (tnum_is_const(reg->var_off)) { 3693 min_off = max_off = reg->var_off.value + reg->off; 3694 err = __check_stack_boundary(env, regno, min_off, access_size, 3695 zero_size_allowed); 3696 if (err) 3697 return err; 3698 } else { 3699 /* Variable offset is prohibited for unprivileged mode for 3700 * simplicity since it requires corresponding support in 3701 * Spectre masking for stack ALU. 3702 * See also retrieve_ptr_limit(). 3703 */ 3704 if (!env->bypass_spec_v1) { 3705 char tn_buf[48]; 3706 3707 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3708 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", 3709 regno, tn_buf); 3710 return -EACCES; 3711 } 3712 /* Only initialized buffer on stack is allowed to be accessed 3713 * with variable offset. With uninitialized buffer it's hard to 3714 * guarantee that whole memory is marked as initialized on 3715 * helper return since specific bounds are unknown what may 3716 * cause uninitialized stack leaking. 3717 */ 3718 if (meta && meta->raw_mode) 3719 meta = NULL; 3720 3721 if (reg->smax_value >= BPF_MAX_VAR_OFF || 3722 reg->smax_value <= -BPF_MAX_VAR_OFF) { 3723 verbose(env, "R%d unbounded indirect variable offset stack access\n", 3724 regno); 3725 return -EACCES; 3726 } 3727 min_off = reg->smin_value + reg->off; 3728 max_off = reg->smax_value + reg->off; 3729 err = __check_stack_boundary(env, regno, min_off, access_size, 3730 zero_size_allowed); 3731 if (err) { 3732 verbose(env, "R%d min value is outside of stack bound\n", 3733 regno); 3734 return err; 3735 } 3736 err = __check_stack_boundary(env, regno, max_off, access_size, 3737 zero_size_allowed); 3738 if (err) { 3739 verbose(env, "R%d max value is outside of stack bound\n", 3740 regno); 3741 return err; 3742 } 3743 } 3744 3745 if (meta && meta->raw_mode) { 3746 meta->access_size = access_size; 3747 meta->regno = regno; 3748 return 0; 3749 } 3750 3751 for (i = min_off; i < max_off + access_size; i++) { 3752 u8 *stype; 3753 3754 slot = -i - 1; 3755 spi = slot / BPF_REG_SIZE; 3756 if (state->allocated_stack <= slot) 3757 goto err; 3758 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3759 if (*stype == STACK_MISC) 3760 goto mark; 3761 if (*stype == STACK_ZERO) { 3762 /* helper can write anything into the stack */ 3763 *stype = STACK_MISC; 3764 goto mark; 3765 } 3766 3767 if (state->stack[spi].slot_type[0] == STACK_SPILL && 3768 state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) 3769 goto mark; 3770 3771 if (state->stack[spi].slot_type[0] == STACK_SPILL && 3772 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 3773 env->allow_ptr_leaks)) { 3774 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 3775 for (j = 0; j < BPF_REG_SIZE; j++) 3776 state->stack[spi].slot_type[j] = STACK_MISC; 3777 goto mark; 3778 } 3779 3780 err: 3781 if (tnum_is_const(reg->var_off)) { 3782 verbose(env, "invalid indirect read from stack off %d+%d size %d\n", 3783 min_off, i - min_off, access_size); 3784 } else { 3785 char tn_buf[48]; 3786 3787 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3788 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", 3789 tn_buf, i - min_off, access_size); 3790 } 3791 return -EACCES; 3792 mark: 3793 /* reading any byte out of 8-byte 'spill_slot' will cause 3794 * the whole slot to be marked as 'read' 3795 */ 3796 mark_reg_read(env, &state->stack[spi].spilled_ptr, 3797 state->stack[spi].spilled_ptr.parent, 3798 REG_LIVE_READ64); 3799 } 3800 return update_stack_depth(env, state, min_off); 3801 } 3802 3803 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 3804 int access_size, bool zero_size_allowed, 3805 struct bpf_call_arg_meta *meta) 3806 { 3807 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 3808 3809 switch (reg->type) { 3810 case PTR_TO_PACKET: 3811 case PTR_TO_PACKET_META: 3812 return check_packet_access(env, regno, reg->off, access_size, 3813 zero_size_allowed); 3814 case PTR_TO_MAP_VALUE: 3815 if (check_map_access_type(env, regno, reg->off, access_size, 3816 meta && meta->raw_mode ? BPF_WRITE : 3817 BPF_READ)) 3818 return -EACCES; 3819 return check_map_access(env, regno, reg->off, access_size, 3820 zero_size_allowed); 3821 case PTR_TO_MEM: 3822 return check_mem_region_access(env, regno, reg->off, 3823 access_size, reg->mem_size, 3824 zero_size_allowed); 3825 case PTR_TO_RDONLY_BUF: 3826 if (meta && meta->raw_mode) 3827 return -EACCES; 3828 return check_buffer_access(env, reg, regno, reg->off, 3829 access_size, zero_size_allowed, 3830 "rdonly", 3831 &env->prog->aux->max_rdonly_access); 3832 case PTR_TO_RDWR_BUF: 3833 return check_buffer_access(env, reg, regno, reg->off, 3834 access_size, zero_size_allowed, 3835 "rdwr", 3836 &env->prog->aux->max_rdwr_access); 3837 case PTR_TO_STACK: 3838 return check_stack_boundary(env, regno, access_size, 3839 zero_size_allowed, meta); 3840 default: /* scalar_value or invalid ptr */ 3841 /* Allow zero-byte read from NULL, regardless of pointer type */ 3842 if (zero_size_allowed && access_size == 0 && 3843 register_is_null(reg)) 3844 return 0; 3845 3846 verbose(env, "R%d type=%s expected=%s\n", regno, 3847 reg_type_str[reg->type], 3848 reg_type_str[PTR_TO_STACK]); 3849 return -EACCES; 3850 } 3851 } 3852 3853 /* Implementation details: 3854 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL 3855 * Two bpf_map_lookups (even with the same key) will have different reg->id. 3856 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after 3857 * value_or_null->value transition, since the verifier only cares about 3858 * the range of access to valid map value pointer and doesn't care about actual 3859 * address of the map element. 3860 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 3861 * reg->id > 0 after value_or_null->value transition. By doing so 3862 * two bpf_map_lookups will be considered two different pointers that 3863 * point to different bpf_spin_locks. 3864 * The verifier allows taking only one bpf_spin_lock at a time to avoid 3865 * dead-locks. 3866 * Since only one bpf_spin_lock is allowed the checks are simpler than 3867 * reg_is_refcounted() logic. The verifier needs to remember only 3868 * one spin_lock instead of array of acquired_refs. 3869 * cur_state->active_spin_lock remembers which map value element got locked 3870 * and clears it after bpf_spin_unlock. 3871 */ 3872 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 3873 bool is_lock) 3874 { 3875 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 3876 struct bpf_verifier_state *cur = env->cur_state; 3877 bool is_const = tnum_is_const(reg->var_off); 3878 struct bpf_map *map = reg->map_ptr; 3879 u64 val = reg->var_off.value; 3880 3881 if (!is_const) { 3882 verbose(env, 3883 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 3884 regno); 3885 return -EINVAL; 3886 } 3887 if (!map->btf) { 3888 verbose(env, 3889 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 3890 map->name); 3891 return -EINVAL; 3892 } 3893 if (!map_value_has_spin_lock(map)) { 3894 if (map->spin_lock_off == -E2BIG) 3895 verbose(env, 3896 "map '%s' has more than one 'struct bpf_spin_lock'\n", 3897 map->name); 3898 else if (map->spin_lock_off == -ENOENT) 3899 verbose(env, 3900 "map '%s' doesn't have 'struct bpf_spin_lock'\n", 3901 map->name); 3902 else 3903 verbose(env, 3904 "map '%s' is not a struct type or bpf_spin_lock is mangled\n", 3905 map->name); 3906 return -EINVAL; 3907 } 3908 if (map->spin_lock_off != val + reg->off) { 3909 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", 3910 val + reg->off); 3911 return -EINVAL; 3912 } 3913 if (is_lock) { 3914 if (cur->active_spin_lock) { 3915 verbose(env, 3916 "Locking two bpf_spin_locks are not allowed\n"); 3917 return -EINVAL; 3918 } 3919 cur->active_spin_lock = reg->id; 3920 } else { 3921 if (!cur->active_spin_lock) { 3922 verbose(env, "bpf_spin_unlock without taking a lock\n"); 3923 return -EINVAL; 3924 } 3925 if (cur->active_spin_lock != reg->id) { 3926 verbose(env, "bpf_spin_unlock of different lock\n"); 3927 return -EINVAL; 3928 } 3929 cur->active_spin_lock = 0; 3930 } 3931 return 0; 3932 } 3933 3934 static bool arg_type_is_mem_ptr(enum bpf_arg_type type) 3935 { 3936 return type == ARG_PTR_TO_MEM || 3937 type == ARG_PTR_TO_MEM_OR_NULL || 3938 type == ARG_PTR_TO_UNINIT_MEM; 3939 } 3940 3941 static bool arg_type_is_mem_size(enum bpf_arg_type type) 3942 { 3943 return type == ARG_CONST_SIZE || 3944 type == ARG_CONST_SIZE_OR_ZERO; 3945 } 3946 3947 static bool arg_type_is_alloc_size(enum bpf_arg_type type) 3948 { 3949 return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; 3950 } 3951 3952 static bool arg_type_is_int_ptr(enum bpf_arg_type type) 3953 { 3954 return type == ARG_PTR_TO_INT || 3955 type == ARG_PTR_TO_LONG; 3956 } 3957 3958 static int int_ptr_type_to_size(enum bpf_arg_type type) 3959 { 3960 if (type == ARG_PTR_TO_INT) 3961 return sizeof(u32); 3962 else if (type == ARG_PTR_TO_LONG) 3963 return sizeof(u64); 3964 3965 return -EINVAL; 3966 } 3967 3968 static int resolve_map_arg_type(struct bpf_verifier_env *env, 3969 const struct bpf_call_arg_meta *meta, 3970 enum bpf_arg_type *arg_type) 3971 { 3972 if (!meta->map_ptr) { 3973 /* kernel subsystem misconfigured verifier */ 3974 verbose(env, "invalid map_ptr to access map->type\n"); 3975 return -EACCES; 3976 } 3977 3978 switch (meta->map_ptr->map_type) { 3979 case BPF_MAP_TYPE_SOCKMAP: 3980 case BPF_MAP_TYPE_SOCKHASH: 3981 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 3982 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 3983 } else { 3984 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 3985 return -EINVAL; 3986 } 3987 break; 3988 3989 default: 3990 break; 3991 } 3992 return 0; 3993 } 3994 3995 struct bpf_reg_types { 3996 const enum bpf_reg_type types[10]; 3997 u32 *btf_id; 3998 }; 3999 4000 static const struct bpf_reg_types map_key_value_types = { 4001 .types = { 4002 PTR_TO_STACK, 4003 PTR_TO_PACKET, 4004 PTR_TO_PACKET_META, 4005 PTR_TO_MAP_VALUE, 4006 }, 4007 }; 4008 4009 static const struct bpf_reg_types sock_types = { 4010 .types = { 4011 PTR_TO_SOCK_COMMON, 4012 PTR_TO_SOCKET, 4013 PTR_TO_TCP_SOCK, 4014 PTR_TO_XDP_SOCK, 4015 }, 4016 }; 4017 4018 #ifdef CONFIG_NET 4019 static const struct bpf_reg_types btf_id_sock_common_types = { 4020 .types = { 4021 PTR_TO_SOCK_COMMON, 4022 PTR_TO_SOCKET, 4023 PTR_TO_TCP_SOCK, 4024 PTR_TO_XDP_SOCK, 4025 PTR_TO_BTF_ID, 4026 }, 4027 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 4028 }; 4029 #endif 4030 4031 static const struct bpf_reg_types mem_types = { 4032 .types = { 4033 PTR_TO_STACK, 4034 PTR_TO_PACKET, 4035 PTR_TO_PACKET_META, 4036 PTR_TO_MAP_VALUE, 4037 PTR_TO_MEM, 4038 PTR_TO_RDONLY_BUF, 4039 PTR_TO_RDWR_BUF, 4040 }, 4041 }; 4042 4043 static const struct bpf_reg_types int_ptr_types = { 4044 .types = { 4045 PTR_TO_STACK, 4046 PTR_TO_PACKET, 4047 PTR_TO_PACKET_META, 4048 PTR_TO_MAP_VALUE, 4049 }, 4050 }; 4051 4052 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 4053 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 4054 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 4055 static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; 4056 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 4057 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; 4058 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; 4059 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; 4060 4061 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 4062 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, 4063 [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, 4064 [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, 4065 [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, 4066 [ARG_CONST_SIZE] = &scalar_types, 4067 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 4068 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 4069 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 4070 [ARG_PTR_TO_CTX] = &context_types, 4071 [ARG_PTR_TO_CTX_OR_NULL] = &context_types, 4072 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 4073 #ifdef CONFIG_NET 4074 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 4075 #endif 4076 [ARG_PTR_TO_SOCKET] = &fullsock_types, 4077 [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, 4078 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 4079 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 4080 [ARG_PTR_TO_MEM] = &mem_types, 4081 [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, 4082 [ARG_PTR_TO_UNINIT_MEM] = &mem_types, 4083 [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, 4084 [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, 4085 [ARG_PTR_TO_INT] = &int_ptr_types, 4086 [ARG_PTR_TO_LONG] = &int_ptr_types, 4087 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 4088 }; 4089 4090 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 4091 enum bpf_arg_type arg_type, 4092 const u32 *arg_btf_id) 4093 { 4094 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4095 enum bpf_reg_type expected, type = reg->type; 4096 const struct bpf_reg_types *compatible; 4097 int i, j; 4098 4099 compatible = compatible_reg_types[arg_type]; 4100 if (!compatible) { 4101 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 4102 return -EFAULT; 4103 } 4104 4105 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 4106 expected = compatible->types[i]; 4107 if (expected == NOT_INIT) 4108 break; 4109 4110 if (type == expected) 4111 goto found; 4112 } 4113 4114 verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); 4115 for (j = 0; j + 1 < i; j++) 4116 verbose(env, "%s, ", reg_type_str[compatible->types[j]]); 4117 verbose(env, "%s\n", reg_type_str[compatible->types[j]]); 4118 return -EACCES; 4119 4120 found: 4121 if (type == PTR_TO_BTF_ID) { 4122 if (!arg_btf_id) { 4123 if (!compatible->btf_id) { 4124 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 4125 return -EFAULT; 4126 } 4127 arg_btf_id = compatible->btf_id; 4128 } 4129 4130 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 4131 btf_vmlinux, *arg_btf_id)) { 4132 verbose(env, "R%d is of type %s but %s is expected\n", 4133 regno, kernel_type_name(reg->btf, reg->btf_id), 4134 kernel_type_name(btf_vmlinux, *arg_btf_id)); 4135 return -EACCES; 4136 } 4137 4138 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4139 verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", 4140 regno); 4141 return -EACCES; 4142 } 4143 } 4144 4145 return 0; 4146 } 4147 4148 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 4149 struct bpf_call_arg_meta *meta, 4150 const struct bpf_func_proto *fn) 4151 { 4152 u32 regno = BPF_REG_1 + arg; 4153 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 4154 enum bpf_arg_type arg_type = fn->arg_type[arg]; 4155 enum bpf_reg_type type = reg->type; 4156 int err = 0; 4157 4158 if (arg_type == ARG_DONTCARE) 4159 return 0; 4160 4161 err = check_reg_arg(env, regno, SRC_OP); 4162 if (err) 4163 return err; 4164 4165 if (arg_type == ARG_ANYTHING) { 4166 if (is_pointer_value(env, regno)) { 4167 verbose(env, "R%d leaks addr into helper function\n", 4168 regno); 4169 return -EACCES; 4170 } 4171 return 0; 4172 } 4173 4174 if (type_is_pkt_pointer(type) && 4175 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 4176 verbose(env, "helper access to the packet is not allowed\n"); 4177 return -EACCES; 4178 } 4179 4180 if (arg_type == ARG_PTR_TO_MAP_VALUE || 4181 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || 4182 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { 4183 err = resolve_map_arg_type(env, meta, &arg_type); 4184 if (err) 4185 return err; 4186 } 4187 4188 if (register_is_null(reg) && arg_type_may_be_null(arg_type)) 4189 /* A NULL register has a SCALAR_VALUE type, so skip 4190 * type checking. 4191 */ 4192 goto skip_type_check; 4193 4194 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); 4195 if (err) 4196 return err; 4197 4198 if (type == PTR_TO_CTX) { 4199 err = check_ctx_reg(env, reg, regno); 4200 if (err < 0) 4201 return err; 4202 } 4203 4204 skip_type_check: 4205 if (reg->ref_obj_id) { 4206 if (meta->ref_obj_id) { 4207 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 4208 regno, reg->ref_obj_id, 4209 meta->ref_obj_id); 4210 return -EFAULT; 4211 } 4212 meta->ref_obj_id = reg->ref_obj_id; 4213 } 4214 4215 if (arg_type == ARG_CONST_MAP_PTR) { 4216 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 4217 meta->map_ptr = reg->map_ptr; 4218 } else if (arg_type == ARG_PTR_TO_MAP_KEY) { 4219 /* bpf_map_xxx(..., map_ptr, ..., key) call: 4220 * check that [key, key + map->key_size) are within 4221 * stack limits and initialized 4222 */ 4223 if (!meta->map_ptr) { 4224 /* in function declaration map_ptr must come before 4225 * map_key, so that it's verified and known before 4226 * we have to check map_key here. Otherwise it means 4227 * that kernel subsystem misconfigured verifier 4228 */ 4229 verbose(env, "invalid map_ptr to access map->key\n"); 4230 return -EACCES; 4231 } 4232 err = check_helper_mem_access(env, regno, 4233 meta->map_ptr->key_size, false, 4234 NULL); 4235 } else if (arg_type == ARG_PTR_TO_MAP_VALUE || 4236 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && 4237 !register_is_null(reg)) || 4238 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { 4239 /* bpf_map_xxx(..., map_ptr, ..., value) call: 4240 * check [value, value + map->value_size) validity 4241 */ 4242 if (!meta->map_ptr) { 4243 /* kernel subsystem misconfigured verifier */ 4244 verbose(env, "invalid map_ptr to access map->value\n"); 4245 return -EACCES; 4246 } 4247 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); 4248 err = check_helper_mem_access(env, regno, 4249 meta->map_ptr->value_size, false, 4250 meta); 4251 } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { 4252 if (!reg->btf_id) { 4253 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 4254 return -EACCES; 4255 } 4256 meta->ret_btf = reg->btf; 4257 meta->ret_btf_id = reg->btf_id; 4258 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { 4259 if (meta->func_id == BPF_FUNC_spin_lock) { 4260 if (process_spin_lock(env, regno, true)) 4261 return -EACCES; 4262 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 4263 if (process_spin_lock(env, regno, false)) 4264 return -EACCES; 4265 } else { 4266 verbose(env, "verifier internal error\n"); 4267 return -EFAULT; 4268 } 4269 } else if (arg_type_is_mem_ptr(arg_type)) { 4270 /* The access to this pointer is only checked when we hit the 4271 * next is_mem_size argument below. 4272 */ 4273 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); 4274 } else if (arg_type_is_mem_size(arg_type)) { 4275 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); 4276 4277 /* This is used to refine r0 return value bounds for helpers 4278 * that enforce this value as an upper bound on return values. 4279 * See do_refine_retval_range() for helpers that can refine 4280 * the return value. C type of helper is u32 so we pull register 4281 * bound from umax_value however, if negative verifier errors 4282 * out. Only upper bounds can be learned because retval is an 4283 * int type and negative retvals are allowed. 4284 */ 4285 meta->msize_max_value = reg->umax_value; 4286 4287 /* The register is SCALAR_VALUE; the access check 4288 * happens using its boundaries. 4289 */ 4290 if (!tnum_is_const(reg->var_off)) 4291 /* For unprivileged variable accesses, disable raw 4292 * mode so that the program is required to 4293 * initialize all the memory that the helper could 4294 * just partially fill up. 4295 */ 4296 meta = NULL; 4297 4298 if (reg->smin_value < 0) { 4299 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 4300 regno); 4301 return -EACCES; 4302 } 4303 4304 if (reg->umin_value == 0) { 4305 err = check_helper_mem_access(env, regno - 1, 0, 4306 zero_size_allowed, 4307 meta); 4308 if (err) 4309 return err; 4310 } 4311 4312 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 4313 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 4314 regno); 4315 return -EACCES; 4316 } 4317 err = check_helper_mem_access(env, regno - 1, 4318 reg->umax_value, 4319 zero_size_allowed, meta); 4320 if (!err) 4321 err = mark_chain_precision(env, regno); 4322 } else if (arg_type_is_alloc_size(arg_type)) { 4323 if (!tnum_is_const(reg->var_off)) { 4324 verbose(env, "R%d unbounded size, use 'var &= const' or 'if (var < const)'\n", 4325 regno); 4326 return -EACCES; 4327 } 4328 meta->mem_size = reg->var_off.value; 4329 } else if (arg_type_is_int_ptr(arg_type)) { 4330 int size = int_ptr_type_to_size(arg_type); 4331 4332 err = check_helper_mem_access(env, regno, size, false, meta); 4333 if (err) 4334 return err; 4335 err = check_ptr_alignment(env, reg, 0, size, true); 4336 } 4337 4338 return err; 4339 } 4340 4341 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 4342 { 4343 enum bpf_attach_type eatype = env->prog->expected_attach_type; 4344 enum bpf_prog_type type = resolve_prog_type(env->prog); 4345 4346 if (func_id != BPF_FUNC_map_update_elem) 4347 return false; 4348 4349 /* It's not possible to get access to a locked struct sock in these 4350 * contexts, so updating is safe. 4351 */ 4352 switch (type) { 4353 case BPF_PROG_TYPE_TRACING: 4354 if (eatype == BPF_TRACE_ITER) 4355 return true; 4356 break; 4357 case BPF_PROG_TYPE_SOCKET_FILTER: 4358 case BPF_PROG_TYPE_SCHED_CLS: 4359 case BPF_PROG_TYPE_SCHED_ACT: 4360 case BPF_PROG_TYPE_XDP: 4361 case BPF_PROG_TYPE_SK_REUSEPORT: 4362 case BPF_PROG_TYPE_FLOW_DISSECTOR: 4363 case BPF_PROG_TYPE_SK_LOOKUP: 4364 return true; 4365 default: 4366 break; 4367 } 4368 4369 verbose(env, "cannot update sockmap in this context\n"); 4370 return false; 4371 } 4372 4373 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 4374 { 4375 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); 4376 } 4377 4378 static int check_map_func_compatibility(struct bpf_verifier_env *env, 4379 struct bpf_map *map, int func_id) 4380 { 4381 if (!map) 4382 return 0; 4383 4384 /* We need a two way check, first is from map perspective ... */ 4385 switch (map->map_type) { 4386 case BPF_MAP_TYPE_PROG_ARRAY: 4387 if (func_id != BPF_FUNC_tail_call) 4388 goto error; 4389 break; 4390 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 4391 if (func_id != BPF_FUNC_perf_event_read && 4392 func_id != BPF_FUNC_perf_event_output && 4393 func_id != BPF_FUNC_skb_output && 4394 func_id != BPF_FUNC_perf_event_read_value && 4395 func_id != BPF_FUNC_xdp_output) 4396 goto error; 4397 break; 4398 case BPF_MAP_TYPE_RINGBUF: 4399 if (func_id != BPF_FUNC_ringbuf_output && 4400 func_id != BPF_FUNC_ringbuf_reserve && 4401 func_id != BPF_FUNC_ringbuf_submit && 4402 func_id != BPF_FUNC_ringbuf_discard && 4403 func_id != BPF_FUNC_ringbuf_query) 4404 goto error; 4405 break; 4406 case BPF_MAP_TYPE_STACK_TRACE: 4407 if (func_id != BPF_FUNC_get_stackid) 4408 goto error; 4409 break; 4410 case BPF_MAP_TYPE_CGROUP_ARRAY: 4411 if (func_id != BPF_FUNC_skb_under_cgroup && 4412 func_id != BPF_FUNC_current_task_under_cgroup) 4413 goto error; 4414 break; 4415 case BPF_MAP_TYPE_CGROUP_STORAGE: 4416 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 4417 if (func_id != BPF_FUNC_get_local_storage) 4418 goto error; 4419 break; 4420 case BPF_MAP_TYPE_DEVMAP: 4421 case BPF_MAP_TYPE_DEVMAP_HASH: 4422 if (func_id != BPF_FUNC_redirect_map && 4423 func_id != BPF_FUNC_map_lookup_elem) 4424 goto error; 4425 break; 4426 /* Restrict bpf side of cpumap and xskmap, open when use-cases 4427 * appear. 4428 */ 4429 case BPF_MAP_TYPE_CPUMAP: 4430 if (func_id != BPF_FUNC_redirect_map) 4431 goto error; 4432 break; 4433 case BPF_MAP_TYPE_XSKMAP: 4434 if (func_id != BPF_FUNC_redirect_map && 4435 func_id != BPF_FUNC_map_lookup_elem) 4436 goto error; 4437 break; 4438 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 4439 case BPF_MAP_TYPE_HASH_OF_MAPS: 4440 if (func_id != BPF_FUNC_map_lookup_elem) 4441 goto error; 4442 break; 4443 case BPF_MAP_TYPE_SOCKMAP: 4444 if (func_id != BPF_FUNC_sk_redirect_map && 4445 func_id != BPF_FUNC_sock_map_update && 4446 func_id != BPF_FUNC_map_delete_elem && 4447 func_id != BPF_FUNC_msg_redirect_map && 4448 func_id != BPF_FUNC_sk_select_reuseport && 4449 func_id != BPF_FUNC_map_lookup_elem && 4450 !may_update_sockmap(env, func_id)) 4451 goto error; 4452 break; 4453 case BPF_MAP_TYPE_SOCKHASH: 4454 if (func_id != BPF_FUNC_sk_redirect_hash && 4455 func_id != BPF_FUNC_sock_hash_update && 4456 func_id != BPF_FUNC_map_delete_elem && 4457 func_id != BPF_FUNC_msg_redirect_hash && 4458 func_id != BPF_FUNC_sk_select_reuseport && 4459 func_id != BPF_FUNC_map_lookup_elem && 4460 !may_update_sockmap(env, func_id)) 4461 goto error; 4462 break; 4463 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 4464 if (func_id != BPF_FUNC_sk_select_reuseport) 4465 goto error; 4466 break; 4467 case BPF_MAP_TYPE_QUEUE: 4468 case BPF_MAP_TYPE_STACK: 4469 if (func_id != BPF_FUNC_map_peek_elem && 4470 func_id != BPF_FUNC_map_pop_elem && 4471 func_id != BPF_FUNC_map_push_elem) 4472 goto error; 4473 break; 4474 case BPF_MAP_TYPE_SK_STORAGE: 4475 if (func_id != BPF_FUNC_sk_storage_get && 4476 func_id != BPF_FUNC_sk_storage_delete) 4477 goto error; 4478 break; 4479 case BPF_MAP_TYPE_INODE_STORAGE: 4480 if (func_id != BPF_FUNC_inode_storage_get && 4481 func_id != BPF_FUNC_inode_storage_delete) 4482 goto error; 4483 break; 4484 case BPF_MAP_TYPE_TASK_STORAGE: 4485 if (func_id != BPF_FUNC_task_storage_get && 4486 func_id != BPF_FUNC_task_storage_delete) 4487 goto error; 4488 break; 4489 default: 4490 break; 4491 } 4492 4493 /* ... and second from the function itself. */ 4494 switch (func_id) { 4495 case BPF_FUNC_tail_call: 4496 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 4497 goto error; 4498 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 4499 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 4500 return -EINVAL; 4501 } 4502 break; 4503 case BPF_FUNC_perf_event_read: 4504 case BPF_FUNC_perf_event_output: 4505 case BPF_FUNC_perf_event_read_value: 4506 case BPF_FUNC_skb_output: 4507 case BPF_FUNC_xdp_output: 4508 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 4509 goto error; 4510 break; 4511 case BPF_FUNC_get_stackid: 4512 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 4513 goto error; 4514 break; 4515 case BPF_FUNC_current_task_under_cgroup: 4516 case BPF_FUNC_skb_under_cgroup: 4517 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 4518 goto error; 4519 break; 4520 case BPF_FUNC_redirect_map: 4521 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 4522 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 4523 map->map_type != BPF_MAP_TYPE_CPUMAP && 4524 map->map_type != BPF_MAP_TYPE_XSKMAP) 4525 goto error; 4526 break; 4527 case BPF_FUNC_sk_redirect_map: 4528 case BPF_FUNC_msg_redirect_map: 4529 case BPF_FUNC_sock_map_update: 4530 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 4531 goto error; 4532 break; 4533 case BPF_FUNC_sk_redirect_hash: 4534 case BPF_FUNC_msg_redirect_hash: 4535 case BPF_FUNC_sock_hash_update: 4536 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 4537 goto error; 4538 break; 4539 case BPF_FUNC_get_local_storage: 4540 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 4541 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 4542 goto error; 4543 break; 4544 case BPF_FUNC_sk_select_reuseport: 4545 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 4546 map->map_type != BPF_MAP_TYPE_SOCKMAP && 4547 map->map_type != BPF_MAP_TYPE_SOCKHASH) 4548 goto error; 4549 break; 4550 case BPF_FUNC_map_peek_elem: 4551 case BPF_FUNC_map_pop_elem: 4552 case BPF_FUNC_map_push_elem: 4553 if (map->map_type != BPF_MAP_TYPE_QUEUE && 4554 map->map_type != BPF_MAP_TYPE_STACK) 4555 goto error; 4556 break; 4557 case BPF_FUNC_sk_storage_get: 4558 case BPF_FUNC_sk_storage_delete: 4559 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 4560 goto error; 4561 break; 4562 case BPF_FUNC_inode_storage_get: 4563 case BPF_FUNC_inode_storage_delete: 4564 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 4565 goto error; 4566 break; 4567 case BPF_FUNC_task_storage_get: 4568 case BPF_FUNC_task_storage_delete: 4569 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 4570 goto error; 4571 break; 4572 default: 4573 break; 4574 } 4575 4576 return 0; 4577 error: 4578 verbose(env, "cannot pass map_type %d into func %s#%d\n", 4579 map->map_type, func_id_name(func_id), func_id); 4580 return -EINVAL; 4581 } 4582 4583 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 4584 { 4585 int count = 0; 4586 4587 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 4588 count++; 4589 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 4590 count++; 4591 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 4592 count++; 4593 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 4594 count++; 4595 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 4596 count++; 4597 4598 /* We only support one arg being in raw mode at the moment, 4599 * which is sufficient for the helper functions we have 4600 * right now. 4601 */ 4602 return count <= 1; 4603 } 4604 4605 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, 4606 enum bpf_arg_type arg_next) 4607 { 4608 return (arg_type_is_mem_ptr(arg_curr) && 4609 !arg_type_is_mem_size(arg_next)) || 4610 (!arg_type_is_mem_ptr(arg_curr) && 4611 arg_type_is_mem_size(arg_next)); 4612 } 4613 4614 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 4615 { 4616 /* bpf_xxx(..., buf, len) call will access 'len' 4617 * bytes from memory 'buf'. Both arg types need 4618 * to be paired, so make sure there's no buggy 4619 * helper function specification. 4620 */ 4621 if (arg_type_is_mem_size(fn->arg1_type) || 4622 arg_type_is_mem_ptr(fn->arg5_type) || 4623 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || 4624 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || 4625 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || 4626 check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) 4627 return false; 4628 4629 return true; 4630 } 4631 4632 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) 4633 { 4634 int count = 0; 4635 4636 if (arg_type_may_be_refcounted(fn->arg1_type)) 4637 count++; 4638 if (arg_type_may_be_refcounted(fn->arg2_type)) 4639 count++; 4640 if (arg_type_may_be_refcounted(fn->arg3_type)) 4641 count++; 4642 if (arg_type_may_be_refcounted(fn->arg4_type)) 4643 count++; 4644 if (arg_type_may_be_refcounted(fn->arg5_type)) 4645 count++; 4646 4647 /* A reference acquiring function cannot acquire 4648 * another refcounted ptr. 4649 */ 4650 if (may_be_acquire_function(func_id) && count) 4651 return false; 4652 4653 /* We only support one arg being unreferenced at the moment, 4654 * which is sufficient for the helper functions we have right now. 4655 */ 4656 return count <= 1; 4657 } 4658 4659 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 4660 { 4661 int i; 4662 4663 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 4664 if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) 4665 return false; 4666 4667 if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) 4668 return false; 4669 } 4670 4671 return true; 4672 } 4673 4674 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 4675 { 4676 return check_raw_mode_ok(fn) && 4677 check_arg_pair_ok(fn) && 4678 check_btf_id_ok(fn) && 4679 check_refcount_ok(fn, func_id) ? 0 : -EINVAL; 4680 } 4681 4682 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 4683 * are now invalid, so turn them into unknown SCALAR_VALUE. 4684 */ 4685 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, 4686 struct bpf_func_state *state) 4687 { 4688 struct bpf_reg_state *regs = state->regs, *reg; 4689 int i; 4690 4691 for (i = 0; i < MAX_BPF_REG; i++) 4692 if (reg_is_pkt_pointer_any(®s[i])) 4693 mark_reg_unknown(env, regs, i); 4694 4695 bpf_for_each_spilled_reg(i, state, reg) { 4696 if (!reg) 4697 continue; 4698 if (reg_is_pkt_pointer_any(reg)) 4699 __mark_reg_unknown(env, reg); 4700 } 4701 } 4702 4703 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 4704 { 4705 struct bpf_verifier_state *vstate = env->cur_state; 4706 int i; 4707 4708 for (i = 0; i <= vstate->curframe; i++) 4709 __clear_all_pkt_pointers(env, vstate->frame[i]); 4710 } 4711 4712 enum { 4713 AT_PKT_END = -1, 4714 BEYOND_PKT_END = -2, 4715 }; 4716 4717 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 4718 { 4719 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4720 struct bpf_reg_state *reg = &state->regs[regn]; 4721 4722 if (reg->type != PTR_TO_PACKET) 4723 /* PTR_TO_PACKET_META is not supported yet */ 4724 return; 4725 4726 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 4727 * How far beyond pkt_end it goes is unknown. 4728 * if (!range_open) it's the case of pkt >= pkt_end 4729 * if (range_open) it's the case of pkt > pkt_end 4730 * hence this pointer is at least 1 byte bigger than pkt_end 4731 */ 4732 if (range_open) 4733 reg->range = BEYOND_PKT_END; 4734 else 4735 reg->range = AT_PKT_END; 4736 } 4737 4738 static void release_reg_references(struct bpf_verifier_env *env, 4739 struct bpf_func_state *state, 4740 int ref_obj_id) 4741 { 4742 struct bpf_reg_state *regs = state->regs, *reg; 4743 int i; 4744 4745 for (i = 0; i < MAX_BPF_REG; i++) 4746 if (regs[i].ref_obj_id == ref_obj_id) 4747 mark_reg_unknown(env, regs, i); 4748 4749 bpf_for_each_spilled_reg(i, state, reg) { 4750 if (!reg) 4751 continue; 4752 if (reg->ref_obj_id == ref_obj_id) 4753 __mark_reg_unknown(env, reg); 4754 } 4755 } 4756 4757 /* The pointer with the specified id has released its reference to kernel 4758 * resources. Identify all copies of the same pointer and clear the reference. 4759 */ 4760 static int release_reference(struct bpf_verifier_env *env, 4761 int ref_obj_id) 4762 { 4763 struct bpf_verifier_state *vstate = env->cur_state; 4764 int err; 4765 int i; 4766 4767 err = release_reference_state(cur_func(env), ref_obj_id); 4768 if (err) 4769 return err; 4770 4771 for (i = 0; i <= vstate->curframe; i++) 4772 release_reg_references(env, vstate->frame[i], ref_obj_id); 4773 4774 return 0; 4775 } 4776 4777 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 4778 struct bpf_reg_state *regs) 4779 { 4780 int i; 4781 4782 /* after the call registers r0 - r5 were scratched */ 4783 for (i = 0; i < CALLER_SAVED_REGS; i++) { 4784 mark_reg_not_init(env, regs, caller_saved[i]); 4785 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 4786 } 4787 } 4788 4789 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 4790 int *insn_idx) 4791 { 4792 struct bpf_verifier_state *state = env->cur_state; 4793 struct bpf_func_info_aux *func_info_aux; 4794 struct bpf_func_state *caller, *callee; 4795 int i, err, subprog, target_insn; 4796 bool is_global = false; 4797 4798 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 4799 verbose(env, "the call stack of %d frames is too deep\n", 4800 state->curframe + 2); 4801 return -E2BIG; 4802 } 4803 4804 target_insn = *insn_idx + insn->imm; 4805 subprog = find_subprog(env, target_insn + 1); 4806 if (subprog < 0) { 4807 verbose(env, "verifier bug. No program starts at insn %d\n", 4808 target_insn + 1); 4809 return -EFAULT; 4810 } 4811 4812 caller = state->frame[state->curframe]; 4813 if (state->frame[state->curframe + 1]) { 4814 verbose(env, "verifier bug. Frame %d already allocated\n", 4815 state->curframe + 1); 4816 return -EFAULT; 4817 } 4818 4819 func_info_aux = env->prog->aux->func_info_aux; 4820 if (func_info_aux) 4821 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 4822 err = btf_check_func_arg_match(env, subprog, caller->regs); 4823 if (err == -EFAULT) 4824 return err; 4825 if (is_global) { 4826 if (err) { 4827 verbose(env, "Caller passes invalid args into func#%d\n", 4828 subprog); 4829 return err; 4830 } else { 4831 if (env->log.level & BPF_LOG_LEVEL) 4832 verbose(env, 4833 "Func#%d is global and valid. Skipping.\n", 4834 subprog); 4835 clear_caller_saved_regs(env, caller->regs); 4836 4837 /* All global functions return SCALAR_VALUE */ 4838 mark_reg_unknown(env, caller->regs, BPF_REG_0); 4839 4840 /* continue with next insn after call */ 4841 return 0; 4842 } 4843 } 4844 4845 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 4846 if (!callee) 4847 return -ENOMEM; 4848 state->frame[state->curframe + 1] = callee; 4849 4850 /* callee cannot access r0, r6 - r9 for reading and has to write 4851 * into its own stack before reading from it. 4852 * callee can read/write into caller's stack 4853 */ 4854 init_func_state(env, callee, 4855 /* remember the callsite, it will be used by bpf_exit */ 4856 *insn_idx /* callsite */, 4857 state->curframe + 1 /* frameno within this callchain */, 4858 subprog /* subprog number within this prog */); 4859 4860 /* Transfer references to the callee */ 4861 err = transfer_reference_state(callee, caller); 4862 if (err) 4863 return err; 4864 4865 /* copy r1 - r5 args that callee can access. The copy includes parent 4866 * pointers, which connects us up to the liveness chain 4867 */ 4868 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 4869 callee->regs[i] = caller->regs[i]; 4870 4871 clear_caller_saved_regs(env, caller->regs); 4872 4873 /* only increment it after check_reg_arg() finished */ 4874 state->curframe++; 4875 4876 /* and go analyze first insn of the callee */ 4877 *insn_idx = target_insn; 4878 4879 if (env->log.level & BPF_LOG_LEVEL) { 4880 verbose(env, "caller:\n"); 4881 print_verifier_state(env, caller); 4882 verbose(env, "callee:\n"); 4883 print_verifier_state(env, callee); 4884 } 4885 return 0; 4886 } 4887 4888 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 4889 { 4890 struct bpf_verifier_state *state = env->cur_state; 4891 struct bpf_func_state *caller, *callee; 4892 struct bpf_reg_state *r0; 4893 int err; 4894 4895 callee = state->frame[state->curframe]; 4896 r0 = &callee->regs[BPF_REG_0]; 4897 if (r0->type == PTR_TO_STACK) { 4898 /* technically it's ok to return caller's stack pointer 4899 * (or caller's caller's pointer) back to the caller, 4900 * since these pointers are valid. Only current stack 4901 * pointer will be invalid as soon as function exits, 4902 * but let's be conservative 4903 */ 4904 verbose(env, "cannot return stack pointer to the caller\n"); 4905 return -EINVAL; 4906 } 4907 4908 state->curframe--; 4909 caller = state->frame[state->curframe]; 4910 /* return to the caller whatever r0 had in the callee */ 4911 caller->regs[BPF_REG_0] = *r0; 4912 4913 /* Transfer references to the caller */ 4914 err = transfer_reference_state(caller, callee); 4915 if (err) 4916 return err; 4917 4918 *insn_idx = callee->callsite + 1; 4919 if (env->log.level & BPF_LOG_LEVEL) { 4920 verbose(env, "returning from callee:\n"); 4921 print_verifier_state(env, callee); 4922 verbose(env, "to caller at %d:\n", *insn_idx); 4923 print_verifier_state(env, caller); 4924 } 4925 /* clear everything in the callee */ 4926 free_func_state(callee); 4927 state->frame[state->curframe + 1] = NULL; 4928 return 0; 4929 } 4930 4931 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 4932 int func_id, 4933 struct bpf_call_arg_meta *meta) 4934 { 4935 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 4936 4937 if (ret_type != RET_INTEGER || 4938 (func_id != BPF_FUNC_get_stack && 4939 func_id != BPF_FUNC_probe_read_str && 4940 func_id != BPF_FUNC_probe_read_kernel_str && 4941 func_id != BPF_FUNC_probe_read_user_str)) 4942 return; 4943 4944 ret_reg->smax_value = meta->msize_max_value; 4945 ret_reg->s32_max_value = meta->msize_max_value; 4946 ret_reg->smin_value = -MAX_ERRNO; 4947 ret_reg->s32_min_value = -MAX_ERRNO; 4948 __reg_deduce_bounds(ret_reg); 4949 __reg_bound_offset(ret_reg); 4950 __update_reg_bounds(ret_reg); 4951 } 4952 4953 static int 4954 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 4955 int func_id, int insn_idx) 4956 { 4957 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 4958 struct bpf_map *map = meta->map_ptr; 4959 4960 if (func_id != BPF_FUNC_tail_call && 4961 func_id != BPF_FUNC_map_lookup_elem && 4962 func_id != BPF_FUNC_map_update_elem && 4963 func_id != BPF_FUNC_map_delete_elem && 4964 func_id != BPF_FUNC_map_push_elem && 4965 func_id != BPF_FUNC_map_pop_elem && 4966 func_id != BPF_FUNC_map_peek_elem) 4967 return 0; 4968 4969 if (map == NULL) { 4970 verbose(env, "kernel subsystem misconfigured verifier\n"); 4971 return -EINVAL; 4972 } 4973 4974 /* In case of read-only, some additional restrictions 4975 * need to be applied in order to prevent altering the 4976 * state of the map from program side. 4977 */ 4978 if ((map->map_flags & BPF_F_RDONLY_PROG) && 4979 (func_id == BPF_FUNC_map_delete_elem || 4980 func_id == BPF_FUNC_map_update_elem || 4981 func_id == BPF_FUNC_map_push_elem || 4982 func_id == BPF_FUNC_map_pop_elem)) { 4983 verbose(env, "write into map forbidden\n"); 4984 return -EACCES; 4985 } 4986 4987 if (!BPF_MAP_PTR(aux->map_ptr_state)) 4988 bpf_map_ptr_store(aux, meta->map_ptr, 4989 !meta->map_ptr->bypass_spec_v1); 4990 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 4991 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 4992 !meta->map_ptr->bypass_spec_v1); 4993 return 0; 4994 } 4995 4996 static int 4997 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 4998 int func_id, int insn_idx) 4999 { 5000 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 5001 struct bpf_reg_state *regs = cur_regs(env), *reg; 5002 struct bpf_map *map = meta->map_ptr; 5003 struct tnum range; 5004 u64 val; 5005 int err; 5006 5007 if (func_id != BPF_FUNC_tail_call) 5008 return 0; 5009 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 5010 verbose(env, "kernel subsystem misconfigured verifier\n"); 5011 return -EINVAL; 5012 } 5013 5014 range = tnum_range(0, map->max_entries - 1); 5015 reg = ®s[BPF_REG_3]; 5016 5017 if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { 5018 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 5019 return 0; 5020 } 5021 5022 err = mark_chain_precision(env, BPF_REG_3); 5023 if (err) 5024 return err; 5025 5026 val = reg->var_off.value; 5027 if (bpf_map_key_unseen(aux)) 5028 bpf_map_key_store(aux, val); 5029 else if (!bpf_map_key_poisoned(aux) && 5030 bpf_map_key_immediate(aux) != val) 5031 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 5032 return 0; 5033 } 5034 5035 static int check_reference_leak(struct bpf_verifier_env *env) 5036 { 5037 struct bpf_func_state *state = cur_func(env); 5038 int i; 5039 5040 for (i = 0; i < state->acquired_refs; i++) { 5041 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 5042 state->refs[i].id, state->refs[i].insn_idx); 5043 } 5044 return state->acquired_refs ? -EINVAL : 0; 5045 } 5046 5047 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) 5048 { 5049 const struct bpf_func_proto *fn = NULL; 5050 struct bpf_reg_state *regs; 5051 struct bpf_call_arg_meta meta; 5052 bool changes_data; 5053 int i, err; 5054 5055 /* find function prototype */ 5056 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 5057 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 5058 func_id); 5059 return -EINVAL; 5060 } 5061 5062 if (env->ops->get_func_proto) 5063 fn = env->ops->get_func_proto(func_id, env->prog); 5064 if (!fn) { 5065 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 5066 func_id); 5067 return -EINVAL; 5068 } 5069 5070 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 5071 if (!env->prog->gpl_compatible && fn->gpl_only) { 5072 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 5073 return -EINVAL; 5074 } 5075 5076 if (fn->allowed && !fn->allowed(env->prog)) { 5077 verbose(env, "helper call is not allowed in probe\n"); 5078 return -EINVAL; 5079 } 5080 5081 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 5082 changes_data = bpf_helper_changes_pkt_data(fn->func); 5083 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 5084 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 5085 func_id_name(func_id), func_id); 5086 return -EINVAL; 5087 } 5088 5089 memset(&meta, 0, sizeof(meta)); 5090 meta.pkt_access = fn->pkt_access; 5091 5092 err = check_func_proto(fn, func_id); 5093 if (err) { 5094 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 5095 func_id_name(func_id), func_id); 5096 return err; 5097 } 5098 5099 meta.func_id = func_id; 5100 /* check args */ 5101 for (i = 0; i < 5; i++) { 5102 err = check_func_arg(env, i, &meta, fn); 5103 if (err) 5104 return err; 5105 } 5106 5107 err = record_func_map(env, &meta, func_id, insn_idx); 5108 if (err) 5109 return err; 5110 5111 err = record_func_key(env, &meta, func_id, insn_idx); 5112 if (err) 5113 return err; 5114 5115 /* Mark slots with STACK_MISC in case of raw mode, stack offset 5116 * is inferred from register state. 5117 */ 5118 for (i = 0; i < meta.access_size; i++) { 5119 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 5120 BPF_WRITE, -1, false); 5121 if (err) 5122 return err; 5123 } 5124 5125 if (func_id == BPF_FUNC_tail_call) { 5126 err = check_reference_leak(env); 5127 if (err) { 5128 verbose(env, "tail_call would lead to reference leak\n"); 5129 return err; 5130 } 5131 } else if (is_release_function(func_id)) { 5132 err = release_reference(env, meta.ref_obj_id); 5133 if (err) { 5134 verbose(env, "func %s#%d reference has not been acquired before\n", 5135 func_id_name(func_id), func_id); 5136 return err; 5137 } 5138 } 5139 5140 regs = cur_regs(env); 5141 5142 /* check that flags argument in get_local_storage(map, flags) is 0, 5143 * this is required because get_local_storage() can't return an error. 5144 */ 5145 if (func_id == BPF_FUNC_get_local_storage && 5146 !register_is_null(®s[BPF_REG_2])) { 5147 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 5148 return -EINVAL; 5149 } 5150 5151 /* reset caller saved regs */ 5152 for (i = 0; i < CALLER_SAVED_REGS; i++) { 5153 mark_reg_not_init(env, regs, caller_saved[i]); 5154 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 5155 } 5156 5157 /* helper call returns 64-bit value. */ 5158 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 5159 5160 /* update return register (already marked as written above) */ 5161 if (fn->ret_type == RET_INTEGER) { 5162 /* sets type to SCALAR_VALUE */ 5163 mark_reg_unknown(env, regs, BPF_REG_0); 5164 } else if (fn->ret_type == RET_VOID) { 5165 regs[BPF_REG_0].type = NOT_INIT; 5166 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || 5167 fn->ret_type == RET_PTR_TO_MAP_VALUE) { 5168 /* There is no offset yet applied, variable or fixed */ 5169 mark_reg_known_zero(env, regs, BPF_REG_0); 5170 /* remember map_ptr, so that check_map_access() 5171 * can check 'value_size' boundary of memory access 5172 * to map element returned from bpf_map_lookup_elem() 5173 */ 5174 if (meta.map_ptr == NULL) { 5175 verbose(env, 5176 "kernel subsystem misconfigured verifier\n"); 5177 return -EINVAL; 5178 } 5179 regs[BPF_REG_0].map_ptr = meta.map_ptr; 5180 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { 5181 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; 5182 if (map_value_has_spin_lock(meta.map_ptr)) 5183 regs[BPF_REG_0].id = ++env->id_gen; 5184 } else { 5185 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; 5186 } 5187 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { 5188 mark_reg_known_zero(env, regs, BPF_REG_0); 5189 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; 5190 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { 5191 mark_reg_known_zero(env, regs, BPF_REG_0); 5192 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; 5193 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { 5194 mark_reg_known_zero(env, regs, BPF_REG_0); 5195 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; 5196 } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { 5197 mark_reg_known_zero(env, regs, BPF_REG_0); 5198 regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; 5199 regs[BPF_REG_0].mem_size = meta.mem_size; 5200 } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || 5201 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { 5202 const struct btf_type *t; 5203 5204 mark_reg_known_zero(env, regs, BPF_REG_0); 5205 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 5206 if (!btf_type_is_struct(t)) { 5207 u32 tsize; 5208 const struct btf_type *ret; 5209 const char *tname; 5210 5211 /* resolve the type size of ksym. */ 5212 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 5213 if (IS_ERR(ret)) { 5214 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 5215 verbose(env, "unable to resolve the size of type '%s': %ld\n", 5216 tname, PTR_ERR(ret)); 5217 return -EINVAL; 5218 } 5219 regs[BPF_REG_0].type = 5220 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 5221 PTR_TO_MEM : PTR_TO_MEM_OR_NULL; 5222 regs[BPF_REG_0].mem_size = tsize; 5223 } else { 5224 regs[BPF_REG_0].type = 5225 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? 5226 PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; 5227 regs[BPF_REG_0].btf = meta.ret_btf; 5228 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 5229 } 5230 } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || 5231 fn->ret_type == RET_PTR_TO_BTF_ID) { 5232 int ret_btf_id; 5233 5234 mark_reg_known_zero(env, regs, BPF_REG_0); 5235 regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? 5236 PTR_TO_BTF_ID : 5237 PTR_TO_BTF_ID_OR_NULL; 5238 ret_btf_id = *fn->ret_btf_id; 5239 if (ret_btf_id == 0) { 5240 verbose(env, "invalid return type %d of func %s#%d\n", 5241 fn->ret_type, func_id_name(func_id), func_id); 5242 return -EINVAL; 5243 } 5244 /* current BPF helper definitions are only coming from 5245 * built-in code with type IDs from vmlinux BTF 5246 */ 5247 regs[BPF_REG_0].btf = btf_vmlinux; 5248 regs[BPF_REG_0].btf_id = ret_btf_id; 5249 } else { 5250 verbose(env, "unknown return type %d of func %s#%d\n", 5251 fn->ret_type, func_id_name(func_id), func_id); 5252 return -EINVAL; 5253 } 5254 5255 if (reg_type_may_be_null(regs[BPF_REG_0].type)) 5256 regs[BPF_REG_0].id = ++env->id_gen; 5257 5258 if (is_ptr_cast_function(func_id)) { 5259 /* For release_reference() */ 5260 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 5261 } else if (is_acquire_function(func_id, meta.map_ptr)) { 5262 int id = acquire_reference_state(env, insn_idx); 5263 5264 if (id < 0) 5265 return id; 5266 /* For mark_ptr_or_null_reg() */ 5267 regs[BPF_REG_0].id = id; 5268 /* For release_reference() */ 5269 regs[BPF_REG_0].ref_obj_id = id; 5270 } 5271 5272 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 5273 5274 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 5275 if (err) 5276 return err; 5277 5278 if ((func_id == BPF_FUNC_get_stack || 5279 func_id == BPF_FUNC_get_task_stack) && 5280 !env->prog->has_callchain_buf) { 5281 const char *err_str; 5282 5283 #ifdef CONFIG_PERF_EVENTS 5284 err = get_callchain_buffers(sysctl_perf_event_max_stack); 5285 err_str = "cannot get callchain buffer for func %s#%d\n"; 5286 #else 5287 err = -ENOTSUPP; 5288 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 5289 #endif 5290 if (err) { 5291 verbose(env, err_str, func_id_name(func_id), func_id); 5292 return err; 5293 } 5294 5295 env->prog->has_callchain_buf = true; 5296 } 5297 5298 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 5299 env->prog->call_get_stack = true; 5300 5301 if (changes_data) 5302 clear_all_pkt_pointers(env); 5303 return 0; 5304 } 5305 5306 static bool signed_add_overflows(s64 a, s64 b) 5307 { 5308 /* Do the add in u64, where overflow is well-defined */ 5309 s64 res = (s64)((u64)a + (u64)b); 5310 5311 if (b < 0) 5312 return res > a; 5313 return res < a; 5314 } 5315 5316 static bool signed_add32_overflows(s32 a, s32 b) 5317 { 5318 /* Do the add in u32, where overflow is well-defined */ 5319 s32 res = (s32)((u32)a + (u32)b); 5320 5321 if (b < 0) 5322 return res > a; 5323 return res < a; 5324 } 5325 5326 static bool signed_sub_overflows(s64 a, s64 b) 5327 { 5328 /* Do the sub in u64, where overflow is well-defined */ 5329 s64 res = (s64)((u64)a - (u64)b); 5330 5331 if (b < 0) 5332 return res < a; 5333 return res > a; 5334 } 5335 5336 static bool signed_sub32_overflows(s32 a, s32 b) 5337 { 5338 /* Do the sub in u32, where overflow is well-defined */ 5339 s32 res = (s32)((u32)a - (u32)b); 5340 5341 if (b < 0) 5342 return res < a; 5343 return res > a; 5344 } 5345 5346 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 5347 const struct bpf_reg_state *reg, 5348 enum bpf_reg_type type) 5349 { 5350 bool known = tnum_is_const(reg->var_off); 5351 s64 val = reg->var_off.value; 5352 s64 smin = reg->smin_value; 5353 5354 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 5355 verbose(env, "math between %s pointer and %lld is not allowed\n", 5356 reg_type_str[type], val); 5357 return false; 5358 } 5359 5360 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 5361 verbose(env, "%s pointer offset %d is not allowed\n", 5362 reg_type_str[type], reg->off); 5363 return false; 5364 } 5365 5366 if (smin == S64_MIN) { 5367 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 5368 reg_type_str[type]); 5369 return false; 5370 } 5371 5372 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 5373 verbose(env, "value %lld makes %s pointer be out of bounds\n", 5374 smin, reg_type_str[type]); 5375 return false; 5376 } 5377 5378 return true; 5379 } 5380 5381 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 5382 { 5383 return &env->insn_aux_data[env->insn_idx]; 5384 } 5385 5386 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 5387 u32 *ptr_limit, u8 opcode, bool off_is_neg) 5388 { 5389 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) || 5390 (opcode == BPF_SUB && !off_is_neg); 5391 u32 off; 5392 5393 switch (ptr_reg->type) { 5394 case PTR_TO_STACK: 5395 /* Indirect variable offset stack access is prohibited in 5396 * unprivileged mode so it's not handled here. 5397 */ 5398 off = ptr_reg->off + ptr_reg->var_off.value; 5399 if (mask_to_left) 5400 *ptr_limit = MAX_BPF_STACK + off; 5401 else 5402 *ptr_limit = -off; 5403 return 0; 5404 case PTR_TO_MAP_VALUE: 5405 if (mask_to_left) { 5406 *ptr_limit = ptr_reg->umax_value + ptr_reg->off; 5407 } else { 5408 off = ptr_reg->smin_value + ptr_reg->off; 5409 *ptr_limit = ptr_reg->map_ptr->value_size - off; 5410 } 5411 return 0; 5412 default: 5413 return -EINVAL; 5414 } 5415 } 5416 5417 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 5418 const struct bpf_insn *insn) 5419 { 5420 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 5421 } 5422 5423 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 5424 u32 alu_state, u32 alu_limit) 5425 { 5426 /* If we arrived here from different branches with different 5427 * state or limits to sanitize, then this won't work. 5428 */ 5429 if (aux->alu_state && 5430 (aux->alu_state != alu_state || 5431 aux->alu_limit != alu_limit)) 5432 return -EACCES; 5433 5434 /* Corresponding fixup done in fixup_bpf_calls(). */ 5435 aux->alu_state = alu_state; 5436 aux->alu_limit = alu_limit; 5437 return 0; 5438 } 5439 5440 static int sanitize_val_alu(struct bpf_verifier_env *env, 5441 struct bpf_insn *insn) 5442 { 5443 struct bpf_insn_aux_data *aux = cur_aux(env); 5444 5445 if (can_skip_alu_sanitation(env, insn)) 5446 return 0; 5447 5448 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 5449 } 5450 5451 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 5452 struct bpf_insn *insn, 5453 const struct bpf_reg_state *ptr_reg, 5454 struct bpf_reg_state *dst_reg, 5455 bool off_is_neg) 5456 { 5457 struct bpf_verifier_state *vstate = env->cur_state; 5458 struct bpf_insn_aux_data *aux = cur_aux(env); 5459 bool ptr_is_dst_reg = ptr_reg == dst_reg; 5460 u8 opcode = BPF_OP(insn->code); 5461 u32 alu_state, alu_limit; 5462 struct bpf_reg_state tmp; 5463 bool ret; 5464 5465 if (can_skip_alu_sanitation(env, insn)) 5466 return 0; 5467 5468 /* We already marked aux for masking from non-speculative 5469 * paths, thus we got here in the first place. We only care 5470 * to explore bad access from here. 5471 */ 5472 if (vstate->speculative) 5473 goto do_sim; 5474 5475 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 5476 alu_state |= ptr_is_dst_reg ? 5477 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 5478 5479 if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg)) 5480 return 0; 5481 if (update_alu_sanitation_state(aux, alu_state, alu_limit)) 5482 return -EACCES; 5483 do_sim: 5484 /* Simulate and find potential out-of-bounds access under 5485 * speculative execution from truncation as a result of 5486 * masking when off was not within expected range. If off 5487 * sits in dst, then we temporarily need to move ptr there 5488 * to simulate dst (== 0) +/-= ptr. Needed, for example, 5489 * for cases where we use K-based arithmetic in one direction 5490 * and truncated reg-based in the other in order to explore 5491 * bad access. 5492 */ 5493 if (!ptr_is_dst_reg) { 5494 tmp = *dst_reg; 5495 *dst_reg = *ptr_reg; 5496 } 5497 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true); 5498 if (!ptr_is_dst_reg && ret) 5499 *dst_reg = tmp; 5500 return !ret ? -EFAULT : 0; 5501 } 5502 5503 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 5504 * Caller should also handle BPF_MOV case separately. 5505 * If we return -EACCES, caller may want to try again treating pointer as a 5506 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 5507 */ 5508 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 5509 struct bpf_insn *insn, 5510 const struct bpf_reg_state *ptr_reg, 5511 const struct bpf_reg_state *off_reg) 5512 { 5513 struct bpf_verifier_state *vstate = env->cur_state; 5514 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 5515 struct bpf_reg_state *regs = state->regs, *dst_reg; 5516 bool known = tnum_is_const(off_reg->var_off); 5517 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 5518 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 5519 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 5520 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 5521 u32 dst = insn->dst_reg, src = insn->src_reg; 5522 u8 opcode = BPF_OP(insn->code); 5523 int ret; 5524 5525 dst_reg = ®s[dst]; 5526 5527 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 5528 smin_val > smax_val || umin_val > umax_val) { 5529 /* Taint dst register if offset had invalid bounds derived from 5530 * e.g. dead branches. 5531 */ 5532 __mark_reg_unknown(env, dst_reg); 5533 return 0; 5534 } 5535 5536 if (BPF_CLASS(insn->code) != BPF_ALU64) { 5537 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 5538 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 5539 __mark_reg_unknown(env, dst_reg); 5540 return 0; 5541 } 5542 5543 verbose(env, 5544 "R%d 32-bit pointer arithmetic prohibited\n", 5545 dst); 5546 return -EACCES; 5547 } 5548 5549 switch (ptr_reg->type) { 5550 case PTR_TO_MAP_VALUE_OR_NULL: 5551 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 5552 dst, reg_type_str[ptr_reg->type]); 5553 return -EACCES; 5554 case CONST_PTR_TO_MAP: 5555 /* smin_val represents the known value */ 5556 if (known && smin_val == 0 && opcode == BPF_ADD) 5557 break; 5558 fallthrough; 5559 case PTR_TO_PACKET_END: 5560 case PTR_TO_SOCKET: 5561 case PTR_TO_SOCKET_OR_NULL: 5562 case PTR_TO_SOCK_COMMON: 5563 case PTR_TO_SOCK_COMMON_OR_NULL: 5564 case PTR_TO_TCP_SOCK: 5565 case PTR_TO_TCP_SOCK_OR_NULL: 5566 case PTR_TO_XDP_SOCK: 5567 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 5568 dst, reg_type_str[ptr_reg->type]); 5569 return -EACCES; 5570 case PTR_TO_MAP_VALUE: 5571 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) { 5572 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n", 5573 off_reg == dst_reg ? dst : src); 5574 return -EACCES; 5575 } 5576 fallthrough; 5577 default: 5578 break; 5579 } 5580 5581 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 5582 * The id may be overwritten later if we create a new variable offset. 5583 */ 5584 dst_reg->type = ptr_reg->type; 5585 dst_reg->id = ptr_reg->id; 5586 5587 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 5588 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 5589 return -EINVAL; 5590 5591 /* pointer types do not carry 32-bit bounds at the moment. */ 5592 __mark_reg32_unbounded(dst_reg); 5593 5594 switch (opcode) { 5595 case BPF_ADD: 5596 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); 5597 if (ret < 0) { 5598 verbose(env, "R%d tried to add from different maps or paths\n", dst); 5599 return ret; 5600 } 5601 /* We can take a fixed offset as long as it doesn't overflow 5602 * the s32 'off' field 5603 */ 5604 if (known && (ptr_reg->off + smin_val == 5605 (s64)(s32)(ptr_reg->off + smin_val))) { 5606 /* pointer += K. Accumulate it into fixed offset */ 5607 dst_reg->smin_value = smin_ptr; 5608 dst_reg->smax_value = smax_ptr; 5609 dst_reg->umin_value = umin_ptr; 5610 dst_reg->umax_value = umax_ptr; 5611 dst_reg->var_off = ptr_reg->var_off; 5612 dst_reg->off = ptr_reg->off + smin_val; 5613 dst_reg->raw = ptr_reg->raw; 5614 break; 5615 } 5616 /* A new variable offset is created. Note that off_reg->off 5617 * == 0, since it's a scalar. 5618 * dst_reg gets the pointer type and since some positive 5619 * integer value was added to the pointer, give it a new 'id' 5620 * if it's a PTR_TO_PACKET. 5621 * this creates a new 'base' pointer, off_reg (variable) gets 5622 * added into the variable offset, and we copy the fixed offset 5623 * from ptr_reg. 5624 */ 5625 if (signed_add_overflows(smin_ptr, smin_val) || 5626 signed_add_overflows(smax_ptr, smax_val)) { 5627 dst_reg->smin_value = S64_MIN; 5628 dst_reg->smax_value = S64_MAX; 5629 } else { 5630 dst_reg->smin_value = smin_ptr + smin_val; 5631 dst_reg->smax_value = smax_ptr + smax_val; 5632 } 5633 if (umin_ptr + umin_val < umin_ptr || 5634 umax_ptr + umax_val < umax_ptr) { 5635 dst_reg->umin_value = 0; 5636 dst_reg->umax_value = U64_MAX; 5637 } else { 5638 dst_reg->umin_value = umin_ptr + umin_val; 5639 dst_reg->umax_value = umax_ptr + umax_val; 5640 } 5641 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 5642 dst_reg->off = ptr_reg->off; 5643 dst_reg->raw = ptr_reg->raw; 5644 if (reg_is_pkt_pointer(ptr_reg)) { 5645 dst_reg->id = ++env->id_gen; 5646 /* something was added to pkt_ptr, set range to zero */ 5647 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 5648 } 5649 break; 5650 case BPF_SUB: 5651 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); 5652 if (ret < 0) { 5653 verbose(env, "R%d tried to sub from different maps or paths\n", dst); 5654 return ret; 5655 } 5656 if (dst_reg == off_reg) { 5657 /* scalar -= pointer. Creates an unknown scalar */ 5658 verbose(env, "R%d tried to subtract pointer from scalar\n", 5659 dst); 5660 return -EACCES; 5661 } 5662 /* We don't allow subtraction from FP, because (according to 5663 * test_verifier.c test "invalid fp arithmetic", JITs might not 5664 * be able to deal with it. 5665 */ 5666 if (ptr_reg->type == PTR_TO_STACK) { 5667 verbose(env, "R%d subtraction from stack pointer prohibited\n", 5668 dst); 5669 return -EACCES; 5670 } 5671 if (known && (ptr_reg->off - smin_val == 5672 (s64)(s32)(ptr_reg->off - smin_val))) { 5673 /* pointer -= K. Subtract it from fixed offset */ 5674 dst_reg->smin_value = smin_ptr; 5675 dst_reg->smax_value = smax_ptr; 5676 dst_reg->umin_value = umin_ptr; 5677 dst_reg->umax_value = umax_ptr; 5678 dst_reg->var_off = ptr_reg->var_off; 5679 dst_reg->id = ptr_reg->id; 5680 dst_reg->off = ptr_reg->off - smin_val; 5681 dst_reg->raw = ptr_reg->raw; 5682 break; 5683 } 5684 /* A new variable offset is created. If the subtrahend is known 5685 * nonnegative, then any reg->range we had before is still good. 5686 */ 5687 if (signed_sub_overflows(smin_ptr, smax_val) || 5688 signed_sub_overflows(smax_ptr, smin_val)) { 5689 /* Overflow possible, we know nothing */ 5690 dst_reg->smin_value = S64_MIN; 5691 dst_reg->smax_value = S64_MAX; 5692 } else { 5693 dst_reg->smin_value = smin_ptr - smax_val; 5694 dst_reg->smax_value = smax_ptr - smin_val; 5695 } 5696 if (umin_ptr < umax_val) { 5697 /* Overflow possible, we know nothing */ 5698 dst_reg->umin_value = 0; 5699 dst_reg->umax_value = U64_MAX; 5700 } else { 5701 /* Cannot overflow (as long as bounds are consistent) */ 5702 dst_reg->umin_value = umin_ptr - umax_val; 5703 dst_reg->umax_value = umax_ptr - umin_val; 5704 } 5705 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 5706 dst_reg->off = ptr_reg->off; 5707 dst_reg->raw = ptr_reg->raw; 5708 if (reg_is_pkt_pointer(ptr_reg)) { 5709 dst_reg->id = ++env->id_gen; 5710 /* something was added to pkt_ptr, set range to zero */ 5711 if (smin_val < 0) 5712 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 5713 } 5714 break; 5715 case BPF_AND: 5716 case BPF_OR: 5717 case BPF_XOR: 5718 /* bitwise ops on pointers are troublesome, prohibit. */ 5719 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 5720 dst, bpf_alu_string[opcode >> 4]); 5721 return -EACCES; 5722 default: 5723 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 5724 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 5725 dst, bpf_alu_string[opcode >> 4]); 5726 return -EACCES; 5727 } 5728 5729 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 5730 return -EINVAL; 5731 5732 __update_reg_bounds(dst_reg); 5733 __reg_deduce_bounds(dst_reg); 5734 __reg_bound_offset(dst_reg); 5735 5736 /* For unprivileged we require that resulting offset must be in bounds 5737 * in order to be able to sanitize access later on. 5738 */ 5739 if (!env->bypass_spec_v1) { 5740 if (dst_reg->type == PTR_TO_MAP_VALUE && 5741 check_map_access(env, dst, dst_reg->off, 1, false)) { 5742 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 5743 "prohibited for !root\n", dst); 5744 return -EACCES; 5745 } else if (dst_reg->type == PTR_TO_STACK && 5746 check_stack_access(env, dst_reg, dst_reg->off + 5747 dst_reg->var_off.value, 1)) { 5748 verbose(env, "R%d stack pointer arithmetic goes out of range, " 5749 "prohibited for !root\n", dst); 5750 return -EACCES; 5751 } 5752 } 5753 5754 return 0; 5755 } 5756 5757 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 5758 struct bpf_reg_state *src_reg) 5759 { 5760 s32 smin_val = src_reg->s32_min_value; 5761 s32 smax_val = src_reg->s32_max_value; 5762 u32 umin_val = src_reg->u32_min_value; 5763 u32 umax_val = src_reg->u32_max_value; 5764 5765 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 5766 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 5767 dst_reg->s32_min_value = S32_MIN; 5768 dst_reg->s32_max_value = S32_MAX; 5769 } else { 5770 dst_reg->s32_min_value += smin_val; 5771 dst_reg->s32_max_value += smax_val; 5772 } 5773 if (dst_reg->u32_min_value + umin_val < umin_val || 5774 dst_reg->u32_max_value + umax_val < umax_val) { 5775 dst_reg->u32_min_value = 0; 5776 dst_reg->u32_max_value = U32_MAX; 5777 } else { 5778 dst_reg->u32_min_value += umin_val; 5779 dst_reg->u32_max_value += umax_val; 5780 } 5781 } 5782 5783 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 5784 struct bpf_reg_state *src_reg) 5785 { 5786 s64 smin_val = src_reg->smin_value; 5787 s64 smax_val = src_reg->smax_value; 5788 u64 umin_val = src_reg->umin_value; 5789 u64 umax_val = src_reg->umax_value; 5790 5791 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 5792 signed_add_overflows(dst_reg->smax_value, smax_val)) { 5793 dst_reg->smin_value = S64_MIN; 5794 dst_reg->smax_value = S64_MAX; 5795 } else { 5796 dst_reg->smin_value += smin_val; 5797 dst_reg->smax_value += smax_val; 5798 } 5799 if (dst_reg->umin_value + umin_val < umin_val || 5800 dst_reg->umax_value + umax_val < umax_val) { 5801 dst_reg->umin_value = 0; 5802 dst_reg->umax_value = U64_MAX; 5803 } else { 5804 dst_reg->umin_value += umin_val; 5805 dst_reg->umax_value += umax_val; 5806 } 5807 } 5808 5809 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 5810 struct bpf_reg_state *src_reg) 5811 { 5812 s32 smin_val = src_reg->s32_min_value; 5813 s32 smax_val = src_reg->s32_max_value; 5814 u32 umin_val = src_reg->u32_min_value; 5815 u32 umax_val = src_reg->u32_max_value; 5816 5817 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 5818 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 5819 /* Overflow possible, we know nothing */ 5820 dst_reg->s32_min_value = S32_MIN; 5821 dst_reg->s32_max_value = S32_MAX; 5822 } else { 5823 dst_reg->s32_min_value -= smax_val; 5824 dst_reg->s32_max_value -= smin_val; 5825 } 5826 if (dst_reg->u32_min_value < umax_val) { 5827 /* Overflow possible, we know nothing */ 5828 dst_reg->u32_min_value = 0; 5829 dst_reg->u32_max_value = U32_MAX; 5830 } else { 5831 /* Cannot overflow (as long as bounds are consistent) */ 5832 dst_reg->u32_min_value -= umax_val; 5833 dst_reg->u32_max_value -= umin_val; 5834 } 5835 } 5836 5837 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 5838 struct bpf_reg_state *src_reg) 5839 { 5840 s64 smin_val = src_reg->smin_value; 5841 s64 smax_val = src_reg->smax_value; 5842 u64 umin_val = src_reg->umin_value; 5843 u64 umax_val = src_reg->umax_value; 5844 5845 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 5846 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 5847 /* Overflow possible, we know nothing */ 5848 dst_reg->smin_value = S64_MIN; 5849 dst_reg->smax_value = S64_MAX; 5850 } else { 5851 dst_reg->smin_value -= smax_val; 5852 dst_reg->smax_value -= smin_val; 5853 } 5854 if (dst_reg->umin_value < umax_val) { 5855 /* Overflow possible, we know nothing */ 5856 dst_reg->umin_value = 0; 5857 dst_reg->umax_value = U64_MAX; 5858 } else { 5859 /* Cannot overflow (as long as bounds are consistent) */ 5860 dst_reg->umin_value -= umax_val; 5861 dst_reg->umax_value -= umin_val; 5862 } 5863 } 5864 5865 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 5866 struct bpf_reg_state *src_reg) 5867 { 5868 s32 smin_val = src_reg->s32_min_value; 5869 u32 umin_val = src_reg->u32_min_value; 5870 u32 umax_val = src_reg->u32_max_value; 5871 5872 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 5873 /* Ain't nobody got time to multiply that sign */ 5874 __mark_reg32_unbounded(dst_reg); 5875 return; 5876 } 5877 /* Both values are positive, so we can work with unsigned and 5878 * copy the result to signed (unless it exceeds S32_MAX). 5879 */ 5880 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 5881 /* Potential overflow, we know nothing */ 5882 __mark_reg32_unbounded(dst_reg); 5883 return; 5884 } 5885 dst_reg->u32_min_value *= umin_val; 5886 dst_reg->u32_max_value *= umax_val; 5887 if (dst_reg->u32_max_value > S32_MAX) { 5888 /* Overflow possible, we know nothing */ 5889 dst_reg->s32_min_value = S32_MIN; 5890 dst_reg->s32_max_value = S32_MAX; 5891 } else { 5892 dst_reg->s32_min_value = dst_reg->u32_min_value; 5893 dst_reg->s32_max_value = dst_reg->u32_max_value; 5894 } 5895 } 5896 5897 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 5898 struct bpf_reg_state *src_reg) 5899 { 5900 s64 smin_val = src_reg->smin_value; 5901 u64 umin_val = src_reg->umin_value; 5902 u64 umax_val = src_reg->umax_value; 5903 5904 if (smin_val < 0 || dst_reg->smin_value < 0) { 5905 /* Ain't nobody got time to multiply that sign */ 5906 __mark_reg64_unbounded(dst_reg); 5907 return; 5908 } 5909 /* Both values are positive, so we can work with unsigned and 5910 * copy the result to signed (unless it exceeds S64_MAX). 5911 */ 5912 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 5913 /* Potential overflow, we know nothing */ 5914 __mark_reg64_unbounded(dst_reg); 5915 return; 5916 } 5917 dst_reg->umin_value *= umin_val; 5918 dst_reg->umax_value *= umax_val; 5919 if (dst_reg->umax_value > S64_MAX) { 5920 /* Overflow possible, we know nothing */ 5921 dst_reg->smin_value = S64_MIN; 5922 dst_reg->smax_value = S64_MAX; 5923 } else { 5924 dst_reg->smin_value = dst_reg->umin_value; 5925 dst_reg->smax_value = dst_reg->umax_value; 5926 } 5927 } 5928 5929 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 5930 struct bpf_reg_state *src_reg) 5931 { 5932 bool src_known = tnum_subreg_is_const(src_reg->var_off); 5933 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 5934 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 5935 s32 smin_val = src_reg->s32_min_value; 5936 u32 umax_val = src_reg->u32_max_value; 5937 5938 /* Assuming scalar64_min_max_and will be called so its safe 5939 * to skip updating register for known 32-bit case. 5940 */ 5941 if (src_known && dst_known) 5942 return; 5943 5944 /* We get our minimum from the var_off, since that's inherently 5945 * bitwise. Our maximum is the minimum of the operands' maxima. 5946 */ 5947 dst_reg->u32_min_value = var32_off.value; 5948 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 5949 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 5950 /* Lose signed bounds when ANDing negative numbers, 5951 * ain't nobody got time for that. 5952 */ 5953 dst_reg->s32_min_value = S32_MIN; 5954 dst_reg->s32_max_value = S32_MAX; 5955 } else { 5956 /* ANDing two positives gives a positive, so safe to 5957 * cast result into s64. 5958 */ 5959 dst_reg->s32_min_value = dst_reg->u32_min_value; 5960 dst_reg->s32_max_value = dst_reg->u32_max_value; 5961 } 5962 5963 } 5964 5965 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 5966 struct bpf_reg_state *src_reg) 5967 { 5968 bool src_known = tnum_is_const(src_reg->var_off); 5969 bool dst_known = tnum_is_const(dst_reg->var_off); 5970 s64 smin_val = src_reg->smin_value; 5971 u64 umax_val = src_reg->umax_value; 5972 5973 if (src_known && dst_known) { 5974 __mark_reg_known(dst_reg, dst_reg->var_off.value); 5975 return; 5976 } 5977 5978 /* We get our minimum from the var_off, since that's inherently 5979 * bitwise. Our maximum is the minimum of the operands' maxima. 5980 */ 5981 dst_reg->umin_value = dst_reg->var_off.value; 5982 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 5983 if (dst_reg->smin_value < 0 || smin_val < 0) { 5984 /* Lose signed bounds when ANDing negative numbers, 5985 * ain't nobody got time for that. 5986 */ 5987 dst_reg->smin_value = S64_MIN; 5988 dst_reg->smax_value = S64_MAX; 5989 } else { 5990 /* ANDing two positives gives a positive, so safe to 5991 * cast result into s64. 5992 */ 5993 dst_reg->smin_value = dst_reg->umin_value; 5994 dst_reg->smax_value = dst_reg->umax_value; 5995 } 5996 /* We may learn something more from the var_off */ 5997 __update_reg_bounds(dst_reg); 5998 } 5999 6000 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 6001 struct bpf_reg_state *src_reg) 6002 { 6003 bool src_known = tnum_subreg_is_const(src_reg->var_off); 6004 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 6005 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 6006 s32 smin_val = src_reg->s32_min_value; 6007 u32 umin_val = src_reg->u32_min_value; 6008 6009 /* Assuming scalar64_min_max_or will be called so it is safe 6010 * to skip updating register for known case. 6011 */ 6012 if (src_known && dst_known) 6013 return; 6014 6015 /* We get our maximum from the var_off, and our minimum is the 6016 * maximum of the operands' minima 6017 */ 6018 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 6019 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 6020 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 6021 /* Lose signed bounds when ORing negative numbers, 6022 * ain't nobody got time for that. 6023 */ 6024 dst_reg->s32_min_value = S32_MIN; 6025 dst_reg->s32_max_value = S32_MAX; 6026 } else { 6027 /* ORing two positives gives a positive, so safe to 6028 * cast result into s64. 6029 */ 6030 dst_reg->s32_min_value = dst_reg->u32_min_value; 6031 dst_reg->s32_max_value = dst_reg->u32_max_value; 6032 } 6033 } 6034 6035 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 6036 struct bpf_reg_state *src_reg) 6037 { 6038 bool src_known = tnum_is_const(src_reg->var_off); 6039 bool dst_known = tnum_is_const(dst_reg->var_off); 6040 s64 smin_val = src_reg->smin_value; 6041 u64 umin_val = src_reg->umin_value; 6042 6043 if (src_known && dst_known) { 6044 __mark_reg_known(dst_reg, dst_reg->var_off.value); 6045 return; 6046 } 6047 6048 /* We get our maximum from the var_off, and our minimum is the 6049 * maximum of the operands' minima 6050 */ 6051 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 6052 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 6053 if (dst_reg->smin_value < 0 || smin_val < 0) { 6054 /* Lose signed bounds when ORing negative numbers, 6055 * ain't nobody got time for that. 6056 */ 6057 dst_reg->smin_value = S64_MIN; 6058 dst_reg->smax_value = S64_MAX; 6059 } else { 6060 /* ORing two positives gives a positive, so safe to 6061 * cast result into s64. 6062 */ 6063 dst_reg->smin_value = dst_reg->umin_value; 6064 dst_reg->smax_value = dst_reg->umax_value; 6065 } 6066 /* We may learn something more from the var_off */ 6067 __update_reg_bounds(dst_reg); 6068 } 6069 6070 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 6071 struct bpf_reg_state *src_reg) 6072 { 6073 bool src_known = tnum_subreg_is_const(src_reg->var_off); 6074 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 6075 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 6076 s32 smin_val = src_reg->s32_min_value; 6077 6078 /* Assuming scalar64_min_max_xor will be called so it is safe 6079 * to skip updating register for known case. 6080 */ 6081 if (src_known && dst_known) 6082 return; 6083 6084 /* We get both minimum and maximum from the var32_off. */ 6085 dst_reg->u32_min_value = var32_off.value; 6086 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 6087 6088 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 6089 /* XORing two positive sign numbers gives a positive, 6090 * so safe to cast u32 result into s32. 6091 */ 6092 dst_reg->s32_min_value = dst_reg->u32_min_value; 6093 dst_reg->s32_max_value = dst_reg->u32_max_value; 6094 } else { 6095 dst_reg->s32_min_value = S32_MIN; 6096 dst_reg->s32_max_value = S32_MAX; 6097 } 6098 } 6099 6100 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 6101 struct bpf_reg_state *src_reg) 6102 { 6103 bool src_known = tnum_is_const(src_reg->var_off); 6104 bool dst_known = tnum_is_const(dst_reg->var_off); 6105 s64 smin_val = src_reg->smin_value; 6106 6107 if (src_known && dst_known) { 6108 /* dst_reg->var_off.value has been updated earlier */ 6109 __mark_reg_known(dst_reg, dst_reg->var_off.value); 6110 return; 6111 } 6112 6113 /* We get both minimum and maximum from the var_off. */ 6114 dst_reg->umin_value = dst_reg->var_off.value; 6115 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 6116 6117 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 6118 /* XORing two positive sign numbers gives a positive, 6119 * so safe to cast u64 result into s64. 6120 */ 6121 dst_reg->smin_value = dst_reg->umin_value; 6122 dst_reg->smax_value = dst_reg->umax_value; 6123 } else { 6124 dst_reg->smin_value = S64_MIN; 6125 dst_reg->smax_value = S64_MAX; 6126 } 6127 6128 __update_reg_bounds(dst_reg); 6129 } 6130 6131 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 6132 u64 umin_val, u64 umax_val) 6133 { 6134 /* We lose all sign bit information (except what we can pick 6135 * up from var_off) 6136 */ 6137 dst_reg->s32_min_value = S32_MIN; 6138 dst_reg->s32_max_value = S32_MAX; 6139 /* If we might shift our top bit out, then we know nothing */ 6140 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 6141 dst_reg->u32_min_value = 0; 6142 dst_reg->u32_max_value = U32_MAX; 6143 } else { 6144 dst_reg->u32_min_value <<= umin_val; 6145 dst_reg->u32_max_value <<= umax_val; 6146 } 6147 } 6148 6149 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 6150 struct bpf_reg_state *src_reg) 6151 { 6152 u32 umax_val = src_reg->u32_max_value; 6153 u32 umin_val = src_reg->u32_min_value; 6154 /* u32 alu operation will zext upper bits */ 6155 struct tnum subreg = tnum_subreg(dst_reg->var_off); 6156 6157 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 6158 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 6159 /* Not required but being careful mark reg64 bounds as unknown so 6160 * that we are forced to pick them up from tnum and zext later and 6161 * if some path skips this step we are still safe. 6162 */ 6163 __mark_reg64_unbounded(dst_reg); 6164 __update_reg32_bounds(dst_reg); 6165 } 6166 6167 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 6168 u64 umin_val, u64 umax_val) 6169 { 6170 /* Special case <<32 because it is a common compiler pattern to sign 6171 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 6172 * positive we know this shift will also be positive so we can track 6173 * bounds correctly. Otherwise we lose all sign bit information except 6174 * what we can pick up from var_off. Perhaps we can generalize this 6175 * later to shifts of any length. 6176 */ 6177 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 6178 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 6179 else 6180 dst_reg->smax_value = S64_MAX; 6181 6182 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 6183 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 6184 else 6185 dst_reg->smin_value = S64_MIN; 6186 6187 /* If we might shift our top bit out, then we know nothing */ 6188 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 6189 dst_reg->umin_value = 0; 6190 dst_reg->umax_value = U64_MAX; 6191 } else { 6192 dst_reg->umin_value <<= umin_val; 6193 dst_reg->umax_value <<= umax_val; 6194 } 6195 } 6196 6197 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 6198 struct bpf_reg_state *src_reg) 6199 { 6200 u64 umax_val = src_reg->umax_value; 6201 u64 umin_val = src_reg->umin_value; 6202 6203 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 6204 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 6205 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 6206 6207 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 6208 /* We may learn something more from the var_off */ 6209 __update_reg_bounds(dst_reg); 6210 } 6211 6212 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 6213 struct bpf_reg_state *src_reg) 6214 { 6215 struct tnum subreg = tnum_subreg(dst_reg->var_off); 6216 u32 umax_val = src_reg->u32_max_value; 6217 u32 umin_val = src_reg->u32_min_value; 6218 6219 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 6220 * be negative, then either: 6221 * 1) src_reg might be zero, so the sign bit of the result is 6222 * unknown, so we lose our signed bounds 6223 * 2) it's known negative, thus the unsigned bounds capture the 6224 * signed bounds 6225 * 3) the signed bounds cross zero, so they tell us nothing 6226 * about the result 6227 * If the value in dst_reg is known nonnegative, then again the 6228 * unsigned bounts capture the signed bounds. 6229 * Thus, in all cases it suffices to blow away our signed bounds 6230 * and rely on inferring new ones from the unsigned bounds and 6231 * var_off of the result. 6232 */ 6233 dst_reg->s32_min_value = S32_MIN; 6234 dst_reg->s32_max_value = S32_MAX; 6235 6236 dst_reg->var_off = tnum_rshift(subreg, umin_val); 6237 dst_reg->u32_min_value >>= umax_val; 6238 dst_reg->u32_max_value >>= umin_val; 6239 6240 __mark_reg64_unbounded(dst_reg); 6241 __update_reg32_bounds(dst_reg); 6242 } 6243 6244 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 6245 struct bpf_reg_state *src_reg) 6246 { 6247 u64 umax_val = src_reg->umax_value; 6248 u64 umin_val = src_reg->umin_value; 6249 6250 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 6251 * be negative, then either: 6252 * 1) src_reg might be zero, so the sign bit of the result is 6253 * unknown, so we lose our signed bounds 6254 * 2) it's known negative, thus the unsigned bounds capture the 6255 * signed bounds 6256 * 3) the signed bounds cross zero, so they tell us nothing 6257 * about the result 6258 * If the value in dst_reg is known nonnegative, then again the 6259 * unsigned bounts capture the signed bounds. 6260 * Thus, in all cases it suffices to blow away our signed bounds 6261 * and rely on inferring new ones from the unsigned bounds and 6262 * var_off of the result. 6263 */ 6264 dst_reg->smin_value = S64_MIN; 6265 dst_reg->smax_value = S64_MAX; 6266 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 6267 dst_reg->umin_value >>= umax_val; 6268 dst_reg->umax_value >>= umin_val; 6269 6270 /* Its not easy to operate on alu32 bounds here because it depends 6271 * on bits being shifted in. Take easy way out and mark unbounded 6272 * so we can recalculate later from tnum. 6273 */ 6274 __mark_reg32_unbounded(dst_reg); 6275 __update_reg_bounds(dst_reg); 6276 } 6277 6278 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 6279 struct bpf_reg_state *src_reg) 6280 { 6281 u64 umin_val = src_reg->u32_min_value; 6282 6283 /* Upon reaching here, src_known is true and 6284 * umax_val is equal to umin_val. 6285 */ 6286 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 6287 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 6288 6289 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 6290 6291 /* blow away the dst_reg umin_value/umax_value and rely on 6292 * dst_reg var_off to refine the result. 6293 */ 6294 dst_reg->u32_min_value = 0; 6295 dst_reg->u32_max_value = U32_MAX; 6296 6297 __mark_reg64_unbounded(dst_reg); 6298 __update_reg32_bounds(dst_reg); 6299 } 6300 6301 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 6302 struct bpf_reg_state *src_reg) 6303 { 6304 u64 umin_val = src_reg->umin_value; 6305 6306 /* Upon reaching here, src_known is true and umax_val is equal 6307 * to umin_val. 6308 */ 6309 dst_reg->smin_value >>= umin_val; 6310 dst_reg->smax_value >>= umin_val; 6311 6312 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 6313 6314 /* blow away the dst_reg umin_value/umax_value and rely on 6315 * dst_reg var_off to refine the result. 6316 */ 6317 dst_reg->umin_value = 0; 6318 dst_reg->umax_value = U64_MAX; 6319 6320 /* Its not easy to operate on alu32 bounds here because it depends 6321 * on bits being shifted in from upper 32-bits. Take easy way out 6322 * and mark unbounded so we can recalculate later from tnum. 6323 */ 6324 __mark_reg32_unbounded(dst_reg); 6325 __update_reg_bounds(dst_reg); 6326 } 6327 6328 /* WARNING: This function does calculations on 64-bit values, but the actual 6329 * execution may occur on 32-bit values. Therefore, things like bitshifts 6330 * need extra checks in the 32-bit case. 6331 */ 6332 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 6333 struct bpf_insn *insn, 6334 struct bpf_reg_state *dst_reg, 6335 struct bpf_reg_state src_reg) 6336 { 6337 struct bpf_reg_state *regs = cur_regs(env); 6338 u8 opcode = BPF_OP(insn->code); 6339 bool src_known; 6340 s64 smin_val, smax_val; 6341 u64 umin_val, umax_val; 6342 s32 s32_min_val, s32_max_val; 6343 u32 u32_min_val, u32_max_val; 6344 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 6345 u32 dst = insn->dst_reg; 6346 int ret; 6347 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 6348 6349 smin_val = src_reg.smin_value; 6350 smax_val = src_reg.smax_value; 6351 umin_val = src_reg.umin_value; 6352 umax_val = src_reg.umax_value; 6353 6354 s32_min_val = src_reg.s32_min_value; 6355 s32_max_val = src_reg.s32_max_value; 6356 u32_min_val = src_reg.u32_min_value; 6357 u32_max_val = src_reg.u32_max_value; 6358 6359 if (alu32) { 6360 src_known = tnum_subreg_is_const(src_reg.var_off); 6361 if ((src_known && 6362 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 6363 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 6364 /* Taint dst register if offset had invalid bounds 6365 * derived from e.g. dead branches. 6366 */ 6367 __mark_reg_unknown(env, dst_reg); 6368 return 0; 6369 } 6370 } else { 6371 src_known = tnum_is_const(src_reg.var_off); 6372 if ((src_known && 6373 (smin_val != smax_val || umin_val != umax_val)) || 6374 smin_val > smax_val || umin_val > umax_val) { 6375 /* Taint dst register if offset had invalid bounds 6376 * derived from e.g. dead branches. 6377 */ 6378 __mark_reg_unknown(env, dst_reg); 6379 return 0; 6380 } 6381 } 6382 6383 if (!src_known && 6384 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 6385 __mark_reg_unknown(env, dst_reg); 6386 return 0; 6387 } 6388 6389 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 6390 * There are two classes of instructions: The first class we track both 6391 * alu32 and alu64 sign/unsigned bounds independently this provides the 6392 * greatest amount of precision when alu operations are mixed with jmp32 6393 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 6394 * and BPF_OR. This is possible because these ops have fairly easy to 6395 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 6396 * See alu32 verifier tests for examples. The second class of 6397 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 6398 * with regards to tracking sign/unsigned bounds because the bits may 6399 * cross subreg boundaries in the alu64 case. When this happens we mark 6400 * the reg unbounded in the subreg bound space and use the resulting 6401 * tnum to calculate an approximation of the sign/unsigned bounds. 6402 */ 6403 switch (opcode) { 6404 case BPF_ADD: 6405 ret = sanitize_val_alu(env, insn); 6406 if (ret < 0) { 6407 verbose(env, "R%d tried to add from different pointers or scalars\n", dst); 6408 return ret; 6409 } 6410 scalar32_min_max_add(dst_reg, &src_reg); 6411 scalar_min_max_add(dst_reg, &src_reg); 6412 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 6413 break; 6414 case BPF_SUB: 6415 ret = sanitize_val_alu(env, insn); 6416 if (ret < 0) { 6417 verbose(env, "R%d tried to sub from different pointers or scalars\n", dst); 6418 return ret; 6419 } 6420 scalar32_min_max_sub(dst_reg, &src_reg); 6421 scalar_min_max_sub(dst_reg, &src_reg); 6422 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 6423 break; 6424 case BPF_MUL: 6425 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 6426 scalar32_min_max_mul(dst_reg, &src_reg); 6427 scalar_min_max_mul(dst_reg, &src_reg); 6428 break; 6429 case BPF_AND: 6430 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 6431 scalar32_min_max_and(dst_reg, &src_reg); 6432 scalar_min_max_and(dst_reg, &src_reg); 6433 break; 6434 case BPF_OR: 6435 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 6436 scalar32_min_max_or(dst_reg, &src_reg); 6437 scalar_min_max_or(dst_reg, &src_reg); 6438 break; 6439 case BPF_XOR: 6440 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 6441 scalar32_min_max_xor(dst_reg, &src_reg); 6442 scalar_min_max_xor(dst_reg, &src_reg); 6443 break; 6444 case BPF_LSH: 6445 if (umax_val >= insn_bitness) { 6446 /* Shifts greater than 31 or 63 are undefined. 6447 * This includes shifts by a negative number. 6448 */ 6449 mark_reg_unknown(env, regs, insn->dst_reg); 6450 break; 6451 } 6452 if (alu32) 6453 scalar32_min_max_lsh(dst_reg, &src_reg); 6454 else 6455 scalar_min_max_lsh(dst_reg, &src_reg); 6456 break; 6457 case BPF_RSH: 6458 if (umax_val >= insn_bitness) { 6459 /* Shifts greater than 31 or 63 are undefined. 6460 * This includes shifts by a negative number. 6461 */ 6462 mark_reg_unknown(env, regs, insn->dst_reg); 6463 break; 6464 } 6465 if (alu32) 6466 scalar32_min_max_rsh(dst_reg, &src_reg); 6467 else 6468 scalar_min_max_rsh(dst_reg, &src_reg); 6469 break; 6470 case BPF_ARSH: 6471 if (umax_val >= insn_bitness) { 6472 /* Shifts greater than 31 or 63 are undefined. 6473 * This includes shifts by a negative number. 6474 */ 6475 mark_reg_unknown(env, regs, insn->dst_reg); 6476 break; 6477 } 6478 if (alu32) 6479 scalar32_min_max_arsh(dst_reg, &src_reg); 6480 else 6481 scalar_min_max_arsh(dst_reg, &src_reg); 6482 break; 6483 default: 6484 mark_reg_unknown(env, regs, insn->dst_reg); 6485 break; 6486 } 6487 6488 /* ALU32 ops are zero extended into 64bit register */ 6489 if (alu32) 6490 zext_32_to_64(dst_reg); 6491 6492 __update_reg_bounds(dst_reg); 6493 __reg_deduce_bounds(dst_reg); 6494 __reg_bound_offset(dst_reg); 6495 return 0; 6496 } 6497 6498 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 6499 * and var_off. 6500 */ 6501 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 6502 struct bpf_insn *insn) 6503 { 6504 struct bpf_verifier_state *vstate = env->cur_state; 6505 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6506 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 6507 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 6508 u8 opcode = BPF_OP(insn->code); 6509 int err; 6510 6511 dst_reg = ®s[insn->dst_reg]; 6512 src_reg = NULL; 6513 if (dst_reg->type != SCALAR_VALUE) 6514 ptr_reg = dst_reg; 6515 else 6516 /* Make sure ID is cleared otherwise dst_reg min/max could be 6517 * incorrectly propagated into other registers by find_equal_scalars() 6518 */ 6519 dst_reg->id = 0; 6520 if (BPF_SRC(insn->code) == BPF_X) { 6521 src_reg = ®s[insn->src_reg]; 6522 if (src_reg->type != SCALAR_VALUE) { 6523 if (dst_reg->type != SCALAR_VALUE) { 6524 /* Combining two pointers by any ALU op yields 6525 * an arbitrary scalar. Disallow all math except 6526 * pointer subtraction 6527 */ 6528 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 6529 mark_reg_unknown(env, regs, insn->dst_reg); 6530 return 0; 6531 } 6532 verbose(env, "R%d pointer %s pointer prohibited\n", 6533 insn->dst_reg, 6534 bpf_alu_string[opcode >> 4]); 6535 return -EACCES; 6536 } else { 6537 /* scalar += pointer 6538 * This is legal, but we have to reverse our 6539 * src/dest handling in computing the range 6540 */ 6541 err = mark_chain_precision(env, insn->dst_reg); 6542 if (err) 6543 return err; 6544 return adjust_ptr_min_max_vals(env, insn, 6545 src_reg, dst_reg); 6546 } 6547 } else if (ptr_reg) { 6548 /* pointer += scalar */ 6549 err = mark_chain_precision(env, insn->src_reg); 6550 if (err) 6551 return err; 6552 return adjust_ptr_min_max_vals(env, insn, 6553 dst_reg, src_reg); 6554 } 6555 } else { 6556 /* Pretend the src is a reg with a known value, since we only 6557 * need to be able to read from this state. 6558 */ 6559 off_reg.type = SCALAR_VALUE; 6560 __mark_reg_known(&off_reg, insn->imm); 6561 src_reg = &off_reg; 6562 if (ptr_reg) /* pointer += K */ 6563 return adjust_ptr_min_max_vals(env, insn, 6564 ptr_reg, src_reg); 6565 } 6566 6567 /* Got here implies adding two SCALAR_VALUEs */ 6568 if (WARN_ON_ONCE(ptr_reg)) { 6569 print_verifier_state(env, state); 6570 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 6571 return -EINVAL; 6572 } 6573 if (WARN_ON(!src_reg)) { 6574 print_verifier_state(env, state); 6575 verbose(env, "verifier internal error: no src_reg\n"); 6576 return -EINVAL; 6577 } 6578 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 6579 } 6580 6581 /* check validity of 32-bit and 64-bit arithmetic operations */ 6582 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 6583 { 6584 struct bpf_reg_state *regs = cur_regs(env); 6585 u8 opcode = BPF_OP(insn->code); 6586 int err; 6587 6588 if (opcode == BPF_END || opcode == BPF_NEG) { 6589 if (opcode == BPF_NEG) { 6590 if (BPF_SRC(insn->code) != 0 || 6591 insn->src_reg != BPF_REG_0 || 6592 insn->off != 0 || insn->imm != 0) { 6593 verbose(env, "BPF_NEG uses reserved fields\n"); 6594 return -EINVAL; 6595 } 6596 } else { 6597 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 6598 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 6599 BPF_CLASS(insn->code) == BPF_ALU64) { 6600 verbose(env, "BPF_END uses reserved fields\n"); 6601 return -EINVAL; 6602 } 6603 } 6604 6605 /* check src operand */ 6606 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 6607 if (err) 6608 return err; 6609 6610 if (is_pointer_value(env, insn->dst_reg)) { 6611 verbose(env, "R%d pointer arithmetic prohibited\n", 6612 insn->dst_reg); 6613 return -EACCES; 6614 } 6615 6616 /* check dest operand */ 6617 err = check_reg_arg(env, insn->dst_reg, DST_OP); 6618 if (err) 6619 return err; 6620 6621 } else if (opcode == BPF_MOV) { 6622 6623 if (BPF_SRC(insn->code) == BPF_X) { 6624 if (insn->imm != 0 || insn->off != 0) { 6625 verbose(env, "BPF_MOV uses reserved fields\n"); 6626 return -EINVAL; 6627 } 6628 6629 /* check src operand */ 6630 err = check_reg_arg(env, insn->src_reg, SRC_OP); 6631 if (err) 6632 return err; 6633 } else { 6634 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 6635 verbose(env, "BPF_MOV uses reserved fields\n"); 6636 return -EINVAL; 6637 } 6638 } 6639 6640 /* check dest operand, mark as required later */ 6641 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 6642 if (err) 6643 return err; 6644 6645 if (BPF_SRC(insn->code) == BPF_X) { 6646 struct bpf_reg_state *src_reg = regs + insn->src_reg; 6647 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 6648 6649 if (BPF_CLASS(insn->code) == BPF_ALU64) { 6650 /* case: R1 = R2 6651 * copy register state to dest reg 6652 */ 6653 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 6654 /* Assign src and dst registers the same ID 6655 * that will be used by find_equal_scalars() 6656 * to propagate min/max range. 6657 */ 6658 src_reg->id = ++env->id_gen; 6659 *dst_reg = *src_reg; 6660 dst_reg->live |= REG_LIVE_WRITTEN; 6661 dst_reg->subreg_def = DEF_NOT_SUBREG; 6662 } else { 6663 /* R1 = (u32) R2 */ 6664 if (is_pointer_value(env, insn->src_reg)) { 6665 verbose(env, 6666 "R%d partial copy of pointer\n", 6667 insn->src_reg); 6668 return -EACCES; 6669 } else if (src_reg->type == SCALAR_VALUE) { 6670 *dst_reg = *src_reg; 6671 /* Make sure ID is cleared otherwise 6672 * dst_reg min/max could be incorrectly 6673 * propagated into src_reg by find_equal_scalars() 6674 */ 6675 dst_reg->id = 0; 6676 dst_reg->live |= REG_LIVE_WRITTEN; 6677 dst_reg->subreg_def = env->insn_idx + 1; 6678 } else { 6679 mark_reg_unknown(env, regs, 6680 insn->dst_reg); 6681 } 6682 zext_32_to_64(dst_reg); 6683 } 6684 } else { 6685 /* case: R = imm 6686 * remember the value we stored into this reg 6687 */ 6688 /* clear any state __mark_reg_known doesn't set */ 6689 mark_reg_unknown(env, regs, insn->dst_reg); 6690 regs[insn->dst_reg].type = SCALAR_VALUE; 6691 if (BPF_CLASS(insn->code) == BPF_ALU64) { 6692 __mark_reg_known(regs + insn->dst_reg, 6693 insn->imm); 6694 } else { 6695 __mark_reg_known(regs + insn->dst_reg, 6696 (u32)insn->imm); 6697 } 6698 } 6699 6700 } else if (opcode > BPF_END) { 6701 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 6702 return -EINVAL; 6703 6704 } else { /* all other ALU ops: and, sub, xor, add, ... */ 6705 6706 if (BPF_SRC(insn->code) == BPF_X) { 6707 if (insn->imm != 0 || insn->off != 0) { 6708 verbose(env, "BPF_ALU uses reserved fields\n"); 6709 return -EINVAL; 6710 } 6711 /* check src1 operand */ 6712 err = check_reg_arg(env, insn->src_reg, SRC_OP); 6713 if (err) 6714 return err; 6715 } else { 6716 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 6717 verbose(env, "BPF_ALU uses reserved fields\n"); 6718 return -EINVAL; 6719 } 6720 } 6721 6722 /* check src2 operand */ 6723 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 6724 if (err) 6725 return err; 6726 6727 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 6728 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 6729 verbose(env, "div by zero\n"); 6730 return -EINVAL; 6731 } 6732 6733 if ((opcode == BPF_LSH || opcode == BPF_RSH || 6734 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 6735 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 6736 6737 if (insn->imm < 0 || insn->imm >= size) { 6738 verbose(env, "invalid shift %d\n", insn->imm); 6739 return -EINVAL; 6740 } 6741 } 6742 6743 /* check dest operand */ 6744 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 6745 if (err) 6746 return err; 6747 6748 return adjust_reg_min_max_vals(env, insn); 6749 } 6750 6751 return 0; 6752 } 6753 6754 static void __find_good_pkt_pointers(struct bpf_func_state *state, 6755 struct bpf_reg_state *dst_reg, 6756 enum bpf_reg_type type, int new_range) 6757 { 6758 struct bpf_reg_state *reg; 6759 int i; 6760 6761 for (i = 0; i < MAX_BPF_REG; i++) { 6762 reg = &state->regs[i]; 6763 if (reg->type == type && reg->id == dst_reg->id) 6764 /* keep the maximum range already checked */ 6765 reg->range = max(reg->range, new_range); 6766 } 6767 6768 bpf_for_each_spilled_reg(i, state, reg) { 6769 if (!reg) 6770 continue; 6771 if (reg->type == type && reg->id == dst_reg->id) 6772 reg->range = max(reg->range, new_range); 6773 } 6774 } 6775 6776 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 6777 struct bpf_reg_state *dst_reg, 6778 enum bpf_reg_type type, 6779 bool range_right_open) 6780 { 6781 int new_range, i; 6782 6783 if (dst_reg->off < 0 || 6784 (dst_reg->off == 0 && range_right_open)) 6785 /* This doesn't give us any range */ 6786 return; 6787 6788 if (dst_reg->umax_value > MAX_PACKET_OFF || 6789 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 6790 /* Risk of overflow. For instance, ptr + (1<<63) may be less 6791 * than pkt_end, but that's because it's also less than pkt. 6792 */ 6793 return; 6794 6795 new_range = dst_reg->off; 6796 if (range_right_open) 6797 new_range--; 6798 6799 /* Examples for register markings: 6800 * 6801 * pkt_data in dst register: 6802 * 6803 * r2 = r3; 6804 * r2 += 8; 6805 * if (r2 > pkt_end) goto <handle exception> 6806 * <access okay> 6807 * 6808 * r2 = r3; 6809 * r2 += 8; 6810 * if (r2 < pkt_end) goto <access okay> 6811 * <handle exception> 6812 * 6813 * Where: 6814 * r2 == dst_reg, pkt_end == src_reg 6815 * r2=pkt(id=n,off=8,r=0) 6816 * r3=pkt(id=n,off=0,r=0) 6817 * 6818 * pkt_data in src register: 6819 * 6820 * r2 = r3; 6821 * r2 += 8; 6822 * if (pkt_end >= r2) goto <access okay> 6823 * <handle exception> 6824 * 6825 * r2 = r3; 6826 * r2 += 8; 6827 * if (pkt_end <= r2) goto <handle exception> 6828 * <access okay> 6829 * 6830 * Where: 6831 * pkt_end == dst_reg, r2 == src_reg 6832 * r2=pkt(id=n,off=8,r=0) 6833 * r3=pkt(id=n,off=0,r=0) 6834 * 6835 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 6836 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 6837 * and [r3, r3 + 8-1) respectively is safe to access depending on 6838 * the check. 6839 */ 6840 6841 /* If our ids match, then we must have the same max_value. And we 6842 * don't care about the other reg's fixed offset, since if it's too big 6843 * the range won't allow anything. 6844 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 6845 */ 6846 for (i = 0; i <= vstate->curframe; i++) 6847 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, 6848 new_range); 6849 } 6850 6851 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 6852 { 6853 struct tnum subreg = tnum_subreg(reg->var_off); 6854 s32 sval = (s32)val; 6855 6856 switch (opcode) { 6857 case BPF_JEQ: 6858 if (tnum_is_const(subreg)) 6859 return !!tnum_equals_const(subreg, val); 6860 break; 6861 case BPF_JNE: 6862 if (tnum_is_const(subreg)) 6863 return !tnum_equals_const(subreg, val); 6864 break; 6865 case BPF_JSET: 6866 if ((~subreg.mask & subreg.value) & val) 6867 return 1; 6868 if (!((subreg.mask | subreg.value) & val)) 6869 return 0; 6870 break; 6871 case BPF_JGT: 6872 if (reg->u32_min_value > val) 6873 return 1; 6874 else if (reg->u32_max_value <= val) 6875 return 0; 6876 break; 6877 case BPF_JSGT: 6878 if (reg->s32_min_value > sval) 6879 return 1; 6880 else if (reg->s32_max_value < sval) 6881 return 0; 6882 break; 6883 case BPF_JLT: 6884 if (reg->u32_max_value < val) 6885 return 1; 6886 else if (reg->u32_min_value >= val) 6887 return 0; 6888 break; 6889 case BPF_JSLT: 6890 if (reg->s32_max_value < sval) 6891 return 1; 6892 else if (reg->s32_min_value >= sval) 6893 return 0; 6894 break; 6895 case BPF_JGE: 6896 if (reg->u32_min_value >= val) 6897 return 1; 6898 else if (reg->u32_max_value < val) 6899 return 0; 6900 break; 6901 case BPF_JSGE: 6902 if (reg->s32_min_value >= sval) 6903 return 1; 6904 else if (reg->s32_max_value < sval) 6905 return 0; 6906 break; 6907 case BPF_JLE: 6908 if (reg->u32_max_value <= val) 6909 return 1; 6910 else if (reg->u32_min_value > val) 6911 return 0; 6912 break; 6913 case BPF_JSLE: 6914 if (reg->s32_max_value <= sval) 6915 return 1; 6916 else if (reg->s32_min_value > sval) 6917 return 0; 6918 break; 6919 } 6920 6921 return -1; 6922 } 6923 6924 6925 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 6926 { 6927 s64 sval = (s64)val; 6928 6929 switch (opcode) { 6930 case BPF_JEQ: 6931 if (tnum_is_const(reg->var_off)) 6932 return !!tnum_equals_const(reg->var_off, val); 6933 break; 6934 case BPF_JNE: 6935 if (tnum_is_const(reg->var_off)) 6936 return !tnum_equals_const(reg->var_off, val); 6937 break; 6938 case BPF_JSET: 6939 if ((~reg->var_off.mask & reg->var_off.value) & val) 6940 return 1; 6941 if (!((reg->var_off.mask | reg->var_off.value) & val)) 6942 return 0; 6943 break; 6944 case BPF_JGT: 6945 if (reg->umin_value > val) 6946 return 1; 6947 else if (reg->umax_value <= val) 6948 return 0; 6949 break; 6950 case BPF_JSGT: 6951 if (reg->smin_value > sval) 6952 return 1; 6953 else if (reg->smax_value < sval) 6954 return 0; 6955 break; 6956 case BPF_JLT: 6957 if (reg->umax_value < val) 6958 return 1; 6959 else if (reg->umin_value >= val) 6960 return 0; 6961 break; 6962 case BPF_JSLT: 6963 if (reg->smax_value < sval) 6964 return 1; 6965 else if (reg->smin_value >= sval) 6966 return 0; 6967 break; 6968 case BPF_JGE: 6969 if (reg->umin_value >= val) 6970 return 1; 6971 else if (reg->umax_value < val) 6972 return 0; 6973 break; 6974 case BPF_JSGE: 6975 if (reg->smin_value >= sval) 6976 return 1; 6977 else if (reg->smax_value < sval) 6978 return 0; 6979 break; 6980 case BPF_JLE: 6981 if (reg->umax_value <= val) 6982 return 1; 6983 else if (reg->umin_value > val) 6984 return 0; 6985 break; 6986 case BPF_JSLE: 6987 if (reg->smax_value <= sval) 6988 return 1; 6989 else if (reg->smin_value > sval) 6990 return 0; 6991 break; 6992 } 6993 6994 return -1; 6995 } 6996 6997 /* compute branch direction of the expression "if (reg opcode val) goto target;" 6998 * and return: 6999 * 1 - branch will be taken and "goto target" will be executed 7000 * 0 - branch will not be taken and fall-through to next insn 7001 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 7002 * range [0,10] 7003 */ 7004 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 7005 bool is_jmp32) 7006 { 7007 if (__is_pointer_value(false, reg)) { 7008 if (!reg_type_not_null(reg->type)) 7009 return -1; 7010 7011 /* If pointer is valid tests against zero will fail so we can 7012 * use this to direct branch taken. 7013 */ 7014 if (val != 0) 7015 return -1; 7016 7017 switch (opcode) { 7018 case BPF_JEQ: 7019 return 0; 7020 case BPF_JNE: 7021 return 1; 7022 default: 7023 return -1; 7024 } 7025 } 7026 7027 if (is_jmp32) 7028 return is_branch32_taken(reg, val, opcode); 7029 return is_branch64_taken(reg, val, opcode); 7030 } 7031 7032 static int flip_opcode(u32 opcode) 7033 { 7034 /* How can we transform "a <op> b" into "b <op> a"? */ 7035 static const u8 opcode_flip[16] = { 7036 /* these stay the same */ 7037 [BPF_JEQ >> 4] = BPF_JEQ, 7038 [BPF_JNE >> 4] = BPF_JNE, 7039 [BPF_JSET >> 4] = BPF_JSET, 7040 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 7041 [BPF_JGE >> 4] = BPF_JLE, 7042 [BPF_JGT >> 4] = BPF_JLT, 7043 [BPF_JLE >> 4] = BPF_JGE, 7044 [BPF_JLT >> 4] = BPF_JGT, 7045 [BPF_JSGE >> 4] = BPF_JSLE, 7046 [BPF_JSGT >> 4] = BPF_JSLT, 7047 [BPF_JSLE >> 4] = BPF_JSGE, 7048 [BPF_JSLT >> 4] = BPF_JSGT 7049 }; 7050 return opcode_flip[opcode >> 4]; 7051 } 7052 7053 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 7054 struct bpf_reg_state *src_reg, 7055 u8 opcode) 7056 { 7057 struct bpf_reg_state *pkt; 7058 7059 if (src_reg->type == PTR_TO_PACKET_END) { 7060 pkt = dst_reg; 7061 } else if (dst_reg->type == PTR_TO_PACKET_END) { 7062 pkt = src_reg; 7063 opcode = flip_opcode(opcode); 7064 } else { 7065 return -1; 7066 } 7067 7068 if (pkt->range >= 0) 7069 return -1; 7070 7071 switch (opcode) { 7072 case BPF_JLE: 7073 /* pkt <= pkt_end */ 7074 fallthrough; 7075 case BPF_JGT: 7076 /* pkt > pkt_end */ 7077 if (pkt->range == BEYOND_PKT_END) 7078 /* pkt has at last one extra byte beyond pkt_end */ 7079 return opcode == BPF_JGT; 7080 break; 7081 case BPF_JLT: 7082 /* pkt < pkt_end */ 7083 fallthrough; 7084 case BPF_JGE: 7085 /* pkt >= pkt_end */ 7086 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 7087 return opcode == BPF_JGE; 7088 break; 7089 } 7090 return -1; 7091 } 7092 7093 /* Adjusts the register min/max values in the case that the dst_reg is the 7094 * variable register that we are working on, and src_reg is a constant or we're 7095 * simply doing a BPF_K check. 7096 * In JEQ/JNE cases we also adjust the var_off values. 7097 */ 7098 static void reg_set_min_max(struct bpf_reg_state *true_reg, 7099 struct bpf_reg_state *false_reg, 7100 u64 val, u32 val32, 7101 u8 opcode, bool is_jmp32) 7102 { 7103 struct tnum false_32off = tnum_subreg(false_reg->var_off); 7104 struct tnum false_64off = false_reg->var_off; 7105 struct tnum true_32off = tnum_subreg(true_reg->var_off); 7106 struct tnum true_64off = true_reg->var_off; 7107 s64 sval = (s64)val; 7108 s32 sval32 = (s32)val32; 7109 7110 /* If the dst_reg is a pointer, we can't learn anything about its 7111 * variable offset from the compare (unless src_reg were a pointer into 7112 * the same object, but we don't bother with that. 7113 * Since false_reg and true_reg have the same type by construction, we 7114 * only need to check one of them for pointerness. 7115 */ 7116 if (__is_pointer_value(false, false_reg)) 7117 return; 7118 7119 switch (opcode) { 7120 case BPF_JEQ: 7121 case BPF_JNE: 7122 { 7123 struct bpf_reg_state *reg = 7124 opcode == BPF_JEQ ? true_reg : false_reg; 7125 7126 /* JEQ/JNE comparison doesn't change the register equivalence. 7127 * r1 = r2; 7128 * if (r1 == 42) goto label; 7129 * ... 7130 * label: // here both r1 and r2 are known to be 42. 7131 * 7132 * Hence when marking register as known preserve it's ID. 7133 */ 7134 if (is_jmp32) 7135 __mark_reg32_known(reg, val32); 7136 else 7137 ___mark_reg_known(reg, val); 7138 break; 7139 } 7140 case BPF_JSET: 7141 if (is_jmp32) { 7142 false_32off = tnum_and(false_32off, tnum_const(~val32)); 7143 if (is_power_of_2(val32)) 7144 true_32off = tnum_or(true_32off, 7145 tnum_const(val32)); 7146 } else { 7147 false_64off = tnum_and(false_64off, tnum_const(~val)); 7148 if (is_power_of_2(val)) 7149 true_64off = tnum_or(true_64off, 7150 tnum_const(val)); 7151 } 7152 break; 7153 case BPF_JGE: 7154 case BPF_JGT: 7155 { 7156 if (is_jmp32) { 7157 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 7158 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 7159 7160 false_reg->u32_max_value = min(false_reg->u32_max_value, 7161 false_umax); 7162 true_reg->u32_min_value = max(true_reg->u32_min_value, 7163 true_umin); 7164 } else { 7165 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 7166 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 7167 7168 false_reg->umax_value = min(false_reg->umax_value, false_umax); 7169 true_reg->umin_value = max(true_reg->umin_value, true_umin); 7170 } 7171 break; 7172 } 7173 case BPF_JSGE: 7174 case BPF_JSGT: 7175 { 7176 if (is_jmp32) { 7177 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 7178 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 7179 7180 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 7181 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 7182 } else { 7183 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 7184 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 7185 7186 false_reg->smax_value = min(false_reg->smax_value, false_smax); 7187 true_reg->smin_value = max(true_reg->smin_value, true_smin); 7188 } 7189 break; 7190 } 7191 case BPF_JLE: 7192 case BPF_JLT: 7193 { 7194 if (is_jmp32) { 7195 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 7196 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 7197 7198 false_reg->u32_min_value = max(false_reg->u32_min_value, 7199 false_umin); 7200 true_reg->u32_max_value = min(true_reg->u32_max_value, 7201 true_umax); 7202 } else { 7203 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 7204 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 7205 7206 false_reg->umin_value = max(false_reg->umin_value, false_umin); 7207 true_reg->umax_value = min(true_reg->umax_value, true_umax); 7208 } 7209 break; 7210 } 7211 case BPF_JSLE: 7212 case BPF_JSLT: 7213 { 7214 if (is_jmp32) { 7215 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 7216 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 7217 7218 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 7219 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 7220 } else { 7221 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 7222 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 7223 7224 false_reg->smin_value = max(false_reg->smin_value, false_smin); 7225 true_reg->smax_value = min(true_reg->smax_value, true_smax); 7226 } 7227 break; 7228 } 7229 default: 7230 return; 7231 } 7232 7233 if (is_jmp32) { 7234 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 7235 tnum_subreg(false_32off)); 7236 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 7237 tnum_subreg(true_32off)); 7238 __reg_combine_32_into_64(false_reg); 7239 __reg_combine_32_into_64(true_reg); 7240 } else { 7241 false_reg->var_off = false_64off; 7242 true_reg->var_off = true_64off; 7243 __reg_combine_64_into_32(false_reg); 7244 __reg_combine_64_into_32(true_reg); 7245 } 7246 } 7247 7248 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 7249 * the variable reg. 7250 */ 7251 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 7252 struct bpf_reg_state *false_reg, 7253 u64 val, u32 val32, 7254 u8 opcode, bool is_jmp32) 7255 { 7256 opcode = flip_opcode(opcode); 7257 /* This uses zero as "not present in table"; luckily the zero opcode, 7258 * BPF_JA, can't get here. 7259 */ 7260 if (opcode) 7261 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 7262 } 7263 7264 /* Regs are known to be equal, so intersect their min/max/var_off */ 7265 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 7266 struct bpf_reg_state *dst_reg) 7267 { 7268 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 7269 dst_reg->umin_value); 7270 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 7271 dst_reg->umax_value); 7272 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 7273 dst_reg->smin_value); 7274 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 7275 dst_reg->smax_value); 7276 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 7277 dst_reg->var_off); 7278 /* We might have learned new bounds from the var_off. */ 7279 __update_reg_bounds(src_reg); 7280 __update_reg_bounds(dst_reg); 7281 /* We might have learned something about the sign bit. */ 7282 __reg_deduce_bounds(src_reg); 7283 __reg_deduce_bounds(dst_reg); 7284 /* We might have learned some bits from the bounds. */ 7285 __reg_bound_offset(src_reg); 7286 __reg_bound_offset(dst_reg); 7287 /* Intersecting with the old var_off might have improved our bounds 7288 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 7289 * then new var_off is (0; 0x7f...fc) which improves our umax. 7290 */ 7291 __update_reg_bounds(src_reg); 7292 __update_reg_bounds(dst_reg); 7293 } 7294 7295 static void reg_combine_min_max(struct bpf_reg_state *true_src, 7296 struct bpf_reg_state *true_dst, 7297 struct bpf_reg_state *false_src, 7298 struct bpf_reg_state *false_dst, 7299 u8 opcode) 7300 { 7301 switch (opcode) { 7302 case BPF_JEQ: 7303 __reg_combine_min_max(true_src, true_dst); 7304 break; 7305 case BPF_JNE: 7306 __reg_combine_min_max(false_src, false_dst); 7307 break; 7308 } 7309 } 7310 7311 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 7312 struct bpf_reg_state *reg, u32 id, 7313 bool is_null) 7314 { 7315 if (reg_type_may_be_null(reg->type) && reg->id == id && 7316 !WARN_ON_ONCE(!reg->id)) { 7317 /* Old offset (both fixed and variable parts) should 7318 * have been known-zero, because we don't allow pointer 7319 * arithmetic on pointers that might be NULL. 7320 */ 7321 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || 7322 !tnum_equals_const(reg->var_off, 0) || 7323 reg->off)) { 7324 __mark_reg_known_zero(reg); 7325 reg->off = 0; 7326 } 7327 if (is_null) { 7328 reg->type = SCALAR_VALUE; 7329 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) { 7330 const struct bpf_map *map = reg->map_ptr; 7331 7332 if (map->inner_map_meta) { 7333 reg->type = CONST_PTR_TO_MAP; 7334 reg->map_ptr = map->inner_map_meta; 7335 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 7336 reg->type = PTR_TO_XDP_SOCK; 7337 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 7338 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 7339 reg->type = PTR_TO_SOCKET; 7340 } else { 7341 reg->type = PTR_TO_MAP_VALUE; 7342 } 7343 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) { 7344 reg->type = PTR_TO_SOCKET; 7345 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) { 7346 reg->type = PTR_TO_SOCK_COMMON; 7347 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { 7348 reg->type = PTR_TO_TCP_SOCK; 7349 } else if (reg->type == PTR_TO_BTF_ID_OR_NULL) { 7350 reg->type = PTR_TO_BTF_ID; 7351 } else if (reg->type == PTR_TO_MEM_OR_NULL) { 7352 reg->type = PTR_TO_MEM; 7353 } else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) { 7354 reg->type = PTR_TO_RDONLY_BUF; 7355 } else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) { 7356 reg->type = PTR_TO_RDWR_BUF; 7357 } 7358 if (is_null) { 7359 /* We don't need id and ref_obj_id from this point 7360 * onwards anymore, thus we should better reset it, 7361 * so that state pruning has chances to take effect. 7362 */ 7363 reg->id = 0; 7364 reg->ref_obj_id = 0; 7365 } else if (!reg_may_point_to_spin_lock(reg)) { 7366 /* For not-NULL ptr, reg->ref_obj_id will be reset 7367 * in release_reg_references(). 7368 * 7369 * reg->id is still used by spin_lock ptr. Other 7370 * than spin_lock ptr type, reg->id can be reset. 7371 */ 7372 reg->id = 0; 7373 } 7374 } 7375 } 7376 7377 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, 7378 bool is_null) 7379 { 7380 struct bpf_reg_state *reg; 7381 int i; 7382 7383 for (i = 0; i < MAX_BPF_REG; i++) 7384 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); 7385 7386 bpf_for_each_spilled_reg(i, state, reg) { 7387 if (!reg) 7388 continue; 7389 mark_ptr_or_null_reg(state, reg, id, is_null); 7390 } 7391 } 7392 7393 /* The logic is similar to find_good_pkt_pointers(), both could eventually 7394 * be folded together at some point. 7395 */ 7396 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 7397 bool is_null) 7398 { 7399 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7400 struct bpf_reg_state *regs = state->regs; 7401 u32 ref_obj_id = regs[regno].ref_obj_id; 7402 u32 id = regs[regno].id; 7403 int i; 7404 7405 if (ref_obj_id && ref_obj_id == id && is_null) 7406 /* regs[regno] is in the " == NULL" branch. 7407 * No one could have freed the reference state before 7408 * doing the NULL check. 7409 */ 7410 WARN_ON_ONCE(release_reference_state(state, id)); 7411 7412 for (i = 0; i <= vstate->curframe; i++) 7413 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); 7414 } 7415 7416 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 7417 struct bpf_reg_state *dst_reg, 7418 struct bpf_reg_state *src_reg, 7419 struct bpf_verifier_state *this_branch, 7420 struct bpf_verifier_state *other_branch) 7421 { 7422 if (BPF_SRC(insn->code) != BPF_X) 7423 return false; 7424 7425 /* Pointers are always 64-bit. */ 7426 if (BPF_CLASS(insn->code) == BPF_JMP32) 7427 return false; 7428 7429 switch (BPF_OP(insn->code)) { 7430 case BPF_JGT: 7431 if ((dst_reg->type == PTR_TO_PACKET && 7432 src_reg->type == PTR_TO_PACKET_END) || 7433 (dst_reg->type == PTR_TO_PACKET_META && 7434 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 7435 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 7436 find_good_pkt_pointers(this_branch, dst_reg, 7437 dst_reg->type, false); 7438 mark_pkt_end(other_branch, insn->dst_reg, true); 7439 } else if ((dst_reg->type == PTR_TO_PACKET_END && 7440 src_reg->type == PTR_TO_PACKET) || 7441 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 7442 src_reg->type == PTR_TO_PACKET_META)) { 7443 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 7444 find_good_pkt_pointers(other_branch, src_reg, 7445 src_reg->type, true); 7446 mark_pkt_end(this_branch, insn->src_reg, false); 7447 } else { 7448 return false; 7449 } 7450 break; 7451 case BPF_JLT: 7452 if ((dst_reg->type == PTR_TO_PACKET && 7453 src_reg->type == PTR_TO_PACKET_END) || 7454 (dst_reg->type == PTR_TO_PACKET_META && 7455 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 7456 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 7457 find_good_pkt_pointers(other_branch, dst_reg, 7458 dst_reg->type, true); 7459 mark_pkt_end(this_branch, insn->dst_reg, false); 7460 } else if ((dst_reg->type == PTR_TO_PACKET_END && 7461 src_reg->type == PTR_TO_PACKET) || 7462 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 7463 src_reg->type == PTR_TO_PACKET_META)) { 7464 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 7465 find_good_pkt_pointers(this_branch, src_reg, 7466 src_reg->type, false); 7467 mark_pkt_end(other_branch, insn->src_reg, true); 7468 } else { 7469 return false; 7470 } 7471 break; 7472 case BPF_JGE: 7473 if ((dst_reg->type == PTR_TO_PACKET && 7474 src_reg->type == PTR_TO_PACKET_END) || 7475 (dst_reg->type == PTR_TO_PACKET_META && 7476 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 7477 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 7478 find_good_pkt_pointers(this_branch, dst_reg, 7479 dst_reg->type, true); 7480 mark_pkt_end(other_branch, insn->dst_reg, false); 7481 } else if ((dst_reg->type == PTR_TO_PACKET_END && 7482 src_reg->type == PTR_TO_PACKET) || 7483 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 7484 src_reg->type == PTR_TO_PACKET_META)) { 7485 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 7486 find_good_pkt_pointers(other_branch, src_reg, 7487 src_reg->type, false); 7488 mark_pkt_end(this_branch, insn->src_reg, true); 7489 } else { 7490 return false; 7491 } 7492 break; 7493 case BPF_JLE: 7494 if ((dst_reg->type == PTR_TO_PACKET && 7495 src_reg->type == PTR_TO_PACKET_END) || 7496 (dst_reg->type == PTR_TO_PACKET_META && 7497 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 7498 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 7499 find_good_pkt_pointers(other_branch, dst_reg, 7500 dst_reg->type, false); 7501 mark_pkt_end(this_branch, insn->dst_reg, true); 7502 } else if ((dst_reg->type == PTR_TO_PACKET_END && 7503 src_reg->type == PTR_TO_PACKET) || 7504 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 7505 src_reg->type == PTR_TO_PACKET_META)) { 7506 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 7507 find_good_pkt_pointers(this_branch, src_reg, 7508 src_reg->type, true); 7509 mark_pkt_end(other_branch, insn->src_reg, false); 7510 } else { 7511 return false; 7512 } 7513 break; 7514 default: 7515 return false; 7516 } 7517 7518 return true; 7519 } 7520 7521 static void find_equal_scalars(struct bpf_verifier_state *vstate, 7522 struct bpf_reg_state *known_reg) 7523 { 7524 struct bpf_func_state *state; 7525 struct bpf_reg_state *reg; 7526 int i, j; 7527 7528 for (i = 0; i <= vstate->curframe; i++) { 7529 state = vstate->frame[i]; 7530 for (j = 0; j < MAX_BPF_REG; j++) { 7531 reg = &state->regs[j]; 7532 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 7533 *reg = *known_reg; 7534 } 7535 7536 bpf_for_each_spilled_reg(j, state, reg) { 7537 if (!reg) 7538 continue; 7539 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 7540 *reg = *known_reg; 7541 } 7542 } 7543 } 7544 7545 static int check_cond_jmp_op(struct bpf_verifier_env *env, 7546 struct bpf_insn *insn, int *insn_idx) 7547 { 7548 struct bpf_verifier_state *this_branch = env->cur_state; 7549 struct bpf_verifier_state *other_branch; 7550 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 7551 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 7552 u8 opcode = BPF_OP(insn->code); 7553 bool is_jmp32; 7554 int pred = -1; 7555 int err; 7556 7557 /* Only conditional jumps are expected to reach here. */ 7558 if (opcode == BPF_JA || opcode > BPF_JSLE) { 7559 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 7560 return -EINVAL; 7561 } 7562 7563 if (BPF_SRC(insn->code) == BPF_X) { 7564 if (insn->imm != 0) { 7565 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 7566 return -EINVAL; 7567 } 7568 7569 /* check src1 operand */ 7570 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7571 if (err) 7572 return err; 7573 7574 if (is_pointer_value(env, insn->src_reg)) { 7575 verbose(env, "R%d pointer comparison prohibited\n", 7576 insn->src_reg); 7577 return -EACCES; 7578 } 7579 src_reg = ®s[insn->src_reg]; 7580 } else { 7581 if (insn->src_reg != BPF_REG_0) { 7582 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 7583 return -EINVAL; 7584 } 7585 } 7586 7587 /* check src2 operand */ 7588 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 7589 if (err) 7590 return err; 7591 7592 dst_reg = ®s[insn->dst_reg]; 7593 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 7594 7595 if (BPF_SRC(insn->code) == BPF_K) { 7596 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 7597 } else if (src_reg->type == SCALAR_VALUE && 7598 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 7599 pred = is_branch_taken(dst_reg, 7600 tnum_subreg(src_reg->var_off).value, 7601 opcode, 7602 is_jmp32); 7603 } else if (src_reg->type == SCALAR_VALUE && 7604 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 7605 pred = is_branch_taken(dst_reg, 7606 src_reg->var_off.value, 7607 opcode, 7608 is_jmp32); 7609 } else if (reg_is_pkt_pointer_any(dst_reg) && 7610 reg_is_pkt_pointer_any(src_reg) && 7611 !is_jmp32) { 7612 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 7613 } 7614 7615 if (pred >= 0) { 7616 /* If we get here with a dst_reg pointer type it is because 7617 * above is_branch_taken() special cased the 0 comparison. 7618 */ 7619 if (!__is_pointer_value(false, dst_reg)) 7620 err = mark_chain_precision(env, insn->dst_reg); 7621 if (BPF_SRC(insn->code) == BPF_X && !err && 7622 !__is_pointer_value(false, src_reg)) 7623 err = mark_chain_precision(env, insn->src_reg); 7624 if (err) 7625 return err; 7626 } 7627 if (pred == 1) { 7628 /* only follow the goto, ignore fall-through */ 7629 *insn_idx += insn->off; 7630 return 0; 7631 } else if (pred == 0) { 7632 /* only follow fall-through branch, since 7633 * that's where the program will go 7634 */ 7635 return 0; 7636 } 7637 7638 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 7639 false); 7640 if (!other_branch) 7641 return -EFAULT; 7642 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 7643 7644 /* detect if we are comparing against a constant value so we can adjust 7645 * our min/max values for our dst register. 7646 * this is only legit if both are scalars (or pointers to the same 7647 * object, I suppose, but we don't support that right now), because 7648 * otherwise the different base pointers mean the offsets aren't 7649 * comparable. 7650 */ 7651 if (BPF_SRC(insn->code) == BPF_X) { 7652 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 7653 7654 if (dst_reg->type == SCALAR_VALUE && 7655 src_reg->type == SCALAR_VALUE) { 7656 if (tnum_is_const(src_reg->var_off) || 7657 (is_jmp32 && 7658 tnum_is_const(tnum_subreg(src_reg->var_off)))) 7659 reg_set_min_max(&other_branch_regs[insn->dst_reg], 7660 dst_reg, 7661 src_reg->var_off.value, 7662 tnum_subreg(src_reg->var_off).value, 7663 opcode, is_jmp32); 7664 else if (tnum_is_const(dst_reg->var_off) || 7665 (is_jmp32 && 7666 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 7667 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 7668 src_reg, 7669 dst_reg->var_off.value, 7670 tnum_subreg(dst_reg->var_off).value, 7671 opcode, is_jmp32); 7672 else if (!is_jmp32 && 7673 (opcode == BPF_JEQ || opcode == BPF_JNE)) 7674 /* Comparing for equality, we can combine knowledge */ 7675 reg_combine_min_max(&other_branch_regs[insn->src_reg], 7676 &other_branch_regs[insn->dst_reg], 7677 src_reg, dst_reg, opcode); 7678 if (src_reg->id && 7679 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 7680 find_equal_scalars(this_branch, src_reg); 7681 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 7682 } 7683 7684 } 7685 } else if (dst_reg->type == SCALAR_VALUE) { 7686 reg_set_min_max(&other_branch_regs[insn->dst_reg], 7687 dst_reg, insn->imm, (u32)insn->imm, 7688 opcode, is_jmp32); 7689 } 7690 7691 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 7692 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 7693 find_equal_scalars(this_branch, dst_reg); 7694 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 7695 } 7696 7697 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 7698 * NOTE: these optimizations below are related with pointer comparison 7699 * which will never be JMP32. 7700 */ 7701 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 7702 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 7703 reg_type_may_be_null(dst_reg->type)) { 7704 /* Mark all identical registers in each branch as either 7705 * safe or unknown depending R == 0 or R != 0 conditional. 7706 */ 7707 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 7708 opcode == BPF_JNE); 7709 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 7710 opcode == BPF_JEQ); 7711 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 7712 this_branch, other_branch) && 7713 is_pointer_value(env, insn->dst_reg)) { 7714 verbose(env, "R%d pointer comparison prohibited\n", 7715 insn->dst_reg); 7716 return -EACCES; 7717 } 7718 if (env->log.level & BPF_LOG_LEVEL) 7719 print_verifier_state(env, this_branch->frame[this_branch->curframe]); 7720 return 0; 7721 } 7722 7723 /* verify BPF_LD_IMM64 instruction */ 7724 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 7725 { 7726 struct bpf_insn_aux_data *aux = cur_aux(env); 7727 struct bpf_reg_state *regs = cur_regs(env); 7728 struct bpf_reg_state *dst_reg; 7729 struct bpf_map *map; 7730 int err; 7731 7732 if (BPF_SIZE(insn->code) != BPF_DW) { 7733 verbose(env, "invalid BPF_LD_IMM insn\n"); 7734 return -EINVAL; 7735 } 7736 if (insn->off != 0) { 7737 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 7738 return -EINVAL; 7739 } 7740 7741 err = check_reg_arg(env, insn->dst_reg, DST_OP); 7742 if (err) 7743 return err; 7744 7745 dst_reg = ®s[insn->dst_reg]; 7746 if (insn->src_reg == 0) { 7747 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 7748 7749 dst_reg->type = SCALAR_VALUE; 7750 __mark_reg_known(®s[insn->dst_reg], imm); 7751 return 0; 7752 } 7753 7754 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 7755 mark_reg_known_zero(env, regs, insn->dst_reg); 7756 7757 dst_reg->type = aux->btf_var.reg_type; 7758 switch (dst_reg->type) { 7759 case PTR_TO_MEM: 7760 dst_reg->mem_size = aux->btf_var.mem_size; 7761 break; 7762 case PTR_TO_BTF_ID: 7763 case PTR_TO_PERCPU_BTF_ID: 7764 dst_reg->btf = aux->btf_var.btf; 7765 dst_reg->btf_id = aux->btf_var.btf_id; 7766 break; 7767 default: 7768 verbose(env, "bpf verifier is misconfigured\n"); 7769 return -EFAULT; 7770 } 7771 return 0; 7772 } 7773 7774 map = env->used_maps[aux->map_index]; 7775 mark_reg_known_zero(env, regs, insn->dst_reg); 7776 dst_reg->map_ptr = map; 7777 7778 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { 7779 dst_reg->type = PTR_TO_MAP_VALUE; 7780 dst_reg->off = aux->map_off; 7781 if (map_value_has_spin_lock(map)) 7782 dst_reg->id = ++env->id_gen; 7783 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 7784 dst_reg->type = CONST_PTR_TO_MAP; 7785 } else { 7786 verbose(env, "bpf verifier is misconfigured\n"); 7787 return -EINVAL; 7788 } 7789 7790 return 0; 7791 } 7792 7793 static bool may_access_skb(enum bpf_prog_type type) 7794 { 7795 switch (type) { 7796 case BPF_PROG_TYPE_SOCKET_FILTER: 7797 case BPF_PROG_TYPE_SCHED_CLS: 7798 case BPF_PROG_TYPE_SCHED_ACT: 7799 return true; 7800 default: 7801 return false; 7802 } 7803 } 7804 7805 /* verify safety of LD_ABS|LD_IND instructions: 7806 * - they can only appear in the programs where ctx == skb 7807 * - since they are wrappers of function calls, they scratch R1-R5 registers, 7808 * preserve R6-R9, and store return value into R0 7809 * 7810 * Implicit input: 7811 * ctx == skb == R6 == CTX 7812 * 7813 * Explicit input: 7814 * SRC == any register 7815 * IMM == 32-bit immediate 7816 * 7817 * Output: 7818 * R0 - 8/16/32-bit skb data converted to cpu endianness 7819 */ 7820 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 7821 { 7822 struct bpf_reg_state *regs = cur_regs(env); 7823 static const int ctx_reg = BPF_REG_6; 7824 u8 mode = BPF_MODE(insn->code); 7825 int i, err; 7826 7827 if (!may_access_skb(resolve_prog_type(env->prog))) { 7828 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 7829 return -EINVAL; 7830 } 7831 7832 if (!env->ops->gen_ld_abs) { 7833 verbose(env, "bpf verifier is misconfigured\n"); 7834 return -EINVAL; 7835 } 7836 7837 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 7838 BPF_SIZE(insn->code) == BPF_DW || 7839 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 7840 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 7841 return -EINVAL; 7842 } 7843 7844 /* check whether implicit source operand (register R6) is readable */ 7845 err = check_reg_arg(env, ctx_reg, SRC_OP); 7846 if (err) 7847 return err; 7848 7849 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 7850 * gen_ld_abs() may terminate the program at runtime, leading to 7851 * reference leak. 7852 */ 7853 err = check_reference_leak(env); 7854 if (err) { 7855 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 7856 return err; 7857 } 7858 7859 if (env->cur_state->active_spin_lock) { 7860 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 7861 return -EINVAL; 7862 } 7863 7864 if (regs[ctx_reg].type != PTR_TO_CTX) { 7865 verbose(env, 7866 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 7867 return -EINVAL; 7868 } 7869 7870 if (mode == BPF_IND) { 7871 /* check explicit source operand */ 7872 err = check_reg_arg(env, insn->src_reg, SRC_OP); 7873 if (err) 7874 return err; 7875 } 7876 7877 err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); 7878 if (err < 0) 7879 return err; 7880 7881 /* reset caller saved regs to unreadable */ 7882 for (i = 0; i < CALLER_SAVED_REGS; i++) { 7883 mark_reg_not_init(env, regs, caller_saved[i]); 7884 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 7885 } 7886 7887 /* mark destination R0 register as readable, since it contains 7888 * the value fetched from the packet. 7889 * Already marked as written above. 7890 */ 7891 mark_reg_unknown(env, regs, BPF_REG_0); 7892 /* ld_abs load up to 32-bit skb data. */ 7893 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 7894 return 0; 7895 } 7896 7897 static int check_return_code(struct bpf_verifier_env *env) 7898 { 7899 struct tnum enforce_attach_type_range = tnum_unknown; 7900 const struct bpf_prog *prog = env->prog; 7901 struct bpf_reg_state *reg; 7902 struct tnum range = tnum_range(0, 1); 7903 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 7904 int err; 7905 const bool is_subprog = env->cur_state->frame[0]->subprogno; 7906 7907 /* LSM and struct_ops func-ptr's return type could be "void" */ 7908 if (!is_subprog && 7909 (prog_type == BPF_PROG_TYPE_STRUCT_OPS || 7910 prog_type == BPF_PROG_TYPE_LSM) && 7911 !prog->aux->attach_func_proto->type) 7912 return 0; 7913 7914 /* eBPF calling convetion is such that R0 is used 7915 * to return the value from eBPF program. 7916 * Make sure that it's readable at this time 7917 * of bpf_exit, which means that program wrote 7918 * something into it earlier 7919 */ 7920 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 7921 if (err) 7922 return err; 7923 7924 if (is_pointer_value(env, BPF_REG_0)) { 7925 verbose(env, "R0 leaks addr as return value\n"); 7926 return -EACCES; 7927 } 7928 7929 reg = cur_regs(env) + BPF_REG_0; 7930 if (is_subprog) { 7931 if (reg->type != SCALAR_VALUE) { 7932 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 7933 reg_type_str[reg->type]); 7934 return -EINVAL; 7935 } 7936 return 0; 7937 } 7938 7939 switch (prog_type) { 7940 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 7941 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 7942 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 7943 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 7944 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 7945 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 7946 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 7947 range = tnum_range(1, 1); 7948 break; 7949 case BPF_PROG_TYPE_CGROUP_SKB: 7950 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 7951 range = tnum_range(0, 3); 7952 enforce_attach_type_range = tnum_range(2, 3); 7953 } 7954 break; 7955 case BPF_PROG_TYPE_CGROUP_SOCK: 7956 case BPF_PROG_TYPE_SOCK_OPS: 7957 case BPF_PROG_TYPE_CGROUP_DEVICE: 7958 case BPF_PROG_TYPE_CGROUP_SYSCTL: 7959 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 7960 break; 7961 case BPF_PROG_TYPE_RAW_TRACEPOINT: 7962 if (!env->prog->aux->attach_btf_id) 7963 return 0; 7964 range = tnum_const(0); 7965 break; 7966 case BPF_PROG_TYPE_TRACING: 7967 switch (env->prog->expected_attach_type) { 7968 case BPF_TRACE_FENTRY: 7969 case BPF_TRACE_FEXIT: 7970 range = tnum_const(0); 7971 break; 7972 case BPF_TRACE_RAW_TP: 7973 case BPF_MODIFY_RETURN: 7974 return 0; 7975 case BPF_TRACE_ITER: 7976 break; 7977 default: 7978 return -ENOTSUPP; 7979 } 7980 break; 7981 case BPF_PROG_TYPE_SK_LOOKUP: 7982 range = tnum_range(SK_DROP, SK_PASS); 7983 break; 7984 case BPF_PROG_TYPE_EXT: 7985 /* freplace program can return anything as its return value 7986 * depends on the to-be-replaced kernel func or bpf program. 7987 */ 7988 default: 7989 return 0; 7990 } 7991 7992 if (reg->type != SCALAR_VALUE) { 7993 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 7994 reg_type_str[reg->type]); 7995 return -EINVAL; 7996 } 7997 7998 if (!tnum_in(range, reg->var_off)) { 7999 char tn_buf[48]; 8000 8001 verbose(env, "At program exit the register R0 "); 8002 if (!tnum_is_unknown(reg->var_off)) { 8003 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 8004 verbose(env, "has value %s", tn_buf); 8005 } else { 8006 verbose(env, "has unknown scalar value"); 8007 } 8008 tnum_strn(tn_buf, sizeof(tn_buf), range); 8009 verbose(env, " should have been in %s\n", tn_buf); 8010 return -EINVAL; 8011 } 8012 8013 if (!tnum_is_unknown(enforce_attach_type_range) && 8014 tnum_in(enforce_attach_type_range, reg->var_off)) 8015 env->prog->enforce_expected_attach_type = 1; 8016 return 0; 8017 } 8018 8019 /* non-recursive DFS pseudo code 8020 * 1 procedure DFS-iterative(G,v): 8021 * 2 label v as discovered 8022 * 3 let S be a stack 8023 * 4 S.push(v) 8024 * 5 while S is not empty 8025 * 6 t <- S.pop() 8026 * 7 if t is what we're looking for: 8027 * 8 return t 8028 * 9 for all edges e in G.adjacentEdges(t) do 8029 * 10 if edge e is already labelled 8030 * 11 continue with the next edge 8031 * 12 w <- G.adjacentVertex(t,e) 8032 * 13 if vertex w is not discovered and not explored 8033 * 14 label e as tree-edge 8034 * 15 label w as discovered 8035 * 16 S.push(w) 8036 * 17 continue at 5 8037 * 18 else if vertex w is discovered 8038 * 19 label e as back-edge 8039 * 20 else 8040 * 21 // vertex w is explored 8041 * 22 label e as forward- or cross-edge 8042 * 23 label t as explored 8043 * 24 S.pop() 8044 * 8045 * convention: 8046 * 0x10 - discovered 8047 * 0x11 - discovered and fall-through edge labelled 8048 * 0x12 - discovered and fall-through and branch edges labelled 8049 * 0x20 - explored 8050 */ 8051 8052 enum { 8053 DISCOVERED = 0x10, 8054 EXPLORED = 0x20, 8055 FALLTHROUGH = 1, 8056 BRANCH = 2, 8057 }; 8058 8059 static u32 state_htab_size(struct bpf_verifier_env *env) 8060 { 8061 return env->prog->len; 8062 } 8063 8064 static struct bpf_verifier_state_list **explored_state( 8065 struct bpf_verifier_env *env, 8066 int idx) 8067 { 8068 struct bpf_verifier_state *cur = env->cur_state; 8069 struct bpf_func_state *state = cur->frame[cur->curframe]; 8070 8071 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 8072 } 8073 8074 static void init_explored_state(struct bpf_verifier_env *env, int idx) 8075 { 8076 env->insn_aux_data[idx].prune_point = true; 8077 } 8078 8079 enum { 8080 DONE_EXPLORING = 0, 8081 KEEP_EXPLORING = 1, 8082 }; 8083 8084 /* t, w, e - match pseudo-code above: 8085 * t - index of current instruction 8086 * w - next instruction 8087 * e - edge 8088 */ 8089 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 8090 bool loop_ok) 8091 { 8092 int *insn_stack = env->cfg.insn_stack; 8093 int *insn_state = env->cfg.insn_state; 8094 8095 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 8096 return DONE_EXPLORING; 8097 8098 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 8099 return DONE_EXPLORING; 8100 8101 if (w < 0 || w >= env->prog->len) { 8102 verbose_linfo(env, t, "%d: ", t); 8103 verbose(env, "jump out of range from insn %d to %d\n", t, w); 8104 return -EINVAL; 8105 } 8106 8107 if (e == BRANCH) 8108 /* mark branch target for state pruning */ 8109 init_explored_state(env, w); 8110 8111 if (insn_state[w] == 0) { 8112 /* tree-edge */ 8113 insn_state[t] = DISCOVERED | e; 8114 insn_state[w] = DISCOVERED; 8115 if (env->cfg.cur_stack >= env->prog->len) 8116 return -E2BIG; 8117 insn_stack[env->cfg.cur_stack++] = w; 8118 return KEEP_EXPLORING; 8119 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 8120 if (loop_ok && env->bpf_capable) 8121 return DONE_EXPLORING; 8122 verbose_linfo(env, t, "%d: ", t); 8123 verbose_linfo(env, w, "%d: ", w); 8124 verbose(env, "back-edge from insn %d to %d\n", t, w); 8125 return -EINVAL; 8126 } else if (insn_state[w] == EXPLORED) { 8127 /* forward- or cross-edge */ 8128 insn_state[t] = DISCOVERED | e; 8129 } else { 8130 verbose(env, "insn state internal bug\n"); 8131 return -EFAULT; 8132 } 8133 return DONE_EXPLORING; 8134 } 8135 8136 /* Visits the instruction at index t and returns one of the following: 8137 * < 0 - an error occurred 8138 * DONE_EXPLORING - the instruction was fully explored 8139 * KEEP_EXPLORING - there is still work to be done before it is fully explored 8140 */ 8141 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 8142 { 8143 struct bpf_insn *insns = env->prog->insnsi; 8144 int ret; 8145 8146 /* All non-branch instructions have a single fall-through edge. */ 8147 if (BPF_CLASS(insns[t].code) != BPF_JMP && 8148 BPF_CLASS(insns[t].code) != BPF_JMP32) 8149 return push_insn(t, t + 1, FALLTHROUGH, env, false); 8150 8151 switch (BPF_OP(insns[t].code)) { 8152 case BPF_EXIT: 8153 return DONE_EXPLORING; 8154 8155 case BPF_CALL: 8156 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 8157 if (ret) 8158 return ret; 8159 8160 if (t + 1 < insn_cnt) 8161 init_explored_state(env, t + 1); 8162 if (insns[t].src_reg == BPF_PSEUDO_CALL) { 8163 init_explored_state(env, t); 8164 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, 8165 env, false); 8166 } 8167 return ret; 8168 8169 case BPF_JA: 8170 if (BPF_SRC(insns[t].code) != BPF_K) 8171 return -EINVAL; 8172 8173 /* unconditional jump with single edge */ 8174 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 8175 true); 8176 if (ret) 8177 return ret; 8178 8179 /* unconditional jmp is not a good pruning point, 8180 * but it's marked, since backtracking needs 8181 * to record jmp history in is_state_visited(). 8182 */ 8183 init_explored_state(env, t + insns[t].off + 1); 8184 /* tell verifier to check for equivalent states 8185 * after every call and jump 8186 */ 8187 if (t + 1 < insn_cnt) 8188 init_explored_state(env, t + 1); 8189 8190 return ret; 8191 8192 default: 8193 /* conditional jump with two edges */ 8194 init_explored_state(env, t); 8195 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 8196 if (ret) 8197 return ret; 8198 8199 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 8200 } 8201 } 8202 8203 /* non-recursive depth-first-search to detect loops in BPF program 8204 * loop == back-edge in directed graph 8205 */ 8206 static int check_cfg(struct bpf_verifier_env *env) 8207 { 8208 int insn_cnt = env->prog->len; 8209 int *insn_stack, *insn_state; 8210 int ret = 0; 8211 int i; 8212 8213 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 8214 if (!insn_state) 8215 return -ENOMEM; 8216 8217 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 8218 if (!insn_stack) { 8219 kvfree(insn_state); 8220 return -ENOMEM; 8221 } 8222 8223 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 8224 insn_stack[0] = 0; /* 0 is the first instruction */ 8225 env->cfg.cur_stack = 1; 8226 8227 while (env->cfg.cur_stack > 0) { 8228 int t = insn_stack[env->cfg.cur_stack - 1]; 8229 8230 ret = visit_insn(t, insn_cnt, env); 8231 switch (ret) { 8232 case DONE_EXPLORING: 8233 insn_state[t] = EXPLORED; 8234 env->cfg.cur_stack--; 8235 break; 8236 case KEEP_EXPLORING: 8237 break; 8238 default: 8239 if (ret > 0) { 8240 verbose(env, "visit_insn internal bug\n"); 8241 ret = -EFAULT; 8242 } 8243 goto err_free; 8244 } 8245 } 8246 8247 if (env->cfg.cur_stack < 0) { 8248 verbose(env, "pop stack internal bug\n"); 8249 ret = -EFAULT; 8250 goto err_free; 8251 } 8252 8253 for (i = 0; i < insn_cnt; i++) { 8254 if (insn_state[i] != EXPLORED) { 8255 verbose(env, "unreachable insn %d\n", i); 8256 ret = -EINVAL; 8257 goto err_free; 8258 } 8259 } 8260 ret = 0; /* cfg looks good */ 8261 8262 err_free: 8263 kvfree(insn_state); 8264 kvfree(insn_stack); 8265 env->cfg.insn_state = env->cfg.insn_stack = NULL; 8266 return ret; 8267 } 8268 8269 static int check_abnormal_return(struct bpf_verifier_env *env) 8270 { 8271 int i; 8272 8273 for (i = 1; i < env->subprog_cnt; i++) { 8274 if (env->subprog_info[i].has_ld_abs) { 8275 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 8276 return -EINVAL; 8277 } 8278 if (env->subprog_info[i].has_tail_call) { 8279 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 8280 return -EINVAL; 8281 } 8282 } 8283 return 0; 8284 } 8285 8286 /* The minimum supported BTF func info size */ 8287 #define MIN_BPF_FUNCINFO_SIZE 8 8288 #define MAX_FUNCINFO_REC_SIZE 252 8289 8290 static int check_btf_func(struct bpf_verifier_env *env, 8291 const union bpf_attr *attr, 8292 union bpf_attr __user *uattr) 8293 { 8294 const struct btf_type *type, *func_proto, *ret_type; 8295 u32 i, nfuncs, urec_size, min_size; 8296 u32 krec_size = sizeof(struct bpf_func_info); 8297 struct bpf_func_info *krecord; 8298 struct bpf_func_info_aux *info_aux = NULL; 8299 struct bpf_prog *prog; 8300 const struct btf *btf; 8301 void __user *urecord; 8302 u32 prev_offset = 0; 8303 bool scalar_return; 8304 int ret = -ENOMEM; 8305 8306 nfuncs = attr->func_info_cnt; 8307 if (!nfuncs) { 8308 if (check_abnormal_return(env)) 8309 return -EINVAL; 8310 return 0; 8311 } 8312 8313 if (nfuncs != env->subprog_cnt) { 8314 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 8315 return -EINVAL; 8316 } 8317 8318 urec_size = attr->func_info_rec_size; 8319 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 8320 urec_size > MAX_FUNCINFO_REC_SIZE || 8321 urec_size % sizeof(u32)) { 8322 verbose(env, "invalid func info rec size %u\n", urec_size); 8323 return -EINVAL; 8324 } 8325 8326 prog = env->prog; 8327 btf = prog->aux->btf; 8328 8329 urecord = u64_to_user_ptr(attr->func_info); 8330 min_size = min_t(u32, krec_size, urec_size); 8331 8332 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 8333 if (!krecord) 8334 return -ENOMEM; 8335 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 8336 if (!info_aux) 8337 goto err_free; 8338 8339 for (i = 0; i < nfuncs; i++) { 8340 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 8341 if (ret) { 8342 if (ret == -E2BIG) { 8343 verbose(env, "nonzero tailing record in func info"); 8344 /* set the size kernel expects so loader can zero 8345 * out the rest of the record. 8346 */ 8347 if (put_user(min_size, &uattr->func_info_rec_size)) 8348 ret = -EFAULT; 8349 } 8350 goto err_free; 8351 } 8352 8353 if (copy_from_user(&krecord[i], urecord, min_size)) { 8354 ret = -EFAULT; 8355 goto err_free; 8356 } 8357 8358 /* check insn_off */ 8359 ret = -EINVAL; 8360 if (i == 0) { 8361 if (krecord[i].insn_off) { 8362 verbose(env, 8363 "nonzero insn_off %u for the first func info record", 8364 krecord[i].insn_off); 8365 goto err_free; 8366 } 8367 } else if (krecord[i].insn_off <= prev_offset) { 8368 verbose(env, 8369 "same or smaller insn offset (%u) than previous func info record (%u)", 8370 krecord[i].insn_off, prev_offset); 8371 goto err_free; 8372 } 8373 8374 if (env->subprog_info[i].start != krecord[i].insn_off) { 8375 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 8376 goto err_free; 8377 } 8378 8379 /* check type_id */ 8380 type = btf_type_by_id(btf, krecord[i].type_id); 8381 if (!type || !btf_type_is_func(type)) { 8382 verbose(env, "invalid type id %d in func info", 8383 krecord[i].type_id); 8384 goto err_free; 8385 } 8386 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 8387 8388 func_proto = btf_type_by_id(btf, type->type); 8389 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 8390 /* btf_func_check() already verified it during BTF load */ 8391 goto err_free; 8392 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 8393 scalar_return = 8394 btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); 8395 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 8396 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 8397 goto err_free; 8398 } 8399 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 8400 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 8401 goto err_free; 8402 } 8403 8404 prev_offset = krecord[i].insn_off; 8405 urecord += urec_size; 8406 } 8407 8408 prog->aux->func_info = krecord; 8409 prog->aux->func_info_cnt = nfuncs; 8410 prog->aux->func_info_aux = info_aux; 8411 return 0; 8412 8413 err_free: 8414 kvfree(krecord); 8415 kfree(info_aux); 8416 return ret; 8417 } 8418 8419 static void adjust_btf_func(struct bpf_verifier_env *env) 8420 { 8421 struct bpf_prog_aux *aux = env->prog->aux; 8422 int i; 8423 8424 if (!aux->func_info) 8425 return; 8426 8427 for (i = 0; i < env->subprog_cnt; i++) 8428 aux->func_info[i].insn_off = env->subprog_info[i].start; 8429 } 8430 8431 #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ 8432 sizeof(((struct bpf_line_info *)(0))->line_col)) 8433 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 8434 8435 static int check_btf_line(struct bpf_verifier_env *env, 8436 const union bpf_attr *attr, 8437 union bpf_attr __user *uattr) 8438 { 8439 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 8440 struct bpf_subprog_info *sub; 8441 struct bpf_line_info *linfo; 8442 struct bpf_prog *prog; 8443 const struct btf *btf; 8444 void __user *ulinfo; 8445 int err; 8446 8447 nr_linfo = attr->line_info_cnt; 8448 if (!nr_linfo) 8449 return 0; 8450 8451 rec_size = attr->line_info_rec_size; 8452 if (rec_size < MIN_BPF_LINEINFO_SIZE || 8453 rec_size > MAX_LINEINFO_REC_SIZE || 8454 rec_size & (sizeof(u32) - 1)) 8455 return -EINVAL; 8456 8457 /* Need to zero it in case the userspace may 8458 * pass in a smaller bpf_line_info object. 8459 */ 8460 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 8461 GFP_KERNEL | __GFP_NOWARN); 8462 if (!linfo) 8463 return -ENOMEM; 8464 8465 prog = env->prog; 8466 btf = prog->aux->btf; 8467 8468 s = 0; 8469 sub = env->subprog_info; 8470 ulinfo = u64_to_user_ptr(attr->line_info); 8471 expected_size = sizeof(struct bpf_line_info); 8472 ncopy = min_t(u32, expected_size, rec_size); 8473 for (i = 0; i < nr_linfo; i++) { 8474 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 8475 if (err) { 8476 if (err == -E2BIG) { 8477 verbose(env, "nonzero tailing record in line_info"); 8478 if (put_user(expected_size, 8479 &uattr->line_info_rec_size)) 8480 err = -EFAULT; 8481 } 8482 goto err_free; 8483 } 8484 8485 if (copy_from_user(&linfo[i], ulinfo, ncopy)) { 8486 err = -EFAULT; 8487 goto err_free; 8488 } 8489 8490 /* 8491 * Check insn_off to ensure 8492 * 1) strictly increasing AND 8493 * 2) bounded by prog->len 8494 * 8495 * The linfo[0].insn_off == 0 check logically falls into 8496 * the later "missing bpf_line_info for func..." case 8497 * because the first linfo[0].insn_off must be the 8498 * first sub also and the first sub must have 8499 * subprog_info[0].start == 0. 8500 */ 8501 if ((i && linfo[i].insn_off <= prev_offset) || 8502 linfo[i].insn_off >= prog->len) { 8503 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 8504 i, linfo[i].insn_off, prev_offset, 8505 prog->len); 8506 err = -EINVAL; 8507 goto err_free; 8508 } 8509 8510 if (!prog->insnsi[linfo[i].insn_off].code) { 8511 verbose(env, 8512 "Invalid insn code at line_info[%u].insn_off\n", 8513 i); 8514 err = -EINVAL; 8515 goto err_free; 8516 } 8517 8518 if (!btf_name_by_offset(btf, linfo[i].line_off) || 8519 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 8520 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 8521 err = -EINVAL; 8522 goto err_free; 8523 } 8524 8525 if (s != env->subprog_cnt) { 8526 if (linfo[i].insn_off == sub[s].start) { 8527 sub[s].linfo_idx = i; 8528 s++; 8529 } else if (sub[s].start < linfo[i].insn_off) { 8530 verbose(env, "missing bpf_line_info for func#%u\n", s); 8531 err = -EINVAL; 8532 goto err_free; 8533 } 8534 } 8535 8536 prev_offset = linfo[i].insn_off; 8537 ulinfo += rec_size; 8538 } 8539 8540 if (s != env->subprog_cnt) { 8541 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 8542 env->subprog_cnt - s, s); 8543 err = -EINVAL; 8544 goto err_free; 8545 } 8546 8547 prog->aux->linfo = linfo; 8548 prog->aux->nr_linfo = nr_linfo; 8549 8550 return 0; 8551 8552 err_free: 8553 kvfree(linfo); 8554 return err; 8555 } 8556 8557 static int check_btf_info(struct bpf_verifier_env *env, 8558 const union bpf_attr *attr, 8559 union bpf_attr __user *uattr) 8560 { 8561 struct btf *btf; 8562 int err; 8563 8564 if (!attr->func_info_cnt && !attr->line_info_cnt) { 8565 if (check_abnormal_return(env)) 8566 return -EINVAL; 8567 return 0; 8568 } 8569 8570 btf = btf_get_by_fd(attr->prog_btf_fd); 8571 if (IS_ERR(btf)) 8572 return PTR_ERR(btf); 8573 env->prog->aux->btf = btf; 8574 8575 err = check_btf_func(env, attr, uattr); 8576 if (err) 8577 return err; 8578 8579 err = check_btf_line(env, attr, uattr); 8580 if (err) 8581 return err; 8582 8583 return 0; 8584 } 8585 8586 /* check %cur's range satisfies %old's */ 8587 static bool range_within(struct bpf_reg_state *old, 8588 struct bpf_reg_state *cur) 8589 { 8590 return old->umin_value <= cur->umin_value && 8591 old->umax_value >= cur->umax_value && 8592 old->smin_value <= cur->smin_value && 8593 old->smax_value >= cur->smax_value; 8594 } 8595 8596 /* Maximum number of register states that can exist at once */ 8597 #define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) 8598 struct idpair { 8599 u32 old; 8600 u32 cur; 8601 }; 8602 8603 /* If in the old state two registers had the same id, then they need to have 8604 * the same id in the new state as well. But that id could be different from 8605 * the old state, so we need to track the mapping from old to new ids. 8606 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 8607 * regs with old id 5 must also have new id 9 for the new state to be safe. But 8608 * regs with a different old id could still have new id 9, we don't care about 8609 * that. 8610 * So we look through our idmap to see if this old id has been seen before. If 8611 * so, we require the new id to match; otherwise, we add the id pair to the map. 8612 */ 8613 static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) 8614 { 8615 unsigned int i; 8616 8617 for (i = 0; i < ID_MAP_SIZE; i++) { 8618 if (!idmap[i].old) { 8619 /* Reached an empty slot; haven't seen this id before */ 8620 idmap[i].old = old_id; 8621 idmap[i].cur = cur_id; 8622 return true; 8623 } 8624 if (idmap[i].old == old_id) 8625 return idmap[i].cur == cur_id; 8626 } 8627 /* We ran out of idmap slots, which should be impossible */ 8628 WARN_ON_ONCE(1); 8629 return false; 8630 } 8631 8632 static void clean_func_state(struct bpf_verifier_env *env, 8633 struct bpf_func_state *st) 8634 { 8635 enum bpf_reg_liveness live; 8636 int i, j; 8637 8638 for (i = 0; i < BPF_REG_FP; i++) { 8639 live = st->regs[i].live; 8640 /* liveness must not touch this register anymore */ 8641 st->regs[i].live |= REG_LIVE_DONE; 8642 if (!(live & REG_LIVE_READ)) 8643 /* since the register is unused, clear its state 8644 * to make further comparison simpler 8645 */ 8646 __mark_reg_not_init(env, &st->regs[i]); 8647 } 8648 8649 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 8650 live = st->stack[i].spilled_ptr.live; 8651 /* liveness must not touch this stack slot anymore */ 8652 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 8653 if (!(live & REG_LIVE_READ)) { 8654 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 8655 for (j = 0; j < BPF_REG_SIZE; j++) 8656 st->stack[i].slot_type[j] = STACK_INVALID; 8657 } 8658 } 8659 } 8660 8661 static void clean_verifier_state(struct bpf_verifier_env *env, 8662 struct bpf_verifier_state *st) 8663 { 8664 int i; 8665 8666 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 8667 /* all regs in this state in all frames were already marked */ 8668 return; 8669 8670 for (i = 0; i <= st->curframe; i++) 8671 clean_func_state(env, st->frame[i]); 8672 } 8673 8674 /* the parentage chains form a tree. 8675 * the verifier states are added to state lists at given insn and 8676 * pushed into state stack for future exploration. 8677 * when the verifier reaches bpf_exit insn some of the verifer states 8678 * stored in the state lists have their final liveness state already, 8679 * but a lot of states will get revised from liveness point of view when 8680 * the verifier explores other branches. 8681 * Example: 8682 * 1: r0 = 1 8683 * 2: if r1 == 100 goto pc+1 8684 * 3: r0 = 2 8685 * 4: exit 8686 * when the verifier reaches exit insn the register r0 in the state list of 8687 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 8688 * of insn 2 and goes exploring further. At the insn 4 it will walk the 8689 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 8690 * 8691 * Since the verifier pushes the branch states as it sees them while exploring 8692 * the program the condition of walking the branch instruction for the second 8693 * time means that all states below this branch were already explored and 8694 * their final liveness markes are already propagated. 8695 * Hence when the verifier completes the search of state list in is_state_visited() 8696 * we can call this clean_live_states() function to mark all liveness states 8697 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 8698 * will not be used. 8699 * This function also clears the registers and stack for states that !READ 8700 * to simplify state merging. 8701 * 8702 * Important note here that walking the same branch instruction in the callee 8703 * doesn't meant that the states are DONE. The verifier has to compare 8704 * the callsites 8705 */ 8706 static void clean_live_states(struct bpf_verifier_env *env, int insn, 8707 struct bpf_verifier_state *cur) 8708 { 8709 struct bpf_verifier_state_list *sl; 8710 int i; 8711 8712 sl = *explored_state(env, insn); 8713 while (sl) { 8714 if (sl->state.branches) 8715 goto next; 8716 if (sl->state.insn_idx != insn || 8717 sl->state.curframe != cur->curframe) 8718 goto next; 8719 for (i = 0; i <= cur->curframe; i++) 8720 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 8721 goto next; 8722 clean_verifier_state(env, &sl->state); 8723 next: 8724 sl = sl->next; 8725 } 8726 } 8727 8728 /* Returns true if (rold safe implies rcur safe) */ 8729 static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, 8730 struct idpair *idmap) 8731 { 8732 bool equal; 8733 8734 if (!(rold->live & REG_LIVE_READ)) 8735 /* explored state didn't use this */ 8736 return true; 8737 8738 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 8739 8740 if (rold->type == PTR_TO_STACK) 8741 /* two stack pointers are equal only if they're pointing to 8742 * the same stack frame, since fp-8 in foo != fp-8 in bar 8743 */ 8744 return equal && rold->frameno == rcur->frameno; 8745 8746 if (equal) 8747 return true; 8748 8749 if (rold->type == NOT_INIT) 8750 /* explored state can't have used this */ 8751 return true; 8752 if (rcur->type == NOT_INIT) 8753 return false; 8754 switch (rold->type) { 8755 case SCALAR_VALUE: 8756 if (rcur->type == SCALAR_VALUE) { 8757 if (!rold->precise && !rcur->precise) 8758 return true; 8759 /* new val must satisfy old val knowledge */ 8760 return range_within(rold, rcur) && 8761 tnum_in(rold->var_off, rcur->var_off); 8762 } else { 8763 /* We're trying to use a pointer in place of a scalar. 8764 * Even if the scalar was unbounded, this could lead to 8765 * pointer leaks because scalars are allowed to leak 8766 * while pointers are not. We could make this safe in 8767 * special cases if root is calling us, but it's 8768 * probably not worth the hassle. 8769 */ 8770 return false; 8771 } 8772 case PTR_TO_MAP_VALUE: 8773 /* If the new min/max/var_off satisfy the old ones and 8774 * everything else matches, we are OK. 8775 * 'id' is not compared, since it's only used for maps with 8776 * bpf_spin_lock inside map element and in such cases if 8777 * the rest of the prog is valid for one map element then 8778 * it's valid for all map elements regardless of the key 8779 * used in bpf_map_lookup() 8780 */ 8781 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 8782 range_within(rold, rcur) && 8783 tnum_in(rold->var_off, rcur->var_off); 8784 case PTR_TO_MAP_VALUE_OR_NULL: 8785 /* a PTR_TO_MAP_VALUE could be safe to use as a 8786 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 8787 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 8788 * checked, doing so could have affected others with the same 8789 * id, and we can't check for that because we lost the id when 8790 * we converted to a PTR_TO_MAP_VALUE. 8791 */ 8792 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) 8793 return false; 8794 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 8795 return false; 8796 /* Check our ids match any regs they're supposed to */ 8797 return check_ids(rold->id, rcur->id, idmap); 8798 case PTR_TO_PACKET_META: 8799 case PTR_TO_PACKET: 8800 if (rcur->type != rold->type) 8801 return false; 8802 /* We must have at least as much range as the old ptr 8803 * did, so that any accesses which were safe before are 8804 * still safe. This is true even if old range < old off, 8805 * since someone could have accessed through (ptr - k), or 8806 * even done ptr -= k in a register, to get a safe access. 8807 */ 8808 if (rold->range > rcur->range) 8809 return false; 8810 /* If the offsets don't match, we can't trust our alignment; 8811 * nor can we be sure that we won't fall out of range. 8812 */ 8813 if (rold->off != rcur->off) 8814 return false; 8815 /* id relations must be preserved */ 8816 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 8817 return false; 8818 /* new val must satisfy old val knowledge */ 8819 return range_within(rold, rcur) && 8820 tnum_in(rold->var_off, rcur->var_off); 8821 case PTR_TO_CTX: 8822 case CONST_PTR_TO_MAP: 8823 case PTR_TO_PACKET_END: 8824 case PTR_TO_FLOW_KEYS: 8825 case PTR_TO_SOCKET: 8826 case PTR_TO_SOCKET_OR_NULL: 8827 case PTR_TO_SOCK_COMMON: 8828 case PTR_TO_SOCK_COMMON_OR_NULL: 8829 case PTR_TO_TCP_SOCK: 8830 case PTR_TO_TCP_SOCK_OR_NULL: 8831 case PTR_TO_XDP_SOCK: 8832 /* Only valid matches are exact, which memcmp() above 8833 * would have accepted 8834 */ 8835 default: 8836 /* Don't know what's going on, just say it's not safe */ 8837 return false; 8838 } 8839 8840 /* Shouldn't get here; if we do, say it's not safe */ 8841 WARN_ON_ONCE(1); 8842 return false; 8843 } 8844 8845 static bool stacksafe(struct bpf_func_state *old, 8846 struct bpf_func_state *cur, 8847 struct idpair *idmap) 8848 { 8849 int i, spi; 8850 8851 /* walk slots of the explored stack and ignore any additional 8852 * slots in the current stack, since explored(safe) state 8853 * didn't use them 8854 */ 8855 for (i = 0; i < old->allocated_stack; i++) { 8856 spi = i / BPF_REG_SIZE; 8857 8858 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 8859 i += BPF_REG_SIZE - 1; 8860 /* explored state didn't use this */ 8861 continue; 8862 } 8863 8864 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 8865 continue; 8866 8867 /* explored stack has more populated slots than current stack 8868 * and these slots were used 8869 */ 8870 if (i >= cur->allocated_stack) 8871 return false; 8872 8873 /* if old state was safe with misc data in the stack 8874 * it will be safe with zero-initialized stack. 8875 * The opposite is not true 8876 */ 8877 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 8878 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 8879 continue; 8880 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 8881 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 8882 /* Ex: old explored (safe) state has STACK_SPILL in 8883 * this stack slot, but current has STACK_MISC -> 8884 * this verifier states are not equivalent, 8885 * return false to continue verification of this path 8886 */ 8887 return false; 8888 if (i % BPF_REG_SIZE) 8889 continue; 8890 if (old->stack[spi].slot_type[0] != STACK_SPILL) 8891 continue; 8892 if (!regsafe(&old->stack[spi].spilled_ptr, 8893 &cur->stack[spi].spilled_ptr, 8894 idmap)) 8895 /* when explored and current stack slot are both storing 8896 * spilled registers, check that stored pointers types 8897 * are the same as well. 8898 * Ex: explored safe path could have stored 8899 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 8900 * but current path has stored: 8901 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 8902 * such verifier states are not equivalent. 8903 * return false to continue verification of this path 8904 */ 8905 return false; 8906 } 8907 return true; 8908 } 8909 8910 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 8911 { 8912 if (old->acquired_refs != cur->acquired_refs) 8913 return false; 8914 return !memcmp(old->refs, cur->refs, 8915 sizeof(*old->refs) * old->acquired_refs); 8916 } 8917 8918 /* compare two verifier states 8919 * 8920 * all states stored in state_list are known to be valid, since 8921 * verifier reached 'bpf_exit' instruction through them 8922 * 8923 * this function is called when verifier exploring different branches of 8924 * execution popped from the state stack. If it sees an old state that has 8925 * more strict register state and more strict stack state then this execution 8926 * branch doesn't need to be explored further, since verifier already 8927 * concluded that more strict state leads to valid finish. 8928 * 8929 * Therefore two states are equivalent if register state is more conservative 8930 * and explored stack state is more conservative than the current one. 8931 * Example: 8932 * explored current 8933 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 8934 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 8935 * 8936 * In other words if current stack state (one being explored) has more 8937 * valid slots than old one that already passed validation, it means 8938 * the verifier can stop exploring and conclude that current state is valid too 8939 * 8940 * Similarly with registers. If explored state has register type as invalid 8941 * whereas register type in current state is meaningful, it means that 8942 * the current state will reach 'bpf_exit' instruction safely 8943 */ 8944 static bool func_states_equal(struct bpf_func_state *old, 8945 struct bpf_func_state *cur) 8946 { 8947 struct idpair *idmap; 8948 bool ret = false; 8949 int i; 8950 8951 idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); 8952 /* If we failed to allocate the idmap, just say it's not safe */ 8953 if (!idmap) 8954 return false; 8955 8956 for (i = 0; i < MAX_BPF_REG; i++) { 8957 if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) 8958 goto out_free; 8959 } 8960 8961 if (!stacksafe(old, cur, idmap)) 8962 goto out_free; 8963 8964 if (!refsafe(old, cur)) 8965 goto out_free; 8966 ret = true; 8967 out_free: 8968 kfree(idmap); 8969 return ret; 8970 } 8971 8972 static bool states_equal(struct bpf_verifier_env *env, 8973 struct bpf_verifier_state *old, 8974 struct bpf_verifier_state *cur) 8975 { 8976 int i; 8977 8978 if (old->curframe != cur->curframe) 8979 return false; 8980 8981 /* Verification state from speculative execution simulation 8982 * must never prune a non-speculative execution one. 8983 */ 8984 if (old->speculative && !cur->speculative) 8985 return false; 8986 8987 if (old->active_spin_lock != cur->active_spin_lock) 8988 return false; 8989 8990 /* for states to be equal callsites have to be the same 8991 * and all frame states need to be equivalent 8992 */ 8993 for (i = 0; i <= old->curframe; i++) { 8994 if (old->frame[i]->callsite != cur->frame[i]->callsite) 8995 return false; 8996 if (!func_states_equal(old->frame[i], cur->frame[i])) 8997 return false; 8998 } 8999 return true; 9000 } 9001 9002 /* Return 0 if no propagation happened. Return negative error code if error 9003 * happened. Otherwise, return the propagated bit. 9004 */ 9005 static int propagate_liveness_reg(struct bpf_verifier_env *env, 9006 struct bpf_reg_state *reg, 9007 struct bpf_reg_state *parent_reg) 9008 { 9009 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 9010 u8 flag = reg->live & REG_LIVE_READ; 9011 int err; 9012 9013 /* When comes here, read flags of PARENT_REG or REG could be any of 9014 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 9015 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 9016 */ 9017 if (parent_flag == REG_LIVE_READ64 || 9018 /* Or if there is no read flag from REG. */ 9019 !flag || 9020 /* Or if the read flag from REG is the same as PARENT_REG. */ 9021 parent_flag == flag) 9022 return 0; 9023 9024 err = mark_reg_read(env, reg, parent_reg, flag); 9025 if (err) 9026 return err; 9027 9028 return flag; 9029 } 9030 9031 /* A write screens off any subsequent reads; but write marks come from the 9032 * straight-line code between a state and its parent. When we arrive at an 9033 * equivalent state (jump target or such) we didn't arrive by the straight-line 9034 * code, so read marks in the state must propagate to the parent regardless 9035 * of the state's write marks. That's what 'parent == state->parent' comparison 9036 * in mark_reg_read() is for. 9037 */ 9038 static int propagate_liveness(struct bpf_verifier_env *env, 9039 const struct bpf_verifier_state *vstate, 9040 struct bpf_verifier_state *vparent) 9041 { 9042 struct bpf_reg_state *state_reg, *parent_reg; 9043 struct bpf_func_state *state, *parent; 9044 int i, frame, err = 0; 9045 9046 if (vparent->curframe != vstate->curframe) { 9047 WARN(1, "propagate_live: parent frame %d current frame %d\n", 9048 vparent->curframe, vstate->curframe); 9049 return -EFAULT; 9050 } 9051 /* Propagate read liveness of registers... */ 9052 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 9053 for (frame = 0; frame <= vstate->curframe; frame++) { 9054 parent = vparent->frame[frame]; 9055 state = vstate->frame[frame]; 9056 parent_reg = parent->regs; 9057 state_reg = state->regs; 9058 /* We don't need to worry about FP liveness, it's read-only */ 9059 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 9060 err = propagate_liveness_reg(env, &state_reg[i], 9061 &parent_reg[i]); 9062 if (err < 0) 9063 return err; 9064 if (err == REG_LIVE_READ64) 9065 mark_insn_zext(env, &parent_reg[i]); 9066 } 9067 9068 /* Propagate stack slots. */ 9069 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 9070 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 9071 parent_reg = &parent->stack[i].spilled_ptr; 9072 state_reg = &state->stack[i].spilled_ptr; 9073 err = propagate_liveness_reg(env, state_reg, 9074 parent_reg); 9075 if (err < 0) 9076 return err; 9077 } 9078 } 9079 return 0; 9080 } 9081 9082 /* find precise scalars in the previous equivalent state and 9083 * propagate them into the current state 9084 */ 9085 static int propagate_precision(struct bpf_verifier_env *env, 9086 const struct bpf_verifier_state *old) 9087 { 9088 struct bpf_reg_state *state_reg; 9089 struct bpf_func_state *state; 9090 int i, err = 0; 9091 9092 state = old->frame[old->curframe]; 9093 state_reg = state->regs; 9094 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 9095 if (state_reg->type != SCALAR_VALUE || 9096 !state_reg->precise) 9097 continue; 9098 if (env->log.level & BPF_LOG_LEVEL2) 9099 verbose(env, "propagating r%d\n", i); 9100 err = mark_chain_precision(env, i); 9101 if (err < 0) 9102 return err; 9103 } 9104 9105 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 9106 if (state->stack[i].slot_type[0] != STACK_SPILL) 9107 continue; 9108 state_reg = &state->stack[i].spilled_ptr; 9109 if (state_reg->type != SCALAR_VALUE || 9110 !state_reg->precise) 9111 continue; 9112 if (env->log.level & BPF_LOG_LEVEL2) 9113 verbose(env, "propagating fp%d\n", 9114 (-i - 1) * BPF_REG_SIZE); 9115 err = mark_chain_precision_stack(env, i); 9116 if (err < 0) 9117 return err; 9118 } 9119 return 0; 9120 } 9121 9122 static bool states_maybe_looping(struct bpf_verifier_state *old, 9123 struct bpf_verifier_state *cur) 9124 { 9125 struct bpf_func_state *fold, *fcur; 9126 int i, fr = cur->curframe; 9127 9128 if (old->curframe != fr) 9129 return false; 9130 9131 fold = old->frame[fr]; 9132 fcur = cur->frame[fr]; 9133 for (i = 0; i < MAX_BPF_REG; i++) 9134 if (memcmp(&fold->regs[i], &fcur->regs[i], 9135 offsetof(struct bpf_reg_state, parent))) 9136 return false; 9137 return true; 9138 } 9139 9140 9141 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 9142 { 9143 struct bpf_verifier_state_list *new_sl; 9144 struct bpf_verifier_state_list *sl, **pprev; 9145 struct bpf_verifier_state *cur = env->cur_state, *new; 9146 int i, j, err, states_cnt = 0; 9147 bool add_new_state = env->test_state_freq ? true : false; 9148 9149 cur->last_insn_idx = env->prev_insn_idx; 9150 if (!env->insn_aux_data[insn_idx].prune_point) 9151 /* this 'insn_idx' instruction wasn't marked, so we will not 9152 * be doing state search here 9153 */ 9154 return 0; 9155 9156 /* bpf progs typically have pruning point every 4 instructions 9157 * http://vger.kernel.org/bpfconf2019.html#session-1 9158 * Do not add new state for future pruning if the verifier hasn't seen 9159 * at least 2 jumps and at least 8 instructions. 9160 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 9161 * In tests that amounts to up to 50% reduction into total verifier 9162 * memory consumption and 20% verifier time speedup. 9163 */ 9164 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 9165 env->insn_processed - env->prev_insn_processed >= 8) 9166 add_new_state = true; 9167 9168 pprev = explored_state(env, insn_idx); 9169 sl = *pprev; 9170 9171 clean_live_states(env, insn_idx, cur); 9172 9173 while (sl) { 9174 states_cnt++; 9175 if (sl->state.insn_idx != insn_idx) 9176 goto next; 9177 if (sl->state.branches) { 9178 if (states_maybe_looping(&sl->state, cur) && 9179 states_equal(env, &sl->state, cur)) { 9180 verbose_linfo(env, insn_idx, "; "); 9181 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 9182 return -EINVAL; 9183 } 9184 /* if the verifier is processing a loop, avoid adding new state 9185 * too often, since different loop iterations have distinct 9186 * states and may not help future pruning. 9187 * This threshold shouldn't be too low to make sure that 9188 * a loop with large bound will be rejected quickly. 9189 * The most abusive loop will be: 9190 * r1 += 1 9191 * if r1 < 1000000 goto pc-2 9192 * 1M insn_procssed limit / 100 == 10k peak states. 9193 * This threshold shouldn't be too high either, since states 9194 * at the end of the loop are likely to be useful in pruning. 9195 */ 9196 if (env->jmps_processed - env->prev_jmps_processed < 20 && 9197 env->insn_processed - env->prev_insn_processed < 100) 9198 add_new_state = false; 9199 goto miss; 9200 } 9201 if (states_equal(env, &sl->state, cur)) { 9202 sl->hit_cnt++; 9203 /* reached equivalent register/stack state, 9204 * prune the search. 9205 * Registers read by the continuation are read by us. 9206 * If we have any write marks in env->cur_state, they 9207 * will prevent corresponding reads in the continuation 9208 * from reaching our parent (an explored_state). Our 9209 * own state will get the read marks recorded, but 9210 * they'll be immediately forgotten as we're pruning 9211 * this state and will pop a new one. 9212 */ 9213 err = propagate_liveness(env, &sl->state, cur); 9214 9215 /* if previous state reached the exit with precision and 9216 * current state is equivalent to it (except precsion marks) 9217 * the precision needs to be propagated back in 9218 * the current state. 9219 */ 9220 err = err ? : push_jmp_history(env, cur); 9221 err = err ? : propagate_precision(env, &sl->state); 9222 if (err) 9223 return err; 9224 return 1; 9225 } 9226 miss: 9227 /* when new state is not going to be added do not increase miss count. 9228 * Otherwise several loop iterations will remove the state 9229 * recorded earlier. The goal of these heuristics is to have 9230 * states from some iterations of the loop (some in the beginning 9231 * and some at the end) to help pruning. 9232 */ 9233 if (add_new_state) 9234 sl->miss_cnt++; 9235 /* heuristic to determine whether this state is beneficial 9236 * to keep checking from state equivalence point of view. 9237 * Higher numbers increase max_states_per_insn and verification time, 9238 * but do not meaningfully decrease insn_processed. 9239 */ 9240 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 9241 /* the state is unlikely to be useful. Remove it to 9242 * speed up verification 9243 */ 9244 *pprev = sl->next; 9245 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 9246 u32 br = sl->state.branches; 9247 9248 WARN_ONCE(br, 9249 "BUG live_done but branches_to_explore %d\n", 9250 br); 9251 free_verifier_state(&sl->state, false); 9252 kfree(sl); 9253 env->peak_states--; 9254 } else { 9255 /* cannot free this state, since parentage chain may 9256 * walk it later. Add it for free_list instead to 9257 * be freed at the end of verification 9258 */ 9259 sl->next = env->free_list; 9260 env->free_list = sl; 9261 } 9262 sl = *pprev; 9263 continue; 9264 } 9265 next: 9266 pprev = &sl->next; 9267 sl = *pprev; 9268 } 9269 9270 if (env->max_states_per_insn < states_cnt) 9271 env->max_states_per_insn = states_cnt; 9272 9273 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 9274 return push_jmp_history(env, cur); 9275 9276 if (!add_new_state) 9277 return push_jmp_history(env, cur); 9278 9279 /* There were no equivalent states, remember the current one. 9280 * Technically the current state is not proven to be safe yet, 9281 * but it will either reach outer most bpf_exit (which means it's safe) 9282 * or it will be rejected. When there are no loops the verifier won't be 9283 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 9284 * again on the way to bpf_exit. 9285 * When looping the sl->state.branches will be > 0 and this state 9286 * will not be considered for equivalence until branches == 0. 9287 */ 9288 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 9289 if (!new_sl) 9290 return -ENOMEM; 9291 env->total_states++; 9292 env->peak_states++; 9293 env->prev_jmps_processed = env->jmps_processed; 9294 env->prev_insn_processed = env->insn_processed; 9295 9296 /* add new state to the head of linked list */ 9297 new = &new_sl->state; 9298 err = copy_verifier_state(new, cur); 9299 if (err) { 9300 free_verifier_state(new, false); 9301 kfree(new_sl); 9302 return err; 9303 } 9304 new->insn_idx = insn_idx; 9305 WARN_ONCE(new->branches != 1, 9306 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 9307 9308 cur->parent = new; 9309 cur->first_insn_idx = insn_idx; 9310 clear_jmp_history(cur); 9311 new_sl->next = *explored_state(env, insn_idx); 9312 *explored_state(env, insn_idx) = new_sl; 9313 /* connect new state to parentage chain. Current frame needs all 9314 * registers connected. Only r6 - r9 of the callers are alive (pushed 9315 * to the stack implicitly by JITs) so in callers' frames connect just 9316 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 9317 * the state of the call instruction (with WRITTEN set), and r0 comes 9318 * from callee with its full parentage chain, anyway. 9319 */ 9320 /* clear write marks in current state: the writes we did are not writes 9321 * our child did, so they don't screen off its reads from us. 9322 * (There are no read marks in current state, because reads always mark 9323 * their parent and current state never has children yet. Only 9324 * explored_states can get read marks.) 9325 */ 9326 for (j = 0; j <= cur->curframe; j++) { 9327 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 9328 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 9329 for (i = 0; i < BPF_REG_FP; i++) 9330 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 9331 } 9332 9333 /* all stack frames are accessible from callee, clear them all */ 9334 for (j = 0; j <= cur->curframe; j++) { 9335 struct bpf_func_state *frame = cur->frame[j]; 9336 struct bpf_func_state *newframe = new->frame[j]; 9337 9338 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 9339 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 9340 frame->stack[i].spilled_ptr.parent = 9341 &newframe->stack[i].spilled_ptr; 9342 } 9343 } 9344 return 0; 9345 } 9346 9347 /* Return true if it's OK to have the same insn return a different type. */ 9348 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 9349 { 9350 switch (type) { 9351 case PTR_TO_CTX: 9352 case PTR_TO_SOCKET: 9353 case PTR_TO_SOCKET_OR_NULL: 9354 case PTR_TO_SOCK_COMMON: 9355 case PTR_TO_SOCK_COMMON_OR_NULL: 9356 case PTR_TO_TCP_SOCK: 9357 case PTR_TO_TCP_SOCK_OR_NULL: 9358 case PTR_TO_XDP_SOCK: 9359 case PTR_TO_BTF_ID: 9360 case PTR_TO_BTF_ID_OR_NULL: 9361 return false; 9362 default: 9363 return true; 9364 } 9365 } 9366 9367 /* If an instruction was previously used with particular pointer types, then we 9368 * need to be careful to avoid cases such as the below, where it may be ok 9369 * for one branch accessing the pointer, but not ok for the other branch: 9370 * 9371 * R1 = sock_ptr 9372 * goto X; 9373 * ... 9374 * R1 = some_other_valid_ptr; 9375 * goto X; 9376 * ... 9377 * R2 = *(u32 *)(R1 + 0); 9378 */ 9379 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 9380 { 9381 return src != prev && (!reg_type_mismatch_ok(src) || 9382 !reg_type_mismatch_ok(prev)); 9383 } 9384 9385 static int do_check(struct bpf_verifier_env *env) 9386 { 9387 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 9388 struct bpf_verifier_state *state = env->cur_state; 9389 struct bpf_insn *insns = env->prog->insnsi; 9390 struct bpf_reg_state *regs; 9391 int insn_cnt = env->prog->len; 9392 bool do_print_state = false; 9393 int prev_insn_idx = -1; 9394 9395 for (;;) { 9396 struct bpf_insn *insn; 9397 u8 class; 9398 int err; 9399 9400 env->prev_insn_idx = prev_insn_idx; 9401 if (env->insn_idx >= insn_cnt) { 9402 verbose(env, "invalid insn idx %d insn_cnt %d\n", 9403 env->insn_idx, insn_cnt); 9404 return -EFAULT; 9405 } 9406 9407 insn = &insns[env->insn_idx]; 9408 class = BPF_CLASS(insn->code); 9409 9410 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 9411 verbose(env, 9412 "BPF program is too large. Processed %d insn\n", 9413 env->insn_processed); 9414 return -E2BIG; 9415 } 9416 9417 err = is_state_visited(env, env->insn_idx); 9418 if (err < 0) 9419 return err; 9420 if (err == 1) { 9421 /* found equivalent state, can prune the search */ 9422 if (env->log.level & BPF_LOG_LEVEL) { 9423 if (do_print_state) 9424 verbose(env, "\nfrom %d to %d%s: safe\n", 9425 env->prev_insn_idx, env->insn_idx, 9426 env->cur_state->speculative ? 9427 " (speculative execution)" : ""); 9428 else 9429 verbose(env, "%d: safe\n", env->insn_idx); 9430 } 9431 goto process_bpf_exit; 9432 } 9433 9434 if (signal_pending(current)) 9435 return -EAGAIN; 9436 9437 if (need_resched()) 9438 cond_resched(); 9439 9440 if (env->log.level & BPF_LOG_LEVEL2 || 9441 (env->log.level & BPF_LOG_LEVEL && do_print_state)) { 9442 if (env->log.level & BPF_LOG_LEVEL2) 9443 verbose(env, "%d:", env->insn_idx); 9444 else 9445 verbose(env, "\nfrom %d to %d%s:", 9446 env->prev_insn_idx, env->insn_idx, 9447 env->cur_state->speculative ? 9448 " (speculative execution)" : ""); 9449 print_verifier_state(env, state->frame[state->curframe]); 9450 do_print_state = false; 9451 } 9452 9453 if (env->log.level & BPF_LOG_LEVEL) { 9454 const struct bpf_insn_cbs cbs = { 9455 .cb_print = verbose, 9456 .private_data = env, 9457 }; 9458 9459 verbose_linfo(env, env->insn_idx, "; "); 9460 verbose(env, "%d: ", env->insn_idx); 9461 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 9462 } 9463 9464 if (bpf_prog_is_dev_bound(env->prog->aux)) { 9465 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 9466 env->prev_insn_idx); 9467 if (err) 9468 return err; 9469 } 9470 9471 regs = cur_regs(env); 9472 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 9473 prev_insn_idx = env->insn_idx; 9474 9475 if (class == BPF_ALU || class == BPF_ALU64) { 9476 err = check_alu_op(env, insn); 9477 if (err) 9478 return err; 9479 9480 } else if (class == BPF_LDX) { 9481 enum bpf_reg_type *prev_src_type, src_reg_type; 9482 9483 /* check for reserved fields is already done */ 9484 9485 /* check src operand */ 9486 err = check_reg_arg(env, insn->src_reg, SRC_OP); 9487 if (err) 9488 return err; 9489 9490 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 9491 if (err) 9492 return err; 9493 9494 src_reg_type = regs[insn->src_reg].type; 9495 9496 /* check that memory (src_reg + off) is readable, 9497 * the state of dst_reg will be updated by this func 9498 */ 9499 err = check_mem_access(env, env->insn_idx, insn->src_reg, 9500 insn->off, BPF_SIZE(insn->code), 9501 BPF_READ, insn->dst_reg, false); 9502 if (err) 9503 return err; 9504 9505 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 9506 9507 if (*prev_src_type == NOT_INIT) { 9508 /* saw a valid insn 9509 * dst_reg = *(u32 *)(src_reg + off) 9510 * save type to validate intersecting paths 9511 */ 9512 *prev_src_type = src_reg_type; 9513 9514 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 9515 /* ABuser program is trying to use the same insn 9516 * dst_reg = *(u32*) (src_reg + off) 9517 * with different pointer types: 9518 * src_reg == ctx in one branch and 9519 * src_reg == stack|map in some other branch. 9520 * Reject it. 9521 */ 9522 verbose(env, "same insn cannot be used with different pointers\n"); 9523 return -EINVAL; 9524 } 9525 9526 } else if (class == BPF_STX) { 9527 enum bpf_reg_type *prev_dst_type, dst_reg_type; 9528 9529 if (BPF_MODE(insn->code) == BPF_XADD) { 9530 err = check_xadd(env, env->insn_idx, insn); 9531 if (err) 9532 return err; 9533 env->insn_idx++; 9534 continue; 9535 } 9536 9537 /* check src1 operand */ 9538 err = check_reg_arg(env, insn->src_reg, SRC_OP); 9539 if (err) 9540 return err; 9541 /* check src2 operand */ 9542 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 9543 if (err) 9544 return err; 9545 9546 dst_reg_type = regs[insn->dst_reg].type; 9547 9548 /* check that memory (dst_reg + off) is writeable */ 9549 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 9550 insn->off, BPF_SIZE(insn->code), 9551 BPF_WRITE, insn->src_reg, false); 9552 if (err) 9553 return err; 9554 9555 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 9556 9557 if (*prev_dst_type == NOT_INIT) { 9558 *prev_dst_type = dst_reg_type; 9559 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 9560 verbose(env, "same insn cannot be used with different pointers\n"); 9561 return -EINVAL; 9562 } 9563 9564 } else if (class == BPF_ST) { 9565 if (BPF_MODE(insn->code) != BPF_MEM || 9566 insn->src_reg != BPF_REG_0) { 9567 verbose(env, "BPF_ST uses reserved fields\n"); 9568 return -EINVAL; 9569 } 9570 /* check src operand */ 9571 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 9572 if (err) 9573 return err; 9574 9575 if (is_ctx_reg(env, insn->dst_reg)) { 9576 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 9577 insn->dst_reg, 9578 reg_type_str[reg_state(env, insn->dst_reg)->type]); 9579 return -EACCES; 9580 } 9581 9582 /* check that memory (dst_reg + off) is writeable */ 9583 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 9584 insn->off, BPF_SIZE(insn->code), 9585 BPF_WRITE, -1, false); 9586 if (err) 9587 return err; 9588 9589 } else if (class == BPF_JMP || class == BPF_JMP32) { 9590 u8 opcode = BPF_OP(insn->code); 9591 9592 env->jmps_processed++; 9593 if (opcode == BPF_CALL) { 9594 if (BPF_SRC(insn->code) != BPF_K || 9595 insn->off != 0 || 9596 (insn->src_reg != BPF_REG_0 && 9597 insn->src_reg != BPF_PSEUDO_CALL) || 9598 insn->dst_reg != BPF_REG_0 || 9599 class == BPF_JMP32) { 9600 verbose(env, "BPF_CALL uses reserved fields\n"); 9601 return -EINVAL; 9602 } 9603 9604 if (env->cur_state->active_spin_lock && 9605 (insn->src_reg == BPF_PSEUDO_CALL || 9606 insn->imm != BPF_FUNC_spin_unlock)) { 9607 verbose(env, "function calls are not allowed while holding a lock\n"); 9608 return -EINVAL; 9609 } 9610 if (insn->src_reg == BPF_PSEUDO_CALL) 9611 err = check_func_call(env, insn, &env->insn_idx); 9612 else 9613 err = check_helper_call(env, insn->imm, env->insn_idx); 9614 if (err) 9615 return err; 9616 9617 } else if (opcode == BPF_JA) { 9618 if (BPF_SRC(insn->code) != BPF_K || 9619 insn->imm != 0 || 9620 insn->src_reg != BPF_REG_0 || 9621 insn->dst_reg != BPF_REG_0 || 9622 class == BPF_JMP32) { 9623 verbose(env, "BPF_JA uses reserved fields\n"); 9624 return -EINVAL; 9625 } 9626 9627 env->insn_idx += insn->off + 1; 9628 continue; 9629 9630 } else if (opcode == BPF_EXIT) { 9631 if (BPF_SRC(insn->code) != BPF_K || 9632 insn->imm != 0 || 9633 insn->src_reg != BPF_REG_0 || 9634 insn->dst_reg != BPF_REG_0 || 9635 class == BPF_JMP32) { 9636 verbose(env, "BPF_EXIT uses reserved fields\n"); 9637 return -EINVAL; 9638 } 9639 9640 if (env->cur_state->active_spin_lock) { 9641 verbose(env, "bpf_spin_unlock is missing\n"); 9642 return -EINVAL; 9643 } 9644 9645 if (state->curframe) { 9646 /* exit from nested function */ 9647 err = prepare_func_exit(env, &env->insn_idx); 9648 if (err) 9649 return err; 9650 do_print_state = true; 9651 continue; 9652 } 9653 9654 err = check_reference_leak(env); 9655 if (err) 9656 return err; 9657 9658 err = check_return_code(env); 9659 if (err) 9660 return err; 9661 process_bpf_exit: 9662 update_branch_counts(env, env->cur_state); 9663 err = pop_stack(env, &prev_insn_idx, 9664 &env->insn_idx, pop_log); 9665 if (err < 0) { 9666 if (err != -ENOENT) 9667 return err; 9668 break; 9669 } else { 9670 do_print_state = true; 9671 continue; 9672 } 9673 } else { 9674 err = check_cond_jmp_op(env, insn, &env->insn_idx); 9675 if (err) 9676 return err; 9677 } 9678 } else if (class == BPF_LD) { 9679 u8 mode = BPF_MODE(insn->code); 9680 9681 if (mode == BPF_ABS || mode == BPF_IND) { 9682 err = check_ld_abs(env, insn); 9683 if (err) 9684 return err; 9685 9686 } else if (mode == BPF_IMM) { 9687 err = check_ld_imm(env, insn); 9688 if (err) 9689 return err; 9690 9691 env->insn_idx++; 9692 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 9693 } else { 9694 verbose(env, "invalid BPF_LD mode\n"); 9695 return -EINVAL; 9696 } 9697 } else { 9698 verbose(env, "unknown insn class %d\n", class); 9699 return -EINVAL; 9700 } 9701 9702 env->insn_idx++; 9703 } 9704 9705 return 0; 9706 } 9707 9708 /* replace pseudo btf_id with kernel symbol address */ 9709 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 9710 struct bpf_insn *insn, 9711 struct bpf_insn_aux_data *aux) 9712 { 9713 const struct btf_var_secinfo *vsi; 9714 const struct btf_type *datasec; 9715 const struct btf_type *t; 9716 const char *sym_name; 9717 bool percpu = false; 9718 u32 type, id = insn->imm; 9719 s32 datasec_id; 9720 u64 addr; 9721 int i; 9722 9723 if (!btf_vmlinux) { 9724 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 9725 return -EINVAL; 9726 } 9727 9728 if (insn[1].imm != 0) { 9729 verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n"); 9730 return -EINVAL; 9731 } 9732 9733 t = btf_type_by_id(btf_vmlinux, id); 9734 if (!t) { 9735 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 9736 return -ENOENT; 9737 } 9738 9739 if (!btf_type_is_var(t)) { 9740 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", 9741 id); 9742 return -EINVAL; 9743 } 9744 9745 sym_name = btf_name_by_offset(btf_vmlinux, t->name_off); 9746 addr = kallsyms_lookup_name(sym_name); 9747 if (!addr) { 9748 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 9749 sym_name); 9750 return -ENOENT; 9751 } 9752 9753 datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu", 9754 BTF_KIND_DATASEC); 9755 if (datasec_id > 0) { 9756 datasec = btf_type_by_id(btf_vmlinux, datasec_id); 9757 for_each_vsi(i, datasec, vsi) { 9758 if (vsi->type == id) { 9759 percpu = true; 9760 break; 9761 } 9762 } 9763 } 9764 9765 insn[0].imm = (u32)addr; 9766 insn[1].imm = addr >> 32; 9767 9768 type = t->type; 9769 t = btf_type_skip_modifiers(btf_vmlinux, type, NULL); 9770 if (percpu) { 9771 aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; 9772 aux->btf_var.btf = btf_vmlinux; 9773 aux->btf_var.btf_id = type; 9774 } else if (!btf_type_is_struct(t)) { 9775 const struct btf_type *ret; 9776 const char *tname; 9777 u32 tsize; 9778 9779 /* resolve the type size of ksym. */ 9780 ret = btf_resolve_size(btf_vmlinux, t, &tsize); 9781 if (IS_ERR(ret)) { 9782 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 9783 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 9784 tname, PTR_ERR(ret)); 9785 return -EINVAL; 9786 } 9787 aux->btf_var.reg_type = PTR_TO_MEM; 9788 aux->btf_var.mem_size = tsize; 9789 } else { 9790 aux->btf_var.reg_type = PTR_TO_BTF_ID; 9791 aux->btf_var.btf = btf_vmlinux; 9792 aux->btf_var.btf_id = type; 9793 } 9794 return 0; 9795 } 9796 9797 static int check_map_prealloc(struct bpf_map *map) 9798 { 9799 return (map->map_type != BPF_MAP_TYPE_HASH && 9800 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 9801 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || 9802 !(map->map_flags & BPF_F_NO_PREALLOC); 9803 } 9804 9805 static bool is_tracing_prog_type(enum bpf_prog_type type) 9806 { 9807 switch (type) { 9808 case BPF_PROG_TYPE_KPROBE: 9809 case BPF_PROG_TYPE_TRACEPOINT: 9810 case BPF_PROG_TYPE_PERF_EVENT: 9811 case BPF_PROG_TYPE_RAW_TRACEPOINT: 9812 return true; 9813 default: 9814 return false; 9815 } 9816 } 9817 9818 static bool is_preallocated_map(struct bpf_map *map) 9819 { 9820 if (!check_map_prealloc(map)) 9821 return false; 9822 if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) 9823 return false; 9824 return true; 9825 } 9826 9827 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 9828 struct bpf_map *map, 9829 struct bpf_prog *prog) 9830 9831 { 9832 enum bpf_prog_type prog_type = resolve_prog_type(prog); 9833 /* 9834 * Validate that trace type programs use preallocated hash maps. 9835 * 9836 * For programs attached to PERF events this is mandatory as the 9837 * perf NMI can hit any arbitrary code sequence. 9838 * 9839 * All other trace types using preallocated hash maps are unsafe as 9840 * well because tracepoint or kprobes can be inside locked regions 9841 * of the memory allocator or at a place where a recursion into the 9842 * memory allocator would see inconsistent state. 9843 * 9844 * On RT enabled kernels run-time allocation of all trace type 9845 * programs is strictly prohibited due to lock type constraints. On 9846 * !RT kernels it is allowed for backwards compatibility reasons for 9847 * now, but warnings are emitted so developers are made aware of 9848 * the unsafety and can fix their programs before this is enforced. 9849 */ 9850 if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { 9851 if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { 9852 verbose(env, "perf_event programs can only use preallocated hash map\n"); 9853 return -EINVAL; 9854 } 9855 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 9856 verbose(env, "trace type programs can only use preallocated hash map\n"); 9857 return -EINVAL; 9858 } 9859 WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); 9860 verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); 9861 } 9862 9863 if (map_value_has_spin_lock(map)) { 9864 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 9865 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 9866 return -EINVAL; 9867 } 9868 9869 if (is_tracing_prog_type(prog_type)) { 9870 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 9871 return -EINVAL; 9872 } 9873 9874 if (prog->aux->sleepable) { 9875 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 9876 return -EINVAL; 9877 } 9878 } 9879 9880 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 9881 !bpf_offload_prog_map_match(prog, map)) { 9882 verbose(env, "offload device mismatch between prog and map\n"); 9883 return -EINVAL; 9884 } 9885 9886 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 9887 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 9888 return -EINVAL; 9889 } 9890 9891 if (prog->aux->sleepable) 9892 switch (map->map_type) { 9893 case BPF_MAP_TYPE_HASH: 9894 case BPF_MAP_TYPE_LRU_HASH: 9895 case BPF_MAP_TYPE_ARRAY: 9896 if (!is_preallocated_map(map)) { 9897 verbose(env, 9898 "Sleepable programs can only use preallocated hash maps\n"); 9899 return -EINVAL; 9900 } 9901 break; 9902 default: 9903 verbose(env, 9904 "Sleepable programs can only use array and hash maps\n"); 9905 return -EINVAL; 9906 } 9907 9908 return 0; 9909 } 9910 9911 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 9912 { 9913 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 9914 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 9915 } 9916 9917 /* find and rewrite pseudo imm in ld_imm64 instructions: 9918 * 9919 * 1. if it accesses map FD, replace it with actual map pointer. 9920 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 9921 * 9922 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 9923 */ 9924 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 9925 { 9926 struct bpf_insn *insn = env->prog->insnsi; 9927 int insn_cnt = env->prog->len; 9928 int i, j, err; 9929 9930 err = bpf_prog_calc_tag(env->prog); 9931 if (err) 9932 return err; 9933 9934 for (i = 0; i < insn_cnt; i++, insn++) { 9935 if (BPF_CLASS(insn->code) == BPF_LDX && 9936 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 9937 verbose(env, "BPF_LDX uses reserved fields\n"); 9938 return -EINVAL; 9939 } 9940 9941 if (BPF_CLASS(insn->code) == BPF_STX && 9942 ((BPF_MODE(insn->code) != BPF_MEM && 9943 BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { 9944 verbose(env, "BPF_STX uses reserved fields\n"); 9945 return -EINVAL; 9946 } 9947 9948 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 9949 struct bpf_insn_aux_data *aux; 9950 struct bpf_map *map; 9951 struct fd f; 9952 u64 addr; 9953 9954 if (i == insn_cnt - 1 || insn[1].code != 0 || 9955 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 9956 insn[1].off != 0) { 9957 verbose(env, "invalid bpf_ld_imm64 insn\n"); 9958 return -EINVAL; 9959 } 9960 9961 if (insn[0].src_reg == 0) 9962 /* valid generic load 64-bit imm */ 9963 goto next_insn; 9964 9965 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 9966 aux = &env->insn_aux_data[i]; 9967 err = check_pseudo_btf_id(env, insn, aux); 9968 if (err) 9969 return err; 9970 goto next_insn; 9971 } 9972 9973 /* In final convert_pseudo_ld_imm64() step, this is 9974 * converted into regular 64-bit imm load insn. 9975 */ 9976 if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && 9977 insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || 9978 (insn[0].src_reg == BPF_PSEUDO_MAP_FD && 9979 insn[1].imm != 0)) { 9980 verbose(env, 9981 "unrecognized bpf_ld_imm64 insn\n"); 9982 return -EINVAL; 9983 } 9984 9985 f = fdget(insn[0].imm); 9986 map = __bpf_map_get(f); 9987 if (IS_ERR(map)) { 9988 verbose(env, "fd %d is not pointing to valid bpf_map\n", 9989 insn[0].imm); 9990 return PTR_ERR(map); 9991 } 9992 9993 err = check_map_prog_compatibility(env, map, env->prog); 9994 if (err) { 9995 fdput(f); 9996 return err; 9997 } 9998 9999 aux = &env->insn_aux_data[i]; 10000 if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 10001 addr = (unsigned long)map; 10002 } else { 10003 u32 off = insn[1].imm; 10004 10005 if (off >= BPF_MAX_VAR_OFF) { 10006 verbose(env, "direct value offset of %u is not allowed\n", off); 10007 fdput(f); 10008 return -EINVAL; 10009 } 10010 10011 if (!map->ops->map_direct_value_addr) { 10012 verbose(env, "no direct value access support for this map type\n"); 10013 fdput(f); 10014 return -EINVAL; 10015 } 10016 10017 err = map->ops->map_direct_value_addr(map, &addr, off); 10018 if (err) { 10019 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 10020 map->value_size, off); 10021 fdput(f); 10022 return err; 10023 } 10024 10025 aux->map_off = off; 10026 addr += off; 10027 } 10028 10029 insn[0].imm = (u32)addr; 10030 insn[1].imm = addr >> 32; 10031 10032 /* check whether we recorded this map already */ 10033 for (j = 0; j < env->used_map_cnt; j++) { 10034 if (env->used_maps[j] == map) { 10035 aux->map_index = j; 10036 fdput(f); 10037 goto next_insn; 10038 } 10039 } 10040 10041 if (env->used_map_cnt >= MAX_USED_MAPS) { 10042 fdput(f); 10043 return -E2BIG; 10044 } 10045 10046 /* hold the map. If the program is rejected by verifier, 10047 * the map will be released by release_maps() or it 10048 * will be used by the valid program until it's unloaded 10049 * and all maps are released in free_used_maps() 10050 */ 10051 bpf_map_inc(map); 10052 10053 aux->map_index = env->used_map_cnt; 10054 env->used_maps[env->used_map_cnt++] = map; 10055 10056 if (bpf_map_is_cgroup_storage(map) && 10057 bpf_cgroup_storage_assign(env->prog->aux, map)) { 10058 verbose(env, "only one cgroup storage of each type is allowed\n"); 10059 fdput(f); 10060 return -EBUSY; 10061 } 10062 10063 fdput(f); 10064 next_insn: 10065 insn++; 10066 i++; 10067 continue; 10068 } 10069 10070 /* Basic sanity check before we invest more work here. */ 10071 if (!bpf_opcode_in_insntable(insn->code)) { 10072 verbose(env, "unknown opcode %02x\n", insn->code); 10073 return -EINVAL; 10074 } 10075 } 10076 10077 /* now all pseudo BPF_LD_IMM64 instructions load valid 10078 * 'struct bpf_map *' into a register instead of user map_fd. 10079 * These pointers will be used later by verifier to validate map access. 10080 */ 10081 return 0; 10082 } 10083 10084 /* drop refcnt of maps used by the rejected program */ 10085 static void release_maps(struct bpf_verifier_env *env) 10086 { 10087 __bpf_free_used_maps(env->prog->aux, env->used_maps, 10088 env->used_map_cnt); 10089 } 10090 10091 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 10092 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 10093 { 10094 struct bpf_insn *insn = env->prog->insnsi; 10095 int insn_cnt = env->prog->len; 10096 int i; 10097 10098 for (i = 0; i < insn_cnt; i++, insn++) 10099 if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) 10100 insn->src_reg = 0; 10101 } 10102 10103 /* single env->prog->insni[off] instruction was replaced with the range 10104 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 10105 * [0, off) and [off, end) to new locations, so the patched range stays zero 10106 */ 10107 static int adjust_insn_aux_data(struct bpf_verifier_env *env, 10108 struct bpf_prog *new_prog, u32 off, u32 cnt) 10109 { 10110 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; 10111 struct bpf_insn *insn = new_prog->insnsi; 10112 u32 prog_len; 10113 int i; 10114 10115 /* aux info at OFF always needs adjustment, no matter fast path 10116 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 10117 * original insn at old prog. 10118 */ 10119 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 10120 10121 if (cnt == 1) 10122 return 0; 10123 prog_len = new_prog->len; 10124 new_data = vzalloc(array_size(prog_len, 10125 sizeof(struct bpf_insn_aux_data))); 10126 if (!new_data) 10127 return -ENOMEM; 10128 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 10129 memcpy(new_data + off + cnt - 1, old_data + off, 10130 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 10131 for (i = off; i < off + cnt - 1; i++) { 10132 new_data[i].seen = env->pass_cnt; 10133 new_data[i].zext_dst = insn_has_def32(env, insn + i); 10134 } 10135 env->insn_aux_data = new_data; 10136 vfree(old_data); 10137 return 0; 10138 } 10139 10140 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 10141 { 10142 int i; 10143 10144 if (len == 1) 10145 return; 10146 /* NOTE: fake 'exit' subprog should be updated as well. */ 10147 for (i = 0; i <= env->subprog_cnt; i++) { 10148 if (env->subprog_info[i].start <= off) 10149 continue; 10150 env->subprog_info[i].start += len - 1; 10151 } 10152 } 10153 10154 static void adjust_poke_descs(struct bpf_prog *prog, u32 len) 10155 { 10156 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 10157 int i, sz = prog->aux->size_poke_tab; 10158 struct bpf_jit_poke_descriptor *desc; 10159 10160 for (i = 0; i < sz; i++) { 10161 desc = &tab[i]; 10162 desc->insn_idx += len - 1; 10163 } 10164 } 10165 10166 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 10167 const struct bpf_insn *patch, u32 len) 10168 { 10169 struct bpf_prog *new_prog; 10170 10171 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 10172 if (IS_ERR(new_prog)) { 10173 if (PTR_ERR(new_prog) == -ERANGE) 10174 verbose(env, 10175 "insn %d cannot be patched due to 16-bit range\n", 10176 env->insn_aux_data[off].orig_idx); 10177 return NULL; 10178 } 10179 if (adjust_insn_aux_data(env, new_prog, off, len)) 10180 return NULL; 10181 adjust_subprog_starts(env, off, len); 10182 adjust_poke_descs(new_prog, len); 10183 return new_prog; 10184 } 10185 10186 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 10187 u32 off, u32 cnt) 10188 { 10189 int i, j; 10190 10191 /* find first prog starting at or after off (first to remove) */ 10192 for (i = 0; i < env->subprog_cnt; i++) 10193 if (env->subprog_info[i].start >= off) 10194 break; 10195 /* find first prog starting at or after off + cnt (first to stay) */ 10196 for (j = i; j < env->subprog_cnt; j++) 10197 if (env->subprog_info[j].start >= off + cnt) 10198 break; 10199 /* if j doesn't start exactly at off + cnt, we are just removing 10200 * the front of previous prog 10201 */ 10202 if (env->subprog_info[j].start != off + cnt) 10203 j--; 10204 10205 if (j > i) { 10206 struct bpf_prog_aux *aux = env->prog->aux; 10207 int move; 10208 10209 /* move fake 'exit' subprog as well */ 10210 move = env->subprog_cnt + 1 - j; 10211 10212 memmove(env->subprog_info + i, 10213 env->subprog_info + j, 10214 sizeof(*env->subprog_info) * move); 10215 env->subprog_cnt -= j - i; 10216 10217 /* remove func_info */ 10218 if (aux->func_info) { 10219 move = aux->func_info_cnt - j; 10220 10221 memmove(aux->func_info + i, 10222 aux->func_info + j, 10223 sizeof(*aux->func_info) * move); 10224 aux->func_info_cnt -= j - i; 10225 /* func_info->insn_off is set after all code rewrites, 10226 * in adjust_btf_func() - no need to adjust 10227 */ 10228 } 10229 } else { 10230 /* convert i from "first prog to remove" to "first to adjust" */ 10231 if (env->subprog_info[i].start == off) 10232 i++; 10233 } 10234 10235 /* update fake 'exit' subprog as well */ 10236 for (; i <= env->subprog_cnt; i++) 10237 env->subprog_info[i].start -= cnt; 10238 10239 return 0; 10240 } 10241 10242 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 10243 u32 cnt) 10244 { 10245 struct bpf_prog *prog = env->prog; 10246 u32 i, l_off, l_cnt, nr_linfo; 10247 struct bpf_line_info *linfo; 10248 10249 nr_linfo = prog->aux->nr_linfo; 10250 if (!nr_linfo) 10251 return 0; 10252 10253 linfo = prog->aux->linfo; 10254 10255 /* find first line info to remove, count lines to be removed */ 10256 for (i = 0; i < nr_linfo; i++) 10257 if (linfo[i].insn_off >= off) 10258 break; 10259 10260 l_off = i; 10261 l_cnt = 0; 10262 for (; i < nr_linfo; i++) 10263 if (linfo[i].insn_off < off + cnt) 10264 l_cnt++; 10265 else 10266 break; 10267 10268 /* First live insn doesn't match first live linfo, it needs to "inherit" 10269 * last removed linfo. prog is already modified, so prog->len == off 10270 * means no live instructions after (tail of the program was removed). 10271 */ 10272 if (prog->len != off && l_cnt && 10273 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 10274 l_cnt--; 10275 linfo[--i].insn_off = off + cnt; 10276 } 10277 10278 /* remove the line info which refer to the removed instructions */ 10279 if (l_cnt) { 10280 memmove(linfo + l_off, linfo + i, 10281 sizeof(*linfo) * (nr_linfo - i)); 10282 10283 prog->aux->nr_linfo -= l_cnt; 10284 nr_linfo = prog->aux->nr_linfo; 10285 } 10286 10287 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 10288 for (i = l_off; i < nr_linfo; i++) 10289 linfo[i].insn_off -= cnt; 10290 10291 /* fix up all subprogs (incl. 'exit') which start >= off */ 10292 for (i = 0; i <= env->subprog_cnt; i++) 10293 if (env->subprog_info[i].linfo_idx > l_off) { 10294 /* program may have started in the removed region but 10295 * may not be fully removed 10296 */ 10297 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 10298 env->subprog_info[i].linfo_idx -= l_cnt; 10299 else 10300 env->subprog_info[i].linfo_idx = l_off; 10301 } 10302 10303 return 0; 10304 } 10305 10306 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 10307 { 10308 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 10309 unsigned int orig_prog_len = env->prog->len; 10310 int err; 10311 10312 if (bpf_prog_is_dev_bound(env->prog->aux)) 10313 bpf_prog_offload_remove_insns(env, off, cnt); 10314 10315 err = bpf_remove_insns(env->prog, off, cnt); 10316 if (err) 10317 return err; 10318 10319 err = adjust_subprog_starts_after_remove(env, off, cnt); 10320 if (err) 10321 return err; 10322 10323 err = bpf_adj_linfo_after_remove(env, off, cnt); 10324 if (err) 10325 return err; 10326 10327 memmove(aux_data + off, aux_data + off + cnt, 10328 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 10329 10330 return 0; 10331 } 10332 10333 /* The verifier does more data flow analysis than llvm and will not 10334 * explore branches that are dead at run time. Malicious programs can 10335 * have dead code too. Therefore replace all dead at-run-time code 10336 * with 'ja -1'. 10337 * 10338 * Just nops are not optimal, e.g. if they would sit at the end of the 10339 * program and through another bug we would manage to jump there, then 10340 * we'd execute beyond program memory otherwise. Returning exception 10341 * code also wouldn't work since we can have subprogs where the dead 10342 * code could be located. 10343 */ 10344 static void sanitize_dead_code(struct bpf_verifier_env *env) 10345 { 10346 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 10347 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 10348 struct bpf_insn *insn = env->prog->insnsi; 10349 const int insn_cnt = env->prog->len; 10350 int i; 10351 10352 for (i = 0; i < insn_cnt; i++) { 10353 if (aux_data[i].seen) 10354 continue; 10355 memcpy(insn + i, &trap, sizeof(trap)); 10356 } 10357 } 10358 10359 static bool insn_is_cond_jump(u8 code) 10360 { 10361 u8 op; 10362 10363 if (BPF_CLASS(code) == BPF_JMP32) 10364 return true; 10365 10366 if (BPF_CLASS(code) != BPF_JMP) 10367 return false; 10368 10369 op = BPF_OP(code); 10370 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 10371 } 10372 10373 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 10374 { 10375 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 10376 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 10377 struct bpf_insn *insn = env->prog->insnsi; 10378 const int insn_cnt = env->prog->len; 10379 int i; 10380 10381 for (i = 0; i < insn_cnt; i++, insn++) { 10382 if (!insn_is_cond_jump(insn->code)) 10383 continue; 10384 10385 if (!aux_data[i + 1].seen) 10386 ja.off = insn->off; 10387 else if (!aux_data[i + 1 + insn->off].seen) 10388 ja.off = 0; 10389 else 10390 continue; 10391 10392 if (bpf_prog_is_dev_bound(env->prog->aux)) 10393 bpf_prog_offload_replace_insn(env, i, &ja); 10394 10395 memcpy(insn, &ja, sizeof(ja)); 10396 } 10397 } 10398 10399 static int opt_remove_dead_code(struct bpf_verifier_env *env) 10400 { 10401 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 10402 int insn_cnt = env->prog->len; 10403 int i, err; 10404 10405 for (i = 0; i < insn_cnt; i++) { 10406 int j; 10407 10408 j = 0; 10409 while (i + j < insn_cnt && !aux_data[i + j].seen) 10410 j++; 10411 if (!j) 10412 continue; 10413 10414 err = verifier_remove_insns(env, i, j); 10415 if (err) 10416 return err; 10417 insn_cnt = env->prog->len; 10418 } 10419 10420 return 0; 10421 } 10422 10423 static int opt_remove_nops(struct bpf_verifier_env *env) 10424 { 10425 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 10426 struct bpf_insn *insn = env->prog->insnsi; 10427 int insn_cnt = env->prog->len; 10428 int i, err; 10429 10430 for (i = 0; i < insn_cnt; i++) { 10431 if (memcmp(&insn[i], &ja, sizeof(ja))) 10432 continue; 10433 10434 err = verifier_remove_insns(env, i, 1); 10435 if (err) 10436 return err; 10437 insn_cnt--; 10438 i--; 10439 } 10440 10441 return 0; 10442 } 10443 10444 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 10445 const union bpf_attr *attr) 10446 { 10447 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 10448 struct bpf_insn_aux_data *aux = env->insn_aux_data; 10449 int i, patch_len, delta = 0, len = env->prog->len; 10450 struct bpf_insn *insns = env->prog->insnsi; 10451 struct bpf_prog *new_prog; 10452 bool rnd_hi32; 10453 10454 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 10455 zext_patch[1] = BPF_ZEXT_REG(0); 10456 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 10457 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 10458 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 10459 for (i = 0; i < len; i++) { 10460 int adj_idx = i + delta; 10461 struct bpf_insn insn; 10462 10463 insn = insns[adj_idx]; 10464 if (!aux[adj_idx].zext_dst) { 10465 u8 code, class; 10466 u32 imm_rnd; 10467 10468 if (!rnd_hi32) 10469 continue; 10470 10471 code = insn.code; 10472 class = BPF_CLASS(code); 10473 if (insn_no_def(&insn)) 10474 continue; 10475 10476 /* NOTE: arg "reg" (the fourth one) is only used for 10477 * BPF_STX which has been ruled out in above 10478 * check, it is safe to pass NULL here. 10479 */ 10480 if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) { 10481 if (class == BPF_LD && 10482 BPF_MODE(code) == BPF_IMM) 10483 i++; 10484 continue; 10485 } 10486 10487 /* ctx load could be transformed into wider load. */ 10488 if (class == BPF_LDX && 10489 aux[adj_idx].ptr_type == PTR_TO_CTX) 10490 continue; 10491 10492 imm_rnd = get_random_int(); 10493 rnd_hi32_patch[0] = insn; 10494 rnd_hi32_patch[1].imm = imm_rnd; 10495 rnd_hi32_patch[3].dst_reg = insn.dst_reg; 10496 patch = rnd_hi32_patch; 10497 patch_len = 4; 10498 goto apply_patch_buffer; 10499 } 10500 10501 if (!bpf_jit_needs_zext()) 10502 continue; 10503 10504 zext_patch[0] = insn; 10505 zext_patch[1].dst_reg = insn.dst_reg; 10506 zext_patch[1].src_reg = insn.dst_reg; 10507 patch = zext_patch; 10508 patch_len = 2; 10509 apply_patch_buffer: 10510 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 10511 if (!new_prog) 10512 return -ENOMEM; 10513 env->prog = new_prog; 10514 insns = new_prog->insnsi; 10515 aux = env->insn_aux_data; 10516 delta += patch_len - 1; 10517 } 10518 10519 return 0; 10520 } 10521 10522 /* convert load instructions that access fields of a context type into a 10523 * sequence of instructions that access fields of the underlying structure: 10524 * struct __sk_buff -> struct sk_buff 10525 * struct bpf_sock_ops -> struct sock 10526 */ 10527 static int convert_ctx_accesses(struct bpf_verifier_env *env) 10528 { 10529 const struct bpf_verifier_ops *ops = env->ops; 10530 int i, cnt, size, ctx_field_size, delta = 0; 10531 const int insn_cnt = env->prog->len; 10532 struct bpf_insn insn_buf[16], *insn; 10533 u32 target_size, size_default, off; 10534 struct bpf_prog *new_prog; 10535 enum bpf_access_type type; 10536 bool is_narrower_load; 10537 10538 if (ops->gen_prologue || env->seen_direct_write) { 10539 if (!ops->gen_prologue) { 10540 verbose(env, "bpf verifier is misconfigured\n"); 10541 return -EINVAL; 10542 } 10543 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 10544 env->prog); 10545 if (cnt >= ARRAY_SIZE(insn_buf)) { 10546 verbose(env, "bpf verifier is misconfigured\n"); 10547 return -EINVAL; 10548 } else if (cnt) { 10549 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 10550 if (!new_prog) 10551 return -ENOMEM; 10552 10553 env->prog = new_prog; 10554 delta += cnt - 1; 10555 } 10556 } 10557 10558 if (bpf_prog_is_dev_bound(env->prog->aux)) 10559 return 0; 10560 10561 insn = env->prog->insnsi + delta; 10562 10563 for (i = 0; i < insn_cnt; i++, insn++) { 10564 bpf_convert_ctx_access_t convert_ctx_access; 10565 10566 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 10567 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 10568 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 10569 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) 10570 type = BPF_READ; 10571 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 10572 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 10573 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 10574 insn->code == (BPF_STX | BPF_MEM | BPF_DW)) 10575 type = BPF_WRITE; 10576 else 10577 continue; 10578 10579 if (type == BPF_WRITE && 10580 env->insn_aux_data[i + delta].sanitize_stack_off) { 10581 struct bpf_insn patch[] = { 10582 /* Sanitize suspicious stack slot with zero. 10583 * There are no memory dependencies for this store, 10584 * since it's only using frame pointer and immediate 10585 * constant of zero 10586 */ 10587 BPF_ST_MEM(BPF_DW, BPF_REG_FP, 10588 env->insn_aux_data[i + delta].sanitize_stack_off, 10589 0), 10590 /* the original STX instruction will immediately 10591 * overwrite the same stack slot with appropriate value 10592 */ 10593 *insn, 10594 }; 10595 10596 cnt = ARRAY_SIZE(patch); 10597 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 10598 if (!new_prog) 10599 return -ENOMEM; 10600 10601 delta += cnt - 1; 10602 env->prog = new_prog; 10603 insn = new_prog->insnsi + i + delta; 10604 continue; 10605 } 10606 10607 switch (env->insn_aux_data[i + delta].ptr_type) { 10608 case PTR_TO_CTX: 10609 if (!ops->convert_ctx_access) 10610 continue; 10611 convert_ctx_access = ops->convert_ctx_access; 10612 break; 10613 case PTR_TO_SOCKET: 10614 case PTR_TO_SOCK_COMMON: 10615 convert_ctx_access = bpf_sock_convert_ctx_access; 10616 break; 10617 case PTR_TO_TCP_SOCK: 10618 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 10619 break; 10620 case PTR_TO_XDP_SOCK: 10621 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 10622 break; 10623 case PTR_TO_BTF_ID: 10624 if (type == BPF_READ) { 10625 insn->code = BPF_LDX | BPF_PROBE_MEM | 10626 BPF_SIZE((insn)->code); 10627 env->prog->aux->num_exentries++; 10628 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { 10629 verbose(env, "Writes through BTF pointers are not allowed\n"); 10630 return -EINVAL; 10631 } 10632 continue; 10633 default: 10634 continue; 10635 } 10636 10637 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 10638 size = BPF_LDST_BYTES(insn); 10639 10640 /* If the read access is a narrower load of the field, 10641 * convert to a 4/8-byte load, to minimum program type specific 10642 * convert_ctx_access changes. If conversion is successful, 10643 * we will apply proper mask to the result. 10644 */ 10645 is_narrower_load = size < ctx_field_size; 10646 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 10647 off = insn->off; 10648 if (is_narrower_load) { 10649 u8 size_code; 10650 10651 if (type == BPF_WRITE) { 10652 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 10653 return -EINVAL; 10654 } 10655 10656 size_code = BPF_H; 10657 if (ctx_field_size == 4) 10658 size_code = BPF_W; 10659 else if (ctx_field_size == 8) 10660 size_code = BPF_DW; 10661 10662 insn->off = off & ~(size_default - 1); 10663 insn->code = BPF_LDX | BPF_MEM | size_code; 10664 } 10665 10666 target_size = 0; 10667 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 10668 &target_size); 10669 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 10670 (ctx_field_size && !target_size)) { 10671 verbose(env, "bpf verifier is misconfigured\n"); 10672 return -EINVAL; 10673 } 10674 10675 if (is_narrower_load && size < target_size) { 10676 u8 shift = bpf_ctx_narrow_access_offset( 10677 off, size, size_default) * 8; 10678 if (ctx_field_size <= 4) { 10679 if (shift) 10680 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 10681 insn->dst_reg, 10682 shift); 10683 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 10684 (1 << size * 8) - 1); 10685 } else { 10686 if (shift) 10687 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 10688 insn->dst_reg, 10689 shift); 10690 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 10691 (1ULL << size * 8) - 1); 10692 } 10693 } 10694 10695 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 10696 if (!new_prog) 10697 return -ENOMEM; 10698 10699 delta += cnt - 1; 10700 10701 /* keep walking new program and skip insns we just inserted */ 10702 env->prog = new_prog; 10703 insn = new_prog->insnsi + i + delta; 10704 } 10705 10706 return 0; 10707 } 10708 10709 static int jit_subprogs(struct bpf_verifier_env *env) 10710 { 10711 struct bpf_prog *prog = env->prog, **func, *tmp; 10712 int i, j, subprog_start, subprog_end = 0, len, subprog; 10713 struct bpf_map *map_ptr; 10714 struct bpf_insn *insn; 10715 void *old_bpf_func; 10716 int err, num_exentries; 10717 10718 if (env->subprog_cnt <= 1) 10719 return 0; 10720 10721 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 10722 if (insn->code != (BPF_JMP | BPF_CALL) || 10723 insn->src_reg != BPF_PSEUDO_CALL) 10724 continue; 10725 /* Upon error here we cannot fall back to interpreter but 10726 * need a hard reject of the program. Thus -EFAULT is 10727 * propagated in any case. 10728 */ 10729 subprog = find_subprog(env, i + insn->imm + 1); 10730 if (subprog < 0) { 10731 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 10732 i + insn->imm + 1); 10733 return -EFAULT; 10734 } 10735 /* temporarily remember subprog id inside insn instead of 10736 * aux_data, since next loop will split up all insns into funcs 10737 */ 10738 insn->off = subprog; 10739 /* remember original imm in case JIT fails and fallback 10740 * to interpreter will be needed 10741 */ 10742 env->insn_aux_data[i].call_imm = insn->imm; 10743 /* point imm to __bpf_call_base+1 from JITs point of view */ 10744 insn->imm = 1; 10745 } 10746 10747 err = bpf_prog_alloc_jited_linfo(prog); 10748 if (err) 10749 goto out_undo_insn; 10750 10751 err = -ENOMEM; 10752 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 10753 if (!func) 10754 goto out_undo_insn; 10755 10756 for (i = 0; i < env->subprog_cnt; i++) { 10757 subprog_start = subprog_end; 10758 subprog_end = env->subprog_info[i + 1].start; 10759 10760 len = subprog_end - subprog_start; 10761 /* BPF_PROG_RUN doesn't call subprogs directly, 10762 * hence main prog stats include the runtime of subprogs. 10763 * subprogs don't have IDs and not reachable via prog_get_next_id 10764 * func[i]->aux->stats will never be accessed and stays NULL 10765 */ 10766 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 10767 if (!func[i]) 10768 goto out_free; 10769 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 10770 len * sizeof(struct bpf_insn)); 10771 func[i]->type = prog->type; 10772 func[i]->len = len; 10773 if (bpf_prog_calc_tag(func[i])) 10774 goto out_free; 10775 func[i]->is_func = 1; 10776 func[i]->aux->func_idx = i; 10777 /* the btf and func_info will be freed only at prog->aux */ 10778 func[i]->aux->btf = prog->aux->btf; 10779 func[i]->aux->func_info = prog->aux->func_info; 10780 10781 for (j = 0; j < prog->aux->size_poke_tab; j++) { 10782 u32 insn_idx = prog->aux->poke_tab[j].insn_idx; 10783 int ret; 10784 10785 if (!(insn_idx >= subprog_start && 10786 insn_idx <= subprog_end)) 10787 continue; 10788 10789 ret = bpf_jit_add_poke_descriptor(func[i], 10790 &prog->aux->poke_tab[j]); 10791 if (ret < 0) { 10792 verbose(env, "adding tail call poke descriptor failed\n"); 10793 goto out_free; 10794 } 10795 10796 func[i]->insnsi[insn_idx - subprog_start].imm = ret + 1; 10797 10798 map_ptr = func[i]->aux->poke_tab[ret].tail_call.map; 10799 ret = map_ptr->ops->map_poke_track(map_ptr, func[i]->aux); 10800 if (ret < 0) { 10801 verbose(env, "tracking tail call prog failed\n"); 10802 goto out_free; 10803 } 10804 } 10805 10806 /* Use bpf_prog_F_tag to indicate functions in stack traces. 10807 * Long term would need debug info to populate names 10808 */ 10809 func[i]->aux->name[0] = 'F'; 10810 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 10811 func[i]->jit_requested = 1; 10812 func[i]->aux->linfo = prog->aux->linfo; 10813 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 10814 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 10815 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 10816 num_exentries = 0; 10817 insn = func[i]->insnsi; 10818 for (j = 0; j < func[i]->len; j++, insn++) { 10819 if (BPF_CLASS(insn->code) == BPF_LDX && 10820 BPF_MODE(insn->code) == BPF_PROBE_MEM) 10821 num_exentries++; 10822 } 10823 func[i]->aux->num_exentries = num_exentries; 10824 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 10825 func[i] = bpf_int_jit_compile(func[i]); 10826 if (!func[i]->jited) { 10827 err = -ENOTSUPP; 10828 goto out_free; 10829 } 10830 cond_resched(); 10831 } 10832 10833 /* Untrack main program's aux structs so that during map_poke_run() 10834 * we will not stumble upon the unfilled poke descriptors; each 10835 * of the main program's poke descs got distributed across subprogs 10836 * and got tracked onto map, so we are sure that none of them will 10837 * be missed after the operation below 10838 */ 10839 for (i = 0; i < prog->aux->size_poke_tab; i++) { 10840 map_ptr = prog->aux->poke_tab[i].tail_call.map; 10841 10842 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 10843 } 10844 10845 /* at this point all bpf functions were successfully JITed 10846 * now populate all bpf_calls with correct addresses and 10847 * run last pass of JIT 10848 */ 10849 for (i = 0; i < env->subprog_cnt; i++) { 10850 insn = func[i]->insnsi; 10851 for (j = 0; j < func[i]->len; j++, insn++) { 10852 if (insn->code != (BPF_JMP | BPF_CALL) || 10853 insn->src_reg != BPF_PSEUDO_CALL) 10854 continue; 10855 subprog = insn->off; 10856 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - 10857 __bpf_call_base; 10858 } 10859 10860 /* we use the aux data to keep a list of the start addresses 10861 * of the JITed images for each function in the program 10862 * 10863 * for some architectures, such as powerpc64, the imm field 10864 * might not be large enough to hold the offset of the start 10865 * address of the callee's JITed image from __bpf_call_base 10866 * 10867 * in such cases, we can lookup the start address of a callee 10868 * by using its subprog id, available from the off field of 10869 * the call instruction, as an index for this list 10870 */ 10871 func[i]->aux->func = func; 10872 func[i]->aux->func_cnt = env->subprog_cnt; 10873 } 10874 for (i = 0; i < env->subprog_cnt; i++) { 10875 old_bpf_func = func[i]->bpf_func; 10876 tmp = bpf_int_jit_compile(func[i]); 10877 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 10878 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 10879 err = -ENOTSUPP; 10880 goto out_free; 10881 } 10882 cond_resched(); 10883 } 10884 10885 /* finally lock prog and jit images for all functions and 10886 * populate kallsysm 10887 */ 10888 for (i = 0; i < env->subprog_cnt; i++) { 10889 bpf_prog_lock_ro(func[i]); 10890 bpf_prog_kallsyms_add(func[i]); 10891 } 10892 10893 /* Last step: make now unused interpreter insns from main 10894 * prog consistent for later dump requests, so they can 10895 * later look the same as if they were interpreted only. 10896 */ 10897 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 10898 if (insn->code != (BPF_JMP | BPF_CALL) || 10899 insn->src_reg != BPF_PSEUDO_CALL) 10900 continue; 10901 insn->off = env->insn_aux_data[i].call_imm; 10902 subprog = find_subprog(env, i + insn->off + 1); 10903 insn->imm = subprog; 10904 } 10905 10906 prog->jited = 1; 10907 prog->bpf_func = func[0]->bpf_func; 10908 prog->aux->func = func; 10909 prog->aux->func_cnt = env->subprog_cnt; 10910 bpf_prog_free_unused_jited_linfo(prog); 10911 return 0; 10912 out_free: 10913 for (i = 0; i < env->subprog_cnt; i++) { 10914 if (!func[i]) 10915 continue; 10916 10917 for (j = 0; j < func[i]->aux->size_poke_tab; j++) { 10918 map_ptr = func[i]->aux->poke_tab[j].tail_call.map; 10919 map_ptr->ops->map_poke_untrack(map_ptr, func[i]->aux); 10920 } 10921 bpf_jit_free(func[i]); 10922 } 10923 kfree(func); 10924 out_undo_insn: 10925 /* cleanup main prog to be interpreted */ 10926 prog->jit_requested = 0; 10927 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 10928 if (insn->code != (BPF_JMP | BPF_CALL) || 10929 insn->src_reg != BPF_PSEUDO_CALL) 10930 continue; 10931 insn->off = 0; 10932 insn->imm = env->insn_aux_data[i].call_imm; 10933 } 10934 bpf_prog_free_jited_linfo(prog); 10935 return err; 10936 } 10937 10938 static int fixup_call_args(struct bpf_verifier_env *env) 10939 { 10940 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 10941 struct bpf_prog *prog = env->prog; 10942 struct bpf_insn *insn = prog->insnsi; 10943 int i, depth; 10944 #endif 10945 int err = 0; 10946 10947 if (env->prog->jit_requested && 10948 !bpf_prog_is_dev_bound(env->prog->aux)) { 10949 err = jit_subprogs(env); 10950 if (err == 0) 10951 return 0; 10952 if (err == -EFAULT) 10953 return err; 10954 } 10955 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 10956 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 10957 /* When JIT fails the progs with bpf2bpf calls and tail_calls 10958 * have to be rejected, since interpreter doesn't support them yet. 10959 */ 10960 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 10961 return -EINVAL; 10962 } 10963 for (i = 0; i < prog->len; i++, insn++) { 10964 if (insn->code != (BPF_JMP | BPF_CALL) || 10965 insn->src_reg != BPF_PSEUDO_CALL) 10966 continue; 10967 depth = get_callee_stack_depth(env, insn, i); 10968 if (depth < 0) 10969 return depth; 10970 bpf_patch_call_args(insn, depth); 10971 } 10972 err = 0; 10973 #endif 10974 return err; 10975 } 10976 10977 /* fixup insn->imm field of bpf_call instructions 10978 * and inline eligible helpers as explicit sequence of BPF instructions 10979 * 10980 * this function is called after eBPF program passed verification 10981 */ 10982 static int fixup_bpf_calls(struct bpf_verifier_env *env) 10983 { 10984 struct bpf_prog *prog = env->prog; 10985 bool expect_blinding = bpf_jit_blinding_enabled(prog); 10986 struct bpf_insn *insn = prog->insnsi; 10987 const struct bpf_func_proto *fn; 10988 const int insn_cnt = prog->len; 10989 const struct bpf_map_ops *ops; 10990 struct bpf_insn_aux_data *aux; 10991 struct bpf_insn insn_buf[16]; 10992 struct bpf_prog *new_prog; 10993 struct bpf_map *map_ptr; 10994 int i, ret, cnt, delta = 0; 10995 10996 for (i = 0; i < insn_cnt; i++, insn++) { 10997 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 10998 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 10999 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 11000 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 11001 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 11002 struct bpf_insn mask_and_div[] = { 11003 BPF_MOV32_REG(insn->src_reg, insn->src_reg), 11004 /* Rx div 0 -> 0 */ 11005 BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2), 11006 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 11007 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 11008 *insn, 11009 }; 11010 struct bpf_insn mask_and_mod[] = { 11011 BPF_MOV32_REG(insn->src_reg, insn->src_reg), 11012 /* Rx mod 0 -> Rx */ 11013 BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1), 11014 *insn, 11015 }; 11016 struct bpf_insn *patchlet; 11017 11018 if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 11019 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 11020 patchlet = mask_and_div + (is64 ? 1 : 0); 11021 cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0); 11022 } else { 11023 patchlet = mask_and_mod + (is64 ? 1 : 0); 11024 cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0); 11025 } 11026 11027 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 11028 if (!new_prog) 11029 return -ENOMEM; 11030 11031 delta += cnt - 1; 11032 env->prog = prog = new_prog; 11033 insn = new_prog->insnsi + i + delta; 11034 continue; 11035 } 11036 11037 if (BPF_CLASS(insn->code) == BPF_LD && 11038 (BPF_MODE(insn->code) == BPF_ABS || 11039 BPF_MODE(insn->code) == BPF_IND)) { 11040 cnt = env->ops->gen_ld_abs(insn, insn_buf); 11041 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 11042 verbose(env, "bpf verifier is misconfigured\n"); 11043 return -EINVAL; 11044 } 11045 11046 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 11047 if (!new_prog) 11048 return -ENOMEM; 11049 11050 delta += cnt - 1; 11051 env->prog = prog = new_prog; 11052 insn = new_prog->insnsi + i + delta; 11053 continue; 11054 } 11055 11056 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 11057 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 11058 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 11059 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 11060 struct bpf_insn insn_buf[16]; 11061 struct bpf_insn *patch = &insn_buf[0]; 11062 bool issrc, isneg; 11063 u32 off_reg; 11064 11065 aux = &env->insn_aux_data[i + delta]; 11066 if (!aux->alu_state || 11067 aux->alu_state == BPF_ALU_NON_POINTER) 11068 continue; 11069 11070 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 11071 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 11072 BPF_ALU_SANITIZE_SRC; 11073 11074 off_reg = issrc ? insn->src_reg : insn->dst_reg; 11075 if (isneg) 11076 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 11077 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1); 11078 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 11079 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 11080 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 11081 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 11082 if (issrc) { 11083 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, 11084 off_reg); 11085 insn->src_reg = BPF_REG_AX; 11086 } else { 11087 *patch++ = BPF_ALU64_REG(BPF_AND, off_reg, 11088 BPF_REG_AX); 11089 } 11090 if (isneg) 11091 insn->code = insn->code == code_add ? 11092 code_sub : code_add; 11093 *patch++ = *insn; 11094 if (issrc && isneg) 11095 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 11096 cnt = patch - insn_buf; 11097 11098 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 11099 if (!new_prog) 11100 return -ENOMEM; 11101 11102 delta += cnt - 1; 11103 env->prog = prog = new_prog; 11104 insn = new_prog->insnsi + i + delta; 11105 continue; 11106 } 11107 11108 if (insn->code != (BPF_JMP | BPF_CALL)) 11109 continue; 11110 if (insn->src_reg == BPF_PSEUDO_CALL) 11111 continue; 11112 11113 if (insn->imm == BPF_FUNC_get_route_realm) 11114 prog->dst_needed = 1; 11115 if (insn->imm == BPF_FUNC_get_prandom_u32) 11116 bpf_user_rnd_init_once(); 11117 if (insn->imm == BPF_FUNC_override_return) 11118 prog->kprobe_override = 1; 11119 if (insn->imm == BPF_FUNC_tail_call) { 11120 /* If we tail call into other programs, we 11121 * cannot make any assumptions since they can 11122 * be replaced dynamically during runtime in 11123 * the program array. 11124 */ 11125 prog->cb_access = 1; 11126 if (!allow_tail_call_in_subprogs(env)) 11127 prog->aux->stack_depth = MAX_BPF_STACK; 11128 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 11129 11130 /* mark bpf_tail_call as different opcode to avoid 11131 * conditional branch in the interpeter for every normal 11132 * call and to prevent accidental JITing by JIT compiler 11133 * that doesn't support bpf_tail_call yet 11134 */ 11135 insn->imm = 0; 11136 insn->code = BPF_JMP | BPF_TAIL_CALL; 11137 11138 aux = &env->insn_aux_data[i + delta]; 11139 if (env->bpf_capable && !expect_blinding && 11140 prog->jit_requested && 11141 !bpf_map_key_poisoned(aux) && 11142 !bpf_map_ptr_poisoned(aux) && 11143 !bpf_map_ptr_unpriv(aux)) { 11144 struct bpf_jit_poke_descriptor desc = { 11145 .reason = BPF_POKE_REASON_TAIL_CALL, 11146 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 11147 .tail_call.key = bpf_map_key_immediate(aux), 11148 .insn_idx = i + delta, 11149 }; 11150 11151 ret = bpf_jit_add_poke_descriptor(prog, &desc); 11152 if (ret < 0) { 11153 verbose(env, "adding tail call poke descriptor failed\n"); 11154 return ret; 11155 } 11156 11157 insn->imm = ret + 1; 11158 continue; 11159 } 11160 11161 if (!bpf_map_ptr_unpriv(aux)) 11162 continue; 11163 11164 /* instead of changing every JIT dealing with tail_call 11165 * emit two extra insns: 11166 * if (index >= max_entries) goto out; 11167 * index &= array->index_mask; 11168 * to avoid out-of-bounds cpu speculation 11169 */ 11170 if (bpf_map_ptr_poisoned(aux)) { 11171 verbose(env, "tail_call abusing map_ptr\n"); 11172 return -EINVAL; 11173 } 11174 11175 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 11176 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 11177 map_ptr->max_entries, 2); 11178 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 11179 container_of(map_ptr, 11180 struct bpf_array, 11181 map)->index_mask); 11182 insn_buf[2] = *insn; 11183 cnt = 3; 11184 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 11185 if (!new_prog) 11186 return -ENOMEM; 11187 11188 delta += cnt - 1; 11189 env->prog = prog = new_prog; 11190 insn = new_prog->insnsi + i + delta; 11191 continue; 11192 } 11193 11194 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 11195 * and other inlining handlers are currently limited to 64 bit 11196 * only. 11197 */ 11198 if (prog->jit_requested && BITS_PER_LONG == 64 && 11199 (insn->imm == BPF_FUNC_map_lookup_elem || 11200 insn->imm == BPF_FUNC_map_update_elem || 11201 insn->imm == BPF_FUNC_map_delete_elem || 11202 insn->imm == BPF_FUNC_map_push_elem || 11203 insn->imm == BPF_FUNC_map_pop_elem || 11204 insn->imm == BPF_FUNC_map_peek_elem)) { 11205 aux = &env->insn_aux_data[i + delta]; 11206 if (bpf_map_ptr_poisoned(aux)) 11207 goto patch_call_imm; 11208 11209 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 11210 ops = map_ptr->ops; 11211 if (insn->imm == BPF_FUNC_map_lookup_elem && 11212 ops->map_gen_lookup) { 11213 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 11214 if (cnt == -EOPNOTSUPP) 11215 goto patch_map_ops_generic; 11216 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 11217 verbose(env, "bpf verifier is misconfigured\n"); 11218 return -EINVAL; 11219 } 11220 11221 new_prog = bpf_patch_insn_data(env, i + delta, 11222 insn_buf, cnt); 11223 if (!new_prog) 11224 return -ENOMEM; 11225 11226 delta += cnt - 1; 11227 env->prog = prog = new_prog; 11228 insn = new_prog->insnsi + i + delta; 11229 continue; 11230 } 11231 11232 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 11233 (void *(*)(struct bpf_map *map, void *key))NULL)); 11234 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 11235 (int (*)(struct bpf_map *map, void *key))NULL)); 11236 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 11237 (int (*)(struct bpf_map *map, void *key, void *value, 11238 u64 flags))NULL)); 11239 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 11240 (int (*)(struct bpf_map *map, void *value, 11241 u64 flags))NULL)); 11242 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 11243 (int (*)(struct bpf_map *map, void *value))NULL)); 11244 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 11245 (int (*)(struct bpf_map *map, void *value))NULL)); 11246 patch_map_ops_generic: 11247 switch (insn->imm) { 11248 case BPF_FUNC_map_lookup_elem: 11249 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - 11250 __bpf_call_base; 11251 continue; 11252 case BPF_FUNC_map_update_elem: 11253 insn->imm = BPF_CAST_CALL(ops->map_update_elem) - 11254 __bpf_call_base; 11255 continue; 11256 case BPF_FUNC_map_delete_elem: 11257 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - 11258 __bpf_call_base; 11259 continue; 11260 case BPF_FUNC_map_push_elem: 11261 insn->imm = BPF_CAST_CALL(ops->map_push_elem) - 11262 __bpf_call_base; 11263 continue; 11264 case BPF_FUNC_map_pop_elem: 11265 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - 11266 __bpf_call_base; 11267 continue; 11268 case BPF_FUNC_map_peek_elem: 11269 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - 11270 __bpf_call_base; 11271 continue; 11272 } 11273 11274 goto patch_call_imm; 11275 } 11276 11277 if (prog->jit_requested && BITS_PER_LONG == 64 && 11278 insn->imm == BPF_FUNC_jiffies64) { 11279 struct bpf_insn ld_jiffies_addr[2] = { 11280 BPF_LD_IMM64(BPF_REG_0, 11281 (unsigned long)&jiffies), 11282 }; 11283 11284 insn_buf[0] = ld_jiffies_addr[0]; 11285 insn_buf[1] = ld_jiffies_addr[1]; 11286 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 11287 BPF_REG_0, 0); 11288 cnt = 3; 11289 11290 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 11291 cnt); 11292 if (!new_prog) 11293 return -ENOMEM; 11294 11295 delta += cnt - 1; 11296 env->prog = prog = new_prog; 11297 insn = new_prog->insnsi + i + delta; 11298 continue; 11299 } 11300 11301 patch_call_imm: 11302 fn = env->ops->get_func_proto(insn->imm, env->prog); 11303 /* all functions that have prototype and verifier allowed 11304 * programs to call them, must be real in-kernel functions 11305 */ 11306 if (!fn->func) { 11307 verbose(env, 11308 "kernel subsystem misconfigured func %s#%d\n", 11309 func_id_name(insn->imm), insn->imm); 11310 return -EFAULT; 11311 } 11312 insn->imm = fn->func - __bpf_call_base; 11313 } 11314 11315 /* Since poke tab is now finalized, publish aux to tracker. */ 11316 for (i = 0; i < prog->aux->size_poke_tab; i++) { 11317 map_ptr = prog->aux->poke_tab[i].tail_call.map; 11318 if (!map_ptr->ops->map_poke_track || 11319 !map_ptr->ops->map_poke_untrack || 11320 !map_ptr->ops->map_poke_run) { 11321 verbose(env, "bpf verifier is misconfigured\n"); 11322 return -EINVAL; 11323 } 11324 11325 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 11326 if (ret < 0) { 11327 verbose(env, "tracking tail call prog failed\n"); 11328 return ret; 11329 } 11330 } 11331 11332 return 0; 11333 } 11334 11335 static void free_states(struct bpf_verifier_env *env) 11336 { 11337 struct bpf_verifier_state_list *sl, *sln; 11338 int i; 11339 11340 sl = env->free_list; 11341 while (sl) { 11342 sln = sl->next; 11343 free_verifier_state(&sl->state, false); 11344 kfree(sl); 11345 sl = sln; 11346 } 11347 env->free_list = NULL; 11348 11349 if (!env->explored_states) 11350 return; 11351 11352 for (i = 0; i < state_htab_size(env); i++) { 11353 sl = env->explored_states[i]; 11354 11355 while (sl) { 11356 sln = sl->next; 11357 free_verifier_state(&sl->state, false); 11358 kfree(sl); 11359 sl = sln; 11360 } 11361 env->explored_states[i] = NULL; 11362 } 11363 } 11364 11365 /* The verifier is using insn_aux_data[] to store temporary data during 11366 * verification and to store information for passes that run after the 11367 * verification like dead code sanitization. do_check_common() for subprogram N 11368 * may analyze many other subprograms. sanitize_insn_aux_data() clears all 11369 * temporary data after do_check_common() finds that subprogram N cannot be 11370 * verified independently. pass_cnt counts the number of times 11371 * do_check_common() was run and insn->aux->seen tells the pass number 11372 * insn_aux_data was touched. These variables are compared to clear temporary 11373 * data from failed pass. For testing and experiments do_check_common() can be 11374 * run multiple times even when prior attempt to verify is unsuccessful. 11375 */ 11376 static void sanitize_insn_aux_data(struct bpf_verifier_env *env) 11377 { 11378 struct bpf_insn *insn = env->prog->insnsi; 11379 struct bpf_insn_aux_data *aux; 11380 int i, class; 11381 11382 for (i = 0; i < env->prog->len; i++) { 11383 class = BPF_CLASS(insn[i].code); 11384 if (class != BPF_LDX && class != BPF_STX) 11385 continue; 11386 aux = &env->insn_aux_data[i]; 11387 if (aux->seen != env->pass_cnt) 11388 continue; 11389 memset(aux, 0, offsetof(typeof(*aux), orig_idx)); 11390 } 11391 } 11392 11393 static int do_check_common(struct bpf_verifier_env *env, int subprog) 11394 { 11395 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 11396 struct bpf_verifier_state *state; 11397 struct bpf_reg_state *regs; 11398 int ret, i; 11399 11400 env->prev_linfo = NULL; 11401 env->pass_cnt++; 11402 11403 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 11404 if (!state) 11405 return -ENOMEM; 11406 state->curframe = 0; 11407 state->speculative = false; 11408 state->branches = 1; 11409 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 11410 if (!state->frame[0]) { 11411 kfree(state); 11412 return -ENOMEM; 11413 } 11414 env->cur_state = state; 11415 init_func_state(env, state->frame[0], 11416 BPF_MAIN_FUNC /* callsite */, 11417 0 /* frameno */, 11418 subprog); 11419 11420 regs = state->frame[state->curframe]->regs; 11421 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 11422 ret = btf_prepare_func_args(env, subprog, regs); 11423 if (ret) 11424 goto out; 11425 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 11426 if (regs[i].type == PTR_TO_CTX) 11427 mark_reg_known_zero(env, regs, i); 11428 else if (regs[i].type == SCALAR_VALUE) 11429 mark_reg_unknown(env, regs, i); 11430 } 11431 } else { 11432 /* 1st arg to a function */ 11433 regs[BPF_REG_1].type = PTR_TO_CTX; 11434 mark_reg_known_zero(env, regs, BPF_REG_1); 11435 ret = btf_check_func_arg_match(env, subprog, regs); 11436 if (ret == -EFAULT) 11437 /* unlikely verifier bug. abort. 11438 * ret == 0 and ret < 0 are sadly acceptable for 11439 * main() function due to backward compatibility. 11440 * Like socket filter program may be written as: 11441 * int bpf_prog(struct pt_regs *ctx) 11442 * and never dereference that ctx in the program. 11443 * 'struct pt_regs' is a type mismatch for socket 11444 * filter that should be using 'struct __sk_buff'. 11445 */ 11446 goto out; 11447 } 11448 11449 ret = do_check(env); 11450 out: 11451 /* check for NULL is necessary, since cur_state can be freed inside 11452 * do_check() under memory pressure. 11453 */ 11454 if (env->cur_state) { 11455 free_verifier_state(env->cur_state, true); 11456 env->cur_state = NULL; 11457 } 11458 while (!pop_stack(env, NULL, NULL, false)); 11459 if (!ret && pop_log) 11460 bpf_vlog_reset(&env->log, 0); 11461 free_states(env); 11462 if (ret) 11463 /* clean aux data in case subprog was rejected */ 11464 sanitize_insn_aux_data(env); 11465 return ret; 11466 } 11467 11468 /* Verify all global functions in a BPF program one by one based on their BTF. 11469 * All global functions must pass verification. Otherwise the whole program is rejected. 11470 * Consider: 11471 * int bar(int); 11472 * int foo(int f) 11473 * { 11474 * return bar(f); 11475 * } 11476 * int bar(int b) 11477 * { 11478 * ... 11479 * } 11480 * foo() will be verified first for R1=any_scalar_value. During verification it 11481 * will be assumed that bar() already verified successfully and call to bar() 11482 * from foo() will be checked for type match only. Later bar() will be verified 11483 * independently to check that it's safe for R1=any_scalar_value. 11484 */ 11485 static int do_check_subprogs(struct bpf_verifier_env *env) 11486 { 11487 struct bpf_prog_aux *aux = env->prog->aux; 11488 int i, ret; 11489 11490 if (!aux->func_info) 11491 return 0; 11492 11493 for (i = 1; i < env->subprog_cnt; i++) { 11494 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 11495 continue; 11496 env->insn_idx = env->subprog_info[i].start; 11497 WARN_ON_ONCE(env->insn_idx == 0); 11498 ret = do_check_common(env, i); 11499 if (ret) { 11500 return ret; 11501 } else if (env->log.level & BPF_LOG_LEVEL) { 11502 verbose(env, 11503 "Func#%d is safe for any args that match its prototype\n", 11504 i); 11505 } 11506 } 11507 return 0; 11508 } 11509 11510 static int do_check_main(struct bpf_verifier_env *env) 11511 { 11512 int ret; 11513 11514 env->insn_idx = 0; 11515 ret = do_check_common(env, 0); 11516 if (!ret) 11517 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 11518 return ret; 11519 } 11520 11521 11522 static void print_verification_stats(struct bpf_verifier_env *env) 11523 { 11524 int i; 11525 11526 if (env->log.level & BPF_LOG_STATS) { 11527 verbose(env, "verification time %lld usec\n", 11528 div_u64(env->verification_time, 1000)); 11529 verbose(env, "stack depth "); 11530 for (i = 0; i < env->subprog_cnt; i++) { 11531 u32 depth = env->subprog_info[i].stack_depth; 11532 11533 verbose(env, "%d", depth); 11534 if (i + 1 < env->subprog_cnt) 11535 verbose(env, "+"); 11536 } 11537 verbose(env, "\n"); 11538 } 11539 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 11540 "total_states %d peak_states %d mark_read %d\n", 11541 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 11542 env->max_states_per_insn, env->total_states, 11543 env->peak_states, env->longest_mark_read_walk); 11544 } 11545 11546 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 11547 { 11548 const struct btf_type *t, *func_proto; 11549 const struct bpf_struct_ops *st_ops; 11550 const struct btf_member *member; 11551 struct bpf_prog *prog = env->prog; 11552 u32 btf_id, member_idx; 11553 const char *mname; 11554 11555 btf_id = prog->aux->attach_btf_id; 11556 st_ops = bpf_struct_ops_find(btf_id); 11557 if (!st_ops) { 11558 verbose(env, "attach_btf_id %u is not a supported struct\n", 11559 btf_id); 11560 return -ENOTSUPP; 11561 } 11562 11563 t = st_ops->type; 11564 member_idx = prog->expected_attach_type; 11565 if (member_idx >= btf_type_vlen(t)) { 11566 verbose(env, "attach to invalid member idx %u of struct %s\n", 11567 member_idx, st_ops->name); 11568 return -EINVAL; 11569 } 11570 11571 member = &btf_type_member(t)[member_idx]; 11572 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 11573 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 11574 NULL); 11575 if (!func_proto) { 11576 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 11577 mname, member_idx, st_ops->name); 11578 return -EINVAL; 11579 } 11580 11581 if (st_ops->check_member) { 11582 int err = st_ops->check_member(t, member); 11583 11584 if (err) { 11585 verbose(env, "attach to unsupported member %s of struct %s\n", 11586 mname, st_ops->name); 11587 return err; 11588 } 11589 } 11590 11591 prog->aux->attach_func_proto = func_proto; 11592 prog->aux->attach_func_name = mname; 11593 env->ops = st_ops->verifier_ops; 11594 11595 return 0; 11596 } 11597 #define SECURITY_PREFIX "security_" 11598 11599 static int check_attach_modify_return(unsigned long addr, const char *func_name) 11600 { 11601 if (within_error_injection_list(addr) || 11602 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 11603 return 0; 11604 11605 return -EINVAL; 11606 } 11607 11608 /* list of non-sleepable functions that are otherwise on 11609 * ALLOW_ERROR_INJECTION list 11610 */ 11611 BTF_SET_START(btf_non_sleepable_error_inject) 11612 /* Three functions below can be called from sleepable and non-sleepable context. 11613 * Assume non-sleepable from bpf safety point of view. 11614 */ 11615 BTF_ID(func, __add_to_page_cache_locked) 11616 BTF_ID(func, should_fail_alloc_page) 11617 BTF_ID(func, should_failslab) 11618 BTF_SET_END(btf_non_sleepable_error_inject) 11619 11620 static int check_non_sleepable_error_inject(u32 btf_id) 11621 { 11622 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 11623 } 11624 11625 int bpf_check_attach_target(struct bpf_verifier_log *log, 11626 const struct bpf_prog *prog, 11627 const struct bpf_prog *tgt_prog, 11628 u32 btf_id, 11629 struct bpf_attach_target_info *tgt_info) 11630 { 11631 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 11632 const char prefix[] = "btf_trace_"; 11633 int ret = 0, subprog = -1, i; 11634 const struct btf_type *t; 11635 bool conservative = true; 11636 const char *tname; 11637 struct btf *btf; 11638 long addr = 0; 11639 11640 if (!btf_id) { 11641 bpf_log(log, "Tracing programs must provide btf_id\n"); 11642 return -EINVAL; 11643 } 11644 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 11645 if (!btf) { 11646 bpf_log(log, 11647 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 11648 return -EINVAL; 11649 } 11650 t = btf_type_by_id(btf, btf_id); 11651 if (!t) { 11652 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 11653 return -EINVAL; 11654 } 11655 tname = btf_name_by_offset(btf, t->name_off); 11656 if (!tname) { 11657 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 11658 return -EINVAL; 11659 } 11660 if (tgt_prog) { 11661 struct bpf_prog_aux *aux = tgt_prog->aux; 11662 11663 for (i = 0; i < aux->func_info_cnt; i++) 11664 if (aux->func_info[i].type_id == btf_id) { 11665 subprog = i; 11666 break; 11667 } 11668 if (subprog == -1) { 11669 bpf_log(log, "Subprog %s doesn't exist\n", tname); 11670 return -EINVAL; 11671 } 11672 conservative = aux->func_info_aux[subprog].unreliable; 11673 if (prog_extension) { 11674 if (conservative) { 11675 bpf_log(log, 11676 "Cannot replace static functions\n"); 11677 return -EINVAL; 11678 } 11679 if (!prog->jit_requested) { 11680 bpf_log(log, 11681 "Extension programs should be JITed\n"); 11682 return -EINVAL; 11683 } 11684 } 11685 if (!tgt_prog->jited) { 11686 bpf_log(log, "Can attach to only JITed progs\n"); 11687 return -EINVAL; 11688 } 11689 if (tgt_prog->type == prog->type) { 11690 /* Cannot fentry/fexit another fentry/fexit program. 11691 * Cannot attach program extension to another extension. 11692 * It's ok to attach fentry/fexit to extension program. 11693 */ 11694 bpf_log(log, "Cannot recursively attach\n"); 11695 return -EINVAL; 11696 } 11697 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 11698 prog_extension && 11699 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 11700 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 11701 /* Program extensions can extend all program types 11702 * except fentry/fexit. The reason is the following. 11703 * The fentry/fexit programs are used for performance 11704 * analysis, stats and can be attached to any program 11705 * type except themselves. When extension program is 11706 * replacing XDP function it is necessary to allow 11707 * performance analysis of all functions. Both original 11708 * XDP program and its program extension. Hence 11709 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 11710 * allowed. If extending of fentry/fexit was allowed it 11711 * would be possible to create long call chain 11712 * fentry->extension->fentry->extension beyond 11713 * reasonable stack size. Hence extending fentry is not 11714 * allowed. 11715 */ 11716 bpf_log(log, "Cannot extend fentry/fexit\n"); 11717 return -EINVAL; 11718 } 11719 } else { 11720 if (prog_extension) { 11721 bpf_log(log, "Cannot replace kernel functions\n"); 11722 return -EINVAL; 11723 } 11724 } 11725 11726 switch (prog->expected_attach_type) { 11727 case BPF_TRACE_RAW_TP: 11728 if (tgt_prog) { 11729 bpf_log(log, 11730 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 11731 return -EINVAL; 11732 } 11733 if (!btf_type_is_typedef(t)) { 11734 bpf_log(log, "attach_btf_id %u is not a typedef\n", 11735 btf_id); 11736 return -EINVAL; 11737 } 11738 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 11739 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 11740 btf_id, tname); 11741 return -EINVAL; 11742 } 11743 tname += sizeof(prefix) - 1; 11744 t = btf_type_by_id(btf, t->type); 11745 if (!btf_type_is_ptr(t)) 11746 /* should never happen in valid vmlinux build */ 11747 return -EINVAL; 11748 t = btf_type_by_id(btf, t->type); 11749 if (!btf_type_is_func_proto(t)) 11750 /* should never happen in valid vmlinux build */ 11751 return -EINVAL; 11752 11753 break; 11754 case BPF_TRACE_ITER: 11755 if (!btf_type_is_func(t)) { 11756 bpf_log(log, "attach_btf_id %u is not a function\n", 11757 btf_id); 11758 return -EINVAL; 11759 } 11760 t = btf_type_by_id(btf, t->type); 11761 if (!btf_type_is_func_proto(t)) 11762 return -EINVAL; 11763 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 11764 if (ret) 11765 return ret; 11766 break; 11767 default: 11768 if (!prog_extension) 11769 return -EINVAL; 11770 fallthrough; 11771 case BPF_MODIFY_RETURN: 11772 case BPF_LSM_MAC: 11773 case BPF_TRACE_FENTRY: 11774 case BPF_TRACE_FEXIT: 11775 if (!btf_type_is_func(t)) { 11776 bpf_log(log, "attach_btf_id %u is not a function\n", 11777 btf_id); 11778 return -EINVAL; 11779 } 11780 if (prog_extension && 11781 btf_check_type_match(log, prog, btf, t)) 11782 return -EINVAL; 11783 t = btf_type_by_id(btf, t->type); 11784 if (!btf_type_is_func_proto(t)) 11785 return -EINVAL; 11786 11787 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 11788 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 11789 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 11790 return -EINVAL; 11791 11792 if (tgt_prog && conservative) 11793 t = NULL; 11794 11795 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 11796 if (ret < 0) 11797 return ret; 11798 11799 if (tgt_prog) { 11800 if (subprog == 0) 11801 addr = (long) tgt_prog->bpf_func; 11802 else 11803 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 11804 } else { 11805 addr = kallsyms_lookup_name(tname); 11806 if (!addr) { 11807 bpf_log(log, 11808 "The address of function %s cannot be found\n", 11809 tname); 11810 return -ENOENT; 11811 } 11812 } 11813 11814 if (prog->aux->sleepable) { 11815 ret = -EINVAL; 11816 switch (prog->type) { 11817 case BPF_PROG_TYPE_TRACING: 11818 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 11819 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 11820 */ 11821 if (!check_non_sleepable_error_inject(btf_id) && 11822 within_error_injection_list(addr)) 11823 ret = 0; 11824 break; 11825 case BPF_PROG_TYPE_LSM: 11826 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 11827 * Only some of them are sleepable. 11828 */ 11829 if (bpf_lsm_is_sleepable_hook(btf_id)) 11830 ret = 0; 11831 break; 11832 default: 11833 break; 11834 } 11835 if (ret) { 11836 bpf_log(log, "%s is not sleepable\n", tname); 11837 return ret; 11838 } 11839 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 11840 if (tgt_prog) { 11841 bpf_log(log, "can't modify return codes of BPF programs\n"); 11842 return -EINVAL; 11843 } 11844 ret = check_attach_modify_return(addr, tname); 11845 if (ret) { 11846 bpf_log(log, "%s() is not modifiable\n", tname); 11847 return ret; 11848 } 11849 } 11850 11851 break; 11852 } 11853 tgt_info->tgt_addr = addr; 11854 tgt_info->tgt_name = tname; 11855 tgt_info->tgt_type = t; 11856 return 0; 11857 } 11858 11859 static int check_attach_btf_id(struct bpf_verifier_env *env) 11860 { 11861 struct bpf_prog *prog = env->prog; 11862 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 11863 struct bpf_attach_target_info tgt_info = {}; 11864 u32 btf_id = prog->aux->attach_btf_id; 11865 struct bpf_trampoline *tr; 11866 int ret; 11867 u64 key; 11868 11869 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 11870 prog->type != BPF_PROG_TYPE_LSM) { 11871 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); 11872 return -EINVAL; 11873 } 11874 11875 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 11876 return check_struct_ops_btf_id(env); 11877 11878 if (prog->type != BPF_PROG_TYPE_TRACING && 11879 prog->type != BPF_PROG_TYPE_LSM && 11880 prog->type != BPF_PROG_TYPE_EXT) 11881 return 0; 11882 11883 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 11884 if (ret) 11885 return ret; 11886 11887 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 11888 /* to make freplace equivalent to their targets, they need to 11889 * inherit env->ops and expected_attach_type for the rest of the 11890 * verification 11891 */ 11892 env->ops = bpf_verifier_ops[tgt_prog->type]; 11893 prog->expected_attach_type = tgt_prog->expected_attach_type; 11894 } 11895 11896 /* store info about the attachment target that will be used later */ 11897 prog->aux->attach_func_proto = tgt_info.tgt_type; 11898 prog->aux->attach_func_name = tgt_info.tgt_name; 11899 11900 if (tgt_prog) { 11901 prog->aux->saved_dst_prog_type = tgt_prog->type; 11902 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 11903 } 11904 11905 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 11906 prog->aux->attach_btf_trace = true; 11907 return 0; 11908 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 11909 if (!bpf_iter_prog_supported(prog)) 11910 return -EINVAL; 11911 return 0; 11912 } 11913 11914 if (prog->type == BPF_PROG_TYPE_LSM) { 11915 ret = bpf_lsm_verify_prog(&env->log, prog); 11916 if (ret < 0) 11917 return ret; 11918 } 11919 11920 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 11921 tr = bpf_trampoline_get(key, &tgt_info); 11922 if (!tr) 11923 return -ENOMEM; 11924 11925 prog->aux->dst_trampoline = tr; 11926 return 0; 11927 } 11928 11929 struct btf *bpf_get_btf_vmlinux(void) 11930 { 11931 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 11932 mutex_lock(&bpf_verifier_lock); 11933 if (!btf_vmlinux) 11934 btf_vmlinux = btf_parse_vmlinux(); 11935 mutex_unlock(&bpf_verifier_lock); 11936 } 11937 return btf_vmlinux; 11938 } 11939 11940 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, 11941 union bpf_attr __user *uattr) 11942 { 11943 u64 start_time = ktime_get_ns(); 11944 struct bpf_verifier_env *env; 11945 struct bpf_verifier_log *log; 11946 int i, len, ret = -EINVAL; 11947 bool is_priv; 11948 11949 /* no program is valid */ 11950 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 11951 return -EINVAL; 11952 11953 /* 'struct bpf_verifier_env' can be global, but since it's not small, 11954 * allocate/free it every time bpf_check() is called 11955 */ 11956 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 11957 if (!env) 11958 return -ENOMEM; 11959 log = &env->log; 11960 11961 len = (*prog)->len; 11962 env->insn_aux_data = 11963 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 11964 ret = -ENOMEM; 11965 if (!env->insn_aux_data) 11966 goto err_free_env; 11967 for (i = 0; i < len; i++) 11968 env->insn_aux_data[i].orig_idx = i; 11969 env->prog = *prog; 11970 env->ops = bpf_verifier_ops[env->prog->type]; 11971 is_priv = bpf_capable(); 11972 11973 bpf_get_btf_vmlinux(); 11974 11975 /* grab the mutex to protect few globals used by verifier */ 11976 if (!is_priv) 11977 mutex_lock(&bpf_verifier_lock); 11978 11979 if (attr->log_level || attr->log_buf || attr->log_size) { 11980 /* user requested verbose verifier output 11981 * and supplied buffer to store the verification trace 11982 */ 11983 log->level = attr->log_level; 11984 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 11985 log->len_total = attr->log_size; 11986 11987 ret = -EINVAL; 11988 /* log attributes have to be sane */ 11989 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || 11990 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) 11991 goto err_unlock; 11992 } 11993 11994 if (IS_ERR(btf_vmlinux)) { 11995 /* Either gcc or pahole or kernel are broken. */ 11996 verbose(env, "in-kernel BTF is malformed\n"); 11997 ret = PTR_ERR(btf_vmlinux); 11998 goto skip_full_check; 11999 } 12000 12001 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 12002 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 12003 env->strict_alignment = true; 12004 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 12005 env->strict_alignment = false; 12006 12007 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 12008 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); 12009 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 12010 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 12011 env->bpf_capable = bpf_capable(); 12012 12013 if (is_priv) 12014 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 12015 12016 if (bpf_prog_is_dev_bound(env->prog->aux)) { 12017 ret = bpf_prog_offload_verifier_prep(env->prog); 12018 if (ret) 12019 goto skip_full_check; 12020 } 12021 12022 env->explored_states = kvcalloc(state_htab_size(env), 12023 sizeof(struct bpf_verifier_state_list *), 12024 GFP_USER); 12025 ret = -ENOMEM; 12026 if (!env->explored_states) 12027 goto skip_full_check; 12028 12029 ret = check_subprogs(env); 12030 if (ret < 0) 12031 goto skip_full_check; 12032 12033 ret = check_btf_info(env, attr, uattr); 12034 if (ret < 0) 12035 goto skip_full_check; 12036 12037 ret = check_attach_btf_id(env); 12038 if (ret) 12039 goto skip_full_check; 12040 12041 ret = resolve_pseudo_ldimm64(env); 12042 if (ret < 0) 12043 goto skip_full_check; 12044 12045 ret = check_cfg(env); 12046 if (ret < 0) 12047 goto skip_full_check; 12048 12049 ret = do_check_subprogs(env); 12050 ret = ret ?: do_check_main(env); 12051 12052 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 12053 ret = bpf_prog_offload_finalize(env); 12054 12055 skip_full_check: 12056 kvfree(env->explored_states); 12057 12058 if (ret == 0) 12059 ret = check_max_stack_depth(env); 12060 12061 /* instruction rewrites happen after this point */ 12062 if (is_priv) { 12063 if (ret == 0) 12064 opt_hard_wire_dead_code_branches(env); 12065 if (ret == 0) 12066 ret = opt_remove_dead_code(env); 12067 if (ret == 0) 12068 ret = opt_remove_nops(env); 12069 } else { 12070 if (ret == 0) 12071 sanitize_dead_code(env); 12072 } 12073 12074 if (ret == 0) 12075 /* program is valid, convert *(u32*)(ctx + off) accesses */ 12076 ret = convert_ctx_accesses(env); 12077 12078 if (ret == 0) 12079 ret = fixup_bpf_calls(env); 12080 12081 /* do 32-bit optimization after insn patching has done so those patched 12082 * insns could be handled correctly. 12083 */ 12084 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 12085 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 12086 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 12087 : false; 12088 } 12089 12090 if (ret == 0) 12091 ret = fixup_call_args(env); 12092 12093 env->verification_time = ktime_get_ns() - start_time; 12094 print_verification_stats(env); 12095 12096 if (log->level && bpf_verifier_log_full(log)) 12097 ret = -ENOSPC; 12098 if (log->level && !log->ubuf) { 12099 ret = -EFAULT; 12100 goto err_release_maps; 12101 } 12102 12103 if (ret == 0 && env->used_map_cnt) { 12104 /* if program passed verifier, update used_maps in bpf_prog_info */ 12105 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 12106 sizeof(env->used_maps[0]), 12107 GFP_KERNEL); 12108 12109 if (!env->prog->aux->used_maps) { 12110 ret = -ENOMEM; 12111 goto err_release_maps; 12112 } 12113 12114 memcpy(env->prog->aux->used_maps, env->used_maps, 12115 sizeof(env->used_maps[0]) * env->used_map_cnt); 12116 env->prog->aux->used_map_cnt = env->used_map_cnt; 12117 12118 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 12119 * bpf_ld_imm64 instructions 12120 */ 12121 convert_pseudo_ld_imm64(env); 12122 } 12123 12124 if (ret == 0) 12125 adjust_btf_func(env); 12126 12127 err_release_maps: 12128 if (!env->prog->aux->used_maps) 12129 /* if we didn't copy map pointers into bpf_prog_info, release 12130 * them now. Otherwise free_used_maps() will release them. 12131 */ 12132 release_maps(env); 12133 12134 /* extension progs temporarily inherit the attach_type of their targets 12135 for verification purposes, so set it back to zero before returning 12136 */ 12137 if (env->prog->type == BPF_PROG_TYPE_EXT) 12138 env->prog->expected_attach_type = 0; 12139 12140 *prog = env->prog; 12141 err_unlock: 12142 if (!is_priv) 12143 mutex_unlock(&bpf_verifier_lock); 12144 vfree(env->insn_aux_data); 12145 err_free_env: 12146 kfree(env); 12147 return ret; 12148 } 12149