1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux Socket Filter - Kernel level socket filtering 4 * 5 * Based on the design of the Berkeley Packet Filter. The new 6 * internal format has been designed by PLUMgrid: 7 * 8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com 9 * 10 * Authors: 11 * 12 * Jay Schulist <jschlst@samba.org> 13 * Alexei Starovoitov <ast@plumgrid.com> 14 * Daniel Borkmann <dborkman@redhat.com> 15 * 16 * Andi Kleen - Fix a few bad bugs and races. 17 * Kris Katterjohn - Added many additional checks in bpf_check_classic() 18 */ 19 20 #include <uapi/linux/btf.h> 21 #include <linux/ctype.h> 22 #include <linux/filter.h> 23 #include <linux/sched/signal.h> 24 #include <linux/skbuff.h> 25 #include <linux/static_call.h> 26 #include <linux/vmalloc.h> 27 #include <linux/prandom.h> 28 #include <linux/bpf.h> 29 #include <linux/btf.h> 30 #include <linux/hex.h> 31 #include <linux/objtool.h> 32 #include <linux/overflow.h> 33 #include <linux/rbtree_latch.h> 34 #include <linux/kallsyms.h> 35 #include <linux/rcupdate.h> 36 #include <linux/perf_event.h> 37 #include <linux/extable.h> 38 #include <linux/log2.h> 39 #include <linux/bpf_verifier.h> 40 #include <linux/nodemask.h> 41 #include <linux/nospec.h> 42 #include <linux/bpf_mem_alloc.h> 43 #include <linux/memcontrol.h> 44 #include <linux/execmem.h> 45 #include <crypto/sha2.h> 46 47 #include <asm/barrier.h> 48 #include <linux/unaligned.h> 49 50 /* Registers */ 51 #define BPF_R0 regs[BPF_REG_0] 52 #define BPF_R1 regs[BPF_REG_1] 53 #define BPF_R2 regs[BPF_REG_2] 54 #define BPF_R3 regs[BPF_REG_3] 55 #define BPF_R4 regs[BPF_REG_4] 56 #define BPF_R5 regs[BPF_REG_5] 57 #define BPF_R6 regs[BPF_REG_6] 58 #define BPF_R7 regs[BPF_REG_7] 59 #define BPF_R8 regs[BPF_REG_8] 60 #define BPF_R9 regs[BPF_REG_9] 61 #define BPF_R10 regs[BPF_REG_10] 62 63 /* Named registers */ 64 #define DST regs[insn->dst_reg] 65 #define SRC regs[insn->src_reg] 66 #define FP regs[BPF_REG_FP] 67 #define AX regs[BPF_REG_AX] 68 #define ARG1 regs[BPF_REG_ARG1] 69 #define CTX regs[BPF_REG_CTX] 70 #define OFF insn->off 71 #define IMM insn->imm 72 73 struct bpf_mem_alloc bpf_global_ma; 74 bool bpf_global_ma_set; 75 76 /* No hurry in this branch 77 * 78 * Exported for the bpf jit load helper. 79 */ 80 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size) 81 { 82 u8 *ptr = NULL; 83 84 if (k >= SKF_NET_OFF) { 85 ptr = skb_network_header(skb) + k - SKF_NET_OFF; 86 } else if (k >= SKF_LL_OFF) { 87 if (unlikely(!skb_mac_header_was_set(skb))) 88 return NULL; 89 ptr = skb_mac_header(skb) + k - SKF_LL_OFF; 90 } 91 if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb)) 92 return ptr; 93 94 return NULL; 95 } 96 97 /* tell bpf programs that include vmlinux.h kernel's PAGE_SIZE */ 98 enum page_size_enum { 99 __PAGE_SIZE = PAGE_SIZE 100 }; 101 102 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) 103 { 104 gfp_t gfp_flags = bpf_memcg_flags(GFP_KERNEL | __GFP_ZERO | gfp_extra_flags); 105 struct bpf_prog_aux *aux; 106 struct bpf_prog *fp; 107 108 size = round_up(size, __PAGE_SIZE); 109 fp = __vmalloc(size, gfp_flags); 110 if (fp == NULL) 111 return NULL; 112 113 aux = kzalloc_obj(*aux, bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); 114 if (aux == NULL) { 115 vfree(fp); 116 return NULL; 117 } 118 fp->active = __alloc_percpu_gfp(sizeof(u8[BPF_NR_CONTEXTS]), 4, 119 bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); 120 if (!fp->active) { 121 vfree(fp); 122 kfree(aux); 123 return NULL; 124 } 125 126 fp->pages = size / PAGE_SIZE; 127 fp->aux = aux; 128 fp->aux->main_prog_aux = aux; 129 fp->aux->prog = fp; 130 fp->jit_requested = ebpf_jit_enabled(); 131 fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); 132 fp->blinding_requested = bpf_jit_blinding_enabled(fp); 133 #ifdef CONFIG_CGROUP_BPF 134 aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; 135 #endif 136 137 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); 138 #ifdef CONFIG_FINEIBT 139 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode); 140 #endif 141 mutex_init(&fp->aux->used_maps_mutex); 142 mutex_init(&fp->aux->ext_mutex); 143 mutex_init(&fp->aux->dst_mutex); 144 mutex_init(&fp->aux->st_ops_assoc_mutex); 145 146 return fp; 147 } 148 149 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) 150 { 151 gfp_t gfp_flags = bpf_memcg_flags(GFP_KERNEL | __GFP_ZERO | gfp_extra_flags); 152 struct bpf_prog *prog; 153 int cpu; 154 155 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); 156 if (!prog) 157 return NULL; 158 159 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); 160 if (!prog->stats) { 161 free_percpu(prog->active); 162 kfree(prog->aux); 163 vfree(prog); 164 return NULL; 165 } 166 167 for_each_possible_cpu(cpu) { 168 struct bpf_prog_stats *pstats; 169 170 pstats = per_cpu_ptr(prog->stats, cpu); 171 u64_stats_init(&pstats->syncp); 172 } 173 return prog; 174 } 175 176 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) 177 { 178 if (!prog->aux->nr_linfo || !prog->jit_requested) 179 return 0; 180 181 prog->aux->jited_linfo = kvzalloc_objs(*prog->aux->jited_linfo, 182 prog->aux->nr_linfo, 183 bpf_memcg_flags(GFP_KERNEL | __GFP_NOWARN)); 184 if (!prog->aux->jited_linfo) 185 return -ENOMEM; 186 187 return 0; 188 } 189 190 void bpf_prog_jit_attempt_done(struct bpf_prog *prog) 191 { 192 if (prog->aux->jited_linfo && 193 (!prog->jited || !prog->aux->jited_linfo[0])) { 194 kvfree(prog->aux->jited_linfo); 195 prog->aux->jited_linfo = NULL; 196 } 197 198 kfree(prog->aux->kfunc_tab); 199 prog->aux->kfunc_tab = NULL; 200 } 201 202 /* The jit engine is responsible to provide an array 203 * for insn_off to the jited_off mapping (insn_to_jit_off). 204 * 205 * The idx to this array is the insn_off. Hence, the insn_off 206 * here is relative to the prog itself instead of the main prog. 207 * This array has one entry for each xlated bpf insn. 208 * 209 * jited_off is the byte off to the end of the jited insn. 210 * 211 * Hence, with 212 * insn_start: 213 * The first bpf insn off of the prog. The insn off 214 * here is relative to the main prog. 215 * e.g. if prog is a subprog, insn_start > 0 216 * linfo_idx: 217 * The prog's idx to prog->aux->linfo and jited_linfo 218 * 219 * jited_linfo[linfo_idx] = prog->bpf_func 220 * 221 * For i > linfo_idx, 222 * 223 * jited_linfo[i] = prog->bpf_func + 224 * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] 225 */ 226 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, 227 const u32 *insn_to_jit_off) 228 { 229 u32 linfo_idx, insn_start, insn_end, nr_linfo, i; 230 const struct bpf_line_info *linfo; 231 void **jited_linfo; 232 233 if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt) 234 /* Userspace did not provide linfo */ 235 return; 236 237 linfo_idx = prog->aux->linfo_idx; 238 linfo = &prog->aux->linfo[linfo_idx]; 239 insn_start = linfo[0].insn_off; 240 insn_end = insn_start + prog->len; 241 242 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; 243 jited_linfo[0] = prog->bpf_func; 244 245 nr_linfo = prog->aux->nr_linfo - linfo_idx; 246 247 for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) 248 /* The verifier ensures that linfo[i].insn_off is 249 * strictly increasing 250 */ 251 jited_linfo[i] = prog->bpf_func + 252 insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; 253 } 254 255 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 256 gfp_t gfp_extra_flags) 257 { 258 gfp_t gfp_flags = bpf_memcg_flags(GFP_KERNEL | __GFP_ZERO | gfp_extra_flags); 259 struct bpf_prog *fp; 260 u32 pages; 261 262 size = round_up(size, PAGE_SIZE); 263 pages = size / PAGE_SIZE; 264 if (pages <= fp_old->pages) 265 return fp_old; 266 267 fp = __vmalloc(size, gfp_flags); 268 if (fp) { 269 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); 270 fp->pages = pages; 271 fp->aux->prog = fp; 272 bpf_prog_dev_bound_moved(fp); 273 274 /* We keep fp->aux from fp_old around in the new 275 * reallocated structure. 276 */ 277 fp_old->aux = NULL; 278 fp_old->stats = NULL; 279 fp_old->active = NULL; 280 __bpf_prog_free(fp_old); 281 } 282 283 return fp; 284 } 285 286 void __bpf_prog_free(struct bpf_prog *fp) 287 { 288 if (fp->aux) { 289 #ifdef CONFIG_BPF_SYSCALL 290 bpf_prog_stream_free(fp); 291 #endif 292 mutex_destroy(&fp->aux->used_maps_mutex); 293 mutex_destroy(&fp->aux->dst_mutex); 294 mutex_destroy(&fp->aux->st_ops_assoc_mutex); 295 kfree(fp->aux->poke_tab); 296 kfree(fp->aux); 297 } 298 free_percpu(fp->stats); 299 free_percpu(fp->active); 300 vfree(fp); 301 } 302 303 int bpf_prog_calc_tag(struct bpf_prog *fp) 304 { 305 size_t size = bpf_prog_insn_size(fp); 306 struct bpf_insn *dst; 307 bool was_ld_map; 308 u32 i; 309 310 dst = __vmalloc(size, GFP_KERNEL_ACCOUNT); 311 if (!dst) 312 return -ENOMEM; 313 314 /* We need to take out the map fd for the digest calculation 315 * since they are unstable from user space side. 316 */ 317 for (i = 0, was_ld_map = false; i < fp->len; i++) { 318 dst[i] = fp->insnsi[i]; 319 if (!was_ld_map && 320 dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && 321 (dst[i].src_reg == BPF_PSEUDO_MAP_FD || 322 dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { 323 was_ld_map = true; 324 dst[i].imm = 0; 325 } else if (was_ld_map && 326 dst[i].code == 0 && 327 dst[i].dst_reg == 0 && 328 dst[i].src_reg == 0 && 329 dst[i].off == 0) { 330 was_ld_map = false; 331 dst[i].imm = 0; 332 } else { 333 was_ld_map = false; 334 } 335 } 336 sha256((u8 *)dst, size, fp->digest); 337 vfree(dst); 338 return 0; 339 } 340 341 static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, 342 s32 end_new, s32 curr, const bool probe_pass) 343 { 344 const s64 imm_min = S32_MIN, imm_max = S32_MAX; 345 s32 delta = end_new - end_old; 346 s64 imm = insn->imm; 347 348 if (curr < pos && curr + imm + 1 >= end_old) 349 imm += delta; 350 else if (curr >= end_new && curr + imm + 1 < end_new) 351 imm -= delta; 352 if (imm < imm_min || imm > imm_max) 353 return -ERANGE; 354 if (!probe_pass) 355 insn->imm = imm; 356 return 0; 357 } 358 359 static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, 360 s32 end_new, s32 curr, const bool probe_pass) 361 { 362 s64 off_min, off_max, off; 363 s32 delta = end_new - end_old; 364 365 if (insn->code == (BPF_JMP32 | BPF_JA)) { 366 off = insn->imm; 367 off_min = S32_MIN; 368 off_max = S32_MAX; 369 } else { 370 off = insn->off; 371 off_min = S16_MIN; 372 off_max = S16_MAX; 373 } 374 375 if (curr < pos && curr + off + 1 >= end_old) 376 off += delta; 377 else if (curr >= end_new && curr + off + 1 < end_new) 378 off -= delta; 379 if (off < off_min || off > off_max) 380 return -ERANGE; 381 if (!probe_pass) { 382 if (insn->code == (BPF_JMP32 | BPF_JA)) 383 insn->imm = off; 384 else 385 insn->off = off; 386 } 387 return 0; 388 } 389 390 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, 391 s32 end_new, const bool probe_pass) 392 { 393 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); 394 struct bpf_insn *insn = prog->insnsi; 395 int ret = 0; 396 397 for (i = 0; i < insn_cnt; i++, insn++) { 398 u8 code; 399 400 /* In the probing pass we still operate on the original, 401 * unpatched image in order to check overflows before we 402 * do any other adjustments. Therefore skip the patchlet. 403 */ 404 if (probe_pass && i == pos) { 405 i = end_new; 406 insn = prog->insnsi + end_old; 407 } 408 if (bpf_pseudo_func(insn)) { 409 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 410 end_new, i, probe_pass); 411 if (ret) 412 return ret; 413 continue; 414 } 415 code = insn->code; 416 if ((BPF_CLASS(code) != BPF_JMP && 417 BPF_CLASS(code) != BPF_JMP32) || 418 BPF_OP(code) == BPF_EXIT) 419 continue; 420 /* Adjust offset of jmps if we cross patch boundaries. */ 421 if (BPF_OP(code) == BPF_CALL) { 422 if (insn->src_reg != BPF_PSEUDO_CALL) 423 continue; 424 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 425 end_new, i, probe_pass); 426 } else { 427 ret = bpf_adj_delta_to_off(insn, pos, end_old, 428 end_new, i, probe_pass); 429 } 430 if (ret) 431 break; 432 } 433 434 return ret; 435 } 436 437 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) 438 { 439 struct bpf_line_info *linfo; 440 u32 i, nr_linfo; 441 442 nr_linfo = prog->aux->nr_linfo; 443 if (!nr_linfo || !delta) 444 return; 445 446 linfo = prog->aux->linfo; 447 448 for (i = 0; i < nr_linfo; i++) 449 if (off < linfo[i].insn_off) 450 break; 451 452 /* Push all off < linfo[i].insn_off by delta */ 453 for (; i < nr_linfo; i++) 454 linfo[i].insn_off += delta; 455 } 456 457 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 458 const struct bpf_insn *patch, u32 len) 459 { 460 u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; 461 const u32 cnt_max = S16_MAX; 462 struct bpf_prog *prog_adj; 463 int err; 464 465 /* Since our patchlet doesn't expand the image, we're done. */ 466 if (insn_delta == 0) { 467 memcpy(prog->insnsi + off, patch, sizeof(*patch)); 468 return prog; 469 } 470 471 insn_adj_cnt = prog->len + insn_delta; 472 473 /* Reject anything that would potentially let the insn->off 474 * target overflow when we have excessive program expansions. 475 * We need to probe here before we do any reallocation where 476 * we afterwards may not fail anymore. 477 */ 478 if (insn_adj_cnt > cnt_max && 479 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) 480 return ERR_PTR(err); 481 482 /* Several new instructions need to be inserted. Make room 483 * for them. Likely, there's no need for a new allocation as 484 * last page could have large enough tailroom. 485 */ 486 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), 487 GFP_USER); 488 if (!prog_adj) 489 return ERR_PTR(-ENOMEM); 490 491 prog_adj->len = insn_adj_cnt; 492 493 /* Patching happens in 3 steps: 494 * 495 * 1) Move over tail of insnsi from next instruction onwards, 496 * so we can patch the single target insn with one or more 497 * new ones (patching is always from 1 to n insns, n > 0). 498 * 2) Inject new instructions at the target location. 499 * 3) Adjust branch offsets if necessary. 500 */ 501 insn_rest = insn_adj_cnt - off - len; 502 503 memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, 504 sizeof(*patch) * insn_rest); 505 memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); 506 507 /* We are guaranteed to not fail at this point, otherwise 508 * the ship has sailed to reverse to the original state. An 509 * overflow cannot happen at this point. 510 */ 511 BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); 512 513 bpf_adj_linfo(prog_adj, off, insn_delta); 514 515 return prog_adj; 516 } 517 518 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) 519 { 520 int err; 521 522 /* Branch offsets can't overflow when program is shrinking, no need 523 * to call bpf_adj_branches(..., true) here 524 */ 525 memmove(prog->insnsi + off, prog->insnsi + off + cnt, 526 sizeof(struct bpf_insn) * (prog->len - off - cnt)); 527 prog->len -= cnt; 528 529 err = bpf_adj_branches(prog, off, off + cnt, off, false); 530 WARN_ON_ONCE(err); 531 return err; 532 } 533 534 static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) 535 { 536 int i; 537 538 for (i = 0; i < fp->aux->real_func_cnt; i++) 539 bpf_prog_kallsyms_del(fp->aux->func[i]); 540 } 541 542 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) 543 { 544 bpf_prog_kallsyms_del_subprogs(fp); 545 bpf_prog_kallsyms_del(fp); 546 } 547 548 #ifdef CONFIG_BPF_JIT 549 /* All BPF JIT sysctl knobs here. */ 550 int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 551 int bpf_jit_kallsyms __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 552 int bpf_jit_harden __read_mostly; 553 long bpf_jit_limit __read_mostly; 554 long bpf_jit_limit_max __read_mostly; 555 556 static void 557 bpf_prog_ksym_set_addr(struct bpf_prog *prog) 558 { 559 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); 560 561 prog->aux->ksym.start = (unsigned long) prog->bpf_func; 562 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; 563 } 564 565 static void 566 bpf_prog_ksym_set_name(struct bpf_prog *prog) 567 { 568 char *sym = prog->aux->ksym.name; 569 const char *end = sym + KSYM_NAME_LEN; 570 const struct btf_type *type; 571 const char *func_name; 572 573 BUILD_BUG_ON(sizeof("bpf_prog_") + 574 sizeof(prog->tag) * 2 + 575 /* name has been null terminated. 576 * We should need +1 for the '_' preceding 577 * the name. However, the null character 578 * is double counted between the name and the 579 * sizeof("bpf_prog_") above, so we omit 580 * the +1 here. 581 */ 582 sizeof(prog->aux->name) > KSYM_NAME_LEN); 583 584 sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); 585 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); 586 587 /* prog->aux->name will be ignored if full btf name is available */ 588 if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) { 589 type = btf_type_by_id(prog->aux->btf, 590 prog->aux->func_info[prog->aux->func_idx].type_id); 591 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); 592 snprintf(sym, (size_t)(end - sym), "_%s", func_name); 593 /* the name of a function of Rust is not an identifier */ 594 for (; *sym; sym++) 595 if (!isalnum(*sym) && *sym != '_' && *sym != '.') 596 *sym = '_'; 597 return; 598 } 599 600 if (prog->aux->name[0]) 601 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); 602 else 603 *sym = 0; 604 } 605 606 static unsigned long bpf_get_ksym_start(struct latch_tree_node *n) 607 { 608 return container_of(n, struct bpf_ksym, tnode)->start; 609 } 610 611 static __always_inline bool bpf_tree_less(struct latch_tree_node *a, 612 struct latch_tree_node *b) 613 { 614 return bpf_get_ksym_start(a) < bpf_get_ksym_start(b); 615 } 616 617 static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) 618 { 619 unsigned long val = (unsigned long)key; 620 const struct bpf_ksym *ksym; 621 622 ksym = container_of(n, struct bpf_ksym, tnode); 623 624 if (val < ksym->start) 625 return -1; 626 /* Ensure that we detect return addresses as part of the program, when 627 * the final instruction is a call for a program part of the stack 628 * trace. Therefore, do val > ksym->end instead of val >= ksym->end. 629 */ 630 if (val > ksym->end) 631 return 1; 632 633 return 0; 634 } 635 636 static const struct latch_tree_ops bpf_tree_ops = { 637 .less = bpf_tree_less, 638 .comp = bpf_tree_comp, 639 }; 640 641 static DEFINE_SPINLOCK(bpf_lock); 642 static LIST_HEAD(bpf_kallsyms); 643 static struct latch_tree_root bpf_tree __cacheline_aligned; 644 645 void bpf_ksym_add(struct bpf_ksym *ksym) 646 { 647 spin_lock_bh(&bpf_lock); 648 WARN_ON_ONCE(!list_empty(&ksym->lnode)); 649 list_add_tail_rcu(&ksym->lnode, &bpf_kallsyms); 650 latch_tree_insert(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 651 spin_unlock_bh(&bpf_lock); 652 } 653 654 static void __bpf_ksym_del(struct bpf_ksym *ksym) 655 { 656 if (list_empty(&ksym->lnode)) 657 return; 658 659 latch_tree_erase(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 660 list_del_rcu(&ksym->lnode); 661 } 662 663 void bpf_ksym_del(struct bpf_ksym *ksym) 664 { 665 spin_lock_bh(&bpf_lock); 666 __bpf_ksym_del(ksym); 667 spin_unlock_bh(&bpf_lock); 668 } 669 670 static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) 671 { 672 return fp->jited && !bpf_prog_was_classic(fp); 673 } 674 675 void bpf_prog_kallsyms_add(struct bpf_prog *fp) 676 { 677 if (!bpf_prog_kallsyms_candidate(fp) || 678 !bpf_token_capable(fp->aux->token, CAP_BPF)) 679 return; 680 681 bpf_prog_ksym_set_addr(fp); 682 bpf_prog_ksym_set_name(fp); 683 fp->aux->ksym.prog = true; 684 685 bpf_ksym_add(&fp->aux->ksym); 686 687 #ifdef CONFIG_FINEIBT 688 /* 689 * When FineIBT, code in the __cfi_foo() symbols can get executed 690 * and hence unwinder needs help. 691 */ 692 if (cfi_mode != CFI_FINEIBT) 693 return; 694 695 snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN, 696 "__cfi_%s", fp->aux->ksym.name); 697 698 fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16; 699 fp->aux->ksym_prefix.end = (unsigned long) fp->bpf_func; 700 701 bpf_ksym_add(&fp->aux->ksym_prefix); 702 #endif 703 } 704 705 void bpf_prog_kallsyms_del(struct bpf_prog *fp) 706 { 707 if (!bpf_prog_kallsyms_candidate(fp)) 708 return; 709 710 bpf_ksym_del(&fp->aux->ksym); 711 #ifdef CONFIG_FINEIBT 712 if (cfi_mode != CFI_FINEIBT) 713 return; 714 bpf_ksym_del(&fp->aux->ksym_prefix); 715 #endif 716 } 717 718 static struct bpf_ksym *bpf_ksym_find(unsigned long addr) 719 { 720 struct latch_tree_node *n; 721 722 n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); 723 return n ? container_of(n, struct bpf_ksym, tnode) : NULL; 724 } 725 726 int bpf_address_lookup(unsigned long addr, unsigned long *size, 727 unsigned long *off, char *sym) 728 { 729 struct bpf_ksym *ksym; 730 int ret = 0; 731 732 rcu_read_lock(); 733 ksym = bpf_ksym_find(addr); 734 if (ksym) { 735 unsigned long symbol_start = ksym->start; 736 unsigned long symbol_end = ksym->end; 737 738 ret = strscpy(sym, ksym->name, KSYM_NAME_LEN); 739 740 if (size) 741 *size = symbol_end - symbol_start; 742 if (off) 743 *off = addr - symbol_start; 744 } 745 rcu_read_unlock(); 746 747 return ret; 748 } 749 750 bool is_bpf_text_address(unsigned long addr) 751 { 752 bool ret; 753 754 rcu_read_lock(); 755 ret = bpf_ksym_find(addr) != NULL; 756 rcu_read_unlock(); 757 758 return ret; 759 } 760 761 struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 762 { 763 struct bpf_ksym *ksym; 764 765 WARN_ON_ONCE(!rcu_read_lock_held()); 766 ksym = bpf_ksym_find(addr); 767 768 return ksym && ksym->prog ? 769 container_of(ksym, struct bpf_prog_aux, ksym)->prog : 770 NULL; 771 } 772 773 bool bpf_has_frame_pointer(unsigned long ip) 774 { 775 struct bpf_ksym *ksym; 776 unsigned long offset; 777 778 guard(rcu)(); 779 780 ksym = bpf_ksym_find(ip); 781 if (!ksym || !ksym->fp_start || !ksym->fp_end) 782 return false; 783 784 offset = ip - ksym->start; 785 786 return offset >= ksym->fp_start && offset < ksym->fp_end; 787 } 788 789 const struct exception_table_entry *search_bpf_extables(unsigned long addr) 790 { 791 const struct exception_table_entry *e = NULL; 792 struct bpf_prog *prog; 793 794 rcu_read_lock(); 795 prog = bpf_prog_ksym_find(addr); 796 if (!prog) 797 goto out; 798 if (!prog->aux->num_exentries) 799 goto out; 800 801 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); 802 out: 803 rcu_read_unlock(); 804 return e; 805 } 806 807 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 808 char *sym) 809 { 810 struct bpf_ksym *ksym; 811 unsigned int it = 0; 812 int ret = -ERANGE; 813 814 if (!bpf_jit_kallsyms_enabled()) 815 return ret; 816 817 rcu_read_lock(); 818 list_for_each_entry_rcu(ksym, &bpf_kallsyms, lnode) { 819 if (it++ != symnum) 820 continue; 821 822 strscpy(sym, ksym->name, KSYM_NAME_LEN); 823 824 *value = ksym->start; 825 *type = BPF_SYM_ELF_TYPE; 826 827 ret = 0; 828 break; 829 } 830 rcu_read_unlock(); 831 832 return ret; 833 } 834 835 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, 836 struct bpf_jit_poke_descriptor *poke) 837 { 838 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 839 static const u32 poke_tab_max = 1024; 840 u32 slot = prog->aux->size_poke_tab; 841 u32 size = slot + 1; 842 843 if (size > poke_tab_max) 844 return -ENOSPC; 845 if (poke->tailcall_target || poke->tailcall_target_stable || 846 poke->tailcall_bypass || poke->adj_off || poke->bypass_addr) 847 return -EINVAL; 848 849 switch (poke->reason) { 850 case BPF_POKE_REASON_TAIL_CALL: 851 if (!poke->tail_call.map) 852 return -EINVAL; 853 break; 854 default: 855 return -EINVAL; 856 } 857 858 tab = krealloc_array(tab, size, sizeof(*poke), GFP_KERNEL); 859 if (!tab) 860 return -ENOMEM; 861 862 memcpy(&tab[slot], poke, sizeof(*poke)); 863 prog->aux->size_poke_tab = size; 864 prog->aux->poke_tab = tab; 865 866 return slot; 867 } 868 869 /* 870 * BPF program pack allocator. 871 * 872 * Most BPF programs are pretty small. Allocating a hole page for each 873 * program is sometime a waste. Many small bpf program also adds pressure 874 * to instruction TLB. To solve this issue, we introduce a BPF program pack 875 * allocator. The prog_pack allocator uses HPAGE_PMD_SIZE page (2MB on x86) 876 * to host BPF programs. 877 */ 878 #define BPF_PROG_CHUNK_SHIFT 6 879 #define BPF_PROG_CHUNK_SIZE (1 << BPF_PROG_CHUNK_SHIFT) 880 #define BPF_PROG_CHUNK_MASK (~(BPF_PROG_CHUNK_SIZE - 1)) 881 882 struct bpf_prog_pack { 883 struct list_head list; 884 void *ptr; 885 bool arch_flush_needed; 886 unsigned long bitmap[]; 887 }; 888 889 void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) 890 { 891 memset(area, 0, size); 892 } 893 894 DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); 895 896 /* 897 * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the 898 * pack allocator test "is a predictor flush wired up at all" with a cheap 899 * static branch instead of repeatedly querying the static call target. 900 */ 901 DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); 902 903 #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) 904 905 static DEFINE_MUTEX(pack_mutex); 906 static LIST_HEAD(pack_list); 907 908 /* PMD_SIZE is not available in some special config, e.g. ARCH=arm with 909 * CONFIG_MMU=n. Use PAGE_SIZE in these cases. 910 */ 911 #ifdef PMD_SIZE 912 /* PMD_SIZE is really big for some archs. It doesn't make sense to 913 * reserve too much memory in one allocation. Hardcode BPF_PROG_PACK_SIZE to 914 * 2MiB * num_possible_nodes(). On most architectures PMD_SIZE will be 915 * greater than or equal to 2MB. 916 */ 917 #define BPF_PROG_PACK_SIZE (SZ_2M * num_possible_nodes()) 918 #else 919 #define BPF_PROG_PACK_SIZE PAGE_SIZE 920 #endif 921 922 #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) 923 924 static bool bpf_jit_mem_is_rox(void) 925 { 926 return execmem_is_rox(EXECMEM_BPF); 927 } 928 929 static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) 930 { 931 struct bpf_prog_pack *pack; 932 int err; 933 934 pack = kzalloc_flex(*pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)); 935 if (!pack) 936 return NULL; 937 pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE); 938 if (!pack->ptr) 939 goto out; 940 bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); 941 942 if (static_branch_unlikely(&bpf_pred_flush_enabled)) 943 pack->arch_flush_needed = true; 944 if (!bpf_jit_mem_is_rox()) { 945 bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); 946 set_vm_flush_reset_perms(pack->ptr); 947 err = set_memory_rox((unsigned long)pack->ptr, 948 BPF_PROG_PACK_SIZE / PAGE_SIZE); 949 if (err) 950 goto out; 951 } 952 list_add_tail(&pack->list, &pack_list); 953 return pack; 954 955 out: 956 bpf_jit_free_exec(pack->ptr); 957 kfree(pack); 958 return NULL; 959 } 960 961 void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) 962 { 963 unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); 964 struct bpf_prog_pack *pack, *fallback_pack = NULL; 965 unsigned long pos, fallback_pos = 0; 966 void *ptr = NULL; 967 968 mutex_lock(&pack_mutex); 969 if (size > BPF_PROG_PACK_SIZE) { 970 /* 971 * Allocations larger than a pack get their own pages, and 972 * predictors are not flushed for such allocation. This is only 973 * safe because cBPF programs (the unprivileged attack surface) 974 * are bounded well below a pack size. 975 */ 976 if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) 977 pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); 978 size = round_up(size, PAGE_SIZE); 979 ptr = bpf_jit_alloc_exec(size); 980 if (ptr && !bpf_jit_mem_is_rox()) { 981 int err; 982 983 bpf_fill_ill_insns(ptr, size); 984 set_vm_flush_reset_perms(ptr); 985 err = set_memory_rox((unsigned long)ptr, 986 size / PAGE_SIZE); 987 if (err) { 988 bpf_jit_free_exec(ptr); 989 ptr = NULL; 990 } 991 } 992 goto out; 993 } 994 list_for_each_entry(pack, &pack_list, list) { 995 pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 996 nbits, 0); 997 if (pos >= BPF_PROG_CHUNK_COUNT) 998 continue; 999 /* Flush not enabled, use any pack */ 1000 if (!static_branch_unlikely(&bpf_pred_flush_enabled)) 1001 goto found_free_area; 1002 /* 1003 * cBPF reuse of a dirty pack triggers a flush, so prefer a 1004 * clean pack for cBPF. eBPF never flushes, so steer it to a 1005 * dirty pack and keep clean packs free for cBPF. 1006 */ 1007 if (was_classic ^ pack->arch_flush_needed) 1008 goto found_free_area; 1009 if (!fallback_pack) { 1010 fallback_pack = pack; 1011 fallback_pos = pos; 1012 } 1013 } 1014 1015 /* No preferred pack found */ 1016 if (fallback_pack) { 1017 pack = fallback_pack; 1018 pos = fallback_pos; 1019 goto found_free_area; 1020 } 1021 1022 pack = alloc_new_pack(bpf_fill_ill_insns); 1023 if (!pack) 1024 goto out; 1025 1026 pos = 0; 1027 1028 found_free_area: 1029 /* Flush only for cBPF as it may contain a crafted gadget */ 1030 if (static_branch_unlikely(&bpf_pred_flush_enabled) && 1031 pack->arch_flush_needed && 1032 was_classic) { 1033 struct bpf_prog_pack *p; 1034 1035 static_call_cond(bpf_arch_pred_flush)(); 1036 list_for_each_entry(p, &pack_list, list) 1037 p->arch_flush_needed = false; 1038 } 1039 bitmap_set(pack->bitmap, pos, nbits); 1040 ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); 1041 1042 out: 1043 mutex_unlock(&pack_mutex); 1044 return ptr; 1045 } 1046 1047 void bpf_prog_pack_free(void *ptr, u32 size) 1048 { 1049 struct bpf_prog_pack *pack = NULL, *tmp; 1050 unsigned int nbits; 1051 unsigned long pos; 1052 1053 mutex_lock(&pack_mutex); 1054 if (size > BPF_PROG_PACK_SIZE) { 1055 bpf_jit_free_exec(ptr); 1056 goto out; 1057 } 1058 1059 list_for_each_entry(tmp, &pack_list, list) { 1060 if (ptr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > ptr) { 1061 pack = tmp; 1062 break; 1063 } 1064 } 1065 1066 if (WARN_ONCE(!pack, "bpf_prog_pack bug\n")) 1067 goto out; 1068 1069 nbits = BPF_PROG_SIZE_TO_NBITS(size); 1070 pos = ((unsigned long)ptr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT; 1071 1072 WARN_ONCE(bpf_arch_text_invalidate(ptr, size), 1073 "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); 1074 1075 bitmap_clear(pack->bitmap, pos, nbits); 1076 1077 if (static_branch_unlikely(&bpf_pred_flush_enabled)) 1078 pack->arch_flush_needed = true; 1079 if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 1080 BPF_PROG_CHUNK_COUNT, 0) == 0) { 1081 list_del(&pack->list); 1082 bpf_jit_free_exec(pack->ptr); 1083 kfree(pack); 1084 } 1085 out: 1086 mutex_unlock(&pack_mutex); 1087 } 1088 1089 static atomic_long_t bpf_jit_current; 1090 1091 /* Can be overridden by an arch's JIT compiler if it has a custom, 1092 * dedicated BPF backend memory area, or if neither of the two 1093 * below apply. 1094 */ 1095 u64 __weak bpf_jit_alloc_exec_limit(void) 1096 { 1097 #if defined(MODULES_VADDR) 1098 return MODULES_END - MODULES_VADDR; 1099 #else 1100 return VMALLOC_END - VMALLOC_START; 1101 #endif 1102 } 1103 1104 static int __init bpf_jit_charge_init(void) 1105 { 1106 /* Only used as heuristic here to derive limit. */ 1107 bpf_jit_limit_max = bpf_jit_alloc_exec_limit(); 1108 bpf_jit_limit = min_t(u64, round_up(bpf_jit_limit_max >> 1, 1109 PAGE_SIZE), LONG_MAX); 1110 return 0; 1111 } 1112 pure_initcall(bpf_jit_charge_init); 1113 1114 int bpf_jit_charge_modmem(u32 size) 1115 { 1116 if (atomic_long_add_return(size, &bpf_jit_current) > READ_ONCE(bpf_jit_limit)) { 1117 if (!bpf_capable()) { 1118 atomic_long_sub(size, &bpf_jit_current); 1119 return -EPERM; 1120 } 1121 } 1122 1123 return 0; 1124 } 1125 1126 void bpf_jit_uncharge_modmem(u32 size) 1127 { 1128 atomic_long_sub(size, &bpf_jit_current); 1129 } 1130 1131 void *bpf_jit_alloc_exec(unsigned long size) 1132 { 1133 return execmem_alloc(EXECMEM_BPF, size); 1134 } 1135 1136 void bpf_jit_free_exec(void *addr) 1137 { 1138 execmem_free(addr); 1139 } 1140 1141 struct bpf_binary_header * 1142 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 1143 unsigned int alignment, 1144 bpf_jit_fill_hole_t bpf_fill_ill_insns) 1145 { 1146 struct bpf_binary_header *hdr; 1147 u32 size, hole, start; 1148 1149 WARN_ON_ONCE(!is_power_of_2(alignment) || 1150 alignment > BPF_IMAGE_ALIGNMENT); 1151 1152 /* Most of BPF filters are really small, but if some of them 1153 * fill a page, allow at least 128 extra bytes to insert a 1154 * random section of illegal instructions. 1155 */ 1156 size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); 1157 1158 if (bpf_jit_charge_modmem(size)) 1159 return NULL; 1160 hdr = bpf_jit_alloc_exec(size); 1161 if (!hdr) { 1162 bpf_jit_uncharge_modmem(size); 1163 return NULL; 1164 } 1165 1166 /* Fill space with illegal/arch-dep instructions. */ 1167 bpf_fill_ill_insns(hdr, size); 1168 1169 hdr->size = size; 1170 hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), 1171 PAGE_SIZE - sizeof(*hdr)); 1172 start = get_random_u32_below(hole) & ~(alignment - 1); 1173 1174 /* Leave a random number of instructions before BPF code. */ 1175 *image_ptr = &hdr->image[start]; 1176 1177 return hdr; 1178 } 1179 1180 void bpf_jit_binary_free(struct bpf_binary_header *hdr) 1181 { 1182 u32 size = hdr->size; 1183 1184 bpf_jit_free_exec(hdr); 1185 bpf_jit_uncharge_modmem(size); 1186 } 1187 1188 /* Allocate jit binary from bpf_prog_pack allocator. 1189 * Since the allocated memory is RO+X, the JIT engine cannot write directly 1190 * to the memory. To solve this problem, a RW buffer is also allocated at 1191 * as the same time. The JIT engine should calculate offsets based on the 1192 * RO memory address, but write JITed program to the RW buffer. Once the 1193 * JIT engine finishes, it calls bpf_jit_binary_pack_finalize, which copies 1194 * the JITed program to the RO memory. 1195 */ 1196 struct bpf_binary_header * 1197 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, 1198 unsigned int alignment, 1199 struct bpf_binary_header **rw_header, 1200 u8 **rw_image, 1201 bpf_jit_fill_hole_t bpf_fill_ill_insns, 1202 bool was_classic) 1203 { 1204 struct bpf_binary_header *ro_header; 1205 u32 size, hole, start; 1206 1207 WARN_ON_ONCE(!is_power_of_2(alignment) || 1208 alignment > BPF_IMAGE_ALIGNMENT); 1209 1210 /* add 16 bytes for a random section of illegal instructions */ 1211 size = round_up(proglen + sizeof(*ro_header) + 16, BPF_PROG_CHUNK_SIZE); 1212 1213 if (bpf_jit_charge_modmem(size)) 1214 return NULL; 1215 ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); 1216 if (!ro_header) { 1217 bpf_jit_uncharge_modmem(size); 1218 return NULL; 1219 } 1220 1221 *rw_header = kvmalloc(size, GFP_KERNEL); 1222 if (!*rw_header) { 1223 bpf_prog_pack_free(ro_header, size); 1224 bpf_jit_uncharge_modmem(size); 1225 return NULL; 1226 } 1227 1228 /* Fill space with illegal/arch-dep instructions. */ 1229 bpf_fill_ill_insns(*rw_header, size); 1230 (*rw_header)->size = size; 1231 1232 hole = min_t(unsigned int, size - (proglen + sizeof(*ro_header)), 1233 BPF_PROG_CHUNK_SIZE - sizeof(*ro_header)); 1234 start = get_random_u32_below(hole) & ~(alignment - 1); 1235 1236 *image_ptr = &ro_header->image[start]; 1237 *rw_image = &(*rw_header)->image[start]; 1238 1239 return ro_header; 1240 } 1241 1242 /* Copy JITed text from rw_header to its final location, the ro_header. */ 1243 int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, 1244 struct bpf_binary_header *rw_header) 1245 { 1246 void *ptr; 1247 1248 ptr = bpf_arch_text_copy(ro_header, rw_header, rw_header->size); 1249 1250 kvfree(rw_header); 1251 1252 if (IS_ERR(ptr)) { 1253 bpf_prog_pack_free(ro_header, ro_header->size); 1254 return PTR_ERR(ptr); 1255 } 1256 return 0; 1257 } 1258 1259 /* bpf_jit_binary_pack_free is called in two different scenarios: 1260 * 1) when the program is freed after; 1261 * 2) when the JIT engine fails (before bpf_jit_binary_pack_finalize). 1262 * For case 2), we need to free both the RO memory and the RW buffer. 1263 * 1264 * bpf_jit_binary_pack_free requires proper ro_header->size. However, 1265 * bpf_jit_binary_pack_alloc does not set it. Therefore, ro_header->size 1266 * must be set with either bpf_jit_binary_pack_finalize (normal path) or 1267 * bpf_arch_text_copy (when jit fails). 1268 */ 1269 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, 1270 struct bpf_binary_header *rw_header) 1271 { 1272 u32 size = ro_header->size; 1273 1274 bpf_prog_pack_free(ro_header, size); 1275 kvfree(rw_header); 1276 bpf_jit_uncharge_modmem(size); 1277 } 1278 1279 struct bpf_binary_header * 1280 bpf_jit_binary_pack_hdr(const struct bpf_prog *fp) 1281 { 1282 unsigned long real_start = (unsigned long)fp->bpf_func; 1283 unsigned long addr; 1284 1285 addr = real_start & BPF_PROG_CHUNK_MASK; 1286 return (void *)addr; 1287 } 1288 1289 static inline struct bpf_binary_header * 1290 bpf_jit_binary_hdr(const struct bpf_prog *fp) 1291 { 1292 unsigned long real_start = (unsigned long)fp->bpf_func; 1293 unsigned long addr; 1294 1295 addr = real_start & PAGE_MASK; 1296 return (void *)addr; 1297 } 1298 1299 /* This symbol is only overridden by archs that have different 1300 * requirements than the usual eBPF JITs, f.e. when they only 1301 * implement cBPF JIT, do not set images read-only, etc. 1302 */ 1303 void __weak bpf_jit_free(struct bpf_prog *fp) 1304 { 1305 if (fp->jited) { 1306 struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); 1307 1308 bpf_jit_binary_free(hdr); 1309 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 1310 } 1311 1312 bpf_prog_unlock_free(fp); 1313 } 1314 1315 int bpf_jit_get_func_addr(const struct bpf_prog *prog, 1316 const struct bpf_insn *insn, bool extra_pass, 1317 u64 *func_addr, bool *func_addr_fixed) 1318 { 1319 s16 off = insn->off; 1320 s32 imm = insn->imm; 1321 u8 *addr; 1322 int err; 1323 1324 *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; 1325 if (!*func_addr_fixed) { 1326 /* Place-holder address till the last pass has collected 1327 * all addresses for JITed subprograms in which case we 1328 * can pick them up from prog->aux. 1329 */ 1330 if (!extra_pass) 1331 addr = NULL; 1332 else if (prog->aux->func && 1333 off >= 0 && off < prog->aux->real_func_cnt) 1334 addr = (u8 *)prog->aux->func[off]->bpf_func; 1335 else 1336 return -EINVAL; 1337 } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 1338 bpf_jit_supports_far_kfunc_call()) { 1339 err = bpf_get_kfunc_addr(prog, insn->imm, insn->off, &addr); 1340 if (err) 1341 return err; 1342 } else { 1343 /* Address of a BPF helper call. Since part of the core 1344 * kernel, it's always at a fixed location. __bpf_call_base 1345 * and the helper with imm relative to it are both in core 1346 * kernel. 1347 */ 1348 addr = (u8 *)__bpf_call_base + imm; 1349 } 1350 1351 *func_addr = (unsigned long)addr; 1352 return 0; 1353 } 1354 1355 const char *bpf_jit_get_prog_name(struct bpf_prog *prog) 1356 { 1357 if (prog->aux->ksym.prog) 1358 return prog->aux->ksym.name; 1359 return prog->aux->name; 1360 } 1361 1362 static int bpf_jit_blind_insn(const struct bpf_insn *from, 1363 const struct bpf_insn *aux, 1364 struct bpf_insn *to_buff, 1365 bool emit_zext) 1366 { 1367 struct bpf_insn *to = to_buff; 1368 u32 imm_rnd = get_random_u32(); 1369 int off; 1370 1371 BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); 1372 BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); 1373 1374 /* Constraints on AX register: 1375 * 1376 * AX register is inaccessible from user space. It is mapped in 1377 * all JITs, and used here for constant blinding rewrites. It is 1378 * typically "stateless" meaning its contents are only valid within 1379 * the executed instruction, but not across several instructions. 1380 * There are a few exceptions however which are further detailed 1381 * below. 1382 * 1383 * Constant blinding is only used by JITs, not in the interpreter. 1384 * The interpreter uses AX in some occasions as a local temporary 1385 * register e.g. in DIV or MOD instructions. 1386 * 1387 * In restricted circumstances, the verifier can also use the AX 1388 * register for rewrites as long as they do not interfere with 1389 * the above cases! 1390 */ 1391 if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) 1392 goto out; 1393 1394 if (from->imm == 0 && 1395 (from->code == (BPF_ALU | BPF_MOV | BPF_K) || 1396 from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { 1397 *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); 1398 goto out; 1399 } 1400 1401 switch (from->code) { 1402 case BPF_ALU | BPF_ADD | BPF_K: 1403 case BPF_ALU | BPF_SUB | BPF_K: 1404 case BPF_ALU | BPF_AND | BPF_K: 1405 case BPF_ALU | BPF_OR | BPF_K: 1406 case BPF_ALU | BPF_XOR | BPF_K: 1407 case BPF_ALU | BPF_MUL | BPF_K: 1408 case BPF_ALU | BPF_MOV | BPF_K: 1409 case BPF_ALU | BPF_DIV | BPF_K: 1410 case BPF_ALU | BPF_MOD | BPF_K: 1411 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1412 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1413 *to++ = BPF_ALU32_REG_OFF(from->code, from->dst_reg, BPF_REG_AX, from->off); 1414 break; 1415 1416 case BPF_ALU64 | BPF_ADD | BPF_K: 1417 case BPF_ALU64 | BPF_SUB | BPF_K: 1418 case BPF_ALU64 | BPF_AND | BPF_K: 1419 case BPF_ALU64 | BPF_OR | BPF_K: 1420 case BPF_ALU64 | BPF_XOR | BPF_K: 1421 case BPF_ALU64 | BPF_MUL | BPF_K: 1422 case BPF_ALU64 | BPF_MOV | BPF_K: 1423 case BPF_ALU64 | BPF_DIV | BPF_K: 1424 case BPF_ALU64 | BPF_MOD | BPF_K: 1425 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1426 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1427 *to++ = BPF_ALU64_REG_OFF(from->code, from->dst_reg, BPF_REG_AX, from->off); 1428 break; 1429 1430 case BPF_JMP | BPF_JEQ | BPF_K: 1431 case BPF_JMP | BPF_JNE | BPF_K: 1432 case BPF_JMP | BPF_JGT | BPF_K: 1433 case BPF_JMP | BPF_JLT | BPF_K: 1434 case BPF_JMP | BPF_JGE | BPF_K: 1435 case BPF_JMP | BPF_JLE | BPF_K: 1436 case BPF_JMP | BPF_JSGT | BPF_K: 1437 case BPF_JMP | BPF_JSLT | BPF_K: 1438 case BPF_JMP | BPF_JSGE | BPF_K: 1439 case BPF_JMP | BPF_JSLE | BPF_K: 1440 case BPF_JMP | BPF_JSET | BPF_K: 1441 /* Accommodate for extra offset in case of a backjump. */ 1442 off = from->off; 1443 if (off < 0) 1444 off -= 2; 1445 if (off < S16_MIN) 1446 return -ERANGE; 1447 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1448 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1449 *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); 1450 break; 1451 1452 case BPF_JMP32 | BPF_JEQ | BPF_K: 1453 case BPF_JMP32 | BPF_JNE | BPF_K: 1454 case BPF_JMP32 | BPF_JGT | BPF_K: 1455 case BPF_JMP32 | BPF_JLT | BPF_K: 1456 case BPF_JMP32 | BPF_JGE | BPF_K: 1457 case BPF_JMP32 | BPF_JLE | BPF_K: 1458 case BPF_JMP32 | BPF_JSGT | BPF_K: 1459 case BPF_JMP32 | BPF_JSLT | BPF_K: 1460 case BPF_JMP32 | BPF_JSGE | BPF_K: 1461 case BPF_JMP32 | BPF_JSLE | BPF_K: 1462 case BPF_JMP32 | BPF_JSET | BPF_K: 1463 /* Accommodate for extra offset in case of a backjump. */ 1464 off = from->off; 1465 if (off < 0) 1466 off -= 2; 1467 if (off < S16_MIN) 1468 return -ERANGE; 1469 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1470 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1471 *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, 1472 off); 1473 break; 1474 1475 case BPF_LD | BPF_IMM | BPF_DW: 1476 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); 1477 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1478 *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 1479 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); 1480 break; 1481 case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ 1482 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); 1483 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1484 if (emit_zext) 1485 *to++ = BPF_ZEXT_REG(BPF_REG_AX); 1486 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); 1487 break; 1488 1489 case BPF_ST | BPF_MEM | BPF_DW: 1490 case BPF_ST | BPF_MEM | BPF_W: 1491 case BPF_ST | BPF_MEM | BPF_H: 1492 case BPF_ST | BPF_MEM | BPF_B: 1493 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1494 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1495 *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); 1496 break; 1497 1498 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1499 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1500 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1501 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1502 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ 1503 from->imm); 1504 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1505 /* 1506 * Cannot use BPF_STX_MEM() macro here as it 1507 * hardcodes BPF_MEM mode, losing PROBE_MEM32 1508 * and breaking arena addressing in the JIT. 1509 */ 1510 *to++ = (struct bpf_insn) { 1511 .code = BPF_STX | BPF_PROBE_MEM32 | 1512 BPF_SIZE(from->code), 1513 .dst_reg = from->dst_reg, 1514 .src_reg = BPF_REG_AX, 1515 .off = from->off, 1516 }; 1517 break; 1518 } 1519 out: 1520 return to - to_buff; 1521 } 1522 1523 static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, 1524 gfp_t gfp_extra_flags) 1525 { 1526 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; 1527 struct bpf_prog *fp; 1528 1529 fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags); 1530 if (fp != NULL) { 1531 /* aux->prog still points to the fp_other one, so 1532 * when promoting the clone to the real program, 1533 * this still needs to be adapted. 1534 */ 1535 memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); 1536 } 1537 1538 return fp; 1539 } 1540 1541 static void bpf_prog_clone_free(struct bpf_prog *fp) 1542 { 1543 /* aux was stolen by the other clone, so we cannot free 1544 * it from this path! It will be freed eventually by the 1545 * other program on release. 1546 * 1547 * At this point, we don't need a deferred release since 1548 * clone is guaranteed to not be locked. 1549 */ 1550 fp->aux = NULL; 1551 fp->stats = NULL; 1552 fp->active = NULL; 1553 __bpf_prog_free(fp); 1554 } 1555 1556 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) 1557 { 1558 /* We have to repoint aux->prog to self, as we don't 1559 * know whether fp here is the clone or the original. 1560 */ 1561 fp->aux->prog = fp; 1562 bpf_prog_dev_bound_moved(fp); 1563 bpf_prog_clone_free(fp_other); 1564 } 1565 1566 /* 1567 * Now this function is used only to blind the main prog and must be invoked only when 1568 * bpf_prog_need_blind() returns true. 1569 */ 1570 struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog) 1571 { 1572 struct bpf_insn insn_buff[16], aux[2]; 1573 struct bpf_prog *clone, *tmp; 1574 int insn_delta, insn_cnt; 1575 struct bpf_insn *insn; 1576 int i, rewritten; 1577 1578 if (WARN_ON_ONCE(env && env->prog != prog)) 1579 return ERR_PTR(-EINVAL); 1580 1581 clone = bpf_prog_clone_create(prog, GFP_USER); 1582 if (!clone) 1583 return ERR_PTR(-ENOMEM); 1584 1585 /* make sure bpf_patch_insn_data() patches the correct prog */ 1586 if (env) 1587 env->prog = clone; 1588 1589 insn_cnt = clone->len; 1590 insn = clone->insnsi; 1591 1592 for (i = 0; i < insn_cnt; i++, insn++) { 1593 if (bpf_pseudo_func(insn)) { 1594 /* ld_imm64 with an address of bpf subprog is not 1595 * a user controlled constant. Don't randomize it, 1596 * since it will conflict with jit_subprogs() logic. 1597 */ 1598 insn++; 1599 i++; 1600 continue; 1601 } 1602 1603 /* We temporarily need to hold the original ld64 insn 1604 * so that we can still access the first part in the 1605 * second blinding run. 1606 */ 1607 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && 1608 insn[1].code == 0) 1609 memcpy(aux, insn, sizeof(aux)); 1610 1611 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, 1612 clone->aux->verifier_zext); 1613 if (!rewritten) 1614 continue; 1615 1616 if (rewritten < 0) 1617 tmp = ERR_PTR(rewritten); 1618 else if (env) 1619 tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); 1620 else 1621 tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); 1622 1623 if (IS_ERR_OR_NULL(tmp)) { 1624 if (env) 1625 /* restore the original prog */ 1626 env->prog = prog; 1627 /* Patching may have repointed aux->prog during 1628 * realloc from the original one, so we need to 1629 * fix it up here on error. 1630 */ 1631 bpf_jit_prog_release_other(prog, clone); 1632 if (env && fatal_signal_pending(current)) 1633 return ERR_PTR(-EINTR); 1634 return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); 1635 } 1636 1637 clone = tmp; 1638 insn_delta = rewritten - 1; 1639 1640 if (env) 1641 env->prog = clone; 1642 1643 /* Walk new program and skip insns we just inserted. */ 1644 insn = clone->insnsi + i + insn_delta; 1645 insn_cnt += insn_delta; 1646 i += insn_delta; 1647 } 1648 1649 clone->blinded = 1; 1650 return clone; 1651 } 1652 1653 bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog, 1654 int insn_idx) 1655 { 1656 if (!env) 1657 return false; 1658 insn_idx += prog->aux->subprog_start; 1659 return env->insn_aux_data[insn_idx].indirect_target; 1660 } 1661 1662 u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog) 1663 { 1664 const struct bpf_subprog_info *sub; 1665 1666 if (!env) 1667 return 0; 1668 sub = &env->subprog_info[prog->aux->func_idx]; 1669 return sub->stack_arg_cnt - bpf_in_stack_arg_cnt(sub); 1670 } 1671 #endif /* CONFIG_BPF_JIT */ 1672 1673 /* Base function for offset calculation. Needs to go into .text section, 1674 * therefore keeping it non-static as well; will also be used by JITs 1675 * anyway later on, so do not let the compiler omit it. This also needs 1676 * to go into kallsyms for correlation from e.g. bpftool, so naming 1677 * must not change. 1678 */ 1679 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1680 { 1681 return 0; 1682 } 1683 1684 /* All UAPI available opcodes. */ 1685 #define BPF_INSN_MAP(INSN_2, INSN_3) \ 1686 /* 32 bit ALU operations. */ \ 1687 /* Register based. */ \ 1688 INSN_3(ALU, ADD, X), \ 1689 INSN_3(ALU, SUB, X), \ 1690 INSN_3(ALU, AND, X), \ 1691 INSN_3(ALU, OR, X), \ 1692 INSN_3(ALU, LSH, X), \ 1693 INSN_3(ALU, RSH, X), \ 1694 INSN_3(ALU, XOR, X), \ 1695 INSN_3(ALU, MUL, X), \ 1696 INSN_3(ALU, MOV, X), \ 1697 INSN_3(ALU, ARSH, X), \ 1698 INSN_3(ALU, DIV, X), \ 1699 INSN_3(ALU, MOD, X), \ 1700 INSN_2(ALU, NEG), \ 1701 INSN_3(ALU, END, TO_BE), \ 1702 INSN_3(ALU, END, TO_LE), \ 1703 /* Immediate based. */ \ 1704 INSN_3(ALU, ADD, K), \ 1705 INSN_3(ALU, SUB, K), \ 1706 INSN_3(ALU, AND, K), \ 1707 INSN_3(ALU, OR, K), \ 1708 INSN_3(ALU, LSH, K), \ 1709 INSN_3(ALU, RSH, K), \ 1710 INSN_3(ALU, XOR, K), \ 1711 INSN_3(ALU, MUL, K), \ 1712 INSN_3(ALU, MOV, K), \ 1713 INSN_3(ALU, ARSH, K), \ 1714 INSN_3(ALU, DIV, K), \ 1715 INSN_3(ALU, MOD, K), \ 1716 /* 64 bit ALU operations. */ \ 1717 /* Register based. */ \ 1718 INSN_3(ALU64, ADD, X), \ 1719 INSN_3(ALU64, SUB, X), \ 1720 INSN_3(ALU64, AND, X), \ 1721 INSN_3(ALU64, OR, X), \ 1722 INSN_3(ALU64, LSH, X), \ 1723 INSN_3(ALU64, RSH, X), \ 1724 INSN_3(ALU64, XOR, X), \ 1725 INSN_3(ALU64, MUL, X), \ 1726 INSN_3(ALU64, MOV, X), \ 1727 INSN_3(ALU64, ARSH, X), \ 1728 INSN_3(ALU64, DIV, X), \ 1729 INSN_3(ALU64, MOD, X), \ 1730 INSN_2(ALU64, NEG), \ 1731 INSN_3(ALU64, END, TO_LE), \ 1732 /* Immediate based. */ \ 1733 INSN_3(ALU64, ADD, K), \ 1734 INSN_3(ALU64, SUB, K), \ 1735 INSN_3(ALU64, AND, K), \ 1736 INSN_3(ALU64, OR, K), \ 1737 INSN_3(ALU64, LSH, K), \ 1738 INSN_3(ALU64, RSH, K), \ 1739 INSN_3(ALU64, XOR, K), \ 1740 INSN_3(ALU64, MUL, K), \ 1741 INSN_3(ALU64, MOV, K), \ 1742 INSN_3(ALU64, ARSH, K), \ 1743 INSN_3(ALU64, DIV, K), \ 1744 INSN_3(ALU64, MOD, K), \ 1745 /* Call instruction. */ \ 1746 INSN_2(JMP, CALL), \ 1747 /* Exit instruction. */ \ 1748 INSN_2(JMP, EXIT), \ 1749 /* 32-bit Jump instructions. */ \ 1750 /* Register based. */ \ 1751 INSN_3(JMP32, JEQ, X), \ 1752 INSN_3(JMP32, JNE, X), \ 1753 INSN_3(JMP32, JGT, X), \ 1754 INSN_3(JMP32, JLT, X), \ 1755 INSN_3(JMP32, JGE, X), \ 1756 INSN_3(JMP32, JLE, X), \ 1757 INSN_3(JMP32, JSGT, X), \ 1758 INSN_3(JMP32, JSLT, X), \ 1759 INSN_3(JMP32, JSGE, X), \ 1760 INSN_3(JMP32, JSLE, X), \ 1761 INSN_3(JMP32, JSET, X), \ 1762 /* Immediate based. */ \ 1763 INSN_3(JMP32, JEQ, K), \ 1764 INSN_3(JMP32, JNE, K), \ 1765 INSN_3(JMP32, JGT, K), \ 1766 INSN_3(JMP32, JLT, K), \ 1767 INSN_3(JMP32, JGE, K), \ 1768 INSN_3(JMP32, JLE, K), \ 1769 INSN_3(JMP32, JSGT, K), \ 1770 INSN_3(JMP32, JSLT, K), \ 1771 INSN_3(JMP32, JSGE, K), \ 1772 INSN_3(JMP32, JSLE, K), \ 1773 INSN_3(JMP32, JSET, K), \ 1774 /* Jump instructions. */ \ 1775 /* Register based. */ \ 1776 INSN_3(JMP, JEQ, X), \ 1777 INSN_3(JMP, JNE, X), \ 1778 INSN_3(JMP, JGT, X), \ 1779 INSN_3(JMP, JLT, X), \ 1780 INSN_3(JMP, JGE, X), \ 1781 INSN_3(JMP, JLE, X), \ 1782 INSN_3(JMP, JSGT, X), \ 1783 INSN_3(JMP, JSLT, X), \ 1784 INSN_3(JMP, JSGE, X), \ 1785 INSN_3(JMP, JSLE, X), \ 1786 INSN_3(JMP, JSET, X), \ 1787 /* Immediate based. */ \ 1788 INSN_3(JMP, JEQ, K), \ 1789 INSN_3(JMP, JNE, K), \ 1790 INSN_3(JMP, JGT, K), \ 1791 INSN_3(JMP, JLT, K), \ 1792 INSN_3(JMP, JGE, K), \ 1793 INSN_3(JMP, JLE, K), \ 1794 INSN_3(JMP, JSGT, K), \ 1795 INSN_3(JMP, JSLT, K), \ 1796 INSN_3(JMP, JSGE, K), \ 1797 INSN_3(JMP, JSLE, K), \ 1798 INSN_3(JMP, JSET, K), \ 1799 INSN_2(JMP, JA), \ 1800 INSN_2(JMP32, JA), \ 1801 /* Atomic operations. */ \ 1802 INSN_3(STX, ATOMIC, B), \ 1803 INSN_3(STX, ATOMIC, H), \ 1804 INSN_3(STX, ATOMIC, W), \ 1805 INSN_3(STX, ATOMIC, DW), \ 1806 /* Store instructions. */ \ 1807 /* Register based. */ \ 1808 INSN_3(STX, MEM, B), \ 1809 INSN_3(STX, MEM, H), \ 1810 INSN_3(STX, MEM, W), \ 1811 INSN_3(STX, MEM, DW), \ 1812 /* Immediate based. */ \ 1813 INSN_3(ST, MEM, B), \ 1814 INSN_3(ST, MEM, H), \ 1815 INSN_3(ST, MEM, W), \ 1816 INSN_3(ST, MEM, DW), \ 1817 /* Load instructions. */ \ 1818 /* Register based. */ \ 1819 INSN_3(LDX, MEM, B), \ 1820 INSN_3(LDX, MEM, H), \ 1821 INSN_3(LDX, MEM, W), \ 1822 INSN_3(LDX, MEM, DW), \ 1823 INSN_3(LDX, MEMSX, B), \ 1824 INSN_3(LDX, MEMSX, H), \ 1825 INSN_3(LDX, MEMSX, W), \ 1826 /* Immediate based. */ \ 1827 INSN_3(LD, IMM, DW) 1828 1829 bool bpf_opcode_in_insntable(u8 code) 1830 { 1831 #define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true 1832 #define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true 1833 static const bool public_insntable[256] = { 1834 [0 ... 255] = false, 1835 /* Now overwrite non-defaults ... */ 1836 BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), 1837 /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ 1838 [BPF_LD | BPF_ABS | BPF_B] = true, 1839 [BPF_LD | BPF_ABS | BPF_H] = true, 1840 [BPF_LD | BPF_ABS | BPF_W] = true, 1841 [BPF_LD | BPF_IND | BPF_B] = true, 1842 [BPF_LD | BPF_IND | BPF_H] = true, 1843 [BPF_LD | BPF_IND | BPF_W] = true, 1844 [BPF_JMP | BPF_JA | BPF_X] = true, 1845 [BPF_JMP | BPF_CALL | BPF_X] = true, 1846 [BPF_JMP | BPF_JCOND] = true, 1847 }; 1848 #undef BPF_INSN_3_TBL 1849 #undef BPF_INSN_2_TBL 1850 return public_insntable[code]; 1851 } 1852 1853 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1854 /* Absolute value of s32 without undefined behavior for S32_MIN */ 1855 static u32 abs_s32(s32 x) 1856 { 1857 return x >= 0 ? (u32)x : -(u32)x; 1858 } 1859 1860 static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, 1861 const struct bpf_insn *insn); 1862 1863 /** 1864 * ___bpf_prog_run - run eBPF program on a given context 1865 * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers 1866 * @insn: is the array of eBPF instructions 1867 * 1868 * Decode and execute eBPF instructions. 1869 * 1870 * Return: whatever value is in %BPF_R0 at program exit 1871 */ 1872 static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) 1873 { 1874 #define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y 1875 #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z 1876 static const void * const jumptable[256] __annotate_jump_table = { 1877 [0 ... 255] = &&default_label, 1878 /* Now overwrite non-defaults ... */ 1879 BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), 1880 /* Non-UAPI available opcodes. */ 1881 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, 1882 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, 1883 [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC, 1884 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B, 1885 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H, 1886 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W, 1887 [BPF_LDX | BPF_PROBE_MEM | BPF_DW] = &&LDX_PROBE_MEM_DW, 1888 [BPF_LDX | BPF_PROBE_MEMSX | BPF_B] = &&LDX_PROBE_MEMSX_B, 1889 [BPF_LDX | BPF_PROBE_MEMSX | BPF_H] = &&LDX_PROBE_MEMSX_H, 1890 [BPF_LDX | BPF_PROBE_MEMSX | BPF_W] = &&LDX_PROBE_MEMSX_W, 1891 }; 1892 #undef BPF_INSN_3_LBL 1893 #undef BPF_INSN_2_LBL 1894 u32 tail_call_cnt = 0; 1895 1896 #define CONT ({ insn++; goto select_insn; }) 1897 #define CONT_JMP ({ insn++; goto select_insn; }) 1898 1899 select_insn: 1900 goto *jumptable[insn->code]; 1901 1902 /* Explicitly mask the register-based shift amounts with 63 or 31 1903 * to avoid undefined behavior. Normally this won't affect the 1904 * generated code, for example, in case of native 64 bit archs such 1905 * as x86-64 or arm64, the compiler is optimizing the AND away for 1906 * the interpreter. In case of JITs, each of the JIT backends compiles 1907 * the BPF shift operations to machine instructions which produce 1908 * implementation-defined results in such a case; the resulting 1909 * contents of the register may be arbitrary, but program behaviour 1910 * as a whole remains defined. In other words, in case of JIT backends, 1911 * the AND must /not/ be added to the emitted LSH/RSH/ARSH translation. 1912 */ 1913 /* ALU (shifts) */ 1914 #define SHT(OPCODE, OP) \ 1915 ALU64_##OPCODE##_X: \ 1916 DST = DST OP (SRC & 63); \ 1917 CONT; \ 1918 ALU_##OPCODE##_X: \ 1919 DST = (u32) DST OP ((u32) SRC & 31); \ 1920 CONT; \ 1921 ALU64_##OPCODE##_K: \ 1922 DST = DST OP IMM; \ 1923 CONT; \ 1924 ALU_##OPCODE##_K: \ 1925 DST = (u32) DST OP (u32) IMM; \ 1926 CONT; 1927 /* ALU (rest) */ 1928 #define ALU(OPCODE, OP) \ 1929 ALU64_##OPCODE##_X: \ 1930 DST = DST OP SRC; \ 1931 CONT; \ 1932 ALU_##OPCODE##_X: \ 1933 DST = (u32) DST OP (u32) SRC; \ 1934 CONT; \ 1935 ALU64_##OPCODE##_K: \ 1936 DST = DST OP IMM; \ 1937 CONT; \ 1938 ALU_##OPCODE##_K: \ 1939 DST = (u32) DST OP (u32) IMM; \ 1940 CONT; 1941 ALU(ADD, +) 1942 ALU(SUB, -) 1943 ALU(AND, &) 1944 ALU(OR, |) 1945 ALU(XOR, ^) 1946 ALU(MUL, *) 1947 SHT(LSH, <<) 1948 SHT(RSH, >>) 1949 #undef SHT 1950 #undef ALU 1951 ALU_NEG: 1952 DST = (u32) -DST; 1953 CONT; 1954 ALU64_NEG: 1955 DST = -DST; 1956 CONT; 1957 ALU_MOV_X: 1958 switch (OFF) { 1959 case 0: 1960 DST = (u32) SRC; 1961 break; 1962 case 8: 1963 DST = (u32)(s8) SRC; 1964 break; 1965 case 16: 1966 DST = (u32)(s16) SRC; 1967 break; 1968 } 1969 CONT; 1970 ALU_MOV_K: 1971 DST = (u32) IMM; 1972 CONT; 1973 ALU64_MOV_X: 1974 switch (OFF) { 1975 case 0: 1976 DST = SRC; 1977 break; 1978 case 8: 1979 DST = (s8) SRC; 1980 break; 1981 case 16: 1982 DST = (s16) SRC; 1983 break; 1984 case 32: 1985 DST = (s32) SRC; 1986 break; 1987 } 1988 CONT; 1989 ALU64_MOV_K: 1990 DST = IMM; 1991 CONT; 1992 LD_IMM_DW: 1993 DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; 1994 insn++; 1995 CONT; 1996 ALU_ARSH_X: 1997 DST = (u64) (u32) (((s32) DST) >> (SRC & 31)); 1998 CONT; 1999 ALU_ARSH_K: 2000 DST = (u64) (u32) (((s32) DST) >> IMM); 2001 CONT; 2002 ALU64_ARSH_X: 2003 (*(s64 *) &DST) >>= (SRC & 63); 2004 CONT; 2005 ALU64_ARSH_K: 2006 (*(s64 *) &DST) >>= IMM; 2007 CONT; 2008 ALU64_MOD_X: 2009 switch (OFF) { 2010 case 0: 2011 div64_u64_rem(DST, SRC, &AX); 2012 DST = AX; 2013 break; 2014 case 1: 2015 AX = div64_s64(DST, SRC); 2016 DST = DST - AX * SRC; 2017 break; 2018 } 2019 CONT; 2020 ALU_MOD_X: 2021 switch (OFF) { 2022 case 0: 2023 AX = (u32) DST; 2024 DST = do_div(AX, (u32) SRC); 2025 break; 2026 case 1: 2027 AX = abs_s32((s32)DST); 2028 AX = do_div(AX, abs_s32((s32)SRC)); 2029 if ((s32)DST < 0) 2030 DST = (u32)-AX; 2031 else 2032 DST = (u32)AX; 2033 break; 2034 } 2035 CONT; 2036 ALU64_MOD_K: 2037 switch (OFF) { 2038 case 0: 2039 div64_u64_rem(DST, IMM, &AX); 2040 DST = AX; 2041 break; 2042 case 1: 2043 AX = div64_s64(DST, IMM); 2044 DST = DST - AX * IMM; 2045 break; 2046 } 2047 CONT; 2048 ALU_MOD_K: 2049 switch (OFF) { 2050 case 0: 2051 AX = (u32) DST; 2052 DST = do_div(AX, (u32) IMM); 2053 break; 2054 case 1: 2055 AX = abs_s32((s32)DST); 2056 AX = do_div(AX, abs_s32((s32)IMM)); 2057 if ((s32)DST < 0) 2058 DST = (u32)-AX; 2059 else 2060 DST = (u32)AX; 2061 break; 2062 } 2063 CONT; 2064 ALU64_DIV_X: 2065 switch (OFF) { 2066 case 0: 2067 DST = div64_u64(DST, SRC); 2068 break; 2069 case 1: 2070 DST = div64_s64(DST, SRC); 2071 break; 2072 } 2073 CONT; 2074 ALU_DIV_X: 2075 switch (OFF) { 2076 case 0: 2077 AX = (u32) DST; 2078 do_div(AX, (u32) SRC); 2079 DST = (u32) AX; 2080 break; 2081 case 1: 2082 AX = abs_s32((s32)DST); 2083 do_div(AX, abs_s32((s32)SRC)); 2084 if (((s32)DST < 0) == ((s32)SRC < 0)) 2085 DST = (u32)AX; 2086 else 2087 DST = (u32)-AX; 2088 break; 2089 } 2090 CONT; 2091 ALU64_DIV_K: 2092 switch (OFF) { 2093 case 0: 2094 DST = div64_u64(DST, IMM); 2095 break; 2096 case 1: 2097 DST = div64_s64(DST, IMM); 2098 break; 2099 } 2100 CONT; 2101 ALU_DIV_K: 2102 switch (OFF) { 2103 case 0: 2104 AX = (u32) DST; 2105 do_div(AX, (u32) IMM); 2106 DST = (u32) AX; 2107 break; 2108 case 1: 2109 AX = abs_s32((s32)DST); 2110 do_div(AX, abs_s32((s32)IMM)); 2111 if (((s32)DST < 0) == ((s32)IMM < 0)) 2112 DST = (u32)AX; 2113 else 2114 DST = (u32)-AX; 2115 break; 2116 } 2117 CONT; 2118 ALU_END_TO_BE: 2119 switch (IMM) { 2120 case 16: 2121 DST = (__force u16) cpu_to_be16(DST); 2122 break; 2123 case 32: 2124 DST = (__force u32) cpu_to_be32(DST); 2125 break; 2126 case 64: 2127 DST = (__force u64) cpu_to_be64(DST); 2128 break; 2129 } 2130 CONT; 2131 ALU_END_TO_LE: 2132 switch (IMM) { 2133 case 16: 2134 DST = (__force u16) cpu_to_le16(DST); 2135 break; 2136 case 32: 2137 DST = (__force u32) cpu_to_le32(DST); 2138 break; 2139 case 64: 2140 DST = (__force u64) cpu_to_le64(DST); 2141 break; 2142 } 2143 CONT; 2144 ALU64_END_TO_LE: 2145 switch (IMM) { 2146 case 16: 2147 DST = (__force u16) __swab16(DST); 2148 break; 2149 case 32: 2150 DST = (__force u32) __swab32(DST); 2151 break; 2152 case 64: 2153 DST = (__force u64) __swab64(DST); 2154 break; 2155 } 2156 CONT; 2157 2158 /* CALL */ 2159 JMP_CALL: 2160 /* Function call scratches BPF_R1-BPF_R5 registers, 2161 * preserves BPF_R6-BPF_R9, and stores return value 2162 * into BPF_R0. 2163 */ 2164 BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3, 2165 BPF_R4, BPF_R5); 2166 CONT; 2167 2168 JMP_CALL_ARGS: 2169 BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3, 2170 BPF_R4, BPF_R5, 2171 insn + insn->imm + 1); 2172 CONT; 2173 2174 JMP_TAIL_CALL: { 2175 struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; 2176 struct bpf_array *array = container_of(map, struct bpf_array, map); 2177 struct bpf_prog *prog; 2178 u32 index = BPF_R3; 2179 2180 if (unlikely(index >= array->map.max_entries)) 2181 goto out; 2182 2183 if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT)) 2184 goto out; 2185 2186 prog = READ_ONCE(array->ptrs[index]); 2187 if (!prog) 2188 goto out; 2189 2190 tail_call_cnt++; 2191 2192 /* ARG1 at this point is guaranteed to point to CTX from 2193 * the verifier side due to the fact that the tail call is 2194 * handled like a helper, that is, bpf_tail_call_proto, 2195 * where arg1_type is ARG_PTR_TO_CTX. 2196 */ 2197 insn = prog->insnsi; 2198 goto select_insn; 2199 out: 2200 CONT; 2201 } 2202 JMP_JA: 2203 insn += insn->off; 2204 CONT; 2205 JMP32_JA: 2206 insn += insn->imm; 2207 CONT; 2208 JMP_EXIT: 2209 return BPF_R0; 2210 /* JMP */ 2211 #define COND_JMP(SIGN, OPCODE, CMP_OP) \ 2212 JMP_##OPCODE##_X: \ 2213 if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ 2214 insn += insn->off; \ 2215 CONT_JMP; \ 2216 } \ 2217 CONT; \ 2218 JMP32_##OPCODE##_X: \ 2219 if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ 2220 insn += insn->off; \ 2221 CONT_JMP; \ 2222 } \ 2223 CONT; \ 2224 JMP_##OPCODE##_K: \ 2225 if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ 2226 insn += insn->off; \ 2227 CONT_JMP; \ 2228 } \ 2229 CONT; \ 2230 JMP32_##OPCODE##_K: \ 2231 if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ 2232 insn += insn->off; \ 2233 CONT_JMP; \ 2234 } \ 2235 CONT; 2236 COND_JMP(u, JEQ, ==) 2237 COND_JMP(u, JNE, !=) 2238 COND_JMP(u, JGT, >) 2239 COND_JMP(u, JLT, <) 2240 COND_JMP(u, JGE, >=) 2241 COND_JMP(u, JLE, <=) 2242 COND_JMP(u, JSET, &) 2243 COND_JMP(s, JSGT, >) 2244 COND_JMP(s, JSLT, <) 2245 COND_JMP(s, JSGE, >=) 2246 COND_JMP(s, JSLE, <=) 2247 #undef COND_JMP 2248 /* ST, STX and LDX*/ 2249 ST_NOSPEC: 2250 /* Speculation barrier for mitigating Speculative Store Bypass, 2251 * Bounds-Check Bypass and Type Confusion. In case of arm64, we 2252 * rely on the firmware mitigation as controlled via the ssbd 2253 * kernel parameter. Whenever the mitigation is enabled, it 2254 * works for all of the kernel code with no need to provide any 2255 * additional instructions here. In case of x86, we use 'lfence' 2256 * insn for mitigation. We reuse preexisting logic from Spectre 2257 * v1 mitigation that happens to produce the required code on 2258 * x86 for v4 as well. 2259 */ 2260 barrier_nospec(); 2261 CONT; 2262 #define LDST(SIZEOP, SIZE) \ 2263 STX_MEM_##SIZEOP: \ 2264 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ 2265 CONT; \ 2266 ST_MEM_##SIZEOP: \ 2267 *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \ 2268 CONT; \ 2269 LDX_MEM_##SIZEOP: \ 2270 DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ 2271 CONT; \ 2272 LDX_PROBE_MEM_##SIZEOP: \ 2273 bpf_probe_read_kernel_common(&DST, sizeof(SIZE), \ 2274 (const void *)(long) (SRC + insn->off)); \ 2275 DST = *((SIZE *)&DST); \ 2276 CONT; 2277 2278 LDST(B, u8) 2279 LDST(H, u16) 2280 LDST(W, u32) 2281 LDST(DW, u64) 2282 #undef LDST 2283 2284 #define LDSX(SIZEOP, SIZE) \ 2285 LDX_MEMSX_##SIZEOP: \ 2286 DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ 2287 CONT; \ 2288 LDX_PROBE_MEMSX_##SIZEOP: \ 2289 bpf_probe_read_kernel_common(&DST, sizeof(SIZE), \ 2290 (const void *)(long) (SRC + insn->off)); \ 2291 DST = *((SIZE *)&DST); \ 2292 CONT; 2293 2294 LDSX(B, s8) 2295 LDSX(H, s16) 2296 LDSX(W, s32) 2297 #undef LDSX 2298 2299 #define ATOMIC_ALU_OP(BOP, KOP) \ 2300 case BOP: \ 2301 if (BPF_SIZE(insn->code) == BPF_W) \ 2302 atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \ 2303 (DST + insn->off)); \ 2304 else if (BPF_SIZE(insn->code) == BPF_DW) \ 2305 atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \ 2306 (DST + insn->off)); \ 2307 else \ 2308 goto default_label; \ 2309 break; \ 2310 case BOP | BPF_FETCH: \ 2311 if (BPF_SIZE(insn->code) == BPF_W) \ 2312 SRC = (u32) atomic_fetch_##KOP( \ 2313 (u32) SRC, \ 2314 (atomic_t *)(unsigned long) (DST + insn->off)); \ 2315 else if (BPF_SIZE(insn->code) == BPF_DW) \ 2316 SRC = (u64) atomic64_fetch_##KOP( \ 2317 (u64) SRC, \ 2318 (atomic64_t *)(unsigned long) (DST + insn->off)); \ 2319 else \ 2320 goto default_label; \ 2321 break; 2322 2323 STX_ATOMIC_DW: 2324 STX_ATOMIC_W: 2325 STX_ATOMIC_H: 2326 STX_ATOMIC_B: 2327 switch (IMM) { 2328 /* Atomic read-modify-write instructions support only W and DW 2329 * size modifiers. 2330 */ 2331 ATOMIC_ALU_OP(BPF_ADD, add) 2332 ATOMIC_ALU_OP(BPF_AND, and) 2333 ATOMIC_ALU_OP(BPF_OR, or) 2334 ATOMIC_ALU_OP(BPF_XOR, xor) 2335 #undef ATOMIC_ALU_OP 2336 2337 case BPF_XCHG: 2338 if (BPF_SIZE(insn->code) == BPF_W) 2339 SRC = (u32) atomic_xchg( 2340 (atomic_t *)(unsigned long) (DST + insn->off), 2341 (u32) SRC); 2342 else if (BPF_SIZE(insn->code) == BPF_DW) 2343 SRC = (u64) atomic64_xchg( 2344 (atomic64_t *)(unsigned long) (DST + insn->off), 2345 (u64) SRC); 2346 else 2347 goto default_label; 2348 break; 2349 case BPF_CMPXCHG: 2350 if (BPF_SIZE(insn->code) == BPF_W) 2351 BPF_R0 = (u32) atomic_cmpxchg( 2352 (atomic_t *)(unsigned long) (DST + insn->off), 2353 (u32) BPF_R0, (u32) SRC); 2354 else if (BPF_SIZE(insn->code) == BPF_DW) 2355 BPF_R0 = (u64) atomic64_cmpxchg( 2356 (atomic64_t *)(unsigned long) (DST + insn->off), 2357 (u64) BPF_R0, (u64) SRC); 2358 else 2359 goto default_label; 2360 break; 2361 /* Atomic load and store instructions support all size 2362 * modifiers. 2363 */ 2364 case BPF_LOAD_ACQ: 2365 switch (BPF_SIZE(insn->code)) { 2366 #define LOAD_ACQUIRE(SIZEOP, SIZE) \ 2367 case BPF_##SIZEOP: \ 2368 DST = (SIZE)smp_load_acquire( \ 2369 (SIZE *)(unsigned long)(SRC + insn->off)); \ 2370 break; 2371 LOAD_ACQUIRE(B, u8) 2372 LOAD_ACQUIRE(H, u16) 2373 LOAD_ACQUIRE(W, u32) 2374 #ifdef CONFIG_64BIT 2375 LOAD_ACQUIRE(DW, u64) 2376 #endif 2377 #undef LOAD_ACQUIRE 2378 default: 2379 goto default_label; 2380 } 2381 break; 2382 case BPF_STORE_REL: 2383 switch (BPF_SIZE(insn->code)) { 2384 #define STORE_RELEASE(SIZEOP, SIZE) \ 2385 case BPF_##SIZEOP: \ 2386 smp_store_release( \ 2387 (SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \ 2388 break; 2389 STORE_RELEASE(B, u8) 2390 STORE_RELEASE(H, u16) 2391 STORE_RELEASE(W, u32) 2392 #ifdef CONFIG_64BIT 2393 STORE_RELEASE(DW, u64) 2394 #endif 2395 #undef STORE_RELEASE 2396 default: 2397 goto default_label; 2398 } 2399 break; 2400 2401 default: 2402 goto default_label; 2403 } 2404 CONT; 2405 2406 default_label: 2407 /* If we ever reach this, we have a bug somewhere. Die hard here 2408 * instead of just returning 0; we could be somewhere in a subprog, 2409 * so execution could continue otherwise which we do /not/ want. 2410 * 2411 * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). 2412 */ 2413 pr_warn("BPF interpreter: unknown opcode %02x (imm: 0x%x)\n", 2414 insn->code, insn->imm); 2415 BUG_ON(1); 2416 return 0; 2417 } 2418 2419 #define PROG_NAME(stack_size) __bpf_prog_run##stack_size 2420 #define DEFINE_BPF_PROG_RUN(stack_size) \ 2421 static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ 2422 { \ 2423 u64 stack[stack_size / sizeof(u64)]; \ 2424 u64 regs[MAX_BPF_EXT_REG] = {}; \ 2425 \ 2426 kmsan_unpoison_memory(stack, sizeof(stack)); \ 2427 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 2428 ARG1 = (u64) (unsigned long) ctx; \ 2429 return ___bpf_prog_run(regs, insn); \ 2430 } 2431 2432 #define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size 2433 #define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ 2434 static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ 2435 const struct bpf_insn *insn) \ 2436 { \ 2437 u64 stack[stack_size / sizeof(u64)]; \ 2438 u64 regs[MAX_BPF_EXT_REG]; \ 2439 \ 2440 kmsan_unpoison_memory(stack, sizeof(stack)); \ 2441 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 2442 BPF_R1 = r1; \ 2443 BPF_R2 = r2; \ 2444 BPF_R3 = r3; \ 2445 BPF_R4 = r4; \ 2446 BPF_R5 = r5; \ 2447 return ___bpf_prog_run(regs, insn); \ 2448 } 2449 2450 #define EVAL1(FN, X) FN(X) 2451 #define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) 2452 #define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) 2453 #define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) 2454 #define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) 2455 #define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) 2456 2457 EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); 2458 EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); 2459 EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); 2460 2461 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); 2462 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); 2463 EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); 2464 2465 #define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), 2466 2467 static unsigned int (*interpreters[])(const void *ctx, 2468 const struct bpf_insn *insn) = { 2469 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2470 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2471 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2472 }; 2473 #undef PROG_NAME_LIST 2474 #define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), 2475 static __maybe_unused 2476 u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, 2477 const struct bpf_insn *insn) = { 2478 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2479 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2480 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2481 }; 2482 #undef PROG_NAME_LIST 2483 2484 #ifdef CONFIG_BPF_SYSCALL 2485 int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) 2486 { 2487 stack_depth = max_t(u32, stack_depth, 1); 2488 /* Prevent out-of-bounds read to interpreters_args */ 2489 if (stack_depth > MAX_BPF_STACK) 2490 return -EINVAL; 2491 insn->off = (round_up(stack_depth, 32) / 32) - 1; 2492 insn->code = BPF_JMP | BPF_CALL_ARGS; 2493 return 0; 2494 } 2495 2496 s32 bpf_call_args_imm(s16 idx) 2497 { 2498 if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args))) 2499 return 0; 2500 return BPF_CALL_IMM(interpreters_args[idx]); 2501 } 2502 #endif 2503 #endif 2504 2505 static unsigned int __bpf_prog_ret0_warn(const void *ctx, 2506 const struct bpf_insn *insn) 2507 { 2508 /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON 2509 * is not working properly, so warn about it! 2510 */ 2511 WARN_ON_ONCE(1); 2512 return 0; 2513 } 2514 2515 static bool __bpf_prog_map_compatible(struct bpf_map *map, 2516 const struct bpf_prog *fp) 2517 { 2518 enum bpf_prog_type prog_type = resolve_prog_type(fp); 2519 struct bpf_prog_aux *aux = fp->aux; 2520 enum bpf_cgroup_storage_type i; 2521 bool ret = false; 2522 u64 cookie; 2523 2524 if (fp->kprobe_override) 2525 return ret; 2526 2527 spin_lock(&map->owner_lock); 2528 /* There's no owner yet where we could check for compatibility. */ 2529 if (!map->owner) { 2530 map->owner = bpf_map_owner_alloc(map); 2531 if (!map->owner) 2532 goto err; 2533 map->owner->type = prog_type; 2534 map->owner->jited = fp->jited; 2535 map->owner->xdp_has_frags = aux->xdp_has_frags; 2536 map->owner->sleepable = fp->sleepable; 2537 map->owner->expected_attach_type = fp->expected_attach_type; 2538 map->owner->attach_func_proto = aux->attach_func_proto; 2539 for_each_cgroup_storage_type(i) { 2540 map->owner->storage_cookie[i] = 2541 aux->cgroup_storage[i] ? 2542 aux->cgroup_storage[i]->cookie : 0; 2543 } 2544 ret = true; 2545 } else { 2546 ret = map->owner->type == prog_type && 2547 map->owner->jited == fp->jited && 2548 map->owner->xdp_has_frags == aux->xdp_has_frags && 2549 map->owner->sleepable == fp->sleepable; 2550 if (ret && 2551 map->map_type == BPF_MAP_TYPE_PROG_ARRAY && 2552 map->owner->expected_attach_type != fp->expected_attach_type) 2553 ret = false; 2554 for_each_cgroup_storage_type(i) { 2555 if (!ret) 2556 break; 2557 cookie = aux->cgroup_storage[i] ? 2558 aux->cgroup_storage[i]->cookie : 0; 2559 ret = map->owner->storage_cookie[i] == cookie || 2560 (!cookie && !aux->tail_call_reachable); 2561 } 2562 if (ret && 2563 map->owner->attach_func_proto != aux->attach_func_proto) { 2564 switch (prog_type) { 2565 case BPF_PROG_TYPE_TRACING: 2566 case BPF_PROG_TYPE_LSM: 2567 case BPF_PROG_TYPE_EXT: 2568 case BPF_PROG_TYPE_STRUCT_OPS: 2569 ret = false; 2570 break; 2571 default: 2572 break; 2573 } 2574 } 2575 } 2576 err: 2577 spin_unlock(&map->owner_lock); 2578 return ret; 2579 } 2580 2581 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp) 2582 { 2583 /* XDP programs inserted into maps are not guaranteed to run on 2584 * a particular netdev (and can run outside driver context entirely 2585 * in the case of devmap and cpumap). Until device checks 2586 * are implemented, prohibit adding dev-bound programs to program maps. 2587 */ 2588 if (bpf_prog_is_dev_bound(fp->aux)) 2589 return false; 2590 2591 return __bpf_prog_map_compatible(map, fp); 2592 } 2593 2594 static int bpf_check_tail_call(const struct bpf_prog *fp) 2595 { 2596 struct bpf_prog_aux *aux = fp->aux; 2597 int i, ret = 0; 2598 2599 mutex_lock(&aux->used_maps_mutex); 2600 for (i = 0; i < aux->used_map_cnt; i++) { 2601 struct bpf_map *map = aux->used_maps[i]; 2602 2603 if (!map_type_contains_progs(map)) 2604 continue; 2605 2606 if (!__bpf_prog_map_compatible(map, fp)) { 2607 ret = -EINVAL; 2608 goto out; 2609 } 2610 } 2611 2612 out: 2613 mutex_unlock(&aux->used_maps_mutex); 2614 return ret; 2615 } 2616 2617 static bool bpf_prog_select_interpreter(struct bpf_prog *fp) 2618 { 2619 bool select_interpreter = false; 2620 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2621 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); 2622 u32 idx = (round_up(stack_depth, 32) / 32) - 1; 2623 2624 /* may_goto may cause stack size > 512, leading to idx out-of-bounds. 2625 * But for non-JITed programs, we don't need bpf_func, so no bounds 2626 * check needed. 2627 */ 2628 if (idx < ARRAY_SIZE(interpreters)) { 2629 fp->bpf_func = interpreters[idx]; 2630 select_interpreter = true; 2631 } else { 2632 fp->bpf_func = __bpf_prog_ret0_warn; 2633 } 2634 #else 2635 fp->bpf_func = __bpf_prog_ret0_warn; 2636 #endif 2637 return select_interpreter; 2638 } 2639 2640 /* 2641 * Whether a function has an insn that works with the arena base, which a JIT 2642 * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on 2643 * the final insns of the function: an access through an arena pointer, or 2644 * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been 2645 * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address 2646 * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, 2647 * which the JIT rebases, has a function model that says so. A load of an 2648 * __arena struct member is rewritten with an immediate and needs none; 2649 * dereferencing the result does. 2650 */ 2651 bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) 2652 { 2653 const struct bpf_insn *insn = prog->insnsi; 2654 const struct btf_func_model *m; 2655 int i, j; 2656 2657 if (!prog->aux->arena) 2658 return false; 2659 2660 for (i = 0; i < prog->len; i++, insn++) { 2661 switch (BPF_CLASS(insn->code)) { 2662 case BPF_LDX: 2663 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 2664 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) 2665 return true; 2666 break; 2667 case BPF_ST: 2668 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) 2669 return true; 2670 break; 2671 case BPF_STX: 2672 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || 2673 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) 2674 return true; 2675 break; 2676 case BPF_ALU64: 2677 if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && 2678 insn->off == BPF_ADDR_SPACE_CAST) 2679 return true; 2680 break; 2681 case BPF_JMP: 2682 if (!bpf_pseudo_kfunc_call(insn)) 2683 break; 2684 m = bpf_jit_find_kfunc_model(prog, insn); 2685 if (!m) 2686 break; 2687 for (j = 0; j < m->nr_args; j++) 2688 if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) 2689 return true; 2690 break; 2691 } 2692 } 2693 return false; 2694 } 2695 2696 static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2697 { 2698 #ifdef CONFIG_BPF_JIT 2699 struct bpf_prog *orig_prog; 2700 2701 prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); 2702 if (!bpf_prog_need_blind(prog)) 2703 return bpf_int_jit_compile(env, prog); 2704 2705 orig_prog = prog; 2706 prog = bpf_jit_blind_constants(env, prog); 2707 /* 2708 * Fall back to the interpreter after blinding failures, except when 2709 * the loader was killed. 2710 */ 2711 if (IS_ERR(prog)) { 2712 if (PTR_ERR(prog) == -EINTR) 2713 return prog; 2714 goto out_restore; 2715 } 2716 2717 prog = bpf_int_jit_compile(env, prog); 2718 if (prog->jited) { 2719 bpf_jit_prog_release_other(prog, orig_prog); 2720 return prog; 2721 } 2722 2723 bpf_jit_prog_release_other(orig_prog, prog); 2724 2725 out_restore: 2726 prog = orig_prog; 2727 #endif 2728 return prog; 2729 } 2730 2731 struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, 2732 int *err) 2733 { 2734 struct bpf_prog *jit_prog; 2735 2736 /* In case of BPF to BPF calls, verifier did all the prep 2737 * work with regards to JITing, etc. 2738 */ 2739 bool jit_needed = fp->jit_required; 2740 2741 if (fp->bpf_func) 2742 goto finalize; 2743 2744 if (!bpf_prog_select_interpreter(fp)) 2745 jit_needed = true; 2746 2747 /* eBPF JITs can rewrite the program in case constant 2748 * blinding is active. However, in case of error during 2749 * blinding, bpf_int_jit_compile() must always return a 2750 * valid program, which in this case would simply not 2751 * be JITed, but falls back to the interpreter. 2752 */ 2753 if (!bpf_prog_is_offloaded(fp->aux)) { 2754 *err = bpf_prog_alloc_jited_linfo(fp); 2755 if (*err) 2756 return fp; 2757 2758 jit_prog = bpf_prog_jit_compile(env, fp); 2759 if (IS_ERR(jit_prog)) { 2760 *err = PTR_ERR(jit_prog); 2761 return fp; 2762 } 2763 fp = jit_prog; 2764 bpf_prog_jit_attempt_done(fp); 2765 if (!fp->jited && jit_needed) { 2766 *err = -ENOTSUPP; 2767 return fp; 2768 } 2769 } else { 2770 *err = bpf_prog_offload_compile(fp); 2771 if (*err) 2772 return fp; 2773 } 2774 2775 finalize: 2776 *err = bpf_prog_lock_ro(fp); 2777 if (*err) 2778 return fp; 2779 2780 /* The tail call compatibility check can only be done at 2781 * this late stage as we need to determine, if we deal 2782 * with JITed or non JITed program concatenations and not 2783 * all eBPF JITs might immediately support all features. 2784 */ 2785 *err = bpf_check_tail_call(fp); 2786 2787 return fp; 2788 } 2789 2790 /** 2791 * bpf_prog_select_runtime - select exec runtime for BPF program 2792 * @fp: bpf_prog populated with BPF program 2793 * @err: pointer to error variable 2794 * 2795 * Try to JIT eBPF program, if JIT is not available, use interpreter. 2796 * The BPF program will be executed via bpf_prog_run() function. 2797 * 2798 * Return: the &fp argument along with &err set to 0 for success or 2799 * a negative errno code on failure 2800 */ 2801 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) 2802 { 2803 return __bpf_prog_select_runtime(NULL, fp, err); 2804 } 2805 2806 static unsigned int __bpf_prog_ret1(const void *ctx, 2807 const struct bpf_insn *insn) 2808 { 2809 return 1; 2810 } 2811 2812 static struct bpf_prog_dummy { 2813 struct bpf_prog prog; 2814 } dummy_bpf_prog = { 2815 .prog = { 2816 .bpf_func = __bpf_prog_ret1, 2817 }, 2818 }; 2819 2820 struct bpf_prog_array bpf_empty_prog_array = { 2821 .items = { 2822 { .prog = NULL }, 2823 }, 2824 }; 2825 EXPORT_SYMBOL(bpf_empty_prog_array); 2826 2827 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) 2828 { 2829 struct bpf_prog_array *p; 2830 2831 if (prog_cnt) 2832 p = kzalloc_flex(*p, items, prog_cnt + 1, flags); 2833 else 2834 p = &bpf_empty_prog_array; 2835 2836 return p; 2837 } 2838 2839 void bpf_prog_array_free(struct bpf_prog_array *progs) 2840 { 2841 if (!progs || progs == &bpf_empty_prog_array) 2842 return; 2843 kfree_rcu(progs, rcu); 2844 } 2845 2846 static void __bpf_prog_array_free_sleepable_cb(struct rcu_head *rcu) 2847 { 2848 struct bpf_prog_array *progs; 2849 2850 /* 2851 * RCU Tasks Trace grace period implies RCU grace period, there is no 2852 * need to call kfree_rcu(), just call kfree() directly. 2853 */ 2854 progs = container_of(rcu, struct bpf_prog_array, rcu); 2855 kfree(progs); 2856 } 2857 2858 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) 2859 { 2860 if (!progs || progs == &bpf_empty_prog_array) 2861 return; 2862 call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); 2863 } 2864 2865 struct bpf_prog *bpf_prog_dummy(void) 2866 { 2867 return &dummy_bpf_prog.prog; 2868 } 2869 2870 int bpf_prog_array_length(struct bpf_prog_array *array) 2871 { 2872 struct bpf_prog_array_item *item; 2873 u32 cnt = 0; 2874 2875 for (item = array->items; item->prog; item++) 2876 if (item->prog != &dummy_bpf_prog.prog) 2877 cnt++; 2878 return cnt; 2879 } 2880 2881 bool bpf_prog_array_is_empty(struct bpf_prog_array *array) 2882 { 2883 struct bpf_prog_array_item *item; 2884 2885 for (item = array->items; item->prog; item++) 2886 if (item->prog != &dummy_bpf_prog.prog) 2887 return false; 2888 return true; 2889 } 2890 2891 static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, 2892 u32 *prog_ids, 2893 u32 request_cnt) 2894 { 2895 struct bpf_prog_array_item *item; 2896 int i = 0; 2897 2898 for (item = array->items; item->prog; item++) { 2899 if (item->prog == &dummy_bpf_prog.prog) 2900 continue; 2901 prog_ids[i] = item->prog->aux->id; 2902 if (++i == request_cnt) { 2903 item++; 2904 break; 2905 } 2906 } 2907 2908 return !!(item->prog); 2909 } 2910 2911 int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, 2912 __u32 __user *prog_ids, u32 cnt) 2913 { 2914 unsigned long err = 0; 2915 bool nospc; 2916 u32 *ids; 2917 2918 /* users of this function are doing: 2919 * cnt = bpf_prog_array_length(); 2920 * if (cnt > 0) 2921 * bpf_prog_array_copy_to_user(..., cnt); 2922 * so below kcalloc doesn't need extra cnt > 0 check. 2923 */ 2924 ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); 2925 if (!ids) 2926 return -ENOMEM; 2927 nospc = bpf_prog_array_copy_core(array, ids, cnt); 2928 err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); 2929 kfree(ids); 2930 if (err) 2931 return -EFAULT; 2932 if (nospc) 2933 return -ENOSPC; 2934 return 0; 2935 } 2936 2937 void bpf_prog_array_delete_safe(struct bpf_prog_array *array, 2938 struct bpf_prog *old_prog) 2939 { 2940 struct bpf_prog_array_item *item; 2941 2942 for (item = array->items; item->prog; item++) 2943 if (item->prog == old_prog) { 2944 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); 2945 break; 2946 } 2947 } 2948 2949 /** 2950 * bpf_prog_array_delete_safe_at() - Replaces the program at the given 2951 * index into the program array with 2952 * a dummy no-op program. 2953 * @array: a bpf_prog_array 2954 * @index: the index of the program to replace 2955 * 2956 * Skips over dummy programs, by not counting them, when calculating 2957 * the position of the program to replace. 2958 * 2959 * Return: 2960 * * 0 - Success 2961 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2962 * * -ENOENT - Index out of range 2963 */ 2964 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index) 2965 { 2966 return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog); 2967 } 2968 2969 /** 2970 * bpf_prog_array_update_at() - Updates the program at the given index 2971 * into the program array. 2972 * @array: a bpf_prog_array 2973 * @index: the index of the program to update 2974 * @prog: the program to insert into the array 2975 * 2976 * Skips over dummy programs, by not counting them, when calculating 2977 * the position of the program to update. 2978 * 2979 * Return: 2980 * * 0 - Success 2981 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2982 * * -ENOENT - Index out of range 2983 */ 2984 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 2985 struct bpf_prog *prog) 2986 { 2987 struct bpf_prog_array_item *item; 2988 2989 if (unlikely(index < 0)) 2990 return -EINVAL; 2991 2992 for (item = array->items; item->prog; item++) { 2993 if (item->prog == &dummy_bpf_prog.prog) 2994 continue; 2995 if (!index) { 2996 WRITE_ONCE(item->prog, prog); 2997 return 0; 2998 } 2999 index--; 3000 } 3001 return -ENOENT; 3002 } 3003 3004 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 3005 struct bpf_prog *exclude_prog, 3006 struct bpf_prog *include_prog, 3007 u64 bpf_cookie, 3008 struct bpf_prog_array **new_array) 3009 { 3010 int new_prog_cnt, carry_prog_cnt = 0; 3011 struct bpf_prog_array_item *existing, *new; 3012 struct bpf_prog_array *array; 3013 bool found_exclude = false; 3014 3015 /* Figure out how many existing progs we need to carry over to 3016 * the new array. 3017 */ 3018 if (old_array) { 3019 existing = old_array->items; 3020 for (; existing->prog; existing++) { 3021 if (existing->prog == exclude_prog) { 3022 found_exclude = true; 3023 continue; 3024 } 3025 if (existing->prog != &dummy_bpf_prog.prog) 3026 carry_prog_cnt++; 3027 if (existing->prog == include_prog) 3028 return -EEXIST; 3029 } 3030 } 3031 3032 if (exclude_prog && !found_exclude) 3033 return -ENOENT; 3034 3035 /* How many progs (not NULL) will be in the new array? */ 3036 new_prog_cnt = carry_prog_cnt; 3037 if (include_prog) 3038 new_prog_cnt += 1; 3039 3040 /* Do we have any prog (not NULL) in the new array? */ 3041 if (!new_prog_cnt) { 3042 *new_array = NULL; 3043 return 0; 3044 } 3045 3046 /* +1 as the end of prog_array is marked with NULL */ 3047 array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); 3048 if (!array) 3049 return -ENOMEM; 3050 new = array->items; 3051 3052 /* Fill in the new prog array */ 3053 if (carry_prog_cnt) { 3054 existing = old_array->items; 3055 for (; existing->prog; existing++) { 3056 if (existing->prog == exclude_prog || 3057 existing->prog == &dummy_bpf_prog.prog) 3058 continue; 3059 3060 new->prog = existing->prog; 3061 new->bpf_cookie = existing->bpf_cookie; 3062 new++; 3063 } 3064 } 3065 if (include_prog) { 3066 new->prog = include_prog; 3067 new->bpf_cookie = bpf_cookie; 3068 new++; 3069 } 3070 new->prog = NULL; 3071 *new_array = array; 3072 return 0; 3073 } 3074 3075 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 3076 u32 *prog_ids, u32 request_cnt, 3077 u32 *prog_cnt) 3078 { 3079 u32 cnt = 0; 3080 3081 if (array) 3082 cnt = bpf_prog_array_length(array); 3083 3084 *prog_cnt = cnt; 3085 3086 /* return early if user requested only program count or nothing to copy */ 3087 if (!request_cnt || !cnt) 3088 return 0; 3089 3090 /* this function is called under trace/bpf_trace.c: bpf_event_mutex */ 3091 return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC 3092 : 0; 3093 } 3094 3095 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 3096 struct bpf_map **used_maps, u32 len) 3097 { 3098 struct bpf_map *map; 3099 bool sleepable; 3100 u32 i; 3101 3102 sleepable = bpf_prog_runs_sleepable(aux->prog); 3103 for (i = 0; i < len; i++) { 3104 map = used_maps[i]; 3105 if (map->ops->map_poke_untrack) 3106 map->ops->map_poke_untrack(map, aux); 3107 if (sleepable) 3108 atomic64_dec(&map->sleepable_refcnt); 3109 /* 3110 * The program that didn't load is not a user of the map. libbpf 3111 * loads the program again to get the log of the verifier. 3112 */ 3113 cmpxchg(&map->user, (unsigned long)aux, 0); 3114 bpf_map_put(map); 3115 } 3116 } 3117 3118 static void bpf_free_used_maps(struct bpf_prog_aux *aux) 3119 { 3120 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt); 3121 kfree(aux->used_maps); 3122 } 3123 3124 void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len) 3125 { 3126 #ifdef CONFIG_BPF_SYSCALL 3127 struct btf_mod_pair *btf_mod; 3128 u32 i; 3129 3130 for (i = 0; i < len; i++) { 3131 btf_mod = &used_btfs[i]; 3132 if (btf_mod->module) 3133 module_put(btf_mod->module); 3134 btf_put(btf_mod->btf); 3135 } 3136 #endif 3137 } 3138 3139 static void bpf_free_used_btfs(struct bpf_prog_aux *aux) 3140 { 3141 __bpf_free_used_btfs(aux->used_btfs, aux->used_btf_cnt); 3142 kfree(aux->used_btfs); 3143 } 3144 3145 static void bpf_prog_free_images(struct work_struct *work) 3146 { 3147 struct bpf_prog_aux *aux; 3148 int i; 3149 3150 aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); 3151 for (i = 0; i < aux->real_func_cnt; i++) { 3152 /* We can just unlink the subprog poke descriptor table as 3153 * it was originally linked to the main program and is also 3154 * released along with it. 3155 */ 3156 aux->func[i]->aux->poke_tab = NULL; 3157 bpf_jit_free(aux->func[i]); 3158 } 3159 if (aux->real_func_cnt) { 3160 kfree(aux->func); 3161 bpf_prog_unlock_free(aux->prog); 3162 } else { 3163 bpf_jit_free(aux->prog); 3164 } 3165 } 3166 3167 static void bpf_prog_free_deferred(struct work_struct *work) 3168 { 3169 struct bpf_prog_aux *aux; 3170 bool ksyms; 3171 3172 aux = container_of(work, struct bpf_prog_aux, work); 3173 /* 3174 * A prog that still runs when its last reference goes needs its symbols, 3175 * by which its exception table is found. It has ended by now. 3176 */ 3177 ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); 3178 bpf_prog_kallsyms_del_all(aux->prog); 3179 #ifdef CONFIG_BPF_SYSCALL 3180 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); 3181 #endif 3182 #ifdef CONFIG_CGROUP_BPF 3183 if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) 3184 bpf_cgroup_atype_put(aux->cgroup_atype); 3185 #endif 3186 bpf_free_used_maps(aux); 3187 bpf_free_used_btfs(aux); 3188 bpf_prog_disassoc_struct_ops(aux->prog); 3189 if (bpf_prog_is_dev_bound(aux)) 3190 bpf_prog_dev_bound_destroy(aux->prog); 3191 #ifdef CONFIG_PERF_EVENTS 3192 if (aux->prog->has_callchain_buf) 3193 put_callchain_buffers(); 3194 #endif 3195 if (aux->dst_trampoline) 3196 bpf_trampoline_put(aux->dst_trampoline); 3197 /* The symbols are looked up under rcu_read_lock() */ 3198 INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); 3199 if (ksyms) 3200 queue_rcu_work(system_percpu_wq, &aux->rwork); 3201 else 3202 bpf_prog_free_images(&aux->rwork.work); 3203 } 3204 3205 void bpf_prog_free(struct bpf_prog *fp) 3206 { 3207 struct bpf_prog_aux *aux = fp->aux; 3208 3209 if (aux->dst_prog) 3210 bpf_prog_put(aux->dst_prog); 3211 bpf_token_put(aux->token); 3212 INIT_WORK(&aux->work, bpf_prog_free_deferred); 3213 schedule_work(&aux->work); 3214 } 3215 3216 /* RNG for unprivileged user space with separated state from prandom_u32(). */ 3217 static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); 3218 3219 void bpf_user_rnd_init_once(void) 3220 { 3221 prandom_init_once(&bpf_user_rnd_state); 3222 } 3223 3224 BPF_CALL_0(bpf_user_rnd_u32) 3225 { 3226 /* Should someone ever have the rather unwise idea to use some 3227 * of the registers passed into this function, then note that 3228 * this function is called from native eBPF and classic-to-eBPF 3229 * transformations. Register assignments from both sides are 3230 * different, f.e. classic always sets fn(ctx, A, X) here. 3231 */ 3232 struct rnd_state *state; 3233 u32 res; 3234 3235 state = &get_cpu_var(bpf_user_rnd_state); 3236 res = prandom_u32_state(state); 3237 put_cpu_var(bpf_user_rnd_state); 3238 3239 return res; 3240 } 3241 3242 BPF_CALL_0(bpf_get_raw_cpu_id) 3243 { 3244 return raw_smp_processor_id(); 3245 } 3246 3247 /* Weak definitions of helper functions in case we don't have bpf syscall. */ 3248 const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; 3249 const struct bpf_func_proto bpf_map_update_elem_proto __weak; 3250 const struct bpf_func_proto bpf_map_delete_elem_proto __weak; 3251 const struct bpf_func_proto bpf_map_push_elem_proto __weak; 3252 const struct bpf_func_proto bpf_map_pop_elem_proto __weak; 3253 const struct bpf_func_proto bpf_map_peek_elem_proto __weak; 3254 const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto __weak; 3255 const struct bpf_func_proto bpf_spin_lock_proto __weak; 3256 const struct bpf_func_proto bpf_spin_unlock_proto __weak; 3257 const struct bpf_func_proto bpf_jiffies64_proto __weak; 3258 3259 const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; 3260 const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; 3261 const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; 3262 const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; 3263 const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak; 3264 const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak; 3265 const struct bpf_func_proto bpf_ktime_get_tai_ns_proto __weak; 3266 3267 const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; 3268 const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; 3269 const struct bpf_func_proto bpf_get_current_comm_proto __weak; 3270 const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; 3271 const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak; 3272 const struct bpf_func_proto bpf_get_local_storage_proto __weak; 3273 const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak; 3274 const struct bpf_func_proto bpf_snprintf_btf_proto __weak; 3275 const struct bpf_func_proto bpf_seq_printf_btf_proto __weak; 3276 const struct bpf_func_proto bpf_set_retval_proto __weak; 3277 const struct bpf_func_proto bpf_get_retval_proto __weak; 3278 3279 const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) 3280 { 3281 return NULL; 3282 } 3283 3284 const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void) 3285 { 3286 return NULL; 3287 } 3288 3289 const struct bpf_func_proto * __weak bpf_get_perf_event_read_value_proto(void) 3290 { 3291 return NULL; 3292 } 3293 3294 u64 __weak 3295 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 3296 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 3297 { 3298 return -ENOTSUPP; 3299 } 3300 EXPORT_SYMBOL_GPL(bpf_event_output); 3301 3302 /* Always built-in helper functions. */ 3303 const struct bpf_func_proto bpf_tail_call_proto = { 3304 /* func is unused for tail_call, we set it to pass the 3305 * get_helper_proto check 3306 */ 3307 .func = BPF_PTR_POISON, 3308 .gpl_only = false, 3309 .ret_type = RET_VOID, 3310 .arg1_type = ARG_PTR_TO_CTX, 3311 .arg2_type = ARG_CONST_MAP_PTR, 3312 .arg3_type = ARG_ANYTHING, 3313 }; 3314 3315 /* Stub for JITs that only support cBPF. eBPF programs are interpreted. 3316 * It is encouraged to implement bpf_int_jit_compile() instead, so that 3317 * eBPF and implicitly also cBPF can get JITed! 3318 */ 3319 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 3320 { 3321 return prog; 3322 } 3323 3324 /* Stub for JITs that support eBPF. All cBPF code gets transformed into 3325 * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). 3326 */ 3327 void __weak bpf_jit_compile(struct bpf_prog *prog) 3328 { 3329 } 3330 3331 bool __weak bpf_helper_changes_pkt_data(enum bpf_func_id func_id) 3332 { 3333 return false; 3334 } 3335 3336 /* Return TRUE if the JIT backend wants verifier to enable sub-register usage 3337 * analysis code and wants explicit zero extension inserted by verifier. 3338 * Otherwise, return FALSE. 3339 * 3340 * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if 3341 * you don't override this. JITs that don't want these extra insns can detect 3342 * them using insn_is_zext. 3343 */ 3344 bool __weak bpf_jit_needs_zext(void) 3345 { 3346 return false; 3347 } 3348 3349 /* By default, enable the verifier's mitigations against Spectre v1 and v4 for 3350 * all archs. The value returned must not change at runtime as there is 3351 * currently no support for reloading programs that were loaded without 3352 * mitigations. 3353 */ 3354 bool __weak bpf_jit_bypass_spec_v1(void) 3355 { 3356 return false; 3357 } 3358 3359 bool __weak bpf_jit_bypass_spec_v4(void) 3360 { 3361 return false; 3362 } 3363 3364 /* Return true if the JIT inlines the call to the helper corresponding to 3365 * the imm. 3366 * 3367 * The verifier will not patch the insn->imm for the call to the helper if 3368 * this returns true. 3369 */ 3370 bool __weak bpf_jit_inlines_helper_call(s32 imm) 3371 { 3372 return false; 3373 } 3374 3375 /* Return TRUE if the JIT backend supports mixing bpf2bpf and tailcalls. */ 3376 bool __weak bpf_jit_supports_subprog_tailcalls(void) 3377 { 3378 return false; 3379 } 3380 3381 bool __weak bpf_jit_supports_percpu_insn(void) 3382 { 3383 return false; 3384 } 3385 3386 bool __weak bpf_jit_supports_kfunc_call(void) 3387 { 3388 return false; 3389 } 3390 3391 /* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ 3392 bool __weak bpf_jit_supports_callx(void) 3393 { 3394 return false; 3395 } 3396 3397 bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) 3398 { 3399 return false; 3400 } 3401 3402 /* unwind_goto and unwind_exit */ 3403 bool __weak bpf_jit_supports_unwind(void) 3404 { 3405 return false; 3406 } 3407 3408 /* 3409 * How this arch places a by-value kfunc argument, or NULL for one that has 3410 * not opted in and so only takes an argument of a single eightbyte, which 3411 * every convention places in slot order. 3412 */ 3413 const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) 3414 { 3415 return NULL; 3416 } 3417 3418 u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, 3419 const struct btf_func_model *fm, u8 *pos_of_slot) 3420 { 3421 u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; 3422 bool on_stack = false; 3423 3424 for (i = 0; i < fm->nr_args; i++) { 3425 bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; 3426 u32 pos; 3427 3428 nslots = btf_func_model_arg_slots(fm, i); 3429 3430 if (align16 && abi->even_reg_align) 3431 nregs_used = round_up(nregs_used, 2); 3432 3433 if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { 3434 /* wholly in registers */ 3435 pos = nregs_used; 3436 nregs_used += nslots; 3437 } else if (!on_stack && abi->split_at_boundary) { 3438 /* the last registers hold what fits, the stack the rest */ 3439 pos = nregs_used; 3440 stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; 3441 nregs_used = abi->nr_arg_regs; 3442 on_stack = true; 3443 } else { 3444 /* wholly on the stack */ 3445 if (align16 && abi->even_stack_align) 3446 stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); 3447 pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; 3448 stack_off += nslots * BPF_REG_SIZE; 3449 if (!abi->backfill_after_stack) 3450 on_stack = true; 3451 } 3452 3453 for (k = 0; k < nslots; k++) 3454 pos_of_slot[slot + k] = pos + k; 3455 slot += nslots; 3456 } 3457 3458 return slot; 3459 } 3460 3461 u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, 3462 const struct btf_func_model *fm, 3463 struct bpf_jit_arg_move *moves) 3464 { 3465 u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; 3466 u32 nslots, n = 0, s, back; 3467 3468 nslots = bpf_jit_place_args(abi, fm, pos_of_slot); 3469 back = nslots; 3470 3471 /* 3472 * An argument is two eightbytes at most, so it frees one register at 3473 * most and only one argument ever moves down. Its destination is 3474 * still in use, so carry it in the scratch. Only a lower slot can 3475 * take the one it leaves, so the walk reaches it first. 3476 */ 3477 for (s = nslots; s > 0; s--) { 3478 u8 slot = s - 1, pos = pos_of_slot[slot]; 3479 3480 if (pos == slot) 3481 continue; 3482 3483 if (pos < slot) { 3484 moves[n].dst = BPF_JIT_ARG_TMP; 3485 back = slot; 3486 } else { 3487 moves[n].dst = pos; 3488 } 3489 moves[n].src = slot; 3490 n++; 3491 } 3492 3493 if (back < nslots) { 3494 moves[n].dst = pos_of_slot[back]; 3495 moves[n].src = BPF_JIT_ARG_TMP; 3496 n++; 3497 } 3498 3499 return n; 3500 } 3501 3502 bool __weak bpf_jit_supports_stack_args(void) 3503 { 3504 return false; 3505 } 3506 3507 bool __weak bpf_jit_supports_arena_args(void) 3508 { 3509 return false; 3510 } 3511 3512 bool __weak bpf_jit_supports_far_kfunc_call(void) 3513 { 3514 return false; 3515 } 3516 3517 bool __weak bpf_jit_supports_arena(void) 3518 { 3519 return false; 3520 } 3521 3522 /* Whether JIT takes BPF_REG_AX as the address of arena access */ 3523 bool __weak bpf_jit_supports_arena_scalar(void) 3524 { 3525 return false; 3526 } 3527 3528 bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 3529 { 3530 return false; 3531 } 3532 3533 bool __weak bpf_jit_supports_fsession(void) 3534 { 3535 return false; 3536 } 3537 3538 u64 __weak bpf_arch_uaddress_limit(void) 3539 { 3540 #if defined(CONFIG_64BIT) && defined(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) 3541 return TASK_SIZE; 3542 #else 3543 return 0; 3544 #endif 3545 } 3546 3547 /* Return TRUE if the JIT backend satisfies the following two conditions: 3548 * 1) JIT backend supports atomic_xchg() on pointer-sized words. 3549 * 2) Under the specific arch, the implementation of xchg() is the same 3550 * as atomic_xchg() on pointer-sized words. 3551 */ 3552 bool __weak bpf_jit_supports_ptr_xchg(void) 3553 { 3554 return false; 3555 } 3556 3557 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call 3558 * skb_copy_bits(), so provide a weak definition of it for NET-less config. 3559 */ 3560 int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, 3561 int len) 3562 { 3563 return -EFAULT; 3564 } 3565 3566 int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 3567 enum bpf_text_poke_type new_t, void *old_addr, 3568 void *new_addr) 3569 { 3570 return -ENOTSUPP; 3571 } 3572 3573 void * __weak bpf_arch_text_copy(void *dst, void *src, size_t len) 3574 { 3575 return ERR_PTR(-ENOTSUPP); 3576 } 3577 3578 int __weak bpf_arch_text_invalidate(void *dst, size_t len) 3579 { 3580 return -ENOTSUPP; 3581 } 3582 3583 bool __weak bpf_jit_supports_exceptions(void) 3584 { 3585 return false; 3586 } 3587 3588 bool __weak bpf_jit_supports_private_stack(void) 3589 { 3590 return false; 3591 } 3592 3593 /* 3594 * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. 3595 * Its prologue, epilogue and tail call sequences must encode such frame 3596 * sizes and a private stack must be sized from the program's depth. The 3597 * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(), 3598 * whose tail calls land before the target sets up its own frame; see 3599 * bpf_prog_stack_limit(). 3600 */ 3601 bool __weak bpf_jit_supports_large_stack(void) 3602 { 3603 return false; 3604 } 3605 3606 void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) 3607 { 3608 } 3609 3610 bool __weak bpf_jit_supports_timed_may_goto(void) 3611 { 3612 return false; 3613 } 3614 3615 u64 __weak arch_bpf_timed_may_goto(void) 3616 { 3617 return 0; 3618 } 3619 3620 static noinline void bpf_prog_report_may_goto_violation(void) 3621 { 3622 #ifdef CONFIG_BPF_SYSCALL 3623 struct bpf_stream_stage ss; 3624 struct bpf_prog *prog; 3625 3626 prog = bpf_prog_find_from_stack(); 3627 if (!prog) 3628 return; 3629 bpf_stream_stage(ss, prog, BPF_STDERR, ({ 3630 bpf_stream_printk(ss, "ERROR: Timeout detected for may_goto instruction\n"); 3631 bpf_stream_dump_stack(ss); 3632 })); 3633 #endif 3634 } 3635 3636 u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p) 3637 { 3638 u64 time = ktime_get_mono_fast_ns(); 3639 3640 /* Populate the timestamp for this stack frame, and refresh count. */ 3641 if (!p->timestamp) { 3642 p->timestamp = time; 3643 return BPF_MAX_TIMED_LOOPS; 3644 } 3645 /* Check if we've exhausted our time slice, and zero count. */ 3646 if (unlikely(time - p->timestamp >= (NSEC_PER_SEC / 4))) { 3647 bpf_prog_report_may_goto_violation(); 3648 return 0; 3649 } 3650 /* Refresh the count for the stack frame. */ 3651 return BPF_MAX_TIMED_LOOPS; 3652 } 3653 3654 /* for configs without MMU or 32-bit */ 3655 __weak const struct bpf_map_ops arena_map_ops; 3656 __weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) 3657 { 3658 return 0; 3659 } 3660 __weak u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena) 3661 { 3662 return 0; 3663 } 3664 3665 #ifdef CONFIG_BPF_SYSCALL 3666 __weak bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, 3667 unsigned long fault_ip) 3668 { 3669 return false; 3670 } 3671 3672 static int __init bpf_global_ma_init(void) 3673 { 3674 int ret; 3675 3676 ret = bpf_mem_alloc_init(&bpf_global_ma, 0, false); 3677 bpf_global_ma_set = !ret; 3678 return ret; 3679 } 3680 late_initcall(bpf_global_ma_init); 3681 #endif 3682 3683 DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); 3684 EXPORT_SYMBOL(bpf_stats_enabled_key); 3685 3686 /* All definitions of tracepoints related to BPF. */ 3687 #define CREATE_TRACE_POINTS 3688 #include <linux/bpf_trace.h> 3689 3690 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); 3691 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); 3692 3693 #ifdef CONFIG_BPF_SYSCALL 3694 3695 void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, 3696 struct bpf_linfo_source *src) 3697 { 3698 src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); 3699 src->line = btf_name_by_offset(btf, linfo->line_off); 3700 src->file_name_off = linfo->file_name_off; 3701 src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); 3702 src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); 3703 } 3704 3705 const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) 3706 { 3707 const struct bpf_line_info *linfo; 3708 u32 nr_linfo; 3709 int l, r, m; 3710 3711 nr_linfo = prog->aux->nr_linfo; 3712 if (!nr_linfo || insn_off >= prog->len) 3713 return NULL; 3714 3715 linfo = prog->aux->linfo; 3716 /* Loop invariant: linfo[l].insn_off <= insns_off. 3717 * linfo[0].insn_off == 0 which always satisfies above condition. 3718 * Binary search is searching for rightmost linfo entry that satisfies 3719 * the above invariant, giving us the desired record that covers given 3720 * instruction offset. 3721 */ 3722 l = 0; 3723 r = nr_linfo - 1; 3724 while (l < r) { 3725 /* (r - l + 1) / 2 means we break a tie to the right, so if: 3726 * l=1, r=2, linfo[l].insn_off <= insn_off, linfo[r].insn_off > insn_off, 3727 * then m=2, we see that linfo[m].insn_off > insn_off, and so 3728 * r becomes 1 and we exit the loop with correct l==1. 3729 * If the tie was broken to the left, m=1 would end us up in 3730 * an endless loop where l and m stay at 1 and r stays at 2. 3731 */ 3732 m = l + (r - l + 1) / 2; 3733 if (linfo[m].insn_off <= insn_off) 3734 l = m; 3735 else 3736 r = m - 1; 3737 } 3738 3739 return &linfo[l]; 3740 } 3741 3742 int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, 3743 const char **linep, int *nump) 3744 { 3745 struct bpf_linfo_source src; 3746 int idx = -1, insn_start, insn_end, len; 3747 struct bpf_line_info *linfo; 3748 void **jited_linfo; 3749 struct btf *btf; 3750 int nr_linfo; 3751 3752 btf = prog->aux->btf; 3753 linfo = prog->aux->linfo; 3754 jited_linfo = prog->aux->jited_linfo; 3755 3756 if (!btf || !linfo || !jited_linfo) 3757 return -EINVAL; 3758 len = prog->aux->func ? prog->aux->func[prog->aux->func_idx]->len : prog->len; 3759 3760 linfo = &prog->aux->linfo[prog->aux->linfo_idx]; 3761 jited_linfo = &prog->aux->jited_linfo[prog->aux->linfo_idx]; 3762 3763 insn_start = linfo[0].insn_off; 3764 insn_end = insn_start + len; 3765 nr_linfo = prog->aux->nr_linfo - prog->aux->linfo_idx; 3766 3767 for (int i = 0; i < nr_linfo && 3768 linfo[i].insn_off >= insn_start && linfo[i].insn_off < insn_end; i++) { 3769 if (jited_linfo[i] >= (void *)ip) 3770 break; 3771 idx = i; 3772 } 3773 3774 if (idx == -1) 3775 return -ENOENT; 3776 3777 bpf_get_linfo_source(btf, &linfo[idx], &src); 3778 while (isspace(*src.line)) 3779 src.line++; 3780 if (filep) 3781 *filep = src.file; 3782 if (linep) 3783 *linep = src.line; 3784 if (nump) 3785 *nump = src.line_num; 3786 return 0; 3787 } 3788 3789 struct walk_stack_ctx { 3790 struct bpf_prog *prog; 3791 }; 3792 3793 static bool find_from_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp) 3794 { 3795 struct walk_stack_ctx *ctxp = cookie; 3796 struct bpf_prog *prog; 3797 3798 /* 3799 * The RCU read lock is held to safely traverse the latch tree, but we 3800 * don't need its protection when accessing the prog, since it has an 3801 * active stack frame on the current stack trace, and won't disappear. 3802 */ 3803 rcu_read_lock(); 3804 prog = bpf_prog_ksym_find(ip); 3805 rcu_read_unlock(); 3806 if (!prog) 3807 return true; 3808 /* Make sure we return the main prog if we found a subprog */ 3809 ctxp->prog = prog->aux->main_prog_aux->prog; 3810 return false; 3811 } 3812 3813 struct bpf_prog *bpf_prog_find_from_stack(void) 3814 { 3815 struct walk_stack_ctx ctx = {}; 3816 3817 arch_bpf_stack_walk(find_from_stack_cb, &ctx); 3818 return ctx.prog; 3819 } 3820 3821 #endif 3822