1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux Socket Filter - Kernel level socket filtering 4 * 5 * Based on the design of the Berkeley Packet Filter. The new 6 * internal format has been designed by PLUMgrid: 7 * 8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com 9 * 10 * Authors: 11 * 12 * Jay Schulist <jschlst@samba.org> 13 * Alexei Starovoitov <ast@plumgrid.com> 14 * Daniel Borkmann <dborkman@redhat.com> 15 * 16 * Andi Kleen - Fix a few bad bugs and races. 17 * Kris Katterjohn - Added many additional checks in bpf_check_classic() 18 */ 19 20 #include <uapi/linux/btf.h> 21 #include <linux/filter.h> 22 #include <linux/skbuff.h> 23 #include <linux/vmalloc.h> 24 #include <linux/random.h> 25 #include <linux/moduleloader.h> 26 #include <linux/bpf.h> 27 #include <linux/btf.h> 28 #include <linux/objtool.h> 29 #include <linux/rbtree_latch.h> 30 #include <linux/kallsyms.h> 31 #include <linux/rcupdate.h> 32 #include <linux/perf_event.h> 33 #include <linux/extable.h> 34 #include <linux/log2.h> 35 #include <linux/bpf_verifier.h> 36 37 #include <asm/barrier.h> 38 #include <asm/unaligned.h> 39 40 /* Registers */ 41 #define BPF_R0 regs[BPF_REG_0] 42 #define BPF_R1 regs[BPF_REG_1] 43 #define BPF_R2 regs[BPF_REG_2] 44 #define BPF_R3 regs[BPF_REG_3] 45 #define BPF_R4 regs[BPF_REG_4] 46 #define BPF_R5 regs[BPF_REG_5] 47 #define BPF_R6 regs[BPF_REG_6] 48 #define BPF_R7 regs[BPF_REG_7] 49 #define BPF_R8 regs[BPF_REG_8] 50 #define BPF_R9 regs[BPF_REG_9] 51 #define BPF_R10 regs[BPF_REG_10] 52 53 /* Named registers */ 54 #define DST regs[insn->dst_reg] 55 #define SRC regs[insn->src_reg] 56 #define FP regs[BPF_REG_FP] 57 #define AX regs[BPF_REG_AX] 58 #define ARG1 regs[BPF_REG_ARG1] 59 #define CTX regs[BPF_REG_CTX] 60 #define IMM insn->imm 61 62 /* No hurry in this branch 63 * 64 * Exported for the bpf jit load helper. 65 */ 66 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size) 67 { 68 u8 *ptr = NULL; 69 70 if (k >= SKF_NET_OFF) 71 ptr = skb_network_header(skb) + k - SKF_NET_OFF; 72 else if (k >= SKF_LL_OFF) 73 ptr = skb_mac_header(skb) + k - SKF_LL_OFF; 74 75 if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb)) 76 return ptr; 77 78 return NULL; 79 } 80 81 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) 82 { 83 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 84 struct bpf_prog_aux *aux; 85 struct bpf_prog *fp; 86 87 size = round_up(size, PAGE_SIZE); 88 fp = __vmalloc(size, gfp_flags); 89 if (fp == NULL) 90 return NULL; 91 92 aux = kzalloc(sizeof(*aux), GFP_KERNEL_ACCOUNT | gfp_extra_flags); 93 if (aux == NULL) { 94 vfree(fp); 95 return NULL; 96 } 97 fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags); 98 if (!fp->active) { 99 vfree(fp); 100 kfree(aux); 101 return NULL; 102 } 103 104 fp->pages = size / PAGE_SIZE; 105 fp->aux = aux; 106 fp->aux->prog = fp; 107 fp->jit_requested = ebpf_jit_enabled(); 108 109 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); 110 mutex_init(&fp->aux->used_maps_mutex); 111 mutex_init(&fp->aux->dst_mutex); 112 113 return fp; 114 } 115 116 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) 117 { 118 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 119 struct bpf_prog *prog; 120 int cpu; 121 122 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); 123 if (!prog) 124 return NULL; 125 126 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); 127 if (!prog->stats) { 128 free_percpu(prog->active); 129 kfree(prog->aux); 130 vfree(prog); 131 return NULL; 132 } 133 134 for_each_possible_cpu(cpu) { 135 struct bpf_prog_stats *pstats; 136 137 pstats = per_cpu_ptr(prog->stats, cpu); 138 u64_stats_init(&pstats->syncp); 139 } 140 return prog; 141 } 142 EXPORT_SYMBOL_GPL(bpf_prog_alloc); 143 144 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) 145 { 146 if (!prog->aux->nr_linfo || !prog->jit_requested) 147 return 0; 148 149 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, 150 sizeof(*prog->aux->jited_linfo), 151 GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 152 if (!prog->aux->jited_linfo) 153 return -ENOMEM; 154 155 return 0; 156 } 157 158 void bpf_prog_jit_attempt_done(struct bpf_prog *prog) 159 { 160 if (prog->aux->jited_linfo && 161 (!prog->jited || !prog->aux->jited_linfo[0])) { 162 kvfree(prog->aux->jited_linfo); 163 prog->aux->jited_linfo = NULL; 164 } 165 166 kfree(prog->aux->kfunc_tab); 167 prog->aux->kfunc_tab = NULL; 168 } 169 170 /* The jit engine is responsible to provide an array 171 * for insn_off to the jited_off mapping (insn_to_jit_off). 172 * 173 * The idx to this array is the insn_off. Hence, the insn_off 174 * here is relative to the prog itself instead of the main prog. 175 * This array has one entry for each xlated bpf insn. 176 * 177 * jited_off is the byte off to the last byte of the jited insn. 178 * 179 * Hence, with 180 * insn_start: 181 * The first bpf insn off of the prog. The insn off 182 * here is relative to the main prog. 183 * e.g. if prog is a subprog, insn_start > 0 184 * linfo_idx: 185 * The prog's idx to prog->aux->linfo and jited_linfo 186 * 187 * jited_linfo[linfo_idx] = prog->bpf_func 188 * 189 * For i > linfo_idx, 190 * 191 * jited_linfo[i] = prog->bpf_func + 192 * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] 193 */ 194 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, 195 const u32 *insn_to_jit_off) 196 { 197 u32 linfo_idx, insn_start, insn_end, nr_linfo, i; 198 const struct bpf_line_info *linfo; 199 void **jited_linfo; 200 201 if (!prog->aux->jited_linfo) 202 /* Userspace did not provide linfo */ 203 return; 204 205 linfo_idx = prog->aux->linfo_idx; 206 linfo = &prog->aux->linfo[linfo_idx]; 207 insn_start = linfo[0].insn_off; 208 insn_end = insn_start + prog->len; 209 210 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; 211 jited_linfo[0] = prog->bpf_func; 212 213 nr_linfo = prog->aux->nr_linfo - linfo_idx; 214 215 for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) 216 /* The verifier ensures that linfo[i].insn_off is 217 * strictly increasing 218 */ 219 jited_linfo[i] = prog->bpf_func + 220 insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; 221 } 222 223 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 224 gfp_t gfp_extra_flags) 225 { 226 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 227 struct bpf_prog *fp; 228 u32 pages; 229 230 size = round_up(size, PAGE_SIZE); 231 pages = size / PAGE_SIZE; 232 if (pages <= fp_old->pages) 233 return fp_old; 234 235 fp = __vmalloc(size, gfp_flags); 236 if (fp) { 237 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); 238 fp->pages = pages; 239 fp->aux->prog = fp; 240 241 /* We keep fp->aux from fp_old around in the new 242 * reallocated structure. 243 */ 244 fp_old->aux = NULL; 245 fp_old->stats = NULL; 246 fp_old->active = NULL; 247 __bpf_prog_free(fp_old); 248 } 249 250 return fp; 251 } 252 253 void __bpf_prog_free(struct bpf_prog *fp) 254 { 255 if (fp->aux) { 256 mutex_destroy(&fp->aux->used_maps_mutex); 257 mutex_destroy(&fp->aux->dst_mutex); 258 kfree(fp->aux->poke_tab); 259 kfree(fp->aux); 260 } 261 free_percpu(fp->stats); 262 free_percpu(fp->active); 263 vfree(fp); 264 } 265 266 int bpf_prog_calc_tag(struct bpf_prog *fp) 267 { 268 const u32 bits_offset = SHA1_BLOCK_SIZE - sizeof(__be64); 269 u32 raw_size = bpf_prog_tag_scratch_size(fp); 270 u32 digest[SHA1_DIGEST_WORDS]; 271 u32 ws[SHA1_WORKSPACE_WORDS]; 272 u32 i, bsize, psize, blocks; 273 struct bpf_insn *dst; 274 bool was_ld_map; 275 u8 *raw, *todo; 276 __be32 *result; 277 __be64 *bits; 278 279 raw = vmalloc(raw_size); 280 if (!raw) 281 return -ENOMEM; 282 283 sha1_init(digest); 284 memset(ws, 0, sizeof(ws)); 285 286 /* We need to take out the map fd for the digest calculation 287 * since they are unstable from user space side. 288 */ 289 dst = (void *)raw; 290 for (i = 0, was_ld_map = false; i < fp->len; i++) { 291 dst[i] = fp->insnsi[i]; 292 if (!was_ld_map && 293 dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && 294 (dst[i].src_reg == BPF_PSEUDO_MAP_FD || 295 dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { 296 was_ld_map = true; 297 dst[i].imm = 0; 298 } else if (was_ld_map && 299 dst[i].code == 0 && 300 dst[i].dst_reg == 0 && 301 dst[i].src_reg == 0 && 302 dst[i].off == 0) { 303 was_ld_map = false; 304 dst[i].imm = 0; 305 } else { 306 was_ld_map = false; 307 } 308 } 309 310 psize = bpf_prog_insn_size(fp); 311 memset(&raw[psize], 0, raw_size - psize); 312 raw[psize++] = 0x80; 313 314 bsize = round_up(psize, SHA1_BLOCK_SIZE); 315 blocks = bsize / SHA1_BLOCK_SIZE; 316 todo = raw; 317 if (bsize - psize >= sizeof(__be64)) { 318 bits = (__be64 *)(todo + bsize - sizeof(__be64)); 319 } else { 320 bits = (__be64 *)(todo + bsize + bits_offset); 321 blocks++; 322 } 323 *bits = cpu_to_be64((psize - 1) << 3); 324 325 while (blocks--) { 326 sha1_transform(digest, todo, ws); 327 todo += SHA1_BLOCK_SIZE; 328 } 329 330 result = (__force __be32 *)digest; 331 for (i = 0; i < SHA1_DIGEST_WORDS; i++) 332 result[i] = cpu_to_be32(digest[i]); 333 memcpy(fp->tag, result, sizeof(fp->tag)); 334 335 vfree(raw); 336 return 0; 337 } 338 339 static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, 340 s32 end_new, s32 curr, const bool probe_pass) 341 { 342 const s64 imm_min = S32_MIN, imm_max = S32_MAX; 343 s32 delta = end_new - end_old; 344 s64 imm = insn->imm; 345 346 if (curr < pos && curr + imm + 1 >= end_old) 347 imm += delta; 348 else if (curr >= end_new && curr + imm + 1 < end_new) 349 imm -= delta; 350 if (imm < imm_min || imm > imm_max) 351 return -ERANGE; 352 if (!probe_pass) 353 insn->imm = imm; 354 return 0; 355 } 356 357 static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, 358 s32 end_new, s32 curr, const bool probe_pass) 359 { 360 const s32 off_min = S16_MIN, off_max = S16_MAX; 361 s32 delta = end_new - end_old; 362 s32 off = insn->off; 363 364 if (curr < pos && curr + off + 1 >= end_old) 365 off += delta; 366 else if (curr >= end_new && curr + off + 1 < end_new) 367 off -= delta; 368 if (off < off_min || off > off_max) 369 return -ERANGE; 370 if (!probe_pass) 371 insn->off = off; 372 return 0; 373 } 374 375 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, 376 s32 end_new, const bool probe_pass) 377 { 378 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); 379 struct bpf_insn *insn = prog->insnsi; 380 int ret = 0; 381 382 for (i = 0; i < insn_cnt; i++, insn++) { 383 u8 code; 384 385 /* In the probing pass we still operate on the original, 386 * unpatched image in order to check overflows before we 387 * do any other adjustments. Therefore skip the patchlet. 388 */ 389 if (probe_pass && i == pos) { 390 i = end_new; 391 insn = prog->insnsi + end_old; 392 } 393 if (bpf_pseudo_func(insn)) { 394 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 395 end_new, i, probe_pass); 396 if (ret) 397 return ret; 398 continue; 399 } 400 code = insn->code; 401 if ((BPF_CLASS(code) != BPF_JMP && 402 BPF_CLASS(code) != BPF_JMP32) || 403 BPF_OP(code) == BPF_EXIT) 404 continue; 405 /* Adjust offset of jmps if we cross patch boundaries. */ 406 if (BPF_OP(code) == BPF_CALL) { 407 if (insn->src_reg != BPF_PSEUDO_CALL) 408 continue; 409 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 410 end_new, i, probe_pass); 411 } else { 412 ret = bpf_adj_delta_to_off(insn, pos, end_old, 413 end_new, i, probe_pass); 414 } 415 if (ret) 416 break; 417 } 418 419 return ret; 420 } 421 422 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) 423 { 424 struct bpf_line_info *linfo; 425 u32 i, nr_linfo; 426 427 nr_linfo = prog->aux->nr_linfo; 428 if (!nr_linfo || !delta) 429 return; 430 431 linfo = prog->aux->linfo; 432 433 for (i = 0; i < nr_linfo; i++) 434 if (off < linfo[i].insn_off) 435 break; 436 437 /* Push all off < linfo[i].insn_off by delta */ 438 for (; i < nr_linfo; i++) 439 linfo[i].insn_off += delta; 440 } 441 442 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 443 const struct bpf_insn *patch, u32 len) 444 { 445 u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; 446 const u32 cnt_max = S16_MAX; 447 struct bpf_prog *prog_adj; 448 int err; 449 450 /* Since our patchlet doesn't expand the image, we're done. */ 451 if (insn_delta == 0) { 452 memcpy(prog->insnsi + off, patch, sizeof(*patch)); 453 return prog; 454 } 455 456 insn_adj_cnt = prog->len + insn_delta; 457 458 /* Reject anything that would potentially let the insn->off 459 * target overflow when we have excessive program expansions. 460 * We need to probe here before we do any reallocation where 461 * we afterwards may not fail anymore. 462 */ 463 if (insn_adj_cnt > cnt_max && 464 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) 465 return ERR_PTR(err); 466 467 /* Several new instructions need to be inserted. Make room 468 * for them. Likely, there's no need for a new allocation as 469 * last page could have large enough tailroom. 470 */ 471 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), 472 GFP_USER); 473 if (!prog_adj) 474 return ERR_PTR(-ENOMEM); 475 476 prog_adj->len = insn_adj_cnt; 477 478 /* Patching happens in 3 steps: 479 * 480 * 1) Move over tail of insnsi from next instruction onwards, 481 * so we can patch the single target insn with one or more 482 * new ones (patching is always from 1 to n insns, n > 0). 483 * 2) Inject new instructions at the target location. 484 * 3) Adjust branch offsets if necessary. 485 */ 486 insn_rest = insn_adj_cnt - off - len; 487 488 memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, 489 sizeof(*patch) * insn_rest); 490 memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); 491 492 /* We are guaranteed to not fail at this point, otherwise 493 * the ship has sailed to reverse to the original state. An 494 * overflow cannot happen at this point. 495 */ 496 BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); 497 498 bpf_adj_linfo(prog_adj, off, insn_delta); 499 500 return prog_adj; 501 } 502 503 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) 504 { 505 /* Branch offsets can't overflow when program is shrinking, no need 506 * to call bpf_adj_branches(..., true) here 507 */ 508 memmove(prog->insnsi + off, prog->insnsi + off + cnt, 509 sizeof(struct bpf_insn) * (prog->len - off - cnt)); 510 prog->len -= cnt; 511 512 return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); 513 } 514 515 static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) 516 { 517 int i; 518 519 for (i = 0; i < fp->aux->func_cnt; i++) 520 bpf_prog_kallsyms_del(fp->aux->func[i]); 521 } 522 523 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) 524 { 525 bpf_prog_kallsyms_del_subprogs(fp); 526 bpf_prog_kallsyms_del(fp); 527 } 528 529 #ifdef CONFIG_BPF_JIT 530 /* All BPF JIT sysctl knobs here. */ 531 int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 532 int bpf_jit_kallsyms __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 533 int bpf_jit_harden __read_mostly; 534 long bpf_jit_limit __read_mostly; 535 long bpf_jit_limit_max __read_mostly; 536 537 static void 538 bpf_prog_ksym_set_addr(struct bpf_prog *prog) 539 { 540 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); 541 542 prog->aux->ksym.start = (unsigned long) prog->bpf_func; 543 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; 544 } 545 546 static void 547 bpf_prog_ksym_set_name(struct bpf_prog *prog) 548 { 549 char *sym = prog->aux->ksym.name; 550 const char *end = sym + KSYM_NAME_LEN; 551 const struct btf_type *type; 552 const char *func_name; 553 554 BUILD_BUG_ON(sizeof("bpf_prog_") + 555 sizeof(prog->tag) * 2 + 556 /* name has been null terminated. 557 * We should need +1 for the '_' preceding 558 * the name. However, the null character 559 * is double counted between the name and the 560 * sizeof("bpf_prog_") above, so we omit 561 * the +1 here. 562 */ 563 sizeof(prog->aux->name) > KSYM_NAME_LEN); 564 565 sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); 566 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); 567 568 /* prog->aux->name will be ignored if full btf name is available */ 569 if (prog->aux->func_info_cnt) { 570 type = btf_type_by_id(prog->aux->btf, 571 prog->aux->func_info[prog->aux->func_idx].type_id); 572 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); 573 snprintf(sym, (size_t)(end - sym), "_%s", func_name); 574 return; 575 } 576 577 if (prog->aux->name[0]) 578 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); 579 else 580 *sym = 0; 581 } 582 583 static unsigned long bpf_get_ksym_start(struct latch_tree_node *n) 584 { 585 return container_of(n, struct bpf_ksym, tnode)->start; 586 } 587 588 static __always_inline bool bpf_tree_less(struct latch_tree_node *a, 589 struct latch_tree_node *b) 590 { 591 return bpf_get_ksym_start(a) < bpf_get_ksym_start(b); 592 } 593 594 static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) 595 { 596 unsigned long val = (unsigned long)key; 597 const struct bpf_ksym *ksym; 598 599 ksym = container_of(n, struct bpf_ksym, tnode); 600 601 if (val < ksym->start) 602 return -1; 603 if (val >= ksym->end) 604 return 1; 605 606 return 0; 607 } 608 609 static const struct latch_tree_ops bpf_tree_ops = { 610 .less = bpf_tree_less, 611 .comp = bpf_tree_comp, 612 }; 613 614 static DEFINE_SPINLOCK(bpf_lock); 615 static LIST_HEAD(bpf_kallsyms); 616 static struct latch_tree_root bpf_tree __cacheline_aligned; 617 618 void bpf_ksym_add(struct bpf_ksym *ksym) 619 { 620 spin_lock_bh(&bpf_lock); 621 WARN_ON_ONCE(!list_empty(&ksym->lnode)); 622 list_add_tail_rcu(&ksym->lnode, &bpf_kallsyms); 623 latch_tree_insert(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 624 spin_unlock_bh(&bpf_lock); 625 } 626 627 static void __bpf_ksym_del(struct bpf_ksym *ksym) 628 { 629 if (list_empty(&ksym->lnode)) 630 return; 631 632 latch_tree_erase(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 633 list_del_rcu(&ksym->lnode); 634 } 635 636 void bpf_ksym_del(struct bpf_ksym *ksym) 637 { 638 spin_lock_bh(&bpf_lock); 639 __bpf_ksym_del(ksym); 640 spin_unlock_bh(&bpf_lock); 641 } 642 643 static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) 644 { 645 return fp->jited && !bpf_prog_was_classic(fp); 646 } 647 648 static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) 649 { 650 return list_empty(&fp->aux->ksym.lnode) || 651 fp->aux->ksym.lnode.prev == LIST_POISON2; 652 } 653 654 void bpf_prog_kallsyms_add(struct bpf_prog *fp) 655 { 656 if (!bpf_prog_kallsyms_candidate(fp) || 657 !bpf_capable()) 658 return; 659 660 bpf_prog_ksym_set_addr(fp); 661 bpf_prog_ksym_set_name(fp); 662 fp->aux->ksym.prog = true; 663 664 bpf_ksym_add(&fp->aux->ksym); 665 } 666 667 void bpf_prog_kallsyms_del(struct bpf_prog *fp) 668 { 669 if (!bpf_prog_kallsyms_candidate(fp)) 670 return; 671 672 bpf_ksym_del(&fp->aux->ksym); 673 } 674 675 static struct bpf_ksym *bpf_ksym_find(unsigned long addr) 676 { 677 struct latch_tree_node *n; 678 679 n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); 680 return n ? container_of(n, struct bpf_ksym, tnode) : NULL; 681 } 682 683 const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, 684 unsigned long *off, char *sym) 685 { 686 struct bpf_ksym *ksym; 687 char *ret = NULL; 688 689 rcu_read_lock(); 690 ksym = bpf_ksym_find(addr); 691 if (ksym) { 692 unsigned long symbol_start = ksym->start; 693 unsigned long symbol_end = ksym->end; 694 695 strncpy(sym, ksym->name, KSYM_NAME_LEN); 696 697 ret = sym; 698 if (size) 699 *size = symbol_end - symbol_start; 700 if (off) 701 *off = addr - symbol_start; 702 } 703 rcu_read_unlock(); 704 705 return ret; 706 } 707 708 bool is_bpf_text_address(unsigned long addr) 709 { 710 bool ret; 711 712 rcu_read_lock(); 713 ret = bpf_ksym_find(addr) != NULL; 714 rcu_read_unlock(); 715 716 return ret; 717 } 718 719 static struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 720 { 721 struct bpf_ksym *ksym = bpf_ksym_find(addr); 722 723 return ksym && ksym->prog ? 724 container_of(ksym, struct bpf_prog_aux, ksym)->prog : 725 NULL; 726 } 727 728 const struct exception_table_entry *search_bpf_extables(unsigned long addr) 729 { 730 const struct exception_table_entry *e = NULL; 731 struct bpf_prog *prog; 732 733 rcu_read_lock(); 734 prog = bpf_prog_ksym_find(addr); 735 if (!prog) 736 goto out; 737 if (!prog->aux->num_exentries) 738 goto out; 739 740 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); 741 out: 742 rcu_read_unlock(); 743 return e; 744 } 745 746 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 747 char *sym) 748 { 749 struct bpf_ksym *ksym; 750 unsigned int it = 0; 751 int ret = -ERANGE; 752 753 if (!bpf_jit_kallsyms_enabled()) 754 return ret; 755 756 rcu_read_lock(); 757 list_for_each_entry_rcu(ksym, &bpf_kallsyms, lnode) { 758 if (it++ != symnum) 759 continue; 760 761 strncpy(sym, ksym->name, KSYM_NAME_LEN); 762 763 *value = ksym->start; 764 *type = BPF_SYM_ELF_TYPE; 765 766 ret = 0; 767 break; 768 } 769 rcu_read_unlock(); 770 771 return ret; 772 } 773 774 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, 775 struct bpf_jit_poke_descriptor *poke) 776 { 777 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 778 static const u32 poke_tab_max = 1024; 779 u32 slot = prog->aux->size_poke_tab; 780 u32 size = slot + 1; 781 782 if (size > poke_tab_max) 783 return -ENOSPC; 784 if (poke->tailcall_target || poke->tailcall_target_stable || 785 poke->tailcall_bypass || poke->adj_off || poke->bypass_addr) 786 return -EINVAL; 787 788 switch (poke->reason) { 789 case BPF_POKE_REASON_TAIL_CALL: 790 if (!poke->tail_call.map) 791 return -EINVAL; 792 break; 793 default: 794 return -EINVAL; 795 } 796 797 tab = krealloc(tab, size * sizeof(*poke), GFP_KERNEL); 798 if (!tab) 799 return -ENOMEM; 800 801 memcpy(&tab[slot], poke, sizeof(*poke)); 802 prog->aux->size_poke_tab = size; 803 prog->aux->poke_tab = tab; 804 805 return slot; 806 } 807 808 /* 809 * BPF program pack allocator. 810 * 811 * Most BPF programs are pretty small. Allocating a hole page for each 812 * program is sometime a waste. Many small bpf program also adds pressure 813 * to instruction TLB. To solve this issue, we introduce a BPF program pack 814 * allocator. The prog_pack allocator uses HPAGE_PMD_SIZE page (2MB on x86) 815 * to host BPF programs. 816 */ 817 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 818 #define BPF_PROG_PACK_SIZE HPAGE_PMD_SIZE 819 #else 820 #define BPF_PROG_PACK_SIZE PAGE_SIZE 821 #endif 822 #define BPF_PROG_CHUNK_SHIFT 6 823 #define BPF_PROG_CHUNK_SIZE (1 << BPF_PROG_CHUNK_SHIFT) 824 #define BPF_PROG_CHUNK_MASK (~(BPF_PROG_CHUNK_SIZE - 1)) 825 #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) 826 827 struct bpf_prog_pack { 828 struct list_head list; 829 void *ptr; 830 unsigned long bitmap[]; 831 }; 832 833 #define BPF_PROG_MAX_PACK_PROG_SIZE BPF_PROG_PACK_SIZE 834 #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) 835 836 static DEFINE_MUTEX(pack_mutex); 837 static LIST_HEAD(pack_list); 838 839 static struct bpf_prog_pack *alloc_new_pack(void) 840 { 841 struct bpf_prog_pack *pack; 842 843 pack = kzalloc(sizeof(*pack) + BITS_TO_BYTES(BPF_PROG_CHUNK_COUNT), GFP_KERNEL); 844 if (!pack) 845 return NULL; 846 pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); 847 if (!pack->ptr) { 848 kfree(pack); 849 return NULL; 850 } 851 bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); 852 list_add_tail(&pack->list, &pack_list); 853 854 set_vm_flush_reset_perms(pack->ptr); 855 set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); 856 set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); 857 return pack; 858 } 859 860 static void *bpf_prog_pack_alloc(u32 size) 861 { 862 unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); 863 struct bpf_prog_pack *pack; 864 unsigned long pos; 865 void *ptr = NULL; 866 867 if (size > BPF_PROG_MAX_PACK_PROG_SIZE) { 868 size = round_up(size, PAGE_SIZE); 869 ptr = module_alloc(size); 870 if (ptr) { 871 set_vm_flush_reset_perms(ptr); 872 set_memory_ro((unsigned long)ptr, size / PAGE_SIZE); 873 set_memory_x((unsigned long)ptr, size / PAGE_SIZE); 874 } 875 return ptr; 876 } 877 mutex_lock(&pack_mutex); 878 list_for_each_entry(pack, &pack_list, list) { 879 pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 880 nbits, 0); 881 if (pos < BPF_PROG_CHUNK_COUNT) 882 goto found_free_area; 883 } 884 885 pack = alloc_new_pack(); 886 if (!pack) 887 goto out; 888 889 pos = 0; 890 891 found_free_area: 892 bitmap_set(pack->bitmap, pos, nbits); 893 ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); 894 895 out: 896 mutex_unlock(&pack_mutex); 897 return ptr; 898 } 899 900 static void bpf_prog_pack_free(struct bpf_binary_header *hdr) 901 { 902 struct bpf_prog_pack *pack = NULL, *tmp; 903 unsigned int nbits; 904 unsigned long pos; 905 void *pack_ptr; 906 907 if (hdr->size > BPF_PROG_MAX_PACK_PROG_SIZE) { 908 module_memfree(hdr); 909 return; 910 } 911 912 pack_ptr = (void *)((unsigned long)hdr & ~(BPF_PROG_PACK_SIZE - 1)); 913 mutex_lock(&pack_mutex); 914 915 list_for_each_entry(tmp, &pack_list, list) { 916 if (tmp->ptr == pack_ptr) { 917 pack = tmp; 918 break; 919 } 920 } 921 922 if (WARN_ONCE(!pack, "bpf_prog_pack bug\n")) 923 goto out; 924 925 nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); 926 pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; 927 928 bitmap_clear(pack->bitmap, pos, nbits); 929 if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 930 BPF_PROG_CHUNK_COUNT, 0) == 0) { 931 list_del(&pack->list); 932 module_memfree(pack->ptr); 933 kfree(pack); 934 } 935 out: 936 mutex_unlock(&pack_mutex); 937 } 938 939 static atomic_long_t bpf_jit_current; 940 941 /* Can be overridden by an arch's JIT compiler if it has a custom, 942 * dedicated BPF backend memory area, or if neither of the two 943 * below apply. 944 */ 945 u64 __weak bpf_jit_alloc_exec_limit(void) 946 { 947 #if defined(MODULES_VADDR) 948 return MODULES_END - MODULES_VADDR; 949 #else 950 return VMALLOC_END - VMALLOC_START; 951 #endif 952 } 953 954 static int __init bpf_jit_charge_init(void) 955 { 956 /* Only used as heuristic here to derive limit. */ 957 bpf_jit_limit_max = bpf_jit_alloc_exec_limit(); 958 bpf_jit_limit = min_t(u64, round_up(bpf_jit_limit_max >> 2, 959 PAGE_SIZE), LONG_MAX); 960 return 0; 961 } 962 pure_initcall(bpf_jit_charge_init); 963 964 int bpf_jit_charge_modmem(u32 size) 965 { 966 if (atomic_long_add_return(size, &bpf_jit_current) > bpf_jit_limit) { 967 if (!bpf_capable()) { 968 atomic_long_sub(size, &bpf_jit_current); 969 return -EPERM; 970 } 971 } 972 973 return 0; 974 } 975 976 void bpf_jit_uncharge_modmem(u32 size) 977 { 978 atomic_long_sub(size, &bpf_jit_current); 979 } 980 981 void *__weak bpf_jit_alloc_exec(unsigned long size) 982 { 983 return module_alloc(size); 984 } 985 986 void __weak bpf_jit_free_exec(void *addr) 987 { 988 module_memfree(addr); 989 } 990 991 struct bpf_binary_header * 992 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 993 unsigned int alignment, 994 bpf_jit_fill_hole_t bpf_fill_ill_insns) 995 { 996 struct bpf_binary_header *hdr; 997 u32 size, hole, start; 998 999 WARN_ON_ONCE(!is_power_of_2(alignment) || 1000 alignment > BPF_IMAGE_ALIGNMENT); 1001 1002 /* Most of BPF filters are really small, but if some of them 1003 * fill a page, allow at least 128 extra bytes to insert a 1004 * random section of illegal instructions. 1005 */ 1006 size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); 1007 1008 if (bpf_jit_charge_modmem(size)) 1009 return NULL; 1010 hdr = bpf_jit_alloc_exec(size); 1011 if (!hdr) { 1012 bpf_jit_uncharge_modmem(size); 1013 return NULL; 1014 } 1015 1016 /* Fill space with illegal/arch-dep instructions. */ 1017 bpf_fill_ill_insns(hdr, size); 1018 1019 hdr->size = size; 1020 hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), 1021 PAGE_SIZE - sizeof(*hdr)); 1022 start = (get_random_int() % hole) & ~(alignment - 1); 1023 1024 /* Leave a random number of instructions before BPF code. */ 1025 *image_ptr = &hdr->image[start]; 1026 1027 return hdr; 1028 } 1029 1030 void bpf_jit_binary_free(struct bpf_binary_header *hdr) 1031 { 1032 u32 size = hdr->size; 1033 1034 bpf_jit_free_exec(hdr); 1035 bpf_jit_uncharge_modmem(size); 1036 } 1037 1038 /* Allocate jit binary from bpf_prog_pack allocator. 1039 * Since the allocated memory is RO+X, the JIT engine cannot write directly 1040 * to the memory. To solve this problem, a RW buffer is also allocated at 1041 * as the same time. The JIT engine should calculate offsets based on the 1042 * RO memory address, but write JITed program to the RW buffer. Once the 1043 * JIT engine finishes, it calls bpf_jit_binary_pack_finalize, which copies 1044 * the JITed program to the RO memory. 1045 */ 1046 struct bpf_binary_header * 1047 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, 1048 unsigned int alignment, 1049 struct bpf_binary_header **rw_header, 1050 u8 **rw_image, 1051 bpf_jit_fill_hole_t bpf_fill_ill_insns) 1052 { 1053 struct bpf_binary_header *ro_header; 1054 u32 size, hole, start; 1055 1056 WARN_ON_ONCE(!is_power_of_2(alignment) || 1057 alignment > BPF_IMAGE_ALIGNMENT); 1058 1059 /* add 16 bytes for a random section of illegal instructions */ 1060 size = round_up(proglen + sizeof(*ro_header) + 16, BPF_PROG_CHUNK_SIZE); 1061 1062 if (bpf_jit_charge_modmem(size)) 1063 return NULL; 1064 ro_header = bpf_prog_pack_alloc(size); 1065 if (!ro_header) { 1066 bpf_jit_uncharge_modmem(size); 1067 return NULL; 1068 } 1069 1070 *rw_header = kvmalloc(size, GFP_KERNEL); 1071 if (!*rw_header) { 1072 bpf_prog_pack_free(ro_header); 1073 bpf_jit_uncharge_modmem(size); 1074 return NULL; 1075 } 1076 1077 /* Fill space with illegal/arch-dep instructions. */ 1078 bpf_fill_ill_insns(*rw_header, size); 1079 (*rw_header)->size = size; 1080 1081 hole = min_t(unsigned int, size - (proglen + sizeof(*ro_header)), 1082 BPF_PROG_CHUNK_SIZE - sizeof(*ro_header)); 1083 start = (get_random_int() % hole) & ~(alignment - 1); 1084 1085 *image_ptr = &ro_header->image[start]; 1086 *rw_image = &(*rw_header)->image[start]; 1087 1088 return ro_header; 1089 } 1090 1091 /* Copy JITed text from rw_header to its final location, the ro_header. */ 1092 int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, 1093 struct bpf_binary_header *ro_header, 1094 struct bpf_binary_header *rw_header) 1095 { 1096 void *ptr; 1097 1098 ptr = bpf_arch_text_copy(ro_header, rw_header, rw_header->size); 1099 1100 kvfree(rw_header); 1101 1102 if (IS_ERR(ptr)) { 1103 bpf_prog_pack_free(ro_header); 1104 return PTR_ERR(ptr); 1105 } 1106 prog->aux->use_bpf_prog_pack = true; 1107 return 0; 1108 } 1109 1110 /* bpf_jit_binary_pack_free is called in two different scenarios: 1111 * 1) when the program is freed after; 1112 * 2) when the JIT engine fails (before bpf_jit_binary_pack_finalize). 1113 * For case 2), we need to free both the RO memory and the RW buffer. 1114 * Also, ro_header->size in 2) is not properly set yet, so rw_header->size 1115 * is used for uncharge. 1116 */ 1117 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, 1118 struct bpf_binary_header *rw_header) 1119 { 1120 u32 size = rw_header ? rw_header->size : ro_header->size; 1121 1122 bpf_prog_pack_free(ro_header); 1123 kvfree(rw_header); 1124 bpf_jit_uncharge_modmem(size); 1125 } 1126 1127 static inline struct bpf_binary_header * 1128 bpf_jit_binary_hdr(const struct bpf_prog *fp) 1129 { 1130 unsigned long real_start = (unsigned long)fp->bpf_func; 1131 unsigned long addr; 1132 1133 if (fp->aux->use_bpf_prog_pack) 1134 addr = real_start & BPF_PROG_CHUNK_MASK; 1135 else 1136 addr = real_start & PAGE_MASK; 1137 1138 return (void *)addr; 1139 } 1140 1141 /* This symbol is only overridden by archs that have different 1142 * requirements than the usual eBPF JITs, f.e. when they only 1143 * implement cBPF JIT, do not set images read-only, etc. 1144 */ 1145 void __weak bpf_jit_free(struct bpf_prog *fp) 1146 { 1147 if (fp->jited) { 1148 struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); 1149 1150 if (fp->aux->use_bpf_prog_pack) 1151 bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */); 1152 else 1153 bpf_jit_binary_free(hdr); 1154 1155 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 1156 } 1157 1158 bpf_prog_unlock_free(fp); 1159 } 1160 1161 int bpf_jit_get_func_addr(const struct bpf_prog *prog, 1162 const struct bpf_insn *insn, bool extra_pass, 1163 u64 *func_addr, bool *func_addr_fixed) 1164 { 1165 s16 off = insn->off; 1166 s32 imm = insn->imm; 1167 u8 *addr; 1168 1169 *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; 1170 if (!*func_addr_fixed) { 1171 /* Place-holder address till the last pass has collected 1172 * all addresses for JITed subprograms in which case we 1173 * can pick them up from prog->aux. 1174 */ 1175 if (!extra_pass) 1176 addr = NULL; 1177 else if (prog->aux->func && 1178 off >= 0 && off < prog->aux->func_cnt) 1179 addr = (u8 *)prog->aux->func[off]->bpf_func; 1180 else 1181 return -EINVAL; 1182 } else { 1183 /* Address of a BPF helper call. Since part of the core 1184 * kernel, it's always at a fixed location. __bpf_call_base 1185 * and the helper with imm relative to it are both in core 1186 * kernel. 1187 */ 1188 addr = (u8 *)__bpf_call_base + imm; 1189 } 1190 1191 *func_addr = (unsigned long)addr; 1192 return 0; 1193 } 1194 1195 static int bpf_jit_blind_insn(const struct bpf_insn *from, 1196 const struct bpf_insn *aux, 1197 struct bpf_insn *to_buff, 1198 bool emit_zext) 1199 { 1200 struct bpf_insn *to = to_buff; 1201 u32 imm_rnd = get_random_int(); 1202 s16 off; 1203 1204 BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); 1205 BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); 1206 1207 /* Constraints on AX register: 1208 * 1209 * AX register is inaccessible from user space. It is mapped in 1210 * all JITs, and used here for constant blinding rewrites. It is 1211 * typically "stateless" meaning its contents are only valid within 1212 * the executed instruction, but not across several instructions. 1213 * There are a few exceptions however which are further detailed 1214 * below. 1215 * 1216 * Constant blinding is only used by JITs, not in the interpreter. 1217 * The interpreter uses AX in some occasions as a local temporary 1218 * register e.g. in DIV or MOD instructions. 1219 * 1220 * In restricted circumstances, the verifier can also use the AX 1221 * register for rewrites as long as they do not interfere with 1222 * the above cases! 1223 */ 1224 if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) 1225 goto out; 1226 1227 if (from->imm == 0 && 1228 (from->code == (BPF_ALU | BPF_MOV | BPF_K) || 1229 from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { 1230 *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); 1231 goto out; 1232 } 1233 1234 switch (from->code) { 1235 case BPF_ALU | BPF_ADD | BPF_K: 1236 case BPF_ALU | BPF_SUB | BPF_K: 1237 case BPF_ALU | BPF_AND | BPF_K: 1238 case BPF_ALU | BPF_OR | BPF_K: 1239 case BPF_ALU | BPF_XOR | BPF_K: 1240 case BPF_ALU | BPF_MUL | BPF_K: 1241 case BPF_ALU | BPF_MOV | BPF_K: 1242 case BPF_ALU | BPF_DIV | BPF_K: 1243 case BPF_ALU | BPF_MOD | BPF_K: 1244 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1245 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1246 *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX); 1247 break; 1248 1249 case BPF_ALU64 | BPF_ADD | BPF_K: 1250 case BPF_ALU64 | BPF_SUB | BPF_K: 1251 case BPF_ALU64 | BPF_AND | BPF_K: 1252 case BPF_ALU64 | BPF_OR | BPF_K: 1253 case BPF_ALU64 | BPF_XOR | BPF_K: 1254 case BPF_ALU64 | BPF_MUL | BPF_K: 1255 case BPF_ALU64 | BPF_MOV | BPF_K: 1256 case BPF_ALU64 | BPF_DIV | BPF_K: 1257 case BPF_ALU64 | BPF_MOD | BPF_K: 1258 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1259 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1260 *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX); 1261 break; 1262 1263 case BPF_JMP | BPF_JEQ | BPF_K: 1264 case BPF_JMP | BPF_JNE | BPF_K: 1265 case BPF_JMP | BPF_JGT | BPF_K: 1266 case BPF_JMP | BPF_JLT | BPF_K: 1267 case BPF_JMP | BPF_JGE | BPF_K: 1268 case BPF_JMP | BPF_JLE | BPF_K: 1269 case BPF_JMP | BPF_JSGT | BPF_K: 1270 case BPF_JMP | BPF_JSLT | BPF_K: 1271 case BPF_JMP | BPF_JSGE | BPF_K: 1272 case BPF_JMP | BPF_JSLE | BPF_K: 1273 case BPF_JMP | BPF_JSET | BPF_K: 1274 /* Accommodate for extra offset in case of a backjump. */ 1275 off = from->off; 1276 if (off < 0) 1277 off -= 2; 1278 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1279 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1280 *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); 1281 break; 1282 1283 case BPF_JMP32 | BPF_JEQ | BPF_K: 1284 case BPF_JMP32 | BPF_JNE | BPF_K: 1285 case BPF_JMP32 | BPF_JGT | BPF_K: 1286 case BPF_JMP32 | BPF_JLT | BPF_K: 1287 case BPF_JMP32 | BPF_JGE | BPF_K: 1288 case BPF_JMP32 | BPF_JLE | BPF_K: 1289 case BPF_JMP32 | BPF_JSGT | BPF_K: 1290 case BPF_JMP32 | BPF_JSLT | BPF_K: 1291 case BPF_JMP32 | BPF_JSGE | BPF_K: 1292 case BPF_JMP32 | BPF_JSLE | BPF_K: 1293 case BPF_JMP32 | BPF_JSET | BPF_K: 1294 /* Accommodate for extra offset in case of a backjump. */ 1295 off = from->off; 1296 if (off < 0) 1297 off -= 2; 1298 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1299 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1300 *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, 1301 off); 1302 break; 1303 1304 case BPF_LD | BPF_IMM | BPF_DW: 1305 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); 1306 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1307 *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 1308 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); 1309 break; 1310 case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ 1311 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); 1312 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1313 if (emit_zext) 1314 *to++ = BPF_ZEXT_REG(BPF_REG_AX); 1315 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); 1316 break; 1317 1318 case BPF_ST | BPF_MEM | BPF_DW: 1319 case BPF_ST | BPF_MEM | BPF_W: 1320 case BPF_ST | BPF_MEM | BPF_H: 1321 case BPF_ST | BPF_MEM | BPF_B: 1322 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1323 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1324 *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); 1325 break; 1326 } 1327 out: 1328 return to - to_buff; 1329 } 1330 1331 static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, 1332 gfp_t gfp_extra_flags) 1333 { 1334 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; 1335 struct bpf_prog *fp; 1336 1337 fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags); 1338 if (fp != NULL) { 1339 /* aux->prog still points to the fp_other one, so 1340 * when promoting the clone to the real program, 1341 * this still needs to be adapted. 1342 */ 1343 memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); 1344 } 1345 1346 return fp; 1347 } 1348 1349 static void bpf_prog_clone_free(struct bpf_prog *fp) 1350 { 1351 /* aux was stolen by the other clone, so we cannot free 1352 * it from this path! It will be freed eventually by the 1353 * other program on release. 1354 * 1355 * At this point, we don't need a deferred release since 1356 * clone is guaranteed to not be locked. 1357 */ 1358 fp->aux = NULL; 1359 fp->stats = NULL; 1360 fp->active = NULL; 1361 __bpf_prog_free(fp); 1362 } 1363 1364 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) 1365 { 1366 /* We have to repoint aux->prog to self, as we don't 1367 * know whether fp here is the clone or the original. 1368 */ 1369 fp->aux->prog = fp; 1370 bpf_prog_clone_free(fp_other); 1371 } 1372 1373 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) 1374 { 1375 struct bpf_insn insn_buff[16], aux[2]; 1376 struct bpf_prog *clone, *tmp; 1377 int insn_delta, insn_cnt; 1378 struct bpf_insn *insn; 1379 int i, rewritten; 1380 1381 if (!bpf_jit_blinding_enabled(prog) || prog->blinded) 1382 return prog; 1383 1384 clone = bpf_prog_clone_create(prog, GFP_USER); 1385 if (!clone) 1386 return ERR_PTR(-ENOMEM); 1387 1388 insn_cnt = clone->len; 1389 insn = clone->insnsi; 1390 1391 for (i = 0; i < insn_cnt; i++, insn++) { 1392 /* We temporarily need to hold the original ld64 insn 1393 * so that we can still access the first part in the 1394 * second blinding run. 1395 */ 1396 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && 1397 insn[1].code == 0) 1398 memcpy(aux, insn, sizeof(aux)); 1399 1400 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, 1401 clone->aux->verifier_zext); 1402 if (!rewritten) 1403 continue; 1404 1405 tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); 1406 if (IS_ERR(tmp)) { 1407 /* Patching may have repointed aux->prog during 1408 * realloc from the original one, so we need to 1409 * fix it up here on error. 1410 */ 1411 bpf_jit_prog_release_other(prog, clone); 1412 return tmp; 1413 } 1414 1415 clone = tmp; 1416 insn_delta = rewritten - 1; 1417 1418 /* Walk new program and skip insns we just inserted. */ 1419 insn = clone->insnsi + i + insn_delta; 1420 insn_cnt += insn_delta; 1421 i += insn_delta; 1422 } 1423 1424 clone->blinded = 1; 1425 return clone; 1426 } 1427 #endif /* CONFIG_BPF_JIT */ 1428 1429 /* Base function for offset calculation. Needs to go into .text section, 1430 * therefore keeping it non-static as well; will also be used by JITs 1431 * anyway later on, so do not let the compiler omit it. This also needs 1432 * to go into kallsyms for correlation from e.g. bpftool, so naming 1433 * must not change. 1434 */ 1435 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1436 { 1437 return 0; 1438 } 1439 EXPORT_SYMBOL_GPL(__bpf_call_base); 1440 1441 /* All UAPI available opcodes. */ 1442 #define BPF_INSN_MAP(INSN_2, INSN_3) \ 1443 /* 32 bit ALU operations. */ \ 1444 /* Register based. */ \ 1445 INSN_3(ALU, ADD, X), \ 1446 INSN_3(ALU, SUB, X), \ 1447 INSN_3(ALU, AND, X), \ 1448 INSN_3(ALU, OR, X), \ 1449 INSN_3(ALU, LSH, X), \ 1450 INSN_3(ALU, RSH, X), \ 1451 INSN_3(ALU, XOR, X), \ 1452 INSN_3(ALU, MUL, X), \ 1453 INSN_3(ALU, MOV, X), \ 1454 INSN_3(ALU, ARSH, X), \ 1455 INSN_3(ALU, DIV, X), \ 1456 INSN_3(ALU, MOD, X), \ 1457 INSN_2(ALU, NEG), \ 1458 INSN_3(ALU, END, TO_BE), \ 1459 INSN_3(ALU, END, TO_LE), \ 1460 /* Immediate based. */ \ 1461 INSN_3(ALU, ADD, K), \ 1462 INSN_3(ALU, SUB, K), \ 1463 INSN_3(ALU, AND, K), \ 1464 INSN_3(ALU, OR, K), \ 1465 INSN_3(ALU, LSH, K), \ 1466 INSN_3(ALU, RSH, K), \ 1467 INSN_3(ALU, XOR, K), \ 1468 INSN_3(ALU, MUL, K), \ 1469 INSN_3(ALU, MOV, K), \ 1470 INSN_3(ALU, ARSH, K), \ 1471 INSN_3(ALU, DIV, K), \ 1472 INSN_3(ALU, MOD, K), \ 1473 /* 64 bit ALU operations. */ \ 1474 /* Register based. */ \ 1475 INSN_3(ALU64, ADD, X), \ 1476 INSN_3(ALU64, SUB, X), \ 1477 INSN_3(ALU64, AND, X), \ 1478 INSN_3(ALU64, OR, X), \ 1479 INSN_3(ALU64, LSH, X), \ 1480 INSN_3(ALU64, RSH, X), \ 1481 INSN_3(ALU64, XOR, X), \ 1482 INSN_3(ALU64, MUL, X), \ 1483 INSN_3(ALU64, MOV, X), \ 1484 INSN_3(ALU64, ARSH, X), \ 1485 INSN_3(ALU64, DIV, X), \ 1486 INSN_3(ALU64, MOD, X), \ 1487 INSN_2(ALU64, NEG), \ 1488 /* Immediate based. */ \ 1489 INSN_3(ALU64, ADD, K), \ 1490 INSN_3(ALU64, SUB, K), \ 1491 INSN_3(ALU64, AND, K), \ 1492 INSN_3(ALU64, OR, K), \ 1493 INSN_3(ALU64, LSH, K), \ 1494 INSN_3(ALU64, RSH, K), \ 1495 INSN_3(ALU64, XOR, K), \ 1496 INSN_3(ALU64, MUL, K), \ 1497 INSN_3(ALU64, MOV, K), \ 1498 INSN_3(ALU64, ARSH, K), \ 1499 INSN_3(ALU64, DIV, K), \ 1500 INSN_3(ALU64, MOD, K), \ 1501 /* Call instruction. */ \ 1502 INSN_2(JMP, CALL), \ 1503 /* Exit instruction. */ \ 1504 INSN_2(JMP, EXIT), \ 1505 /* 32-bit Jump instructions. */ \ 1506 /* Register based. */ \ 1507 INSN_3(JMP32, JEQ, X), \ 1508 INSN_3(JMP32, JNE, X), \ 1509 INSN_3(JMP32, JGT, X), \ 1510 INSN_3(JMP32, JLT, X), \ 1511 INSN_3(JMP32, JGE, X), \ 1512 INSN_3(JMP32, JLE, X), \ 1513 INSN_3(JMP32, JSGT, X), \ 1514 INSN_3(JMP32, JSLT, X), \ 1515 INSN_3(JMP32, JSGE, X), \ 1516 INSN_3(JMP32, JSLE, X), \ 1517 INSN_3(JMP32, JSET, X), \ 1518 /* Immediate based. */ \ 1519 INSN_3(JMP32, JEQ, K), \ 1520 INSN_3(JMP32, JNE, K), \ 1521 INSN_3(JMP32, JGT, K), \ 1522 INSN_3(JMP32, JLT, K), \ 1523 INSN_3(JMP32, JGE, K), \ 1524 INSN_3(JMP32, JLE, K), \ 1525 INSN_3(JMP32, JSGT, K), \ 1526 INSN_3(JMP32, JSLT, K), \ 1527 INSN_3(JMP32, JSGE, K), \ 1528 INSN_3(JMP32, JSLE, K), \ 1529 INSN_3(JMP32, JSET, K), \ 1530 /* Jump instructions. */ \ 1531 /* Register based. */ \ 1532 INSN_3(JMP, JEQ, X), \ 1533 INSN_3(JMP, JNE, X), \ 1534 INSN_3(JMP, JGT, X), \ 1535 INSN_3(JMP, JLT, X), \ 1536 INSN_3(JMP, JGE, X), \ 1537 INSN_3(JMP, JLE, X), \ 1538 INSN_3(JMP, JSGT, X), \ 1539 INSN_3(JMP, JSLT, X), \ 1540 INSN_3(JMP, JSGE, X), \ 1541 INSN_3(JMP, JSLE, X), \ 1542 INSN_3(JMP, JSET, X), \ 1543 /* Immediate based. */ \ 1544 INSN_3(JMP, JEQ, K), \ 1545 INSN_3(JMP, JNE, K), \ 1546 INSN_3(JMP, JGT, K), \ 1547 INSN_3(JMP, JLT, K), \ 1548 INSN_3(JMP, JGE, K), \ 1549 INSN_3(JMP, JLE, K), \ 1550 INSN_3(JMP, JSGT, K), \ 1551 INSN_3(JMP, JSLT, K), \ 1552 INSN_3(JMP, JSGE, K), \ 1553 INSN_3(JMP, JSLE, K), \ 1554 INSN_3(JMP, JSET, K), \ 1555 INSN_2(JMP, JA), \ 1556 /* Store instructions. */ \ 1557 /* Register based. */ \ 1558 INSN_3(STX, MEM, B), \ 1559 INSN_3(STX, MEM, H), \ 1560 INSN_3(STX, MEM, W), \ 1561 INSN_3(STX, MEM, DW), \ 1562 INSN_3(STX, ATOMIC, W), \ 1563 INSN_3(STX, ATOMIC, DW), \ 1564 /* Immediate based. */ \ 1565 INSN_3(ST, MEM, B), \ 1566 INSN_3(ST, MEM, H), \ 1567 INSN_3(ST, MEM, W), \ 1568 INSN_3(ST, MEM, DW), \ 1569 /* Load instructions. */ \ 1570 /* Register based. */ \ 1571 INSN_3(LDX, MEM, B), \ 1572 INSN_3(LDX, MEM, H), \ 1573 INSN_3(LDX, MEM, W), \ 1574 INSN_3(LDX, MEM, DW), \ 1575 /* Immediate based. */ \ 1576 INSN_3(LD, IMM, DW) 1577 1578 bool bpf_opcode_in_insntable(u8 code) 1579 { 1580 #define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true 1581 #define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true 1582 static const bool public_insntable[256] = { 1583 [0 ... 255] = false, 1584 /* Now overwrite non-defaults ... */ 1585 BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), 1586 /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ 1587 [BPF_LD | BPF_ABS | BPF_B] = true, 1588 [BPF_LD | BPF_ABS | BPF_H] = true, 1589 [BPF_LD | BPF_ABS | BPF_W] = true, 1590 [BPF_LD | BPF_IND | BPF_B] = true, 1591 [BPF_LD | BPF_IND | BPF_H] = true, 1592 [BPF_LD | BPF_IND | BPF_W] = true, 1593 }; 1594 #undef BPF_INSN_3_TBL 1595 #undef BPF_INSN_2_TBL 1596 return public_insntable[code]; 1597 } 1598 1599 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1600 u64 __weak bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr) 1601 { 1602 memset(dst, 0, size); 1603 return -EFAULT; 1604 } 1605 1606 /** 1607 * ___bpf_prog_run - run eBPF program on a given context 1608 * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers 1609 * @insn: is the array of eBPF instructions 1610 * 1611 * Decode and execute eBPF instructions. 1612 * 1613 * Return: whatever value is in %BPF_R0 at program exit 1614 */ 1615 static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) 1616 { 1617 #define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y 1618 #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z 1619 static const void * const jumptable[256] __annotate_jump_table = { 1620 [0 ... 255] = &&default_label, 1621 /* Now overwrite non-defaults ... */ 1622 BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), 1623 /* Non-UAPI available opcodes. */ 1624 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, 1625 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, 1626 [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC, 1627 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B, 1628 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H, 1629 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W, 1630 [BPF_LDX | BPF_PROBE_MEM | BPF_DW] = &&LDX_PROBE_MEM_DW, 1631 }; 1632 #undef BPF_INSN_3_LBL 1633 #undef BPF_INSN_2_LBL 1634 u32 tail_call_cnt = 0; 1635 1636 #define CONT ({ insn++; goto select_insn; }) 1637 #define CONT_JMP ({ insn++; goto select_insn; }) 1638 1639 select_insn: 1640 goto *jumptable[insn->code]; 1641 1642 /* Explicitly mask the register-based shift amounts with 63 or 31 1643 * to avoid undefined behavior. Normally this won't affect the 1644 * generated code, for example, in case of native 64 bit archs such 1645 * as x86-64 or arm64, the compiler is optimizing the AND away for 1646 * the interpreter. In case of JITs, each of the JIT backends compiles 1647 * the BPF shift operations to machine instructions which produce 1648 * implementation-defined results in such a case; the resulting 1649 * contents of the register may be arbitrary, but program behaviour 1650 * as a whole remains defined. In other words, in case of JIT backends, 1651 * the AND must /not/ be added to the emitted LSH/RSH/ARSH translation. 1652 */ 1653 /* ALU (shifts) */ 1654 #define SHT(OPCODE, OP) \ 1655 ALU64_##OPCODE##_X: \ 1656 DST = DST OP (SRC & 63); \ 1657 CONT; \ 1658 ALU_##OPCODE##_X: \ 1659 DST = (u32) DST OP ((u32) SRC & 31); \ 1660 CONT; \ 1661 ALU64_##OPCODE##_K: \ 1662 DST = DST OP IMM; \ 1663 CONT; \ 1664 ALU_##OPCODE##_K: \ 1665 DST = (u32) DST OP (u32) IMM; \ 1666 CONT; 1667 /* ALU (rest) */ 1668 #define ALU(OPCODE, OP) \ 1669 ALU64_##OPCODE##_X: \ 1670 DST = DST OP SRC; \ 1671 CONT; \ 1672 ALU_##OPCODE##_X: \ 1673 DST = (u32) DST OP (u32) SRC; \ 1674 CONT; \ 1675 ALU64_##OPCODE##_K: \ 1676 DST = DST OP IMM; \ 1677 CONT; \ 1678 ALU_##OPCODE##_K: \ 1679 DST = (u32) DST OP (u32) IMM; \ 1680 CONT; 1681 ALU(ADD, +) 1682 ALU(SUB, -) 1683 ALU(AND, &) 1684 ALU(OR, |) 1685 ALU(XOR, ^) 1686 ALU(MUL, *) 1687 SHT(LSH, <<) 1688 SHT(RSH, >>) 1689 #undef SHT 1690 #undef ALU 1691 ALU_NEG: 1692 DST = (u32) -DST; 1693 CONT; 1694 ALU64_NEG: 1695 DST = -DST; 1696 CONT; 1697 ALU_MOV_X: 1698 DST = (u32) SRC; 1699 CONT; 1700 ALU_MOV_K: 1701 DST = (u32) IMM; 1702 CONT; 1703 ALU64_MOV_X: 1704 DST = SRC; 1705 CONT; 1706 ALU64_MOV_K: 1707 DST = IMM; 1708 CONT; 1709 LD_IMM_DW: 1710 DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; 1711 insn++; 1712 CONT; 1713 ALU_ARSH_X: 1714 DST = (u64) (u32) (((s32) DST) >> (SRC & 31)); 1715 CONT; 1716 ALU_ARSH_K: 1717 DST = (u64) (u32) (((s32) DST) >> IMM); 1718 CONT; 1719 ALU64_ARSH_X: 1720 (*(s64 *) &DST) >>= (SRC & 63); 1721 CONT; 1722 ALU64_ARSH_K: 1723 (*(s64 *) &DST) >>= IMM; 1724 CONT; 1725 ALU64_MOD_X: 1726 div64_u64_rem(DST, SRC, &AX); 1727 DST = AX; 1728 CONT; 1729 ALU_MOD_X: 1730 AX = (u32) DST; 1731 DST = do_div(AX, (u32) SRC); 1732 CONT; 1733 ALU64_MOD_K: 1734 div64_u64_rem(DST, IMM, &AX); 1735 DST = AX; 1736 CONT; 1737 ALU_MOD_K: 1738 AX = (u32) DST; 1739 DST = do_div(AX, (u32) IMM); 1740 CONT; 1741 ALU64_DIV_X: 1742 DST = div64_u64(DST, SRC); 1743 CONT; 1744 ALU_DIV_X: 1745 AX = (u32) DST; 1746 do_div(AX, (u32) SRC); 1747 DST = (u32) AX; 1748 CONT; 1749 ALU64_DIV_K: 1750 DST = div64_u64(DST, IMM); 1751 CONT; 1752 ALU_DIV_K: 1753 AX = (u32) DST; 1754 do_div(AX, (u32) IMM); 1755 DST = (u32) AX; 1756 CONT; 1757 ALU_END_TO_BE: 1758 switch (IMM) { 1759 case 16: 1760 DST = (__force u16) cpu_to_be16(DST); 1761 break; 1762 case 32: 1763 DST = (__force u32) cpu_to_be32(DST); 1764 break; 1765 case 64: 1766 DST = (__force u64) cpu_to_be64(DST); 1767 break; 1768 } 1769 CONT; 1770 ALU_END_TO_LE: 1771 switch (IMM) { 1772 case 16: 1773 DST = (__force u16) cpu_to_le16(DST); 1774 break; 1775 case 32: 1776 DST = (__force u32) cpu_to_le32(DST); 1777 break; 1778 case 64: 1779 DST = (__force u64) cpu_to_le64(DST); 1780 break; 1781 } 1782 CONT; 1783 1784 /* CALL */ 1785 JMP_CALL: 1786 /* Function call scratches BPF_R1-BPF_R5 registers, 1787 * preserves BPF_R6-BPF_R9, and stores return value 1788 * into BPF_R0. 1789 */ 1790 BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3, 1791 BPF_R4, BPF_R5); 1792 CONT; 1793 1794 JMP_CALL_ARGS: 1795 BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, 1796 BPF_R3, BPF_R4, 1797 BPF_R5, 1798 insn + insn->off + 1); 1799 CONT; 1800 1801 JMP_TAIL_CALL: { 1802 struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; 1803 struct bpf_array *array = container_of(map, struct bpf_array, map); 1804 struct bpf_prog *prog; 1805 u32 index = BPF_R3; 1806 1807 if (unlikely(index >= array->map.max_entries)) 1808 goto out; 1809 1810 if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT)) 1811 goto out; 1812 1813 tail_call_cnt++; 1814 1815 prog = READ_ONCE(array->ptrs[index]); 1816 if (!prog) 1817 goto out; 1818 1819 /* ARG1 at this point is guaranteed to point to CTX from 1820 * the verifier side due to the fact that the tail call is 1821 * handled like a helper, that is, bpf_tail_call_proto, 1822 * where arg1_type is ARG_PTR_TO_CTX. 1823 */ 1824 insn = prog->insnsi; 1825 goto select_insn; 1826 out: 1827 CONT; 1828 } 1829 JMP_JA: 1830 insn += insn->off; 1831 CONT; 1832 JMP_EXIT: 1833 return BPF_R0; 1834 /* JMP */ 1835 #define COND_JMP(SIGN, OPCODE, CMP_OP) \ 1836 JMP_##OPCODE##_X: \ 1837 if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ 1838 insn += insn->off; \ 1839 CONT_JMP; \ 1840 } \ 1841 CONT; \ 1842 JMP32_##OPCODE##_X: \ 1843 if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ 1844 insn += insn->off; \ 1845 CONT_JMP; \ 1846 } \ 1847 CONT; \ 1848 JMP_##OPCODE##_K: \ 1849 if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ 1850 insn += insn->off; \ 1851 CONT_JMP; \ 1852 } \ 1853 CONT; \ 1854 JMP32_##OPCODE##_K: \ 1855 if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ 1856 insn += insn->off; \ 1857 CONT_JMP; \ 1858 } \ 1859 CONT; 1860 COND_JMP(u, JEQ, ==) 1861 COND_JMP(u, JNE, !=) 1862 COND_JMP(u, JGT, >) 1863 COND_JMP(u, JLT, <) 1864 COND_JMP(u, JGE, >=) 1865 COND_JMP(u, JLE, <=) 1866 COND_JMP(u, JSET, &) 1867 COND_JMP(s, JSGT, >) 1868 COND_JMP(s, JSLT, <) 1869 COND_JMP(s, JSGE, >=) 1870 COND_JMP(s, JSLE, <=) 1871 #undef COND_JMP 1872 /* ST, STX and LDX*/ 1873 ST_NOSPEC: 1874 /* Speculation barrier for mitigating Speculative Store Bypass. 1875 * In case of arm64, we rely on the firmware mitigation as 1876 * controlled via the ssbd kernel parameter. Whenever the 1877 * mitigation is enabled, it works for all of the kernel code 1878 * with no need to provide any additional instructions here. 1879 * In case of x86, we use 'lfence' insn for mitigation. We 1880 * reuse preexisting logic from Spectre v1 mitigation that 1881 * happens to produce the required code on x86 for v4 as well. 1882 */ 1883 #ifdef CONFIG_X86 1884 barrier_nospec(); 1885 #endif 1886 CONT; 1887 #define LDST(SIZEOP, SIZE) \ 1888 STX_MEM_##SIZEOP: \ 1889 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ 1890 CONT; \ 1891 ST_MEM_##SIZEOP: \ 1892 *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \ 1893 CONT; \ 1894 LDX_MEM_##SIZEOP: \ 1895 DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ 1896 CONT; 1897 1898 LDST(B, u8) 1899 LDST(H, u16) 1900 LDST(W, u32) 1901 LDST(DW, u64) 1902 #undef LDST 1903 #define LDX_PROBE(SIZEOP, SIZE) \ 1904 LDX_PROBE_MEM_##SIZEOP: \ 1905 bpf_probe_read_kernel(&DST, SIZE, (const void *)(long) (SRC + insn->off)); \ 1906 CONT; 1907 LDX_PROBE(B, 1) 1908 LDX_PROBE(H, 2) 1909 LDX_PROBE(W, 4) 1910 LDX_PROBE(DW, 8) 1911 #undef LDX_PROBE 1912 1913 #define ATOMIC_ALU_OP(BOP, KOP) \ 1914 case BOP: \ 1915 if (BPF_SIZE(insn->code) == BPF_W) \ 1916 atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \ 1917 (DST + insn->off)); \ 1918 else \ 1919 atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \ 1920 (DST + insn->off)); \ 1921 break; \ 1922 case BOP | BPF_FETCH: \ 1923 if (BPF_SIZE(insn->code) == BPF_W) \ 1924 SRC = (u32) atomic_fetch_##KOP( \ 1925 (u32) SRC, \ 1926 (atomic_t *)(unsigned long) (DST + insn->off)); \ 1927 else \ 1928 SRC = (u64) atomic64_fetch_##KOP( \ 1929 (u64) SRC, \ 1930 (atomic64_t *)(unsigned long) (DST + insn->off)); \ 1931 break; 1932 1933 STX_ATOMIC_DW: 1934 STX_ATOMIC_W: 1935 switch (IMM) { 1936 ATOMIC_ALU_OP(BPF_ADD, add) 1937 ATOMIC_ALU_OP(BPF_AND, and) 1938 ATOMIC_ALU_OP(BPF_OR, or) 1939 ATOMIC_ALU_OP(BPF_XOR, xor) 1940 #undef ATOMIC_ALU_OP 1941 1942 case BPF_XCHG: 1943 if (BPF_SIZE(insn->code) == BPF_W) 1944 SRC = (u32) atomic_xchg( 1945 (atomic_t *)(unsigned long) (DST + insn->off), 1946 (u32) SRC); 1947 else 1948 SRC = (u64) atomic64_xchg( 1949 (atomic64_t *)(unsigned long) (DST + insn->off), 1950 (u64) SRC); 1951 break; 1952 case BPF_CMPXCHG: 1953 if (BPF_SIZE(insn->code) == BPF_W) 1954 BPF_R0 = (u32) atomic_cmpxchg( 1955 (atomic_t *)(unsigned long) (DST + insn->off), 1956 (u32) BPF_R0, (u32) SRC); 1957 else 1958 BPF_R0 = (u64) atomic64_cmpxchg( 1959 (atomic64_t *)(unsigned long) (DST + insn->off), 1960 (u64) BPF_R0, (u64) SRC); 1961 break; 1962 1963 default: 1964 goto default_label; 1965 } 1966 CONT; 1967 1968 default_label: 1969 /* If we ever reach this, we have a bug somewhere. Die hard here 1970 * instead of just returning 0; we could be somewhere in a subprog, 1971 * so execution could continue otherwise which we do /not/ want. 1972 * 1973 * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). 1974 */ 1975 pr_warn("BPF interpreter: unknown opcode %02x (imm: 0x%x)\n", 1976 insn->code, insn->imm); 1977 BUG_ON(1); 1978 return 0; 1979 } 1980 1981 #define PROG_NAME(stack_size) __bpf_prog_run##stack_size 1982 #define DEFINE_BPF_PROG_RUN(stack_size) \ 1983 static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ 1984 { \ 1985 u64 stack[stack_size / sizeof(u64)]; \ 1986 u64 regs[MAX_BPF_EXT_REG]; \ 1987 \ 1988 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 1989 ARG1 = (u64) (unsigned long) ctx; \ 1990 return ___bpf_prog_run(regs, insn); \ 1991 } 1992 1993 #define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size 1994 #define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ 1995 static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ 1996 const struct bpf_insn *insn) \ 1997 { \ 1998 u64 stack[stack_size / sizeof(u64)]; \ 1999 u64 regs[MAX_BPF_EXT_REG]; \ 2000 \ 2001 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 2002 BPF_R1 = r1; \ 2003 BPF_R2 = r2; \ 2004 BPF_R3 = r3; \ 2005 BPF_R4 = r4; \ 2006 BPF_R5 = r5; \ 2007 return ___bpf_prog_run(regs, insn); \ 2008 } 2009 2010 #define EVAL1(FN, X) FN(X) 2011 #define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) 2012 #define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) 2013 #define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) 2014 #define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) 2015 #define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) 2016 2017 EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); 2018 EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); 2019 EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); 2020 2021 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); 2022 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); 2023 EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); 2024 2025 #define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), 2026 2027 static unsigned int (*interpreters[])(const void *ctx, 2028 const struct bpf_insn *insn) = { 2029 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2030 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2031 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2032 }; 2033 #undef PROG_NAME_LIST 2034 #define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), 2035 static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, 2036 const struct bpf_insn *insn) = { 2037 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2038 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2039 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2040 }; 2041 #undef PROG_NAME_LIST 2042 2043 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) 2044 { 2045 stack_depth = max_t(u32, stack_depth, 1); 2046 insn->off = (s16) insn->imm; 2047 insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - 2048 __bpf_call_base_args; 2049 insn->code = BPF_JMP | BPF_CALL_ARGS; 2050 } 2051 2052 #else 2053 static unsigned int __bpf_prog_ret0_warn(const void *ctx, 2054 const struct bpf_insn *insn) 2055 { 2056 /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON 2057 * is not working properly, so warn about it! 2058 */ 2059 WARN_ON_ONCE(1); 2060 return 0; 2061 } 2062 #endif 2063 2064 bool bpf_prog_map_compatible(struct bpf_map *map, 2065 const struct bpf_prog *fp) 2066 { 2067 bool ret; 2068 2069 if (fp->kprobe_override) 2070 return false; 2071 2072 spin_lock(&map->owner.lock); 2073 if (!map->owner.type) { 2074 /* There's no owner yet where we could check for 2075 * compatibility. 2076 */ 2077 map->owner.type = fp->type; 2078 map->owner.jited = fp->jited; 2079 map->owner.xdp_has_frags = fp->aux->xdp_has_frags; 2080 ret = true; 2081 } else { 2082 ret = map->owner.type == fp->type && 2083 map->owner.jited == fp->jited && 2084 map->owner.xdp_has_frags == fp->aux->xdp_has_frags; 2085 } 2086 spin_unlock(&map->owner.lock); 2087 2088 return ret; 2089 } 2090 2091 static int bpf_check_tail_call(const struct bpf_prog *fp) 2092 { 2093 struct bpf_prog_aux *aux = fp->aux; 2094 int i, ret = 0; 2095 2096 mutex_lock(&aux->used_maps_mutex); 2097 for (i = 0; i < aux->used_map_cnt; i++) { 2098 struct bpf_map *map = aux->used_maps[i]; 2099 2100 if (!map_type_contains_progs(map)) 2101 continue; 2102 2103 if (!bpf_prog_map_compatible(map, fp)) { 2104 ret = -EINVAL; 2105 goto out; 2106 } 2107 } 2108 2109 out: 2110 mutex_unlock(&aux->used_maps_mutex); 2111 return ret; 2112 } 2113 2114 static void bpf_prog_select_func(struct bpf_prog *fp) 2115 { 2116 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2117 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); 2118 2119 fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; 2120 #else 2121 fp->bpf_func = __bpf_prog_ret0_warn; 2122 #endif 2123 } 2124 2125 /** 2126 * bpf_prog_select_runtime - select exec runtime for BPF program 2127 * @fp: bpf_prog populated with BPF program 2128 * @err: pointer to error variable 2129 * 2130 * Try to JIT eBPF program, if JIT is not available, use interpreter. 2131 * The BPF program will be executed via bpf_prog_run() function. 2132 * 2133 * Return: the &fp argument along with &err set to 0 for success or 2134 * a negative errno code on failure 2135 */ 2136 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) 2137 { 2138 /* In case of BPF to BPF calls, verifier did all the prep 2139 * work with regards to JITing, etc. 2140 */ 2141 bool jit_needed = false; 2142 2143 if (fp->bpf_func) 2144 goto finalize; 2145 2146 if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || 2147 bpf_prog_has_kfunc_call(fp)) 2148 jit_needed = true; 2149 2150 bpf_prog_select_func(fp); 2151 2152 /* eBPF JITs can rewrite the program in case constant 2153 * blinding is active. However, in case of error during 2154 * blinding, bpf_int_jit_compile() must always return a 2155 * valid program, which in this case would simply not 2156 * be JITed, but falls back to the interpreter. 2157 */ 2158 if (!bpf_prog_is_dev_bound(fp->aux)) { 2159 *err = bpf_prog_alloc_jited_linfo(fp); 2160 if (*err) 2161 return fp; 2162 2163 fp = bpf_int_jit_compile(fp); 2164 bpf_prog_jit_attempt_done(fp); 2165 if (!fp->jited && jit_needed) { 2166 *err = -ENOTSUPP; 2167 return fp; 2168 } 2169 } else { 2170 *err = bpf_prog_offload_compile(fp); 2171 if (*err) 2172 return fp; 2173 } 2174 2175 finalize: 2176 bpf_prog_lock_ro(fp); 2177 2178 /* The tail call compatibility check can only be done at 2179 * this late stage as we need to determine, if we deal 2180 * with JITed or non JITed program concatenations and not 2181 * all eBPF JITs might immediately support all features. 2182 */ 2183 *err = bpf_check_tail_call(fp); 2184 2185 return fp; 2186 } 2187 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); 2188 2189 static unsigned int __bpf_prog_ret1(const void *ctx, 2190 const struct bpf_insn *insn) 2191 { 2192 return 1; 2193 } 2194 2195 static struct bpf_prog_dummy { 2196 struct bpf_prog prog; 2197 } dummy_bpf_prog = { 2198 .prog = { 2199 .bpf_func = __bpf_prog_ret1, 2200 }, 2201 }; 2202 2203 struct bpf_empty_prog_array bpf_empty_prog_array = { 2204 .null_prog = NULL, 2205 }; 2206 EXPORT_SYMBOL(bpf_empty_prog_array); 2207 2208 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) 2209 { 2210 if (prog_cnt) 2211 return kzalloc(sizeof(struct bpf_prog_array) + 2212 sizeof(struct bpf_prog_array_item) * 2213 (prog_cnt + 1), 2214 flags); 2215 2216 return &bpf_empty_prog_array.hdr; 2217 } 2218 2219 void bpf_prog_array_free(struct bpf_prog_array *progs) 2220 { 2221 if (!progs || progs == &bpf_empty_prog_array.hdr) 2222 return; 2223 kfree_rcu(progs, rcu); 2224 } 2225 2226 int bpf_prog_array_length(struct bpf_prog_array *array) 2227 { 2228 struct bpf_prog_array_item *item; 2229 u32 cnt = 0; 2230 2231 for (item = array->items; item->prog; item++) 2232 if (item->prog != &dummy_bpf_prog.prog) 2233 cnt++; 2234 return cnt; 2235 } 2236 2237 bool bpf_prog_array_is_empty(struct bpf_prog_array *array) 2238 { 2239 struct bpf_prog_array_item *item; 2240 2241 for (item = array->items; item->prog; item++) 2242 if (item->prog != &dummy_bpf_prog.prog) 2243 return false; 2244 return true; 2245 } 2246 2247 static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, 2248 u32 *prog_ids, 2249 u32 request_cnt) 2250 { 2251 struct bpf_prog_array_item *item; 2252 int i = 0; 2253 2254 for (item = array->items; item->prog; item++) { 2255 if (item->prog == &dummy_bpf_prog.prog) 2256 continue; 2257 prog_ids[i] = item->prog->aux->id; 2258 if (++i == request_cnt) { 2259 item++; 2260 break; 2261 } 2262 } 2263 2264 return !!(item->prog); 2265 } 2266 2267 int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, 2268 __u32 __user *prog_ids, u32 cnt) 2269 { 2270 unsigned long err = 0; 2271 bool nospc; 2272 u32 *ids; 2273 2274 /* users of this function are doing: 2275 * cnt = bpf_prog_array_length(); 2276 * if (cnt > 0) 2277 * bpf_prog_array_copy_to_user(..., cnt); 2278 * so below kcalloc doesn't need extra cnt > 0 check. 2279 */ 2280 ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); 2281 if (!ids) 2282 return -ENOMEM; 2283 nospc = bpf_prog_array_copy_core(array, ids, cnt); 2284 err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); 2285 kfree(ids); 2286 if (err) 2287 return -EFAULT; 2288 if (nospc) 2289 return -ENOSPC; 2290 return 0; 2291 } 2292 2293 void bpf_prog_array_delete_safe(struct bpf_prog_array *array, 2294 struct bpf_prog *old_prog) 2295 { 2296 struct bpf_prog_array_item *item; 2297 2298 for (item = array->items; item->prog; item++) 2299 if (item->prog == old_prog) { 2300 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); 2301 break; 2302 } 2303 } 2304 2305 /** 2306 * bpf_prog_array_delete_safe_at() - Replaces the program at the given 2307 * index into the program array with 2308 * a dummy no-op program. 2309 * @array: a bpf_prog_array 2310 * @index: the index of the program to replace 2311 * 2312 * Skips over dummy programs, by not counting them, when calculating 2313 * the position of the program to replace. 2314 * 2315 * Return: 2316 * * 0 - Success 2317 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2318 * * -ENOENT - Index out of range 2319 */ 2320 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index) 2321 { 2322 return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog); 2323 } 2324 2325 /** 2326 * bpf_prog_array_update_at() - Updates the program at the given index 2327 * into the program array. 2328 * @array: a bpf_prog_array 2329 * @index: the index of the program to update 2330 * @prog: the program to insert into the array 2331 * 2332 * Skips over dummy programs, by not counting them, when calculating 2333 * the position of the program to update. 2334 * 2335 * Return: 2336 * * 0 - Success 2337 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2338 * * -ENOENT - Index out of range 2339 */ 2340 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 2341 struct bpf_prog *prog) 2342 { 2343 struct bpf_prog_array_item *item; 2344 2345 if (unlikely(index < 0)) 2346 return -EINVAL; 2347 2348 for (item = array->items; item->prog; item++) { 2349 if (item->prog == &dummy_bpf_prog.prog) 2350 continue; 2351 if (!index) { 2352 WRITE_ONCE(item->prog, prog); 2353 return 0; 2354 } 2355 index--; 2356 } 2357 return -ENOENT; 2358 } 2359 2360 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 2361 struct bpf_prog *exclude_prog, 2362 struct bpf_prog *include_prog, 2363 u64 bpf_cookie, 2364 struct bpf_prog_array **new_array) 2365 { 2366 int new_prog_cnt, carry_prog_cnt = 0; 2367 struct bpf_prog_array_item *existing, *new; 2368 struct bpf_prog_array *array; 2369 bool found_exclude = false; 2370 2371 /* Figure out how many existing progs we need to carry over to 2372 * the new array. 2373 */ 2374 if (old_array) { 2375 existing = old_array->items; 2376 for (; existing->prog; existing++) { 2377 if (existing->prog == exclude_prog) { 2378 found_exclude = true; 2379 continue; 2380 } 2381 if (existing->prog != &dummy_bpf_prog.prog) 2382 carry_prog_cnt++; 2383 if (existing->prog == include_prog) 2384 return -EEXIST; 2385 } 2386 } 2387 2388 if (exclude_prog && !found_exclude) 2389 return -ENOENT; 2390 2391 /* How many progs (not NULL) will be in the new array? */ 2392 new_prog_cnt = carry_prog_cnt; 2393 if (include_prog) 2394 new_prog_cnt += 1; 2395 2396 /* Do we have any prog (not NULL) in the new array? */ 2397 if (!new_prog_cnt) { 2398 *new_array = NULL; 2399 return 0; 2400 } 2401 2402 /* +1 as the end of prog_array is marked with NULL */ 2403 array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); 2404 if (!array) 2405 return -ENOMEM; 2406 new = array->items; 2407 2408 /* Fill in the new prog array */ 2409 if (carry_prog_cnt) { 2410 existing = old_array->items; 2411 for (; existing->prog; existing++) { 2412 if (existing->prog == exclude_prog || 2413 existing->prog == &dummy_bpf_prog.prog) 2414 continue; 2415 2416 new->prog = existing->prog; 2417 new->bpf_cookie = existing->bpf_cookie; 2418 new++; 2419 } 2420 } 2421 if (include_prog) { 2422 new->prog = include_prog; 2423 new->bpf_cookie = bpf_cookie; 2424 new++; 2425 } 2426 new->prog = NULL; 2427 *new_array = array; 2428 return 0; 2429 } 2430 2431 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 2432 u32 *prog_ids, u32 request_cnt, 2433 u32 *prog_cnt) 2434 { 2435 u32 cnt = 0; 2436 2437 if (array) 2438 cnt = bpf_prog_array_length(array); 2439 2440 *prog_cnt = cnt; 2441 2442 /* return early if user requested only program count or nothing to copy */ 2443 if (!request_cnt || !cnt) 2444 return 0; 2445 2446 /* this function is called under trace/bpf_trace.c: bpf_event_mutex */ 2447 return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC 2448 : 0; 2449 } 2450 2451 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 2452 struct bpf_map **used_maps, u32 len) 2453 { 2454 struct bpf_map *map; 2455 u32 i; 2456 2457 for (i = 0; i < len; i++) { 2458 map = used_maps[i]; 2459 if (map->ops->map_poke_untrack) 2460 map->ops->map_poke_untrack(map, aux); 2461 bpf_map_put(map); 2462 } 2463 } 2464 2465 static void bpf_free_used_maps(struct bpf_prog_aux *aux) 2466 { 2467 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt); 2468 kfree(aux->used_maps); 2469 } 2470 2471 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 2472 struct btf_mod_pair *used_btfs, u32 len) 2473 { 2474 #ifdef CONFIG_BPF_SYSCALL 2475 struct btf_mod_pair *btf_mod; 2476 u32 i; 2477 2478 for (i = 0; i < len; i++) { 2479 btf_mod = &used_btfs[i]; 2480 if (btf_mod->module) 2481 module_put(btf_mod->module); 2482 btf_put(btf_mod->btf); 2483 } 2484 #endif 2485 } 2486 2487 static void bpf_free_used_btfs(struct bpf_prog_aux *aux) 2488 { 2489 __bpf_free_used_btfs(aux, aux->used_btfs, aux->used_btf_cnt); 2490 kfree(aux->used_btfs); 2491 } 2492 2493 static void bpf_prog_free_deferred(struct work_struct *work) 2494 { 2495 struct bpf_prog_aux *aux; 2496 int i; 2497 2498 aux = container_of(work, struct bpf_prog_aux, work); 2499 #ifdef CONFIG_BPF_SYSCALL 2500 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); 2501 #endif 2502 bpf_free_used_maps(aux); 2503 bpf_free_used_btfs(aux); 2504 if (bpf_prog_is_dev_bound(aux)) 2505 bpf_prog_offload_destroy(aux->prog); 2506 #ifdef CONFIG_PERF_EVENTS 2507 if (aux->prog->has_callchain_buf) 2508 put_callchain_buffers(); 2509 #endif 2510 if (aux->dst_trampoline) 2511 bpf_trampoline_put(aux->dst_trampoline); 2512 for (i = 0; i < aux->func_cnt; i++) { 2513 /* We can just unlink the subprog poke descriptor table as 2514 * it was originally linked to the main program and is also 2515 * released along with it. 2516 */ 2517 aux->func[i]->aux->poke_tab = NULL; 2518 bpf_jit_free(aux->func[i]); 2519 } 2520 if (aux->func_cnt) { 2521 kfree(aux->func); 2522 bpf_prog_unlock_free(aux->prog); 2523 } else { 2524 bpf_jit_free(aux->prog); 2525 } 2526 } 2527 2528 void bpf_prog_free(struct bpf_prog *fp) 2529 { 2530 struct bpf_prog_aux *aux = fp->aux; 2531 2532 if (aux->dst_prog) 2533 bpf_prog_put(aux->dst_prog); 2534 INIT_WORK(&aux->work, bpf_prog_free_deferred); 2535 schedule_work(&aux->work); 2536 } 2537 EXPORT_SYMBOL_GPL(bpf_prog_free); 2538 2539 /* RNG for unpriviledged user space with separated state from prandom_u32(). */ 2540 static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); 2541 2542 void bpf_user_rnd_init_once(void) 2543 { 2544 prandom_init_once(&bpf_user_rnd_state); 2545 } 2546 2547 BPF_CALL_0(bpf_user_rnd_u32) 2548 { 2549 /* Should someone ever have the rather unwise idea to use some 2550 * of the registers passed into this function, then note that 2551 * this function is called from native eBPF and classic-to-eBPF 2552 * transformations. Register assignments from both sides are 2553 * different, f.e. classic always sets fn(ctx, A, X) here. 2554 */ 2555 struct rnd_state *state; 2556 u32 res; 2557 2558 state = &get_cpu_var(bpf_user_rnd_state); 2559 res = prandom_u32_state(state); 2560 put_cpu_var(bpf_user_rnd_state); 2561 2562 return res; 2563 } 2564 2565 BPF_CALL_0(bpf_get_raw_cpu_id) 2566 { 2567 return raw_smp_processor_id(); 2568 } 2569 2570 /* Weak definitions of helper functions in case we don't have bpf syscall. */ 2571 const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; 2572 const struct bpf_func_proto bpf_map_update_elem_proto __weak; 2573 const struct bpf_func_proto bpf_map_delete_elem_proto __weak; 2574 const struct bpf_func_proto bpf_map_push_elem_proto __weak; 2575 const struct bpf_func_proto bpf_map_pop_elem_proto __weak; 2576 const struct bpf_func_proto bpf_map_peek_elem_proto __weak; 2577 const struct bpf_func_proto bpf_spin_lock_proto __weak; 2578 const struct bpf_func_proto bpf_spin_unlock_proto __weak; 2579 const struct bpf_func_proto bpf_jiffies64_proto __weak; 2580 2581 const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; 2582 const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; 2583 const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; 2584 const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; 2585 const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak; 2586 const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak; 2587 2588 const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; 2589 const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; 2590 const struct bpf_func_proto bpf_get_current_comm_proto __weak; 2591 const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; 2592 const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak; 2593 const struct bpf_func_proto bpf_get_local_storage_proto __weak; 2594 const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak; 2595 const struct bpf_func_proto bpf_snprintf_btf_proto __weak; 2596 const struct bpf_func_proto bpf_seq_printf_btf_proto __weak; 2597 2598 const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) 2599 { 2600 return NULL; 2601 } 2602 2603 const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void) 2604 { 2605 return NULL; 2606 } 2607 2608 u64 __weak 2609 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 2610 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 2611 { 2612 return -ENOTSUPP; 2613 } 2614 EXPORT_SYMBOL_GPL(bpf_event_output); 2615 2616 /* Always built-in helper functions. */ 2617 const struct bpf_func_proto bpf_tail_call_proto = { 2618 .func = NULL, 2619 .gpl_only = false, 2620 .ret_type = RET_VOID, 2621 .arg1_type = ARG_PTR_TO_CTX, 2622 .arg2_type = ARG_CONST_MAP_PTR, 2623 .arg3_type = ARG_ANYTHING, 2624 }; 2625 2626 /* Stub for JITs that only support cBPF. eBPF programs are interpreted. 2627 * It is encouraged to implement bpf_int_jit_compile() instead, so that 2628 * eBPF and implicitly also cBPF can get JITed! 2629 */ 2630 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) 2631 { 2632 return prog; 2633 } 2634 2635 /* Stub for JITs that support eBPF. All cBPF code gets transformed into 2636 * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). 2637 */ 2638 void __weak bpf_jit_compile(struct bpf_prog *prog) 2639 { 2640 } 2641 2642 bool __weak bpf_helper_changes_pkt_data(void *func) 2643 { 2644 return false; 2645 } 2646 2647 /* Return TRUE if the JIT backend wants verifier to enable sub-register usage 2648 * analysis code and wants explicit zero extension inserted by verifier. 2649 * Otherwise, return FALSE. 2650 * 2651 * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if 2652 * you don't override this. JITs that don't want these extra insns can detect 2653 * them using insn_is_zext. 2654 */ 2655 bool __weak bpf_jit_needs_zext(void) 2656 { 2657 return false; 2658 } 2659 2660 bool __weak bpf_jit_supports_kfunc_call(void) 2661 { 2662 return false; 2663 } 2664 2665 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call 2666 * skb_copy_bits(), so provide a weak definition of it for NET-less config. 2667 */ 2668 int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, 2669 int len) 2670 { 2671 return -EFAULT; 2672 } 2673 2674 int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2675 void *addr1, void *addr2) 2676 { 2677 return -ENOTSUPP; 2678 } 2679 2680 void * __weak bpf_arch_text_copy(void *dst, void *src, size_t len) 2681 { 2682 return ERR_PTR(-ENOTSUPP); 2683 } 2684 2685 DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); 2686 EXPORT_SYMBOL(bpf_stats_enabled_key); 2687 2688 /* All definitions of tracepoints related to BPF. */ 2689 #define CREATE_TRACE_POINTS 2690 #include <linux/bpf_trace.h> 2691 2692 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); 2693 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); 2694