1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Linux Socket Filter - Kernel level socket filtering 4 * 5 * Based on the design of the Berkeley Packet Filter. The new 6 * internal format has been designed by PLUMgrid: 7 * 8 * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com 9 * 10 * Authors: 11 * 12 * Jay Schulist <jschlst@samba.org> 13 * Alexei Starovoitov <ast@plumgrid.com> 14 * Daniel Borkmann <dborkman@redhat.com> 15 * 16 * Andi Kleen - Fix a few bad bugs and races. 17 * Kris Katterjohn - Added many additional checks in bpf_check_classic() 18 */ 19 20 #include <uapi/linux/btf.h> 21 #include <linux/filter.h> 22 #include <linux/skbuff.h> 23 #include <linux/vmalloc.h> 24 #include <linux/random.h> 25 #include <linux/moduleloader.h> 26 #include <linux/bpf.h> 27 #include <linux/btf.h> 28 #include <linux/objtool.h> 29 #include <linux/rbtree_latch.h> 30 #include <linux/kallsyms.h> 31 #include <linux/rcupdate.h> 32 #include <linux/perf_event.h> 33 #include <linux/extable.h> 34 #include <linux/log2.h> 35 #include <linux/bpf_verifier.h> 36 37 #include <asm/barrier.h> 38 #include <asm/unaligned.h> 39 40 /* Registers */ 41 #define BPF_R0 regs[BPF_REG_0] 42 #define BPF_R1 regs[BPF_REG_1] 43 #define BPF_R2 regs[BPF_REG_2] 44 #define BPF_R3 regs[BPF_REG_3] 45 #define BPF_R4 regs[BPF_REG_4] 46 #define BPF_R5 regs[BPF_REG_5] 47 #define BPF_R6 regs[BPF_REG_6] 48 #define BPF_R7 regs[BPF_REG_7] 49 #define BPF_R8 regs[BPF_REG_8] 50 #define BPF_R9 regs[BPF_REG_9] 51 #define BPF_R10 regs[BPF_REG_10] 52 53 /* Named registers */ 54 #define DST regs[insn->dst_reg] 55 #define SRC regs[insn->src_reg] 56 #define FP regs[BPF_REG_FP] 57 #define AX regs[BPF_REG_AX] 58 #define ARG1 regs[BPF_REG_ARG1] 59 #define CTX regs[BPF_REG_CTX] 60 #define IMM insn->imm 61 62 /* No hurry in this branch 63 * 64 * Exported for the bpf jit load helper. 65 */ 66 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size) 67 { 68 u8 *ptr = NULL; 69 70 if (k >= SKF_NET_OFF) 71 ptr = skb_network_header(skb) + k - SKF_NET_OFF; 72 else if (k >= SKF_LL_OFF) 73 ptr = skb_mac_header(skb) + k - SKF_LL_OFF; 74 75 if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb)) 76 return ptr; 77 78 return NULL; 79 } 80 81 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) 82 { 83 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 84 struct bpf_prog_aux *aux; 85 struct bpf_prog *fp; 86 87 size = round_up(size, PAGE_SIZE); 88 fp = __vmalloc(size, gfp_flags); 89 if (fp == NULL) 90 return NULL; 91 92 aux = kzalloc(sizeof(*aux), GFP_KERNEL_ACCOUNT | gfp_extra_flags); 93 if (aux == NULL) { 94 vfree(fp); 95 return NULL; 96 } 97 fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags); 98 if (!fp->active) { 99 vfree(fp); 100 kfree(aux); 101 return NULL; 102 } 103 104 fp->pages = size / PAGE_SIZE; 105 fp->aux = aux; 106 fp->aux->prog = fp; 107 fp->jit_requested = ebpf_jit_enabled(); 108 109 INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode); 110 mutex_init(&fp->aux->used_maps_mutex); 111 mutex_init(&fp->aux->dst_mutex); 112 113 return fp; 114 } 115 116 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) 117 { 118 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 119 struct bpf_prog *prog; 120 int cpu; 121 122 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); 123 if (!prog) 124 return NULL; 125 126 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); 127 if (!prog->stats) { 128 free_percpu(prog->active); 129 kfree(prog->aux); 130 vfree(prog); 131 return NULL; 132 } 133 134 for_each_possible_cpu(cpu) { 135 struct bpf_prog_stats *pstats; 136 137 pstats = per_cpu_ptr(prog->stats, cpu); 138 u64_stats_init(&pstats->syncp); 139 } 140 return prog; 141 } 142 EXPORT_SYMBOL_GPL(bpf_prog_alloc); 143 144 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) 145 { 146 if (!prog->aux->nr_linfo || !prog->jit_requested) 147 return 0; 148 149 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, 150 sizeof(*prog->aux->jited_linfo), 151 GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 152 if (!prog->aux->jited_linfo) 153 return -ENOMEM; 154 155 return 0; 156 } 157 158 void bpf_prog_jit_attempt_done(struct bpf_prog *prog) 159 { 160 if (prog->aux->jited_linfo && 161 (!prog->jited || !prog->aux->jited_linfo[0])) { 162 kvfree(prog->aux->jited_linfo); 163 prog->aux->jited_linfo = NULL; 164 } 165 166 kfree(prog->aux->kfunc_tab); 167 prog->aux->kfunc_tab = NULL; 168 } 169 170 /* The jit engine is responsible to provide an array 171 * for insn_off to the jited_off mapping (insn_to_jit_off). 172 * 173 * The idx to this array is the insn_off. Hence, the insn_off 174 * here is relative to the prog itself instead of the main prog. 175 * This array has one entry for each xlated bpf insn. 176 * 177 * jited_off is the byte off to the last byte of the jited insn. 178 * 179 * Hence, with 180 * insn_start: 181 * The first bpf insn off of the prog. The insn off 182 * here is relative to the main prog. 183 * e.g. if prog is a subprog, insn_start > 0 184 * linfo_idx: 185 * The prog's idx to prog->aux->linfo and jited_linfo 186 * 187 * jited_linfo[linfo_idx] = prog->bpf_func 188 * 189 * For i > linfo_idx, 190 * 191 * jited_linfo[i] = prog->bpf_func + 192 * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] 193 */ 194 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, 195 const u32 *insn_to_jit_off) 196 { 197 u32 linfo_idx, insn_start, insn_end, nr_linfo, i; 198 const struct bpf_line_info *linfo; 199 void **jited_linfo; 200 201 if (!prog->aux->jited_linfo) 202 /* Userspace did not provide linfo */ 203 return; 204 205 linfo_idx = prog->aux->linfo_idx; 206 linfo = &prog->aux->linfo[linfo_idx]; 207 insn_start = linfo[0].insn_off; 208 insn_end = insn_start + prog->len; 209 210 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; 211 jited_linfo[0] = prog->bpf_func; 212 213 nr_linfo = prog->aux->nr_linfo - linfo_idx; 214 215 for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) 216 /* The verifier ensures that linfo[i].insn_off is 217 * strictly increasing 218 */ 219 jited_linfo[i] = prog->bpf_func + 220 insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; 221 } 222 223 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 224 gfp_t gfp_extra_flags) 225 { 226 gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; 227 struct bpf_prog *fp; 228 u32 pages; 229 230 size = round_up(size, PAGE_SIZE); 231 pages = size / PAGE_SIZE; 232 if (pages <= fp_old->pages) 233 return fp_old; 234 235 fp = __vmalloc(size, gfp_flags); 236 if (fp) { 237 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); 238 fp->pages = pages; 239 fp->aux->prog = fp; 240 241 /* We keep fp->aux from fp_old around in the new 242 * reallocated structure. 243 */ 244 fp_old->aux = NULL; 245 fp_old->stats = NULL; 246 fp_old->active = NULL; 247 __bpf_prog_free(fp_old); 248 } 249 250 return fp; 251 } 252 253 void __bpf_prog_free(struct bpf_prog *fp) 254 { 255 if (fp->aux) { 256 mutex_destroy(&fp->aux->used_maps_mutex); 257 mutex_destroy(&fp->aux->dst_mutex); 258 kfree(fp->aux->poke_tab); 259 kfree(fp->aux); 260 } 261 free_percpu(fp->stats); 262 free_percpu(fp->active); 263 vfree(fp); 264 } 265 266 int bpf_prog_calc_tag(struct bpf_prog *fp) 267 { 268 const u32 bits_offset = SHA1_BLOCK_SIZE - sizeof(__be64); 269 u32 raw_size = bpf_prog_tag_scratch_size(fp); 270 u32 digest[SHA1_DIGEST_WORDS]; 271 u32 ws[SHA1_WORKSPACE_WORDS]; 272 u32 i, bsize, psize, blocks; 273 struct bpf_insn *dst; 274 bool was_ld_map; 275 u8 *raw, *todo; 276 __be32 *result; 277 __be64 *bits; 278 279 raw = vmalloc(raw_size); 280 if (!raw) 281 return -ENOMEM; 282 283 sha1_init(digest); 284 memset(ws, 0, sizeof(ws)); 285 286 /* We need to take out the map fd for the digest calculation 287 * since they are unstable from user space side. 288 */ 289 dst = (void *)raw; 290 for (i = 0, was_ld_map = false; i < fp->len; i++) { 291 dst[i] = fp->insnsi[i]; 292 if (!was_ld_map && 293 dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && 294 (dst[i].src_reg == BPF_PSEUDO_MAP_FD || 295 dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { 296 was_ld_map = true; 297 dst[i].imm = 0; 298 } else if (was_ld_map && 299 dst[i].code == 0 && 300 dst[i].dst_reg == 0 && 301 dst[i].src_reg == 0 && 302 dst[i].off == 0) { 303 was_ld_map = false; 304 dst[i].imm = 0; 305 } else { 306 was_ld_map = false; 307 } 308 } 309 310 psize = bpf_prog_insn_size(fp); 311 memset(&raw[psize], 0, raw_size - psize); 312 raw[psize++] = 0x80; 313 314 bsize = round_up(psize, SHA1_BLOCK_SIZE); 315 blocks = bsize / SHA1_BLOCK_SIZE; 316 todo = raw; 317 if (bsize - psize >= sizeof(__be64)) { 318 bits = (__be64 *)(todo + bsize - sizeof(__be64)); 319 } else { 320 bits = (__be64 *)(todo + bsize + bits_offset); 321 blocks++; 322 } 323 *bits = cpu_to_be64((psize - 1) << 3); 324 325 while (blocks--) { 326 sha1_transform(digest, todo, ws); 327 todo += SHA1_BLOCK_SIZE; 328 } 329 330 result = (__force __be32 *)digest; 331 for (i = 0; i < SHA1_DIGEST_WORDS; i++) 332 result[i] = cpu_to_be32(digest[i]); 333 memcpy(fp->tag, result, sizeof(fp->tag)); 334 335 vfree(raw); 336 return 0; 337 } 338 339 static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, 340 s32 end_new, s32 curr, const bool probe_pass) 341 { 342 const s64 imm_min = S32_MIN, imm_max = S32_MAX; 343 s32 delta = end_new - end_old; 344 s64 imm = insn->imm; 345 346 if (curr < pos && curr + imm + 1 >= end_old) 347 imm += delta; 348 else if (curr >= end_new && curr + imm + 1 < end_new) 349 imm -= delta; 350 if (imm < imm_min || imm > imm_max) 351 return -ERANGE; 352 if (!probe_pass) 353 insn->imm = imm; 354 return 0; 355 } 356 357 static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, 358 s32 end_new, s32 curr, const bool probe_pass) 359 { 360 const s32 off_min = S16_MIN, off_max = S16_MAX; 361 s32 delta = end_new - end_old; 362 s32 off = insn->off; 363 364 if (curr < pos && curr + off + 1 >= end_old) 365 off += delta; 366 else if (curr >= end_new && curr + off + 1 < end_new) 367 off -= delta; 368 if (off < off_min || off > off_max) 369 return -ERANGE; 370 if (!probe_pass) 371 insn->off = off; 372 return 0; 373 } 374 375 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, 376 s32 end_new, const bool probe_pass) 377 { 378 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); 379 struct bpf_insn *insn = prog->insnsi; 380 int ret = 0; 381 382 for (i = 0; i < insn_cnt; i++, insn++) { 383 u8 code; 384 385 /* In the probing pass we still operate on the original, 386 * unpatched image in order to check overflows before we 387 * do any other adjustments. Therefore skip the patchlet. 388 */ 389 if (probe_pass && i == pos) { 390 i = end_new; 391 insn = prog->insnsi + end_old; 392 } 393 if (bpf_pseudo_func(insn)) { 394 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 395 end_new, i, probe_pass); 396 if (ret) 397 return ret; 398 continue; 399 } 400 code = insn->code; 401 if ((BPF_CLASS(code) != BPF_JMP && 402 BPF_CLASS(code) != BPF_JMP32) || 403 BPF_OP(code) == BPF_EXIT) 404 continue; 405 /* Adjust offset of jmps if we cross patch boundaries. */ 406 if (BPF_OP(code) == BPF_CALL) { 407 if (insn->src_reg != BPF_PSEUDO_CALL) 408 continue; 409 ret = bpf_adj_delta_to_imm(insn, pos, end_old, 410 end_new, i, probe_pass); 411 } else { 412 ret = bpf_adj_delta_to_off(insn, pos, end_old, 413 end_new, i, probe_pass); 414 } 415 if (ret) 416 break; 417 } 418 419 return ret; 420 } 421 422 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) 423 { 424 struct bpf_line_info *linfo; 425 u32 i, nr_linfo; 426 427 nr_linfo = prog->aux->nr_linfo; 428 if (!nr_linfo || !delta) 429 return; 430 431 linfo = prog->aux->linfo; 432 433 for (i = 0; i < nr_linfo; i++) 434 if (off < linfo[i].insn_off) 435 break; 436 437 /* Push all off < linfo[i].insn_off by delta */ 438 for (; i < nr_linfo; i++) 439 linfo[i].insn_off += delta; 440 } 441 442 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 443 const struct bpf_insn *patch, u32 len) 444 { 445 u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; 446 const u32 cnt_max = S16_MAX; 447 struct bpf_prog *prog_adj; 448 int err; 449 450 /* Since our patchlet doesn't expand the image, we're done. */ 451 if (insn_delta == 0) { 452 memcpy(prog->insnsi + off, patch, sizeof(*patch)); 453 return prog; 454 } 455 456 insn_adj_cnt = prog->len + insn_delta; 457 458 /* Reject anything that would potentially let the insn->off 459 * target overflow when we have excessive program expansions. 460 * We need to probe here before we do any reallocation where 461 * we afterwards may not fail anymore. 462 */ 463 if (insn_adj_cnt > cnt_max && 464 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) 465 return ERR_PTR(err); 466 467 /* Several new instructions need to be inserted. Make room 468 * for them. Likely, there's no need for a new allocation as 469 * last page could have large enough tailroom. 470 */ 471 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), 472 GFP_USER); 473 if (!prog_adj) 474 return ERR_PTR(-ENOMEM); 475 476 prog_adj->len = insn_adj_cnt; 477 478 /* Patching happens in 3 steps: 479 * 480 * 1) Move over tail of insnsi from next instruction onwards, 481 * so we can patch the single target insn with one or more 482 * new ones (patching is always from 1 to n insns, n > 0). 483 * 2) Inject new instructions at the target location. 484 * 3) Adjust branch offsets if necessary. 485 */ 486 insn_rest = insn_adj_cnt - off - len; 487 488 memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, 489 sizeof(*patch) * insn_rest); 490 memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); 491 492 /* We are guaranteed to not fail at this point, otherwise 493 * the ship has sailed to reverse to the original state. An 494 * overflow cannot happen at this point. 495 */ 496 BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); 497 498 bpf_adj_linfo(prog_adj, off, insn_delta); 499 500 return prog_adj; 501 } 502 503 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) 504 { 505 /* Branch offsets can't overflow when program is shrinking, no need 506 * to call bpf_adj_branches(..., true) here 507 */ 508 memmove(prog->insnsi + off, prog->insnsi + off + cnt, 509 sizeof(struct bpf_insn) * (prog->len - off - cnt)); 510 prog->len -= cnt; 511 512 return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); 513 } 514 515 static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) 516 { 517 int i; 518 519 for (i = 0; i < fp->aux->func_cnt; i++) 520 bpf_prog_kallsyms_del(fp->aux->func[i]); 521 } 522 523 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) 524 { 525 bpf_prog_kallsyms_del_subprogs(fp); 526 bpf_prog_kallsyms_del(fp); 527 } 528 529 #ifdef CONFIG_BPF_JIT 530 /* All BPF JIT sysctl knobs here. */ 531 int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 532 int bpf_jit_kallsyms __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON); 533 int bpf_jit_harden __read_mostly; 534 long bpf_jit_limit __read_mostly; 535 long bpf_jit_limit_max __read_mostly; 536 537 static void 538 bpf_prog_ksym_set_addr(struct bpf_prog *prog) 539 { 540 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); 541 542 prog->aux->ksym.start = (unsigned long) prog->bpf_func; 543 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; 544 } 545 546 static void 547 bpf_prog_ksym_set_name(struct bpf_prog *prog) 548 { 549 char *sym = prog->aux->ksym.name; 550 const char *end = sym + KSYM_NAME_LEN; 551 const struct btf_type *type; 552 const char *func_name; 553 554 BUILD_BUG_ON(sizeof("bpf_prog_") + 555 sizeof(prog->tag) * 2 + 556 /* name has been null terminated. 557 * We should need +1 for the '_' preceding 558 * the name. However, the null character 559 * is double counted between the name and the 560 * sizeof("bpf_prog_") above, so we omit 561 * the +1 here. 562 */ 563 sizeof(prog->aux->name) > KSYM_NAME_LEN); 564 565 sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); 566 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); 567 568 /* prog->aux->name will be ignored if full btf name is available */ 569 if (prog->aux->func_info_cnt) { 570 type = btf_type_by_id(prog->aux->btf, 571 prog->aux->func_info[prog->aux->func_idx].type_id); 572 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); 573 snprintf(sym, (size_t)(end - sym), "_%s", func_name); 574 return; 575 } 576 577 if (prog->aux->name[0]) 578 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); 579 else 580 *sym = 0; 581 } 582 583 static unsigned long bpf_get_ksym_start(struct latch_tree_node *n) 584 { 585 return container_of(n, struct bpf_ksym, tnode)->start; 586 } 587 588 static __always_inline bool bpf_tree_less(struct latch_tree_node *a, 589 struct latch_tree_node *b) 590 { 591 return bpf_get_ksym_start(a) < bpf_get_ksym_start(b); 592 } 593 594 static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) 595 { 596 unsigned long val = (unsigned long)key; 597 const struct bpf_ksym *ksym; 598 599 ksym = container_of(n, struct bpf_ksym, tnode); 600 601 if (val < ksym->start) 602 return -1; 603 if (val >= ksym->end) 604 return 1; 605 606 return 0; 607 } 608 609 static const struct latch_tree_ops bpf_tree_ops = { 610 .less = bpf_tree_less, 611 .comp = bpf_tree_comp, 612 }; 613 614 static DEFINE_SPINLOCK(bpf_lock); 615 static LIST_HEAD(bpf_kallsyms); 616 static struct latch_tree_root bpf_tree __cacheline_aligned; 617 618 void bpf_ksym_add(struct bpf_ksym *ksym) 619 { 620 spin_lock_bh(&bpf_lock); 621 WARN_ON_ONCE(!list_empty(&ksym->lnode)); 622 list_add_tail_rcu(&ksym->lnode, &bpf_kallsyms); 623 latch_tree_insert(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 624 spin_unlock_bh(&bpf_lock); 625 } 626 627 static void __bpf_ksym_del(struct bpf_ksym *ksym) 628 { 629 if (list_empty(&ksym->lnode)) 630 return; 631 632 latch_tree_erase(&ksym->tnode, &bpf_tree, &bpf_tree_ops); 633 list_del_rcu(&ksym->lnode); 634 } 635 636 void bpf_ksym_del(struct bpf_ksym *ksym) 637 { 638 spin_lock_bh(&bpf_lock); 639 __bpf_ksym_del(ksym); 640 spin_unlock_bh(&bpf_lock); 641 } 642 643 static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) 644 { 645 return fp->jited && !bpf_prog_was_classic(fp); 646 } 647 648 static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) 649 { 650 return list_empty(&fp->aux->ksym.lnode) || 651 fp->aux->ksym.lnode.prev == LIST_POISON2; 652 } 653 654 void bpf_prog_kallsyms_add(struct bpf_prog *fp) 655 { 656 if (!bpf_prog_kallsyms_candidate(fp) || 657 !bpf_capable()) 658 return; 659 660 bpf_prog_ksym_set_addr(fp); 661 bpf_prog_ksym_set_name(fp); 662 fp->aux->ksym.prog = true; 663 664 bpf_ksym_add(&fp->aux->ksym); 665 } 666 667 void bpf_prog_kallsyms_del(struct bpf_prog *fp) 668 { 669 if (!bpf_prog_kallsyms_candidate(fp)) 670 return; 671 672 bpf_ksym_del(&fp->aux->ksym); 673 } 674 675 static struct bpf_ksym *bpf_ksym_find(unsigned long addr) 676 { 677 struct latch_tree_node *n; 678 679 n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); 680 return n ? container_of(n, struct bpf_ksym, tnode) : NULL; 681 } 682 683 const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, 684 unsigned long *off, char *sym) 685 { 686 struct bpf_ksym *ksym; 687 char *ret = NULL; 688 689 rcu_read_lock(); 690 ksym = bpf_ksym_find(addr); 691 if (ksym) { 692 unsigned long symbol_start = ksym->start; 693 unsigned long symbol_end = ksym->end; 694 695 strncpy(sym, ksym->name, KSYM_NAME_LEN); 696 697 ret = sym; 698 if (size) 699 *size = symbol_end - symbol_start; 700 if (off) 701 *off = addr - symbol_start; 702 } 703 rcu_read_unlock(); 704 705 return ret; 706 } 707 708 bool is_bpf_text_address(unsigned long addr) 709 { 710 bool ret; 711 712 rcu_read_lock(); 713 ret = bpf_ksym_find(addr) != NULL; 714 rcu_read_unlock(); 715 716 return ret; 717 } 718 719 static struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 720 { 721 struct bpf_ksym *ksym = bpf_ksym_find(addr); 722 723 return ksym && ksym->prog ? 724 container_of(ksym, struct bpf_prog_aux, ksym)->prog : 725 NULL; 726 } 727 728 const struct exception_table_entry *search_bpf_extables(unsigned long addr) 729 { 730 const struct exception_table_entry *e = NULL; 731 struct bpf_prog *prog; 732 733 rcu_read_lock(); 734 prog = bpf_prog_ksym_find(addr); 735 if (!prog) 736 goto out; 737 if (!prog->aux->num_exentries) 738 goto out; 739 740 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); 741 out: 742 rcu_read_unlock(); 743 return e; 744 } 745 746 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 747 char *sym) 748 { 749 struct bpf_ksym *ksym; 750 unsigned int it = 0; 751 int ret = -ERANGE; 752 753 if (!bpf_jit_kallsyms_enabled()) 754 return ret; 755 756 rcu_read_lock(); 757 list_for_each_entry_rcu(ksym, &bpf_kallsyms, lnode) { 758 if (it++ != symnum) 759 continue; 760 761 strncpy(sym, ksym->name, KSYM_NAME_LEN); 762 763 *value = ksym->start; 764 *type = BPF_SYM_ELF_TYPE; 765 766 ret = 0; 767 break; 768 } 769 rcu_read_unlock(); 770 771 return ret; 772 } 773 774 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, 775 struct bpf_jit_poke_descriptor *poke) 776 { 777 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 778 static const u32 poke_tab_max = 1024; 779 u32 slot = prog->aux->size_poke_tab; 780 u32 size = slot + 1; 781 782 if (size > poke_tab_max) 783 return -ENOSPC; 784 if (poke->tailcall_target || poke->tailcall_target_stable || 785 poke->tailcall_bypass || poke->adj_off || poke->bypass_addr) 786 return -EINVAL; 787 788 switch (poke->reason) { 789 case BPF_POKE_REASON_TAIL_CALL: 790 if (!poke->tail_call.map) 791 return -EINVAL; 792 break; 793 default: 794 return -EINVAL; 795 } 796 797 tab = krealloc(tab, size * sizeof(*poke), GFP_KERNEL); 798 if (!tab) 799 return -ENOMEM; 800 801 memcpy(&tab[slot], poke, sizeof(*poke)); 802 prog->aux->size_poke_tab = size; 803 prog->aux->poke_tab = tab; 804 805 return slot; 806 } 807 808 /* 809 * BPF program pack allocator. 810 * 811 * Most BPF programs are pretty small. Allocating a hole page for each 812 * program is sometime a waste. Many small bpf program also adds pressure 813 * to instruction TLB. To solve this issue, we introduce a BPF program pack 814 * allocator. The prog_pack allocator uses HPAGE_PMD_SIZE page (2MB on x86) 815 * to host BPF programs. 816 */ 817 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 818 #define BPF_PROG_PACK_SIZE HPAGE_PMD_SIZE 819 #else 820 #define BPF_PROG_PACK_SIZE PAGE_SIZE 821 #endif 822 #define BPF_PROG_CHUNK_SHIFT 6 823 #define BPF_PROG_CHUNK_SIZE (1 << BPF_PROG_CHUNK_SHIFT) 824 #define BPF_PROG_CHUNK_MASK (~(BPF_PROG_CHUNK_SIZE - 1)) 825 #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) 826 827 struct bpf_prog_pack { 828 struct list_head list; 829 void *ptr; 830 unsigned long bitmap[]; 831 }; 832 833 #define BPF_PROG_MAX_PACK_PROG_SIZE BPF_PROG_PACK_SIZE 834 #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) 835 836 static DEFINE_MUTEX(pack_mutex); 837 static LIST_HEAD(pack_list); 838 839 static struct bpf_prog_pack *alloc_new_pack(void) 840 { 841 struct bpf_prog_pack *pack; 842 843 pack = kzalloc(sizeof(*pack) + BITS_TO_BYTES(BPF_PROG_CHUNK_COUNT), GFP_KERNEL); 844 if (!pack) 845 return NULL; 846 pack->ptr = module_alloc(BPF_PROG_PACK_SIZE); 847 if (!pack->ptr) { 848 kfree(pack); 849 return NULL; 850 } 851 bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); 852 list_add_tail(&pack->list, &pack_list); 853 854 set_vm_flush_reset_perms(pack->ptr); 855 set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); 856 set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE); 857 return pack; 858 } 859 860 static void *bpf_prog_pack_alloc(u32 size) 861 { 862 unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); 863 struct bpf_prog_pack *pack; 864 unsigned long pos; 865 void *ptr = NULL; 866 867 if (size > BPF_PROG_MAX_PACK_PROG_SIZE) { 868 size = round_up(size, PAGE_SIZE); 869 ptr = module_alloc(size); 870 if (ptr) { 871 set_vm_flush_reset_perms(ptr); 872 set_memory_ro((unsigned long)ptr, size / PAGE_SIZE); 873 set_memory_x((unsigned long)ptr, size / PAGE_SIZE); 874 } 875 return ptr; 876 } 877 mutex_lock(&pack_mutex); 878 list_for_each_entry(pack, &pack_list, list) { 879 pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 880 nbits, 0); 881 if (pos < BPF_PROG_CHUNK_COUNT) 882 goto found_free_area; 883 } 884 885 pack = alloc_new_pack(); 886 if (!pack) 887 goto out; 888 889 pos = 0; 890 891 found_free_area: 892 bitmap_set(pack->bitmap, pos, nbits); 893 ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); 894 895 out: 896 mutex_unlock(&pack_mutex); 897 return ptr; 898 } 899 900 static void bpf_prog_pack_free(struct bpf_binary_header *hdr) 901 { 902 struct bpf_prog_pack *pack = NULL, *tmp; 903 unsigned int nbits; 904 unsigned long pos; 905 void *pack_ptr; 906 907 if (hdr->size > BPF_PROG_MAX_PACK_PROG_SIZE) { 908 module_memfree(hdr); 909 return; 910 } 911 912 pack_ptr = (void *)((unsigned long)hdr & ~(BPF_PROG_PACK_SIZE - 1)); 913 mutex_lock(&pack_mutex); 914 915 list_for_each_entry(tmp, &pack_list, list) { 916 if (tmp->ptr == pack_ptr) { 917 pack = tmp; 918 break; 919 } 920 } 921 922 if (WARN_ONCE(!pack, "bpf_prog_pack bug\n")) 923 goto out; 924 925 nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); 926 pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; 927 928 bitmap_clear(pack->bitmap, pos, nbits); 929 if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0, 930 BPF_PROG_CHUNK_COUNT, 0) == 0) { 931 list_del(&pack->list); 932 module_memfree(pack->ptr); 933 kfree(pack); 934 } 935 out: 936 mutex_unlock(&pack_mutex); 937 } 938 939 static atomic_long_t bpf_jit_current; 940 941 /* Can be overridden by an arch's JIT compiler if it has a custom, 942 * dedicated BPF backend memory area, or if neither of the two 943 * below apply. 944 */ 945 u64 __weak bpf_jit_alloc_exec_limit(void) 946 { 947 #if defined(MODULES_VADDR) 948 return MODULES_END - MODULES_VADDR; 949 #else 950 return VMALLOC_END - VMALLOC_START; 951 #endif 952 } 953 954 static int __init bpf_jit_charge_init(void) 955 { 956 /* Only used as heuristic here to derive limit. */ 957 bpf_jit_limit_max = bpf_jit_alloc_exec_limit(); 958 bpf_jit_limit = min_t(u64, round_up(bpf_jit_limit_max >> 2, 959 PAGE_SIZE), LONG_MAX); 960 return 0; 961 } 962 pure_initcall(bpf_jit_charge_init); 963 964 int bpf_jit_charge_modmem(u32 size) 965 { 966 if (atomic_long_add_return(size, &bpf_jit_current) > bpf_jit_limit) { 967 if (!bpf_capable()) { 968 atomic_long_sub(size, &bpf_jit_current); 969 return -EPERM; 970 } 971 } 972 973 return 0; 974 } 975 976 void bpf_jit_uncharge_modmem(u32 size) 977 { 978 atomic_long_sub(size, &bpf_jit_current); 979 } 980 981 void *__weak bpf_jit_alloc_exec(unsigned long size) 982 { 983 return module_alloc(size); 984 } 985 986 void __weak bpf_jit_free_exec(void *addr) 987 { 988 module_memfree(addr); 989 } 990 991 struct bpf_binary_header * 992 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 993 unsigned int alignment, 994 bpf_jit_fill_hole_t bpf_fill_ill_insns) 995 { 996 struct bpf_binary_header *hdr; 997 u32 size, hole, start; 998 999 WARN_ON_ONCE(!is_power_of_2(alignment) || 1000 alignment > BPF_IMAGE_ALIGNMENT); 1001 1002 /* Most of BPF filters are really small, but if some of them 1003 * fill a page, allow at least 128 extra bytes to insert a 1004 * random section of illegal instructions. 1005 */ 1006 size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); 1007 1008 if (bpf_jit_charge_modmem(size)) 1009 return NULL; 1010 hdr = bpf_jit_alloc_exec(size); 1011 if (!hdr) { 1012 bpf_jit_uncharge_modmem(size); 1013 return NULL; 1014 } 1015 1016 /* Fill space with illegal/arch-dep instructions. */ 1017 bpf_fill_ill_insns(hdr, size); 1018 1019 hdr->size = size; 1020 hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), 1021 PAGE_SIZE - sizeof(*hdr)); 1022 start = (get_random_int() % hole) & ~(alignment - 1); 1023 1024 /* Leave a random number of instructions before BPF code. */ 1025 *image_ptr = &hdr->image[start]; 1026 1027 return hdr; 1028 } 1029 1030 void bpf_jit_binary_free(struct bpf_binary_header *hdr) 1031 { 1032 u32 size = hdr->size; 1033 1034 bpf_jit_free_exec(hdr); 1035 bpf_jit_uncharge_modmem(size); 1036 } 1037 1038 /* Allocate jit binary from bpf_prog_pack allocator. 1039 * Since the allocated memory is RO+X, the JIT engine cannot write directly 1040 * to the memory. To solve this problem, a RW buffer is also allocated at 1041 * as the same time. The JIT engine should calculate offsets based on the 1042 * RO memory address, but write JITed program to the RW buffer. Once the 1043 * JIT engine finishes, it calls bpf_jit_binary_pack_finalize, which copies 1044 * the JITed program to the RO memory. 1045 */ 1046 struct bpf_binary_header * 1047 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, 1048 unsigned int alignment, 1049 struct bpf_binary_header **rw_header, 1050 u8 **rw_image, 1051 bpf_jit_fill_hole_t bpf_fill_ill_insns) 1052 { 1053 struct bpf_binary_header *ro_header; 1054 u32 size, hole, start; 1055 1056 WARN_ON_ONCE(!is_power_of_2(alignment) || 1057 alignment > BPF_IMAGE_ALIGNMENT); 1058 1059 /* add 16 bytes for a random section of illegal instructions */ 1060 size = round_up(proglen + sizeof(*ro_header) + 16, BPF_PROG_CHUNK_SIZE); 1061 1062 if (bpf_jit_charge_modmem(size)) 1063 return NULL; 1064 ro_header = bpf_prog_pack_alloc(size); 1065 if (!ro_header) { 1066 bpf_jit_uncharge_modmem(size); 1067 return NULL; 1068 } 1069 1070 *rw_header = kvmalloc(size, GFP_KERNEL); 1071 if (!*rw_header) { 1072 bpf_arch_text_copy(&ro_header->size, &size, sizeof(size)); 1073 bpf_prog_pack_free(ro_header); 1074 bpf_jit_uncharge_modmem(size); 1075 return NULL; 1076 } 1077 1078 /* Fill space with illegal/arch-dep instructions. */ 1079 bpf_fill_ill_insns(*rw_header, size); 1080 (*rw_header)->size = size; 1081 1082 hole = min_t(unsigned int, size - (proglen + sizeof(*ro_header)), 1083 BPF_PROG_CHUNK_SIZE - sizeof(*ro_header)); 1084 start = (get_random_int() % hole) & ~(alignment - 1); 1085 1086 *image_ptr = &ro_header->image[start]; 1087 *rw_image = &(*rw_header)->image[start]; 1088 1089 return ro_header; 1090 } 1091 1092 /* Copy JITed text from rw_header to its final location, the ro_header. */ 1093 int bpf_jit_binary_pack_finalize(struct bpf_prog *prog, 1094 struct bpf_binary_header *ro_header, 1095 struct bpf_binary_header *rw_header) 1096 { 1097 void *ptr; 1098 1099 ptr = bpf_arch_text_copy(ro_header, rw_header, rw_header->size); 1100 1101 kvfree(rw_header); 1102 1103 if (IS_ERR(ptr)) { 1104 bpf_prog_pack_free(ro_header); 1105 return PTR_ERR(ptr); 1106 } 1107 prog->aux->use_bpf_prog_pack = true; 1108 return 0; 1109 } 1110 1111 /* bpf_jit_binary_pack_free is called in two different scenarios: 1112 * 1) when the program is freed after; 1113 * 2) when the JIT engine fails (before bpf_jit_binary_pack_finalize). 1114 * For case 2), we need to free both the RO memory and the RW buffer. 1115 * Also, ro_header->size in 2) is not properly set yet, so rw_header->size 1116 * is used for uncharge. 1117 */ 1118 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, 1119 struct bpf_binary_header *rw_header) 1120 { 1121 u32 size = rw_header ? rw_header->size : ro_header->size; 1122 1123 bpf_prog_pack_free(ro_header); 1124 kvfree(rw_header); 1125 bpf_jit_uncharge_modmem(size); 1126 } 1127 1128 static inline struct bpf_binary_header * 1129 bpf_jit_binary_hdr(const struct bpf_prog *fp) 1130 { 1131 unsigned long real_start = (unsigned long)fp->bpf_func; 1132 unsigned long addr; 1133 1134 if (fp->aux->use_bpf_prog_pack) 1135 addr = real_start & BPF_PROG_CHUNK_MASK; 1136 else 1137 addr = real_start & PAGE_MASK; 1138 1139 return (void *)addr; 1140 } 1141 1142 /* This symbol is only overridden by archs that have different 1143 * requirements than the usual eBPF JITs, f.e. when they only 1144 * implement cBPF JIT, do not set images read-only, etc. 1145 */ 1146 void __weak bpf_jit_free(struct bpf_prog *fp) 1147 { 1148 if (fp->jited) { 1149 struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); 1150 1151 if (fp->aux->use_bpf_prog_pack) 1152 bpf_jit_binary_pack_free(hdr, NULL /* rw_buffer */); 1153 else 1154 bpf_jit_binary_free(hdr); 1155 1156 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 1157 } 1158 1159 bpf_prog_unlock_free(fp); 1160 } 1161 1162 int bpf_jit_get_func_addr(const struct bpf_prog *prog, 1163 const struct bpf_insn *insn, bool extra_pass, 1164 u64 *func_addr, bool *func_addr_fixed) 1165 { 1166 s16 off = insn->off; 1167 s32 imm = insn->imm; 1168 u8 *addr; 1169 1170 *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; 1171 if (!*func_addr_fixed) { 1172 /* Place-holder address till the last pass has collected 1173 * all addresses for JITed subprograms in which case we 1174 * can pick them up from prog->aux. 1175 */ 1176 if (!extra_pass) 1177 addr = NULL; 1178 else if (prog->aux->func && 1179 off >= 0 && off < prog->aux->func_cnt) 1180 addr = (u8 *)prog->aux->func[off]->bpf_func; 1181 else 1182 return -EINVAL; 1183 } else { 1184 /* Address of a BPF helper call. Since part of the core 1185 * kernel, it's always at a fixed location. __bpf_call_base 1186 * and the helper with imm relative to it are both in core 1187 * kernel. 1188 */ 1189 addr = (u8 *)__bpf_call_base + imm; 1190 } 1191 1192 *func_addr = (unsigned long)addr; 1193 return 0; 1194 } 1195 1196 static int bpf_jit_blind_insn(const struct bpf_insn *from, 1197 const struct bpf_insn *aux, 1198 struct bpf_insn *to_buff, 1199 bool emit_zext) 1200 { 1201 struct bpf_insn *to = to_buff; 1202 u32 imm_rnd = get_random_int(); 1203 s16 off; 1204 1205 BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); 1206 BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); 1207 1208 /* Constraints on AX register: 1209 * 1210 * AX register is inaccessible from user space. It is mapped in 1211 * all JITs, and used here for constant blinding rewrites. It is 1212 * typically "stateless" meaning its contents are only valid within 1213 * the executed instruction, but not across several instructions. 1214 * There are a few exceptions however which are further detailed 1215 * below. 1216 * 1217 * Constant blinding is only used by JITs, not in the interpreter. 1218 * The interpreter uses AX in some occasions as a local temporary 1219 * register e.g. in DIV or MOD instructions. 1220 * 1221 * In restricted circumstances, the verifier can also use the AX 1222 * register for rewrites as long as they do not interfere with 1223 * the above cases! 1224 */ 1225 if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) 1226 goto out; 1227 1228 if (from->imm == 0 && 1229 (from->code == (BPF_ALU | BPF_MOV | BPF_K) || 1230 from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { 1231 *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); 1232 goto out; 1233 } 1234 1235 switch (from->code) { 1236 case BPF_ALU | BPF_ADD | BPF_K: 1237 case BPF_ALU | BPF_SUB | BPF_K: 1238 case BPF_ALU | BPF_AND | BPF_K: 1239 case BPF_ALU | BPF_OR | BPF_K: 1240 case BPF_ALU | BPF_XOR | BPF_K: 1241 case BPF_ALU | BPF_MUL | BPF_K: 1242 case BPF_ALU | BPF_MOV | BPF_K: 1243 case BPF_ALU | BPF_DIV | BPF_K: 1244 case BPF_ALU | BPF_MOD | BPF_K: 1245 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1246 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1247 *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX); 1248 break; 1249 1250 case BPF_ALU64 | BPF_ADD | BPF_K: 1251 case BPF_ALU64 | BPF_SUB | BPF_K: 1252 case BPF_ALU64 | BPF_AND | BPF_K: 1253 case BPF_ALU64 | BPF_OR | BPF_K: 1254 case BPF_ALU64 | BPF_XOR | BPF_K: 1255 case BPF_ALU64 | BPF_MUL | BPF_K: 1256 case BPF_ALU64 | BPF_MOV | BPF_K: 1257 case BPF_ALU64 | BPF_DIV | BPF_K: 1258 case BPF_ALU64 | BPF_MOD | BPF_K: 1259 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1260 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1261 *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX); 1262 break; 1263 1264 case BPF_JMP | BPF_JEQ | BPF_K: 1265 case BPF_JMP | BPF_JNE | BPF_K: 1266 case BPF_JMP | BPF_JGT | BPF_K: 1267 case BPF_JMP | BPF_JLT | BPF_K: 1268 case BPF_JMP | BPF_JGE | BPF_K: 1269 case BPF_JMP | BPF_JLE | BPF_K: 1270 case BPF_JMP | BPF_JSGT | BPF_K: 1271 case BPF_JMP | BPF_JSLT | BPF_K: 1272 case BPF_JMP | BPF_JSGE | BPF_K: 1273 case BPF_JMP | BPF_JSLE | BPF_K: 1274 case BPF_JMP | BPF_JSET | BPF_K: 1275 /* Accommodate for extra offset in case of a backjump. */ 1276 off = from->off; 1277 if (off < 0) 1278 off -= 2; 1279 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1280 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1281 *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); 1282 break; 1283 1284 case BPF_JMP32 | BPF_JEQ | BPF_K: 1285 case BPF_JMP32 | BPF_JNE | BPF_K: 1286 case BPF_JMP32 | BPF_JGT | BPF_K: 1287 case BPF_JMP32 | BPF_JLT | BPF_K: 1288 case BPF_JMP32 | BPF_JGE | BPF_K: 1289 case BPF_JMP32 | BPF_JLE | BPF_K: 1290 case BPF_JMP32 | BPF_JSGT | BPF_K: 1291 case BPF_JMP32 | BPF_JSLT | BPF_K: 1292 case BPF_JMP32 | BPF_JSGE | BPF_K: 1293 case BPF_JMP32 | BPF_JSLE | BPF_K: 1294 case BPF_JMP32 | BPF_JSET | BPF_K: 1295 /* Accommodate for extra offset in case of a backjump. */ 1296 off = from->off; 1297 if (off < 0) 1298 off -= 2; 1299 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1300 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1301 *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, 1302 off); 1303 break; 1304 1305 case BPF_LD | BPF_IMM | BPF_DW: 1306 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); 1307 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1308 *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 1309 *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); 1310 break; 1311 case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ 1312 *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); 1313 *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1314 if (emit_zext) 1315 *to++ = BPF_ZEXT_REG(BPF_REG_AX); 1316 *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); 1317 break; 1318 1319 case BPF_ST | BPF_MEM | BPF_DW: 1320 case BPF_ST | BPF_MEM | BPF_W: 1321 case BPF_ST | BPF_MEM | BPF_H: 1322 case BPF_ST | BPF_MEM | BPF_B: 1323 *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); 1324 *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); 1325 *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); 1326 break; 1327 } 1328 out: 1329 return to - to_buff; 1330 } 1331 1332 static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, 1333 gfp_t gfp_extra_flags) 1334 { 1335 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; 1336 struct bpf_prog *fp; 1337 1338 fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags); 1339 if (fp != NULL) { 1340 /* aux->prog still points to the fp_other one, so 1341 * when promoting the clone to the real program, 1342 * this still needs to be adapted. 1343 */ 1344 memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); 1345 } 1346 1347 return fp; 1348 } 1349 1350 static void bpf_prog_clone_free(struct bpf_prog *fp) 1351 { 1352 /* aux was stolen by the other clone, so we cannot free 1353 * it from this path! It will be freed eventually by the 1354 * other program on release. 1355 * 1356 * At this point, we don't need a deferred release since 1357 * clone is guaranteed to not be locked. 1358 */ 1359 fp->aux = NULL; 1360 fp->stats = NULL; 1361 fp->active = NULL; 1362 __bpf_prog_free(fp); 1363 } 1364 1365 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) 1366 { 1367 /* We have to repoint aux->prog to self, as we don't 1368 * know whether fp here is the clone or the original. 1369 */ 1370 fp->aux->prog = fp; 1371 bpf_prog_clone_free(fp_other); 1372 } 1373 1374 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) 1375 { 1376 struct bpf_insn insn_buff[16], aux[2]; 1377 struct bpf_prog *clone, *tmp; 1378 int insn_delta, insn_cnt; 1379 struct bpf_insn *insn; 1380 int i, rewritten; 1381 1382 if (!bpf_jit_blinding_enabled(prog) || prog->blinded) 1383 return prog; 1384 1385 clone = bpf_prog_clone_create(prog, GFP_USER); 1386 if (!clone) 1387 return ERR_PTR(-ENOMEM); 1388 1389 insn_cnt = clone->len; 1390 insn = clone->insnsi; 1391 1392 for (i = 0; i < insn_cnt; i++, insn++) { 1393 /* We temporarily need to hold the original ld64 insn 1394 * so that we can still access the first part in the 1395 * second blinding run. 1396 */ 1397 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && 1398 insn[1].code == 0) 1399 memcpy(aux, insn, sizeof(aux)); 1400 1401 rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, 1402 clone->aux->verifier_zext); 1403 if (!rewritten) 1404 continue; 1405 1406 tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); 1407 if (IS_ERR(tmp)) { 1408 /* Patching may have repointed aux->prog during 1409 * realloc from the original one, so we need to 1410 * fix it up here on error. 1411 */ 1412 bpf_jit_prog_release_other(prog, clone); 1413 return tmp; 1414 } 1415 1416 clone = tmp; 1417 insn_delta = rewritten - 1; 1418 1419 /* Walk new program and skip insns we just inserted. */ 1420 insn = clone->insnsi + i + insn_delta; 1421 insn_cnt += insn_delta; 1422 i += insn_delta; 1423 } 1424 1425 clone->blinded = 1; 1426 return clone; 1427 } 1428 #endif /* CONFIG_BPF_JIT */ 1429 1430 /* Base function for offset calculation. Needs to go into .text section, 1431 * therefore keeping it non-static as well; will also be used by JITs 1432 * anyway later on, so do not let the compiler omit it. This also needs 1433 * to go into kallsyms for correlation from e.g. bpftool, so naming 1434 * must not change. 1435 */ 1436 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 1437 { 1438 return 0; 1439 } 1440 EXPORT_SYMBOL_GPL(__bpf_call_base); 1441 1442 /* All UAPI available opcodes. */ 1443 #define BPF_INSN_MAP(INSN_2, INSN_3) \ 1444 /* 32 bit ALU operations. */ \ 1445 /* Register based. */ \ 1446 INSN_3(ALU, ADD, X), \ 1447 INSN_3(ALU, SUB, X), \ 1448 INSN_3(ALU, AND, X), \ 1449 INSN_3(ALU, OR, X), \ 1450 INSN_3(ALU, LSH, X), \ 1451 INSN_3(ALU, RSH, X), \ 1452 INSN_3(ALU, XOR, X), \ 1453 INSN_3(ALU, MUL, X), \ 1454 INSN_3(ALU, MOV, X), \ 1455 INSN_3(ALU, ARSH, X), \ 1456 INSN_3(ALU, DIV, X), \ 1457 INSN_3(ALU, MOD, X), \ 1458 INSN_2(ALU, NEG), \ 1459 INSN_3(ALU, END, TO_BE), \ 1460 INSN_3(ALU, END, TO_LE), \ 1461 /* Immediate based. */ \ 1462 INSN_3(ALU, ADD, K), \ 1463 INSN_3(ALU, SUB, K), \ 1464 INSN_3(ALU, AND, K), \ 1465 INSN_3(ALU, OR, K), \ 1466 INSN_3(ALU, LSH, K), \ 1467 INSN_3(ALU, RSH, K), \ 1468 INSN_3(ALU, XOR, K), \ 1469 INSN_3(ALU, MUL, K), \ 1470 INSN_3(ALU, MOV, K), \ 1471 INSN_3(ALU, ARSH, K), \ 1472 INSN_3(ALU, DIV, K), \ 1473 INSN_3(ALU, MOD, K), \ 1474 /* 64 bit ALU operations. */ \ 1475 /* Register based. */ \ 1476 INSN_3(ALU64, ADD, X), \ 1477 INSN_3(ALU64, SUB, X), \ 1478 INSN_3(ALU64, AND, X), \ 1479 INSN_3(ALU64, OR, X), \ 1480 INSN_3(ALU64, LSH, X), \ 1481 INSN_3(ALU64, RSH, X), \ 1482 INSN_3(ALU64, XOR, X), \ 1483 INSN_3(ALU64, MUL, X), \ 1484 INSN_3(ALU64, MOV, X), \ 1485 INSN_3(ALU64, ARSH, X), \ 1486 INSN_3(ALU64, DIV, X), \ 1487 INSN_3(ALU64, MOD, X), \ 1488 INSN_2(ALU64, NEG), \ 1489 /* Immediate based. */ \ 1490 INSN_3(ALU64, ADD, K), \ 1491 INSN_3(ALU64, SUB, K), \ 1492 INSN_3(ALU64, AND, K), \ 1493 INSN_3(ALU64, OR, K), \ 1494 INSN_3(ALU64, LSH, K), \ 1495 INSN_3(ALU64, RSH, K), \ 1496 INSN_3(ALU64, XOR, K), \ 1497 INSN_3(ALU64, MUL, K), \ 1498 INSN_3(ALU64, MOV, K), \ 1499 INSN_3(ALU64, ARSH, K), \ 1500 INSN_3(ALU64, DIV, K), \ 1501 INSN_3(ALU64, MOD, K), \ 1502 /* Call instruction. */ \ 1503 INSN_2(JMP, CALL), \ 1504 /* Exit instruction. */ \ 1505 INSN_2(JMP, EXIT), \ 1506 /* 32-bit Jump instructions. */ \ 1507 /* Register based. */ \ 1508 INSN_3(JMP32, JEQ, X), \ 1509 INSN_3(JMP32, JNE, X), \ 1510 INSN_3(JMP32, JGT, X), \ 1511 INSN_3(JMP32, JLT, X), \ 1512 INSN_3(JMP32, JGE, X), \ 1513 INSN_3(JMP32, JLE, X), \ 1514 INSN_3(JMP32, JSGT, X), \ 1515 INSN_3(JMP32, JSLT, X), \ 1516 INSN_3(JMP32, JSGE, X), \ 1517 INSN_3(JMP32, JSLE, X), \ 1518 INSN_3(JMP32, JSET, X), \ 1519 /* Immediate based. */ \ 1520 INSN_3(JMP32, JEQ, K), \ 1521 INSN_3(JMP32, JNE, K), \ 1522 INSN_3(JMP32, JGT, K), \ 1523 INSN_3(JMP32, JLT, K), \ 1524 INSN_3(JMP32, JGE, K), \ 1525 INSN_3(JMP32, JLE, K), \ 1526 INSN_3(JMP32, JSGT, K), \ 1527 INSN_3(JMP32, JSLT, K), \ 1528 INSN_3(JMP32, JSGE, K), \ 1529 INSN_3(JMP32, JSLE, K), \ 1530 INSN_3(JMP32, JSET, K), \ 1531 /* Jump instructions. */ \ 1532 /* Register based. */ \ 1533 INSN_3(JMP, JEQ, X), \ 1534 INSN_3(JMP, JNE, X), \ 1535 INSN_3(JMP, JGT, X), \ 1536 INSN_3(JMP, JLT, X), \ 1537 INSN_3(JMP, JGE, X), \ 1538 INSN_3(JMP, JLE, X), \ 1539 INSN_3(JMP, JSGT, X), \ 1540 INSN_3(JMP, JSLT, X), \ 1541 INSN_3(JMP, JSGE, X), \ 1542 INSN_3(JMP, JSLE, X), \ 1543 INSN_3(JMP, JSET, X), \ 1544 /* Immediate based. */ \ 1545 INSN_3(JMP, JEQ, K), \ 1546 INSN_3(JMP, JNE, K), \ 1547 INSN_3(JMP, JGT, K), \ 1548 INSN_3(JMP, JLT, K), \ 1549 INSN_3(JMP, JGE, K), \ 1550 INSN_3(JMP, JLE, K), \ 1551 INSN_3(JMP, JSGT, K), \ 1552 INSN_3(JMP, JSLT, K), \ 1553 INSN_3(JMP, JSGE, K), \ 1554 INSN_3(JMP, JSLE, K), \ 1555 INSN_3(JMP, JSET, K), \ 1556 INSN_2(JMP, JA), \ 1557 /* Store instructions. */ \ 1558 /* Register based. */ \ 1559 INSN_3(STX, MEM, B), \ 1560 INSN_3(STX, MEM, H), \ 1561 INSN_3(STX, MEM, W), \ 1562 INSN_3(STX, MEM, DW), \ 1563 INSN_3(STX, ATOMIC, W), \ 1564 INSN_3(STX, ATOMIC, DW), \ 1565 /* Immediate based. */ \ 1566 INSN_3(ST, MEM, B), \ 1567 INSN_3(ST, MEM, H), \ 1568 INSN_3(ST, MEM, W), \ 1569 INSN_3(ST, MEM, DW), \ 1570 /* Load instructions. */ \ 1571 /* Register based. */ \ 1572 INSN_3(LDX, MEM, B), \ 1573 INSN_3(LDX, MEM, H), \ 1574 INSN_3(LDX, MEM, W), \ 1575 INSN_3(LDX, MEM, DW), \ 1576 /* Immediate based. */ \ 1577 INSN_3(LD, IMM, DW) 1578 1579 bool bpf_opcode_in_insntable(u8 code) 1580 { 1581 #define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true 1582 #define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true 1583 static const bool public_insntable[256] = { 1584 [0 ... 255] = false, 1585 /* Now overwrite non-defaults ... */ 1586 BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), 1587 /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ 1588 [BPF_LD | BPF_ABS | BPF_B] = true, 1589 [BPF_LD | BPF_ABS | BPF_H] = true, 1590 [BPF_LD | BPF_ABS | BPF_W] = true, 1591 [BPF_LD | BPF_IND | BPF_B] = true, 1592 [BPF_LD | BPF_IND | BPF_H] = true, 1593 [BPF_LD | BPF_IND | BPF_W] = true, 1594 }; 1595 #undef BPF_INSN_3_TBL 1596 #undef BPF_INSN_2_TBL 1597 return public_insntable[code]; 1598 } 1599 1600 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1601 u64 __weak bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr) 1602 { 1603 memset(dst, 0, size); 1604 return -EFAULT; 1605 } 1606 1607 /** 1608 * ___bpf_prog_run - run eBPF program on a given context 1609 * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers 1610 * @insn: is the array of eBPF instructions 1611 * 1612 * Decode and execute eBPF instructions. 1613 * 1614 * Return: whatever value is in %BPF_R0 at program exit 1615 */ 1616 static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn) 1617 { 1618 #define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y 1619 #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z 1620 static const void * const jumptable[256] __annotate_jump_table = { 1621 [0 ... 255] = &&default_label, 1622 /* Now overwrite non-defaults ... */ 1623 BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), 1624 /* Non-UAPI available opcodes. */ 1625 [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, 1626 [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, 1627 [BPF_ST | BPF_NOSPEC] = &&ST_NOSPEC, 1628 [BPF_LDX | BPF_PROBE_MEM | BPF_B] = &&LDX_PROBE_MEM_B, 1629 [BPF_LDX | BPF_PROBE_MEM | BPF_H] = &&LDX_PROBE_MEM_H, 1630 [BPF_LDX | BPF_PROBE_MEM | BPF_W] = &&LDX_PROBE_MEM_W, 1631 [BPF_LDX | BPF_PROBE_MEM | BPF_DW] = &&LDX_PROBE_MEM_DW, 1632 }; 1633 #undef BPF_INSN_3_LBL 1634 #undef BPF_INSN_2_LBL 1635 u32 tail_call_cnt = 0; 1636 1637 #define CONT ({ insn++; goto select_insn; }) 1638 #define CONT_JMP ({ insn++; goto select_insn; }) 1639 1640 select_insn: 1641 goto *jumptable[insn->code]; 1642 1643 /* Explicitly mask the register-based shift amounts with 63 or 31 1644 * to avoid undefined behavior. Normally this won't affect the 1645 * generated code, for example, in case of native 64 bit archs such 1646 * as x86-64 or arm64, the compiler is optimizing the AND away for 1647 * the interpreter. In case of JITs, each of the JIT backends compiles 1648 * the BPF shift operations to machine instructions which produce 1649 * implementation-defined results in such a case; the resulting 1650 * contents of the register may be arbitrary, but program behaviour 1651 * as a whole remains defined. In other words, in case of JIT backends, 1652 * the AND must /not/ be added to the emitted LSH/RSH/ARSH translation. 1653 */ 1654 /* ALU (shifts) */ 1655 #define SHT(OPCODE, OP) \ 1656 ALU64_##OPCODE##_X: \ 1657 DST = DST OP (SRC & 63); \ 1658 CONT; \ 1659 ALU_##OPCODE##_X: \ 1660 DST = (u32) DST OP ((u32) SRC & 31); \ 1661 CONT; \ 1662 ALU64_##OPCODE##_K: \ 1663 DST = DST OP IMM; \ 1664 CONT; \ 1665 ALU_##OPCODE##_K: \ 1666 DST = (u32) DST OP (u32) IMM; \ 1667 CONT; 1668 /* ALU (rest) */ 1669 #define ALU(OPCODE, OP) \ 1670 ALU64_##OPCODE##_X: \ 1671 DST = DST OP SRC; \ 1672 CONT; \ 1673 ALU_##OPCODE##_X: \ 1674 DST = (u32) DST OP (u32) SRC; \ 1675 CONT; \ 1676 ALU64_##OPCODE##_K: \ 1677 DST = DST OP IMM; \ 1678 CONT; \ 1679 ALU_##OPCODE##_K: \ 1680 DST = (u32) DST OP (u32) IMM; \ 1681 CONT; 1682 ALU(ADD, +) 1683 ALU(SUB, -) 1684 ALU(AND, &) 1685 ALU(OR, |) 1686 ALU(XOR, ^) 1687 ALU(MUL, *) 1688 SHT(LSH, <<) 1689 SHT(RSH, >>) 1690 #undef SHT 1691 #undef ALU 1692 ALU_NEG: 1693 DST = (u32) -DST; 1694 CONT; 1695 ALU64_NEG: 1696 DST = -DST; 1697 CONT; 1698 ALU_MOV_X: 1699 DST = (u32) SRC; 1700 CONT; 1701 ALU_MOV_K: 1702 DST = (u32) IMM; 1703 CONT; 1704 ALU64_MOV_X: 1705 DST = SRC; 1706 CONT; 1707 ALU64_MOV_K: 1708 DST = IMM; 1709 CONT; 1710 LD_IMM_DW: 1711 DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; 1712 insn++; 1713 CONT; 1714 ALU_ARSH_X: 1715 DST = (u64) (u32) (((s32) DST) >> (SRC & 31)); 1716 CONT; 1717 ALU_ARSH_K: 1718 DST = (u64) (u32) (((s32) DST) >> IMM); 1719 CONT; 1720 ALU64_ARSH_X: 1721 (*(s64 *) &DST) >>= (SRC & 63); 1722 CONT; 1723 ALU64_ARSH_K: 1724 (*(s64 *) &DST) >>= IMM; 1725 CONT; 1726 ALU64_MOD_X: 1727 div64_u64_rem(DST, SRC, &AX); 1728 DST = AX; 1729 CONT; 1730 ALU_MOD_X: 1731 AX = (u32) DST; 1732 DST = do_div(AX, (u32) SRC); 1733 CONT; 1734 ALU64_MOD_K: 1735 div64_u64_rem(DST, IMM, &AX); 1736 DST = AX; 1737 CONT; 1738 ALU_MOD_K: 1739 AX = (u32) DST; 1740 DST = do_div(AX, (u32) IMM); 1741 CONT; 1742 ALU64_DIV_X: 1743 DST = div64_u64(DST, SRC); 1744 CONT; 1745 ALU_DIV_X: 1746 AX = (u32) DST; 1747 do_div(AX, (u32) SRC); 1748 DST = (u32) AX; 1749 CONT; 1750 ALU64_DIV_K: 1751 DST = div64_u64(DST, IMM); 1752 CONT; 1753 ALU_DIV_K: 1754 AX = (u32) DST; 1755 do_div(AX, (u32) IMM); 1756 DST = (u32) AX; 1757 CONT; 1758 ALU_END_TO_BE: 1759 switch (IMM) { 1760 case 16: 1761 DST = (__force u16) cpu_to_be16(DST); 1762 break; 1763 case 32: 1764 DST = (__force u32) cpu_to_be32(DST); 1765 break; 1766 case 64: 1767 DST = (__force u64) cpu_to_be64(DST); 1768 break; 1769 } 1770 CONT; 1771 ALU_END_TO_LE: 1772 switch (IMM) { 1773 case 16: 1774 DST = (__force u16) cpu_to_le16(DST); 1775 break; 1776 case 32: 1777 DST = (__force u32) cpu_to_le32(DST); 1778 break; 1779 case 64: 1780 DST = (__force u64) cpu_to_le64(DST); 1781 break; 1782 } 1783 CONT; 1784 1785 /* CALL */ 1786 JMP_CALL: 1787 /* Function call scratches BPF_R1-BPF_R5 registers, 1788 * preserves BPF_R6-BPF_R9, and stores return value 1789 * into BPF_R0. 1790 */ 1791 BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3, 1792 BPF_R4, BPF_R5); 1793 CONT; 1794 1795 JMP_CALL_ARGS: 1796 BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, 1797 BPF_R3, BPF_R4, 1798 BPF_R5, 1799 insn + insn->off + 1); 1800 CONT; 1801 1802 JMP_TAIL_CALL: { 1803 struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; 1804 struct bpf_array *array = container_of(map, struct bpf_array, map); 1805 struct bpf_prog *prog; 1806 u32 index = BPF_R3; 1807 1808 if (unlikely(index >= array->map.max_entries)) 1809 goto out; 1810 1811 if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT)) 1812 goto out; 1813 1814 tail_call_cnt++; 1815 1816 prog = READ_ONCE(array->ptrs[index]); 1817 if (!prog) 1818 goto out; 1819 1820 /* ARG1 at this point is guaranteed to point to CTX from 1821 * the verifier side due to the fact that the tail call is 1822 * handled like a helper, that is, bpf_tail_call_proto, 1823 * where arg1_type is ARG_PTR_TO_CTX. 1824 */ 1825 insn = prog->insnsi; 1826 goto select_insn; 1827 out: 1828 CONT; 1829 } 1830 JMP_JA: 1831 insn += insn->off; 1832 CONT; 1833 JMP_EXIT: 1834 return BPF_R0; 1835 /* JMP */ 1836 #define COND_JMP(SIGN, OPCODE, CMP_OP) \ 1837 JMP_##OPCODE##_X: \ 1838 if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ 1839 insn += insn->off; \ 1840 CONT_JMP; \ 1841 } \ 1842 CONT; \ 1843 JMP32_##OPCODE##_X: \ 1844 if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ 1845 insn += insn->off; \ 1846 CONT_JMP; \ 1847 } \ 1848 CONT; \ 1849 JMP_##OPCODE##_K: \ 1850 if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ 1851 insn += insn->off; \ 1852 CONT_JMP; \ 1853 } \ 1854 CONT; \ 1855 JMP32_##OPCODE##_K: \ 1856 if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ 1857 insn += insn->off; \ 1858 CONT_JMP; \ 1859 } \ 1860 CONT; 1861 COND_JMP(u, JEQ, ==) 1862 COND_JMP(u, JNE, !=) 1863 COND_JMP(u, JGT, >) 1864 COND_JMP(u, JLT, <) 1865 COND_JMP(u, JGE, >=) 1866 COND_JMP(u, JLE, <=) 1867 COND_JMP(u, JSET, &) 1868 COND_JMP(s, JSGT, >) 1869 COND_JMP(s, JSLT, <) 1870 COND_JMP(s, JSGE, >=) 1871 COND_JMP(s, JSLE, <=) 1872 #undef COND_JMP 1873 /* ST, STX and LDX*/ 1874 ST_NOSPEC: 1875 /* Speculation barrier for mitigating Speculative Store Bypass. 1876 * In case of arm64, we rely on the firmware mitigation as 1877 * controlled via the ssbd kernel parameter. Whenever the 1878 * mitigation is enabled, it works for all of the kernel code 1879 * with no need to provide any additional instructions here. 1880 * In case of x86, we use 'lfence' insn for mitigation. We 1881 * reuse preexisting logic from Spectre v1 mitigation that 1882 * happens to produce the required code on x86 for v4 as well. 1883 */ 1884 #ifdef CONFIG_X86 1885 barrier_nospec(); 1886 #endif 1887 CONT; 1888 #define LDST(SIZEOP, SIZE) \ 1889 STX_MEM_##SIZEOP: \ 1890 *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ 1891 CONT; \ 1892 ST_MEM_##SIZEOP: \ 1893 *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \ 1894 CONT; \ 1895 LDX_MEM_##SIZEOP: \ 1896 DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ 1897 CONT; 1898 1899 LDST(B, u8) 1900 LDST(H, u16) 1901 LDST(W, u32) 1902 LDST(DW, u64) 1903 #undef LDST 1904 #define LDX_PROBE(SIZEOP, SIZE) \ 1905 LDX_PROBE_MEM_##SIZEOP: \ 1906 bpf_probe_read_kernel(&DST, SIZE, (const void *)(long) (SRC + insn->off)); \ 1907 CONT; 1908 LDX_PROBE(B, 1) 1909 LDX_PROBE(H, 2) 1910 LDX_PROBE(W, 4) 1911 LDX_PROBE(DW, 8) 1912 #undef LDX_PROBE 1913 1914 #define ATOMIC_ALU_OP(BOP, KOP) \ 1915 case BOP: \ 1916 if (BPF_SIZE(insn->code) == BPF_W) \ 1917 atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \ 1918 (DST + insn->off)); \ 1919 else \ 1920 atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \ 1921 (DST + insn->off)); \ 1922 break; \ 1923 case BOP | BPF_FETCH: \ 1924 if (BPF_SIZE(insn->code) == BPF_W) \ 1925 SRC = (u32) atomic_fetch_##KOP( \ 1926 (u32) SRC, \ 1927 (atomic_t *)(unsigned long) (DST + insn->off)); \ 1928 else \ 1929 SRC = (u64) atomic64_fetch_##KOP( \ 1930 (u64) SRC, \ 1931 (atomic64_t *)(unsigned long) (DST + insn->off)); \ 1932 break; 1933 1934 STX_ATOMIC_DW: 1935 STX_ATOMIC_W: 1936 switch (IMM) { 1937 ATOMIC_ALU_OP(BPF_ADD, add) 1938 ATOMIC_ALU_OP(BPF_AND, and) 1939 ATOMIC_ALU_OP(BPF_OR, or) 1940 ATOMIC_ALU_OP(BPF_XOR, xor) 1941 #undef ATOMIC_ALU_OP 1942 1943 case BPF_XCHG: 1944 if (BPF_SIZE(insn->code) == BPF_W) 1945 SRC = (u32) atomic_xchg( 1946 (atomic_t *)(unsigned long) (DST + insn->off), 1947 (u32) SRC); 1948 else 1949 SRC = (u64) atomic64_xchg( 1950 (atomic64_t *)(unsigned long) (DST + insn->off), 1951 (u64) SRC); 1952 break; 1953 case BPF_CMPXCHG: 1954 if (BPF_SIZE(insn->code) == BPF_W) 1955 BPF_R0 = (u32) atomic_cmpxchg( 1956 (atomic_t *)(unsigned long) (DST + insn->off), 1957 (u32) BPF_R0, (u32) SRC); 1958 else 1959 BPF_R0 = (u64) atomic64_cmpxchg( 1960 (atomic64_t *)(unsigned long) (DST + insn->off), 1961 (u64) BPF_R0, (u64) SRC); 1962 break; 1963 1964 default: 1965 goto default_label; 1966 } 1967 CONT; 1968 1969 default_label: 1970 /* If we ever reach this, we have a bug somewhere. Die hard here 1971 * instead of just returning 0; we could be somewhere in a subprog, 1972 * so execution could continue otherwise which we do /not/ want. 1973 * 1974 * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). 1975 */ 1976 pr_warn("BPF interpreter: unknown opcode %02x (imm: 0x%x)\n", 1977 insn->code, insn->imm); 1978 BUG_ON(1); 1979 return 0; 1980 } 1981 1982 #define PROG_NAME(stack_size) __bpf_prog_run##stack_size 1983 #define DEFINE_BPF_PROG_RUN(stack_size) \ 1984 static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ 1985 { \ 1986 u64 stack[stack_size / sizeof(u64)]; \ 1987 u64 regs[MAX_BPF_EXT_REG]; \ 1988 \ 1989 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 1990 ARG1 = (u64) (unsigned long) ctx; \ 1991 return ___bpf_prog_run(regs, insn); \ 1992 } 1993 1994 #define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size 1995 #define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ 1996 static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ 1997 const struct bpf_insn *insn) \ 1998 { \ 1999 u64 stack[stack_size / sizeof(u64)]; \ 2000 u64 regs[MAX_BPF_EXT_REG]; \ 2001 \ 2002 FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ 2003 BPF_R1 = r1; \ 2004 BPF_R2 = r2; \ 2005 BPF_R3 = r3; \ 2006 BPF_R4 = r4; \ 2007 BPF_R5 = r5; \ 2008 return ___bpf_prog_run(regs, insn); \ 2009 } 2010 2011 #define EVAL1(FN, X) FN(X) 2012 #define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) 2013 #define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) 2014 #define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) 2015 #define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) 2016 #define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) 2017 2018 EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); 2019 EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); 2020 EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); 2021 2022 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); 2023 EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); 2024 EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); 2025 2026 #define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), 2027 2028 static unsigned int (*interpreters[])(const void *ctx, 2029 const struct bpf_insn *insn) = { 2030 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2031 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2032 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2033 }; 2034 #undef PROG_NAME_LIST 2035 #define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), 2036 static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, 2037 const struct bpf_insn *insn) = { 2038 EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) 2039 EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) 2040 EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) 2041 }; 2042 #undef PROG_NAME_LIST 2043 2044 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) 2045 { 2046 stack_depth = max_t(u32, stack_depth, 1); 2047 insn->off = (s16) insn->imm; 2048 insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - 2049 __bpf_call_base_args; 2050 insn->code = BPF_JMP | BPF_CALL_ARGS; 2051 } 2052 2053 #else 2054 static unsigned int __bpf_prog_ret0_warn(const void *ctx, 2055 const struct bpf_insn *insn) 2056 { 2057 /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON 2058 * is not working properly, so warn about it! 2059 */ 2060 WARN_ON_ONCE(1); 2061 return 0; 2062 } 2063 #endif 2064 2065 bool bpf_prog_map_compatible(struct bpf_map *map, 2066 const struct bpf_prog *fp) 2067 { 2068 bool ret; 2069 2070 if (fp->kprobe_override) 2071 return false; 2072 2073 spin_lock(&map->owner.lock); 2074 if (!map->owner.type) { 2075 /* There's no owner yet where we could check for 2076 * compatibility. 2077 */ 2078 map->owner.type = fp->type; 2079 map->owner.jited = fp->jited; 2080 map->owner.xdp_has_frags = fp->aux->xdp_has_frags; 2081 ret = true; 2082 } else { 2083 ret = map->owner.type == fp->type && 2084 map->owner.jited == fp->jited && 2085 map->owner.xdp_has_frags == fp->aux->xdp_has_frags; 2086 } 2087 spin_unlock(&map->owner.lock); 2088 2089 return ret; 2090 } 2091 2092 static int bpf_check_tail_call(const struct bpf_prog *fp) 2093 { 2094 struct bpf_prog_aux *aux = fp->aux; 2095 int i, ret = 0; 2096 2097 mutex_lock(&aux->used_maps_mutex); 2098 for (i = 0; i < aux->used_map_cnt; i++) { 2099 struct bpf_map *map = aux->used_maps[i]; 2100 2101 if (!map_type_contains_progs(map)) 2102 continue; 2103 2104 if (!bpf_prog_map_compatible(map, fp)) { 2105 ret = -EINVAL; 2106 goto out; 2107 } 2108 } 2109 2110 out: 2111 mutex_unlock(&aux->used_maps_mutex); 2112 return ret; 2113 } 2114 2115 static void bpf_prog_select_func(struct bpf_prog *fp) 2116 { 2117 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2118 u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); 2119 2120 fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; 2121 #else 2122 fp->bpf_func = __bpf_prog_ret0_warn; 2123 #endif 2124 } 2125 2126 /** 2127 * bpf_prog_select_runtime - select exec runtime for BPF program 2128 * @fp: bpf_prog populated with BPF program 2129 * @err: pointer to error variable 2130 * 2131 * Try to JIT eBPF program, if JIT is not available, use interpreter. 2132 * The BPF program will be executed via bpf_prog_run() function. 2133 * 2134 * Return: the &fp argument along with &err set to 0 for success or 2135 * a negative errno code on failure 2136 */ 2137 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) 2138 { 2139 /* In case of BPF to BPF calls, verifier did all the prep 2140 * work with regards to JITing, etc. 2141 */ 2142 bool jit_needed = false; 2143 2144 if (fp->bpf_func) 2145 goto finalize; 2146 2147 if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || 2148 bpf_prog_has_kfunc_call(fp)) 2149 jit_needed = true; 2150 2151 bpf_prog_select_func(fp); 2152 2153 /* eBPF JITs can rewrite the program in case constant 2154 * blinding is active. However, in case of error during 2155 * blinding, bpf_int_jit_compile() must always return a 2156 * valid program, which in this case would simply not 2157 * be JITed, but falls back to the interpreter. 2158 */ 2159 if (!bpf_prog_is_dev_bound(fp->aux)) { 2160 *err = bpf_prog_alloc_jited_linfo(fp); 2161 if (*err) 2162 return fp; 2163 2164 fp = bpf_int_jit_compile(fp); 2165 bpf_prog_jit_attempt_done(fp); 2166 if (!fp->jited && jit_needed) { 2167 *err = -ENOTSUPP; 2168 return fp; 2169 } 2170 } else { 2171 *err = bpf_prog_offload_compile(fp); 2172 if (*err) 2173 return fp; 2174 } 2175 2176 finalize: 2177 bpf_prog_lock_ro(fp); 2178 2179 /* The tail call compatibility check can only be done at 2180 * this late stage as we need to determine, if we deal 2181 * with JITed or non JITed program concatenations and not 2182 * all eBPF JITs might immediately support all features. 2183 */ 2184 *err = bpf_check_tail_call(fp); 2185 2186 return fp; 2187 } 2188 EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); 2189 2190 static unsigned int __bpf_prog_ret1(const void *ctx, 2191 const struct bpf_insn *insn) 2192 { 2193 return 1; 2194 } 2195 2196 static struct bpf_prog_dummy { 2197 struct bpf_prog prog; 2198 } dummy_bpf_prog = { 2199 .prog = { 2200 .bpf_func = __bpf_prog_ret1, 2201 }, 2202 }; 2203 2204 struct bpf_empty_prog_array bpf_empty_prog_array = { 2205 .null_prog = NULL, 2206 }; 2207 EXPORT_SYMBOL(bpf_empty_prog_array); 2208 2209 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) 2210 { 2211 if (prog_cnt) 2212 return kzalloc(sizeof(struct bpf_prog_array) + 2213 sizeof(struct bpf_prog_array_item) * 2214 (prog_cnt + 1), 2215 flags); 2216 2217 return &bpf_empty_prog_array.hdr; 2218 } 2219 2220 void bpf_prog_array_free(struct bpf_prog_array *progs) 2221 { 2222 if (!progs || progs == &bpf_empty_prog_array.hdr) 2223 return; 2224 kfree_rcu(progs, rcu); 2225 } 2226 2227 int bpf_prog_array_length(struct bpf_prog_array *array) 2228 { 2229 struct bpf_prog_array_item *item; 2230 u32 cnt = 0; 2231 2232 for (item = array->items; item->prog; item++) 2233 if (item->prog != &dummy_bpf_prog.prog) 2234 cnt++; 2235 return cnt; 2236 } 2237 2238 bool bpf_prog_array_is_empty(struct bpf_prog_array *array) 2239 { 2240 struct bpf_prog_array_item *item; 2241 2242 for (item = array->items; item->prog; item++) 2243 if (item->prog != &dummy_bpf_prog.prog) 2244 return false; 2245 return true; 2246 } 2247 2248 static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, 2249 u32 *prog_ids, 2250 u32 request_cnt) 2251 { 2252 struct bpf_prog_array_item *item; 2253 int i = 0; 2254 2255 for (item = array->items; item->prog; item++) { 2256 if (item->prog == &dummy_bpf_prog.prog) 2257 continue; 2258 prog_ids[i] = item->prog->aux->id; 2259 if (++i == request_cnt) { 2260 item++; 2261 break; 2262 } 2263 } 2264 2265 return !!(item->prog); 2266 } 2267 2268 int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, 2269 __u32 __user *prog_ids, u32 cnt) 2270 { 2271 unsigned long err = 0; 2272 bool nospc; 2273 u32 *ids; 2274 2275 /* users of this function are doing: 2276 * cnt = bpf_prog_array_length(); 2277 * if (cnt > 0) 2278 * bpf_prog_array_copy_to_user(..., cnt); 2279 * so below kcalloc doesn't need extra cnt > 0 check. 2280 */ 2281 ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); 2282 if (!ids) 2283 return -ENOMEM; 2284 nospc = bpf_prog_array_copy_core(array, ids, cnt); 2285 err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); 2286 kfree(ids); 2287 if (err) 2288 return -EFAULT; 2289 if (nospc) 2290 return -ENOSPC; 2291 return 0; 2292 } 2293 2294 void bpf_prog_array_delete_safe(struct bpf_prog_array *array, 2295 struct bpf_prog *old_prog) 2296 { 2297 struct bpf_prog_array_item *item; 2298 2299 for (item = array->items; item->prog; item++) 2300 if (item->prog == old_prog) { 2301 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); 2302 break; 2303 } 2304 } 2305 2306 /** 2307 * bpf_prog_array_delete_safe_at() - Replaces the program at the given 2308 * index into the program array with 2309 * a dummy no-op program. 2310 * @array: a bpf_prog_array 2311 * @index: the index of the program to replace 2312 * 2313 * Skips over dummy programs, by not counting them, when calculating 2314 * the position of the program to replace. 2315 * 2316 * Return: 2317 * * 0 - Success 2318 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2319 * * -ENOENT - Index out of range 2320 */ 2321 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index) 2322 { 2323 return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog); 2324 } 2325 2326 /** 2327 * bpf_prog_array_update_at() - Updates the program at the given index 2328 * into the program array. 2329 * @array: a bpf_prog_array 2330 * @index: the index of the program to update 2331 * @prog: the program to insert into the array 2332 * 2333 * Skips over dummy programs, by not counting them, when calculating 2334 * the position of the program to update. 2335 * 2336 * Return: 2337 * * 0 - Success 2338 * * -EINVAL - Invalid index value. Must be a non-negative integer. 2339 * * -ENOENT - Index out of range 2340 */ 2341 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 2342 struct bpf_prog *prog) 2343 { 2344 struct bpf_prog_array_item *item; 2345 2346 if (unlikely(index < 0)) 2347 return -EINVAL; 2348 2349 for (item = array->items; item->prog; item++) { 2350 if (item->prog == &dummy_bpf_prog.prog) 2351 continue; 2352 if (!index) { 2353 WRITE_ONCE(item->prog, prog); 2354 return 0; 2355 } 2356 index--; 2357 } 2358 return -ENOENT; 2359 } 2360 2361 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 2362 struct bpf_prog *exclude_prog, 2363 struct bpf_prog *include_prog, 2364 u64 bpf_cookie, 2365 struct bpf_prog_array **new_array) 2366 { 2367 int new_prog_cnt, carry_prog_cnt = 0; 2368 struct bpf_prog_array_item *existing, *new; 2369 struct bpf_prog_array *array; 2370 bool found_exclude = false; 2371 2372 /* Figure out how many existing progs we need to carry over to 2373 * the new array. 2374 */ 2375 if (old_array) { 2376 existing = old_array->items; 2377 for (; existing->prog; existing++) { 2378 if (existing->prog == exclude_prog) { 2379 found_exclude = true; 2380 continue; 2381 } 2382 if (existing->prog != &dummy_bpf_prog.prog) 2383 carry_prog_cnt++; 2384 if (existing->prog == include_prog) 2385 return -EEXIST; 2386 } 2387 } 2388 2389 if (exclude_prog && !found_exclude) 2390 return -ENOENT; 2391 2392 /* How many progs (not NULL) will be in the new array? */ 2393 new_prog_cnt = carry_prog_cnt; 2394 if (include_prog) 2395 new_prog_cnt += 1; 2396 2397 /* Do we have any prog (not NULL) in the new array? */ 2398 if (!new_prog_cnt) { 2399 *new_array = NULL; 2400 return 0; 2401 } 2402 2403 /* +1 as the end of prog_array is marked with NULL */ 2404 array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); 2405 if (!array) 2406 return -ENOMEM; 2407 new = array->items; 2408 2409 /* Fill in the new prog array */ 2410 if (carry_prog_cnt) { 2411 existing = old_array->items; 2412 for (; existing->prog; existing++) { 2413 if (existing->prog == exclude_prog || 2414 existing->prog == &dummy_bpf_prog.prog) 2415 continue; 2416 2417 new->prog = existing->prog; 2418 new->bpf_cookie = existing->bpf_cookie; 2419 new++; 2420 } 2421 } 2422 if (include_prog) { 2423 new->prog = include_prog; 2424 new->bpf_cookie = bpf_cookie; 2425 new++; 2426 } 2427 new->prog = NULL; 2428 *new_array = array; 2429 return 0; 2430 } 2431 2432 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 2433 u32 *prog_ids, u32 request_cnt, 2434 u32 *prog_cnt) 2435 { 2436 u32 cnt = 0; 2437 2438 if (array) 2439 cnt = bpf_prog_array_length(array); 2440 2441 *prog_cnt = cnt; 2442 2443 /* return early if user requested only program count or nothing to copy */ 2444 if (!request_cnt || !cnt) 2445 return 0; 2446 2447 /* this function is called under trace/bpf_trace.c: bpf_event_mutex */ 2448 return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC 2449 : 0; 2450 } 2451 2452 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 2453 struct bpf_map **used_maps, u32 len) 2454 { 2455 struct bpf_map *map; 2456 u32 i; 2457 2458 for (i = 0; i < len; i++) { 2459 map = used_maps[i]; 2460 if (map->ops->map_poke_untrack) 2461 map->ops->map_poke_untrack(map, aux); 2462 bpf_map_put(map); 2463 } 2464 } 2465 2466 static void bpf_free_used_maps(struct bpf_prog_aux *aux) 2467 { 2468 __bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt); 2469 kfree(aux->used_maps); 2470 } 2471 2472 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 2473 struct btf_mod_pair *used_btfs, u32 len) 2474 { 2475 #ifdef CONFIG_BPF_SYSCALL 2476 struct btf_mod_pair *btf_mod; 2477 u32 i; 2478 2479 for (i = 0; i < len; i++) { 2480 btf_mod = &used_btfs[i]; 2481 if (btf_mod->module) 2482 module_put(btf_mod->module); 2483 btf_put(btf_mod->btf); 2484 } 2485 #endif 2486 } 2487 2488 static void bpf_free_used_btfs(struct bpf_prog_aux *aux) 2489 { 2490 __bpf_free_used_btfs(aux, aux->used_btfs, aux->used_btf_cnt); 2491 kfree(aux->used_btfs); 2492 } 2493 2494 static void bpf_prog_free_deferred(struct work_struct *work) 2495 { 2496 struct bpf_prog_aux *aux; 2497 int i; 2498 2499 aux = container_of(work, struct bpf_prog_aux, work); 2500 #ifdef CONFIG_BPF_SYSCALL 2501 bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); 2502 #endif 2503 bpf_free_used_maps(aux); 2504 bpf_free_used_btfs(aux); 2505 if (bpf_prog_is_dev_bound(aux)) 2506 bpf_prog_offload_destroy(aux->prog); 2507 #ifdef CONFIG_PERF_EVENTS 2508 if (aux->prog->has_callchain_buf) 2509 put_callchain_buffers(); 2510 #endif 2511 if (aux->dst_trampoline) 2512 bpf_trampoline_put(aux->dst_trampoline); 2513 for (i = 0; i < aux->func_cnt; i++) { 2514 /* We can just unlink the subprog poke descriptor table as 2515 * it was originally linked to the main program and is also 2516 * released along with it. 2517 */ 2518 aux->func[i]->aux->poke_tab = NULL; 2519 bpf_jit_free(aux->func[i]); 2520 } 2521 if (aux->func_cnt) { 2522 kfree(aux->func); 2523 bpf_prog_unlock_free(aux->prog); 2524 } else { 2525 bpf_jit_free(aux->prog); 2526 } 2527 } 2528 2529 void bpf_prog_free(struct bpf_prog *fp) 2530 { 2531 struct bpf_prog_aux *aux = fp->aux; 2532 2533 if (aux->dst_prog) 2534 bpf_prog_put(aux->dst_prog); 2535 INIT_WORK(&aux->work, bpf_prog_free_deferred); 2536 schedule_work(&aux->work); 2537 } 2538 EXPORT_SYMBOL_GPL(bpf_prog_free); 2539 2540 /* RNG for unpriviledged user space with separated state from prandom_u32(). */ 2541 static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); 2542 2543 void bpf_user_rnd_init_once(void) 2544 { 2545 prandom_init_once(&bpf_user_rnd_state); 2546 } 2547 2548 BPF_CALL_0(bpf_user_rnd_u32) 2549 { 2550 /* Should someone ever have the rather unwise idea to use some 2551 * of the registers passed into this function, then note that 2552 * this function is called from native eBPF and classic-to-eBPF 2553 * transformations. Register assignments from both sides are 2554 * different, f.e. classic always sets fn(ctx, A, X) here. 2555 */ 2556 struct rnd_state *state; 2557 u32 res; 2558 2559 state = &get_cpu_var(bpf_user_rnd_state); 2560 res = prandom_u32_state(state); 2561 put_cpu_var(bpf_user_rnd_state); 2562 2563 return res; 2564 } 2565 2566 BPF_CALL_0(bpf_get_raw_cpu_id) 2567 { 2568 return raw_smp_processor_id(); 2569 } 2570 2571 /* Weak definitions of helper functions in case we don't have bpf syscall. */ 2572 const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; 2573 const struct bpf_func_proto bpf_map_update_elem_proto __weak; 2574 const struct bpf_func_proto bpf_map_delete_elem_proto __weak; 2575 const struct bpf_func_proto bpf_map_push_elem_proto __weak; 2576 const struct bpf_func_proto bpf_map_pop_elem_proto __weak; 2577 const struct bpf_func_proto bpf_map_peek_elem_proto __weak; 2578 const struct bpf_func_proto bpf_spin_lock_proto __weak; 2579 const struct bpf_func_proto bpf_spin_unlock_proto __weak; 2580 const struct bpf_func_proto bpf_jiffies64_proto __weak; 2581 2582 const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; 2583 const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; 2584 const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; 2585 const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; 2586 const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak; 2587 const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak; 2588 2589 const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; 2590 const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; 2591 const struct bpf_func_proto bpf_get_current_comm_proto __weak; 2592 const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; 2593 const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak; 2594 const struct bpf_func_proto bpf_get_local_storage_proto __weak; 2595 const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak; 2596 const struct bpf_func_proto bpf_snprintf_btf_proto __weak; 2597 const struct bpf_func_proto bpf_seq_printf_btf_proto __weak; 2598 2599 const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) 2600 { 2601 return NULL; 2602 } 2603 2604 const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void) 2605 { 2606 return NULL; 2607 } 2608 2609 u64 __weak 2610 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 2611 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 2612 { 2613 return -ENOTSUPP; 2614 } 2615 EXPORT_SYMBOL_GPL(bpf_event_output); 2616 2617 /* Always built-in helper functions. */ 2618 const struct bpf_func_proto bpf_tail_call_proto = { 2619 .func = NULL, 2620 .gpl_only = false, 2621 .ret_type = RET_VOID, 2622 .arg1_type = ARG_PTR_TO_CTX, 2623 .arg2_type = ARG_CONST_MAP_PTR, 2624 .arg3_type = ARG_ANYTHING, 2625 }; 2626 2627 /* Stub for JITs that only support cBPF. eBPF programs are interpreted. 2628 * It is encouraged to implement bpf_int_jit_compile() instead, so that 2629 * eBPF and implicitly also cBPF can get JITed! 2630 */ 2631 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) 2632 { 2633 return prog; 2634 } 2635 2636 /* Stub for JITs that support eBPF. All cBPF code gets transformed into 2637 * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). 2638 */ 2639 void __weak bpf_jit_compile(struct bpf_prog *prog) 2640 { 2641 } 2642 2643 bool __weak bpf_helper_changes_pkt_data(void *func) 2644 { 2645 return false; 2646 } 2647 2648 /* Return TRUE if the JIT backend wants verifier to enable sub-register usage 2649 * analysis code and wants explicit zero extension inserted by verifier. 2650 * Otherwise, return FALSE. 2651 * 2652 * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if 2653 * you don't override this. JITs that don't want these extra insns can detect 2654 * them using insn_is_zext. 2655 */ 2656 bool __weak bpf_jit_needs_zext(void) 2657 { 2658 return false; 2659 } 2660 2661 bool __weak bpf_jit_supports_kfunc_call(void) 2662 { 2663 return false; 2664 } 2665 2666 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call 2667 * skb_copy_bits(), so provide a weak definition of it for NET-less config. 2668 */ 2669 int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, 2670 int len) 2671 { 2672 return -EFAULT; 2673 } 2674 2675 int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2676 void *addr1, void *addr2) 2677 { 2678 return -ENOTSUPP; 2679 } 2680 2681 void * __weak bpf_arch_text_copy(void *dst, void *src, size_t len) 2682 { 2683 return ERR_PTR(-ENOTSUPP); 2684 } 2685 2686 DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); 2687 EXPORT_SYMBOL(bpf_stats_enabled_key); 2688 2689 /* All definitions of tracepoints related to BPF. */ 2690 #define CREATE_TRACE_POINTS 2691 #include <linux/bpf_trace.h> 2692 2693 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); 2694 EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); 2695