1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Linux Socket Filter Data Structures 4 */ 5 #ifndef __LINUX_FILTER_H__ 6 #define __LINUX_FILTER_H__ 7 8 #include <linux/atomic.h> 9 #include <linux/bpf.h> 10 #include <linux/refcount.h> 11 #include <linux/compat.h> 12 #include <linux/skbuff.h> 13 #include <linux/linkage.h> 14 #include <linux/printk.h> 15 #include <linux/workqueue.h> 16 #include <linux/sched.h> 17 #include <linux/sched/clock.h> 18 #include <linux/capability.h> 19 #include <linux/set_memory.h> 20 #include <linux/kallsyms.h> 21 #include <linux/if_vlan.h> 22 #include <linux/vmalloc.h> 23 #include <linux/sockptr.h> 24 #include <linux/static_call.h> 25 #include <linux/u64_stats_sync.h> 26 27 #include <net/sch_generic.h> 28 29 #include <asm/byteorder.h> 30 #include <uapi/linux/filter.h> 31 32 struct sk_buff; 33 struct sock; 34 struct seccomp_data; 35 struct bpf_prog_aux; 36 struct xdp_rxq_info; 37 struct xdp_buff; 38 struct sock_reuseport; 39 struct ctl_table; 40 struct ctl_table_header; 41 42 /* ArgX, context and stack frame pointer register positions. Note, 43 * Arg1, Arg2, Arg3, etc are used as argument mappings of function 44 * calls in BPF_CALL instruction. 45 */ 46 #define BPF_REG_ARG1 BPF_REG_1 47 #define BPF_REG_ARG2 BPF_REG_2 48 #define BPF_REG_ARG3 BPF_REG_3 49 #define BPF_REG_ARG4 BPF_REG_4 50 #define BPF_REG_ARG5 BPF_REG_5 51 #define BPF_REG_CTX BPF_REG_6 52 #define BPF_REG_FP BPF_REG_10 53 54 /* Additional register mappings for converted user programs. */ 55 #define BPF_REG_A BPF_REG_0 56 #define BPF_REG_X BPF_REG_7 57 #define BPF_REG_TMP BPF_REG_2 /* scratch reg */ 58 #define BPF_REG_D BPF_REG_8 /* data, callee-saved */ 59 #define BPF_REG_H BPF_REG_9 /* hlen, callee-saved */ 60 61 /* Kernel hidden auxiliary/helper register. */ 62 #define BPF_REG_PARAMS MAX_BPF_REG 63 #define BPF_REG_AX (MAX_BPF_REG + 1) 64 #define MAX_BPF_EXT_REG (MAX_BPF_REG + 2) 65 #define MAX_BPF_JIT_REG MAX_BPF_EXT_REG 66 67 /* unused opcode to mark special call to bpf_tail_call() helper */ 68 #define BPF_TAIL_CALL 0xf0 69 70 /* unused opcode to mark special load instruction. Same as BPF_ABS */ 71 #define BPF_PROBE_MEM 0x20 72 73 /* unused opcode to mark special ldsx instruction. Same as BPF_IND */ 74 #define BPF_PROBE_MEMSX 0x40 75 76 /* unused opcode to mark special load instruction. Same as BPF_MSH */ 77 #define BPF_PROBE_MEM32 0xa0 78 79 /* unused opcode to mark special atomic instruction */ 80 #define BPF_PROBE_ATOMIC 0xe0 81 82 /* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */ 83 #define BPF_PROBE_MEM32SX 0xc0 84 85 /* unused opcode to mark call to interpreter with arguments */ 86 #define BPF_CALL_ARGS 0xe0 87 88 /* unused opcode to mark speculation barrier for mitigating 89 * Spectre v1 and v4 90 */ 91 #define BPF_NOSPEC 0xc0 92 93 /* As per nm, we expose JITed images as text (code) section for 94 * kallsyms. That way, tools like perf can find it to match 95 * addresses. 96 */ 97 #define BPF_SYM_ELF_TYPE 't' 98 99 /* BPF program can access up to 512 bytes of stack space. */ 100 #define MAX_BPF_STACK 512 101 /* 102 * Stack budget of a program on a JIT that lays out frames of that size. 103 * The interpreter and JITs without such support keep MAX_BPF_STACK. 104 */ 105 #define MAX_BPF_STACK_JIT 2048 106 107 /* Helper macros for filter block array initializers. */ 108 109 /* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */ 110 111 #define BPF_ALU64_REG_OFF(OP, DST, SRC, OFF) \ 112 ((struct bpf_insn) { \ 113 .code = BPF_ALU64 | BPF_OP(OP) | BPF_X, \ 114 .dst_reg = DST, \ 115 .src_reg = SRC, \ 116 .off = OFF, \ 117 .imm = 0 }) 118 119 #define BPF_ALU64_REG(OP, DST, SRC) \ 120 BPF_ALU64_REG_OFF(OP, DST, SRC, 0) 121 122 #define BPF_ALU32_REG_OFF(OP, DST, SRC, OFF) \ 123 ((struct bpf_insn) { \ 124 .code = BPF_ALU | BPF_OP(OP) | BPF_X, \ 125 .dst_reg = DST, \ 126 .src_reg = SRC, \ 127 .off = OFF, \ 128 .imm = 0 }) 129 130 #define BPF_ALU32_REG(OP, DST, SRC) \ 131 BPF_ALU32_REG_OFF(OP, DST, SRC, 0) 132 133 /* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */ 134 135 #define BPF_ALU64_IMM_OFF(OP, DST, IMM, OFF) \ 136 ((struct bpf_insn) { \ 137 .code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \ 138 .dst_reg = DST, \ 139 .src_reg = 0, \ 140 .off = OFF, \ 141 .imm = IMM }) 142 #define BPF_ALU64_IMM(OP, DST, IMM) \ 143 BPF_ALU64_IMM_OFF(OP, DST, IMM, 0) 144 145 #define BPF_ALU32_IMM_OFF(OP, DST, IMM, OFF) \ 146 ((struct bpf_insn) { \ 147 .code = BPF_ALU | BPF_OP(OP) | BPF_K, \ 148 .dst_reg = DST, \ 149 .src_reg = 0, \ 150 .off = OFF, \ 151 .imm = IMM }) 152 #define BPF_ALU32_IMM(OP, DST, IMM) \ 153 BPF_ALU32_IMM_OFF(OP, DST, IMM, 0) 154 155 /* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */ 156 157 #define BPF_ENDIAN(TYPE, DST, LEN) \ 158 ((struct bpf_insn) { \ 159 .code = BPF_ALU | BPF_END | BPF_SRC(TYPE), \ 160 .dst_reg = DST, \ 161 .src_reg = 0, \ 162 .off = 0, \ 163 .imm = LEN }) 164 165 /* Byte Swap, bswap16/32/64 */ 166 167 #define BPF_BSWAP(DST, LEN) \ 168 ((struct bpf_insn) { \ 169 .code = BPF_ALU64 | BPF_END | BPF_SRC(BPF_TO_LE), \ 170 .dst_reg = DST, \ 171 .src_reg = 0, \ 172 .off = 0, \ 173 .imm = LEN }) 174 175 /* Short form of mov, dst_reg = src_reg */ 176 177 #define BPF_MOV64_REG(DST, SRC) \ 178 ((struct bpf_insn) { \ 179 .code = BPF_ALU64 | BPF_MOV | BPF_X, \ 180 .dst_reg = DST, \ 181 .src_reg = SRC, \ 182 .off = 0, \ 183 .imm = 0 }) 184 185 #define BPF_MOV32_REG(DST, SRC) \ 186 ((struct bpf_insn) { \ 187 .code = BPF_ALU | BPF_MOV | BPF_X, \ 188 .dst_reg = DST, \ 189 .src_reg = SRC, \ 190 .off = 0, \ 191 .imm = 0 }) 192 193 /* Special (internal-only) form of mov, used to resolve per-CPU addrs: 194 * dst_reg = src_reg + <percpu_base_off> 195 * BPF_ADDR_PERCPU is used as a special insn->off value. 196 */ 197 #define BPF_ADDR_PERCPU (-1) 198 199 #define BPF_MOV64_PERCPU_REG(DST, SRC) \ 200 ((struct bpf_insn) { \ 201 .code = BPF_ALU64 | BPF_MOV | BPF_X, \ 202 .dst_reg = DST, \ 203 .src_reg = SRC, \ 204 .off = BPF_ADDR_PERCPU, \ 205 .imm = 0 }) 206 207 static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn) 208 { 209 return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->off == BPF_ADDR_PERCPU; 210 } 211 212 /* Short form of mov, dst_reg = imm32 */ 213 214 #define BPF_MOV64_IMM(DST, IMM) \ 215 ((struct bpf_insn) { \ 216 .code = BPF_ALU64 | BPF_MOV | BPF_K, \ 217 .dst_reg = DST, \ 218 .src_reg = 0, \ 219 .off = 0, \ 220 .imm = IMM }) 221 222 #define BPF_MOV32_IMM(DST, IMM) \ 223 ((struct bpf_insn) { \ 224 .code = BPF_ALU | BPF_MOV | BPF_K, \ 225 .dst_reg = DST, \ 226 .src_reg = 0, \ 227 .off = 0, \ 228 .imm = IMM }) 229 230 /* Short form of movsx, dst_reg = (s8,s16,s32)src_reg */ 231 232 #define BPF_MOVSX64_REG(DST, SRC, OFF) \ 233 ((struct bpf_insn) { \ 234 .code = BPF_ALU64 | BPF_MOV | BPF_X, \ 235 .dst_reg = DST, \ 236 .src_reg = SRC, \ 237 .off = OFF, \ 238 .imm = 0 }) 239 240 #define BPF_MOVSX32_REG(DST, SRC, OFF) \ 241 ((struct bpf_insn) { \ 242 .code = BPF_ALU | BPF_MOV | BPF_X, \ 243 .dst_reg = DST, \ 244 .src_reg = SRC, \ 245 .off = OFF, \ 246 .imm = 0 }) 247 248 /* Special form of mov32, used for doing explicit zero extension on dst. */ 249 #define BPF_ZEXT_REG(DST) \ 250 ((struct bpf_insn) { \ 251 .code = BPF_ALU | BPF_MOV | BPF_X, \ 252 .dst_reg = DST, \ 253 .src_reg = DST, \ 254 .off = 0, \ 255 .imm = 1 }) 256 257 static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn) 258 { 259 return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO; 260 } 261 262 static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) 263 { 264 return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT; 265 } 266 267 /* 268 * The caller of the main prog doesn't tell unwind_exit from exit. An extension 269 * is different: its caller is the caller of the subprog that it replaced. 270 * An exception callback returns from the frame of the main prog. After freplace 271 * or a tail call it returns to a prog that wasn't verified with the callback. 272 */ 273 static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) 274 { 275 if (prog->aux->exception_cb) 276 return false; 277 return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; 278 } 279 280 static inline bool insn_is_zext(const struct bpf_insn *insn) 281 { 282 return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; 283 } 284 285 /* addr_space_cast from as(0) to as(1) is for converting bpf arena pointers 286 * to pointers in user vma. 287 */ 288 static inline bool insn_is_cast_user(const struct bpf_insn *insn) 289 { 290 return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && 291 insn->off == BPF_ADDR_SPACE_CAST && 292 insn->imm == 1U << 16; 293 } 294 295 /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */ 296 #define BPF_LD_IMM64(DST, IMM) \ 297 BPF_LD_IMM64_RAW(DST, 0, IMM) 298 299 #define BPF_LD_IMM64_RAW(DST, SRC, IMM) \ 300 ((struct bpf_insn) { \ 301 .code = BPF_LD | BPF_DW | BPF_IMM, \ 302 .dst_reg = DST, \ 303 .src_reg = SRC, \ 304 .off = 0, \ 305 .imm = (__u32) (IMM) }), \ 306 ((struct bpf_insn) { \ 307 .code = 0, /* zero is reserved opcode */ \ 308 .dst_reg = 0, \ 309 .src_reg = 0, \ 310 .off = 0, \ 311 .imm = ((__u64) (IMM)) >> 32 }) 312 313 /* pseudo BPF_LD_IMM64 insn used to refer to process-local map_fd */ 314 #define BPF_LD_MAP_FD(DST, MAP_FD) \ 315 BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD) 316 317 /* Short form of mov based on type, BPF_X: dst_reg = src_reg, BPF_K: dst_reg = imm32 */ 318 319 #define BPF_MOV64_RAW(TYPE, DST, SRC, IMM) \ 320 ((struct bpf_insn) { \ 321 .code = BPF_ALU64 | BPF_MOV | BPF_SRC(TYPE), \ 322 .dst_reg = DST, \ 323 .src_reg = SRC, \ 324 .off = 0, \ 325 .imm = IMM }) 326 327 #define BPF_MOV32_RAW(TYPE, DST, SRC, IMM) \ 328 ((struct bpf_insn) { \ 329 .code = BPF_ALU | BPF_MOV | BPF_SRC(TYPE), \ 330 .dst_reg = DST, \ 331 .src_reg = SRC, \ 332 .off = 0, \ 333 .imm = IMM }) 334 335 /* Direct packet access, R0 = *(uint *) (skb->data + imm32) */ 336 337 #define BPF_LD_ABS(SIZE, IMM) \ 338 ((struct bpf_insn) { \ 339 .code = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS, \ 340 .dst_reg = 0, \ 341 .src_reg = 0, \ 342 .off = 0, \ 343 .imm = IMM }) 344 345 /* Indirect packet access, R0 = *(uint *) (skb->data + src_reg + imm32) */ 346 347 #define BPF_LD_IND(SIZE, SRC, IMM) \ 348 ((struct bpf_insn) { \ 349 .code = BPF_LD | BPF_SIZE(SIZE) | BPF_IND, \ 350 .dst_reg = 0, \ 351 .src_reg = SRC, \ 352 .off = 0, \ 353 .imm = IMM }) 354 355 /* Memory load, dst_reg = *(uint *) (src_reg + off16) */ 356 357 #define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \ 358 ((struct bpf_insn) { \ 359 .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \ 360 .dst_reg = DST, \ 361 .src_reg = SRC, \ 362 .off = OFF, \ 363 .imm = 0 }) 364 365 /* Memory load, dst_reg = *(signed size *) (src_reg + off16) */ 366 367 #define BPF_LDX_MEMSX(SIZE, DST, SRC, OFF) \ 368 ((struct bpf_insn) { \ 369 .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEMSX, \ 370 .dst_reg = DST, \ 371 .src_reg = SRC, \ 372 .off = OFF, \ 373 .imm = 0 }) 374 375 /* Memory store, *(uint *) (dst_reg + off16) = src_reg */ 376 377 #define BPF_STX_MEM(SIZE, DST, SRC, OFF) \ 378 ((struct bpf_insn) { \ 379 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM, \ 380 .dst_reg = DST, \ 381 .src_reg = SRC, \ 382 .off = OFF, \ 383 .imm = 0 }) 384 385 386 /* 387 * Atomic operations: 388 * 389 * BPF_ADD *(uint *) (dst_reg + off16) += src_reg 390 * BPF_AND *(uint *) (dst_reg + off16) &= src_reg 391 * BPF_OR *(uint *) (dst_reg + off16) |= src_reg 392 * BPF_XOR *(uint *) (dst_reg + off16) ^= src_reg 393 * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); 394 * BPF_AND | BPF_FETCH src_reg = atomic_fetch_and(dst_reg + off16, src_reg); 395 * BPF_OR | BPF_FETCH src_reg = atomic_fetch_or(dst_reg + off16, src_reg); 396 * BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg); 397 * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg) 398 * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg) 399 * BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16) 400 * BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg) 401 */ 402 403 #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ 404 ((struct bpf_insn) { \ 405 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \ 406 .dst_reg = DST, \ 407 .src_reg = SRC, \ 408 .off = OFF, \ 409 .imm = OP }) 410 411 /* Legacy alias */ 412 #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) 413 414 /* 415 * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an 416 * atomic load or store, and false if it is a read-modify-write instruction. 417 */ 418 static inline bool 419 bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) 420 { 421 switch (atomic_insn->imm) { 422 case BPF_LOAD_ACQ: 423 case BPF_STORE_REL: 424 return true; 425 default: 426 return false; 427 } 428 } 429 430 /* 431 * A load-acquire is the only BPF_STX class instruction that reads into 432 * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. 433 * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC 434 * instruction here, so that callers which walk all instruction classes can 435 * use this directly. 436 */ 437 static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) 438 { 439 return BPF_CLASS(insn->code) == BPF_STX && 440 (BPF_MODE(insn->code) == BPF_ATOMIC || 441 BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && 442 insn->imm == BPF_LOAD_ACQ; 443 } 444 445 /* 446 * Given an instruction @insn, return the number of the BPF register that a 447 * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is 448 * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when 449 * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to 450 * be a BPF_ATOMIC here. 451 */ 452 static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) 453 { 454 if (BPF_CLASS(insn->code) != BPF_STX || 455 (BPF_MODE(insn->code) != BPF_ATOMIC && 456 BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) 457 return -1; 458 459 switch (insn->imm) { 460 case BPF_LOAD_ACQ: 461 return insn->dst_reg; 462 case BPF_CMPXCHG: 463 return BPF_REG_0; 464 default: 465 return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; 466 } 467 } 468 469 /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ 470 471 #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ 472 ((struct bpf_insn) { \ 473 .code = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM, \ 474 .dst_reg = DST, \ 475 .src_reg = 0, \ 476 .off = OFF, \ 477 .imm = IMM }) 478 479 /* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */ 480 481 #define BPF_JMP_REG(OP, DST, SRC, OFF) \ 482 ((struct bpf_insn) { \ 483 .code = BPF_JMP | BPF_OP(OP) | BPF_X, \ 484 .dst_reg = DST, \ 485 .src_reg = SRC, \ 486 .off = OFF, \ 487 .imm = 0 }) 488 489 /* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */ 490 491 #define BPF_JMP_IMM(OP, DST, IMM, OFF) \ 492 ((struct bpf_insn) { \ 493 .code = BPF_JMP | BPF_OP(OP) | BPF_K, \ 494 .dst_reg = DST, \ 495 .src_reg = 0, \ 496 .off = OFF, \ 497 .imm = IMM }) 498 499 /* Like BPF_JMP_REG, but with 32-bit wide operands for comparison. */ 500 501 #define BPF_JMP32_REG(OP, DST, SRC, OFF) \ 502 ((struct bpf_insn) { \ 503 .code = BPF_JMP32 | BPF_OP(OP) | BPF_X, \ 504 .dst_reg = DST, \ 505 .src_reg = SRC, \ 506 .off = OFF, \ 507 .imm = 0 }) 508 509 /* Like BPF_JMP_IMM, but with 32-bit wide operands for comparison. */ 510 511 #define BPF_JMP32_IMM(OP, DST, IMM, OFF) \ 512 ((struct bpf_insn) { \ 513 .code = BPF_JMP32 | BPF_OP(OP) | BPF_K, \ 514 .dst_reg = DST, \ 515 .src_reg = 0, \ 516 .off = OFF, \ 517 .imm = IMM }) 518 519 /* Unconditional jumps, goto pc + off16 */ 520 521 #define BPF_JMP_A(OFF) \ 522 ((struct bpf_insn) { \ 523 .code = BPF_JMP | BPF_JA, \ 524 .dst_reg = 0, \ 525 .src_reg = 0, \ 526 .off = OFF, \ 527 .imm = 0 }) 528 529 /* Unconditional jumps, gotol pc + imm32 */ 530 531 #define BPF_JMP32_A(IMM) \ 532 ((struct bpf_insn) { \ 533 .code = BPF_JMP32 | BPF_JA, \ 534 .dst_reg = 0, \ 535 .src_reg = 0, \ 536 .off = 0, \ 537 .imm = IMM }) 538 539 /* Relative call */ 540 541 #define BPF_CALL_REL(TGT) \ 542 ((struct bpf_insn) { \ 543 .code = BPF_JMP | BPF_CALL, \ 544 .dst_reg = 0, \ 545 .src_reg = BPF_PSEUDO_CALL, \ 546 .off = 0, \ 547 .imm = TGT }) 548 549 /* Convert function address to BPF immediate */ 550 551 #define BPF_CALL_IMM(x) ((void *)(x) - (void *)__bpf_call_base) 552 553 #define BPF_EMIT_CALL(FUNC) \ 554 ((struct bpf_insn) { \ 555 .code = BPF_JMP | BPF_CALL, \ 556 .dst_reg = 0, \ 557 .src_reg = 0, \ 558 .off = 0, \ 559 .imm = BPF_CALL_IMM(FUNC) }) 560 561 /* Kfunc call */ 562 563 #define BPF_CALL_KFUNC(OFF, IMM) \ 564 ((struct bpf_insn) { \ 565 .code = BPF_JMP | BPF_CALL, \ 566 .dst_reg = 0, \ 567 .src_reg = BPF_PSEUDO_KFUNC_CALL, \ 568 .off = OFF, \ 569 .imm = IMM }) 570 571 /* Raw code statement block */ 572 573 #define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \ 574 ((struct bpf_insn) { \ 575 .code = CODE, \ 576 .dst_reg = DST, \ 577 .src_reg = SRC, \ 578 .off = OFF, \ 579 .imm = IMM }) 580 581 /* Program exit */ 582 583 #define BPF_EXIT_INSN() \ 584 ((struct bpf_insn) { \ 585 .code = BPF_JMP | BPF_EXIT, \ 586 .dst_reg = 0, \ 587 .src_reg = 0, \ 588 .off = 0, \ 589 .imm = 0 }) 590 591 /* Speculation barrier */ 592 593 #define BPF_ST_NOSPEC() \ 594 ((struct bpf_insn) { \ 595 .code = BPF_ST | BPF_NOSPEC, \ 596 .dst_reg = 0, \ 597 .src_reg = 0, \ 598 .off = 0, \ 599 .imm = 0 }) 600 601 /* Internal classic blocks for direct assignment */ 602 603 #define __BPF_STMT(CODE, K) \ 604 ((struct sock_filter) BPF_STMT(CODE, K)) 605 606 #define __BPF_JUMP(CODE, K, JT, JF) \ 607 ((struct sock_filter) BPF_JUMP(CODE, K, JT, JF)) 608 609 #define bytes_to_bpf_size(bytes) \ 610 ({ \ 611 int bpf_size = -EINVAL; \ 612 \ 613 if (bytes == sizeof(u8)) \ 614 bpf_size = BPF_B; \ 615 else if (bytes == sizeof(u16)) \ 616 bpf_size = BPF_H; \ 617 else if (bytes == sizeof(u32)) \ 618 bpf_size = BPF_W; \ 619 else if (bytes == sizeof(u64)) \ 620 bpf_size = BPF_DW; \ 621 \ 622 bpf_size; \ 623 }) 624 625 #define bpf_size_to_bytes(bpf_size) \ 626 ({ \ 627 int bytes = -EINVAL; \ 628 \ 629 if (bpf_size == BPF_B) \ 630 bytes = sizeof(u8); \ 631 else if (bpf_size == BPF_H) \ 632 bytes = sizeof(u16); \ 633 else if (bpf_size == BPF_W) \ 634 bytes = sizeof(u32); \ 635 else if (bpf_size == BPF_DW) \ 636 bytes = sizeof(u64); \ 637 \ 638 bytes; \ 639 }) 640 641 #define BPF_SIZEOF(type) \ 642 ({ \ 643 const int __size = bytes_to_bpf_size(sizeof(type)); \ 644 BUILD_BUG_ON(__size < 0); \ 645 __size; \ 646 }) 647 648 #define BPF_FIELD_SIZEOF(type, field) \ 649 ({ \ 650 const int __size = bytes_to_bpf_size(sizeof_field(type, field)); \ 651 BUILD_BUG_ON(__size < 0); \ 652 __size; \ 653 }) 654 655 #define BPF_LDST_BYTES(insn) \ 656 ({ \ 657 const int __size = bpf_size_to_bytes(BPF_SIZE((insn)->code)); \ 658 WARN_ON(__size < 0); \ 659 __size; \ 660 }) 661 662 #define __BPF_MAP_0(m, v, ...) v 663 #define __BPF_MAP_1(m, v, t, a, ...) m(t, a) 664 #define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__) 665 #define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__) 666 #define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__) 667 #define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__) 668 669 #define __BPF_REG_0(...) __BPF_PAD(5) 670 #define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4) 671 #define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3) 672 #define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2) 673 #define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1) 674 #define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__) 675 676 #define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__) 677 #define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__) 678 679 #define __BPF_CAST(t, a) \ 680 (__force t) \ 681 (__force \ 682 typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long), \ 683 (unsigned long)0, (t)0))) a 684 #define __BPF_V void 685 #define __BPF_N 686 687 #define __BPF_DECL_ARGS(t, a) t a 688 #define __BPF_DECL_REGS(t, a) u64 a 689 690 #define __BPF_PAD(n) \ 691 __BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ 692 u64, __ur_3, u64, __ur_4, u64, __ur_5) 693 694 #define BPF_CALL_x(x, attr, name, ...) \ 695 static __always_inline \ 696 u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ 697 typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ 698 attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ 699 attr u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ 700 { \ 701 return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ 702 } \ 703 static __always_inline \ 704 u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) 705 706 #define __NOATTR 707 #define BPF_CALL_0(name, ...) BPF_CALL_x(0, __NOATTR, name, __VA_ARGS__) 708 #define BPF_CALL_1(name, ...) BPF_CALL_x(1, __NOATTR, name, __VA_ARGS__) 709 #define BPF_CALL_2(name, ...) BPF_CALL_x(2, __NOATTR, name, __VA_ARGS__) 710 #define BPF_CALL_3(name, ...) BPF_CALL_x(3, __NOATTR, name, __VA_ARGS__) 711 #define BPF_CALL_4(name, ...) BPF_CALL_x(4, __NOATTR, name, __VA_ARGS__) 712 #define BPF_CALL_5(name, ...) BPF_CALL_x(5, __NOATTR, name, __VA_ARGS__) 713 714 #define NOTRACE_BPF_CALL_1(name, ...) BPF_CALL_x(1, notrace, name, __VA_ARGS__) 715 716 #define bpf_ctx_range(TYPE, MEMBER) \ 717 offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 718 #define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2) \ 719 offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1 720 #if BITS_PER_LONG == 64 721 # define bpf_ctx_range_ptr(TYPE, MEMBER) \ 722 offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 723 #else 724 # define bpf_ctx_range_ptr(TYPE, MEMBER) \ 725 offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1 726 #endif /* BITS_PER_LONG == 64 */ 727 728 #define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE) \ 729 ({ \ 730 BUILD_BUG_ON(sizeof_field(TYPE, MEMBER) != (SIZE)); \ 731 *(PTR_SIZE) = (SIZE); \ 732 offsetof(TYPE, MEMBER); \ 733 }) 734 735 /* A struct sock_filter is architecture independent. */ 736 struct compat_sock_fprog { 737 u16 len; 738 compat_uptr_t filter; /* struct sock_filter * */ 739 }; 740 741 struct sock_fprog_kern { 742 u16 len; 743 struct sock_filter *filter; 744 }; 745 746 /* Some arches need doubleword alignment for their instructions and/or data */ 747 #define BPF_IMAGE_ALIGNMENT 8 748 749 struct bpf_binary_header { 750 u32 size; 751 u8 image[] __aligned(BPF_IMAGE_ALIGNMENT); 752 }; 753 754 struct bpf_prog_stats { 755 u64_stats_t cnt; 756 u64_stats_t nsecs; 757 u64_stats_t misses; 758 struct u64_stats_sync syncp; 759 } __aligned(2 * sizeof(u64)); 760 761 struct bpf_timed_may_goto { 762 u64 count; 763 u64 timestamp; 764 }; 765 766 struct sk_filter { 767 refcount_t refcnt; 768 struct rcu_head rcu; 769 struct bpf_prog *prog; 770 }; 771 772 DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); 773 774 extern struct mutex nf_conn_btf_access_lock; 775 extern int (*nfct_btf_struct_access)(struct bpf_verifier_log *log, 776 const struct bpf_reg_state *reg, 777 int off, int size); 778 779 typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx, 780 const struct bpf_insn *insnsi, 781 unsigned int (*bpf_func)(const void *, 782 const struct bpf_insn *)); 783 784 static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, 785 const void *ctx, 786 bpf_dispatcher_fn dfunc) 787 { 788 u32 ret; 789 790 cant_migrate(); 791 if (static_branch_unlikely(&bpf_stats_enabled_key)) { 792 struct bpf_prog_stats *stats; 793 u64 duration, start = sched_clock(); 794 unsigned long flags; 795 796 ret = dfunc(ctx, prog->insnsi, prog->bpf_func); 797 798 duration = sched_clock() - start; 799 if (likely(prog->stats)) { 800 stats = this_cpu_ptr(prog->stats); 801 flags = u64_stats_update_begin_irqsave(&stats->syncp); 802 u64_stats_inc(&stats->cnt); 803 u64_stats_add(&stats->nsecs, duration); 804 u64_stats_update_end_irqrestore(&stats->syncp, flags); 805 } 806 } else { 807 ret = dfunc(ctx, prog->insnsi, prog->bpf_func); 808 } 809 return ret; 810 } 811 812 static __always_inline u32 bpf_prog_run(const struct bpf_prog *prog, const void *ctx) 813 { 814 return __bpf_prog_run(prog, ctx, bpf_dispatcher_nop_func); 815 } 816 817 /* 818 * Use in preemptible and therefore migratable context to make sure that 819 * the execution of the BPF program runs on one CPU. 820 * 821 * This uses migrate_disable/enable() explicitly to document that the 822 * invocation of a BPF program does not require reentrancy protection 823 * against a BPF program which is invoked from a preempting task. 824 */ 825 static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog, 826 const void *ctx) 827 { 828 u32 ret; 829 830 migrate_disable(); 831 ret = bpf_prog_run(prog, ctx); 832 migrate_enable(); 833 return ret; 834 } 835 836 static inline bool is_stack_arg_ldx(const struct bpf_insn *insn) 837 { 838 return insn->code == (BPF_LDX | BPF_MEM | BPF_DW) && 839 insn->src_reg == BPF_REG_PARAMS && 840 insn->off > 0 && insn->off % 8 == 0; 841 } 842 843 static inline bool is_stack_arg_st(const struct bpf_insn *insn) 844 { 845 return insn->code == (BPF_ST | BPF_MEM | BPF_DW) && 846 insn->dst_reg == BPF_REG_PARAMS && 847 insn->off < 0 && insn->off % 8 == 0; 848 } 849 850 static inline bool is_stack_arg_stx(const struct bpf_insn *insn) 851 { 852 return insn->code == (BPF_STX | BPF_MEM | BPF_DW) && 853 insn->dst_reg == BPF_REG_PARAMS && 854 insn->off < 0 && insn->off % 8 == 0; 855 } 856 857 #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN 858 859 struct bpf_skb_data_end { 860 struct qdisc_skb_cb qdisc_cb; 861 void *data_meta; 862 void *data_end; 863 }; 864 865 struct bpf_nh_params { 866 u32 nh_family; 867 union { 868 u32 ipv4_nh; 869 struct in6_addr ipv6_nh; 870 }; 871 }; 872 873 /* flags for bpf_redirect_info kern_flags */ 874 #define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */ 875 #define BPF_RI_F_RI_INIT BIT(1) 876 #define BPF_RI_F_CPU_MAP_INIT BIT(2) 877 #define BPF_RI_F_DEV_MAP_INIT BIT(3) 878 #define BPF_RI_F_XSK_MAP_INIT BIT(4) 879 880 struct bpf_redirect_info { 881 u64 tgt_index; 882 void *tgt_value; 883 struct bpf_map *map; 884 u32 flags; 885 u32 map_id; 886 enum bpf_map_type map_type; 887 struct bpf_nh_params nh; 888 u32 kern_flags; 889 }; 890 891 struct bpf_net_context { 892 struct bpf_redirect_info ri; 893 struct list_head cpu_map_flush_list; 894 struct list_head dev_map_flush_list; 895 struct list_head xskmap_map_flush_list; 896 }; 897 898 static inline struct bpf_net_context *bpf_net_ctx_set(struct bpf_net_context *bpf_net_ctx) 899 { 900 struct task_struct *tsk = current; 901 902 if (tsk->bpf_net_context != NULL) 903 return NULL; 904 bpf_net_ctx->ri.kern_flags = 0; 905 906 tsk->bpf_net_context = bpf_net_ctx; 907 return bpf_net_ctx; 908 } 909 910 static inline void bpf_net_ctx_clear(struct bpf_net_context *bpf_net_ctx) 911 { 912 if (bpf_net_ctx) 913 current->bpf_net_context = NULL; 914 } 915 916 static inline struct bpf_net_context *bpf_net_ctx_get(void) 917 { 918 return current->bpf_net_context; 919 } 920 921 static inline struct bpf_redirect_info *bpf_net_ctx_get_ri(void) 922 { 923 struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get(); 924 925 if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_RI_INIT)) { 926 memset(&bpf_net_ctx->ri, 0, offsetof(struct bpf_net_context, ri.nh)); 927 bpf_net_ctx->ri.kern_flags |= BPF_RI_F_RI_INIT; 928 } 929 930 return &bpf_net_ctx->ri; 931 } 932 933 static inline struct list_head *bpf_net_ctx_get_cpu_map_flush_list(void) 934 { 935 struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get(); 936 937 if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_CPU_MAP_INIT)) { 938 INIT_LIST_HEAD(&bpf_net_ctx->cpu_map_flush_list); 939 bpf_net_ctx->ri.kern_flags |= BPF_RI_F_CPU_MAP_INIT; 940 } 941 942 return &bpf_net_ctx->cpu_map_flush_list; 943 } 944 945 static inline struct list_head *bpf_net_ctx_get_dev_flush_list(void) 946 { 947 struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get(); 948 949 if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_DEV_MAP_INIT)) { 950 INIT_LIST_HEAD(&bpf_net_ctx->dev_map_flush_list); 951 bpf_net_ctx->ri.kern_flags |= BPF_RI_F_DEV_MAP_INIT; 952 } 953 954 return &bpf_net_ctx->dev_map_flush_list; 955 } 956 957 static inline struct list_head *bpf_net_ctx_get_xskmap_flush_list(void) 958 { 959 struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get(); 960 961 if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_XSK_MAP_INIT)) { 962 INIT_LIST_HEAD(&bpf_net_ctx->xskmap_map_flush_list); 963 bpf_net_ctx->ri.kern_flags |= BPF_RI_F_XSK_MAP_INIT; 964 } 965 966 return &bpf_net_ctx->xskmap_map_flush_list; 967 } 968 969 static inline void bpf_net_ctx_get_all_used_flush_lists(struct list_head **lh_map, 970 struct list_head **lh_dev, 971 struct list_head **lh_xsk) 972 { 973 struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get(); 974 u32 kern_flags = bpf_net_ctx->ri.kern_flags; 975 struct list_head *lh; 976 977 *lh_map = *lh_dev = *lh_xsk = NULL; 978 979 if (!IS_ENABLED(CONFIG_BPF_SYSCALL)) 980 return; 981 982 lh = &bpf_net_ctx->dev_map_flush_list; 983 if (kern_flags & BPF_RI_F_DEV_MAP_INIT && !list_empty(lh)) 984 *lh_dev = lh; 985 986 lh = &bpf_net_ctx->cpu_map_flush_list; 987 if (kern_flags & BPF_RI_F_CPU_MAP_INIT && !list_empty(lh)) 988 *lh_map = lh; 989 990 lh = &bpf_net_ctx->xskmap_map_flush_list; 991 if (IS_ENABLED(CONFIG_XDP_SOCKETS) && 992 kern_flags & BPF_RI_F_XSK_MAP_INIT && !list_empty(lh)) 993 *lh_xsk = lh; 994 } 995 996 /* Compute the linear packet data range [data, data_end) which 997 * will be accessed by various program types (cls_bpf, act_bpf, 998 * lwt, ...). Subsystems allowing direct data access must (!) 999 * ensure that cb[] area can be written to when BPF program is 1000 * invoked (otherwise cb[] save/restore is necessary). 1001 */ 1002 static inline void bpf_compute_data_pointers(struct sk_buff *skb) 1003 { 1004 struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; 1005 1006 BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb)); 1007 cb->data_meta = skb->data - skb_metadata_len(skb); 1008 cb->data_end = skb->data + skb_headlen(skb); 1009 } 1010 1011 static inline int bpf_prog_run_data_pointers( 1012 const struct bpf_prog *prog, 1013 struct sk_buff *skb) 1014 { 1015 struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; 1016 void *save_data_meta, *save_data_end; 1017 int res; 1018 1019 save_data_meta = cb->data_meta; 1020 save_data_end = cb->data_end; 1021 1022 bpf_compute_data_pointers(skb); 1023 res = bpf_prog_run(prog, skb); 1024 1025 cb->data_meta = save_data_meta; 1026 cb->data_end = save_data_end; 1027 1028 return res; 1029 } 1030 1031 /* Similar to bpf_compute_data_pointers(), except that save orginal 1032 * data in cb->data and cb->meta_data for restore. 1033 */ 1034 static inline void bpf_compute_and_save_data_end( 1035 struct sk_buff *skb, void **saved_data_end) 1036 { 1037 struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; 1038 1039 *saved_data_end = cb->data_end; 1040 cb->data_end = skb->data + skb_headlen(skb); 1041 } 1042 1043 /* Restore data saved by bpf_compute_and_save_data_end(). */ 1044 static inline void bpf_restore_data_end( 1045 struct sk_buff *skb, void *saved_data_end) 1046 { 1047 struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; 1048 1049 cb->data_end = saved_data_end; 1050 } 1051 1052 static inline u8 *bpf_skb_cb(const struct sk_buff *skb) 1053 { 1054 /* eBPF programs may read/write skb->cb[] area to transfer meta 1055 * data between tail calls. Since this also needs to work with 1056 * tc, that scratch memory is mapped to qdisc_skb_cb's data area. 1057 * 1058 * In some socket filter cases, the cb unfortunately needs to be 1059 * saved/restored so that protocol specific skb->cb[] data won't 1060 * be lost. In any case, due to unpriviledged eBPF programs 1061 * attached to sockets, we need to clear the bpf_skb_cb() area 1062 * to not leak previous contents to user space. 1063 */ 1064 BUILD_BUG_ON(sizeof_field(struct __sk_buff, cb) != BPF_SKB_CB_LEN); 1065 BUILD_BUG_ON(sizeof_field(struct __sk_buff, cb) != 1066 sizeof_field(struct qdisc_skb_cb, data)); 1067 1068 return qdisc_skb_cb(skb)->data; 1069 } 1070 1071 /* Must be invoked with migration disabled */ 1072 static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog, 1073 const void *ctx) 1074 { 1075 const struct sk_buff *skb = ctx; 1076 u8 *cb_data = bpf_skb_cb(skb); 1077 u8 cb_saved[BPF_SKB_CB_LEN]; 1078 u32 res; 1079 1080 if (unlikely(prog->cb_access)) { 1081 memcpy(cb_saved, cb_data, sizeof(cb_saved)); 1082 memset(cb_data, 0, sizeof(cb_saved)); 1083 } 1084 1085 res = bpf_prog_run(prog, skb); 1086 1087 if (unlikely(prog->cb_access)) 1088 memcpy(cb_data, cb_saved, sizeof(cb_saved)); 1089 1090 return res; 1091 } 1092 1093 static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog, 1094 struct sk_buff *skb) 1095 { 1096 u32 res; 1097 1098 migrate_disable(); 1099 res = __bpf_prog_run_save_cb(prog, skb); 1100 migrate_enable(); 1101 return res; 1102 } 1103 1104 static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog, 1105 struct sk_buff *skb) 1106 { 1107 u8 *cb_data = bpf_skb_cb(skb); 1108 u32 res; 1109 1110 if (unlikely(prog->cb_access)) 1111 memset(cb_data, 0, BPF_SKB_CB_LEN); 1112 1113 res = bpf_prog_run_pin_on_cpu(prog, skb); 1114 return res; 1115 } 1116 1117 DECLARE_BPF_DISPATCHER(xdp) 1118 1119 DECLARE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key); 1120 1121 u32 xdp_master_redirect(struct xdp_buff *xdp); 1122 1123 void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog); 1124 1125 static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog) 1126 { 1127 return prog->len * sizeof(struct bpf_insn); 1128 } 1129 1130 static inline unsigned int bpf_prog_size(unsigned int proglen) 1131 { 1132 return max(sizeof(struct bpf_prog), 1133 offsetof(struct bpf_prog, insns[proglen])); 1134 } 1135 1136 bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); 1137 1138 /* 1139 * Whether a JIT has to set up the arena base for @prog. A program bound to 1140 * an arena need not touch it, libbpf binds the arena of an object to every 1141 * program in it. The flag caches bpf_prog_has_arena_insns() for the function 1142 * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). 1143 */ 1144 static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) 1145 { 1146 return prog->aux->uses_arena_insns; 1147 } 1148 1149 static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) 1150 { 1151 /* When classic BPF programs have been loaded and the arch 1152 * does not have a classic BPF JIT (anymore), they have been 1153 * converted via bpf_migrate_filter() to eBPF and thus always 1154 * have an unspec program type. 1155 */ 1156 return prog->type == BPF_PROG_TYPE_UNSPEC; 1157 } 1158 1159 static inline u32 bpf_ctx_off_adjust_machine(u32 size) 1160 { 1161 const u32 size_machine = sizeof(unsigned long); 1162 1163 if (size > size_machine && size % size_machine == 0) 1164 size = size_machine; 1165 1166 return size; 1167 } 1168 1169 static inline bool 1170 bpf_ctx_narrow_access_ok(u32 off, u32 size, u32 size_default) 1171 { 1172 return size <= size_default && (size & (size - 1)) == 0; 1173 } 1174 1175 static inline u8 1176 bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default) 1177 { 1178 u8 access_off = off & (size_default - 1); 1179 1180 #ifdef __LITTLE_ENDIAN 1181 return access_off; 1182 #else 1183 return size_default - (access_off + size); 1184 #endif 1185 } 1186 1187 #define bpf_ctx_wide_access_ok(off, size, type, field) \ 1188 (size == sizeof(__u64) && \ 1189 off >= offsetof(type, field) && \ 1190 off + sizeof(__u64) <= offsetofend(type, field) && \ 1191 off % sizeof(__u64) == 0) 1192 1193 #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0])) 1194 1195 static inline int __must_check bpf_prog_lock_ro(struct bpf_prog *fp) 1196 { 1197 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1198 if (!fp->jited) { 1199 set_vm_flush_reset_perms(fp); 1200 return set_memory_ro((unsigned long)fp, fp->pages); 1201 } 1202 #endif 1203 return 0; 1204 } 1205 1206 static inline int __must_check 1207 bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr) 1208 { 1209 set_vm_flush_reset_perms(hdr); 1210 return set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT); 1211 } 1212 1213 enum skb_drop_reason 1214 sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap); 1215 1216 static inline int sk_filter(struct sock *sk, struct sk_buff *skb) 1217 { 1218 enum skb_drop_reason drop_reason; 1219 1220 drop_reason = sk_filter_trim_cap(sk, skb, 1); 1221 return drop_reason ? -EPERM : 0; 1222 } 1223 1224 static inline enum skb_drop_reason 1225 sk_filter_reason(struct sock *sk, struct sk_buff *skb) 1226 { 1227 return sk_filter_trim_cap(sk, skb, 1); 1228 } 1229 1230 struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, 1231 int *err); 1232 struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); 1233 void bpf_prog_free(struct bpf_prog *fp); 1234 1235 bool bpf_opcode_in_insntable(u8 code); 1236 1237 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, 1238 const u32 *insn_to_jit_off); 1239 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog); 1240 void bpf_prog_jit_attempt_done(struct bpf_prog *prog); 1241 1242 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags); 1243 struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags); 1244 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 1245 gfp_t gfp_extra_flags); 1246 void __bpf_prog_free(struct bpf_prog *fp); 1247 1248 static inline void bpf_prog_unlock_free(struct bpf_prog *fp) 1249 { 1250 __bpf_prog_free(fp); 1251 } 1252 1253 typedef int (*bpf_aux_classic_check_t)(struct sock_filter *filter, 1254 unsigned int flen); 1255 1256 int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog); 1257 int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog, 1258 bpf_aux_classic_check_t trans, bool save_orig); 1259 void bpf_prog_destroy(struct bpf_prog *fp); 1260 1261 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk); 1262 int sk_attach_bpf(u32 ufd, struct sock *sk); 1263 int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk); 1264 int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk); 1265 void sk_reuseport_prog_free(struct bpf_prog *prog); 1266 int sk_detach_filter(struct sock *sk); 1267 int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len); 1268 1269 bool sk_filter_charge(struct sock *sk, struct sk_filter *fp); 1270 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); 1271 1272 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1273 1274 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); 1275 void bpf_jit_compile(struct bpf_prog *prog); 1276 bool bpf_jit_needs_zext(void); 1277 bool bpf_jit_inlines_helper_call(s32 imm); 1278 bool bpf_jit_supports_subprog_tailcalls(void); 1279 bool bpf_jit_supports_percpu_insn(void); 1280 bool bpf_jit_supports_kfunc_call(void); 1281 bool bpf_jit_supports_callx(void); 1282 bool bpf_jit_supports_kfunc_ret_reg_pair(void); 1283 bool bpf_jit_supports_unwind(void); 1284 bool bpf_jit_supports_stack_args(void); 1285 bool bpf_jit_supports_arena_args(void); 1286 bool bpf_jit_supports_far_kfunc_call(void); 1287 bool bpf_jit_supports_exceptions(void); 1288 bool bpf_jit_supports_ptr_xchg(void); 1289 bool bpf_jit_supports_arena(void); 1290 bool bpf_jit_supports_arena_scalar(void); 1291 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); 1292 bool bpf_jit_supports_private_stack(void); 1293 bool bpf_jit_supports_large_stack(void); 1294 bool bpf_jit_supports_timed_may_goto(void); 1295 bool bpf_jit_supports_fsession(void); 1296 1297 struct bpf_jit_arg_abi { 1298 /* Argument registers of the kernel convention. */ 1299 u8 nr_arg_regs; 1300 /* Round the register number up to an even one for 16-byte alignment. */ 1301 bool even_reg_align; 1302 /* Round the stack slot up to an even one for 16-byte alignment. */ 1303 bool even_stack_align; 1304 /* An argument may straddle the last register and the stack. */ 1305 bool split_at_boundary; 1306 /* A later argument may reuse a register a stack-passed one skipped. */ 1307 bool backfill_after_stack; 1308 }; 1309 1310 const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); 1311 u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, 1312 const struct btf_func_model *fm, u8 *pos_of_slot); 1313 1314 /* The JIT's scratch register, in place of an argument slot. */ 1315 #define BPF_JIT_ARG_TMP 0xff 1316 1317 /* Every argument slot moves at most once, and the scratch goes out and back. */ 1318 #define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) 1319 1320 struct bpf_jit_arg_move { 1321 u8 dst; 1322 u8 src; 1323 }; 1324 1325 u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, 1326 const struct btf_func_model *fm, 1327 struct bpf_jit_arg_move *moves); 1328 u64 bpf_arch_uaddress_limit(void); 1329 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); 1330 u64 arch_bpf_timed_may_goto(void); 1331 u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *); 1332 bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id); 1333 1334 static inline bool bpf_dump_raw_ok(const struct cred *cred) 1335 { 1336 /* Reconstruction of call-sites is dependent on kallsyms, 1337 * thus make dump the same restriction. 1338 */ 1339 return kallsyms_show_value(cred); 1340 } 1341 1342 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, 1343 const struct bpf_insn *patch, u32 len); 1344 1345 #ifdef CONFIG_BPF_SYSCALL 1346 struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 1347 const struct bpf_insn *patch, u32 len); 1348 #else 1349 static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 1350 const struct bpf_insn *patch, u32 len) 1351 { 1352 return ERR_PTR(-ENOTSUPP); 1353 } 1354 #endif /* CONFIG_BPF_SYSCALL */ 1355 1356 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); 1357 1358 static inline bool xdp_return_frame_no_direct(void) 1359 { 1360 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 1361 1362 return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT; 1363 } 1364 1365 static inline void xdp_set_return_frame_no_direct(void) 1366 { 1367 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 1368 1369 ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT; 1370 } 1371 1372 static inline void xdp_clear_return_frame_no_direct(void) 1373 { 1374 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 1375 1376 ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT; 1377 } 1378 1379 static inline int xdp_ok_fwd_dev(const struct net_device *fwd, 1380 unsigned int pktlen) 1381 { 1382 unsigned int len; 1383 1384 if (unlikely(!(fwd->flags & IFF_UP))) 1385 return -ENETDOWN; 1386 1387 len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN; 1388 if (pktlen > len) 1389 return -EMSGSIZE; 1390 1391 return 0; 1392 } 1393 1394 /* The pair of xdp_do_redirect and xdp_do_flush MUST be called in the 1395 * same cpu context. Further for best results no more than a single map 1396 * for the do_redirect/do_flush pair should be used. This limitation is 1397 * because we only track one map and force a flush when the map changes. 1398 * This does not appear to be a real limitation for existing software. 1399 */ 1400 int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, 1401 struct xdp_buff *xdp, const struct bpf_prog *prog); 1402 int xdp_do_redirect(struct net_device *dev, 1403 struct xdp_buff *xdp, 1404 const struct bpf_prog *prog); 1405 int xdp_do_redirect_frame(struct net_device *dev, 1406 struct xdp_buff *xdp, 1407 struct xdp_frame *xdpf, 1408 const struct bpf_prog *prog); 1409 void xdp_do_flush(void); 1410 1411 void bpf_warn_invalid_xdp_action(const struct net_device *dev, 1412 const struct bpf_prog *prog, u32 act); 1413 1414 #ifdef CONFIG_INET 1415 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, 1416 struct bpf_prog *prog, struct sk_buff *skb, 1417 struct sock *migrating_sk, 1418 u32 hash); 1419 #else 1420 static inline struct sock * 1421 bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, 1422 struct bpf_prog *prog, struct sk_buff *skb, 1423 struct sock *migrating_sk, 1424 u32 hash) 1425 { 1426 return NULL; 1427 } 1428 #endif 1429 1430 #ifdef CONFIG_BPF_JIT 1431 extern int bpf_jit_enable; 1432 extern int bpf_jit_harden; 1433 extern int bpf_jit_kallsyms; 1434 extern long bpf_jit_limit; 1435 extern long bpf_jit_limit_max; 1436 1437 typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); 1438 1439 /* 1440 * Flush the indirect branch predictors before reusing JIT memory, so that 1441 * indirect jumps into a newly written program don't reuse predictions left 1442 * behind by an old program that occupied the same space. 1443 */ 1444 void bpf_arch_pred_flush(void); 1445 DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); 1446 DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); 1447 1448 void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); 1449 1450 struct bpf_binary_header * 1451 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 1452 unsigned int alignment, 1453 bpf_jit_fill_hole_t bpf_fill_ill_insns); 1454 void bpf_jit_binary_free(struct bpf_binary_header *hdr); 1455 u64 bpf_jit_alloc_exec_limit(void); 1456 void *bpf_jit_alloc_exec(unsigned long size); 1457 void bpf_jit_free_exec(void *addr); 1458 void bpf_jit_free(struct bpf_prog *fp); 1459 struct bpf_binary_header * 1460 bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); 1461 1462 void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); 1463 void bpf_prog_pack_free(void *ptr, u32 size); 1464 1465 static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) 1466 { 1467 return list_empty(&fp->aux->ksym.lnode) || 1468 fp->aux->ksym.lnode.prev == LIST_POISON2; 1469 } 1470 1471 struct bpf_binary_header * 1472 bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, 1473 unsigned int alignment, 1474 struct bpf_binary_header **rw_hdr, 1475 u8 **rw_image, 1476 bpf_jit_fill_hole_t bpf_fill_ill_insns, 1477 bool was_classic); 1478 int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, 1479 struct bpf_binary_header *rw_header); 1480 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, 1481 struct bpf_binary_header *rw_header); 1482 1483 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, 1484 struct bpf_jit_poke_descriptor *poke); 1485 1486 int bpf_jit_get_func_addr(const struct bpf_prog *prog, 1487 const struct bpf_insn *insn, bool extra_pass, 1488 u64 *func_addr, bool *func_addr_fixed); 1489 1490 const char *bpf_jit_get_prog_name(struct bpf_prog *prog); 1491 1492 struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog); 1493 void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); 1494 1495 static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) 1496 { 1497 return prog->blinding_requested && !prog->blinded; 1498 } 1499 1500 static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, 1501 u32 pass, void *image) 1502 { 1503 pr_err("flen=%u proglen=%u pass=%u image=%p from=%s pid=%d\n", flen, 1504 proglen, pass, image, current->comm, task_pid_nr(current)); 1505 1506 if (image) 1507 print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET, 1508 16, 1, image, proglen, false); 1509 } 1510 1511 static inline bool bpf_jit_is_ebpf(void) 1512 { 1513 # ifdef CONFIG_HAVE_EBPF_JIT 1514 return true; 1515 # else 1516 return false; 1517 # endif 1518 } 1519 1520 static inline bool ebpf_jit_enabled(void) 1521 { 1522 return bpf_jit_enable && bpf_jit_is_ebpf(); 1523 } 1524 1525 static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) 1526 { 1527 return fp->jited && bpf_jit_is_ebpf(); 1528 } 1529 1530 static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) 1531 { 1532 /* These are the prerequisites, should someone ever have the 1533 * idea to call blinding outside of them, we make sure to 1534 * bail out. 1535 */ 1536 if (!bpf_jit_is_ebpf()) 1537 return false; 1538 if (!prog->jit_requested) 1539 return false; 1540 if (!bpf_jit_harden) 1541 return false; 1542 if (bpf_jit_harden == 1 && bpf_token_capable(prog->aux->token, CAP_BPF)) 1543 return false; 1544 1545 return true; 1546 } 1547 1548 static inline bool bpf_jit_kallsyms_enabled(void) 1549 { 1550 /* There are a couple of corner cases where kallsyms should 1551 * not be enabled f.e. on hardening. 1552 */ 1553 if (bpf_jit_harden) 1554 return false; 1555 if (!bpf_jit_kallsyms) 1556 return false; 1557 if (bpf_jit_kallsyms == 1) 1558 return true; 1559 1560 return false; 1561 } 1562 1563 int bpf_address_lookup(unsigned long addr, unsigned long *size, 1564 unsigned long *off, char *sym); 1565 bool is_bpf_text_address(unsigned long addr); 1566 int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 1567 char *sym); 1568 struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); 1569 1570 void bpf_prog_kallsyms_add(struct bpf_prog *fp); 1571 void bpf_prog_kallsyms_del(struct bpf_prog *fp); 1572 1573 #else /* CONFIG_BPF_JIT */ 1574 1575 static inline bool ebpf_jit_enabled(void) 1576 { 1577 return false; 1578 } 1579 1580 static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) 1581 { 1582 return false; 1583 } 1584 1585 static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) 1586 { 1587 return false; 1588 } 1589 1590 static inline int 1591 bpf_jit_add_poke_descriptor(struct bpf_prog *prog, 1592 struct bpf_jit_poke_descriptor *poke) 1593 { 1594 return -ENOTSUPP; 1595 } 1596 1597 static inline void bpf_jit_free(struct bpf_prog *fp) 1598 { 1599 bpf_prog_unlock_free(fp); 1600 } 1601 1602 static inline bool bpf_jit_kallsyms_enabled(void) 1603 { 1604 return false; 1605 } 1606 1607 static inline int 1608 bpf_address_lookup(unsigned long addr, unsigned long *size, 1609 unsigned long *off, char *sym) 1610 { 1611 return 0; 1612 } 1613 1614 static inline bool is_bpf_text_address(unsigned long addr) 1615 { 1616 return false; 1617 } 1618 1619 static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value, 1620 char *type, char *sym) 1621 { 1622 return -ERANGE; 1623 } 1624 1625 static inline struct bpf_prog *bpf_prog_ksym_find(unsigned long addr) 1626 { 1627 return NULL; 1628 } 1629 1630 static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) 1631 { 1632 } 1633 1634 static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) 1635 { 1636 } 1637 1638 static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) 1639 { 1640 return false; 1641 } 1642 1643 static inline 1644 struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog) 1645 { 1646 return prog; 1647 } 1648 1649 static inline void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) 1650 { 1651 } 1652 #endif /* CONFIG_BPF_JIT */ 1653 1654 void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); 1655 1656 #define BPF_ANC BIT(15) 1657 1658 static inline bool bpf_needs_clear_a(const struct sock_filter *first) 1659 { 1660 switch (first->code) { 1661 case BPF_RET | BPF_K: 1662 case BPF_LD | BPF_W | BPF_LEN: 1663 return false; 1664 1665 case BPF_LD | BPF_W | BPF_ABS: 1666 case BPF_LD | BPF_H | BPF_ABS: 1667 case BPF_LD | BPF_B | BPF_ABS: 1668 if (first->k == SKF_AD_OFF + SKF_AD_ALU_XOR_X) 1669 return true; 1670 return false; 1671 1672 default: 1673 return true; 1674 } 1675 } 1676 1677 static inline u16 bpf_anc_helper(const struct sock_filter *ftest) 1678 { 1679 BUG_ON(ftest->code & BPF_ANC); 1680 1681 switch (ftest->code) { 1682 case BPF_LD | BPF_W | BPF_ABS: 1683 case BPF_LD | BPF_H | BPF_ABS: 1684 case BPF_LD | BPF_B | BPF_ABS: 1685 #define BPF_ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE: \ 1686 return BPF_ANC | SKF_AD_##CODE 1687 switch (ftest->k) { 1688 BPF_ANCILLARY(PROTOCOL); 1689 BPF_ANCILLARY(PKTTYPE); 1690 BPF_ANCILLARY(IFINDEX); 1691 BPF_ANCILLARY(NLATTR); 1692 BPF_ANCILLARY(NLATTR_NEST); 1693 BPF_ANCILLARY(MARK); 1694 BPF_ANCILLARY(QUEUE); 1695 BPF_ANCILLARY(HATYPE); 1696 BPF_ANCILLARY(RXHASH); 1697 BPF_ANCILLARY(CPU); 1698 BPF_ANCILLARY(ALU_XOR_X); 1699 BPF_ANCILLARY(VLAN_TAG); 1700 BPF_ANCILLARY(VLAN_TAG_PRESENT); 1701 BPF_ANCILLARY(PAY_OFFSET); 1702 BPF_ANCILLARY(RANDOM); 1703 BPF_ANCILLARY(VLAN_TPID); 1704 } 1705 fallthrough; 1706 default: 1707 return ftest->code; 1708 } 1709 } 1710 1711 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, 1712 int k, unsigned int size); 1713 1714 static inline int bpf_tell_extensions(void) 1715 { 1716 return SKF_AD_MAX; 1717 } 1718 1719 struct bpf_sock_addr_kern { 1720 struct sock *sk; 1721 struct sockaddr_unsized *uaddr; 1722 /* Temporary "register" to make indirect stores to nested structures 1723 * defined above. We need three registers to make such a store, but 1724 * only two (src and dst) are available at convert_ctx_access time 1725 */ 1726 u64 tmp_reg; 1727 void *t_ctx; /* Attach type specific context. */ 1728 u32 uaddrlen; 1729 }; 1730 1731 struct bpf_sock_ops_kern { 1732 struct sock *sk; 1733 union { 1734 u32 args[4]; 1735 u32 reply; 1736 u32 replylong[4]; 1737 }; 1738 struct sk_buff *syn_skb; 1739 struct sk_buff *skb; 1740 void *skb_data_end; 1741 u8 op; 1742 u8 is_fullsock; 1743 u8 is_locked_tcp_sock; 1744 u8 remaining_opt_len; 1745 u64 temp; /* temp and everything after is not 1746 * initialized to 0 before calling 1747 * the BPF program. New fields that 1748 * should be initialized to 0 should 1749 * be inserted before temp. 1750 * temp is scratch storage used by 1751 * sock_ops_convert_ctx_access 1752 * as temporary storage of a register. 1753 */ 1754 }; 1755 1756 struct bpf_sysctl_kern { 1757 struct ctl_table_header *head; 1758 const struct ctl_table *table; 1759 void *cur_val; 1760 size_t cur_len; 1761 void *new_val; 1762 size_t new_len; 1763 int new_updated; 1764 int write; 1765 loff_t *ppos; 1766 /* Temporary "register" for indirect stores to ppos. */ 1767 u64 tmp_reg; 1768 }; 1769 1770 #define BPF_SOCKOPT_KERN_BUF_SIZE 32 1771 struct bpf_sockopt_buf { 1772 u8 data[BPF_SOCKOPT_KERN_BUF_SIZE]; 1773 }; 1774 1775 struct bpf_sockopt_kern { 1776 struct sock *sk; 1777 u8 *optval; 1778 u8 *optval_end; 1779 s32 level; 1780 s32 optname; 1781 s32 optlen; 1782 /* for retval in struct bpf_cg_run_ctx */ 1783 struct task_struct *current_task; 1784 /* Temporary "register" for indirect stores to ppos. */ 1785 u64 tmp_reg; 1786 }; 1787 1788 int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len); 1789 1790 struct bpf_sk_lookup_kern { 1791 u16 family; 1792 u16 protocol; 1793 __be16 sport; 1794 u16 dport; 1795 struct { 1796 __be32 saddr; 1797 __be32 daddr; 1798 } v4; 1799 struct { 1800 const struct in6_addr *saddr; 1801 const struct in6_addr *daddr; 1802 } v6; 1803 struct sock *selected_sk; 1804 u32 ingress_ifindex; 1805 bool no_reuseport; 1806 }; 1807 1808 extern struct static_key_false bpf_sk_lookup_enabled; 1809 1810 /* Runners for BPF_SK_LOOKUP programs to invoke on socket lookup. 1811 * 1812 * Allowed return values for a BPF SK_LOOKUP program are SK_PASS and 1813 * SK_DROP. Their meaning is as follows: 1814 * 1815 * SK_PASS && ctx.selected_sk != NULL: use selected_sk as lookup result 1816 * SK_PASS && ctx.selected_sk == NULL: continue to htable-based socket lookup 1817 * SK_DROP : terminate lookup with -ECONNREFUSED 1818 * 1819 * This macro aggregates return values and selected sockets from 1820 * multiple BPF programs according to following rules in order: 1821 * 1822 * 1. If any program returned SK_PASS and a non-NULL ctx.selected_sk, 1823 * macro result is SK_PASS and last ctx.selected_sk is used. 1824 * 2. If any program returned SK_DROP return value, 1825 * macro result is SK_DROP. 1826 * 3. Otherwise result is SK_PASS and ctx.selected_sk is NULL. 1827 * 1828 * Caller must ensure that the prog array is non-NULL, and that the 1829 * array as well as the programs it contains remain valid. 1830 */ 1831 #define BPF_PROG_SK_LOOKUP_RUN_ARRAY(array, ctx, func) \ 1832 ({ \ 1833 struct bpf_sk_lookup_kern *_ctx = &(ctx); \ 1834 struct bpf_prog_array_item *_item; \ 1835 struct sock *_selected_sk = NULL; \ 1836 bool _no_reuseport = false; \ 1837 struct bpf_prog *_prog; \ 1838 bool _all_pass = true; \ 1839 u32 _ret; \ 1840 \ 1841 migrate_disable(); \ 1842 _item = &(array)->items[0]; \ 1843 while ((_prog = READ_ONCE(_item->prog))) { \ 1844 /* restore most recent selection */ \ 1845 _ctx->selected_sk = _selected_sk; \ 1846 _ctx->no_reuseport = _no_reuseport; \ 1847 \ 1848 _ret = func(_prog, _ctx); \ 1849 if (_ret == SK_PASS && _ctx->selected_sk) { \ 1850 /* remember last non-NULL socket */ \ 1851 _selected_sk = _ctx->selected_sk; \ 1852 _no_reuseport = _ctx->no_reuseport; \ 1853 } else if (_ret == SK_DROP && _all_pass) { \ 1854 _all_pass = false; \ 1855 } \ 1856 _item++; \ 1857 } \ 1858 _ctx->selected_sk = _selected_sk; \ 1859 _ctx->no_reuseport = _no_reuseport; \ 1860 migrate_enable(); \ 1861 _all_pass || _selected_sk ? SK_PASS : SK_DROP; \ 1862 }) 1863 1864 static inline bool bpf_sk_lookup_run_v4(const struct net *net, int protocol, 1865 const __be32 saddr, const __be16 sport, 1866 const __be32 daddr, const u16 dport, 1867 const int ifindex, struct sock **psk) 1868 { 1869 struct bpf_prog_array *run_array; 1870 struct sock *selected_sk = NULL; 1871 bool no_reuseport = false; 1872 1873 rcu_read_lock(); 1874 run_array = rcu_dereference(net->bpf.run_array[NETNS_BPF_SK_LOOKUP]); 1875 if (run_array) { 1876 struct bpf_sk_lookup_kern ctx = { 1877 .family = AF_INET, 1878 .protocol = protocol, 1879 .v4.saddr = saddr, 1880 .v4.daddr = daddr, 1881 .sport = sport, 1882 .dport = dport, 1883 .ingress_ifindex = ifindex, 1884 }; 1885 u32 act; 1886 1887 act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, bpf_prog_run); 1888 if (act == SK_PASS) { 1889 selected_sk = ctx.selected_sk; 1890 no_reuseport = ctx.no_reuseport; 1891 } else { 1892 selected_sk = ERR_PTR(-ECONNREFUSED); 1893 } 1894 } 1895 rcu_read_unlock(); 1896 *psk = selected_sk; 1897 return no_reuseport; 1898 } 1899 1900 #if IS_ENABLED(CONFIG_IPV6) 1901 static inline bool bpf_sk_lookup_run_v6(const struct net *net, int protocol, 1902 const struct in6_addr *saddr, 1903 const __be16 sport, 1904 const struct in6_addr *daddr, 1905 const u16 dport, 1906 const int ifindex, struct sock **psk) 1907 { 1908 struct bpf_prog_array *run_array; 1909 struct sock *selected_sk = NULL; 1910 bool no_reuseport = false; 1911 1912 rcu_read_lock(); 1913 run_array = rcu_dereference(net->bpf.run_array[NETNS_BPF_SK_LOOKUP]); 1914 if (run_array) { 1915 struct bpf_sk_lookup_kern ctx = { 1916 .family = AF_INET6, 1917 .protocol = protocol, 1918 .v6.saddr = saddr, 1919 .v6.daddr = daddr, 1920 .sport = sport, 1921 .dport = dport, 1922 .ingress_ifindex = ifindex, 1923 }; 1924 u32 act; 1925 1926 act = BPF_PROG_SK_LOOKUP_RUN_ARRAY(run_array, ctx, bpf_prog_run); 1927 if (act == SK_PASS) { 1928 selected_sk = ctx.selected_sk; 1929 no_reuseport = ctx.no_reuseport; 1930 } else { 1931 selected_sk = ERR_PTR(-ECONNREFUSED); 1932 } 1933 } 1934 rcu_read_unlock(); 1935 *psk = selected_sk; 1936 return no_reuseport; 1937 } 1938 #endif /* IS_ENABLED(CONFIG_IPV6) */ 1939 1940 static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 index, 1941 u64 flags, const u64 flag_mask, 1942 void *lookup_elem(struct bpf_map *map, u32 key)) 1943 { 1944 struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); 1945 const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX; 1946 1947 /* Lower bits of the flags are used as return code on lookup failure */ 1948 if (unlikely(flags & ~(action_mask | flag_mask))) 1949 return XDP_ABORTED; 1950 1951 ri->tgt_value = lookup_elem(map, index); 1952 if (unlikely(!ri->tgt_value) && !(flags & BPF_F_BROADCAST)) { 1953 /* If the lookup fails we want to clear out the state in the 1954 * redirect_info struct completely, so that if an eBPF program 1955 * performs multiple lookups, the last one always takes 1956 * precedence. 1957 */ 1958 ri->map_id = INT_MAX; /* Valid map id idr range: [1,INT_MAX[ */ 1959 ri->map_type = BPF_MAP_TYPE_UNSPEC; 1960 return flags & action_mask; 1961 } 1962 1963 ri->tgt_index = index; 1964 ri->map_id = map->id; 1965 ri->map_type = map->map_type; 1966 1967 if (flags & BPF_F_BROADCAST) { 1968 WRITE_ONCE(ri->map, map); 1969 ri->flags = flags; 1970 } else { 1971 WRITE_ONCE(ri->map, NULL); 1972 ri->flags = 0; 1973 } 1974 1975 return XDP_REDIRECT; 1976 } 1977 1978 int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, 1979 void *search_res, u32 len, u64 flags); 1980 int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, 1981 const void *from, u32 len, u64 flags); 1982 1983 #ifdef CONFIG_NET 1984 int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); 1985 int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, 1986 u32 len, u64 flags); 1987 int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len); 1988 int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len); 1989 void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len); 1990 void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off, 1991 void *buf, unsigned long len, bool flush); 1992 int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset, 1993 const void *from, u32 len, u64 flags); 1994 void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset); 1995 #else /* CONFIG_NET */ 1996 static inline int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, 1997 void *to, u32 len) 1998 { 1999 return -EOPNOTSUPP; 2000 } 2001 2002 static inline int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, 2003 const void *from, u32 len, u64 flags) 2004 { 2005 return -EOPNOTSUPP; 2006 } 2007 2008 static inline int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, 2009 void *buf, u32 len) 2010 { 2011 return -EOPNOTSUPP; 2012 } 2013 2014 static inline int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, 2015 void *buf, u32 len) 2016 { 2017 return -EOPNOTSUPP; 2018 } 2019 2020 static inline void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len) 2021 { 2022 return NULL; 2023 } 2024 2025 static inline void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off, void *buf, 2026 unsigned long len, bool flush) 2027 { 2028 } 2029 2030 static inline int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset, 2031 const void *from, u32 len, 2032 u64 flags) 2033 { 2034 return -EOPNOTSUPP; 2035 } 2036 2037 static inline void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset) 2038 { 2039 return ERR_PTR(-EOPNOTSUPP); 2040 } 2041 #endif /* CONFIG_NET */ 2042 2043 #endif /* __LINUX_FILTER_H__ */ 2044