1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * bpf_jit_comp64.c: eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <asm/kprobes.h> 17 #include <linux/bpf.h> 18 #include <asm/security_features.h> 19 20 #include "bpf_jit.h" 21 22 /* 23 * Stack layout with frame: 24 * Layout when setting up our own stack frame. 25 * Note: r1 at bottom, component offsets positive wrt r1. 26 * Ensure the top half (upto local_tmp_var) stays consistent 27 * with our redzone usage. 28 * 29 * tail_call_info - stores tailcall count value in main program's 30 * frame, stores reference to tail_call_info of 31 * main's frame in sub-prog's frame. 32 * 33 * [ prev sp ] <------------- 34 * [ tail_call_info ] 8 | 35 * [ nv gpr save area ] (6 * 8) | 36 * [ addl. nv gpr save area] (12 * 8) | <--- exception boundary/callback program 37 * [ local_tmp_var ] 24 | 38 * fp (r31) --> [ ebpf stack space ] upto 512 | 39 * [ frame header ] 32/112 | 40 * sp (r1) ---> [ stack pointer ] -------------- 41 * 42 * Additional (12 * 8) in 'nv gpr save area' only in case of 43 * exception boundary/callback. 44 */ 45 46 /* BPF non-volatile registers save area size */ 47 #define BPF_PPC_STACK_SAVE (6 * 8) 48 49 /* for bpf JIT code internal usage */ 50 #define BPF_PPC_STACK_LOCALS 24 51 /* 52 * for additional non volatile registers(r14-r25) to be saved 53 * at exception boundary 54 */ 55 #define BPF_PPC_EXC_STACK_SAVE (12 * 8) 56 57 /* stack frame excluding BPF stack, ensure this is quadword aligned */ 58 #define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \ 59 BPF_PPC_STACK_LOCALS + \ 60 BPF_PPC_STACK_SAVE + \ 61 BPF_PPC_TAILCALL) 62 63 /* 64 * same as BPF_PPC_STACKFRAME with save area for additional 65 * non volatile registers saved at exception boundary. 66 * This is quad-word aligned. 67 */ 68 #define BPF_PPC_EXC_STACKFRAME (BPF_PPC_STACKFRAME + BPF_PPC_EXC_STACK_SAVE) 69 70 /* BPF register usage */ 71 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 72 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 73 #define ARENA_VM_START (MAX_BPF_JIT_REG + 2) 74 75 /* BPF to ppc register mappings */ 76 void bpf_jit_init_reg_mapping(struct codegen_context *ctx) 77 { 78 /* function return value */ 79 ctx->b2p[BPF_REG_0] = _R8; 80 /* function arguments */ 81 ctx->b2p[BPF_REG_1] = _R3; 82 ctx->b2p[BPF_REG_2] = _R4; 83 ctx->b2p[BPF_REG_3] = _R5; 84 ctx->b2p[BPF_REG_4] = _R6; 85 ctx->b2p[BPF_REG_5] = _R7; 86 /* non volatile registers */ 87 ctx->b2p[BPF_REG_6] = _R27; 88 ctx->b2p[BPF_REG_7] = _R28; 89 ctx->b2p[BPF_REG_8] = _R29; 90 ctx->b2p[BPF_REG_9] = _R30; 91 /* frame pointer aka BPF_REG_10 */ 92 ctx->b2p[BPF_REG_FP] = _R31; 93 /* eBPF jit internal registers */ 94 ctx->b2p[BPF_REG_AX] = _R12; 95 ctx->b2p[TMP_REG_1] = _R9; 96 ctx->b2p[TMP_REG_2] = _R10; 97 /* non volatile register for kern_vm_start address */ 98 ctx->b2p[ARENA_VM_START] = _R26; 99 } 100 101 /* PPC NVR range -- update this if we ever use NVRs below r26 */ 102 #define BPF_PPC_NVR_MIN _R26 103 104 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 105 { 106 /* 107 * We only need a stack frame if: 108 * - we call other functions (kernel helpers), or 109 * - the bpf program uses its stack area 110 * The latter condition is deduced from the usage of BPF_REG_FP 111 * 112 * bpf_throw() leads to exception callback from a BPF (sub)program. 113 * The (sub)program is always marked as SEEN_FUNC, creating a stack 114 * frame. The exception callback uses the frame of the exception 115 * boundary, so the exception boundary program must have a frame. 116 */ 117 return ctx->seen & SEEN_FUNC || 118 bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)) || 119 ctx->exception_cb || 120 ctx->exception_boundary; 121 } 122 123 /* 124 * Stack layout with redzone: 125 * When not setting up our own stackframe, the redzone (288 bytes) usage is: 126 * Note: r1 from prev frame. Component offset negative wrt r1. 127 * 128 * [ prev sp ] <------------- 129 * [ ... ] | 130 * sp (r1) ---> [ stack pointer ] -------------- 131 * [ tail_call_info ] 8 132 * [ nv gpr save area ] (6 * 8) 133 * [ addl. nv gpr save area] (12 * 8) <--- exception boundary/callback program 134 * [ local_tmp_var ] 24 135 * [ unused red zone ] 224 136 * 137 * Additional (12 * 8) in 'nv gpr save area' only in case of 138 * exception boundary/callback. 139 */ 140 static int bpf_jit_stack_local(struct codegen_context *ctx) 141 { 142 if (bpf_has_stack_frame(ctx)) { 143 /* Stack layout with frame */ 144 return STACK_FRAME_MIN_SIZE + ctx->stack_size; 145 } else { 146 /* Stack layout with redzone */ 147 return -(BPF_PPC_TAILCALL 148 +BPF_PPC_STACK_SAVE 149 +(ctx->exception_boundary || ctx->exception_cb ? 150 BPF_PPC_EXC_STACK_SAVE : 0) 151 +BPF_PPC_STACK_LOCALS 152 ); 153 } 154 } 155 156 static int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx) 157 { 158 return bpf_jit_stack_local(ctx) + BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE; 159 } 160 161 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 162 { 163 int min_valid_nvreg = BPF_PPC_NVR_MIN; 164 /* Default frame size for all cases except exception boundary */ 165 int frame_nvr_size = BPF_PPC_STACKFRAME; 166 167 /* Consider all nv regs for handling exceptions */ 168 if (ctx->exception_boundary || ctx->exception_cb) { 169 min_valid_nvreg = _R14; 170 frame_nvr_size = BPF_PPC_EXC_STACKFRAME; 171 } 172 173 if (reg >= min_valid_nvreg && reg < 32) 174 return (bpf_has_stack_frame(ctx) ? 175 (frame_nvr_size + ctx->stack_size) : 0) 176 - (8 * (32 - reg)) - BPF_PPC_TAILCALL; 177 178 pr_err("BPF JIT is asking about unknown registers"); 179 BUG(); 180 } 181 182 void bpf_jit_realloc_regs(struct codegen_context *ctx) 183 { 184 } 185 186 /* 187 * For exception boundary & exception_cb progs: 188 * return increased size to accommodate additional NVRs. 189 */ 190 static int bpf_jit_stack_size(struct codegen_context *ctx) 191 { 192 return ctx->exception_boundary || ctx->exception_cb ? 193 BPF_PPC_EXC_STACKFRAME : 194 BPF_PPC_STACKFRAME; 195 } 196 197 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 198 { 199 int i; 200 201 /* Instruction for trampoline attach */ 202 EMIT(PPC_RAW_NOP()); 203 204 #ifndef CONFIG_PPC_KERNEL_PCREL 205 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) 206 EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc))); 207 #endif 208 209 /* 210 * Tail call count(tcc) is saved & updated only in main 211 * program's frame and the address of tcc in main program's 212 * frame (tcc_ptr) is saved in subprogs frame. 213 * 214 * Offset of tail_call_info on any frame will be interpreted 215 * as either tcc_ptr or tcc value depending on whether it is 216 * greater than MAX_TAIL_CALL_CNT or not. 217 */ 218 if (!ctx->is_subprog) { 219 EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); 220 /* this goes in the redzone */ 221 EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); 222 } else if (!ctx->exception_cb) { 223 /* 224 * Tailcall jitting for non exception_cb progs only. 225 * exception_cb won't require tail_call_info to be setup. 226 * 227 * tail_call_info interpretation logic: 228 * 229 * if tail_call_info < MAX_TAIL_CALL_CNT 230 * main prog calling first subprog -> copy reference 231 * else 232 * subsequent subprog calling another subprog -> directly copy content 233 */ 234 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, 0)); 235 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), -(BPF_PPC_TAILCALL))); 236 EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); 237 PPC_BCC_CONST_SHORT(COND_GT, 8); 238 EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 239 -(BPF_PPC_TAILCALL))); 240 EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); 241 } 242 243 if (bpf_has_stack_frame(ctx) && !ctx->exception_cb) { 244 /* 245 * We need a stack frame, but we don't necessarily need to 246 * save/restore LR unless we call other functions 247 */ 248 if (ctx->seen & SEEN_FUNC) { 249 EMIT(PPC_RAW_MFLR(_R0)); 250 EMIT(PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF)); 251 } 252 253 EMIT(PPC_RAW_STDU(_R1, _R1, 254 -(bpf_jit_stack_size(ctx) + ctx->stack_size))); 255 } 256 257 /* 258 * Program acting as exception boundary pushes R14..R25 in addition to 259 * BPF callee-saved non volatile registers. Exception callback uses 260 * the boundary program's stack frame, recover additionally saved 261 * registers in epilogue of exception callback. 262 */ 263 if (ctx->exception_boundary) { 264 for (i = _R14; i <= _R25; i++) 265 EMIT(PPC_RAW_STD(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 266 } 267 268 if (!ctx->exception_cb) { 269 /* 270 * Back up non-volatile regs -- BPF registers 6-10 271 * If we haven't created our own stack frame, we save these 272 * in the protected zone below the previous stack frame 273 */ 274 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 275 if (ctx->exception_boundary || bpf_is_seen_register(ctx, bpf_to_ppc(i))) 276 EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, 277 bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); 278 279 if (ctx->exception_boundary || ctx->arena_vm_start) 280 EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, 281 bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); 282 } else { 283 /* 284 * Exception callback receives Frame Pointer of boundary 285 * program(main prog) as third arg 286 */ 287 EMIT(PPC_RAW_MR(_R1, _R5)); 288 /* 289 * Exception callback reuses the stack frame of exception boundary. 290 * But BPF stack depth of exception callback and exception boundary 291 * don't have to be same. If BPF stack depth is different, adjust the 292 * stack frame size considering BPF stack depth of exception callback. 293 * The non-volatile register save area remains unchanged. These non- 294 * volatile registers are restored in exception callback's epilogue. 295 */ 296 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R5, 0)); 297 EMIT(PPC_RAW_SUB(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_1), _R1)); 298 EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), 299 -BPF_PPC_EXC_STACKFRAME)); 300 EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), ctx->stack_size)); 301 PPC_BCC_CONST_SHORT(COND_EQ, 12); 302 EMIT(PPC_RAW_MR(_R1, bpf_to_ppc(TMP_REG_1))); 303 EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_EXC_STACKFRAME + ctx->stack_size))); 304 } 305 306 /* 307 * Exception_cb not restricted from using stack area or arena. 308 * Setup frame pointer to point to the bpf stack area 309 */ 310 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) 311 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, 312 STACK_FRAME_MIN_SIZE + ctx->stack_size)); 313 314 if (ctx->arena_vm_start) 315 PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start); 316 } 317 318 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 319 { 320 int i; 321 322 /* Restore NVRs */ 323 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 324 if (ctx->exception_cb || bpf_is_seen_register(ctx, bpf_to_ppc(i))) 325 EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); 326 327 if (ctx->exception_cb || ctx->arena_vm_start) 328 EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1, 329 bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); 330 331 if (ctx->exception_cb) { 332 /* 333 * Recover additionally saved non volatile registers from stack 334 * frame of exception boundary program. 335 */ 336 for (i = _R14; i <= _R25; i++) 337 EMIT(PPC_RAW_LD(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 338 } 339 340 /* Tear down our stack frame */ 341 if (bpf_has_stack_frame(ctx)) { 342 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_jit_stack_size(ctx) + ctx->stack_size)); 343 344 if (ctx->seen & SEEN_FUNC || ctx->exception_cb) { 345 EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF)); 346 EMIT(PPC_RAW_MTLR(_R0)); 347 } 348 } 349 } 350 351 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 352 { 353 bpf_jit_emit_common_epilogue(image, ctx); 354 355 /* Move result to r3 */ 356 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0))); 357 358 EMIT(PPC_RAW_BLR()); 359 360 bpf_jit_build_fentry_stubs(image, ctx); 361 } 362 363 /* 364 * arch_bpf_stack_walk() - BPF stack walker for PowerPC 365 * 366 * Based on arch_stack_walk() from stacktrace.c. 367 * PowerPC uses stack frames rather than stack pointers. See [1] for 368 * the equivalence between frame pointers and stack pointers. 369 * Additional reference at [2]. 370 * TODO: refactor with arch_stack_walk() 371 * 372 * [1]: https://lore.kernel.org/all/20200220115141.2707-1-mpe@ellerman.id.au/ 373 * [2]: https://lore.kernel.org/bpf/20260122211854.5508-5-adubey@linux.ibm.com/ 374 */ 375 376 void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), void *cookie) 377 { 378 // callback processing always in current context 379 unsigned long sp = current_stack_frame(); 380 381 for (;;) { 382 unsigned long *stack = (unsigned long *) sp; 383 unsigned long ip; 384 385 if (!validate_sp(sp, current)) 386 return; 387 388 ip = stack[STACK_FRAME_LR_SAVE]; 389 if (!ip) 390 break; 391 392 /* 393 * consume_fn common code expects stack pointer in third 394 * argument. There is no sp in ppc64, rather pass frame 395 * pointer(named sp here). 396 */ 397 if (ip && !consume_fn(cookie, ip, sp, sp)) 398 break; 399 400 sp = stack[0]; 401 } 402 } 403 404 int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) 405 { 406 unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0; 407 long reladdr; 408 409 /* bpf to bpf call, func is not known in the initial pass. Emit 5 nops as a placeholder */ 410 if (!func) { 411 for (int i = 0; i < 5; i++) 412 EMIT(PPC_RAW_NOP()); 413 /* elfv1 needs an additional instruction to load addr from descriptor */ 414 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) 415 EMIT(PPC_RAW_NOP()); 416 EMIT(PPC_RAW_MTCTR(_R12)); 417 EMIT(PPC_RAW_BCTRL()); 418 return 0; 419 } 420 421 #ifdef CONFIG_PPC_KERNEL_PCREL 422 reladdr = func_addr - local_paca->kernelbase; 423 424 /* 425 * If fimage is NULL (the initial pass to find image size), 426 * account for the maximum no. of instructions possible. 427 */ 428 if (!fimage) { 429 ctx->idx += 7; 430 return 0; 431 } else if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) { 432 EMIT(PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase))); 433 /* Align for subsequent prefix instruction */ 434 if (!IS_ALIGNED((unsigned long)fimage + CTX_NIA(ctx), 8)) 435 EMIT(PPC_RAW_NOP()); 436 /* paddi r12,r12,addr */ 437 EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(0) | IMM_H18(reladdr)); 438 EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12) | IMM_L(reladdr)); 439 } else { 440 unsigned long pc = (unsigned long)fimage + CTX_NIA(ctx); 441 bool alignment_needed = !IS_ALIGNED(pc, 8); 442 443 reladdr = func_addr - (alignment_needed ? pc + 4 : pc); 444 445 if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) { 446 if (alignment_needed) 447 EMIT(PPC_RAW_NOP()); 448 /* pla r12,addr */ 449 EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(1) | IMM_H18(reladdr)); 450 EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | IMM_L(reladdr)); 451 } else { 452 /* We can clobber r12 */ 453 PPC_LI64(_R12, func); 454 } 455 } 456 EMIT(PPC_RAW_MTCTR(_R12)); 457 EMIT(PPC_RAW_BCTRL()); 458 #else 459 if (core_kernel_text(func_addr)) { 460 reladdr = func_addr - kernel_toc_addr(); 461 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 462 pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func); 463 return -ERANGE; 464 } 465 466 EMIT(PPC_RAW_ADDIS(_R12, _R2, PPC_HA(reladdr))); 467 EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr))); 468 EMIT(PPC_RAW_MTCTR(_R12)); 469 EMIT(PPC_RAW_BCTRL()); 470 } else { 471 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) { 472 /* func points to the function descriptor */ 473 PPC_LI64(bpf_to_ppc(TMP_REG_2), func); 474 /* Load actual entry point from function descriptor */ 475 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0)); 476 /* ... and move it to CTR */ 477 EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1))); 478 /* 479 * Load TOC from function descriptor at offset 8. 480 * We can clobber r2 since we get called through a 481 * function pointer (so caller will save/restore r2). 482 */ 483 if (is_module_text_address(func_addr)) 484 EMIT(PPC_RAW_LD(_R2, bpf_to_ppc(TMP_REG_2), 8)); 485 } else { 486 PPC_LI64(_R12, func); 487 EMIT(PPC_RAW_MTCTR(_R12)); 488 } 489 EMIT(PPC_RAW_BCTRL()); 490 /* 491 * Load r2 with kernel TOC as kernel TOC is used if function address falls 492 * within core kernel text. 493 */ 494 if (is_module_text_address(func_addr)) 495 EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc))); 496 } 497 #endif 498 499 return 0; 500 } 501 502 static int zero_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) 503 { 504 switch (size) { 505 case 1: 506 /* zero-extend 8 bits into 64 bits */ 507 EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 56)); 508 return 0; 509 case 2: 510 /* zero-extend 16 bits into 64 bits */ 511 EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 48)); 512 return 0; 513 case 4: 514 /* zero-extend 32 bits into 64 bits */ 515 EMIT(PPC_RAW_RLDICL(dst_reg, src_reg, 0, 32)); 516 fallthrough; 517 case 8: 518 /* Nothing to do */ 519 return 0; 520 default: 521 return -1; 522 } 523 } 524 525 static int sign_extend(u32 *image, struct codegen_context *ctx, u32 src_reg, u32 dst_reg, u32 size) 526 { 527 switch (size) { 528 case 1: 529 /* sign-extend 8 bits into 64 bits */ 530 EMIT(PPC_RAW_EXTSB(dst_reg, src_reg)); 531 return 0; 532 case 2: 533 /* sign-extend 16 bits into 64 bits */ 534 EMIT(PPC_RAW_EXTSH(dst_reg, src_reg)); 535 return 0; 536 case 4: 537 /* sign-extend 32 bits into 64 bits */ 538 EMIT(PPC_RAW_EXTSW(dst_reg, src_reg)); 539 fallthrough; 540 case 8: 541 /* Nothing to do */ 542 return 0; 543 default: 544 return -1; 545 } 546 } 547 548 /* 549 * Handle powerpc ABI expectations from caller: 550 * - Unsigned arguments are zero-extended. 551 * - Signed arguments are sign-extended. 552 */ 553 static int prepare_for_kfunc_call(const struct bpf_prog *fp, u32 *image, 554 struct codegen_context *ctx, 555 const struct bpf_insn *insn) 556 { 557 const struct btf_func_model *m = bpf_jit_find_kfunc_model(fp, insn); 558 int i; 559 560 if (!m) 561 return -1; 562 563 for (i = 0; i < m->nr_args; i++) { 564 /* Note that BPF ABI only allows up to 5 args for kfuncs */ 565 u32 reg = bpf_to_ppc(BPF_REG_1 + i), size = m->arg_size[i]; 566 567 if (!(m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG)) { 568 if (zero_extend(image, ctx, reg, reg, size)) 569 return -1; 570 } else { 571 if (sign_extend(image, ctx, reg, reg, size)) 572 return -1; 573 } 574 } 575 576 return 0; 577 } 578 579 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 580 { 581 /* 582 * By now, the eBPF program has already setup parameters in r3, r4 and r5 583 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 584 * r4/BPF_REG_2 - pointer to bpf_array 585 * r5/BPF_REG_3 - index in bpf_array 586 */ 587 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2); 588 int b2p_index = bpf_to_ppc(BPF_REG_3); 589 int bpf_tailcall_prologue_size = 12; 590 591 if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL) && IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) 592 bpf_tailcall_prologue_size += 4; /* skip past the toc load */ 593 594 /* 595 * if (index >= array->map.max_entries) 596 * goto out; 597 */ 598 EMIT(PPC_RAW_LWZ(bpf_to_ppc(TMP_REG_1), b2p_bpf_array, offsetof(struct bpf_array, map.max_entries))); 599 EMIT(PPC_RAW_RLWINM(b2p_index, b2p_index, 0, 0, 31)); 600 EMIT(PPC_RAW_CMPLW(b2p_index, bpf_to_ppc(TMP_REG_1))); 601 PPC_BCC_SHORT(COND_GE, out); 602 603 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); 604 EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); 605 PPC_BCC_CONST_SHORT(COND_LE, 8); 606 607 /* dereference TMP_REG_1 */ 608 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 0)); 609 610 /* 611 * if (tail_call_info == MAX_TAIL_CALL_CNT) 612 * goto out; 613 */ 614 EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); 615 PPC_BCC_SHORT(COND_EQ, out); 616 617 /* 618 * tail_call_info++; <- Actual value of tcc here 619 * Writeback this updated value only if tailcall succeeds. 620 */ 621 EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1)); 622 623 /* prog = array->ptrs[index]; */ 624 EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_2), b2p_index, 8)); 625 EMIT(PPC_RAW_ADD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), b2p_bpf_array)); 626 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), 627 offsetof(struct bpf_array, ptrs))); 628 629 /* 630 * if (prog == NULL) 631 * goto out; 632 */ 633 EMIT(PPC_RAW_CMPLDI(bpf_to_ppc(TMP_REG_2), 0)); 634 PPC_BCC_SHORT(COND_EQ, out); 635 636 /* goto *(prog->bpf_func + prologue_size); */ 637 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), 638 offsetof(struct bpf_prog, bpf_func))); 639 EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), bpf_to_ppc(TMP_REG_2), 640 FUNCTION_DESCR_SIZE + bpf_tailcall_prologue_size)); 641 EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_2))); 642 643 /* 644 * Before writing updated tail_call_info, distinguish if current frame 645 * is storing a reference to tail_call_info or actual tcc value in 646 * tail_call_info. 647 */ 648 EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); 649 EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_2), MAX_TAIL_CALL_CNT)); 650 PPC_BCC_CONST_SHORT(COND_GT, 8); 651 652 /* First get address of tail_call_info */ 653 EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); 654 /* Writeback updated value to tail_call_info */ 655 EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0)); 656 657 /* tear down stack, restore NVRs, ... */ 658 bpf_jit_emit_common_epilogue(image, ctx); 659 660 EMIT(PPC_RAW_BCTR()); 661 662 /* out: */ 663 return 0; 664 } 665 666 bool bpf_jit_bypass_spec_v1(void) 667 { 668 #if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64) 669 return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && 670 security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)); 671 #else 672 return true; 673 #endif 674 } 675 676 bool bpf_jit_bypass_spec_v4(void) 677 { 678 return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && 679 security_ftr_enabled(SEC_FTR_STF_BARRIER) && 680 stf_barrier_type_get() != STF_BARRIER_NONE); 681 } 682 683 /* 684 * We spill into the redzone always, even if the bpf program has its own stackframe. 685 * Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local() 686 */ 687 void bpf_stf_barrier(void); 688 689 asm ( 690 " .global bpf_stf_barrier ;" 691 " bpf_stf_barrier: ;" 692 " std 21,-80(1) ;" 693 " std 22,-72(1) ;" 694 " sync ;" 695 " ld 21,-80(1) ;" 696 " ld 22,-72(1) ;" 697 " ori 31,31,0 ;" 698 " .rept 14 ;" 699 " b 1f ;" 700 " 1: ;" 701 " .endr ;" 702 " blr ;" 703 ); 704 705 static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx, 706 const struct bpf_insn *insn, u32 *jmp_off, 707 u32 *tmp_idx, u32 *addrp) 708 { 709 u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); 710 u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); 711 u32 size = BPF_SIZE(insn->code); 712 u32 src_reg = bpf_to_ppc(insn->src_reg); 713 u32 dst_reg = bpf_to_ppc(insn->dst_reg); 714 s32 imm = insn->imm; 715 716 u32 save_reg = tmp2_reg; 717 u32 ret_reg = src_reg; 718 u32 fixup_idx; 719 720 /* Get offset into TMP_REG_1 */ 721 EMIT(PPC_RAW_LI(tmp1_reg, insn->off)); 722 /* 723 * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync' 724 * before and after the operation. 725 * 726 * This is a requirement in the Linux Kernel Memory Model. 727 * See __cmpxchg_u64() in asm/cmpxchg.h as an example. 728 */ 729 if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP)) 730 EMIT(PPC_RAW_SYNC()); 731 732 *tmp_idx = ctx->idx; 733 734 /* load value from memory into TMP_REG_2 */ 735 if (size == BPF_DW) 736 EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0)); 737 else 738 EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0)); 739 /* Save old value in _R0 */ 740 if (imm & BPF_FETCH) 741 EMIT(PPC_RAW_MR(_R0, tmp2_reg)); 742 743 switch (imm) { 744 case BPF_ADD: 745 case BPF_ADD | BPF_FETCH: 746 EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg)); 747 break; 748 case BPF_AND: 749 case BPF_AND | BPF_FETCH: 750 EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg)); 751 break; 752 case BPF_OR: 753 case BPF_OR | BPF_FETCH: 754 EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg)); 755 break; 756 case BPF_XOR: 757 case BPF_XOR | BPF_FETCH: 758 EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg)); 759 break; 760 case BPF_CMPXCHG: 761 /* 762 * Return old value in BPF_REG_0 for BPF_CMPXCHG & 763 * in src_reg for other cases. 764 */ 765 ret_reg = bpf_to_ppc(BPF_REG_0); 766 767 /* Compare with old value in BPF_R0 */ 768 if (size == BPF_DW) 769 EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg)); 770 else 771 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg)); 772 /* Don't set if different from old value */ 773 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4); 774 fallthrough; 775 case BPF_XCHG: 776 save_reg = src_reg; 777 break; 778 default: 779 return -EOPNOTSUPP; 780 } 781 782 /* store new value */ 783 if (size == BPF_DW) 784 EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg)); 785 else 786 EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg)); 787 /* we're done if this succeeded */ 788 PPC_BCC_SHORT(COND_NE, *tmp_idx * 4); 789 fixup_idx = ctx->idx; 790 791 if (imm & BPF_FETCH) { 792 /* Emit 'sync' to enforce full ordering */ 793 if (IS_ENABLED(CONFIG_SMP)) 794 EMIT(PPC_RAW_SYNC()); 795 EMIT(PPC_RAW_MR(ret_reg, _R0)); 796 /* 797 * Skip unnecessary zero-extension for 32-bit cmpxchg. 798 * For context, see commit 39491867ace5. 799 */ 800 if (size != BPF_DW && imm == BPF_CMPXCHG && 801 insn_is_zext(insn + 1)) 802 *addrp = ctx->idx * 4; 803 } 804 805 *jmp_off = (fixup_idx - *tmp_idx) * 4; 806 807 return 0; 808 } 809 810 static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off, 811 u32 code, u32 *image) 812 { 813 u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); 814 u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); 815 816 switch (BPF_SIZE(code)) { 817 case BPF_B: 818 EMIT(PPC_RAW_STB(src_reg, tmp1_reg, off)); 819 break; 820 case BPF_H: 821 EMIT(PPC_RAW_STH(src_reg, tmp1_reg, off)); 822 break; 823 case BPF_W: 824 EMIT(PPC_RAW_STW(src_reg, tmp1_reg, off)); 825 break; 826 case BPF_DW: 827 if (off % 4) { 828 EMIT(PPC_RAW_LI(tmp2_reg, off)); 829 EMIT(PPC_RAW_STDX(src_reg, tmp1_reg, tmp2_reg)); 830 } else { 831 EMIT(PPC_RAW_STD(src_reg, tmp1_reg, off)); 832 } 833 break; 834 default: 835 return -EINVAL; 836 } 837 return 0; 838 } 839 840 static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image) 841 { 842 u32 code = insn.code; 843 u32 dst_reg = bpf_to_ppc(insn.dst_reg); 844 u32 src_reg = bpf_to_ppc(insn.src_reg); 845 u32 size = BPF_SIZE(code); 846 u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); 847 u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); 848 s16 off = insn.off; 849 s32 imm = insn.imm; 850 851 switch (imm) { 852 case BPF_LOAD_ACQ: 853 switch (size) { 854 case BPF_B: 855 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 856 break; 857 case BPF_H: 858 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 859 break; 860 case BPF_W: 861 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 862 break; 863 case BPF_DW: 864 if (off % 4) { 865 EMIT(PPC_RAW_LI(tmp1_reg, off)); 866 EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg)); 867 } else { 868 EMIT(PPC_RAW_LD(dst_reg, src_reg, off)); 869 } 870 break; 871 } 872 EMIT(PPC_RAW_LWSYNC()); 873 break; 874 case BPF_STORE_REL: 875 EMIT(PPC_RAW_LWSYNC()); 876 switch (size) { 877 case BPF_B: 878 EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 879 break; 880 case BPF_H: 881 EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 882 break; 883 case BPF_W: 884 EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 885 break; 886 case BPF_DW: 887 if (off % 4) { 888 EMIT(PPC_RAW_LI(tmp2_reg, off)); 889 EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg)); 890 } else { 891 EMIT(PPC_RAW_STD(src_reg, dst_reg, off)); 892 } 893 break; 894 } 895 break; 896 default: 897 pr_err_ratelimited("unexpected atomic load/store op code %02x\n", 898 imm); 899 return -EINVAL; 900 } 901 902 return 0; 903 } 904 905 /* Assemble the body code between the prologue & epilogue */ 906 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, 907 u32 *addrs, int pass, bool extra_pass) 908 { 909 enum stf_barrier_type stf_barrier = stf_barrier_type_get(); 910 bool sync_emitted, ori31_emitted; 911 const struct bpf_insn *insn = fp->insnsi; 912 int flen = fp->len; 913 int i, ret; 914 915 /* Start of epilogue code - will only be valid 2nd pass onwards */ 916 u32 exit_addr = addrs[flen]; 917 918 for (i = 0; i < flen; i++) { 919 u32 code = insn[i].code; 920 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg); 921 u32 src_reg = bpf_to_ppc(insn[i].src_reg); 922 u32 size = BPF_SIZE(code); 923 u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); 924 u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); 925 s16 off = insn[i].off; 926 s32 imm = insn[i].imm; 927 bool func_addr_fixed; 928 u64 func_addr; 929 u64 imm64; 930 u32 true_cond; 931 u32 tmp_idx; 932 u32 jmp_off; 933 934 /* 935 * addrs[] maps a BPF bytecode address into a real offset from 936 * the start of the body code. 937 */ 938 addrs[i] = ctx->idx * 4; 939 940 /* 941 * As an optimization, we note down which non-volatile registers 942 * are used so that we can only save/restore those in our 943 * prologue and epilogue. We do this here regardless of whether 944 * the actual BPF instruction uses src/dst registers or not 945 * (for instance, BPF_CALL does not use them). The expectation 946 * is that those instructions will have src_reg/dst_reg set to 947 * 0. Even otherwise, we just lose some prologue/epilogue 948 * optimization but everything else should work without 949 * any issues. 950 */ 951 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 952 bpf_set_seen_register(ctx, dst_reg); 953 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 954 bpf_set_seen_register(ctx, src_reg); 955 956 switch (code) { 957 /* 958 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 959 */ 960 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 961 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 962 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg)); 963 goto bpf_alu32_trunc; 964 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 965 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 966 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg)); 967 goto bpf_alu32_trunc; 968 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 969 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 970 if (!imm) { 971 goto bpf_alu32_trunc; 972 } else if (imm >= -32768 && imm < 32768) { 973 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm))); 974 } else { 975 PPC_LI32(tmp1_reg, imm); 976 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, tmp1_reg)); 977 } 978 goto bpf_alu32_trunc; 979 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 980 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 981 if (!imm) { 982 goto bpf_alu32_trunc; 983 } else if (imm > -32768 && imm <= 32768) { 984 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm))); 985 } else { 986 PPC_LI32(tmp1_reg, imm); 987 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg)); 988 } 989 goto bpf_alu32_trunc; 990 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 991 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 992 if (BPF_CLASS(code) == BPF_ALU) 993 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 994 else 995 EMIT(PPC_RAW_MULD(dst_reg, dst_reg, src_reg)); 996 goto bpf_alu32_trunc; 997 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 998 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 999 if (imm >= -32768 && imm < 32768) 1000 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm))); 1001 else { 1002 PPC_LI32(tmp1_reg, imm); 1003 if (BPF_CLASS(code) == BPF_ALU) 1004 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp1_reg)); 1005 else 1006 EMIT(PPC_RAW_MULD(dst_reg, dst_reg, tmp1_reg)); 1007 } 1008 goto bpf_alu32_trunc; 1009 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 1010 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 1011 if (BPF_OP(code) == BPF_MOD) { 1012 if (off) 1013 EMIT(PPC_RAW_DIVW(tmp1_reg, dst_reg, src_reg)); 1014 else 1015 EMIT(PPC_RAW_DIVWU(tmp1_reg, dst_reg, src_reg)); 1016 1017 EMIT(PPC_RAW_MULW(tmp1_reg, src_reg, tmp1_reg)); 1018 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg)); 1019 } else 1020 if (off) 1021 EMIT(PPC_RAW_DIVW(dst_reg, dst_reg, src_reg)); 1022 else 1023 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg)); 1024 goto bpf_alu32_trunc; 1025 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 1026 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 1027 if (BPF_OP(code) == BPF_MOD) { 1028 if (off) 1029 EMIT(PPC_RAW_DIVD(tmp1_reg, dst_reg, src_reg)); 1030 else 1031 EMIT(PPC_RAW_DIVDU(tmp1_reg, dst_reg, src_reg)); 1032 EMIT(PPC_RAW_MULD(tmp1_reg, src_reg, tmp1_reg)); 1033 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg)); 1034 } else 1035 if (off) 1036 EMIT(PPC_RAW_DIVD(dst_reg, dst_reg, src_reg)); 1037 else 1038 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg)); 1039 break; 1040 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 1041 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 1042 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 1043 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 1044 if (imm == 0) 1045 return -EINVAL; 1046 if (imm == 1) { 1047 if (BPF_OP(code) == BPF_DIV) { 1048 goto bpf_alu32_trunc; 1049 } else { 1050 EMIT(PPC_RAW_LI(dst_reg, 0)); 1051 break; 1052 } 1053 } 1054 1055 PPC_LI32(tmp1_reg, imm); 1056 switch (BPF_CLASS(code)) { 1057 case BPF_ALU: 1058 if (BPF_OP(code) == BPF_MOD) { 1059 if (off) 1060 EMIT(PPC_RAW_DIVW(tmp2_reg, dst_reg, tmp1_reg)); 1061 else 1062 EMIT(PPC_RAW_DIVWU(tmp2_reg, dst_reg, tmp1_reg)); 1063 EMIT(PPC_RAW_MULW(tmp1_reg, tmp1_reg, tmp2_reg)); 1064 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg)); 1065 } else 1066 if (off) 1067 EMIT(PPC_RAW_DIVW(dst_reg, dst_reg, tmp1_reg)); 1068 else 1069 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, tmp1_reg)); 1070 break; 1071 case BPF_ALU64: 1072 if (BPF_OP(code) == BPF_MOD) { 1073 if (off) 1074 EMIT(PPC_RAW_DIVD(tmp2_reg, dst_reg, tmp1_reg)); 1075 else 1076 EMIT(PPC_RAW_DIVDU(tmp2_reg, dst_reg, tmp1_reg)); 1077 EMIT(PPC_RAW_MULD(tmp1_reg, tmp1_reg, tmp2_reg)); 1078 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg)); 1079 } else 1080 if (off) 1081 EMIT(PPC_RAW_DIVD(dst_reg, dst_reg, tmp1_reg)); 1082 else 1083 EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, tmp1_reg)); 1084 break; 1085 } 1086 goto bpf_alu32_trunc; 1087 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 1088 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 1089 EMIT(PPC_RAW_NEG(dst_reg, dst_reg)); 1090 goto bpf_alu32_trunc; 1091 1092 /* 1093 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 1094 */ 1095 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 1096 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 1097 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 1098 goto bpf_alu32_trunc; 1099 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 1100 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 1101 if (!IMM_H(imm)) 1102 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm))); 1103 else { 1104 /* Sign-extended */ 1105 PPC_LI32(tmp1_reg, imm); 1106 EMIT(PPC_RAW_AND(dst_reg, dst_reg, tmp1_reg)); 1107 } 1108 goto bpf_alu32_trunc; 1109 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 1110 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 1111 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 1112 goto bpf_alu32_trunc; 1113 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 1114 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 1115 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 1116 /* Sign-extended */ 1117 PPC_LI32(tmp1_reg, imm); 1118 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp1_reg)); 1119 } else { 1120 if (IMM_L(imm)) 1121 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm))); 1122 if (IMM_H(imm)) 1123 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm))); 1124 } 1125 goto bpf_alu32_trunc; 1126 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 1127 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 1128 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 1129 goto bpf_alu32_trunc; 1130 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 1131 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 1132 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 1133 /* Sign-extended */ 1134 PPC_LI32(tmp1_reg, imm); 1135 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, tmp1_reg)); 1136 } else { 1137 if (IMM_L(imm)) 1138 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm))); 1139 if (IMM_H(imm)) 1140 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm))); 1141 } 1142 goto bpf_alu32_trunc; 1143 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 1144 /* slw clears top 32 bits */ 1145 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 1146 /* skip zero extension move, but set address map. */ 1147 if (insn_is_zext(&insn[i + 1])) 1148 addrs[++i] = ctx->idx * 4; 1149 break; 1150 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 1151 EMIT(PPC_RAW_SLD(dst_reg, dst_reg, src_reg)); 1152 break; 1153 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 1154 /* with imm 0, we still need to clear top 32 bits */ 1155 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm)); 1156 if (insn_is_zext(&insn[i + 1])) 1157 addrs[++i] = ctx->idx * 4; 1158 break; 1159 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 1160 if (imm != 0) 1161 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, imm)); 1162 break; 1163 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 1164 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 1165 if (insn_is_zext(&insn[i + 1])) 1166 addrs[++i] = ctx->idx * 4; 1167 break; 1168 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 1169 EMIT(PPC_RAW_SRD(dst_reg, dst_reg, src_reg)); 1170 break; 1171 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 1172 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm)); 1173 if (insn_is_zext(&insn[i + 1])) 1174 addrs[++i] = ctx->idx * 4; 1175 break; 1176 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 1177 if (imm != 0) 1178 EMIT(PPC_RAW_SRDI(dst_reg, dst_reg, imm)); 1179 break; 1180 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 1181 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg)); 1182 goto bpf_alu32_trunc; 1183 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 1184 EMIT(PPC_RAW_SRAD(dst_reg, dst_reg, src_reg)); 1185 break; 1186 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 1187 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm)); 1188 goto bpf_alu32_trunc; 1189 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 1190 if (imm != 0) 1191 EMIT(PPC_RAW_SRADI(dst_reg, dst_reg, imm)); 1192 break; 1193 1194 /* 1195 * MOV 1196 */ 1197 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 1198 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 1199 1200 if (insn_is_mov_percpu_addr(&insn[i])) { 1201 if (IS_ENABLED(CONFIG_SMP)) { 1202 EMIT(PPC_RAW_LD(tmp1_reg, _R13, offsetof(struct paca_struct, data_offset))); 1203 EMIT(PPC_RAW_ADD(dst_reg, src_reg, tmp1_reg)); 1204 } else if (src_reg != dst_reg) { 1205 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 1206 } 1207 break; 1208 } 1209 1210 if (insn_is_cast_user(&insn[i])) { 1211 EMIT(PPC_RAW_RLDICL_DOT(tmp1_reg, src_reg, 0, 32)); 1212 PPC_LI64(dst_reg, (ctx->user_vm_start & 0xffffffff00000000UL)); 1213 PPC_BCC_SHORT(COND_EQ, (ctx->idx + 2) * 4); 1214 EMIT(PPC_RAW_OR(tmp1_reg, dst_reg, tmp1_reg)); 1215 EMIT(PPC_RAW_MR(dst_reg, tmp1_reg)); 1216 break; 1217 } 1218 1219 if (imm == 1) { 1220 /* special mov32 for zext */ 1221 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); 1222 break; 1223 } 1224 if (off == 0) { 1225 /* MOV */ 1226 if (dst_reg != src_reg) 1227 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 1228 } else { 1229 /* MOVSX: dst = (s8,s16,s32)src (off = 8,16,32) */ 1230 if (sign_extend(image, ctx, src_reg, dst_reg, off / 8)) 1231 return -1; 1232 } 1233 goto bpf_alu32_trunc; 1234 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 1235 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 1236 PPC_LI32(dst_reg, imm); 1237 if (imm < 0) 1238 goto bpf_alu32_trunc; 1239 else if (insn_is_zext(&insn[i + 1])) 1240 addrs[++i] = ctx->idx * 4; 1241 break; 1242 1243 bpf_alu32_trunc: 1244 /* Truncate to 32-bits */ 1245 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext) 1246 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31)); 1247 break; 1248 1249 /* 1250 * BPF_FROM_BE/LE 1251 */ 1252 case BPF_ALU | BPF_END | BPF_FROM_LE: 1253 case BPF_ALU | BPF_END | BPF_FROM_BE: 1254 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1255 #ifdef __BIG_ENDIAN__ 1256 if (BPF_SRC(code) == BPF_FROM_BE) 1257 goto emit_clear; 1258 #else /* !__BIG_ENDIAN__ */ 1259 if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_LE) 1260 goto emit_clear; 1261 #endif 1262 switch (imm) { 1263 case 16: 1264 /* Rotate 8 bits left & mask with 0x0000ff00 */ 1265 EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 16, 23)); 1266 /* Rotate 8 bits right & insert LSB to reg */ 1267 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 24, 31)); 1268 /* Move result back to dst_reg */ 1269 EMIT(PPC_RAW_MR(dst_reg, tmp1_reg)); 1270 break; 1271 case 32: 1272 /* 1273 * Rotate word left by 8 bits: 1274 * 2 bytes are already in their final position 1275 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 1276 */ 1277 EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 0, 31)); 1278 /* Rotate 24 bits and insert byte 1 */ 1279 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 0, 7)); 1280 /* Rotate 24 bits and insert byte 3 */ 1281 EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 16, 23)); 1282 EMIT(PPC_RAW_MR(dst_reg, tmp1_reg)); 1283 break; 1284 case 64: 1285 /* Store the value to stack and then use byte-reverse loads */ 1286 EMIT(PPC_RAW_STD(dst_reg, _R1, bpf_jit_stack_local(ctx))); 1287 EMIT(PPC_RAW_ADDI(tmp1_reg, _R1, bpf_jit_stack_local(ctx))); 1288 if (cpu_has_feature(CPU_FTR_ARCH_206)) { 1289 EMIT(PPC_RAW_LDBRX(dst_reg, 0, tmp1_reg)); 1290 } else { 1291 EMIT(PPC_RAW_LWBRX(dst_reg, 0, tmp1_reg)); 1292 if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) 1293 EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32)); 1294 EMIT(PPC_RAW_LI(tmp2_reg, 4)); 1295 EMIT(PPC_RAW_LWBRX(tmp2_reg, tmp2_reg, tmp1_reg)); 1296 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 1297 EMIT(PPC_RAW_SLDI(tmp2_reg, tmp2_reg, 32)); 1298 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp2_reg)); 1299 } 1300 break; 1301 } 1302 break; 1303 1304 emit_clear: 1305 switch (imm) { 1306 case 16: 1307 /* zero-extend 16 bits into 64 bits */ 1308 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 48)); 1309 if (insn_is_zext(&insn[i + 1])) 1310 addrs[++i] = ctx->idx * 4; 1311 break; 1312 case 32: 1313 if (!fp->aux->verifier_zext) 1314 /* zero-extend 32 bits into 64 bits */ 1315 EMIT(PPC_RAW_RLDICL(dst_reg, dst_reg, 0, 32)); 1316 break; 1317 case 64: 1318 /* nop */ 1319 break; 1320 } 1321 break; 1322 1323 /* 1324 * BPF_ST NOSPEC (speculation barrier) 1325 * 1326 * The following must act as a barrier against both Spectre v1 1327 * and v4 if we requested both mitigations. Therefore, also emit 1328 * 'isync; sync' on E500 or 'ori31' on BOOK3S_64 in addition to 1329 * the insns needed for a Spectre v4 barrier. 1330 * 1331 * If we requested only !bypass_spec_v1 OR only !bypass_spec_v4, 1332 * we can skip the respective other barrier type as an 1333 * optimization. 1334 */ 1335 case BPF_ST | BPF_NOSPEC: 1336 sync_emitted = false; 1337 ori31_emitted = false; 1338 if (IS_ENABLED(CONFIG_PPC_E500) && 1339 !bpf_jit_bypass_spec_v1()) { 1340 EMIT(PPC_RAW_ISYNC()); 1341 EMIT(PPC_RAW_SYNC()); 1342 sync_emitted = true; 1343 } 1344 if (!bpf_jit_bypass_spec_v4()) { 1345 switch (stf_barrier) { 1346 case STF_BARRIER_EIEIO: 1347 EMIT(PPC_RAW_EIEIO() | 0x02000000); 1348 break; 1349 case STF_BARRIER_SYNC_ORI: 1350 if (!sync_emitted) 1351 EMIT(PPC_RAW_SYNC()); 1352 EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0)); 1353 EMIT(PPC_RAW_ORI(_R31, _R31, 0)); 1354 ori31_emitted = true; 1355 break; 1356 case STF_BARRIER_FALLBACK: 1357 ctx->seen |= SEEN_FUNC; 1358 PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier)); 1359 EMIT(PPC_RAW_MTCTR(_R12)); 1360 EMIT(PPC_RAW_BCTRL()); 1361 break; 1362 case STF_BARRIER_NONE: 1363 break; 1364 } 1365 } 1366 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && 1367 !bpf_jit_bypass_spec_v1() && 1368 !ori31_emitted) 1369 EMIT(PPC_RAW_ORI(_R31, _R31, 0)); 1370 break; 1371 1372 /* 1373 * BPF_ST(X) 1374 */ 1375 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 1376 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 1377 if (BPF_CLASS(code) == BPF_ST) { 1378 EMIT(PPC_RAW_LI(tmp1_reg, imm)); 1379 src_reg = tmp1_reg; 1380 } 1381 EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 1382 break; 1383 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 1384 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 1385 if (BPF_CLASS(code) == BPF_ST) { 1386 EMIT(PPC_RAW_LI(tmp1_reg, imm)); 1387 src_reg = tmp1_reg; 1388 } 1389 EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 1390 break; 1391 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 1392 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 1393 if (BPF_CLASS(code) == BPF_ST) { 1394 PPC_LI32(tmp1_reg, imm); 1395 src_reg = tmp1_reg; 1396 } 1397 EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 1398 break; 1399 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 1400 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 1401 if (BPF_CLASS(code) == BPF_ST) { 1402 PPC_LI32(tmp1_reg, imm); 1403 src_reg = tmp1_reg; 1404 } 1405 if (off % 4) { 1406 EMIT(PPC_RAW_LI(tmp2_reg, off)); 1407 EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg)); 1408 } else { 1409 EMIT(PPC_RAW_STD(src_reg, dst_reg, off)); 1410 } 1411 break; 1412 1413 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1414 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1415 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1416 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1417 1418 EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); 1419 1420 ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image); 1421 if (ret) 1422 return ret; 1423 1424 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, 1425 ctx->idx - 1, 4, -1, code); 1426 if (ret) 1427 return ret; 1428 1429 break; 1430 1431 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1432 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1433 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1434 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1435 1436 EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); 1437 1438 if (BPF_SIZE(code) == BPF_W || BPF_SIZE(code) == BPF_DW) { 1439 PPC_LI32(tmp2_reg, imm); 1440 src_reg = tmp2_reg; 1441 } else { 1442 EMIT(PPC_RAW_LI(tmp2_reg, imm)); 1443 src_reg = tmp2_reg; 1444 } 1445 1446 ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image); 1447 if (ret) 1448 return ret; 1449 1450 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, 1451 ctx->idx - 1, 4, -1, code); 1452 if (ret) 1453 return ret; 1454 1455 break; 1456 1457 /* 1458 * BPF_STX PROBE_ATOMIC (arena atomic ops) 1459 */ 1460 case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: 1461 case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: 1462 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); 1463 ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i], 1464 &jmp_off, &tmp_idx, &addrs[i + 1]); 1465 if (ret) { 1466 if (ret == -EOPNOTSUPP) { 1467 pr_err_ratelimited( 1468 "eBPF filter atomic op code %02x (@%d) unsupported\n", 1469 code, i); 1470 } 1471 return ret; 1472 } 1473 /* LDARX/LWARX should land here on exception. */ 1474 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, 1475 tmp_idx, jmp_off, dst_reg, code); 1476 if (ret) 1477 return ret; 1478 1479 /* Retrieve the dst_reg */ 1480 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); 1481 break; 1482 1483 /* 1484 * BPF_STX ATOMIC (atomic ops) 1485 */ 1486 case BPF_STX | BPF_ATOMIC | BPF_B: 1487 case BPF_STX | BPF_ATOMIC | BPF_H: 1488 case BPF_STX | BPF_ATOMIC | BPF_W: 1489 case BPF_STX | BPF_ATOMIC | BPF_DW: 1490 if (bpf_atomic_is_load_store(&insn[i])) { 1491 ret = emit_atomic_ld_st(insn[i], ctx, image); 1492 if (ret) 1493 return ret; 1494 1495 if (size != BPF_DW && insn_is_zext(&insn[i + 1])) 1496 addrs[++i] = ctx->idx * 4; 1497 break; 1498 } else if (size == BPF_B || size == BPF_H) { 1499 pr_err_ratelimited( 1500 "eBPF filter atomic op code %02x (@%d) unsupported\n", 1501 code, i); 1502 return -EOPNOTSUPP; 1503 } 1504 1505 ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i], 1506 &jmp_off, &tmp_idx, &addrs[i + 1]); 1507 if (ret) { 1508 if (ret == -EOPNOTSUPP) { 1509 pr_err_ratelimited( 1510 "eBPF filter atomic op code %02x (@%d) unsupported\n", 1511 code, i); 1512 } 1513 return ret; 1514 } 1515 break; 1516 1517 /* 1518 * BPF_LDX 1519 */ 1520 /* dst = *(u8 *)(ul) (src + off) */ 1521 case BPF_LDX | BPF_MEM | BPF_B: 1522 case BPF_LDX | BPF_MEMSX | BPF_B: 1523 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1524 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1525 /* dst = *(u16 *)(ul) (src + off) */ 1526 case BPF_LDX | BPF_MEM | BPF_H: 1527 case BPF_LDX | BPF_MEMSX | BPF_H: 1528 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1529 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1530 /* dst = *(u32 *)(ul) (src + off) */ 1531 case BPF_LDX | BPF_MEM | BPF_W: 1532 case BPF_LDX | BPF_MEMSX | BPF_W: 1533 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1534 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1535 /* dst = *(u64 *)(ul) (src + off) */ 1536 case BPF_LDX | BPF_MEM | BPF_DW: 1537 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1538 /* 1539 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid 1540 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM 1541 * load only if addr is kernel address (see is_kernel_addr()), otherwise 1542 * set dst_reg=0 and move on. 1543 */ 1544 if (BPF_MODE(code) == BPF_PROBE_MEM || BPF_MODE(code) == BPF_PROBE_MEMSX) { 1545 EMIT(PPC_RAW_ADDI(tmp1_reg, src_reg, off)); 1546 if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) 1547 PPC_LI64(tmp2_reg, 0x8000000000000000ul); 1548 else /* BOOK3S_64 */ 1549 PPC_LI64(tmp2_reg, PAGE_OFFSET); 1550 EMIT(PPC_RAW_CMPLD(tmp1_reg, tmp2_reg)); 1551 PPC_BCC_SHORT(COND_GT, (ctx->idx + 3) * 4); 1552 EMIT(PPC_RAW_LI(dst_reg, 0)); 1553 /* 1554 * Check if 'off' is word aligned for BPF_DW, because 1555 * we might generate two instructions. 1556 */ 1557 if ((BPF_SIZE(code) == BPF_DW && (off & 3)) || 1558 (BPF_SIZE(code) == BPF_B && 1559 BPF_MODE(code) == BPF_PROBE_MEMSX) || 1560 (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_MEMSX)) 1561 PPC_JMP((ctx->idx + 3) * 4); 1562 else 1563 PPC_JMP((ctx->idx + 2) * 4); 1564 } 1565 1566 if (BPF_MODE(code) == BPF_MEMSX || BPF_MODE(code) == BPF_PROBE_MEMSX) { 1567 switch (size) { 1568 case BPF_B: 1569 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 1570 EMIT(PPC_RAW_EXTSB(dst_reg, dst_reg)); 1571 break; 1572 case BPF_H: 1573 EMIT(PPC_RAW_LHA(dst_reg, src_reg, off)); 1574 break; 1575 case BPF_W: 1576 EMIT(PPC_RAW_LWA(dst_reg, src_reg, off)); 1577 break; 1578 } 1579 } else { 1580 switch (size) { 1581 case BPF_B: 1582 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 1583 break; 1584 case BPF_H: 1585 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 1586 break; 1587 case BPF_W: 1588 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 1589 break; 1590 case BPF_DW: 1591 if (off % 4) { 1592 EMIT(PPC_RAW_LI(tmp1_reg, off)); 1593 EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg)); 1594 } else { 1595 EMIT(PPC_RAW_LD(dst_reg, src_reg, off)); 1596 } 1597 break; 1598 } 1599 } 1600 1601 if (size != BPF_DW && insn_is_zext(&insn[i + 1])) 1602 addrs[++i] = ctx->idx * 4; 1603 1604 if (BPF_MODE(code) == BPF_PROBE_MEM) { 1605 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, 1606 ctx->idx - 1, 4, dst_reg, code); 1607 if (ret) 1608 return ret; 1609 } 1610 break; 1611 1612 /* dst = *(u64 *)(ul) (src + ARENA_VM_START + off) */ 1613 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1614 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1615 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1616 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1617 1618 EMIT(PPC_RAW_ADD(tmp1_reg, src_reg, bpf_to_ppc(ARENA_VM_START))); 1619 1620 switch (size) { 1621 case BPF_B: 1622 EMIT(PPC_RAW_LBZ(dst_reg, tmp1_reg, off)); 1623 break; 1624 case BPF_H: 1625 EMIT(PPC_RAW_LHZ(dst_reg, tmp1_reg, off)); 1626 break; 1627 case BPF_W: 1628 EMIT(PPC_RAW_LWZ(dst_reg, tmp1_reg, off)); 1629 break; 1630 case BPF_DW: 1631 if (off % 4) { 1632 EMIT(PPC_RAW_LI(tmp2_reg, off)); 1633 EMIT(PPC_RAW_LDX(dst_reg, tmp1_reg, tmp2_reg)); 1634 } else { 1635 EMIT(PPC_RAW_LD(dst_reg, tmp1_reg, off)); 1636 } 1637 break; 1638 } 1639 1640 if (size != BPF_DW && insn_is_zext(&insn[i + 1])) 1641 addrs[++i] = ctx->idx * 4; 1642 1643 ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, 1644 ctx->idx - 1, 4, dst_reg, code); 1645 if (ret) 1646 return ret; 1647 break; 1648 1649 /* 1650 * Doubleword load 1651 * 16 byte instruction that uses two 'struct bpf_insn' 1652 */ 1653 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 1654 imm64 = ((u64)(u32) insn[i].imm) | 1655 (((u64)(u32) insn[i+1].imm) << 32); 1656 PPC_LI64(dst_reg, imm64); 1657 /* Adjust for two bpf instructions */ 1658 addrs[++i] = ctx->idx * 4; 1659 break; 1660 1661 /* 1662 * Return/Exit 1663 */ 1664 case BPF_JMP | BPF_EXIT: 1665 /* 1666 * If this isn't the very last instruction, branch to 1667 * the epilogue. If we _are_ the last instruction, 1668 * we'll just fall through to the epilogue. 1669 */ 1670 if (i != flen - 1) { 1671 ret = bpf_jit_emit_exit_insn(image, ctx, tmp1_reg, exit_addr); 1672 if (ret) 1673 return ret; 1674 } 1675 /* else fall through to the epilogue */ 1676 break; 1677 1678 /* 1679 * Call kernel helper or bpf function 1680 */ 1681 case BPF_JMP | BPF_CALL: 1682 ctx->seen |= SEEN_FUNC; 1683 1684 if (src_reg == bpf_to_ppc(BPF_REG_0)) { 1685 if (imm == BPF_FUNC_get_smp_processor_id) { 1686 EMIT(PPC_RAW_LHZ(src_reg, _R13, offsetof(struct paca_struct, paca_index))); 1687 break; 1688 } else if (imm == BPF_FUNC_get_current_task || 1689 imm == BPF_FUNC_get_current_task_btf) { 1690 EMIT(PPC_RAW_LD(src_reg, _R13, offsetof(struct paca_struct, __current))); 1691 break; 1692 } 1693 } 1694 1695 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, 1696 &func_addr, &func_addr_fixed); 1697 if (ret < 0) 1698 return ret; 1699 1700 /* Take care of powerpc ABI requirements before kfunc call */ 1701 if (insn[i].src_reg == BPF_PSEUDO_KFUNC_CALL) { 1702 if (prepare_for_kfunc_call(fp, image, ctx, &insn[i])) 1703 return -1; 1704 } 1705 1706 ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); 1707 if (ret) 1708 return ret; 1709 1710 /* move return value from r3 to BPF_REG_0 */ 1711 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R3)); 1712 break; 1713 1714 /* 1715 * Jumps and branches 1716 */ 1717 case BPF_JMP | BPF_JA: 1718 PPC_JMP(addrs[i + 1 + off]); 1719 break; 1720 case BPF_JMP32 | BPF_JA: 1721 PPC_JMP(addrs[i + 1 + imm]); 1722 break; 1723 1724 case BPF_JMP | BPF_JGT | BPF_K: 1725 case BPF_JMP | BPF_JGT | BPF_X: 1726 case BPF_JMP | BPF_JSGT | BPF_K: 1727 case BPF_JMP | BPF_JSGT | BPF_X: 1728 case BPF_JMP32 | BPF_JGT | BPF_K: 1729 case BPF_JMP32 | BPF_JGT | BPF_X: 1730 case BPF_JMP32 | BPF_JSGT | BPF_K: 1731 case BPF_JMP32 | BPF_JSGT | BPF_X: 1732 true_cond = COND_GT; 1733 goto cond_branch; 1734 case BPF_JMP | BPF_JLT | BPF_K: 1735 case BPF_JMP | BPF_JLT | BPF_X: 1736 case BPF_JMP | BPF_JSLT | BPF_K: 1737 case BPF_JMP | BPF_JSLT | BPF_X: 1738 case BPF_JMP32 | BPF_JLT | BPF_K: 1739 case BPF_JMP32 | BPF_JLT | BPF_X: 1740 case BPF_JMP32 | BPF_JSLT | BPF_K: 1741 case BPF_JMP32 | BPF_JSLT | BPF_X: 1742 true_cond = COND_LT; 1743 goto cond_branch; 1744 case BPF_JMP | BPF_JGE | BPF_K: 1745 case BPF_JMP | BPF_JGE | BPF_X: 1746 case BPF_JMP | BPF_JSGE | BPF_K: 1747 case BPF_JMP | BPF_JSGE | BPF_X: 1748 case BPF_JMP32 | BPF_JGE | BPF_K: 1749 case BPF_JMP32 | BPF_JGE | BPF_X: 1750 case BPF_JMP32 | BPF_JSGE | BPF_K: 1751 case BPF_JMP32 | BPF_JSGE | BPF_X: 1752 true_cond = COND_GE; 1753 goto cond_branch; 1754 case BPF_JMP | BPF_JLE | BPF_K: 1755 case BPF_JMP | BPF_JLE | BPF_X: 1756 case BPF_JMP | BPF_JSLE | BPF_K: 1757 case BPF_JMP | BPF_JSLE | BPF_X: 1758 case BPF_JMP32 | BPF_JLE | BPF_K: 1759 case BPF_JMP32 | BPF_JLE | BPF_X: 1760 case BPF_JMP32 | BPF_JSLE | BPF_K: 1761 case BPF_JMP32 | BPF_JSLE | BPF_X: 1762 true_cond = COND_LE; 1763 goto cond_branch; 1764 case BPF_JMP | BPF_JEQ | BPF_K: 1765 case BPF_JMP | BPF_JEQ | BPF_X: 1766 case BPF_JMP32 | BPF_JEQ | BPF_K: 1767 case BPF_JMP32 | BPF_JEQ | BPF_X: 1768 true_cond = COND_EQ; 1769 goto cond_branch; 1770 case BPF_JMP | BPF_JNE | BPF_K: 1771 case BPF_JMP | BPF_JNE | BPF_X: 1772 case BPF_JMP32 | BPF_JNE | BPF_K: 1773 case BPF_JMP32 | BPF_JNE | BPF_X: 1774 true_cond = COND_NE; 1775 goto cond_branch; 1776 case BPF_JMP | BPF_JSET | BPF_K: 1777 case BPF_JMP | BPF_JSET | BPF_X: 1778 case BPF_JMP32 | BPF_JSET | BPF_K: 1779 case BPF_JMP32 | BPF_JSET | BPF_X: 1780 true_cond = COND_NE; 1781 /* Fall through */ 1782 1783 cond_branch: 1784 switch (code) { 1785 case BPF_JMP | BPF_JGT | BPF_X: 1786 case BPF_JMP | BPF_JLT | BPF_X: 1787 case BPF_JMP | BPF_JGE | BPF_X: 1788 case BPF_JMP | BPF_JLE | BPF_X: 1789 case BPF_JMP | BPF_JEQ | BPF_X: 1790 case BPF_JMP | BPF_JNE | BPF_X: 1791 case BPF_JMP32 | BPF_JGT | BPF_X: 1792 case BPF_JMP32 | BPF_JLT | BPF_X: 1793 case BPF_JMP32 | BPF_JGE | BPF_X: 1794 case BPF_JMP32 | BPF_JLE | BPF_X: 1795 case BPF_JMP32 | BPF_JEQ | BPF_X: 1796 case BPF_JMP32 | BPF_JNE | BPF_X: 1797 /* unsigned comparison */ 1798 if (BPF_CLASS(code) == BPF_JMP32) 1799 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1800 else 1801 EMIT(PPC_RAW_CMPLD(dst_reg, src_reg)); 1802 break; 1803 case BPF_JMP | BPF_JSGT | BPF_X: 1804 case BPF_JMP | BPF_JSLT | BPF_X: 1805 case BPF_JMP | BPF_JSGE | BPF_X: 1806 case BPF_JMP | BPF_JSLE | BPF_X: 1807 case BPF_JMP32 | BPF_JSGT | BPF_X: 1808 case BPF_JMP32 | BPF_JSLT | BPF_X: 1809 case BPF_JMP32 | BPF_JSGE | BPF_X: 1810 case BPF_JMP32 | BPF_JSLE | BPF_X: 1811 /* signed comparison */ 1812 if (BPF_CLASS(code) == BPF_JMP32) 1813 EMIT(PPC_RAW_CMPW(dst_reg, src_reg)); 1814 else 1815 EMIT(PPC_RAW_CMPD(dst_reg, src_reg)); 1816 break; 1817 case BPF_JMP | BPF_JSET | BPF_X: 1818 case BPF_JMP32 | BPF_JSET | BPF_X: 1819 if (BPF_CLASS(code) == BPF_JMP) { 1820 EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, src_reg)); 1821 } else { 1822 EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, src_reg)); 1823 EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 0, 0, 31)); 1824 } 1825 break; 1826 case BPF_JMP | BPF_JNE | BPF_K: 1827 case BPF_JMP | BPF_JEQ | BPF_K: 1828 case BPF_JMP | BPF_JGT | BPF_K: 1829 case BPF_JMP | BPF_JLT | BPF_K: 1830 case BPF_JMP | BPF_JGE | BPF_K: 1831 case BPF_JMP | BPF_JLE | BPF_K: 1832 case BPF_JMP32 | BPF_JNE | BPF_K: 1833 case BPF_JMP32 | BPF_JEQ | BPF_K: 1834 case BPF_JMP32 | BPF_JGT | BPF_K: 1835 case BPF_JMP32 | BPF_JLT | BPF_K: 1836 case BPF_JMP32 | BPF_JGE | BPF_K: 1837 case BPF_JMP32 | BPF_JLE | BPF_K: 1838 { 1839 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 1840 1841 /* 1842 * Need sign-extended load, so only positive 1843 * values can be used as imm in cmpldi 1844 */ 1845 if (imm >= 0 && imm < 32768) { 1846 if (is_jmp32) 1847 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1848 else 1849 EMIT(PPC_RAW_CMPLDI(dst_reg, imm)); 1850 } else { 1851 /* sign-extending load */ 1852 PPC_LI32(tmp1_reg, imm); 1853 /* ... but unsigned comparison */ 1854 if (is_jmp32) 1855 EMIT(PPC_RAW_CMPLW(dst_reg, tmp1_reg)); 1856 else 1857 EMIT(PPC_RAW_CMPLD(dst_reg, tmp1_reg)); 1858 } 1859 break; 1860 } 1861 case BPF_JMP | BPF_JSGT | BPF_K: 1862 case BPF_JMP | BPF_JSLT | BPF_K: 1863 case BPF_JMP | BPF_JSGE | BPF_K: 1864 case BPF_JMP | BPF_JSLE | BPF_K: 1865 case BPF_JMP32 | BPF_JSGT | BPF_K: 1866 case BPF_JMP32 | BPF_JSLT | BPF_K: 1867 case BPF_JMP32 | BPF_JSGE | BPF_K: 1868 case BPF_JMP32 | BPF_JSLE | BPF_K: 1869 { 1870 bool is_jmp32 = BPF_CLASS(code) == BPF_JMP32; 1871 1872 /* 1873 * signed comparison, so any 16-bit value 1874 * can be used in cmpdi 1875 */ 1876 if (imm >= -32768 && imm < 32768) { 1877 if (is_jmp32) 1878 EMIT(PPC_RAW_CMPWI(dst_reg, imm)); 1879 else 1880 EMIT(PPC_RAW_CMPDI(dst_reg, imm)); 1881 } else { 1882 PPC_LI32(tmp1_reg, imm); 1883 if (is_jmp32) 1884 EMIT(PPC_RAW_CMPW(dst_reg, tmp1_reg)); 1885 else 1886 EMIT(PPC_RAW_CMPD(dst_reg, tmp1_reg)); 1887 } 1888 break; 1889 } 1890 case BPF_JMP | BPF_JSET | BPF_K: 1891 case BPF_JMP32 | BPF_JSET | BPF_K: 1892 /* andi does not sign-extend the immediate */ 1893 if (imm >= 0 && imm < 32768) 1894 /* PPC_ANDI is _only/always_ dot-form */ 1895 EMIT(PPC_RAW_ANDI(tmp1_reg, dst_reg, imm)); 1896 else { 1897 PPC_LI32(tmp1_reg, imm); 1898 if (BPF_CLASS(code) == BPF_JMP) { 1899 EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, 1900 tmp1_reg)); 1901 } else { 1902 EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, tmp1_reg)); 1903 EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 1904 0, 0, 31)); 1905 } 1906 } 1907 break; 1908 } 1909 PPC_BCC(true_cond, addrs[i + 1 + off]); 1910 break; 1911 1912 /* 1913 * Tail call 1914 */ 1915 case BPF_JMP | BPF_TAIL_CALL: 1916 ctx->seen |= SEEN_TAILCALL; 1917 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 1918 if (ret < 0) 1919 return ret; 1920 break; 1921 1922 default: 1923 /* 1924 * The filter contains something cruel & unusual. 1925 * We don't handle it, but also there shouldn't be 1926 * anything missing from our list. 1927 */ 1928 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 1929 code, i); 1930 return -ENOTSUPP; 1931 } 1932 } 1933 1934 /* Set end-of-body-code address for exit. */ 1935 addrs[i] = ctx->idx * 4; 1936 1937 return 0; 1938 } 1939