1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <linux/kernel.h> 17 #include <linux/memory.h> 18 #include <linux/bpf.h> 19 20 #include <asm/kprobes.h> 21 #include <asm/text-patching.h> 22 23 #include "bpf_jit.h" 24 25 /* These offsets are from bpf prog end and stay the same across progs */ 26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub; 27 28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 29 { 30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 31 } 32 33 void dummy_tramp(void); 34 35 asm ( 36 " .pushsection .text, \"ax\", @progbits ;" 37 " .global dummy_tramp ;" 38 " .type dummy_tramp, @function ;" 39 "dummy_tramp: ;" 40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 41 " blr ;" 42 #else 43 /* LR is always in r11, so we don't need a 'mflr r11' here */ 44 " mtctr 11 ;" 45 " mtlr 0 ;" 46 " bctr ;" 47 #endif 48 " .size dummy_tramp, .-dummy_tramp ;" 49 " .popsection ;" 50 ); 51 52 void bpf_jit_build_fentry_stubs(u32 *image, u32 *fimage, struct codegen_context *ctx) 53 { 54 int ool_stub_idx, long_branch_stub_idx; 55 int stub_sz; 56 57 /* 58 * The dummy_tramp_addr field is placed at bottom of Long branch stub. 59 * Align the mis-aligned dummy_tramp_addr field in the fimage. 60 * The alignment NOP must appear before OOL stub, to make 61 * ool_stub_idx & long_branch_stub_idx constant from end. 62 * 63 * The fimage can be non 8-byte aligned, so final alignment depends 64 * on start of fimage and the stub's instruction count. The 65 * stubs block has 11 instructions (with CONFIG_PPC_FTRACE_OUT_OF_LINE) 66 * or 10 instructions (without) before dummy_tramp_addr field. Emit a 67 * NOP if the address of dummy_tramp_addr is non aligned. 68 * 69 * In pass=0 when image==NULL, conservatively account for space 70 * required to accommodate alignment NOP. In case final pass skips 71 * emitting alignment NOP, the image buffer have 4 spare bytes and 72 * jited_len signifies correct program size. 73 */ 74 75 stub_sz = IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) ? 44 : 40; 76 if (!image || !IS_ALIGNED((unsigned long)fimage + ctx->idx*4 + stub_sz, SZL)) 77 EMIT(PPC_RAW_NOP()); 78 79 /* 80 * nop // optional, for alignment of dummy_tramp_addr 81 * Out-of-line stub: 82 * mflr r0 83 * [b|bl] tramp 84 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE 85 * b bpf_func + 4 86 */ 87 ool_stub_idx = ctx->idx; 88 EMIT(PPC_RAW_MFLR(_R0)); 89 EMIT(PPC_RAW_NOP()); 90 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 91 EMIT(PPC_RAW_MTLR(_R0)); 92 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4)); 93 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4)); 94 95 /* 96 * Long branch stub: 97 * mflr r11 98 * bcl 20,31,$+4 99 * mflr r12 // lr/r12 stores pc of current(this) inst. 100 * ld r12, 20(r12) // offset(dummy_tramp_addr) from prev inst. is 20 101 * mtctr r12 102 * mtlr r11 // needed to retain ftrace ABI 103 * bctr 104 * .long <dummy_tramp_addr> // SZL bytes aligned 105 */ 106 long_branch_stub_idx = ctx->idx; 107 EMIT(PPC_RAW_MFLR(_R11)); 108 EMIT(PPC_RAW_BCL4()); 109 EMIT(PPC_RAW_MFLR(_R12)); 110 EMIT(PPC_RAW_LL(_R12, _R12, 20)); 111 EMIT(PPC_RAW_MTCTR(_R12)); 112 EMIT(PPC_RAW_MTLR(_R11)); 113 EMIT(PPC_RAW_BCTR()); 114 115 if (image) 116 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp; 117 118 ctx->idx += SZL / 4; 119 120 if (!bpf_jit_ool_stub) { 121 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4; 122 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4; 123 } 124 } 125 126 int bpf_jit_emit_exit_insn(u32 *image, u32 *fimage, struct codegen_context *ctx, 127 int tmp_reg, long exit_addr) 128 { 129 if (exit_addr && is_offset_in_branch_range(exit_addr - (long)(ctx->idx * 4))) { 130 PPC_JMP(exit_addr); 131 } else if (ctx->alt_exit_addr && is_offset_in_branch_range( 132 (long)(ctx->alt_exit_addr) - (long)(ctx->idx * 4))) { 133 PPC_JMP(ctx->alt_exit_addr); 134 } else { 135 ctx->alt_exit_addr = ctx->idx * 4; 136 bpf_jit_build_epilogue(image, fimage, ctx); 137 } 138 139 return 0; 140 } 141 142 struct powerpc_jit_data { 143 /* address of rw header */ 144 struct bpf_binary_header *hdr; 145 /* address of ro final header */ 146 struct bpf_binary_header *fhdr; 147 u32 *addrs; 148 u8 *fimage; 149 u32 proglen; 150 struct codegen_context ctx; 151 }; 152 153 bool bpf_jit_needs_zext(void) 154 { 155 return true; 156 } 157 158 static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) 159 { 160 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 161 u64 *stack_ptr; 162 163 for_each_possible_cpu(cpu) { 164 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 165 stack_ptr[0] = PRIV_STACK_GUARD_VAL; 166 stack_ptr[1] = PRIV_STACK_GUARD_VAL; 167 stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL; 168 stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL; 169 } 170 } 171 172 static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size, 173 struct bpf_prog *fp) 174 { 175 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 176 u64 *stack_ptr; 177 178 for_each_possible_cpu(cpu) { 179 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 180 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL || 181 stack_ptr[1] != PRIV_STACK_GUARD_VAL || 182 stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL || 183 stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) { 184 pr_err("BPF private stack overflow/underflow detected for prog %s\n", 185 bpf_jit_get_prog_name(fp)); 186 break; 187 } 188 } 189 } 190 191 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 192 { 193 u32 proglen; 194 u32 alloclen; 195 u8 *image = NULL; 196 u32 *code_base = NULL; 197 u32 *addrs = NULL; 198 struct powerpc_jit_data *jit_data = NULL; 199 struct codegen_context cgctx; 200 int pass; 201 int flen; 202 int priv_stack_alloc_size; 203 void __percpu *priv_stack_ptr = NULL; 204 struct bpf_binary_header *fhdr = NULL; 205 struct bpf_binary_header *hdr = NULL; 206 bool extra_pass = false; 207 u8 *fimage = NULL; 208 u32 *fcode_base = NULL; 209 u32 extable_len; 210 u32 fixup_len; 211 212 if (!fp->jit_requested) 213 return fp; 214 215 jit_data = fp->aux->jit_data; 216 if (!jit_data) { 217 jit_data = kzalloc_obj(*jit_data); 218 if (!jit_data) 219 return fp; 220 fp->aux->jit_data = jit_data; 221 } 222 223 priv_stack_ptr = fp->aux->priv_stack_ptr; 224 if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) { 225 /* 226 * Allocate private stack of size equivalent to 227 * verifier-calculated stack size plus two memory 228 * guard regions to detect private stack overflow 229 * and underflow. 230 */ 231 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 232 2 * PRIV_STACK_GUARD_SZ; 233 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); 234 if (!priv_stack_ptr) 235 goto out_priv_stack; 236 237 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); 238 fp->aux->priv_stack_ptr = priv_stack_ptr; 239 } 240 241 flen = fp->len; 242 addrs = jit_data->addrs; 243 if (addrs) { 244 cgctx = jit_data->ctx; 245 /* 246 * JIT compiled to a writable location (image/code_base) first. 247 * It is then moved to the readonly final location (fimage/fcode_base) 248 * using instruction patching. 249 */ 250 fimage = jit_data->fimage; 251 fhdr = jit_data->fhdr; 252 proglen = jit_data->proglen; 253 hdr = jit_data->hdr; 254 image = (void *)hdr + ((void *)fimage - (void *)fhdr); 255 extra_pass = true; 256 /* During extra pass, ensure index is reset before repopulating extable entries */ 257 cgctx.exentry_idx = 0; 258 goto skip_init_ctx; 259 } 260 261 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 262 if (addrs == NULL) 263 goto out_err; 264 265 memset(&cgctx, 0, sizeof(struct codegen_context)); 266 bpf_jit_init_reg_mapping(&cgctx); 267 268 /* Make sure that the stack is quadword aligned. */ 269 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 270 if (bpf_prog_uses_arena_insns(fp)) { 271 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); 272 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); 273 } 274 cgctx.is_subprog = bpf_is_subprog(fp); 275 cgctx.exception_boundary = fp->aux->exception_boundary; 276 cgctx.exception_cb = fp->aux->exception_cb; 277 cgctx.priv_sp = priv_stack_ptr; 278 cgctx.priv_stack_size = 0; 279 if (priv_stack_ptr) { 280 /* 281 * priv_stack_size required for setting bpf FP inside 282 * percpu allocation. 283 * stack_size is marked 0 to prevent allocation on 284 * general stack and offset calculation don't go for 285 * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local() 286 */ 287 cgctx.priv_stack_size = cgctx.stack_size; 288 cgctx.stack_size = 0; 289 } 290 291 /* Scouting faux-generate pass 0 */ 292 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 293 /* We hit something illegal or unsupported. */ 294 goto out_err; 295 296 /* 297 * If we have seen a tail call, we need a second pass. 298 * This is because bpf_jit_emit_common_epilogue() is called 299 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 300 * We also need a second pass if we ended up with too large 301 * a program so as to ensure BPF_EXIT branches are in range. 302 */ 303 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 304 cgctx.idx = 0; 305 cgctx.alt_exit_addr = 0; 306 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 307 goto out_err; 308 } 309 310 bpf_jit_realloc_regs(&cgctx); 311 /* 312 * Pretend to build prologue, given the features we've seen. This will 313 * update ctgtx.idx as it pretends to output instructions, then we can 314 * calculate total size from idx. 315 */ 316 bpf_jit_build_prologue(NULL, &cgctx); 317 addrs[fp->len] = cgctx.idx * 4; 318 bpf_jit_build_epilogue(NULL, NULL, &cgctx); 319 320 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 321 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 322 323 proglen = cgctx.idx * 4; 324 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 325 326 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 327 bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); 328 if (!fhdr) 329 goto out_err; 330 331 if (extable_len) 332 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; 333 334 skip_init_ctx: 335 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 336 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); 337 338 /* Code generation passes 1-2+, loop until program size converges. */ 339 for (pass = 1; pass <= CODEGEN_MAX_PASSES; pass++) { 340 u32 prev_proglen = proglen; 341 342 /* Now build the prologue, body code & epilogue for real. */ 343 cgctx.idx = 0; 344 cgctx.exentry_idx = 0; 345 cgctx.alt_exit_addr = 0; 346 bpf_jit_build_prologue(code_base, &cgctx); 347 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, 348 extra_pass)) { 349 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 350 bpf_jit_binary_pack_free(fhdr, hdr); 351 goto out_err; 352 } 353 addrs[fp->len] = cgctx.idx * 4; 354 bpf_jit_build_epilogue(code_base, fcode_base, &cgctx); 355 356 proglen = cgctx.idx * 4; 357 358 if (bpf_jit_enable > 1) 359 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 360 prev_proglen - proglen, cgctx.seen); 361 362 /* Check if program size has converged, but ensure minimum passes */ 363 if (pass >= CODEGEN_MIN_PASSES && proglen == prev_proglen) 364 break; 365 366 if (pass == CODEGEN_MAX_PASSES && proglen != prev_proglen) { 367 pr_err("BPF JIT: Program did not converge after %d passes\n", 368 CODEGEN_MAX_PASSES); 369 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 370 bpf_jit_binary_pack_free(fhdr, hdr); 371 goto out_err; 372 } 373 } 374 375 if (bpf_jit_enable > 1) 376 /* 377 * Note that we output the base address of the code_base 378 * rather than image, since opcodes are in code_base. 379 */ 380 bpf_jit_dump(flen, proglen, pass, code_base); 381 382 #ifdef CONFIG_PPC64_ELF_ABI_V1 383 /* Function descriptor nastiness: Address + TOC */ 384 ((u64 *)image)[0] = (u64)fcode_base; 385 ((u64 *)image)[1] = local_paca->kernel_toc; 386 #endif 387 388 if (!fp->is_func || extra_pass) { 389 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) 390 goto out_err; 391 } 392 393 fp->bpf_func = (void *)fimage; 394 fp->jited = 1; 395 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; 396 397 if (!fp->is_func || extra_pass) { 398 bpf_prog_fill_jited_linfo(fp, addrs); 399 /* 400 * On ABI V1, executable code starts after the function 401 * descriptor, so adjust base accordingly. 402 */ 403 bpf_prog_update_insn_ptrs(fp, addrs, 404 (void *)fimage + FUNCTION_DESCR_SIZE); 405 406 out_addrs: 407 if (!fp->jited && priv_stack_ptr) { 408 fp->aux->priv_stack_ptr = NULL; 409 free_percpu(priv_stack_ptr); 410 } 411 out_priv_stack: 412 kfree(addrs); 413 kfree(jit_data); 414 fp->aux->jit_data = NULL; 415 } else { 416 jit_data->addrs = addrs; 417 jit_data->ctx = cgctx; 418 jit_data->proglen = proglen; 419 jit_data->fimage = fimage; 420 jit_data->fhdr = fhdr; 421 jit_data->hdr = hdr; 422 } 423 424 return fp; 425 426 out_err: 427 if (extra_pass) { 428 fp->bpf_func = NULL; 429 fp->jited = 0; 430 fp->jited_len = 0; 431 } 432 goto out_addrs; 433 } 434 435 /* 436 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 437 * this function, as this only applies to BPF_PROBE_MEM, for now. 438 */ 439 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, 440 struct codegen_context *ctx, int insn_idx, int jmp_off, 441 int dst_reg, u32 code) 442 { 443 off_t offset; 444 unsigned long pc; 445 struct exception_table_entry *ex, *ex_entry; 446 u32 *fixup; 447 448 /* Populate extable entries only in the last pass */ 449 if (pass < CODEGEN_MIN_PASSES) 450 return 0; 451 452 if (!fp->aux->extable || 453 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 454 return -EINVAL; 455 456 /* 457 * Program is first written to image before copying to the 458 * final location (fimage). Accordingly, update in the image first. 459 * As all offsets used are relative, copying as is to the 460 * final location should be alright. 461 */ 462 pc = (unsigned long)&image[insn_idx]; 463 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; 464 465 fixup = (void *)ex - 466 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 467 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 468 469 fixup[0] = PPC_RAW_LI(dst_reg, 0); 470 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX) 471 fixup[0] = PPC_RAW_NOP(); 472 473 if (IS_ENABLED(CONFIG_PPC32)) 474 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 475 476 fixup[BPF_FIXUP_LEN - 1] = 477 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 478 479 ex_entry = &ex[ctx->exentry_idx]; 480 481 offset = pc - (long)&ex_entry->insn; 482 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 483 return -ERANGE; 484 ex_entry->insn = offset; 485 486 offset = (long)fixup - (long)&ex_entry->fixup; 487 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 488 return -ERANGE; 489 ex_entry->fixup = offset; 490 491 ctx->exentry_idx++; 492 return 0; 493 } 494 495 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 496 { 497 int err; 498 499 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 500 return ERR_PTR(-EINVAL); 501 502 mutex_lock(&text_mutex); 503 err = patch_instructions(dst, src, len, false); 504 mutex_unlock(&text_mutex); 505 506 return err ? ERR_PTR(err) : dst; 507 } 508 509 int bpf_arch_text_invalidate(void *dst, size_t len) 510 { 511 u32 insn = BREAKPOINT_INSTRUCTION; 512 int ret; 513 514 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 515 return -EINVAL; 516 517 mutex_lock(&text_mutex); 518 ret = patch_instructions(dst, &insn, len, true); 519 mutex_unlock(&text_mutex); 520 521 return ret; 522 } 523 524 void bpf_jit_free(struct bpf_prog *fp) 525 { 526 if (fp->jited) { 527 struct powerpc_jit_data *jit_data = fp->aux->jit_data; 528 struct bpf_binary_header *hdr; 529 void __percpu *priv_stack_ptr; 530 int priv_stack_alloc_size; 531 532 /* 533 * If we fail the final pass of JIT (from jit_subprogs), 534 * the program may not be finalized yet. Call finalize here 535 * before freeing it. 536 */ 537 if (jit_data) { 538 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr); 539 kvfree(jit_data->addrs); 540 kfree(jit_data); 541 } 542 hdr = bpf_jit_binary_pack_hdr(fp); 543 bpf_jit_binary_pack_free(hdr, NULL); 544 priv_stack_ptr = fp->aux->priv_stack_ptr; 545 if (priv_stack_ptr) { 546 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 547 2 * PRIV_STACK_GUARD_SZ; 548 priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp); 549 free_percpu(priv_stack_ptr); 550 } 551 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 552 } 553 554 bpf_prog_unlock_free(fp); 555 } 556 557 bool bpf_jit_supports_exceptions(void) 558 { 559 return IS_ENABLED(CONFIG_PPC64); 560 } 561 562 bool bpf_jit_supports_subprog_tailcalls(void) 563 { 564 return IS_ENABLED(CONFIG_PPC64); 565 } 566 567 bool bpf_jit_supports_timed_may_goto(void) 568 { 569 return IS_ENABLED(CONFIG_PPC64); 570 } 571 572 bool bpf_jit_supports_kfunc_call(void) 573 { 574 return IS_ENABLED(CONFIG_PPC64); 575 } 576 577 bool bpf_jit_supports_private_stack(void) 578 { 579 return IS_ENABLED(CONFIG_PPC64); 580 } 581 582 bool bpf_jit_supports_fsession(void) 583 { 584 /* 585 * TODO: Remove after validating support 586 * for fsession and trampoline on ppc32. 587 */ 588 if (IS_ENABLED(CONFIG_PPC32)) 589 return -EOPNOTSUPP; 590 return true; 591 } 592 593 bool bpf_jit_supports_arena(void) 594 { 595 return IS_ENABLED(CONFIG_PPC64); 596 } 597 598 bool bpf_jit_supports_far_kfunc_call(void) 599 { 600 return IS_ENABLED(CONFIG_PPC64); 601 } 602 603 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 604 { 605 if (!in_arena) 606 return true; 607 switch (insn->code) { 608 case BPF_STX | BPF_ATOMIC | BPF_H: 609 case BPF_STX | BPF_ATOMIC | BPF_B: 610 case BPF_STX | BPF_ATOMIC | BPF_W: 611 case BPF_STX | BPF_ATOMIC | BPF_DW: 612 if (bpf_atomic_is_load_store(insn)) 613 return false; 614 return IS_ENABLED(CONFIG_PPC64); 615 } 616 return true; 617 } 618 619 bool bpf_jit_supports_percpu_insn(void) 620 { 621 return IS_ENABLED(CONFIG_PPC64); 622 } 623 624 bool bpf_jit_inlines_helper_call(s32 imm) 625 { 626 switch (imm) { 627 case BPF_FUNC_get_smp_processor_id: 628 case BPF_FUNC_get_current_task: 629 case BPF_FUNC_get_current_task_btf: 630 return true; 631 default: 632 return false; 633 } 634 } 635 636 void *arch_alloc_bpf_trampoline(unsigned int size) 637 { 638 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); 639 } 640 641 void arch_free_bpf_trampoline(void *image, unsigned int size) 642 { 643 bpf_prog_pack_free(image, size); 644 } 645 646 int arch_protect_bpf_trampoline(void *image, unsigned int size) 647 { 648 return 0; 649 } 650 651 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, 652 struct bpf_tramp_image *im, struct bpf_tramp_node *n, 653 int regs_off, int retval_off, int run_ctx_off, bool save_ret) 654 { 655 struct bpf_prog *p = n->link->prog; 656 ppc_inst_t branch_insn; 657 u32 jmp_idx, skip_idx; 658 int ret = 0; 659 660 /* nop, patched to skip this prog when it is detached */ 661 skip_idx = ctx->idx; 662 EMIT(PPC_RAW_NOP()); 663 664 /* Save cookie */ 665 if (IS_ENABLED(CONFIG_PPC64)) { 666 PPC_LI64(_R3, n->cookie); 667 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx, 668 bpf_cookie))); 669 } else { 670 PPC_LI32(_R3, n->cookie >> 32); 671 PPC_LI32(_R4, n->cookie); 672 EMIT(PPC_RAW_STW(_R3, _R1, 673 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie))); 674 EMIT(PPC_RAW_STW(_R4, _R1, 675 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4)); 676 } 677 678 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */ 679 PPC_LI_ADDR(_R3, p); 680 EMIT(PPC_RAW_MR(_R25, _R3)); 681 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off)); 682 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 683 (unsigned long)bpf_trampoline_enter(p)); 684 if (ret) 685 return ret; 686 687 /* Remember prog start time returned by __bpf_prog_enter */ 688 EMIT(PPC_RAW_MR(_R26, _R3)); 689 690 /* 691 * if (__bpf_prog_enter(p) == 0) 692 * goto skip_exec_of_prog; 693 * 694 * Emit a nop to be later patched with conditional branch, once offset is known 695 */ 696 EMIT(PPC_RAW_CMPLI(_R3, 0)); 697 jmp_idx = ctx->idx; 698 EMIT(PPC_RAW_NOP()); 699 700 /* p->bpf_func(ctx) */ 701 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off)); 702 if (!p->jited) 703 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi); 704 /* Account for max possible instructions during dummy pass for size calculation */ 705 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], 706 (unsigned long)p->bpf_func, 707 BRANCH_SET_LINK)) { 708 image[ctx->idx] = ppc_inst_val(branch_insn); 709 ctx->idx++; 710 } else { 711 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func))); 712 EMIT(PPC_RAW_MTCTR(_R12)); 713 EMIT(PPC_RAW_BCTRL()); 714 } 715 716 if (save_ret) 717 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 718 719 /* Fix up branch */ 720 if (image) { 721 if (create_cond_branch(&branch_insn, &image[jmp_idx], 722 (unsigned long)&image[ctx->idx], COND_EQ << 16)) 723 return -EINVAL; 724 image[jmp_idx] = ppc_inst_val(branch_insn); 725 } 726 727 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */ 728 EMIT(PPC_RAW_MR(_R3, _R25)); 729 EMIT(PPC_RAW_MR(_R4, _R26)); 730 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); 731 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 732 (unsigned long)bpf_trampoline_exit(p)); 733 if (ret) 734 return ret; 735 736 if (ro_image) /* image is NULL for dummy pass */ 737 bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); 738 return 0; 739 } 740 741 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, 742 struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, 743 int regs_off, int retval_off, int run_ctx_off, u32 *branches) 744 { 745 int i; 746 747 /* 748 * The first fmod_ret program will receive a garbage return value. 749 * Set this to 0 to avoid confusing the program. 750 */ 751 EMIT(PPC_RAW_LI(_R3, 0)); 752 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 753 for (i = 0; i < tn->nr_nodes; i++) { 754 if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, 755 retval_off, run_ctx_off, true)) 756 return -EINVAL; 757 758 /* 759 * mod_ret prog stored return value after prog ctx. Emit: 760 * if (*(u64 *)(ret_val) != 0) 761 * goto do_fexit; 762 */ 763 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 764 EMIT(PPC_RAW_CMPLI(_R3, 0)); 765 766 /* 767 * Save the location of the branch and generate a nop, which is 768 * replaced with a conditional jump once do_fexit (i.e. the 769 * start of the fexit invocation) is finalized. 770 */ 771 branches[i] = ctx->idx; 772 EMIT(PPC_RAW_NOP()); 773 } 774 775 return 0; 776 } 777 778 /* 779 * Refer __arch_prepare_bpf_trampoline() for stack component details. 780 * 781 * The tailcall count/reference is present in caller's stack frame. The 782 * tail_call_info is saved at the same offset on the trampoline frame 783 * for the traced function (BPF subprog/callee) to fetch it. 784 */ 785 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, 786 int bpf_frame_size, int r4_off) 787 { 788 if (IS_ENABLED(CONFIG_PPC64)) { 789 EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size)); 790 /* Refer to trampoline's Generated stack layout */ 791 EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL)); 792 793 /* 794 * Setting the tail_call_info in trampoline's frame 795 * depending on if previous frame had value or reference. 796 */ 797 EMIT(PPC_RAW_CMPLLI(_R3, MAX_TAIL_CALL_CNT)); 798 PPC_BCC_CONST_SHORT(COND_GT, 8); 799 EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL)); 800 801 /* 802 * Trampoline's tail_call_info is at the same offset, as that of 803 * any bpf program, with reference to previous frame. Update the 804 * address of main's tail_call_info in trampoline frame. 805 */ 806 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL)); 807 } else { 808 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ 809 EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); 810 } 811 } 812 813 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx, 814 int bpf_frame_size, int r4_off) 815 { 816 if (IS_ENABLED(CONFIG_PPC32)) { 817 /* 818 * Restore tailcall for 32-bit powerpc 819 * See bpf_jit_stack_offsetof() and BPF_PPC_TC 820 */ 821 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 822 } 823 } 824 825 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, 826 int bpf_frame_size, int nr_regs, int regs_off) 827 { 828 int param_save_area_offset; 829 830 param_save_area_offset = bpf_frame_size; 831 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 832 833 for (int i = 0; i < nr_regs; i++) { 834 if (i < 8) { 835 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL)); 836 } else { 837 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 838 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL)); 839 } 840 } 841 } 842 843 /* Used when restoring just the register parameters when returning back */ 844 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx, 845 int nr_regs, int regs_off) 846 { 847 for (int i = 0; i < nr_regs && i < 8; i++) 848 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL)); 849 } 850 851 /* Used when we call into the traced function. Replicate parameter save area */ 852 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx, 853 int bpf_frame_size, int nr_regs, int regs_off) 854 { 855 int param_save_area_offset; 856 857 param_save_area_offset = bpf_frame_size; 858 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 859 860 for (int i = 8; i < nr_regs; i++) { 861 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 862 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL)); 863 } 864 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 865 } 866 867 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image, 868 void *rw_image_end, void *ro_image, 869 const struct btf_func_model *m, u32 flags, 870 struct bpf_tramp_nodes *tnodes, 871 void *func_addr) 872 { 873 int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off; 874 int nvr_off, alt_lr_off, r4_off = 0; 875 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 876 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 877 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 878 int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; 879 struct codegen_context codegen_ctx, *ctx; 880 int cookie_off, cookie_cnt, cookie_ctx_off; 881 int fsession_cnt = bpf_fsession_cnt(tnodes); 882 u64 func_meta; 883 u32 *image = (u32 *)rw_image; 884 ppc_inst_t branch_insn; 885 u32 *branches = NULL; 886 bool save_ret; 887 888 if (IS_ENABLED(CONFIG_PPC32)) 889 return -EOPNOTSUPP; 890 891 nr_regs = m->nr_args; 892 /* Extra registers for struct arguments */ 893 for (i = 0; i < m->nr_args; i++) 894 if (m->arg_size[i] > SZL) 895 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1; 896 897 if (nr_regs > MAX_BPF_FUNC_ARGS) 898 return -EOPNOTSUPP; 899 900 ctx = &codegen_ctx; 901 memset(ctx, 0, sizeof(*ctx)); 902 903 /* 904 * Generated stack layout: 905 * 906 * func prev back chain [ back chain ] 907 * [ tail_call_info ] optional - 64-bit powerpc 908 * [ padding ] align stack frame 909 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc 910 * alt_lr_off [ real lr (ool stub)] optional - actual lr 911 * retaddr_off [ return address ] 912 * [ r26 ] 913 * nvr_off [ r25 ] nvr save area 914 * retval_off [ return value ] 915 * [ reg argN ] 916 * [ ... ] 917 * regs_off [ reg_arg1 ] prog_ctx 918 * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1] 919 * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] 920 * [ stack cookieN ] 921 * [ ... ] 922 * cookie_off [ stack cookie1 ] 923 * run_ctx_off [ bpf_tramp_run_ctx ] 924 * [ reg argN ] 925 * [ ... ] 926 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI 927 * [ TOC save (64-bit) ] -- 928 * [ LR save (64-bit) ] | header 929 * [ LR save (32-bit) ] | 930 * bpf trampoline frame [ back chain 2 ] -- 931 * 932 */ 933 934 /* Minimum stack frame header */ 935 bpf_frame_size = STACK_FRAME_MIN_SIZE; 936 937 /* 938 * Room for parameter save area. 939 * 940 * As per the ABI, this is required if we call into the traced 941 * function (BPF_TRAMP_F_CALL_ORIG): 942 * - if the function takes more than 8 arguments for the rest to spill onto the stack 943 * - or, if the function has variadic arguments 944 * - or, if this functions's prototype was not available to the caller 945 * 946 * Reserve space for at least 8 registers for now. This can be optimized later. 947 */ 948 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL; 949 950 /* Room for struct bpf_tramp_run_ctx */ 951 run_ctx_off = bpf_frame_size; 952 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); 953 954 /* room for session cookies */ 955 cookie_off = bpf_frame_size; 956 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 957 bpf_frame_size += cookie_cnt * 8; 958 959 /* Room for IP address argument */ 960 ip_off = bpf_frame_size; 961 if (flags & BPF_TRAMP_F_IP_ARG) 962 bpf_frame_size += SZL; 963 964 /* Room for function metadata, arg regs count */ 965 func_meta_off = bpf_frame_size; 966 bpf_frame_size += SZL; 967 968 /* Room for arg regs */ 969 regs_off = bpf_frame_size; 970 bpf_frame_size += nr_regs * SZL; 971 972 /* Room for return value of func_addr or fentry prog */ 973 retval_off = bpf_frame_size; 974 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 975 if (save_ret) 976 bpf_frame_size += SZL; 977 978 /* Room for nvr save area */ 979 nvr_off = bpf_frame_size; 980 bpf_frame_size += 2 * SZL; 981 982 /* Save area for return address */ 983 retaddr_off = bpf_frame_size; 984 bpf_frame_size += SZL; 985 986 /* Optional save area for actual LR in case of ool ftrace */ 987 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 988 alt_lr_off = bpf_frame_size; 989 bpf_frame_size += SZL; 990 } 991 992 if (IS_ENABLED(CONFIG_PPC32)) { 993 if (nr_regs < 2) { 994 r4_off = bpf_frame_size; 995 bpf_frame_size += SZL; 996 } else { 997 r4_off = regs_off + SZL; 998 } 999 } 1000 1001 /* 1002 * Save tailcall count pointer at the same offset on the 1003 * stack where subprogs expect it 1004 */ 1005 if ((flags & BPF_TRAMP_F_CALL_ORIG) && 1006 (flags & BPF_TRAMP_F_TAIL_CALL_CTX)) 1007 bpf_frame_size += BPF_PPC_TAILCALL; 1008 1009 /* Padding to align stack frame, if any */ 1010 bpf_frame_size = round_up(bpf_frame_size, SZL * 2); 1011 1012 /* Store original return value */ 1013 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF)); 1014 1015 /* Create our stack frame */ 1016 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size)); 1017 1018 /* 64-bit: Save TOC and load kernel TOC */ 1019 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 1020 EMIT(PPC_RAW_STD(_R2, _R1, 24)); 1021 PPC64_LOAD_PACA(); 1022 } 1023 1024 /* 32-bit: save tail call count in r4 */ 1025 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2) 1026 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 1027 1028 bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off); 1029 1030 /* Save our LR/return address */ 1031 EMIT(PPC_RAW_MFLR(_R3)); 1032 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 1033 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); 1034 else 1035 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 1036 1037 /* 1038 * Derive IP address of the traced function. 1039 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction 1040 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and 1041 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. 1042 * Relevant stub sequence: 1043 * 1044 * bl <tramp> 1045 * LR (R3) => mtlr r0 1046 * b <func_addr+4> 1047 * 1048 * Recover kernel function/bpf program address from the unconditional 1049 * branch instruction at the end of OOL stub. 1050 */ 1051 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { 1052 EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); 1053 EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); 1054 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); 1055 EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); 1056 } 1057 1058 if (flags & BPF_TRAMP_F_IP_ARG) 1059 EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); 1060 1061 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1062 /* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ 1063 EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); 1064 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 1065 } 1066 1067 /* Save function arg regs count -- see bpf_get_func_arg_cnt() */ 1068 func_meta = nr_regs; 1069 store_func_meta(image, ctx, func_meta, func_meta_off); 1070 1071 /* Save nv regs */ 1072 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); 1073 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL)); 1074 1075 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1076 PPC_LI_ADDR(_R3, (unsigned long)im); 1077 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1078 (unsigned long)__bpf_tramp_enter); 1079 if (ret) 1080 return ret; 1081 } 1082 1083 if (fsession_cnt) { 1084 /* 1085 * Clear all the session cookies' values 1086 * Clear the return value to make sure fentry always get 0 1087 */ 1088 prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off); 1089 } 1090 1091 cookie_ctx_off = (regs_off - cookie_off) / 8; 1092 1093 for (i = 0; i < fentry->nr_nodes; i++) { 1094 if (bpf_prog_calls_session_cookie(fentry->nodes[i])) { 1095 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1096 1097 store_func_meta(image, ctx, meta, func_meta_off); 1098 cookie_ctx_off--; 1099 } 1100 1101 if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, 1102 retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) 1103 return -EINVAL; 1104 } 1105 1106 if (fmod_ret->nr_nodes) { 1107 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32), GFP_KERNEL); 1108 if (!branches) 1109 return -ENOMEM; 1110 1111 if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, 1112 run_ctx_off, branches)) { 1113 ret = -EINVAL; 1114 goto cleanup; 1115 } 1116 } 1117 1118 /* Call the traced function */ 1119 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1120 /* 1121 * retaddr on trampoline stack points to the correct point in the original function 1122 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction 1123 * sequence 1124 */ 1125 EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off)); 1126 EMIT(PPC_RAW_MTCTR(_R3)); 1127 1128 /* Replicate tail_call_cnt before calling the original BPF prog */ 1129 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1130 bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off); 1131 1132 /* Restore args */ 1133 bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off); 1134 1135 /* Restore TOC for 64-bit */ 1136 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1137 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1138 EMIT(PPC_RAW_BCTRL()); 1139 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1140 PPC64_LOAD_PACA(); 1141 1142 /* Store return value for bpf prog to access */ 1143 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 1144 1145 /* Restore updated tail_call_cnt */ 1146 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1147 bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); 1148 } 1149 1150 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ 1151 for (i = 0; i < fmod_ret->nr_nodes && image; i++) { 1152 if (create_cond_branch(&branch_insn, &image[branches[i]], 1153 (unsigned long)&image[ctx->idx], COND_NE << 16)) { 1154 ret = -EINVAL; 1155 goto cleanup; 1156 } 1157 1158 image[branches[i]] = ppc_inst_val(branch_insn); 1159 } 1160 1161 /* set the "is_return" flag for fsession */ 1162 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 1163 if (fsession_cnt) 1164 store_func_meta(image, ctx, func_meta, func_meta_off); 1165 1166 cookie_ctx_off = (regs_off - cookie_off) / 8; 1167 1168 for (i = 0; i < fexit->nr_nodes; i++) { 1169 if (bpf_prog_calls_session_cookie(fexit->nodes[i])) { 1170 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1171 1172 store_func_meta(image, ctx, meta, func_meta_off); 1173 cookie_ctx_off--; 1174 } 1175 1176 if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, 1177 retval_off, run_ctx_off, false)) { 1178 ret = -EINVAL; 1179 goto cleanup; 1180 } 1181 } 1182 1183 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1184 PPC_LI_ADDR(_R3, im); 1185 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1186 (unsigned long)__bpf_tramp_exit); 1187 if (ret) 1188 goto cleanup; 1189 } 1190 1191 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1192 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 1193 1194 /* Restore return value of func_addr or fentry prog */ 1195 if (save_ret) 1196 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 1197 1198 /* Restore nv regs */ 1199 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL)); 1200 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off)); 1201 1202 /* Epilogue */ 1203 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1204 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1205 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 1206 /* Skip the traced function and return to parent */ 1207 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1208 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1209 EMIT(PPC_RAW_MTLR(_R0)); 1210 EMIT(PPC_RAW_BLR()); 1211 } else { 1212 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1213 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off)); 1214 EMIT(PPC_RAW_MTLR(_R0)); 1215 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1216 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1217 EMIT(PPC_RAW_BLR()); 1218 } else { 1219 EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off)); 1220 EMIT(PPC_RAW_MTCTR(_R0)); 1221 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1222 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1223 EMIT(PPC_RAW_MTLR(_R0)); 1224 EMIT(PPC_RAW_BCTR()); 1225 } 1226 } 1227 1228 /* Make sure the trampoline generation logic doesn't overflow */ 1229 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) { 1230 ret = -EFAULT; 1231 goto cleanup; 1232 } 1233 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4; 1234 1235 cleanup: 1236 kfree(branches); 1237 return ret; 1238 } 1239 1240 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1241 struct bpf_tramp_nodes *tnodes, void *func_addr) 1242 { 1243 struct bpf_tramp_image im = {}; 1244 int ret; 1245 1246 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); 1247 return ret; 1248 } 1249 1250 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, 1251 const struct btf_func_model *m, u32 flags, 1252 struct bpf_tramp_nodes *tnodes, 1253 void *func_addr) 1254 { 1255 u32 size = image_end - image; 1256 void *rw_image, *tmp; 1257 int ret; 1258 1259 /* 1260 * rw_image doesn't need to be in module memory range, so we can 1261 * use kvmalloc. 1262 */ 1263 rw_image = kvmalloc(size, GFP_KERNEL); 1264 if (!rw_image) 1265 return -ENOMEM; 1266 1267 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m, 1268 flags, tnodes, func_addr); 1269 if (ret < 0) 1270 goto out; 1271 1272 if (bpf_jit_enable > 1) 1273 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image); 1274 1275 tmp = bpf_arch_text_copy(image, rw_image, size); 1276 if (IS_ERR(tmp)) 1277 ret = PTR_ERR(tmp); 1278 1279 out: 1280 kvfree(rw_image); 1281 return ret; 1282 } 1283 1284 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst) 1285 { 1286 ppc_inst_t org_inst; 1287 1288 if (copy_inst_from_kernel_nofault(&org_inst, ip)) { 1289 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip); 1290 return -EFAULT; 1291 } 1292 1293 if (!ppc_inst_equal(org_inst, old_inst)) { 1294 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n", 1295 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst)); 1296 return -EINVAL; 1297 } 1298 1299 if (ppc_inst_equal(old_inst, new_inst)) 1300 return 0; 1301 1302 return patch_instruction(ip, new_inst); 1303 } 1304 1305 static void do_isync(void *info __maybe_unused) 1306 { 1307 isync(); 1308 } 1309 1310 /* 1311 * A 3-step process for bpf prog entry: 1312 * 1. At bpf prog entry, a single nop/b: 1313 * bpf_func: 1314 * [nop|b] ool_stub 1315 * 2. Out-of-line stub: 1316 * nop // optional nop for alignment 1317 * ool_stub: 1318 * mflr r0 1319 * [b|bl] <bpf_prog>/<long_branch_stub> 1320 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only 1321 * b bpf_func + 4 1322 * 3. Long branch stub: 1323 * long_branch_stub: 1324 * mflr r11 1325 * bcl 20,31,$+4 1326 * mflr r12 1327 * ld r12, 20(r12) 1328 * mtctr r12 1329 * mtlr r11 // needed to retain ftrace ABI 1330 * bctr 1331 * .long <branch_addr>/<dummy_tramp> 1332 * 1333 * dummy_tramp is used to reduce synchronization requirements. 1334 * 1335 * When attaching a bpf trampoline to a bpf prog, we do not need any 1336 * synchronization here since we always have a valid branch target regardless 1337 * of the order in which the above stores are seen. dummy_tramp ensures that 1338 * the long_branch stub goes to a valid destination on other cpus, even when 1339 * the branch to the long_branch stub is seen before the updated trampoline 1340 * address. 1341 * 1342 * However, when detaching a bpf trampoline from a bpf prog, or if changing 1343 * the bpf trampoline address, we need synchronization to ensure that other 1344 * cpus can no longer branch into the older trampoline so that it can be 1345 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus 1346 * make forward progress, but we still need to ensure that other cpus 1347 * execute isync (or some CSI) so that they don't go back into the 1348 * trampoline again. 1349 */ 1350 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1351 enum bpf_text_poke_type new_t, void *old_addr, 1352 void *new_addr) 1353 { 1354 unsigned long bpf_func, bpf_func_end, size, offset; 1355 ppc_inst_t old_inst, new_inst; 1356 int ret = 0, branch_flags; 1357 char name[KSYM_NAME_LEN]; 1358 1359 if (IS_ENABLED(CONFIG_PPC32)) 1360 return -EOPNOTSUPP; 1361 1362 bpf_func = (unsigned long)ip; 1363 1364 /* We currently only support poking bpf programs */ 1365 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { 1366 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); 1367 return -EOPNOTSUPP; 1368 } 1369 1370 /* 1371 * If we are not poking at bpf prog entry, then we are simply patching in/out 1372 * an unconditional branch instruction in a trampoline image 1373 */ 1374 if (offset) { 1375 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { 1376 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__, 1377 bpf_func); 1378 return -EOPNOTSUPP; 1379 } 1380 old_inst = ppc_inst(PPC_RAW_NOP()); 1381 if (old_addr) 1382 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0)) 1383 return -ERANGE; 1384 new_inst = ppc_inst(PPC_RAW_NOP()); 1385 if (new_addr) 1386 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0)) 1387 return -ERANGE; 1388 mutex_lock(&text_mutex); 1389 ret = bpf_modify_inst(ip, old_inst, new_inst); 1390 mutex_unlock(&text_mutex); 1391 1392 /* Make sure all cpus see the new instruction */ 1393 smp_call_function(do_isync, NULL, 1); 1394 return ret; 1395 } 1396 1397 bpf_func_end = bpf_func + size; 1398 1399 /* Address of the jmp/call instruction in the out-of-line stub */ 1400 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4); 1401 1402 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) { 1403 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__, 1404 bpf_func); 1405 return -ERANGE; 1406 } 1407 1408 old_inst = ppc_inst(PPC_RAW_NOP()); 1409 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1410 if (old_addr) { 1411 if (is_offset_in_branch_range(ip - old_addr)) 1412 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags); 1413 else 1414 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1415 branch_flags); 1416 } 1417 new_inst = ppc_inst(PPC_RAW_NOP()); 1418 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1419 if (new_addr) { 1420 if (is_offset_in_branch_range(ip - new_addr)) 1421 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags); 1422 else 1423 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1424 branch_flags); 1425 } 1426 1427 mutex_lock(&text_mutex); 1428 1429 /* 1430 * 1. Update the address in the long branch stub: 1431 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr 1432 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here. 1433 * 1434 * dummy_tramp_addr moved to bottom of long branch stub. 1435 */ 1436 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) || 1437 (old_addr && !is_offset_in_branch_range(old_addr - ip))) 1438 ret = patch_ulong((void *)(bpf_func_end - SZL), /* SZL: dummy_tramp_addr offset */ 1439 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ? 1440 (unsigned long)new_addr : (unsigned long)dummy_tramp); 1441 if (ret) 1442 goto out; 1443 1444 /* 2. Update the branch/call in the out-of-line stub */ 1445 ret = bpf_modify_inst(ip, old_inst, new_inst); 1446 if (ret) 1447 goto out; 1448 1449 /* 3. Update instruction at bpf prog entry */ 1450 ip = (void *)bpf_func; 1451 if (!old_addr || !new_addr) { 1452 if (!old_addr) { 1453 old_inst = ppc_inst(PPC_RAW_NOP()); 1454 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1455 } else { 1456 new_inst = ppc_inst(PPC_RAW_NOP()); 1457 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1458 } 1459 ret = bpf_modify_inst(ip, old_inst, new_inst); 1460 } 1461 1462 out: 1463 mutex_unlock(&text_mutex); 1464 1465 /* 1466 * Sync only if we are not attaching a trampoline to a bpf prog so the older 1467 * trampoline can be freed safely. 1468 */ 1469 if (old_addr) 1470 smp_call_function(do_isync, NULL, 1); 1471 1472 return ret; 1473 } 1474