1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <linux/kernel.h> 17 #include <linux/memory.h> 18 #include <linux/bpf.h> 19 20 #include <asm/kprobes.h> 21 #include <asm/text-patching.h> 22 23 #include "bpf_jit.h" 24 25 /* These offsets are from bpf prog end and stay the same across progs */ 26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub; 27 28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 29 { 30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 31 } 32 33 void dummy_tramp(void); 34 35 asm ( 36 " .pushsection .text, \"ax\", @progbits ;" 37 " .global dummy_tramp ;" 38 " .type dummy_tramp, @function ;" 39 "dummy_tramp: ;" 40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 41 " blr ;" 42 #else 43 /* LR is always in r11, so we don't need a 'mflr r11' here */ 44 " mtctr 11 ;" 45 " mtlr 0 ;" 46 " bctr ;" 47 #endif 48 " .size dummy_tramp, .-dummy_tramp ;" 49 " .popsection ;" 50 ); 51 52 void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx) 53 { 54 int ool_stub_idx, long_branch_stub_idx; 55 56 /* 57 * Out-of-line stub: 58 * mflr r0 59 * [b|bl] tramp 60 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE 61 * b bpf_func + 4 62 */ 63 ool_stub_idx = ctx->idx; 64 EMIT(PPC_RAW_MFLR(_R0)); 65 EMIT(PPC_RAW_NOP()); 66 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 67 EMIT(PPC_RAW_MTLR(_R0)); 68 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4)); 69 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4)); 70 71 /* 72 * Long branch stub: 73 * .long <dummy_tramp_addr> 74 * mflr r11 75 * bcl 20,31,$+4 76 * mflr r12 77 * ld r12, -8-SZL(r12) 78 * mtctr r12 79 * mtlr r11 // needed to retain ftrace ABI 80 * bctr 81 */ 82 if (image) 83 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp; 84 ctx->idx += SZL / 4; 85 long_branch_stub_idx = ctx->idx; 86 EMIT(PPC_RAW_MFLR(_R11)); 87 EMIT(PPC_RAW_BCL4()); 88 EMIT(PPC_RAW_MFLR(_R12)); 89 EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL)); 90 EMIT(PPC_RAW_MTCTR(_R12)); 91 EMIT(PPC_RAW_MTLR(_R11)); 92 EMIT(PPC_RAW_BCTR()); 93 94 if (!bpf_jit_ool_stub) { 95 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4; 96 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4; 97 } 98 } 99 100 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) 101 { 102 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { 103 PPC_JMP(exit_addr); 104 } else if (ctx->alt_exit_addr) { 105 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) 106 return -1; 107 PPC_JMP(ctx->alt_exit_addr); 108 } else { 109 ctx->alt_exit_addr = ctx->idx * 4; 110 bpf_jit_build_epilogue(image, ctx); 111 } 112 113 return 0; 114 } 115 116 struct powerpc_jit_data { 117 /* address of rw header */ 118 struct bpf_binary_header *hdr; 119 /* address of ro final header */ 120 struct bpf_binary_header *fhdr; 121 u32 *addrs; 122 u8 *fimage; 123 u32 proglen; 124 struct codegen_context ctx; 125 }; 126 127 bool bpf_jit_needs_zext(void) 128 { 129 return true; 130 } 131 132 static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) 133 { 134 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 135 u64 *stack_ptr; 136 137 for_each_possible_cpu(cpu) { 138 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 139 stack_ptr[0] = PRIV_STACK_GUARD_VAL; 140 stack_ptr[1] = PRIV_STACK_GUARD_VAL; 141 stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL; 142 stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL; 143 } 144 } 145 146 static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size, 147 struct bpf_prog *fp) 148 { 149 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 150 u64 *stack_ptr; 151 152 for_each_possible_cpu(cpu) { 153 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 154 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL || 155 stack_ptr[1] != PRIV_STACK_GUARD_VAL || 156 stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL || 157 stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) { 158 pr_err("BPF private stack overflow/underflow detected for prog %s\n", 159 bpf_jit_get_prog_name(fp)); 160 break; 161 } 162 } 163 } 164 165 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 166 { 167 u32 proglen; 168 u32 alloclen; 169 u8 *image = NULL; 170 u32 *code_base = NULL; 171 u32 *addrs = NULL; 172 struct powerpc_jit_data *jit_data = NULL; 173 struct codegen_context cgctx; 174 int pass; 175 int flen; 176 int priv_stack_alloc_size; 177 void __percpu *priv_stack_ptr = NULL; 178 struct bpf_binary_header *fhdr = NULL; 179 struct bpf_binary_header *hdr = NULL; 180 bool extra_pass = false; 181 u8 *fimage = NULL; 182 u32 *fcode_base = NULL; 183 u32 extable_len; 184 u32 fixup_len; 185 186 if (!fp->jit_requested) 187 return fp; 188 189 jit_data = fp->aux->jit_data; 190 if (!jit_data) { 191 jit_data = kzalloc_obj(*jit_data); 192 if (!jit_data) 193 return fp; 194 fp->aux->jit_data = jit_data; 195 } 196 197 priv_stack_ptr = fp->aux->priv_stack_ptr; 198 if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) { 199 /* 200 * Allocate private stack of size equivalent to 201 * verifier-calculated stack size plus two memory 202 * guard regions to detect private stack overflow 203 * and underflow. 204 */ 205 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 206 2 * PRIV_STACK_GUARD_SZ; 207 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); 208 if (!priv_stack_ptr) 209 goto out_priv_stack; 210 211 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); 212 fp->aux->priv_stack_ptr = priv_stack_ptr; 213 } 214 215 flen = fp->len; 216 addrs = jit_data->addrs; 217 if (addrs) { 218 cgctx = jit_data->ctx; 219 /* 220 * JIT compiled to a writable location (image/code_base) first. 221 * It is then moved to the readonly final location (fimage/fcode_base) 222 * using instruction patching. 223 */ 224 fimage = jit_data->fimage; 225 fhdr = jit_data->fhdr; 226 proglen = jit_data->proglen; 227 hdr = jit_data->hdr; 228 image = (void *)hdr + ((void *)fimage - (void *)fhdr); 229 extra_pass = true; 230 /* During extra pass, ensure index is reset before repopulating extable entries */ 231 cgctx.exentry_idx = 0; 232 goto skip_init_ctx; 233 } 234 235 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 236 if (addrs == NULL) 237 goto out_err; 238 239 memset(&cgctx, 0, sizeof(struct codegen_context)); 240 bpf_jit_init_reg_mapping(&cgctx); 241 242 /* Make sure that the stack is quadword aligned. */ 243 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 244 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); 245 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); 246 cgctx.is_subprog = bpf_is_subprog(fp); 247 cgctx.exception_boundary = fp->aux->exception_boundary; 248 cgctx.exception_cb = fp->aux->exception_cb; 249 cgctx.priv_sp = priv_stack_ptr; 250 cgctx.priv_stack_size = 0; 251 if (priv_stack_ptr) { 252 /* 253 * priv_stack_size required for setting bpf FP inside 254 * percpu allocation. 255 * stack_size is marked 0 to prevent allocation on 256 * general stack and offset calculation don't go for 257 * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local() 258 */ 259 cgctx.priv_stack_size = cgctx.stack_size; 260 cgctx.stack_size = 0; 261 } 262 263 /* Scouting faux-generate pass 0 */ 264 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 265 /* We hit something illegal or unsupported. */ 266 goto out_err; 267 268 /* 269 * If we have seen a tail call, we need a second pass. 270 * This is because bpf_jit_emit_common_epilogue() is called 271 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 272 * We also need a second pass if we ended up with too large 273 * a program so as to ensure BPF_EXIT branches are in range. 274 */ 275 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 276 cgctx.idx = 0; 277 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 278 goto out_err; 279 } 280 281 bpf_jit_realloc_regs(&cgctx); 282 /* 283 * Pretend to build prologue, given the features we've seen. This will 284 * update ctgtx.idx as it pretends to output instructions, then we can 285 * calculate total size from idx. 286 */ 287 bpf_jit_build_prologue(NULL, &cgctx); 288 addrs[fp->len] = cgctx.idx * 4; 289 bpf_jit_build_epilogue(NULL, &cgctx); 290 291 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 292 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 293 294 proglen = cgctx.idx * 4; 295 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 296 297 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 298 bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); 299 if (!fhdr) 300 goto out_err; 301 302 if (extable_len) 303 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; 304 305 skip_init_ctx: 306 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 307 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); 308 309 /* Code generation passes 1-2 */ 310 for (pass = 1; pass < 3; pass++) { 311 /* Now build the prologue, body code & epilogue for real. */ 312 cgctx.idx = 0; 313 cgctx.alt_exit_addr = 0; 314 bpf_jit_build_prologue(code_base, &cgctx); 315 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, 316 extra_pass)) { 317 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 318 bpf_jit_binary_pack_free(fhdr, hdr); 319 goto out_err; 320 } 321 bpf_jit_build_epilogue(code_base, &cgctx); 322 323 if (bpf_jit_enable > 1) 324 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 325 proglen - (cgctx.idx * 4), cgctx.seen); 326 } 327 328 if (bpf_jit_enable > 1) 329 /* 330 * Note that we output the base address of the code_base 331 * rather than image, since opcodes are in code_base. 332 */ 333 bpf_jit_dump(flen, proglen, pass, code_base); 334 335 #ifdef CONFIG_PPC64_ELF_ABI_V1 336 /* Function descriptor nastiness: Address + TOC */ 337 ((u64 *)image)[0] = (u64)fcode_base; 338 ((u64 *)image)[1] = local_paca->kernel_toc; 339 #endif 340 341 if (!fp->is_func || extra_pass) { 342 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) 343 goto out_err; 344 } 345 346 fp->bpf_func = (void *)fimage; 347 fp->jited = 1; 348 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; 349 350 if (!fp->is_func || extra_pass) { 351 bpf_prog_fill_jited_linfo(fp, addrs); 352 /* 353 * On ABI V1, executable code starts after the function 354 * descriptor, so adjust base accordingly. 355 */ 356 bpf_prog_update_insn_ptrs(fp, addrs, 357 (void *)fimage + FUNCTION_DESCR_SIZE); 358 359 out_addrs: 360 if (!image && priv_stack_ptr) { 361 fp->aux->priv_stack_ptr = NULL; 362 free_percpu(priv_stack_ptr); 363 } 364 out_priv_stack: 365 kfree(addrs); 366 kfree(jit_data); 367 fp->aux->jit_data = NULL; 368 } else { 369 jit_data->addrs = addrs; 370 jit_data->ctx = cgctx; 371 jit_data->proglen = proglen; 372 jit_data->fimage = fimage; 373 jit_data->fhdr = fhdr; 374 jit_data->hdr = hdr; 375 } 376 377 return fp; 378 379 out_err: 380 if (extra_pass) { 381 fp->bpf_func = NULL; 382 fp->jited = 0; 383 fp->jited_len = 0; 384 } 385 goto out_addrs; 386 } 387 388 /* 389 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 390 * this function, as this only applies to BPF_PROBE_MEM, for now. 391 */ 392 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, 393 struct codegen_context *ctx, int insn_idx, int jmp_off, 394 int dst_reg, u32 code) 395 { 396 off_t offset; 397 unsigned long pc; 398 struct exception_table_entry *ex, *ex_entry; 399 u32 *fixup; 400 401 /* Populate extable entries only in the last pass */ 402 if (pass != 2) 403 return 0; 404 405 if (!fp->aux->extable || 406 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 407 return -EINVAL; 408 409 /* 410 * Program is first written to image before copying to the 411 * final location (fimage). Accordingly, update in the image first. 412 * As all offsets used are relative, copying as is to the 413 * final location should be alright. 414 */ 415 pc = (unsigned long)&image[insn_idx]; 416 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; 417 418 fixup = (void *)ex - 419 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 420 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 421 422 fixup[0] = PPC_RAW_LI(dst_reg, 0); 423 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX) 424 fixup[0] = PPC_RAW_NOP(); 425 426 if (IS_ENABLED(CONFIG_PPC32)) 427 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 428 429 fixup[BPF_FIXUP_LEN - 1] = 430 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 431 432 ex_entry = &ex[ctx->exentry_idx]; 433 434 offset = pc - (long)&ex_entry->insn; 435 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 436 return -ERANGE; 437 ex_entry->insn = offset; 438 439 offset = (long)fixup - (long)&ex_entry->fixup; 440 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 441 return -ERANGE; 442 ex_entry->fixup = offset; 443 444 ctx->exentry_idx++; 445 return 0; 446 } 447 448 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 449 { 450 int err; 451 452 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 453 return ERR_PTR(-EINVAL); 454 455 mutex_lock(&text_mutex); 456 err = patch_instructions(dst, src, len, false); 457 mutex_unlock(&text_mutex); 458 459 return err ? ERR_PTR(err) : dst; 460 } 461 462 int bpf_arch_text_invalidate(void *dst, size_t len) 463 { 464 u32 insn = BREAKPOINT_INSTRUCTION; 465 int ret; 466 467 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 468 return -EINVAL; 469 470 mutex_lock(&text_mutex); 471 ret = patch_instructions(dst, &insn, len, true); 472 mutex_unlock(&text_mutex); 473 474 return ret; 475 } 476 477 void bpf_jit_free(struct bpf_prog *fp) 478 { 479 if (fp->jited) { 480 struct powerpc_jit_data *jit_data = fp->aux->jit_data; 481 struct bpf_binary_header *hdr; 482 void __percpu *priv_stack_ptr; 483 int priv_stack_alloc_size; 484 485 /* 486 * If we fail the final pass of JIT (from jit_subprogs), 487 * the program may not be finalized yet. Call finalize here 488 * before freeing it. 489 */ 490 if (jit_data) { 491 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr); 492 kvfree(jit_data->addrs); 493 kfree(jit_data); 494 } 495 hdr = bpf_jit_binary_pack_hdr(fp); 496 bpf_jit_binary_pack_free(hdr, NULL); 497 priv_stack_ptr = fp->aux->priv_stack_ptr; 498 if (priv_stack_ptr) { 499 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 500 2 * PRIV_STACK_GUARD_SZ; 501 priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp); 502 free_percpu(priv_stack_ptr); 503 } 504 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 505 } 506 507 bpf_prog_unlock_free(fp); 508 } 509 510 bool bpf_jit_supports_exceptions(void) 511 { 512 return IS_ENABLED(CONFIG_PPC64); 513 } 514 515 bool bpf_jit_supports_subprog_tailcalls(void) 516 { 517 return IS_ENABLED(CONFIG_PPC64); 518 } 519 520 bool bpf_jit_supports_timed_may_goto(void) 521 { 522 return IS_ENABLED(CONFIG_PPC64); 523 } 524 525 bool bpf_jit_supports_kfunc_call(void) 526 { 527 return IS_ENABLED(CONFIG_PPC64); 528 } 529 530 bool bpf_jit_supports_private_stack(void) 531 { 532 return IS_ENABLED(CONFIG_PPC64); 533 } 534 535 bool bpf_jit_supports_fsession(void) 536 { 537 /* 538 * TODO: Remove after validating support 539 * for fsession and trampoline on ppc32. 540 */ 541 if (IS_ENABLED(CONFIG_PPC32)) 542 return -EOPNOTSUPP; 543 return true; 544 } 545 546 bool bpf_jit_supports_arena(void) 547 { 548 return IS_ENABLED(CONFIG_PPC64); 549 } 550 551 bool bpf_jit_supports_far_kfunc_call(void) 552 { 553 return IS_ENABLED(CONFIG_PPC64); 554 } 555 556 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 557 { 558 if (!in_arena) 559 return true; 560 switch (insn->code) { 561 case BPF_STX | BPF_ATOMIC | BPF_H: 562 case BPF_STX | BPF_ATOMIC | BPF_B: 563 case BPF_STX | BPF_ATOMIC | BPF_W: 564 case BPF_STX | BPF_ATOMIC | BPF_DW: 565 if (bpf_atomic_is_load_store(insn)) 566 return false; 567 return IS_ENABLED(CONFIG_PPC64); 568 } 569 return true; 570 } 571 572 bool bpf_jit_supports_percpu_insn(void) 573 { 574 return IS_ENABLED(CONFIG_PPC64); 575 } 576 577 bool bpf_jit_inlines_helper_call(s32 imm) 578 { 579 switch (imm) { 580 case BPF_FUNC_get_smp_processor_id: 581 case BPF_FUNC_get_current_task: 582 case BPF_FUNC_get_current_task_btf: 583 return true; 584 default: 585 return false; 586 } 587 } 588 589 void *arch_alloc_bpf_trampoline(unsigned int size) 590 { 591 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); 592 } 593 594 void arch_free_bpf_trampoline(void *image, unsigned int size) 595 { 596 bpf_prog_pack_free(image, size); 597 } 598 599 int arch_protect_bpf_trampoline(void *image, unsigned int size) 600 { 601 return 0; 602 } 603 604 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, 605 struct bpf_tramp_image *im, struct bpf_tramp_node *n, 606 int regs_off, int retval_off, int run_ctx_off, bool save_ret) 607 { 608 struct bpf_prog *p = n->link->prog; 609 ppc_inst_t branch_insn; 610 u32 jmp_idx, skip_idx; 611 int ret = 0; 612 613 /* nop, patched to skip this prog when it is detached */ 614 skip_idx = ctx->idx; 615 EMIT(PPC_RAW_NOP()); 616 617 /* Save cookie */ 618 if (IS_ENABLED(CONFIG_PPC64)) { 619 PPC_LI64(_R3, n->cookie); 620 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx, 621 bpf_cookie))); 622 } else { 623 PPC_LI32(_R3, n->cookie >> 32); 624 PPC_LI32(_R4, n->cookie); 625 EMIT(PPC_RAW_STW(_R3, _R1, 626 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie))); 627 EMIT(PPC_RAW_STW(_R4, _R1, 628 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4)); 629 } 630 631 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */ 632 PPC_LI_ADDR(_R3, p); 633 EMIT(PPC_RAW_MR(_R25, _R3)); 634 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off)); 635 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 636 (unsigned long)bpf_trampoline_enter(p)); 637 if (ret) 638 return ret; 639 640 /* Remember prog start time returned by __bpf_prog_enter */ 641 EMIT(PPC_RAW_MR(_R26, _R3)); 642 643 /* 644 * if (__bpf_prog_enter(p) == 0) 645 * goto skip_exec_of_prog; 646 * 647 * Emit a nop to be later patched with conditional branch, once offset is known 648 */ 649 EMIT(PPC_RAW_CMPLI(_R3, 0)); 650 jmp_idx = ctx->idx; 651 EMIT(PPC_RAW_NOP()); 652 653 /* p->bpf_func(ctx) */ 654 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off)); 655 if (!p->jited) 656 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi); 657 /* Account for max possible instructions during dummy pass for size calculation */ 658 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], 659 (unsigned long)p->bpf_func, 660 BRANCH_SET_LINK)) { 661 image[ctx->idx] = ppc_inst_val(branch_insn); 662 ctx->idx++; 663 } else { 664 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func))); 665 EMIT(PPC_RAW_MTCTR(_R12)); 666 EMIT(PPC_RAW_BCTRL()); 667 } 668 669 if (save_ret) 670 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 671 672 /* Fix up branch */ 673 if (image) { 674 if (create_cond_branch(&branch_insn, &image[jmp_idx], 675 (unsigned long)&image[ctx->idx], COND_EQ << 16)) 676 return -EINVAL; 677 image[jmp_idx] = ppc_inst_val(branch_insn); 678 } 679 680 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */ 681 EMIT(PPC_RAW_MR(_R3, _R25)); 682 EMIT(PPC_RAW_MR(_R4, _R26)); 683 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); 684 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 685 (unsigned long)bpf_trampoline_exit(p)); 686 if (ret) 687 return ret; 688 689 if (ro_image) /* image is NULL for dummy pass */ 690 bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); 691 return 0; 692 } 693 694 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, 695 struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, 696 int regs_off, int retval_off, int run_ctx_off, u32 *branches) 697 { 698 int i; 699 700 /* 701 * The first fmod_ret program will receive a garbage return value. 702 * Set this to 0 to avoid confusing the program. 703 */ 704 EMIT(PPC_RAW_LI(_R3, 0)); 705 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 706 for (i = 0; i < tn->nr_nodes; i++) { 707 if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, 708 retval_off, run_ctx_off, true)) 709 return -EINVAL; 710 711 /* 712 * mod_ret prog stored return value after prog ctx. Emit: 713 * if (*(u64 *)(ret_val) != 0) 714 * goto do_fexit; 715 */ 716 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 717 EMIT(PPC_RAW_CMPLI(_R3, 0)); 718 719 /* 720 * Save the location of the branch and generate a nop, which is 721 * replaced with a conditional jump once do_fexit (i.e. the 722 * start of the fexit invocation) is finalized. 723 */ 724 branches[i] = ctx->idx; 725 EMIT(PPC_RAW_NOP()); 726 } 727 728 return 0; 729 } 730 731 /* 732 * Refer __arch_prepare_bpf_trampoline() for stack component details. 733 * 734 * The tailcall count/reference is present in caller's stack frame. The 735 * tail_call_info is saved at the same offset on the trampoline frame 736 * for the traced function (BPF subprog/callee) to fetch it. 737 */ 738 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, 739 int bpf_frame_size, int r4_off) 740 { 741 if (IS_ENABLED(CONFIG_PPC64)) { 742 EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size)); 743 /* Refer to trampoline's Generated stack layout */ 744 EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL)); 745 746 /* 747 * Setting the tail_call_info in trampoline's frame 748 * depending on if previous frame had value or reference. 749 */ 750 EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT)); 751 PPC_BCC_CONST_SHORT(COND_GT, 8); 752 EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL)); 753 754 /* 755 * Trampoline's tail_call_info is at the same offset, as that of 756 * any bpf program, with reference to previous frame. Update the 757 * address of main's tail_call_info in trampoline frame. 758 */ 759 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL)); 760 } else { 761 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ 762 EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); 763 } 764 } 765 766 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx, 767 int bpf_frame_size, int r4_off) 768 { 769 if (IS_ENABLED(CONFIG_PPC32)) { 770 /* 771 * Restore tailcall for 32-bit powerpc 772 * See bpf_jit_stack_offsetof() and BPF_PPC_TC 773 */ 774 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 775 } 776 } 777 778 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, 779 int bpf_frame_size, int nr_regs, int regs_off) 780 { 781 int param_save_area_offset; 782 783 param_save_area_offset = bpf_frame_size; 784 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 785 786 for (int i = 0; i < nr_regs; i++) { 787 if (i < 8) { 788 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL)); 789 } else { 790 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 791 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL)); 792 } 793 } 794 } 795 796 /* Used when restoring just the register parameters when returning back */ 797 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx, 798 int nr_regs, int regs_off) 799 { 800 for (int i = 0; i < nr_regs && i < 8; i++) 801 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL)); 802 } 803 804 /* Used when we call into the traced function. Replicate parameter save area */ 805 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx, 806 int bpf_frame_size, int nr_regs, int regs_off) 807 { 808 int param_save_area_offset; 809 810 param_save_area_offset = bpf_frame_size; 811 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 812 813 for (int i = 8; i < nr_regs; i++) { 814 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 815 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL)); 816 } 817 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 818 } 819 820 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image, 821 void *rw_image_end, void *ro_image, 822 const struct btf_func_model *m, u32 flags, 823 struct bpf_tramp_nodes *tnodes, 824 void *func_addr) 825 { 826 int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off; 827 int nvr_off, alt_lr_off, r4_off = 0; 828 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 829 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 830 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 831 int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; 832 struct codegen_context codegen_ctx, *ctx; 833 int cookie_off, cookie_cnt, cookie_ctx_off; 834 int fsession_cnt = bpf_fsession_cnt(tnodes); 835 u64 func_meta; 836 u32 *image = (u32 *)rw_image; 837 ppc_inst_t branch_insn; 838 u32 *branches = NULL; 839 bool save_ret; 840 841 if (IS_ENABLED(CONFIG_PPC32)) 842 return -EOPNOTSUPP; 843 844 nr_regs = m->nr_args; 845 /* Extra registers for struct arguments */ 846 for (i = 0; i < m->nr_args; i++) 847 if (m->arg_size[i] > SZL) 848 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1; 849 850 if (nr_regs > MAX_BPF_FUNC_ARGS) 851 return -EOPNOTSUPP; 852 853 ctx = &codegen_ctx; 854 memset(ctx, 0, sizeof(*ctx)); 855 856 /* 857 * Generated stack layout: 858 * 859 * func prev back chain [ back chain ] 860 * [ tail_call_info ] optional - 64-bit powerpc 861 * [ padding ] align stack frame 862 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc 863 * alt_lr_off [ real lr (ool stub)] optional - actual lr 864 * retaddr_off [ return address ] 865 * [ r26 ] 866 * nvr_off [ r25 ] nvr save area 867 * retval_off [ return value ] 868 * [ reg argN ] 869 * [ ... ] 870 * regs_off [ reg_arg1 ] prog_ctx 871 * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1] 872 * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] 873 * [ stack cookieN ] 874 * [ ... ] 875 * cookie_off [ stack cookie1 ] 876 * run_ctx_off [ bpf_tramp_run_ctx ] 877 * [ reg argN ] 878 * [ ... ] 879 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI 880 * [ TOC save (64-bit) ] -- 881 * [ LR save (64-bit) ] | header 882 * [ LR save (32-bit) ] | 883 * bpf trampoline frame [ back chain 2 ] -- 884 * 885 */ 886 887 /* Minimum stack frame header */ 888 bpf_frame_size = STACK_FRAME_MIN_SIZE; 889 890 /* 891 * Room for parameter save area. 892 * 893 * As per the ABI, this is required if we call into the traced 894 * function (BPF_TRAMP_F_CALL_ORIG): 895 * - if the function takes more than 8 arguments for the rest to spill onto the stack 896 * - or, if the function has variadic arguments 897 * - or, if this functions's prototype was not available to the caller 898 * 899 * Reserve space for at least 8 registers for now. This can be optimized later. 900 */ 901 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL; 902 903 /* Room for struct bpf_tramp_run_ctx */ 904 run_ctx_off = bpf_frame_size; 905 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); 906 907 /* room for session cookies */ 908 cookie_off = bpf_frame_size; 909 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 910 bpf_frame_size += cookie_cnt * 8; 911 912 /* Room for IP address argument */ 913 ip_off = bpf_frame_size; 914 if (flags & BPF_TRAMP_F_IP_ARG) 915 bpf_frame_size += SZL; 916 917 /* Room for function metadata, arg regs count */ 918 func_meta_off = bpf_frame_size; 919 bpf_frame_size += SZL; 920 921 /* Room for arg regs */ 922 regs_off = bpf_frame_size; 923 bpf_frame_size += nr_regs * SZL; 924 925 /* Room for return value of func_addr or fentry prog */ 926 retval_off = bpf_frame_size; 927 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 928 if (save_ret) 929 bpf_frame_size += SZL; 930 931 /* Room for nvr save area */ 932 nvr_off = bpf_frame_size; 933 bpf_frame_size += 2 * SZL; 934 935 /* Save area for return address */ 936 retaddr_off = bpf_frame_size; 937 bpf_frame_size += SZL; 938 939 /* Optional save area for actual LR in case of ool ftrace */ 940 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 941 alt_lr_off = bpf_frame_size; 942 bpf_frame_size += SZL; 943 } 944 945 if (IS_ENABLED(CONFIG_PPC32)) { 946 if (nr_regs < 2) { 947 r4_off = bpf_frame_size; 948 bpf_frame_size += SZL; 949 } else { 950 r4_off = regs_off + SZL; 951 } 952 } 953 954 /* 955 * Save tailcall count pointer at the same offset on the 956 * stack where subprogs expect it 957 */ 958 if ((flags & BPF_TRAMP_F_CALL_ORIG) && 959 (flags & BPF_TRAMP_F_TAIL_CALL_CTX)) 960 bpf_frame_size += BPF_PPC_TAILCALL; 961 962 /* Padding to align stack frame, if any */ 963 bpf_frame_size = round_up(bpf_frame_size, SZL * 2); 964 965 /* Store original return value */ 966 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF)); 967 968 /* Create our stack frame */ 969 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size)); 970 971 /* 64-bit: Save TOC and load kernel TOC */ 972 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 973 EMIT(PPC_RAW_STD(_R2, _R1, 24)); 974 PPC64_LOAD_PACA(); 975 } 976 977 /* 32-bit: save tail call count in r4 */ 978 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2) 979 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 980 981 bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off); 982 983 /* Save our LR/return address */ 984 EMIT(PPC_RAW_MFLR(_R3)); 985 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 986 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); 987 else 988 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 989 990 /* 991 * Derive IP address of the traced function. 992 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction 993 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and 994 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. 995 * Relevant stub sequence: 996 * 997 * bl <tramp> 998 * LR (R3) => mtlr r0 999 * b <func_addr+4> 1000 * 1001 * Recover kernel function/bpf program address from the unconditional 1002 * branch instruction at the end of OOL stub. 1003 */ 1004 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { 1005 EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); 1006 EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); 1007 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); 1008 EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); 1009 } 1010 1011 if (flags & BPF_TRAMP_F_IP_ARG) 1012 EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); 1013 1014 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1015 /* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ 1016 EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); 1017 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 1018 } 1019 1020 /* Save function arg regs count -- see bpf_get_func_arg_cnt() */ 1021 func_meta = nr_regs; 1022 store_func_meta(image, ctx, func_meta, func_meta_off); 1023 1024 /* Save nv regs */ 1025 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); 1026 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL)); 1027 1028 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1029 PPC_LI_ADDR(_R3, (unsigned long)im); 1030 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1031 (unsigned long)__bpf_tramp_enter); 1032 if (ret) 1033 return ret; 1034 } 1035 1036 if (fsession_cnt) { 1037 /* 1038 * Clear all the session cookies' values 1039 * Clear the return value to make sure fentry always get 0 1040 */ 1041 prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off); 1042 } 1043 1044 cookie_ctx_off = (regs_off - cookie_off) / 8; 1045 1046 for (i = 0; i < fentry->nr_nodes; i++) { 1047 if (bpf_prog_calls_session_cookie(fentry->nodes[i])) { 1048 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1049 1050 store_func_meta(image, ctx, meta, func_meta_off); 1051 cookie_ctx_off--; 1052 } 1053 1054 if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, 1055 retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) 1056 return -EINVAL; 1057 } 1058 1059 if (fmod_ret->nr_nodes) { 1060 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32), GFP_KERNEL); 1061 if (!branches) 1062 return -ENOMEM; 1063 1064 if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, 1065 run_ctx_off, branches)) { 1066 ret = -EINVAL; 1067 goto cleanup; 1068 } 1069 } 1070 1071 /* Call the traced function */ 1072 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1073 /* 1074 * retaddr on trampoline stack points to the correct point in the original function 1075 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction 1076 * sequence 1077 */ 1078 EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off)); 1079 EMIT(PPC_RAW_MTCTR(_R3)); 1080 1081 /* Replicate tail_call_cnt before calling the original BPF prog */ 1082 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1083 bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off); 1084 1085 /* Restore args */ 1086 bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off); 1087 1088 /* Restore TOC for 64-bit */ 1089 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1090 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1091 EMIT(PPC_RAW_BCTRL()); 1092 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1093 PPC64_LOAD_PACA(); 1094 1095 /* Store return value for bpf prog to access */ 1096 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 1097 1098 /* Restore updated tail_call_cnt */ 1099 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1100 bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); 1101 } 1102 1103 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ 1104 for (i = 0; i < fmod_ret->nr_nodes && image; i++) { 1105 if (create_cond_branch(&branch_insn, &image[branches[i]], 1106 (unsigned long)&image[ctx->idx], COND_NE << 16)) { 1107 ret = -EINVAL; 1108 goto cleanup; 1109 } 1110 1111 image[branches[i]] = ppc_inst_val(branch_insn); 1112 } 1113 1114 /* set the "is_return" flag for fsession */ 1115 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 1116 if (fsession_cnt) 1117 store_func_meta(image, ctx, func_meta, func_meta_off); 1118 1119 cookie_ctx_off = (regs_off - cookie_off) / 8; 1120 1121 for (i = 0; i < fexit->nr_nodes; i++) { 1122 if (bpf_prog_calls_session_cookie(fexit->nodes[i])) { 1123 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1124 1125 store_func_meta(image, ctx, meta, func_meta_off); 1126 cookie_ctx_off--; 1127 } 1128 1129 if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, 1130 retval_off, run_ctx_off, false)) { 1131 ret = -EINVAL; 1132 goto cleanup; 1133 } 1134 } 1135 1136 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1137 PPC_LI_ADDR(_R3, im); 1138 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1139 (unsigned long)__bpf_tramp_exit); 1140 if (ret) 1141 goto cleanup; 1142 } 1143 1144 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1145 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 1146 1147 /* Restore return value of func_addr or fentry prog */ 1148 if (save_ret) 1149 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 1150 1151 /* Restore nv regs */ 1152 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL)); 1153 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off)); 1154 1155 /* Epilogue */ 1156 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1157 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1158 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 1159 /* Skip the traced function and return to parent */ 1160 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1161 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1162 EMIT(PPC_RAW_MTLR(_R0)); 1163 EMIT(PPC_RAW_BLR()); 1164 } else { 1165 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1166 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off)); 1167 EMIT(PPC_RAW_MTLR(_R0)); 1168 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1169 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1170 EMIT(PPC_RAW_BLR()); 1171 } else { 1172 EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off)); 1173 EMIT(PPC_RAW_MTCTR(_R0)); 1174 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1175 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1176 EMIT(PPC_RAW_MTLR(_R0)); 1177 EMIT(PPC_RAW_BCTR()); 1178 } 1179 } 1180 1181 /* Make sure the trampoline generation logic doesn't overflow */ 1182 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) { 1183 ret = -EFAULT; 1184 goto cleanup; 1185 } 1186 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4; 1187 1188 cleanup: 1189 kfree(branches); 1190 return ret; 1191 } 1192 1193 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1194 struct bpf_tramp_nodes *tnodes, void *func_addr) 1195 { 1196 struct bpf_tramp_image im = {}; 1197 int ret; 1198 1199 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); 1200 return ret; 1201 } 1202 1203 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, 1204 const struct btf_func_model *m, u32 flags, 1205 struct bpf_tramp_nodes *tnodes, 1206 void *func_addr) 1207 { 1208 u32 size = image_end - image; 1209 void *rw_image, *tmp; 1210 int ret; 1211 1212 /* 1213 * rw_image doesn't need to be in module memory range, so we can 1214 * use kvmalloc. 1215 */ 1216 rw_image = kvmalloc(size, GFP_KERNEL); 1217 if (!rw_image) 1218 return -ENOMEM; 1219 1220 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m, 1221 flags, tnodes, func_addr); 1222 if (ret < 0) 1223 goto out; 1224 1225 if (bpf_jit_enable > 1) 1226 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image); 1227 1228 tmp = bpf_arch_text_copy(image, rw_image, size); 1229 if (IS_ERR(tmp)) 1230 ret = PTR_ERR(tmp); 1231 1232 out: 1233 kvfree(rw_image); 1234 return ret; 1235 } 1236 1237 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst) 1238 { 1239 ppc_inst_t org_inst; 1240 1241 if (copy_inst_from_kernel_nofault(&org_inst, ip)) { 1242 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip); 1243 return -EFAULT; 1244 } 1245 1246 if (!ppc_inst_equal(org_inst, old_inst)) { 1247 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n", 1248 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst)); 1249 return -EINVAL; 1250 } 1251 1252 if (ppc_inst_equal(old_inst, new_inst)) 1253 return 0; 1254 1255 return patch_instruction(ip, new_inst); 1256 } 1257 1258 static void do_isync(void *info __maybe_unused) 1259 { 1260 isync(); 1261 } 1262 1263 /* 1264 * A 3-step process for bpf prog entry: 1265 * 1. At bpf prog entry, a single nop/b: 1266 * bpf_func: 1267 * [nop|b] ool_stub 1268 * 2. Out-of-line stub: 1269 * ool_stub: 1270 * mflr r0 1271 * [b|bl] <bpf_prog>/<long_branch_stub> 1272 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only 1273 * b bpf_func + 4 1274 * 3. Long branch stub: 1275 * long_branch_stub: 1276 * .long <branch_addr>/<dummy_tramp> 1277 * mflr r11 1278 * bcl 20,31,$+4 1279 * mflr r12 1280 * ld r12, -16(r12) 1281 * mtctr r12 1282 * mtlr r11 // needed to retain ftrace ABI 1283 * bctr 1284 * 1285 * dummy_tramp is used to reduce synchronization requirements. 1286 * 1287 * When attaching a bpf trampoline to a bpf prog, we do not need any 1288 * synchronization here since we always have a valid branch target regardless 1289 * of the order in which the above stores are seen. dummy_tramp ensures that 1290 * the long_branch stub goes to a valid destination on other cpus, even when 1291 * the branch to the long_branch stub is seen before the updated trampoline 1292 * address. 1293 * 1294 * However, when detaching a bpf trampoline from a bpf prog, or if changing 1295 * the bpf trampoline address, we need synchronization to ensure that other 1296 * cpus can no longer branch into the older trampoline so that it can be 1297 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus 1298 * make forward progress, but we still need to ensure that other cpus 1299 * execute isync (or some CSI) so that they don't go back into the 1300 * trampoline again. 1301 */ 1302 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1303 enum bpf_text_poke_type new_t, void *old_addr, 1304 void *new_addr) 1305 { 1306 unsigned long bpf_func, bpf_func_end, size, offset; 1307 ppc_inst_t old_inst, new_inst; 1308 int ret = 0, branch_flags; 1309 char name[KSYM_NAME_LEN]; 1310 1311 if (IS_ENABLED(CONFIG_PPC32)) 1312 return -EOPNOTSUPP; 1313 1314 bpf_func = (unsigned long)ip; 1315 1316 /* We currently only support poking bpf programs */ 1317 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { 1318 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); 1319 return -EOPNOTSUPP; 1320 } 1321 1322 /* 1323 * If we are not poking at bpf prog entry, then we are simply patching in/out 1324 * an unconditional branch instruction in a trampoline image 1325 */ 1326 if (offset) { 1327 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { 1328 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__, 1329 bpf_func); 1330 return -EOPNOTSUPP; 1331 } 1332 old_inst = ppc_inst(PPC_RAW_NOP()); 1333 if (old_addr) 1334 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0)) 1335 return -ERANGE; 1336 new_inst = ppc_inst(PPC_RAW_NOP()); 1337 if (new_addr) 1338 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0)) 1339 return -ERANGE; 1340 mutex_lock(&text_mutex); 1341 ret = bpf_modify_inst(ip, old_inst, new_inst); 1342 mutex_unlock(&text_mutex); 1343 1344 /* Make sure all cpus see the new instruction */ 1345 smp_call_function(do_isync, NULL, 1); 1346 return ret; 1347 } 1348 1349 bpf_func_end = bpf_func + size; 1350 1351 /* Address of the jmp/call instruction in the out-of-line stub */ 1352 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4); 1353 1354 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) { 1355 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__, 1356 bpf_func); 1357 return -ERANGE; 1358 } 1359 1360 old_inst = ppc_inst(PPC_RAW_NOP()); 1361 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1362 if (old_addr) { 1363 if (is_offset_in_branch_range(ip - old_addr)) 1364 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags); 1365 else 1366 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1367 branch_flags); 1368 } 1369 new_inst = ppc_inst(PPC_RAW_NOP()); 1370 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1371 if (new_addr) { 1372 if (is_offset_in_branch_range(ip - new_addr)) 1373 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags); 1374 else 1375 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1376 branch_flags); 1377 } 1378 1379 mutex_lock(&text_mutex); 1380 1381 /* 1382 * 1. Update the address in the long branch stub: 1383 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr 1384 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here. 1385 */ 1386 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) || 1387 (old_addr && !is_offset_in_branch_range(old_addr - ip))) 1388 ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL), 1389 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ? 1390 (unsigned long)new_addr : (unsigned long)dummy_tramp); 1391 if (ret) 1392 goto out; 1393 1394 /* 2. Update the branch/call in the out-of-line stub */ 1395 ret = bpf_modify_inst(ip, old_inst, new_inst); 1396 if (ret) 1397 goto out; 1398 1399 /* 3. Update instruction at bpf prog entry */ 1400 ip = (void *)bpf_func; 1401 if (!old_addr || !new_addr) { 1402 if (!old_addr) { 1403 old_inst = ppc_inst(PPC_RAW_NOP()); 1404 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1405 } else { 1406 new_inst = ppc_inst(PPC_RAW_NOP()); 1407 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1408 } 1409 ret = bpf_modify_inst(ip, old_inst, new_inst); 1410 } 1411 1412 out: 1413 mutex_unlock(&text_mutex); 1414 1415 /* 1416 * Sync only if we are not attaching a trampoline to a bpf prog so the older 1417 * trampoline can be freed safely. 1418 */ 1419 if (old_addr) 1420 smp_call_function(do_isync, NULL, 1); 1421 1422 return ret; 1423 } 1424