1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <linux/kernel.h> 17 #include <linux/memory.h> 18 #include <linux/bpf.h> 19 20 #include <asm/kprobes.h> 21 #include <asm/text-patching.h> 22 23 #include "bpf_jit.h" 24 25 /* These offsets are from bpf prog end and stay the same across progs */ 26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub; 27 28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 29 { 30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 31 } 32 33 void dummy_tramp(void); 34 35 asm ( 36 " .pushsection .text, \"ax\", @progbits ;" 37 " .global dummy_tramp ;" 38 " .type dummy_tramp, @function ;" 39 "dummy_tramp: ;" 40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 41 " blr ;" 42 #else 43 /* LR is always in r11, so we don't need a 'mflr r11' here */ 44 " mtctr 11 ;" 45 " mtlr 0 ;" 46 " bctr ;" 47 #endif 48 " .size dummy_tramp, .-dummy_tramp ;" 49 " .popsection ;" 50 ); 51 52 void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx) 53 { 54 int ool_stub_idx, long_branch_stub_idx; 55 56 /* 57 * Out-of-line stub: 58 * mflr r0 59 * [b|bl] tramp 60 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE 61 * b bpf_func + 4 62 */ 63 ool_stub_idx = ctx->idx; 64 EMIT(PPC_RAW_MFLR(_R0)); 65 EMIT(PPC_RAW_NOP()); 66 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 67 EMIT(PPC_RAW_MTLR(_R0)); 68 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4)); 69 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4)); 70 71 /* 72 * Long branch stub: 73 * .long <dummy_tramp_addr> 74 * mflr r11 75 * bcl 20,31,$+4 76 * mflr r12 77 * ld r12, -8-SZL(r12) 78 * mtctr r12 79 * mtlr r11 // needed to retain ftrace ABI 80 * bctr 81 */ 82 if (image) 83 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp; 84 ctx->idx += SZL / 4; 85 long_branch_stub_idx = ctx->idx; 86 EMIT(PPC_RAW_MFLR(_R11)); 87 EMIT(PPC_RAW_BCL4()); 88 EMIT(PPC_RAW_MFLR(_R12)); 89 EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL)); 90 EMIT(PPC_RAW_MTCTR(_R12)); 91 EMIT(PPC_RAW_MTLR(_R11)); 92 EMIT(PPC_RAW_BCTR()); 93 94 if (!bpf_jit_ool_stub) { 95 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4; 96 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4; 97 } 98 } 99 100 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) 101 { 102 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { 103 PPC_JMP(exit_addr); 104 } else if (ctx->alt_exit_addr) { 105 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) 106 return -1; 107 PPC_JMP(ctx->alt_exit_addr); 108 } else { 109 ctx->alt_exit_addr = ctx->idx * 4; 110 bpf_jit_build_epilogue(image, ctx); 111 } 112 113 return 0; 114 } 115 116 struct powerpc_jit_data { 117 /* address of rw header */ 118 struct bpf_binary_header *hdr; 119 /* address of ro final header */ 120 struct bpf_binary_header *fhdr; 121 u32 *addrs; 122 u8 *fimage; 123 u32 proglen; 124 struct codegen_context ctx; 125 }; 126 127 bool bpf_jit_needs_zext(void) 128 { 129 return true; 130 } 131 132 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 133 { 134 u32 proglen; 135 u32 alloclen; 136 u8 *image = NULL; 137 u32 *code_base; 138 u32 *addrs; 139 struct powerpc_jit_data *jit_data; 140 struct codegen_context cgctx; 141 int pass; 142 int flen; 143 struct bpf_binary_header *fhdr = NULL; 144 struct bpf_binary_header *hdr = NULL; 145 struct bpf_prog *org_fp = fp; 146 struct bpf_prog *tmp_fp; 147 bool bpf_blinded = false; 148 bool extra_pass = false; 149 u8 *fimage = NULL; 150 u32 *fcode_base; 151 u32 extable_len; 152 u32 fixup_len; 153 154 if (!fp->jit_requested) 155 return org_fp; 156 157 tmp_fp = bpf_jit_blind_constants(org_fp); 158 if (IS_ERR(tmp_fp)) 159 return org_fp; 160 161 if (tmp_fp != org_fp) { 162 bpf_blinded = true; 163 fp = tmp_fp; 164 } 165 166 jit_data = fp->aux->jit_data; 167 if (!jit_data) { 168 jit_data = kzalloc_obj(*jit_data); 169 if (!jit_data) { 170 fp = org_fp; 171 goto out; 172 } 173 fp->aux->jit_data = jit_data; 174 } 175 176 flen = fp->len; 177 addrs = jit_data->addrs; 178 if (addrs) { 179 cgctx = jit_data->ctx; 180 /* 181 * JIT compiled to a writable location (image/code_base) first. 182 * It is then moved to the readonly final location (fimage/fcode_base) 183 * using instruction patching. 184 */ 185 fimage = jit_data->fimage; 186 fhdr = jit_data->fhdr; 187 proglen = jit_data->proglen; 188 hdr = jit_data->hdr; 189 image = (void *)hdr + ((void *)fimage - (void *)fhdr); 190 extra_pass = true; 191 /* During extra pass, ensure index is reset before repopulating extable entries */ 192 cgctx.exentry_idx = 0; 193 goto skip_init_ctx; 194 } 195 196 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 197 if (addrs == NULL) { 198 fp = org_fp; 199 goto out_addrs; 200 } 201 202 memset(&cgctx, 0, sizeof(struct codegen_context)); 203 bpf_jit_init_reg_mapping(&cgctx); 204 205 /* Make sure that the stack is quadword aligned. */ 206 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 207 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); 208 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); 209 cgctx.is_subprog = bpf_is_subprog(fp); 210 cgctx.exception_boundary = fp->aux->exception_boundary; 211 cgctx.exception_cb = fp->aux->exception_cb; 212 213 /* Scouting faux-generate pass 0 */ 214 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 215 /* We hit something illegal or unsupported. */ 216 fp = org_fp; 217 goto out_addrs; 218 } 219 220 /* 221 * If we have seen a tail call, we need a second pass. 222 * This is because bpf_jit_emit_common_epilogue() is called 223 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 224 * We also need a second pass if we ended up with too large 225 * a program so as to ensure BPF_EXIT branches are in range. 226 */ 227 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 228 cgctx.idx = 0; 229 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 230 fp = org_fp; 231 goto out_addrs; 232 } 233 } 234 235 bpf_jit_realloc_regs(&cgctx); 236 /* 237 * Pretend to build prologue, given the features we've seen. This will 238 * update ctgtx.idx as it pretends to output instructions, then we can 239 * calculate total size from idx. 240 */ 241 bpf_jit_build_prologue(NULL, &cgctx); 242 addrs[fp->len] = cgctx.idx * 4; 243 bpf_jit_build_epilogue(NULL, &cgctx); 244 245 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 246 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 247 248 proglen = cgctx.idx * 4; 249 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 250 251 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 252 bpf_jit_fill_ill_insns); 253 if (!fhdr) { 254 fp = org_fp; 255 goto out_addrs; 256 } 257 258 if (extable_len) 259 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; 260 261 skip_init_ctx: 262 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 263 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); 264 265 /* Code generation passes 1-2 */ 266 for (pass = 1; pass < 3; pass++) { 267 /* Now build the prologue, body code & epilogue for real. */ 268 cgctx.idx = 0; 269 cgctx.alt_exit_addr = 0; 270 bpf_jit_build_prologue(code_base, &cgctx); 271 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, 272 extra_pass)) { 273 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 274 bpf_jit_binary_pack_free(fhdr, hdr); 275 fp = org_fp; 276 goto out_addrs; 277 } 278 bpf_jit_build_epilogue(code_base, &cgctx); 279 280 if (bpf_jit_enable > 1) 281 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 282 proglen - (cgctx.idx * 4), cgctx.seen); 283 } 284 285 if (bpf_jit_enable > 1) 286 /* 287 * Note that we output the base address of the code_base 288 * rather than image, since opcodes are in code_base. 289 */ 290 bpf_jit_dump(flen, proglen, pass, code_base); 291 292 #ifdef CONFIG_PPC64_ELF_ABI_V1 293 /* Function descriptor nastiness: Address + TOC */ 294 ((u64 *)image)[0] = (u64)fcode_base; 295 ((u64 *)image)[1] = local_paca->kernel_toc; 296 #endif 297 298 fp->bpf_func = (void *)fimage; 299 fp->jited = 1; 300 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; 301 302 if (!fp->is_func || extra_pass) { 303 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) { 304 fp = org_fp; 305 goto out_addrs; 306 } 307 bpf_prog_fill_jited_linfo(fp, addrs); 308 out_addrs: 309 kfree(addrs); 310 kfree(jit_data); 311 fp->aux->jit_data = NULL; 312 } else { 313 jit_data->addrs = addrs; 314 jit_data->ctx = cgctx; 315 jit_data->proglen = proglen; 316 jit_data->fimage = fimage; 317 jit_data->fhdr = fhdr; 318 jit_data->hdr = hdr; 319 } 320 321 out: 322 if (bpf_blinded) 323 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 324 325 return fp; 326 } 327 328 /* 329 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 330 * this function, as this only applies to BPF_PROBE_MEM, for now. 331 */ 332 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, 333 struct codegen_context *ctx, int insn_idx, int jmp_off, 334 int dst_reg, u32 code) 335 { 336 off_t offset; 337 unsigned long pc; 338 struct exception_table_entry *ex, *ex_entry; 339 u32 *fixup; 340 341 /* Populate extable entries only in the last pass */ 342 if (pass != 2) 343 return 0; 344 345 if (!fp->aux->extable || 346 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 347 return -EINVAL; 348 349 /* 350 * Program is first written to image before copying to the 351 * final location (fimage). Accordingly, update in the image first. 352 * As all offsets used are relative, copying as is to the 353 * final location should be alright. 354 */ 355 pc = (unsigned long)&image[insn_idx]; 356 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; 357 358 fixup = (void *)ex - 359 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 360 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 361 362 fixup[0] = PPC_RAW_LI(dst_reg, 0); 363 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX) 364 fixup[0] = PPC_RAW_NOP(); 365 366 if (IS_ENABLED(CONFIG_PPC32)) 367 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 368 369 fixup[BPF_FIXUP_LEN - 1] = 370 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 371 372 ex_entry = &ex[ctx->exentry_idx]; 373 374 offset = pc - (long)&ex_entry->insn; 375 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 376 return -ERANGE; 377 ex_entry->insn = offset; 378 379 offset = (long)fixup - (long)&ex_entry->fixup; 380 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 381 return -ERANGE; 382 ex_entry->fixup = offset; 383 384 ctx->exentry_idx++; 385 return 0; 386 } 387 388 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 389 { 390 int err; 391 392 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 393 return ERR_PTR(-EINVAL); 394 395 mutex_lock(&text_mutex); 396 err = patch_instructions(dst, src, len, false); 397 mutex_unlock(&text_mutex); 398 399 return err ? ERR_PTR(err) : dst; 400 } 401 402 int bpf_arch_text_invalidate(void *dst, size_t len) 403 { 404 u32 insn = BREAKPOINT_INSTRUCTION; 405 int ret; 406 407 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 408 return -EINVAL; 409 410 mutex_lock(&text_mutex); 411 ret = patch_instructions(dst, &insn, len, true); 412 mutex_unlock(&text_mutex); 413 414 return ret; 415 } 416 417 void bpf_jit_free(struct bpf_prog *fp) 418 { 419 if (fp->jited) { 420 struct powerpc_jit_data *jit_data = fp->aux->jit_data; 421 struct bpf_binary_header *hdr; 422 423 /* 424 * If we fail the final pass of JIT (from jit_subprogs), 425 * the program may not be finalized yet. Call finalize here 426 * before freeing it. 427 */ 428 if (jit_data) { 429 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr); 430 kvfree(jit_data->addrs); 431 kfree(jit_data); 432 } 433 hdr = bpf_jit_binary_pack_hdr(fp); 434 bpf_jit_binary_pack_free(hdr, NULL); 435 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 436 } 437 438 bpf_prog_unlock_free(fp); 439 } 440 441 bool bpf_jit_supports_exceptions(void) 442 { 443 return IS_ENABLED(CONFIG_PPC64); 444 } 445 446 bool bpf_jit_supports_subprog_tailcalls(void) 447 { 448 return IS_ENABLED(CONFIG_PPC64); 449 } 450 451 bool bpf_jit_supports_kfunc_call(void) 452 { 453 return IS_ENABLED(CONFIG_PPC64); 454 } 455 456 bool bpf_jit_supports_arena(void) 457 { 458 return IS_ENABLED(CONFIG_PPC64); 459 } 460 461 bool bpf_jit_supports_far_kfunc_call(void) 462 { 463 return IS_ENABLED(CONFIG_PPC64); 464 } 465 466 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 467 { 468 if (!in_arena) 469 return true; 470 switch (insn->code) { 471 case BPF_STX | BPF_ATOMIC | BPF_H: 472 case BPF_STX | BPF_ATOMIC | BPF_B: 473 case BPF_STX | BPF_ATOMIC | BPF_W: 474 case BPF_STX | BPF_ATOMIC | BPF_DW: 475 if (bpf_atomic_is_load_store(insn)) 476 return false; 477 return IS_ENABLED(CONFIG_PPC64); 478 } 479 return true; 480 } 481 482 bool bpf_jit_supports_percpu_insn(void) 483 { 484 return IS_ENABLED(CONFIG_PPC64); 485 } 486 487 bool bpf_jit_inlines_helper_call(s32 imm) 488 { 489 switch (imm) { 490 case BPF_FUNC_get_smp_processor_id: 491 case BPF_FUNC_get_current_task: 492 case BPF_FUNC_get_current_task_btf: 493 return true; 494 default: 495 return false; 496 } 497 } 498 499 void *arch_alloc_bpf_trampoline(unsigned int size) 500 { 501 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); 502 } 503 504 void arch_free_bpf_trampoline(void *image, unsigned int size) 505 { 506 bpf_prog_pack_free(image, size); 507 } 508 509 int arch_protect_bpf_trampoline(void *image, unsigned int size) 510 { 511 return 0; 512 } 513 514 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, 515 struct bpf_tramp_link *l, int regs_off, int retval_off, 516 int run_ctx_off, bool save_ret) 517 { 518 struct bpf_prog *p = l->link.prog; 519 ppc_inst_t branch_insn; 520 u32 jmp_idx; 521 int ret = 0; 522 523 /* Save cookie */ 524 if (IS_ENABLED(CONFIG_PPC64)) { 525 PPC_LI64(_R3, l->cookie); 526 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx, 527 bpf_cookie))); 528 } else { 529 PPC_LI32(_R3, l->cookie >> 32); 530 PPC_LI32(_R4, l->cookie); 531 EMIT(PPC_RAW_STW(_R3, _R1, 532 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie))); 533 EMIT(PPC_RAW_STW(_R4, _R1, 534 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4)); 535 } 536 537 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */ 538 PPC_LI_ADDR(_R3, p); 539 EMIT(PPC_RAW_MR(_R25, _R3)); 540 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off)); 541 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 542 (unsigned long)bpf_trampoline_enter(p)); 543 if (ret) 544 return ret; 545 546 /* Remember prog start time returned by __bpf_prog_enter */ 547 EMIT(PPC_RAW_MR(_R26, _R3)); 548 549 /* 550 * if (__bpf_prog_enter(p) == 0) 551 * goto skip_exec_of_prog; 552 * 553 * Emit a nop to be later patched with conditional branch, once offset is known 554 */ 555 EMIT(PPC_RAW_CMPLI(_R3, 0)); 556 jmp_idx = ctx->idx; 557 EMIT(PPC_RAW_NOP()); 558 559 /* p->bpf_func(ctx) */ 560 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off)); 561 if (!p->jited) 562 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi); 563 /* Account for max possible instructions during dummy pass for size calculation */ 564 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], 565 (unsigned long)p->bpf_func, 566 BRANCH_SET_LINK)) { 567 image[ctx->idx] = ppc_inst_val(branch_insn); 568 ctx->idx++; 569 } else { 570 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func))); 571 EMIT(PPC_RAW_MTCTR(_R12)); 572 EMIT(PPC_RAW_BCTRL()); 573 } 574 575 if (save_ret) 576 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 577 578 /* Fix up branch */ 579 if (image) { 580 if (create_cond_branch(&branch_insn, &image[jmp_idx], 581 (unsigned long)&image[ctx->idx], COND_EQ << 16)) 582 return -EINVAL; 583 image[jmp_idx] = ppc_inst_val(branch_insn); 584 } 585 586 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */ 587 EMIT(PPC_RAW_MR(_R3, _R25)); 588 EMIT(PPC_RAW_MR(_R4, _R26)); 589 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); 590 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 591 (unsigned long)bpf_trampoline_exit(p)); 592 593 return ret; 594 } 595 596 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, 597 struct bpf_tramp_links *tl, int regs_off, int retval_off, 598 int run_ctx_off, u32 *branches) 599 { 600 int i; 601 602 /* 603 * The first fmod_ret program will receive a garbage return value. 604 * Set this to 0 to avoid confusing the program. 605 */ 606 EMIT(PPC_RAW_LI(_R3, 0)); 607 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 608 for (i = 0; i < tl->nr_links; i++) { 609 if (invoke_bpf_prog(image, ro_image, ctx, tl->links[i], regs_off, retval_off, 610 run_ctx_off, true)) 611 return -EINVAL; 612 613 /* 614 * mod_ret prog stored return value after prog ctx. Emit: 615 * if (*(u64 *)(ret_val) != 0) 616 * goto do_fexit; 617 */ 618 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 619 EMIT(PPC_RAW_CMPLI(_R3, 0)); 620 621 /* 622 * Save the location of the branch and generate a nop, which is 623 * replaced with a conditional jump once do_fexit (i.e. the 624 * start of the fexit invocation) is finalized. 625 */ 626 branches[i] = ctx->idx; 627 EMIT(PPC_RAW_NOP()); 628 } 629 630 return 0; 631 } 632 633 /* 634 * Refer __arch_prepare_bpf_trampoline() for stack component details. 635 * 636 * The tailcall count/reference is present in caller's stack frame. The 637 * tail_call_info is saved at the same offset on the trampoline frame 638 * for the traced function (BPF subprog/callee) to fetch it. 639 */ 640 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, 641 int bpf_frame_size, int r4_off) 642 { 643 if (IS_ENABLED(CONFIG_PPC64)) { 644 EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size)); 645 /* Refer to trampoline's Generated stack layout */ 646 EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL)); 647 648 /* 649 * Setting the tail_call_info in trampoline's frame 650 * depending on if previous frame had value or reference. 651 */ 652 EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT)); 653 PPC_BCC_CONST_SHORT(COND_GT, 8); 654 EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL)); 655 656 /* 657 * Trampoline's tail_call_info is at the same offset, as that of 658 * any bpf program, with reference to previous frame. Update the 659 * address of main's tail_call_info in trampoline frame. 660 */ 661 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL)); 662 } else { 663 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ 664 EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); 665 } 666 } 667 668 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx, 669 int bpf_frame_size, int r4_off) 670 { 671 if (IS_ENABLED(CONFIG_PPC32)) { 672 /* 673 * Restore tailcall for 32-bit powerpc 674 * See bpf_jit_stack_offsetof() and BPF_PPC_TC 675 */ 676 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 677 } 678 } 679 680 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, 681 int bpf_frame_size, int nr_regs, int regs_off) 682 { 683 int param_save_area_offset; 684 685 param_save_area_offset = bpf_frame_size; 686 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 687 688 for (int i = 0; i < nr_regs; i++) { 689 if (i < 8) { 690 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL)); 691 } else { 692 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 693 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL)); 694 } 695 } 696 } 697 698 /* Used when restoring just the register parameters when returning back */ 699 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx, 700 int nr_regs, int regs_off) 701 { 702 for (int i = 0; i < nr_regs && i < 8; i++) 703 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL)); 704 } 705 706 /* Used when we call into the traced function. Replicate parameter save area */ 707 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx, 708 int bpf_frame_size, int nr_regs, int regs_off) 709 { 710 int param_save_area_offset; 711 712 param_save_area_offset = bpf_frame_size; 713 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 714 715 for (int i = 8; i < nr_regs; i++) { 716 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 717 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL)); 718 } 719 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 720 } 721 722 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image, 723 void *rw_image_end, void *ro_image, 724 const struct btf_func_model *m, u32 flags, 725 struct bpf_tramp_links *tlinks, 726 void *func_addr) 727 { 728 int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0; 729 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 730 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 731 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 732 int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; 733 struct codegen_context codegen_ctx, *ctx; 734 u32 *image = (u32 *)rw_image; 735 ppc_inst_t branch_insn; 736 u32 *branches = NULL; 737 bool save_ret; 738 739 if (IS_ENABLED(CONFIG_PPC32)) 740 return -EOPNOTSUPP; 741 742 nr_regs = m->nr_args; 743 /* Extra registers for struct arguments */ 744 for (i = 0; i < m->nr_args; i++) 745 if (m->arg_size[i] > SZL) 746 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1; 747 748 if (nr_regs > MAX_BPF_FUNC_ARGS) 749 return -EOPNOTSUPP; 750 751 ctx = &codegen_ctx; 752 memset(ctx, 0, sizeof(*ctx)); 753 754 /* 755 * Generated stack layout: 756 * 757 * func prev back chain [ back chain ] 758 * [ tail_call_info ] optional - 64-bit powerpc 759 * [ padding ] align stack frame 760 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc 761 * alt_lr_off [ real lr (ool stub)] optional - actual lr 762 * retaddr_off [ return address ] 763 * [ r26 ] 764 * nvr_off [ r25 ] nvr save area 765 * retval_off [ return value ] 766 * [ reg argN ] 767 * [ ... ] 768 * regs_off [ reg_arg1 ] prog_ctx 769 * nregs_off [ args count ] ((u64 *)prog_ctx)[-1] 770 * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] 771 * [ ... ] 772 * run_ctx_off [ bpf_tramp_run_ctx ] 773 * [ reg argN ] 774 * [ ... ] 775 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI 776 * [ TOC save (64-bit) ] -- 777 * [ LR save (64-bit) ] | header 778 * [ LR save (32-bit) ] | 779 * bpf trampoline frame [ back chain 2 ] -- 780 * 781 */ 782 783 /* Minimum stack frame header */ 784 bpf_frame_size = STACK_FRAME_MIN_SIZE; 785 786 /* 787 * Room for parameter save area. 788 * 789 * As per the ABI, this is required if we call into the traced 790 * function (BPF_TRAMP_F_CALL_ORIG): 791 * - if the function takes more than 8 arguments for the rest to spill onto the stack 792 * - or, if the function has variadic arguments 793 * - or, if this functions's prototype was not available to the caller 794 * 795 * Reserve space for at least 8 registers for now. This can be optimized later. 796 */ 797 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL; 798 799 /* Room for struct bpf_tramp_run_ctx */ 800 run_ctx_off = bpf_frame_size; 801 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); 802 803 /* Room for IP address argument */ 804 ip_off = bpf_frame_size; 805 if (flags & BPF_TRAMP_F_IP_ARG) 806 bpf_frame_size += SZL; 807 808 /* Room for args count */ 809 nregs_off = bpf_frame_size; 810 bpf_frame_size += SZL; 811 812 /* Room for args */ 813 regs_off = bpf_frame_size; 814 bpf_frame_size += nr_regs * SZL; 815 816 /* Room for return value of func_addr or fentry prog */ 817 retval_off = bpf_frame_size; 818 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 819 if (save_ret) 820 bpf_frame_size += SZL; 821 822 /* Room for nvr save area */ 823 nvr_off = bpf_frame_size; 824 bpf_frame_size += 2 * SZL; 825 826 /* Save area for return address */ 827 retaddr_off = bpf_frame_size; 828 bpf_frame_size += SZL; 829 830 /* Optional save area for actual LR in case of ool ftrace */ 831 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 832 alt_lr_off = bpf_frame_size; 833 bpf_frame_size += SZL; 834 } 835 836 if (IS_ENABLED(CONFIG_PPC32)) { 837 if (nr_regs < 2) { 838 r4_off = bpf_frame_size; 839 bpf_frame_size += SZL; 840 } else { 841 r4_off = regs_off + SZL; 842 } 843 } 844 845 /* 846 * Save tailcall count pointer at the same offset on the 847 * stack where subprogs expect it 848 */ 849 if ((flags & BPF_TRAMP_F_CALL_ORIG) && 850 (flags & BPF_TRAMP_F_TAIL_CALL_CTX)) 851 bpf_frame_size += BPF_PPC_TAILCALL; 852 853 /* Padding to align stack frame, if any */ 854 bpf_frame_size = round_up(bpf_frame_size, SZL * 2); 855 856 /* Store original return value */ 857 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF)); 858 859 /* Create our stack frame */ 860 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size)); 861 862 /* 64-bit: Save TOC and load kernel TOC */ 863 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 864 EMIT(PPC_RAW_STD(_R2, _R1, 24)); 865 PPC64_LOAD_PACA(); 866 } 867 868 /* 32-bit: save tail call count in r4 */ 869 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2) 870 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 871 872 bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off); 873 874 /* Save our LR/return address */ 875 EMIT(PPC_RAW_MFLR(_R3)); 876 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 877 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); 878 else 879 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 880 881 /* 882 * Derive IP address of the traced function. 883 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction 884 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and 885 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. 886 * Relevant stub sequence: 887 * 888 * bl <tramp> 889 * LR (R3) => mtlr r0 890 * b <func_addr+4> 891 * 892 * Recover kernel function/bpf program address from the unconditional 893 * branch instruction at the end of OOL stub. 894 */ 895 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { 896 EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); 897 EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); 898 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); 899 EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); 900 } 901 902 if (flags & BPF_TRAMP_F_IP_ARG) 903 EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); 904 905 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 906 /* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ 907 EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); 908 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 909 } 910 911 /* Save function arg count -- see bpf_get_func_arg_cnt() */ 912 EMIT(PPC_RAW_LI(_R3, nr_regs)); 913 EMIT(PPC_RAW_STL(_R3, _R1, nregs_off)); 914 915 /* Save nv regs */ 916 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); 917 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL)); 918 919 if (flags & BPF_TRAMP_F_CALL_ORIG) { 920 PPC_LI_ADDR(_R3, (unsigned long)im); 921 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 922 (unsigned long)__bpf_tramp_enter); 923 if (ret) 924 return ret; 925 } 926 927 for (i = 0; i < fentry->nr_links; i++) 928 if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off, 929 run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) 930 return -EINVAL; 931 932 if (fmod_ret->nr_links) { 933 branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL); 934 if (!branches) 935 return -ENOMEM; 936 937 if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off, 938 run_ctx_off, branches)) { 939 ret = -EINVAL; 940 goto cleanup; 941 } 942 } 943 944 /* Call the traced function */ 945 if (flags & BPF_TRAMP_F_CALL_ORIG) { 946 /* 947 * retaddr on trampoline stack points to the correct point in the original function 948 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction 949 * sequence 950 */ 951 EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off)); 952 EMIT(PPC_RAW_MTCTR(_R3)); 953 954 /* Replicate tail_call_cnt before calling the original BPF prog */ 955 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 956 bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off); 957 958 /* Restore args */ 959 bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off); 960 961 /* Restore TOC for 64-bit */ 962 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 963 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 964 EMIT(PPC_RAW_BCTRL()); 965 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 966 PPC64_LOAD_PACA(); 967 968 /* Store return value for bpf prog to access */ 969 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 970 971 /* Restore updated tail_call_cnt */ 972 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 973 bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); 974 975 /* Reserve space to patch branch instruction to skip fexit progs */ 976 if (ro_image) /* image is NULL for dummy pass */ 977 im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; 978 EMIT(PPC_RAW_NOP()); 979 } 980 981 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ 982 for (i = 0; i < fmod_ret->nr_links && image; i++) { 983 if (create_cond_branch(&branch_insn, &image[branches[i]], 984 (unsigned long)&image[ctx->idx], COND_NE << 16)) { 985 ret = -EINVAL; 986 goto cleanup; 987 } 988 989 image[branches[i]] = ppc_inst_val(branch_insn); 990 } 991 992 for (i = 0; i < fexit->nr_links; i++) 993 if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off, 994 run_ctx_off, false)) { 995 ret = -EINVAL; 996 goto cleanup; 997 } 998 999 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1000 if (ro_image) /* image is NULL for dummy pass */ 1001 im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; 1002 PPC_LI_ADDR(_R3, im); 1003 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1004 (unsigned long)__bpf_tramp_exit); 1005 if (ret) 1006 goto cleanup; 1007 } 1008 1009 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1010 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 1011 1012 /* Restore return value of func_addr or fentry prog */ 1013 if (save_ret) 1014 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 1015 1016 /* Restore nv regs */ 1017 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL)); 1018 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off)); 1019 1020 /* Epilogue */ 1021 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1022 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1023 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 1024 /* Skip the traced function and return to parent */ 1025 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1026 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1027 EMIT(PPC_RAW_MTLR(_R0)); 1028 EMIT(PPC_RAW_BLR()); 1029 } else { 1030 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1031 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off)); 1032 EMIT(PPC_RAW_MTLR(_R0)); 1033 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1034 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1035 EMIT(PPC_RAW_BLR()); 1036 } else { 1037 EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off)); 1038 EMIT(PPC_RAW_MTCTR(_R0)); 1039 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1040 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1041 EMIT(PPC_RAW_MTLR(_R0)); 1042 EMIT(PPC_RAW_BCTR()); 1043 } 1044 } 1045 1046 /* Make sure the trampoline generation logic doesn't overflow */ 1047 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) { 1048 ret = -EFAULT; 1049 goto cleanup; 1050 } 1051 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4; 1052 1053 cleanup: 1054 kfree(branches); 1055 return ret; 1056 } 1057 1058 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1059 struct bpf_tramp_links *tlinks, void *func_addr) 1060 { 1061 struct bpf_tramp_image im; 1062 int ret; 1063 1064 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tlinks, func_addr); 1065 return ret; 1066 } 1067 1068 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, 1069 const struct btf_func_model *m, u32 flags, 1070 struct bpf_tramp_links *tlinks, 1071 void *func_addr) 1072 { 1073 u32 size = image_end - image; 1074 void *rw_image, *tmp; 1075 int ret; 1076 1077 /* 1078 * rw_image doesn't need to be in module memory range, so we can 1079 * use kvmalloc. 1080 */ 1081 rw_image = kvmalloc(size, GFP_KERNEL); 1082 if (!rw_image) 1083 return -ENOMEM; 1084 1085 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m, 1086 flags, tlinks, func_addr); 1087 if (ret < 0) 1088 goto out; 1089 1090 if (bpf_jit_enable > 1) 1091 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image); 1092 1093 tmp = bpf_arch_text_copy(image, rw_image, size); 1094 if (IS_ERR(tmp)) 1095 ret = PTR_ERR(tmp); 1096 1097 out: 1098 kvfree(rw_image); 1099 return ret; 1100 } 1101 1102 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst) 1103 { 1104 ppc_inst_t org_inst; 1105 1106 if (copy_inst_from_kernel_nofault(&org_inst, ip)) { 1107 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip); 1108 return -EFAULT; 1109 } 1110 1111 if (!ppc_inst_equal(org_inst, old_inst)) { 1112 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n", 1113 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst)); 1114 return -EINVAL; 1115 } 1116 1117 if (ppc_inst_equal(old_inst, new_inst)) 1118 return 0; 1119 1120 return patch_instruction(ip, new_inst); 1121 } 1122 1123 static void do_isync(void *info __maybe_unused) 1124 { 1125 isync(); 1126 } 1127 1128 /* 1129 * A 3-step process for bpf prog entry: 1130 * 1. At bpf prog entry, a single nop/b: 1131 * bpf_func: 1132 * [nop|b] ool_stub 1133 * 2. Out-of-line stub: 1134 * ool_stub: 1135 * mflr r0 1136 * [b|bl] <bpf_prog>/<long_branch_stub> 1137 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only 1138 * b bpf_func + 4 1139 * 3. Long branch stub: 1140 * long_branch_stub: 1141 * .long <branch_addr>/<dummy_tramp> 1142 * mflr r11 1143 * bcl 20,31,$+4 1144 * mflr r12 1145 * ld r12, -16(r12) 1146 * mtctr r12 1147 * mtlr r11 // needed to retain ftrace ABI 1148 * bctr 1149 * 1150 * dummy_tramp is used to reduce synchronization requirements. 1151 * 1152 * When attaching a bpf trampoline to a bpf prog, we do not need any 1153 * synchronization here since we always have a valid branch target regardless 1154 * of the order in which the above stores are seen. dummy_tramp ensures that 1155 * the long_branch stub goes to a valid destination on other cpus, even when 1156 * the branch to the long_branch stub is seen before the updated trampoline 1157 * address. 1158 * 1159 * However, when detaching a bpf trampoline from a bpf prog, or if changing 1160 * the bpf trampoline address, we need synchronization to ensure that other 1161 * cpus can no longer branch into the older trampoline so that it can be 1162 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus 1163 * make forward progress, but we still need to ensure that other cpus 1164 * execute isync (or some CSI) so that they don't go back into the 1165 * trampoline again. 1166 */ 1167 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1168 enum bpf_text_poke_type new_t, void *old_addr, 1169 void *new_addr) 1170 { 1171 unsigned long bpf_func, bpf_func_end, size, offset; 1172 ppc_inst_t old_inst, new_inst; 1173 int ret = 0, branch_flags; 1174 char name[KSYM_NAME_LEN]; 1175 1176 if (IS_ENABLED(CONFIG_PPC32)) 1177 return -EOPNOTSUPP; 1178 1179 bpf_func = (unsigned long)ip; 1180 1181 /* We currently only support poking bpf programs */ 1182 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { 1183 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); 1184 return -EOPNOTSUPP; 1185 } 1186 1187 /* 1188 * If we are not poking at bpf prog entry, then we are simply patching in/out 1189 * an unconditional branch instruction at im->ip_after_call 1190 */ 1191 if (offset) { 1192 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { 1193 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__, 1194 bpf_func); 1195 return -EOPNOTSUPP; 1196 } 1197 old_inst = ppc_inst(PPC_RAW_NOP()); 1198 if (old_addr) 1199 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0)) 1200 return -ERANGE; 1201 new_inst = ppc_inst(PPC_RAW_NOP()); 1202 if (new_addr) 1203 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0)) 1204 return -ERANGE; 1205 mutex_lock(&text_mutex); 1206 ret = bpf_modify_inst(ip, old_inst, new_inst); 1207 mutex_unlock(&text_mutex); 1208 1209 /* Make sure all cpus see the new instruction */ 1210 smp_call_function(do_isync, NULL, 1); 1211 return ret; 1212 } 1213 1214 bpf_func_end = bpf_func + size; 1215 1216 /* Address of the jmp/call instruction in the out-of-line stub */ 1217 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4); 1218 1219 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) { 1220 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__, 1221 bpf_func); 1222 return -ERANGE; 1223 } 1224 1225 old_inst = ppc_inst(PPC_RAW_NOP()); 1226 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1227 if (old_addr) { 1228 if (is_offset_in_branch_range(ip - old_addr)) 1229 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags); 1230 else 1231 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1232 branch_flags); 1233 } 1234 new_inst = ppc_inst(PPC_RAW_NOP()); 1235 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1236 if (new_addr) { 1237 if (is_offset_in_branch_range(ip - new_addr)) 1238 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags); 1239 else 1240 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1241 branch_flags); 1242 } 1243 1244 mutex_lock(&text_mutex); 1245 1246 /* 1247 * 1. Update the address in the long branch stub: 1248 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr 1249 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here. 1250 */ 1251 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) || 1252 (old_addr && !is_offset_in_branch_range(old_addr - ip))) 1253 ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL), 1254 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ? 1255 (unsigned long)new_addr : (unsigned long)dummy_tramp); 1256 if (ret) 1257 goto out; 1258 1259 /* 2. Update the branch/call in the out-of-line stub */ 1260 ret = bpf_modify_inst(ip, old_inst, new_inst); 1261 if (ret) 1262 goto out; 1263 1264 /* 3. Update instruction at bpf prog entry */ 1265 ip = (void *)bpf_func; 1266 if (!old_addr || !new_addr) { 1267 if (!old_addr) { 1268 old_inst = ppc_inst(PPC_RAW_NOP()); 1269 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1270 } else { 1271 new_inst = ppc_inst(PPC_RAW_NOP()); 1272 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1273 } 1274 ret = bpf_modify_inst(ip, old_inst, new_inst); 1275 } 1276 1277 out: 1278 mutex_unlock(&text_mutex); 1279 1280 /* 1281 * Sync only if we are not attaching a trampoline to a bpf prog so the older 1282 * trampoline can be freed safely. 1283 */ 1284 if (old_addr) 1285 smp_call_function(do_isync, NULL, 1); 1286 1287 return ret; 1288 } 1289