1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <linux/kernel.h> 17 #include <linux/memory.h> 18 #include <linux/bpf.h> 19 20 #include <asm/kprobes.h> 21 #include <asm/text-patching.h> 22 23 #include "bpf_jit.h" 24 25 /* These offsets are from bpf prog end and stay the same across progs */ 26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub; 27 28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 29 { 30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 31 } 32 33 void dummy_tramp(void); 34 35 asm ( 36 " .pushsection .text, \"ax\", @progbits ;" 37 " .global dummy_tramp ;" 38 " .type dummy_tramp, @function ;" 39 "dummy_tramp: ;" 40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 41 " blr ;" 42 #else 43 /* LR is always in r11, so we don't need a 'mflr r11' here */ 44 " mtctr 11 ;" 45 " mtlr 0 ;" 46 " bctr ;" 47 #endif 48 " .size dummy_tramp, .-dummy_tramp ;" 49 " .popsection ;" 50 ); 51 52 void bpf_jit_build_fentry_stubs(u32 *image, u32 *fimage, struct codegen_context *ctx) 53 { 54 int ool_stub_idx, long_branch_stub_idx; 55 int stub_sz; 56 57 /* 58 * The dummy_tramp_addr field is placed at bottom of Long branch stub. 59 * Align the mis-aligned dummy_tramp_addr field in the fimage. 60 * The alignment NOP must appear before OOL stub, to make 61 * ool_stub_idx & long_branch_stub_idx constant from end. 62 * 63 * The fimage can be non 8-byte aligned, so final alignment depends 64 * on start of fimage and the stub's instruction count. The 65 * stubs block has 11 instructions (with CONFIG_PPC_FTRACE_OUT_OF_LINE) 66 * or 10 instructions (without) before dummy_tramp_addr field. Emit a 67 * NOP if the address of dummy_tramp_addr is non aligned. 68 * 69 * In pass=0 when image==NULL, conservatively account for space 70 * required to accommodate alignment NOP. In case final pass skips 71 * emitting alignment NOP, the image buffer have 4 spare bytes and 72 * jited_len signifies correct program size. 73 */ 74 75 stub_sz = IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) ? 44 : 40; 76 if (!image || !IS_ALIGNED((unsigned long)fimage + ctx->idx*4 + stub_sz, SZL)) 77 EMIT(PPC_RAW_NOP()); 78 79 /* 80 * nop // optional, for alignment of dummy_tramp_addr 81 * Out-of-line stub: 82 * mflr r0 83 * [b|bl] tramp 84 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE 85 * b bpf_func + 4 86 */ 87 ool_stub_idx = ctx->idx; 88 EMIT(PPC_RAW_MFLR(_R0)); 89 EMIT(PPC_RAW_NOP()); 90 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 91 EMIT(PPC_RAW_MTLR(_R0)); 92 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4)); 93 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4)); 94 95 /* 96 * Long branch stub: 97 * mflr r11 98 * bcl 20,31,$+4 99 * mflr r12 // lr/r12 stores pc of current(this) inst. 100 * ld r12, 20(r12) // offset(dummy_tramp_addr) from prev inst. is 20 101 * mtctr r12 102 * mtlr r11 // needed to retain ftrace ABI 103 * bctr 104 * .long <dummy_tramp_addr> // SZL bytes aligned 105 */ 106 long_branch_stub_idx = ctx->idx; 107 EMIT(PPC_RAW_MFLR(_R11)); 108 EMIT(PPC_RAW_BCL4()); 109 EMIT(PPC_RAW_MFLR(_R12)); 110 EMIT(PPC_RAW_LL(_R12, _R12, 20)); 111 EMIT(PPC_RAW_MTCTR(_R12)); 112 EMIT(PPC_RAW_MTLR(_R11)); 113 EMIT(PPC_RAW_BCTR()); 114 115 if (image) 116 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp; 117 118 ctx->idx += SZL / 4; 119 120 if (!bpf_jit_ool_stub) { 121 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4; 122 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4; 123 } 124 } 125 126 int bpf_jit_emit_exit_insn(u32 *image, u32 *fimage, struct codegen_context *ctx, 127 int tmp_reg, long exit_addr) 128 { 129 if (exit_addr && is_offset_in_branch_range(exit_addr - (long)(ctx->idx * 4))) { 130 PPC_JMP(exit_addr); 131 } else if (ctx->alt_exit_addr && is_offset_in_branch_range( 132 (long)(ctx->alt_exit_addr) - (long)(ctx->idx * 4))) { 133 PPC_JMP(ctx->alt_exit_addr); 134 } else { 135 ctx->alt_exit_addr = ctx->idx * 4; 136 bpf_jit_build_epilogue(image, fimage, ctx); 137 } 138 139 return 0; 140 } 141 142 struct powerpc_jit_data { 143 /* address of rw header */ 144 struct bpf_binary_header *hdr; 145 /* address of ro final header */ 146 struct bpf_binary_header *fhdr; 147 u32 *addrs; 148 u8 *fimage; 149 u32 proglen; 150 struct codegen_context ctx; 151 }; 152 153 bool bpf_jit_needs_zext(void) 154 { 155 return true; 156 } 157 158 static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) 159 { 160 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 161 u64 *stack_ptr; 162 163 for_each_possible_cpu(cpu) { 164 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 165 stack_ptr[0] = PRIV_STACK_GUARD_VAL; 166 stack_ptr[1] = PRIV_STACK_GUARD_VAL; 167 stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL; 168 stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL; 169 } 170 } 171 172 static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size, 173 struct bpf_prog *fp) 174 { 175 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; 176 u64 *stack_ptr; 177 178 for_each_possible_cpu(cpu) { 179 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); 180 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL || 181 stack_ptr[1] != PRIV_STACK_GUARD_VAL || 182 stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL || 183 stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) { 184 pr_err("BPF private stack overflow/underflow detected for prog %s\n", 185 bpf_jit_get_prog_name(fp)); 186 break; 187 } 188 } 189 } 190 191 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp) 192 { 193 u32 proglen; 194 u32 alloclen; 195 u8 *image = NULL; 196 u32 *code_base = NULL; 197 u32 *addrs = NULL; 198 struct powerpc_jit_data *jit_data = NULL; 199 struct codegen_context cgctx; 200 int pass; 201 int flen; 202 int priv_stack_alloc_size; 203 void __percpu *priv_stack_ptr = NULL; 204 struct bpf_binary_header *fhdr = NULL; 205 struct bpf_binary_header *hdr = NULL; 206 bool extra_pass = false; 207 u8 *fimage = NULL; 208 u32 *fcode_base = NULL; 209 u32 extable_len; 210 u32 fixup_len; 211 212 if (!fp->jit_requested) 213 return fp; 214 215 jit_data = fp->aux->jit_data; 216 if (!jit_data) { 217 jit_data = kzalloc_obj(*jit_data); 218 if (!jit_data) 219 return fp; 220 fp->aux->jit_data = jit_data; 221 } 222 223 priv_stack_ptr = fp->aux->priv_stack_ptr; 224 if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) { 225 /* 226 * Allocate private stack of size equivalent to 227 * verifier-calculated stack size plus two memory 228 * guard regions to detect private stack overflow 229 * and underflow. 230 */ 231 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 232 2 * PRIV_STACK_GUARD_SZ; 233 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); 234 if (!priv_stack_ptr) 235 goto out_priv_stack; 236 237 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); 238 fp->aux->priv_stack_ptr = priv_stack_ptr; 239 } 240 241 flen = fp->len; 242 addrs = jit_data->addrs; 243 if (addrs) { 244 cgctx = jit_data->ctx; 245 /* 246 * JIT compiled to a writable location (image/code_base) first. 247 * It is then moved to the readonly final location (fimage/fcode_base) 248 * using instruction patching. 249 */ 250 fimage = jit_data->fimage; 251 fhdr = jit_data->fhdr; 252 proglen = jit_data->proglen; 253 hdr = jit_data->hdr; 254 image = (void *)hdr + ((void *)fimage - (void *)fhdr); 255 extra_pass = true; 256 /* During extra pass, ensure index is reset before repopulating extable entries */ 257 cgctx.exentry_idx = 0; 258 goto skip_init_ctx; 259 } 260 261 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 262 if (addrs == NULL) 263 goto out_err; 264 265 memset(&cgctx, 0, sizeof(struct codegen_context)); 266 bpf_jit_init_reg_mapping(&cgctx); 267 268 /* Make sure that the stack is quadword aligned. */ 269 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 270 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); 271 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); 272 cgctx.is_subprog = bpf_is_subprog(fp); 273 cgctx.exception_boundary = fp->aux->exception_boundary; 274 cgctx.exception_cb = fp->aux->exception_cb; 275 cgctx.priv_sp = priv_stack_ptr; 276 cgctx.priv_stack_size = 0; 277 if (priv_stack_ptr) { 278 /* 279 * priv_stack_size required for setting bpf FP inside 280 * percpu allocation. 281 * stack_size is marked 0 to prevent allocation on 282 * general stack and offset calculation don't go for 283 * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local() 284 */ 285 cgctx.priv_stack_size = cgctx.stack_size; 286 cgctx.stack_size = 0; 287 } 288 289 /* Scouting faux-generate pass 0 */ 290 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 291 /* We hit something illegal or unsupported. */ 292 goto out_err; 293 294 /* 295 * If we have seen a tail call, we need a second pass. 296 * This is because bpf_jit_emit_common_epilogue() is called 297 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 298 * We also need a second pass if we ended up with too large 299 * a program so as to ensure BPF_EXIT branches are in range. 300 */ 301 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 302 cgctx.idx = 0; 303 cgctx.alt_exit_addr = 0; 304 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) 305 goto out_err; 306 } 307 308 bpf_jit_realloc_regs(&cgctx); 309 /* 310 * Pretend to build prologue, given the features we've seen. This will 311 * update ctgtx.idx as it pretends to output instructions, then we can 312 * calculate total size from idx. 313 */ 314 bpf_jit_build_prologue(NULL, &cgctx); 315 addrs[fp->len] = cgctx.idx * 4; 316 bpf_jit_build_epilogue(NULL, NULL, &cgctx); 317 318 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 319 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 320 321 proglen = cgctx.idx * 4; 322 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 323 324 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 325 bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); 326 if (!fhdr) 327 goto out_err; 328 329 if (extable_len) 330 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; 331 332 skip_init_ctx: 333 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 334 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); 335 336 /* Code generation passes 1-2+, loop until program size converges. */ 337 for (pass = 1; pass <= CODEGEN_MAX_PASSES; pass++) { 338 u32 prev_proglen = proglen; 339 340 /* Now build the prologue, body code & epilogue for real. */ 341 cgctx.idx = 0; 342 cgctx.exentry_idx = 0; 343 cgctx.alt_exit_addr = 0; 344 bpf_jit_build_prologue(code_base, &cgctx); 345 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, 346 extra_pass)) { 347 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 348 bpf_jit_binary_pack_free(fhdr, hdr); 349 goto out_err; 350 } 351 addrs[fp->len] = cgctx.idx * 4; 352 bpf_jit_build_epilogue(code_base, fcode_base, &cgctx); 353 354 proglen = cgctx.idx * 4; 355 356 if (bpf_jit_enable > 1) 357 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 358 prev_proglen - proglen, cgctx.seen); 359 360 /* Check if program size has converged, but ensure minimum passes */ 361 if (pass >= CODEGEN_MIN_PASSES && proglen == prev_proglen) 362 break; 363 364 if (pass == CODEGEN_MAX_PASSES && proglen != prev_proglen) { 365 pr_err("BPF JIT: Program did not converge after %d passes\n", 366 CODEGEN_MAX_PASSES); 367 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 368 bpf_jit_binary_pack_free(fhdr, hdr); 369 goto out_err; 370 } 371 } 372 373 if (bpf_jit_enable > 1) 374 /* 375 * Note that we output the base address of the code_base 376 * rather than image, since opcodes are in code_base. 377 */ 378 bpf_jit_dump(flen, proglen, pass, code_base); 379 380 #ifdef CONFIG_PPC64_ELF_ABI_V1 381 /* Function descriptor nastiness: Address + TOC */ 382 ((u64 *)image)[0] = (u64)fcode_base; 383 ((u64 *)image)[1] = local_paca->kernel_toc; 384 #endif 385 386 if (!fp->is_func || extra_pass) { 387 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) 388 goto out_err; 389 } 390 391 fp->bpf_func = (void *)fimage; 392 fp->jited = 1; 393 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; 394 395 if (!fp->is_func || extra_pass) { 396 bpf_prog_fill_jited_linfo(fp, addrs); 397 /* 398 * On ABI V1, executable code starts after the function 399 * descriptor, so adjust base accordingly. 400 */ 401 bpf_prog_update_insn_ptrs(fp, addrs, 402 (void *)fimage + FUNCTION_DESCR_SIZE); 403 404 out_addrs: 405 if (!fp->jited && priv_stack_ptr) { 406 fp->aux->priv_stack_ptr = NULL; 407 free_percpu(priv_stack_ptr); 408 } 409 out_priv_stack: 410 kfree(addrs); 411 kfree(jit_data); 412 fp->aux->jit_data = NULL; 413 } else { 414 jit_data->addrs = addrs; 415 jit_data->ctx = cgctx; 416 jit_data->proglen = proglen; 417 jit_data->fimage = fimage; 418 jit_data->fhdr = fhdr; 419 jit_data->hdr = hdr; 420 } 421 422 return fp; 423 424 out_err: 425 if (extra_pass) { 426 fp->bpf_func = NULL; 427 fp->jited = 0; 428 fp->jited_len = 0; 429 } 430 goto out_addrs; 431 } 432 433 /* 434 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 435 * this function, as this only applies to BPF_PROBE_MEM, for now. 436 */ 437 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, 438 struct codegen_context *ctx, int insn_idx, int jmp_off, 439 int dst_reg, u32 code) 440 { 441 off_t offset; 442 unsigned long pc; 443 struct exception_table_entry *ex, *ex_entry; 444 u32 *fixup; 445 446 /* Populate extable entries only in the last pass */ 447 if (pass < CODEGEN_MIN_PASSES) 448 return 0; 449 450 if (!fp->aux->extable || 451 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 452 return -EINVAL; 453 454 /* 455 * Program is first written to image before copying to the 456 * final location (fimage). Accordingly, update in the image first. 457 * As all offsets used are relative, copying as is to the 458 * final location should be alright. 459 */ 460 pc = (unsigned long)&image[insn_idx]; 461 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; 462 463 fixup = (void *)ex - 464 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 465 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 466 467 fixup[0] = PPC_RAW_LI(dst_reg, 0); 468 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX) 469 fixup[0] = PPC_RAW_NOP(); 470 471 if (IS_ENABLED(CONFIG_PPC32)) 472 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 473 474 fixup[BPF_FIXUP_LEN - 1] = 475 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 476 477 ex_entry = &ex[ctx->exentry_idx]; 478 479 offset = pc - (long)&ex_entry->insn; 480 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 481 return -ERANGE; 482 ex_entry->insn = offset; 483 484 offset = (long)fixup - (long)&ex_entry->fixup; 485 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 486 return -ERANGE; 487 ex_entry->fixup = offset; 488 489 ctx->exentry_idx++; 490 return 0; 491 } 492 493 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 494 { 495 int err; 496 497 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 498 return ERR_PTR(-EINVAL); 499 500 mutex_lock(&text_mutex); 501 err = patch_instructions(dst, src, len, false); 502 mutex_unlock(&text_mutex); 503 504 return err ? ERR_PTR(err) : dst; 505 } 506 507 int bpf_arch_text_invalidate(void *dst, size_t len) 508 { 509 u32 insn = BREAKPOINT_INSTRUCTION; 510 int ret; 511 512 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) 513 return -EINVAL; 514 515 mutex_lock(&text_mutex); 516 ret = patch_instructions(dst, &insn, len, true); 517 mutex_unlock(&text_mutex); 518 519 return ret; 520 } 521 522 void bpf_jit_free(struct bpf_prog *fp) 523 { 524 if (fp->jited) { 525 struct powerpc_jit_data *jit_data = fp->aux->jit_data; 526 struct bpf_binary_header *hdr; 527 void __percpu *priv_stack_ptr; 528 int priv_stack_alloc_size; 529 530 /* 531 * If we fail the final pass of JIT (from jit_subprogs), 532 * the program may not be finalized yet. Call finalize here 533 * before freeing it. 534 */ 535 if (jit_data) { 536 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr); 537 kvfree(jit_data->addrs); 538 kfree(jit_data); 539 } 540 hdr = bpf_jit_binary_pack_hdr(fp); 541 bpf_jit_binary_pack_free(hdr, NULL); 542 priv_stack_ptr = fp->aux->priv_stack_ptr; 543 if (priv_stack_ptr) { 544 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 545 2 * PRIV_STACK_GUARD_SZ; 546 priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp); 547 free_percpu(priv_stack_ptr); 548 } 549 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 550 } 551 552 bpf_prog_unlock_free(fp); 553 } 554 555 bool bpf_jit_supports_exceptions(void) 556 { 557 return IS_ENABLED(CONFIG_PPC64); 558 } 559 560 bool bpf_jit_supports_subprog_tailcalls(void) 561 { 562 return IS_ENABLED(CONFIG_PPC64); 563 } 564 565 bool bpf_jit_supports_timed_may_goto(void) 566 { 567 return IS_ENABLED(CONFIG_PPC64); 568 } 569 570 bool bpf_jit_supports_kfunc_call(void) 571 { 572 return IS_ENABLED(CONFIG_PPC64); 573 } 574 575 bool bpf_jit_supports_private_stack(void) 576 { 577 return IS_ENABLED(CONFIG_PPC64); 578 } 579 580 bool bpf_jit_supports_fsession(void) 581 { 582 /* 583 * TODO: Remove after validating support 584 * for fsession and trampoline on ppc32. 585 */ 586 if (IS_ENABLED(CONFIG_PPC32)) 587 return -EOPNOTSUPP; 588 return true; 589 } 590 591 bool bpf_jit_supports_arena(void) 592 { 593 return IS_ENABLED(CONFIG_PPC64); 594 } 595 596 bool bpf_jit_supports_far_kfunc_call(void) 597 { 598 return IS_ENABLED(CONFIG_PPC64); 599 } 600 601 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) 602 { 603 if (!in_arena) 604 return true; 605 switch (insn->code) { 606 case BPF_STX | BPF_ATOMIC | BPF_H: 607 case BPF_STX | BPF_ATOMIC | BPF_B: 608 case BPF_STX | BPF_ATOMIC | BPF_W: 609 case BPF_STX | BPF_ATOMIC | BPF_DW: 610 if (bpf_atomic_is_load_store(insn)) 611 return false; 612 return IS_ENABLED(CONFIG_PPC64); 613 } 614 return true; 615 } 616 617 bool bpf_jit_supports_percpu_insn(void) 618 { 619 return IS_ENABLED(CONFIG_PPC64); 620 } 621 622 bool bpf_jit_inlines_helper_call(s32 imm) 623 { 624 switch (imm) { 625 case BPF_FUNC_get_smp_processor_id: 626 case BPF_FUNC_get_current_task: 627 case BPF_FUNC_get_current_task_btf: 628 return true; 629 default: 630 return false; 631 } 632 } 633 634 void *arch_alloc_bpf_trampoline(unsigned int size) 635 { 636 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); 637 } 638 639 void arch_free_bpf_trampoline(void *image, unsigned int size) 640 { 641 bpf_prog_pack_free(image, size); 642 } 643 644 int arch_protect_bpf_trampoline(void *image, unsigned int size) 645 { 646 return 0; 647 } 648 649 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, 650 struct bpf_tramp_image *im, struct bpf_tramp_node *n, 651 int regs_off, int retval_off, int run_ctx_off, bool save_ret) 652 { 653 struct bpf_prog *p = n->link->prog; 654 ppc_inst_t branch_insn; 655 u32 jmp_idx, skip_idx; 656 int ret = 0; 657 658 /* nop, patched to skip this prog when it is detached */ 659 skip_idx = ctx->idx; 660 EMIT(PPC_RAW_NOP()); 661 662 /* Save cookie */ 663 if (IS_ENABLED(CONFIG_PPC64)) { 664 PPC_LI64(_R3, n->cookie); 665 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx, 666 bpf_cookie))); 667 } else { 668 PPC_LI32(_R3, n->cookie >> 32); 669 PPC_LI32(_R4, n->cookie); 670 EMIT(PPC_RAW_STW(_R3, _R1, 671 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie))); 672 EMIT(PPC_RAW_STW(_R4, _R1, 673 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4)); 674 } 675 676 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */ 677 PPC_LI_ADDR(_R3, p); 678 EMIT(PPC_RAW_MR(_R25, _R3)); 679 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off)); 680 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 681 (unsigned long)bpf_trampoline_enter(p)); 682 if (ret) 683 return ret; 684 685 /* Remember prog start time returned by __bpf_prog_enter */ 686 EMIT(PPC_RAW_MR(_R26, _R3)); 687 688 /* 689 * if (__bpf_prog_enter(p) == 0) 690 * goto skip_exec_of_prog; 691 * 692 * Emit a nop to be later patched with conditional branch, once offset is known 693 */ 694 EMIT(PPC_RAW_CMPLI(_R3, 0)); 695 jmp_idx = ctx->idx; 696 EMIT(PPC_RAW_NOP()); 697 698 /* p->bpf_func(ctx) */ 699 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off)); 700 if (!p->jited) 701 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi); 702 /* Account for max possible instructions during dummy pass for size calculation */ 703 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], 704 (unsigned long)p->bpf_func, 705 BRANCH_SET_LINK)) { 706 image[ctx->idx] = ppc_inst_val(branch_insn); 707 ctx->idx++; 708 } else { 709 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func))); 710 EMIT(PPC_RAW_MTCTR(_R12)); 711 EMIT(PPC_RAW_BCTRL()); 712 } 713 714 if (save_ret) 715 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 716 717 /* Fix up branch */ 718 if (image) { 719 if (create_cond_branch(&branch_insn, &image[jmp_idx], 720 (unsigned long)&image[ctx->idx], COND_EQ << 16)) 721 return -EINVAL; 722 image[jmp_idx] = ppc_inst_val(branch_insn); 723 } 724 725 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */ 726 EMIT(PPC_RAW_MR(_R3, _R25)); 727 EMIT(PPC_RAW_MR(_R4, _R26)); 728 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); 729 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 730 (unsigned long)bpf_trampoline_exit(p)); 731 if (ret) 732 return ret; 733 734 if (ro_image) /* image is NULL for dummy pass */ 735 bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); 736 return 0; 737 } 738 739 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, 740 struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, 741 int regs_off, int retval_off, int run_ctx_off, u32 *branches) 742 { 743 int i; 744 745 /* 746 * The first fmod_ret program will receive a garbage return value. 747 * Set this to 0 to avoid confusing the program. 748 */ 749 EMIT(PPC_RAW_LI(_R3, 0)); 750 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 751 for (i = 0; i < tn->nr_nodes; i++) { 752 if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, 753 retval_off, run_ctx_off, true)) 754 return -EINVAL; 755 756 /* 757 * mod_ret prog stored return value after prog ctx. Emit: 758 * if (*(u64 *)(ret_val) != 0) 759 * goto do_fexit; 760 */ 761 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 762 EMIT(PPC_RAW_CMPLI(_R3, 0)); 763 764 /* 765 * Save the location of the branch and generate a nop, which is 766 * replaced with a conditional jump once do_fexit (i.e. the 767 * start of the fexit invocation) is finalized. 768 */ 769 branches[i] = ctx->idx; 770 EMIT(PPC_RAW_NOP()); 771 } 772 773 return 0; 774 } 775 776 /* 777 * Refer __arch_prepare_bpf_trampoline() for stack component details. 778 * 779 * The tailcall count/reference is present in caller's stack frame. The 780 * tail_call_info is saved at the same offset on the trampoline frame 781 * for the traced function (BPF subprog/callee) to fetch it. 782 */ 783 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, 784 int bpf_frame_size, int r4_off) 785 { 786 if (IS_ENABLED(CONFIG_PPC64)) { 787 EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size)); 788 /* Refer to trampoline's Generated stack layout */ 789 EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL)); 790 791 /* 792 * Setting the tail_call_info in trampoline's frame 793 * depending on if previous frame had value or reference. 794 */ 795 EMIT(PPC_RAW_CMPLLI(_R3, MAX_TAIL_CALL_CNT)); 796 PPC_BCC_CONST_SHORT(COND_GT, 8); 797 EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL)); 798 799 /* 800 * Trampoline's tail_call_info is at the same offset, as that of 801 * any bpf program, with reference to previous frame. Update the 802 * address of main's tail_call_info in trampoline frame. 803 */ 804 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL)); 805 } else { 806 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ 807 EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); 808 } 809 } 810 811 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx, 812 int bpf_frame_size, int r4_off) 813 { 814 if (IS_ENABLED(CONFIG_PPC32)) { 815 /* 816 * Restore tailcall for 32-bit powerpc 817 * See bpf_jit_stack_offsetof() and BPF_PPC_TC 818 */ 819 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 820 } 821 } 822 823 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, 824 int bpf_frame_size, int nr_regs, int regs_off) 825 { 826 int param_save_area_offset; 827 828 param_save_area_offset = bpf_frame_size; 829 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 830 831 for (int i = 0; i < nr_regs; i++) { 832 if (i < 8) { 833 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL)); 834 } else { 835 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 836 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL)); 837 } 838 } 839 } 840 841 /* Used when restoring just the register parameters when returning back */ 842 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx, 843 int nr_regs, int regs_off) 844 { 845 for (int i = 0; i < nr_regs && i < 8; i++) 846 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL)); 847 } 848 849 /* Used when we call into the traced function. Replicate parameter save area */ 850 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx, 851 int bpf_frame_size, int nr_regs, int regs_off) 852 { 853 int param_save_area_offset; 854 855 param_save_area_offset = bpf_frame_size; 856 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */ 857 858 for (int i = 8; i < nr_regs; i++) { 859 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL)); 860 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL)); 861 } 862 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 863 } 864 865 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image, 866 void *rw_image_end, void *ro_image, 867 const struct btf_func_model *m, u32 flags, 868 struct bpf_tramp_nodes *tnodes, 869 void *func_addr) 870 { 871 int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off; 872 int nvr_off, alt_lr_off, r4_off = 0; 873 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 874 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 875 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 876 int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; 877 struct codegen_context codegen_ctx, *ctx; 878 int cookie_off, cookie_cnt, cookie_ctx_off; 879 int fsession_cnt = bpf_fsession_cnt(tnodes); 880 u64 func_meta; 881 u32 *image = (u32 *)rw_image; 882 ppc_inst_t branch_insn; 883 u32 *branches = NULL; 884 bool save_ret; 885 886 if (IS_ENABLED(CONFIG_PPC32)) 887 return -EOPNOTSUPP; 888 889 nr_regs = m->nr_args; 890 /* Extra registers for struct arguments */ 891 for (i = 0; i < m->nr_args; i++) 892 if (m->arg_size[i] > SZL) 893 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1; 894 895 if (nr_regs > MAX_BPF_FUNC_ARGS) 896 return -EOPNOTSUPP; 897 898 ctx = &codegen_ctx; 899 memset(ctx, 0, sizeof(*ctx)); 900 901 /* 902 * Generated stack layout: 903 * 904 * func prev back chain [ back chain ] 905 * [ tail_call_info ] optional - 64-bit powerpc 906 * [ padding ] align stack frame 907 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc 908 * alt_lr_off [ real lr (ool stub)] optional - actual lr 909 * retaddr_off [ return address ] 910 * [ r26 ] 911 * nvr_off [ r25 ] nvr save area 912 * retval_off [ return value ] 913 * [ reg argN ] 914 * [ ... ] 915 * regs_off [ reg_arg1 ] prog_ctx 916 * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1] 917 * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] 918 * [ stack cookieN ] 919 * [ ... ] 920 * cookie_off [ stack cookie1 ] 921 * run_ctx_off [ bpf_tramp_run_ctx ] 922 * [ reg argN ] 923 * [ ... ] 924 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI 925 * [ TOC save (64-bit) ] -- 926 * [ LR save (64-bit) ] | header 927 * [ LR save (32-bit) ] | 928 * bpf trampoline frame [ back chain 2 ] -- 929 * 930 */ 931 932 /* Minimum stack frame header */ 933 bpf_frame_size = STACK_FRAME_MIN_SIZE; 934 935 /* 936 * Room for parameter save area. 937 * 938 * As per the ABI, this is required if we call into the traced 939 * function (BPF_TRAMP_F_CALL_ORIG): 940 * - if the function takes more than 8 arguments for the rest to spill onto the stack 941 * - or, if the function has variadic arguments 942 * - or, if this functions's prototype was not available to the caller 943 * 944 * Reserve space for at least 8 registers for now. This can be optimized later. 945 */ 946 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL; 947 948 /* Room for struct bpf_tramp_run_ctx */ 949 run_ctx_off = bpf_frame_size; 950 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); 951 952 /* room for session cookies */ 953 cookie_off = bpf_frame_size; 954 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 955 bpf_frame_size += cookie_cnt * 8; 956 957 /* Room for IP address argument */ 958 ip_off = bpf_frame_size; 959 if (flags & BPF_TRAMP_F_IP_ARG) 960 bpf_frame_size += SZL; 961 962 /* Room for function metadata, arg regs count */ 963 func_meta_off = bpf_frame_size; 964 bpf_frame_size += SZL; 965 966 /* Room for arg regs */ 967 regs_off = bpf_frame_size; 968 bpf_frame_size += nr_regs * SZL; 969 970 /* Room for return value of func_addr or fentry prog */ 971 retval_off = bpf_frame_size; 972 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 973 if (save_ret) 974 bpf_frame_size += SZL; 975 976 /* Room for nvr save area */ 977 nvr_off = bpf_frame_size; 978 bpf_frame_size += 2 * SZL; 979 980 /* Save area for return address */ 981 retaddr_off = bpf_frame_size; 982 bpf_frame_size += SZL; 983 984 /* Optional save area for actual LR in case of ool ftrace */ 985 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 986 alt_lr_off = bpf_frame_size; 987 bpf_frame_size += SZL; 988 } 989 990 if (IS_ENABLED(CONFIG_PPC32)) { 991 if (nr_regs < 2) { 992 r4_off = bpf_frame_size; 993 bpf_frame_size += SZL; 994 } else { 995 r4_off = regs_off + SZL; 996 } 997 } 998 999 /* 1000 * Save tailcall count pointer at the same offset on the 1001 * stack where subprogs expect it 1002 */ 1003 if ((flags & BPF_TRAMP_F_CALL_ORIG) && 1004 (flags & BPF_TRAMP_F_TAIL_CALL_CTX)) 1005 bpf_frame_size += BPF_PPC_TAILCALL; 1006 1007 /* Padding to align stack frame, if any */ 1008 bpf_frame_size = round_up(bpf_frame_size, SZL * 2); 1009 1010 /* Store original return value */ 1011 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF)); 1012 1013 /* Create our stack frame */ 1014 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size)); 1015 1016 /* 64-bit: Save TOC and load kernel TOC */ 1017 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 1018 EMIT(PPC_RAW_STD(_R2, _R1, 24)); 1019 PPC64_LOAD_PACA(); 1020 } 1021 1022 /* 32-bit: save tail call count in r4 */ 1023 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2) 1024 EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); 1025 1026 bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off); 1027 1028 /* Save our LR/return address */ 1029 EMIT(PPC_RAW_MFLR(_R3)); 1030 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) 1031 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off)); 1032 else 1033 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 1034 1035 /* 1036 * Derive IP address of the traced function. 1037 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction 1038 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and 1039 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively. 1040 * Relevant stub sequence: 1041 * 1042 * bl <tramp> 1043 * LR (R3) => mtlr r0 1044 * b <func_addr+4> 1045 * 1046 * Recover kernel function/bpf program address from the unconditional 1047 * branch instruction at the end of OOL stub. 1048 */ 1049 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) { 1050 EMIT(PPC_RAW_LWZ(_R4, _R3, 4)); 1051 EMIT(PPC_RAW_SLWI(_R4, _R4, 6)); 1052 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6)); 1053 EMIT(PPC_RAW_ADD(_R3, _R3, _R4)); 1054 } 1055 1056 if (flags & BPF_TRAMP_F_IP_ARG) 1057 EMIT(PPC_RAW_STL(_R3, _R1, ip_off)); 1058 1059 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1060 /* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */ 1061 EMIT(PPC_RAW_ADDI(_R3, _R3, 4)); 1062 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); 1063 } 1064 1065 /* Save function arg regs count -- see bpf_get_func_arg_cnt() */ 1066 func_meta = nr_regs; 1067 store_func_meta(image, ctx, func_meta, func_meta_off); 1068 1069 /* Save nv regs */ 1070 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); 1071 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL)); 1072 1073 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1074 PPC_LI_ADDR(_R3, (unsigned long)im); 1075 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1076 (unsigned long)__bpf_tramp_enter); 1077 if (ret) 1078 return ret; 1079 } 1080 1081 if (fsession_cnt) { 1082 /* 1083 * Clear all the session cookies' values 1084 * Clear the return value to make sure fentry always get 0 1085 */ 1086 prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off); 1087 } 1088 1089 cookie_ctx_off = (regs_off - cookie_off) / 8; 1090 1091 for (i = 0; i < fentry->nr_nodes; i++) { 1092 if (bpf_prog_calls_session_cookie(fentry->nodes[i])) { 1093 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1094 1095 store_func_meta(image, ctx, meta, func_meta_off); 1096 cookie_ctx_off--; 1097 } 1098 1099 if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, 1100 retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) 1101 return -EINVAL; 1102 } 1103 1104 if (fmod_ret->nr_nodes) { 1105 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32), GFP_KERNEL); 1106 if (!branches) 1107 return -ENOMEM; 1108 1109 if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, 1110 run_ctx_off, branches)) { 1111 ret = -EINVAL; 1112 goto cleanup; 1113 } 1114 } 1115 1116 /* Call the traced function */ 1117 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1118 /* 1119 * retaddr on trampoline stack points to the correct point in the original function 1120 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction 1121 * sequence 1122 */ 1123 EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off)); 1124 EMIT(PPC_RAW_MTCTR(_R3)); 1125 1126 /* Replicate tail_call_cnt before calling the original BPF prog */ 1127 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1128 bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off); 1129 1130 /* Restore args */ 1131 bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off); 1132 1133 /* Restore TOC for 64-bit */ 1134 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1135 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1136 EMIT(PPC_RAW_BCTRL()); 1137 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1138 PPC64_LOAD_PACA(); 1139 1140 /* Store return value for bpf prog to access */ 1141 EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); 1142 1143 /* Restore updated tail_call_cnt */ 1144 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1145 bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); 1146 } 1147 1148 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ 1149 for (i = 0; i < fmod_ret->nr_nodes && image; i++) { 1150 if (create_cond_branch(&branch_insn, &image[branches[i]], 1151 (unsigned long)&image[ctx->idx], COND_NE << 16)) { 1152 ret = -EINVAL; 1153 goto cleanup; 1154 } 1155 1156 image[branches[i]] = ppc_inst_val(branch_insn); 1157 } 1158 1159 /* set the "is_return" flag for fsession */ 1160 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 1161 if (fsession_cnt) 1162 store_func_meta(image, ctx, func_meta, func_meta_off); 1163 1164 cookie_ctx_off = (regs_off - cookie_off) / 8; 1165 1166 for (i = 0; i < fexit->nr_nodes; i++) { 1167 if (bpf_prog_calls_session_cookie(fexit->nodes[i])) { 1168 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1169 1170 store_func_meta(image, ctx, meta, func_meta_off); 1171 cookie_ctx_off--; 1172 } 1173 1174 if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, 1175 retval_off, run_ctx_off, false)) { 1176 ret = -EINVAL; 1177 goto cleanup; 1178 } 1179 } 1180 1181 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1182 PPC_LI_ADDR(_R3, im); 1183 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, 1184 (unsigned long)__bpf_tramp_exit); 1185 if (ret) 1186 goto cleanup; 1187 } 1188 1189 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1190 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off); 1191 1192 /* Restore return value of func_addr or fentry prog */ 1193 if (save_ret) 1194 EMIT(PPC_RAW_LL(_R3, _R1, retval_off)); 1195 1196 /* Restore nv regs */ 1197 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL)); 1198 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off)); 1199 1200 /* Epilogue */ 1201 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1202 EMIT(PPC_RAW_LD(_R2, _R1, 24)); 1203 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 1204 /* Skip the traced function and return to parent */ 1205 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1206 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1207 EMIT(PPC_RAW_MTLR(_R0)); 1208 EMIT(PPC_RAW_BLR()); 1209 } else { 1210 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) { 1211 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off)); 1212 EMIT(PPC_RAW_MTLR(_R0)); 1213 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1214 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1215 EMIT(PPC_RAW_BLR()); 1216 } else { 1217 EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off)); 1218 EMIT(PPC_RAW_MTCTR(_R0)); 1219 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size)); 1220 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF)); 1221 EMIT(PPC_RAW_MTLR(_R0)); 1222 EMIT(PPC_RAW_BCTR()); 1223 } 1224 } 1225 1226 /* Make sure the trampoline generation logic doesn't overflow */ 1227 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) { 1228 ret = -EFAULT; 1229 goto cleanup; 1230 } 1231 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4; 1232 1233 cleanup: 1234 kfree(branches); 1235 return ret; 1236 } 1237 1238 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1239 struct bpf_tramp_nodes *tnodes, void *func_addr) 1240 { 1241 struct bpf_tramp_image im = {}; 1242 int ret; 1243 1244 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); 1245 return ret; 1246 } 1247 1248 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, 1249 const struct btf_func_model *m, u32 flags, 1250 struct bpf_tramp_nodes *tnodes, 1251 void *func_addr) 1252 { 1253 u32 size = image_end - image; 1254 void *rw_image, *tmp; 1255 int ret; 1256 1257 /* 1258 * rw_image doesn't need to be in module memory range, so we can 1259 * use kvmalloc. 1260 */ 1261 rw_image = kvmalloc(size, GFP_KERNEL); 1262 if (!rw_image) 1263 return -ENOMEM; 1264 1265 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m, 1266 flags, tnodes, func_addr); 1267 if (ret < 0) 1268 goto out; 1269 1270 if (bpf_jit_enable > 1) 1271 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image); 1272 1273 tmp = bpf_arch_text_copy(image, rw_image, size); 1274 if (IS_ERR(tmp)) 1275 ret = PTR_ERR(tmp); 1276 1277 out: 1278 kvfree(rw_image); 1279 return ret; 1280 } 1281 1282 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst) 1283 { 1284 ppc_inst_t org_inst; 1285 1286 if (copy_inst_from_kernel_nofault(&org_inst, ip)) { 1287 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip); 1288 return -EFAULT; 1289 } 1290 1291 if (!ppc_inst_equal(org_inst, old_inst)) { 1292 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n", 1293 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst)); 1294 return -EINVAL; 1295 } 1296 1297 if (ppc_inst_equal(old_inst, new_inst)) 1298 return 0; 1299 1300 return patch_instruction(ip, new_inst); 1301 } 1302 1303 static void do_isync(void *info __maybe_unused) 1304 { 1305 isync(); 1306 } 1307 1308 /* 1309 * A 3-step process for bpf prog entry: 1310 * 1. At bpf prog entry, a single nop/b: 1311 * bpf_func: 1312 * [nop|b] ool_stub 1313 * 2. Out-of-line stub: 1314 * nop // optional nop for alignment 1315 * ool_stub: 1316 * mflr r0 1317 * [b|bl] <bpf_prog>/<long_branch_stub> 1318 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only 1319 * b bpf_func + 4 1320 * 3. Long branch stub: 1321 * long_branch_stub: 1322 * mflr r11 1323 * bcl 20,31,$+4 1324 * mflr r12 1325 * ld r12, 20(r12) 1326 * mtctr r12 1327 * mtlr r11 // needed to retain ftrace ABI 1328 * bctr 1329 * .long <branch_addr>/<dummy_tramp> 1330 * 1331 * dummy_tramp is used to reduce synchronization requirements. 1332 * 1333 * When attaching a bpf trampoline to a bpf prog, we do not need any 1334 * synchronization here since we always have a valid branch target regardless 1335 * of the order in which the above stores are seen. dummy_tramp ensures that 1336 * the long_branch stub goes to a valid destination on other cpus, even when 1337 * the branch to the long_branch stub is seen before the updated trampoline 1338 * address. 1339 * 1340 * However, when detaching a bpf trampoline from a bpf prog, or if changing 1341 * the bpf trampoline address, we need synchronization to ensure that other 1342 * cpus can no longer branch into the older trampoline so that it can be 1343 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus 1344 * make forward progress, but we still need to ensure that other cpus 1345 * execute isync (or some CSI) so that they don't go back into the 1346 * trampoline again. 1347 */ 1348 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1349 enum bpf_text_poke_type new_t, void *old_addr, 1350 void *new_addr) 1351 { 1352 unsigned long bpf_func, bpf_func_end, size, offset; 1353 ppc_inst_t old_inst, new_inst; 1354 int ret = 0, branch_flags; 1355 char name[KSYM_NAME_LEN]; 1356 1357 if (IS_ENABLED(CONFIG_PPC32)) 1358 return -EOPNOTSUPP; 1359 1360 bpf_func = (unsigned long)ip; 1361 1362 /* We currently only support poking bpf programs */ 1363 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) { 1364 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func); 1365 return -EOPNOTSUPP; 1366 } 1367 1368 /* 1369 * If we are not poking at bpf prog entry, then we are simply patching in/out 1370 * an unconditional branch instruction in a trampoline image 1371 */ 1372 if (offset) { 1373 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { 1374 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__, 1375 bpf_func); 1376 return -EOPNOTSUPP; 1377 } 1378 old_inst = ppc_inst(PPC_RAW_NOP()); 1379 if (old_addr) 1380 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0)) 1381 return -ERANGE; 1382 new_inst = ppc_inst(PPC_RAW_NOP()); 1383 if (new_addr) 1384 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0)) 1385 return -ERANGE; 1386 mutex_lock(&text_mutex); 1387 ret = bpf_modify_inst(ip, old_inst, new_inst); 1388 mutex_unlock(&text_mutex); 1389 1390 /* Make sure all cpus see the new instruction */ 1391 smp_call_function(do_isync, NULL, 1); 1392 return ret; 1393 } 1394 1395 bpf_func_end = bpf_func + size; 1396 1397 /* Address of the jmp/call instruction in the out-of-line stub */ 1398 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4); 1399 1400 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) { 1401 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__, 1402 bpf_func); 1403 return -ERANGE; 1404 } 1405 1406 old_inst = ppc_inst(PPC_RAW_NOP()); 1407 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1408 if (old_addr) { 1409 if (is_offset_in_branch_range(ip - old_addr)) 1410 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags); 1411 else 1412 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1413 branch_flags); 1414 } 1415 new_inst = ppc_inst(PPC_RAW_NOP()); 1416 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0; 1417 if (new_addr) { 1418 if (is_offset_in_branch_range(ip - new_addr)) 1419 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags); 1420 else 1421 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub, 1422 branch_flags); 1423 } 1424 1425 mutex_lock(&text_mutex); 1426 1427 /* 1428 * 1. Update the address in the long branch stub: 1429 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr 1430 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here. 1431 * 1432 * dummy_tramp_addr moved to bottom of long branch stub. 1433 */ 1434 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) || 1435 (old_addr && !is_offset_in_branch_range(old_addr - ip))) 1436 ret = patch_ulong((void *)(bpf_func_end - SZL), /* SZL: dummy_tramp_addr offset */ 1437 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ? 1438 (unsigned long)new_addr : (unsigned long)dummy_tramp); 1439 if (ret) 1440 goto out; 1441 1442 /* 2. Update the branch/call in the out-of-line stub */ 1443 ret = bpf_modify_inst(ip, old_inst, new_inst); 1444 if (ret) 1445 goto out; 1446 1447 /* 3. Update instruction at bpf prog entry */ 1448 ip = (void *)bpf_func; 1449 if (!old_addr || !new_addr) { 1450 if (!old_addr) { 1451 old_inst = ppc_inst(PPC_RAW_NOP()); 1452 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1453 } else { 1454 new_inst = ppc_inst(PPC_RAW_NOP()); 1455 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0); 1456 } 1457 ret = bpf_modify_inst(ip, old_inst, new_inst); 1458 } 1459 1460 out: 1461 mutex_unlock(&text_mutex); 1462 1463 /* 1464 * Sync only if we are not attaching a trampoline to a bpf prog so the older 1465 * trampoline can be freed safely. 1466 */ 1467 if (old_addr) 1468 smp_call_function(do_isync, NULL, 1); 1469 1470 return ret; 1471 } 1472