1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <linux/kernel.h>
17 #include <linux/memory.h>
18 #include <linux/bpf.h>
19
20 #include <asm/kprobes.h>
21 #include <asm/text-patching.h>
22
23 #include "bpf_jit.h"
24
25 /* These offsets are from bpf prog end and stay the same across progs */
26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub;
27
bpf_jit_fill_ill_insns(void * area,unsigned int size)28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
29 {
30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
31 }
32
33 void dummy_tramp(void);
34
35 asm (
36 " .pushsection .text, \"ax\", @progbits ;"
37 " .global dummy_tramp ;"
38 " .type dummy_tramp, @function ;"
39 "dummy_tramp: ;"
40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
41 " blr ;"
42 #else
43 /* LR is always in r11, so we don't need a 'mflr r11' here */
44 " mtctr 11 ;"
45 " mtlr 0 ;"
46 " bctr ;"
47 #endif
48 " .size dummy_tramp, .-dummy_tramp ;"
49 " .popsection ;"
50 );
51
bpf_jit_build_fentry_stubs(u32 * image,struct codegen_context * ctx)52 void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
53 {
54 int ool_stub_idx, long_branch_stub_idx;
55
56 /*
57 * Out-of-line stub:
58 * mflr r0
59 * [b|bl] tramp
60 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE
61 * b bpf_func + 4
62 */
63 ool_stub_idx = ctx->idx;
64 EMIT(PPC_RAW_MFLR(_R0));
65 EMIT(PPC_RAW_NOP());
66 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
67 EMIT(PPC_RAW_MTLR(_R0));
68 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4));
69 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4));
70
71 /*
72 * Long branch stub:
73 * .long <dummy_tramp_addr>
74 * mflr r11
75 * bcl 20,31,$+4
76 * mflr r12
77 * ld r12, -8-SZL(r12)
78 * mtctr r12
79 * mtlr r11 // needed to retain ftrace ABI
80 * bctr
81 */
82 if (image)
83 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;
84 ctx->idx += SZL / 4;
85 long_branch_stub_idx = ctx->idx;
86 EMIT(PPC_RAW_MFLR(_R11));
87 EMIT(PPC_RAW_BCL4());
88 EMIT(PPC_RAW_MFLR(_R12));
89 EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL));
90 EMIT(PPC_RAW_MTCTR(_R12));
91 EMIT(PPC_RAW_MTLR(_R11));
92 EMIT(PPC_RAW_BCTR());
93
94 if (!bpf_jit_ool_stub) {
95 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4;
96 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4;
97 }
98 }
99
bpf_jit_emit_exit_insn(u32 * image,struct codegen_context * ctx,int tmp_reg,long exit_addr)100 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
101 {
102 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
103 PPC_JMP(exit_addr);
104 } else if (ctx->alt_exit_addr) {
105 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
106 return -1;
107 PPC_JMP(ctx->alt_exit_addr);
108 } else {
109 ctx->alt_exit_addr = ctx->idx * 4;
110 bpf_jit_build_epilogue(image, ctx);
111 }
112
113 return 0;
114 }
115
116 struct powerpc_jit_data {
117 /* address of rw header */
118 struct bpf_binary_header *hdr;
119 /* address of ro final header */
120 struct bpf_binary_header *fhdr;
121 u32 *addrs;
122 u8 *fimage;
123 u32 proglen;
124 struct codegen_context ctx;
125 };
126
bpf_jit_needs_zext(void)127 bool bpf_jit_needs_zext(void)
128 {
129 return true;
130 }
131
priv_stack_init_guard(void __percpu * priv_stack_ptr,int alloc_size)132 static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
133 {
134 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
135 u64 *stack_ptr;
136
137 for_each_possible_cpu(cpu) {
138 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
139 stack_ptr[0] = PRIV_STACK_GUARD_VAL;
140 stack_ptr[1] = PRIV_STACK_GUARD_VAL;
141 stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
142 stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL;
143 }
144 }
145
priv_stack_check_guard(void __percpu * priv_stack_ptr,int alloc_size,struct bpf_prog * fp)146 static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
147 struct bpf_prog *fp)
148 {
149 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
150 u64 *stack_ptr;
151
152 for_each_possible_cpu(cpu) {
153 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
154 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
155 stack_ptr[1] != PRIV_STACK_GUARD_VAL ||
156 stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL ||
157 stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) {
158 pr_err("BPF private stack overflow/underflow detected for prog %s\n",
159 bpf_jit_get_prog_name(fp));
160 break;
161 }
162 }
163 }
164
bpf_int_jit_compile(struct bpf_verifier_env * env,struct bpf_prog * fp)165 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
166 {
167 u32 proglen;
168 u32 alloclen;
169 u8 *image = NULL;
170 u32 *code_base = NULL;
171 u32 *addrs = NULL;
172 struct powerpc_jit_data *jit_data = NULL;
173 struct codegen_context cgctx;
174 int pass;
175 int flen;
176 int priv_stack_alloc_size;
177 void __percpu *priv_stack_ptr = NULL;
178 struct bpf_binary_header *fhdr = NULL;
179 struct bpf_binary_header *hdr = NULL;
180 bool extra_pass = false;
181 u8 *fimage = NULL;
182 u32 *fcode_base = NULL;
183 u32 extable_len;
184 u32 fixup_len;
185
186 if (!fp->jit_requested)
187 return fp;
188
189 jit_data = fp->aux->jit_data;
190 if (!jit_data) {
191 jit_data = kzalloc_obj(*jit_data);
192 if (!jit_data)
193 return fp;
194 fp->aux->jit_data = jit_data;
195 }
196
197 priv_stack_ptr = fp->aux->priv_stack_ptr;
198 if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) {
199 /*
200 * Allocate private stack of size equivalent to
201 * verifier-calculated stack size plus two memory
202 * guard regions to detect private stack overflow
203 * and underflow.
204 */
205 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
206 2 * PRIV_STACK_GUARD_SZ;
207 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL);
208 if (!priv_stack_ptr)
209 goto out_priv_stack;
210
211 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size);
212 fp->aux->priv_stack_ptr = priv_stack_ptr;
213 }
214
215 flen = fp->len;
216 addrs = jit_data->addrs;
217 if (addrs) {
218 cgctx = jit_data->ctx;
219 /*
220 * JIT compiled to a writable location (image/code_base) first.
221 * It is then moved to the readonly final location (fimage/fcode_base)
222 * using instruction patching.
223 */
224 fimage = jit_data->fimage;
225 fhdr = jit_data->fhdr;
226 proglen = jit_data->proglen;
227 hdr = jit_data->hdr;
228 image = (void *)hdr + ((void *)fimage - (void *)fhdr);
229 extra_pass = true;
230 /* During extra pass, ensure index is reset before repopulating extable entries */
231 cgctx.exentry_idx = 0;
232 goto skip_init_ctx;
233 }
234
235 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
236 if (addrs == NULL)
237 goto out_err;
238
239 memset(&cgctx, 0, sizeof(struct codegen_context));
240 bpf_jit_init_reg_mapping(&cgctx);
241
242 /* Make sure that the stack is quadword aligned. */
243 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
244 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
245 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
246 cgctx.is_subprog = bpf_is_subprog(fp);
247 cgctx.exception_boundary = fp->aux->exception_boundary;
248 cgctx.exception_cb = fp->aux->exception_cb;
249 cgctx.priv_sp = priv_stack_ptr;
250 cgctx.priv_stack_size = 0;
251 if (priv_stack_ptr) {
252 /*
253 * priv_stack_size required for setting bpf FP inside
254 * percpu allocation.
255 * stack_size is marked 0 to prevent allocation on
256 * general stack and offset calculation don't go for
257 * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local()
258 */
259 cgctx.priv_stack_size = cgctx.stack_size;
260 cgctx.stack_size = 0;
261 }
262
263 /* Scouting faux-generate pass 0 */
264 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
265 /* We hit something illegal or unsupported. */
266 goto out_err;
267
268 /*
269 * If we have seen a tail call, we need a second pass.
270 * This is because bpf_jit_emit_common_epilogue() is called
271 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
272 * We also need a second pass if we ended up with too large
273 * a program so as to ensure BPF_EXIT branches are in range.
274 */
275 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
276 cgctx.idx = 0;
277 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
278 goto out_err;
279 }
280
281 bpf_jit_realloc_regs(&cgctx);
282 /*
283 * Pretend to build prologue, given the features we've seen. This will
284 * update ctgtx.idx as it pretends to output instructions, then we can
285 * calculate total size from idx.
286 */
287 bpf_jit_build_prologue(NULL, &cgctx);
288 addrs[fp->len] = cgctx.idx * 4;
289 bpf_jit_build_epilogue(NULL, &cgctx);
290
291 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
292 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
293
294 proglen = cgctx.idx * 4;
295 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
296
297 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
298 bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp));
299 if (!fhdr)
300 goto out_err;
301
302 if (extable_len)
303 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
304
305 skip_init_ctx:
306 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
307 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
308
309 /* Code generation passes 1-2 */
310 for (pass = 1; pass < 3; pass++) {
311 /* Now build the prologue, body code & epilogue for real. */
312 cgctx.idx = 0;
313 cgctx.alt_exit_addr = 0;
314 bpf_jit_build_prologue(code_base, &cgctx);
315 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
316 extra_pass)) {
317 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
318 bpf_jit_binary_pack_free(fhdr, hdr);
319 goto out_err;
320 }
321 bpf_jit_build_epilogue(code_base, &cgctx);
322
323 if (bpf_jit_enable > 1)
324 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
325 proglen - (cgctx.idx * 4), cgctx.seen);
326 }
327
328 if (bpf_jit_enable > 1)
329 /*
330 * Note that we output the base address of the code_base
331 * rather than image, since opcodes are in code_base.
332 */
333 bpf_jit_dump(flen, proglen, pass, code_base);
334
335 #ifdef CONFIG_PPC64_ELF_ABI_V1
336 /* Function descriptor nastiness: Address + TOC */
337 ((u64 *)image)[0] = (u64)fcode_base;
338 ((u64 *)image)[1] = local_paca->kernel_toc;
339 #endif
340
341 if (!fp->is_func || extra_pass) {
342 if (bpf_jit_binary_pack_finalize(fhdr, hdr))
343 goto out_err;
344 }
345
346 fp->bpf_func = (void *)fimage;
347 fp->jited = 1;
348 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;
349
350 if (!fp->is_func || extra_pass) {
351 bpf_prog_fill_jited_linfo(fp, addrs);
352 /*
353 * On ABI V1, executable code starts after the function
354 * descriptor, so adjust base accordingly.
355 */
356 bpf_prog_update_insn_ptrs(fp, addrs,
357 (void *)fimage + FUNCTION_DESCR_SIZE);
358
359 out_addrs:
360 if (!image && priv_stack_ptr) {
361 fp->aux->priv_stack_ptr = NULL;
362 free_percpu(priv_stack_ptr);
363 }
364 out_priv_stack:
365 kfree(addrs);
366 kfree(jit_data);
367 fp->aux->jit_data = NULL;
368 } else {
369 jit_data->addrs = addrs;
370 jit_data->ctx = cgctx;
371 jit_data->proglen = proglen;
372 jit_data->fimage = fimage;
373 jit_data->fhdr = fhdr;
374 jit_data->hdr = hdr;
375 }
376
377 return fp;
378
379 out_err:
380 if (extra_pass) {
381 fp->bpf_func = NULL;
382 fp->jited = 0;
383 fp->jited_len = 0;
384 }
385 goto out_addrs;
386 }
387
388 /*
389 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
390 * this function, as this only applies to BPF_PROBE_MEM, for now.
391 */
bpf_add_extable_entry(struct bpf_prog * fp,u32 * image,u32 * fimage,int pass,struct codegen_context * ctx,int insn_idx,int jmp_off,int dst_reg,u32 code)392 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
393 struct codegen_context *ctx, int insn_idx, int jmp_off,
394 int dst_reg, u32 code)
395 {
396 off_t offset;
397 unsigned long pc;
398 struct exception_table_entry *ex, *ex_entry;
399 u32 *fixup;
400
401 /* Populate extable entries only in the last pass */
402 if (pass != 2)
403 return 0;
404
405 if (!fp->aux->extable ||
406 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
407 return -EINVAL;
408
409 /*
410 * Program is first written to image before copying to the
411 * final location (fimage). Accordingly, update in the image first.
412 * As all offsets used are relative, copying as is to the
413 * final location should be alright.
414 */
415 pc = (unsigned long)&image[insn_idx];
416 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
417
418 fixup = (void *)ex -
419 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
420 (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
421
422 fixup[0] = PPC_RAW_LI(dst_reg, 0);
423 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX)
424 fixup[0] = PPC_RAW_NOP();
425
426 if (IS_ENABLED(CONFIG_PPC32))
427 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
428
429 fixup[BPF_FIXUP_LEN - 1] =
430 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
431
432 ex_entry = &ex[ctx->exentry_idx];
433
434 offset = pc - (long)&ex_entry->insn;
435 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
436 return -ERANGE;
437 ex_entry->insn = offset;
438
439 offset = (long)fixup - (long)&ex_entry->fixup;
440 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
441 return -ERANGE;
442 ex_entry->fixup = offset;
443
444 ctx->exentry_idx++;
445 return 0;
446 }
447
bpf_arch_text_copy(void * dst,void * src,size_t len)448 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
449 {
450 int err;
451
452 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
453 return ERR_PTR(-EINVAL);
454
455 mutex_lock(&text_mutex);
456 err = patch_instructions(dst, src, len, false);
457 mutex_unlock(&text_mutex);
458
459 return err ? ERR_PTR(err) : dst;
460 }
461
bpf_arch_text_invalidate(void * dst,size_t len)462 int bpf_arch_text_invalidate(void *dst, size_t len)
463 {
464 u32 insn = BREAKPOINT_INSTRUCTION;
465 int ret;
466
467 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
468 return -EINVAL;
469
470 mutex_lock(&text_mutex);
471 ret = patch_instructions(dst, &insn, len, true);
472 mutex_unlock(&text_mutex);
473
474 return ret;
475 }
476
bpf_jit_free(struct bpf_prog * fp)477 void bpf_jit_free(struct bpf_prog *fp)
478 {
479 if (fp->jited) {
480 struct powerpc_jit_data *jit_data = fp->aux->jit_data;
481 struct bpf_binary_header *hdr;
482 void __percpu *priv_stack_ptr;
483 int priv_stack_alloc_size;
484
485 /*
486 * If we fail the final pass of JIT (from jit_subprogs),
487 * the program may not be finalized yet. Call finalize here
488 * before freeing it.
489 */
490 if (jit_data) {
491 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
492 kvfree(jit_data->addrs);
493 kfree(jit_data);
494 }
495 hdr = bpf_jit_binary_pack_hdr(fp);
496 bpf_jit_binary_pack_free(hdr, NULL);
497 priv_stack_ptr = fp->aux->priv_stack_ptr;
498 if (priv_stack_ptr) {
499 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
500 2 * PRIV_STACK_GUARD_SZ;
501 priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp);
502 free_percpu(priv_stack_ptr);
503 }
504 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
505 }
506
507 bpf_prog_unlock_free(fp);
508 }
509
bpf_jit_supports_exceptions(void)510 bool bpf_jit_supports_exceptions(void)
511 {
512 return IS_ENABLED(CONFIG_PPC64);
513 }
514
bpf_jit_supports_subprog_tailcalls(void)515 bool bpf_jit_supports_subprog_tailcalls(void)
516 {
517 return IS_ENABLED(CONFIG_PPC64);
518 }
519
bpf_jit_supports_timed_may_goto(void)520 bool bpf_jit_supports_timed_may_goto(void)
521 {
522 return IS_ENABLED(CONFIG_PPC64);
523 }
524
bpf_jit_supports_kfunc_call(void)525 bool bpf_jit_supports_kfunc_call(void)
526 {
527 return IS_ENABLED(CONFIG_PPC64);
528 }
529
bpf_jit_supports_private_stack(void)530 bool bpf_jit_supports_private_stack(void)
531 {
532 return IS_ENABLED(CONFIG_PPC64);
533 }
534
bpf_jit_supports_fsession(void)535 bool bpf_jit_supports_fsession(void)
536 {
537 /*
538 * TODO: Remove after validating support
539 * for fsession and trampoline on ppc32.
540 */
541 if (IS_ENABLED(CONFIG_PPC32))
542 return -EOPNOTSUPP;
543 return true;
544 }
545
bpf_jit_supports_arena(void)546 bool bpf_jit_supports_arena(void)
547 {
548 return IS_ENABLED(CONFIG_PPC64);
549 }
550
bpf_jit_supports_far_kfunc_call(void)551 bool bpf_jit_supports_far_kfunc_call(void)
552 {
553 return IS_ENABLED(CONFIG_PPC64);
554 }
555
bpf_jit_supports_insn(struct bpf_insn * insn,bool in_arena)556 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
557 {
558 if (!in_arena)
559 return true;
560 switch (insn->code) {
561 case BPF_STX | BPF_ATOMIC | BPF_H:
562 case BPF_STX | BPF_ATOMIC | BPF_B:
563 case BPF_STX | BPF_ATOMIC | BPF_W:
564 case BPF_STX | BPF_ATOMIC | BPF_DW:
565 if (bpf_atomic_is_load_store(insn))
566 return false;
567 return IS_ENABLED(CONFIG_PPC64);
568 }
569 return true;
570 }
571
bpf_jit_supports_percpu_insn(void)572 bool bpf_jit_supports_percpu_insn(void)
573 {
574 return IS_ENABLED(CONFIG_PPC64);
575 }
576
bpf_jit_inlines_helper_call(s32 imm)577 bool bpf_jit_inlines_helper_call(s32 imm)
578 {
579 switch (imm) {
580 case BPF_FUNC_get_smp_processor_id:
581 case BPF_FUNC_get_current_task:
582 case BPF_FUNC_get_current_task_btf:
583 return true;
584 default:
585 return false;
586 }
587 }
588
arch_alloc_bpf_trampoline(unsigned int size)589 void *arch_alloc_bpf_trampoline(unsigned int size)
590 {
591 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false);
592 }
593
arch_free_bpf_trampoline(void * image,unsigned int size)594 void arch_free_bpf_trampoline(void *image, unsigned int size)
595 {
596 bpf_prog_pack_free(image, size);
597 }
598
arch_protect_bpf_trampoline(void * image,unsigned int size)599 int arch_protect_bpf_trampoline(void *image, unsigned int size)
600 {
601 return 0;
602 }
603
invoke_bpf_prog(u32 * image,u32 * ro_image,struct codegen_context * ctx,struct bpf_tramp_node * n,int regs_off,int retval_off,int run_ctx_off,bool save_ret)604 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,
605 struct bpf_tramp_node *n, int regs_off, int retval_off,
606 int run_ctx_off, bool save_ret)
607 {
608 struct bpf_prog *p = n->link->prog;
609 ppc_inst_t branch_insn;
610 u32 jmp_idx;
611 int ret = 0;
612
613 /* Save cookie */
614 if (IS_ENABLED(CONFIG_PPC64)) {
615 PPC_LI64(_R3, n->cookie);
616 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx,
617 bpf_cookie)));
618 } else {
619 PPC_LI32(_R3, n->cookie >> 32);
620 PPC_LI32(_R4, n->cookie);
621 EMIT(PPC_RAW_STW(_R3, _R1,
622 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie)));
623 EMIT(PPC_RAW_STW(_R4, _R1,
624 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4));
625 }
626
627 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */
628 PPC_LI_ADDR(_R3, p);
629 EMIT(PPC_RAW_MR(_R25, _R3));
630 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off));
631 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
632 (unsigned long)bpf_trampoline_enter(p));
633 if (ret)
634 return ret;
635
636 /* Remember prog start time returned by __bpf_prog_enter */
637 EMIT(PPC_RAW_MR(_R26, _R3));
638
639 /*
640 * if (__bpf_prog_enter(p) == 0)
641 * goto skip_exec_of_prog;
642 *
643 * Emit a nop to be later patched with conditional branch, once offset is known
644 */
645 EMIT(PPC_RAW_CMPLI(_R3, 0));
646 jmp_idx = ctx->idx;
647 EMIT(PPC_RAW_NOP());
648
649 /* p->bpf_func(ctx) */
650 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off));
651 if (!p->jited)
652 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi);
653 /* Account for max possible instructions during dummy pass for size calculation */
654 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx],
655 (unsigned long)p->bpf_func,
656 BRANCH_SET_LINK)) {
657 image[ctx->idx] = ppc_inst_val(branch_insn);
658 ctx->idx++;
659 } else {
660 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func)));
661 EMIT(PPC_RAW_MTCTR(_R12));
662 EMIT(PPC_RAW_BCTRL());
663 }
664
665 if (save_ret)
666 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
667
668 /* Fix up branch */
669 if (image) {
670 if (create_cond_branch(&branch_insn, &image[jmp_idx],
671 (unsigned long)&image[ctx->idx], COND_EQ << 16))
672 return -EINVAL;
673 image[jmp_idx] = ppc_inst_val(branch_insn);
674 }
675
676 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */
677 EMIT(PPC_RAW_MR(_R3, _R25));
678 EMIT(PPC_RAW_MR(_R4, _R26));
679 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off));
680 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
681 (unsigned long)bpf_trampoline_exit(p));
682
683 return ret;
684 }
685
invoke_bpf_mod_ret(u32 * image,u32 * ro_image,struct codegen_context * ctx,struct bpf_tramp_nodes * tn,int regs_off,int retval_off,int run_ctx_off,u32 * branches)686 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,
687 struct bpf_tramp_nodes *tn, int regs_off, int retval_off,
688 int run_ctx_off, u32 *branches)
689 {
690 int i;
691
692 /*
693 * The first fmod_ret program will receive a garbage return value.
694 * Set this to 0 to avoid confusing the program.
695 */
696 EMIT(PPC_RAW_LI(_R3, 0));
697 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
698 for (i = 0; i < tn->nr_nodes; i++) {
699 if (invoke_bpf_prog(image, ro_image, ctx, tn->nodes[i], regs_off, retval_off,
700 run_ctx_off, true))
701 return -EINVAL;
702
703 /*
704 * mod_ret prog stored return value after prog ctx. Emit:
705 * if (*(u64 *)(ret_val) != 0)
706 * goto do_fexit;
707 */
708 EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
709 EMIT(PPC_RAW_CMPLI(_R3, 0));
710
711 /*
712 * Save the location of the branch and generate a nop, which is
713 * replaced with a conditional jump once do_fexit (i.e. the
714 * start of the fexit invocation) is finalized.
715 */
716 branches[i] = ctx->idx;
717 EMIT(PPC_RAW_NOP());
718 }
719
720 return 0;
721 }
722
723 /*
724 * Refer __arch_prepare_bpf_trampoline() for stack component details.
725 *
726 * The tailcall count/reference is present in caller's stack frame. The
727 * tail_call_info is saved at the same offset on the trampoline frame
728 * for the traced function (BPF subprog/callee) to fetch it.
729 */
bpf_trampoline_setup_tail_call_info(u32 * image,struct codegen_context * ctx,int bpf_frame_size,int r4_off)730 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx,
731 int bpf_frame_size, int r4_off)
732 {
733 if (IS_ENABLED(CONFIG_PPC64)) {
734 EMIT(PPC_RAW_LD(_R4, _R1, bpf_frame_size));
735 /* Refer to trampoline's Generated stack layout */
736 EMIT(PPC_RAW_LD(_R3, _R4, -BPF_PPC_TAILCALL));
737
738 /*
739 * Setting the tail_call_info in trampoline's frame
740 * depending on if previous frame had value or reference.
741 */
742 EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT));
743 PPC_BCC_CONST_SHORT(COND_GT, 8);
744 EMIT(PPC_RAW_ADDI(_R3, _R4, -BPF_PPC_TAILCALL));
745
746 /*
747 * Trampoline's tail_call_info is at the same offset, as that of
748 * any bpf program, with reference to previous frame. Update the
749 * address of main's tail_call_info in trampoline frame.
750 */
751 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size - BPF_PPC_TAILCALL));
752 } else {
753 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
754 EMIT(PPC_RAW_LL(_R4, _R1, r4_off));
755 }
756 }
757
bpf_trampoline_restore_tail_call_cnt(u32 * image,struct codegen_context * ctx,int bpf_frame_size,int r4_off)758 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx,
759 int bpf_frame_size, int r4_off)
760 {
761 if (IS_ENABLED(CONFIG_PPC32)) {
762 /*
763 * Restore tailcall for 32-bit powerpc
764 * See bpf_jit_stack_offsetof() and BPF_PPC_TC
765 */
766 EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
767 }
768 }
769
bpf_trampoline_save_args(u32 * image,struct codegen_context * ctx,int bpf_frame_size,int nr_regs,int regs_off)770 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx,
771 int bpf_frame_size, int nr_regs, int regs_off)
772 {
773 int param_save_area_offset;
774
775 param_save_area_offset = bpf_frame_size;
776 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
777
778 for (int i = 0; i < nr_regs; i++) {
779 if (i < 8) {
780 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL));
781 } else {
782 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
783 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL));
784 }
785 }
786 }
787
788 /* Used when restoring just the register parameters when returning back */
bpf_trampoline_restore_args_regs(u32 * image,struct codegen_context * ctx,int nr_regs,int regs_off)789 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx,
790 int nr_regs, int regs_off)
791 {
792 for (int i = 0; i < nr_regs && i < 8; i++)
793 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL));
794 }
795
796 /* Used when we call into the traced function. Replicate parameter save area */
bpf_trampoline_restore_args_stack(u32 * image,struct codegen_context * ctx,int bpf_frame_size,int nr_regs,int regs_off)797 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx,
798 int bpf_frame_size, int nr_regs, int regs_off)
799 {
800 int param_save_area_offset;
801
802 param_save_area_offset = bpf_frame_size;
803 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
804
805 for (int i = 8; i < nr_regs; i++) {
806 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
807 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL));
808 }
809 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
810 }
811
__arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * rw_image,void * rw_image_end,void * ro_image,const struct btf_func_model * m,u32 flags,struct bpf_tramp_nodes * tnodes,void * func_addr)812 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image,
813 void *rw_image_end, void *ro_image,
814 const struct btf_func_model *m, u32 flags,
815 struct bpf_tramp_nodes *tnodes,
816 void *func_addr)
817 {
818 int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off;
819 int nvr_off, alt_lr_off, r4_off = 0;
820 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN];
821 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY];
822 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT];
823 int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0;
824 struct codegen_context codegen_ctx, *ctx;
825 int cookie_off, cookie_cnt, cookie_ctx_off;
826 int fsession_cnt = bpf_fsession_cnt(tnodes);
827 u64 func_meta;
828 u32 *image = (u32 *)rw_image;
829 ppc_inst_t branch_insn;
830 u32 *branches = NULL;
831 bool save_ret;
832
833 if (IS_ENABLED(CONFIG_PPC32))
834 return -EOPNOTSUPP;
835
836 nr_regs = m->nr_args;
837 /* Extra registers for struct arguments */
838 for (i = 0; i < m->nr_args; i++)
839 if (m->arg_size[i] > SZL)
840 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1;
841
842 if (nr_regs > MAX_BPF_FUNC_ARGS)
843 return -EOPNOTSUPP;
844
845 ctx = &codegen_ctx;
846 memset(ctx, 0, sizeof(*ctx));
847
848 /*
849 * Generated stack layout:
850 *
851 * func prev back chain [ back chain ]
852 * [ tail_call_info ] optional - 64-bit powerpc
853 * [ padding ] align stack frame
854 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc
855 * alt_lr_off [ real lr (ool stub)] optional - actual lr
856 * retaddr_off [ return address ]
857 * [ r26 ]
858 * nvr_off [ r25 ] nvr save area
859 * retval_off [ return value ]
860 * [ reg argN ]
861 * [ ... ]
862 * regs_off [ reg_arg1 ] prog_ctx
863 * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1]
864 * ip_off [ traced function ] ((u64 *)prog_ctx)[-2]
865 * [ stack cookieN ]
866 * [ ... ]
867 * cookie_off [ stack cookie1 ]
868 * run_ctx_off [ bpf_tramp_run_ctx ]
869 * [ reg argN ]
870 * [ ... ]
871 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI
872 * [ TOC save (64-bit) ] --
873 * [ LR save (64-bit) ] | header
874 * [ LR save (32-bit) ] |
875 * bpf trampoline frame [ back chain 2 ] --
876 *
877 */
878
879 /* Minimum stack frame header */
880 bpf_frame_size = STACK_FRAME_MIN_SIZE;
881
882 /*
883 * Room for parameter save area.
884 *
885 * As per the ABI, this is required if we call into the traced
886 * function (BPF_TRAMP_F_CALL_ORIG):
887 * - if the function takes more than 8 arguments for the rest to spill onto the stack
888 * - or, if the function has variadic arguments
889 * - or, if this functions's prototype was not available to the caller
890 *
891 * Reserve space for at least 8 registers for now. This can be optimized later.
892 */
893 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL;
894
895 /* Room for struct bpf_tramp_run_ctx */
896 run_ctx_off = bpf_frame_size;
897 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL);
898
899 /* room for session cookies */
900 cookie_off = bpf_frame_size;
901 cookie_cnt = bpf_fsession_cookie_cnt(tnodes);
902 bpf_frame_size += cookie_cnt * 8;
903
904 /* Room for IP address argument */
905 ip_off = bpf_frame_size;
906 if (flags & BPF_TRAMP_F_IP_ARG)
907 bpf_frame_size += SZL;
908
909 /* Room for function metadata, arg regs count */
910 func_meta_off = bpf_frame_size;
911 bpf_frame_size += SZL;
912
913 /* Room for arg regs */
914 regs_off = bpf_frame_size;
915 bpf_frame_size += nr_regs * SZL;
916
917 /* Room for return value of func_addr or fentry prog */
918 retval_off = bpf_frame_size;
919 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
920 if (save_ret)
921 bpf_frame_size += SZL;
922
923 /* Room for nvr save area */
924 nvr_off = bpf_frame_size;
925 bpf_frame_size += 2 * SZL;
926
927 /* Save area for return address */
928 retaddr_off = bpf_frame_size;
929 bpf_frame_size += SZL;
930
931 /* Optional save area for actual LR in case of ool ftrace */
932 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
933 alt_lr_off = bpf_frame_size;
934 bpf_frame_size += SZL;
935 }
936
937 if (IS_ENABLED(CONFIG_PPC32)) {
938 if (nr_regs < 2) {
939 r4_off = bpf_frame_size;
940 bpf_frame_size += SZL;
941 } else {
942 r4_off = regs_off + SZL;
943 }
944 }
945
946 /*
947 * Save tailcall count pointer at the same offset on the
948 * stack where subprogs expect it
949 */
950 if ((flags & BPF_TRAMP_F_CALL_ORIG) &&
951 (flags & BPF_TRAMP_F_TAIL_CALL_CTX))
952 bpf_frame_size += BPF_PPC_TAILCALL;
953
954 /* Padding to align stack frame, if any */
955 bpf_frame_size = round_up(bpf_frame_size, SZL * 2);
956
957 /* Store original return value */
958 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF));
959
960 /* Create our stack frame */
961 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size));
962
963 /* 64-bit: Save TOC and load kernel TOC */
964 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
965 EMIT(PPC_RAW_STD(_R2, _R1, 24));
966 PPC64_LOAD_PACA();
967 }
968
969 /* 32-bit: save tail call count in r4 */
970 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2)
971 EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
972
973 bpf_trampoline_save_args(image, ctx, bpf_frame_size, nr_regs, regs_off);
974
975 /* Save our LR/return address */
976 EMIT(PPC_RAW_MFLR(_R3));
977 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
978 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off));
979 else
980 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off));
981
982 /*
983 * Derive IP address of the traced function.
984 * In case of CONFIG_PPC_FTRACE_OUT_OF_LINE or BPF program, LR points to the instruction
985 * after the 'bl' instruction in the OOL stub. Refer to ftrace_init_ool_stub() and
986 * bpf_arch_text_poke() for OOL stub of kernel functions and bpf programs respectively.
987 * Relevant stub sequence:
988 *
989 * bl <tramp>
990 * LR (R3) => mtlr r0
991 * b <func_addr+4>
992 *
993 * Recover kernel function/bpf program address from the unconditional
994 * branch instruction at the end of OOL stub.
995 */
996 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) {
997 EMIT(PPC_RAW_LWZ(_R4, _R3, 4));
998 EMIT(PPC_RAW_SLWI(_R4, _R4, 6));
999 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6));
1000 EMIT(PPC_RAW_ADD(_R3, _R3, _R4));
1001 }
1002
1003 if (flags & BPF_TRAMP_F_IP_ARG)
1004 EMIT(PPC_RAW_STL(_R3, _R1, ip_off));
1005
1006 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
1007 /* Fake our LR for BPF_TRAMP_F_CALL_ORIG case */
1008 EMIT(PPC_RAW_ADDI(_R3, _R3, 4));
1009 EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off));
1010 }
1011
1012 /* Save function arg regs count -- see bpf_get_func_arg_cnt() */
1013 func_meta = nr_regs;
1014 store_func_meta(image, ctx, func_meta, func_meta_off);
1015
1016 /* Save nv regs */
1017 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off));
1018 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL));
1019
1020 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1021 PPC_LI_ADDR(_R3, (unsigned long)im);
1022 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
1023 (unsigned long)__bpf_tramp_enter);
1024 if (ret)
1025 return ret;
1026 }
1027
1028 if (fsession_cnt) {
1029 /*
1030 * Clear all the session cookies' values
1031 * Clear the return value to make sure fentry always get 0
1032 */
1033 prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off);
1034 }
1035
1036 cookie_ctx_off = (regs_off - cookie_off) / 8;
1037
1038 for (i = 0; i < fentry->nr_nodes; i++) {
1039 if (bpf_prog_calls_session_cookie(fentry->nodes[i])) {
1040 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
1041
1042 store_func_meta(image, ctx, meta, func_meta_off);
1043 cookie_ctx_off--;
1044 }
1045
1046 if (invoke_bpf_prog(image, ro_image, ctx, fentry->nodes[i], regs_off, retval_off,
1047 run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
1048 return -EINVAL;
1049 }
1050
1051 if (fmod_ret->nr_nodes) {
1052 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32), GFP_KERNEL);
1053 if (!branches)
1054 return -ENOMEM;
1055
1056 if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off,
1057 run_ctx_off, branches)) {
1058 ret = -EINVAL;
1059 goto cleanup;
1060 }
1061 }
1062
1063 /* Call the traced function */
1064 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1065 /*
1066 * retaddr on trampoline stack points to the correct point in the original function
1067 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction
1068 * sequence
1069 */
1070 EMIT(PPC_RAW_LL(_R3, _R1, retaddr_off));
1071 EMIT(PPC_RAW_MTCTR(_R3));
1072
1073 /* Replicate tail_call_cnt before calling the original BPF prog */
1074 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1075 bpf_trampoline_setup_tail_call_info(image, ctx, bpf_frame_size, r4_off);
1076
1077 /* Restore args */
1078 bpf_trampoline_restore_args_stack(image, ctx, bpf_frame_size, nr_regs, regs_off);
1079
1080 /* Restore TOC for 64-bit */
1081 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
1082 EMIT(PPC_RAW_LD(_R2, _R1, 24));
1083 EMIT(PPC_RAW_BCTRL());
1084 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
1085 PPC64_LOAD_PACA();
1086
1087 /* Store return value for bpf prog to access */
1088 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
1089
1090 /* Restore updated tail_call_cnt */
1091 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1092 bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off);
1093
1094 /* Reserve space to patch branch instruction to skip fexit progs */
1095 if (ro_image) /* image is NULL for dummy pass */
1096 im->ip_after_call = &((u32 *)ro_image)[ctx->idx];
1097 EMIT(PPC_RAW_NOP());
1098 }
1099
1100 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */
1101 for (i = 0; i < fmod_ret->nr_nodes && image; i++) {
1102 if (create_cond_branch(&branch_insn, &image[branches[i]],
1103 (unsigned long)&image[ctx->idx], COND_NE << 16)) {
1104 ret = -EINVAL;
1105 goto cleanup;
1106 }
1107
1108 image[branches[i]] = ppc_inst_val(branch_insn);
1109 }
1110
1111 /* set the "is_return" flag for fsession */
1112 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
1113 if (fsession_cnt)
1114 store_func_meta(image, ctx, func_meta, func_meta_off);
1115
1116 cookie_ctx_off = (regs_off - cookie_off) / 8;
1117
1118 for (i = 0; i < fexit->nr_nodes; i++) {
1119 if (bpf_prog_calls_session_cookie(fexit->nodes[i])) {
1120 u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
1121
1122 store_func_meta(image, ctx, meta, func_meta_off);
1123 cookie_ctx_off--;
1124 }
1125
1126 if (invoke_bpf_prog(image, ro_image, ctx, fexit->nodes[i], regs_off, retval_off,
1127 run_ctx_off, false)) {
1128 ret = -EINVAL;
1129 goto cleanup;
1130 }
1131 }
1132
1133 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1134 if (ro_image) /* image is NULL for dummy pass */
1135 im->ip_epilogue = &((u32 *)ro_image)[ctx->idx];
1136 PPC_LI_ADDR(_R3, im);
1137 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
1138 (unsigned long)__bpf_tramp_exit);
1139 if (ret)
1140 goto cleanup;
1141 }
1142
1143 if (flags & BPF_TRAMP_F_RESTORE_REGS)
1144 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
1145
1146 /* Restore return value of func_addr or fentry prog */
1147 if (save_ret)
1148 EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
1149
1150 /* Restore nv regs */
1151 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL));
1152 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off));
1153
1154 /* Epilogue */
1155 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
1156 EMIT(PPC_RAW_LD(_R2, _R1, 24));
1157 if (flags & BPF_TRAMP_F_SKIP_FRAME) {
1158 /* Skip the traced function and return to parent */
1159 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
1160 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1161 EMIT(PPC_RAW_MTLR(_R0));
1162 EMIT(PPC_RAW_BLR());
1163 } else {
1164 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
1165 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off));
1166 EMIT(PPC_RAW_MTLR(_R0));
1167 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
1168 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1169 EMIT(PPC_RAW_BLR());
1170 } else {
1171 EMIT(PPC_RAW_LL(_R0, _R1, retaddr_off));
1172 EMIT(PPC_RAW_MTCTR(_R0));
1173 EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_frame_size));
1174 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1175 EMIT(PPC_RAW_MTLR(_R0));
1176 EMIT(PPC_RAW_BCTR());
1177 }
1178 }
1179
1180 /* Make sure the trampoline generation logic doesn't overflow */
1181 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) {
1182 ret = -EFAULT;
1183 goto cleanup;
1184 }
1185 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4;
1186
1187 cleanup:
1188 kfree(branches);
1189 return ret;
1190 }
1191
arch_bpf_trampoline_size(const struct btf_func_model * m,u32 flags,struct bpf_tramp_nodes * tnodes,void * func_addr)1192 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1193 struct bpf_tramp_nodes *tnodes, void *func_addr)
1194 {
1195 struct bpf_tramp_image im;
1196 int ret;
1197
1198 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr);
1199 return ret;
1200 }
1201
arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * image,void * image_end,const struct btf_func_model * m,u32 flags,struct bpf_tramp_nodes * tnodes,void * func_addr)1202 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1203 const struct btf_func_model *m, u32 flags,
1204 struct bpf_tramp_nodes *tnodes,
1205 void *func_addr)
1206 {
1207 u32 size = image_end - image;
1208 void *rw_image, *tmp;
1209 int ret;
1210
1211 /*
1212 * rw_image doesn't need to be in module memory range, so we can
1213 * use kvmalloc.
1214 */
1215 rw_image = kvmalloc(size, GFP_KERNEL);
1216 if (!rw_image)
1217 return -ENOMEM;
1218
1219 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,
1220 flags, tnodes, func_addr);
1221 if (ret < 0)
1222 goto out;
1223
1224 if (bpf_jit_enable > 1)
1225 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image);
1226
1227 tmp = bpf_arch_text_copy(image, rw_image, size);
1228 if (IS_ERR(tmp))
1229 ret = PTR_ERR(tmp);
1230
1231 out:
1232 kvfree(rw_image);
1233 return ret;
1234 }
1235
bpf_modify_inst(void * ip,ppc_inst_t old_inst,ppc_inst_t new_inst)1236 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst)
1237 {
1238 ppc_inst_t org_inst;
1239
1240 if (copy_inst_from_kernel_nofault(&org_inst, ip)) {
1241 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip);
1242 return -EFAULT;
1243 }
1244
1245 if (!ppc_inst_equal(org_inst, old_inst)) {
1246 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
1247 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst));
1248 return -EINVAL;
1249 }
1250
1251 if (ppc_inst_equal(old_inst, new_inst))
1252 return 0;
1253
1254 return patch_instruction(ip, new_inst);
1255 }
1256
do_isync(void * info __maybe_unused)1257 static void do_isync(void *info __maybe_unused)
1258 {
1259 isync();
1260 }
1261
1262 /*
1263 * A 3-step process for bpf prog entry:
1264 * 1. At bpf prog entry, a single nop/b:
1265 * bpf_func:
1266 * [nop|b] ool_stub
1267 * 2. Out-of-line stub:
1268 * ool_stub:
1269 * mflr r0
1270 * [b|bl] <bpf_prog>/<long_branch_stub>
1271 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only
1272 * b bpf_func + 4
1273 * 3. Long branch stub:
1274 * long_branch_stub:
1275 * .long <branch_addr>/<dummy_tramp>
1276 * mflr r11
1277 * bcl 20,31,$+4
1278 * mflr r12
1279 * ld r12, -16(r12)
1280 * mtctr r12
1281 * mtlr r11 // needed to retain ftrace ABI
1282 * bctr
1283 *
1284 * dummy_tramp is used to reduce synchronization requirements.
1285 *
1286 * When attaching a bpf trampoline to a bpf prog, we do not need any
1287 * synchronization here since we always have a valid branch target regardless
1288 * of the order in which the above stores are seen. dummy_tramp ensures that
1289 * the long_branch stub goes to a valid destination on other cpus, even when
1290 * the branch to the long_branch stub is seen before the updated trampoline
1291 * address.
1292 *
1293 * However, when detaching a bpf trampoline from a bpf prog, or if changing
1294 * the bpf trampoline address, we need synchronization to ensure that other
1295 * cpus can no longer branch into the older trampoline so that it can be
1296 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus
1297 * make forward progress, but we still need to ensure that other cpus
1298 * execute isync (or some CSI) so that they don't go back into the
1299 * trampoline again.
1300 */
bpf_arch_text_poke(void * ip,enum bpf_text_poke_type old_t,enum bpf_text_poke_type new_t,void * old_addr,void * new_addr)1301 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1302 enum bpf_text_poke_type new_t, void *old_addr,
1303 void *new_addr)
1304 {
1305 unsigned long bpf_func, bpf_func_end, size, offset;
1306 ppc_inst_t old_inst, new_inst;
1307 int ret = 0, branch_flags;
1308 char name[KSYM_NAME_LEN];
1309
1310 if (IS_ENABLED(CONFIG_PPC32))
1311 return -EOPNOTSUPP;
1312
1313 bpf_func = (unsigned long)ip;
1314
1315 /* We currently only support poking bpf programs */
1316 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) {
1317 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func);
1318 return -EOPNOTSUPP;
1319 }
1320
1321 /*
1322 * If we are not poking at bpf prog entry, then we are simply patching in/out
1323 * an unconditional branch instruction at im->ip_after_call
1324 */
1325 if (offset) {
1326 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) {
1327 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__,
1328 bpf_func);
1329 return -EOPNOTSUPP;
1330 }
1331 old_inst = ppc_inst(PPC_RAW_NOP());
1332 if (old_addr)
1333 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0))
1334 return -ERANGE;
1335 new_inst = ppc_inst(PPC_RAW_NOP());
1336 if (new_addr)
1337 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0))
1338 return -ERANGE;
1339 mutex_lock(&text_mutex);
1340 ret = bpf_modify_inst(ip, old_inst, new_inst);
1341 mutex_unlock(&text_mutex);
1342
1343 /* Make sure all cpus see the new instruction */
1344 smp_call_function(do_isync, NULL, 1);
1345 return ret;
1346 }
1347
1348 bpf_func_end = bpf_func + size;
1349
1350 /* Address of the jmp/call instruction in the out-of-line stub */
1351 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4);
1352
1353 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) {
1354 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__,
1355 bpf_func);
1356 return -ERANGE;
1357 }
1358
1359 old_inst = ppc_inst(PPC_RAW_NOP());
1360 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1361 if (old_addr) {
1362 if (is_offset_in_branch_range(ip - old_addr))
1363 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags);
1364 else
1365 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1366 branch_flags);
1367 }
1368 new_inst = ppc_inst(PPC_RAW_NOP());
1369 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1370 if (new_addr) {
1371 if (is_offset_in_branch_range(ip - new_addr))
1372 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags);
1373 else
1374 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1375 branch_flags);
1376 }
1377
1378 mutex_lock(&text_mutex);
1379
1380 /*
1381 * 1. Update the address in the long branch stub:
1382 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr
1383 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here.
1384 */
1385 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) ||
1386 (old_addr && !is_offset_in_branch_range(old_addr - ip)))
1387 ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL),
1388 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ?
1389 (unsigned long)new_addr : (unsigned long)dummy_tramp);
1390 if (ret)
1391 goto out;
1392
1393 /* 2. Update the branch/call in the out-of-line stub */
1394 ret = bpf_modify_inst(ip, old_inst, new_inst);
1395 if (ret)
1396 goto out;
1397
1398 /* 3. Update instruction at bpf prog entry */
1399 ip = (void *)bpf_func;
1400 if (!old_addr || !new_addr) {
1401 if (!old_addr) {
1402 old_inst = ppc_inst(PPC_RAW_NOP());
1403 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1404 } else {
1405 new_inst = ppc_inst(PPC_RAW_NOP());
1406 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1407 }
1408 ret = bpf_modify_inst(ip, old_inst, new_inst);
1409 }
1410
1411 out:
1412 mutex_unlock(&text_mutex);
1413
1414 /*
1415 * Sync only if we are not attaching a trampoline to a bpf prog so the older
1416 * trampoline can be freed safely.
1417 */
1418 if (old_addr)
1419 smp_call_function(do_isync, NULL, 1);
1420
1421 return ret;
1422 }
1423