1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler
4 *
5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6 * IBM Corporation
7 *
8 * Based on the powerpc classic BPF JIT compiler by Matt Evans
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <linux/kernel.h>
17 #include <linux/memory.h>
18 #include <linux/bpf.h>
19
20 #include <asm/kprobes.h>
21 #include <asm/text-patching.h>
22
23 #include "bpf_jit.h"
24
25 /* These offsets are from bpf prog end and stay the same across progs */
26 static int bpf_jit_ool_stub, bpf_jit_long_branch_stub;
27
bpf_jit_fill_ill_insns(void * area,unsigned int size)28 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
29 {
30 memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
31 }
32
33 void dummy_tramp(void);
34
35 asm (
36 " .pushsection .text, \"ax\", @progbits ;"
37 " .global dummy_tramp ;"
38 " .type dummy_tramp, @function ;"
39 "dummy_tramp: ;"
40 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
41 " blr ;"
42 #else
43 /* LR is always in r11, so we don't need a 'mflr r11' here */
44 " mtctr 11 ;"
45 " mtlr 0 ;"
46 " bctr ;"
47 #endif
48 " .size dummy_tramp, .-dummy_tramp ;"
49 " .popsection ;"
50 );
51
bpf_jit_build_fentry_stubs(u32 * image,struct codegen_context * ctx)52 void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
53 {
54 int ool_stub_idx, long_branch_stub_idx;
55
56 /*
57 * Out-of-line stub:
58 * mflr r0
59 * [b|bl] tramp
60 * mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE
61 * b bpf_func + 4
62 */
63 ool_stub_idx = ctx->idx;
64 EMIT(PPC_RAW_MFLR(_R0));
65 EMIT(PPC_RAW_NOP());
66 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
67 EMIT(PPC_RAW_MTLR(_R0));
68 WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4));
69 EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4));
70
71 /*
72 * Long branch stub:
73 * .long <dummy_tramp_addr>
74 * mflr r11
75 * bcl 20,31,$+4
76 * mflr r12
77 * ld r12, -8-SZL(r12)
78 * mtctr r12
79 * mtlr r11 // needed to retain ftrace ABI
80 * bctr
81 */
82 if (image)
83 *((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;
84 ctx->idx += SZL / 4;
85 long_branch_stub_idx = ctx->idx;
86 EMIT(PPC_RAW_MFLR(_R11));
87 EMIT(PPC_RAW_BCL4());
88 EMIT(PPC_RAW_MFLR(_R12));
89 EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL));
90 EMIT(PPC_RAW_MTCTR(_R12));
91 EMIT(PPC_RAW_MTLR(_R11));
92 EMIT(PPC_RAW_BCTR());
93
94 if (!bpf_jit_ool_stub) {
95 bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4;
96 bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4;
97 }
98 }
99
bpf_jit_emit_exit_insn(u32 * image,struct codegen_context * ctx,int tmp_reg,long exit_addr)100 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
101 {
102 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
103 PPC_JMP(exit_addr);
104 } else if (ctx->alt_exit_addr) {
105 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
106 return -1;
107 PPC_JMP(ctx->alt_exit_addr);
108 } else {
109 ctx->alt_exit_addr = ctx->idx * 4;
110 bpf_jit_build_epilogue(image, ctx);
111 }
112
113 return 0;
114 }
115
116 struct powerpc_jit_data {
117 /* address of rw header */
118 struct bpf_binary_header *hdr;
119 /* address of ro final header */
120 struct bpf_binary_header *fhdr;
121 u32 *addrs;
122 u8 *fimage;
123 u32 proglen;
124 struct codegen_context ctx;
125 };
126
bpf_jit_needs_zext(void)127 bool bpf_jit_needs_zext(void)
128 {
129 return true;
130 }
131
bpf_int_jit_compile(struct bpf_prog * fp)132 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
133 {
134 u32 proglen;
135 u32 alloclen;
136 u8 *image = NULL;
137 u32 *code_base;
138 u32 *addrs;
139 struct powerpc_jit_data *jit_data;
140 struct codegen_context cgctx;
141 int pass;
142 int flen;
143 struct bpf_binary_header *fhdr = NULL;
144 struct bpf_binary_header *hdr = NULL;
145 struct bpf_prog *org_fp = fp;
146 struct bpf_prog *tmp_fp;
147 bool bpf_blinded = false;
148 bool extra_pass = false;
149 u8 *fimage = NULL;
150 u32 *fcode_base;
151 u32 extable_len;
152 u32 fixup_len;
153
154 if (!fp->jit_requested)
155 return org_fp;
156
157 tmp_fp = bpf_jit_blind_constants(org_fp);
158 if (IS_ERR(tmp_fp))
159 return org_fp;
160
161 if (tmp_fp != org_fp) {
162 bpf_blinded = true;
163 fp = tmp_fp;
164 }
165
166 jit_data = fp->aux->jit_data;
167 if (!jit_data) {
168 jit_data = kzalloc_obj(*jit_data);
169 if (!jit_data) {
170 fp = org_fp;
171 goto out;
172 }
173 fp->aux->jit_data = jit_data;
174 }
175
176 flen = fp->len;
177 addrs = jit_data->addrs;
178 if (addrs) {
179 cgctx = jit_data->ctx;
180 /*
181 * JIT compiled to a writable location (image/code_base) first.
182 * It is then moved to the readonly final location (fimage/fcode_base)
183 * using instruction patching.
184 */
185 fimage = jit_data->fimage;
186 fhdr = jit_data->fhdr;
187 proglen = jit_data->proglen;
188 hdr = jit_data->hdr;
189 image = (void *)hdr + ((void *)fimage - (void *)fhdr);
190 extra_pass = true;
191 /* During extra pass, ensure index is reset before repopulating extable entries */
192 cgctx.exentry_idx = 0;
193 goto skip_init_ctx;
194 }
195
196 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
197 if (addrs == NULL) {
198 fp = org_fp;
199 goto out_addrs;
200 }
201
202 memset(&cgctx, 0, sizeof(struct codegen_context));
203 bpf_jit_init_reg_mapping(&cgctx);
204
205 /* Make sure that the stack is quadword aligned. */
206 cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
207 cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
208 cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
209 cgctx.is_subprog = bpf_is_subprog(fp);
210 cgctx.exception_boundary = fp->aux->exception_boundary;
211 cgctx.exception_cb = fp->aux->exception_cb;
212
213 /* Scouting faux-generate pass 0 */
214 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
215 /* We hit something illegal or unsupported. */
216 fp = org_fp;
217 goto out_addrs;
218 }
219
220 /*
221 * If we have seen a tail call, we need a second pass.
222 * This is because bpf_jit_emit_common_epilogue() is called
223 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
224 * We also need a second pass if we ended up with too large
225 * a program so as to ensure BPF_EXIT branches are in range.
226 */
227 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
228 cgctx.idx = 0;
229 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
230 fp = org_fp;
231 goto out_addrs;
232 }
233 }
234
235 bpf_jit_realloc_regs(&cgctx);
236 /*
237 * Pretend to build prologue, given the features we've seen. This will
238 * update ctgtx.idx as it pretends to output instructions, then we can
239 * calculate total size from idx.
240 */
241 bpf_jit_build_prologue(NULL, &cgctx);
242 addrs[fp->len] = cgctx.idx * 4;
243 bpf_jit_build_epilogue(NULL, &cgctx);
244
245 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
246 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
247
248 proglen = cgctx.idx * 4;
249 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
250
251 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
252 bpf_jit_fill_ill_insns);
253 if (!fhdr) {
254 fp = org_fp;
255 goto out_addrs;
256 }
257
258 if (extable_len)
259 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
260
261 skip_init_ctx:
262 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
263 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
264
265 /* Code generation passes 1-2 */
266 for (pass = 1; pass < 3; pass++) {
267 /* Now build the prologue, body code & epilogue for real. */
268 cgctx.idx = 0;
269 cgctx.alt_exit_addr = 0;
270 bpf_jit_build_prologue(code_base, &cgctx);
271 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
272 extra_pass)) {
273 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
274 bpf_jit_binary_pack_free(fhdr, hdr);
275 fp = org_fp;
276 goto out_addrs;
277 }
278 bpf_jit_build_epilogue(code_base, &cgctx);
279
280 if (bpf_jit_enable > 1)
281 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
282 proglen - (cgctx.idx * 4), cgctx.seen);
283 }
284
285 if (bpf_jit_enable > 1)
286 /*
287 * Note that we output the base address of the code_base
288 * rather than image, since opcodes are in code_base.
289 */
290 bpf_jit_dump(flen, proglen, pass, code_base);
291
292 #ifdef CONFIG_PPC64_ELF_ABI_V1
293 /* Function descriptor nastiness: Address + TOC */
294 ((u64 *)image)[0] = (u64)fcode_base;
295 ((u64 *)image)[1] = local_paca->kernel_toc;
296 #endif
297
298 fp->bpf_func = (void *)fimage;
299 fp->jited = 1;
300 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;
301
302 if (!fp->is_func || extra_pass) {
303 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
304 fp = org_fp;
305 goto out_addrs;
306 }
307 bpf_prog_fill_jited_linfo(fp, addrs);
308 out_addrs:
309 kfree(addrs);
310 kfree(jit_data);
311 fp->aux->jit_data = NULL;
312 } else {
313 jit_data->addrs = addrs;
314 jit_data->ctx = cgctx;
315 jit_data->proglen = proglen;
316 jit_data->fimage = fimage;
317 jit_data->fhdr = fhdr;
318 jit_data->hdr = hdr;
319 }
320
321 out:
322 if (bpf_blinded)
323 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
324
325 return fp;
326 }
327
328 /*
329 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
330 * this function, as this only applies to BPF_PROBE_MEM, for now.
331 */
bpf_add_extable_entry(struct bpf_prog * fp,u32 * image,u32 * fimage,int pass,struct codegen_context * ctx,int insn_idx,int jmp_off,int dst_reg,u32 code)332 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
333 struct codegen_context *ctx, int insn_idx, int jmp_off,
334 int dst_reg, u32 code)
335 {
336 off_t offset;
337 unsigned long pc;
338 struct exception_table_entry *ex, *ex_entry;
339 u32 *fixup;
340
341 /* Populate extable entries only in the last pass */
342 if (pass != 2)
343 return 0;
344
345 if (!fp->aux->extable ||
346 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
347 return -EINVAL;
348
349 /*
350 * Program is first written to image before copying to the
351 * final location (fimage). Accordingly, update in the image first.
352 * As all offsets used are relative, copying as is to the
353 * final location should be alright.
354 */
355 pc = (unsigned long)&image[insn_idx];
356 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
357
358 fixup = (void *)ex -
359 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
360 (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
361
362 fixup[0] = PPC_RAW_LI(dst_reg, 0);
363 if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX)
364 fixup[0] = PPC_RAW_NOP();
365
366 if (IS_ENABLED(CONFIG_PPC32))
367 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
368
369 fixup[BPF_FIXUP_LEN - 1] =
370 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
371
372 ex_entry = &ex[ctx->exentry_idx];
373
374 offset = pc - (long)&ex_entry->insn;
375 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
376 return -ERANGE;
377 ex_entry->insn = offset;
378
379 offset = (long)fixup - (long)&ex_entry->fixup;
380 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
381 return -ERANGE;
382 ex_entry->fixup = offset;
383
384 ctx->exentry_idx++;
385 return 0;
386 }
387
bpf_arch_text_copy(void * dst,void * src,size_t len)388 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
389 {
390 int err;
391
392 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
393 return ERR_PTR(-EINVAL);
394
395 mutex_lock(&text_mutex);
396 err = patch_instructions(dst, src, len, false);
397 mutex_unlock(&text_mutex);
398
399 return err ? ERR_PTR(err) : dst;
400 }
401
bpf_arch_text_invalidate(void * dst,size_t len)402 int bpf_arch_text_invalidate(void *dst, size_t len)
403 {
404 u32 insn = BREAKPOINT_INSTRUCTION;
405 int ret;
406
407 if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
408 return -EINVAL;
409
410 mutex_lock(&text_mutex);
411 ret = patch_instructions(dst, &insn, len, true);
412 mutex_unlock(&text_mutex);
413
414 return ret;
415 }
416
bpf_jit_free(struct bpf_prog * fp)417 void bpf_jit_free(struct bpf_prog *fp)
418 {
419 if (fp->jited) {
420 struct powerpc_jit_data *jit_data = fp->aux->jit_data;
421 struct bpf_binary_header *hdr;
422
423 /*
424 * If we fail the final pass of JIT (from jit_subprogs),
425 * the program may not be finalized yet. Call finalize here
426 * before freeing it.
427 */
428 if (jit_data) {
429 bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
430 kvfree(jit_data->addrs);
431 kfree(jit_data);
432 }
433 hdr = bpf_jit_binary_pack_hdr(fp);
434 bpf_jit_binary_pack_free(hdr, NULL);
435 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
436 }
437
438 bpf_prog_unlock_free(fp);
439 }
440
bpf_jit_supports_exceptions(void)441 bool bpf_jit_supports_exceptions(void)
442 {
443 return IS_ENABLED(CONFIG_PPC64);
444 }
445
bpf_jit_supports_subprog_tailcalls(void)446 bool bpf_jit_supports_subprog_tailcalls(void)
447 {
448 return IS_ENABLED(CONFIG_PPC64);
449 }
450
bpf_jit_supports_kfunc_call(void)451 bool bpf_jit_supports_kfunc_call(void)
452 {
453 return true;
454 }
455
bpf_jit_supports_arena(void)456 bool bpf_jit_supports_arena(void)
457 {
458 return IS_ENABLED(CONFIG_PPC64);
459 }
460
bpf_jit_supports_far_kfunc_call(void)461 bool bpf_jit_supports_far_kfunc_call(void)
462 {
463 return IS_ENABLED(CONFIG_PPC64);
464 }
465
bpf_jit_supports_insn(struct bpf_insn * insn,bool in_arena)466 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
467 {
468 if (!in_arena)
469 return true;
470 switch (insn->code) {
471 case BPF_STX | BPF_ATOMIC | BPF_H:
472 case BPF_STX | BPF_ATOMIC | BPF_B:
473 case BPF_STX | BPF_ATOMIC | BPF_W:
474 case BPF_STX | BPF_ATOMIC | BPF_DW:
475 if (bpf_atomic_is_load_store(insn))
476 return false;
477 return IS_ENABLED(CONFIG_PPC64);
478 }
479 return true;
480 }
481
bpf_jit_supports_percpu_insn(void)482 bool bpf_jit_supports_percpu_insn(void)
483 {
484 return IS_ENABLED(CONFIG_PPC64);
485 }
486
bpf_jit_inlines_helper_call(s32 imm)487 bool bpf_jit_inlines_helper_call(s32 imm)
488 {
489 switch (imm) {
490 case BPF_FUNC_get_smp_processor_id:
491 case BPF_FUNC_get_current_task:
492 case BPF_FUNC_get_current_task_btf:
493 return true;
494 default:
495 return false;
496 }
497 }
498
arch_alloc_bpf_trampoline(unsigned int size)499 void *arch_alloc_bpf_trampoline(unsigned int size)
500 {
501 return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns);
502 }
503
arch_free_bpf_trampoline(void * image,unsigned int size)504 void arch_free_bpf_trampoline(void *image, unsigned int size)
505 {
506 bpf_prog_pack_free(image, size);
507 }
508
arch_protect_bpf_trampoline(void * image,unsigned int size)509 int arch_protect_bpf_trampoline(void *image, unsigned int size)
510 {
511 return 0;
512 }
513
invoke_bpf_prog(u32 * image,u32 * ro_image,struct codegen_context * ctx,struct bpf_tramp_link * l,int regs_off,int retval_off,int run_ctx_off,bool save_ret)514 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,
515 struct bpf_tramp_link *l, int regs_off, int retval_off,
516 int run_ctx_off, bool save_ret)
517 {
518 struct bpf_prog *p = l->link.prog;
519 ppc_inst_t branch_insn;
520 u32 jmp_idx;
521 int ret = 0;
522
523 /* Save cookie */
524 if (IS_ENABLED(CONFIG_PPC64)) {
525 PPC_LI64(_R3, l->cookie);
526 EMIT(PPC_RAW_STD(_R3, _R1, run_ctx_off + offsetof(struct bpf_tramp_run_ctx,
527 bpf_cookie)));
528 } else {
529 PPC_LI32(_R3, l->cookie >> 32);
530 PPC_LI32(_R4, l->cookie);
531 EMIT(PPC_RAW_STW(_R3, _R1,
532 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie)));
533 EMIT(PPC_RAW_STW(_R4, _R1,
534 run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie) + 4));
535 }
536
537 /* __bpf_prog_enter(p, &bpf_tramp_run_ctx) */
538 PPC_LI_ADDR(_R3, p);
539 EMIT(PPC_RAW_MR(_R25, _R3));
540 EMIT(PPC_RAW_ADDI(_R4, _R1, run_ctx_off));
541 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
542 (unsigned long)bpf_trampoline_enter(p));
543 if (ret)
544 return ret;
545
546 /* Remember prog start time returned by __bpf_prog_enter */
547 EMIT(PPC_RAW_MR(_R26, _R3));
548
549 /*
550 * if (__bpf_prog_enter(p) == 0)
551 * goto skip_exec_of_prog;
552 *
553 * Emit a nop to be later patched with conditional branch, once offset is known
554 */
555 EMIT(PPC_RAW_CMPLI(_R3, 0));
556 jmp_idx = ctx->idx;
557 EMIT(PPC_RAW_NOP());
558
559 /* p->bpf_func(ctx) */
560 EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off));
561 if (!p->jited)
562 PPC_LI_ADDR(_R4, (unsigned long)p->insnsi);
563 /* Account for max possible instructions during dummy pass for size calculation */
564 if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx],
565 (unsigned long)p->bpf_func,
566 BRANCH_SET_LINK)) {
567 image[ctx->idx] = ppc_inst_val(branch_insn);
568 ctx->idx++;
569 } else {
570 EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func)));
571 EMIT(PPC_RAW_MTCTR(_R12));
572 EMIT(PPC_RAW_BCTRL());
573 }
574
575 if (save_ret)
576 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
577
578 /* Fix up branch */
579 if (image) {
580 if (create_cond_branch(&branch_insn, &image[jmp_idx],
581 (unsigned long)&image[ctx->idx], COND_EQ << 16))
582 return -EINVAL;
583 image[jmp_idx] = ppc_inst_val(branch_insn);
584 }
585
586 /* __bpf_prog_exit(p, start_time, &bpf_tramp_run_ctx) */
587 EMIT(PPC_RAW_MR(_R3, _R25));
588 EMIT(PPC_RAW_MR(_R4, _R26));
589 EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off));
590 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
591 (unsigned long)bpf_trampoline_exit(p));
592
593 return ret;
594 }
595
invoke_bpf_mod_ret(u32 * image,u32 * ro_image,struct codegen_context * ctx,struct bpf_tramp_links * tl,int regs_off,int retval_off,int run_ctx_off,u32 * branches)596 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,
597 struct bpf_tramp_links *tl, int regs_off, int retval_off,
598 int run_ctx_off, u32 *branches)
599 {
600 int i;
601
602 /*
603 * The first fmod_ret program will receive a garbage return value.
604 * Set this to 0 to avoid confusing the program.
605 */
606 EMIT(PPC_RAW_LI(_R3, 0));
607 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
608 for (i = 0; i < tl->nr_links; i++) {
609 if (invoke_bpf_prog(image, ro_image, ctx, tl->links[i], regs_off, retval_off,
610 run_ctx_off, true))
611 return -EINVAL;
612
613 /*
614 * mod_ret prog stored return value after prog ctx. Emit:
615 * if (*(u64 *)(ret_val) != 0)
616 * goto do_fexit;
617 */
618 EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
619 EMIT(PPC_RAW_CMPLI(_R3, 0));
620
621 /*
622 * Save the location of the branch and generate a nop, which is
623 * replaced with a conditional jump once do_fexit (i.e. the
624 * start of the fexit invocation) is finalized.
625 */
626 branches[i] = ctx->idx;
627 EMIT(PPC_RAW_NOP());
628 }
629
630 return 0;
631 }
632
633 /*
634 * Refer __arch_prepare_bpf_trampoline() for stack component details.
635 *
636 * The tailcall count/reference is present in caller's stack frame. The
637 * tail_call_info is saved at the same offset on the trampoline frame
638 * for the traced function (BPF subprog/callee) to fetch it.
639 */
bpf_trampoline_setup_tail_call_info(u32 * image,struct codegen_context * ctx,int func_frame_offset,int bpf_dummy_frame_size,int r4_off)640 static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx,
641 int func_frame_offset,
642 int bpf_dummy_frame_size, int r4_off)
643 {
644 if (IS_ENABLED(CONFIG_PPC64)) {
645 /* See Generated stack layout */
646 int tailcallinfo_offset = BPF_PPC_TAILCALL;
647
648 /*
649 * func_frame_offset = ...(1)
650 * bpf_dummy_frame_size + trampoline_frame_size
651 */
652 EMIT(PPC_RAW_LD(_R4, _R1, func_frame_offset));
653 EMIT(PPC_RAW_LD(_R3, _R4, -tailcallinfo_offset));
654
655 /*
656 * Setting the tail_call_info in trampoline's frame
657 * depending on if previous frame had value or reference.
658 */
659 EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT));
660 PPC_BCC_CONST_SHORT(COND_GT, 8);
661 EMIT(PPC_RAW_ADDI(_R3, _R4, bpf_jit_stack_tailcallinfo_offset(ctx)));
662 /*
663 * From ...(1) above:
664 * trampoline_frame_bottom = ...(2)
665 * func_frame_offset - bpf_dummy_frame_size
666 *
667 * Using ...(2) derived above:
668 * trampoline_tail_call_info_offset = ...(3)
669 * trampoline_frame_bottom - tailcallinfo_offset
670 *
671 * From ...(3):
672 * Use trampoline_tail_call_info_offset to write reference of main's
673 * tail_call_info in trampoline frame.
674 */
675 EMIT(PPC_RAW_STL(_R3, _R1, (func_frame_offset - bpf_dummy_frame_size)
676 - tailcallinfo_offset));
677 } else {
678 /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */
679 EMIT(PPC_RAW_LL(_R4, _R1, r4_off));
680 }
681 }
682
bpf_trampoline_restore_tail_call_cnt(u32 * image,struct codegen_context * ctx,int func_frame_offset,int r4_off)683 static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx,
684 int func_frame_offset, int r4_off)
685 {
686 if (IS_ENABLED(CONFIG_PPC32)) {
687 /*
688 * Restore tailcall for 32-bit powerpc
689 * See bpf_jit_stack_offsetof() and BPF_PPC_TC
690 */
691 EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
692 }
693 }
694
bpf_trampoline_save_args(u32 * image,struct codegen_context * ctx,int func_frame_offset,int nr_regs,int regs_off)695 static void bpf_trampoline_save_args(u32 *image, struct codegen_context *ctx, int func_frame_offset,
696 int nr_regs, int regs_off)
697 {
698 int param_save_area_offset;
699
700 param_save_area_offset = func_frame_offset; /* the two frames we alloted */
701 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
702
703 for (int i = 0; i < nr_regs; i++) {
704 if (i < 8) {
705 EMIT(PPC_RAW_STL(_R3 + i, _R1, regs_off + i * SZL));
706 } else {
707 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
708 EMIT(PPC_RAW_STL(_R3, _R1, regs_off + i * SZL));
709 }
710 }
711 }
712
713 /* Used when restoring just the register parameters when returning back */
bpf_trampoline_restore_args_regs(u32 * image,struct codegen_context * ctx,int nr_regs,int regs_off)714 static void bpf_trampoline_restore_args_regs(u32 *image, struct codegen_context *ctx,
715 int nr_regs, int regs_off)
716 {
717 for (int i = 0; i < nr_regs && i < 8; i++)
718 EMIT(PPC_RAW_LL(_R3 + i, _R1, regs_off + i * SZL));
719 }
720
721 /* Used when we call into the traced function. Replicate parameter save area */
bpf_trampoline_restore_args_stack(u32 * image,struct codegen_context * ctx,int func_frame_offset,int nr_regs,int regs_off)722 static void bpf_trampoline_restore_args_stack(u32 *image, struct codegen_context *ctx,
723 int func_frame_offset, int nr_regs, int regs_off)
724 {
725 int param_save_area_offset;
726
727 param_save_area_offset = func_frame_offset; /* the two frames we alloted */
728 param_save_area_offset += STACK_FRAME_MIN_SIZE; /* param save area is past frame header */
729
730 for (int i = 8; i < nr_regs; i++) {
731 EMIT(PPC_RAW_LL(_R3, _R1, param_save_area_offset + i * SZL));
732 EMIT(PPC_RAW_STL(_R3, _R1, STACK_FRAME_MIN_SIZE + i * SZL));
733 }
734 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
735 }
736
__arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * rw_image,void * rw_image_end,void * ro_image,const struct btf_func_model * m,u32 flags,struct bpf_tramp_links * tlinks,void * func_addr)737 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image,
738 void *rw_image_end, void *ro_image,
739 const struct btf_func_model *m, u32 flags,
740 struct bpf_tramp_links *tlinks,
741 void *func_addr)
742 {
743 int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0;
744 int i, ret, nr_regs, bpf_frame_size = 0, bpf_dummy_frame_size = 0, func_frame_offset;
745 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
746 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
747 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
748 struct codegen_context codegen_ctx, *ctx;
749 u32 *image = (u32 *)rw_image;
750 ppc_inst_t branch_insn;
751 u32 *branches = NULL;
752 bool save_ret;
753
754 if (IS_ENABLED(CONFIG_PPC32))
755 return -EOPNOTSUPP;
756
757 nr_regs = m->nr_args;
758 /* Extra registers for struct arguments */
759 for (i = 0; i < m->nr_args; i++)
760 if (m->arg_size[i] > SZL)
761 nr_regs += round_up(m->arg_size[i], SZL) / SZL - 1;
762
763 if (nr_regs > MAX_BPF_FUNC_ARGS)
764 return -EOPNOTSUPP;
765
766 ctx = &codegen_ctx;
767 memset(ctx, 0, sizeof(*ctx));
768
769 /*
770 * Generated stack layout:
771 *
772 * func prev back chain [ back chain ]
773 * [ ]
774 * bpf prog redzone/tailcallcnt [ ... ] 64 bytes (64-bit powerpc)
775 * [ ] --
776 * LR save area [ r0 save (64-bit) ] | header
777 * [ r0 save (32-bit) ] |
778 * dummy frame for unwind [ back chain 1 ] --
779 * [ tail_call_info ] optional - 64-bit powerpc
780 * [ padding ] align stack frame
781 * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc
782 * alt_lr_off [ real lr (ool stub)] optional - actual lr
783 * [ r26 ]
784 * nvr_off [ r25 ] nvr save area
785 * retval_off [ return value ]
786 * [ reg argN ]
787 * [ ... ]
788 * regs_off [ reg_arg1 ] prog ctx context
789 * nregs_off [ args count ]
790 * ip_off [ traced function ]
791 * [ ... ]
792 * run_ctx_off [ bpf_tramp_run_ctx ]
793 * [ reg argN ]
794 * [ ... ]
795 * param_save_area [ reg_arg1 ] min 8 doublewords, per ABI
796 * [ TOC save (64-bit) ] --
797 * [ LR save (64-bit) ] | header
798 * [ LR save (32-bit) ] |
799 * bpf trampoline frame [ back chain 2 ] --
800 *
801 */
802
803 /* Minimum stack frame header */
804 bpf_frame_size = STACK_FRAME_MIN_SIZE;
805
806 /*
807 * Room for parameter save area.
808 *
809 * As per the ABI, this is required if we call into the traced
810 * function (BPF_TRAMP_F_CALL_ORIG):
811 * - if the function takes more than 8 arguments for the rest to spill onto the stack
812 * - or, if the function has variadic arguments
813 * - or, if this functions's prototype was not available to the caller
814 *
815 * Reserve space for at least 8 registers for now. This can be optimized later.
816 */
817 bpf_frame_size += (nr_regs > 8 ? nr_regs : 8) * SZL;
818
819 /* Room for struct bpf_tramp_run_ctx */
820 run_ctx_off = bpf_frame_size;
821 bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL);
822
823 /* Room for IP address argument */
824 ip_off = bpf_frame_size;
825 if (flags & BPF_TRAMP_F_IP_ARG)
826 bpf_frame_size += SZL;
827
828 /* Room for args count */
829 nregs_off = bpf_frame_size;
830 bpf_frame_size += SZL;
831
832 /* Room for args */
833 regs_off = bpf_frame_size;
834 bpf_frame_size += nr_regs * SZL;
835
836 /* Room for return value of func_addr or fentry prog */
837 retval_off = bpf_frame_size;
838 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
839 if (save_ret)
840 bpf_frame_size += SZL;
841
842 /* Room for nvr save area */
843 nvr_off = bpf_frame_size;
844 bpf_frame_size += 2 * SZL;
845
846 /* Optional save area for actual LR in case of ool ftrace */
847 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
848 alt_lr_off = bpf_frame_size;
849 bpf_frame_size += SZL;
850 }
851
852 if (IS_ENABLED(CONFIG_PPC32)) {
853 if (nr_regs < 2) {
854 r4_off = bpf_frame_size;
855 bpf_frame_size += SZL;
856 } else {
857 r4_off = regs_off + SZL;
858 }
859 }
860
861 /*
862 * Save tailcall count pointer at the same offset on the
863 * stack where subprogs expect it
864 */
865 if ((flags & BPF_TRAMP_F_CALL_ORIG) &&
866 (flags & BPF_TRAMP_F_TAIL_CALL_CTX))
867 bpf_frame_size += BPF_PPC_TAILCALL;
868
869 /* Padding to align stack frame, if any */
870 bpf_frame_size = round_up(bpf_frame_size, SZL * 2);
871
872 /* Dummy frame size for proper unwind - includes 64-bytes red zone for 64-bit powerpc */
873 bpf_dummy_frame_size = STACK_FRAME_MIN_SIZE + 64;
874
875 /* Offset to the traced function's stack frame */
876 func_frame_offset = bpf_dummy_frame_size + bpf_frame_size;
877
878 /* Create dummy frame for unwind, store original return value */
879 EMIT(PPC_RAW_STL(_R0, _R1, PPC_LR_STKOFF));
880 /* Protect red zone where tail call count goes */
881 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_dummy_frame_size));
882
883 /* Create our stack frame */
884 EMIT(PPC_RAW_STLU(_R1, _R1, -bpf_frame_size));
885
886 /* 64-bit: Save TOC and load kernel TOC */
887 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
888 EMIT(PPC_RAW_STD(_R2, _R1, 24));
889 PPC64_LOAD_PACA();
890 }
891
892 /* 32-bit: save tail call count in r4 */
893 if (IS_ENABLED(CONFIG_PPC32) && nr_regs < 2)
894 EMIT(PPC_RAW_STL(_R4, _R1, r4_off));
895
896 bpf_trampoline_save_args(image, ctx, func_frame_offset, nr_regs, regs_off);
897
898 /* Save our return address */
899 EMIT(PPC_RAW_MFLR(_R3));
900 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
901 EMIT(PPC_RAW_STL(_R3, _R1, alt_lr_off));
902 else
903 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
904
905 /*
906 * Save ip address of the traced function.
907 * We could recover this from LR, but we will need to address for OOL trampoline,
908 * and optional GEP area.
909 */
910 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE) || flags & BPF_TRAMP_F_IP_ARG) {
911 EMIT(PPC_RAW_LWZ(_R4, _R3, 4));
912 EMIT(PPC_RAW_SLWI(_R4, _R4, 6));
913 EMIT(PPC_RAW_SRAWI(_R4, _R4, 6));
914 EMIT(PPC_RAW_ADD(_R3, _R3, _R4));
915 EMIT(PPC_RAW_ADDI(_R3, _R3, 4));
916 }
917
918 if (flags & BPF_TRAMP_F_IP_ARG)
919 EMIT(PPC_RAW_STL(_R3, _R1, ip_off));
920
921 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
922 /* Fake our LR for unwind */
923 EMIT(PPC_RAW_STL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
924
925 /* Save function arg count -- see bpf_get_func_arg_cnt() */
926 EMIT(PPC_RAW_LI(_R3, nr_regs));
927 EMIT(PPC_RAW_STL(_R3, _R1, nregs_off));
928
929 /* Save nv regs */
930 EMIT(PPC_RAW_STL(_R25, _R1, nvr_off));
931 EMIT(PPC_RAW_STL(_R26, _R1, nvr_off + SZL));
932
933 if (flags & BPF_TRAMP_F_CALL_ORIG) {
934 PPC_LI_ADDR(_R3, (unsigned long)im);
935 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
936 (unsigned long)__bpf_tramp_enter);
937 if (ret)
938 return ret;
939 }
940
941 for (i = 0; i < fentry->nr_links; i++)
942 if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off,
943 run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
944 return -EINVAL;
945
946 if (fmod_ret->nr_links) {
947 branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL);
948 if (!branches)
949 return -ENOMEM;
950
951 if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off,
952 run_ctx_off, branches)) {
953 ret = -EINVAL;
954 goto cleanup;
955 }
956 }
957
958 /* Call the traced function */
959 if (flags & BPF_TRAMP_F_CALL_ORIG) {
960 /*
961 * The address in LR save area points to the correct point in the original function
962 * with both PPC_FTRACE_OUT_OF_LINE as well as with traditional ftrace instruction
963 * sequence
964 */
965 EMIT(PPC_RAW_LL(_R3, _R1, bpf_frame_size + PPC_LR_STKOFF));
966 EMIT(PPC_RAW_MTCTR(_R3));
967
968 /* Replicate tail_call_cnt before calling the original BPF prog */
969 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
970 bpf_trampoline_setup_tail_call_info(image, ctx, func_frame_offset,
971 bpf_dummy_frame_size, r4_off);
972
973 /* Restore args */
974 bpf_trampoline_restore_args_stack(image, ctx, func_frame_offset, nr_regs, regs_off);
975
976 /* Restore TOC for 64-bit */
977 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
978 EMIT(PPC_RAW_LD(_R2, _R1, 24));
979 EMIT(PPC_RAW_BCTRL());
980 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
981 PPC64_LOAD_PACA();
982
983 /* Store return value for bpf prog to access */
984 EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
985
986 /* Restore updated tail_call_cnt */
987 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
988 bpf_trampoline_restore_tail_call_cnt(image, ctx, func_frame_offset, r4_off);
989
990 /* Reserve space to patch branch instruction to skip fexit progs */
991 if (ro_image) /* image is NULL for dummy pass */
992 im->ip_after_call = &((u32 *)ro_image)[ctx->idx];
993 EMIT(PPC_RAW_NOP());
994 }
995
996 /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */
997 for (i = 0; i < fmod_ret->nr_links && image; i++) {
998 if (create_cond_branch(&branch_insn, &image[branches[i]],
999 (unsigned long)&image[ctx->idx], COND_NE << 16)) {
1000 ret = -EINVAL;
1001 goto cleanup;
1002 }
1003
1004 image[branches[i]] = ppc_inst_val(branch_insn);
1005 }
1006
1007 for (i = 0; i < fexit->nr_links; i++)
1008 if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off,
1009 run_ctx_off, false)) {
1010 ret = -EINVAL;
1011 goto cleanup;
1012 }
1013
1014 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1015 if (ro_image) /* image is NULL for dummy pass */
1016 im->ip_epilogue = &((u32 *)ro_image)[ctx->idx];
1017 PPC_LI_ADDR(_R3, im);
1018 ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
1019 (unsigned long)__bpf_tramp_exit);
1020 if (ret)
1021 goto cleanup;
1022 }
1023
1024 if (flags & BPF_TRAMP_F_RESTORE_REGS)
1025 bpf_trampoline_restore_args_regs(image, ctx, nr_regs, regs_off);
1026
1027 /* Restore return value of func_addr or fentry prog */
1028 if (save_ret)
1029 EMIT(PPC_RAW_LL(_R3, _R1, retval_off));
1030
1031 /* Restore nv regs */
1032 EMIT(PPC_RAW_LL(_R26, _R1, nvr_off + SZL));
1033 EMIT(PPC_RAW_LL(_R25, _R1, nvr_off));
1034
1035 /* Epilogue */
1036 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
1037 EMIT(PPC_RAW_LD(_R2, _R1, 24));
1038 if (flags & BPF_TRAMP_F_SKIP_FRAME) {
1039 /* Skip the traced function and return to parent */
1040 EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
1041 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1042 EMIT(PPC_RAW_MTLR(_R0));
1043 EMIT(PPC_RAW_BLR());
1044 } else {
1045 if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE)) {
1046 EMIT(PPC_RAW_LL(_R0, _R1, alt_lr_off));
1047 EMIT(PPC_RAW_MTLR(_R0));
1048 EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
1049 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1050 EMIT(PPC_RAW_BLR());
1051 } else {
1052 EMIT(PPC_RAW_LL(_R0, _R1, bpf_frame_size + PPC_LR_STKOFF));
1053 EMIT(PPC_RAW_MTCTR(_R0));
1054 EMIT(PPC_RAW_ADDI(_R1, _R1, func_frame_offset));
1055 EMIT(PPC_RAW_LL(_R0, _R1, PPC_LR_STKOFF));
1056 EMIT(PPC_RAW_MTLR(_R0));
1057 EMIT(PPC_RAW_BCTR());
1058 }
1059 }
1060
1061 /* Make sure the trampoline generation logic doesn't overflow */
1062 if (image && WARN_ON_ONCE(&image[ctx->idx] > (u32 *)rw_image_end - BPF_INSN_SAFETY)) {
1063 ret = -EFAULT;
1064 goto cleanup;
1065 }
1066 ret = ctx->idx * 4 + BPF_INSN_SAFETY * 4;
1067
1068 cleanup:
1069 kfree(branches);
1070 return ret;
1071 }
1072
arch_bpf_trampoline_size(const struct btf_func_model * m,u32 flags,struct bpf_tramp_links * tlinks,void * func_addr)1073 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1074 struct bpf_tramp_links *tlinks, void *func_addr)
1075 {
1076 struct bpf_tramp_image im;
1077 int ret;
1078
1079 ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tlinks, func_addr);
1080 return ret;
1081 }
1082
arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * image,void * image_end,const struct btf_func_model * m,u32 flags,struct bpf_tramp_links * tlinks,void * func_addr)1083 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1084 const struct btf_func_model *m, u32 flags,
1085 struct bpf_tramp_links *tlinks,
1086 void *func_addr)
1087 {
1088 u32 size = image_end - image;
1089 void *rw_image, *tmp;
1090 int ret;
1091
1092 /*
1093 * rw_image doesn't need to be in module memory range, so we can
1094 * use kvmalloc.
1095 */
1096 rw_image = kvmalloc(size, GFP_KERNEL);
1097 if (!rw_image)
1098 return -ENOMEM;
1099
1100 ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,
1101 flags, tlinks, func_addr);
1102 if (ret < 0)
1103 goto out;
1104
1105 if (bpf_jit_enable > 1)
1106 bpf_jit_dump(1, ret - BPF_INSN_SAFETY * 4, 1, rw_image);
1107
1108 tmp = bpf_arch_text_copy(image, rw_image, size);
1109 if (IS_ERR(tmp))
1110 ret = PTR_ERR(tmp);
1111
1112 out:
1113 kvfree(rw_image);
1114 return ret;
1115 }
1116
bpf_modify_inst(void * ip,ppc_inst_t old_inst,ppc_inst_t new_inst)1117 static int bpf_modify_inst(void *ip, ppc_inst_t old_inst, ppc_inst_t new_inst)
1118 {
1119 ppc_inst_t org_inst;
1120
1121 if (copy_inst_from_kernel_nofault(&org_inst, ip)) {
1122 pr_err("0x%lx: fetching instruction failed\n", (unsigned long)ip);
1123 return -EFAULT;
1124 }
1125
1126 if (!ppc_inst_equal(org_inst, old_inst)) {
1127 pr_err("0x%lx: expected (%08lx) != found (%08lx)\n",
1128 (unsigned long)ip, ppc_inst_as_ulong(old_inst), ppc_inst_as_ulong(org_inst));
1129 return -EINVAL;
1130 }
1131
1132 if (ppc_inst_equal(old_inst, new_inst))
1133 return 0;
1134
1135 return patch_instruction(ip, new_inst);
1136 }
1137
do_isync(void * info __maybe_unused)1138 static void do_isync(void *info __maybe_unused)
1139 {
1140 isync();
1141 }
1142
1143 /*
1144 * A 3-step process for bpf prog entry:
1145 * 1. At bpf prog entry, a single nop/b:
1146 * bpf_func:
1147 * [nop|b] ool_stub
1148 * 2. Out-of-line stub:
1149 * ool_stub:
1150 * mflr r0
1151 * [b|bl] <bpf_prog>/<long_branch_stub>
1152 * mtlr r0 // CONFIG_PPC_FTRACE_OUT_OF_LINE only
1153 * b bpf_func + 4
1154 * 3. Long branch stub:
1155 * long_branch_stub:
1156 * .long <branch_addr>/<dummy_tramp>
1157 * mflr r11
1158 * bcl 20,31,$+4
1159 * mflr r12
1160 * ld r12, -16(r12)
1161 * mtctr r12
1162 * mtlr r11 // needed to retain ftrace ABI
1163 * bctr
1164 *
1165 * dummy_tramp is used to reduce synchronization requirements.
1166 *
1167 * When attaching a bpf trampoline to a bpf prog, we do not need any
1168 * synchronization here since we always have a valid branch target regardless
1169 * of the order in which the above stores are seen. dummy_tramp ensures that
1170 * the long_branch stub goes to a valid destination on other cpus, even when
1171 * the branch to the long_branch stub is seen before the updated trampoline
1172 * address.
1173 *
1174 * However, when detaching a bpf trampoline from a bpf prog, or if changing
1175 * the bpf trampoline address, we need synchronization to ensure that other
1176 * cpus can no longer branch into the older trampoline so that it can be
1177 * safely freed. bpf_tramp_image_put() uses rcu_tasks to ensure all cpus
1178 * make forward progress, but we still need to ensure that other cpus
1179 * execute isync (or some CSI) so that they don't go back into the
1180 * trampoline again.
1181 */
bpf_arch_text_poke(void * ip,enum bpf_text_poke_type old_t,enum bpf_text_poke_type new_t,void * old_addr,void * new_addr)1182 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1183 enum bpf_text_poke_type new_t, void *old_addr,
1184 void *new_addr)
1185 {
1186 unsigned long bpf_func, bpf_func_end, size, offset;
1187 ppc_inst_t old_inst, new_inst;
1188 int ret = 0, branch_flags;
1189 char name[KSYM_NAME_LEN];
1190
1191 if (IS_ENABLED(CONFIG_PPC32))
1192 return -EOPNOTSUPP;
1193
1194 bpf_func = (unsigned long)ip;
1195
1196 /* We currently only support poking bpf programs */
1197 if (!bpf_address_lookup(bpf_func, &size, &offset, name)) {
1198 pr_err("%s (0x%lx): kernel/modules are not supported\n", __func__, bpf_func);
1199 return -EOPNOTSUPP;
1200 }
1201
1202 /*
1203 * If we are not poking at bpf prog entry, then we are simply patching in/out
1204 * an unconditional branch instruction at im->ip_after_call
1205 */
1206 if (offset) {
1207 if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) {
1208 pr_err("%s (0x%lx): calls are not supported in bpf prog body\n", __func__,
1209 bpf_func);
1210 return -EOPNOTSUPP;
1211 }
1212 old_inst = ppc_inst(PPC_RAW_NOP());
1213 if (old_addr)
1214 if (create_branch(&old_inst, ip, (unsigned long)old_addr, 0))
1215 return -ERANGE;
1216 new_inst = ppc_inst(PPC_RAW_NOP());
1217 if (new_addr)
1218 if (create_branch(&new_inst, ip, (unsigned long)new_addr, 0))
1219 return -ERANGE;
1220 mutex_lock(&text_mutex);
1221 ret = bpf_modify_inst(ip, old_inst, new_inst);
1222 mutex_unlock(&text_mutex);
1223
1224 /* Make sure all cpus see the new instruction */
1225 smp_call_function(do_isync, NULL, 1);
1226 return ret;
1227 }
1228
1229 bpf_func_end = bpf_func + size;
1230
1231 /* Address of the jmp/call instruction in the out-of-line stub */
1232 ip = (void *)(bpf_func_end - bpf_jit_ool_stub + 4);
1233
1234 if (!is_offset_in_branch_range((long)ip - 4 - bpf_func)) {
1235 pr_err("%s (0x%lx): bpf prog too large, ool stub out of branch range\n", __func__,
1236 bpf_func);
1237 return -ERANGE;
1238 }
1239
1240 old_inst = ppc_inst(PPC_RAW_NOP());
1241 branch_flags = old_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1242 if (old_addr) {
1243 if (is_offset_in_branch_range(ip - old_addr))
1244 create_branch(&old_inst, ip, (unsigned long)old_addr, branch_flags);
1245 else
1246 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1247 branch_flags);
1248 }
1249 new_inst = ppc_inst(PPC_RAW_NOP());
1250 branch_flags = new_t == BPF_MOD_CALL ? BRANCH_SET_LINK : 0;
1251 if (new_addr) {
1252 if (is_offset_in_branch_range(ip - new_addr))
1253 create_branch(&new_inst, ip, (unsigned long)new_addr, branch_flags);
1254 else
1255 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_long_branch_stub,
1256 branch_flags);
1257 }
1258
1259 mutex_lock(&text_mutex);
1260
1261 /*
1262 * 1. Update the address in the long branch stub:
1263 * If new_addr is out of range, we will have to use the long branch stub, so patch new_addr
1264 * here. Otherwise, revert to dummy_tramp, but only if we had patched old_addr here.
1265 */
1266 if ((new_addr && !is_offset_in_branch_range(new_addr - ip)) ||
1267 (old_addr && !is_offset_in_branch_range(old_addr - ip)))
1268 ret = patch_ulong((void *)(bpf_func_end - bpf_jit_long_branch_stub - SZL),
1269 (new_addr && !is_offset_in_branch_range(new_addr - ip)) ?
1270 (unsigned long)new_addr : (unsigned long)dummy_tramp);
1271 if (ret)
1272 goto out;
1273
1274 /* 2. Update the branch/call in the out-of-line stub */
1275 ret = bpf_modify_inst(ip, old_inst, new_inst);
1276 if (ret)
1277 goto out;
1278
1279 /* 3. Update instruction at bpf prog entry */
1280 ip = (void *)bpf_func;
1281 if (!old_addr || !new_addr) {
1282 if (!old_addr) {
1283 old_inst = ppc_inst(PPC_RAW_NOP());
1284 create_branch(&new_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1285 } else {
1286 new_inst = ppc_inst(PPC_RAW_NOP());
1287 create_branch(&old_inst, ip, bpf_func_end - bpf_jit_ool_stub, 0);
1288 }
1289 ret = bpf_modify_inst(ip, old_inst, new_inst);
1290 }
1291
1292 out:
1293 mutex_unlock(&text_mutex);
1294
1295 /*
1296 * Sync only if we are not attaching a trampoline to a bpf prog so the older
1297 * trampoline can be freed safely.
1298 */
1299 if (old_addr)
1300 smp_call_function(do_isync, NULL, 1);
1301
1302 return ret;
1303 }
1304