14ea76e90SChristophe Leroy // SPDX-License-Identifier: GPL-2.0-only
24ea76e90SChristophe Leroy /*
34ea76e90SChristophe Leroy * eBPF JIT compiler
44ea76e90SChristophe Leroy *
54ea76e90SChristophe Leroy * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
64ea76e90SChristophe Leroy * IBM Corporation
74ea76e90SChristophe Leroy *
84ea76e90SChristophe Leroy * Based on the powerpc classic BPF JIT compiler by Matt Evans
94ea76e90SChristophe Leroy */
104ea76e90SChristophe Leroy #include <linux/moduleloader.h>
114ea76e90SChristophe Leroy #include <asm/cacheflush.h>
124ea76e90SChristophe Leroy #include <asm/asm-compat.h>
134ea76e90SChristophe Leroy #include <linux/netdevice.h>
144ea76e90SChristophe Leroy #include <linux/filter.h>
154ea76e90SChristophe Leroy #include <linux/if_vlan.h>
166efc1675SHari Bathini #include <linux/kernel.h>
176efc1675SHari Bathini #include <linux/memory.h>
184ea76e90SChristophe Leroy #include <linux/bpf.h>
194ea76e90SChristophe Leroy
206efc1675SHari Bathini #include <asm/kprobes.h>
216efc1675SHari Bathini #include <asm/code-patching.h>
226efc1675SHari Bathini
234ea76e90SChristophe Leroy #include "bpf_jit.h"
244ea76e90SChristophe Leroy
bpf_jit_fill_ill_insns(void * area,unsigned int size)254ea76e90SChristophe Leroy static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
264ea76e90SChristophe Leroy {
274ea76e90SChristophe Leroy memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
284ea76e90SChristophe Leroy }
294ea76e90SChristophe Leroy
bpf_jit_emit_exit_insn(u32 * image,struct codegen_context * ctx,int tmp_reg,long exit_addr)300ffdbce6SNaveen N. Rao int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
310ffdbce6SNaveen N. Rao {
320ffdbce6SNaveen N. Rao if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
330ffdbce6SNaveen N. Rao PPC_JMP(exit_addr);
340ffdbce6SNaveen N. Rao } else if (ctx->alt_exit_addr) {
350ffdbce6SNaveen N. Rao if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
360ffdbce6SNaveen N. Rao return -1;
370ffdbce6SNaveen N. Rao PPC_JMP(ctx->alt_exit_addr);
380ffdbce6SNaveen N. Rao } else {
390ffdbce6SNaveen N. Rao ctx->alt_exit_addr = ctx->idx * 4;
400ffdbce6SNaveen N. Rao bpf_jit_build_epilogue(image, ctx);
410ffdbce6SNaveen N. Rao }
420ffdbce6SNaveen N. Rao
430ffdbce6SNaveen N. Rao return 0;
440ffdbce6SNaveen N. Rao }
450ffdbce6SNaveen N. Rao
46de04e406SHari Bathini struct powerpc_jit_data {
4790d862f3SHari Bathini /* address of rw header */
4890d862f3SHari Bathini struct bpf_binary_header *hdr;
4990d862f3SHari Bathini /* address of ro final header */
5090d862f3SHari Bathini struct bpf_binary_header *fhdr;
514ea76e90SChristophe Leroy u32 *addrs;
5290d862f3SHari Bathini u8 *fimage;
534ea76e90SChristophe Leroy u32 proglen;
544ea76e90SChristophe Leroy struct codegen_context ctx;
554ea76e90SChristophe Leroy };
564ea76e90SChristophe Leroy
bpf_jit_needs_zext(void)574ea76e90SChristophe Leroy bool bpf_jit_needs_zext(void)
584ea76e90SChristophe Leroy {
594ea76e90SChristophe Leroy return true;
604ea76e90SChristophe Leroy }
614ea76e90SChristophe Leroy
bpf_int_jit_compile(struct bpf_prog * fp)624ea76e90SChristophe Leroy struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
634ea76e90SChristophe Leroy {
644ea76e90SChristophe Leroy u32 proglen;
654ea76e90SChristophe Leroy u32 alloclen;
664ea76e90SChristophe Leroy u8 *image = NULL;
674ea76e90SChristophe Leroy u32 *code_base;
684ea76e90SChristophe Leroy u32 *addrs;
69de04e406SHari Bathini struct powerpc_jit_data *jit_data;
704ea76e90SChristophe Leroy struct codegen_context cgctx;
714ea76e90SChristophe Leroy int pass;
724ea76e90SChristophe Leroy int flen;
7390d862f3SHari Bathini struct bpf_binary_header *fhdr = NULL;
7490d862f3SHari Bathini struct bpf_binary_header *hdr = NULL;
754ea76e90SChristophe Leroy struct bpf_prog *org_fp = fp;
764ea76e90SChristophe Leroy struct bpf_prog *tmp_fp;
774ea76e90SChristophe Leroy bool bpf_blinded = false;
784ea76e90SChristophe Leroy bool extra_pass = false;
7990d862f3SHari Bathini u8 *fimage = NULL;
8090d862f3SHari Bathini u32 *fcode_base;
81983bdc02SRavi Bangoria u32 extable_len;
82983bdc02SRavi Bangoria u32 fixup_len;
834ea76e90SChristophe Leroy
844ea76e90SChristophe Leroy if (!fp->jit_requested)
854ea76e90SChristophe Leroy return org_fp;
864ea76e90SChristophe Leroy
874ea76e90SChristophe Leroy tmp_fp = bpf_jit_blind_constants(org_fp);
884ea76e90SChristophe Leroy if (IS_ERR(tmp_fp))
894ea76e90SChristophe Leroy return org_fp;
904ea76e90SChristophe Leroy
914ea76e90SChristophe Leroy if (tmp_fp != org_fp) {
924ea76e90SChristophe Leroy bpf_blinded = true;
934ea76e90SChristophe Leroy fp = tmp_fp;
944ea76e90SChristophe Leroy }
954ea76e90SChristophe Leroy
964ea76e90SChristophe Leroy jit_data = fp->aux->jit_data;
974ea76e90SChristophe Leroy if (!jit_data) {
984ea76e90SChristophe Leroy jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
994ea76e90SChristophe Leroy if (!jit_data) {
1004ea76e90SChristophe Leroy fp = org_fp;
1014ea76e90SChristophe Leroy goto out;
1024ea76e90SChristophe Leroy }
1034ea76e90SChristophe Leroy fp->aux->jit_data = jit_data;
1044ea76e90SChristophe Leroy }
1054ea76e90SChristophe Leroy
1064ea76e90SChristophe Leroy flen = fp->len;
1074ea76e90SChristophe Leroy addrs = jit_data->addrs;
1084ea76e90SChristophe Leroy if (addrs) {
1094ea76e90SChristophe Leroy cgctx = jit_data->ctx;
11090d862f3SHari Bathini /*
11190d862f3SHari Bathini * JIT compiled to a writable location (image/code_base) first.
11290d862f3SHari Bathini * It is then moved to the readonly final location (fimage/fcode_base)
11390d862f3SHari Bathini * using instruction patching.
11490d862f3SHari Bathini */
11590d862f3SHari Bathini fimage = jit_data->fimage;
11690d862f3SHari Bathini fhdr = jit_data->fhdr;
1174ea76e90SChristophe Leroy proglen = jit_data->proglen;
11890d862f3SHari Bathini hdr = jit_data->hdr;
11990d862f3SHari Bathini image = (void *)hdr + ((void *)fimage - (void *)fhdr);
1204ea76e90SChristophe Leroy extra_pass = true;
12135a4b8ceSHari Bathini /* During extra pass, ensure index is reset before repopulating extable entries */
12235a4b8ceSHari Bathini cgctx.exentry_idx = 0;
1234ea76e90SChristophe Leroy goto skip_init_ctx;
1244ea76e90SChristophe Leroy }
1254ea76e90SChristophe Leroy
1264ea76e90SChristophe Leroy addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
1274ea76e90SChristophe Leroy if (addrs == NULL) {
1284ea76e90SChristophe Leroy fp = org_fp;
1294ea76e90SChristophe Leroy goto out_addrs;
1304ea76e90SChristophe Leroy }
1314ea76e90SChristophe Leroy
1324ea76e90SChristophe Leroy memset(&cgctx, 0, sizeof(struct codegen_context));
13349c3af43SNaveen N. Rao bpf_jit_init_reg_mapping(&cgctx);
1344ea76e90SChristophe Leroy
1354ea76e90SChristophe Leroy /* Make sure that the stack is quadword aligned. */
1364ea76e90SChristophe Leroy cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
1374ea76e90SChristophe Leroy
1384ea76e90SChristophe Leroy /* Scouting faux-generate pass 0 */
13990d862f3SHari Bathini if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
1404ea76e90SChristophe Leroy /* We hit something illegal or unsupported. */
1414ea76e90SChristophe Leroy fp = org_fp;
1424ea76e90SChristophe Leroy goto out_addrs;
1434ea76e90SChristophe Leroy }
1444ea76e90SChristophe Leroy
1454ea76e90SChristophe Leroy /*
1464ea76e90SChristophe Leroy * If we have seen a tail call, we need a second pass.
1474ea76e90SChristophe Leroy * This is because bpf_jit_emit_common_epilogue() is called
1484ea76e90SChristophe Leroy * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
1490ffdbce6SNaveen N. Rao * We also need a second pass if we ended up with too large
1500ffdbce6SNaveen N. Rao * a program so as to ensure BPF_EXIT branches are in range.
1514ea76e90SChristophe Leroy */
1520ffdbce6SNaveen N. Rao if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
1534ea76e90SChristophe Leroy cgctx.idx = 0;
15490d862f3SHari Bathini if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
1554ea76e90SChristophe Leroy fp = org_fp;
1564ea76e90SChristophe Leroy goto out_addrs;
1574ea76e90SChristophe Leroy }
1584ea76e90SChristophe Leroy }
1594ea76e90SChristophe Leroy
16040272035SChristophe Leroy bpf_jit_realloc_regs(&cgctx);
1614ea76e90SChristophe Leroy /*
1624ea76e90SChristophe Leroy * Pretend to build prologue, given the features we've seen. This will
1634ea76e90SChristophe Leroy * update ctgtx.idx as it pretends to output instructions, then we can
1644ea76e90SChristophe Leroy * calculate total size from idx.
1654ea76e90SChristophe Leroy */
166ddfb7d9dSBenjamin Gray bpf_jit_build_prologue(NULL, &cgctx);
1670ffdbce6SNaveen N. Rao addrs[fp->len] = cgctx.idx * 4;
168ddfb7d9dSBenjamin Gray bpf_jit_build_epilogue(NULL, &cgctx);
1694ea76e90SChristophe Leroy
170983bdc02SRavi Bangoria fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
171983bdc02SRavi Bangoria extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
172983bdc02SRavi Bangoria
1734ea76e90SChristophe Leroy proglen = cgctx.idx * 4;
174983bdc02SRavi Bangoria alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
1754ea76e90SChristophe Leroy
17690d862f3SHari Bathini fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
17790d862f3SHari Bathini bpf_jit_fill_ill_insns);
17890d862f3SHari Bathini if (!fhdr) {
1794ea76e90SChristophe Leroy fp = org_fp;
1804ea76e90SChristophe Leroy goto out_addrs;
1814ea76e90SChristophe Leroy }
1824ea76e90SChristophe Leroy
183983bdc02SRavi Bangoria if (extable_len)
18490d862f3SHari Bathini fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
185983bdc02SRavi Bangoria
1864ea76e90SChristophe Leroy skip_init_ctx:
1874ea76e90SChristophe Leroy code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
18890d862f3SHari Bathini fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
1894ea76e90SChristophe Leroy
1904ea76e90SChristophe Leroy /* Code generation passes 1-2 */
1914ea76e90SChristophe Leroy for (pass = 1; pass < 3; pass++) {
1924ea76e90SChristophe Leroy /* Now build the prologue, body code & epilogue for real. */
1934ea76e90SChristophe Leroy cgctx.idx = 0;
1940ffdbce6SNaveen N. Rao cgctx.alt_exit_addr = 0;
1954ea76e90SChristophe Leroy bpf_jit_build_prologue(code_base, &cgctx);
19690d862f3SHari Bathini if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
19790d862f3SHari Bathini extra_pass)) {
19890d862f3SHari Bathini bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
19990d862f3SHari Bathini bpf_jit_binary_pack_free(fhdr, hdr);
2003832ba4eSNaveen N. Rao fp = org_fp;
2013832ba4eSNaveen N. Rao goto out_addrs;
2023832ba4eSNaveen N. Rao }
2034ea76e90SChristophe Leroy bpf_jit_build_epilogue(code_base, &cgctx);
2044ea76e90SChristophe Leroy
2054ea76e90SChristophe Leroy if (bpf_jit_enable > 1)
2064ea76e90SChristophe Leroy pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
2074ea76e90SChristophe Leroy proglen - (cgctx.idx * 4), cgctx.seen);
2084ea76e90SChristophe Leroy }
2094ea76e90SChristophe Leroy
2104ea76e90SChristophe Leroy if (bpf_jit_enable > 1)
2114ea76e90SChristophe Leroy /*
2124ea76e90SChristophe Leroy * Note that we output the base address of the code_base
2134ea76e90SChristophe Leroy * rather than image, since opcodes are in code_base.
2144ea76e90SChristophe Leroy */
2154ea76e90SChristophe Leroy bpf_jit_dump(flen, proglen, pass, code_base);
2164ea76e90SChristophe Leroy
2177d40aff8SChristophe Leroy #ifdef CONFIG_PPC64_ELF_ABI_V1
2184ea76e90SChristophe Leroy /* Function descriptor nastiness: Address + TOC */
21990d862f3SHari Bathini ((u64 *)image)[0] = (u64)fcode_base;
2204ea76e90SChristophe Leroy ((u64 *)image)[1] = local_paca->kernel_toc;
2214ea76e90SChristophe Leroy #endif
2224ea76e90SChristophe Leroy
22390d862f3SHari Bathini fp->bpf_func = (void *)fimage;
2244ea76e90SChristophe Leroy fp->jited = 1;
225983bdc02SRavi Bangoria fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
2264ea76e90SChristophe Leroy
2274ea76e90SChristophe Leroy if (!fp->is_func || extra_pass) {
228*9919c5c9SRafael Passos if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
22990d862f3SHari Bathini fp = org_fp;
23090d862f3SHari Bathini goto out_addrs;
23190d862f3SHari Bathini }
2324ea76e90SChristophe Leroy bpf_prog_fill_jited_linfo(fp, addrs);
2334ea76e90SChristophe Leroy out_addrs:
2344ea76e90SChristophe Leroy kfree(addrs);
2354ea76e90SChristophe Leroy kfree(jit_data);
2364ea76e90SChristophe Leroy fp->aux->jit_data = NULL;
2374ea76e90SChristophe Leroy } else {
2384ea76e90SChristophe Leroy jit_data->addrs = addrs;
2394ea76e90SChristophe Leroy jit_data->ctx = cgctx;
2404ea76e90SChristophe Leroy jit_data->proglen = proglen;
24190d862f3SHari Bathini jit_data->fimage = fimage;
24290d862f3SHari Bathini jit_data->fhdr = fhdr;
24390d862f3SHari Bathini jit_data->hdr = hdr;
2444ea76e90SChristophe Leroy }
2454ea76e90SChristophe Leroy
2464ea76e90SChristophe Leroy out:
2474ea76e90SChristophe Leroy if (bpf_blinded)
2484ea76e90SChristophe Leroy bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
2494ea76e90SChristophe Leroy
2504ea76e90SChristophe Leroy return fp;
2514ea76e90SChristophe Leroy }
252983bdc02SRavi Bangoria
253983bdc02SRavi Bangoria /*
254983bdc02SRavi Bangoria * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
255983bdc02SRavi Bangoria * this function, as this only applies to BPF_PROBE_MEM, for now.
256983bdc02SRavi Bangoria */
bpf_add_extable_entry(struct bpf_prog * fp,u32 * image,u32 * fimage,int pass,struct codegen_context * ctx,int insn_idx,int jmp_off,int dst_reg)25790d862f3SHari Bathini int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
25890d862f3SHari Bathini struct codegen_context *ctx, int insn_idx, int jmp_off,
25990d862f3SHari Bathini int dst_reg)
260983bdc02SRavi Bangoria {
261983bdc02SRavi Bangoria off_t offset;
262983bdc02SRavi Bangoria unsigned long pc;
26390d862f3SHari Bathini struct exception_table_entry *ex, *ex_entry;
264983bdc02SRavi Bangoria u32 *fixup;
265983bdc02SRavi Bangoria
266983bdc02SRavi Bangoria /* Populate extable entries only in the last pass */
267983bdc02SRavi Bangoria if (pass != 2)
268983bdc02SRavi Bangoria return 0;
269983bdc02SRavi Bangoria
270983bdc02SRavi Bangoria if (!fp->aux->extable ||
271983bdc02SRavi Bangoria WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
272983bdc02SRavi Bangoria return -EINVAL;
273983bdc02SRavi Bangoria
27490d862f3SHari Bathini /*
27590d862f3SHari Bathini * Program is first written to image before copying to the
27690d862f3SHari Bathini * final location (fimage). Accordingly, update in the image first.
27790d862f3SHari Bathini * As all offsets used are relative, copying as is to the
27890d862f3SHari Bathini * final location should be alright.
27990d862f3SHari Bathini */
280983bdc02SRavi Bangoria pc = (unsigned long)&image[insn_idx];
28190d862f3SHari Bathini ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
282983bdc02SRavi Bangoria
28390d862f3SHari Bathini fixup = (void *)ex -
284983bdc02SRavi Bangoria (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
285983bdc02SRavi Bangoria (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
286983bdc02SRavi Bangoria
287983bdc02SRavi Bangoria fixup[0] = PPC_RAW_LI(dst_reg, 0);
28823b51916SHari Bathini if (IS_ENABLED(CONFIG_PPC32))
28923b51916SHari Bathini fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
290983bdc02SRavi Bangoria
291983bdc02SRavi Bangoria fixup[BPF_FIXUP_LEN - 1] =
292983bdc02SRavi Bangoria PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
293983bdc02SRavi Bangoria
29490d862f3SHari Bathini ex_entry = &ex[ctx->exentry_idx];
295983bdc02SRavi Bangoria
29690d862f3SHari Bathini offset = pc - (long)&ex_entry->insn;
297983bdc02SRavi Bangoria if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
298983bdc02SRavi Bangoria return -ERANGE;
29990d862f3SHari Bathini ex_entry->insn = offset;
300983bdc02SRavi Bangoria
30190d862f3SHari Bathini offset = (long)fixup - (long)&ex_entry->fixup;
302983bdc02SRavi Bangoria if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
303983bdc02SRavi Bangoria return -ERANGE;
30490d862f3SHari Bathini ex_entry->fixup = offset;
305983bdc02SRavi Bangoria
306983bdc02SRavi Bangoria ctx->exentry_idx++;
307983bdc02SRavi Bangoria return 0;
308983bdc02SRavi Bangoria }
3096efc1675SHari Bathini
bpf_arch_text_copy(void * dst,void * src,size_t len)3106efc1675SHari Bathini void *bpf_arch_text_copy(void *dst, void *src, size_t len)
3116efc1675SHari Bathini {
3126efc1675SHari Bathini int err;
3136efc1675SHari Bathini
3146efc1675SHari Bathini if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
3156efc1675SHari Bathini return ERR_PTR(-EINVAL);
3166efc1675SHari Bathini
3176efc1675SHari Bathini mutex_lock(&text_mutex);
3186efc1675SHari Bathini err = patch_instructions(dst, src, len, false);
3196efc1675SHari Bathini mutex_unlock(&text_mutex);
3206efc1675SHari Bathini
3216efc1675SHari Bathini return err ? ERR_PTR(err) : dst;
3226efc1675SHari Bathini }
323033ffaf0SHari Bathini
bpf_arch_text_invalidate(void * dst,size_t len)324033ffaf0SHari Bathini int bpf_arch_text_invalidate(void *dst, size_t len)
325033ffaf0SHari Bathini {
326033ffaf0SHari Bathini u32 insn = BREAKPOINT_INSTRUCTION;
327033ffaf0SHari Bathini int ret;
328033ffaf0SHari Bathini
329033ffaf0SHari Bathini if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
330033ffaf0SHari Bathini return -EINVAL;
331033ffaf0SHari Bathini
332033ffaf0SHari Bathini mutex_lock(&text_mutex);
333033ffaf0SHari Bathini ret = patch_instructions(dst, &insn, len, true);
334033ffaf0SHari Bathini mutex_unlock(&text_mutex);
335033ffaf0SHari Bathini
336033ffaf0SHari Bathini return ret;
337033ffaf0SHari Bathini }
33890d862f3SHari Bathini
bpf_jit_free(struct bpf_prog * fp)33990d862f3SHari Bathini void bpf_jit_free(struct bpf_prog *fp)
34090d862f3SHari Bathini {
34190d862f3SHari Bathini if (fp->jited) {
34290d862f3SHari Bathini struct powerpc_jit_data *jit_data = fp->aux->jit_data;
34390d862f3SHari Bathini struct bpf_binary_header *hdr;
34490d862f3SHari Bathini
34590d862f3SHari Bathini /*
34690d862f3SHari Bathini * If we fail the final pass of JIT (from jit_subprogs),
34790d862f3SHari Bathini * the program may not be finalized yet. Call finalize here
34890d862f3SHari Bathini * before freeing it.
34990d862f3SHari Bathini */
35090d862f3SHari Bathini if (jit_data) {
351*9919c5c9SRafael Passos bpf_jit_binary_pack_finalize(jit_data->fhdr, jit_data->hdr);
35290d862f3SHari Bathini kvfree(jit_data->addrs);
35390d862f3SHari Bathini kfree(jit_data);
35490d862f3SHari Bathini }
35590d862f3SHari Bathini hdr = bpf_jit_binary_pack_hdr(fp);
35690d862f3SHari Bathini bpf_jit_binary_pack_free(hdr, NULL);
35790d862f3SHari Bathini WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
35890d862f3SHari Bathini }
35990d862f3SHari Bathini
36090d862f3SHari Bathini bpf_prog_unlock_free(fp);
36190d862f3SHari Bathini }
36261688a82SHari Bathini
bpf_jit_supports_kfunc_call(void)36361688a82SHari Bathini bool bpf_jit_supports_kfunc_call(void)
36461688a82SHari Bathini {
36561688a82SHari Bathini return true;
36661688a82SHari Bathini }
36761688a82SHari Bathini
bpf_jit_supports_far_kfunc_call(void)36861688a82SHari Bathini bool bpf_jit_supports_far_kfunc_call(void)
36961688a82SHari Bathini {
37061688a82SHari Bathini return IS_ENABLED(CONFIG_PPC64);
37161688a82SHari Bathini }
372