xref: /linux/arch/powerpc/net/bpf_jit_comp.c (revision 5027ec19f1049a07df5b0a37b1f462514cf2724b)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * eBPF JIT compiler
4  *
5  * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
6  *		  IBM Corporation
7  *
8  * Based on the powerpc classic BPF JIT compiler by Matt Evans
9  */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <linux/kernel.h>
17 #include <linux/memory.h>
18 #include <linux/bpf.h>
19 
20 #include <asm/kprobes.h>
21 #include <asm/code-patching.h>
22 
23 #include "bpf_jit.h"
24 
25 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
26 {
27 	memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
28 }
29 
30 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
31 {
32 	if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
33 		PPC_JMP(exit_addr);
34 	} else if (ctx->alt_exit_addr) {
35 		if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
36 			return -1;
37 		PPC_JMP(ctx->alt_exit_addr);
38 	} else {
39 		ctx->alt_exit_addr = ctx->idx * 4;
40 		bpf_jit_build_epilogue(image, ctx);
41 	}
42 
43 	return 0;
44 }
45 
46 struct powerpc_jit_data {
47 	/* address of rw header */
48 	struct bpf_binary_header *hdr;
49 	/* address of ro final header */
50 	struct bpf_binary_header *fhdr;
51 	u32 *addrs;
52 	u8 *fimage;
53 	u32 proglen;
54 	struct codegen_context ctx;
55 };
56 
57 bool bpf_jit_needs_zext(void)
58 {
59 	return true;
60 }
61 
62 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
63 {
64 	u32 proglen;
65 	u32 alloclen;
66 	u8 *image = NULL;
67 	u32 *code_base;
68 	u32 *addrs;
69 	struct powerpc_jit_data *jit_data;
70 	struct codegen_context cgctx;
71 	int pass;
72 	int flen;
73 	struct bpf_binary_header *fhdr = NULL;
74 	struct bpf_binary_header *hdr = NULL;
75 	struct bpf_prog *org_fp = fp;
76 	struct bpf_prog *tmp_fp;
77 	bool bpf_blinded = false;
78 	bool extra_pass = false;
79 	u8 *fimage = NULL;
80 	u32 *fcode_base;
81 	u32 extable_len;
82 	u32 fixup_len;
83 
84 	if (!fp->jit_requested)
85 		return org_fp;
86 
87 	tmp_fp = bpf_jit_blind_constants(org_fp);
88 	if (IS_ERR(tmp_fp))
89 		return org_fp;
90 
91 	if (tmp_fp != org_fp) {
92 		bpf_blinded = true;
93 		fp = tmp_fp;
94 	}
95 
96 	jit_data = fp->aux->jit_data;
97 	if (!jit_data) {
98 		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
99 		if (!jit_data) {
100 			fp = org_fp;
101 			goto out;
102 		}
103 		fp->aux->jit_data = jit_data;
104 	}
105 
106 	flen = fp->len;
107 	addrs = jit_data->addrs;
108 	if (addrs) {
109 		cgctx = jit_data->ctx;
110 		/*
111 		 * JIT compiled to a writable location (image/code_base) first.
112 		 * It is then moved to the readonly final location (fimage/fcode_base)
113 		 * using instruction patching.
114 		 */
115 		fimage = jit_data->fimage;
116 		fhdr = jit_data->fhdr;
117 		proglen = jit_data->proglen;
118 		hdr = jit_data->hdr;
119 		image = (void *)hdr + ((void *)fimage - (void *)fhdr);
120 		extra_pass = true;
121 		/* During extra pass, ensure index is reset before repopulating extable entries */
122 		cgctx.exentry_idx = 0;
123 		goto skip_init_ctx;
124 	}
125 
126 	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
127 	if (addrs == NULL) {
128 		fp = org_fp;
129 		goto out_addrs;
130 	}
131 
132 	memset(&cgctx, 0, sizeof(struct codegen_context));
133 	bpf_jit_init_reg_mapping(&cgctx);
134 
135 	/* Make sure that the stack is quadword aligned. */
136 	cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
137 
138 	/* Scouting faux-generate pass 0 */
139 	if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
140 		/* We hit something illegal or unsupported. */
141 		fp = org_fp;
142 		goto out_addrs;
143 	}
144 
145 	/*
146 	 * If we have seen a tail call, we need a second pass.
147 	 * This is because bpf_jit_emit_common_epilogue() is called
148 	 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
149 	 * We also need a second pass if we ended up with too large
150 	 * a program so as to ensure BPF_EXIT branches are in range.
151 	 */
152 	if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
153 		cgctx.idx = 0;
154 		if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
155 			fp = org_fp;
156 			goto out_addrs;
157 		}
158 	}
159 
160 	bpf_jit_realloc_regs(&cgctx);
161 	/*
162 	 * Pretend to build prologue, given the features we've seen.  This will
163 	 * update ctgtx.idx as it pretends to output instructions, then we can
164 	 * calculate total size from idx.
165 	 */
166 	bpf_jit_build_prologue(NULL, &cgctx);
167 	addrs[fp->len] = cgctx.idx * 4;
168 	bpf_jit_build_epilogue(NULL, &cgctx);
169 
170 	fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
171 	extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);
172 
173 	proglen = cgctx.idx * 4;
174 	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
175 
176 	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
177 					      bpf_jit_fill_ill_insns);
178 	if (!fhdr) {
179 		fp = org_fp;
180 		goto out_addrs;
181 	}
182 
183 	if (extable_len)
184 		fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
185 
186 skip_init_ctx:
187 	code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
188 	fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE);
189 
190 	/* Code generation passes 1-2 */
191 	for (pass = 1; pass < 3; pass++) {
192 		/* Now build the prologue, body code & epilogue for real. */
193 		cgctx.idx = 0;
194 		cgctx.alt_exit_addr = 0;
195 		bpf_jit_build_prologue(code_base, &cgctx);
196 		if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass,
197 				       extra_pass)) {
198 			bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
199 			bpf_jit_binary_pack_free(fhdr, hdr);
200 			fp = org_fp;
201 			goto out_addrs;
202 		}
203 		bpf_jit_build_epilogue(code_base, &cgctx);
204 
205 		if (bpf_jit_enable > 1)
206 			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
207 				proglen - (cgctx.idx * 4), cgctx.seen);
208 	}
209 
210 	if (bpf_jit_enable > 1)
211 		/*
212 		 * Note that we output the base address of the code_base
213 		 * rather than image, since opcodes are in code_base.
214 		 */
215 		bpf_jit_dump(flen, proglen, pass, code_base);
216 
217 #ifdef CONFIG_PPC64_ELF_ABI_V1
218 	/* Function descriptor nastiness: Address + TOC */
219 	((u64 *)image)[0] = (u64)fcode_base;
220 	((u64 *)image)[1] = local_paca->kernel_toc;
221 #endif
222 
223 	fp->bpf_func = (void *)fimage;
224 	fp->jited = 1;
225 	fp->jited_len = proglen + FUNCTION_DESCR_SIZE;
226 
227 	if (!fp->is_func || extra_pass) {
228 		if (bpf_jit_binary_pack_finalize(fp, fhdr, hdr)) {
229 			fp = org_fp;
230 			goto out_addrs;
231 		}
232 		bpf_prog_fill_jited_linfo(fp, addrs);
233 out_addrs:
234 		kfree(addrs);
235 		kfree(jit_data);
236 		fp->aux->jit_data = NULL;
237 	} else {
238 		jit_data->addrs = addrs;
239 		jit_data->ctx = cgctx;
240 		jit_data->proglen = proglen;
241 		jit_data->fimage = fimage;
242 		jit_data->fhdr = fhdr;
243 		jit_data->hdr = hdr;
244 	}
245 
246 out:
247 	if (bpf_blinded)
248 		bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
249 
250 	return fp;
251 }
252 
253 /*
254  * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
255  * this function, as this only applies to BPF_PROBE_MEM, for now.
256  */
257 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
258 			  struct codegen_context *ctx, int insn_idx, int jmp_off,
259 			  int dst_reg)
260 {
261 	off_t offset;
262 	unsigned long pc;
263 	struct exception_table_entry *ex, *ex_entry;
264 	u32 *fixup;
265 
266 	/* Populate extable entries only in the last pass */
267 	if (pass != 2)
268 		return 0;
269 
270 	if (!fp->aux->extable ||
271 	    WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
272 		return -EINVAL;
273 
274 	/*
275 	 * Program is first written to image before copying to the
276 	 * final location (fimage). Accordingly, update in the image first.
277 	 * As all offsets used are relative, copying as is to the
278 	 * final location should be alright.
279 	 */
280 	pc = (unsigned long)&image[insn_idx];
281 	ex = (void *)fp->aux->extable - (void *)fimage + (void *)image;
282 
283 	fixup = (void *)ex -
284 		(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
285 		(ctx->exentry_idx * BPF_FIXUP_LEN * 4);
286 
287 	fixup[0] = PPC_RAW_LI(dst_reg, 0);
288 	if (IS_ENABLED(CONFIG_PPC32))
289 		fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
290 
291 	fixup[BPF_FIXUP_LEN - 1] =
292 		PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);
293 
294 	ex_entry = &ex[ctx->exentry_idx];
295 
296 	offset = pc - (long)&ex_entry->insn;
297 	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
298 		return -ERANGE;
299 	ex_entry->insn = offset;
300 
301 	offset = (long)fixup - (long)&ex_entry->fixup;
302 	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
303 		return -ERANGE;
304 	ex_entry->fixup = offset;
305 
306 	ctx->exentry_idx++;
307 	return 0;
308 }
309 
310 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
311 {
312 	int err;
313 
314 	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
315 		return ERR_PTR(-EINVAL);
316 
317 	mutex_lock(&text_mutex);
318 	err = patch_instructions(dst, src, len, false);
319 	mutex_unlock(&text_mutex);
320 
321 	return err ? ERR_PTR(err) : dst;
322 }
323 
324 int bpf_arch_text_invalidate(void *dst, size_t len)
325 {
326 	u32 insn = BREAKPOINT_INSTRUCTION;
327 	int ret;
328 
329 	if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst)))
330 		return -EINVAL;
331 
332 	mutex_lock(&text_mutex);
333 	ret = patch_instructions(dst, &insn, len, true);
334 	mutex_unlock(&text_mutex);
335 
336 	return ret;
337 }
338 
339 void bpf_jit_free(struct bpf_prog *fp)
340 {
341 	if (fp->jited) {
342 		struct powerpc_jit_data *jit_data = fp->aux->jit_data;
343 		struct bpf_binary_header *hdr;
344 
345 		/*
346 		 * If we fail the final pass of JIT (from jit_subprogs),
347 		 * the program may not be finalized yet. Call finalize here
348 		 * before freeing it.
349 		 */
350 		if (jit_data) {
351 			bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr);
352 			kvfree(jit_data->addrs);
353 			kfree(jit_data);
354 		}
355 		hdr = bpf_jit_binary_pack_hdr(fp);
356 		bpf_jit_binary_pack_free(hdr, NULL);
357 		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
358 	}
359 
360 	bpf_prog_unlock_free(fp);
361 }
362