xref: /linux/arch/loongarch/net/bpf_jit.c (revision e7d759f31ca295d589f7420719c311870bb3166f)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * BPF JIT compiler for LoongArch
4  *
5  * Copyright (C) 2022 Loongson Technology Corporation Limited
6  */
7 #include "bpf_jit.h"
8 
9 #define REG_TCC		LOONGARCH_GPR_A6
10 #define TCC_SAVED	LOONGARCH_GPR_S5
11 
12 #define SAVE_RA		BIT(0)
13 #define SAVE_TCC	BIT(1)
14 
15 static const int regmap[] = {
16 	/* return value from in-kernel function, and exit value for eBPF program */
17 	[BPF_REG_0] = LOONGARCH_GPR_A5,
18 	/* arguments from eBPF program to in-kernel function */
19 	[BPF_REG_1] = LOONGARCH_GPR_A0,
20 	[BPF_REG_2] = LOONGARCH_GPR_A1,
21 	[BPF_REG_3] = LOONGARCH_GPR_A2,
22 	[BPF_REG_4] = LOONGARCH_GPR_A3,
23 	[BPF_REG_5] = LOONGARCH_GPR_A4,
24 	/* callee saved registers that in-kernel function will preserve */
25 	[BPF_REG_6] = LOONGARCH_GPR_S0,
26 	[BPF_REG_7] = LOONGARCH_GPR_S1,
27 	[BPF_REG_8] = LOONGARCH_GPR_S2,
28 	[BPF_REG_9] = LOONGARCH_GPR_S3,
29 	/* read-only frame pointer to access stack */
30 	[BPF_REG_FP] = LOONGARCH_GPR_S4,
31 	/* temporary register for blinding constants */
32 	[BPF_REG_AX] = LOONGARCH_GPR_T0,
33 };
34 
35 static void mark_call(struct jit_ctx *ctx)
36 {
37 	ctx->flags |= SAVE_RA;
38 }
39 
40 static void mark_tail_call(struct jit_ctx *ctx)
41 {
42 	ctx->flags |= SAVE_TCC;
43 }
44 
45 static bool seen_call(struct jit_ctx *ctx)
46 {
47 	return (ctx->flags & SAVE_RA);
48 }
49 
50 static bool seen_tail_call(struct jit_ctx *ctx)
51 {
52 	return (ctx->flags & SAVE_TCC);
53 }
54 
55 static u8 tail_call_reg(struct jit_ctx *ctx)
56 {
57 	if (seen_call(ctx))
58 		return TCC_SAVED;
59 
60 	return REG_TCC;
61 }
62 
63 /*
64  * eBPF prog stack layout:
65  *
66  *                                        high
67  * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
68  *                            |           $ra           |
69  *                            +-------------------------+
70  *                            |           $fp           |
71  *                            +-------------------------+
72  *                            |           $s0           |
73  *                            +-------------------------+
74  *                            |           $s1           |
75  *                            +-------------------------+
76  *                            |           $s2           |
77  *                            +-------------------------+
78  *                            |           $s3           |
79  *                            +-------------------------+
80  *                            |           $s4           |
81  *                            +-------------------------+
82  *                            |           $s5           |
83  *                            +-------------------------+ <--BPF_REG_FP
84  *                            |  prog->aux->stack_depth |
85  *                            |        (optional)       |
86  * current $sp -------------> +-------------------------+
87  *                                        low
88  */
89 static void build_prologue(struct jit_ctx *ctx)
90 {
91 	int stack_adjust = 0, store_offset, bpf_stack_adjust;
92 
93 	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
94 
95 	/* To store ra, fp, s0, s1, s2, s3, s4 and s5. */
96 	stack_adjust += sizeof(long) * 8;
97 
98 	stack_adjust = round_up(stack_adjust, 16);
99 	stack_adjust += bpf_stack_adjust;
100 
101 	/*
102 	 * First instruction initializes the tail call count (TCC).
103 	 * On tail call we skip this instruction, and the TCC is
104 	 * passed in REG_TCC from the caller.
105 	 */
106 	emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
107 
108 	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
109 
110 	store_offset = stack_adjust - sizeof(long);
111 	emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
112 
113 	store_offset -= sizeof(long);
114 	emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
115 
116 	store_offset -= sizeof(long);
117 	emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
118 
119 	store_offset -= sizeof(long);
120 	emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
121 
122 	store_offset -= sizeof(long);
123 	emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
124 
125 	store_offset -= sizeof(long);
126 	emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
127 
128 	store_offset -= sizeof(long);
129 	emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
130 
131 	store_offset -= sizeof(long);
132 	emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
133 
134 	emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
135 
136 	if (bpf_stack_adjust)
137 		emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
138 
139 	/*
140 	 * Program contains calls and tail calls, so REG_TCC need
141 	 * to be saved across calls.
142 	 */
143 	if (seen_tail_call(ctx) && seen_call(ctx))
144 		move_reg(ctx, TCC_SAVED, REG_TCC);
145 
146 	ctx->stack_size = stack_adjust;
147 }
148 
149 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
150 {
151 	int stack_adjust = ctx->stack_size;
152 	int load_offset;
153 
154 	load_offset = stack_adjust - sizeof(long);
155 	emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
156 
157 	load_offset -= sizeof(long);
158 	emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
159 
160 	load_offset -= sizeof(long);
161 	emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
162 
163 	load_offset -= sizeof(long);
164 	emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
165 
166 	load_offset -= sizeof(long);
167 	emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
168 
169 	load_offset -= sizeof(long);
170 	emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
171 
172 	load_offset -= sizeof(long);
173 	emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
174 
175 	load_offset -= sizeof(long);
176 	emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
177 
178 	emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
179 
180 	if (!is_tail_call) {
181 		/* Set return value */
182 		move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]);
183 		/* Return to the caller */
184 		emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0);
185 	} else {
186 		/*
187 		 * Call the next bpf prog and skip the first instruction
188 		 * of TCC initialization.
189 		 */
190 		emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1);
191 	}
192 }
193 
194 static void build_epilogue(struct jit_ctx *ctx)
195 {
196 	__build_epilogue(ctx, false);
197 }
198 
199 bool bpf_jit_supports_kfunc_call(void)
200 {
201 	return true;
202 }
203 
204 bool bpf_jit_supports_far_kfunc_call(void)
205 {
206 	return true;
207 }
208 
209 /* initialized on the first pass of build_body() */
210 static int out_offset = -1;
211 static int emit_bpf_tail_call(struct jit_ctx *ctx)
212 {
213 	int off;
214 	u8 tcc = tail_call_reg(ctx);
215 	u8 a1 = LOONGARCH_GPR_A1;
216 	u8 a2 = LOONGARCH_GPR_A2;
217 	u8 t1 = LOONGARCH_GPR_T1;
218 	u8 t2 = LOONGARCH_GPR_T2;
219 	u8 t3 = LOONGARCH_GPR_T3;
220 	const int idx0 = ctx->idx;
221 
222 #define cur_offset (ctx->idx - idx0)
223 #define jmp_offset (out_offset - (cur_offset))
224 
225 	/*
226 	 * a0: &ctx
227 	 * a1: &array
228 	 * a2: index
229 	 *
230 	 * if (index >= array->map.max_entries)
231 	 *	 goto out;
232 	 */
233 	off = offsetof(struct bpf_array, map.max_entries);
234 	emit_insn(ctx, ldwu, t1, a1, off);
235 	/* bgeu $a2, $t1, jmp_offset */
236 	if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
237 		goto toofar;
238 
239 	/*
240 	 * if (--TCC < 0)
241 	 *	 goto out;
242 	 */
243 	emit_insn(ctx, addid, REG_TCC, tcc, -1);
244 	if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
245 		goto toofar;
246 
247 	/*
248 	 * prog = array->ptrs[index];
249 	 * if (!prog)
250 	 *	 goto out;
251 	 */
252 	emit_insn(ctx, alsld, t2, a2, a1, 2);
253 	off = offsetof(struct bpf_array, ptrs);
254 	emit_insn(ctx, ldd, t2, t2, off);
255 	/* beq $t2, $zero, jmp_offset */
256 	if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
257 		goto toofar;
258 
259 	/* goto *(prog->bpf_func + 4); */
260 	off = offsetof(struct bpf_prog, bpf_func);
261 	emit_insn(ctx, ldd, t3, t2, off);
262 	__build_epilogue(ctx, true);
263 
264 	/* out: */
265 	if (out_offset == -1)
266 		out_offset = cur_offset;
267 	if (cur_offset != out_offset) {
268 		pr_err_once("tail_call out_offset = %d, expected %d!\n",
269 			    cur_offset, out_offset);
270 		return -1;
271 	}
272 
273 	return 0;
274 
275 toofar:
276 	pr_info_once("tail_call: jump too far\n");
277 	return -1;
278 #undef cur_offset
279 #undef jmp_offset
280 }
281 
282 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
283 {
284 	const u8 t1 = LOONGARCH_GPR_T1;
285 	const u8 t2 = LOONGARCH_GPR_T2;
286 	const u8 t3 = LOONGARCH_GPR_T3;
287 	const u8 r0 = regmap[BPF_REG_0];
288 	const u8 src = regmap[insn->src_reg];
289 	const u8 dst = regmap[insn->dst_reg];
290 	const s16 off = insn->off;
291 	const s32 imm = insn->imm;
292 	const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
293 
294 	move_imm(ctx, t1, off, false);
295 	emit_insn(ctx, addd, t1, dst, t1);
296 	move_reg(ctx, t3, src);
297 
298 	switch (imm) {
299 	/* lock *(size *)(dst + off) <op>= src */
300 	case BPF_ADD:
301 		if (isdw)
302 			emit_insn(ctx, amaddd, t2, t1, src);
303 		else
304 			emit_insn(ctx, amaddw, t2, t1, src);
305 		break;
306 	case BPF_AND:
307 		if (isdw)
308 			emit_insn(ctx, amandd, t2, t1, src);
309 		else
310 			emit_insn(ctx, amandw, t2, t1, src);
311 		break;
312 	case BPF_OR:
313 		if (isdw)
314 			emit_insn(ctx, amord, t2, t1, src);
315 		else
316 			emit_insn(ctx, amorw, t2, t1, src);
317 		break;
318 	case BPF_XOR:
319 		if (isdw)
320 			emit_insn(ctx, amxord, t2, t1, src);
321 		else
322 			emit_insn(ctx, amxorw, t2, t1, src);
323 		break;
324 	/* src = atomic_fetch_<op>(dst + off, src) */
325 	case BPF_ADD | BPF_FETCH:
326 		if (isdw) {
327 			emit_insn(ctx, amaddd, src, t1, t3);
328 		} else {
329 			emit_insn(ctx, amaddw, src, t1, t3);
330 			emit_zext_32(ctx, src, true);
331 		}
332 		break;
333 	case BPF_AND | BPF_FETCH:
334 		if (isdw) {
335 			emit_insn(ctx, amandd, src, t1, t3);
336 		} else {
337 			emit_insn(ctx, amandw, src, t1, t3);
338 			emit_zext_32(ctx, src, true);
339 		}
340 		break;
341 	case BPF_OR | BPF_FETCH:
342 		if (isdw) {
343 			emit_insn(ctx, amord, src, t1, t3);
344 		} else {
345 			emit_insn(ctx, amorw, src, t1, t3);
346 			emit_zext_32(ctx, src, true);
347 		}
348 		break;
349 	case BPF_XOR | BPF_FETCH:
350 		if (isdw) {
351 			emit_insn(ctx, amxord, src, t1, t3);
352 		} else {
353 			emit_insn(ctx, amxorw, src, t1, t3);
354 			emit_zext_32(ctx, src, true);
355 		}
356 		break;
357 	/* src = atomic_xchg(dst + off, src); */
358 	case BPF_XCHG:
359 		if (isdw) {
360 			emit_insn(ctx, amswapd, src, t1, t3);
361 		} else {
362 			emit_insn(ctx, amswapw, src, t1, t3);
363 			emit_zext_32(ctx, src, true);
364 		}
365 		break;
366 	/* r0 = atomic_cmpxchg(dst + off, r0, src); */
367 	case BPF_CMPXCHG:
368 		move_reg(ctx, t2, r0);
369 		if (isdw) {
370 			emit_insn(ctx, lld, r0, t1, 0);
371 			emit_insn(ctx, bne, t2, r0, 4);
372 			move_reg(ctx, t3, src);
373 			emit_insn(ctx, scd, t3, t1, 0);
374 			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
375 		} else {
376 			emit_insn(ctx, llw, r0, t1, 0);
377 			emit_zext_32(ctx, t2, true);
378 			emit_zext_32(ctx, r0, true);
379 			emit_insn(ctx, bne, t2, r0, 4);
380 			move_reg(ctx, t3, src);
381 			emit_insn(ctx, scw, t3, t1, 0);
382 			emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
383 			emit_zext_32(ctx, r0, true);
384 		}
385 		break;
386 	}
387 }
388 
389 static bool is_signed_bpf_cond(u8 cond)
390 {
391 	return cond == BPF_JSGT || cond == BPF_JSLT ||
392 	       cond == BPF_JSGE || cond == BPF_JSLE;
393 }
394 
395 #define BPF_FIXUP_REG_MASK	GENMASK(31, 27)
396 #define BPF_FIXUP_OFFSET_MASK	GENMASK(26, 0)
397 
398 bool ex_handler_bpf(const struct exception_table_entry *ex,
399 		    struct pt_regs *regs)
400 {
401 	int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
402 	off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
403 
404 	regs->regs[dst_reg] = 0;
405 	regs->csr_era = (unsigned long)&ex->fixup - offset;
406 
407 	return true;
408 }
409 
410 /* For accesses to BTF pointers, add an entry to the exception table */
411 static int add_exception_handler(const struct bpf_insn *insn,
412 				 struct jit_ctx *ctx,
413 				 int dst_reg)
414 {
415 	unsigned long pc;
416 	off_t offset;
417 	struct exception_table_entry *ex;
418 
419 	if (!ctx->image || !ctx->prog->aux->extable)
420 		return 0;
421 
422 	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
423 	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
424 		return 0;
425 
426 	if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
427 		return -EINVAL;
428 
429 	ex = &ctx->prog->aux->extable[ctx->num_exentries];
430 	pc = (unsigned long)&ctx->image[ctx->idx - 1];
431 
432 	offset = pc - (long)&ex->insn;
433 	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
434 		return -ERANGE;
435 
436 	ex->insn = offset;
437 
438 	/*
439 	 * Since the extable follows the program, the fixup offset is always
440 	 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
441 	 * to keep things simple, and put the destination register in the upper
442 	 * bits. We don't need to worry about buildtime or runtime sort
443 	 * modifying the upper bits because the table is already sorted, and
444 	 * isn't part of the main exception table.
445 	 */
446 	offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
447 	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
448 		return -ERANGE;
449 
450 	ex->type = EX_TYPE_BPF;
451 	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
452 
453 	ctx->num_exentries++;
454 
455 	return 0;
456 }
457 
458 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
459 {
460 	u8 tm = -1;
461 	u64 func_addr;
462 	bool func_addr_fixed, sign_extend;
463 	int i = insn - ctx->prog->insnsi;
464 	int ret, jmp_offset;
465 	const u8 code = insn->code;
466 	const u8 cond = BPF_OP(code);
467 	const u8 t1 = LOONGARCH_GPR_T1;
468 	const u8 t2 = LOONGARCH_GPR_T2;
469 	const u8 src = regmap[insn->src_reg];
470 	const u8 dst = regmap[insn->dst_reg];
471 	const s16 off = insn->off;
472 	const s32 imm = insn->imm;
473 	const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
474 
475 	switch (code) {
476 	/* dst = src */
477 	case BPF_ALU | BPF_MOV | BPF_X:
478 	case BPF_ALU64 | BPF_MOV | BPF_X:
479 		switch (off) {
480 		case 0:
481 			move_reg(ctx, dst, src);
482 			emit_zext_32(ctx, dst, is32);
483 			break;
484 		case 8:
485 			move_reg(ctx, t1, src);
486 			emit_insn(ctx, extwb, dst, t1);
487 			emit_zext_32(ctx, dst, is32);
488 			break;
489 		case 16:
490 			move_reg(ctx, t1, src);
491 			emit_insn(ctx, extwh, dst, t1);
492 			emit_zext_32(ctx, dst, is32);
493 			break;
494 		case 32:
495 			emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
496 			break;
497 		}
498 		break;
499 
500 	/* dst = imm */
501 	case BPF_ALU | BPF_MOV | BPF_K:
502 	case BPF_ALU64 | BPF_MOV | BPF_K:
503 		move_imm(ctx, dst, imm, is32);
504 		break;
505 
506 	/* dst = dst + src */
507 	case BPF_ALU | BPF_ADD | BPF_X:
508 	case BPF_ALU64 | BPF_ADD | BPF_X:
509 		emit_insn(ctx, addd, dst, dst, src);
510 		emit_zext_32(ctx, dst, is32);
511 		break;
512 
513 	/* dst = dst + imm */
514 	case BPF_ALU | BPF_ADD | BPF_K:
515 	case BPF_ALU64 | BPF_ADD | BPF_K:
516 		if (is_signed_imm12(imm)) {
517 			emit_insn(ctx, addid, dst, dst, imm);
518 		} else {
519 			move_imm(ctx, t1, imm, is32);
520 			emit_insn(ctx, addd, dst, dst, t1);
521 		}
522 		emit_zext_32(ctx, dst, is32);
523 		break;
524 
525 	/* dst = dst - src */
526 	case BPF_ALU | BPF_SUB | BPF_X:
527 	case BPF_ALU64 | BPF_SUB | BPF_X:
528 		emit_insn(ctx, subd, dst, dst, src);
529 		emit_zext_32(ctx, dst, is32);
530 		break;
531 
532 	/* dst = dst - imm */
533 	case BPF_ALU | BPF_SUB | BPF_K:
534 	case BPF_ALU64 | BPF_SUB | BPF_K:
535 		if (is_signed_imm12(-imm)) {
536 			emit_insn(ctx, addid, dst, dst, -imm);
537 		} else {
538 			move_imm(ctx, t1, imm, is32);
539 			emit_insn(ctx, subd, dst, dst, t1);
540 		}
541 		emit_zext_32(ctx, dst, is32);
542 		break;
543 
544 	/* dst = dst * src */
545 	case BPF_ALU | BPF_MUL | BPF_X:
546 	case BPF_ALU64 | BPF_MUL | BPF_X:
547 		emit_insn(ctx, muld, dst, dst, src);
548 		emit_zext_32(ctx, dst, is32);
549 		break;
550 
551 	/* dst = dst * imm */
552 	case BPF_ALU | BPF_MUL | BPF_K:
553 	case BPF_ALU64 | BPF_MUL | BPF_K:
554 		move_imm(ctx, t1, imm, is32);
555 		emit_insn(ctx, muld, dst, dst, t1);
556 		emit_zext_32(ctx, dst, is32);
557 		break;
558 
559 	/* dst = dst / src */
560 	case BPF_ALU | BPF_DIV | BPF_X:
561 	case BPF_ALU64 | BPF_DIV | BPF_X:
562 		if (!off) {
563 			emit_zext_32(ctx, dst, is32);
564 			move_reg(ctx, t1, src);
565 			emit_zext_32(ctx, t1, is32);
566 			emit_insn(ctx, divdu, dst, dst, t1);
567 			emit_zext_32(ctx, dst, is32);
568 		} else {
569 			emit_sext_32(ctx, dst, is32);
570 			move_reg(ctx, t1, src);
571 			emit_sext_32(ctx, t1, is32);
572 			emit_insn(ctx, divd, dst, dst, t1);
573 			emit_sext_32(ctx, dst, is32);
574 		}
575 		break;
576 
577 	/* dst = dst / imm */
578 	case BPF_ALU | BPF_DIV | BPF_K:
579 	case BPF_ALU64 | BPF_DIV | BPF_K:
580 		if (!off) {
581 			move_imm(ctx, t1, imm, is32);
582 			emit_zext_32(ctx, dst, is32);
583 			emit_insn(ctx, divdu, dst, dst, t1);
584 			emit_zext_32(ctx, dst, is32);
585 		} else {
586 			move_imm(ctx, t1, imm, false);
587 			emit_sext_32(ctx, t1, is32);
588 			emit_sext_32(ctx, dst, is32);
589 			emit_insn(ctx, divd, dst, dst, t1);
590 			emit_sext_32(ctx, dst, is32);
591 		}
592 		break;
593 
594 	/* dst = dst % src */
595 	case BPF_ALU | BPF_MOD | BPF_X:
596 	case BPF_ALU64 | BPF_MOD | BPF_X:
597 		if (!off) {
598 			emit_zext_32(ctx, dst, is32);
599 			move_reg(ctx, t1, src);
600 			emit_zext_32(ctx, t1, is32);
601 			emit_insn(ctx, moddu, dst, dst, t1);
602 			emit_zext_32(ctx, dst, is32);
603 		} else {
604 			emit_sext_32(ctx, dst, is32);
605 			move_reg(ctx, t1, src);
606 			emit_sext_32(ctx, t1, is32);
607 			emit_insn(ctx, modd, dst, dst, t1);
608 			emit_sext_32(ctx, dst, is32);
609 		}
610 		break;
611 
612 	/* dst = dst % imm */
613 	case BPF_ALU | BPF_MOD | BPF_K:
614 	case BPF_ALU64 | BPF_MOD | BPF_K:
615 		if (!off) {
616 			move_imm(ctx, t1, imm, is32);
617 			emit_zext_32(ctx, dst, is32);
618 			emit_insn(ctx, moddu, dst, dst, t1);
619 			emit_zext_32(ctx, dst, is32);
620 		} else {
621 			move_imm(ctx, t1, imm, false);
622 			emit_sext_32(ctx, t1, is32);
623 			emit_sext_32(ctx, dst, is32);
624 			emit_insn(ctx, modd, dst, dst, t1);
625 			emit_sext_32(ctx, dst, is32);
626 		}
627 		break;
628 
629 	/* dst = -dst */
630 	case BPF_ALU | BPF_NEG:
631 	case BPF_ALU64 | BPF_NEG:
632 		move_imm(ctx, t1, imm, is32);
633 		emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
634 		emit_zext_32(ctx, dst, is32);
635 		break;
636 
637 	/* dst = dst & src */
638 	case BPF_ALU | BPF_AND | BPF_X:
639 	case BPF_ALU64 | BPF_AND | BPF_X:
640 		emit_insn(ctx, and, dst, dst, src);
641 		emit_zext_32(ctx, dst, is32);
642 		break;
643 
644 	/* dst = dst & imm */
645 	case BPF_ALU | BPF_AND | BPF_K:
646 	case BPF_ALU64 | BPF_AND | BPF_K:
647 		if (is_unsigned_imm12(imm)) {
648 			emit_insn(ctx, andi, dst, dst, imm);
649 		} else {
650 			move_imm(ctx, t1, imm, is32);
651 			emit_insn(ctx, and, dst, dst, t1);
652 		}
653 		emit_zext_32(ctx, dst, is32);
654 		break;
655 
656 	/* dst = dst | src */
657 	case BPF_ALU | BPF_OR | BPF_X:
658 	case BPF_ALU64 | BPF_OR | BPF_X:
659 		emit_insn(ctx, or, dst, dst, src);
660 		emit_zext_32(ctx, dst, is32);
661 		break;
662 
663 	/* dst = dst | imm */
664 	case BPF_ALU | BPF_OR | BPF_K:
665 	case BPF_ALU64 | BPF_OR | BPF_K:
666 		if (is_unsigned_imm12(imm)) {
667 			emit_insn(ctx, ori, dst, dst, imm);
668 		} else {
669 			move_imm(ctx, t1, imm, is32);
670 			emit_insn(ctx, or, dst, dst, t1);
671 		}
672 		emit_zext_32(ctx, dst, is32);
673 		break;
674 
675 	/* dst = dst ^ src */
676 	case BPF_ALU | BPF_XOR | BPF_X:
677 	case BPF_ALU64 | BPF_XOR | BPF_X:
678 		emit_insn(ctx, xor, dst, dst, src);
679 		emit_zext_32(ctx, dst, is32);
680 		break;
681 
682 	/* dst = dst ^ imm */
683 	case BPF_ALU | BPF_XOR | BPF_K:
684 	case BPF_ALU64 | BPF_XOR | BPF_K:
685 		if (is_unsigned_imm12(imm)) {
686 			emit_insn(ctx, xori, dst, dst, imm);
687 		} else {
688 			move_imm(ctx, t1, imm, is32);
689 			emit_insn(ctx, xor, dst, dst, t1);
690 		}
691 		emit_zext_32(ctx, dst, is32);
692 		break;
693 
694 	/* dst = dst << src (logical) */
695 	case BPF_ALU | BPF_LSH | BPF_X:
696 		emit_insn(ctx, sllw, dst, dst, src);
697 		emit_zext_32(ctx, dst, is32);
698 		break;
699 
700 	case BPF_ALU64 | BPF_LSH | BPF_X:
701 		emit_insn(ctx, slld, dst, dst, src);
702 		break;
703 
704 	/* dst = dst << imm (logical) */
705 	case BPF_ALU | BPF_LSH | BPF_K:
706 		emit_insn(ctx, slliw, dst, dst, imm);
707 		emit_zext_32(ctx, dst, is32);
708 		break;
709 
710 	case BPF_ALU64 | BPF_LSH | BPF_K:
711 		emit_insn(ctx, sllid, dst, dst, imm);
712 		break;
713 
714 	/* dst = dst >> src (logical) */
715 	case BPF_ALU | BPF_RSH | BPF_X:
716 		emit_insn(ctx, srlw, dst, dst, src);
717 		emit_zext_32(ctx, dst, is32);
718 		break;
719 
720 	case BPF_ALU64 | BPF_RSH | BPF_X:
721 		emit_insn(ctx, srld, dst, dst, src);
722 		break;
723 
724 	/* dst = dst >> imm (logical) */
725 	case BPF_ALU | BPF_RSH | BPF_K:
726 		emit_insn(ctx, srliw, dst, dst, imm);
727 		emit_zext_32(ctx, dst, is32);
728 		break;
729 
730 	case BPF_ALU64 | BPF_RSH | BPF_K:
731 		emit_insn(ctx, srlid, dst, dst, imm);
732 		break;
733 
734 	/* dst = dst >> src (arithmetic) */
735 	case BPF_ALU | BPF_ARSH | BPF_X:
736 		emit_insn(ctx, sraw, dst, dst, src);
737 		emit_zext_32(ctx, dst, is32);
738 		break;
739 
740 	case BPF_ALU64 | BPF_ARSH | BPF_X:
741 		emit_insn(ctx, srad, dst, dst, src);
742 		break;
743 
744 	/* dst = dst >> imm (arithmetic) */
745 	case BPF_ALU | BPF_ARSH | BPF_K:
746 		emit_insn(ctx, sraiw, dst, dst, imm);
747 		emit_zext_32(ctx, dst, is32);
748 		break;
749 
750 	case BPF_ALU64 | BPF_ARSH | BPF_K:
751 		emit_insn(ctx, sraid, dst, dst, imm);
752 		break;
753 
754 	/* dst = BSWAP##imm(dst) */
755 	case BPF_ALU | BPF_END | BPF_FROM_LE:
756 		switch (imm) {
757 		case 16:
758 			/* zero-extend 16 bits into 64 bits */
759 			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
760 			break;
761 		case 32:
762 			/* zero-extend 32 bits into 64 bits */
763 			emit_zext_32(ctx, dst, is32);
764 			break;
765 		case 64:
766 			/* do nothing */
767 			break;
768 		}
769 		break;
770 
771 	case BPF_ALU | BPF_END | BPF_FROM_BE:
772 	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
773 		switch (imm) {
774 		case 16:
775 			emit_insn(ctx, revb2h, dst, dst);
776 			/* zero-extend 16 bits into 64 bits */
777 			emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
778 			break;
779 		case 32:
780 			emit_insn(ctx, revb2w, dst, dst);
781 			/* clear the upper 32 bits */
782 			emit_zext_32(ctx, dst, true);
783 			break;
784 		case 64:
785 			emit_insn(ctx, revbd, dst, dst);
786 			break;
787 		}
788 		break;
789 
790 	/* PC += off if dst cond src */
791 	case BPF_JMP | BPF_JEQ | BPF_X:
792 	case BPF_JMP | BPF_JNE | BPF_X:
793 	case BPF_JMP | BPF_JGT | BPF_X:
794 	case BPF_JMP | BPF_JGE | BPF_X:
795 	case BPF_JMP | BPF_JLT | BPF_X:
796 	case BPF_JMP | BPF_JLE | BPF_X:
797 	case BPF_JMP | BPF_JSGT | BPF_X:
798 	case BPF_JMP | BPF_JSGE | BPF_X:
799 	case BPF_JMP | BPF_JSLT | BPF_X:
800 	case BPF_JMP | BPF_JSLE | BPF_X:
801 	case BPF_JMP32 | BPF_JEQ | BPF_X:
802 	case BPF_JMP32 | BPF_JNE | BPF_X:
803 	case BPF_JMP32 | BPF_JGT | BPF_X:
804 	case BPF_JMP32 | BPF_JGE | BPF_X:
805 	case BPF_JMP32 | BPF_JLT | BPF_X:
806 	case BPF_JMP32 | BPF_JLE | BPF_X:
807 	case BPF_JMP32 | BPF_JSGT | BPF_X:
808 	case BPF_JMP32 | BPF_JSGE | BPF_X:
809 	case BPF_JMP32 | BPF_JSLT | BPF_X:
810 	case BPF_JMP32 | BPF_JSLE | BPF_X:
811 		jmp_offset = bpf2la_offset(i, off, ctx);
812 		move_reg(ctx, t1, dst);
813 		move_reg(ctx, t2, src);
814 		if (is_signed_bpf_cond(BPF_OP(code))) {
815 			emit_sext_32(ctx, t1, is32);
816 			emit_sext_32(ctx, t2, is32);
817 		} else {
818 			emit_zext_32(ctx, t1, is32);
819 			emit_zext_32(ctx, t2, is32);
820 		}
821 		if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
822 			goto toofar;
823 		break;
824 
825 	/* PC += off if dst cond imm */
826 	case BPF_JMP | BPF_JEQ | BPF_K:
827 	case BPF_JMP | BPF_JNE | BPF_K:
828 	case BPF_JMP | BPF_JGT | BPF_K:
829 	case BPF_JMP | BPF_JGE | BPF_K:
830 	case BPF_JMP | BPF_JLT | BPF_K:
831 	case BPF_JMP | BPF_JLE | BPF_K:
832 	case BPF_JMP | BPF_JSGT | BPF_K:
833 	case BPF_JMP | BPF_JSGE | BPF_K:
834 	case BPF_JMP | BPF_JSLT | BPF_K:
835 	case BPF_JMP | BPF_JSLE | BPF_K:
836 	case BPF_JMP32 | BPF_JEQ | BPF_K:
837 	case BPF_JMP32 | BPF_JNE | BPF_K:
838 	case BPF_JMP32 | BPF_JGT | BPF_K:
839 	case BPF_JMP32 | BPF_JGE | BPF_K:
840 	case BPF_JMP32 | BPF_JLT | BPF_K:
841 	case BPF_JMP32 | BPF_JLE | BPF_K:
842 	case BPF_JMP32 | BPF_JSGT | BPF_K:
843 	case BPF_JMP32 | BPF_JSGE | BPF_K:
844 	case BPF_JMP32 | BPF_JSLT | BPF_K:
845 	case BPF_JMP32 | BPF_JSLE | BPF_K:
846 		jmp_offset = bpf2la_offset(i, off, ctx);
847 		if (imm) {
848 			move_imm(ctx, t1, imm, false);
849 			tm = t1;
850 		} else {
851 			/* If imm is 0, simply use zero register. */
852 			tm = LOONGARCH_GPR_ZERO;
853 		}
854 		move_reg(ctx, t2, dst);
855 		if (is_signed_bpf_cond(BPF_OP(code))) {
856 			emit_sext_32(ctx, tm, is32);
857 			emit_sext_32(ctx, t2, is32);
858 		} else {
859 			emit_zext_32(ctx, tm, is32);
860 			emit_zext_32(ctx, t2, is32);
861 		}
862 		if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
863 			goto toofar;
864 		break;
865 
866 	/* PC += off if dst & src */
867 	case BPF_JMP | BPF_JSET | BPF_X:
868 	case BPF_JMP32 | BPF_JSET | BPF_X:
869 		jmp_offset = bpf2la_offset(i, off, ctx);
870 		emit_insn(ctx, and, t1, dst, src);
871 		emit_zext_32(ctx, t1, is32);
872 		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
873 			goto toofar;
874 		break;
875 
876 	/* PC += off if dst & imm */
877 	case BPF_JMP | BPF_JSET | BPF_K:
878 	case BPF_JMP32 | BPF_JSET | BPF_K:
879 		jmp_offset = bpf2la_offset(i, off, ctx);
880 		move_imm(ctx, t1, imm, is32);
881 		emit_insn(ctx, and, t1, dst, t1);
882 		emit_zext_32(ctx, t1, is32);
883 		if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
884 			goto toofar;
885 		break;
886 
887 	/* PC += off */
888 	case BPF_JMP | BPF_JA:
889 	case BPF_JMP32 | BPF_JA:
890 		if (BPF_CLASS(code) == BPF_JMP)
891 			jmp_offset = bpf2la_offset(i, off, ctx);
892 		else
893 			jmp_offset = bpf2la_offset(i, imm, ctx);
894 		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
895 			goto toofar;
896 		break;
897 
898 	/* function call */
899 	case BPF_JMP | BPF_CALL:
900 		mark_call(ctx);
901 		ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
902 					    &func_addr, &func_addr_fixed);
903 		if (ret < 0)
904 			return ret;
905 
906 		move_addr(ctx, t1, func_addr);
907 		emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0);
908 		move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
909 		break;
910 
911 	/* tail call */
912 	case BPF_JMP | BPF_TAIL_CALL:
913 		mark_tail_call(ctx);
914 		if (emit_bpf_tail_call(ctx) < 0)
915 			return -EINVAL;
916 		break;
917 
918 	/* function return */
919 	case BPF_JMP | BPF_EXIT:
920 		if (i == ctx->prog->len - 1)
921 			break;
922 
923 		jmp_offset = epilogue_offset(ctx);
924 		if (emit_uncond_jmp(ctx, jmp_offset) < 0)
925 			goto toofar;
926 		break;
927 
928 	/* dst = imm64 */
929 	case BPF_LD | BPF_IMM | BPF_DW:
930 	{
931 		const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
932 
933 		move_imm(ctx, dst, imm64, is32);
934 		return 1;
935 	}
936 
937 	/* dst = *(size *)(src + off) */
938 	case BPF_LDX | BPF_MEM | BPF_B:
939 	case BPF_LDX | BPF_MEM | BPF_H:
940 	case BPF_LDX | BPF_MEM | BPF_W:
941 	case BPF_LDX | BPF_MEM | BPF_DW:
942 	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
943 	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
944 	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
945 	case BPF_LDX | BPF_PROBE_MEM | BPF_B:
946 	/* dst_reg = (s64)*(signed size *)(src_reg + off) */
947 	case BPF_LDX | BPF_MEMSX | BPF_B:
948 	case BPF_LDX | BPF_MEMSX | BPF_H:
949 	case BPF_LDX | BPF_MEMSX | BPF_W:
950 	case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
951 	case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
952 	case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
953 		sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
954 			      BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
955 		switch (BPF_SIZE(code)) {
956 		case BPF_B:
957 			if (is_signed_imm12(off)) {
958 				if (sign_extend)
959 					emit_insn(ctx, ldb, dst, src, off);
960 				else
961 					emit_insn(ctx, ldbu, dst, src, off);
962 			} else {
963 				move_imm(ctx, t1, off, is32);
964 				if (sign_extend)
965 					emit_insn(ctx, ldxb, dst, src, t1);
966 				else
967 					emit_insn(ctx, ldxbu, dst, src, t1);
968 			}
969 			break;
970 		case BPF_H:
971 			if (is_signed_imm12(off)) {
972 				if (sign_extend)
973 					emit_insn(ctx, ldh, dst, src, off);
974 				else
975 					emit_insn(ctx, ldhu, dst, src, off);
976 			} else {
977 				move_imm(ctx, t1, off, is32);
978 				if (sign_extend)
979 					emit_insn(ctx, ldxh, dst, src, t1);
980 				else
981 					emit_insn(ctx, ldxhu, dst, src, t1);
982 			}
983 			break;
984 		case BPF_W:
985 			if (is_signed_imm12(off)) {
986 				if (sign_extend)
987 					emit_insn(ctx, ldw, dst, src, off);
988 				else
989 					emit_insn(ctx, ldwu, dst, src, off);
990 			} else {
991 				move_imm(ctx, t1, off, is32);
992 				if (sign_extend)
993 					emit_insn(ctx, ldxw, dst, src, t1);
994 				else
995 					emit_insn(ctx, ldxwu, dst, src, t1);
996 			}
997 			break;
998 		case BPF_DW:
999 			move_imm(ctx, t1, off, is32);
1000 			emit_insn(ctx, ldxd, dst, src, t1);
1001 			break;
1002 		}
1003 
1004 		ret = add_exception_handler(insn, ctx, dst);
1005 		if (ret)
1006 			return ret;
1007 		break;
1008 
1009 	/* *(size *)(dst + off) = imm */
1010 	case BPF_ST | BPF_MEM | BPF_B:
1011 	case BPF_ST | BPF_MEM | BPF_H:
1012 	case BPF_ST | BPF_MEM | BPF_W:
1013 	case BPF_ST | BPF_MEM | BPF_DW:
1014 		switch (BPF_SIZE(code)) {
1015 		case BPF_B:
1016 			move_imm(ctx, t1, imm, is32);
1017 			if (is_signed_imm12(off)) {
1018 				emit_insn(ctx, stb, t1, dst, off);
1019 			} else {
1020 				move_imm(ctx, t2, off, is32);
1021 				emit_insn(ctx, stxb, t1, dst, t2);
1022 			}
1023 			break;
1024 		case BPF_H:
1025 			move_imm(ctx, t1, imm, is32);
1026 			if (is_signed_imm12(off)) {
1027 				emit_insn(ctx, sth, t1, dst, off);
1028 			} else {
1029 				move_imm(ctx, t2, off, is32);
1030 				emit_insn(ctx, stxh, t1, dst, t2);
1031 			}
1032 			break;
1033 		case BPF_W:
1034 			move_imm(ctx, t1, imm, is32);
1035 			if (is_signed_imm12(off)) {
1036 				emit_insn(ctx, stw, t1, dst, off);
1037 			} else if (is_signed_imm14(off)) {
1038 				emit_insn(ctx, stptrw, t1, dst, off);
1039 			} else {
1040 				move_imm(ctx, t2, off, is32);
1041 				emit_insn(ctx, stxw, t1, dst, t2);
1042 			}
1043 			break;
1044 		case BPF_DW:
1045 			move_imm(ctx, t1, imm, is32);
1046 			if (is_signed_imm12(off)) {
1047 				emit_insn(ctx, std, t1, dst, off);
1048 			} else if (is_signed_imm14(off)) {
1049 				emit_insn(ctx, stptrd, t1, dst, off);
1050 			} else {
1051 				move_imm(ctx, t2, off, is32);
1052 				emit_insn(ctx, stxd, t1, dst, t2);
1053 			}
1054 			break;
1055 		}
1056 		break;
1057 
1058 	/* *(size *)(dst + off) = src */
1059 	case BPF_STX | BPF_MEM | BPF_B:
1060 	case BPF_STX | BPF_MEM | BPF_H:
1061 	case BPF_STX | BPF_MEM | BPF_W:
1062 	case BPF_STX | BPF_MEM | BPF_DW:
1063 		switch (BPF_SIZE(code)) {
1064 		case BPF_B:
1065 			if (is_signed_imm12(off)) {
1066 				emit_insn(ctx, stb, src, dst, off);
1067 			} else {
1068 				move_imm(ctx, t1, off, is32);
1069 				emit_insn(ctx, stxb, src, dst, t1);
1070 			}
1071 			break;
1072 		case BPF_H:
1073 			if (is_signed_imm12(off)) {
1074 				emit_insn(ctx, sth, src, dst, off);
1075 			} else {
1076 				move_imm(ctx, t1, off, is32);
1077 				emit_insn(ctx, stxh, src, dst, t1);
1078 			}
1079 			break;
1080 		case BPF_W:
1081 			if (is_signed_imm12(off)) {
1082 				emit_insn(ctx, stw, src, dst, off);
1083 			} else if (is_signed_imm14(off)) {
1084 				emit_insn(ctx, stptrw, src, dst, off);
1085 			} else {
1086 				move_imm(ctx, t1, off, is32);
1087 				emit_insn(ctx, stxw, src, dst, t1);
1088 			}
1089 			break;
1090 		case BPF_DW:
1091 			if (is_signed_imm12(off)) {
1092 				emit_insn(ctx, std, src, dst, off);
1093 			} else if (is_signed_imm14(off)) {
1094 				emit_insn(ctx, stptrd, src, dst, off);
1095 			} else {
1096 				move_imm(ctx, t1, off, is32);
1097 				emit_insn(ctx, stxd, src, dst, t1);
1098 			}
1099 			break;
1100 		}
1101 		break;
1102 
1103 	case BPF_STX | BPF_ATOMIC | BPF_W:
1104 	case BPF_STX | BPF_ATOMIC | BPF_DW:
1105 		emit_atomic(insn, ctx);
1106 		break;
1107 
1108 	/* Speculation barrier */
1109 	case BPF_ST | BPF_NOSPEC:
1110 		break;
1111 
1112 	default:
1113 		pr_err("bpf_jit: unknown opcode %02x\n", code);
1114 		return -EINVAL;
1115 	}
1116 
1117 	return 0;
1118 
1119 toofar:
1120 	pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1121 	return -E2BIG;
1122 }
1123 
1124 static int build_body(struct jit_ctx *ctx, bool extra_pass)
1125 {
1126 	int i;
1127 	const struct bpf_prog *prog = ctx->prog;
1128 
1129 	for (i = 0; i < prog->len; i++) {
1130 		const struct bpf_insn *insn = &prog->insnsi[i];
1131 		int ret;
1132 
1133 		if (ctx->image == NULL)
1134 			ctx->offset[i] = ctx->idx;
1135 
1136 		ret = build_insn(insn, ctx, extra_pass);
1137 		if (ret > 0) {
1138 			i++;
1139 			if (ctx->image == NULL)
1140 				ctx->offset[i] = ctx->idx;
1141 			continue;
1142 		}
1143 		if (ret)
1144 			return ret;
1145 	}
1146 
1147 	if (ctx->image == NULL)
1148 		ctx->offset[i] = ctx->idx;
1149 
1150 	return 0;
1151 }
1152 
1153 /* Fill space with break instructions */
1154 static void jit_fill_hole(void *area, unsigned int size)
1155 {
1156 	u32 *ptr;
1157 
1158 	/* We are guaranteed to have aligned memory */
1159 	for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1160 		*ptr++ = INSN_BREAK;
1161 }
1162 
1163 static int validate_code(struct jit_ctx *ctx)
1164 {
1165 	int i;
1166 	union loongarch_instruction insn;
1167 
1168 	for (i = 0; i < ctx->idx; i++) {
1169 		insn = ctx->image[i];
1170 		/* Check INSN_BREAK */
1171 		if (insn.word == INSN_BREAK)
1172 			return -1;
1173 	}
1174 
1175 	if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1176 		return -1;
1177 
1178 	return 0;
1179 }
1180 
1181 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1182 {
1183 	bool tmp_blinded = false, extra_pass = false;
1184 	u8 *image_ptr;
1185 	int image_size, prog_size, extable_size;
1186 	struct jit_ctx ctx;
1187 	struct jit_data *jit_data;
1188 	struct bpf_binary_header *header;
1189 	struct bpf_prog *tmp, *orig_prog = prog;
1190 
1191 	/*
1192 	 * If BPF JIT was not enabled then we must fall back to
1193 	 * the interpreter.
1194 	 */
1195 	if (!prog->jit_requested)
1196 		return orig_prog;
1197 
1198 	tmp = bpf_jit_blind_constants(prog);
1199 	/*
1200 	 * If blinding was requested and we failed during blinding,
1201 	 * we must fall back to the interpreter. Otherwise, we save
1202 	 * the new JITed code.
1203 	 */
1204 	if (IS_ERR(tmp))
1205 		return orig_prog;
1206 
1207 	if (tmp != prog) {
1208 		tmp_blinded = true;
1209 		prog = tmp;
1210 	}
1211 
1212 	jit_data = prog->aux->jit_data;
1213 	if (!jit_data) {
1214 		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1215 		if (!jit_data) {
1216 			prog = orig_prog;
1217 			goto out;
1218 		}
1219 		prog->aux->jit_data = jit_data;
1220 	}
1221 	if (jit_data->ctx.offset) {
1222 		ctx = jit_data->ctx;
1223 		image_ptr = jit_data->image;
1224 		header = jit_data->header;
1225 		extra_pass = true;
1226 		prog_size = sizeof(u32) * ctx.idx;
1227 		goto skip_init_ctx;
1228 	}
1229 
1230 	memset(&ctx, 0, sizeof(ctx));
1231 	ctx.prog = prog;
1232 
1233 	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1234 	if (ctx.offset == NULL) {
1235 		prog = orig_prog;
1236 		goto out_offset;
1237 	}
1238 
1239 	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1240 	build_prologue(&ctx);
1241 	if (build_body(&ctx, extra_pass)) {
1242 		prog = orig_prog;
1243 		goto out_offset;
1244 	}
1245 	ctx.epilogue_offset = ctx.idx;
1246 	build_epilogue(&ctx);
1247 
1248 	extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1249 
1250 	/* Now we know the actual image size.
1251 	 * As each LoongArch instruction is of length 32bit,
1252 	 * we are translating number of JITed intructions into
1253 	 * the size required to store these JITed code.
1254 	 */
1255 	prog_size = sizeof(u32) * ctx.idx;
1256 	image_size = prog_size + extable_size;
1257 	/* Now we know the size of the structure to make */
1258 	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1259 				      sizeof(u32), jit_fill_hole);
1260 	if (header == NULL) {
1261 		prog = orig_prog;
1262 		goto out_offset;
1263 	}
1264 
1265 	/* 2. Now, the actual pass to generate final JIT code */
1266 	ctx.image = (union loongarch_instruction *)image_ptr;
1267 	if (extable_size)
1268 		prog->aux->extable = (void *)image_ptr + prog_size;
1269 
1270 skip_init_ctx:
1271 	ctx.idx = 0;
1272 	ctx.num_exentries = 0;
1273 
1274 	build_prologue(&ctx);
1275 	if (build_body(&ctx, extra_pass)) {
1276 		bpf_jit_binary_free(header);
1277 		prog = orig_prog;
1278 		goto out_offset;
1279 	}
1280 	build_epilogue(&ctx);
1281 
1282 	/* 3. Extra pass to validate JITed code */
1283 	if (validate_code(&ctx)) {
1284 		bpf_jit_binary_free(header);
1285 		prog = orig_prog;
1286 		goto out_offset;
1287 	}
1288 
1289 	/* And we're done */
1290 	if (bpf_jit_enable > 1)
1291 		bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1292 
1293 	/* Update the icache */
1294 	flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1295 
1296 	if (!prog->is_func || extra_pass) {
1297 		if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1298 			pr_err_once("multi-func JIT bug %d != %d\n",
1299 				    ctx.idx, jit_data->ctx.idx);
1300 			bpf_jit_binary_free(header);
1301 			prog->bpf_func = NULL;
1302 			prog->jited = 0;
1303 			prog->jited_len = 0;
1304 			goto out_offset;
1305 		}
1306 		bpf_jit_binary_lock_ro(header);
1307 	} else {
1308 		jit_data->ctx = ctx;
1309 		jit_data->image = image_ptr;
1310 		jit_data->header = header;
1311 	}
1312 	prog->jited = 1;
1313 	prog->jited_len = prog_size;
1314 	prog->bpf_func = (void *)ctx.image;
1315 
1316 	if (!prog->is_func || extra_pass) {
1317 		int i;
1318 
1319 		/* offset[prog->len] is the size of program */
1320 		for (i = 0; i <= prog->len; i++)
1321 			ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1322 		bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1323 
1324 out_offset:
1325 		kvfree(ctx.offset);
1326 		kfree(jit_data);
1327 		prog->aux->jit_data = NULL;
1328 	}
1329 
1330 out:
1331 	if (tmp_blinded)
1332 		bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
1333 
1334 	out_offset = -1;
1335 
1336 	return prog;
1337 }
1338 
1339 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
1340 bool bpf_jit_supports_subprog_tailcalls(void)
1341 {
1342 	return true;
1343 }
1344