1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * BPF JIT compiler for LoongArch
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7 #include "bpf_jit.h"
8
9 #define REG_TCC LOONGARCH_GPR_A6
10 #define TCC_SAVED LOONGARCH_GPR_S5
11
12 #define SAVE_RA BIT(0)
13 #define SAVE_TCC BIT(1)
14
15 static const int regmap[] = {
16 /* return value from in-kernel function, and exit value for eBPF program */
17 [BPF_REG_0] = LOONGARCH_GPR_A5,
18 /* arguments from eBPF program to in-kernel function */
19 [BPF_REG_1] = LOONGARCH_GPR_A0,
20 [BPF_REG_2] = LOONGARCH_GPR_A1,
21 [BPF_REG_3] = LOONGARCH_GPR_A2,
22 [BPF_REG_4] = LOONGARCH_GPR_A3,
23 [BPF_REG_5] = LOONGARCH_GPR_A4,
24 /* callee saved registers that in-kernel function will preserve */
25 [BPF_REG_6] = LOONGARCH_GPR_S0,
26 [BPF_REG_7] = LOONGARCH_GPR_S1,
27 [BPF_REG_8] = LOONGARCH_GPR_S2,
28 [BPF_REG_9] = LOONGARCH_GPR_S3,
29 /* read-only frame pointer to access stack */
30 [BPF_REG_FP] = LOONGARCH_GPR_S4,
31 /* temporary register for blinding constants */
32 [BPF_REG_AX] = LOONGARCH_GPR_T0,
33 };
34
mark_call(struct jit_ctx * ctx)35 static void mark_call(struct jit_ctx *ctx)
36 {
37 ctx->flags |= SAVE_RA;
38 }
39
mark_tail_call(struct jit_ctx * ctx)40 static void mark_tail_call(struct jit_ctx *ctx)
41 {
42 ctx->flags |= SAVE_TCC;
43 }
44
seen_call(struct jit_ctx * ctx)45 static bool seen_call(struct jit_ctx *ctx)
46 {
47 return (ctx->flags & SAVE_RA);
48 }
49
seen_tail_call(struct jit_ctx * ctx)50 static bool seen_tail_call(struct jit_ctx *ctx)
51 {
52 return (ctx->flags & SAVE_TCC);
53 }
54
tail_call_reg(struct jit_ctx * ctx)55 static u8 tail_call_reg(struct jit_ctx *ctx)
56 {
57 if (seen_call(ctx))
58 return TCC_SAVED;
59
60 return REG_TCC;
61 }
62
63 /*
64 * eBPF prog stack layout:
65 *
66 * high
67 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
68 * | $ra |
69 * +-------------------------+
70 * | $fp |
71 * +-------------------------+
72 * | $s0 |
73 * +-------------------------+
74 * | $s1 |
75 * +-------------------------+
76 * | $s2 |
77 * +-------------------------+
78 * | $s3 |
79 * +-------------------------+
80 * | $s4 |
81 * +-------------------------+
82 * | $s5 |
83 * +-------------------------+ <--BPF_REG_FP
84 * | prog->aux->stack_depth |
85 * | (optional) |
86 * current $sp -------------> +-------------------------+
87 * low
88 */
build_prologue(struct jit_ctx * ctx)89 static void build_prologue(struct jit_ctx *ctx)
90 {
91 int stack_adjust = 0, store_offset, bpf_stack_adjust;
92
93 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
94
95 /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */
96 stack_adjust += sizeof(long) * 8;
97
98 stack_adjust = round_up(stack_adjust, 16);
99 stack_adjust += bpf_stack_adjust;
100
101 /*
102 * First instruction initializes the tail call count (TCC).
103 * On tail call we skip this instruction, and the TCC is
104 * passed in REG_TCC from the caller.
105 */
106 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
107
108 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
109
110 store_offset = stack_adjust - sizeof(long);
111 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
112
113 store_offset -= sizeof(long);
114 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
115
116 store_offset -= sizeof(long);
117 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
118
119 store_offset -= sizeof(long);
120 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
121
122 store_offset -= sizeof(long);
123 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
124
125 store_offset -= sizeof(long);
126 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
127
128 store_offset -= sizeof(long);
129 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
130
131 store_offset -= sizeof(long);
132 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
133
134 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
135
136 if (bpf_stack_adjust)
137 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
138
139 /*
140 * Program contains calls and tail calls, so REG_TCC need
141 * to be saved across calls.
142 */
143 if (seen_tail_call(ctx) && seen_call(ctx))
144 move_reg(ctx, TCC_SAVED, REG_TCC);
145
146 ctx->stack_size = stack_adjust;
147 }
148
__build_epilogue(struct jit_ctx * ctx,bool is_tail_call)149 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
150 {
151 int stack_adjust = ctx->stack_size;
152 int load_offset;
153
154 load_offset = stack_adjust - sizeof(long);
155 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
156
157 load_offset -= sizeof(long);
158 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
159
160 load_offset -= sizeof(long);
161 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
162
163 load_offset -= sizeof(long);
164 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
165
166 load_offset -= sizeof(long);
167 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
168
169 load_offset -= sizeof(long);
170 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
171
172 load_offset -= sizeof(long);
173 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
174
175 load_offset -= sizeof(long);
176 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
177
178 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
179
180 if (!is_tail_call) {
181 /* Set return value */
182 move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]);
183 /* Return to the caller */
184 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0);
185 } else {
186 /*
187 * Call the next bpf prog and skip the first instruction
188 * of TCC initialization.
189 */
190 emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1);
191 }
192 }
193
build_epilogue(struct jit_ctx * ctx)194 static void build_epilogue(struct jit_ctx *ctx)
195 {
196 __build_epilogue(ctx, false);
197 }
198
bpf_jit_supports_kfunc_call(void)199 bool bpf_jit_supports_kfunc_call(void)
200 {
201 return true;
202 }
203
bpf_jit_supports_far_kfunc_call(void)204 bool bpf_jit_supports_far_kfunc_call(void)
205 {
206 return true;
207 }
208
209 /* initialized on the first pass of build_body() */
210 static int out_offset = -1;
emit_bpf_tail_call(struct jit_ctx * ctx)211 static int emit_bpf_tail_call(struct jit_ctx *ctx)
212 {
213 int off;
214 u8 tcc = tail_call_reg(ctx);
215 u8 a1 = LOONGARCH_GPR_A1;
216 u8 a2 = LOONGARCH_GPR_A2;
217 u8 t1 = LOONGARCH_GPR_T1;
218 u8 t2 = LOONGARCH_GPR_T2;
219 u8 t3 = LOONGARCH_GPR_T3;
220 const int idx0 = ctx->idx;
221
222 #define cur_offset (ctx->idx - idx0)
223 #define jmp_offset (out_offset - (cur_offset))
224
225 /*
226 * a0: &ctx
227 * a1: &array
228 * a2: index
229 *
230 * if (index >= array->map.max_entries)
231 * goto out;
232 */
233 off = offsetof(struct bpf_array, map.max_entries);
234 emit_insn(ctx, ldwu, t1, a1, off);
235 /* bgeu $a2, $t1, jmp_offset */
236 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
237 goto toofar;
238
239 /*
240 * if (--TCC < 0)
241 * goto out;
242 */
243 emit_insn(ctx, addid, REG_TCC, tcc, -1);
244 if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
245 goto toofar;
246
247 /*
248 * prog = array->ptrs[index];
249 * if (!prog)
250 * goto out;
251 */
252 emit_insn(ctx, alsld, t2, a2, a1, 2);
253 off = offsetof(struct bpf_array, ptrs);
254 emit_insn(ctx, ldd, t2, t2, off);
255 /* beq $t2, $zero, jmp_offset */
256 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
257 goto toofar;
258
259 /* goto *(prog->bpf_func + 4); */
260 off = offsetof(struct bpf_prog, bpf_func);
261 emit_insn(ctx, ldd, t3, t2, off);
262 __build_epilogue(ctx, true);
263
264 /* out: */
265 if (out_offset == -1)
266 out_offset = cur_offset;
267 if (cur_offset != out_offset) {
268 pr_err_once("tail_call out_offset = %d, expected %d!\n",
269 cur_offset, out_offset);
270 return -1;
271 }
272
273 return 0;
274
275 toofar:
276 pr_info_once("tail_call: jump too far\n");
277 return -1;
278 #undef cur_offset
279 #undef jmp_offset
280 }
281
emit_atomic(const struct bpf_insn * insn,struct jit_ctx * ctx)282 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
283 {
284 const u8 t1 = LOONGARCH_GPR_T1;
285 const u8 t2 = LOONGARCH_GPR_T2;
286 const u8 t3 = LOONGARCH_GPR_T3;
287 const u8 r0 = regmap[BPF_REG_0];
288 const u8 src = regmap[insn->src_reg];
289 const u8 dst = regmap[insn->dst_reg];
290 const s16 off = insn->off;
291 const s32 imm = insn->imm;
292 const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
293
294 move_imm(ctx, t1, off, false);
295 emit_insn(ctx, addd, t1, dst, t1);
296 move_reg(ctx, t3, src);
297
298 switch (imm) {
299 /* lock *(size *)(dst + off) <op>= src */
300 case BPF_ADD:
301 if (isdw)
302 emit_insn(ctx, amaddd, t2, t1, src);
303 else
304 emit_insn(ctx, amaddw, t2, t1, src);
305 break;
306 case BPF_AND:
307 if (isdw)
308 emit_insn(ctx, amandd, t2, t1, src);
309 else
310 emit_insn(ctx, amandw, t2, t1, src);
311 break;
312 case BPF_OR:
313 if (isdw)
314 emit_insn(ctx, amord, t2, t1, src);
315 else
316 emit_insn(ctx, amorw, t2, t1, src);
317 break;
318 case BPF_XOR:
319 if (isdw)
320 emit_insn(ctx, amxord, t2, t1, src);
321 else
322 emit_insn(ctx, amxorw, t2, t1, src);
323 break;
324 /* src = atomic_fetch_<op>(dst + off, src) */
325 case BPF_ADD | BPF_FETCH:
326 if (isdw) {
327 emit_insn(ctx, amaddd, src, t1, t3);
328 } else {
329 emit_insn(ctx, amaddw, src, t1, t3);
330 emit_zext_32(ctx, src, true);
331 }
332 break;
333 case BPF_AND | BPF_FETCH:
334 if (isdw) {
335 emit_insn(ctx, amandd, src, t1, t3);
336 } else {
337 emit_insn(ctx, amandw, src, t1, t3);
338 emit_zext_32(ctx, src, true);
339 }
340 break;
341 case BPF_OR | BPF_FETCH:
342 if (isdw) {
343 emit_insn(ctx, amord, src, t1, t3);
344 } else {
345 emit_insn(ctx, amorw, src, t1, t3);
346 emit_zext_32(ctx, src, true);
347 }
348 break;
349 case BPF_XOR | BPF_FETCH:
350 if (isdw) {
351 emit_insn(ctx, amxord, src, t1, t3);
352 } else {
353 emit_insn(ctx, amxorw, src, t1, t3);
354 emit_zext_32(ctx, src, true);
355 }
356 break;
357 /* src = atomic_xchg(dst + off, src); */
358 case BPF_XCHG:
359 if (isdw) {
360 emit_insn(ctx, amswapd, src, t1, t3);
361 } else {
362 emit_insn(ctx, amswapw, src, t1, t3);
363 emit_zext_32(ctx, src, true);
364 }
365 break;
366 /* r0 = atomic_cmpxchg(dst + off, r0, src); */
367 case BPF_CMPXCHG:
368 move_reg(ctx, t2, r0);
369 if (isdw) {
370 emit_insn(ctx, lld, r0, t1, 0);
371 emit_insn(ctx, bne, t2, r0, 4);
372 move_reg(ctx, t3, src);
373 emit_insn(ctx, scd, t3, t1, 0);
374 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
375 } else {
376 emit_insn(ctx, llw, r0, t1, 0);
377 emit_zext_32(ctx, t2, true);
378 emit_zext_32(ctx, r0, true);
379 emit_insn(ctx, bne, t2, r0, 4);
380 move_reg(ctx, t3, src);
381 emit_insn(ctx, scw, t3, t1, 0);
382 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
383 emit_zext_32(ctx, r0, true);
384 }
385 break;
386 }
387 }
388
is_signed_bpf_cond(u8 cond)389 static bool is_signed_bpf_cond(u8 cond)
390 {
391 return cond == BPF_JSGT || cond == BPF_JSLT ||
392 cond == BPF_JSGE || cond == BPF_JSLE;
393 }
394
395 #define BPF_FIXUP_REG_MASK GENMASK(31, 27)
396 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
397
ex_handler_bpf(const struct exception_table_entry * ex,struct pt_regs * regs)398 bool ex_handler_bpf(const struct exception_table_entry *ex,
399 struct pt_regs *regs)
400 {
401 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
402 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
403
404 regs->regs[dst_reg] = 0;
405 regs->csr_era = (unsigned long)&ex->fixup - offset;
406
407 return true;
408 }
409
410 /* For accesses to BTF pointers, add an entry to the exception table */
add_exception_handler(const struct bpf_insn * insn,struct jit_ctx * ctx,int dst_reg)411 static int add_exception_handler(const struct bpf_insn *insn,
412 struct jit_ctx *ctx,
413 int dst_reg)
414 {
415 unsigned long pc;
416 off_t offset;
417 struct exception_table_entry *ex;
418
419 if (!ctx->image || !ctx->prog->aux->extable)
420 return 0;
421
422 if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
423 BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
424 return 0;
425
426 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
427 return -EINVAL;
428
429 ex = &ctx->prog->aux->extable[ctx->num_exentries];
430 pc = (unsigned long)&ctx->image[ctx->idx - 1];
431
432 offset = pc - (long)&ex->insn;
433 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
434 return -ERANGE;
435
436 ex->insn = offset;
437
438 /*
439 * Since the extable follows the program, the fixup offset is always
440 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
441 * to keep things simple, and put the destination register in the upper
442 * bits. We don't need to worry about buildtime or runtime sort
443 * modifying the upper bits because the table is already sorted, and
444 * isn't part of the main exception table.
445 */
446 offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
447 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
448 return -ERANGE;
449
450 ex->type = EX_TYPE_BPF;
451 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
452
453 ctx->num_exentries++;
454
455 return 0;
456 }
457
build_insn(const struct bpf_insn * insn,struct jit_ctx * ctx,bool extra_pass)458 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
459 {
460 u8 tm = -1;
461 u64 func_addr;
462 bool func_addr_fixed, sign_extend;
463 int i = insn - ctx->prog->insnsi;
464 int ret, jmp_offset;
465 const u8 code = insn->code;
466 const u8 cond = BPF_OP(code);
467 const u8 t1 = LOONGARCH_GPR_T1;
468 const u8 t2 = LOONGARCH_GPR_T2;
469 const u8 src = regmap[insn->src_reg];
470 const u8 dst = regmap[insn->dst_reg];
471 const s16 off = insn->off;
472 const s32 imm = insn->imm;
473 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
474
475 switch (code) {
476 /* dst = src */
477 case BPF_ALU | BPF_MOV | BPF_X:
478 case BPF_ALU64 | BPF_MOV | BPF_X:
479 switch (off) {
480 case 0:
481 move_reg(ctx, dst, src);
482 emit_zext_32(ctx, dst, is32);
483 break;
484 case 8:
485 move_reg(ctx, t1, src);
486 emit_insn(ctx, extwb, dst, t1);
487 emit_zext_32(ctx, dst, is32);
488 break;
489 case 16:
490 move_reg(ctx, t1, src);
491 emit_insn(ctx, extwh, dst, t1);
492 emit_zext_32(ctx, dst, is32);
493 break;
494 case 32:
495 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
496 break;
497 }
498 break;
499
500 /* dst = imm */
501 case BPF_ALU | BPF_MOV | BPF_K:
502 case BPF_ALU64 | BPF_MOV | BPF_K:
503 move_imm(ctx, dst, imm, is32);
504 break;
505
506 /* dst = dst + src */
507 case BPF_ALU | BPF_ADD | BPF_X:
508 case BPF_ALU64 | BPF_ADD | BPF_X:
509 emit_insn(ctx, addd, dst, dst, src);
510 emit_zext_32(ctx, dst, is32);
511 break;
512
513 /* dst = dst + imm */
514 case BPF_ALU | BPF_ADD | BPF_K:
515 case BPF_ALU64 | BPF_ADD | BPF_K:
516 if (is_signed_imm12(imm)) {
517 emit_insn(ctx, addid, dst, dst, imm);
518 } else {
519 move_imm(ctx, t1, imm, is32);
520 emit_insn(ctx, addd, dst, dst, t1);
521 }
522 emit_zext_32(ctx, dst, is32);
523 break;
524
525 /* dst = dst - src */
526 case BPF_ALU | BPF_SUB | BPF_X:
527 case BPF_ALU64 | BPF_SUB | BPF_X:
528 emit_insn(ctx, subd, dst, dst, src);
529 emit_zext_32(ctx, dst, is32);
530 break;
531
532 /* dst = dst - imm */
533 case BPF_ALU | BPF_SUB | BPF_K:
534 case BPF_ALU64 | BPF_SUB | BPF_K:
535 if (is_signed_imm12(-imm)) {
536 emit_insn(ctx, addid, dst, dst, -imm);
537 } else {
538 move_imm(ctx, t1, imm, is32);
539 emit_insn(ctx, subd, dst, dst, t1);
540 }
541 emit_zext_32(ctx, dst, is32);
542 break;
543
544 /* dst = dst * src */
545 case BPF_ALU | BPF_MUL | BPF_X:
546 case BPF_ALU64 | BPF_MUL | BPF_X:
547 emit_insn(ctx, muld, dst, dst, src);
548 emit_zext_32(ctx, dst, is32);
549 break;
550
551 /* dst = dst * imm */
552 case BPF_ALU | BPF_MUL | BPF_K:
553 case BPF_ALU64 | BPF_MUL | BPF_K:
554 move_imm(ctx, t1, imm, is32);
555 emit_insn(ctx, muld, dst, dst, t1);
556 emit_zext_32(ctx, dst, is32);
557 break;
558
559 /* dst = dst / src */
560 case BPF_ALU | BPF_DIV | BPF_X:
561 case BPF_ALU64 | BPF_DIV | BPF_X:
562 if (!off) {
563 emit_zext_32(ctx, dst, is32);
564 move_reg(ctx, t1, src);
565 emit_zext_32(ctx, t1, is32);
566 emit_insn(ctx, divdu, dst, dst, t1);
567 emit_zext_32(ctx, dst, is32);
568 } else {
569 emit_sext_32(ctx, dst, is32);
570 move_reg(ctx, t1, src);
571 emit_sext_32(ctx, t1, is32);
572 emit_insn(ctx, divd, dst, dst, t1);
573 emit_sext_32(ctx, dst, is32);
574 }
575 break;
576
577 /* dst = dst / imm */
578 case BPF_ALU | BPF_DIV | BPF_K:
579 case BPF_ALU64 | BPF_DIV | BPF_K:
580 if (!off) {
581 move_imm(ctx, t1, imm, is32);
582 emit_zext_32(ctx, dst, is32);
583 emit_insn(ctx, divdu, dst, dst, t1);
584 emit_zext_32(ctx, dst, is32);
585 } else {
586 move_imm(ctx, t1, imm, false);
587 emit_sext_32(ctx, t1, is32);
588 emit_sext_32(ctx, dst, is32);
589 emit_insn(ctx, divd, dst, dst, t1);
590 emit_sext_32(ctx, dst, is32);
591 }
592 break;
593
594 /* dst = dst % src */
595 case BPF_ALU | BPF_MOD | BPF_X:
596 case BPF_ALU64 | BPF_MOD | BPF_X:
597 if (!off) {
598 emit_zext_32(ctx, dst, is32);
599 move_reg(ctx, t1, src);
600 emit_zext_32(ctx, t1, is32);
601 emit_insn(ctx, moddu, dst, dst, t1);
602 emit_zext_32(ctx, dst, is32);
603 } else {
604 emit_sext_32(ctx, dst, is32);
605 move_reg(ctx, t1, src);
606 emit_sext_32(ctx, t1, is32);
607 emit_insn(ctx, modd, dst, dst, t1);
608 emit_sext_32(ctx, dst, is32);
609 }
610 break;
611
612 /* dst = dst % imm */
613 case BPF_ALU | BPF_MOD | BPF_K:
614 case BPF_ALU64 | BPF_MOD | BPF_K:
615 if (!off) {
616 move_imm(ctx, t1, imm, is32);
617 emit_zext_32(ctx, dst, is32);
618 emit_insn(ctx, moddu, dst, dst, t1);
619 emit_zext_32(ctx, dst, is32);
620 } else {
621 move_imm(ctx, t1, imm, false);
622 emit_sext_32(ctx, t1, is32);
623 emit_sext_32(ctx, dst, is32);
624 emit_insn(ctx, modd, dst, dst, t1);
625 emit_sext_32(ctx, dst, is32);
626 }
627 break;
628
629 /* dst = -dst */
630 case BPF_ALU | BPF_NEG:
631 case BPF_ALU64 | BPF_NEG:
632 move_imm(ctx, t1, imm, is32);
633 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
634 emit_zext_32(ctx, dst, is32);
635 break;
636
637 /* dst = dst & src */
638 case BPF_ALU | BPF_AND | BPF_X:
639 case BPF_ALU64 | BPF_AND | BPF_X:
640 emit_insn(ctx, and, dst, dst, src);
641 emit_zext_32(ctx, dst, is32);
642 break;
643
644 /* dst = dst & imm */
645 case BPF_ALU | BPF_AND | BPF_K:
646 case BPF_ALU64 | BPF_AND | BPF_K:
647 if (is_unsigned_imm12(imm)) {
648 emit_insn(ctx, andi, dst, dst, imm);
649 } else {
650 move_imm(ctx, t1, imm, is32);
651 emit_insn(ctx, and, dst, dst, t1);
652 }
653 emit_zext_32(ctx, dst, is32);
654 break;
655
656 /* dst = dst | src */
657 case BPF_ALU | BPF_OR | BPF_X:
658 case BPF_ALU64 | BPF_OR | BPF_X:
659 emit_insn(ctx, or, dst, dst, src);
660 emit_zext_32(ctx, dst, is32);
661 break;
662
663 /* dst = dst | imm */
664 case BPF_ALU | BPF_OR | BPF_K:
665 case BPF_ALU64 | BPF_OR | BPF_K:
666 if (is_unsigned_imm12(imm)) {
667 emit_insn(ctx, ori, dst, dst, imm);
668 } else {
669 move_imm(ctx, t1, imm, is32);
670 emit_insn(ctx, or, dst, dst, t1);
671 }
672 emit_zext_32(ctx, dst, is32);
673 break;
674
675 /* dst = dst ^ src */
676 case BPF_ALU | BPF_XOR | BPF_X:
677 case BPF_ALU64 | BPF_XOR | BPF_X:
678 emit_insn(ctx, xor, dst, dst, src);
679 emit_zext_32(ctx, dst, is32);
680 break;
681
682 /* dst = dst ^ imm */
683 case BPF_ALU | BPF_XOR | BPF_K:
684 case BPF_ALU64 | BPF_XOR | BPF_K:
685 if (is_unsigned_imm12(imm)) {
686 emit_insn(ctx, xori, dst, dst, imm);
687 } else {
688 move_imm(ctx, t1, imm, is32);
689 emit_insn(ctx, xor, dst, dst, t1);
690 }
691 emit_zext_32(ctx, dst, is32);
692 break;
693
694 /* dst = dst << src (logical) */
695 case BPF_ALU | BPF_LSH | BPF_X:
696 emit_insn(ctx, sllw, dst, dst, src);
697 emit_zext_32(ctx, dst, is32);
698 break;
699
700 case BPF_ALU64 | BPF_LSH | BPF_X:
701 emit_insn(ctx, slld, dst, dst, src);
702 break;
703
704 /* dst = dst << imm (logical) */
705 case BPF_ALU | BPF_LSH | BPF_K:
706 emit_insn(ctx, slliw, dst, dst, imm);
707 emit_zext_32(ctx, dst, is32);
708 break;
709
710 case BPF_ALU64 | BPF_LSH | BPF_K:
711 emit_insn(ctx, sllid, dst, dst, imm);
712 break;
713
714 /* dst = dst >> src (logical) */
715 case BPF_ALU | BPF_RSH | BPF_X:
716 emit_insn(ctx, srlw, dst, dst, src);
717 emit_zext_32(ctx, dst, is32);
718 break;
719
720 case BPF_ALU64 | BPF_RSH | BPF_X:
721 emit_insn(ctx, srld, dst, dst, src);
722 break;
723
724 /* dst = dst >> imm (logical) */
725 case BPF_ALU | BPF_RSH | BPF_K:
726 emit_insn(ctx, srliw, dst, dst, imm);
727 emit_zext_32(ctx, dst, is32);
728 break;
729
730 case BPF_ALU64 | BPF_RSH | BPF_K:
731 emit_insn(ctx, srlid, dst, dst, imm);
732 break;
733
734 /* dst = dst >> src (arithmetic) */
735 case BPF_ALU | BPF_ARSH | BPF_X:
736 emit_insn(ctx, sraw, dst, dst, src);
737 emit_zext_32(ctx, dst, is32);
738 break;
739
740 case BPF_ALU64 | BPF_ARSH | BPF_X:
741 emit_insn(ctx, srad, dst, dst, src);
742 break;
743
744 /* dst = dst >> imm (arithmetic) */
745 case BPF_ALU | BPF_ARSH | BPF_K:
746 emit_insn(ctx, sraiw, dst, dst, imm);
747 emit_zext_32(ctx, dst, is32);
748 break;
749
750 case BPF_ALU64 | BPF_ARSH | BPF_K:
751 emit_insn(ctx, sraid, dst, dst, imm);
752 break;
753
754 /* dst = BSWAP##imm(dst) */
755 case BPF_ALU | BPF_END | BPF_FROM_LE:
756 switch (imm) {
757 case 16:
758 /* zero-extend 16 bits into 64 bits */
759 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
760 break;
761 case 32:
762 /* zero-extend 32 bits into 64 bits */
763 emit_zext_32(ctx, dst, is32);
764 break;
765 case 64:
766 /* do nothing */
767 break;
768 }
769 break;
770
771 case BPF_ALU | BPF_END | BPF_FROM_BE:
772 case BPF_ALU64 | BPF_END | BPF_FROM_LE:
773 switch (imm) {
774 case 16:
775 emit_insn(ctx, revb2h, dst, dst);
776 /* zero-extend 16 bits into 64 bits */
777 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
778 break;
779 case 32:
780 emit_insn(ctx, revb2w, dst, dst);
781 /* clear the upper 32 bits */
782 emit_zext_32(ctx, dst, true);
783 break;
784 case 64:
785 emit_insn(ctx, revbd, dst, dst);
786 break;
787 }
788 break;
789
790 /* PC += off if dst cond src */
791 case BPF_JMP | BPF_JEQ | BPF_X:
792 case BPF_JMP | BPF_JNE | BPF_X:
793 case BPF_JMP | BPF_JGT | BPF_X:
794 case BPF_JMP | BPF_JGE | BPF_X:
795 case BPF_JMP | BPF_JLT | BPF_X:
796 case BPF_JMP | BPF_JLE | BPF_X:
797 case BPF_JMP | BPF_JSGT | BPF_X:
798 case BPF_JMP | BPF_JSGE | BPF_X:
799 case BPF_JMP | BPF_JSLT | BPF_X:
800 case BPF_JMP | BPF_JSLE | BPF_X:
801 case BPF_JMP32 | BPF_JEQ | BPF_X:
802 case BPF_JMP32 | BPF_JNE | BPF_X:
803 case BPF_JMP32 | BPF_JGT | BPF_X:
804 case BPF_JMP32 | BPF_JGE | BPF_X:
805 case BPF_JMP32 | BPF_JLT | BPF_X:
806 case BPF_JMP32 | BPF_JLE | BPF_X:
807 case BPF_JMP32 | BPF_JSGT | BPF_X:
808 case BPF_JMP32 | BPF_JSGE | BPF_X:
809 case BPF_JMP32 | BPF_JSLT | BPF_X:
810 case BPF_JMP32 | BPF_JSLE | BPF_X:
811 jmp_offset = bpf2la_offset(i, off, ctx);
812 move_reg(ctx, t1, dst);
813 move_reg(ctx, t2, src);
814 if (is_signed_bpf_cond(BPF_OP(code))) {
815 emit_sext_32(ctx, t1, is32);
816 emit_sext_32(ctx, t2, is32);
817 } else {
818 emit_zext_32(ctx, t1, is32);
819 emit_zext_32(ctx, t2, is32);
820 }
821 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
822 goto toofar;
823 break;
824
825 /* PC += off if dst cond imm */
826 case BPF_JMP | BPF_JEQ | BPF_K:
827 case BPF_JMP | BPF_JNE | BPF_K:
828 case BPF_JMP | BPF_JGT | BPF_K:
829 case BPF_JMP | BPF_JGE | BPF_K:
830 case BPF_JMP | BPF_JLT | BPF_K:
831 case BPF_JMP | BPF_JLE | BPF_K:
832 case BPF_JMP | BPF_JSGT | BPF_K:
833 case BPF_JMP | BPF_JSGE | BPF_K:
834 case BPF_JMP | BPF_JSLT | BPF_K:
835 case BPF_JMP | BPF_JSLE | BPF_K:
836 case BPF_JMP32 | BPF_JEQ | BPF_K:
837 case BPF_JMP32 | BPF_JNE | BPF_K:
838 case BPF_JMP32 | BPF_JGT | BPF_K:
839 case BPF_JMP32 | BPF_JGE | BPF_K:
840 case BPF_JMP32 | BPF_JLT | BPF_K:
841 case BPF_JMP32 | BPF_JLE | BPF_K:
842 case BPF_JMP32 | BPF_JSGT | BPF_K:
843 case BPF_JMP32 | BPF_JSGE | BPF_K:
844 case BPF_JMP32 | BPF_JSLT | BPF_K:
845 case BPF_JMP32 | BPF_JSLE | BPF_K:
846 jmp_offset = bpf2la_offset(i, off, ctx);
847 if (imm) {
848 move_imm(ctx, t1, imm, false);
849 tm = t1;
850 } else {
851 /* If imm is 0, simply use zero register. */
852 tm = LOONGARCH_GPR_ZERO;
853 }
854 move_reg(ctx, t2, dst);
855 if (is_signed_bpf_cond(BPF_OP(code))) {
856 emit_sext_32(ctx, tm, is32);
857 emit_sext_32(ctx, t2, is32);
858 } else {
859 emit_zext_32(ctx, tm, is32);
860 emit_zext_32(ctx, t2, is32);
861 }
862 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
863 goto toofar;
864 break;
865
866 /* PC += off if dst & src */
867 case BPF_JMP | BPF_JSET | BPF_X:
868 case BPF_JMP32 | BPF_JSET | BPF_X:
869 jmp_offset = bpf2la_offset(i, off, ctx);
870 emit_insn(ctx, and, t1, dst, src);
871 emit_zext_32(ctx, t1, is32);
872 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
873 goto toofar;
874 break;
875
876 /* PC += off if dst & imm */
877 case BPF_JMP | BPF_JSET | BPF_K:
878 case BPF_JMP32 | BPF_JSET | BPF_K:
879 jmp_offset = bpf2la_offset(i, off, ctx);
880 move_imm(ctx, t1, imm, is32);
881 emit_insn(ctx, and, t1, dst, t1);
882 emit_zext_32(ctx, t1, is32);
883 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
884 goto toofar;
885 break;
886
887 /* PC += off */
888 case BPF_JMP | BPF_JA:
889 case BPF_JMP32 | BPF_JA:
890 if (BPF_CLASS(code) == BPF_JMP)
891 jmp_offset = bpf2la_offset(i, off, ctx);
892 else
893 jmp_offset = bpf2la_offset(i, imm, ctx);
894 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
895 goto toofar;
896 break;
897
898 /* function call */
899 case BPF_JMP | BPF_CALL:
900 mark_call(ctx);
901 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
902 &func_addr, &func_addr_fixed);
903 if (ret < 0)
904 return ret;
905
906 move_addr(ctx, t1, func_addr);
907 emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0);
908 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
909 break;
910
911 /* tail call */
912 case BPF_JMP | BPF_TAIL_CALL:
913 mark_tail_call(ctx);
914 if (emit_bpf_tail_call(ctx) < 0)
915 return -EINVAL;
916 break;
917
918 /* function return */
919 case BPF_JMP | BPF_EXIT:
920 if (i == ctx->prog->len - 1)
921 break;
922
923 jmp_offset = epilogue_offset(ctx);
924 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
925 goto toofar;
926 break;
927
928 /* dst = imm64 */
929 case BPF_LD | BPF_IMM | BPF_DW:
930 {
931 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
932
933 move_imm(ctx, dst, imm64, is32);
934 return 1;
935 }
936
937 /* dst = *(size *)(src + off) */
938 case BPF_LDX | BPF_MEM | BPF_B:
939 case BPF_LDX | BPF_MEM | BPF_H:
940 case BPF_LDX | BPF_MEM | BPF_W:
941 case BPF_LDX | BPF_MEM | BPF_DW:
942 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
943 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
944 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
945 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
946 /* dst_reg = (s64)*(signed size *)(src_reg + off) */
947 case BPF_LDX | BPF_MEMSX | BPF_B:
948 case BPF_LDX | BPF_MEMSX | BPF_H:
949 case BPF_LDX | BPF_MEMSX | BPF_W:
950 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
951 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
952 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
953 sign_extend = BPF_MODE(insn->code) == BPF_MEMSX ||
954 BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
955 switch (BPF_SIZE(code)) {
956 case BPF_B:
957 if (is_signed_imm12(off)) {
958 if (sign_extend)
959 emit_insn(ctx, ldb, dst, src, off);
960 else
961 emit_insn(ctx, ldbu, dst, src, off);
962 } else {
963 move_imm(ctx, t1, off, is32);
964 if (sign_extend)
965 emit_insn(ctx, ldxb, dst, src, t1);
966 else
967 emit_insn(ctx, ldxbu, dst, src, t1);
968 }
969 break;
970 case BPF_H:
971 if (is_signed_imm12(off)) {
972 if (sign_extend)
973 emit_insn(ctx, ldh, dst, src, off);
974 else
975 emit_insn(ctx, ldhu, dst, src, off);
976 } else {
977 move_imm(ctx, t1, off, is32);
978 if (sign_extend)
979 emit_insn(ctx, ldxh, dst, src, t1);
980 else
981 emit_insn(ctx, ldxhu, dst, src, t1);
982 }
983 break;
984 case BPF_W:
985 if (is_signed_imm12(off)) {
986 if (sign_extend)
987 emit_insn(ctx, ldw, dst, src, off);
988 else
989 emit_insn(ctx, ldwu, dst, src, off);
990 } else {
991 move_imm(ctx, t1, off, is32);
992 if (sign_extend)
993 emit_insn(ctx, ldxw, dst, src, t1);
994 else
995 emit_insn(ctx, ldxwu, dst, src, t1);
996 }
997 break;
998 case BPF_DW:
999 move_imm(ctx, t1, off, is32);
1000 emit_insn(ctx, ldxd, dst, src, t1);
1001 break;
1002 }
1003
1004 ret = add_exception_handler(insn, ctx, dst);
1005 if (ret)
1006 return ret;
1007 break;
1008
1009 /* *(size *)(dst + off) = imm */
1010 case BPF_ST | BPF_MEM | BPF_B:
1011 case BPF_ST | BPF_MEM | BPF_H:
1012 case BPF_ST | BPF_MEM | BPF_W:
1013 case BPF_ST | BPF_MEM | BPF_DW:
1014 switch (BPF_SIZE(code)) {
1015 case BPF_B:
1016 move_imm(ctx, t1, imm, is32);
1017 if (is_signed_imm12(off)) {
1018 emit_insn(ctx, stb, t1, dst, off);
1019 } else {
1020 move_imm(ctx, t2, off, is32);
1021 emit_insn(ctx, stxb, t1, dst, t2);
1022 }
1023 break;
1024 case BPF_H:
1025 move_imm(ctx, t1, imm, is32);
1026 if (is_signed_imm12(off)) {
1027 emit_insn(ctx, sth, t1, dst, off);
1028 } else {
1029 move_imm(ctx, t2, off, is32);
1030 emit_insn(ctx, stxh, t1, dst, t2);
1031 }
1032 break;
1033 case BPF_W:
1034 move_imm(ctx, t1, imm, is32);
1035 if (is_signed_imm12(off)) {
1036 emit_insn(ctx, stw, t1, dst, off);
1037 } else if (is_signed_imm14(off)) {
1038 emit_insn(ctx, stptrw, t1, dst, off);
1039 } else {
1040 move_imm(ctx, t2, off, is32);
1041 emit_insn(ctx, stxw, t1, dst, t2);
1042 }
1043 break;
1044 case BPF_DW:
1045 move_imm(ctx, t1, imm, is32);
1046 if (is_signed_imm12(off)) {
1047 emit_insn(ctx, std, t1, dst, off);
1048 } else if (is_signed_imm14(off)) {
1049 emit_insn(ctx, stptrd, t1, dst, off);
1050 } else {
1051 move_imm(ctx, t2, off, is32);
1052 emit_insn(ctx, stxd, t1, dst, t2);
1053 }
1054 break;
1055 }
1056 break;
1057
1058 /* *(size *)(dst + off) = src */
1059 case BPF_STX | BPF_MEM | BPF_B:
1060 case BPF_STX | BPF_MEM | BPF_H:
1061 case BPF_STX | BPF_MEM | BPF_W:
1062 case BPF_STX | BPF_MEM | BPF_DW:
1063 switch (BPF_SIZE(code)) {
1064 case BPF_B:
1065 if (is_signed_imm12(off)) {
1066 emit_insn(ctx, stb, src, dst, off);
1067 } else {
1068 move_imm(ctx, t1, off, is32);
1069 emit_insn(ctx, stxb, src, dst, t1);
1070 }
1071 break;
1072 case BPF_H:
1073 if (is_signed_imm12(off)) {
1074 emit_insn(ctx, sth, src, dst, off);
1075 } else {
1076 move_imm(ctx, t1, off, is32);
1077 emit_insn(ctx, stxh, src, dst, t1);
1078 }
1079 break;
1080 case BPF_W:
1081 if (is_signed_imm12(off)) {
1082 emit_insn(ctx, stw, src, dst, off);
1083 } else if (is_signed_imm14(off)) {
1084 emit_insn(ctx, stptrw, src, dst, off);
1085 } else {
1086 move_imm(ctx, t1, off, is32);
1087 emit_insn(ctx, stxw, src, dst, t1);
1088 }
1089 break;
1090 case BPF_DW:
1091 if (is_signed_imm12(off)) {
1092 emit_insn(ctx, std, src, dst, off);
1093 } else if (is_signed_imm14(off)) {
1094 emit_insn(ctx, stptrd, src, dst, off);
1095 } else {
1096 move_imm(ctx, t1, off, is32);
1097 emit_insn(ctx, stxd, src, dst, t1);
1098 }
1099 break;
1100 }
1101 break;
1102
1103 case BPF_STX | BPF_ATOMIC | BPF_W:
1104 case BPF_STX | BPF_ATOMIC | BPF_DW:
1105 emit_atomic(insn, ctx);
1106 break;
1107
1108 /* Speculation barrier */
1109 case BPF_ST | BPF_NOSPEC:
1110 break;
1111
1112 default:
1113 pr_err("bpf_jit: unknown opcode %02x\n", code);
1114 return -EINVAL;
1115 }
1116
1117 return 0;
1118
1119 toofar:
1120 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1121 return -E2BIG;
1122 }
1123
build_body(struct jit_ctx * ctx,bool extra_pass)1124 static int build_body(struct jit_ctx *ctx, bool extra_pass)
1125 {
1126 int i;
1127 const struct bpf_prog *prog = ctx->prog;
1128
1129 for (i = 0; i < prog->len; i++) {
1130 const struct bpf_insn *insn = &prog->insnsi[i];
1131 int ret;
1132
1133 if (ctx->image == NULL)
1134 ctx->offset[i] = ctx->idx;
1135
1136 ret = build_insn(insn, ctx, extra_pass);
1137 if (ret > 0) {
1138 i++;
1139 if (ctx->image == NULL)
1140 ctx->offset[i] = ctx->idx;
1141 continue;
1142 }
1143 if (ret)
1144 return ret;
1145 }
1146
1147 if (ctx->image == NULL)
1148 ctx->offset[i] = ctx->idx;
1149
1150 return 0;
1151 }
1152
1153 /* Fill space with break instructions */
jit_fill_hole(void * area,unsigned int size)1154 static void jit_fill_hole(void *area, unsigned int size)
1155 {
1156 u32 *ptr;
1157
1158 /* We are guaranteed to have aligned memory */
1159 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1160 *ptr++ = INSN_BREAK;
1161 }
1162
validate_code(struct jit_ctx * ctx)1163 static int validate_code(struct jit_ctx *ctx)
1164 {
1165 int i;
1166 union loongarch_instruction insn;
1167
1168 for (i = 0; i < ctx->idx; i++) {
1169 insn = ctx->image[i];
1170 /* Check INSN_BREAK */
1171 if (insn.word == INSN_BREAK)
1172 return -1;
1173 }
1174
1175 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1176 return -1;
1177
1178 return 0;
1179 }
1180
bpf_int_jit_compile(struct bpf_prog * prog)1181 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1182 {
1183 bool tmp_blinded = false, extra_pass = false;
1184 u8 *image_ptr;
1185 int image_size, prog_size, extable_size;
1186 struct jit_ctx ctx;
1187 struct jit_data *jit_data;
1188 struct bpf_binary_header *header;
1189 struct bpf_prog *tmp, *orig_prog = prog;
1190
1191 /*
1192 * If BPF JIT was not enabled then we must fall back to
1193 * the interpreter.
1194 */
1195 if (!prog->jit_requested)
1196 return orig_prog;
1197
1198 tmp = bpf_jit_blind_constants(prog);
1199 /*
1200 * If blinding was requested and we failed during blinding,
1201 * we must fall back to the interpreter. Otherwise, we save
1202 * the new JITed code.
1203 */
1204 if (IS_ERR(tmp))
1205 return orig_prog;
1206
1207 if (tmp != prog) {
1208 tmp_blinded = true;
1209 prog = tmp;
1210 }
1211
1212 jit_data = prog->aux->jit_data;
1213 if (!jit_data) {
1214 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1215 if (!jit_data) {
1216 prog = orig_prog;
1217 goto out;
1218 }
1219 prog->aux->jit_data = jit_data;
1220 }
1221 if (jit_data->ctx.offset) {
1222 ctx = jit_data->ctx;
1223 image_ptr = jit_data->image;
1224 header = jit_data->header;
1225 extra_pass = true;
1226 prog_size = sizeof(u32) * ctx.idx;
1227 goto skip_init_ctx;
1228 }
1229
1230 memset(&ctx, 0, sizeof(ctx));
1231 ctx.prog = prog;
1232
1233 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1234 if (ctx.offset == NULL) {
1235 prog = orig_prog;
1236 goto out_offset;
1237 }
1238
1239 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1240 build_prologue(&ctx);
1241 if (build_body(&ctx, extra_pass)) {
1242 prog = orig_prog;
1243 goto out_offset;
1244 }
1245 ctx.epilogue_offset = ctx.idx;
1246 build_epilogue(&ctx);
1247
1248 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
1249
1250 /* Now we know the actual image size.
1251 * As each LoongArch instruction is of length 32bit,
1252 * we are translating number of JITed intructions into
1253 * the size required to store these JITed code.
1254 */
1255 prog_size = sizeof(u32) * ctx.idx;
1256 image_size = prog_size + extable_size;
1257 /* Now we know the size of the structure to make */
1258 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1259 sizeof(u32), jit_fill_hole);
1260 if (header == NULL) {
1261 prog = orig_prog;
1262 goto out_offset;
1263 }
1264
1265 /* 2. Now, the actual pass to generate final JIT code */
1266 ctx.image = (union loongarch_instruction *)image_ptr;
1267 if (extable_size)
1268 prog->aux->extable = (void *)image_ptr + prog_size;
1269
1270 skip_init_ctx:
1271 ctx.idx = 0;
1272 ctx.num_exentries = 0;
1273
1274 build_prologue(&ctx);
1275 if (build_body(&ctx, extra_pass)) {
1276 bpf_jit_binary_free(header);
1277 prog = orig_prog;
1278 goto out_offset;
1279 }
1280 build_epilogue(&ctx);
1281
1282 /* 3. Extra pass to validate JITed code */
1283 if (validate_code(&ctx)) {
1284 bpf_jit_binary_free(header);
1285 prog = orig_prog;
1286 goto out_offset;
1287 }
1288
1289 /* And we're done */
1290 if (bpf_jit_enable > 1)
1291 bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1292
1293 /* Update the icache */
1294 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
1295
1296 if (!prog->is_func || extra_pass) {
1297 int err;
1298
1299 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1300 pr_err_once("multi-func JIT bug %d != %d\n",
1301 ctx.idx, jit_data->ctx.idx);
1302 goto out_free;
1303 }
1304 err = bpf_jit_binary_lock_ro(header);
1305 if (err) {
1306 pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
1307 err);
1308 goto out_free;
1309 }
1310 } else {
1311 jit_data->ctx = ctx;
1312 jit_data->image = image_ptr;
1313 jit_data->header = header;
1314 }
1315 prog->jited = 1;
1316 prog->jited_len = prog_size;
1317 prog->bpf_func = (void *)ctx.image;
1318
1319 if (!prog->is_func || extra_pass) {
1320 int i;
1321
1322 /* offset[prog->len] is the size of program */
1323 for (i = 0; i <= prog->len; i++)
1324 ctx.offset[i] *= LOONGARCH_INSN_SIZE;
1325 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1326
1327 out_offset:
1328 kvfree(ctx.offset);
1329 kfree(jit_data);
1330 prog->aux->jit_data = NULL;
1331 }
1332
1333 out:
1334 if (tmp_blinded)
1335 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
1336
1337 out_offset = -1;
1338
1339 return prog;
1340
1341 out_free:
1342 bpf_jit_binary_free(header);
1343 prog->bpf_func = NULL;
1344 prog->jited = 0;
1345 prog->jited_len = 0;
1346 goto out_offset;
1347 }
1348
1349 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bpf_jit_supports_subprog_tailcalls(void)1350 bool bpf_jit_supports_subprog_tailcalls(void)
1351 {
1352 return true;
1353 }
1354