1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * BPF JIT compiler for LoongArch
4 *
5 * Copyright (C) 2022 Loongson Technology Corporation Limited
6 */
7 #include <linux/memory.h>
8 #include <asm/asm-offsets.h>
9 #include "bpf_jit.h"
10
11 /* DBAR hint for LL/SC completion ordering, see __WEAK_LLSC_MB */
12 #define DBAR_LLSC_MB 0x700
13
14 #define LOONGARCH_MAX_REG_ARGS 8
15
16 #define LOONGARCH_SAVE_RA_NINSNS 1
17 #define LOONGARCH_LONG_JUMP_NINSNS 5
18 #define LOONGARCH_TCC_SLOT_NINSNS 1
19
20 #define LOONGARCH_PROLOGUE_SKIP_INSNS \
21 (LOONGARCH_SAVE_RA_NINSNS + LOONGARCH_LONG_JUMP_NINSNS + LOONGARCH_TCC_SLOT_NINSNS)
22
23 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
24
25 #define LOONGARCH_FENTRY_NINSNS 2
26 #define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
27 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
28
29 #define REG_TCC LOONGARCH_GPR_A6
30 #define REG_ARENA LOONGARCH_GPR_S6 /* For storing arena_vm_start */
31 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
32
33 static const int regmap[] = {
34 /* return value from in-kernel function, and exit value for eBPF program */
35 [BPF_REG_0] = LOONGARCH_GPR_A5,
36 /* arguments from eBPF program to in-kernel function */
37 [BPF_REG_1] = LOONGARCH_GPR_A0,
38 [BPF_REG_2] = LOONGARCH_GPR_A1,
39 [BPF_REG_3] = LOONGARCH_GPR_A2,
40 [BPF_REG_4] = LOONGARCH_GPR_A3,
41 [BPF_REG_5] = LOONGARCH_GPR_A4,
42 /* callee saved registers that in-kernel function will preserve */
43 [BPF_REG_6] = LOONGARCH_GPR_S0,
44 [BPF_REG_7] = LOONGARCH_GPR_S1,
45 [BPF_REG_8] = LOONGARCH_GPR_S2,
46 [BPF_REG_9] = LOONGARCH_GPR_S3,
47 /* read-only frame pointer to access stack */
48 [BPF_REG_FP] = LOONGARCH_GPR_S4,
49 /* temporary register for blinding constants */
50 [BPF_REG_AX] = LOONGARCH_GPR_T0,
51 };
52
prepare_bpf_tail_call_cnt(struct jit_ctx * ctx,int * store_offset)53 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
54 {
55 const struct bpf_prog *prog = ctx->prog;
56 const bool is_main_prog = !bpf_is_subprog(prog);
57
58 *store_offset -= sizeof(long);
59 if (is_main_prog) {
60 /* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */
61 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
62
63 /* Compute the absolute pointer to the local 'tcc' slot */
64 emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset);
65
66 /*
67 * Branchless classification and blending:
68 * Combine interleaved inputs between a scalar count (0 to 33)
69 * and a kernel pointer address without runtime branching.
70 */
71 emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1);
72 emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8);
73 emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8);
74 emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7);
75 } else {
76 /* Subprograms: backup the verified TCC pointer inherited via REG_TCC */
77 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
78 }
79
80 /* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */
81 *store_offset -= sizeof(long);
82 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
83 }
84
85 /*
86 * eBPF prog stack layout:
87 *
88 * high
89 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP
90 * | $ra |
91 * +-------------------------+
92 * | $fp |
93 * +-------------------------+
94 * | $s0 |
95 * +-------------------------+
96 * | $s1 |
97 * +-------------------------+
98 * | $s2 |
99 * +-------------------------+
100 * | $s3 |
101 * +-------------------------+
102 * | $s4 |
103 * +-------------------------+
104 * | $s5 |
105 * +-------------------------+
106 * | tcc |
107 * +-------------------------+
108 * | tcc_ptr |
109 * +-------------------------+
110 * | arena |
111 * | (optional) |
112 * +-------------------------+ <--BPF_REG_FP
113 * | prog->aux->stack_depth |
114 * | (optional) |
115 * current $sp -------------> +-------------------------+
116 * low
117 */
build_prologue(struct jit_ctx * ctx)118 static void build_prologue(struct jit_ctx *ctx)
119 {
120 int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
121 const struct bpf_prog *prog = ctx->prog;
122 const bool is_main_prog = !bpf_is_subprog(prog);
123
124 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
125
126 /* To store ra, fp, s0, s1, s2, s3, s4, s5 */
127 stack_adjust += sizeof(long) * 8;
128
129 /* To store tcc and tcc_ptr */
130 stack_adjust += sizeof(long) * 2;
131
132 if (ctx->arena_vm_start)
133 stack_adjust += sizeof(long);
134
135 stack_adjust = round_up(stack_adjust, 16);
136 stack_adjust += bpf_stack_adjust;
137
138 /*
139 * Save the original return address to a temporary register to prevent
140 * it from being overwritten, then reserve space for the long jump and
141 * fentry trampoline slot for dynamically patching by ftrace at runtime.
142 * These instructions are bypassed during a tail call invocation.
143 */
144 move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);
145 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
146 emit_insn(ctx, nop);
147
148 /*
149 * First instruction initializes the tail call count (TCC)
150 * register to zero. On tail call we skip this instruction,
151 * and the TCC is passed in REG_TCC from the caller.
152 */
153 if (is_main_prog)
154 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
155
156 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
157
158 store_offset = stack_adjust - sizeof(long);
159 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
160
161 store_offset -= sizeof(long);
162 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
163
164 store_offset -= sizeof(long);
165 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
166
167 store_offset -= sizeof(long);
168 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
169
170 store_offset -= sizeof(long);
171 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
172
173 store_offset -= sizeof(long);
174 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
175
176 store_offset -= sizeof(long);
177 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
178
179 store_offset -= sizeof(long);
180 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
181
182 prepare_bpf_tail_call_cnt(ctx, &store_offset);
183
184 if (ctx->arena_vm_start) {
185 store_offset -= sizeof(long);
186 emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset);
187 }
188
189 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
190
191 if (bpf_stack_adjust)
192 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
193
194 ctx->stack_size = stack_adjust;
195
196 if (ctx->arena_vm_start)
197 move_imm(ctx, REG_ARENA, ctx->arena_vm_start, false);
198 }
199
__build_epilogue(struct jit_ctx * ctx,bool is_tail_call)200 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
201 {
202 int stack_adjust = ctx->stack_size;
203 int load_offset;
204
205 load_offset = stack_adjust - sizeof(long);
206 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
207
208 load_offset -= sizeof(long);
209 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
210
211 load_offset -= sizeof(long);
212 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
213
214 load_offset -= sizeof(long);
215 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
216
217 load_offset -= sizeof(long);
218 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
219
220 load_offset -= sizeof(long);
221 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
222
223 load_offset -= sizeof(long);
224 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
225
226 load_offset -= sizeof(long);
227 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
228
229 /* Only restore the TCC state into REG_TCC from the higher slot */
230 load_offset -= sizeof(long);
231 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
232
233 /* Skip the unused local 'tcc_ptr' slot to align with arena */
234 load_offset -= sizeof(long);
235
236 if (ctx->arena_vm_start) {
237 load_offset -= sizeof(long);
238 emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset);
239 }
240
241 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
242
243 if (!is_tail_call) {
244 /* Set return value */
245 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0);
246 /* Return to the caller */
247 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
248 } else {
249 /*
250 * Tail call to the next BPF program, passing offset in number
251 * of instructions to jirl to bypass the initial setup slots.
252 */
253 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO,
254 LOONGARCH_GPR_T3, LOONGARCH_PROLOGUE_SKIP_INSNS);
255 }
256 }
257
build_epilogue(struct jit_ctx * ctx)258 static void build_epilogue(struct jit_ctx *ctx)
259 {
260 __build_epilogue(ctx, false);
261 }
262
bpf_jit_supports_kfunc_call(void)263 bool bpf_jit_supports_kfunc_call(void)
264 {
265 return true;
266 }
267
bpf_jit_supports_far_kfunc_call(void)268 bool bpf_jit_supports_far_kfunc_call(void)
269 {
270 return true;
271 }
272
emit_bpf_tail_call(struct jit_ctx * ctx,int insn)273 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
274 {
275 int off, jmp_offset;
276 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
277 u8 a1 = LOONGARCH_GPR_A1;
278 u8 a2 = LOONGARCH_GPR_A2;
279 u8 t1 = LOONGARCH_GPR_T1;
280 u8 t2 = LOONGARCH_GPR_T2;
281 u8 t3 = LOONGARCH_GPR_T3;
282
283 /*
284 * a0: &ctx
285 * a1: &array
286 * a2: index
287 *
288 * if (index >= array->map.max_entries)
289 * goto out;
290 */
291 emit_zext_32(ctx, a2, true);
292
293 off = offsetof(struct bpf_array, map.max_entries);
294 emit_insn(ctx, ldwu, t1, a1, off);
295 /* bgeu $a2, $t1, jmp_offset */
296 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
297 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0)
298 goto toofar;
299
300 /*
301 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
302 * goto out;
303 */
304 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
305 emit_insn(ctx, ldd, t3, REG_TCC, 0);
306 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
307 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
308 if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0)
309 goto toofar;
310
311 emit_insn(ctx, addid, t3, t3, 1);
312
313 /*
314 * prog = array->ptrs[index];
315 * if (!prog)
316 * goto out;
317 */
318 emit_insn(ctx, alsld, t2, a2, a1, 2);
319 off = offsetof(struct bpf_array, ptrs);
320 emit_insn(ctx, ldd, t2, t2, off);
321 /* beq $t2, $zero, jmp_offset */
322 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0;
323 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
324 goto toofar;
325
326 emit_insn(ctx, std, t3, REG_TCC, 0);
327
328 /* goto *(prog->bpf_func + 4); */
329 off = offsetof(struct bpf_prog, bpf_func);
330 emit_insn(ctx, ldd, t3, t2, off);
331 __build_epilogue(ctx, true);
332
333 return 0;
334
335 toofar:
336 pr_info_once("tail_call: jump too far\n");
337 return -1;
338 }
339
emit_store_stack_imm64(struct jit_ctx * ctx,int reg,int stack_off,u64 imm64)340 static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64)
341 {
342 move_imm(ctx, reg, imm64, false);
343 emit_insn(ctx, std, reg, LOONGARCH_GPR_FP, stack_off);
344 }
345
emit_atomic_rmw(const struct bpf_insn * insn,struct jit_ctx * ctx)346 static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
347 {
348 const u8 t1 = LOONGARCH_GPR_T1;
349 const u8 t2 = LOONGARCH_GPR_T2;
350 const u8 t3 = LOONGARCH_GPR_T3;
351 const u8 r0 = regmap[BPF_REG_0];
352 const u8 src = regmap[insn->src_reg];
353 const u8 dst = regmap[insn->dst_reg];
354 const s16 off = insn->off;
355 const s32 imm = insn->imm;
356 const bool isdw = BPF_SIZE(insn->code) == BPF_DW;
357
358 move_imm(ctx, t1, off, false);
359 emit_insn(ctx, addd, t1, dst, t1);
360 move_reg(ctx, t3, src);
361
362 switch (imm) {
363 /* lock *(size *)(dst + off) <op>= src */
364 case BPF_ADD:
365 switch (BPF_SIZE(insn->code)) {
366 case BPF_B:
367 if (!cpu_has_lam_bh) {
368 pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
369 return -EINVAL;
370 }
371 emit_insn(ctx, amaddb, t2, t1, src);
372 break;
373 case BPF_H:
374 if (!cpu_has_lam_bh) {
375 pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
376 return -EINVAL;
377 }
378 emit_insn(ctx, amaddh, t2, t1, src);
379 break;
380 case BPF_W:
381 emit_insn(ctx, amaddw, t2, t1, src);
382 break;
383 case BPF_DW:
384 emit_insn(ctx, amaddd, t2, t1, src);
385 break;
386 }
387 break;
388 case BPF_AND:
389 if (isdw)
390 emit_insn(ctx, amandd, t2, t1, src);
391 else
392 emit_insn(ctx, amandw, t2, t1, src);
393 break;
394 case BPF_OR:
395 if (isdw)
396 emit_insn(ctx, amord, t2, t1, src);
397 else
398 emit_insn(ctx, amorw, t2, t1, src);
399 break;
400 case BPF_XOR:
401 if (isdw)
402 emit_insn(ctx, amxord, t2, t1, src);
403 else
404 emit_insn(ctx, amxorw, t2, t1, src);
405 break;
406 /* src = atomic_fetch_<op>(dst + off, src) */
407 case BPF_ADD | BPF_FETCH:
408 switch (BPF_SIZE(insn->code)) {
409 case BPF_B:
410 if (!cpu_has_lam_bh) {
411 pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
412 return -EINVAL;
413 }
414 emit_insn(ctx, amadddbb, src, t1, t3);
415 emit_zext_32(ctx, src, true);
416 break;
417 case BPF_H:
418 if (!cpu_has_lam_bh) {
419 pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
420 return -EINVAL;
421 }
422 emit_insn(ctx, amadddbh, src, t1, t3);
423 emit_zext_32(ctx, src, true);
424 break;
425 case BPF_W:
426 emit_insn(ctx, amadddbw, src, t1, t3);
427 emit_zext_32(ctx, src, true);
428 break;
429 case BPF_DW:
430 emit_insn(ctx, amadddbd, src, t1, t3);
431 break;
432 }
433 break;
434 case BPF_AND | BPF_FETCH:
435 if (isdw) {
436 emit_insn(ctx, amanddbd, src, t1, t3);
437 } else {
438 emit_insn(ctx, amanddbw, src, t1, t3);
439 emit_zext_32(ctx, src, true);
440 }
441 break;
442 case BPF_OR | BPF_FETCH:
443 if (isdw) {
444 emit_insn(ctx, amordbd, src, t1, t3);
445 } else {
446 emit_insn(ctx, amordbw, src, t1, t3);
447 emit_zext_32(ctx, src, true);
448 }
449 break;
450 case BPF_XOR | BPF_FETCH:
451 if (isdw) {
452 emit_insn(ctx, amxordbd, src, t1, t3);
453 } else {
454 emit_insn(ctx, amxordbw, src, t1, t3);
455 emit_zext_32(ctx, src, true);
456 }
457 break;
458 /* src = atomic_xchg(dst + off, src); */
459 case BPF_XCHG:
460 switch (BPF_SIZE(insn->code)) {
461 case BPF_B:
462 if (!cpu_has_lam_bh) {
463 pr_err_once("bpf-jit: amswap.b instruction is not supported\n");
464 return -EINVAL;
465 }
466 emit_insn(ctx, amswapdbb, src, t1, t3);
467 emit_zext_32(ctx, src, true);
468 break;
469 case BPF_H:
470 if (!cpu_has_lam_bh) {
471 pr_err_once("bpf-jit: amswap.h instruction is not supported\n");
472 return -EINVAL;
473 }
474 emit_insn(ctx, amswapdbh, src, t1, t3);
475 emit_zext_32(ctx, src, true);
476 break;
477 case BPF_W:
478 emit_insn(ctx, amswapdbw, src, t1, t3);
479 emit_zext_32(ctx, src, true);
480 break;
481 case BPF_DW:
482 emit_insn(ctx, amswapdbd, src, t1, t3);
483 break;
484 }
485 break;
486 /* r0 = atomic_cmpxchg(dst + off, r0, src); */
487 case BPF_CMPXCHG:
488 move_reg(ctx, t2, r0);
489 if (isdw) {
490 emit_insn(ctx, lld, r0, t1, 0);
491 emit_insn(ctx, bne, t2, r0, 4);
492 move_reg(ctx, t3, src);
493 emit_insn(ctx, scd, t3, t1, 0);
494 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
495 } else {
496 emit_insn(ctx, llw, r0, t1, 0);
497 emit_zext_32(ctx, t2, true);
498 emit_zext_32(ctx, r0, true);
499 emit_insn(ctx, bne, t2, r0, 4);
500 move_reg(ctx, t3, src);
501 emit_insn(ctx, scw, t3, t1, 0);
502 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
503 emit_zext_32(ctx, r0, true);
504 }
505 emit_insn(ctx, dbar, DBAR_LLSC_MB);
506 break;
507 default:
508 pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm);
509 return -EINVAL;
510 }
511
512 return 0;
513 }
514
emit_atomic_ld_st(const struct bpf_insn * insn,struct jit_ctx * ctx)515 static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
516 {
517 const u8 t1 = LOONGARCH_GPR_T1;
518 const u8 src = regmap[insn->src_reg];
519 const u8 dst = regmap[insn->dst_reg];
520 const s16 off = insn->off;
521 const s32 imm = insn->imm;
522
523 switch (imm) {
524 /* dst_reg = load_acquire(src_reg + off16) */
525 case BPF_LOAD_ACQ:
526 switch (BPF_SIZE(insn->code)) {
527 case BPF_B:
528 if (is_signed_imm12(off)) {
529 emit_insn(ctx, ldbu, dst, src, off);
530 } else {
531 move_imm(ctx, t1, off, false);
532 emit_insn(ctx, ldxbu, dst, src, t1);
533 }
534 break;
535 case BPF_H:
536 if (is_signed_imm12(off)) {
537 emit_insn(ctx, ldhu, dst, src, off);
538 } else {
539 move_imm(ctx, t1, off, false);
540 emit_insn(ctx, ldxhu, dst, src, t1);
541 }
542 break;
543 case BPF_W:
544 if (is_signed_imm12(off)) {
545 emit_insn(ctx, ldwu, dst, src, off);
546 } else {
547 move_imm(ctx, t1, off, false);
548 emit_insn(ctx, ldxwu, dst, src, t1);
549 }
550 break;
551 case BPF_DW:
552 if (is_signed_imm12(off)) {
553 emit_insn(ctx, ldd, dst, src, off);
554 } else {
555 move_imm(ctx, t1, off, false);
556 emit_insn(ctx, ldxd, dst, src, t1);
557 }
558 break;
559 }
560 emit_insn(ctx, dbar, 0b10100);
561 break;
562 /* store_release(dst_reg + off16, src_reg) */
563 case BPF_STORE_REL:
564 emit_insn(ctx, dbar, 0b10010);
565 switch (BPF_SIZE(insn->code)) {
566 case BPF_B:
567 if (is_signed_imm12(off)) {
568 emit_insn(ctx, stb, src, dst, off);
569 } else {
570 move_imm(ctx, t1, off, false);
571 emit_insn(ctx, stxb, src, dst, t1);
572 }
573 break;
574 case BPF_H:
575 if (is_signed_imm12(off)) {
576 emit_insn(ctx, sth, src, dst, off);
577 } else {
578 move_imm(ctx, t1, off, false);
579 emit_insn(ctx, stxh, src, dst, t1);
580 }
581 break;
582 case BPF_W:
583 if (is_signed_imm12(off)) {
584 emit_insn(ctx, stw, src, dst, off);
585 } else {
586 move_imm(ctx, t1, off, false);
587 emit_insn(ctx, stxw, src, dst, t1);
588 }
589 break;
590 case BPF_DW:
591 if (is_signed_imm12(off)) {
592 emit_insn(ctx, std, src, dst, off);
593 } else {
594 move_imm(ctx, t1, off, false);
595 emit_insn(ctx, stxd, src, dst, t1);
596 }
597 break;
598 }
599 break;
600 default:
601 pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm);
602 return -EINVAL;
603 }
604
605 return 0;
606 }
607
is_signed_bpf_cond(u8 cond)608 static bool is_signed_bpf_cond(u8 cond)
609 {
610 return cond == BPF_JSGT || cond == BPF_JSLT ||
611 cond == BPF_JSGE || cond == BPF_JSLE;
612 }
613
614 #define BPF_FIXUP_REG_MASK GENMASK(31, 27)
615 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
616 #define REG_DONT_CLEAR_MARKER 0
617
ex_handler_bpf(const struct exception_table_entry * ex,struct pt_regs * regs)618 bool ex_handler_bpf(const struct exception_table_entry *ex,
619 struct pt_regs *regs)
620 {
621 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
622 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
623
624 if (dst_reg != REG_DONT_CLEAR_MARKER)
625 regs->regs[dst_reg] = 0;
626 regs->csr_era = (unsigned long)&ex->fixup - offset;
627
628 return true;
629 }
630
631 /* For accesses to BTF pointers, add an entry to the exception table */
add_exception_handler(const struct bpf_insn * insn,struct jit_ctx * ctx,int dst_reg)632 static int add_exception_handler(const struct bpf_insn *insn,
633 struct jit_ctx *ctx,
634 int dst_reg)
635 {
636 unsigned long pc;
637 off_t ins_offset, fixup_offset;
638 struct exception_table_entry *ex;
639
640 if (!ctx->image || !ctx->ro_image || !ctx->prog->aux->extable)
641 return 0;
642
643 if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
644 BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
645 BPF_MODE(insn->code) != BPF_PROBE_MEM32)
646 return 0;
647
648 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries))
649 return -EINVAL;
650
651 ex = &ctx->prog->aux->extable[ctx->num_exentries];
652 pc = (unsigned long)&ctx->ro_image[ctx->idx - 1];
653
654 /*
655 * This is the relative offset of the instruction that may fault from
656 * the exception table itself. This will be written to the exception
657 * table and if this instruction faults, the destination register will
658 * be set to '0' and the execution will jump to the next instruction.
659 */
660 ins_offset = pc - (long)&ex->insn;
661 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
662 return -ERANGE;
663
664 /*
665 * Since the extable follows the program, the fixup offset is always
666 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
667 * to keep things simple, and put the destination register in the upper
668 * bits. We don't need to worry about buildtime or runtime sort
669 * modifying the upper bits because the table is already sorted, and
670 * isn't part of the main exception table.
671 *
672 * The fixup_offset is set to the next instruction from the instruction
673 * that may fault. The execution will jump to this after handling the fault.
674 */
675 fixup_offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE);
676 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
677 return -ERANGE;
678
679 /*
680 * The offsets above have been calculated using the RO buffer but we
681 * need to use the R/W buffer for writes. Switch ex to rw buffer for writing.
682 */
683 ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image);
684 ex->insn = ins_offset;
685 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
686 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
687 ex->type = EX_TYPE_BPF;
688
689 ctx->num_exentries++;
690
691 return 0;
692 }
693
build_insn(const struct bpf_insn * insn,struct jit_ctx * ctx,bool extra_pass)694 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
695 {
696 u8 tm = -1;
697 u64 func_addr;
698 bool func_addr_fixed, sign_extend;
699 int i = insn - ctx->prog->insnsi;
700 int ret, jmp_offset, tcc_ptr_off;
701 const u8 code = insn->code;
702 const u8 cond = BPF_OP(code);
703 const u8 t1 = LOONGARCH_GPR_T1;
704 const u8 t2 = LOONGARCH_GPR_T2;
705 const u8 t3 = LOONGARCH_GPR_T3;
706 u8 src = regmap[insn->src_reg];
707 u8 dst = regmap[insn->dst_reg];
708 const s16 off = insn->off;
709 const s32 imm = insn->imm;
710 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
711
712 switch (code) {
713 /* dst = src */
714 case BPF_ALU | BPF_MOV | BPF_X:
715 case BPF_ALU64 | BPF_MOV | BPF_X:
716 if (insn_is_cast_user(insn)) {
717 move_reg(ctx, t1, src);
718 emit_zext_32(ctx, t1, true);
719 move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32, false);
720 emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 2);
721 emit_insn(ctx, or, t1, dst, t1);
722 move_reg(ctx, dst, t1);
723 break;
724 }
725 if (insn_is_mov_percpu_addr(insn)) {
726 if (dst != src)
727 move_reg(ctx, dst, src);
728 #ifdef CONFIG_SMP
729 /* dst += __my_cpu_offset, held in $r21 */
730 emit_insn(ctx, addd, dst, dst, LOONGARCH_GPR_U0);
731 #endif
732 break;
733 }
734 switch (off) {
735 case 0:
736 move_reg(ctx, dst, src);
737 emit_zext_32(ctx, dst, is32);
738 break;
739 case 8:
740 emit_insn(ctx, extwb, dst, src);
741 emit_zext_32(ctx, dst, is32);
742 break;
743 case 16:
744 emit_insn(ctx, extwh, dst, src);
745 emit_zext_32(ctx, dst, is32);
746 break;
747 case 32:
748 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
749 break;
750 }
751 break;
752
753 /* dst = imm */
754 case BPF_ALU | BPF_MOV | BPF_K:
755 case BPF_ALU64 | BPF_MOV | BPF_K:
756 move_imm(ctx, dst, imm, is32);
757 break;
758
759 /* dst = dst + src */
760 case BPF_ALU | BPF_ADD | BPF_X:
761 case BPF_ALU64 | BPF_ADD | BPF_X:
762 emit_insn(ctx, addd, dst, dst, src);
763 emit_zext_32(ctx, dst, is32);
764 break;
765
766 /* dst = dst + imm */
767 case BPF_ALU | BPF_ADD | BPF_K:
768 case BPF_ALU64 | BPF_ADD | BPF_K:
769 if (is_signed_imm12(imm)) {
770 emit_insn(ctx, addid, dst, dst, imm);
771 } else {
772 move_imm(ctx, t1, imm, is32);
773 emit_insn(ctx, addd, dst, dst, t1);
774 }
775 emit_zext_32(ctx, dst, is32);
776 break;
777
778 /* dst = dst - src */
779 case BPF_ALU | BPF_SUB | BPF_X:
780 case BPF_ALU64 | BPF_SUB | BPF_X:
781 emit_insn(ctx, subd, dst, dst, src);
782 emit_zext_32(ctx, dst, is32);
783 break;
784
785 /* dst = dst - imm */
786 case BPF_ALU | BPF_SUB | BPF_K:
787 case BPF_ALU64 | BPF_SUB | BPF_K:
788 if (is_signed_imm12(-imm)) {
789 emit_insn(ctx, addid, dst, dst, -imm);
790 } else {
791 move_imm(ctx, t1, imm, is32);
792 emit_insn(ctx, subd, dst, dst, t1);
793 }
794 emit_zext_32(ctx, dst, is32);
795 break;
796
797 /* dst = dst * src */
798 case BPF_ALU | BPF_MUL | BPF_X:
799 case BPF_ALU64 | BPF_MUL | BPF_X:
800 emit_insn(ctx, muld, dst, dst, src);
801 emit_zext_32(ctx, dst, is32);
802 break;
803
804 /* dst = dst * imm */
805 case BPF_ALU | BPF_MUL | BPF_K:
806 case BPF_ALU64 | BPF_MUL | BPF_K:
807 move_imm(ctx, t1, imm, is32);
808 emit_insn(ctx, muld, dst, dst, t1);
809 emit_zext_32(ctx, dst, is32);
810 break;
811
812 /* dst = dst / src */
813 case BPF_ALU | BPF_DIV | BPF_X:
814 case BPF_ALU64 | BPF_DIV | BPF_X:
815 if (!off) {
816 emit_zext_32(ctx, dst, is32);
817 move_reg(ctx, t1, src);
818 emit_zext_32(ctx, t1, is32);
819 emit_insn(ctx, divdu, dst, dst, t1);
820 emit_zext_32(ctx, dst, is32);
821 } else {
822 emit_sext_32(ctx, dst, is32);
823 move_reg(ctx, t1, src);
824 emit_sext_32(ctx, t1, is32);
825 emit_insn(ctx, divd, dst, dst, t1);
826 emit_zext_32(ctx, dst, is32);
827 }
828 break;
829
830 /* dst = dst / imm */
831 case BPF_ALU | BPF_DIV | BPF_K:
832 case BPF_ALU64 | BPF_DIV | BPF_K:
833 if (!off) {
834 move_imm(ctx, t1, imm, is32);
835 emit_zext_32(ctx, dst, is32);
836 emit_insn(ctx, divdu, dst, dst, t1);
837 emit_zext_32(ctx, dst, is32);
838 } else {
839 move_imm(ctx, t1, imm, false);
840 emit_sext_32(ctx, t1, is32);
841 emit_sext_32(ctx, dst, is32);
842 emit_insn(ctx, divd, dst, dst, t1);
843 emit_zext_32(ctx, dst, is32);
844 }
845 break;
846
847 /* dst = dst % src */
848 case BPF_ALU | BPF_MOD | BPF_X:
849 case BPF_ALU64 | BPF_MOD | BPF_X:
850 if (!off) {
851 emit_zext_32(ctx, dst, is32);
852 move_reg(ctx, t1, src);
853 emit_zext_32(ctx, t1, is32);
854 emit_insn(ctx, moddu, dst, dst, t1);
855 emit_zext_32(ctx, dst, is32);
856 } else {
857 emit_sext_32(ctx, dst, is32);
858 move_reg(ctx, t1, src);
859 emit_sext_32(ctx, t1, is32);
860 emit_insn(ctx, modd, dst, dst, t1);
861 emit_zext_32(ctx, dst, is32);
862 }
863 break;
864
865 /* dst = dst % imm */
866 case BPF_ALU | BPF_MOD | BPF_K:
867 case BPF_ALU64 | BPF_MOD | BPF_K:
868 if (!off) {
869 move_imm(ctx, t1, imm, is32);
870 emit_zext_32(ctx, dst, is32);
871 emit_insn(ctx, moddu, dst, dst, t1);
872 emit_zext_32(ctx, dst, is32);
873 } else {
874 move_imm(ctx, t1, imm, false);
875 emit_sext_32(ctx, t1, is32);
876 emit_sext_32(ctx, dst, is32);
877 emit_insn(ctx, modd, dst, dst, t1);
878 emit_zext_32(ctx, dst, is32);
879 }
880 break;
881
882 /* dst = -dst */
883 case BPF_ALU | BPF_NEG:
884 case BPF_ALU64 | BPF_NEG:
885 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
886 emit_zext_32(ctx, dst, is32);
887 break;
888
889 /* dst = dst & src */
890 case BPF_ALU | BPF_AND | BPF_X:
891 case BPF_ALU64 | BPF_AND | BPF_X:
892 emit_insn(ctx, and, dst, dst, src);
893 emit_zext_32(ctx, dst, is32);
894 break;
895
896 /* dst = dst & imm */
897 case BPF_ALU | BPF_AND | BPF_K:
898 case BPF_ALU64 | BPF_AND | BPF_K:
899 if (is_unsigned_imm12(imm)) {
900 emit_insn(ctx, andi, dst, dst, imm);
901 } else {
902 move_imm(ctx, t1, imm, is32);
903 emit_insn(ctx, and, dst, dst, t1);
904 }
905 emit_zext_32(ctx, dst, is32);
906 break;
907
908 /* dst = dst | src */
909 case BPF_ALU | BPF_OR | BPF_X:
910 case BPF_ALU64 | BPF_OR | BPF_X:
911 emit_insn(ctx, or, dst, dst, src);
912 emit_zext_32(ctx, dst, is32);
913 break;
914
915 /* dst = dst | imm */
916 case BPF_ALU | BPF_OR | BPF_K:
917 case BPF_ALU64 | BPF_OR | BPF_K:
918 if (is_unsigned_imm12(imm)) {
919 emit_insn(ctx, ori, dst, dst, imm);
920 } else {
921 move_imm(ctx, t1, imm, is32);
922 emit_insn(ctx, or, dst, dst, t1);
923 }
924 emit_zext_32(ctx, dst, is32);
925 break;
926
927 /* dst = dst ^ src */
928 case BPF_ALU | BPF_XOR | BPF_X:
929 case BPF_ALU64 | BPF_XOR | BPF_X:
930 emit_insn(ctx, xor, dst, dst, src);
931 emit_zext_32(ctx, dst, is32);
932 break;
933
934 /* dst = dst ^ imm */
935 case BPF_ALU | BPF_XOR | BPF_K:
936 case BPF_ALU64 | BPF_XOR | BPF_K:
937 if (is_unsigned_imm12(imm)) {
938 emit_insn(ctx, xori, dst, dst, imm);
939 } else {
940 move_imm(ctx, t1, imm, is32);
941 emit_insn(ctx, xor, dst, dst, t1);
942 }
943 emit_zext_32(ctx, dst, is32);
944 break;
945
946 /* dst = dst << src (logical) */
947 case BPF_ALU | BPF_LSH | BPF_X:
948 emit_insn(ctx, sllw, dst, dst, src);
949 emit_zext_32(ctx, dst, is32);
950 break;
951
952 case BPF_ALU64 | BPF_LSH | BPF_X:
953 emit_insn(ctx, slld, dst, dst, src);
954 break;
955
956 /* dst = dst << imm (logical) */
957 case BPF_ALU | BPF_LSH | BPF_K:
958 emit_insn(ctx, slliw, dst, dst, imm);
959 emit_zext_32(ctx, dst, is32);
960 break;
961
962 case BPF_ALU64 | BPF_LSH | BPF_K:
963 emit_insn(ctx, sllid, dst, dst, imm);
964 break;
965
966 /* dst = dst >> src (logical) */
967 case BPF_ALU | BPF_RSH | BPF_X:
968 emit_insn(ctx, srlw, dst, dst, src);
969 emit_zext_32(ctx, dst, is32);
970 break;
971
972 case BPF_ALU64 | BPF_RSH | BPF_X:
973 emit_insn(ctx, srld, dst, dst, src);
974 break;
975
976 /* dst = dst >> imm (logical) */
977 case BPF_ALU | BPF_RSH | BPF_K:
978 emit_insn(ctx, srliw, dst, dst, imm);
979 emit_zext_32(ctx, dst, is32);
980 break;
981
982 case BPF_ALU64 | BPF_RSH | BPF_K:
983 emit_insn(ctx, srlid, dst, dst, imm);
984 break;
985
986 /* dst = dst >> src (arithmetic) */
987 case BPF_ALU | BPF_ARSH | BPF_X:
988 emit_insn(ctx, sraw, dst, dst, src);
989 emit_zext_32(ctx, dst, is32);
990 break;
991
992 case BPF_ALU64 | BPF_ARSH | BPF_X:
993 emit_insn(ctx, srad, dst, dst, src);
994 break;
995
996 /* dst = dst >> imm (arithmetic) */
997 case BPF_ALU | BPF_ARSH | BPF_K:
998 emit_insn(ctx, sraiw, dst, dst, imm);
999 emit_zext_32(ctx, dst, is32);
1000 break;
1001
1002 case BPF_ALU64 | BPF_ARSH | BPF_K:
1003 emit_insn(ctx, sraid, dst, dst, imm);
1004 break;
1005
1006 /* dst = BSWAP##imm(dst) */
1007 case BPF_ALU | BPF_END | BPF_FROM_LE:
1008 switch (imm) {
1009 case 16:
1010 /* zero-extend 16 bits into 64 bits */
1011 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
1012 break;
1013 case 32:
1014 /* zero-extend 32 bits into 64 bits */
1015 emit_zext_32(ctx, dst, is32);
1016 break;
1017 case 64:
1018 /* do nothing */
1019 break;
1020 }
1021 break;
1022
1023 case BPF_ALU | BPF_END | BPF_FROM_BE:
1024 case BPF_ALU64 | BPF_END | BPF_FROM_LE:
1025 switch (imm) {
1026 case 16:
1027 emit_insn(ctx, revb2h, dst, dst);
1028 /* zero-extend 16 bits into 64 bits */
1029 emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
1030 break;
1031 case 32:
1032 emit_insn(ctx, revb2w, dst, dst);
1033 /* clear the upper 32 bits */
1034 emit_zext_32(ctx, dst, true);
1035 break;
1036 case 64:
1037 emit_insn(ctx, revbd, dst, dst);
1038 break;
1039 }
1040 break;
1041
1042 /* PC += off if dst cond src */
1043 case BPF_JMP | BPF_JEQ | BPF_X:
1044 case BPF_JMP | BPF_JNE | BPF_X:
1045 case BPF_JMP | BPF_JGT | BPF_X:
1046 case BPF_JMP | BPF_JGE | BPF_X:
1047 case BPF_JMP | BPF_JLT | BPF_X:
1048 case BPF_JMP | BPF_JLE | BPF_X:
1049 case BPF_JMP | BPF_JSGT | BPF_X:
1050 case BPF_JMP | BPF_JSGE | BPF_X:
1051 case BPF_JMP | BPF_JSLT | BPF_X:
1052 case BPF_JMP | BPF_JSLE | BPF_X:
1053 case BPF_JMP32 | BPF_JEQ | BPF_X:
1054 case BPF_JMP32 | BPF_JNE | BPF_X:
1055 case BPF_JMP32 | BPF_JGT | BPF_X:
1056 case BPF_JMP32 | BPF_JGE | BPF_X:
1057 case BPF_JMP32 | BPF_JLT | BPF_X:
1058 case BPF_JMP32 | BPF_JLE | BPF_X:
1059 case BPF_JMP32 | BPF_JSGT | BPF_X:
1060 case BPF_JMP32 | BPF_JSGE | BPF_X:
1061 case BPF_JMP32 | BPF_JSLT | BPF_X:
1062 case BPF_JMP32 | BPF_JSLE | BPF_X:
1063 jmp_offset = bpf2la_offset(i, off, ctx);
1064 move_reg(ctx, t1, dst);
1065 move_reg(ctx, t2, src);
1066 if (is_signed_bpf_cond(BPF_OP(code))) {
1067 emit_sext_32(ctx, t1, is32);
1068 emit_sext_32(ctx, t2, is32);
1069 } else {
1070 emit_zext_32(ctx, t1, is32);
1071 emit_zext_32(ctx, t2, is32);
1072 }
1073 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
1074 goto toofar;
1075 break;
1076
1077 /* PC += off if dst cond imm */
1078 case BPF_JMP | BPF_JEQ | BPF_K:
1079 case BPF_JMP | BPF_JNE | BPF_K:
1080 case BPF_JMP | BPF_JGT | BPF_K:
1081 case BPF_JMP | BPF_JGE | BPF_K:
1082 case BPF_JMP | BPF_JLT | BPF_K:
1083 case BPF_JMP | BPF_JLE | BPF_K:
1084 case BPF_JMP | BPF_JSGT | BPF_K:
1085 case BPF_JMP | BPF_JSGE | BPF_K:
1086 case BPF_JMP | BPF_JSLT | BPF_K:
1087 case BPF_JMP | BPF_JSLE | BPF_K:
1088 case BPF_JMP32 | BPF_JEQ | BPF_K:
1089 case BPF_JMP32 | BPF_JNE | BPF_K:
1090 case BPF_JMP32 | BPF_JGT | BPF_K:
1091 case BPF_JMP32 | BPF_JGE | BPF_K:
1092 case BPF_JMP32 | BPF_JLT | BPF_K:
1093 case BPF_JMP32 | BPF_JLE | BPF_K:
1094 case BPF_JMP32 | BPF_JSGT | BPF_K:
1095 case BPF_JMP32 | BPF_JSGE | BPF_K:
1096 case BPF_JMP32 | BPF_JSLT | BPF_K:
1097 case BPF_JMP32 | BPF_JSLE | BPF_K:
1098 jmp_offset = bpf2la_offset(i, off, ctx);
1099 if (imm) {
1100 move_imm(ctx, t1, imm, false);
1101 tm = t1;
1102 } else {
1103 /* If imm is 0, simply use zero register. */
1104 tm = LOONGARCH_GPR_ZERO;
1105 }
1106 move_reg(ctx, t2, dst);
1107 if (is_signed_bpf_cond(BPF_OP(code))) {
1108 emit_sext_32(ctx, tm, is32);
1109 emit_sext_32(ctx, t2, is32);
1110 } else {
1111 emit_zext_32(ctx, tm, is32);
1112 emit_zext_32(ctx, t2, is32);
1113 }
1114 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
1115 goto toofar;
1116 break;
1117
1118 /* PC += off if dst & src */
1119 case BPF_JMP | BPF_JSET | BPF_X:
1120 case BPF_JMP32 | BPF_JSET | BPF_X:
1121 jmp_offset = bpf2la_offset(i, off, ctx);
1122 emit_insn(ctx, and, t1, dst, src);
1123 emit_zext_32(ctx, t1, is32);
1124 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
1125 goto toofar;
1126 break;
1127
1128 /* PC += off if dst & imm */
1129 case BPF_JMP | BPF_JSET | BPF_K:
1130 case BPF_JMP32 | BPF_JSET | BPF_K:
1131 jmp_offset = bpf2la_offset(i, off, ctx);
1132 move_imm(ctx, t1, imm, is32);
1133 emit_insn(ctx, and, t1, dst, t1);
1134 emit_zext_32(ctx, t1, is32);
1135 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
1136 goto toofar;
1137 break;
1138
1139 /* PC += off */
1140 case BPF_JMP | BPF_JA:
1141 jmp_offset = bpf2la_offset(i, off, ctx);
1142 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
1143 goto toofar;
1144 break;
1145 case BPF_JMP32 | BPF_JA:
1146 jmp_offset = bpf2la_offset(i, imm, ctx);
1147 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
1148 goto toofar;
1149 break;
1150
1151 /* function call */
1152 case BPF_JMP | BPF_CALL:
1153 /* Implement helper call to bpf_get_current_task/_btf() inline */
1154 if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
1155 insn->imm == BPF_FUNC_get_current_task_btf)) {
1156 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_TP);
1157 break;
1158 }
1159
1160 /* Implement helper call to bpf_get_smp_processor_id() inline */
1161 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) {
1162 emit_insn(ctx, ldwu, regmap[BPF_REG_0], LOONGARCH_GPR_TP, TI_CPU);
1163 break;
1164 }
1165
1166 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
1167 &func_addr, &func_addr_fixed);
1168 if (ret < 0)
1169 return ret;
1170
1171 if (insn->src_reg == BPF_PSEUDO_CALL) {
1172 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
1173 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
1174 }
1175
1176 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
1177 const struct btf_func_model *m;
1178 int i;
1179
1180 m = bpf_jit_find_kfunc_model(ctx->prog, insn);
1181 if (!m)
1182 return -EINVAL;
1183
1184 for (i = 0; i < m->nr_args; i++) {
1185 u8 reg = regmap[BPF_REG_1 + i];
1186 bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG;
1187
1188 emit_abi_ext(ctx, reg, m->arg_size[i], sign);
1189 }
1190 }
1191
1192 move_addr(ctx, t1, func_addr);
1193 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
1194
1195 /*
1196 * Call to arch_bpf_timed_may_goto() uses a custom calling
1197 * convention with the argument and return value in BPF_REG_AX,
1198 * so skip moving the C return value into BPF_REG_0.
1199 */
1200 if (insn->src_reg != BPF_PSEUDO_CALL &&
1201 func_addr != (u64)arch_bpf_timed_may_goto)
1202 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
1203
1204 break;
1205
1206 /* tail call */
1207 case BPF_JMP | BPF_TAIL_CALL:
1208 if (emit_bpf_tail_call(ctx, i) < 0)
1209 return -EINVAL;
1210 break;
1211
1212 /* function return */
1213 case BPF_JMP | BPF_EXIT:
1214 if (i == ctx->prog->len - 1)
1215 break;
1216
1217 jmp_offset = epilogue_offset(ctx);
1218 if (emit_uncond_jmp(ctx, jmp_offset) < 0)
1219 goto toofar;
1220 break;
1221
1222 /* dst = imm64 */
1223 case BPF_LD | BPF_IMM | BPF_DW:
1224 {
1225 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm;
1226
1227 if (bpf_pseudo_func(insn))
1228 move_addr(ctx, dst, imm64);
1229 else
1230 move_imm(ctx, dst, imm64, is32);
1231 return 1;
1232 }
1233
1234 /* dst = *(size *)(src + off) */
1235 case BPF_LDX | BPF_MEM | BPF_B:
1236 case BPF_LDX | BPF_MEM | BPF_H:
1237 case BPF_LDX | BPF_MEM | BPF_W:
1238 case BPF_LDX | BPF_MEM | BPF_DW:
1239 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
1240 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
1241 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
1242 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
1243 /* dst_reg = (s64)*(signed size *)(src_reg + off) */
1244 case BPF_LDX | BPF_MEMSX | BPF_B:
1245 case BPF_LDX | BPF_MEMSX | BPF_H:
1246 case BPF_LDX | BPF_MEMSX | BPF_W:
1247 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
1248 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
1249 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
1250 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + REG_ARENA + off) */
1251 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
1252 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
1253 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
1254 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
1255 sign_extend = BPF_MODE(code) == BPF_MEMSX ||
1256 BPF_MODE(code) == BPF_PROBE_MEMSX;
1257
1258 if (BPF_MODE(code) == BPF_PROBE_MEM32) {
1259 emit_insn(ctx, addd, t2, src, REG_ARENA);
1260 src = t2;
1261 }
1262
1263 switch (BPF_SIZE(code)) {
1264 case BPF_B:
1265 if (is_signed_imm12(off)) {
1266 if (sign_extend)
1267 emit_insn(ctx, ldb, dst, src, off);
1268 else
1269 emit_insn(ctx, ldbu, dst, src, off);
1270 } else {
1271 move_imm(ctx, t1, off, is32);
1272 if (sign_extend)
1273 emit_insn(ctx, ldxb, dst, src, t1);
1274 else
1275 emit_insn(ctx, ldxbu, dst, src, t1);
1276 }
1277 break;
1278 case BPF_H:
1279 if (is_signed_imm12(off)) {
1280 if (sign_extend)
1281 emit_insn(ctx, ldh, dst, src, off);
1282 else
1283 emit_insn(ctx, ldhu, dst, src, off);
1284 } else {
1285 move_imm(ctx, t1, off, is32);
1286 if (sign_extend)
1287 emit_insn(ctx, ldxh, dst, src, t1);
1288 else
1289 emit_insn(ctx, ldxhu, dst, src, t1);
1290 }
1291 break;
1292 case BPF_W:
1293 if (is_signed_imm12(off)) {
1294 if (sign_extend)
1295 emit_insn(ctx, ldw, dst, src, off);
1296 else
1297 emit_insn(ctx, ldwu, dst, src, off);
1298 } else {
1299 move_imm(ctx, t1, off, is32);
1300 if (sign_extend)
1301 emit_insn(ctx, ldxw, dst, src, t1);
1302 else
1303 emit_insn(ctx, ldxwu, dst, src, t1);
1304 }
1305 break;
1306 case BPF_DW:
1307 move_imm(ctx, t1, off, is32);
1308 emit_insn(ctx, ldxd, dst, src, t1);
1309 break;
1310 }
1311
1312 ret = add_exception_handler(insn, ctx, dst);
1313 if (ret)
1314 return ret;
1315 break;
1316
1317 /* *(size *)(dst + off) = imm */
1318 case BPF_ST | BPF_MEM | BPF_B:
1319 case BPF_ST | BPF_MEM | BPF_H:
1320 case BPF_ST | BPF_MEM | BPF_W:
1321 case BPF_ST | BPF_MEM | BPF_DW:
1322 /* ST | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = imm */
1323 case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
1324 case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
1325 case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
1326 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
1327 if (BPF_MODE(code) == BPF_PROBE_MEM32) {
1328 emit_insn(ctx, addd, t3, dst, REG_ARENA);
1329 dst = t3;
1330 }
1331
1332 switch (BPF_SIZE(code)) {
1333 case BPF_B:
1334 move_imm(ctx, t1, imm, is32);
1335 if (is_signed_imm12(off)) {
1336 emit_insn(ctx, stb, t1, dst, off);
1337 } else {
1338 move_imm(ctx, t2, off, is32);
1339 emit_insn(ctx, stxb, t1, dst, t2);
1340 }
1341 break;
1342 case BPF_H:
1343 move_imm(ctx, t1, imm, is32);
1344 if (is_signed_imm12(off)) {
1345 emit_insn(ctx, sth, t1, dst, off);
1346 } else {
1347 move_imm(ctx, t2, off, is32);
1348 emit_insn(ctx, stxh, t1, dst, t2);
1349 }
1350 break;
1351 case BPF_W:
1352 move_imm(ctx, t1, imm, is32);
1353 if (is_signed_imm12(off)) {
1354 emit_insn(ctx, stw, t1, dst, off);
1355 } else if (is_signed_imm14(off)) {
1356 emit_insn(ctx, stptrw, t1, dst, off);
1357 } else {
1358 move_imm(ctx, t2, off, is32);
1359 emit_insn(ctx, stxw, t1, dst, t2);
1360 }
1361 break;
1362 case BPF_DW:
1363 move_imm(ctx, t1, imm, is32);
1364 if (is_signed_imm12(off)) {
1365 emit_insn(ctx, std, t1, dst, off);
1366 } else if (is_signed_imm14(off)) {
1367 emit_insn(ctx, stptrd, t1, dst, off);
1368 } else {
1369 move_imm(ctx, t2, off, is32);
1370 emit_insn(ctx, stxd, t1, dst, t2);
1371 }
1372 break;
1373 }
1374
1375 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER);
1376 if (ret)
1377 return ret;
1378 break;
1379
1380 /* *(size *)(dst + off) = src */
1381 case BPF_STX | BPF_MEM | BPF_B:
1382 case BPF_STX | BPF_MEM | BPF_H:
1383 case BPF_STX | BPF_MEM | BPF_W:
1384 case BPF_STX | BPF_MEM | BPF_DW:
1385 /* STX | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = src */
1386 case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
1387 case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
1388 case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
1389 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
1390 if (BPF_MODE(code) == BPF_PROBE_MEM32) {
1391 emit_insn(ctx, addd, t2, dst, REG_ARENA);
1392 dst = t2;
1393 }
1394
1395 switch (BPF_SIZE(code)) {
1396 case BPF_B:
1397 if (is_signed_imm12(off)) {
1398 emit_insn(ctx, stb, src, dst, off);
1399 } else {
1400 move_imm(ctx, t1, off, is32);
1401 emit_insn(ctx, stxb, src, dst, t1);
1402 }
1403 break;
1404 case BPF_H:
1405 if (is_signed_imm12(off)) {
1406 emit_insn(ctx, sth, src, dst, off);
1407 } else {
1408 move_imm(ctx, t1, off, is32);
1409 emit_insn(ctx, stxh, src, dst, t1);
1410 }
1411 break;
1412 case BPF_W:
1413 if (is_signed_imm12(off)) {
1414 emit_insn(ctx, stw, src, dst, off);
1415 } else if (is_signed_imm14(off)) {
1416 emit_insn(ctx, stptrw, src, dst, off);
1417 } else {
1418 move_imm(ctx, t1, off, is32);
1419 emit_insn(ctx, stxw, src, dst, t1);
1420 }
1421 break;
1422 case BPF_DW:
1423 if (is_signed_imm12(off)) {
1424 emit_insn(ctx, std, src, dst, off);
1425 } else if (is_signed_imm14(off)) {
1426 emit_insn(ctx, stptrd, src, dst, off);
1427 } else {
1428 move_imm(ctx, t1, off, is32);
1429 emit_insn(ctx, stxd, src, dst, t1);
1430 }
1431 break;
1432 }
1433
1434 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER);
1435 if (ret)
1436 return ret;
1437 break;
1438
1439 /* Atomics */
1440 case BPF_STX | BPF_ATOMIC | BPF_B:
1441 case BPF_STX | BPF_ATOMIC | BPF_H:
1442 case BPF_STX | BPF_ATOMIC | BPF_W:
1443 case BPF_STX | BPF_ATOMIC | BPF_DW:
1444 if (!bpf_atomic_is_load_store(insn))
1445 ret = emit_atomic_rmw(insn, ctx);
1446 else
1447 ret = emit_atomic_ld_st(insn, ctx);
1448 if (ret)
1449 return ret;
1450 break;
1451
1452 /* Speculation barrier */
1453 case BPF_ST | BPF_NOSPEC:
1454 break;
1455
1456 default:
1457 pr_err("bpf_jit: unknown opcode %02x\n", code);
1458 return -EINVAL;
1459 }
1460
1461 return 0;
1462
1463 toofar:
1464 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code);
1465 return -E2BIG;
1466 }
1467
build_body(struct jit_ctx * ctx,bool extra_pass)1468 static int build_body(struct jit_ctx *ctx, bool extra_pass)
1469 {
1470 int i;
1471 const struct bpf_prog *prog = ctx->prog;
1472
1473 for (i = 0; i < prog->len; i++) {
1474 const struct bpf_insn *insn = &prog->insnsi[i];
1475 int ret;
1476
1477 if (ctx->image == NULL)
1478 ctx->offset[i] = ctx->idx;
1479
1480 ret = build_insn(insn, ctx, extra_pass);
1481 if (ret > 0) {
1482 i++;
1483 if (ctx->image == NULL)
1484 ctx->offset[i] = ctx->idx;
1485 continue;
1486 }
1487 if (ret)
1488 return ret;
1489 }
1490
1491 if (ctx->image == NULL)
1492 ctx->offset[i] = ctx->idx;
1493
1494 return 0;
1495 }
1496
1497 /* Fill space with break instructions */
jit_fill_hole(void * area,unsigned int size)1498 static void jit_fill_hole(void *area, unsigned int size)
1499 {
1500 u32 *ptr;
1501
1502 /* We are guaranteed to have aligned memory */
1503 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32))
1504 *ptr++ = INSN_BREAK;
1505 }
1506
validate_code(struct jit_ctx * ctx)1507 static int validate_code(struct jit_ctx *ctx)
1508 {
1509 int i;
1510 union loongarch_instruction insn;
1511
1512 for (i = 0; i < ctx->idx; i++) {
1513 insn = ctx->image[i];
1514 /* Check INSN_BREAK */
1515 if (insn.word == INSN_BREAK)
1516 return -1;
1517 }
1518
1519 return 0;
1520 }
1521
validate_ctx(struct jit_ctx * ctx)1522 static int validate_ctx(struct jit_ctx *ctx)
1523 {
1524 if (validate_code(ctx))
1525 return -1;
1526
1527 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
1528 return -1;
1529
1530 return 0;
1531 }
1532
emit_jump_and_link(struct jit_ctx * ctx,u8 rd,u64 target)1533 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
1534 {
1535 if (!target) {
1536 pr_err("bpf_jit: jump target address is error\n");
1537 return -EFAULT;
1538 }
1539
1540 move_imm(ctx, LOONGARCH_GPR_T1, target, false);
1541 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
1542
1543 return 0;
1544 }
1545
emit_jump_or_nops(void * target,void * ip,u32 * insns,bool is_call)1546 static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
1547 {
1548 int i;
1549 struct jit_ctx ctx;
1550
1551 ctx.idx = 0;
1552 ctx.image = (union loongarch_instruction *)insns;
1553
1554 if (!target) {
1555 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
1556 emit_insn((&ctx), nop);
1557 return 0;
1558 }
1559
1560 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target);
1561 }
1562
emit_call(struct jit_ctx * ctx,u64 addr)1563 static int emit_call(struct jit_ctx *ctx, u64 addr)
1564 {
1565 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
1566 }
1567
bpf_arch_text_copy(void * dst,void * src,size_t len)1568 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
1569 {
1570 int ret;
1571
1572 cpus_read_lock();
1573 mutex_lock(&text_mutex);
1574 ret = larch_insn_text_copy(dst, src, len);
1575 mutex_unlock(&text_mutex);
1576 cpus_read_unlock();
1577
1578 return ret ? ERR_PTR(-EINVAL) : dst;
1579 }
1580
bpf_arch_text_poke(void * ip,enum bpf_text_poke_type old_t,enum bpf_text_poke_type new_t,void * old_addr,void * new_addr)1581 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
1582 enum bpf_text_poke_type new_t, void *old_addr,
1583 void *new_addr)
1584 {
1585 int ret;
1586 bool is_call;
1587 unsigned long size = 0;
1588 unsigned long offset = 0;
1589 void *image = NULL;
1590 char namebuf[KSYM_NAME_LEN];
1591 u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1592 u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
1593
1594 /* Only poking bpf text is supported. Since kernel function entry
1595 * is set up by ftrace, we rely on ftrace to poke kernel functions.
1596 */
1597 if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf))
1598 return -ENOTSUPP;
1599
1600 image = ip - offset;
1601
1602 /* zero offset means we're poking bpf prog entry */
1603 if (offset == 0) {
1604 /* skip to the nop instruction in bpf prog entry:
1605 * move t0, ra
1606 * nop
1607 */
1608 ip = image + LOONGARCH_INSN_SIZE;
1609 }
1610
1611 is_call = old_t == BPF_MOD_CALL;
1612 ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call);
1613 if (ret)
1614 return ret;
1615
1616 if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES))
1617 return -EFAULT;
1618
1619 is_call = new_t == BPF_MOD_CALL;
1620 ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call);
1621 if (ret)
1622 return ret;
1623
1624 cpus_read_lock();
1625 mutex_lock(&text_mutex);
1626 if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
1627 ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
1628 mutex_unlock(&text_mutex);
1629 cpus_read_unlock();
1630
1631 return ret;
1632 }
1633
bpf_arch_text_invalidate(void * dst,size_t len)1634 int bpf_arch_text_invalidate(void *dst, size_t len)
1635 {
1636 int i;
1637 int ret = 0;
1638 u32 *inst;
1639
1640 inst = kvmalloc(len, GFP_KERNEL);
1641 if (!inst)
1642 return -ENOMEM;
1643
1644 for (i = 0; i < (len / sizeof(u32)); i++)
1645 inst[i] = INSN_BREAK;
1646
1647 cpus_read_lock();
1648 mutex_lock(&text_mutex);
1649 if (larch_insn_text_copy(dst, inst, len))
1650 ret = -EINVAL;
1651 mutex_unlock(&text_mutex);
1652 cpus_read_unlock();
1653
1654 kvfree(inst);
1655
1656 return ret;
1657 }
1658
store_args(struct jit_ctx * ctx,int nr_arg_slots,int args_off)1659 static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off)
1660 {
1661 int i;
1662
1663 for (i = 0; i < nr_arg_slots; i++) {
1664 if (i < LOONGARCH_MAX_REG_ARGS)
1665 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1666 else {
1667 /* Skip slots for T0 and FP of traced function */
1668 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,
1669 16 + (i - LOONGARCH_MAX_REG_ARGS) * 8);
1670 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off);
1671 }
1672 args_off -= 8;
1673 }
1674 }
1675
restore_args(struct jit_ctx * ctx,int nr_reg_args,int args_off)1676 static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off)
1677 {
1678 int i;
1679
1680 for (i = 0; i < nr_reg_args; i++) {
1681 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
1682 args_off -= 8;
1683 }
1684 }
1685
restore_stk_args(struct jit_ctx * ctx,int nr_stk_args,int args_off,int stk_args_off)1686 static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, int stk_args_off)
1687 {
1688 int i;
1689
1690 for (i = 0; i < nr_stk_args; i++) {
1691 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP,
1692 -(args_off - LOONGARCH_MAX_REG_ARGS * 8));
1693 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -stk_args_off);
1694 args_off -= 8;
1695 stk_args_off -= 8;
1696 }
1697 }
1698
invoke_bpf_prog(struct jit_ctx * ctx,struct bpf_tramp_node * n,int args_off,int retval_off,int run_ctx_off,bool save_ret)1699 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n,
1700 int args_off, int retval_off, int run_ctx_off, bool save_ret)
1701 {
1702 int ret;
1703 u32 *branch;
1704 struct bpf_prog *p = n->link->prog;
1705 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
1706
1707 if (n->cookie)
1708 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1,
1709 -run_ctx_off + cookie_off, n->cookie);
1710 else
1711 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
1712
1713 /* arg1: prog */
1714 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1715 /* arg2: &run_ctx */
1716 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
1717 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
1718 if (ret)
1719 return ret;
1720
1721 /* store prog start time */
1722 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
1723
1724 /*
1725 * if (__bpf_prog_enter(prog) == 0)
1726 * goto skip_exec_of_prog;
1727 */
1728 branch = (u32 *)ctx->image + ctx->idx;
1729 /* nop reserved for conditional jump */
1730 emit_insn(ctx, nop);
1731
1732 /* arg1: &args_off */
1733 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
1734 if (!p->jited)
1735 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
1736 ret = emit_call(ctx, (const u64)p->bpf_func);
1737 if (ret)
1738 return ret;
1739
1740 if (save_ret) {
1741 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
1742 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
1743 }
1744
1745 /* update branch with beqz */
1746 if (ctx->image) {
1747 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
1748 *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
1749 }
1750
1751 /* arg1: prog */
1752 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
1753 /* arg2: prog start time */
1754 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
1755 /* arg3: &run_ctx */
1756 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
1757 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
1758
1759 return ret;
1760 }
1761
invoke_bpf(struct jit_ctx * ctx,struct bpf_tramp_nodes * tn,int args_off,int retval_off,int run_ctx_off,int func_meta_off,bool save_ret,u64 func_meta,int cookie_off)1762 static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
1763 int args_off, int retval_off, int run_ctx_off,
1764 int func_meta_off, bool save_ret, u64 func_meta, int cookie_off)
1765 {
1766 int i, cur_cookie = (cookie_off - args_off) / 8;
1767
1768 for (i = 0; i < tn->nr_nodes; i++) {
1769 int err;
1770
1771 if (bpf_prog_calls_session_cookie(tn->nodes[i])) {
1772 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
1773
1774 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta);
1775 cur_cookie--;
1776 }
1777 err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret);
1778 if (err)
1779 return err;
1780 }
1781
1782 return 0;
1783 }
1784
arch_alloc_bpf_trampoline(unsigned int size)1785 void *arch_alloc_bpf_trampoline(unsigned int size)
1786 {
1787 return bpf_prog_pack_alloc(size, jit_fill_hole, false);
1788 }
1789
arch_free_bpf_trampoline(void * image,unsigned int size)1790 void arch_free_bpf_trampoline(void *image, unsigned int size)
1791 {
1792 bpf_prog_pack_free(image, size);
1793 }
1794
arch_protect_bpf_trampoline(void * image,unsigned int size)1795 int arch_protect_bpf_trampoline(void *image, unsigned int size)
1796 {
1797 return 0;
1798 }
1799
1800 /*
1801 * Sign-extend the register if necessary
1802 */
sign_extend(struct jit_ctx * ctx,int rd,int rj,u8 size,bool sign)1803 static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign)
1804 {
1805 /* ABI requires unsigned char/short to be zero-extended */
1806 if (!sign && (size == 1 || size == 2)) {
1807 if (rd != rj)
1808 move_reg(ctx, rd, rj);
1809 return;
1810 }
1811
1812 switch (size) {
1813 case 1:
1814 emit_insn(ctx, extwb, rd, rj);
1815 break;
1816 case 2:
1817 emit_insn(ctx, extwh, rd, rj);
1818 break;
1819 case 4:
1820 emit_insn(ctx, addiw, rd, rj, 0);
1821 break;
1822 case 8:
1823 if (rd != rj)
1824 move_reg(ctx, rd, rj);
1825 break;
1826 default:
1827 pr_warn("bpf_jit: invalid size %d for sign_extend\n", size);
1828 }
1829 }
1830
__arch_prepare_bpf_trampoline(struct jit_ctx * ctx,struct bpf_tramp_image * im,const struct btf_func_model * m,struct bpf_tramp_nodes * tnodes,void * func_addr,u32 flags)1831 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
1832 const struct btf_func_model *m, struct bpf_tramp_nodes *tnodes,
1833 void *func_addr, u32 flags)
1834 {
1835 int i, ret, save_ret;
1836 int cookie_cnt, cookie_off;
1837 int stack_size, args_off, stk_args_off, nr_arg_slots = 0;
1838 int retval_off, func_meta_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
1839 unsigned long long func_meta;
1840 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
1841 void *orig_call = func_addr;
1842 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY];
1843 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT];
1844 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN];
1845 u32 **branches = NULL;
1846
1847 /*
1848 * FP + 8 [ RA to parent func ] return address to parent
1849 * function
1850 * FP + 0 [ FP of parent func ] frame pointer of parent
1851 * function
1852 * FP - 8 [ T0 to traced func ] return address of traced
1853 * function
1854 * FP - 16 [ FP of traced func ] frame pointer of traced
1855 * function
1856 *
1857 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
1858 * BPF_TRAMP_F_RET_FENTRY_RET
1859 * [ arg regN ]
1860 * [ ... ]
1861 * FP - args_off [ arg reg1 ]
1862 *
1863 * FP - func_meta_off [ regs count, etc ]
1864 *
1865 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
1866 *
1867 * [ stack cookie N ]
1868 * [ ... ]
1869 * FP - cookie_off [ stack cookie 1 ]
1870 *
1871 * FP - run_ctx_off [ bpf_tramp_run_ctx ]
1872 *
1873 * FP - sreg_off [ callee saved reg ]
1874 *
1875 * FP - tcc_ptr_off [ tail_call_cnt_ptr ]
1876 *
1877 * [ stack_argN ]
1878 * [ ... ]
1879 * FP - stk_args_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG
1880 */
1881
1882 if (m->nr_args > MAX_BPF_FUNC_ARGS)
1883 return -ENOTSUPP;
1884
1885 /* Extra registers for struct arguments */
1886 for (i = 0; i < m->nr_args; i++) {
1887 /*
1888 * The struct argument size is at most 16 bytes,
1889 * enforced by the verifier. The struct argument
1890 * may be passed in a pair of registers if its
1891 * size is more than 8 bytes and no more than 16
1892 * bytes.
1893 */
1894 nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
1895 }
1896
1897 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
1898 return -ENOTSUPP;
1899
1900 /* Room of trampoline frame to store return address and frame pointer */
1901 stack_size = 16;
1902
1903 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
1904 if (save_ret)
1905 stack_size += 16; /* Save BPF R0 and A0 */
1906
1907 retval_off = stack_size;
1908
1909 /* Room of trampoline frame to store args */
1910 stack_size += nr_arg_slots * 8;
1911 args_off = stack_size;
1912
1913 /* Room of function metadata, such as regs count */
1914 stack_size += 8;
1915 func_meta_off = stack_size;
1916
1917 /* Room of trampoline frame to store ip address */
1918 if (flags & BPF_TRAMP_F_IP_ARG) {
1919 stack_size += 8;
1920 ip_off = stack_size;
1921 }
1922
1923 cookie_cnt = bpf_fsession_cookie_cnt(tnodes);
1924
1925 /* Room for session cookies */
1926 stack_size += cookie_cnt * 8;
1927 cookie_off = stack_size;
1928
1929 /* Room of trampoline frame to store struct bpf_tramp_run_ctx */
1930 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
1931 run_ctx_off = stack_size;
1932
1933 stack_size += 8;
1934 sreg_off = stack_size;
1935
1936 /* Room of trampoline frame to store tail_call_cnt_ptr */
1937 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
1938 stack_size += 8;
1939 tcc_ptr_off = stack_size;
1940 }
1941
1942 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0))
1943 stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8;
1944
1945 stack_size = round_up(stack_size, 16);
1946
1947 /* Room for args on stack must be at the top of stack */
1948 stk_args_off = stack_size;
1949
1950 if (is_struct_ops) {
1951 /*
1952 * For the trampoline called directly, just handle
1953 * the frame of trampoline.
1954 */
1955 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1956 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
1957 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1958 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1959 } else {
1960 /*
1961 * For the trampoline called from function entry,
1962 * the frame of traced function and the frame of
1963 * trampoline need to be considered.
1964 */
1965 /* RA and FP for parent function */
1966 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
1967 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
1968 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
1969 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
1970
1971 /* RA and FP for traced function */
1972 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
1973 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
1974 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
1975 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
1976 }
1977
1978 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
1979 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
1980
1981 /* callee saved register S1 to pass start time */
1982 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
1983
1984 /* store ip address of the traced function */
1985 if (flags & BPF_TRAMP_F_IP_ARG)
1986 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -ip_off, (u64)func_addr);
1987
1988 /* store arg regs count */
1989 func_meta = nr_arg_slots;
1990 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta);
1991
1992 store_args(ctx, nr_arg_slots, args_off);
1993
1994 if (bpf_fsession_cnt(tnodes)) {
1995 /* clear all session cookies' value */
1996 for (i = 0; i < cookie_cnt; i++)
1997 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -cookie_off + 8 * i);
1998
1999 /* clear return value to make sure fentry always get 0 */
2000 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
2001 }
2002
2003 /* To traced function */
2004 /* Ftrace jump skips 2 NOP instructions */
2005 if (is_kernel_text((unsigned long)orig_call) ||
2006 is_module_text_address((unsigned long)orig_call))
2007 orig_call += LOONGARCH_FENTRY_NBYTES;
2008 /* Direct jump skips 5 NOP instructions */
2009 else if (is_bpf_text_address((unsigned long)orig_call))
2010 orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
2011
2012 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2013 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
2014 ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
2015 if (ret)
2016 return ret;
2017 }
2018
2019 if (fentry->nr_nodes) {
2020 ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off,
2021 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off);
2022 if (ret)
2023 return ret;
2024 }
2025 if (fmod_ret->nr_nodes) {
2026 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32 *), GFP_KERNEL);
2027 if (!branches)
2028 return -ENOMEM;
2029
2030 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
2031 for (i = 0; i < fmod_ret->nr_nodes; i++) {
2032 ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i],
2033 args_off, retval_off, run_ctx_off, true);
2034 if (ret)
2035 goto out;
2036 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
2037 branches[i] = (u32 *)ctx->image + ctx->idx;
2038 emit_insn(ctx, nop);
2039 }
2040 }
2041
2042 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2043 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);
2044 restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_args_off);
2045
2046 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
2047 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
2048
2049 ret = emit_call(ctx, (const u64)orig_call);
2050 if (ret)
2051 goto out;
2052 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
2053 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
2054 im->ip_after_call = ctx->ro_image + ctx->idx;
2055 /* Reserve space for the move_imm + jirl instruction */
2056 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
2057 emit_insn(ctx, nop);
2058 }
2059
2060 for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) {
2061 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
2062 *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
2063 }
2064
2065 /* Set "is_return" flag for fsession */
2066 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
2067 if (bpf_fsession_cnt(tnodes))
2068 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta);
2069
2070 if (fexit->nr_nodes) {
2071 ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off,
2072 func_meta_off, false, func_meta, cookie_off);
2073 if (ret)
2074 goto out;
2075 }
2076
2077 if (flags & BPF_TRAMP_F_CALL_ORIG) {
2078 im->ip_epilogue = ctx->ro_image + ctx->idx;
2079 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
2080 ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
2081 if (ret)
2082 goto out;
2083 }
2084
2085 if (flags & BPF_TRAMP_F_RESTORE_REGS)
2086 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off);
2087
2088 if (save_ret) {
2089 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
2090 if (is_struct_ops)
2091 sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0],
2092 m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG);
2093 else
2094 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
2095 }
2096
2097 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
2098
2099 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
2100 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
2101
2102 if (is_struct_ops) {
2103 /* trampoline called directly */
2104 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
2105 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
2106 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
2107
2108 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
2109 } else {
2110 /* trampoline called from function entry */
2111 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
2112 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
2113 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
2114
2115 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
2116 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
2117 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
2118
2119 if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2120 /* return to parent function */
2121 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
2122 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
2123 } else {
2124 /* return to traced function */
2125 move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA);
2126 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0);
2127 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0);
2128 }
2129 }
2130
2131 ret = ctx->idx;
2132 out:
2133 kfree(branches);
2134
2135 return ret;
2136 }
2137
arch_prepare_bpf_trampoline(struct bpf_tramp_image * im,void * ro_image,void * ro_image_end,const struct btf_func_model * m,u32 flags,struct bpf_tramp_nodes * tnodes,void * func_addr)2138 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
2139 void *ro_image_end, const struct btf_func_model *m,
2140 u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr)
2141 {
2142 int ret, size;
2143 void *image, *tmp;
2144 struct jit_ctx ctx;
2145
2146 size = ro_image_end - ro_image;
2147 image = kvmalloc(size, GFP_KERNEL);
2148 if (!image)
2149 return -ENOMEM;
2150
2151 ctx.image = (union loongarch_instruction *)image;
2152 ctx.ro_image = (union loongarch_instruction *)ro_image;
2153 ctx.idx = 0;
2154
2155 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
2156 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tnodes, func_addr, flags);
2157 if (ret < 0)
2158 goto out;
2159
2160 if (validate_code(&ctx) < 0) {
2161 ret = -EINVAL;
2162 goto out;
2163 }
2164
2165 tmp = bpf_arch_text_copy(ro_image, image, size);
2166 if (IS_ERR(tmp)) {
2167 ret = PTR_ERR(tmp);
2168 goto out;
2169 }
2170
2171 out:
2172 kvfree(image);
2173 return ret < 0 ? ret : size;
2174 }
2175
arch_bpf_trampoline_size(const struct btf_func_model * m,u32 flags,struct bpf_tramp_nodes * tnodes,void * func_addr)2176 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
2177 struct bpf_tramp_nodes *tnodes, void *func_addr)
2178 {
2179 int ret;
2180 struct jit_ctx ctx;
2181 struct bpf_tramp_image im;
2182
2183 ctx.image = NULL;
2184 ctx.idx = 0;
2185
2186 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags);
2187
2188 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
2189 }
2190
bpf_int_jit_compile(struct bpf_verifier_env * env,struct bpf_prog * prog)2191 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
2192 {
2193 bool extra_pass = false;
2194 u8 *image_ptr, *ro_image_ptr;
2195 int image_size, prog_size, extable_size;
2196 struct jit_ctx ctx;
2197 struct jit_data *jit_data;
2198 struct bpf_binary_header *header;
2199 struct bpf_binary_header *ro_header;
2200
2201 /*
2202 * If BPF JIT was not enabled then we must fall back to
2203 * the interpreter.
2204 */
2205 if (!prog->jit_requested)
2206 return prog;
2207
2208 jit_data = prog->aux->jit_data;
2209 if (!jit_data) {
2210 jit_data = kzalloc_obj(*jit_data);
2211 if (!jit_data)
2212 return prog;
2213 prog->aux->jit_data = jit_data;
2214 }
2215 if (jit_data->ctx.offset) {
2216 ctx = jit_data->ctx;
2217 ro_header = jit_data->ro_header;
2218 ro_image_ptr = (void *)ctx.ro_image;
2219 header = jit_data->header;
2220 image_ptr = (void *)header + ((void *)ro_image_ptr - (void *)ro_header);
2221 extra_pass = true;
2222 prog_size = sizeof(u32) * ctx.idx;
2223 goto skip_init_ctx;
2224 }
2225
2226 memset(&ctx, 0, sizeof(ctx));
2227 ctx.prog = prog;
2228 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
2229 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
2230
2231 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
2232 if (ctx.offset == NULL)
2233 goto out_offset;
2234
2235 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
2236 build_prologue(&ctx);
2237 if (build_body(&ctx, extra_pass))
2238 goto out_offset;
2239 ctx.epilogue_offset = ctx.idx;
2240 build_epilogue(&ctx);
2241
2242 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry);
2243
2244 /* Now we know the actual image size.
2245 * As each LoongArch instruction is of length 32bit,
2246 * we are translating number of JITed intructions into
2247 * the size required to store these JITed code.
2248 */
2249 prog_size = sizeof(u32) * ctx.idx;
2250 image_size = prog_size + extable_size;
2251 /* Now we know the size of the structure to make */
2252 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
2253 &header, &image_ptr, jit_fill_hole,
2254 bpf_prog_was_classic(prog));
2255 if (!ro_header)
2256 goto out_offset;
2257
2258 /* 2. Now, the actual pass to generate final JIT code */
2259 /*
2260 * Use the image (RW) for writing the JITed instructions. But also save
2261 * the ro_image (RX) for calculating the offsets in the image. The RW
2262 * image will be later copied to the RX image from where the program will
2263 * run. The bpf_jit_binary_pack_finalize() will do this copy in the final
2264 * step.
2265 */
2266 ctx.image = (union loongarch_instruction *)image_ptr;
2267 ctx.ro_image = (union loongarch_instruction *)ro_image_ptr;
2268 if (extable_size)
2269 prog->aux->extable = (void *)ro_image_ptr + prog_size;
2270
2271 skip_init_ctx:
2272 ctx.idx = 0;
2273 ctx.num_exentries = 0;
2274
2275 build_prologue(&ctx);
2276 if (build_body(&ctx, extra_pass))
2277 goto out_free;
2278 build_epilogue(&ctx);
2279
2280 /* 3. Extra pass to validate JITed code */
2281 if (validate_ctx(&ctx))
2282 goto out_free;
2283
2284 /* And we're done */
2285 if (bpf_jit_enable > 1)
2286 bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
2287
2288 if (!prog->is_func || extra_pass) {
2289 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
2290 pr_err_once("multi-func JIT bug %d != %d\n",
2291 ctx.idx, jit_data->ctx.idx);
2292 goto out_free;
2293 }
2294 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2295 /* ro_header and header have been freed */
2296 ro_header = NULL;
2297 header = NULL;
2298 goto out_free;
2299 }
2300 /*
2301 * The instructions have now been copied to the ROX region from
2302 * where they will execute. Now the data cache has to be cleaned
2303 * to the PoU and the I-cache has to be invalidated for the VAs.
2304 */
2305 bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx);
2306 } else {
2307 jit_data->ctx = ctx;
2308 jit_data->header = header;
2309 jit_data->ro_header = ro_header;
2310 }
2311 prog->jited = 1;
2312 prog->jited_len = prog_size;
2313 prog->bpf_func = (void *)ctx.ro_image;
2314
2315 if (!prog->is_func || extra_pass) {
2316 int i;
2317
2318 /* offset[prog->len] is the size of program */
2319 for (i = 0; i <= prog->len; i++)
2320 ctx.offset[i] *= LOONGARCH_INSN_SIZE;
2321 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
2322
2323 out_offset:
2324 kvfree(ctx.offset);
2325 kfree(jit_data);
2326 prog->aux->jit_data = NULL;
2327 }
2328
2329 return prog;
2330
2331 out_free:
2332 if (extra_pass) {
2333 prog->bpf_func = NULL;
2334 prog->jited = 0;
2335 prog->jited_len = 0;
2336 }
2337
2338 if (header) {
2339 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));
2340 bpf_jit_binary_pack_free(ro_header, header);
2341 }
2342 goto out_offset;
2343 }
2344
bpf_jit_free(struct bpf_prog * prog)2345 void bpf_jit_free(struct bpf_prog *prog)
2346 {
2347 if (prog->jited) {
2348 struct jit_data *jit_data = prog->aux->jit_data;
2349 struct bpf_binary_header *hdr;
2350
2351 /*
2352 * If we fail the final pass of JIT (from jit_subprogs), the
2353 * program may not be finalized yet. Call finalize here before
2354 * freeing it.
2355 */
2356 if (jit_data) {
2357 bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header);
2358 kvfree(jit_data->ctx.offset);
2359 kfree(jit_data);
2360 }
2361 hdr = bpf_jit_binary_pack_hdr(prog);
2362 bpf_jit_binary_pack_free(hdr, NULL);
2363 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
2364 }
2365
2366 bpf_prog_unlock_free(prog);
2367 }
2368
2369 #if defined(CONFIG_UNWINDER_ORC)
2370 #include <asm/unwind.h>
2371
walk_bpf_stackframe(bool (* consume_fn)(void * cookie,u64 ip,u64 sp,u64 bp),void * cookie,unsigned long fp)2372 static noinline void walk_bpf_stackframe(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp),
2373 void *cookie, unsigned long fp)
2374 {
2375 unsigned long addr;
2376 struct unwind_state state;
2377 struct pt_regs dummyregs;
2378 struct pt_regs *regs = &dummyregs;
2379
2380 regs->regs[1] = 0;
2381 regs->regs[22] = fp;
2382 regs->regs[3] = (unsigned long)__builtin_frame_address(0);
2383 regs->csr_era = (unsigned long)__builtin_return_address(0);
2384
2385 for (unwind_start(&state, current, regs);
2386 !unwind_done(&state); unwind_next_frame(&state)) {
2387 addr = unwind_get_return_address(&state);
2388 if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.fp))
2389 break;
2390 }
2391 }
2392
arch_bpf_stack_walk(bool (* consume_fn)(void * cookie,u64 ip,u64 sp,u64 bp),void * cookie)2393 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
2394 {
2395 unsigned long fp;
2396
2397 /*
2398 * Capture the live frame pointer ($r22) at the very front-line before
2399 * any kernel C code clobbers it. This must be a thin wrapper with no
2400 * large stack locals to prevent the compiler from reusing $r22 early.
2401 */
2402 asm volatile("move %0, $r22" : "=r"(fp));
2403 walk_bpf_stackframe(consume_fn, cookie, fp);
2404 }
2405 #endif /* CONFIG_UNWINDER_ORC */
2406
bpf_jit_bypass_spec_v1(void)2407 bool bpf_jit_bypass_spec_v1(void)
2408 {
2409 return true;
2410 }
2411
bpf_jit_bypass_spec_v4(void)2412 bool bpf_jit_bypass_spec_v4(void)
2413 {
2414 return true;
2415 }
2416
bpf_jit_supports_arena(void)2417 bool bpf_jit_supports_arena(void)
2418 {
2419 return true;
2420 }
2421
bpf_jit_supports_fsession(void)2422 bool bpf_jit_supports_fsession(void)
2423 {
2424 return true;
2425 }
2426
bpf_jit_supports_percpu_insn(void)2427 bool bpf_jit_supports_percpu_insn(void)
2428 {
2429 return true;
2430 }
2431
bpf_jit_supports_ptr_xchg(void)2432 bool bpf_jit_supports_ptr_xchg(void)
2433 {
2434 return true;
2435 }
2436
2437 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bpf_jit_supports_subprog_tailcalls(void)2438 bool bpf_jit_supports_subprog_tailcalls(void)
2439 {
2440 return true;
2441 }
2442
bpf_jit_supports_timed_may_goto(void)2443 bool bpf_jit_supports_timed_may_goto(void)
2444 {
2445 return true;
2446 }
2447
bpf_jit_inlines_helper_call(s32 imm)2448 bool bpf_jit_inlines_helper_call(s32 imm)
2449 {
2450 switch (imm) {
2451 case BPF_FUNC_get_current_task:
2452 case BPF_FUNC_get_current_task_btf:
2453 case BPF_FUNC_get_smp_processor_id:
2454 return true;
2455 default:
2456 return false;
2457 }
2458 }
2459