Lines Matching +full:src +full:-
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Just-In-Time compiler for eBPF bytecode on MIPS.
4 * Implementation of JIT functions for 64-bit CPUs.
18 #include <asm/cpu-features.h>
19 #include <asm/isa-rev.h>
24 /* MIPS t0-t3 are not available in the n64 ABI */
30 /* Stack is 16-byte aligned in n64 ABI */
33 /* Extra 64-bit eBPF registers used by JIT */
40 /* Callee-saved CPU registers that the JIT must preserve */
54 /* Caller-saved CPU registers available for JIT use */
60 * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers.
61 * MIPS registers t4 - t7 may be used by the JIT as temporary registers.
62 * MIPS registers t8 - t9 are reserved for single-register common functions.
65 /* Return value from in-kernel function, and exit value from eBPF */
67 /* Arguments from eBPF program to in-kernel function */
73 /* Callee-saved registers that in-kernel function will preserve */
78 /* Read-only frame pointer to access the eBPF stack */
82 /* Tail call count register, caller-saved */
84 /* Constant for register zero-extension */
89 * MIPS 32-bit operations on 64-bit registers generate a sign-extended
90 * result. However, the eBPF ISA mandates zero-extension, so we rely on the
92 * operations, right shift and byte swap require properly sign-extended
93 * operands or the result is unpredictable. We emit explicit sign-extensions
98 static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) in emit_sext() argument
100 emit(ctx, sll, dst, src, 0); in emit_sext()
119 if (!ctx->program->aux->verifier_zext) in emit_zext_ver()
123 /* dst = imm (64-bit) */
138 u16 half = imm64 >> (48 - 16 * k); in emit_mov_i64()
157 /* ALU immediate operation (64-bit) */
169 /* dst = -dst */ in emit_alu_i64()
189 /* dst = dst - imm */ in emit_alu_i64()
191 emit(ctx, daddiu, dst, dst, -imm); in emit_alu_i64()
194 /* Width-generic operations */ in emit_alu_i64()
200 /* ALU register operation (64-bit) */
201 static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) in emit_alu_r64() argument
204 /* dst = dst << src */ in emit_alu_r64()
206 emit(ctx, dsllv, dst, dst, src); in emit_alu_r64()
208 /* dst = dst >> src */ in emit_alu_r64()
210 emit(ctx, dsrlv, dst, dst, src); in emit_alu_r64()
212 /* dst = dst >> src (arithmetic) */ in emit_alu_r64()
214 emit(ctx, dsrav, dst, dst, src); in emit_alu_r64()
216 /* dst = dst + src */ in emit_alu_r64()
218 emit(ctx, daddu, dst, dst, src); in emit_alu_r64()
220 /* dst = dst - src */ in emit_alu_r64()
222 emit(ctx, dsubu, dst, dst, src); in emit_alu_r64()
224 /* dst = dst * src */ in emit_alu_r64()
227 emit(ctx, dmulu, dst, dst, src); in emit_alu_r64()
229 emit(ctx, dmultu, dst, src); in emit_alu_r64()
236 /* dst = dst / src */ in emit_alu_r64()
239 emit(ctx, ddivu_r6, dst, dst, src); in emit_alu_r64()
241 emit(ctx, ddivu, dst, src); in emit_alu_r64()
245 /* dst = dst % src */ in emit_alu_r64()
248 emit(ctx, dmodu, dst, dst, src); in emit_alu_r64()
250 emit(ctx, ddivu, dst, src); in emit_alu_r64()
255 /* Width-generic operations */ in emit_alu_r64()
256 emit_alu_r(ctx, dst, src, op); in emit_alu_r64()
321 /* Zero-extend a word */ in emit_trunc_r64()
325 /* Zero-extend a half word */ in emit_trunc_r64()
333 /* Load operation: dst = *(size*)(src + off) */
334 static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) in emit_ldx() argument
339 emit(ctx, lbu, dst, off, src); in emit_ldx()
343 emit(ctx, lhu, dst, off, src); in emit_ldx()
347 emit(ctx, lwu, dst, off, src); in emit_ldx()
351 emit(ctx, ld, dst, off, src); in emit_ldx()
357 /* Store operation: *(size *)(dst + off) = src */
358 static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) in emit_stx() argument
363 emit(ctx, sb, src, off, dst); in emit_stx()
367 emit(ctx, sh, src, off, dst); in emit_stx()
371 emit(ctx, sw, src, off, dst); in emit_stx()
375 emit(ctx, sd, src, off, dst); in emit_stx()
380 /* Atomic read-modify-write */
382 u8 dst, u8 src, s16 off, u8 code) in emit_atomic_r64() argument
392 emit(ctx, daddu, t2, t1, src); in emit_atomic_r64()
396 emit(ctx, and, t2, t1, src); in emit_atomic_r64()
400 emit(ctx, or, t2, t1, src); in emit_atomic_r64()
404 emit(ctx, xor, t2, t1, src); in emit_atomic_r64()
407 emit(ctx, move, t2, src); in emit_atomic_r64()
411 emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); in emit_atomic_r64()
415 emit(ctx, move, src, t1); in emit_atomic_r64()
416 clobber_reg(ctx, src); in emit_atomic_r64()
420 /* Atomic compare-and-exchange */
421 static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) in emit_cmpxchg_r64() argument
430 emit(ctx, move, t2, src); /* Delay slot */ in emit_cmpxchg_r64()
432 emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset); in emit_cmpxchg_r64()
447 if (bpf_jit_get_func_addr(ctx->program, insn, false, in emit_call()
449 return -1; in emit_call()
451 return -1; in emit_call()
453 /* Push caller-saved registers on stack */ in emit_call()
454 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); in emit_call()
461 /* Restore caller-saved registers */ in emit_call()
462 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); in emit_call()
464 /* Re-initialize the JIT zero-extension register if accessed */ in emit_call()
465 if (ctx->accessed & BIT(JIT_REG_ZX)) { in emit_call()
466 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); in emit_call()
487 * eBPF R1 - function argument (context ptr), passed in a0-a1 in emit_tail_call()
488 * eBPF R2 - ptr to object with array of function entry points in emit_tail_call()
489 * eBPF R3 - array index of function to be called in emit_tail_call()
492 /* if (ind >= ary->map.max_entries) goto out */ in emit_tail_call()
495 return -1; in emit_tail_call()
496 emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ in emit_tail_call()
500 /* if (--TCC < 0) goto out */ in emit_tail_call()
501 emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ in emit_tail_call()
504 /* prog = ary->ptrs[ind] */ in emit_tail_call()
507 return -1; in emit_tail_call()
516 /* func = prog->bpf_func + 8 (prologue skip offset) */ in emit_tail_call()
519 return -1; in emit_tail_call()
533 * +===========================+ <--- MIPS sp before call
534 * | Callee-saved registers, |
536 * +---------------------------+ <--- eBPF FP (MIPS fp)
539 * +---------------------------+
540 * | Reserved for caller-saved |
542 * Lower address +===========================+ <--- MIPS sp
568 /* === Entry-point for tail calls === */ in build_prologue()
575 if (ctx->accessed & BIT(BPF_REG_FP)) in build_prologue()
577 if (ctx->accessed & BIT(JIT_REG_TC)) in build_prologue()
579 if (ctx->accessed & BIT(JIT_REG_ZX)) in build_prologue()
582 /* Compute the stack space needed for callee-saved registers */ in build_prologue()
583 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); in build_prologue()
587 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); in build_prologue()
591 * caller-saved registers needed by the JIT. The required space is in build_prologue()
594 reserved = ctx->stack_used; in build_prologue()
599 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); in build_prologue()
601 /* Store callee-saved registers on stack */ in build_prologue()
602 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); in build_prologue()
605 if (ctx->accessed & BIT(BPF_REG_FP)) in build_prologue()
606 emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); in build_prologue()
608 /* Initialize the ePF JIT zero-extension register if accessed */ in build_prologue()
609 if (ctx->accessed & BIT(JIT_REG_ZX)) { in build_prologue()
610 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); in build_prologue()
614 ctx->saved_size = saved; in build_prologue()
615 ctx->stack_size = stack; in build_prologue()
621 /* Restore callee-saved registers from stack */ in build_epilogue()
622 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, in build_epilogue()
623 ctx->stack_size - ctx->saved_size); in build_epilogue()
626 if (ctx->stack_size) in build_epilogue()
627 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); in build_epilogue()
629 /* Jump to return address and sign-extend the 32-bit return value */ in build_epilogue()
637 u8 dst = bpf2mips64[insn->dst_reg]; in build_insn()
638 u8 src = bpf2mips64[insn->src_reg]; in build_insn() local
640 u8 code = insn->code; in build_insn()
641 s16 off = insn->off; in build_insn()
642 s32 imm = insn->imm; in build_insn()
653 /* dst = src */ in build_insn()
659 emit_mov_r(ctx, dst, src); in build_insn()
663 /* dst = -dst */ in build_insn()
688 /* dst = dst - imm */ in build_insn()
709 /* dst = dst & src */ in build_insn()
710 /* dst = dst | src */ in build_insn()
711 /* dst = dst ^ src */ in build_insn()
712 /* dst = dst << src */ in build_insn()
717 emit_alu_r(ctx, dst, src, BPF_OP(code)); in build_insn()
720 /* dst = dst >> src */ in build_insn()
721 /* dst = dst >> src (arithmetic) */ in build_insn()
722 /* dst = dst + src */ in build_insn()
723 /* dst = dst - src */ in build_insn()
724 /* dst = dst * src */ in build_insn()
725 /* dst = dst / src */ in build_insn()
726 /* dst = dst % src */ in build_insn()
735 emit_sext(ctx, MIPS_R_T4, src); in build_insn()
739 /* dst = imm (64-bit) */ in build_insn()
743 /* dst = src (64-bit) */ in build_insn()
745 emit_mov_r(ctx, dst, src); in build_insn()
747 /* dst = -dst (64-bit) */ in build_insn()
751 /* dst = dst & imm (64-bit) */ in build_insn()
752 /* dst = dst | imm (64-bit) */ in build_insn()
753 /* dst = dst ^ imm (64-bit) */ in build_insn()
754 /* dst = dst << imm (64-bit) */ in build_insn()
755 /* dst = dst >> imm (64-bit) */ in build_insn()
756 /* dst = dst >> imm ((64-bit, arithmetic) */ in build_insn()
757 /* dst = dst + imm (64-bit) */ in build_insn()
758 /* dst = dst - imm (64-bit) */ in build_insn()
759 /* dst = dst * imm (64-bit) */ in build_insn()
760 /* dst = dst / imm (64-bit) */ in build_insn()
761 /* dst = dst % imm (64-bit) */ in build_insn()
780 /* dst = dst & src (64-bit) */ in build_insn()
781 /* dst = dst | src (64-bit) */ in build_insn()
782 /* dst = dst ^ src (64-bit) */ in build_insn()
783 /* dst = dst << src (64-bit) */ in build_insn()
784 /* dst = dst >> src (64-bit) */ in build_insn()
785 /* dst = dst >> src (64-bit, arithmetic) */ in build_insn()
786 /* dst = dst + src (64-bit) */ in build_insn()
787 /* dst = dst - src (64-bit) */ in build_insn()
788 /* dst = dst * src (64-bit) */ in build_insn()
789 /* dst = dst / src (64-bit) */ in build_insn()
790 /* dst = dst % src (64-bit) */ in build_insn()
802 emit_alu_r64(ctx, dst, src, BPF_OP(code)); in build_insn()
823 /* LDX: dst = *(size *)(src + off) */ in build_insn()
828 emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); in build_insn()
838 /* STX: *(size *)(dst + off) = src */ in build_insn()
843 emit_stx(ctx, dst, src, off, BPF_SIZE(code)); in build_insn()
862 emit_atomic_r64(ctx, dst, src, off, imm); in build_insn()
866 if (src == dst) { /* Don't overwrite dst */ in build_insn()
870 emit_sext(ctx, src, src); in build_insn()
871 emit_atomic_r(ctx, tmp, src, off, imm); in build_insn()
872 emit_zext_ver(ctx, src); in build_insn()
873 } else { /* 32-bit, no fetch */ in build_insn()
874 emit_sext(ctx, MIPS_R_T4, src); in build_insn()
880 emit_cmpxchg_r64(ctx, dst, src, off); in build_insn()
887 emit_sext(ctx, MIPS_R_T5, src); in build_insn()
898 /* PC += off if dst == src */ in build_insn()
899 /* PC += off if dst != src */ in build_insn()
900 /* PC += off if dst & src */ in build_insn()
901 /* PC += off if dst > src */ in build_insn()
902 /* PC += off if dst >= src */ in build_insn()
903 /* PC += off if dst < src */ in build_insn()
904 /* PC += off if dst <= src */ in build_insn()
905 /* PC += off if dst > src (signed) */ in build_insn()
906 /* PC += off if dst >= src (signed) */ in build_insn()
907 /* PC += off if dst < src (signed) */ in build_insn()
908 /* PC += off if dst <= src (signed) */ in build_insn()
922 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
923 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ in build_insn()
924 emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ in build_insn()
954 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ in build_insn()
958 /* Move large immediate to register, sign-extended */ in build_insn()
965 /* PC += off if dst == src */ in build_insn()
966 /* PC += off if dst != src */ in build_insn()
967 /* PC += off if dst & src */ in build_insn()
968 /* PC += off if dst > src */ in build_insn()
969 /* PC += off if dst >= src */ in build_insn()
970 /* PC += off if dst < src */ in build_insn()
971 /* PC += off if dst <= src */ in build_insn()
972 /* PC += off if dst > src (signed) */ in build_insn()
973 /* PC += off if dst >= src (signed) */ in build_insn()
974 /* PC += off if dst < src (signed) */ in build_insn()
975 /* PC += off if dst <= src (signed) */ in build_insn()
989 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
990 emit_jmp_r(ctx, dst, src, rel, jmp); in build_insn()
1052 if (ctx->bpf_index == ctx->program->len - 1) in build_insn()
1061 return -EINVAL; in build_insn()
1064 return -EFAULT; in build_insn()
1067 ctx->bpf_index, code); in build_insn()
1068 return -E2BIG; in build_insn()