xref: /linux/arch/mips/net/bpf_jit_comp.h (revision 03ab8e6297acd1bc0eedaa050e2a1635c576fd11)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS.
4  *
5  * Copyright (c) 2021 Anyfi Networks AB.
6  * Author: Johan Almbladh <johan.almbladh@gmail.com>
7  *
8  * Based on code and ideas from
9  * Copyright (c) 2017 Cavium, Inc.
10  * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
11  * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
12  */
13 
14 #ifndef _BPF_JIT_COMP_H
15 #define _BPF_JIT_COMP_H
16 
17 /* MIPS registers */
18 #define MIPS_R_ZERO	0   /* Const zero */
19 #define MIPS_R_AT	1   /* Asm temp   */
20 #define MIPS_R_V0	2   /* Result     */
21 #define MIPS_R_V1	3   /* Result     */
22 #define MIPS_R_A0	4   /* Argument   */
23 #define MIPS_R_A1	5   /* Argument   */
24 #define MIPS_R_A2	6   /* Argument   */
25 #define MIPS_R_A3	7   /* Argument   */
26 #define MIPS_R_A4	8   /* Arg (n64)  */
27 #define MIPS_R_A5	9   /* Arg (n64)  */
28 #define MIPS_R_A6	10  /* Arg (n64)  */
29 #define MIPS_R_A7	11  /* Arg (n64)  */
30 #define MIPS_R_T0	8   /* Temp (o32) */
31 #define MIPS_R_T1	9   /* Temp (o32) */
32 #define MIPS_R_T2	10  /* Temp (o32) */
33 #define MIPS_R_T3	11  /* Temp (o32) */
34 #define MIPS_R_T4	12  /* Temporary  */
35 #define MIPS_R_T5	13  /* Temporary  */
36 #define MIPS_R_T6	14  /* Temporary  */
37 #define MIPS_R_T7	15  /* Temporary  */
38 #define MIPS_R_S0	16  /* Saved      */
39 #define MIPS_R_S1	17  /* Saved      */
40 #define MIPS_R_S2	18  /* Saved      */
41 #define MIPS_R_S3	19  /* Saved      */
42 #define MIPS_R_S4	20  /* Saved      */
43 #define MIPS_R_S5	21  /* Saved      */
44 #define MIPS_R_S6	22  /* Saved      */
45 #define MIPS_R_S7	23  /* Saved      */
46 #define MIPS_R_T8	24  /* Temporary  */
47 #define MIPS_R_T9	25  /* Temporary  */
48 /*      MIPS_R_K0	26     Reserved   */
49 /*      MIPS_R_K1	27     Reserved   */
50 #define MIPS_R_GP	28  /* Global ptr */
51 #define MIPS_R_SP	29  /* Stack ptr  */
52 #define MIPS_R_FP	30  /* Frame ptr  */
53 #define MIPS_R_RA	31  /* Return     */
54 
55 /*
56  * Jump address mask for immediate jumps. The four most significant bits
57  * must be equal to PC.
58  */
59 #define MIPS_JMP_MASK	0x0fffffffUL
60 
61 /* Maximum number of iterations in offset table computation */
62 #define JIT_MAX_ITERATIONS	8
63 
64 /*
65  * Jump pseudo-instructions used internally
66  * for branch conversion and branch optimization.
67  */
68 #define JIT_JNSET	0xe0
69 #define JIT_JNOP	0xf0
70 
71 /* Descriptor flag for PC-relative branch conversion */
72 #define JIT_DESC_CONVERT	BIT(31)
73 
74 /* JIT context for an eBPF program */
75 struct jit_context {
76 	struct bpf_prog *program;     /* The eBPF program being JITed        */
77 	u32 *descriptors;             /* eBPF to JITed CPU insn descriptors  */
78 	u32 *target;                  /* JITed code buffer                   */
79 	u32 bpf_index;                /* Index of current BPF program insn   */
80 	u32 jit_index;                /* Index of current JIT target insn    */
81 	u32 changes;                  /* Number of PC-relative branch conv   */
82 	u32 accessed;                 /* Bit mask of read eBPF registers     */
83 	u32 clobbered;                /* Bit mask of modified CPU registers  */
84 	u32 stack_size;               /* Total allocated stack size in bytes */
85 	u32 saved_size;               /* Size of callee-saved registers      */
86 	u32 stack_used;               /* Stack size used for function calls  */
87 };
88 
89 /* Emit the instruction if the JIT memory space has been allocated */
90 #define __emit(ctx, func, ...)					\
91 do {								\
92 	if ((ctx)->target != NULL) {				\
93 		u32 *p = &(ctx)->target[ctx->jit_index];	\
94 		uasm_i_##func(&p, ##__VA_ARGS__);		\
95 	}							\
96 	(ctx)->jit_index++;					\
97 } while (0)
98 #define emit(...) __emit(__VA_ARGS__)
99 
100 /* Workaround for R10000 ll/sc errata */
101 #ifdef CONFIG_WAR_R10000_LLSC
102 #define LLSC_beqz	beqzl
103 #else
104 #define LLSC_beqz	beqz
105 #endif
106 
107 /* Workaround for Loongson-3 ll/sc errata */
108 #ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
109 #define LLSC_sync(ctx)	emit(ctx, sync, 0)
110 #define LLSC_offset	4
111 #else
112 #define LLSC_sync(ctx)
113 #define LLSC_offset	0
114 #endif
115 
116 /* Workaround for Loongson-2F jump errata */
117 #ifdef CONFIG_CPU_JUMP_WORKAROUNDS
118 #define JALR_MASK	0xffffffffcfffffffULL
119 #else
120 #define JALR_MASK	(~0ULL)
121 #endif
122 
123 /*
124  * Mark a BPF register as accessed, it needs to be
125  * initialized by the program if expected, e.g. FP.
126  */
access_reg(struct jit_context * ctx,u8 reg)127 static inline void access_reg(struct jit_context *ctx, u8 reg)
128 {
129 	ctx->accessed |= BIT(reg);
130 }
131 
132 /*
133  * Mark a CPU register as clobbered, it needs to be
134  * saved/restored by the program if callee-saved.
135  */
clobber_reg(struct jit_context * ctx,u8 reg)136 static inline void clobber_reg(struct jit_context *ctx, u8 reg)
137 {
138 	ctx->clobbered |= BIT(reg);
139 }
140 
141 /*
142  * Push registers on the stack, starting at a given depth from the stack
143  * pointer and increasing. The next depth to be written is returned.
144  */
145 int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
146 
147 /*
148  * Pop registers from the stack, starting at a given depth from the stack
149  * pointer and increasing. The next depth to be read is returned.
150  */
151 int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
152 
153 /* Compute the 28-bit jump target address from a BPF program location */
154 int get_target(struct jit_context *ctx, u32 loc);
155 
156 /* Compute the PC-relative offset to relative BPF program offset */
157 int get_offset(const struct jit_context *ctx, int off);
158 
159 /* dst = imm (32-bit) */
160 void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm);
161 
162 /* dst = src (32-bit) */
163 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src);
164 
165 /* Validate ALU/ALU64 immediate range */
166 bool valid_alu_i(u8 op, s32 imm);
167 
168 /* Rewrite ALU/ALU64 immediate operation */
169 bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
170 
171 /* ALU immediate operation (32-bit) */
172 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op);
173 
174 /* ALU register operation (32-bit) */
175 void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op);
176 
177 /* Atomic read-modify-write (32-bit) */
178 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code);
179 
180 /* Atomic compare-and-exchange (32-bit) */
181 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off);
182 
183 /* Swap bytes and truncate a register word or half word */
184 void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width);
185 
186 /* Validate JMP/JMP32 immediate range */
187 bool valid_jmp_i(u8 op, s32 imm);
188 
189 /* Prepare a PC-relative jump operation with immediate conditional */
190 void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width,
191 		 u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
192 
193 /* Prepare a PC-relative jump operation with register conditional */
194 void setup_jmp_r(struct jit_context *ctx, bool same_reg,
195 		 u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
196 
197 /* Finish a PC-relative jump operation */
198 int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off);
199 
200 /* Conditional JMP/JMP32 immediate */
201 void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op);
202 
203 /* Conditional JMP/JMP32 register */
204 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op);
205 
206 /* Jump always */
207 int emit_ja(struct jit_context *ctx, s16 off);
208 
209 /* Jump to epilogue */
210 int emit_exit(struct jit_context *ctx);
211 
212 /*
213  * Build program prologue to set up the stack and registers.
214  * This function is implemented separately for 32-bit and 64-bit JITs.
215  */
216 void build_prologue(struct jit_context *ctx);
217 
218 /*
219  * Build the program epilogue to restore the stack and registers.
220  * This function is implemented separately for 32-bit and 64-bit JITs.
221  */
222 void build_epilogue(struct jit_context *ctx, int dest_reg);
223 
224 /*
225  * Convert an eBPF instruction to native instruction, i.e
226  * JITs an eBPF instruction.
227  * Returns :
228  *	0  - Successfully JITed an 8-byte eBPF instruction
229  *	>0 - Successfully JITed a 16-byte eBPF instruction
230  *	<0 - Failed to JIT.
231  * This function is implemented separately for 32-bit and 64-bit JITs.
232  */
233 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx);
234 
235 #endif /* _BPF_JIT_COMP_H */
236