1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * BPF JIT compiler for RV32G
4 *
5 * Copyright (c) 2020 Luke Nelson <luke.r.nels@gmail.com>
6 * Copyright (c) 2020 Xi Wang <xi.wang@gmail.com>
7 *
8 * The code is based on the BPF JIT compiler for RV64G by Björn Töpel and
9 * the BPF JIT compiler for 32-bit ARM by Shubham Bansal and Mircea Gherzan.
10 */
11
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include "bpf_jit.h"
15
16 /*
17 * Stack layout during BPF program execution:
18 *
19 * high
20 * RV32 fp => +----------+
21 * | saved ra |
22 * | saved fp | RV32 callee-saved registers
23 * | ... |
24 * +----------+ <= (fp - 4 * NR_SAVED_REGISTERS)
25 * | hi(R6) |
26 * | lo(R6) |
27 * | hi(R7) | JIT scratch space for BPF registers
28 * | lo(R7) |
29 * | ... |
30 * BPF_REG_FP => +----------+ <= (fp - 4 * NR_SAVED_REGISTERS
31 * | | - 4 * BPF_JIT_SCRATCH_REGS)
32 * | |
33 * | ... | BPF program stack
34 * | |
35 * RV32 sp => +----------+
36 * | |
37 * | ... | Function call stack
38 * | |
39 * +----------+
40 * low
41 */
42
43 enum {
44 /* Stack layout - these are offsets from top of JIT scratch space. */
45 BPF_R6_HI,
46 BPF_R6_LO,
47 BPF_R7_HI,
48 BPF_R7_LO,
49 BPF_R8_HI,
50 BPF_R8_LO,
51 BPF_R9_HI,
52 BPF_R9_LO,
53 BPF_AX_HI,
54 BPF_AX_LO,
55 /* Stack space for BPF_REG_6 through BPF_REG_9 and BPF_REG_AX. */
56 BPF_JIT_SCRATCH_REGS,
57 };
58
59 /* Number of callee-saved registers stored to stack: ra, fp, s1--s7. */
60 #define NR_SAVED_REGISTERS 9
61
62 /* Offset from fp for BPF registers stored on stack. */
63 #define STACK_OFFSET(k) (-4 - (4 * NR_SAVED_REGISTERS) - (4 * (k)))
64
65 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
66 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
67
68 #define RV_REG_TCC RV_REG_T6
69 #define RV_REG_TCC_SAVED RV_REG_S7
70
71 static const s8 bpf2rv32[][2] = {
72 /* Return value from in-kernel function, and exit value from eBPF. */
73 [BPF_REG_0] = {RV_REG_S2, RV_REG_S1},
74 /* Arguments from eBPF program to in-kernel function. */
75 [BPF_REG_1] = {RV_REG_A1, RV_REG_A0},
76 [BPF_REG_2] = {RV_REG_A3, RV_REG_A2},
77 [BPF_REG_3] = {RV_REG_A5, RV_REG_A4},
78 [BPF_REG_4] = {RV_REG_A7, RV_REG_A6},
79 [BPF_REG_5] = {RV_REG_S4, RV_REG_S3},
80 /*
81 * Callee-saved registers that in-kernel function will preserve.
82 * Stored on the stack.
83 */
84 [BPF_REG_6] = {STACK_OFFSET(BPF_R6_HI), STACK_OFFSET(BPF_R6_LO)},
85 [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)},
86 [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)},
87 [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)},
88 /* Read-only frame pointer to access BPF stack. */
89 [BPF_REG_FP] = {RV_REG_S6, RV_REG_S5},
90 /* Temporary register for blinding constants. Stored on the stack. */
91 [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)},
92 /*
93 * Temporary registers used by the JIT to operate on registers stored
94 * on the stack. Save t0 and t1 to be used as temporaries in generated
95 * code.
96 */
97 [TMP_REG_1] = {RV_REG_T3, RV_REG_T2},
98 [TMP_REG_2] = {RV_REG_T5, RV_REG_T4},
99 };
100
hi(const s8 * r)101 static s8 hi(const s8 *r)
102 {
103 return r[0];
104 }
105
lo(const s8 * r)106 static s8 lo(const s8 *r)
107 {
108 return r[1];
109 }
110
emit_imm(const s8 rd,s32 imm,struct rv_jit_context * ctx)111 static void emit_imm(const s8 rd, s32 imm, struct rv_jit_context *ctx)
112 {
113 u32 upper = (imm + (1 << 11)) >> 12;
114 u32 lower = imm & 0xfff;
115
116 if (upper) {
117 emit(rv_lui(rd, upper), ctx);
118 emit(rv_addi(rd, rd, lower), ctx);
119 } else {
120 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
121 }
122 }
123
emit_imm32(const s8 * rd,s32 imm,struct rv_jit_context * ctx)124 static void emit_imm32(const s8 *rd, s32 imm, struct rv_jit_context *ctx)
125 {
126 /* Emit immediate into lower bits. */
127 emit_imm(lo(rd), imm, ctx);
128
129 /* Sign-extend into upper bits. */
130 if (imm >= 0)
131 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
132 else
133 emit(rv_addi(hi(rd), RV_REG_ZERO, -1), ctx);
134 }
135
emit_imm64(const s8 * rd,s32 imm_hi,s32 imm_lo,struct rv_jit_context * ctx)136 static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo,
137 struct rv_jit_context *ctx)
138 {
139 emit_imm(lo(rd), imm_lo, ctx);
140 emit_imm(hi(rd), imm_hi, ctx);
141 }
142
__build_epilogue(bool is_tail_call,struct rv_jit_context * ctx)143 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
144 {
145 int stack_adjust = ctx->stack_size;
146 const s8 *r0 = bpf2rv32[BPF_REG_0];
147
148 /* Set return value if not tail call. */
149 if (!is_tail_call) {
150 emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx);
151 emit(rv_addi(RV_REG_A1, hi(r0), 0), ctx);
152 }
153
154 /* Restore callee-saved registers. */
155 emit(rv_lw(RV_REG_RA, stack_adjust - 4, RV_REG_SP), ctx);
156 emit(rv_lw(RV_REG_FP, stack_adjust - 8, RV_REG_SP), ctx);
157 emit(rv_lw(RV_REG_S1, stack_adjust - 12, RV_REG_SP), ctx);
158 emit(rv_lw(RV_REG_S2, stack_adjust - 16, RV_REG_SP), ctx);
159 emit(rv_lw(RV_REG_S3, stack_adjust - 20, RV_REG_SP), ctx);
160 emit(rv_lw(RV_REG_S4, stack_adjust - 24, RV_REG_SP), ctx);
161 emit(rv_lw(RV_REG_S5, stack_adjust - 28, RV_REG_SP), ctx);
162 emit(rv_lw(RV_REG_S6, stack_adjust - 32, RV_REG_SP), ctx);
163 emit(rv_lw(RV_REG_S7, stack_adjust - 36, RV_REG_SP), ctx);
164
165 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
166
167 if (is_tail_call) {
168 /*
169 * goto *(t0 + 4);
170 * Skips first instruction of prologue which initializes tail
171 * call counter. Assumes t0 contains address of target program,
172 * see emit_bpf_tail_call.
173 */
174 emit(rv_jalr(RV_REG_ZERO, RV_REG_T0, 4), ctx);
175 } else {
176 emit(rv_jalr(RV_REG_ZERO, RV_REG_RA, 0), ctx);
177 }
178 }
179
is_stacked(s8 reg)180 static bool is_stacked(s8 reg)
181 {
182 return reg < 0;
183 }
184
bpf_get_reg64(const s8 * reg,const s8 * tmp,struct rv_jit_context * ctx)185 static const s8 *bpf_get_reg64(const s8 *reg, const s8 *tmp,
186 struct rv_jit_context *ctx)
187 {
188 if (is_stacked(hi(reg))) {
189 emit(rv_lw(hi(tmp), hi(reg), RV_REG_FP), ctx);
190 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
191 reg = tmp;
192 }
193 return reg;
194 }
195
bpf_put_reg64(const s8 * reg,const s8 * src,struct rv_jit_context * ctx)196 static void bpf_put_reg64(const s8 *reg, const s8 *src,
197 struct rv_jit_context *ctx)
198 {
199 if (is_stacked(hi(reg))) {
200 emit(rv_sw(RV_REG_FP, hi(reg), hi(src)), ctx);
201 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
202 }
203 }
204
bpf_get_reg32(const s8 * reg,const s8 * tmp,struct rv_jit_context * ctx)205 static const s8 *bpf_get_reg32(const s8 *reg, const s8 *tmp,
206 struct rv_jit_context *ctx)
207 {
208 if (is_stacked(lo(reg))) {
209 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
210 reg = tmp;
211 }
212 return reg;
213 }
214
bpf_put_reg32(const s8 * reg,const s8 * src,struct rv_jit_context * ctx)215 static void bpf_put_reg32(const s8 *reg, const s8 *src,
216 struct rv_jit_context *ctx)
217 {
218 if (is_stacked(lo(reg))) {
219 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
220 if (!ctx->prog->aux->verifier_zext)
221 emit(rv_sw(RV_REG_FP, hi(reg), RV_REG_ZERO), ctx);
222 } else if (!ctx->prog->aux->verifier_zext) {
223 emit(rv_addi(hi(reg), RV_REG_ZERO, 0), ctx);
224 }
225 }
226
emit_jump_and_link(u8 rd,s32 rvoff,bool force_jalr,struct rv_jit_context * ctx)227 static void emit_jump_and_link(u8 rd, s32 rvoff, bool force_jalr,
228 struct rv_jit_context *ctx)
229 {
230 s32 upper, lower;
231
232 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
233 emit(rv_jal(rd, rvoff >> 1), ctx);
234 return;
235 }
236
237 upper = (rvoff + (1 << 11)) >> 12;
238 lower = rvoff & 0xfff;
239 emit(rv_auipc(RV_REG_T1, upper), ctx);
240 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
241 }
242
emit_alu_i64(const s8 * dst,s32 imm,struct rv_jit_context * ctx,const u8 op)243 static void emit_alu_i64(const s8 *dst, s32 imm,
244 struct rv_jit_context *ctx, const u8 op)
245 {
246 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
247 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
248
249 switch (op) {
250 case BPF_MOV:
251 emit_imm32(rd, imm, ctx);
252 break;
253 case BPF_AND:
254 if (is_12b_int(imm)) {
255 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
256 } else {
257 emit_imm(RV_REG_T0, imm, ctx);
258 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
259 }
260 if (imm >= 0)
261 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
262 break;
263 case BPF_OR:
264 if (is_12b_int(imm)) {
265 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
266 } else {
267 emit_imm(RV_REG_T0, imm, ctx);
268 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
269 }
270 if (imm < 0)
271 emit(rv_ori(hi(rd), RV_REG_ZERO, -1), ctx);
272 break;
273 case BPF_XOR:
274 if (is_12b_int(imm)) {
275 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
276 } else {
277 emit_imm(RV_REG_T0, imm, ctx);
278 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
279 }
280 if (imm < 0)
281 emit(rv_xori(hi(rd), hi(rd), -1), ctx);
282 break;
283 case BPF_LSH:
284 if (imm >= 32) {
285 emit(rv_slli(hi(rd), lo(rd), imm - 32), ctx);
286 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
287 } else if (imm == 0) {
288 /* Do nothing. */
289 } else {
290 emit(rv_srli(RV_REG_T0, lo(rd), 32 - imm), ctx);
291 emit(rv_slli(hi(rd), hi(rd), imm), ctx);
292 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
293 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
294 }
295 break;
296 case BPF_RSH:
297 if (imm >= 32) {
298 emit(rv_srli(lo(rd), hi(rd), imm - 32), ctx);
299 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
300 } else if (imm == 0) {
301 /* Do nothing. */
302 } else {
303 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
304 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
305 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
306 emit(rv_srli(hi(rd), hi(rd), imm), ctx);
307 }
308 break;
309 case BPF_ARSH:
310 if (imm >= 32) {
311 emit(rv_srai(lo(rd), hi(rd), imm - 32), ctx);
312 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
313 } else if (imm == 0) {
314 /* Do nothing. */
315 } else {
316 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
317 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
318 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
319 emit(rv_srai(hi(rd), hi(rd), imm), ctx);
320 }
321 break;
322 }
323
324 bpf_put_reg64(dst, rd, ctx);
325 }
326
emit_alu_i32(const s8 * dst,s32 imm,struct rv_jit_context * ctx,const u8 op)327 static void emit_alu_i32(const s8 *dst, s32 imm,
328 struct rv_jit_context *ctx, const u8 op)
329 {
330 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
331 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
332
333 switch (op) {
334 case BPF_MOV:
335 emit_imm(lo(rd), imm, ctx);
336 break;
337 case BPF_ADD:
338 if (is_12b_int(imm)) {
339 emit(rv_addi(lo(rd), lo(rd), imm), ctx);
340 } else {
341 emit_imm(RV_REG_T0, imm, ctx);
342 emit(rv_add(lo(rd), lo(rd), RV_REG_T0), ctx);
343 }
344 break;
345 case BPF_SUB:
346 if (is_12b_int(-imm)) {
347 emit(rv_addi(lo(rd), lo(rd), -imm), ctx);
348 } else {
349 emit_imm(RV_REG_T0, imm, ctx);
350 emit(rv_sub(lo(rd), lo(rd), RV_REG_T0), ctx);
351 }
352 break;
353 case BPF_AND:
354 if (is_12b_int(imm)) {
355 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
356 } else {
357 emit_imm(RV_REG_T0, imm, ctx);
358 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
359 }
360 break;
361 case BPF_OR:
362 if (is_12b_int(imm)) {
363 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
364 } else {
365 emit_imm(RV_REG_T0, imm, ctx);
366 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
367 }
368 break;
369 case BPF_XOR:
370 if (is_12b_int(imm)) {
371 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
372 } else {
373 emit_imm(RV_REG_T0, imm, ctx);
374 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
375 }
376 break;
377 case BPF_LSH:
378 if (is_12b_int(imm)) {
379 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
380 } else {
381 emit_imm(RV_REG_T0, imm, ctx);
382 emit(rv_sll(lo(rd), lo(rd), RV_REG_T0), ctx);
383 }
384 break;
385 case BPF_RSH:
386 if (is_12b_int(imm)) {
387 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
388 } else {
389 emit_imm(RV_REG_T0, imm, ctx);
390 emit(rv_srl(lo(rd), lo(rd), RV_REG_T0), ctx);
391 }
392 break;
393 case BPF_ARSH:
394 if (is_12b_int(imm)) {
395 emit(rv_srai(lo(rd), lo(rd), imm), ctx);
396 } else {
397 emit_imm(RV_REG_T0, imm, ctx);
398 emit(rv_sra(lo(rd), lo(rd), RV_REG_T0), ctx);
399 }
400 break;
401 }
402
403 bpf_put_reg32(dst, rd, ctx);
404 }
405
emit_alu_r64(const s8 * dst,const s8 * src,struct rv_jit_context * ctx,const u8 op)406 static void emit_alu_r64(const s8 *dst, const s8 *src,
407 struct rv_jit_context *ctx, const u8 op)
408 {
409 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
410 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
411 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
412 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
413
414 switch (op) {
415 case BPF_MOV:
416 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
417 emit(rv_addi(hi(rd), hi(rs), 0), ctx);
418 break;
419 case BPF_ADD:
420 if (rd == rs) {
421 emit(rv_srli(RV_REG_T0, lo(rd), 31), ctx);
422 emit(rv_slli(hi(rd), hi(rd), 1), ctx);
423 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
424 emit(rv_slli(lo(rd), lo(rd), 1), ctx);
425 } else {
426 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
427 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
428 emit(rv_add(hi(rd), hi(rd), hi(rs)), ctx);
429 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
430 }
431 break;
432 case BPF_SUB:
433 emit(rv_sub(RV_REG_T1, hi(rd), hi(rs)), ctx);
434 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
435 emit(rv_sub(hi(rd), RV_REG_T1, RV_REG_T0), ctx);
436 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
437 break;
438 case BPF_AND:
439 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
440 emit(rv_and(hi(rd), hi(rd), hi(rs)), ctx);
441 break;
442 case BPF_OR:
443 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
444 emit(rv_or(hi(rd), hi(rd), hi(rs)), ctx);
445 break;
446 case BPF_XOR:
447 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
448 emit(rv_xor(hi(rd), hi(rd), hi(rs)), ctx);
449 break;
450 case BPF_MUL:
451 emit(rv_mul(RV_REG_T0, hi(rs), lo(rd)), ctx);
452 emit(rv_mul(hi(rd), hi(rd), lo(rs)), ctx);
453 emit(rv_mulhu(RV_REG_T1, lo(rd), lo(rs)), ctx);
454 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
455 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
456 emit(rv_add(hi(rd), hi(rd), RV_REG_T1), ctx);
457 break;
458 case BPF_LSH:
459 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
460 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
461 emit(rv_sll(hi(rd), lo(rd), RV_REG_T0), ctx);
462 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
463 emit(rv_jal(RV_REG_ZERO, 16), ctx);
464 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
465 emit(rv_srli(RV_REG_T0, lo(rd), 1), ctx);
466 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
467 emit(rv_srl(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
468 emit(rv_sll(hi(rd), hi(rd), lo(rs)), ctx);
469 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
470 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
471 break;
472 case BPF_RSH:
473 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
474 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
475 emit(rv_srl(lo(rd), hi(rd), RV_REG_T0), ctx);
476 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
477 emit(rv_jal(RV_REG_ZERO, 16), ctx);
478 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
479 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
480 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
481 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
482 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
483 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
484 emit(rv_srl(hi(rd), hi(rd), lo(rs)), ctx);
485 break;
486 case BPF_ARSH:
487 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
488 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
489 emit(rv_sra(lo(rd), hi(rd), RV_REG_T0), ctx);
490 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
491 emit(rv_jal(RV_REG_ZERO, 16), ctx);
492 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
493 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
494 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
495 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
496 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
497 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
498 emit(rv_sra(hi(rd), hi(rd), lo(rs)), ctx);
499 break;
500 case BPF_NEG:
501 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
502 emit(rv_sltu(RV_REG_T0, RV_REG_ZERO, lo(rd)), ctx);
503 emit(rv_sub(hi(rd), RV_REG_ZERO, hi(rd)), ctx);
504 emit(rv_sub(hi(rd), hi(rd), RV_REG_T0), ctx);
505 break;
506 }
507
508 bpf_put_reg64(dst, rd, ctx);
509 }
510
emit_alu_r32(const s8 * dst,const s8 * src,struct rv_jit_context * ctx,const struct bpf_insn * insn)511 static void emit_alu_r32(const s8 *dst, const s8 *src,
512 struct rv_jit_context *ctx,
513 const struct bpf_insn *insn)
514 {
515 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
516 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
517 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
518 const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
519 u8 op = BPF_OP(insn->code);
520 bool is_signed = insn->off == 1;
521
522 switch (op) {
523 case BPF_MOV:
524 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
525 break;
526 case BPF_ADD:
527 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
528 break;
529 case BPF_SUB:
530 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
531 break;
532 case BPF_AND:
533 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
534 break;
535 case BPF_OR:
536 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
537 break;
538 case BPF_XOR:
539 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
540 break;
541 case BPF_MUL:
542 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
543 break;
544 case BPF_DIV:
545 emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) :
546 rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
547 break;
548 case BPF_MOD:
549 emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) :
550 rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
551 break;
552 case BPF_LSH:
553 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
554 break;
555 case BPF_RSH:
556 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
557 break;
558 case BPF_ARSH:
559 emit(rv_sra(lo(rd), lo(rd), lo(rs)), ctx);
560 break;
561 case BPF_NEG:
562 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
563 break;
564 }
565
566 bpf_put_reg32(dst, rd, ctx);
567 }
568
emit_branch_r64(const s8 * src1,const s8 * src2,s32 rvoff,struct rv_jit_context * ctx,const u8 op)569 static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff,
570 struct rv_jit_context *ctx, const u8 op)
571 {
572 int e, s = ctx->ninsns;
573 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
574 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
575
576 const s8 *rs1 = bpf_get_reg64(src1, tmp1, ctx);
577 const s8 *rs2 = bpf_get_reg64(src2, tmp2, ctx);
578
579 /*
580 * NO_JUMP skips over the rest of the instructions and the
581 * emit_jump_and_link, meaning the BPF branch is not taken.
582 * JUMP skips directly to the emit_jump_and_link, meaning
583 * the BPF branch is taken.
584 *
585 * The fallthrough case results in the BPF branch being taken.
586 */
587 #define NO_JUMP(idx) (6 + (2 * (idx)))
588 #define JUMP(idx) (2 + (2 * (idx)))
589
590 switch (op) {
591 case BPF_JEQ:
592 emit(rv_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
593 emit(rv_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
594 break;
595 case BPF_JGT:
596 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
597 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
598 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
599 break;
600 case BPF_JLT:
601 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
602 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
603 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
604 break;
605 case BPF_JGE:
606 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
607 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
608 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
609 break;
610 case BPF_JLE:
611 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
612 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
613 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
614 break;
615 case BPF_JNE:
616 emit(rv_bne(hi(rs1), hi(rs2), JUMP(1)), ctx);
617 emit(rv_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
618 break;
619 case BPF_JSGT:
620 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
621 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
622 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
623 break;
624 case BPF_JSLT:
625 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
626 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
627 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
628 break;
629 case BPF_JSGE:
630 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
631 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
632 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
633 break;
634 case BPF_JSLE:
635 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
636 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
637 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
638 break;
639 case BPF_JSET:
640 emit(rv_and(RV_REG_T0, hi(rs1), hi(rs2)), ctx);
641 emit(rv_bne(RV_REG_T0, RV_REG_ZERO, JUMP(2)), ctx);
642 emit(rv_and(RV_REG_T0, lo(rs1), lo(rs2)), ctx);
643 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, NO_JUMP(0)), ctx);
644 break;
645 }
646
647 #undef NO_JUMP
648 #undef JUMP
649
650 e = ctx->ninsns;
651 /* Adjust for extra insns. */
652 rvoff -= ninsns_rvoff(e - s);
653 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
654 return 0;
655 }
656
emit_bcc(u8 op,u8 rd,u8 rs,int rvoff,struct rv_jit_context * ctx)657 static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx)
658 {
659 int e, s = ctx->ninsns;
660 bool far = false;
661 int off;
662
663 if (op == BPF_JSET) {
664 /*
665 * BPF_JSET is a special case: it has no inverse so we always
666 * treat it as a far branch.
667 */
668 far = true;
669 } else if (!is_13b_int(rvoff)) {
670 op = invert_bpf_cond(op);
671 far = true;
672 }
673
674 /*
675 * For a far branch, the condition is negated and we jump over the
676 * branch itself, and the two instructions from emit_jump_and_link.
677 * For a near branch, just use rvoff.
678 */
679 off = far ? 6 : (rvoff >> 1);
680
681 switch (op) {
682 case BPF_JEQ:
683 emit(rv_beq(rd, rs, off), ctx);
684 break;
685 case BPF_JGT:
686 emit(rv_bgtu(rd, rs, off), ctx);
687 break;
688 case BPF_JLT:
689 emit(rv_bltu(rd, rs, off), ctx);
690 break;
691 case BPF_JGE:
692 emit(rv_bgeu(rd, rs, off), ctx);
693 break;
694 case BPF_JLE:
695 emit(rv_bleu(rd, rs, off), ctx);
696 break;
697 case BPF_JNE:
698 emit(rv_bne(rd, rs, off), ctx);
699 break;
700 case BPF_JSGT:
701 emit(rv_bgt(rd, rs, off), ctx);
702 break;
703 case BPF_JSLT:
704 emit(rv_blt(rd, rs, off), ctx);
705 break;
706 case BPF_JSGE:
707 emit(rv_bge(rd, rs, off), ctx);
708 break;
709 case BPF_JSLE:
710 emit(rv_ble(rd, rs, off), ctx);
711 break;
712 case BPF_JSET:
713 emit(rv_and(RV_REG_T0, rd, rs), ctx);
714 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, off), ctx);
715 break;
716 }
717
718 if (far) {
719 e = ctx->ninsns;
720 /* Adjust for extra insns. */
721 rvoff -= ninsns_rvoff(e - s);
722 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
723 }
724 return 0;
725 }
726
emit_branch_r32(const s8 * src1,const s8 * src2,s32 rvoff,struct rv_jit_context * ctx,const u8 op)727 static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff,
728 struct rv_jit_context *ctx, const u8 op)
729 {
730 int e, s = ctx->ninsns;
731 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
732 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
733
734 const s8 *rs1 = bpf_get_reg32(src1, tmp1, ctx);
735 const s8 *rs2 = bpf_get_reg32(src2, tmp2, ctx);
736
737 e = ctx->ninsns;
738 /* Adjust for extra insns. */
739 rvoff -= ninsns_rvoff(e - s);
740
741 if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx))
742 return -1;
743
744 return 0;
745 }
746
emit_call(bool fixed,u64 addr,struct rv_jit_context * ctx)747 static void emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
748 {
749 const s8 *r0 = bpf2rv32[BPF_REG_0];
750 const s8 *r5 = bpf2rv32[BPF_REG_5];
751 u32 upper = ((u32)addr + (1 << 11)) >> 12;
752 u32 lower = addr & 0xfff;
753
754 /* R1-R4 already in correct registers---need to push R5 to stack. */
755 emit(rv_addi(RV_REG_SP, RV_REG_SP, -16), ctx);
756 emit(rv_sw(RV_REG_SP, 0, lo(r5)), ctx);
757 emit(rv_sw(RV_REG_SP, 4, hi(r5)), ctx);
758
759 /* Backup TCC. */
760 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
761
762 /*
763 * Use lui/jalr pair to jump to absolute address. Don't use emit_imm as
764 * the number of emitted instructions should not depend on the value of
765 * addr.
766 */
767 emit(rv_lui(RV_REG_T1, upper), ctx);
768 emit(rv_jalr(RV_REG_RA, RV_REG_T1, lower), ctx);
769
770 /* Restore TCC. */
771 emit(rv_addi(RV_REG_TCC, RV_REG_TCC_SAVED, 0), ctx);
772
773 /* Set return value and restore stack. */
774 emit(rv_addi(lo(r0), RV_REG_A0, 0), ctx);
775 emit(rv_addi(hi(r0), RV_REG_A1, 0), ctx);
776 emit(rv_addi(RV_REG_SP, RV_REG_SP, 16), ctx);
777 }
778
emit_bpf_tail_call(int insn,struct rv_jit_context * ctx)779 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
780 {
781 /*
782 * R1 -> &ctx
783 * R2 -> &array
784 * R3 -> index
785 */
786 int tc_ninsn, off, start_insn = ctx->ninsns;
787 const s8 *arr_reg = bpf2rv32[BPF_REG_2];
788 const s8 *idx_reg = bpf2rv32[BPF_REG_3];
789
790 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
791 ctx->offset[0];
792
793 /* max_entries = array->map.max_entries; */
794 off = offsetof(struct bpf_array, map.max_entries);
795 if (is_12b_check(off, insn))
796 return -1;
797 emit(rv_lw(RV_REG_T1, off, lo(arr_reg)), ctx);
798
799 /*
800 * if (index >= max_entries)
801 * goto out;
802 */
803 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
804 emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx);
805
806 /*
807 * if (--tcc < 0)
808 * goto out;
809 */
810 emit(rv_addi(RV_REG_TCC, RV_REG_TCC, -1), ctx);
811 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
812 emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx);
813
814 /*
815 * prog = array->ptrs[index];
816 * if (!prog)
817 * goto out;
818 */
819 emit_sh2add(RV_REG_T0, lo(idx_reg), lo(arr_reg), ctx);
820 off = offsetof(struct bpf_array, ptrs);
821 if (is_12b_check(off, insn))
822 return -1;
823 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
824 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn));
825 emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx);
826
827 /*
828 * tcc = temp_tcc;
829 * goto *(prog->bpf_func + 4);
830 */
831 off = offsetof(struct bpf_prog, bpf_func);
832 if (is_12b_check(off, insn))
833 return -1;
834 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
835 /* Epilogue jumps to *(t0 + 4). */
836 __build_epilogue(true, ctx);
837 return 0;
838 }
839
emit_load_r64(const s8 * dst,const s8 * src,s16 off,struct rv_jit_context * ctx,const u8 size)840 static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,
841 struct rv_jit_context *ctx, const u8 size)
842 {
843 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
844 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
845 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
846 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
847
848 emit_imm(RV_REG_T0, off, ctx);
849 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rs)), ctx);
850
851 switch (size) {
852 case BPF_B:
853 emit(rv_lbu(lo(rd), 0, RV_REG_T0), ctx);
854 if (!ctx->prog->aux->verifier_zext)
855 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
856 break;
857 case BPF_H:
858 emit(rv_lhu(lo(rd), 0, RV_REG_T0), ctx);
859 if (!ctx->prog->aux->verifier_zext)
860 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
861 break;
862 case BPF_W:
863 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
864 if (!ctx->prog->aux->verifier_zext)
865 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
866 break;
867 case BPF_DW:
868 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
869 emit(rv_lw(hi(rd), 4, RV_REG_T0), ctx);
870 break;
871 }
872
873 bpf_put_reg64(dst, rd, ctx);
874 return 0;
875 }
876
emit_bpf_atomic(s8 dst,const s8 * src,const s8 * rs,struct rv_jit_context * ctx,const struct bpf_insn * insn)877 static int emit_bpf_atomic(s8 dst, const s8 *src, const s8 *rs,
878 struct rv_jit_context *ctx,
879 const struct bpf_insn *insn)
880 {
881 s32 imm = insn->imm;
882 bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG);
883 s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO;
884 int aq = is_fetch ? 1 : 0;
885 int rl = is_fetch ? 1 : 0;
886
887 switch (imm) {
888 case BPF_ADD:
889 case BPF_ADD | BPF_FETCH:
890 emit(rv_amoadd_w(fetch_reg, lo(rs), dst, aq, rl), ctx);
891 break;
892 case BPF_AND:
893 case BPF_AND | BPF_FETCH:
894 emit(rv_amoand_w(fetch_reg, lo(rs), dst, aq, rl), ctx);
895 break;
896 case BPF_OR:
897 case BPF_OR | BPF_FETCH:
898 emit(rv_amoor_w(fetch_reg, lo(rs), dst, aq, rl), ctx);
899 break;
900 case BPF_XOR:
901 case BPF_XOR | BPF_FETCH:
902 emit(rv_amoxor_w(fetch_reg, lo(rs), dst, aq, rl), ctx);
903 break;
904 case BPF_XCHG:
905 emit(rv_amoswap_w(fetch_reg, lo(rs), dst, aq, rl), ctx);
906 break;
907 default:
908 return -1;
909 }
910
911 if (is_fetch) {
912 emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx);
913 bpf_put_reg64(src, rs, ctx);
914 }
915 return 0;
916 }
917
emit_store_r64(const s8 * dst,const s8 * src,struct rv_jit_context * ctx,const struct bpf_insn * insn)918 static int emit_store_r64(const s8 *dst, const s8 *src,
919 struct rv_jit_context *ctx,
920 const struct bpf_insn *insn)
921 {
922 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
923 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
924 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
925 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
926 u8 size = BPF_SIZE(insn->code);
927 u8 mode = BPF_MODE(insn->code);
928 s16 off = insn->off;
929
930 if (mode == BPF_ATOMIC && size != BPF_W)
931 return -1;
932
933 emit_imm(RV_REG_T0, off, ctx);
934 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rd)), ctx);
935
936 switch (size) {
937 case BPF_B:
938 emit(rv_sb(RV_REG_T0, 0, lo(rs)), ctx);
939 break;
940 case BPF_H:
941 emit(rv_sh(RV_REG_T0, 0, lo(rs)), ctx);
942 break;
943 case BPF_W:
944 switch (mode) {
945 case BPF_MEM:
946 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
947 break;
948 case BPF_ATOMIC:
949 if (emit_bpf_atomic(RV_REG_T0, src, rs, ctx, insn))
950 return -1;
951 break;
952 }
953 break;
954 case BPF_DW:
955 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
956 emit(rv_sw(RV_REG_T0, 4, hi(rs)), ctx);
957 break;
958 }
959
960 return 0;
961 }
962
emit_rev16(const s8 rd,struct rv_jit_context * ctx)963 static void emit_rev16(const s8 rd, struct rv_jit_context *ctx)
964 {
965 emit(rv_slli(rd, rd, 16), ctx);
966 emit(rv_slli(RV_REG_T1, rd, 8), ctx);
967 emit(rv_srli(rd, rd, 8), ctx);
968 emit(rv_add(RV_REG_T1, rd, RV_REG_T1), ctx);
969 emit(rv_srli(rd, RV_REG_T1, 16), ctx);
970 }
971
emit_rev32(const s8 rd,struct rv_jit_context * ctx)972 static void emit_rev32(const s8 rd, struct rv_jit_context *ctx)
973 {
974 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 0), ctx);
975 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
976 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
977 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
978 emit(rv_srli(rd, rd, 8), ctx);
979 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
980 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
981 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
982 emit(rv_srli(rd, rd, 8), ctx);
983 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
984 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
985 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
986 emit(rv_srli(rd, rd, 8), ctx);
987 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
988 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
989 emit(rv_addi(rd, RV_REG_T1, 0), ctx);
990 }
991
emit_zext64(const s8 * dst,struct rv_jit_context * ctx)992 static void emit_zext64(const s8 *dst, struct rv_jit_context *ctx)
993 {
994 const s8 *rd;
995 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
996
997 rd = bpf_get_reg64(dst, tmp1, ctx);
998 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
999 bpf_put_reg64(dst, rd, ctx);
1000 }
1001
bpf_jit_emit_insn(const struct bpf_insn * insn,struct rv_jit_context * ctx,bool extra_pass)1002 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1003 bool extra_pass)
1004 {
1005 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
1006 BPF_CLASS(insn->code) == BPF_JMP;
1007 int s, e, rvoff, i = insn - ctx->prog->insnsi;
1008 u8 code = insn->code;
1009 s16 off = insn->off;
1010 s32 imm = insn->imm;
1011
1012 const s8 *dst = bpf2rv32[insn->dst_reg];
1013 const s8 *src = bpf2rv32[insn->src_reg];
1014 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
1015 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
1016
1017 switch (code) {
1018 case BPF_ALU64 | BPF_MOV | BPF_X:
1019 if (insn->off != 0) {
1020 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1021 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
1022
1023 if (insn->off == 8) {
1024 emit(rv_slli(lo(rd), lo(rs), 24), ctx);
1025 emit(rv_srai(lo(rd), lo(rd), 24), ctx);
1026 } else if (insn->off == 16) {
1027 emit(rv_slli(lo(rd), lo(rs), 16), ctx);
1028 emit(rv_srai(lo(rd), lo(rd), 16), ctx);
1029 } else {
1030 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
1031 }
1032 emit(rv_srai(hi(rd), lo(rd), 31), ctx);
1033 bpf_put_reg64(dst, rd, ctx);
1034 break;
1035 }
1036 fallthrough;
1037
1038 case BPF_ALU64 | BPF_ADD | BPF_X:
1039 case BPF_ALU64 | BPF_ADD | BPF_K:
1040
1041 case BPF_ALU64 | BPF_SUB | BPF_X:
1042 case BPF_ALU64 | BPF_SUB | BPF_K:
1043
1044 case BPF_ALU64 | BPF_AND | BPF_X:
1045 case BPF_ALU64 | BPF_OR | BPF_X:
1046 case BPF_ALU64 | BPF_XOR | BPF_X:
1047
1048 case BPF_ALU64 | BPF_MUL | BPF_X:
1049 case BPF_ALU64 | BPF_MUL | BPF_K:
1050
1051 case BPF_ALU64 | BPF_LSH | BPF_X:
1052 case BPF_ALU64 | BPF_RSH | BPF_X:
1053 case BPF_ALU64 | BPF_ARSH | BPF_X:
1054 if (BPF_SRC(code) == BPF_K) {
1055 emit_imm32(tmp2, imm, ctx);
1056 src = tmp2;
1057 }
1058 emit_alu_r64(dst, src, ctx, BPF_OP(code));
1059 break;
1060
1061 case BPF_ALU64 | BPF_NEG:
1062 emit_alu_r64(dst, tmp2, ctx, BPF_OP(code));
1063 break;
1064
1065 case BPF_ALU64 | BPF_DIV | BPF_X:
1066 case BPF_ALU64 | BPF_DIV | BPF_K:
1067 case BPF_ALU64 | BPF_MOD | BPF_X:
1068 case BPF_ALU64 | BPF_MOD | BPF_K:
1069 goto notsupported;
1070
1071 case BPF_ALU64 | BPF_MOV | BPF_K:
1072 case BPF_ALU64 | BPF_AND | BPF_K:
1073 case BPF_ALU64 | BPF_OR | BPF_K:
1074 case BPF_ALU64 | BPF_XOR | BPF_K:
1075 case BPF_ALU64 | BPF_LSH | BPF_K:
1076 case BPF_ALU64 | BPF_RSH | BPF_K:
1077 case BPF_ALU64 | BPF_ARSH | BPF_K:
1078 emit_alu_i64(dst, imm, ctx, BPF_OP(code));
1079 break;
1080
1081 case BPF_ALU | BPF_MOV | BPF_X:
1082 if (imm == 1) {
1083 /* Special mov32 for zext. */
1084 emit_zext64(dst, ctx);
1085 break;
1086 }
1087 if (insn->off != 0) {
1088 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
1089 const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
1090
1091 if (insn->off == 8) {
1092 emit(rv_slli(lo(rd), lo(rs), 24), ctx);
1093 emit(rv_srai(lo(rd), lo(rd), 24), ctx);
1094 } else if (insn->off == 16) {
1095 emit(rv_slli(lo(rd), lo(rs), 16), ctx);
1096 emit(rv_srai(lo(rd), lo(rd), 16), ctx);
1097 }
1098 bpf_put_reg32(dst, rd, ctx);
1099 break;
1100 }
1101 fallthrough;
1102
1103 case BPF_ALU | BPF_ADD | BPF_X:
1104 case BPF_ALU | BPF_SUB | BPF_X:
1105 case BPF_ALU | BPF_AND | BPF_X:
1106 case BPF_ALU | BPF_OR | BPF_X:
1107 case BPF_ALU | BPF_XOR | BPF_X:
1108
1109 case BPF_ALU | BPF_MUL | BPF_X:
1110 case BPF_ALU | BPF_MUL | BPF_K:
1111
1112 case BPF_ALU | BPF_DIV | BPF_X:
1113 case BPF_ALU | BPF_DIV | BPF_K:
1114
1115 case BPF_ALU | BPF_MOD | BPF_X:
1116 case BPF_ALU | BPF_MOD | BPF_K:
1117
1118 case BPF_ALU | BPF_LSH | BPF_X:
1119 case BPF_ALU | BPF_RSH | BPF_X:
1120 case BPF_ALU | BPF_ARSH | BPF_X:
1121 if (BPF_SRC(code) == BPF_K) {
1122 emit_imm32(tmp2, imm, ctx);
1123 src = tmp2;
1124 }
1125 emit_alu_r32(dst, src, ctx, insn);
1126 break;
1127
1128 case BPF_ALU | BPF_MOV | BPF_K:
1129 case BPF_ALU | BPF_ADD | BPF_K:
1130 case BPF_ALU | BPF_SUB | BPF_K:
1131 case BPF_ALU | BPF_AND | BPF_K:
1132 case BPF_ALU | BPF_OR | BPF_K:
1133 case BPF_ALU | BPF_XOR | BPF_K:
1134 case BPF_ALU | BPF_LSH | BPF_K:
1135 case BPF_ALU | BPF_RSH | BPF_K:
1136 case BPF_ALU | BPF_ARSH | BPF_K:
1137 /*
1138 * mul,div,mod are handled in the BPF_X case since there are
1139 * no RISC-V I-type equivalents.
1140 */
1141 emit_alu_i32(dst, imm, ctx, BPF_OP(code));
1142 break;
1143
1144 case BPF_ALU | BPF_NEG:
1145 /*
1146 * src is ignored---choose tmp2 as a dummy register since it
1147 * is not on the stack.
1148 */
1149 emit_alu_r32(dst, tmp2, ctx, insn);
1150 break;
1151
1152 case BPF_ALU | BPF_END | BPF_FROM_LE:
1153 {
1154 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1155
1156 switch (imm) {
1157 case 16:
1158 emit(rv_slli(lo(rd), lo(rd), 16), ctx);
1159 emit(rv_srli(lo(rd), lo(rd), 16), ctx);
1160 fallthrough;
1161 case 32:
1162 if (!ctx->prog->aux->verifier_zext)
1163 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1164 break;
1165 case 64:
1166 /* Do nothing. */
1167 break;
1168 default:
1169 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm);
1170 return -1;
1171 }
1172
1173 bpf_put_reg64(dst, rd, ctx);
1174 break;
1175 }
1176
1177 case BPF_ALU | BPF_END | BPF_FROM_BE:
1178 {
1179 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1180
1181 switch (imm) {
1182 case 16:
1183 emit_rev16(lo(rd), ctx);
1184 if (!ctx->prog->aux->verifier_zext)
1185 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1186 break;
1187 case 32:
1188 emit_rev32(lo(rd), ctx);
1189 if (!ctx->prog->aux->verifier_zext)
1190 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1191 break;
1192 case 64:
1193 /* Swap upper and lower halves. */
1194 emit(rv_addi(RV_REG_T0, lo(rd), 0), ctx);
1195 emit(rv_addi(lo(rd), hi(rd), 0), ctx);
1196 emit(rv_addi(hi(rd), RV_REG_T0, 0), ctx);
1197
1198 /* Swap each half. */
1199 emit_rev32(lo(rd), ctx);
1200 emit_rev32(hi(rd), ctx);
1201 break;
1202 default:
1203 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm);
1204 return -1;
1205 }
1206
1207 bpf_put_reg64(dst, rd, ctx);
1208 break;
1209 }
1210
1211 case BPF_JMP | BPF_JA:
1212 rvoff = rv_offset(i, off, ctx);
1213 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1214 break;
1215
1216 case BPF_JMP | BPF_CALL:
1217 {
1218 bool fixed;
1219 int ret;
1220 u64 addr;
1221
1222 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1223 &fixed);
1224 if (ret < 0)
1225 return ret;
1226 emit_call(fixed, addr, ctx);
1227 break;
1228 }
1229
1230 case BPF_JMP | BPF_TAIL_CALL:
1231 if (emit_bpf_tail_call(i, ctx))
1232 return -1;
1233 break;
1234
1235 case BPF_JMP | BPF_JEQ | BPF_X:
1236 case BPF_JMP | BPF_JEQ | BPF_K:
1237 case BPF_JMP32 | BPF_JEQ | BPF_X:
1238 case BPF_JMP32 | BPF_JEQ | BPF_K:
1239
1240 case BPF_JMP | BPF_JNE | BPF_X:
1241 case BPF_JMP | BPF_JNE | BPF_K:
1242 case BPF_JMP32 | BPF_JNE | BPF_X:
1243 case BPF_JMP32 | BPF_JNE | BPF_K:
1244
1245 case BPF_JMP | BPF_JLE | BPF_X:
1246 case BPF_JMP | BPF_JLE | BPF_K:
1247 case BPF_JMP32 | BPF_JLE | BPF_X:
1248 case BPF_JMP32 | BPF_JLE | BPF_K:
1249
1250 case BPF_JMP | BPF_JLT | BPF_X:
1251 case BPF_JMP | BPF_JLT | BPF_K:
1252 case BPF_JMP32 | BPF_JLT | BPF_X:
1253 case BPF_JMP32 | BPF_JLT | BPF_K:
1254
1255 case BPF_JMP | BPF_JGE | BPF_X:
1256 case BPF_JMP | BPF_JGE | BPF_K:
1257 case BPF_JMP32 | BPF_JGE | BPF_X:
1258 case BPF_JMP32 | BPF_JGE | BPF_K:
1259
1260 case BPF_JMP | BPF_JGT | BPF_X:
1261 case BPF_JMP | BPF_JGT | BPF_K:
1262 case BPF_JMP32 | BPF_JGT | BPF_X:
1263 case BPF_JMP32 | BPF_JGT | BPF_K:
1264
1265 case BPF_JMP | BPF_JSLE | BPF_X:
1266 case BPF_JMP | BPF_JSLE | BPF_K:
1267 case BPF_JMP32 | BPF_JSLE | BPF_X:
1268 case BPF_JMP32 | BPF_JSLE | BPF_K:
1269
1270 case BPF_JMP | BPF_JSLT | BPF_X:
1271 case BPF_JMP | BPF_JSLT | BPF_K:
1272 case BPF_JMP32 | BPF_JSLT | BPF_X:
1273 case BPF_JMP32 | BPF_JSLT | BPF_K:
1274
1275 case BPF_JMP | BPF_JSGE | BPF_X:
1276 case BPF_JMP | BPF_JSGE | BPF_K:
1277 case BPF_JMP32 | BPF_JSGE | BPF_X:
1278 case BPF_JMP32 | BPF_JSGE | BPF_K:
1279
1280 case BPF_JMP | BPF_JSGT | BPF_X:
1281 case BPF_JMP | BPF_JSGT | BPF_K:
1282 case BPF_JMP32 | BPF_JSGT | BPF_X:
1283 case BPF_JMP32 | BPF_JSGT | BPF_K:
1284
1285 case BPF_JMP | BPF_JSET | BPF_X:
1286 case BPF_JMP | BPF_JSET | BPF_K:
1287 case BPF_JMP32 | BPF_JSET | BPF_X:
1288 case BPF_JMP32 | BPF_JSET | BPF_K:
1289 rvoff = rv_offset(i, off, ctx);
1290 if (BPF_SRC(code) == BPF_K) {
1291 s = ctx->ninsns;
1292 emit_imm32(tmp2, imm, ctx);
1293 src = tmp2;
1294 e = ctx->ninsns;
1295 rvoff -= ninsns_rvoff(e - s);
1296 }
1297
1298 if (is64)
1299 emit_branch_r64(dst, src, rvoff, ctx, BPF_OP(code));
1300 else
1301 emit_branch_r32(dst, src, rvoff, ctx, BPF_OP(code));
1302 break;
1303
1304 case BPF_JMP | BPF_EXIT:
1305 if (i == ctx->prog->len - 1)
1306 break;
1307
1308 rvoff = epilogue_offset(ctx);
1309 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1310 break;
1311
1312 case BPF_LD | BPF_IMM | BPF_DW:
1313 {
1314 struct bpf_insn insn1 = insn[1];
1315 s32 imm_lo = imm;
1316 s32 imm_hi = insn1.imm;
1317 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1318
1319 emit_imm64(rd, imm_hi, imm_lo, ctx);
1320 bpf_put_reg64(dst, rd, ctx);
1321 return 1;
1322 }
1323
1324 case BPF_LDX | BPF_MEM | BPF_B:
1325 case BPF_LDX | BPF_MEM | BPF_H:
1326 case BPF_LDX | BPF_MEM | BPF_W:
1327 case BPF_LDX | BPF_MEM | BPF_DW:
1328 if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code)))
1329 return -1;
1330 break;
1331
1332 /* speculation barrier */
1333 case BPF_ST | BPF_NOSPEC:
1334 break;
1335
1336 case BPF_ST | BPF_MEM | BPF_B:
1337 case BPF_ST | BPF_MEM | BPF_H:
1338 case BPF_ST | BPF_MEM | BPF_W:
1339 case BPF_ST | BPF_MEM | BPF_DW:
1340
1341 case BPF_STX | BPF_MEM | BPF_B:
1342 case BPF_STX | BPF_MEM | BPF_H:
1343 case BPF_STX | BPF_MEM | BPF_W:
1344 case BPF_STX | BPF_MEM | BPF_DW:
1345 if (BPF_CLASS(code) == BPF_ST) {
1346 emit_imm32(tmp2, imm, ctx);
1347 src = tmp2;
1348 }
1349
1350 if (emit_store_r64(dst, src, ctx, insn))
1351 return -1;
1352 break;
1353
1354 case BPF_STX | BPF_ATOMIC | BPF_W:
1355 if (insn->imm == BPF_CMPXCHG) {
1356 pr_info_once(
1357 "bpf-jit: not supported: atomic operation %02x ***\n",
1358 insn->imm);
1359 return -EFAULT;
1360 }
1361
1362 if (emit_store_r64(dst, src, ctx, insn))
1363 return -1;
1364 break;
1365
1366 /* No hardware support for 8-byte atomics in RV32. */
1367 case BPF_STX | BPF_ATOMIC | BPF_DW:
1368 /* Fallthrough. */
1369
1370 notsupported:
1371 pr_info_once("bpf-jit: not supported: opcode %02x ***\n", code);
1372 return -EFAULT;
1373
1374 default:
1375 pr_err("bpf-jit: unknown opcode %02x\n", code);
1376 return -EINVAL;
1377 }
1378
1379 return 0;
1380 }
1381
bpf_jit_build_prologue(struct rv_jit_context * ctx,bool is_subprog)1382 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
1383 {
1384 const s8 *fp = bpf2rv32[BPF_REG_FP];
1385 const s8 *r1 = bpf2rv32[BPF_REG_1];
1386 int stack_adjust = 0;
1387 int bpf_stack_adjust =
1388 round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
1389
1390 /* Make space for callee-saved registers. */
1391 stack_adjust += NR_SAVED_REGISTERS * sizeof(u32);
1392 /* Make space for BPF registers on stack. */
1393 stack_adjust += BPF_JIT_SCRATCH_REGS * sizeof(u32);
1394 /* Make space for BPF stack. */
1395 stack_adjust += bpf_stack_adjust;
1396 /* Round up for stack alignment. */
1397 stack_adjust = round_up(stack_adjust, STACK_ALIGN);
1398
1399 /*
1400 * The first instruction sets the tail-call-counter (TCC) register.
1401 * This instruction is skipped by tail calls.
1402 */
1403 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1404
1405 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1406
1407 /* Save callee-save registers. */
1408 emit(rv_sw(RV_REG_SP, stack_adjust - 4, RV_REG_RA), ctx);
1409 emit(rv_sw(RV_REG_SP, stack_adjust - 8, RV_REG_FP), ctx);
1410 emit(rv_sw(RV_REG_SP, stack_adjust - 12, RV_REG_S1), ctx);
1411 emit(rv_sw(RV_REG_SP, stack_adjust - 16, RV_REG_S2), ctx);
1412 emit(rv_sw(RV_REG_SP, stack_adjust - 20, RV_REG_S3), ctx);
1413 emit(rv_sw(RV_REG_SP, stack_adjust - 24, RV_REG_S4), ctx);
1414 emit(rv_sw(RV_REG_SP, stack_adjust - 28, RV_REG_S5), ctx);
1415 emit(rv_sw(RV_REG_SP, stack_adjust - 32, RV_REG_S6), ctx);
1416 emit(rv_sw(RV_REG_SP, stack_adjust - 36, RV_REG_S7), ctx);
1417
1418 /* Set fp: used as the base address for stacked BPF registers. */
1419 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1420
1421 /* Set up BPF frame pointer. */
1422 emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx);
1423 emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx);
1424
1425 /* Set up BPF context pointer. */
1426 emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx);
1427 emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx);
1428
1429 ctx->stack_size = stack_adjust;
1430 }
1431
bpf_jit_build_epilogue(struct rv_jit_context * ctx)1432 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1433 {
1434 __build_epilogue(false, ctx);
1435 }
1436