1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * BPF JIT compiler for RV32G 4 * 5 * Copyright (c) 2020 Luke Nelson <luke.r.nels@gmail.com> 6 * Copyright (c) 2020 Xi Wang <xi.wang@gmail.com> 7 * 8 * The code is based on the BPF JIT compiler for RV64G by Björn Töpel and 9 * the BPF JIT compiler for 32-bit ARM by Shubham Bansal and Mircea Gherzan. 10 */ 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include "bpf_jit.h" 15 16 /* 17 * Stack layout during BPF program execution: 18 * 19 * high 20 * RV32 fp => +----------+ 21 * | saved ra | 22 * | saved fp | RV32 callee-saved registers 23 * | ... | 24 * +----------+ <= (fp - 4 * NR_SAVED_REGISTERS) 25 * | hi(R6) | 26 * | lo(R6) | 27 * | hi(R7) | JIT scratch space for BPF registers 28 * | lo(R7) | 29 * | ... | 30 * BPF_REG_FP => +----------+ <= (fp - 4 * NR_SAVED_REGISTERS 31 * | | - 4 * BPF_JIT_SCRATCH_REGS) 32 * | | 33 * | ... | BPF program stack 34 * | | 35 * RV32 sp => +----------+ 36 * | | 37 * | ... | Function call stack 38 * | | 39 * +----------+ 40 * low 41 */ 42 43 enum { 44 /* Stack layout - these are offsets from top of JIT scratch space. */ 45 BPF_R6_HI, 46 BPF_R6_LO, 47 BPF_R7_HI, 48 BPF_R7_LO, 49 BPF_R8_HI, 50 BPF_R8_LO, 51 BPF_R9_HI, 52 BPF_R9_LO, 53 BPF_AX_HI, 54 BPF_AX_LO, 55 /* Stack space for BPF_REG_6 through BPF_REG_9 and BPF_REG_AX. */ 56 BPF_JIT_SCRATCH_REGS, 57 }; 58 59 /* Number of callee-saved registers stored to stack: ra, fp, s1--s7. */ 60 #define NR_SAVED_REGISTERS 9 61 62 /* Offset from fp for BPF registers stored on stack. */ 63 #define STACK_OFFSET(k) (-4 - (4 * NR_SAVED_REGISTERS) - (4 * (k))) 64 65 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 66 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 67 68 #define RV_REG_TCC RV_REG_T6 69 #define RV_REG_TCC_SAVED RV_REG_S7 70 71 static const s8 bpf2rv32[][2] = { 72 /* Return value from in-kernel function, and exit value from eBPF. */ 73 [BPF_REG_0] = {RV_REG_S2, RV_REG_S1}, 74 /* Arguments from eBPF program to in-kernel function. */ 75 [BPF_REG_1] = {RV_REG_A1, RV_REG_A0}, 76 [BPF_REG_2] = {RV_REG_A3, RV_REG_A2}, 77 [BPF_REG_3] = {RV_REG_A5, RV_REG_A4}, 78 [BPF_REG_4] = {RV_REG_A7, RV_REG_A6}, 79 [BPF_REG_5] = {RV_REG_S4, RV_REG_S3}, 80 /* 81 * Callee-saved registers that in-kernel function will preserve. 82 * Stored on the stack. 83 */ 84 [BPF_REG_6] = {STACK_OFFSET(BPF_R6_HI), STACK_OFFSET(BPF_R6_LO)}, 85 [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)}, 86 [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, 87 [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, 88 /* Read-only frame pointer to access BPF stack. */ 89 [BPF_REG_FP] = {RV_REG_S6, RV_REG_S5}, 90 /* Temporary register for blinding constants. Stored on the stack. */ 91 [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, 92 /* 93 * Temporary registers used by the JIT to operate on registers stored 94 * on the stack. Save t0 and t1 to be used as temporaries in generated 95 * code. 96 */ 97 [TMP_REG_1] = {RV_REG_T3, RV_REG_T2}, 98 [TMP_REG_2] = {RV_REG_T5, RV_REG_T4}, 99 }; 100 101 static s8 hi(const s8 *r) 102 { 103 return r[0]; 104 } 105 106 static s8 lo(const s8 *r) 107 { 108 return r[1]; 109 } 110 111 static void emit_imm(const s8 rd, s32 imm, struct rv_jit_context *ctx) 112 { 113 u32 upper = (imm + (1 << 11)) >> 12; 114 u32 lower = imm & 0xfff; 115 116 if (upper) { 117 emit(rv_lui(rd, upper), ctx); 118 emit(rv_addi(rd, rd, lower), ctx); 119 } else { 120 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx); 121 } 122 } 123 124 static void emit_imm32(const s8 *rd, s32 imm, struct rv_jit_context *ctx) 125 { 126 /* Emit immediate into lower bits. */ 127 emit_imm(lo(rd), imm, ctx); 128 129 /* Sign-extend into upper bits. */ 130 if (imm >= 0) 131 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 132 else 133 emit(rv_addi(hi(rd), RV_REG_ZERO, -1), ctx); 134 } 135 136 static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo, 137 struct rv_jit_context *ctx) 138 { 139 emit_imm(lo(rd), imm_lo, ctx); 140 emit_imm(hi(rd), imm_hi, ctx); 141 } 142 143 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 144 { 145 int stack_adjust = ctx->stack_size; 146 const s8 *r0 = bpf2rv32[BPF_REG_0]; 147 148 /* Set return value if not tail call. */ 149 if (!is_tail_call) { 150 emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx); 151 emit(rv_addi(RV_REG_A1, hi(r0), 0), ctx); 152 } 153 154 /* Restore callee-saved registers. */ 155 emit(rv_lw(RV_REG_RA, stack_adjust - 4, RV_REG_SP), ctx); 156 emit(rv_lw(RV_REG_FP, stack_adjust - 8, RV_REG_SP), ctx); 157 emit(rv_lw(RV_REG_S1, stack_adjust - 12, RV_REG_SP), ctx); 158 emit(rv_lw(RV_REG_S2, stack_adjust - 16, RV_REG_SP), ctx); 159 emit(rv_lw(RV_REG_S3, stack_adjust - 20, RV_REG_SP), ctx); 160 emit(rv_lw(RV_REG_S4, stack_adjust - 24, RV_REG_SP), ctx); 161 emit(rv_lw(RV_REG_S5, stack_adjust - 28, RV_REG_SP), ctx); 162 emit(rv_lw(RV_REG_S6, stack_adjust - 32, RV_REG_SP), ctx); 163 emit(rv_lw(RV_REG_S7, stack_adjust - 36, RV_REG_SP), ctx); 164 165 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); 166 167 if (is_tail_call) { 168 /* 169 * goto *(t0 + 4); 170 * Skips first instruction of prologue which initializes tail 171 * call counter. Assumes t0 contains address of target program, 172 * see emit_bpf_tail_call. 173 */ 174 emit(rv_jalr(RV_REG_ZERO, RV_REG_T0, 4), ctx); 175 } else { 176 emit(rv_jalr(RV_REG_ZERO, RV_REG_RA, 0), ctx); 177 } 178 } 179 180 static bool is_stacked(s8 reg) 181 { 182 return reg < 0; 183 } 184 185 static const s8 *bpf_get_reg64(const s8 *reg, const s8 *tmp, 186 struct rv_jit_context *ctx) 187 { 188 if (is_stacked(hi(reg))) { 189 emit(rv_lw(hi(tmp), hi(reg), RV_REG_FP), ctx); 190 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx); 191 reg = tmp; 192 } 193 return reg; 194 } 195 196 static void bpf_put_reg64(const s8 *reg, const s8 *src, 197 struct rv_jit_context *ctx) 198 { 199 if (is_stacked(hi(reg))) { 200 emit(rv_sw(RV_REG_FP, hi(reg), hi(src)), ctx); 201 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx); 202 } 203 } 204 205 static const s8 *bpf_get_reg32(const s8 *reg, const s8 *tmp, 206 struct rv_jit_context *ctx) 207 { 208 if (is_stacked(lo(reg))) { 209 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx); 210 reg = tmp; 211 } 212 return reg; 213 } 214 215 static void bpf_put_reg32(const s8 *reg, const s8 *src, 216 struct rv_jit_context *ctx) 217 { 218 if (is_stacked(lo(reg))) { 219 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx); 220 if (!ctx->prog->aux->verifier_zext) 221 emit(rv_sw(RV_REG_FP, hi(reg), RV_REG_ZERO), ctx); 222 } else if (!ctx->prog->aux->verifier_zext) { 223 emit(rv_addi(hi(reg), RV_REG_ZERO, 0), ctx); 224 } 225 } 226 227 static void emit_jump_and_link(u8 rd, s32 rvoff, bool force_jalr, 228 struct rv_jit_context *ctx) 229 { 230 s32 upper, lower; 231 232 if (rvoff && is_21b_int(rvoff) && !force_jalr) { 233 emit(rv_jal(rd, rvoff >> 1), ctx); 234 return; 235 } 236 237 upper = (rvoff + (1 << 11)) >> 12; 238 lower = rvoff & 0xfff; 239 emit(rv_auipc(RV_REG_T1, upper), ctx); 240 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 241 } 242 243 static void emit_alu_i64(const s8 *dst, s32 imm, 244 struct rv_jit_context *ctx, const u8 op) 245 { 246 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 247 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 248 249 switch (op) { 250 case BPF_MOV: 251 emit_imm32(rd, imm, ctx); 252 break; 253 case BPF_AND: 254 if (is_12b_int(imm)) { 255 emit(rv_andi(lo(rd), lo(rd), imm), ctx); 256 } else { 257 emit_imm(RV_REG_T0, imm, ctx); 258 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx); 259 } 260 if (imm >= 0) 261 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 262 break; 263 case BPF_OR: 264 if (is_12b_int(imm)) { 265 emit(rv_ori(lo(rd), lo(rd), imm), ctx); 266 } else { 267 emit_imm(RV_REG_T0, imm, ctx); 268 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx); 269 } 270 if (imm < 0) 271 emit(rv_ori(hi(rd), RV_REG_ZERO, -1), ctx); 272 break; 273 case BPF_XOR: 274 if (is_12b_int(imm)) { 275 emit(rv_xori(lo(rd), lo(rd), imm), ctx); 276 } else { 277 emit_imm(RV_REG_T0, imm, ctx); 278 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx); 279 } 280 if (imm < 0) 281 emit(rv_xori(hi(rd), hi(rd), -1), ctx); 282 break; 283 case BPF_LSH: 284 if (imm >= 32) { 285 emit(rv_slli(hi(rd), lo(rd), imm - 32), ctx); 286 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx); 287 } else if (imm == 0) { 288 /* Do nothing. */ 289 } else { 290 emit(rv_srli(RV_REG_T0, lo(rd), 32 - imm), ctx); 291 emit(rv_slli(hi(rd), hi(rd), imm), ctx); 292 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx); 293 emit(rv_slli(lo(rd), lo(rd), imm), ctx); 294 } 295 break; 296 case BPF_RSH: 297 if (imm >= 32) { 298 emit(rv_srli(lo(rd), hi(rd), imm - 32), ctx); 299 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 300 } else if (imm == 0) { 301 /* Do nothing. */ 302 } else { 303 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx); 304 emit(rv_srli(lo(rd), lo(rd), imm), ctx); 305 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx); 306 emit(rv_srli(hi(rd), hi(rd), imm), ctx); 307 } 308 break; 309 case BPF_ARSH: 310 if (imm >= 32) { 311 emit(rv_srai(lo(rd), hi(rd), imm - 32), ctx); 312 emit(rv_srai(hi(rd), hi(rd), 31), ctx); 313 } else if (imm == 0) { 314 /* Do nothing. */ 315 } else { 316 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx); 317 emit(rv_srli(lo(rd), lo(rd), imm), ctx); 318 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx); 319 emit(rv_srai(hi(rd), hi(rd), imm), ctx); 320 } 321 break; 322 } 323 324 bpf_put_reg64(dst, rd, ctx); 325 } 326 327 static void emit_alu_i32(const s8 *dst, s32 imm, 328 struct rv_jit_context *ctx, const u8 op) 329 { 330 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 331 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); 332 333 switch (op) { 334 case BPF_MOV: 335 emit_imm(lo(rd), imm, ctx); 336 break; 337 case BPF_ADD: 338 if (is_12b_int(imm)) { 339 emit(rv_addi(lo(rd), lo(rd), imm), ctx); 340 } else { 341 emit_imm(RV_REG_T0, imm, ctx); 342 emit(rv_add(lo(rd), lo(rd), RV_REG_T0), ctx); 343 } 344 break; 345 case BPF_SUB: 346 if (is_12b_int(-imm)) { 347 emit(rv_addi(lo(rd), lo(rd), -imm), ctx); 348 } else { 349 emit_imm(RV_REG_T0, imm, ctx); 350 emit(rv_sub(lo(rd), lo(rd), RV_REG_T0), ctx); 351 } 352 break; 353 case BPF_AND: 354 if (is_12b_int(imm)) { 355 emit(rv_andi(lo(rd), lo(rd), imm), ctx); 356 } else { 357 emit_imm(RV_REG_T0, imm, ctx); 358 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx); 359 } 360 break; 361 case BPF_OR: 362 if (is_12b_int(imm)) { 363 emit(rv_ori(lo(rd), lo(rd), imm), ctx); 364 } else { 365 emit_imm(RV_REG_T0, imm, ctx); 366 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx); 367 } 368 break; 369 case BPF_XOR: 370 if (is_12b_int(imm)) { 371 emit(rv_xori(lo(rd), lo(rd), imm), ctx); 372 } else { 373 emit_imm(RV_REG_T0, imm, ctx); 374 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx); 375 } 376 break; 377 case BPF_LSH: 378 if (is_12b_int(imm)) { 379 emit(rv_slli(lo(rd), lo(rd), imm), ctx); 380 } else { 381 emit_imm(RV_REG_T0, imm, ctx); 382 emit(rv_sll(lo(rd), lo(rd), RV_REG_T0), ctx); 383 } 384 break; 385 case BPF_RSH: 386 if (is_12b_int(imm)) { 387 emit(rv_srli(lo(rd), lo(rd), imm), ctx); 388 } else { 389 emit_imm(RV_REG_T0, imm, ctx); 390 emit(rv_srl(lo(rd), lo(rd), RV_REG_T0), ctx); 391 } 392 break; 393 case BPF_ARSH: 394 if (is_12b_int(imm)) { 395 emit(rv_srai(lo(rd), lo(rd), imm), ctx); 396 } else { 397 emit_imm(RV_REG_T0, imm, ctx); 398 emit(rv_sra(lo(rd), lo(rd), RV_REG_T0), ctx); 399 } 400 break; 401 } 402 403 bpf_put_reg32(dst, rd, ctx); 404 } 405 406 static void emit_alu_r64(const s8 *dst, const s8 *src, 407 struct rv_jit_context *ctx, const u8 op) 408 { 409 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 410 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 411 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 412 const s8 *rs = bpf_get_reg64(src, tmp2, ctx); 413 414 switch (op) { 415 case BPF_MOV: 416 emit(rv_addi(lo(rd), lo(rs), 0), ctx); 417 emit(rv_addi(hi(rd), hi(rs), 0), ctx); 418 break; 419 case BPF_ADD: 420 if (rd == rs) { 421 emit(rv_srli(RV_REG_T0, lo(rd), 31), ctx); 422 emit(rv_slli(hi(rd), hi(rd), 1), ctx); 423 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx); 424 emit(rv_slli(lo(rd), lo(rd), 1), ctx); 425 } else { 426 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx); 427 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx); 428 emit(rv_add(hi(rd), hi(rd), hi(rs)), ctx); 429 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx); 430 } 431 break; 432 case BPF_SUB: 433 emit(rv_sub(RV_REG_T1, hi(rd), hi(rs)), ctx); 434 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx); 435 emit(rv_sub(hi(rd), RV_REG_T1, RV_REG_T0), ctx); 436 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx); 437 break; 438 case BPF_AND: 439 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx); 440 emit(rv_and(hi(rd), hi(rd), hi(rs)), ctx); 441 break; 442 case BPF_OR: 443 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx); 444 emit(rv_or(hi(rd), hi(rd), hi(rs)), ctx); 445 break; 446 case BPF_XOR: 447 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx); 448 emit(rv_xor(hi(rd), hi(rd), hi(rs)), ctx); 449 break; 450 case BPF_MUL: 451 emit(rv_mul(RV_REG_T0, hi(rs), lo(rd)), ctx); 452 emit(rv_mul(hi(rd), hi(rd), lo(rs)), ctx); 453 emit(rv_mulhu(RV_REG_T1, lo(rd), lo(rs)), ctx); 454 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx); 455 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); 456 emit(rv_add(hi(rd), hi(rd), RV_REG_T1), ctx); 457 break; 458 case BPF_LSH: 459 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx); 460 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx); 461 emit(rv_sll(hi(rd), lo(rd), RV_REG_T0), ctx); 462 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx); 463 emit(rv_jal(RV_REG_ZERO, 16), ctx); 464 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx); 465 emit(rv_srli(RV_REG_T0, lo(rd), 1), ctx); 466 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx); 467 emit(rv_srl(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx); 468 emit(rv_sll(hi(rd), hi(rd), lo(rs)), ctx); 469 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx); 470 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); 471 break; 472 case BPF_RSH: 473 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx); 474 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx); 475 emit(rv_srl(lo(rd), hi(rd), RV_REG_T0), ctx); 476 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 477 emit(rv_jal(RV_REG_ZERO, 16), ctx); 478 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx); 479 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx); 480 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx); 481 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx); 482 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx); 483 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx); 484 emit(rv_srl(hi(rd), hi(rd), lo(rs)), ctx); 485 break; 486 case BPF_ARSH: 487 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx); 488 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx); 489 emit(rv_sra(lo(rd), hi(rd), RV_REG_T0), ctx); 490 emit(rv_srai(hi(rd), hi(rd), 31), ctx); 491 emit(rv_jal(RV_REG_ZERO, 16), ctx); 492 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx); 493 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx); 494 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx); 495 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx); 496 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx); 497 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx); 498 emit(rv_sra(hi(rd), hi(rd), lo(rs)), ctx); 499 break; 500 case BPF_NEG: 501 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx); 502 emit(rv_sltu(RV_REG_T0, RV_REG_ZERO, lo(rd)), ctx); 503 emit(rv_sub(hi(rd), RV_REG_ZERO, hi(rd)), ctx); 504 emit(rv_sub(hi(rd), hi(rd), RV_REG_T0), ctx); 505 break; 506 } 507 508 bpf_put_reg64(dst, rd, ctx); 509 } 510 511 static void emit_alu_r32(const s8 *dst, const s8 *src, 512 struct rv_jit_context *ctx, 513 const struct bpf_insn *insn) 514 { 515 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 516 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 517 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); 518 const s8 *rs = bpf_get_reg32(src, tmp2, ctx); 519 u8 op = BPF_OP(insn->code); 520 bool is_signed = insn->off == 1; 521 522 switch (op) { 523 case BPF_MOV: 524 emit(rv_addi(lo(rd), lo(rs), 0), ctx); 525 break; 526 case BPF_ADD: 527 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx); 528 break; 529 case BPF_SUB: 530 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx); 531 break; 532 case BPF_AND: 533 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx); 534 break; 535 case BPF_OR: 536 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx); 537 break; 538 case BPF_XOR: 539 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx); 540 break; 541 case BPF_MUL: 542 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); 543 break; 544 case BPF_DIV: 545 emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) : 546 rv_divu(lo(rd), lo(rd), lo(rs)), ctx); 547 break; 548 case BPF_MOD: 549 emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) : 550 rv_remu(lo(rd), lo(rd), lo(rs)), ctx); 551 break; 552 case BPF_LSH: 553 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); 554 break; 555 case BPF_RSH: 556 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx); 557 break; 558 case BPF_ARSH: 559 emit(rv_sra(lo(rd), lo(rd), lo(rs)), ctx); 560 break; 561 case BPF_NEG: 562 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx); 563 break; 564 } 565 566 bpf_put_reg32(dst, rd, ctx); 567 } 568 569 static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff, 570 struct rv_jit_context *ctx, const u8 op) 571 { 572 int e, s = ctx->ninsns; 573 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 574 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 575 576 const s8 *rs1 = bpf_get_reg64(src1, tmp1, ctx); 577 const s8 *rs2 = bpf_get_reg64(src2, tmp2, ctx); 578 579 /* 580 * NO_JUMP skips over the rest of the instructions and the 581 * emit_jump_and_link, meaning the BPF branch is not taken. 582 * JUMP skips directly to the emit_jump_and_link, meaning 583 * the BPF branch is taken. 584 * 585 * The fallthrough case results in the BPF branch being taken. 586 */ 587 #define NO_JUMP(idx) (6 + (2 * (idx))) 588 #define JUMP(idx) (2 + (2 * (idx))) 589 590 switch (op) { 591 case BPF_JEQ: 592 emit(rv_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 593 emit(rv_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 594 break; 595 case BPF_JGT: 596 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); 597 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 598 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 599 break; 600 case BPF_JLT: 601 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); 602 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 603 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 604 break; 605 case BPF_JGE: 606 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx); 607 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 608 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 609 break; 610 case BPF_JLE: 611 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx); 612 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 613 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 614 break; 615 case BPF_JNE: 616 emit(rv_bne(hi(rs1), hi(rs2), JUMP(1)), ctx); 617 emit(rv_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 618 break; 619 case BPF_JSGT: 620 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); 621 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 622 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 623 break; 624 case BPF_JSLT: 625 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); 626 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 627 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 628 break; 629 case BPF_JSGE: 630 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx); 631 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 632 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 633 break; 634 case BPF_JSLE: 635 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx); 636 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx); 637 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx); 638 break; 639 case BPF_JSET: 640 emit(rv_and(RV_REG_T0, hi(rs1), hi(rs2)), ctx); 641 emit(rv_bne(RV_REG_T0, RV_REG_ZERO, JUMP(2)), ctx); 642 emit(rv_and(RV_REG_T0, lo(rs1), lo(rs2)), ctx); 643 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, NO_JUMP(0)), ctx); 644 break; 645 } 646 647 #undef NO_JUMP 648 #undef JUMP 649 650 e = ctx->ninsns; 651 /* Adjust for extra insns. */ 652 rvoff -= ninsns_rvoff(e - s); 653 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 654 return 0; 655 } 656 657 static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx) 658 { 659 int e, s = ctx->ninsns; 660 bool far = false; 661 int off; 662 663 if (op == BPF_JSET) { 664 /* 665 * BPF_JSET is a special case: it has no inverse so we always 666 * treat it as a far branch. 667 */ 668 far = true; 669 } else if (!is_13b_int(rvoff)) { 670 op = invert_bpf_cond(op); 671 far = true; 672 } 673 674 /* 675 * For a far branch, the condition is negated and we jump over the 676 * branch itself, and the two instructions from emit_jump_and_link. 677 * For a near branch, just use rvoff. 678 */ 679 off = far ? 6 : (rvoff >> 1); 680 681 switch (op) { 682 case BPF_JEQ: 683 emit(rv_beq(rd, rs, off), ctx); 684 break; 685 case BPF_JGT: 686 emit(rv_bgtu(rd, rs, off), ctx); 687 break; 688 case BPF_JLT: 689 emit(rv_bltu(rd, rs, off), ctx); 690 break; 691 case BPF_JGE: 692 emit(rv_bgeu(rd, rs, off), ctx); 693 break; 694 case BPF_JLE: 695 emit(rv_bleu(rd, rs, off), ctx); 696 break; 697 case BPF_JNE: 698 emit(rv_bne(rd, rs, off), ctx); 699 break; 700 case BPF_JSGT: 701 emit(rv_bgt(rd, rs, off), ctx); 702 break; 703 case BPF_JSLT: 704 emit(rv_blt(rd, rs, off), ctx); 705 break; 706 case BPF_JSGE: 707 emit(rv_bge(rd, rs, off), ctx); 708 break; 709 case BPF_JSLE: 710 emit(rv_ble(rd, rs, off), ctx); 711 break; 712 case BPF_JSET: 713 emit(rv_and(RV_REG_T0, rd, rs), ctx); 714 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, off), ctx); 715 break; 716 } 717 718 if (far) { 719 e = ctx->ninsns; 720 /* Adjust for extra insns. */ 721 rvoff -= ninsns_rvoff(e - s); 722 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 723 } 724 return 0; 725 } 726 727 static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff, 728 struct rv_jit_context *ctx, const u8 op) 729 { 730 int e, s = ctx->ninsns; 731 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 732 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 733 734 const s8 *rs1 = bpf_get_reg32(src1, tmp1, ctx); 735 const s8 *rs2 = bpf_get_reg32(src2, tmp2, ctx); 736 737 e = ctx->ninsns; 738 /* Adjust for extra insns. */ 739 rvoff -= ninsns_rvoff(e - s); 740 741 if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx)) 742 return -1; 743 744 return 0; 745 } 746 747 static void emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 748 { 749 const s8 *r0 = bpf2rv32[BPF_REG_0]; 750 const s8 *r5 = bpf2rv32[BPF_REG_5]; 751 u32 upper = ((u32)addr + (1 << 11)) >> 12; 752 u32 lower = addr & 0xfff; 753 754 /* R1-R4 already in correct registers---need to push R5 to stack. */ 755 emit(rv_addi(RV_REG_SP, RV_REG_SP, -16), ctx); 756 emit(rv_sw(RV_REG_SP, 0, lo(r5)), ctx); 757 emit(rv_sw(RV_REG_SP, 4, hi(r5)), ctx); 758 759 /* Backup TCC. */ 760 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx); 761 762 /* 763 * Use lui/jalr pair to jump to absolute address. Don't use emit_imm as 764 * the number of emitted instructions should not depend on the value of 765 * addr. 766 */ 767 emit(rv_lui(RV_REG_T1, upper), ctx); 768 emit(rv_jalr(RV_REG_RA, RV_REG_T1, lower), ctx); 769 770 /* Restore TCC. */ 771 emit(rv_addi(RV_REG_TCC, RV_REG_TCC_SAVED, 0), ctx); 772 773 /* Set return value and restore stack. */ 774 emit(rv_addi(lo(r0), RV_REG_A0, 0), ctx); 775 emit(rv_addi(hi(r0), RV_REG_A1, 0), ctx); 776 emit(rv_addi(RV_REG_SP, RV_REG_SP, 16), ctx); 777 } 778 779 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 780 { 781 /* 782 * R1 -> &ctx 783 * R2 -> &array 784 * R3 -> index 785 */ 786 int tc_ninsn, off, start_insn = ctx->ninsns; 787 const s8 *arr_reg = bpf2rv32[BPF_REG_2]; 788 const s8 *idx_reg = bpf2rv32[BPF_REG_3]; 789 790 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 791 ctx->offset[0]; 792 793 /* max_entries = array->map.max_entries; */ 794 off = offsetof(struct bpf_array, map.max_entries); 795 if (is_12b_check(off, insn)) 796 return -1; 797 emit(rv_lw(RV_REG_T1, off, lo(arr_reg)), ctx); 798 799 /* 800 * if (index >= max_entries) 801 * goto out; 802 */ 803 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 804 emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx); 805 806 /* 807 * if (--tcc < 0) 808 * goto out; 809 */ 810 emit(rv_addi(RV_REG_TCC, RV_REG_TCC, -1), ctx); 811 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 812 emit_bcc(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 813 814 /* 815 * prog = array->ptrs[index]; 816 * if (!prog) 817 * goto out; 818 */ 819 emit_sh2add(RV_REG_T0, lo(idx_reg), lo(arr_reg), ctx); 820 off = offsetof(struct bpf_array, ptrs); 821 if (is_12b_check(off, insn)) 822 return -1; 823 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); 824 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 825 emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx); 826 827 /* 828 * tcc = temp_tcc; 829 * goto *(prog->bpf_func + 4); 830 */ 831 off = offsetof(struct bpf_prog, bpf_func); 832 if (is_12b_check(off, insn)) 833 return -1; 834 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx); 835 /* Epilogue jumps to *(t0 + 4). */ 836 __build_epilogue(true, ctx); 837 return 0; 838 } 839 840 static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, 841 struct rv_jit_context *ctx, const u8 size) 842 { 843 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 844 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 845 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 846 const s8 *rs = bpf_get_reg64(src, tmp2, ctx); 847 848 emit_imm(RV_REG_T0, off, ctx); 849 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rs)), ctx); 850 851 switch (size) { 852 case BPF_B: 853 emit(rv_lbu(lo(rd), 0, RV_REG_T0), ctx); 854 if (!ctx->prog->aux->verifier_zext) 855 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 856 break; 857 case BPF_H: 858 emit(rv_lhu(lo(rd), 0, RV_REG_T0), ctx); 859 if (!ctx->prog->aux->verifier_zext) 860 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 861 break; 862 case BPF_W: 863 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx); 864 if (!ctx->prog->aux->verifier_zext) 865 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 866 break; 867 case BPF_DW: 868 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx); 869 emit(rv_lw(hi(rd), 4, RV_REG_T0), ctx); 870 break; 871 } 872 873 bpf_put_reg64(dst, rd, ctx); 874 return 0; 875 } 876 877 static int emit_bpf_atomic(s8 dst, const s8 *src, const s8 *rs, 878 struct rv_jit_context *ctx, 879 const struct bpf_insn *insn) 880 { 881 s32 imm = insn->imm; 882 bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG); 883 s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO; 884 int aq = is_fetch ? 1 : 0; 885 int rl = is_fetch ? 1 : 0; 886 887 switch (imm) { 888 case BPF_ADD: 889 case BPF_ADD | BPF_FETCH: 890 emit(rv_amoadd_w(fetch_reg, lo(rs), dst, aq, rl), ctx); 891 break; 892 case BPF_AND: 893 case BPF_AND | BPF_FETCH: 894 emit(rv_amoand_w(fetch_reg, lo(rs), dst, aq, rl), ctx); 895 break; 896 case BPF_OR: 897 case BPF_OR | BPF_FETCH: 898 emit(rv_amoor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); 899 break; 900 case BPF_XOR: 901 case BPF_XOR | BPF_FETCH: 902 emit(rv_amoxor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); 903 break; 904 case BPF_XCHG: 905 emit(rv_amoswap_w(fetch_reg, lo(rs), dst, aq, rl), ctx); 906 break; 907 default: 908 return -1; 909 } 910 911 if (is_fetch) { 912 emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx); 913 bpf_put_reg64(src, rs, ctx); 914 } 915 return 0; 916 } 917 918 static int emit_store_r64(const s8 *dst, const s8 *src, 919 struct rv_jit_context *ctx, 920 const struct bpf_insn *insn) 921 { 922 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 923 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 924 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 925 const s8 *rs = bpf_get_reg64(src, tmp2, ctx); 926 u8 size = BPF_SIZE(insn->code); 927 u8 mode = BPF_MODE(insn->code); 928 s16 off = insn->off; 929 930 if (mode == BPF_ATOMIC && size != BPF_W) 931 return -1; 932 933 emit_imm(RV_REG_T0, off, ctx); 934 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rd)), ctx); 935 936 switch (size) { 937 case BPF_B: 938 emit(rv_sb(RV_REG_T0, 0, lo(rs)), ctx); 939 break; 940 case BPF_H: 941 emit(rv_sh(RV_REG_T0, 0, lo(rs)), ctx); 942 break; 943 case BPF_W: 944 switch (mode) { 945 case BPF_MEM: 946 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); 947 break; 948 case BPF_ATOMIC: 949 if (emit_bpf_atomic(RV_REG_T0, src, rs, ctx, insn)) 950 return -1; 951 break; 952 } 953 break; 954 case BPF_DW: 955 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); 956 emit(rv_sw(RV_REG_T0, 4, hi(rs)), ctx); 957 break; 958 } 959 960 return 0; 961 } 962 963 static void emit_rev16(const s8 rd, struct rv_jit_context *ctx) 964 { 965 emit(rv_slli(rd, rd, 16), ctx); 966 emit(rv_slli(RV_REG_T1, rd, 8), ctx); 967 emit(rv_srli(rd, rd, 8), ctx); 968 emit(rv_add(RV_REG_T1, rd, RV_REG_T1), ctx); 969 emit(rv_srli(rd, RV_REG_T1, 16), ctx); 970 } 971 972 static void emit_rev32(const s8 rd, struct rv_jit_context *ctx) 973 { 974 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 0), ctx); 975 emit(rv_andi(RV_REG_T0, rd, 255), ctx); 976 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx); 977 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx); 978 emit(rv_srli(rd, rd, 8), ctx); 979 emit(rv_andi(RV_REG_T0, rd, 255), ctx); 980 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx); 981 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx); 982 emit(rv_srli(rd, rd, 8), ctx); 983 emit(rv_andi(RV_REG_T0, rd, 255), ctx); 984 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx); 985 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx); 986 emit(rv_srli(rd, rd, 8), ctx); 987 emit(rv_andi(RV_REG_T0, rd, 255), ctx); 988 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx); 989 emit(rv_addi(rd, RV_REG_T1, 0), ctx); 990 } 991 992 static void emit_zext64(const s8 *dst, struct rv_jit_context *ctx) 993 { 994 const s8 *rd; 995 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 996 997 rd = bpf_get_reg64(dst, tmp1, ctx); 998 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 999 bpf_put_reg64(dst, rd, ctx); 1000 } 1001 1002 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1003 bool extra_pass) 1004 { 1005 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1006 BPF_CLASS(insn->code) == BPF_JMP; 1007 int s, e, rvoff, i = insn - ctx->prog->insnsi; 1008 u8 code = insn->code; 1009 s16 off = insn->off; 1010 s32 imm = insn->imm; 1011 1012 const s8 *dst = bpf2rv32[insn->dst_reg]; 1013 const s8 *src = bpf2rv32[insn->src_reg]; 1014 const s8 *tmp1 = bpf2rv32[TMP_REG_1]; 1015 const s8 *tmp2 = bpf2rv32[TMP_REG_2]; 1016 1017 switch (code) { 1018 case BPF_ALU64 | BPF_MOV | BPF_X: 1019 if (insn->off != 0) { 1020 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 1021 const s8 *rs = bpf_get_reg64(src, tmp2, ctx); 1022 1023 if (insn->off == 8) { 1024 emit(rv_slli(lo(rd), lo(rs), 24), ctx); 1025 emit(rv_srai(lo(rd), lo(rd), 24), ctx); 1026 } else if (insn->off == 16) { 1027 emit(rv_slli(lo(rd), lo(rs), 16), ctx); 1028 emit(rv_srai(lo(rd), lo(rd), 16), ctx); 1029 } else { 1030 emit(rv_addi(lo(rd), lo(rs), 0), ctx); 1031 } 1032 emit(rv_srai(hi(rd), lo(rd), 31), ctx); 1033 bpf_put_reg64(dst, rd, ctx); 1034 break; 1035 } 1036 fallthrough; 1037 1038 case BPF_ALU64 | BPF_ADD | BPF_X: 1039 case BPF_ALU64 | BPF_ADD | BPF_K: 1040 1041 case BPF_ALU64 | BPF_SUB | BPF_X: 1042 case BPF_ALU64 | BPF_SUB | BPF_K: 1043 1044 case BPF_ALU64 | BPF_AND | BPF_X: 1045 case BPF_ALU64 | BPF_OR | BPF_X: 1046 case BPF_ALU64 | BPF_XOR | BPF_X: 1047 1048 case BPF_ALU64 | BPF_MUL | BPF_X: 1049 case BPF_ALU64 | BPF_MUL | BPF_K: 1050 1051 case BPF_ALU64 | BPF_LSH | BPF_X: 1052 case BPF_ALU64 | BPF_RSH | BPF_X: 1053 case BPF_ALU64 | BPF_ARSH | BPF_X: 1054 if (BPF_SRC(code) == BPF_K) { 1055 emit_imm32(tmp2, imm, ctx); 1056 src = tmp2; 1057 } 1058 emit_alu_r64(dst, src, ctx, BPF_OP(code)); 1059 break; 1060 1061 case BPF_ALU64 | BPF_NEG: 1062 emit_alu_r64(dst, tmp2, ctx, BPF_OP(code)); 1063 break; 1064 1065 case BPF_ALU64 | BPF_DIV | BPF_X: 1066 case BPF_ALU64 | BPF_DIV | BPF_K: 1067 case BPF_ALU64 | BPF_MOD | BPF_X: 1068 case BPF_ALU64 | BPF_MOD | BPF_K: 1069 goto notsupported; 1070 1071 case BPF_ALU64 | BPF_MOV | BPF_K: 1072 case BPF_ALU64 | BPF_AND | BPF_K: 1073 case BPF_ALU64 | BPF_OR | BPF_K: 1074 case BPF_ALU64 | BPF_XOR | BPF_K: 1075 case BPF_ALU64 | BPF_LSH | BPF_K: 1076 case BPF_ALU64 | BPF_RSH | BPF_K: 1077 case BPF_ALU64 | BPF_ARSH | BPF_K: 1078 emit_alu_i64(dst, imm, ctx, BPF_OP(code)); 1079 break; 1080 1081 case BPF_ALU | BPF_MOV | BPF_X: 1082 if (imm == 1) { 1083 /* Special mov32 for zext. */ 1084 emit_zext64(dst, ctx); 1085 break; 1086 } 1087 if (insn->off != 0) { 1088 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); 1089 const s8 *rs = bpf_get_reg32(src, tmp2, ctx); 1090 1091 if (insn->off == 8) { 1092 emit(rv_slli(lo(rd), lo(rs), 24), ctx); 1093 emit(rv_srai(lo(rd), lo(rd), 24), ctx); 1094 } else if (insn->off == 16) { 1095 emit(rv_slli(lo(rd), lo(rs), 16), ctx); 1096 emit(rv_srai(lo(rd), lo(rd), 16), ctx); 1097 } 1098 bpf_put_reg32(dst, rd, ctx); 1099 break; 1100 } 1101 fallthrough; 1102 1103 case BPF_ALU | BPF_ADD | BPF_X: 1104 case BPF_ALU | BPF_SUB | BPF_X: 1105 case BPF_ALU | BPF_AND | BPF_X: 1106 case BPF_ALU | BPF_OR | BPF_X: 1107 case BPF_ALU | BPF_XOR | BPF_X: 1108 1109 case BPF_ALU | BPF_MUL | BPF_X: 1110 case BPF_ALU | BPF_MUL | BPF_K: 1111 1112 case BPF_ALU | BPF_DIV | BPF_X: 1113 case BPF_ALU | BPF_DIV | BPF_K: 1114 1115 case BPF_ALU | BPF_MOD | BPF_X: 1116 case BPF_ALU | BPF_MOD | BPF_K: 1117 1118 case BPF_ALU | BPF_LSH | BPF_X: 1119 case BPF_ALU | BPF_RSH | BPF_X: 1120 case BPF_ALU | BPF_ARSH | BPF_X: 1121 if (BPF_SRC(code) == BPF_K) { 1122 emit_imm32(tmp2, imm, ctx); 1123 src = tmp2; 1124 } 1125 emit_alu_r32(dst, src, ctx, insn); 1126 break; 1127 1128 case BPF_ALU | BPF_MOV | BPF_K: 1129 case BPF_ALU | BPF_ADD | BPF_K: 1130 case BPF_ALU | BPF_SUB | BPF_K: 1131 case BPF_ALU | BPF_AND | BPF_K: 1132 case BPF_ALU | BPF_OR | BPF_K: 1133 case BPF_ALU | BPF_XOR | BPF_K: 1134 case BPF_ALU | BPF_LSH | BPF_K: 1135 case BPF_ALU | BPF_RSH | BPF_K: 1136 case BPF_ALU | BPF_ARSH | BPF_K: 1137 /* 1138 * mul,div,mod are handled in the BPF_X case since there are 1139 * no RISC-V I-type equivalents. 1140 */ 1141 emit_alu_i32(dst, imm, ctx, BPF_OP(code)); 1142 break; 1143 1144 case BPF_ALU | BPF_NEG: 1145 /* 1146 * src is ignored---choose tmp2 as a dummy register since it 1147 * is not on the stack. 1148 */ 1149 emit_alu_r32(dst, tmp2, ctx, insn); 1150 break; 1151 1152 case BPF_ALU | BPF_END | BPF_FROM_LE: 1153 { 1154 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 1155 1156 switch (imm) { 1157 case 16: 1158 emit(rv_slli(lo(rd), lo(rd), 16), ctx); 1159 emit(rv_srli(lo(rd), lo(rd), 16), ctx); 1160 fallthrough; 1161 case 32: 1162 if (!ctx->prog->aux->verifier_zext) 1163 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 1164 break; 1165 case 64: 1166 /* Do nothing. */ 1167 break; 1168 default: 1169 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); 1170 return -1; 1171 } 1172 1173 bpf_put_reg64(dst, rd, ctx); 1174 break; 1175 } 1176 1177 case BPF_ALU | BPF_END | BPF_FROM_BE: 1178 { 1179 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 1180 1181 switch (imm) { 1182 case 16: 1183 emit_rev16(lo(rd), ctx); 1184 if (!ctx->prog->aux->verifier_zext) 1185 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 1186 break; 1187 case 32: 1188 emit_rev32(lo(rd), ctx); 1189 if (!ctx->prog->aux->verifier_zext) 1190 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx); 1191 break; 1192 case 64: 1193 /* Swap upper and lower halves. */ 1194 emit(rv_addi(RV_REG_T0, lo(rd), 0), ctx); 1195 emit(rv_addi(lo(rd), hi(rd), 0), ctx); 1196 emit(rv_addi(hi(rd), RV_REG_T0, 0), ctx); 1197 1198 /* Swap each half. */ 1199 emit_rev32(lo(rd), ctx); 1200 emit_rev32(hi(rd), ctx); 1201 break; 1202 default: 1203 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); 1204 return -1; 1205 } 1206 1207 bpf_put_reg64(dst, rd, ctx); 1208 break; 1209 } 1210 1211 case BPF_JMP | BPF_JA: 1212 rvoff = rv_offset(i, off, ctx); 1213 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 1214 break; 1215 1216 case BPF_JMP | BPF_CALL: 1217 { 1218 bool fixed; 1219 int ret; 1220 u64 addr; 1221 1222 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, 1223 &fixed); 1224 if (ret < 0) 1225 return ret; 1226 emit_call(fixed, addr, ctx); 1227 break; 1228 } 1229 1230 case BPF_JMP | BPF_TAIL_CALL: 1231 if (emit_bpf_tail_call(i, ctx)) 1232 return -1; 1233 break; 1234 1235 case BPF_JMP | BPF_JEQ | BPF_X: 1236 case BPF_JMP | BPF_JEQ | BPF_K: 1237 case BPF_JMP32 | BPF_JEQ | BPF_X: 1238 case BPF_JMP32 | BPF_JEQ | BPF_K: 1239 1240 case BPF_JMP | BPF_JNE | BPF_X: 1241 case BPF_JMP | BPF_JNE | BPF_K: 1242 case BPF_JMP32 | BPF_JNE | BPF_X: 1243 case BPF_JMP32 | BPF_JNE | BPF_K: 1244 1245 case BPF_JMP | BPF_JLE | BPF_X: 1246 case BPF_JMP | BPF_JLE | BPF_K: 1247 case BPF_JMP32 | BPF_JLE | BPF_X: 1248 case BPF_JMP32 | BPF_JLE | BPF_K: 1249 1250 case BPF_JMP | BPF_JLT | BPF_X: 1251 case BPF_JMP | BPF_JLT | BPF_K: 1252 case BPF_JMP32 | BPF_JLT | BPF_X: 1253 case BPF_JMP32 | BPF_JLT | BPF_K: 1254 1255 case BPF_JMP | BPF_JGE | BPF_X: 1256 case BPF_JMP | BPF_JGE | BPF_K: 1257 case BPF_JMP32 | BPF_JGE | BPF_X: 1258 case BPF_JMP32 | BPF_JGE | BPF_K: 1259 1260 case BPF_JMP | BPF_JGT | BPF_X: 1261 case BPF_JMP | BPF_JGT | BPF_K: 1262 case BPF_JMP32 | BPF_JGT | BPF_X: 1263 case BPF_JMP32 | BPF_JGT | BPF_K: 1264 1265 case BPF_JMP | BPF_JSLE | BPF_X: 1266 case BPF_JMP | BPF_JSLE | BPF_K: 1267 case BPF_JMP32 | BPF_JSLE | BPF_X: 1268 case BPF_JMP32 | BPF_JSLE | BPF_K: 1269 1270 case BPF_JMP | BPF_JSLT | BPF_X: 1271 case BPF_JMP | BPF_JSLT | BPF_K: 1272 case BPF_JMP32 | BPF_JSLT | BPF_X: 1273 case BPF_JMP32 | BPF_JSLT | BPF_K: 1274 1275 case BPF_JMP | BPF_JSGE | BPF_X: 1276 case BPF_JMP | BPF_JSGE | BPF_K: 1277 case BPF_JMP32 | BPF_JSGE | BPF_X: 1278 case BPF_JMP32 | BPF_JSGE | BPF_K: 1279 1280 case BPF_JMP | BPF_JSGT | BPF_X: 1281 case BPF_JMP | BPF_JSGT | BPF_K: 1282 case BPF_JMP32 | BPF_JSGT | BPF_X: 1283 case BPF_JMP32 | BPF_JSGT | BPF_K: 1284 1285 case BPF_JMP | BPF_JSET | BPF_X: 1286 case BPF_JMP | BPF_JSET | BPF_K: 1287 case BPF_JMP32 | BPF_JSET | BPF_X: 1288 case BPF_JMP32 | BPF_JSET | BPF_K: 1289 rvoff = rv_offset(i, off, ctx); 1290 if (BPF_SRC(code) == BPF_K) { 1291 s = ctx->ninsns; 1292 emit_imm32(tmp2, imm, ctx); 1293 src = tmp2; 1294 e = ctx->ninsns; 1295 rvoff -= ninsns_rvoff(e - s); 1296 } 1297 1298 if (is64) 1299 emit_branch_r64(dst, src, rvoff, ctx, BPF_OP(code)); 1300 else 1301 emit_branch_r32(dst, src, rvoff, ctx, BPF_OP(code)); 1302 break; 1303 1304 case BPF_JMP | BPF_EXIT: 1305 if (i == ctx->prog->len - 1) 1306 break; 1307 1308 rvoff = epilogue_offset(ctx); 1309 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 1310 break; 1311 1312 case BPF_LD | BPF_IMM | BPF_DW: 1313 { 1314 struct bpf_insn insn1 = insn[1]; 1315 s32 imm_lo = imm; 1316 s32 imm_hi = insn1.imm; 1317 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); 1318 1319 emit_imm64(rd, imm_hi, imm_lo, ctx); 1320 bpf_put_reg64(dst, rd, ctx); 1321 return 1; 1322 } 1323 1324 case BPF_LDX | BPF_MEM | BPF_B: 1325 case BPF_LDX | BPF_MEM | BPF_H: 1326 case BPF_LDX | BPF_MEM | BPF_W: 1327 case BPF_LDX | BPF_MEM | BPF_DW: 1328 if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code))) 1329 return -1; 1330 break; 1331 1332 /* speculation barrier */ 1333 case BPF_ST | BPF_NOSPEC: 1334 break; 1335 1336 case BPF_ST | BPF_MEM | BPF_B: 1337 case BPF_ST | BPF_MEM | BPF_H: 1338 case BPF_ST | BPF_MEM | BPF_W: 1339 case BPF_ST | BPF_MEM | BPF_DW: 1340 1341 case BPF_STX | BPF_MEM | BPF_B: 1342 case BPF_STX | BPF_MEM | BPF_H: 1343 case BPF_STX | BPF_MEM | BPF_W: 1344 case BPF_STX | BPF_MEM | BPF_DW: 1345 if (BPF_CLASS(code) == BPF_ST) { 1346 emit_imm32(tmp2, imm, ctx); 1347 src = tmp2; 1348 } 1349 1350 if (emit_store_r64(dst, src, ctx, insn)) 1351 return -1; 1352 break; 1353 1354 case BPF_STX | BPF_ATOMIC | BPF_W: 1355 if (insn->imm == BPF_CMPXCHG) { 1356 pr_info_once( 1357 "bpf-jit: not supported: atomic operation %02x ***\n", 1358 insn->imm); 1359 return -EFAULT; 1360 } 1361 1362 if (emit_store_r64(dst, src, ctx, insn)) 1363 return -1; 1364 break; 1365 1366 /* No hardware support for 8-byte atomics in RV32. */ 1367 case BPF_STX | BPF_ATOMIC | BPF_DW: 1368 /* Fallthrough. */ 1369 1370 notsupported: 1371 pr_info_once("bpf-jit: not supported: opcode %02x ***\n", code); 1372 return -EFAULT; 1373 1374 default: 1375 pr_err("bpf-jit: unknown opcode %02x\n", code); 1376 return -EINVAL; 1377 } 1378 1379 return 0; 1380 } 1381 1382 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) 1383 { 1384 const s8 *fp = bpf2rv32[BPF_REG_FP]; 1385 const s8 *r1 = bpf2rv32[BPF_REG_1]; 1386 int stack_adjust = 0; 1387 int bpf_stack_adjust = 1388 round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); 1389 1390 /* Make space for callee-saved registers. */ 1391 stack_adjust += NR_SAVED_REGISTERS * sizeof(u32); 1392 /* Make space for BPF registers on stack. */ 1393 stack_adjust += BPF_JIT_SCRATCH_REGS * sizeof(u32); 1394 /* Make space for BPF stack. */ 1395 stack_adjust += bpf_stack_adjust; 1396 /* Round up for stack alignment. */ 1397 stack_adjust = round_up(stack_adjust, STACK_ALIGN); 1398 1399 /* 1400 * The first instruction sets the tail-call-counter (TCC) register. 1401 * This instruction is skipped by tail calls. 1402 */ 1403 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1404 1405 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); 1406 1407 /* Save callee-save registers. */ 1408 emit(rv_sw(RV_REG_SP, stack_adjust - 4, RV_REG_RA), ctx); 1409 emit(rv_sw(RV_REG_SP, stack_adjust - 8, RV_REG_FP), ctx); 1410 emit(rv_sw(RV_REG_SP, stack_adjust - 12, RV_REG_S1), ctx); 1411 emit(rv_sw(RV_REG_SP, stack_adjust - 16, RV_REG_S2), ctx); 1412 emit(rv_sw(RV_REG_SP, stack_adjust - 20, RV_REG_S3), ctx); 1413 emit(rv_sw(RV_REG_SP, stack_adjust - 24, RV_REG_S4), ctx); 1414 emit(rv_sw(RV_REG_SP, stack_adjust - 28, RV_REG_S5), ctx); 1415 emit(rv_sw(RV_REG_SP, stack_adjust - 32, RV_REG_S6), ctx); 1416 emit(rv_sw(RV_REG_SP, stack_adjust - 36, RV_REG_S7), ctx); 1417 1418 /* Set fp: used as the base address for stacked BPF registers. */ 1419 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); 1420 1421 /* Set up BPF frame pointer. */ 1422 emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx); 1423 emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx); 1424 1425 /* Set up BPF context pointer. */ 1426 emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx); 1427 emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx); 1428 1429 ctx->stack_size = stack_adjust; 1430 } 1431 1432 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1433 { 1434 __build_epilogue(false, ctx); 1435 } 1436