1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 15 #define RV_MAX_REG_ARGS 8 16 #define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS) 17 18 /* verify runtime detection extension status */ 19 #define rv_ext_enabled(ext) \ 20 (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) 21 22 static inline bool rvc_enabled(void) 23 { 24 return IS_ENABLED(CONFIG_RISCV_ISA_C); 25 } 26 27 enum { 28 RV_REG_ZERO = 0, /* The constant value 0 */ 29 RV_REG_RA = 1, /* Return address */ 30 RV_REG_SP = 2, /* Stack pointer */ 31 RV_REG_GP = 3, /* Global pointer */ 32 RV_REG_TP = 4, /* Thread pointer */ 33 RV_REG_T0 = 5, /* Temporaries */ 34 RV_REG_T1 = 6, 35 RV_REG_T2 = 7, 36 RV_REG_FP = 8, /* Saved register/frame pointer */ 37 RV_REG_S1 = 9, /* Saved register */ 38 RV_REG_A0 = 10, /* Function argument/return values */ 39 RV_REG_A1 = 11, /* Function arguments */ 40 RV_REG_A2 = 12, 41 RV_REG_A3 = 13, 42 RV_REG_A4 = 14, 43 RV_REG_A5 = 15, 44 RV_REG_A6 = 16, 45 RV_REG_A7 = 17, 46 RV_REG_S2 = 18, /* Saved registers */ 47 RV_REG_S3 = 19, 48 RV_REG_S4 = 20, 49 RV_REG_S5 = 21, 50 RV_REG_S6 = 22, 51 RV_REG_S7 = 23, 52 RV_REG_S8 = 24, 53 RV_REG_S9 = 25, 54 RV_REG_S10 = 26, 55 RV_REG_S11 = 27, 56 RV_REG_T3 = 28, /* Temporaries */ 57 RV_REG_T4 = 29, 58 RV_REG_T5 = 30, 59 RV_REG_T6 = 31, 60 }; 61 62 static inline bool is_creg(u8 reg) 63 { 64 return (1 << reg) & (BIT(RV_REG_FP) | 65 BIT(RV_REG_S1) | 66 BIT(RV_REG_A0) | 67 BIT(RV_REG_A1) | 68 BIT(RV_REG_A2) | 69 BIT(RV_REG_A3) | 70 BIT(RV_REG_A4) | 71 BIT(RV_REG_A5)); 72 } 73 74 struct rv_jit_context { 75 struct bpf_prog *prog; 76 u16 *insns; /* RV insns */ 77 u16 *ro_insns; 78 int ninsns; 79 int prologue_len; 80 int epilogue_offset; 81 int *offset; /* BPF to RV */ 82 int nexentries; 83 int ex_insn_off; 84 int ex_jmp_off; 85 unsigned long flags; 86 int stack_size; 87 int tcc_offset; 88 int stack_arg_sz; 89 u64 arena_vm_start; 90 u64 user_vm_start; 91 }; 92 93 /* Convert from ninsns to bytes. */ 94 static inline int ninsns_rvoff(int ninsns) 95 { 96 return ninsns << 1; 97 } 98 99 struct rv_jit_data { 100 struct bpf_binary_header *header; 101 struct bpf_binary_header *ro_header; 102 u8 *image; 103 u8 *ro_image; 104 struct rv_jit_context ctx; 105 }; 106 107 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 108 { 109 memset(area, 0, size); 110 } 111 112 /* Emit a 4-byte riscv instruction. */ 113 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 114 { 115 if (ctx->insns) { 116 ctx->insns[ctx->ninsns] = insn; 117 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 118 } 119 120 ctx->ninsns += 2; 121 } 122 123 /* Emit a 2-byte riscv compressed instruction. */ 124 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 125 { 126 BUILD_BUG_ON(!rvc_enabled()); 127 128 if (ctx->insns) 129 ctx->insns[ctx->ninsns] = insn; 130 131 ctx->ninsns++; 132 } 133 134 static inline int epilogue_offset(struct rv_jit_context *ctx) 135 { 136 int to = ctx->epilogue_offset, from = ctx->ninsns; 137 138 return ninsns_rvoff(to - from); 139 } 140 141 /* Return -1 or inverted cond. */ 142 static inline int invert_bpf_cond(u8 cond) 143 { 144 switch (cond) { 145 case BPF_JEQ: 146 return BPF_JNE; 147 case BPF_JGT: 148 return BPF_JLE; 149 case BPF_JLT: 150 return BPF_JGE; 151 case BPF_JGE: 152 return BPF_JLT; 153 case BPF_JLE: 154 return BPF_JGT; 155 case BPF_JNE: 156 return BPF_JEQ; 157 case BPF_JSGT: 158 return BPF_JSLE; 159 case BPF_JSLT: 160 return BPF_JSGE; 161 case BPF_JSGE: 162 return BPF_JSLT; 163 case BPF_JSLE: 164 return BPF_JSGT; 165 } 166 return -1; 167 } 168 169 static inline bool is_6b_int(long val) 170 { 171 return -(1L << 5) <= val && val < (1L << 5); 172 } 173 174 static inline bool is_7b_uint(unsigned long val) 175 { 176 return val < (1UL << 7); 177 } 178 179 static inline bool is_8b_uint(unsigned long val) 180 { 181 return val < (1UL << 8); 182 } 183 184 static inline bool is_9b_uint(unsigned long val) 185 { 186 return val < (1UL << 9); 187 } 188 189 static inline bool is_10b_int(long val) 190 { 191 return -(1L << 9) <= val && val < (1L << 9); 192 } 193 194 static inline bool is_10b_uint(unsigned long val) 195 { 196 return val < (1UL << 10); 197 } 198 199 static inline bool is_12b_int(long val) 200 { 201 return -(1L << 11) <= val && val < (1L << 11); 202 } 203 204 static inline int is_12b_check(int off, int insn) 205 { 206 if (!is_12b_int(off)) { 207 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 208 insn, (int)off); 209 return -1; 210 } 211 return 0; 212 } 213 214 static inline bool is_13b_int(long val) 215 { 216 return -(1L << 12) <= val && val < (1L << 12); 217 } 218 219 static inline bool is_21b_int(long val) 220 { 221 return -(1L << 20) <= val && val < (1L << 20); 222 } 223 224 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 225 { 226 int from, to; 227 228 off++; /* BPF branch is from PC+1, RV is from PC */ 229 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; 230 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; 231 return ninsns_rvoff(to - from); 232 } 233 234 /* Instruction formats. */ 235 236 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 237 u8 opcode) 238 { 239 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 240 (rd << 7) | opcode; 241 } 242 243 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 244 { 245 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 246 opcode; 247 } 248 249 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 250 { 251 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 252 253 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 254 (imm4_0 << 7) | opcode; 255 } 256 257 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 258 { 259 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 260 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 261 262 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 263 (imm4_1 << 7) | opcode; 264 } 265 266 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 267 { 268 return (imm31_12 << 12) | (rd << 7) | opcode; 269 } 270 271 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 272 { 273 u32 imm; 274 275 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 276 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 277 278 return (imm << 12) | (rd << 7) | opcode; 279 } 280 281 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 282 u8 funct3, u8 rd, u8 opcode) 283 { 284 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 285 286 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 287 } 288 289 /* RISC-V compressed instruction formats. */ 290 291 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 292 { 293 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 294 } 295 296 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 297 { 298 u32 imm; 299 300 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 301 return (funct3 << 13) | (rd << 7) | op | imm; 302 } 303 304 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 305 { 306 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 307 } 308 309 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 310 { 311 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 312 } 313 314 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 315 u8 op) 316 { 317 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 318 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 319 } 320 321 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 322 u8 op) 323 { 324 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 325 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 326 } 327 328 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 329 { 330 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 331 ((rs2 & 0x7) << 2) | op; 332 } 333 334 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 335 { 336 u32 imm; 337 338 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 339 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 340 } 341 342 /* Instructions shared by both RV32 and RV64. */ 343 344 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 345 { 346 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 347 } 348 349 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 350 { 351 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 352 } 353 354 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 355 { 356 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 357 } 358 359 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 360 { 361 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 362 } 363 364 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 365 { 366 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 367 } 368 369 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 370 { 371 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 372 } 373 374 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 375 { 376 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 377 } 378 379 static inline u32 rv_lui(u8 rd, u32 imm31_12) 380 { 381 return rv_u_insn(imm31_12, rd, 0x37); 382 } 383 384 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 385 { 386 return rv_u_insn(imm31_12, rd, 0x17); 387 } 388 389 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 390 { 391 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 392 } 393 394 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 395 { 396 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 397 } 398 399 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 400 { 401 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 402 } 403 404 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 405 { 406 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 407 } 408 409 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 410 { 411 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 412 } 413 414 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 415 { 416 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 417 } 418 419 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 420 { 421 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 422 } 423 424 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 425 { 426 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 427 } 428 429 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 430 { 431 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 432 } 433 434 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 435 { 436 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 437 } 438 439 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 440 { 441 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 442 } 443 444 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) 445 { 446 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33); 447 } 448 449 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 450 { 451 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 452 } 453 454 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) 455 { 456 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33); 457 } 458 459 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 460 { 461 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 462 } 463 464 static inline u32 rv_jal(u8 rd, u32 imm20_1) 465 { 466 return rv_j_insn(imm20_1, rd, 0x6f); 467 } 468 469 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 470 { 471 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 472 } 473 474 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 475 { 476 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 477 } 478 479 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 480 { 481 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 482 } 483 484 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 485 { 486 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 487 } 488 489 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 490 { 491 return rv_bltu(rs2, rs1, imm12_1); 492 } 493 494 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 495 { 496 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 497 } 498 499 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 500 { 501 return rv_bgeu(rs2, rs1, imm12_1); 502 } 503 504 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 505 { 506 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 507 } 508 509 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 510 { 511 return rv_blt(rs2, rs1, imm12_1); 512 } 513 514 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 515 { 516 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 517 } 518 519 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 520 { 521 return rv_bge(rs2, rs1, imm12_1); 522 } 523 524 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) 525 { 526 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03); 527 } 528 529 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) 530 { 531 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03); 532 } 533 534 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 535 { 536 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 537 } 538 539 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 540 { 541 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 542 } 543 544 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 545 { 546 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 547 } 548 549 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 550 { 551 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 552 } 553 554 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 555 { 556 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 557 } 558 559 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 560 { 561 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 562 } 563 564 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 565 { 566 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 567 } 568 569 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 570 { 571 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 572 } 573 574 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 575 { 576 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 577 } 578 579 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 580 { 581 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 582 } 583 584 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 585 { 586 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 587 } 588 589 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 590 { 591 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 592 } 593 594 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 595 { 596 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 597 } 598 599 static inline u32 rv_fence(u8 pred, u8 succ) 600 { 601 u16 imm11_0 = pred << 4 | succ; 602 603 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 604 } 605 606 static inline void emit_fence_r_rw(struct rv_jit_context *ctx) 607 { 608 emit(rv_fence(0x2, 0x3), ctx); 609 } 610 611 static inline void emit_fence_rw_w(struct rv_jit_context *ctx) 612 { 613 emit(rv_fence(0x3, 0x1), ctx); 614 } 615 616 static inline void emit_fence_rw_rw(struct rv_jit_context *ctx) 617 { 618 emit(rv_fence(0x3, 0x3), ctx); 619 } 620 621 static inline u32 rv_nop(void) 622 { 623 return rv_i_insn(0, 0, 0, 0, 0x13); 624 } 625 626 /* RVC instructions. */ 627 628 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 629 { 630 u32 imm; 631 632 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 633 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 634 return rv_ciw_insn(0x0, imm, rd, 0x0); 635 } 636 637 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 638 { 639 u32 imm_hi, imm_lo; 640 641 imm_hi = (imm7 & 0x38) >> 3; 642 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 643 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 644 } 645 646 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 647 { 648 u32 imm_hi, imm_lo; 649 650 imm_hi = (imm7 & 0x38) >> 3; 651 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 652 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 653 } 654 655 static inline u16 rvc_addi(u8 rd, u32 imm6) 656 { 657 return rv_ci_insn(0, imm6, rd, 0x1); 658 } 659 660 static inline u16 rvc_li(u8 rd, u32 imm6) 661 { 662 return rv_ci_insn(0x2, imm6, rd, 0x1); 663 } 664 665 static inline u16 rvc_addi16sp(u32 imm10) 666 { 667 u32 imm; 668 669 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 670 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 671 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 672 } 673 674 static inline u16 rvc_lui(u8 rd, u32 imm6) 675 { 676 return rv_ci_insn(0x3, imm6, rd, 0x1); 677 } 678 679 static inline u16 rvc_srli(u8 rd, u32 imm6) 680 { 681 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 682 } 683 684 static inline u16 rvc_srai(u8 rd, u32 imm6) 685 { 686 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 687 } 688 689 static inline u16 rvc_andi(u8 rd, u32 imm6) 690 { 691 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 692 } 693 694 static inline u16 rvc_sub(u8 rd, u8 rs) 695 { 696 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 697 } 698 699 static inline u16 rvc_xor(u8 rd, u8 rs) 700 { 701 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 702 } 703 704 static inline u16 rvc_or(u8 rd, u8 rs) 705 { 706 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 707 } 708 709 static inline u16 rvc_and(u8 rd, u8 rs) 710 { 711 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 712 } 713 714 static inline u16 rvc_slli(u8 rd, u32 imm6) 715 { 716 return rv_ci_insn(0, imm6, rd, 0x2); 717 } 718 719 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 720 { 721 u32 imm; 722 723 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 724 return rv_ci_insn(0x2, imm, rd, 0x2); 725 } 726 727 static inline u16 rvc_jr(u8 rs1) 728 { 729 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 730 } 731 732 static inline u16 rvc_mv(u8 rd, u8 rs) 733 { 734 return rv_cr_insn(0x8, rd, rs, 0x2); 735 } 736 737 static inline u16 rvc_jalr(u8 rs1) 738 { 739 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 740 } 741 742 static inline u16 rvc_add(u8 rd, u8 rs) 743 { 744 return rv_cr_insn(0x9, rd, rs, 0x2); 745 } 746 747 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 748 { 749 u32 imm; 750 751 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 752 return rv_css_insn(0x6, imm, rs2, 0x2); 753 } 754 755 /* RVZACAS instructions. */ 756 static inline u32 rvzacas_amocas_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 757 { 758 return rv_amo_insn(0x5, aq, rl, rs2, rs1, 2, rd, 0x2f); 759 } 760 761 static inline u32 rvzacas_amocas_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 762 { 763 return rv_amo_insn(0x5, aq, rl, rs2, rs1, 3, rd, 0x2f); 764 } 765 766 /* RVZBA instructions. */ 767 static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2) 768 { 769 return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33); 770 } 771 772 static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2) 773 { 774 return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33); 775 } 776 777 /* RVZBB instructions. */ 778 static inline u32 rvzbb_sextb(u8 rd, u8 rs1) 779 { 780 return rv_i_insn(0x604, rs1, 1, rd, 0x13); 781 } 782 783 static inline u32 rvzbb_sexth(u8 rd, u8 rs1) 784 { 785 return rv_i_insn(0x605, rs1, 1, rd, 0x13); 786 } 787 788 static inline u32 rvzbb_zexth(u8 rd, u8 rs) 789 { 790 if (IS_ENABLED(CONFIG_64BIT)) 791 return rv_i_insn(0x80, rs, 4, rd, 0x3b); 792 793 return rv_i_insn(0x80, rs, 4, rd, 0x33); 794 } 795 796 static inline u32 rvzbb_rev8(u8 rd, u8 rs) 797 { 798 if (IS_ENABLED(CONFIG_64BIT)) 799 return rv_i_insn(0x6b8, rs, 5, rd, 0x13); 800 801 return rv_i_insn(0x698, rs, 5, rd, 0x13); 802 } 803 804 /* 805 * RV64-only instructions. 806 * 807 * These instructions are not available on RV32. Wrap them below a #if to 808 * ensure that the RV32 JIT doesn't emit any of these instructions. 809 */ 810 811 #if __riscv_xlen == 64 812 813 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 814 { 815 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 816 } 817 818 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 819 { 820 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 821 } 822 823 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 824 { 825 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 826 } 827 828 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 829 { 830 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 831 } 832 833 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 834 { 835 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 836 } 837 838 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 839 { 840 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 841 } 842 843 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 844 { 845 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 846 } 847 848 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 849 { 850 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 851 } 852 853 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 854 { 855 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 856 } 857 858 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 859 { 860 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 861 } 862 863 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) 864 { 865 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); 866 } 867 868 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 869 { 870 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 871 } 872 873 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) 874 { 875 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); 876 } 877 878 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 879 { 880 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 881 } 882 883 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 884 { 885 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 886 } 887 888 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 889 { 890 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 891 } 892 893 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 894 { 895 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 896 } 897 898 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 899 { 900 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 901 } 902 903 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 904 { 905 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 906 } 907 908 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 909 { 910 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 911 } 912 913 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 914 { 915 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 916 } 917 918 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 919 { 920 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 921 } 922 923 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 924 { 925 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 926 } 927 928 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 929 { 930 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 931 } 932 933 /* RV64-only RVC instructions. */ 934 935 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 936 { 937 u32 imm_hi, imm_lo; 938 939 imm_hi = (imm8 & 0x38) >> 3; 940 imm_lo = (imm8 & 0xc0) >> 6; 941 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 942 } 943 944 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 945 { 946 u32 imm_hi, imm_lo; 947 948 imm_hi = (imm8 & 0x38) >> 3; 949 imm_lo = (imm8 & 0xc0) >> 6; 950 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 951 } 952 953 static inline u16 rvc_subw(u8 rd, u8 rs) 954 { 955 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 956 } 957 958 static inline u16 rvc_addiw(u8 rd, u32 imm6) 959 { 960 return rv_ci_insn(0x1, imm6, rd, 0x1); 961 } 962 963 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 964 { 965 u32 imm; 966 967 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 968 return rv_ci_insn(0x3, imm, rd, 0x2); 969 } 970 971 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 972 { 973 u32 imm; 974 975 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 976 return rv_css_insn(0x7, imm, rs2, 0x2); 977 } 978 979 /* RV64-only ZBA instructions. */ 980 981 static inline u32 rvzba_zextw(u8 rd, u8 rs1) 982 { 983 /* add.uw rd, rs1, ZERO */ 984 return rv_r_insn(0x04, RV_REG_ZERO, rs1, 0, rd, 0x3b); 985 } 986 987 #endif /* __riscv_xlen == 64 */ 988 989 /* Helper functions that emit RVC instructions when possible. */ 990 991 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 992 { 993 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 994 emitc(rvc_jalr(rs), ctx); 995 else if (rvc_enabled() && !rd && rs && !imm) 996 emitc(rvc_jr(rs), ctx); 997 else 998 emit(rv_jalr(rd, rs, imm), ctx); 999 } 1000 1001 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 1002 { 1003 if (rvc_enabled() && rd && rs) 1004 emitc(rvc_mv(rd, rs), ctx); 1005 else 1006 emit(rv_addi(rd, rs, 0), ctx); 1007 } 1008 1009 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1010 { 1011 if (rvc_enabled() && rd && rd == rs1 && rs2) 1012 emitc(rvc_add(rd, rs2), ctx); 1013 else 1014 emit(rv_add(rd, rs1, rs2), ctx); 1015 } 1016 1017 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1018 { 1019 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 1020 emitc(rvc_addi16sp(imm), ctx); 1021 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 1022 !(imm & 0x3) && imm) 1023 emitc(rvc_addi4spn(rd, imm), ctx); 1024 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 1025 emitc(rvc_addi(rd, imm), ctx); 1026 else 1027 emit(rv_addi(rd, rs, imm), ctx); 1028 } 1029 1030 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 1031 { 1032 if (rvc_enabled() && rd && is_6b_int(imm)) 1033 emitc(rvc_li(rd, imm), ctx); 1034 else 1035 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 1036 } 1037 1038 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 1039 { 1040 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 1041 emitc(rvc_lui(rd, imm), ctx); 1042 else 1043 emit(rv_lui(rd, imm), ctx); 1044 } 1045 1046 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1047 { 1048 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 1049 emitc(rvc_slli(rd, imm), ctx); 1050 else 1051 emit(rv_slli(rd, rs, imm), ctx); 1052 } 1053 1054 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1055 { 1056 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 1057 emitc(rvc_andi(rd, imm), ctx); 1058 else 1059 emit(rv_andi(rd, rs, imm), ctx); 1060 } 1061 1062 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1063 { 1064 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1065 emitc(rvc_srli(rd, imm), ctx); 1066 else 1067 emit(rv_srli(rd, rs, imm), ctx); 1068 } 1069 1070 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1071 { 1072 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1073 emitc(rvc_srai(rd, imm), ctx); 1074 else 1075 emit(rv_srai(rd, rs, imm), ctx); 1076 } 1077 1078 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1079 { 1080 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1081 emitc(rvc_sub(rd, rs2), ctx); 1082 else 1083 emit(rv_sub(rd, rs1, rs2), ctx); 1084 } 1085 1086 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1087 { 1088 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1089 emitc(rvc_or(rd, rs2), ctx); 1090 else 1091 emit(rv_or(rd, rs1, rs2), ctx); 1092 } 1093 1094 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1095 { 1096 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1097 emitc(rvc_and(rd, rs2), ctx); 1098 else 1099 emit(rv_and(rd, rs1, rs2), ctx); 1100 } 1101 1102 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1103 { 1104 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1105 emitc(rvc_xor(rd, rs2), ctx); 1106 else 1107 emit(rv_xor(rd, rs1, rs2), ctx); 1108 } 1109 1110 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1111 { 1112 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 1113 emitc(rvc_lwsp(rd, off), ctx); 1114 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 1115 emitc(rvc_lw(rd, off, rs1), ctx); 1116 else 1117 emit(rv_lw(rd, off, rs1), ctx); 1118 } 1119 1120 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1121 { 1122 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1123 emitc(rvc_swsp(off, rs2), ctx); 1124 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1125 emitc(rvc_sw(rs1, off, rs2), ctx); 1126 else 1127 emit(rv_sw(rs1, off, rs2), ctx); 1128 } 1129 1130 static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1131 { 1132 if (rv_ext_enabled(ZBA)) { 1133 emit(rvzba_sh2add(rd, rs1, rs2), ctx); 1134 return; 1135 } 1136 1137 emit_slli(rd, rs1, 2, ctx); 1138 emit_add(rd, rd, rs2, ctx); 1139 } 1140 1141 static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1142 { 1143 if (rv_ext_enabled(ZBA)) { 1144 emit(rvzba_sh3add(rd, rs1, rs2), ctx); 1145 return; 1146 } 1147 1148 emit_slli(rd, rs1, 3, ctx); 1149 emit_add(rd, rd, rs2, ctx); 1150 } 1151 1152 /* RV64-only helper functions. */ 1153 #if __riscv_xlen == 64 1154 1155 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1156 { 1157 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1158 emitc(rvc_addiw(rd, imm), ctx); 1159 else 1160 emit(rv_addiw(rd, rs, imm), ctx); 1161 } 1162 1163 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1164 { 1165 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1166 emitc(rvc_ldsp(rd, off), ctx); 1167 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1168 emitc(rvc_ld(rd, off, rs1), ctx); 1169 else 1170 emit(rv_ld(rd, off, rs1), ctx); 1171 } 1172 1173 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1174 { 1175 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1176 emitc(rvc_sdsp(off, rs2), ctx); 1177 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1178 emitc(rvc_sd(rs1, off, rs2), ctx); 1179 else 1180 emit(rv_sd(rs1, off, rs2), ctx); 1181 } 1182 1183 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1184 { 1185 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1186 emitc(rvc_subw(rd, rs2), ctx); 1187 else 1188 emit(rv_subw(rd, rs1, rs2), ctx); 1189 } 1190 1191 static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) 1192 { 1193 if (rv_ext_enabled(ZBB)) { 1194 emit(rvzbb_sextb(rd, rs), ctx); 1195 return; 1196 } 1197 1198 emit_slli(rd, rs, 56, ctx); 1199 emit_srai(rd, rd, 56, ctx); 1200 } 1201 1202 static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1203 { 1204 if (rv_ext_enabled(ZBB)) { 1205 emit(rvzbb_sexth(rd, rs), ctx); 1206 return; 1207 } 1208 1209 emit_slli(rd, rs, 48, ctx); 1210 emit_srai(rd, rd, 48, ctx); 1211 } 1212 1213 static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1214 { 1215 emit_addiw(rd, rs, 0, ctx); 1216 } 1217 1218 static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1219 { 1220 if (rv_ext_enabled(ZBB)) { 1221 emit(rvzbb_zexth(rd, rs), ctx); 1222 return; 1223 } 1224 1225 emit_slli(rd, rs, 48, ctx); 1226 emit_srli(rd, rd, 48, ctx); 1227 } 1228 1229 static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1230 { 1231 if (rv_ext_enabled(ZBA)) { 1232 emit(rvzba_zextw(rd, rs), ctx); 1233 return; 1234 } 1235 1236 emit_slli(rd, rs, 32, ctx); 1237 emit_srli(rd, rd, 32, ctx); 1238 } 1239 1240 static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) 1241 { 1242 if (rv_ext_enabled(ZBB)) { 1243 int bits = 64 - imm; 1244 1245 emit(rvzbb_rev8(rd, rd), ctx); 1246 if (bits) 1247 emit_srli(rd, rd, bits, ctx); 1248 return; 1249 } 1250 1251 emit_li(RV_REG_T2, 0, ctx); 1252 1253 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1254 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1255 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1256 emit_srli(rd, rd, 8, ctx); 1257 if (imm == 16) 1258 goto out_be; 1259 1260 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1261 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1262 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1263 emit_srli(rd, rd, 8, ctx); 1264 1265 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1266 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1267 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1268 emit_srli(rd, rd, 8, ctx); 1269 if (imm == 32) 1270 goto out_be; 1271 1272 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1273 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1274 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1275 emit_srli(rd, rd, 8, ctx); 1276 1277 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1278 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1279 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1280 emit_srli(rd, rd, 8, ctx); 1281 1282 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1283 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1284 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1285 emit_srli(rd, rd, 8, ctx); 1286 1287 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1288 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1289 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1290 emit_srli(rd, rd, 8, ctx); 1291 out_be: 1292 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1293 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1294 1295 emit_mv(rd, RV_REG_T2, ctx); 1296 } 1297 1298 static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx) 1299 { 1300 int jmp_offset; 1301 1302 if (rv_ext_enabled(ZACAS)) { 1303 ctx->ex_insn_off = ctx->ninsns; 1304 emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) : 1305 rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx); 1306 ctx->ex_jmp_off = ctx->ninsns; 1307 if (!is64) 1308 emit_zextw(r0, r0, ctx); 1309 return; 1310 } 1311 1312 if (is64) 1313 emit_mv(RV_REG_T2, r0, ctx); 1314 else 1315 emit_addiw(RV_REG_T2, r0, 0, ctx); 1316 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 1317 rv_lr_w(r0, 0, rd, 0, 0), ctx); 1318 jmp_offset = ninsns_rvoff(8); 1319 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 1320 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) : 1321 rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); 1322 jmp_offset = ninsns_rvoff(-6); 1323 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 1324 emit_fence_rw_rw(ctx); 1325 } 1326 1327 #endif /* __riscv_xlen == 64 */ 1328 1329 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); 1330 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1331 1332 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1333 bool extra_pass); 1334 1335 #endif /* _BPF_JIT_H */ 1336