1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 15 /* verify runtime detection extension status */ 16 #define rv_ext_enabled(ext) \ 17 (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) 18 19 static inline bool rvc_enabled(void) 20 { 21 return IS_ENABLED(CONFIG_RISCV_ISA_C); 22 } 23 24 enum { 25 RV_REG_ZERO = 0, /* The constant value 0 */ 26 RV_REG_RA = 1, /* Return address */ 27 RV_REG_SP = 2, /* Stack pointer */ 28 RV_REG_GP = 3, /* Global pointer */ 29 RV_REG_TP = 4, /* Thread pointer */ 30 RV_REG_T0 = 5, /* Temporaries */ 31 RV_REG_T1 = 6, 32 RV_REG_T2 = 7, 33 RV_REG_FP = 8, /* Saved register/frame pointer */ 34 RV_REG_S1 = 9, /* Saved register */ 35 RV_REG_A0 = 10, /* Function argument/return values */ 36 RV_REG_A1 = 11, /* Function arguments */ 37 RV_REG_A2 = 12, 38 RV_REG_A3 = 13, 39 RV_REG_A4 = 14, 40 RV_REG_A5 = 15, 41 RV_REG_A6 = 16, 42 RV_REG_A7 = 17, 43 RV_REG_S2 = 18, /* Saved registers */ 44 RV_REG_S3 = 19, 45 RV_REG_S4 = 20, 46 RV_REG_S5 = 21, 47 RV_REG_S6 = 22, 48 RV_REG_S7 = 23, 49 RV_REG_S8 = 24, 50 RV_REG_S9 = 25, 51 RV_REG_S10 = 26, 52 RV_REG_S11 = 27, 53 RV_REG_T3 = 28, /* Temporaries */ 54 RV_REG_T4 = 29, 55 RV_REG_T5 = 30, 56 RV_REG_T6 = 31, 57 }; 58 59 static inline bool is_creg(u8 reg) 60 { 61 return (1 << reg) & (BIT(RV_REG_FP) | 62 BIT(RV_REG_S1) | 63 BIT(RV_REG_A0) | 64 BIT(RV_REG_A1) | 65 BIT(RV_REG_A2) | 66 BIT(RV_REG_A3) | 67 BIT(RV_REG_A4) | 68 BIT(RV_REG_A5)); 69 } 70 71 struct rv_jit_context { 72 struct bpf_prog *prog; 73 u16 *insns; /* RV insns */ 74 u16 *ro_insns; 75 int ninsns; 76 int prologue_len; 77 int epilogue_offset; 78 int *offset; /* BPF to RV */ 79 int nexentries; 80 int ex_insn_off; 81 int ex_jmp_off; 82 unsigned long flags; 83 int stack_size; 84 int tcc_offset; 85 u64 arena_vm_start; 86 u64 user_vm_start; 87 }; 88 89 /* Convert from ninsns to bytes. */ 90 static inline int ninsns_rvoff(int ninsns) 91 { 92 return ninsns << 1; 93 } 94 95 struct rv_jit_data { 96 struct bpf_binary_header *header; 97 struct bpf_binary_header *ro_header; 98 u8 *image; 99 u8 *ro_image; 100 struct rv_jit_context ctx; 101 }; 102 103 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 104 { 105 memset(area, 0, size); 106 } 107 108 /* Emit a 4-byte riscv instruction. */ 109 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 110 { 111 if (ctx->insns) { 112 ctx->insns[ctx->ninsns] = insn; 113 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 114 } 115 116 ctx->ninsns += 2; 117 } 118 119 /* Emit a 2-byte riscv compressed instruction. */ 120 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 121 { 122 BUILD_BUG_ON(!rvc_enabled()); 123 124 if (ctx->insns) 125 ctx->insns[ctx->ninsns] = insn; 126 127 ctx->ninsns++; 128 } 129 130 static inline int epilogue_offset(struct rv_jit_context *ctx) 131 { 132 int to = ctx->epilogue_offset, from = ctx->ninsns; 133 134 return ninsns_rvoff(to - from); 135 } 136 137 /* Return -1 or inverted cond. */ 138 static inline int invert_bpf_cond(u8 cond) 139 { 140 switch (cond) { 141 case BPF_JEQ: 142 return BPF_JNE; 143 case BPF_JGT: 144 return BPF_JLE; 145 case BPF_JLT: 146 return BPF_JGE; 147 case BPF_JGE: 148 return BPF_JLT; 149 case BPF_JLE: 150 return BPF_JGT; 151 case BPF_JNE: 152 return BPF_JEQ; 153 case BPF_JSGT: 154 return BPF_JSLE; 155 case BPF_JSLT: 156 return BPF_JSGE; 157 case BPF_JSGE: 158 return BPF_JSLT; 159 case BPF_JSLE: 160 return BPF_JSGT; 161 } 162 return -1; 163 } 164 165 static inline bool is_6b_int(long val) 166 { 167 return -(1L << 5) <= val && val < (1L << 5); 168 } 169 170 static inline bool is_7b_uint(unsigned long val) 171 { 172 return val < (1UL << 7); 173 } 174 175 static inline bool is_8b_uint(unsigned long val) 176 { 177 return val < (1UL << 8); 178 } 179 180 static inline bool is_9b_uint(unsigned long val) 181 { 182 return val < (1UL << 9); 183 } 184 185 static inline bool is_10b_int(long val) 186 { 187 return -(1L << 9) <= val && val < (1L << 9); 188 } 189 190 static inline bool is_10b_uint(unsigned long val) 191 { 192 return val < (1UL << 10); 193 } 194 195 static inline bool is_12b_int(long val) 196 { 197 return -(1L << 11) <= val && val < (1L << 11); 198 } 199 200 static inline int is_12b_check(int off, int insn) 201 { 202 if (!is_12b_int(off)) { 203 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 204 insn, (int)off); 205 return -1; 206 } 207 return 0; 208 } 209 210 static inline bool is_13b_int(long val) 211 { 212 return -(1L << 12) <= val && val < (1L << 12); 213 } 214 215 static inline bool is_21b_int(long val) 216 { 217 return -(1L << 20) <= val && val < (1L << 20); 218 } 219 220 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 221 { 222 int from, to; 223 224 off++; /* BPF branch is from PC+1, RV is from PC */ 225 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; 226 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; 227 return ninsns_rvoff(to - from); 228 } 229 230 /* Instruction formats. */ 231 232 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 233 u8 opcode) 234 { 235 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 236 (rd << 7) | opcode; 237 } 238 239 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 240 { 241 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 242 opcode; 243 } 244 245 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 246 { 247 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 248 249 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 250 (imm4_0 << 7) | opcode; 251 } 252 253 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 254 { 255 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 256 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 257 258 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 259 (imm4_1 << 7) | opcode; 260 } 261 262 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 263 { 264 return (imm31_12 << 12) | (rd << 7) | opcode; 265 } 266 267 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 268 { 269 u32 imm; 270 271 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 272 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 273 274 return (imm << 12) | (rd << 7) | opcode; 275 } 276 277 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 278 u8 funct3, u8 rd, u8 opcode) 279 { 280 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 281 282 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 283 } 284 285 /* RISC-V compressed instruction formats. */ 286 287 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 288 { 289 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 290 } 291 292 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 293 { 294 u32 imm; 295 296 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 297 return (funct3 << 13) | (rd << 7) | op | imm; 298 } 299 300 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 301 { 302 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 303 } 304 305 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 306 { 307 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 308 } 309 310 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 311 u8 op) 312 { 313 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 314 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 315 } 316 317 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 318 u8 op) 319 { 320 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 321 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 322 } 323 324 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 325 { 326 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 327 ((rs2 & 0x7) << 2) | op; 328 } 329 330 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 331 { 332 u32 imm; 333 334 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 335 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 336 } 337 338 /* Instructions shared by both RV32 and RV64. */ 339 340 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 341 { 342 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 343 } 344 345 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 346 { 347 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 348 } 349 350 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 351 { 352 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 353 } 354 355 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 356 { 357 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 358 } 359 360 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 361 { 362 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 363 } 364 365 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 366 { 367 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 368 } 369 370 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 371 { 372 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 373 } 374 375 static inline u32 rv_lui(u8 rd, u32 imm31_12) 376 { 377 return rv_u_insn(imm31_12, rd, 0x37); 378 } 379 380 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 381 { 382 return rv_u_insn(imm31_12, rd, 0x17); 383 } 384 385 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 386 { 387 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 388 } 389 390 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 391 { 392 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 393 } 394 395 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 396 { 397 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 398 } 399 400 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 401 { 402 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 403 } 404 405 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 406 { 407 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 408 } 409 410 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 411 { 412 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 413 } 414 415 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 416 { 417 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 418 } 419 420 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 421 { 422 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 423 } 424 425 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 426 { 427 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 428 } 429 430 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 431 { 432 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 433 } 434 435 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 436 { 437 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 438 } 439 440 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) 441 { 442 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33); 443 } 444 445 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 446 { 447 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 448 } 449 450 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) 451 { 452 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33); 453 } 454 455 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 456 { 457 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 458 } 459 460 static inline u32 rv_jal(u8 rd, u32 imm20_1) 461 { 462 return rv_j_insn(imm20_1, rd, 0x6f); 463 } 464 465 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 466 { 467 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 468 } 469 470 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 471 { 472 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 473 } 474 475 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 476 { 477 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 478 } 479 480 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 481 { 482 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 483 } 484 485 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 486 { 487 return rv_bltu(rs2, rs1, imm12_1); 488 } 489 490 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 491 { 492 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 493 } 494 495 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 496 { 497 return rv_bgeu(rs2, rs1, imm12_1); 498 } 499 500 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 501 { 502 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 503 } 504 505 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 506 { 507 return rv_blt(rs2, rs1, imm12_1); 508 } 509 510 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 511 { 512 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 513 } 514 515 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 516 { 517 return rv_bge(rs2, rs1, imm12_1); 518 } 519 520 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) 521 { 522 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03); 523 } 524 525 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) 526 { 527 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03); 528 } 529 530 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 531 { 532 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 533 } 534 535 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 536 { 537 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 538 } 539 540 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 541 { 542 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 543 } 544 545 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 546 { 547 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 548 } 549 550 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 551 { 552 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 553 } 554 555 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 556 { 557 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 558 } 559 560 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 561 { 562 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 563 } 564 565 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 566 { 567 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 568 } 569 570 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 571 { 572 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 573 } 574 575 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 576 { 577 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 578 } 579 580 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 581 { 582 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 583 } 584 585 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 586 { 587 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 588 } 589 590 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 591 { 592 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 593 } 594 595 static inline u32 rv_fence(u8 pred, u8 succ) 596 { 597 u16 imm11_0 = pred << 4 | succ; 598 599 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 600 } 601 602 static inline void emit_fence_r_rw(struct rv_jit_context *ctx) 603 { 604 emit(rv_fence(0x2, 0x3), ctx); 605 } 606 607 static inline void emit_fence_rw_w(struct rv_jit_context *ctx) 608 { 609 emit(rv_fence(0x3, 0x1), ctx); 610 } 611 612 static inline void emit_fence_rw_rw(struct rv_jit_context *ctx) 613 { 614 emit(rv_fence(0x3, 0x3), ctx); 615 } 616 617 static inline u32 rv_nop(void) 618 { 619 return rv_i_insn(0, 0, 0, 0, 0x13); 620 } 621 622 /* RVC instructions. */ 623 624 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 625 { 626 u32 imm; 627 628 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 629 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 630 return rv_ciw_insn(0x0, imm, rd, 0x0); 631 } 632 633 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 634 { 635 u32 imm_hi, imm_lo; 636 637 imm_hi = (imm7 & 0x38) >> 3; 638 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 639 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 640 } 641 642 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 643 { 644 u32 imm_hi, imm_lo; 645 646 imm_hi = (imm7 & 0x38) >> 3; 647 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 648 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 649 } 650 651 static inline u16 rvc_addi(u8 rd, u32 imm6) 652 { 653 return rv_ci_insn(0, imm6, rd, 0x1); 654 } 655 656 static inline u16 rvc_li(u8 rd, u32 imm6) 657 { 658 return rv_ci_insn(0x2, imm6, rd, 0x1); 659 } 660 661 static inline u16 rvc_addi16sp(u32 imm10) 662 { 663 u32 imm; 664 665 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 666 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 667 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 668 } 669 670 static inline u16 rvc_lui(u8 rd, u32 imm6) 671 { 672 return rv_ci_insn(0x3, imm6, rd, 0x1); 673 } 674 675 static inline u16 rvc_srli(u8 rd, u32 imm6) 676 { 677 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 678 } 679 680 static inline u16 rvc_srai(u8 rd, u32 imm6) 681 { 682 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 683 } 684 685 static inline u16 rvc_andi(u8 rd, u32 imm6) 686 { 687 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 688 } 689 690 static inline u16 rvc_sub(u8 rd, u8 rs) 691 { 692 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 693 } 694 695 static inline u16 rvc_xor(u8 rd, u8 rs) 696 { 697 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 698 } 699 700 static inline u16 rvc_or(u8 rd, u8 rs) 701 { 702 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 703 } 704 705 static inline u16 rvc_and(u8 rd, u8 rs) 706 { 707 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 708 } 709 710 static inline u16 rvc_slli(u8 rd, u32 imm6) 711 { 712 return rv_ci_insn(0, imm6, rd, 0x2); 713 } 714 715 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 716 { 717 u32 imm; 718 719 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 720 return rv_ci_insn(0x2, imm, rd, 0x2); 721 } 722 723 static inline u16 rvc_jr(u8 rs1) 724 { 725 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 726 } 727 728 static inline u16 rvc_mv(u8 rd, u8 rs) 729 { 730 return rv_cr_insn(0x8, rd, rs, 0x2); 731 } 732 733 static inline u16 rvc_jalr(u8 rs1) 734 { 735 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 736 } 737 738 static inline u16 rvc_add(u8 rd, u8 rs) 739 { 740 return rv_cr_insn(0x9, rd, rs, 0x2); 741 } 742 743 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 744 { 745 u32 imm; 746 747 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 748 return rv_css_insn(0x6, imm, rs2, 0x2); 749 } 750 751 /* RVZACAS instructions. */ 752 static inline u32 rvzacas_amocas_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 753 { 754 return rv_amo_insn(0x5, aq, rl, rs2, rs1, 2, rd, 0x2f); 755 } 756 757 static inline u32 rvzacas_amocas_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 758 { 759 return rv_amo_insn(0x5, aq, rl, rs2, rs1, 3, rd, 0x2f); 760 } 761 762 /* RVZBA instructions. */ 763 static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2) 764 { 765 return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33); 766 } 767 768 static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2) 769 { 770 return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33); 771 } 772 773 /* RVZBB instructions. */ 774 static inline u32 rvzbb_sextb(u8 rd, u8 rs1) 775 { 776 return rv_i_insn(0x604, rs1, 1, rd, 0x13); 777 } 778 779 static inline u32 rvzbb_sexth(u8 rd, u8 rs1) 780 { 781 return rv_i_insn(0x605, rs1, 1, rd, 0x13); 782 } 783 784 static inline u32 rvzbb_zexth(u8 rd, u8 rs) 785 { 786 if (IS_ENABLED(CONFIG_64BIT)) 787 return rv_i_insn(0x80, rs, 4, rd, 0x3b); 788 789 return rv_i_insn(0x80, rs, 4, rd, 0x33); 790 } 791 792 static inline u32 rvzbb_rev8(u8 rd, u8 rs) 793 { 794 if (IS_ENABLED(CONFIG_64BIT)) 795 return rv_i_insn(0x6b8, rs, 5, rd, 0x13); 796 797 return rv_i_insn(0x698, rs, 5, rd, 0x13); 798 } 799 800 /* 801 * RV64-only instructions. 802 * 803 * These instructions are not available on RV32. Wrap them below a #if to 804 * ensure that the RV32 JIT doesn't emit any of these instructions. 805 */ 806 807 #if __riscv_xlen == 64 808 809 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 810 { 811 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 812 } 813 814 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 815 { 816 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 817 } 818 819 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 820 { 821 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 822 } 823 824 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 825 { 826 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 827 } 828 829 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 830 { 831 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 832 } 833 834 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 835 { 836 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 837 } 838 839 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 840 { 841 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 842 } 843 844 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 845 { 846 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 847 } 848 849 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 850 { 851 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 852 } 853 854 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 855 { 856 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 857 } 858 859 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) 860 { 861 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); 862 } 863 864 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 865 { 866 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 867 } 868 869 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) 870 { 871 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); 872 } 873 874 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 875 { 876 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 877 } 878 879 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 880 { 881 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 882 } 883 884 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 885 { 886 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 887 } 888 889 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 890 { 891 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 892 } 893 894 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 895 { 896 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 897 } 898 899 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 900 { 901 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 902 } 903 904 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 905 { 906 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 907 } 908 909 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 910 { 911 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 912 } 913 914 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 915 { 916 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 917 } 918 919 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 920 { 921 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 922 } 923 924 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 925 { 926 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 927 } 928 929 /* RV64-only RVC instructions. */ 930 931 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 932 { 933 u32 imm_hi, imm_lo; 934 935 imm_hi = (imm8 & 0x38) >> 3; 936 imm_lo = (imm8 & 0xc0) >> 6; 937 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 938 } 939 940 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 941 { 942 u32 imm_hi, imm_lo; 943 944 imm_hi = (imm8 & 0x38) >> 3; 945 imm_lo = (imm8 & 0xc0) >> 6; 946 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 947 } 948 949 static inline u16 rvc_subw(u8 rd, u8 rs) 950 { 951 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 952 } 953 954 static inline u16 rvc_addiw(u8 rd, u32 imm6) 955 { 956 return rv_ci_insn(0x1, imm6, rd, 0x1); 957 } 958 959 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 960 { 961 u32 imm; 962 963 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 964 return rv_ci_insn(0x3, imm, rd, 0x2); 965 } 966 967 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 968 { 969 u32 imm; 970 971 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 972 return rv_css_insn(0x7, imm, rs2, 0x2); 973 } 974 975 /* RV64-only ZBA instructions. */ 976 977 static inline u32 rvzba_zextw(u8 rd, u8 rs1) 978 { 979 /* add.uw rd, rs1, ZERO */ 980 return rv_r_insn(0x04, RV_REG_ZERO, rs1, 0, rd, 0x3b); 981 } 982 983 #endif /* __riscv_xlen == 64 */ 984 985 /* Helper functions that emit RVC instructions when possible. */ 986 987 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 988 { 989 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 990 emitc(rvc_jalr(rs), ctx); 991 else if (rvc_enabled() && !rd && rs && !imm) 992 emitc(rvc_jr(rs), ctx); 993 else 994 emit(rv_jalr(rd, rs, imm), ctx); 995 } 996 997 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 998 { 999 if (rvc_enabled() && rd && rs) 1000 emitc(rvc_mv(rd, rs), ctx); 1001 else 1002 emit(rv_addi(rd, rs, 0), ctx); 1003 } 1004 1005 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1006 { 1007 if (rvc_enabled() && rd && rd == rs1 && rs2) 1008 emitc(rvc_add(rd, rs2), ctx); 1009 else 1010 emit(rv_add(rd, rs1, rs2), ctx); 1011 } 1012 1013 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1014 { 1015 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 1016 emitc(rvc_addi16sp(imm), ctx); 1017 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 1018 !(imm & 0x3) && imm) 1019 emitc(rvc_addi4spn(rd, imm), ctx); 1020 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 1021 emitc(rvc_addi(rd, imm), ctx); 1022 else 1023 emit(rv_addi(rd, rs, imm), ctx); 1024 } 1025 1026 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 1027 { 1028 if (rvc_enabled() && rd && is_6b_int(imm)) 1029 emitc(rvc_li(rd, imm), ctx); 1030 else 1031 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 1032 } 1033 1034 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 1035 { 1036 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 1037 emitc(rvc_lui(rd, imm), ctx); 1038 else 1039 emit(rv_lui(rd, imm), ctx); 1040 } 1041 1042 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1043 { 1044 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 1045 emitc(rvc_slli(rd, imm), ctx); 1046 else 1047 emit(rv_slli(rd, rs, imm), ctx); 1048 } 1049 1050 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1051 { 1052 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 1053 emitc(rvc_andi(rd, imm), ctx); 1054 else 1055 emit(rv_andi(rd, rs, imm), ctx); 1056 } 1057 1058 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1059 { 1060 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1061 emitc(rvc_srli(rd, imm), ctx); 1062 else 1063 emit(rv_srli(rd, rs, imm), ctx); 1064 } 1065 1066 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1067 { 1068 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 1069 emitc(rvc_srai(rd, imm), ctx); 1070 else 1071 emit(rv_srai(rd, rs, imm), ctx); 1072 } 1073 1074 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1075 { 1076 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1077 emitc(rvc_sub(rd, rs2), ctx); 1078 else 1079 emit(rv_sub(rd, rs1, rs2), ctx); 1080 } 1081 1082 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1083 { 1084 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1085 emitc(rvc_or(rd, rs2), ctx); 1086 else 1087 emit(rv_or(rd, rs1, rs2), ctx); 1088 } 1089 1090 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1091 { 1092 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1093 emitc(rvc_and(rd, rs2), ctx); 1094 else 1095 emit(rv_and(rd, rs1, rs2), ctx); 1096 } 1097 1098 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1099 { 1100 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1101 emitc(rvc_xor(rd, rs2), ctx); 1102 else 1103 emit(rv_xor(rd, rs1, rs2), ctx); 1104 } 1105 1106 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1107 { 1108 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 1109 emitc(rvc_lwsp(rd, off), ctx); 1110 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 1111 emitc(rvc_lw(rd, off, rs1), ctx); 1112 else 1113 emit(rv_lw(rd, off, rs1), ctx); 1114 } 1115 1116 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1117 { 1118 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1119 emitc(rvc_swsp(off, rs2), ctx); 1120 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1121 emitc(rvc_sw(rs1, off, rs2), ctx); 1122 else 1123 emit(rv_sw(rs1, off, rs2), ctx); 1124 } 1125 1126 static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1127 { 1128 if (rv_ext_enabled(ZBA)) { 1129 emit(rvzba_sh2add(rd, rs1, rs2), ctx); 1130 return; 1131 } 1132 1133 emit_slli(rd, rs1, 2, ctx); 1134 emit_add(rd, rd, rs2, ctx); 1135 } 1136 1137 static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1138 { 1139 if (rv_ext_enabled(ZBA)) { 1140 emit(rvzba_sh3add(rd, rs1, rs2), ctx); 1141 return; 1142 } 1143 1144 emit_slli(rd, rs1, 3, ctx); 1145 emit_add(rd, rd, rs2, ctx); 1146 } 1147 1148 /* RV64-only helper functions. */ 1149 #if __riscv_xlen == 64 1150 1151 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1152 { 1153 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1154 emitc(rvc_addiw(rd, imm), ctx); 1155 else 1156 emit(rv_addiw(rd, rs, imm), ctx); 1157 } 1158 1159 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1160 { 1161 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1162 emitc(rvc_ldsp(rd, off), ctx); 1163 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1164 emitc(rvc_ld(rd, off, rs1), ctx); 1165 else 1166 emit(rv_ld(rd, off, rs1), ctx); 1167 } 1168 1169 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1170 { 1171 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1172 emitc(rvc_sdsp(off, rs2), ctx); 1173 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1174 emitc(rvc_sd(rs1, off, rs2), ctx); 1175 else 1176 emit(rv_sd(rs1, off, rs2), ctx); 1177 } 1178 1179 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1180 { 1181 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1182 emitc(rvc_subw(rd, rs2), ctx); 1183 else 1184 emit(rv_subw(rd, rs1, rs2), ctx); 1185 } 1186 1187 static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) 1188 { 1189 if (rv_ext_enabled(ZBB)) { 1190 emit(rvzbb_sextb(rd, rs), ctx); 1191 return; 1192 } 1193 1194 emit_slli(rd, rs, 56, ctx); 1195 emit_srai(rd, rd, 56, ctx); 1196 } 1197 1198 static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1199 { 1200 if (rv_ext_enabled(ZBB)) { 1201 emit(rvzbb_sexth(rd, rs), ctx); 1202 return; 1203 } 1204 1205 emit_slli(rd, rs, 48, ctx); 1206 emit_srai(rd, rd, 48, ctx); 1207 } 1208 1209 static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1210 { 1211 emit_addiw(rd, rs, 0, ctx); 1212 } 1213 1214 static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) 1215 { 1216 if (rv_ext_enabled(ZBB)) { 1217 emit(rvzbb_zexth(rd, rs), ctx); 1218 return; 1219 } 1220 1221 emit_slli(rd, rs, 48, ctx); 1222 emit_srli(rd, rd, 48, ctx); 1223 } 1224 1225 static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) 1226 { 1227 if (rv_ext_enabled(ZBA)) { 1228 emit(rvzba_zextw(rd, rs), ctx); 1229 return; 1230 } 1231 1232 emit_slli(rd, rs, 32, ctx); 1233 emit_srli(rd, rd, 32, ctx); 1234 } 1235 1236 static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) 1237 { 1238 if (rv_ext_enabled(ZBB)) { 1239 int bits = 64 - imm; 1240 1241 emit(rvzbb_rev8(rd, rd), ctx); 1242 if (bits) 1243 emit_srli(rd, rd, bits, ctx); 1244 return; 1245 } 1246 1247 emit_li(RV_REG_T2, 0, ctx); 1248 1249 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1250 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1251 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1252 emit_srli(rd, rd, 8, ctx); 1253 if (imm == 16) 1254 goto out_be; 1255 1256 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1257 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1258 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1259 emit_srli(rd, rd, 8, ctx); 1260 1261 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1262 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1263 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1264 emit_srli(rd, rd, 8, ctx); 1265 if (imm == 32) 1266 goto out_be; 1267 1268 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1269 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1270 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1271 emit_srli(rd, rd, 8, ctx); 1272 1273 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1274 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1275 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1276 emit_srli(rd, rd, 8, ctx); 1277 1278 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1279 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1280 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1281 emit_srli(rd, rd, 8, ctx); 1282 1283 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1284 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1285 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 1286 emit_srli(rd, rd, 8, ctx); 1287 out_be: 1288 emit_andi(RV_REG_T1, rd, 0xff, ctx); 1289 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1290 1291 emit_mv(rd, RV_REG_T2, ctx); 1292 } 1293 1294 static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx) 1295 { 1296 int jmp_offset; 1297 1298 if (rv_ext_enabled(ZACAS)) { 1299 ctx->ex_insn_off = ctx->ninsns; 1300 emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) : 1301 rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx); 1302 ctx->ex_jmp_off = ctx->ninsns; 1303 if (!is64) 1304 emit_zextw(r0, r0, ctx); 1305 return; 1306 } 1307 1308 if (is64) 1309 emit_mv(RV_REG_T2, r0, ctx); 1310 else 1311 emit_addiw(RV_REG_T2, r0, 0, ctx); 1312 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 1313 rv_lr_w(r0, 0, rd, 0, 0), ctx); 1314 jmp_offset = ninsns_rvoff(8); 1315 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 1316 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) : 1317 rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx); 1318 jmp_offset = ninsns_rvoff(-6); 1319 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 1320 emit_fence_rw_rw(ctx); 1321 } 1322 1323 #endif /* __riscv_xlen == 64 */ 1324 1325 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); 1326 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1327 1328 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1329 bool extra_pass); 1330 1331 #endif /* _BPF_JIT_H */ 1332