1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <asm/patch.h> 14 #include "bpf_jit.h" 15 16 #define RV_FENTRY_NINSNS 2 17 18 #define RV_REG_TCC RV_REG_A6 19 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 20 21 static const int regmap[] = { 22 [BPF_REG_0] = RV_REG_A5, 23 [BPF_REG_1] = RV_REG_A0, 24 [BPF_REG_2] = RV_REG_A1, 25 [BPF_REG_3] = RV_REG_A2, 26 [BPF_REG_4] = RV_REG_A3, 27 [BPF_REG_5] = RV_REG_A4, 28 [BPF_REG_6] = RV_REG_S1, 29 [BPF_REG_7] = RV_REG_S2, 30 [BPF_REG_8] = RV_REG_S3, 31 [BPF_REG_9] = RV_REG_S4, 32 [BPF_REG_FP] = RV_REG_S5, 33 [BPF_REG_AX] = RV_REG_T0, 34 }; 35 36 static const int pt_regmap[] = { 37 [RV_REG_A0] = offsetof(struct pt_regs, a0), 38 [RV_REG_A1] = offsetof(struct pt_regs, a1), 39 [RV_REG_A2] = offsetof(struct pt_regs, a2), 40 [RV_REG_A3] = offsetof(struct pt_regs, a3), 41 [RV_REG_A4] = offsetof(struct pt_regs, a4), 42 [RV_REG_A5] = offsetof(struct pt_regs, a5), 43 [RV_REG_S1] = offsetof(struct pt_regs, s1), 44 [RV_REG_S2] = offsetof(struct pt_regs, s2), 45 [RV_REG_S3] = offsetof(struct pt_regs, s3), 46 [RV_REG_S4] = offsetof(struct pt_regs, s4), 47 [RV_REG_S5] = offsetof(struct pt_regs, s5), 48 [RV_REG_T0] = offsetof(struct pt_regs, t0), 49 }; 50 51 enum { 52 RV_CTX_F_SEEN_TAIL_CALL = 0, 53 RV_CTX_F_SEEN_CALL = RV_REG_RA, 54 RV_CTX_F_SEEN_S1 = RV_REG_S1, 55 RV_CTX_F_SEEN_S2 = RV_REG_S2, 56 RV_CTX_F_SEEN_S3 = RV_REG_S3, 57 RV_CTX_F_SEEN_S4 = RV_REG_S4, 58 RV_CTX_F_SEEN_S5 = RV_REG_S5, 59 RV_CTX_F_SEEN_S6 = RV_REG_S6, 60 }; 61 62 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 63 { 64 u8 reg = regmap[bpf_reg]; 65 66 switch (reg) { 67 case RV_CTX_F_SEEN_S1: 68 case RV_CTX_F_SEEN_S2: 69 case RV_CTX_F_SEEN_S3: 70 case RV_CTX_F_SEEN_S4: 71 case RV_CTX_F_SEEN_S5: 72 case RV_CTX_F_SEEN_S6: 73 __set_bit(reg, &ctx->flags); 74 } 75 return reg; 76 }; 77 78 static bool seen_reg(int reg, struct rv_jit_context *ctx) 79 { 80 switch (reg) { 81 case RV_CTX_F_SEEN_CALL: 82 case RV_CTX_F_SEEN_S1: 83 case RV_CTX_F_SEEN_S2: 84 case RV_CTX_F_SEEN_S3: 85 case RV_CTX_F_SEEN_S4: 86 case RV_CTX_F_SEEN_S5: 87 case RV_CTX_F_SEEN_S6: 88 return test_bit(reg, &ctx->flags); 89 } 90 return false; 91 } 92 93 static void mark_fp(struct rv_jit_context *ctx) 94 { 95 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 96 } 97 98 static void mark_call(struct rv_jit_context *ctx) 99 { 100 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 101 } 102 103 static bool seen_call(struct rv_jit_context *ctx) 104 { 105 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 106 } 107 108 static void mark_tail_call(struct rv_jit_context *ctx) 109 { 110 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 111 } 112 113 static bool seen_tail_call(struct rv_jit_context *ctx) 114 { 115 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 116 } 117 118 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 119 { 120 mark_tail_call(ctx); 121 122 if (seen_call(ctx)) { 123 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 124 return RV_REG_S6; 125 } 126 return RV_REG_A6; 127 } 128 129 static bool is_32b_int(s64 val) 130 { 131 return -(1L << 31) <= val && val < (1L << 31); 132 } 133 134 static bool in_auipc_jalr_range(s64 val) 135 { 136 /* 137 * auipc+jalr can reach any signed PC-relative offset in the range 138 * [-2^31 - 2^11, 2^31 - 2^11). 139 */ 140 return (-(1L << 31) - (1L << 11)) <= val && 141 val < ((1L << 31) - (1L << 11)); 142 } 143 144 /* Modify rd pointer to alternate reg to avoid corrupting original reg */ 145 static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 146 { 147 emit_sextw(ra, *rd, ctx); 148 *rd = ra; 149 } 150 151 static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 152 { 153 emit_zextw(ra, *rd, ctx); 154 *rd = ra; 155 } 156 157 /* Emit fixed-length instructions for address */ 158 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 159 { 160 /* 161 * Use the ro_insns(RX) to calculate the offset as the BPF program will 162 * finally run from this memory region. 163 */ 164 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); 165 s64 off = addr - ip; 166 s64 upper = (off + (1 << 11)) >> 12; 167 s64 lower = off & 0xfff; 168 169 if (extra_pass && !in_auipc_jalr_range(off)) { 170 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 171 return -ERANGE; 172 } 173 174 emit(rv_auipc(rd, upper), ctx); 175 emit(rv_addi(rd, rd, lower), ctx); 176 return 0; 177 } 178 179 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 180 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 181 { 182 /* Note that the immediate from the add is sign-extended, 183 * which means that we need to compensate this by adding 2^12, 184 * when the 12th bit is set. A simpler way of doing this, and 185 * getting rid of the check, is to just add 2**11 before the 186 * shift. The "Loading a 32-Bit constant" example from the 187 * "Computer Organization and Design, RISC-V edition" book by 188 * Patterson/Hennessy highlights this fact. 189 * 190 * This also means that we need to process LSB to MSB. 191 */ 192 s64 upper = (val + (1 << 11)) >> 12; 193 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 194 * and addi are signed and RVC checks will perform signed comparisons. 195 */ 196 s64 lower = ((val & 0xfff) << 52) >> 52; 197 int shift; 198 199 if (is_32b_int(val)) { 200 if (upper) 201 emit_lui(rd, upper, ctx); 202 203 if (!upper) { 204 emit_li(rd, lower, ctx); 205 return; 206 } 207 208 emit_addiw(rd, rd, lower, ctx); 209 return; 210 } 211 212 shift = __ffs(upper); 213 upper >>= shift; 214 shift += 12; 215 216 emit_imm(rd, upper, ctx); 217 218 emit_slli(rd, rd, shift, ctx); 219 if (lower) 220 emit_addi(rd, rd, lower, ctx); 221 } 222 223 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 224 { 225 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 226 227 if (seen_reg(RV_REG_RA, ctx)) { 228 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 229 store_offset -= 8; 230 } 231 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 232 store_offset -= 8; 233 if (seen_reg(RV_REG_S1, ctx)) { 234 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 235 store_offset -= 8; 236 } 237 if (seen_reg(RV_REG_S2, ctx)) { 238 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 239 store_offset -= 8; 240 } 241 if (seen_reg(RV_REG_S3, ctx)) { 242 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 243 store_offset -= 8; 244 } 245 if (seen_reg(RV_REG_S4, ctx)) { 246 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 247 store_offset -= 8; 248 } 249 if (seen_reg(RV_REG_S5, ctx)) { 250 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 251 store_offset -= 8; 252 } 253 if (seen_reg(RV_REG_S6, ctx)) { 254 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 255 store_offset -= 8; 256 } 257 258 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 259 /* Set return value. */ 260 if (!is_tail_call) 261 emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); 262 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 263 is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 264 ctx); 265 } 266 267 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 268 struct rv_jit_context *ctx) 269 { 270 switch (cond) { 271 case BPF_JEQ: 272 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 273 return; 274 case BPF_JGT: 275 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 276 return; 277 case BPF_JLT: 278 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 279 return; 280 case BPF_JGE: 281 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 282 return; 283 case BPF_JLE: 284 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 285 return; 286 case BPF_JNE: 287 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 288 return; 289 case BPF_JSGT: 290 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 291 return; 292 case BPF_JSLT: 293 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 294 return; 295 case BPF_JSGE: 296 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 297 return; 298 case BPF_JSLE: 299 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 300 } 301 } 302 303 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 304 struct rv_jit_context *ctx) 305 { 306 s64 upper, lower; 307 308 if (is_13b_int(rvoff)) { 309 emit_bcc(cond, rd, rs, rvoff, ctx); 310 return; 311 } 312 313 /* Adjust for jal */ 314 rvoff -= 4; 315 316 /* Transform, e.g.: 317 * bne rd,rs,foo 318 * to 319 * beq rd,rs,<.L1> 320 * (auipc foo) 321 * jal(r) foo 322 * .L1 323 */ 324 cond = invert_bpf_cond(cond); 325 if (is_21b_int(rvoff)) { 326 emit_bcc(cond, rd, rs, 8, ctx); 327 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 328 return; 329 } 330 331 /* 32b No need for an additional rvoff adjustment, since we 332 * get that from the auipc at PC', where PC = PC' + 4. 333 */ 334 upper = (rvoff + (1 << 11)) >> 12; 335 lower = rvoff & 0xfff; 336 337 emit_bcc(cond, rd, rs, 12, ctx); 338 emit(rv_auipc(RV_REG_T1, upper), ctx); 339 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 340 } 341 342 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 343 { 344 int tc_ninsn, off, start_insn = ctx->ninsns; 345 u8 tcc = rv_tail_call_reg(ctx); 346 347 /* a0: &ctx 348 * a1: &array 349 * a2: index 350 * 351 * if (index >= array->map.max_entries) 352 * goto out; 353 */ 354 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 355 ctx->offset[0]; 356 emit_zextw(RV_REG_A2, RV_REG_A2, ctx); 357 358 off = offsetof(struct bpf_array, map.max_entries); 359 if (is_12b_check(off, insn)) 360 return -1; 361 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 362 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 363 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 364 365 /* if (--TCC < 0) 366 * goto out; 367 */ 368 emit_addi(RV_REG_TCC, tcc, -1, ctx); 369 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 370 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 371 372 /* prog = array->ptrs[index]; 373 * if (!prog) 374 * goto out; 375 */ 376 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 377 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 378 off = offsetof(struct bpf_array, ptrs); 379 if (is_12b_check(off, insn)) 380 return -1; 381 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 382 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 383 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 384 385 /* goto *(prog->bpf_func + 4); */ 386 off = offsetof(struct bpf_prog, bpf_func); 387 if (is_12b_check(off, insn)) 388 return -1; 389 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 390 __build_epilogue(true, ctx); 391 return 0; 392 } 393 394 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 395 struct rv_jit_context *ctx) 396 { 397 u8 code = insn->code; 398 399 switch (code) { 400 case BPF_JMP | BPF_JA: 401 case BPF_JMP | BPF_CALL: 402 case BPF_JMP | BPF_EXIT: 403 case BPF_JMP | BPF_TAIL_CALL: 404 break; 405 default: 406 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 407 } 408 409 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 410 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 411 code & BPF_LDX || code & BPF_STX) 412 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 413 } 414 415 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 416 struct rv_jit_context *ctx) 417 { 418 s64 upper, lower; 419 420 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 421 emit(rv_jal(rd, rvoff >> 1), ctx); 422 return 0; 423 } else if (in_auipc_jalr_range(rvoff)) { 424 upper = (rvoff + (1 << 11)) >> 12; 425 lower = rvoff & 0xfff; 426 emit(rv_auipc(RV_REG_T1, upper), ctx); 427 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 428 return 0; 429 } 430 431 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 432 return -ERANGE; 433 } 434 435 static bool is_signed_bpf_cond(u8 cond) 436 { 437 return cond == BPF_JSGT || cond == BPF_JSLT || 438 cond == BPF_JSGE || cond == BPF_JSLE; 439 } 440 441 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 442 { 443 s64 off = 0; 444 u64 ip; 445 446 if (addr && ctx->insns && ctx->ro_insns) { 447 /* 448 * Use the ro_insns(RX) to calculate the offset as the BPF 449 * program will finally run from this memory region. 450 */ 451 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); 452 off = addr - ip; 453 } 454 455 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 456 } 457 458 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 459 struct rv_jit_context *ctx) 460 { 461 u8 r0; 462 int jmp_offset; 463 464 if (off) { 465 if (is_12b_int(off)) { 466 emit_addi(RV_REG_T1, rd, off, ctx); 467 } else { 468 emit_imm(RV_REG_T1, off, ctx); 469 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 470 } 471 rd = RV_REG_T1; 472 } 473 474 switch (imm) { 475 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 476 case BPF_ADD: 477 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 478 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 479 break; 480 case BPF_AND: 481 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 482 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 483 break; 484 case BPF_OR: 485 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 486 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 487 break; 488 case BPF_XOR: 489 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 490 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 491 break; 492 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 493 case BPF_ADD | BPF_FETCH: 494 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : 495 rv_amoadd_w(rs, rs, rd, 0, 0), ctx); 496 if (!is64) 497 emit_zextw(rs, rs, ctx); 498 break; 499 case BPF_AND | BPF_FETCH: 500 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : 501 rv_amoand_w(rs, rs, rd, 0, 0), ctx); 502 if (!is64) 503 emit_zextw(rs, rs, ctx); 504 break; 505 case BPF_OR | BPF_FETCH: 506 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : 507 rv_amoor_w(rs, rs, rd, 0, 0), ctx); 508 if (!is64) 509 emit_zextw(rs, rs, ctx); 510 break; 511 case BPF_XOR | BPF_FETCH: 512 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : 513 rv_amoxor_w(rs, rs, rd, 0, 0), ctx); 514 if (!is64) 515 emit_zextw(rs, rs, ctx); 516 break; 517 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 518 case BPF_XCHG: 519 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : 520 rv_amoswap_w(rs, rs, rd, 0, 0), ctx); 521 if (!is64) 522 emit_zextw(rs, rs, ctx); 523 break; 524 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 525 case BPF_CMPXCHG: 526 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 527 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 528 rv_addiw(RV_REG_T2, r0, 0), ctx); 529 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 530 rv_lr_w(r0, 0, rd, 0, 0), ctx); 531 jmp_offset = ninsns_rvoff(8); 532 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 533 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 534 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 535 jmp_offset = ninsns_rvoff(-6); 536 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 537 emit(rv_fence(0x3, 0x3), ctx); 538 break; 539 } 540 } 541 542 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 543 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 544 545 bool ex_handler_bpf(const struct exception_table_entry *ex, 546 struct pt_regs *regs) 547 { 548 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 549 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 550 551 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 552 regs->epc = (unsigned long)&ex->fixup - offset; 553 554 return true; 555 } 556 557 /* For accesses to BTF pointers, add an entry to the exception table */ 558 static int add_exception_handler(const struct bpf_insn *insn, 559 struct rv_jit_context *ctx, 560 int dst_reg, int insn_len) 561 { 562 struct exception_table_entry *ex; 563 unsigned long pc; 564 off_t ins_offset; 565 off_t fixup_offset; 566 567 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 568 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX)) 569 return 0; 570 571 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 572 return -EINVAL; 573 574 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 575 return -EINVAL; 576 577 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 578 return -EINVAL; 579 580 ex = &ctx->prog->aux->extable[ctx->nexentries]; 581 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 582 583 /* 584 * This is the relative offset of the instruction that may fault from 585 * the exception table itself. This will be written to the exception 586 * table and if this instruction faults, the destination register will 587 * be set to '0' and the execution will jump to the next instruction. 588 */ 589 ins_offset = pc - (long)&ex->insn; 590 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 591 return -ERANGE; 592 593 /* 594 * Since the extable follows the program, the fixup offset is always 595 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 596 * to keep things simple, and put the destination register in the upper 597 * bits. We don't need to worry about buildtime or runtime sort 598 * modifying the upper bits because the table is already sorted, and 599 * isn't part of the main exception table. 600 * 601 * The fixup_offset is set to the next instruction from the instruction 602 * that may fault. The execution will jump to this after handling the 603 * fault. 604 */ 605 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 606 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 607 return -ERANGE; 608 609 /* 610 * The offsets above have been calculated using the RO buffer but we 611 * need to use the R/W buffer for writes. 612 * switch ex to rw buffer for writing. 613 */ 614 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); 615 616 ex->insn = ins_offset; 617 618 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 619 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 620 ex->type = EX_TYPE_BPF; 621 622 ctx->nexentries++; 623 return 0; 624 } 625 626 static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 627 { 628 s64 rvoff; 629 struct rv_jit_context ctx; 630 631 ctx.ninsns = 0; 632 ctx.insns = (u16 *)insns; 633 634 if (!target) { 635 emit(rv_nop(), &ctx); 636 emit(rv_nop(), &ctx); 637 return 0; 638 } 639 640 rvoff = (s64)(target - ip); 641 return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 642 } 643 644 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 645 void *old_addr, void *new_addr) 646 { 647 u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 648 bool is_call = poke_type == BPF_MOD_CALL; 649 int ret; 650 651 if (!is_kernel_text((unsigned long)ip) && 652 !is_bpf_text_address((unsigned long)ip)) 653 return -ENOTSUPP; 654 655 ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 656 if (ret) 657 return ret; 658 659 if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) 660 return -EFAULT; 661 662 ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 663 if (ret) 664 return ret; 665 666 cpus_read_lock(); 667 mutex_lock(&text_mutex); 668 if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) 669 ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); 670 mutex_unlock(&text_mutex); 671 cpus_read_unlock(); 672 673 return ret; 674 } 675 676 static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) 677 { 678 int i; 679 680 for (i = 0; i < nregs; i++) { 681 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 682 args_off -= 8; 683 } 684 } 685 686 static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) 687 { 688 int i; 689 690 for (i = 0; i < nregs; i++) { 691 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 692 args_off -= 8; 693 } 694 } 695 696 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 697 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 698 { 699 int ret, branch_off; 700 struct bpf_prog *p = l->link.prog; 701 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 702 703 if (l->cookie) { 704 emit_imm(RV_REG_T1, l->cookie, ctx); 705 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 706 } else { 707 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 708 } 709 710 /* arg1: prog */ 711 emit_imm(RV_REG_A0, (const s64)p, ctx); 712 /* arg2: &run_ctx */ 713 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 714 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 715 if (ret) 716 return ret; 717 718 /* if (__bpf_prog_enter(prog) == 0) 719 * goto skip_exec_of_prog; 720 */ 721 branch_off = ctx->ninsns; 722 /* nop reserved for conditional jump */ 723 emit(rv_nop(), ctx); 724 725 /* store prog start time */ 726 emit_mv(RV_REG_S1, RV_REG_A0, ctx); 727 728 /* arg1: &args_off */ 729 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 730 if (!p->jited) 731 /* arg2: progs[i]->insnsi for interpreter */ 732 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 733 ret = emit_call((const u64)p->bpf_func, true, ctx); 734 if (ret) 735 return ret; 736 737 if (save_ret) { 738 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 739 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 740 } 741 742 /* update branch with beqz */ 743 if (ctx->insns) { 744 int offset = ninsns_rvoff(ctx->ninsns - branch_off); 745 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 746 *(u32 *)(ctx->insns + branch_off) = insn; 747 } 748 749 /* arg1: prog */ 750 emit_imm(RV_REG_A0, (const s64)p, ctx); 751 /* arg2: prog start time */ 752 emit_mv(RV_REG_A1, RV_REG_S1, ctx); 753 /* arg3: &run_ctx */ 754 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 755 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 756 757 return ret; 758 } 759 760 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 761 const struct btf_func_model *m, 762 struct bpf_tramp_links *tlinks, 763 void *func_addr, u32 flags, 764 struct rv_jit_context *ctx) 765 { 766 int i, ret, offset; 767 int *branches_off = NULL; 768 int stack_size = 0, nregs = m->nr_args; 769 int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; 770 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 771 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 772 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 773 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 774 void *orig_call = func_addr; 775 bool save_ret; 776 u32 insn; 777 778 /* Two types of generated trampoline stack layout: 779 * 780 * 1. trampoline called from function entry 781 * -------------------------------------- 782 * FP + 8 [ RA to parent func ] return address to parent 783 * function 784 * FP + 0 [ FP of parent func ] frame pointer of parent 785 * function 786 * FP - 8 [ T0 to traced func ] return address of traced 787 * function 788 * FP - 16 [ FP of traced func ] frame pointer of traced 789 * function 790 * -------------------------------------- 791 * 792 * 2. trampoline called directly 793 * -------------------------------------- 794 * FP - 8 [ RA to caller func ] return address to caller 795 * function 796 * FP - 16 [ FP of caller func ] frame pointer of caller 797 * function 798 * -------------------------------------- 799 * 800 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 801 * BPF_TRAMP_F_RET_FENTRY_RET 802 * [ argN ] 803 * [ ... ] 804 * FP - args_off [ arg1 ] 805 * 806 * FP - nregs_off [ regs count ] 807 * 808 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 809 * 810 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 811 * 812 * FP - sreg_off [ callee saved reg ] 813 * 814 * [ pads ] pads for 16 bytes alignment 815 */ 816 817 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 818 return -ENOTSUPP; 819 820 /* extra regiters for struct arguments */ 821 for (i = 0; i < m->nr_args; i++) 822 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 823 nregs += round_up(m->arg_size[i], 8) / 8 - 1; 824 825 /* 8 arguments passed by registers */ 826 if (nregs > 8) 827 return -ENOTSUPP; 828 829 /* room of trampoline frame to store return address and frame pointer */ 830 stack_size += 16; 831 832 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 833 if (save_ret) { 834 stack_size += 16; /* Save both A5 (BPF R0) and A0 */ 835 retval_off = stack_size; 836 } 837 838 stack_size += nregs * 8; 839 args_off = stack_size; 840 841 stack_size += 8; 842 nregs_off = stack_size; 843 844 if (flags & BPF_TRAMP_F_IP_ARG) { 845 stack_size += 8; 846 ip_off = stack_size; 847 } 848 849 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 850 run_ctx_off = stack_size; 851 852 stack_size += 8; 853 sreg_off = stack_size; 854 855 stack_size = round_up(stack_size, 16); 856 857 if (!is_struct_ops) { 858 /* For the trampoline called from function entry, 859 * the frame of traced function and the frame of 860 * trampoline need to be considered. 861 */ 862 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 863 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 864 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 865 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 866 867 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 868 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 869 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 870 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 871 } else { 872 /* For the trampoline called directly, just handle 873 * the frame of trampoline. 874 */ 875 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 876 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 877 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 878 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 879 } 880 881 /* callee saved register S1 to pass start time */ 882 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 883 884 /* store ip address of the traced function */ 885 if (flags & BPF_TRAMP_F_IP_ARG) { 886 emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 887 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 888 } 889 890 emit_li(RV_REG_T1, nregs, ctx); 891 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 892 893 store_args(nregs, args_off, ctx); 894 895 /* skip to actual body of traced function */ 896 if (flags & BPF_TRAMP_F_SKIP_FRAME) 897 orig_call += RV_FENTRY_NINSNS * 4; 898 899 if (flags & BPF_TRAMP_F_CALL_ORIG) { 900 emit_imm(RV_REG_A0, (const s64)im, ctx); 901 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 902 if (ret) 903 return ret; 904 } 905 906 for (i = 0; i < fentry->nr_links; i++) { 907 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 908 flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 909 if (ret) 910 return ret; 911 } 912 913 if (fmod_ret->nr_links) { 914 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 915 if (!branches_off) 916 return -ENOMEM; 917 918 /* cleanup to avoid garbage return value confusion */ 919 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 920 for (i = 0; i < fmod_ret->nr_links; i++) { 921 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 922 run_ctx_off, true, ctx); 923 if (ret) 924 goto out; 925 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 926 branches_off[i] = ctx->ninsns; 927 /* nop reserved for conditional jump */ 928 emit(rv_nop(), ctx); 929 } 930 } 931 932 if (flags & BPF_TRAMP_F_CALL_ORIG) { 933 restore_args(nregs, args_off, ctx); 934 ret = emit_call((const u64)orig_call, true, ctx); 935 if (ret) 936 goto out; 937 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 938 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 939 im->ip_after_call = ctx->insns + ctx->ninsns; 940 /* 2 nops reserved for auipc+jalr pair */ 941 emit(rv_nop(), ctx); 942 emit(rv_nop(), ctx); 943 } 944 945 /* update branches saved in invoke_bpf_mod_ret with bnez */ 946 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 947 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 948 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 949 *(u32 *)(ctx->insns + branches_off[i]) = insn; 950 } 951 952 for (i = 0; i < fexit->nr_links; i++) { 953 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 954 run_ctx_off, false, ctx); 955 if (ret) 956 goto out; 957 } 958 959 if (flags & BPF_TRAMP_F_CALL_ORIG) { 960 im->ip_epilogue = ctx->insns + ctx->ninsns; 961 emit_imm(RV_REG_A0, (const s64)im, ctx); 962 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 963 if (ret) 964 goto out; 965 } 966 967 if (flags & BPF_TRAMP_F_RESTORE_REGS) 968 restore_args(nregs, args_off, ctx); 969 970 if (save_ret) { 971 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 972 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); 973 } 974 975 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 976 977 if (!is_struct_ops) { 978 /* trampoline called from function entry */ 979 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 980 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 981 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 982 983 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 984 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 985 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 986 987 if (flags & BPF_TRAMP_F_SKIP_FRAME) 988 /* return to parent function */ 989 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 990 else 991 /* return to traced function */ 992 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 993 } else { 994 /* trampoline called directly */ 995 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 996 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 997 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 998 999 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1000 } 1001 1002 ret = ctx->ninsns; 1003 out: 1004 kfree(branches_off); 1005 return ret; 1006 } 1007 1008 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1009 struct bpf_tramp_links *tlinks, void *func_addr) 1010 { 1011 struct bpf_tramp_image im; 1012 struct rv_jit_context ctx; 1013 int ret; 1014 1015 ctx.ninsns = 0; 1016 ctx.insns = NULL; 1017 ctx.ro_insns = NULL; 1018 ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); 1019 1020 return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); 1021 } 1022 1023 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 1024 void *image_end, const struct btf_func_model *m, 1025 u32 flags, struct bpf_tramp_links *tlinks, 1026 void *func_addr) 1027 { 1028 int ret; 1029 struct rv_jit_context ctx; 1030 1031 ctx.ninsns = 0; 1032 /* 1033 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the 1034 * JITed instructions and later copies it to a RX region (ctx.ro_insns). 1035 * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the 1036 * trampoline image uses the same memory area for writing and execution, 1037 * both ctx.insns and ctx.ro_insns can be set to image. 1038 */ 1039 ctx.insns = image; 1040 ctx.ro_insns = image; 1041 ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 1042 if (ret < 0) 1043 return ret; 1044 1045 bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); 1046 1047 return ninsns_rvoff(ret); 1048 } 1049 1050 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1051 bool extra_pass) 1052 { 1053 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1054 BPF_CLASS(insn->code) == BPF_JMP; 1055 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1056 struct bpf_prog_aux *aux = ctx->prog->aux; 1057 u8 rd = -1, rs = -1, code = insn->code; 1058 s16 off = insn->off; 1059 s32 imm = insn->imm; 1060 1061 init_regs(&rd, &rs, insn, ctx); 1062 1063 switch (code) { 1064 /* dst = src */ 1065 case BPF_ALU | BPF_MOV | BPF_X: 1066 case BPF_ALU64 | BPF_MOV | BPF_X: 1067 if (imm == 1) { 1068 /* Special mov32 for zext */ 1069 emit_zextw(rd, rd, ctx); 1070 break; 1071 } 1072 switch (insn->off) { 1073 case 0: 1074 emit_mv(rd, rs, ctx); 1075 break; 1076 case 8: 1077 emit_sextb(rd, rs, ctx); 1078 break; 1079 case 16: 1080 emit_sexth(rd, rs, ctx); 1081 break; 1082 case 32: 1083 emit_sextw(rd, rs, ctx); 1084 break; 1085 } 1086 if (!is64 && !aux->verifier_zext) 1087 emit_zextw(rd, rd, ctx); 1088 break; 1089 1090 /* dst = dst OP src */ 1091 case BPF_ALU | BPF_ADD | BPF_X: 1092 case BPF_ALU64 | BPF_ADD | BPF_X: 1093 emit_add(rd, rd, rs, ctx); 1094 if (!is64 && !aux->verifier_zext) 1095 emit_zextw(rd, rd, ctx); 1096 break; 1097 case BPF_ALU | BPF_SUB | BPF_X: 1098 case BPF_ALU64 | BPF_SUB | BPF_X: 1099 if (is64) 1100 emit_sub(rd, rd, rs, ctx); 1101 else 1102 emit_subw(rd, rd, rs, ctx); 1103 1104 if (!is64 && !aux->verifier_zext) 1105 emit_zextw(rd, rd, ctx); 1106 break; 1107 case BPF_ALU | BPF_AND | BPF_X: 1108 case BPF_ALU64 | BPF_AND | BPF_X: 1109 emit_and(rd, rd, rs, ctx); 1110 if (!is64 && !aux->verifier_zext) 1111 emit_zextw(rd, rd, ctx); 1112 break; 1113 case BPF_ALU | BPF_OR | BPF_X: 1114 case BPF_ALU64 | BPF_OR | BPF_X: 1115 emit_or(rd, rd, rs, ctx); 1116 if (!is64 && !aux->verifier_zext) 1117 emit_zextw(rd, rd, ctx); 1118 break; 1119 case BPF_ALU | BPF_XOR | BPF_X: 1120 case BPF_ALU64 | BPF_XOR | BPF_X: 1121 emit_xor(rd, rd, rs, ctx); 1122 if (!is64 && !aux->verifier_zext) 1123 emit_zextw(rd, rd, ctx); 1124 break; 1125 case BPF_ALU | BPF_MUL | BPF_X: 1126 case BPF_ALU64 | BPF_MUL | BPF_X: 1127 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1128 if (!is64 && !aux->verifier_zext) 1129 emit_zextw(rd, rd, ctx); 1130 break; 1131 case BPF_ALU | BPF_DIV | BPF_X: 1132 case BPF_ALU64 | BPF_DIV | BPF_X: 1133 if (off) 1134 emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1135 else 1136 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1137 if (!is64 && !aux->verifier_zext) 1138 emit_zextw(rd, rd, ctx); 1139 break; 1140 case BPF_ALU | BPF_MOD | BPF_X: 1141 case BPF_ALU64 | BPF_MOD | BPF_X: 1142 if (off) 1143 emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1144 else 1145 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1146 if (!is64 && !aux->verifier_zext) 1147 emit_zextw(rd, rd, ctx); 1148 break; 1149 case BPF_ALU | BPF_LSH | BPF_X: 1150 case BPF_ALU64 | BPF_LSH | BPF_X: 1151 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 1152 if (!is64 && !aux->verifier_zext) 1153 emit_zextw(rd, rd, ctx); 1154 break; 1155 case BPF_ALU | BPF_RSH | BPF_X: 1156 case BPF_ALU64 | BPF_RSH | BPF_X: 1157 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1158 if (!is64 && !aux->verifier_zext) 1159 emit_zextw(rd, rd, ctx); 1160 break; 1161 case BPF_ALU | BPF_ARSH | BPF_X: 1162 case BPF_ALU64 | BPF_ARSH | BPF_X: 1163 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1164 if (!is64 && !aux->verifier_zext) 1165 emit_zextw(rd, rd, ctx); 1166 break; 1167 1168 /* dst = -dst */ 1169 case BPF_ALU | BPF_NEG: 1170 case BPF_ALU64 | BPF_NEG: 1171 emit_sub(rd, RV_REG_ZERO, rd, ctx); 1172 if (!is64 && !aux->verifier_zext) 1173 emit_zextw(rd, rd, ctx); 1174 break; 1175 1176 /* dst = BSWAP##imm(dst) */ 1177 case BPF_ALU | BPF_END | BPF_FROM_LE: 1178 switch (imm) { 1179 case 16: 1180 emit_zexth(rd, rd, ctx); 1181 break; 1182 case 32: 1183 if (!aux->verifier_zext) 1184 emit_zextw(rd, rd, ctx); 1185 break; 1186 case 64: 1187 /* Do nothing */ 1188 break; 1189 } 1190 break; 1191 case BPF_ALU | BPF_END | BPF_FROM_BE: 1192 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1193 emit_bswap(rd, imm, ctx); 1194 break; 1195 1196 /* dst = imm */ 1197 case BPF_ALU | BPF_MOV | BPF_K: 1198 case BPF_ALU64 | BPF_MOV | BPF_K: 1199 emit_imm(rd, imm, ctx); 1200 if (!is64 && !aux->verifier_zext) 1201 emit_zextw(rd, rd, ctx); 1202 break; 1203 1204 /* dst = dst OP imm */ 1205 case BPF_ALU | BPF_ADD | BPF_K: 1206 case BPF_ALU64 | BPF_ADD | BPF_K: 1207 if (is_12b_int(imm)) { 1208 emit_addi(rd, rd, imm, ctx); 1209 } else { 1210 emit_imm(RV_REG_T1, imm, ctx); 1211 emit_add(rd, rd, RV_REG_T1, ctx); 1212 } 1213 if (!is64 && !aux->verifier_zext) 1214 emit_zextw(rd, rd, ctx); 1215 break; 1216 case BPF_ALU | BPF_SUB | BPF_K: 1217 case BPF_ALU64 | BPF_SUB | BPF_K: 1218 if (is_12b_int(-imm)) { 1219 emit_addi(rd, rd, -imm, ctx); 1220 } else { 1221 emit_imm(RV_REG_T1, imm, ctx); 1222 emit_sub(rd, rd, RV_REG_T1, ctx); 1223 } 1224 if (!is64 && !aux->verifier_zext) 1225 emit_zextw(rd, rd, ctx); 1226 break; 1227 case BPF_ALU | BPF_AND | BPF_K: 1228 case BPF_ALU64 | BPF_AND | BPF_K: 1229 if (is_12b_int(imm)) { 1230 emit_andi(rd, rd, imm, ctx); 1231 } else { 1232 emit_imm(RV_REG_T1, imm, ctx); 1233 emit_and(rd, rd, RV_REG_T1, ctx); 1234 } 1235 if (!is64 && !aux->verifier_zext) 1236 emit_zextw(rd, rd, ctx); 1237 break; 1238 case BPF_ALU | BPF_OR | BPF_K: 1239 case BPF_ALU64 | BPF_OR | BPF_K: 1240 if (is_12b_int(imm)) { 1241 emit(rv_ori(rd, rd, imm), ctx); 1242 } else { 1243 emit_imm(RV_REG_T1, imm, ctx); 1244 emit_or(rd, rd, RV_REG_T1, ctx); 1245 } 1246 if (!is64 && !aux->verifier_zext) 1247 emit_zextw(rd, rd, ctx); 1248 break; 1249 case BPF_ALU | BPF_XOR | BPF_K: 1250 case BPF_ALU64 | BPF_XOR | BPF_K: 1251 if (is_12b_int(imm)) { 1252 emit(rv_xori(rd, rd, imm), ctx); 1253 } else { 1254 emit_imm(RV_REG_T1, imm, ctx); 1255 emit_xor(rd, rd, RV_REG_T1, ctx); 1256 } 1257 if (!is64 && !aux->verifier_zext) 1258 emit_zextw(rd, rd, ctx); 1259 break; 1260 case BPF_ALU | BPF_MUL | BPF_K: 1261 case BPF_ALU64 | BPF_MUL | BPF_K: 1262 emit_imm(RV_REG_T1, imm, ctx); 1263 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1264 rv_mulw(rd, rd, RV_REG_T1), ctx); 1265 if (!is64 && !aux->verifier_zext) 1266 emit_zextw(rd, rd, ctx); 1267 break; 1268 case BPF_ALU | BPF_DIV | BPF_K: 1269 case BPF_ALU64 | BPF_DIV | BPF_K: 1270 emit_imm(RV_REG_T1, imm, ctx); 1271 if (off) 1272 emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1273 rv_divw(rd, rd, RV_REG_T1), ctx); 1274 else 1275 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1276 rv_divuw(rd, rd, RV_REG_T1), ctx); 1277 if (!is64 && !aux->verifier_zext) 1278 emit_zextw(rd, rd, ctx); 1279 break; 1280 case BPF_ALU | BPF_MOD | BPF_K: 1281 case BPF_ALU64 | BPF_MOD | BPF_K: 1282 emit_imm(RV_REG_T1, imm, ctx); 1283 if (off) 1284 emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1285 rv_remw(rd, rd, RV_REG_T1), ctx); 1286 else 1287 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1288 rv_remuw(rd, rd, RV_REG_T1), ctx); 1289 if (!is64 && !aux->verifier_zext) 1290 emit_zextw(rd, rd, ctx); 1291 break; 1292 case BPF_ALU | BPF_LSH | BPF_K: 1293 case BPF_ALU64 | BPF_LSH | BPF_K: 1294 emit_slli(rd, rd, imm, ctx); 1295 1296 if (!is64 && !aux->verifier_zext) 1297 emit_zextw(rd, rd, ctx); 1298 break; 1299 case BPF_ALU | BPF_RSH | BPF_K: 1300 case BPF_ALU64 | BPF_RSH | BPF_K: 1301 if (is64) 1302 emit_srli(rd, rd, imm, ctx); 1303 else 1304 emit(rv_srliw(rd, rd, imm), ctx); 1305 1306 if (!is64 && !aux->verifier_zext) 1307 emit_zextw(rd, rd, ctx); 1308 break; 1309 case BPF_ALU | BPF_ARSH | BPF_K: 1310 case BPF_ALU64 | BPF_ARSH | BPF_K: 1311 if (is64) 1312 emit_srai(rd, rd, imm, ctx); 1313 else 1314 emit(rv_sraiw(rd, rd, imm), ctx); 1315 1316 if (!is64 && !aux->verifier_zext) 1317 emit_zextw(rd, rd, ctx); 1318 break; 1319 1320 /* JUMP off */ 1321 case BPF_JMP | BPF_JA: 1322 case BPF_JMP32 | BPF_JA: 1323 if (BPF_CLASS(code) == BPF_JMP) 1324 rvoff = rv_offset(i, off, ctx); 1325 else 1326 rvoff = rv_offset(i, imm, ctx); 1327 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1328 if (ret) 1329 return ret; 1330 break; 1331 1332 /* IF (dst COND src) JUMP off */ 1333 case BPF_JMP | BPF_JEQ | BPF_X: 1334 case BPF_JMP32 | BPF_JEQ | BPF_X: 1335 case BPF_JMP | BPF_JGT | BPF_X: 1336 case BPF_JMP32 | BPF_JGT | BPF_X: 1337 case BPF_JMP | BPF_JLT | BPF_X: 1338 case BPF_JMP32 | BPF_JLT | BPF_X: 1339 case BPF_JMP | BPF_JGE | BPF_X: 1340 case BPF_JMP32 | BPF_JGE | BPF_X: 1341 case BPF_JMP | BPF_JLE | BPF_X: 1342 case BPF_JMP32 | BPF_JLE | BPF_X: 1343 case BPF_JMP | BPF_JNE | BPF_X: 1344 case BPF_JMP32 | BPF_JNE | BPF_X: 1345 case BPF_JMP | BPF_JSGT | BPF_X: 1346 case BPF_JMP32 | BPF_JSGT | BPF_X: 1347 case BPF_JMP | BPF_JSLT | BPF_X: 1348 case BPF_JMP32 | BPF_JSLT | BPF_X: 1349 case BPF_JMP | BPF_JSGE | BPF_X: 1350 case BPF_JMP32 | BPF_JSGE | BPF_X: 1351 case BPF_JMP | BPF_JSLE | BPF_X: 1352 case BPF_JMP32 | BPF_JSLE | BPF_X: 1353 case BPF_JMP | BPF_JSET | BPF_X: 1354 case BPF_JMP32 | BPF_JSET | BPF_X: 1355 rvoff = rv_offset(i, off, ctx); 1356 if (!is64) { 1357 s = ctx->ninsns; 1358 if (is_signed_bpf_cond(BPF_OP(code))) { 1359 emit_sextw_alt(&rs, RV_REG_T1, ctx); 1360 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1361 } else { 1362 emit_zextw_alt(&rs, RV_REG_T1, ctx); 1363 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1364 } 1365 e = ctx->ninsns; 1366 1367 /* Adjust for extra insns */ 1368 rvoff -= ninsns_rvoff(e - s); 1369 } 1370 1371 if (BPF_OP(code) == BPF_JSET) { 1372 /* Adjust for and */ 1373 rvoff -= 4; 1374 emit_and(RV_REG_T1, rd, rs, ctx); 1375 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1376 } else { 1377 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1378 } 1379 break; 1380 1381 /* IF (dst COND imm) JUMP off */ 1382 case BPF_JMP | BPF_JEQ | BPF_K: 1383 case BPF_JMP32 | BPF_JEQ | BPF_K: 1384 case BPF_JMP | BPF_JGT | BPF_K: 1385 case BPF_JMP32 | BPF_JGT | BPF_K: 1386 case BPF_JMP | BPF_JLT | BPF_K: 1387 case BPF_JMP32 | BPF_JLT | BPF_K: 1388 case BPF_JMP | BPF_JGE | BPF_K: 1389 case BPF_JMP32 | BPF_JGE | BPF_K: 1390 case BPF_JMP | BPF_JLE | BPF_K: 1391 case BPF_JMP32 | BPF_JLE | BPF_K: 1392 case BPF_JMP | BPF_JNE | BPF_K: 1393 case BPF_JMP32 | BPF_JNE | BPF_K: 1394 case BPF_JMP | BPF_JSGT | BPF_K: 1395 case BPF_JMP32 | BPF_JSGT | BPF_K: 1396 case BPF_JMP | BPF_JSLT | BPF_K: 1397 case BPF_JMP32 | BPF_JSLT | BPF_K: 1398 case BPF_JMP | BPF_JSGE | BPF_K: 1399 case BPF_JMP32 | BPF_JSGE | BPF_K: 1400 case BPF_JMP | BPF_JSLE | BPF_K: 1401 case BPF_JMP32 | BPF_JSLE | BPF_K: 1402 rvoff = rv_offset(i, off, ctx); 1403 s = ctx->ninsns; 1404 if (imm) 1405 emit_imm(RV_REG_T1, imm, ctx); 1406 rs = imm ? RV_REG_T1 : RV_REG_ZERO; 1407 if (!is64) { 1408 if (is_signed_bpf_cond(BPF_OP(code))) { 1409 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1410 /* rs has been sign extended */ 1411 } else { 1412 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1413 if (imm) 1414 emit_zextw(rs, rs, ctx); 1415 } 1416 } 1417 e = ctx->ninsns; 1418 1419 /* Adjust for extra insns */ 1420 rvoff -= ninsns_rvoff(e - s); 1421 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1422 break; 1423 1424 case BPF_JMP | BPF_JSET | BPF_K: 1425 case BPF_JMP32 | BPF_JSET | BPF_K: 1426 rvoff = rv_offset(i, off, ctx); 1427 s = ctx->ninsns; 1428 if (is_12b_int(imm)) { 1429 emit_andi(RV_REG_T1, rd, imm, ctx); 1430 } else { 1431 emit_imm(RV_REG_T1, imm, ctx); 1432 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1433 } 1434 /* For jset32, we should clear the upper 32 bits of t1, but 1435 * sign-extension is sufficient here and saves one instruction, 1436 * as t1 is used only in comparison against zero. 1437 */ 1438 if (!is64 && imm < 0) 1439 emit_sextw(RV_REG_T1, RV_REG_T1, ctx); 1440 e = ctx->ninsns; 1441 rvoff -= ninsns_rvoff(e - s); 1442 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1443 break; 1444 1445 /* function call */ 1446 case BPF_JMP | BPF_CALL: 1447 { 1448 bool fixed_addr; 1449 u64 addr; 1450 1451 mark_call(ctx); 1452 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1453 &addr, &fixed_addr); 1454 if (ret < 0) 1455 return ret; 1456 1457 ret = emit_call(addr, fixed_addr, ctx); 1458 if (ret) 1459 return ret; 1460 1461 if (insn->src_reg != BPF_PSEUDO_CALL) 1462 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1463 break; 1464 } 1465 /* tail call */ 1466 case BPF_JMP | BPF_TAIL_CALL: 1467 if (emit_bpf_tail_call(i, ctx)) 1468 return -1; 1469 break; 1470 1471 /* function return */ 1472 case BPF_JMP | BPF_EXIT: 1473 if (i == ctx->prog->len - 1) 1474 break; 1475 1476 rvoff = epilogue_offset(ctx); 1477 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1478 if (ret) 1479 return ret; 1480 break; 1481 1482 /* dst = imm64 */ 1483 case BPF_LD | BPF_IMM | BPF_DW: 1484 { 1485 struct bpf_insn insn1 = insn[1]; 1486 u64 imm64; 1487 1488 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1489 if (bpf_pseudo_func(insn)) { 1490 /* fixed-length insns for extra jit pass */ 1491 ret = emit_addr(rd, imm64, extra_pass, ctx); 1492 if (ret) 1493 return ret; 1494 } else { 1495 emit_imm(rd, imm64, ctx); 1496 } 1497 1498 return 1; 1499 } 1500 1501 /* LDX: dst = *(unsigned size *)(src + off) */ 1502 case BPF_LDX | BPF_MEM | BPF_B: 1503 case BPF_LDX | BPF_MEM | BPF_H: 1504 case BPF_LDX | BPF_MEM | BPF_W: 1505 case BPF_LDX | BPF_MEM | BPF_DW: 1506 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1507 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1508 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1509 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1510 /* LDSX: dst = *(signed size *)(src + off) */ 1511 case BPF_LDX | BPF_MEMSX | BPF_B: 1512 case BPF_LDX | BPF_MEMSX | BPF_H: 1513 case BPF_LDX | BPF_MEMSX | BPF_W: 1514 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1515 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1516 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1517 { 1518 int insn_len, insns_start; 1519 bool sign_ext; 1520 1521 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1522 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1523 1524 switch (BPF_SIZE(code)) { 1525 case BPF_B: 1526 if (is_12b_int(off)) { 1527 insns_start = ctx->ninsns; 1528 if (sign_ext) 1529 emit(rv_lb(rd, off, rs), ctx); 1530 else 1531 emit(rv_lbu(rd, off, rs), ctx); 1532 insn_len = ctx->ninsns - insns_start; 1533 break; 1534 } 1535 1536 emit_imm(RV_REG_T1, off, ctx); 1537 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1538 insns_start = ctx->ninsns; 1539 if (sign_ext) 1540 emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1541 else 1542 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1543 insn_len = ctx->ninsns - insns_start; 1544 break; 1545 case BPF_H: 1546 if (is_12b_int(off)) { 1547 insns_start = ctx->ninsns; 1548 if (sign_ext) 1549 emit(rv_lh(rd, off, rs), ctx); 1550 else 1551 emit(rv_lhu(rd, off, rs), ctx); 1552 insn_len = ctx->ninsns - insns_start; 1553 break; 1554 } 1555 1556 emit_imm(RV_REG_T1, off, ctx); 1557 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1558 insns_start = ctx->ninsns; 1559 if (sign_ext) 1560 emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1561 else 1562 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1563 insn_len = ctx->ninsns - insns_start; 1564 break; 1565 case BPF_W: 1566 if (is_12b_int(off)) { 1567 insns_start = ctx->ninsns; 1568 if (sign_ext) 1569 emit(rv_lw(rd, off, rs), ctx); 1570 else 1571 emit(rv_lwu(rd, off, rs), ctx); 1572 insn_len = ctx->ninsns - insns_start; 1573 break; 1574 } 1575 1576 emit_imm(RV_REG_T1, off, ctx); 1577 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1578 insns_start = ctx->ninsns; 1579 if (sign_ext) 1580 emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1581 else 1582 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1583 insn_len = ctx->ninsns - insns_start; 1584 break; 1585 case BPF_DW: 1586 if (is_12b_int(off)) { 1587 insns_start = ctx->ninsns; 1588 emit_ld(rd, off, rs, ctx); 1589 insn_len = ctx->ninsns - insns_start; 1590 break; 1591 } 1592 1593 emit_imm(RV_REG_T1, off, ctx); 1594 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1595 insns_start = ctx->ninsns; 1596 emit_ld(rd, 0, RV_REG_T1, ctx); 1597 insn_len = ctx->ninsns - insns_start; 1598 break; 1599 } 1600 1601 ret = add_exception_handler(insn, ctx, rd, insn_len); 1602 if (ret) 1603 return ret; 1604 1605 if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1606 return 1; 1607 break; 1608 } 1609 /* speculation barrier */ 1610 case BPF_ST | BPF_NOSPEC: 1611 break; 1612 1613 /* ST: *(size *)(dst + off) = imm */ 1614 case BPF_ST | BPF_MEM | BPF_B: 1615 emit_imm(RV_REG_T1, imm, ctx); 1616 if (is_12b_int(off)) { 1617 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1618 break; 1619 } 1620 1621 emit_imm(RV_REG_T2, off, ctx); 1622 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1623 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1624 break; 1625 1626 case BPF_ST | BPF_MEM | BPF_H: 1627 emit_imm(RV_REG_T1, imm, ctx); 1628 if (is_12b_int(off)) { 1629 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1630 break; 1631 } 1632 1633 emit_imm(RV_REG_T2, off, ctx); 1634 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1635 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1636 break; 1637 case BPF_ST | BPF_MEM | BPF_W: 1638 emit_imm(RV_REG_T1, imm, ctx); 1639 if (is_12b_int(off)) { 1640 emit_sw(rd, off, RV_REG_T1, ctx); 1641 break; 1642 } 1643 1644 emit_imm(RV_REG_T2, off, ctx); 1645 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1646 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1647 break; 1648 case BPF_ST | BPF_MEM | BPF_DW: 1649 emit_imm(RV_REG_T1, imm, ctx); 1650 if (is_12b_int(off)) { 1651 emit_sd(rd, off, RV_REG_T1, ctx); 1652 break; 1653 } 1654 1655 emit_imm(RV_REG_T2, off, ctx); 1656 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1657 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1658 break; 1659 1660 /* STX: *(size *)(dst + off) = src */ 1661 case BPF_STX | BPF_MEM | BPF_B: 1662 if (is_12b_int(off)) { 1663 emit(rv_sb(rd, off, rs), ctx); 1664 break; 1665 } 1666 1667 emit_imm(RV_REG_T1, off, ctx); 1668 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1669 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1670 break; 1671 case BPF_STX | BPF_MEM | BPF_H: 1672 if (is_12b_int(off)) { 1673 emit(rv_sh(rd, off, rs), ctx); 1674 break; 1675 } 1676 1677 emit_imm(RV_REG_T1, off, ctx); 1678 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1679 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1680 break; 1681 case BPF_STX | BPF_MEM | BPF_W: 1682 if (is_12b_int(off)) { 1683 emit_sw(rd, off, rs, ctx); 1684 break; 1685 } 1686 1687 emit_imm(RV_REG_T1, off, ctx); 1688 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1689 emit_sw(RV_REG_T1, 0, rs, ctx); 1690 break; 1691 case BPF_STX | BPF_MEM | BPF_DW: 1692 if (is_12b_int(off)) { 1693 emit_sd(rd, off, rs, ctx); 1694 break; 1695 } 1696 1697 emit_imm(RV_REG_T1, off, ctx); 1698 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1699 emit_sd(RV_REG_T1, 0, rs, ctx); 1700 break; 1701 case BPF_STX | BPF_ATOMIC | BPF_W: 1702 case BPF_STX | BPF_ATOMIC | BPF_DW: 1703 emit_atomic(rd, rs, off, imm, 1704 BPF_SIZE(code) == BPF_DW, ctx); 1705 break; 1706 default: 1707 pr_err("bpf-jit: unknown opcode %02x\n", code); 1708 return -EINVAL; 1709 } 1710 1711 return 0; 1712 } 1713 1714 void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1715 { 1716 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1717 1718 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1719 if (bpf_stack_adjust) 1720 mark_fp(ctx); 1721 1722 if (seen_reg(RV_REG_RA, ctx)) 1723 stack_adjust += 8; 1724 stack_adjust += 8; /* RV_REG_FP */ 1725 if (seen_reg(RV_REG_S1, ctx)) 1726 stack_adjust += 8; 1727 if (seen_reg(RV_REG_S2, ctx)) 1728 stack_adjust += 8; 1729 if (seen_reg(RV_REG_S3, ctx)) 1730 stack_adjust += 8; 1731 if (seen_reg(RV_REG_S4, ctx)) 1732 stack_adjust += 8; 1733 if (seen_reg(RV_REG_S5, ctx)) 1734 stack_adjust += 8; 1735 if (seen_reg(RV_REG_S6, ctx)) 1736 stack_adjust += 8; 1737 1738 stack_adjust = round_up(stack_adjust, 16); 1739 stack_adjust += bpf_stack_adjust; 1740 1741 store_offset = stack_adjust - 8; 1742 1743 /* nops reserved for auipc+jalr pair */ 1744 for (i = 0; i < RV_FENTRY_NINSNS; i++) 1745 emit(rv_nop(), ctx); 1746 1747 /* First instruction is always setting the tail-call-counter 1748 * (TCC) register. This instruction is skipped for tail calls. 1749 * Force using a 4-byte (non-compressed) instruction. 1750 */ 1751 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1752 1753 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1754 1755 if (seen_reg(RV_REG_RA, ctx)) { 1756 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1757 store_offset -= 8; 1758 } 1759 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1760 store_offset -= 8; 1761 if (seen_reg(RV_REG_S1, ctx)) { 1762 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1763 store_offset -= 8; 1764 } 1765 if (seen_reg(RV_REG_S2, ctx)) { 1766 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1767 store_offset -= 8; 1768 } 1769 if (seen_reg(RV_REG_S3, ctx)) { 1770 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1771 store_offset -= 8; 1772 } 1773 if (seen_reg(RV_REG_S4, ctx)) { 1774 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1775 store_offset -= 8; 1776 } 1777 if (seen_reg(RV_REG_S5, ctx)) { 1778 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1779 store_offset -= 8; 1780 } 1781 if (seen_reg(RV_REG_S6, ctx)) { 1782 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1783 store_offset -= 8; 1784 } 1785 1786 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1787 1788 if (bpf_stack_adjust) 1789 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1790 1791 /* Program contains calls and tail calls, so RV_REG_TCC need 1792 * to be saved across calls. 1793 */ 1794 if (seen_tail_call(ctx) && seen_call(ctx)) 1795 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1796 1797 ctx->stack_size = stack_adjust; 1798 } 1799 1800 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1801 { 1802 __build_epilogue(false, ctx); 1803 } 1804 1805 bool bpf_jit_supports_kfunc_call(void) 1806 { 1807 return true; 1808 } 1809 1810 bool bpf_jit_supports_ptr_xchg(void) 1811 { 1812 return true; 1813 } 1814