1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <asm/patch.h> 14 #include <asm/cfi.h> 15 #include "bpf_jit.h" 16 17 #define RV_FENTRY_NINSNS 2 18 19 #define RV_REG_TCC RV_REG_A6 20 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 21 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ 22 23 static const int regmap[] = { 24 [BPF_REG_0] = RV_REG_A5, 25 [BPF_REG_1] = RV_REG_A0, 26 [BPF_REG_2] = RV_REG_A1, 27 [BPF_REG_3] = RV_REG_A2, 28 [BPF_REG_4] = RV_REG_A3, 29 [BPF_REG_5] = RV_REG_A4, 30 [BPF_REG_6] = RV_REG_S1, 31 [BPF_REG_7] = RV_REG_S2, 32 [BPF_REG_8] = RV_REG_S3, 33 [BPF_REG_9] = RV_REG_S4, 34 [BPF_REG_FP] = RV_REG_S5, 35 [BPF_REG_AX] = RV_REG_T0, 36 }; 37 38 static const int pt_regmap[] = { 39 [RV_REG_A0] = offsetof(struct pt_regs, a0), 40 [RV_REG_A1] = offsetof(struct pt_regs, a1), 41 [RV_REG_A2] = offsetof(struct pt_regs, a2), 42 [RV_REG_A3] = offsetof(struct pt_regs, a3), 43 [RV_REG_A4] = offsetof(struct pt_regs, a4), 44 [RV_REG_A5] = offsetof(struct pt_regs, a5), 45 [RV_REG_S1] = offsetof(struct pt_regs, s1), 46 [RV_REG_S2] = offsetof(struct pt_regs, s2), 47 [RV_REG_S3] = offsetof(struct pt_regs, s3), 48 [RV_REG_S4] = offsetof(struct pt_regs, s4), 49 [RV_REG_S5] = offsetof(struct pt_regs, s5), 50 [RV_REG_T0] = offsetof(struct pt_regs, t0), 51 }; 52 53 enum { 54 RV_CTX_F_SEEN_TAIL_CALL = 0, 55 RV_CTX_F_SEEN_CALL = RV_REG_RA, 56 RV_CTX_F_SEEN_S1 = RV_REG_S1, 57 RV_CTX_F_SEEN_S2 = RV_REG_S2, 58 RV_CTX_F_SEEN_S3 = RV_REG_S3, 59 RV_CTX_F_SEEN_S4 = RV_REG_S4, 60 RV_CTX_F_SEEN_S5 = RV_REG_S5, 61 RV_CTX_F_SEEN_S6 = RV_REG_S6, 62 }; 63 64 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 65 { 66 u8 reg = regmap[bpf_reg]; 67 68 switch (reg) { 69 case RV_CTX_F_SEEN_S1: 70 case RV_CTX_F_SEEN_S2: 71 case RV_CTX_F_SEEN_S3: 72 case RV_CTX_F_SEEN_S4: 73 case RV_CTX_F_SEEN_S5: 74 case RV_CTX_F_SEEN_S6: 75 __set_bit(reg, &ctx->flags); 76 } 77 return reg; 78 }; 79 80 static bool seen_reg(int reg, struct rv_jit_context *ctx) 81 { 82 switch (reg) { 83 case RV_CTX_F_SEEN_CALL: 84 case RV_CTX_F_SEEN_S1: 85 case RV_CTX_F_SEEN_S2: 86 case RV_CTX_F_SEEN_S3: 87 case RV_CTX_F_SEEN_S4: 88 case RV_CTX_F_SEEN_S5: 89 case RV_CTX_F_SEEN_S6: 90 return test_bit(reg, &ctx->flags); 91 } 92 return false; 93 } 94 95 static void mark_fp(struct rv_jit_context *ctx) 96 { 97 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 98 } 99 100 static void mark_call(struct rv_jit_context *ctx) 101 { 102 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 103 } 104 105 static bool seen_call(struct rv_jit_context *ctx) 106 { 107 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 108 } 109 110 static void mark_tail_call(struct rv_jit_context *ctx) 111 { 112 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 113 } 114 115 static bool seen_tail_call(struct rv_jit_context *ctx) 116 { 117 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 118 } 119 120 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 121 { 122 mark_tail_call(ctx); 123 124 if (seen_call(ctx)) { 125 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 126 return RV_REG_S6; 127 } 128 return RV_REG_A6; 129 } 130 131 static bool is_32b_int(s64 val) 132 { 133 return -(1L << 31) <= val && val < (1L << 31); 134 } 135 136 static bool in_auipc_jalr_range(s64 val) 137 { 138 /* 139 * auipc+jalr can reach any signed PC-relative offset in the range 140 * [-2^31 - 2^11, 2^31 - 2^11). 141 */ 142 return (-(1L << 31) - (1L << 11)) <= val && 143 val < ((1L << 31) - (1L << 11)); 144 } 145 146 /* Modify rd pointer to alternate reg to avoid corrupting original reg */ 147 static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 148 { 149 emit_sextw(ra, *rd, ctx); 150 *rd = ra; 151 } 152 153 static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 154 { 155 emit_zextw(ra, *rd, ctx); 156 *rd = ra; 157 } 158 159 /* Emit fixed-length instructions for address */ 160 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 161 { 162 /* 163 * Use the ro_insns(RX) to calculate the offset as the BPF program will 164 * finally run from this memory region. 165 */ 166 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); 167 s64 off = addr - ip; 168 s64 upper = (off + (1 << 11)) >> 12; 169 s64 lower = off & 0xfff; 170 171 if (extra_pass && !in_auipc_jalr_range(off)) { 172 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 173 return -ERANGE; 174 } 175 176 emit(rv_auipc(rd, upper), ctx); 177 emit(rv_addi(rd, rd, lower), ctx); 178 return 0; 179 } 180 181 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 182 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 183 { 184 /* Note that the immediate from the add is sign-extended, 185 * which means that we need to compensate this by adding 2^12, 186 * when the 12th bit is set. A simpler way of doing this, and 187 * getting rid of the check, is to just add 2**11 before the 188 * shift. The "Loading a 32-Bit constant" example from the 189 * "Computer Organization and Design, RISC-V edition" book by 190 * Patterson/Hennessy highlights this fact. 191 * 192 * This also means that we need to process LSB to MSB. 193 */ 194 s64 upper = (val + (1 << 11)) >> 12; 195 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 196 * and addi are signed and RVC checks will perform signed comparisons. 197 */ 198 s64 lower = ((val & 0xfff) << 52) >> 52; 199 int shift; 200 201 if (is_32b_int(val)) { 202 if (upper) 203 emit_lui(rd, upper, ctx); 204 205 if (!upper) { 206 emit_li(rd, lower, ctx); 207 return; 208 } 209 210 emit_addiw(rd, rd, lower, ctx); 211 return; 212 } 213 214 shift = __ffs(upper); 215 upper >>= shift; 216 shift += 12; 217 218 emit_imm(rd, upper, ctx); 219 220 emit_slli(rd, rd, shift, ctx); 221 if (lower) 222 emit_addi(rd, rd, lower, ctx); 223 } 224 225 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 226 { 227 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 228 229 if (seen_reg(RV_REG_RA, ctx)) { 230 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 231 store_offset -= 8; 232 } 233 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 234 store_offset -= 8; 235 if (seen_reg(RV_REG_S1, ctx)) { 236 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 237 store_offset -= 8; 238 } 239 if (seen_reg(RV_REG_S2, ctx)) { 240 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 241 store_offset -= 8; 242 } 243 if (seen_reg(RV_REG_S3, ctx)) { 244 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 245 store_offset -= 8; 246 } 247 if (seen_reg(RV_REG_S4, ctx)) { 248 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 249 store_offset -= 8; 250 } 251 if (seen_reg(RV_REG_S5, ctx)) { 252 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 253 store_offset -= 8; 254 } 255 if (seen_reg(RV_REG_S6, ctx)) { 256 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 257 store_offset -= 8; 258 } 259 if (ctx->arena_vm_start) { 260 emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); 261 store_offset -= 8; 262 } 263 264 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 265 /* Set return value. */ 266 if (!is_tail_call) 267 emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); 268 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 269 is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 270 ctx); 271 } 272 273 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 274 struct rv_jit_context *ctx) 275 { 276 switch (cond) { 277 case BPF_JEQ: 278 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 279 return; 280 case BPF_JGT: 281 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 282 return; 283 case BPF_JLT: 284 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 285 return; 286 case BPF_JGE: 287 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 288 return; 289 case BPF_JLE: 290 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 291 return; 292 case BPF_JNE: 293 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 294 return; 295 case BPF_JSGT: 296 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 297 return; 298 case BPF_JSLT: 299 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 300 return; 301 case BPF_JSGE: 302 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 303 return; 304 case BPF_JSLE: 305 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 306 } 307 } 308 309 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 310 struct rv_jit_context *ctx) 311 { 312 s64 upper, lower; 313 314 if (is_13b_int(rvoff)) { 315 emit_bcc(cond, rd, rs, rvoff, ctx); 316 return; 317 } 318 319 /* Adjust for jal */ 320 rvoff -= 4; 321 322 /* Transform, e.g.: 323 * bne rd,rs,foo 324 * to 325 * beq rd,rs,<.L1> 326 * (auipc foo) 327 * jal(r) foo 328 * .L1 329 */ 330 cond = invert_bpf_cond(cond); 331 if (is_21b_int(rvoff)) { 332 emit_bcc(cond, rd, rs, 8, ctx); 333 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 334 return; 335 } 336 337 /* 32b No need for an additional rvoff adjustment, since we 338 * get that from the auipc at PC', where PC = PC' + 4. 339 */ 340 upper = (rvoff + (1 << 11)) >> 12; 341 lower = rvoff & 0xfff; 342 343 emit_bcc(cond, rd, rs, 12, ctx); 344 emit(rv_auipc(RV_REG_T1, upper), ctx); 345 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 346 } 347 348 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 349 { 350 int tc_ninsn, off, start_insn = ctx->ninsns; 351 u8 tcc = rv_tail_call_reg(ctx); 352 353 /* a0: &ctx 354 * a1: &array 355 * a2: index 356 * 357 * if (index >= array->map.max_entries) 358 * goto out; 359 */ 360 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 361 ctx->offset[0]; 362 emit_zextw(RV_REG_A2, RV_REG_A2, ctx); 363 364 off = offsetof(struct bpf_array, map.max_entries); 365 if (is_12b_check(off, insn)) 366 return -1; 367 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 368 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 369 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 370 371 /* if (--TCC < 0) 372 * goto out; 373 */ 374 emit_addi(RV_REG_TCC, tcc, -1, ctx); 375 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 376 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 377 378 /* prog = array->ptrs[index]; 379 * if (!prog) 380 * goto out; 381 */ 382 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 383 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 384 off = offsetof(struct bpf_array, ptrs); 385 if (is_12b_check(off, insn)) 386 return -1; 387 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 388 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 389 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 390 391 /* goto *(prog->bpf_func + 4); */ 392 off = offsetof(struct bpf_prog, bpf_func); 393 if (is_12b_check(off, insn)) 394 return -1; 395 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 396 __build_epilogue(true, ctx); 397 return 0; 398 } 399 400 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 401 struct rv_jit_context *ctx) 402 { 403 u8 code = insn->code; 404 405 switch (code) { 406 case BPF_JMP | BPF_JA: 407 case BPF_JMP | BPF_CALL: 408 case BPF_JMP | BPF_EXIT: 409 case BPF_JMP | BPF_TAIL_CALL: 410 break; 411 default: 412 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 413 } 414 415 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 416 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 417 code & BPF_LDX || code & BPF_STX) 418 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 419 } 420 421 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 422 struct rv_jit_context *ctx) 423 { 424 s64 upper, lower; 425 426 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 427 emit(rv_jal(rd, rvoff >> 1), ctx); 428 return 0; 429 } else if (in_auipc_jalr_range(rvoff)) { 430 upper = (rvoff + (1 << 11)) >> 12; 431 lower = rvoff & 0xfff; 432 emit(rv_auipc(RV_REG_T1, upper), ctx); 433 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 434 return 0; 435 } 436 437 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 438 return -ERANGE; 439 } 440 441 static bool is_signed_bpf_cond(u8 cond) 442 { 443 return cond == BPF_JSGT || cond == BPF_JSLT || 444 cond == BPF_JSGE || cond == BPF_JSLE; 445 } 446 447 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 448 { 449 s64 off = 0; 450 u64 ip; 451 452 if (addr && ctx->insns && ctx->ro_insns) { 453 /* 454 * Use the ro_insns(RX) to calculate the offset as the BPF 455 * program will finally run from this memory region. 456 */ 457 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); 458 off = addr - ip; 459 } 460 461 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 462 } 463 464 static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) 465 { 466 if (IS_ENABLED(CONFIG_CFI_CLANG)) 467 emit(hash, ctx); 468 } 469 470 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 471 struct rv_jit_context *ctx) 472 { 473 u8 r0; 474 int jmp_offset; 475 476 if (off) { 477 if (is_12b_int(off)) { 478 emit_addi(RV_REG_T1, rd, off, ctx); 479 } else { 480 emit_imm(RV_REG_T1, off, ctx); 481 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 482 } 483 rd = RV_REG_T1; 484 } 485 486 switch (imm) { 487 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 488 case BPF_ADD: 489 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 490 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 491 break; 492 case BPF_AND: 493 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 494 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 495 break; 496 case BPF_OR: 497 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 498 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 499 break; 500 case BPF_XOR: 501 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 502 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 503 break; 504 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 505 case BPF_ADD | BPF_FETCH: 506 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : 507 rv_amoadd_w(rs, rs, rd, 0, 0), ctx); 508 if (!is64) 509 emit_zextw(rs, rs, ctx); 510 break; 511 case BPF_AND | BPF_FETCH: 512 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : 513 rv_amoand_w(rs, rs, rd, 0, 0), ctx); 514 if (!is64) 515 emit_zextw(rs, rs, ctx); 516 break; 517 case BPF_OR | BPF_FETCH: 518 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : 519 rv_amoor_w(rs, rs, rd, 0, 0), ctx); 520 if (!is64) 521 emit_zextw(rs, rs, ctx); 522 break; 523 case BPF_XOR | BPF_FETCH: 524 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : 525 rv_amoxor_w(rs, rs, rd, 0, 0), ctx); 526 if (!is64) 527 emit_zextw(rs, rs, ctx); 528 break; 529 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 530 case BPF_XCHG: 531 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : 532 rv_amoswap_w(rs, rs, rd, 0, 0), ctx); 533 if (!is64) 534 emit_zextw(rs, rs, ctx); 535 break; 536 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 537 case BPF_CMPXCHG: 538 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 539 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 540 rv_addiw(RV_REG_T2, r0, 0), ctx); 541 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 542 rv_lr_w(r0, 0, rd, 0, 0), ctx); 543 jmp_offset = ninsns_rvoff(8); 544 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 545 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 546 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 547 jmp_offset = ninsns_rvoff(-6); 548 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 549 emit(rv_fence(0x3, 0x3), ctx); 550 break; 551 } 552 } 553 554 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 555 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 556 #define REG_DONT_CLEAR_MARKER 0 /* RV_REG_ZERO unused in pt_regmap */ 557 558 bool ex_handler_bpf(const struct exception_table_entry *ex, 559 struct pt_regs *regs) 560 { 561 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 562 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 563 564 if (regs_offset != REG_DONT_CLEAR_MARKER) 565 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 566 regs->epc = (unsigned long)&ex->fixup - offset; 567 568 return true; 569 } 570 571 /* For accesses to BTF pointers, add an entry to the exception table */ 572 static int add_exception_handler(const struct bpf_insn *insn, 573 struct rv_jit_context *ctx, 574 int dst_reg, int insn_len) 575 { 576 struct exception_table_entry *ex; 577 unsigned long pc; 578 off_t ins_offset; 579 off_t fixup_offset; 580 581 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 582 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 583 BPF_MODE(insn->code) != BPF_PROBE_MEM32)) 584 return 0; 585 586 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 587 return -EINVAL; 588 589 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 590 return -EINVAL; 591 592 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 593 return -EINVAL; 594 595 ex = &ctx->prog->aux->extable[ctx->nexentries]; 596 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 597 598 /* 599 * This is the relative offset of the instruction that may fault from 600 * the exception table itself. This will be written to the exception 601 * table and if this instruction faults, the destination register will 602 * be set to '0' and the execution will jump to the next instruction. 603 */ 604 ins_offset = pc - (long)&ex->insn; 605 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 606 return -ERANGE; 607 608 /* 609 * Since the extable follows the program, the fixup offset is always 610 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 611 * to keep things simple, and put the destination register in the upper 612 * bits. We don't need to worry about buildtime or runtime sort 613 * modifying the upper bits because the table is already sorted, and 614 * isn't part of the main exception table. 615 * 616 * The fixup_offset is set to the next instruction from the instruction 617 * that may fault. The execution will jump to this after handling the 618 * fault. 619 */ 620 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 621 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 622 return -ERANGE; 623 624 /* 625 * The offsets above have been calculated using the RO buffer but we 626 * need to use the R/W buffer for writes. 627 * switch ex to rw buffer for writing. 628 */ 629 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); 630 631 ex->insn = ins_offset; 632 633 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 634 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 635 ex->type = EX_TYPE_BPF; 636 637 ctx->nexentries++; 638 return 0; 639 } 640 641 static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 642 { 643 s64 rvoff; 644 struct rv_jit_context ctx; 645 646 ctx.ninsns = 0; 647 ctx.insns = (u16 *)insns; 648 649 if (!target) { 650 emit(rv_nop(), &ctx); 651 emit(rv_nop(), &ctx); 652 return 0; 653 } 654 655 rvoff = (s64)(target - ip); 656 return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 657 } 658 659 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 660 void *old_addr, void *new_addr) 661 { 662 u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 663 bool is_call = poke_type == BPF_MOD_CALL; 664 int ret; 665 666 if (!is_kernel_text((unsigned long)ip) && 667 !is_bpf_text_address((unsigned long)ip)) 668 return -ENOTSUPP; 669 670 ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 671 if (ret) 672 return ret; 673 674 if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) 675 return -EFAULT; 676 677 ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 678 if (ret) 679 return ret; 680 681 cpus_read_lock(); 682 mutex_lock(&text_mutex); 683 if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) 684 ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); 685 mutex_unlock(&text_mutex); 686 cpus_read_unlock(); 687 688 return ret; 689 } 690 691 static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) 692 { 693 int i; 694 695 for (i = 0; i < nregs; i++) { 696 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 697 args_off -= 8; 698 } 699 } 700 701 static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) 702 { 703 int i; 704 705 for (i = 0; i < nregs; i++) { 706 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 707 args_off -= 8; 708 } 709 } 710 711 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 712 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 713 { 714 int ret, branch_off; 715 struct bpf_prog *p = l->link.prog; 716 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 717 718 if (l->cookie) { 719 emit_imm(RV_REG_T1, l->cookie, ctx); 720 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 721 } else { 722 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 723 } 724 725 /* arg1: prog */ 726 emit_imm(RV_REG_A0, (const s64)p, ctx); 727 /* arg2: &run_ctx */ 728 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 729 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 730 if (ret) 731 return ret; 732 733 /* store prog start time */ 734 emit_mv(RV_REG_S1, RV_REG_A0, ctx); 735 736 /* if (__bpf_prog_enter(prog) == 0) 737 * goto skip_exec_of_prog; 738 */ 739 branch_off = ctx->ninsns; 740 /* nop reserved for conditional jump */ 741 emit(rv_nop(), ctx); 742 743 /* arg1: &args_off */ 744 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 745 if (!p->jited) 746 /* arg2: progs[i]->insnsi for interpreter */ 747 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 748 ret = emit_call((const u64)p->bpf_func, true, ctx); 749 if (ret) 750 return ret; 751 752 if (save_ret) { 753 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 754 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 755 } 756 757 /* update branch with beqz */ 758 if (ctx->insns) { 759 int offset = ninsns_rvoff(ctx->ninsns - branch_off); 760 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 761 *(u32 *)(ctx->insns + branch_off) = insn; 762 } 763 764 /* arg1: prog */ 765 emit_imm(RV_REG_A0, (const s64)p, ctx); 766 /* arg2: prog start time */ 767 emit_mv(RV_REG_A1, RV_REG_S1, ctx); 768 /* arg3: &run_ctx */ 769 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 770 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 771 772 return ret; 773 } 774 775 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 776 const struct btf_func_model *m, 777 struct bpf_tramp_links *tlinks, 778 void *func_addr, u32 flags, 779 struct rv_jit_context *ctx) 780 { 781 int i, ret, offset; 782 int *branches_off = NULL; 783 int stack_size = 0, nregs = m->nr_args; 784 int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; 785 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 786 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 787 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 788 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 789 void *orig_call = func_addr; 790 bool save_ret; 791 u32 insn; 792 793 /* Two types of generated trampoline stack layout: 794 * 795 * 1. trampoline called from function entry 796 * -------------------------------------- 797 * FP + 8 [ RA to parent func ] return address to parent 798 * function 799 * FP + 0 [ FP of parent func ] frame pointer of parent 800 * function 801 * FP - 8 [ T0 to traced func ] return address of traced 802 * function 803 * FP - 16 [ FP of traced func ] frame pointer of traced 804 * function 805 * -------------------------------------- 806 * 807 * 2. trampoline called directly 808 * -------------------------------------- 809 * FP - 8 [ RA to caller func ] return address to caller 810 * function 811 * FP - 16 [ FP of caller func ] frame pointer of caller 812 * function 813 * -------------------------------------- 814 * 815 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 816 * BPF_TRAMP_F_RET_FENTRY_RET 817 * [ argN ] 818 * [ ... ] 819 * FP - args_off [ arg1 ] 820 * 821 * FP - nregs_off [ regs count ] 822 * 823 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 824 * 825 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 826 * 827 * FP - sreg_off [ callee saved reg ] 828 * 829 * [ pads ] pads for 16 bytes alignment 830 */ 831 832 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 833 return -ENOTSUPP; 834 835 /* extra regiters for struct arguments */ 836 for (i = 0; i < m->nr_args; i++) 837 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 838 nregs += round_up(m->arg_size[i], 8) / 8 - 1; 839 840 /* 8 arguments passed by registers */ 841 if (nregs > 8) 842 return -ENOTSUPP; 843 844 /* room of trampoline frame to store return address and frame pointer */ 845 stack_size += 16; 846 847 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 848 if (save_ret) { 849 stack_size += 16; /* Save both A5 (BPF R0) and A0 */ 850 retval_off = stack_size; 851 } 852 853 stack_size += nregs * 8; 854 args_off = stack_size; 855 856 stack_size += 8; 857 nregs_off = stack_size; 858 859 if (flags & BPF_TRAMP_F_IP_ARG) { 860 stack_size += 8; 861 ip_off = stack_size; 862 } 863 864 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 865 run_ctx_off = stack_size; 866 867 stack_size += 8; 868 sreg_off = stack_size; 869 870 stack_size = round_up(stack_size, 16); 871 872 if (!is_struct_ops) { 873 /* For the trampoline called from function entry, 874 * the frame of traced function and the frame of 875 * trampoline need to be considered. 876 */ 877 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 878 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 879 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 880 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 881 882 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 883 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 884 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 885 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 886 } else { 887 /* emit kcfi hash */ 888 emit_kcfi(cfi_get_func_hash(func_addr), ctx); 889 /* For the trampoline called directly, just handle 890 * the frame of trampoline. 891 */ 892 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 893 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 894 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 895 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 896 } 897 898 /* callee saved register S1 to pass start time */ 899 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 900 901 /* store ip address of the traced function */ 902 if (flags & BPF_TRAMP_F_IP_ARG) { 903 emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 904 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 905 } 906 907 emit_li(RV_REG_T1, nregs, ctx); 908 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 909 910 store_args(nregs, args_off, ctx); 911 912 /* skip to actual body of traced function */ 913 if (flags & BPF_TRAMP_F_SKIP_FRAME) 914 orig_call += RV_FENTRY_NINSNS * 4; 915 916 if (flags & BPF_TRAMP_F_CALL_ORIG) { 917 emit_imm(RV_REG_A0, (const s64)im, ctx); 918 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 919 if (ret) 920 return ret; 921 } 922 923 for (i = 0; i < fentry->nr_links; i++) { 924 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 925 flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 926 if (ret) 927 return ret; 928 } 929 930 if (fmod_ret->nr_links) { 931 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 932 if (!branches_off) 933 return -ENOMEM; 934 935 /* cleanup to avoid garbage return value confusion */ 936 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 937 for (i = 0; i < fmod_ret->nr_links; i++) { 938 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 939 run_ctx_off, true, ctx); 940 if (ret) 941 goto out; 942 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 943 branches_off[i] = ctx->ninsns; 944 /* nop reserved for conditional jump */ 945 emit(rv_nop(), ctx); 946 } 947 } 948 949 if (flags & BPF_TRAMP_F_CALL_ORIG) { 950 restore_args(nregs, args_off, ctx); 951 ret = emit_call((const u64)orig_call, true, ctx); 952 if (ret) 953 goto out; 954 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 955 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 956 im->ip_after_call = ctx->insns + ctx->ninsns; 957 /* 2 nops reserved for auipc+jalr pair */ 958 emit(rv_nop(), ctx); 959 emit(rv_nop(), ctx); 960 } 961 962 /* update branches saved in invoke_bpf_mod_ret with bnez */ 963 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 964 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 965 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 966 *(u32 *)(ctx->insns + branches_off[i]) = insn; 967 } 968 969 for (i = 0; i < fexit->nr_links; i++) { 970 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 971 run_ctx_off, false, ctx); 972 if (ret) 973 goto out; 974 } 975 976 if (flags & BPF_TRAMP_F_CALL_ORIG) { 977 im->ip_epilogue = ctx->insns + ctx->ninsns; 978 emit_imm(RV_REG_A0, (const s64)im, ctx); 979 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 980 if (ret) 981 goto out; 982 } 983 984 if (flags & BPF_TRAMP_F_RESTORE_REGS) 985 restore_args(nregs, args_off, ctx); 986 987 if (save_ret) { 988 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 989 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); 990 } 991 992 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 993 994 if (!is_struct_ops) { 995 /* trampoline called from function entry */ 996 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 997 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 998 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 999 1000 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 1001 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 1002 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 1003 1004 if (flags & BPF_TRAMP_F_SKIP_FRAME) 1005 /* return to parent function */ 1006 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1007 else 1008 /* return to traced function */ 1009 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 1010 } else { 1011 /* trampoline called directly */ 1012 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 1013 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1014 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1015 1016 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1017 } 1018 1019 ret = ctx->ninsns; 1020 out: 1021 kfree(branches_off); 1022 return ret; 1023 } 1024 1025 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1026 struct bpf_tramp_links *tlinks, void *func_addr) 1027 { 1028 struct bpf_tramp_image im; 1029 struct rv_jit_context ctx; 1030 int ret; 1031 1032 ctx.ninsns = 0; 1033 ctx.insns = NULL; 1034 ctx.ro_insns = NULL; 1035 ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); 1036 1037 return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); 1038 } 1039 1040 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 1041 void *image_end, const struct btf_func_model *m, 1042 u32 flags, struct bpf_tramp_links *tlinks, 1043 void *func_addr) 1044 { 1045 int ret; 1046 struct rv_jit_context ctx; 1047 1048 ctx.ninsns = 0; 1049 /* 1050 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the 1051 * JITed instructions and later copies it to a RX region (ctx.ro_insns). 1052 * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the 1053 * trampoline image uses the same memory area for writing and execution, 1054 * both ctx.insns and ctx.ro_insns can be set to image. 1055 */ 1056 ctx.insns = image; 1057 ctx.ro_insns = image; 1058 ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 1059 if (ret < 0) 1060 return ret; 1061 1062 bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); 1063 1064 return ninsns_rvoff(ret); 1065 } 1066 1067 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1068 bool extra_pass) 1069 { 1070 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1071 BPF_CLASS(insn->code) == BPF_JMP; 1072 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1073 struct bpf_prog_aux *aux = ctx->prog->aux; 1074 u8 rd = -1, rs = -1, code = insn->code; 1075 s16 off = insn->off; 1076 s32 imm = insn->imm; 1077 1078 init_regs(&rd, &rs, insn, ctx); 1079 1080 switch (code) { 1081 /* dst = src */ 1082 case BPF_ALU | BPF_MOV | BPF_X: 1083 case BPF_ALU64 | BPF_MOV | BPF_X: 1084 if (insn_is_cast_user(insn)) { 1085 emit_mv(RV_REG_T1, rs, ctx); 1086 emit_zextw(RV_REG_T1, RV_REG_T1, ctx); 1087 emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx); 1088 emit(rv_beq(RV_REG_T1, RV_REG_ZERO, 4), ctx); 1089 emit_or(RV_REG_T1, rd, RV_REG_T1, ctx); 1090 emit_mv(rd, RV_REG_T1, ctx); 1091 break; 1092 } 1093 if (imm == 1) { 1094 /* Special mov32 for zext */ 1095 emit_zextw(rd, rd, ctx); 1096 break; 1097 } 1098 switch (insn->off) { 1099 case 0: 1100 emit_mv(rd, rs, ctx); 1101 break; 1102 case 8: 1103 emit_sextb(rd, rs, ctx); 1104 break; 1105 case 16: 1106 emit_sexth(rd, rs, ctx); 1107 break; 1108 case 32: 1109 emit_sextw(rd, rs, ctx); 1110 break; 1111 } 1112 if (!is64 && !aux->verifier_zext) 1113 emit_zextw(rd, rd, ctx); 1114 break; 1115 1116 /* dst = dst OP src */ 1117 case BPF_ALU | BPF_ADD | BPF_X: 1118 case BPF_ALU64 | BPF_ADD | BPF_X: 1119 emit_add(rd, rd, rs, ctx); 1120 if (!is64 && !aux->verifier_zext) 1121 emit_zextw(rd, rd, ctx); 1122 break; 1123 case BPF_ALU | BPF_SUB | BPF_X: 1124 case BPF_ALU64 | BPF_SUB | BPF_X: 1125 if (is64) 1126 emit_sub(rd, rd, rs, ctx); 1127 else 1128 emit_subw(rd, rd, rs, ctx); 1129 1130 if (!is64 && !aux->verifier_zext) 1131 emit_zextw(rd, rd, ctx); 1132 break; 1133 case BPF_ALU | BPF_AND | BPF_X: 1134 case BPF_ALU64 | BPF_AND | BPF_X: 1135 emit_and(rd, rd, rs, ctx); 1136 if (!is64 && !aux->verifier_zext) 1137 emit_zextw(rd, rd, ctx); 1138 break; 1139 case BPF_ALU | BPF_OR | BPF_X: 1140 case BPF_ALU64 | BPF_OR | BPF_X: 1141 emit_or(rd, rd, rs, ctx); 1142 if (!is64 && !aux->verifier_zext) 1143 emit_zextw(rd, rd, ctx); 1144 break; 1145 case BPF_ALU | BPF_XOR | BPF_X: 1146 case BPF_ALU64 | BPF_XOR | BPF_X: 1147 emit_xor(rd, rd, rs, ctx); 1148 if (!is64 && !aux->verifier_zext) 1149 emit_zextw(rd, rd, ctx); 1150 break; 1151 case BPF_ALU | BPF_MUL | BPF_X: 1152 case BPF_ALU64 | BPF_MUL | BPF_X: 1153 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1154 if (!is64 && !aux->verifier_zext) 1155 emit_zextw(rd, rd, ctx); 1156 break; 1157 case BPF_ALU | BPF_DIV | BPF_X: 1158 case BPF_ALU64 | BPF_DIV | BPF_X: 1159 if (off) 1160 emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1161 else 1162 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1163 if (!is64 && !aux->verifier_zext) 1164 emit_zextw(rd, rd, ctx); 1165 break; 1166 case BPF_ALU | BPF_MOD | BPF_X: 1167 case BPF_ALU64 | BPF_MOD | BPF_X: 1168 if (off) 1169 emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1170 else 1171 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1172 if (!is64 && !aux->verifier_zext) 1173 emit_zextw(rd, rd, ctx); 1174 break; 1175 case BPF_ALU | BPF_LSH | BPF_X: 1176 case BPF_ALU64 | BPF_LSH | BPF_X: 1177 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 1178 if (!is64 && !aux->verifier_zext) 1179 emit_zextw(rd, rd, ctx); 1180 break; 1181 case BPF_ALU | BPF_RSH | BPF_X: 1182 case BPF_ALU64 | BPF_RSH | BPF_X: 1183 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1184 if (!is64 && !aux->verifier_zext) 1185 emit_zextw(rd, rd, ctx); 1186 break; 1187 case BPF_ALU | BPF_ARSH | BPF_X: 1188 case BPF_ALU64 | BPF_ARSH | BPF_X: 1189 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1190 if (!is64 && !aux->verifier_zext) 1191 emit_zextw(rd, rd, ctx); 1192 break; 1193 1194 /* dst = -dst */ 1195 case BPF_ALU | BPF_NEG: 1196 case BPF_ALU64 | BPF_NEG: 1197 emit_sub(rd, RV_REG_ZERO, rd, ctx); 1198 if (!is64 && !aux->verifier_zext) 1199 emit_zextw(rd, rd, ctx); 1200 break; 1201 1202 /* dst = BSWAP##imm(dst) */ 1203 case BPF_ALU | BPF_END | BPF_FROM_LE: 1204 switch (imm) { 1205 case 16: 1206 emit_zexth(rd, rd, ctx); 1207 break; 1208 case 32: 1209 if (!aux->verifier_zext) 1210 emit_zextw(rd, rd, ctx); 1211 break; 1212 case 64: 1213 /* Do nothing */ 1214 break; 1215 } 1216 break; 1217 case BPF_ALU | BPF_END | BPF_FROM_BE: 1218 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1219 emit_bswap(rd, imm, ctx); 1220 break; 1221 1222 /* dst = imm */ 1223 case BPF_ALU | BPF_MOV | BPF_K: 1224 case BPF_ALU64 | BPF_MOV | BPF_K: 1225 emit_imm(rd, imm, ctx); 1226 if (!is64 && !aux->verifier_zext) 1227 emit_zextw(rd, rd, ctx); 1228 break; 1229 1230 /* dst = dst OP imm */ 1231 case BPF_ALU | BPF_ADD | BPF_K: 1232 case BPF_ALU64 | BPF_ADD | BPF_K: 1233 if (is_12b_int(imm)) { 1234 emit_addi(rd, rd, imm, ctx); 1235 } else { 1236 emit_imm(RV_REG_T1, imm, ctx); 1237 emit_add(rd, rd, RV_REG_T1, ctx); 1238 } 1239 if (!is64 && !aux->verifier_zext) 1240 emit_zextw(rd, rd, ctx); 1241 break; 1242 case BPF_ALU | BPF_SUB | BPF_K: 1243 case BPF_ALU64 | BPF_SUB | BPF_K: 1244 if (is_12b_int(-imm)) { 1245 emit_addi(rd, rd, -imm, ctx); 1246 } else { 1247 emit_imm(RV_REG_T1, imm, ctx); 1248 emit_sub(rd, rd, RV_REG_T1, ctx); 1249 } 1250 if (!is64 && !aux->verifier_zext) 1251 emit_zextw(rd, rd, ctx); 1252 break; 1253 case BPF_ALU | BPF_AND | BPF_K: 1254 case BPF_ALU64 | BPF_AND | BPF_K: 1255 if (is_12b_int(imm)) { 1256 emit_andi(rd, rd, imm, ctx); 1257 } else { 1258 emit_imm(RV_REG_T1, imm, ctx); 1259 emit_and(rd, rd, RV_REG_T1, ctx); 1260 } 1261 if (!is64 && !aux->verifier_zext) 1262 emit_zextw(rd, rd, ctx); 1263 break; 1264 case BPF_ALU | BPF_OR | BPF_K: 1265 case BPF_ALU64 | BPF_OR | BPF_K: 1266 if (is_12b_int(imm)) { 1267 emit(rv_ori(rd, rd, imm), ctx); 1268 } else { 1269 emit_imm(RV_REG_T1, imm, ctx); 1270 emit_or(rd, rd, RV_REG_T1, ctx); 1271 } 1272 if (!is64 && !aux->verifier_zext) 1273 emit_zextw(rd, rd, ctx); 1274 break; 1275 case BPF_ALU | BPF_XOR | BPF_K: 1276 case BPF_ALU64 | BPF_XOR | BPF_K: 1277 if (is_12b_int(imm)) { 1278 emit(rv_xori(rd, rd, imm), ctx); 1279 } else { 1280 emit_imm(RV_REG_T1, imm, ctx); 1281 emit_xor(rd, rd, RV_REG_T1, ctx); 1282 } 1283 if (!is64 && !aux->verifier_zext) 1284 emit_zextw(rd, rd, ctx); 1285 break; 1286 case BPF_ALU | BPF_MUL | BPF_K: 1287 case BPF_ALU64 | BPF_MUL | BPF_K: 1288 emit_imm(RV_REG_T1, imm, ctx); 1289 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1290 rv_mulw(rd, rd, RV_REG_T1), ctx); 1291 if (!is64 && !aux->verifier_zext) 1292 emit_zextw(rd, rd, ctx); 1293 break; 1294 case BPF_ALU | BPF_DIV | BPF_K: 1295 case BPF_ALU64 | BPF_DIV | BPF_K: 1296 emit_imm(RV_REG_T1, imm, ctx); 1297 if (off) 1298 emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1299 rv_divw(rd, rd, RV_REG_T1), ctx); 1300 else 1301 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1302 rv_divuw(rd, rd, RV_REG_T1), ctx); 1303 if (!is64 && !aux->verifier_zext) 1304 emit_zextw(rd, rd, ctx); 1305 break; 1306 case BPF_ALU | BPF_MOD | BPF_K: 1307 case BPF_ALU64 | BPF_MOD | BPF_K: 1308 emit_imm(RV_REG_T1, imm, ctx); 1309 if (off) 1310 emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1311 rv_remw(rd, rd, RV_REG_T1), ctx); 1312 else 1313 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1314 rv_remuw(rd, rd, RV_REG_T1), ctx); 1315 if (!is64 && !aux->verifier_zext) 1316 emit_zextw(rd, rd, ctx); 1317 break; 1318 case BPF_ALU | BPF_LSH | BPF_K: 1319 case BPF_ALU64 | BPF_LSH | BPF_K: 1320 emit_slli(rd, rd, imm, ctx); 1321 1322 if (!is64 && !aux->verifier_zext) 1323 emit_zextw(rd, rd, ctx); 1324 break; 1325 case BPF_ALU | BPF_RSH | BPF_K: 1326 case BPF_ALU64 | BPF_RSH | BPF_K: 1327 if (is64) 1328 emit_srli(rd, rd, imm, ctx); 1329 else 1330 emit(rv_srliw(rd, rd, imm), ctx); 1331 1332 if (!is64 && !aux->verifier_zext) 1333 emit_zextw(rd, rd, ctx); 1334 break; 1335 case BPF_ALU | BPF_ARSH | BPF_K: 1336 case BPF_ALU64 | BPF_ARSH | BPF_K: 1337 if (is64) 1338 emit_srai(rd, rd, imm, ctx); 1339 else 1340 emit(rv_sraiw(rd, rd, imm), ctx); 1341 1342 if (!is64 && !aux->verifier_zext) 1343 emit_zextw(rd, rd, ctx); 1344 break; 1345 1346 /* JUMP off */ 1347 case BPF_JMP | BPF_JA: 1348 case BPF_JMP32 | BPF_JA: 1349 if (BPF_CLASS(code) == BPF_JMP) 1350 rvoff = rv_offset(i, off, ctx); 1351 else 1352 rvoff = rv_offset(i, imm, ctx); 1353 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1354 if (ret) 1355 return ret; 1356 break; 1357 1358 /* IF (dst COND src) JUMP off */ 1359 case BPF_JMP | BPF_JEQ | BPF_X: 1360 case BPF_JMP32 | BPF_JEQ | BPF_X: 1361 case BPF_JMP | BPF_JGT | BPF_X: 1362 case BPF_JMP32 | BPF_JGT | BPF_X: 1363 case BPF_JMP | BPF_JLT | BPF_X: 1364 case BPF_JMP32 | BPF_JLT | BPF_X: 1365 case BPF_JMP | BPF_JGE | BPF_X: 1366 case BPF_JMP32 | BPF_JGE | BPF_X: 1367 case BPF_JMP | BPF_JLE | BPF_X: 1368 case BPF_JMP32 | BPF_JLE | BPF_X: 1369 case BPF_JMP | BPF_JNE | BPF_X: 1370 case BPF_JMP32 | BPF_JNE | BPF_X: 1371 case BPF_JMP | BPF_JSGT | BPF_X: 1372 case BPF_JMP32 | BPF_JSGT | BPF_X: 1373 case BPF_JMP | BPF_JSLT | BPF_X: 1374 case BPF_JMP32 | BPF_JSLT | BPF_X: 1375 case BPF_JMP | BPF_JSGE | BPF_X: 1376 case BPF_JMP32 | BPF_JSGE | BPF_X: 1377 case BPF_JMP | BPF_JSLE | BPF_X: 1378 case BPF_JMP32 | BPF_JSLE | BPF_X: 1379 case BPF_JMP | BPF_JSET | BPF_X: 1380 case BPF_JMP32 | BPF_JSET | BPF_X: 1381 rvoff = rv_offset(i, off, ctx); 1382 if (!is64) { 1383 s = ctx->ninsns; 1384 if (is_signed_bpf_cond(BPF_OP(code))) { 1385 emit_sextw_alt(&rs, RV_REG_T1, ctx); 1386 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1387 } else { 1388 emit_zextw_alt(&rs, RV_REG_T1, ctx); 1389 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1390 } 1391 e = ctx->ninsns; 1392 1393 /* Adjust for extra insns */ 1394 rvoff -= ninsns_rvoff(e - s); 1395 } 1396 1397 if (BPF_OP(code) == BPF_JSET) { 1398 /* Adjust for and */ 1399 rvoff -= 4; 1400 emit_and(RV_REG_T1, rd, rs, ctx); 1401 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1402 } else { 1403 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1404 } 1405 break; 1406 1407 /* IF (dst COND imm) JUMP off */ 1408 case BPF_JMP | BPF_JEQ | BPF_K: 1409 case BPF_JMP32 | BPF_JEQ | BPF_K: 1410 case BPF_JMP | BPF_JGT | BPF_K: 1411 case BPF_JMP32 | BPF_JGT | BPF_K: 1412 case BPF_JMP | BPF_JLT | BPF_K: 1413 case BPF_JMP32 | BPF_JLT | BPF_K: 1414 case BPF_JMP | BPF_JGE | BPF_K: 1415 case BPF_JMP32 | BPF_JGE | BPF_K: 1416 case BPF_JMP | BPF_JLE | BPF_K: 1417 case BPF_JMP32 | BPF_JLE | BPF_K: 1418 case BPF_JMP | BPF_JNE | BPF_K: 1419 case BPF_JMP32 | BPF_JNE | BPF_K: 1420 case BPF_JMP | BPF_JSGT | BPF_K: 1421 case BPF_JMP32 | BPF_JSGT | BPF_K: 1422 case BPF_JMP | BPF_JSLT | BPF_K: 1423 case BPF_JMP32 | BPF_JSLT | BPF_K: 1424 case BPF_JMP | BPF_JSGE | BPF_K: 1425 case BPF_JMP32 | BPF_JSGE | BPF_K: 1426 case BPF_JMP | BPF_JSLE | BPF_K: 1427 case BPF_JMP32 | BPF_JSLE | BPF_K: 1428 rvoff = rv_offset(i, off, ctx); 1429 s = ctx->ninsns; 1430 if (imm) 1431 emit_imm(RV_REG_T1, imm, ctx); 1432 rs = imm ? RV_REG_T1 : RV_REG_ZERO; 1433 if (!is64) { 1434 if (is_signed_bpf_cond(BPF_OP(code))) { 1435 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1436 /* rs has been sign extended */ 1437 } else { 1438 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1439 if (imm) 1440 emit_zextw(rs, rs, ctx); 1441 } 1442 } 1443 e = ctx->ninsns; 1444 1445 /* Adjust for extra insns */ 1446 rvoff -= ninsns_rvoff(e - s); 1447 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1448 break; 1449 1450 case BPF_JMP | BPF_JSET | BPF_K: 1451 case BPF_JMP32 | BPF_JSET | BPF_K: 1452 rvoff = rv_offset(i, off, ctx); 1453 s = ctx->ninsns; 1454 if (is_12b_int(imm)) { 1455 emit_andi(RV_REG_T1, rd, imm, ctx); 1456 } else { 1457 emit_imm(RV_REG_T1, imm, ctx); 1458 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1459 } 1460 /* For jset32, we should clear the upper 32 bits of t1, but 1461 * sign-extension is sufficient here and saves one instruction, 1462 * as t1 is used only in comparison against zero. 1463 */ 1464 if (!is64 && imm < 0) 1465 emit_sextw(RV_REG_T1, RV_REG_T1, ctx); 1466 e = ctx->ninsns; 1467 rvoff -= ninsns_rvoff(e - s); 1468 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1469 break; 1470 1471 /* function call */ 1472 case BPF_JMP | BPF_CALL: 1473 { 1474 bool fixed_addr; 1475 u64 addr; 1476 1477 mark_call(ctx); 1478 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1479 &addr, &fixed_addr); 1480 if (ret < 0) 1481 return ret; 1482 1483 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1484 const struct btf_func_model *fm; 1485 int idx; 1486 1487 fm = bpf_jit_find_kfunc_model(ctx->prog, insn); 1488 if (!fm) 1489 return -EINVAL; 1490 1491 for (idx = 0; idx < fm->nr_args; idx++) { 1492 u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); 1493 1494 if (fm->arg_size[idx] == sizeof(int)) 1495 emit_sextw(reg, reg, ctx); 1496 } 1497 } 1498 1499 ret = emit_call(addr, fixed_addr, ctx); 1500 if (ret) 1501 return ret; 1502 1503 if (insn->src_reg != BPF_PSEUDO_CALL) 1504 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1505 break; 1506 } 1507 /* tail call */ 1508 case BPF_JMP | BPF_TAIL_CALL: 1509 if (emit_bpf_tail_call(i, ctx)) 1510 return -1; 1511 break; 1512 1513 /* function return */ 1514 case BPF_JMP | BPF_EXIT: 1515 if (i == ctx->prog->len - 1) 1516 break; 1517 1518 rvoff = epilogue_offset(ctx); 1519 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1520 if (ret) 1521 return ret; 1522 break; 1523 1524 /* dst = imm64 */ 1525 case BPF_LD | BPF_IMM | BPF_DW: 1526 { 1527 struct bpf_insn insn1 = insn[1]; 1528 u64 imm64; 1529 1530 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1531 if (bpf_pseudo_func(insn)) { 1532 /* fixed-length insns for extra jit pass */ 1533 ret = emit_addr(rd, imm64, extra_pass, ctx); 1534 if (ret) 1535 return ret; 1536 } else { 1537 emit_imm(rd, imm64, ctx); 1538 } 1539 1540 return 1; 1541 } 1542 1543 /* LDX: dst = *(unsigned size *)(src + off) */ 1544 case BPF_LDX | BPF_MEM | BPF_B: 1545 case BPF_LDX | BPF_MEM | BPF_H: 1546 case BPF_LDX | BPF_MEM | BPF_W: 1547 case BPF_LDX | BPF_MEM | BPF_DW: 1548 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1549 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1550 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1551 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1552 /* LDSX: dst = *(signed size *)(src + off) */ 1553 case BPF_LDX | BPF_MEMSX | BPF_B: 1554 case BPF_LDX | BPF_MEMSX | BPF_H: 1555 case BPF_LDX | BPF_MEMSX | BPF_W: 1556 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1557 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1558 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1559 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + RV_REG_ARENA + off) */ 1560 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1561 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1562 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1563 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1564 { 1565 int insn_len, insns_start; 1566 bool sign_ext; 1567 1568 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1569 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1570 1571 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1572 emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); 1573 rs = RV_REG_T2; 1574 } 1575 1576 switch (BPF_SIZE(code)) { 1577 case BPF_B: 1578 if (is_12b_int(off)) { 1579 insns_start = ctx->ninsns; 1580 if (sign_ext) 1581 emit(rv_lb(rd, off, rs), ctx); 1582 else 1583 emit(rv_lbu(rd, off, rs), ctx); 1584 insn_len = ctx->ninsns - insns_start; 1585 break; 1586 } 1587 1588 emit_imm(RV_REG_T1, off, ctx); 1589 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1590 insns_start = ctx->ninsns; 1591 if (sign_ext) 1592 emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1593 else 1594 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1595 insn_len = ctx->ninsns - insns_start; 1596 break; 1597 case BPF_H: 1598 if (is_12b_int(off)) { 1599 insns_start = ctx->ninsns; 1600 if (sign_ext) 1601 emit(rv_lh(rd, off, rs), ctx); 1602 else 1603 emit(rv_lhu(rd, off, rs), ctx); 1604 insn_len = ctx->ninsns - insns_start; 1605 break; 1606 } 1607 1608 emit_imm(RV_REG_T1, off, ctx); 1609 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1610 insns_start = ctx->ninsns; 1611 if (sign_ext) 1612 emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1613 else 1614 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1615 insn_len = ctx->ninsns - insns_start; 1616 break; 1617 case BPF_W: 1618 if (is_12b_int(off)) { 1619 insns_start = ctx->ninsns; 1620 if (sign_ext) 1621 emit(rv_lw(rd, off, rs), ctx); 1622 else 1623 emit(rv_lwu(rd, off, rs), ctx); 1624 insn_len = ctx->ninsns - insns_start; 1625 break; 1626 } 1627 1628 emit_imm(RV_REG_T1, off, ctx); 1629 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1630 insns_start = ctx->ninsns; 1631 if (sign_ext) 1632 emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1633 else 1634 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1635 insn_len = ctx->ninsns - insns_start; 1636 break; 1637 case BPF_DW: 1638 if (is_12b_int(off)) { 1639 insns_start = ctx->ninsns; 1640 emit_ld(rd, off, rs, ctx); 1641 insn_len = ctx->ninsns - insns_start; 1642 break; 1643 } 1644 1645 emit_imm(RV_REG_T1, off, ctx); 1646 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1647 insns_start = ctx->ninsns; 1648 emit_ld(rd, 0, RV_REG_T1, ctx); 1649 insn_len = ctx->ninsns - insns_start; 1650 break; 1651 } 1652 1653 ret = add_exception_handler(insn, ctx, rd, insn_len); 1654 if (ret) 1655 return ret; 1656 1657 if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1658 return 1; 1659 break; 1660 } 1661 /* speculation barrier */ 1662 case BPF_ST | BPF_NOSPEC: 1663 break; 1664 1665 /* ST: *(size *)(dst + off) = imm */ 1666 case BPF_ST | BPF_MEM | BPF_B: 1667 emit_imm(RV_REG_T1, imm, ctx); 1668 if (is_12b_int(off)) { 1669 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1670 break; 1671 } 1672 1673 emit_imm(RV_REG_T2, off, ctx); 1674 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1675 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1676 break; 1677 1678 case BPF_ST | BPF_MEM | BPF_H: 1679 emit_imm(RV_REG_T1, imm, ctx); 1680 if (is_12b_int(off)) { 1681 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1682 break; 1683 } 1684 1685 emit_imm(RV_REG_T2, off, ctx); 1686 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1687 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1688 break; 1689 case BPF_ST | BPF_MEM | BPF_W: 1690 emit_imm(RV_REG_T1, imm, ctx); 1691 if (is_12b_int(off)) { 1692 emit_sw(rd, off, RV_REG_T1, ctx); 1693 break; 1694 } 1695 1696 emit_imm(RV_REG_T2, off, ctx); 1697 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1698 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1699 break; 1700 case BPF_ST | BPF_MEM | BPF_DW: 1701 emit_imm(RV_REG_T1, imm, ctx); 1702 if (is_12b_int(off)) { 1703 emit_sd(rd, off, RV_REG_T1, ctx); 1704 break; 1705 } 1706 1707 emit_imm(RV_REG_T2, off, ctx); 1708 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1709 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1710 break; 1711 1712 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1713 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1714 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1715 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1716 { 1717 int insn_len, insns_start; 1718 1719 emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); 1720 rd = RV_REG_T3; 1721 1722 /* Load imm to a register then store it */ 1723 emit_imm(RV_REG_T1, imm, ctx); 1724 1725 switch (BPF_SIZE(code)) { 1726 case BPF_B: 1727 if (is_12b_int(off)) { 1728 insns_start = ctx->ninsns; 1729 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1730 insn_len = ctx->ninsns - insns_start; 1731 break; 1732 } 1733 1734 emit_imm(RV_REG_T2, off, ctx); 1735 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1736 insns_start = ctx->ninsns; 1737 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1738 insn_len = ctx->ninsns - insns_start; 1739 break; 1740 case BPF_H: 1741 if (is_12b_int(off)) { 1742 insns_start = ctx->ninsns; 1743 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1744 insn_len = ctx->ninsns - insns_start; 1745 break; 1746 } 1747 1748 emit_imm(RV_REG_T2, off, ctx); 1749 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1750 insns_start = ctx->ninsns; 1751 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1752 insn_len = ctx->ninsns - insns_start; 1753 break; 1754 case BPF_W: 1755 if (is_12b_int(off)) { 1756 insns_start = ctx->ninsns; 1757 emit_sw(rd, off, RV_REG_T1, ctx); 1758 insn_len = ctx->ninsns - insns_start; 1759 break; 1760 } 1761 1762 emit_imm(RV_REG_T2, off, ctx); 1763 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1764 insns_start = ctx->ninsns; 1765 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1766 insn_len = ctx->ninsns - insns_start; 1767 break; 1768 case BPF_DW: 1769 if (is_12b_int(off)) { 1770 insns_start = ctx->ninsns; 1771 emit_sd(rd, off, RV_REG_T1, ctx); 1772 insn_len = ctx->ninsns - insns_start; 1773 break; 1774 } 1775 1776 emit_imm(RV_REG_T2, off, ctx); 1777 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1778 insns_start = ctx->ninsns; 1779 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1780 insn_len = ctx->ninsns - insns_start; 1781 break; 1782 } 1783 1784 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1785 insn_len); 1786 if (ret) 1787 return ret; 1788 1789 break; 1790 } 1791 1792 /* STX: *(size *)(dst + off) = src */ 1793 case BPF_STX | BPF_MEM | BPF_B: 1794 if (is_12b_int(off)) { 1795 emit(rv_sb(rd, off, rs), ctx); 1796 break; 1797 } 1798 1799 emit_imm(RV_REG_T1, off, ctx); 1800 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1801 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1802 break; 1803 case BPF_STX | BPF_MEM | BPF_H: 1804 if (is_12b_int(off)) { 1805 emit(rv_sh(rd, off, rs), ctx); 1806 break; 1807 } 1808 1809 emit_imm(RV_REG_T1, off, ctx); 1810 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1811 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1812 break; 1813 case BPF_STX | BPF_MEM | BPF_W: 1814 if (is_12b_int(off)) { 1815 emit_sw(rd, off, rs, ctx); 1816 break; 1817 } 1818 1819 emit_imm(RV_REG_T1, off, ctx); 1820 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1821 emit_sw(RV_REG_T1, 0, rs, ctx); 1822 break; 1823 case BPF_STX | BPF_MEM | BPF_DW: 1824 if (is_12b_int(off)) { 1825 emit_sd(rd, off, rs, ctx); 1826 break; 1827 } 1828 1829 emit_imm(RV_REG_T1, off, ctx); 1830 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1831 emit_sd(RV_REG_T1, 0, rs, ctx); 1832 break; 1833 case BPF_STX | BPF_ATOMIC | BPF_W: 1834 case BPF_STX | BPF_ATOMIC | BPF_DW: 1835 emit_atomic(rd, rs, off, imm, 1836 BPF_SIZE(code) == BPF_DW, ctx); 1837 break; 1838 1839 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1840 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1841 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1842 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1843 { 1844 int insn_len, insns_start; 1845 1846 emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 1847 rd = RV_REG_T2; 1848 1849 switch (BPF_SIZE(code)) { 1850 case BPF_B: 1851 if (is_12b_int(off)) { 1852 insns_start = ctx->ninsns; 1853 emit(rv_sb(rd, off, rs), ctx); 1854 insn_len = ctx->ninsns - insns_start; 1855 break; 1856 } 1857 1858 emit_imm(RV_REG_T1, off, ctx); 1859 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1860 insns_start = ctx->ninsns; 1861 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1862 insn_len = ctx->ninsns - insns_start; 1863 break; 1864 case BPF_H: 1865 if (is_12b_int(off)) { 1866 insns_start = ctx->ninsns; 1867 emit(rv_sh(rd, off, rs), ctx); 1868 insn_len = ctx->ninsns - insns_start; 1869 break; 1870 } 1871 1872 emit_imm(RV_REG_T1, off, ctx); 1873 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1874 insns_start = ctx->ninsns; 1875 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1876 insn_len = ctx->ninsns - insns_start; 1877 break; 1878 case BPF_W: 1879 if (is_12b_int(off)) { 1880 insns_start = ctx->ninsns; 1881 emit_sw(rd, off, rs, ctx); 1882 insn_len = ctx->ninsns - insns_start; 1883 break; 1884 } 1885 1886 emit_imm(RV_REG_T1, off, ctx); 1887 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1888 insns_start = ctx->ninsns; 1889 emit_sw(RV_REG_T1, 0, rs, ctx); 1890 insn_len = ctx->ninsns - insns_start; 1891 break; 1892 case BPF_DW: 1893 if (is_12b_int(off)) { 1894 insns_start = ctx->ninsns; 1895 emit_sd(rd, off, rs, ctx); 1896 insn_len = ctx->ninsns - insns_start; 1897 break; 1898 } 1899 1900 emit_imm(RV_REG_T1, off, ctx); 1901 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1902 insns_start = ctx->ninsns; 1903 emit_sd(RV_REG_T1, 0, rs, ctx); 1904 insn_len = ctx->ninsns - insns_start; 1905 break; 1906 } 1907 1908 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1909 insn_len); 1910 if (ret) 1911 return ret; 1912 1913 break; 1914 } 1915 1916 default: 1917 pr_err("bpf-jit: unknown opcode %02x\n", code); 1918 return -EINVAL; 1919 } 1920 1921 return 0; 1922 } 1923 1924 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) 1925 { 1926 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1927 1928 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1929 if (bpf_stack_adjust) 1930 mark_fp(ctx); 1931 1932 if (seen_reg(RV_REG_RA, ctx)) 1933 stack_adjust += 8; 1934 stack_adjust += 8; /* RV_REG_FP */ 1935 if (seen_reg(RV_REG_S1, ctx)) 1936 stack_adjust += 8; 1937 if (seen_reg(RV_REG_S2, ctx)) 1938 stack_adjust += 8; 1939 if (seen_reg(RV_REG_S3, ctx)) 1940 stack_adjust += 8; 1941 if (seen_reg(RV_REG_S4, ctx)) 1942 stack_adjust += 8; 1943 if (seen_reg(RV_REG_S5, ctx)) 1944 stack_adjust += 8; 1945 if (seen_reg(RV_REG_S6, ctx)) 1946 stack_adjust += 8; 1947 if (ctx->arena_vm_start) 1948 stack_adjust += 8; 1949 1950 stack_adjust = round_up(stack_adjust, 16); 1951 stack_adjust += bpf_stack_adjust; 1952 1953 store_offset = stack_adjust - 8; 1954 1955 /* emit kcfi type preamble immediately before the first insn */ 1956 emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); 1957 1958 /* nops reserved for auipc+jalr pair */ 1959 for (i = 0; i < RV_FENTRY_NINSNS; i++) 1960 emit(rv_nop(), ctx); 1961 1962 /* First instruction is always setting the tail-call-counter 1963 * (TCC) register. This instruction is skipped for tail calls. 1964 * Force using a 4-byte (non-compressed) instruction. 1965 */ 1966 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1967 1968 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1969 1970 if (seen_reg(RV_REG_RA, ctx)) { 1971 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1972 store_offset -= 8; 1973 } 1974 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1975 store_offset -= 8; 1976 if (seen_reg(RV_REG_S1, ctx)) { 1977 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1978 store_offset -= 8; 1979 } 1980 if (seen_reg(RV_REG_S2, ctx)) { 1981 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1982 store_offset -= 8; 1983 } 1984 if (seen_reg(RV_REG_S3, ctx)) { 1985 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1986 store_offset -= 8; 1987 } 1988 if (seen_reg(RV_REG_S4, ctx)) { 1989 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1990 store_offset -= 8; 1991 } 1992 if (seen_reg(RV_REG_S5, ctx)) { 1993 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1994 store_offset -= 8; 1995 } 1996 if (seen_reg(RV_REG_S6, ctx)) { 1997 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1998 store_offset -= 8; 1999 } 2000 if (ctx->arena_vm_start) { 2001 emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); 2002 store_offset -= 8; 2003 } 2004 2005 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 2006 2007 if (bpf_stack_adjust) 2008 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 2009 2010 /* Program contains calls and tail calls, so RV_REG_TCC need 2011 * to be saved across calls. 2012 */ 2013 if (seen_tail_call(ctx) && seen_call(ctx)) 2014 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 2015 2016 ctx->stack_size = stack_adjust; 2017 2018 if (ctx->arena_vm_start) 2019 emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx); 2020 } 2021 2022 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 2023 { 2024 __build_epilogue(false, ctx); 2025 } 2026 2027 bool bpf_jit_supports_kfunc_call(void) 2028 { 2029 return true; 2030 } 2031 2032 bool bpf_jit_supports_ptr_xchg(void) 2033 { 2034 return true; 2035 } 2036 2037 bool bpf_jit_supports_arena(void) 2038 { 2039 return true; 2040 } 2041