1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <asm/patch.h> 14 #include <asm/cfi.h> 15 #include <asm/percpu.h> 16 #include "bpf_jit.h" 17 18 #define RV_FENTRY_NINSNS 2 19 20 #define RV_REG_TCC RV_REG_A6 21 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 22 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ 23 24 static const int regmap[] = { 25 [BPF_REG_0] = RV_REG_A5, 26 [BPF_REG_1] = RV_REG_A0, 27 [BPF_REG_2] = RV_REG_A1, 28 [BPF_REG_3] = RV_REG_A2, 29 [BPF_REG_4] = RV_REG_A3, 30 [BPF_REG_5] = RV_REG_A4, 31 [BPF_REG_6] = RV_REG_S1, 32 [BPF_REG_7] = RV_REG_S2, 33 [BPF_REG_8] = RV_REG_S3, 34 [BPF_REG_9] = RV_REG_S4, 35 [BPF_REG_FP] = RV_REG_S5, 36 [BPF_REG_AX] = RV_REG_T0, 37 }; 38 39 static const int pt_regmap[] = { 40 [RV_REG_A0] = offsetof(struct pt_regs, a0), 41 [RV_REG_A1] = offsetof(struct pt_regs, a1), 42 [RV_REG_A2] = offsetof(struct pt_regs, a2), 43 [RV_REG_A3] = offsetof(struct pt_regs, a3), 44 [RV_REG_A4] = offsetof(struct pt_regs, a4), 45 [RV_REG_A5] = offsetof(struct pt_regs, a5), 46 [RV_REG_S1] = offsetof(struct pt_regs, s1), 47 [RV_REG_S2] = offsetof(struct pt_regs, s2), 48 [RV_REG_S3] = offsetof(struct pt_regs, s3), 49 [RV_REG_S4] = offsetof(struct pt_regs, s4), 50 [RV_REG_S5] = offsetof(struct pt_regs, s5), 51 [RV_REG_T0] = offsetof(struct pt_regs, t0), 52 }; 53 54 enum { 55 RV_CTX_F_SEEN_TAIL_CALL = 0, 56 RV_CTX_F_SEEN_CALL = RV_REG_RA, 57 RV_CTX_F_SEEN_S1 = RV_REG_S1, 58 RV_CTX_F_SEEN_S2 = RV_REG_S2, 59 RV_CTX_F_SEEN_S3 = RV_REG_S3, 60 RV_CTX_F_SEEN_S4 = RV_REG_S4, 61 RV_CTX_F_SEEN_S5 = RV_REG_S5, 62 RV_CTX_F_SEEN_S6 = RV_REG_S6, 63 }; 64 65 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 66 { 67 u8 reg = regmap[bpf_reg]; 68 69 switch (reg) { 70 case RV_CTX_F_SEEN_S1: 71 case RV_CTX_F_SEEN_S2: 72 case RV_CTX_F_SEEN_S3: 73 case RV_CTX_F_SEEN_S4: 74 case RV_CTX_F_SEEN_S5: 75 case RV_CTX_F_SEEN_S6: 76 __set_bit(reg, &ctx->flags); 77 } 78 return reg; 79 }; 80 81 static bool seen_reg(int reg, struct rv_jit_context *ctx) 82 { 83 switch (reg) { 84 case RV_CTX_F_SEEN_CALL: 85 case RV_CTX_F_SEEN_S1: 86 case RV_CTX_F_SEEN_S2: 87 case RV_CTX_F_SEEN_S3: 88 case RV_CTX_F_SEEN_S4: 89 case RV_CTX_F_SEEN_S5: 90 case RV_CTX_F_SEEN_S6: 91 return test_bit(reg, &ctx->flags); 92 } 93 return false; 94 } 95 96 static void mark_fp(struct rv_jit_context *ctx) 97 { 98 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 99 } 100 101 static void mark_call(struct rv_jit_context *ctx) 102 { 103 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 104 } 105 106 static bool seen_call(struct rv_jit_context *ctx) 107 { 108 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 109 } 110 111 static void mark_tail_call(struct rv_jit_context *ctx) 112 { 113 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 114 } 115 116 static bool seen_tail_call(struct rv_jit_context *ctx) 117 { 118 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 119 } 120 121 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 122 { 123 mark_tail_call(ctx); 124 125 if (seen_call(ctx)) { 126 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 127 return RV_REG_S6; 128 } 129 return RV_REG_A6; 130 } 131 132 static bool is_32b_int(s64 val) 133 { 134 return -(1L << 31) <= val && val < (1L << 31); 135 } 136 137 static bool in_auipc_jalr_range(s64 val) 138 { 139 /* 140 * auipc+jalr can reach any signed PC-relative offset in the range 141 * [-2^31 - 2^11, 2^31 - 2^11). 142 */ 143 return (-(1L << 31) - (1L << 11)) <= val && 144 val < ((1L << 31) - (1L << 11)); 145 } 146 147 /* Modify rd pointer to alternate reg to avoid corrupting original reg */ 148 static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 149 { 150 emit_sextw(ra, *rd, ctx); 151 *rd = ra; 152 } 153 154 static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 155 { 156 emit_zextw(ra, *rd, ctx); 157 *rd = ra; 158 } 159 160 /* Emit fixed-length instructions for address */ 161 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 162 { 163 /* 164 * Use the ro_insns(RX) to calculate the offset as the BPF program will 165 * finally run from this memory region. 166 */ 167 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); 168 s64 off = addr - ip; 169 s64 upper = (off + (1 << 11)) >> 12; 170 s64 lower = off & 0xfff; 171 172 if (extra_pass && !in_auipc_jalr_range(off)) { 173 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 174 return -ERANGE; 175 } 176 177 emit(rv_auipc(rd, upper), ctx); 178 emit(rv_addi(rd, rd, lower), ctx); 179 return 0; 180 } 181 182 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 183 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 184 { 185 /* Note that the immediate from the add is sign-extended, 186 * which means that we need to compensate this by adding 2^12, 187 * when the 12th bit is set. A simpler way of doing this, and 188 * getting rid of the check, is to just add 2**11 before the 189 * shift. The "Loading a 32-Bit constant" example from the 190 * "Computer Organization and Design, RISC-V edition" book by 191 * Patterson/Hennessy highlights this fact. 192 * 193 * This also means that we need to process LSB to MSB. 194 */ 195 s64 upper = (val + (1 << 11)) >> 12; 196 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 197 * and addi are signed and RVC checks will perform signed comparisons. 198 */ 199 s64 lower = ((val & 0xfff) << 52) >> 52; 200 int shift; 201 202 if (is_32b_int(val)) { 203 if (upper) 204 emit_lui(rd, upper, ctx); 205 206 if (!upper) { 207 emit_li(rd, lower, ctx); 208 return; 209 } 210 211 emit_addiw(rd, rd, lower, ctx); 212 return; 213 } 214 215 shift = __ffs(upper); 216 upper >>= shift; 217 shift += 12; 218 219 emit_imm(rd, upper, ctx); 220 221 emit_slli(rd, rd, shift, ctx); 222 if (lower) 223 emit_addi(rd, rd, lower, ctx); 224 } 225 226 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 227 { 228 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 229 230 if (seen_reg(RV_REG_RA, ctx)) { 231 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 232 store_offset -= 8; 233 } 234 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 235 store_offset -= 8; 236 if (seen_reg(RV_REG_S1, ctx)) { 237 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 238 store_offset -= 8; 239 } 240 if (seen_reg(RV_REG_S2, ctx)) { 241 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 242 store_offset -= 8; 243 } 244 if (seen_reg(RV_REG_S3, ctx)) { 245 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 246 store_offset -= 8; 247 } 248 if (seen_reg(RV_REG_S4, ctx)) { 249 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 250 store_offset -= 8; 251 } 252 if (seen_reg(RV_REG_S5, ctx)) { 253 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 254 store_offset -= 8; 255 } 256 if (seen_reg(RV_REG_S6, ctx)) { 257 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 258 store_offset -= 8; 259 } 260 if (ctx->arena_vm_start) { 261 emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); 262 store_offset -= 8; 263 } 264 265 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 266 /* Set return value. */ 267 if (!is_tail_call) 268 emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); 269 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 270 is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 271 ctx); 272 } 273 274 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 275 struct rv_jit_context *ctx) 276 { 277 switch (cond) { 278 case BPF_JEQ: 279 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 280 return; 281 case BPF_JGT: 282 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 283 return; 284 case BPF_JLT: 285 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 286 return; 287 case BPF_JGE: 288 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 289 return; 290 case BPF_JLE: 291 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 292 return; 293 case BPF_JNE: 294 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 295 return; 296 case BPF_JSGT: 297 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 298 return; 299 case BPF_JSLT: 300 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 301 return; 302 case BPF_JSGE: 303 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 304 return; 305 case BPF_JSLE: 306 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 307 } 308 } 309 310 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 311 struct rv_jit_context *ctx) 312 { 313 s64 upper, lower; 314 315 if (is_13b_int(rvoff)) { 316 emit_bcc(cond, rd, rs, rvoff, ctx); 317 return; 318 } 319 320 /* Adjust for jal */ 321 rvoff -= 4; 322 323 /* Transform, e.g.: 324 * bne rd,rs,foo 325 * to 326 * beq rd,rs,<.L1> 327 * (auipc foo) 328 * jal(r) foo 329 * .L1 330 */ 331 cond = invert_bpf_cond(cond); 332 if (is_21b_int(rvoff)) { 333 emit_bcc(cond, rd, rs, 8, ctx); 334 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 335 return; 336 } 337 338 /* 32b No need for an additional rvoff adjustment, since we 339 * get that from the auipc at PC', where PC = PC' + 4. 340 */ 341 upper = (rvoff + (1 << 11)) >> 12; 342 lower = rvoff & 0xfff; 343 344 emit_bcc(cond, rd, rs, 12, ctx); 345 emit(rv_auipc(RV_REG_T1, upper), ctx); 346 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 347 } 348 349 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 350 { 351 int tc_ninsn, off, start_insn = ctx->ninsns; 352 u8 tcc = rv_tail_call_reg(ctx); 353 354 /* a0: &ctx 355 * a1: &array 356 * a2: index 357 * 358 * if (index >= array->map.max_entries) 359 * goto out; 360 */ 361 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 362 ctx->offset[0]; 363 emit_zextw(RV_REG_A2, RV_REG_A2, ctx); 364 365 off = offsetof(struct bpf_array, map.max_entries); 366 if (is_12b_check(off, insn)) 367 return -1; 368 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 369 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 370 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 371 372 /* if (--TCC < 0) 373 * goto out; 374 */ 375 emit_addi(RV_REG_TCC, tcc, -1, ctx); 376 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 377 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 378 379 /* prog = array->ptrs[index]; 380 * if (!prog) 381 * goto out; 382 */ 383 emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx); 384 off = offsetof(struct bpf_array, ptrs); 385 if (is_12b_check(off, insn)) 386 return -1; 387 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 388 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 389 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 390 391 /* goto *(prog->bpf_func + 4); */ 392 off = offsetof(struct bpf_prog, bpf_func); 393 if (is_12b_check(off, insn)) 394 return -1; 395 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 396 __build_epilogue(true, ctx); 397 return 0; 398 } 399 400 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 401 struct rv_jit_context *ctx) 402 { 403 u8 code = insn->code; 404 405 switch (code) { 406 case BPF_JMP | BPF_JA: 407 case BPF_JMP | BPF_CALL: 408 case BPF_JMP | BPF_EXIT: 409 case BPF_JMP | BPF_TAIL_CALL: 410 break; 411 default: 412 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 413 } 414 415 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 416 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 417 code & BPF_LDX || code & BPF_STX) 418 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 419 } 420 421 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 422 struct rv_jit_context *ctx) 423 { 424 s64 upper, lower; 425 426 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 427 emit(rv_jal(rd, rvoff >> 1), ctx); 428 return 0; 429 } else if (in_auipc_jalr_range(rvoff)) { 430 upper = (rvoff + (1 << 11)) >> 12; 431 lower = rvoff & 0xfff; 432 emit(rv_auipc(RV_REG_T1, upper), ctx); 433 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 434 return 0; 435 } 436 437 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 438 return -ERANGE; 439 } 440 441 static bool is_signed_bpf_cond(u8 cond) 442 { 443 return cond == BPF_JSGT || cond == BPF_JSLT || 444 cond == BPF_JSGE || cond == BPF_JSLE; 445 } 446 447 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 448 { 449 s64 off = 0; 450 u64 ip; 451 452 if (addr && ctx->insns && ctx->ro_insns) { 453 /* 454 * Use the ro_insns(RX) to calculate the offset as the BPF 455 * program will finally run from this memory region. 456 */ 457 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); 458 off = addr - ip; 459 } 460 461 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 462 } 463 464 static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) 465 { 466 if (IS_ENABLED(CONFIG_CFI_CLANG)) 467 emit(hash, ctx); 468 } 469 470 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 471 struct rv_jit_context *ctx) 472 { 473 u8 r0; 474 int jmp_offset; 475 476 if (off) { 477 if (is_12b_int(off)) { 478 emit_addi(RV_REG_T1, rd, off, ctx); 479 } else { 480 emit_imm(RV_REG_T1, off, ctx); 481 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 482 } 483 rd = RV_REG_T1; 484 } 485 486 switch (imm) { 487 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 488 case BPF_ADD: 489 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 490 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 491 break; 492 case BPF_AND: 493 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 494 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 495 break; 496 case BPF_OR: 497 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 498 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 499 break; 500 case BPF_XOR: 501 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 502 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 503 break; 504 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 505 case BPF_ADD | BPF_FETCH: 506 emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) : 507 rv_amoadd_w(rs, rs, rd, 1, 1), ctx); 508 if (!is64) 509 emit_zextw(rs, rs, ctx); 510 break; 511 case BPF_AND | BPF_FETCH: 512 emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) : 513 rv_amoand_w(rs, rs, rd, 1, 1), ctx); 514 if (!is64) 515 emit_zextw(rs, rs, ctx); 516 break; 517 case BPF_OR | BPF_FETCH: 518 emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) : 519 rv_amoor_w(rs, rs, rd, 1, 1), ctx); 520 if (!is64) 521 emit_zextw(rs, rs, ctx); 522 break; 523 case BPF_XOR | BPF_FETCH: 524 emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) : 525 rv_amoxor_w(rs, rs, rd, 1, 1), ctx); 526 if (!is64) 527 emit_zextw(rs, rs, ctx); 528 break; 529 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 530 case BPF_XCHG: 531 emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) : 532 rv_amoswap_w(rs, rs, rd, 1, 1), ctx); 533 if (!is64) 534 emit_zextw(rs, rs, ctx); 535 break; 536 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 537 case BPF_CMPXCHG: 538 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 539 if (is64) 540 emit_mv(RV_REG_T2, r0, ctx); 541 else 542 emit_addiw(RV_REG_T2, r0, 0, ctx); 543 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 544 rv_lr_w(r0, 0, rd, 0, 0), ctx); 545 jmp_offset = ninsns_rvoff(8); 546 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 547 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 548 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 549 jmp_offset = ninsns_rvoff(-6); 550 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 551 emit(rv_fence(0x3, 0x3), ctx); 552 break; 553 } 554 } 555 556 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 557 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 558 #define REG_DONT_CLEAR_MARKER 0 /* RV_REG_ZERO unused in pt_regmap */ 559 560 bool ex_handler_bpf(const struct exception_table_entry *ex, 561 struct pt_regs *regs) 562 { 563 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 564 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 565 566 if (regs_offset != REG_DONT_CLEAR_MARKER) 567 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 568 regs->epc = (unsigned long)&ex->fixup - offset; 569 570 return true; 571 } 572 573 /* For accesses to BTF pointers, add an entry to the exception table */ 574 static int add_exception_handler(const struct bpf_insn *insn, 575 struct rv_jit_context *ctx, 576 int dst_reg, int insn_len) 577 { 578 struct exception_table_entry *ex; 579 unsigned long pc; 580 off_t ins_offset; 581 off_t fixup_offset; 582 583 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 584 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 585 BPF_MODE(insn->code) != BPF_PROBE_MEM32)) 586 return 0; 587 588 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 589 return -EINVAL; 590 591 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 592 return -EINVAL; 593 594 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 595 return -EINVAL; 596 597 ex = &ctx->prog->aux->extable[ctx->nexentries]; 598 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 599 600 /* 601 * This is the relative offset of the instruction that may fault from 602 * the exception table itself. This will be written to the exception 603 * table and if this instruction faults, the destination register will 604 * be set to '0' and the execution will jump to the next instruction. 605 */ 606 ins_offset = pc - (long)&ex->insn; 607 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 608 return -ERANGE; 609 610 /* 611 * Since the extable follows the program, the fixup offset is always 612 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 613 * to keep things simple, and put the destination register in the upper 614 * bits. We don't need to worry about buildtime or runtime sort 615 * modifying the upper bits because the table is already sorted, and 616 * isn't part of the main exception table. 617 * 618 * The fixup_offset is set to the next instruction from the instruction 619 * that may fault. The execution will jump to this after handling the 620 * fault. 621 */ 622 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 623 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 624 return -ERANGE; 625 626 /* 627 * The offsets above have been calculated using the RO buffer but we 628 * need to use the R/W buffer for writes. 629 * switch ex to rw buffer for writing. 630 */ 631 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); 632 633 ex->insn = ins_offset; 634 635 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 636 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 637 ex->type = EX_TYPE_BPF; 638 639 ctx->nexentries++; 640 return 0; 641 } 642 643 static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 644 { 645 s64 rvoff; 646 struct rv_jit_context ctx; 647 648 ctx.ninsns = 0; 649 ctx.insns = (u16 *)insns; 650 651 if (!target) { 652 emit(rv_nop(), &ctx); 653 emit(rv_nop(), &ctx); 654 return 0; 655 } 656 657 rvoff = (s64)(target - ip); 658 return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 659 } 660 661 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 662 void *old_addr, void *new_addr) 663 { 664 u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 665 bool is_call = poke_type == BPF_MOD_CALL; 666 int ret; 667 668 if (!is_kernel_text((unsigned long)ip) && 669 !is_bpf_text_address((unsigned long)ip)) 670 return -ENOTSUPP; 671 672 ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 673 if (ret) 674 return ret; 675 676 if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) 677 return -EFAULT; 678 679 ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 680 if (ret) 681 return ret; 682 683 cpus_read_lock(); 684 mutex_lock(&text_mutex); 685 if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) 686 ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); 687 mutex_unlock(&text_mutex); 688 cpus_read_unlock(); 689 690 return ret; 691 } 692 693 static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) 694 { 695 int i; 696 697 for (i = 0; i < nregs; i++) { 698 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 699 args_off -= 8; 700 } 701 } 702 703 static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) 704 { 705 int i; 706 707 for (i = 0; i < nregs; i++) { 708 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 709 args_off -= 8; 710 } 711 } 712 713 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 714 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 715 { 716 int ret, branch_off; 717 struct bpf_prog *p = l->link.prog; 718 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 719 720 if (l->cookie) { 721 emit_imm(RV_REG_T1, l->cookie, ctx); 722 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 723 } else { 724 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 725 } 726 727 /* arg1: prog */ 728 emit_imm(RV_REG_A0, (const s64)p, ctx); 729 /* arg2: &run_ctx */ 730 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 731 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 732 if (ret) 733 return ret; 734 735 /* store prog start time */ 736 emit_mv(RV_REG_S1, RV_REG_A0, ctx); 737 738 /* if (__bpf_prog_enter(prog) == 0) 739 * goto skip_exec_of_prog; 740 */ 741 branch_off = ctx->ninsns; 742 /* nop reserved for conditional jump */ 743 emit(rv_nop(), ctx); 744 745 /* arg1: &args_off */ 746 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 747 if (!p->jited) 748 /* arg2: progs[i]->insnsi for interpreter */ 749 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 750 ret = emit_call((const u64)p->bpf_func, true, ctx); 751 if (ret) 752 return ret; 753 754 if (save_ret) { 755 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 756 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 757 } 758 759 /* update branch with beqz */ 760 if (ctx->insns) { 761 int offset = ninsns_rvoff(ctx->ninsns - branch_off); 762 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 763 *(u32 *)(ctx->insns + branch_off) = insn; 764 } 765 766 /* arg1: prog */ 767 emit_imm(RV_REG_A0, (const s64)p, ctx); 768 /* arg2: prog start time */ 769 emit_mv(RV_REG_A1, RV_REG_S1, ctx); 770 /* arg3: &run_ctx */ 771 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 772 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 773 774 return ret; 775 } 776 777 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 778 const struct btf_func_model *m, 779 struct bpf_tramp_links *tlinks, 780 void *func_addr, u32 flags, 781 struct rv_jit_context *ctx) 782 { 783 int i, ret, offset; 784 int *branches_off = NULL; 785 int stack_size = 0, nregs = m->nr_args; 786 int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; 787 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 788 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 789 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 790 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 791 void *orig_call = func_addr; 792 bool save_ret; 793 u32 insn; 794 795 /* Two types of generated trampoline stack layout: 796 * 797 * 1. trampoline called from function entry 798 * -------------------------------------- 799 * FP + 8 [ RA to parent func ] return address to parent 800 * function 801 * FP + 0 [ FP of parent func ] frame pointer of parent 802 * function 803 * FP - 8 [ T0 to traced func ] return address of traced 804 * function 805 * FP - 16 [ FP of traced func ] frame pointer of traced 806 * function 807 * -------------------------------------- 808 * 809 * 2. trampoline called directly 810 * -------------------------------------- 811 * FP - 8 [ RA to caller func ] return address to caller 812 * function 813 * FP - 16 [ FP of caller func ] frame pointer of caller 814 * function 815 * -------------------------------------- 816 * 817 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 818 * BPF_TRAMP_F_RET_FENTRY_RET 819 * [ argN ] 820 * [ ... ] 821 * FP - args_off [ arg1 ] 822 * 823 * FP - nregs_off [ regs count ] 824 * 825 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 826 * 827 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 828 * 829 * FP - sreg_off [ callee saved reg ] 830 * 831 * [ pads ] pads for 16 bytes alignment 832 */ 833 834 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 835 return -ENOTSUPP; 836 837 /* extra regiters for struct arguments */ 838 for (i = 0; i < m->nr_args; i++) 839 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 840 nregs += round_up(m->arg_size[i], 8) / 8 - 1; 841 842 /* 8 arguments passed by registers */ 843 if (nregs > 8) 844 return -ENOTSUPP; 845 846 /* room of trampoline frame to store return address and frame pointer */ 847 stack_size += 16; 848 849 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 850 if (save_ret) { 851 stack_size += 16; /* Save both A5 (BPF R0) and A0 */ 852 retval_off = stack_size; 853 } 854 855 stack_size += nregs * 8; 856 args_off = stack_size; 857 858 stack_size += 8; 859 nregs_off = stack_size; 860 861 if (flags & BPF_TRAMP_F_IP_ARG) { 862 stack_size += 8; 863 ip_off = stack_size; 864 } 865 866 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 867 run_ctx_off = stack_size; 868 869 stack_size += 8; 870 sreg_off = stack_size; 871 872 stack_size = round_up(stack_size, STACK_ALIGN); 873 874 if (!is_struct_ops) { 875 /* For the trampoline called from function entry, 876 * the frame of traced function and the frame of 877 * trampoline need to be considered. 878 */ 879 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 880 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 881 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 882 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 883 884 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 885 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 886 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 887 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 888 } else { 889 /* emit kcfi hash */ 890 emit_kcfi(cfi_get_func_hash(func_addr), ctx); 891 /* For the trampoline called directly, just handle 892 * the frame of trampoline. 893 */ 894 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 895 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 896 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 897 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 898 } 899 900 /* callee saved register S1 to pass start time */ 901 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 902 903 /* store ip address of the traced function */ 904 if (flags & BPF_TRAMP_F_IP_ARG) { 905 emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 906 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 907 } 908 909 emit_li(RV_REG_T1, nregs, ctx); 910 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 911 912 store_args(nregs, args_off, ctx); 913 914 /* skip to actual body of traced function */ 915 if (flags & BPF_TRAMP_F_SKIP_FRAME) 916 orig_call += RV_FENTRY_NINSNS * 4; 917 918 if (flags & BPF_TRAMP_F_CALL_ORIG) { 919 emit_imm(RV_REG_A0, (const s64)im, ctx); 920 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 921 if (ret) 922 return ret; 923 } 924 925 for (i = 0; i < fentry->nr_links; i++) { 926 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 927 flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 928 if (ret) 929 return ret; 930 } 931 932 if (fmod_ret->nr_links) { 933 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 934 if (!branches_off) 935 return -ENOMEM; 936 937 /* cleanup to avoid garbage return value confusion */ 938 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 939 for (i = 0; i < fmod_ret->nr_links; i++) { 940 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 941 run_ctx_off, true, ctx); 942 if (ret) 943 goto out; 944 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 945 branches_off[i] = ctx->ninsns; 946 /* nop reserved for conditional jump */ 947 emit(rv_nop(), ctx); 948 } 949 } 950 951 if (flags & BPF_TRAMP_F_CALL_ORIG) { 952 restore_args(nregs, args_off, ctx); 953 ret = emit_call((const u64)orig_call, true, ctx); 954 if (ret) 955 goto out; 956 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 957 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 958 im->ip_after_call = ctx->insns + ctx->ninsns; 959 /* 2 nops reserved for auipc+jalr pair */ 960 emit(rv_nop(), ctx); 961 emit(rv_nop(), ctx); 962 } 963 964 /* update branches saved in invoke_bpf_mod_ret with bnez */ 965 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 966 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 967 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 968 *(u32 *)(ctx->insns + branches_off[i]) = insn; 969 } 970 971 for (i = 0; i < fexit->nr_links; i++) { 972 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 973 run_ctx_off, false, ctx); 974 if (ret) 975 goto out; 976 } 977 978 if (flags & BPF_TRAMP_F_CALL_ORIG) { 979 im->ip_epilogue = ctx->insns + ctx->ninsns; 980 emit_imm(RV_REG_A0, (const s64)im, ctx); 981 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 982 if (ret) 983 goto out; 984 } 985 986 if (flags & BPF_TRAMP_F_RESTORE_REGS) 987 restore_args(nregs, args_off, ctx); 988 989 if (save_ret) { 990 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 991 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); 992 } 993 994 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 995 996 if (!is_struct_ops) { 997 /* trampoline called from function entry */ 998 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 999 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1000 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1001 1002 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 1003 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 1004 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 1005 1006 if (flags & BPF_TRAMP_F_SKIP_FRAME) 1007 /* return to parent function */ 1008 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1009 else 1010 /* return to traced function */ 1011 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 1012 } else { 1013 /* trampoline called directly */ 1014 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 1015 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1016 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1017 1018 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1019 } 1020 1021 ret = ctx->ninsns; 1022 out: 1023 kfree(branches_off); 1024 return ret; 1025 } 1026 1027 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1028 struct bpf_tramp_links *tlinks, void *func_addr) 1029 { 1030 struct bpf_tramp_image im; 1031 struct rv_jit_context ctx; 1032 int ret; 1033 1034 ctx.ninsns = 0; 1035 ctx.insns = NULL; 1036 ctx.ro_insns = NULL; 1037 ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); 1038 1039 return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); 1040 } 1041 1042 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 1043 void *image_end, const struct btf_func_model *m, 1044 u32 flags, struct bpf_tramp_links *tlinks, 1045 void *func_addr) 1046 { 1047 int ret; 1048 struct rv_jit_context ctx; 1049 1050 ctx.ninsns = 0; 1051 /* 1052 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the 1053 * JITed instructions and later copies it to a RX region (ctx.ro_insns). 1054 * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the 1055 * trampoline image uses the same memory area for writing and execution, 1056 * both ctx.insns and ctx.ro_insns can be set to image. 1057 */ 1058 ctx.insns = image; 1059 ctx.ro_insns = image; 1060 ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 1061 if (ret < 0) 1062 return ret; 1063 1064 bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); 1065 1066 return ninsns_rvoff(ret); 1067 } 1068 1069 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1070 bool extra_pass) 1071 { 1072 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1073 BPF_CLASS(insn->code) == BPF_JMP; 1074 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1075 struct bpf_prog_aux *aux = ctx->prog->aux; 1076 u8 rd = -1, rs = -1, code = insn->code; 1077 s16 off = insn->off; 1078 s32 imm = insn->imm; 1079 1080 init_regs(&rd, &rs, insn, ctx); 1081 1082 switch (code) { 1083 /* dst = src */ 1084 case BPF_ALU | BPF_MOV | BPF_X: 1085 case BPF_ALU64 | BPF_MOV | BPF_X: 1086 if (insn_is_cast_user(insn)) { 1087 emit_mv(RV_REG_T1, rs, ctx); 1088 emit_zextw(RV_REG_T1, RV_REG_T1, ctx); 1089 emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx); 1090 emit(rv_beq(RV_REG_T1, RV_REG_ZERO, 4), ctx); 1091 emit_or(RV_REG_T1, rd, RV_REG_T1, ctx); 1092 emit_mv(rd, RV_REG_T1, ctx); 1093 break; 1094 } else if (insn_is_mov_percpu_addr(insn)) { 1095 if (rd != rs) 1096 emit_mv(rd, rs, ctx); 1097 #ifdef CONFIG_SMP 1098 /* Load current CPU number in T1 */ 1099 emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu), 1100 RV_REG_TP, ctx); 1101 /* Load address of __per_cpu_offset array in T2 */ 1102 emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx); 1103 /* Get address of __per_cpu_offset[cpu] in T1 */ 1104 emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx); 1105 /* Load __per_cpu_offset[cpu] in T1 */ 1106 emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx); 1107 /* Add the offset to Rd */ 1108 emit_add(rd, rd, RV_REG_T1, ctx); 1109 #endif 1110 } 1111 if (imm == 1) { 1112 /* Special mov32 for zext */ 1113 emit_zextw(rd, rd, ctx); 1114 break; 1115 } 1116 switch (insn->off) { 1117 case 0: 1118 emit_mv(rd, rs, ctx); 1119 break; 1120 case 8: 1121 emit_sextb(rd, rs, ctx); 1122 break; 1123 case 16: 1124 emit_sexth(rd, rs, ctx); 1125 break; 1126 case 32: 1127 emit_sextw(rd, rs, ctx); 1128 break; 1129 } 1130 if (!is64 && !aux->verifier_zext) 1131 emit_zextw(rd, rd, ctx); 1132 break; 1133 1134 /* dst = dst OP src */ 1135 case BPF_ALU | BPF_ADD | BPF_X: 1136 case BPF_ALU64 | BPF_ADD | BPF_X: 1137 emit_add(rd, rd, rs, ctx); 1138 if (!is64 && !aux->verifier_zext) 1139 emit_zextw(rd, rd, ctx); 1140 break; 1141 case BPF_ALU | BPF_SUB | BPF_X: 1142 case BPF_ALU64 | BPF_SUB | BPF_X: 1143 if (is64) 1144 emit_sub(rd, rd, rs, ctx); 1145 else 1146 emit_subw(rd, rd, rs, ctx); 1147 1148 if (!is64 && !aux->verifier_zext) 1149 emit_zextw(rd, rd, ctx); 1150 break; 1151 case BPF_ALU | BPF_AND | BPF_X: 1152 case BPF_ALU64 | BPF_AND | BPF_X: 1153 emit_and(rd, rd, rs, ctx); 1154 if (!is64 && !aux->verifier_zext) 1155 emit_zextw(rd, rd, ctx); 1156 break; 1157 case BPF_ALU | BPF_OR | BPF_X: 1158 case BPF_ALU64 | BPF_OR | BPF_X: 1159 emit_or(rd, rd, rs, ctx); 1160 if (!is64 && !aux->verifier_zext) 1161 emit_zextw(rd, rd, ctx); 1162 break; 1163 case BPF_ALU | BPF_XOR | BPF_X: 1164 case BPF_ALU64 | BPF_XOR | BPF_X: 1165 emit_xor(rd, rd, rs, ctx); 1166 if (!is64 && !aux->verifier_zext) 1167 emit_zextw(rd, rd, ctx); 1168 break; 1169 case BPF_ALU | BPF_MUL | BPF_X: 1170 case BPF_ALU64 | BPF_MUL | BPF_X: 1171 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1172 if (!is64 && !aux->verifier_zext) 1173 emit_zextw(rd, rd, ctx); 1174 break; 1175 case BPF_ALU | BPF_DIV | BPF_X: 1176 case BPF_ALU64 | BPF_DIV | BPF_X: 1177 if (off) 1178 emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1179 else 1180 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1181 if (!is64 && !aux->verifier_zext) 1182 emit_zextw(rd, rd, ctx); 1183 break; 1184 case BPF_ALU | BPF_MOD | BPF_X: 1185 case BPF_ALU64 | BPF_MOD | BPF_X: 1186 if (off) 1187 emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1188 else 1189 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1190 if (!is64 && !aux->verifier_zext) 1191 emit_zextw(rd, rd, ctx); 1192 break; 1193 case BPF_ALU | BPF_LSH | BPF_X: 1194 case BPF_ALU64 | BPF_LSH | BPF_X: 1195 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 1196 if (!is64 && !aux->verifier_zext) 1197 emit_zextw(rd, rd, ctx); 1198 break; 1199 case BPF_ALU | BPF_RSH | BPF_X: 1200 case BPF_ALU64 | BPF_RSH | BPF_X: 1201 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1202 if (!is64 && !aux->verifier_zext) 1203 emit_zextw(rd, rd, ctx); 1204 break; 1205 case BPF_ALU | BPF_ARSH | BPF_X: 1206 case BPF_ALU64 | BPF_ARSH | BPF_X: 1207 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1208 if (!is64 && !aux->verifier_zext) 1209 emit_zextw(rd, rd, ctx); 1210 break; 1211 1212 /* dst = -dst */ 1213 case BPF_ALU | BPF_NEG: 1214 case BPF_ALU64 | BPF_NEG: 1215 emit_sub(rd, RV_REG_ZERO, rd, ctx); 1216 if (!is64 && !aux->verifier_zext) 1217 emit_zextw(rd, rd, ctx); 1218 break; 1219 1220 /* dst = BSWAP##imm(dst) */ 1221 case BPF_ALU | BPF_END | BPF_FROM_LE: 1222 switch (imm) { 1223 case 16: 1224 emit_zexth(rd, rd, ctx); 1225 break; 1226 case 32: 1227 if (!aux->verifier_zext) 1228 emit_zextw(rd, rd, ctx); 1229 break; 1230 case 64: 1231 /* Do nothing */ 1232 break; 1233 } 1234 break; 1235 case BPF_ALU | BPF_END | BPF_FROM_BE: 1236 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1237 emit_bswap(rd, imm, ctx); 1238 break; 1239 1240 /* dst = imm */ 1241 case BPF_ALU | BPF_MOV | BPF_K: 1242 case BPF_ALU64 | BPF_MOV | BPF_K: 1243 emit_imm(rd, imm, ctx); 1244 if (!is64 && !aux->verifier_zext) 1245 emit_zextw(rd, rd, ctx); 1246 break; 1247 1248 /* dst = dst OP imm */ 1249 case BPF_ALU | BPF_ADD | BPF_K: 1250 case BPF_ALU64 | BPF_ADD | BPF_K: 1251 if (is_12b_int(imm)) { 1252 emit_addi(rd, rd, imm, ctx); 1253 } else { 1254 emit_imm(RV_REG_T1, imm, ctx); 1255 emit_add(rd, rd, RV_REG_T1, ctx); 1256 } 1257 if (!is64 && !aux->verifier_zext) 1258 emit_zextw(rd, rd, ctx); 1259 break; 1260 case BPF_ALU | BPF_SUB | BPF_K: 1261 case BPF_ALU64 | BPF_SUB | BPF_K: 1262 if (is_12b_int(-imm)) { 1263 emit_addi(rd, rd, -imm, ctx); 1264 } else { 1265 emit_imm(RV_REG_T1, imm, ctx); 1266 emit_sub(rd, rd, RV_REG_T1, ctx); 1267 } 1268 if (!is64 && !aux->verifier_zext) 1269 emit_zextw(rd, rd, ctx); 1270 break; 1271 case BPF_ALU | BPF_AND | BPF_K: 1272 case BPF_ALU64 | BPF_AND | BPF_K: 1273 if (is_12b_int(imm)) { 1274 emit_andi(rd, rd, imm, ctx); 1275 } else { 1276 emit_imm(RV_REG_T1, imm, ctx); 1277 emit_and(rd, rd, RV_REG_T1, ctx); 1278 } 1279 if (!is64 && !aux->verifier_zext) 1280 emit_zextw(rd, rd, ctx); 1281 break; 1282 case BPF_ALU | BPF_OR | BPF_K: 1283 case BPF_ALU64 | BPF_OR | BPF_K: 1284 if (is_12b_int(imm)) { 1285 emit(rv_ori(rd, rd, imm), ctx); 1286 } else { 1287 emit_imm(RV_REG_T1, imm, ctx); 1288 emit_or(rd, rd, RV_REG_T1, ctx); 1289 } 1290 if (!is64 && !aux->verifier_zext) 1291 emit_zextw(rd, rd, ctx); 1292 break; 1293 case BPF_ALU | BPF_XOR | BPF_K: 1294 case BPF_ALU64 | BPF_XOR | BPF_K: 1295 if (is_12b_int(imm)) { 1296 emit(rv_xori(rd, rd, imm), ctx); 1297 } else { 1298 emit_imm(RV_REG_T1, imm, ctx); 1299 emit_xor(rd, rd, RV_REG_T1, ctx); 1300 } 1301 if (!is64 && !aux->verifier_zext) 1302 emit_zextw(rd, rd, ctx); 1303 break; 1304 case BPF_ALU | BPF_MUL | BPF_K: 1305 case BPF_ALU64 | BPF_MUL | BPF_K: 1306 emit_imm(RV_REG_T1, imm, ctx); 1307 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1308 rv_mulw(rd, rd, RV_REG_T1), ctx); 1309 if (!is64 && !aux->verifier_zext) 1310 emit_zextw(rd, rd, ctx); 1311 break; 1312 case BPF_ALU | BPF_DIV | BPF_K: 1313 case BPF_ALU64 | BPF_DIV | BPF_K: 1314 emit_imm(RV_REG_T1, imm, ctx); 1315 if (off) 1316 emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1317 rv_divw(rd, rd, RV_REG_T1), ctx); 1318 else 1319 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1320 rv_divuw(rd, rd, RV_REG_T1), ctx); 1321 if (!is64 && !aux->verifier_zext) 1322 emit_zextw(rd, rd, ctx); 1323 break; 1324 case BPF_ALU | BPF_MOD | BPF_K: 1325 case BPF_ALU64 | BPF_MOD | BPF_K: 1326 emit_imm(RV_REG_T1, imm, ctx); 1327 if (off) 1328 emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1329 rv_remw(rd, rd, RV_REG_T1), ctx); 1330 else 1331 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1332 rv_remuw(rd, rd, RV_REG_T1), ctx); 1333 if (!is64 && !aux->verifier_zext) 1334 emit_zextw(rd, rd, ctx); 1335 break; 1336 case BPF_ALU | BPF_LSH | BPF_K: 1337 case BPF_ALU64 | BPF_LSH | BPF_K: 1338 emit_slli(rd, rd, imm, ctx); 1339 1340 if (!is64 && !aux->verifier_zext) 1341 emit_zextw(rd, rd, ctx); 1342 break; 1343 case BPF_ALU | BPF_RSH | BPF_K: 1344 case BPF_ALU64 | BPF_RSH | BPF_K: 1345 if (is64) 1346 emit_srli(rd, rd, imm, ctx); 1347 else 1348 emit(rv_srliw(rd, rd, imm), ctx); 1349 1350 if (!is64 && !aux->verifier_zext) 1351 emit_zextw(rd, rd, ctx); 1352 break; 1353 case BPF_ALU | BPF_ARSH | BPF_K: 1354 case BPF_ALU64 | BPF_ARSH | BPF_K: 1355 if (is64) 1356 emit_srai(rd, rd, imm, ctx); 1357 else 1358 emit(rv_sraiw(rd, rd, imm), ctx); 1359 1360 if (!is64 && !aux->verifier_zext) 1361 emit_zextw(rd, rd, ctx); 1362 break; 1363 1364 /* JUMP off */ 1365 case BPF_JMP | BPF_JA: 1366 case BPF_JMP32 | BPF_JA: 1367 if (BPF_CLASS(code) == BPF_JMP) 1368 rvoff = rv_offset(i, off, ctx); 1369 else 1370 rvoff = rv_offset(i, imm, ctx); 1371 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1372 if (ret) 1373 return ret; 1374 break; 1375 1376 /* IF (dst COND src) JUMP off */ 1377 case BPF_JMP | BPF_JEQ | BPF_X: 1378 case BPF_JMP32 | BPF_JEQ | BPF_X: 1379 case BPF_JMP | BPF_JGT | BPF_X: 1380 case BPF_JMP32 | BPF_JGT | BPF_X: 1381 case BPF_JMP | BPF_JLT | BPF_X: 1382 case BPF_JMP32 | BPF_JLT | BPF_X: 1383 case BPF_JMP | BPF_JGE | BPF_X: 1384 case BPF_JMP32 | BPF_JGE | BPF_X: 1385 case BPF_JMP | BPF_JLE | BPF_X: 1386 case BPF_JMP32 | BPF_JLE | BPF_X: 1387 case BPF_JMP | BPF_JNE | BPF_X: 1388 case BPF_JMP32 | BPF_JNE | BPF_X: 1389 case BPF_JMP | BPF_JSGT | BPF_X: 1390 case BPF_JMP32 | BPF_JSGT | BPF_X: 1391 case BPF_JMP | BPF_JSLT | BPF_X: 1392 case BPF_JMP32 | BPF_JSLT | BPF_X: 1393 case BPF_JMP | BPF_JSGE | BPF_X: 1394 case BPF_JMP32 | BPF_JSGE | BPF_X: 1395 case BPF_JMP | BPF_JSLE | BPF_X: 1396 case BPF_JMP32 | BPF_JSLE | BPF_X: 1397 case BPF_JMP | BPF_JSET | BPF_X: 1398 case BPF_JMP32 | BPF_JSET | BPF_X: 1399 rvoff = rv_offset(i, off, ctx); 1400 if (!is64) { 1401 s = ctx->ninsns; 1402 if (is_signed_bpf_cond(BPF_OP(code))) { 1403 emit_sextw_alt(&rs, RV_REG_T1, ctx); 1404 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1405 } else { 1406 emit_zextw_alt(&rs, RV_REG_T1, ctx); 1407 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1408 } 1409 e = ctx->ninsns; 1410 1411 /* Adjust for extra insns */ 1412 rvoff -= ninsns_rvoff(e - s); 1413 } 1414 1415 if (BPF_OP(code) == BPF_JSET) { 1416 /* Adjust for and */ 1417 rvoff -= 4; 1418 emit_and(RV_REG_T1, rd, rs, ctx); 1419 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1420 } else { 1421 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1422 } 1423 break; 1424 1425 /* IF (dst COND imm) JUMP off */ 1426 case BPF_JMP | BPF_JEQ | BPF_K: 1427 case BPF_JMP32 | BPF_JEQ | BPF_K: 1428 case BPF_JMP | BPF_JGT | BPF_K: 1429 case BPF_JMP32 | BPF_JGT | BPF_K: 1430 case BPF_JMP | BPF_JLT | BPF_K: 1431 case BPF_JMP32 | BPF_JLT | BPF_K: 1432 case BPF_JMP | BPF_JGE | BPF_K: 1433 case BPF_JMP32 | BPF_JGE | BPF_K: 1434 case BPF_JMP | BPF_JLE | BPF_K: 1435 case BPF_JMP32 | BPF_JLE | BPF_K: 1436 case BPF_JMP | BPF_JNE | BPF_K: 1437 case BPF_JMP32 | BPF_JNE | BPF_K: 1438 case BPF_JMP | BPF_JSGT | BPF_K: 1439 case BPF_JMP32 | BPF_JSGT | BPF_K: 1440 case BPF_JMP | BPF_JSLT | BPF_K: 1441 case BPF_JMP32 | BPF_JSLT | BPF_K: 1442 case BPF_JMP | BPF_JSGE | BPF_K: 1443 case BPF_JMP32 | BPF_JSGE | BPF_K: 1444 case BPF_JMP | BPF_JSLE | BPF_K: 1445 case BPF_JMP32 | BPF_JSLE | BPF_K: 1446 rvoff = rv_offset(i, off, ctx); 1447 s = ctx->ninsns; 1448 if (imm) 1449 emit_imm(RV_REG_T1, imm, ctx); 1450 rs = imm ? RV_REG_T1 : RV_REG_ZERO; 1451 if (!is64) { 1452 if (is_signed_bpf_cond(BPF_OP(code))) { 1453 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1454 /* rs has been sign extended */ 1455 } else { 1456 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1457 if (imm) 1458 emit_zextw(rs, rs, ctx); 1459 } 1460 } 1461 e = ctx->ninsns; 1462 1463 /* Adjust for extra insns */ 1464 rvoff -= ninsns_rvoff(e - s); 1465 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1466 break; 1467 1468 case BPF_JMP | BPF_JSET | BPF_K: 1469 case BPF_JMP32 | BPF_JSET | BPF_K: 1470 rvoff = rv_offset(i, off, ctx); 1471 s = ctx->ninsns; 1472 if (is_12b_int(imm)) { 1473 emit_andi(RV_REG_T1, rd, imm, ctx); 1474 } else { 1475 emit_imm(RV_REG_T1, imm, ctx); 1476 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1477 } 1478 /* For jset32, we should clear the upper 32 bits of t1, but 1479 * sign-extension is sufficient here and saves one instruction, 1480 * as t1 is used only in comparison against zero. 1481 */ 1482 if (!is64 && imm < 0) 1483 emit_sextw(RV_REG_T1, RV_REG_T1, ctx); 1484 e = ctx->ninsns; 1485 rvoff -= ninsns_rvoff(e - s); 1486 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1487 break; 1488 1489 /* function call */ 1490 case BPF_JMP | BPF_CALL: 1491 { 1492 bool fixed_addr; 1493 u64 addr; 1494 1495 /* Inline calls to bpf_get_smp_processor_id() 1496 * 1497 * RV_REG_TP holds the address of the current CPU's task_struct and thread_info is 1498 * at offset 0 in task_struct. 1499 * Load cpu from thread_info: 1500 * Set R0 to ((struct thread_info *)(RV_REG_TP))->cpu 1501 * 1502 * This replicates the implementation of raw_smp_processor_id() on RISCV 1503 */ 1504 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { 1505 /* Load current CPU number in R0 */ 1506 emit_ld(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu), 1507 RV_REG_TP, ctx); 1508 break; 1509 } 1510 1511 mark_call(ctx); 1512 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1513 &addr, &fixed_addr); 1514 if (ret < 0) 1515 return ret; 1516 1517 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1518 const struct btf_func_model *fm; 1519 int idx; 1520 1521 fm = bpf_jit_find_kfunc_model(ctx->prog, insn); 1522 if (!fm) 1523 return -EINVAL; 1524 1525 for (idx = 0; idx < fm->nr_args; idx++) { 1526 u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); 1527 1528 if (fm->arg_size[idx] == sizeof(int)) 1529 emit_sextw(reg, reg, ctx); 1530 } 1531 } 1532 1533 ret = emit_call(addr, fixed_addr, ctx); 1534 if (ret) 1535 return ret; 1536 1537 if (insn->src_reg != BPF_PSEUDO_CALL) 1538 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1539 break; 1540 } 1541 /* tail call */ 1542 case BPF_JMP | BPF_TAIL_CALL: 1543 if (emit_bpf_tail_call(i, ctx)) 1544 return -1; 1545 break; 1546 1547 /* function return */ 1548 case BPF_JMP | BPF_EXIT: 1549 if (i == ctx->prog->len - 1) 1550 break; 1551 1552 rvoff = epilogue_offset(ctx); 1553 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1554 if (ret) 1555 return ret; 1556 break; 1557 1558 /* dst = imm64 */ 1559 case BPF_LD | BPF_IMM | BPF_DW: 1560 { 1561 struct bpf_insn insn1 = insn[1]; 1562 u64 imm64; 1563 1564 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1565 if (bpf_pseudo_func(insn)) { 1566 /* fixed-length insns for extra jit pass */ 1567 ret = emit_addr(rd, imm64, extra_pass, ctx); 1568 if (ret) 1569 return ret; 1570 } else { 1571 emit_imm(rd, imm64, ctx); 1572 } 1573 1574 return 1; 1575 } 1576 1577 /* LDX: dst = *(unsigned size *)(src + off) */ 1578 case BPF_LDX | BPF_MEM | BPF_B: 1579 case BPF_LDX | BPF_MEM | BPF_H: 1580 case BPF_LDX | BPF_MEM | BPF_W: 1581 case BPF_LDX | BPF_MEM | BPF_DW: 1582 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1583 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1584 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1585 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1586 /* LDSX: dst = *(signed size *)(src + off) */ 1587 case BPF_LDX | BPF_MEMSX | BPF_B: 1588 case BPF_LDX | BPF_MEMSX | BPF_H: 1589 case BPF_LDX | BPF_MEMSX | BPF_W: 1590 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1591 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1592 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1593 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + RV_REG_ARENA + off) */ 1594 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1595 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1596 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1597 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1598 { 1599 int insn_len, insns_start; 1600 bool sign_ext; 1601 1602 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1603 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1604 1605 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1606 emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); 1607 rs = RV_REG_T2; 1608 } 1609 1610 switch (BPF_SIZE(code)) { 1611 case BPF_B: 1612 if (is_12b_int(off)) { 1613 insns_start = ctx->ninsns; 1614 if (sign_ext) 1615 emit(rv_lb(rd, off, rs), ctx); 1616 else 1617 emit(rv_lbu(rd, off, rs), ctx); 1618 insn_len = ctx->ninsns - insns_start; 1619 break; 1620 } 1621 1622 emit_imm(RV_REG_T1, off, ctx); 1623 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1624 insns_start = ctx->ninsns; 1625 if (sign_ext) 1626 emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1627 else 1628 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1629 insn_len = ctx->ninsns - insns_start; 1630 break; 1631 case BPF_H: 1632 if (is_12b_int(off)) { 1633 insns_start = ctx->ninsns; 1634 if (sign_ext) 1635 emit(rv_lh(rd, off, rs), ctx); 1636 else 1637 emit(rv_lhu(rd, off, rs), ctx); 1638 insn_len = ctx->ninsns - insns_start; 1639 break; 1640 } 1641 1642 emit_imm(RV_REG_T1, off, ctx); 1643 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1644 insns_start = ctx->ninsns; 1645 if (sign_ext) 1646 emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1647 else 1648 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1649 insn_len = ctx->ninsns - insns_start; 1650 break; 1651 case BPF_W: 1652 if (is_12b_int(off)) { 1653 insns_start = ctx->ninsns; 1654 if (sign_ext) 1655 emit(rv_lw(rd, off, rs), ctx); 1656 else 1657 emit(rv_lwu(rd, off, rs), ctx); 1658 insn_len = ctx->ninsns - insns_start; 1659 break; 1660 } 1661 1662 emit_imm(RV_REG_T1, off, ctx); 1663 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1664 insns_start = ctx->ninsns; 1665 if (sign_ext) 1666 emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1667 else 1668 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1669 insn_len = ctx->ninsns - insns_start; 1670 break; 1671 case BPF_DW: 1672 if (is_12b_int(off)) { 1673 insns_start = ctx->ninsns; 1674 emit_ld(rd, off, rs, ctx); 1675 insn_len = ctx->ninsns - insns_start; 1676 break; 1677 } 1678 1679 emit_imm(RV_REG_T1, off, ctx); 1680 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1681 insns_start = ctx->ninsns; 1682 emit_ld(rd, 0, RV_REG_T1, ctx); 1683 insn_len = ctx->ninsns - insns_start; 1684 break; 1685 } 1686 1687 ret = add_exception_handler(insn, ctx, rd, insn_len); 1688 if (ret) 1689 return ret; 1690 1691 if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1692 return 1; 1693 break; 1694 } 1695 /* speculation barrier */ 1696 case BPF_ST | BPF_NOSPEC: 1697 break; 1698 1699 /* ST: *(size *)(dst + off) = imm */ 1700 case BPF_ST | BPF_MEM | BPF_B: 1701 emit_imm(RV_REG_T1, imm, ctx); 1702 if (is_12b_int(off)) { 1703 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1704 break; 1705 } 1706 1707 emit_imm(RV_REG_T2, off, ctx); 1708 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1709 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1710 break; 1711 1712 case BPF_ST | BPF_MEM | BPF_H: 1713 emit_imm(RV_REG_T1, imm, ctx); 1714 if (is_12b_int(off)) { 1715 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1716 break; 1717 } 1718 1719 emit_imm(RV_REG_T2, off, ctx); 1720 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1721 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1722 break; 1723 case BPF_ST | BPF_MEM | BPF_W: 1724 emit_imm(RV_REG_T1, imm, ctx); 1725 if (is_12b_int(off)) { 1726 emit_sw(rd, off, RV_REG_T1, ctx); 1727 break; 1728 } 1729 1730 emit_imm(RV_REG_T2, off, ctx); 1731 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1732 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1733 break; 1734 case BPF_ST | BPF_MEM | BPF_DW: 1735 emit_imm(RV_REG_T1, imm, ctx); 1736 if (is_12b_int(off)) { 1737 emit_sd(rd, off, RV_REG_T1, ctx); 1738 break; 1739 } 1740 1741 emit_imm(RV_REG_T2, off, ctx); 1742 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1743 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1744 break; 1745 1746 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1747 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1748 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1749 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1750 { 1751 int insn_len, insns_start; 1752 1753 emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); 1754 rd = RV_REG_T3; 1755 1756 /* Load imm to a register then store it */ 1757 emit_imm(RV_REG_T1, imm, ctx); 1758 1759 switch (BPF_SIZE(code)) { 1760 case BPF_B: 1761 if (is_12b_int(off)) { 1762 insns_start = ctx->ninsns; 1763 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1764 insn_len = ctx->ninsns - insns_start; 1765 break; 1766 } 1767 1768 emit_imm(RV_REG_T2, off, ctx); 1769 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1770 insns_start = ctx->ninsns; 1771 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1772 insn_len = ctx->ninsns - insns_start; 1773 break; 1774 case BPF_H: 1775 if (is_12b_int(off)) { 1776 insns_start = ctx->ninsns; 1777 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1778 insn_len = ctx->ninsns - insns_start; 1779 break; 1780 } 1781 1782 emit_imm(RV_REG_T2, off, ctx); 1783 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1784 insns_start = ctx->ninsns; 1785 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1786 insn_len = ctx->ninsns - insns_start; 1787 break; 1788 case BPF_W: 1789 if (is_12b_int(off)) { 1790 insns_start = ctx->ninsns; 1791 emit_sw(rd, off, RV_REG_T1, ctx); 1792 insn_len = ctx->ninsns - insns_start; 1793 break; 1794 } 1795 1796 emit_imm(RV_REG_T2, off, ctx); 1797 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1798 insns_start = ctx->ninsns; 1799 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1800 insn_len = ctx->ninsns - insns_start; 1801 break; 1802 case BPF_DW: 1803 if (is_12b_int(off)) { 1804 insns_start = ctx->ninsns; 1805 emit_sd(rd, off, RV_REG_T1, ctx); 1806 insn_len = ctx->ninsns - insns_start; 1807 break; 1808 } 1809 1810 emit_imm(RV_REG_T2, off, ctx); 1811 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1812 insns_start = ctx->ninsns; 1813 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1814 insn_len = ctx->ninsns - insns_start; 1815 break; 1816 } 1817 1818 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1819 insn_len); 1820 if (ret) 1821 return ret; 1822 1823 break; 1824 } 1825 1826 /* STX: *(size *)(dst + off) = src */ 1827 case BPF_STX | BPF_MEM | BPF_B: 1828 if (is_12b_int(off)) { 1829 emit(rv_sb(rd, off, rs), ctx); 1830 break; 1831 } 1832 1833 emit_imm(RV_REG_T1, off, ctx); 1834 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1835 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1836 break; 1837 case BPF_STX | BPF_MEM | BPF_H: 1838 if (is_12b_int(off)) { 1839 emit(rv_sh(rd, off, rs), ctx); 1840 break; 1841 } 1842 1843 emit_imm(RV_REG_T1, off, ctx); 1844 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1845 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1846 break; 1847 case BPF_STX | BPF_MEM | BPF_W: 1848 if (is_12b_int(off)) { 1849 emit_sw(rd, off, rs, ctx); 1850 break; 1851 } 1852 1853 emit_imm(RV_REG_T1, off, ctx); 1854 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1855 emit_sw(RV_REG_T1, 0, rs, ctx); 1856 break; 1857 case BPF_STX | BPF_MEM | BPF_DW: 1858 if (is_12b_int(off)) { 1859 emit_sd(rd, off, rs, ctx); 1860 break; 1861 } 1862 1863 emit_imm(RV_REG_T1, off, ctx); 1864 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1865 emit_sd(RV_REG_T1, 0, rs, ctx); 1866 break; 1867 case BPF_STX | BPF_ATOMIC | BPF_W: 1868 case BPF_STX | BPF_ATOMIC | BPF_DW: 1869 emit_atomic(rd, rs, off, imm, 1870 BPF_SIZE(code) == BPF_DW, ctx); 1871 break; 1872 1873 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1874 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1875 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1876 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1877 { 1878 int insn_len, insns_start; 1879 1880 emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 1881 rd = RV_REG_T2; 1882 1883 switch (BPF_SIZE(code)) { 1884 case BPF_B: 1885 if (is_12b_int(off)) { 1886 insns_start = ctx->ninsns; 1887 emit(rv_sb(rd, off, rs), ctx); 1888 insn_len = ctx->ninsns - insns_start; 1889 break; 1890 } 1891 1892 emit_imm(RV_REG_T1, off, ctx); 1893 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1894 insns_start = ctx->ninsns; 1895 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1896 insn_len = ctx->ninsns - insns_start; 1897 break; 1898 case BPF_H: 1899 if (is_12b_int(off)) { 1900 insns_start = ctx->ninsns; 1901 emit(rv_sh(rd, off, rs), ctx); 1902 insn_len = ctx->ninsns - insns_start; 1903 break; 1904 } 1905 1906 emit_imm(RV_REG_T1, off, ctx); 1907 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1908 insns_start = ctx->ninsns; 1909 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1910 insn_len = ctx->ninsns - insns_start; 1911 break; 1912 case BPF_W: 1913 if (is_12b_int(off)) { 1914 insns_start = ctx->ninsns; 1915 emit_sw(rd, off, rs, ctx); 1916 insn_len = ctx->ninsns - insns_start; 1917 break; 1918 } 1919 1920 emit_imm(RV_REG_T1, off, ctx); 1921 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1922 insns_start = ctx->ninsns; 1923 emit_sw(RV_REG_T1, 0, rs, ctx); 1924 insn_len = ctx->ninsns - insns_start; 1925 break; 1926 case BPF_DW: 1927 if (is_12b_int(off)) { 1928 insns_start = ctx->ninsns; 1929 emit_sd(rd, off, rs, ctx); 1930 insn_len = ctx->ninsns - insns_start; 1931 break; 1932 } 1933 1934 emit_imm(RV_REG_T1, off, ctx); 1935 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1936 insns_start = ctx->ninsns; 1937 emit_sd(RV_REG_T1, 0, rs, ctx); 1938 insn_len = ctx->ninsns - insns_start; 1939 break; 1940 } 1941 1942 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1943 insn_len); 1944 if (ret) 1945 return ret; 1946 1947 break; 1948 } 1949 1950 default: 1951 pr_err("bpf-jit: unknown opcode %02x\n", code); 1952 return -EINVAL; 1953 } 1954 1955 return 0; 1956 } 1957 1958 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) 1959 { 1960 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1961 1962 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); 1963 if (bpf_stack_adjust) 1964 mark_fp(ctx); 1965 1966 if (seen_reg(RV_REG_RA, ctx)) 1967 stack_adjust += 8; 1968 stack_adjust += 8; /* RV_REG_FP */ 1969 if (seen_reg(RV_REG_S1, ctx)) 1970 stack_adjust += 8; 1971 if (seen_reg(RV_REG_S2, ctx)) 1972 stack_adjust += 8; 1973 if (seen_reg(RV_REG_S3, ctx)) 1974 stack_adjust += 8; 1975 if (seen_reg(RV_REG_S4, ctx)) 1976 stack_adjust += 8; 1977 if (seen_reg(RV_REG_S5, ctx)) 1978 stack_adjust += 8; 1979 if (seen_reg(RV_REG_S6, ctx)) 1980 stack_adjust += 8; 1981 if (ctx->arena_vm_start) 1982 stack_adjust += 8; 1983 1984 stack_adjust = round_up(stack_adjust, STACK_ALIGN); 1985 stack_adjust += bpf_stack_adjust; 1986 1987 store_offset = stack_adjust - 8; 1988 1989 /* emit kcfi type preamble immediately before the first insn */ 1990 emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); 1991 1992 /* nops reserved for auipc+jalr pair */ 1993 for (i = 0; i < RV_FENTRY_NINSNS; i++) 1994 emit(rv_nop(), ctx); 1995 1996 /* First instruction is always setting the tail-call-counter 1997 * (TCC) register. This instruction is skipped for tail calls. 1998 * Force using a 4-byte (non-compressed) instruction. 1999 */ 2000 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 2001 2002 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 2003 2004 if (seen_reg(RV_REG_RA, ctx)) { 2005 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 2006 store_offset -= 8; 2007 } 2008 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 2009 store_offset -= 8; 2010 if (seen_reg(RV_REG_S1, ctx)) { 2011 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 2012 store_offset -= 8; 2013 } 2014 if (seen_reg(RV_REG_S2, ctx)) { 2015 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 2016 store_offset -= 8; 2017 } 2018 if (seen_reg(RV_REG_S3, ctx)) { 2019 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 2020 store_offset -= 8; 2021 } 2022 if (seen_reg(RV_REG_S4, ctx)) { 2023 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 2024 store_offset -= 8; 2025 } 2026 if (seen_reg(RV_REG_S5, ctx)) { 2027 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 2028 store_offset -= 8; 2029 } 2030 if (seen_reg(RV_REG_S6, ctx)) { 2031 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 2032 store_offset -= 8; 2033 } 2034 if (ctx->arena_vm_start) { 2035 emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); 2036 store_offset -= 8; 2037 } 2038 2039 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 2040 2041 if (bpf_stack_adjust) 2042 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 2043 2044 /* Program contains calls and tail calls, so RV_REG_TCC need 2045 * to be saved across calls. 2046 */ 2047 if (seen_tail_call(ctx) && seen_call(ctx)) 2048 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 2049 2050 ctx->stack_size = stack_adjust; 2051 2052 if (ctx->arena_vm_start) 2053 emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx); 2054 } 2055 2056 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 2057 { 2058 __build_epilogue(false, ctx); 2059 } 2060 2061 bool bpf_jit_supports_kfunc_call(void) 2062 { 2063 return true; 2064 } 2065 2066 bool bpf_jit_supports_ptr_xchg(void) 2067 { 2068 return true; 2069 } 2070 2071 bool bpf_jit_supports_arena(void) 2072 { 2073 return true; 2074 } 2075 2076 bool bpf_jit_supports_percpu_insn(void) 2077 { 2078 return true; 2079 } 2080 2081 bool bpf_jit_inlines_helper_call(s32 imm) 2082 { 2083 switch (imm) { 2084 case BPF_FUNC_get_smp_processor_id: 2085 return true; 2086 default: 2087 return false; 2088 } 2089 } 2090