1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <asm/patch.h> 14 #include <asm/cfi.h> 15 #include <asm/percpu.h> 16 #include "bpf_jit.h" 17 18 #define RV_FENTRY_NINSNS 2 19 20 #define RV_REG_TCC RV_REG_A6 21 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 22 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ 23 24 static const int regmap[] = { 25 [BPF_REG_0] = RV_REG_A5, 26 [BPF_REG_1] = RV_REG_A0, 27 [BPF_REG_2] = RV_REG_A1, 28 [BPF_REG_3] = RV_REG_A2, 29 [BPF_REG_4] = RV_REG_A3, 30 [BPF_REG_5] = RV_REG_A4, 31 [BPF_REG_6] = RV_REG_S1, 32 [BPF_REG_7] = RV_REG_S2, 33 [BPF_REG_8] = RV_REG_S3, 34 [BPF_REG_9] = RV_REG_S4, 35 [BPF_REG_FP] = RV_REG_S5, 36 [BPF_REG_AX] = RV_REG_T0, 37 }; 38 39 static const int pt_regmap[] = { 40 [RV_REG_A0] = offsetof(struct pt_regs, a0), 41 [RV_REG_A1] = offsetof(struct pt_regs, a1), 42 [RV_REG_A2] = offsetof(struct pt_regs, a2), 43 [RV_REG_A3] = offsetof(struct pt_regs, a3), 44 [RV_REG_A4] = offsetof(struct pt_regs, a4), 45 [RV_REG_A5] = offsetof(struct pt_regs, a5), 46 [RV_REG_S1] = offsetof(struct pt_regs, s1), 47 [RV_REG_S2] = offsetof(struct pt_regs, s2), 48 [RV_REG_S3] = offsetof(struct pt_regs, s3), 49 [RV_REG_S4] = offsetof(struct pt_regs, s4), 50 [RV_REG_S5] = offsetof(struct pt_regs, s5), 51 [RV_REG_T0] = offsetof(struct pt_regs, t0), 52 }; 53 54 enum { 55 RV_CTX_F_SEEN_TAIL_CALL = 0, 56 RV_CTX_F_SEEN_CALL = RV_REG_RA, 57 RV_CTX_F_SEEN_S1 = RV_REG_S1, 58 RV_CTX_F_SEEN_S2 = RV_REG_S2, 59 RV_CTX_F_SEEN_S3 = RV_REG_S3, 60 RV_CTX_F_SEEN_S4 = RV_REG_S4, 61 RV_CTX_F_SEEN_S5 = RV_REG_S5, 62 RV_CTX_F_SEEN_S6 = RV_REG_S6, 63 }; 64 65 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 66 { 67 u8 reg = regmap[bpf_reg]; 68 69 switch (reg) { 70 case RV_CTX_F_SEEN_S1: 71 case RV_CTX_F_SEEN_S2: 72 case RV_CTX_F_SEEN_S3: 73 case RV_CTX_F_SEEN_S4: 74 case RV_CTX_F_SEEN_S5: 75 case RV_CTX_F_SEEN_S6: 76 __set_bit(reg, &ctx->flags); 77 } 78 return reg; 79 }; 80 81 static bool seen_reg(int reg, struct rv_jit_context *ctx) 82 { 83 switch (reg) { 84 case RV_CTX_F_SEEN_CALL: 85 case RV_CTX_F_SEEN_S1: 86 case RV_CTX_F_SEEN_S2: 87 case RV_CTX_F_SEEN_S3: 88 case RV_CTX_F_SEEN_S4: 89 case RV_CTX_F_SEEN_S5: 90 case RV_CTX_F_SEEN_S6: 91 return test_bit(reg, &ctx->flags); 92 } 93 return false; 94 } 95 96 static void mark_fp(struct rv_jit_context *ctx) 97 { 98 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 99 } 100 101 static void mark_call(struct rv_jit_context *ctx) 102 { 103 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 104 } 105 106 static bool seen_call(struct rv_jit_context *ctx) 107 { 108 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 109 } 110 111 static void mark_tail_call(struct rv_jit_context *ctx) 112 { 113 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 114 } 115 116 static bool seen_tail_call(struct rv_jit_context *ctx) 117 { 118 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 119 } 120 121 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 122 { 123 mark_tail_call(ctx); 124 125 if (seen_call(ctx)) { 126 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 127 return RV_REG_S6; 128 } 129 return RV_REG_A6; 130 } 131 132 static bool is_32b_int(s64 val) 133 { 134 return -(1L << 31) <= val && val < (1L << 31); 135 } 136 137 static bool in_auipc_jalr_range(s64 val) 138 { 139 /* 140 * auipc+jalr can reach any signed PC-relative offset in the range 141 * [-2^31 - 2^11, 2^31 - 2^11). 142 */ 143 return (-(1L << 31) - (1L << 11)) <= val && 144 val < ((1L << 31) - (1L << 11)); 145 } 146 147 /* Modify rd pointer to alternate reg to avoid corrupting original reg */ 148 static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 149 { 150 emit_sextw(ra, *rd, ctx); 151 *rd = ra; 152 } 153 154 static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 155 { 156 emit_zextw(ra, *rd, ctx); 157 *rd = ra; 158 } 159 160 /* Emit fixed-length instructions for address */ 161 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 162 { 163 /* 164 * Use the ro_insns(RX) to calculate the offset as the BPF program will 165 * finally run from this memory region. 166 */ 167 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); 168 s64 off = addr - ip; 169 s64 upper = (off + (1 << 11)) >> 12; 170 s64 lower = off & 0xfff; 171 172 if (extra_pass && !in_auipc_jalr_range(off)) { 173 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 174 return -ERANGE; 175 } 176 177 emit(rv_auipc(rd, upper), ctx); 178 emit(rv_addi(rd, rd, lower), ctx); 179 return 0; 180 } 181 182 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 183 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 184 { 185 /* Note that the immediate from the add is sign-extended, 186 * which means that we need to compensate this by adding 2^12, 187 * when the 12th bit is set. A simpler way of doing this, and 188 * getting rid of the check, is to just add 2**11 before the 189 * shift. The "Loading a 32-Bit constant" example from the 190 * "Computer Organization and Design, RISC-V edition" book by 191 * Patterson/Hennessy highlights this fact. 192 * 193 * This also means that we need to process LSB to MSB. 194 */ 195 s64 upper = (val + (1 << 11)) >> 12; 196 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 197 * and addi are signed and RVC checks will perform signed comparisons. 198 */ 199 s64 lower = ((val & 0xfff) << 52) >> 52; 200 int shift; 201 202 if (is_32b_int(val)) { 203 if (upper) 204 emit_lui(rd, upper, ctx); 205 206 if (!upper) { 207 emit_li(rd, lower, ctx); 208 return; 209 } 210 211 emit_addiw(rd, rd, lower, ctx); 212 return; 213 } 214 215 shift = __ffs(upper); 216 upper >>= shift; 217 shift += 12; 218 219 emit_imm(rd, upper, ctx); 220 221 emit_slli(rd, rd, shift, ctx); 222 if (lower) 223 emit_addi(rd, rd, lower, ctx); 224 } 225 226 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 227 { 228 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 229 230 if (seen_reg(RV_REG_RA, ctx)) { 231 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 232 store_offset -= 8; 233 } 234 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 235 store_offset -= 8; 236 if (seen_reg(RV_REG_S1, ctx)) { 237 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 238 store_offset -= 8; 239 } 240 if (seen_reg(RV_REG_S2, ctx)) { 241 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 242 store_offset -= 8; 243 } 244 if (seen_reg(RV_REG_S3, ctx)) { 245 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 246 store_offset -= 8; 247 } 248 if (seen_reg(RV_REG_S4, ctx)) { 249 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 250 store_offset -= 8; 251 } 252 if (seen_reg(RV_REG_S5, ctx)) { 253 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 254 store_offset -= 8; 255 } 256 if (seen_reg(RV_REG_S6, ctx)) { 257 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 258 store_offset -= 8; 259 } 260 if (ctx->arena_vm_start) { 261 emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); 262 store_offset -= 8; 263 } 264 265 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 266 /* Set return value. */ 267 if (!is_tail_call) 268 emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); 269 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 270 is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 271 ctx); 272 } 273 274 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 275 struct rv_jit_context *ctx) 276 { 277 switch (cond) { 278 case BPF_JEQ: 279 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 280 return; 281 case BPF_JGT: 282 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 283 return; 284 case BPF_JLT: 285 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 286 return; 287 case BPF_JGE: 288 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 289 return; 290 case BPF_JLE: 291 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 292 return; 293 case BPF_JNE: 294 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 295 return; 296 case BPF_JSGT: 297 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 298 return; 299 case BPF_JSLT: 300 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 301 return; 302 case BPF_JSGE: 303 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 304 return; 305 case BPF_JSLE: 306 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 307 } 308 } 309 310 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 311 struct rv_jit_context *ctx) 312 { 313 s64 upper, lower; 314 315 if (is_13b_int(rvoff)) { 316 emit_bcc(cond, rd, rs, rvoff, ctx); 317 return; 318 } 319 320 /* Adjust for jal */ 321 rvoff -= 4; 322 323 /* Transform, e.g.: 324 * bne rd,rs,foo 325 * to 326 * beq rd,rs,<.L1> 327 * (auipc foo) 328 * jal(r) foo 329 * .L1 330 */ 331 cond = invert_bpf_cond(cond); 332 if (is_21b_int(rvoff)) { 333 emit_bcc(cond, rd, rs, 8, ctx); 334 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 335 return; 336 } 337 338 /* 32b No need for an additional rvoff adjustment, since we 339 * get that from the auipc at PC', where PC = PC' + 4. 340 */ 341 upper = (rvoff + (1 << 11)) >> 12; 342 lower = rvoff & 0xfff; 343 344 emit_bcc(cond, rd, rs, 12, ctx); 345 emit(rv_auipc(RV_REG_T1, upper), ctx); 346 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 347 } 348 349 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 350 { 351 int tc_ninsn, off, start_insn = ctx->ninsns; 352 u8 tcc = rv_tail_call_reg(ctx); 353 354 /* a0: &ctx 355 * a1: &array 356 * a2: index 357 * 358 * if (index >= array->map.max_entries) 359 * goto out; 360 */ 361 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 362 ctx->offset[0]; 363 emit_zextw(RV_REG_A2, RV_REG_A2, ctx); 364 365 off = offsetof(struct bpf_array, map.max_entries); 366 if (is_12b_check(off, insn)) 367 return -1; 368 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 369 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 370 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 371 372 /* if (--TCC < 0) 373 * goto out; 374 */ 375 emit_addi(RV_REG_TCC, tcc, -1, ctx); 376 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 377 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 378 379 /* prog = array->ptrs[index]; 380 * if (!prog) 381 * goto out; 382 */ 383 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 384 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 385 off = offsetof(struct bpf_array, ptrs); 386 if (is_12b_check(off, insn)) 387 return -1; 388 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 389 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 390 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 391 392 /* goto *(prog->bpf_func + 4); */ 393 off = offsetof(struct bpf_prog, bpf_func); 394 if (is_12b_check(off, insn)) 395 return -1; 396 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 397 __build_epilogue(true, ctx); 398 return 0; 399 } 400 401 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 402 struct rv_jit_context *ctx) 403 { 404 u8 code = insn->code; 405 406 switch (code) { 407 case BPF_JMP | BPF_JA: 408 case BPF_JMP | BPF_CALL: 409 case BPF_JMP | BPF_EXIT: 410 case BPF_JMP | BPF_TAIL_CALL: 411 break; 412 default: 413 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 414 } 415 416 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 417 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 418 code & BPF_LDX || code & BPF_STX) 419 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 420 } 421 422 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 423 struct rv_jit_context *ctx) 424 { 425 s64 upper, lower; 426 427 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 428 emit(rv_jal(rd, rvoff >> 1), ctx); 429 return 0; 430 } else if (in_auipc_jalr_range(rvoff)) { 431 upper = (rvoff + (1 << 11)) >> 12; 432 lower = rvoff & 0xfff; 433 emit(rv_auipc(RV_REG_T1, upper), ctx); 434 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 435 return 0; 436 } 437 438 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 439 return -ERANGE; 440 } 441 442 static bool is_signed_bpf_cond(u8 cond) 443 { 444 return cond == BPF_JSGT || cond == BPF_JSLT || 445 cond == BPF_JSGE || cond == BPF_JSLE; 446 } 447 448 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 449 { 450 s64 off = 0; 451 u64 ip; 452 453 if (addr && ctx->insns && ctx->ro_insns) { 454 /* 455 * Use the ro_insns(RX) to calculate the offset as the BPF 456 * program will finally run from this memory region. 457 */ 458 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); 459 off = addr - ip; 460 } 461 462 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 463 } 464 465 static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) 466 { 467 if (IS_ENABLED(CONFIG_CFI_CLANG)) 468 emit(hash, ctx); 469 } 470 471 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 472 struct rv_jit_context *ctx) 473 { 474 u8 r0; 475 int jmp_offset; 476 477 if (off) { 478 if (is_12b_int(off)) { 479 emit_addi(RV_REG_T1, rd, off, ctx); 480 } else { 481 emit_imm(RV_REG_T1, off, ctx); 482 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 483 } 484 rd = RV_REG_T1; 485 } 486 487 switch (imm) { 488 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 489 case BPF_ADD: 490 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 491 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 492 break; 493 case BPF_AND: 494 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 495 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 496 break; 497 case BPF_OR: 498 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 499 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 500 break; 501 case BPF_XOR: 502 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 503 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 504 break; 505 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 506 case BPF_ADD | BPF_FETCH: 507 emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) : 508 rv_amoadd_w(rs, rs, rd, 1, 1), ctx); 509 if (!is64) 510 emit_zextw(rs, rs, ctx); 511 break; 512 case BPF_AND | BPF_FETCH: 513 emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) : 514 rv_amoand_w(rs, rs, rd, 1, 1), ctx); 515 if (!is64) 516 emit_zextw(rs, rs, ctx); 517 break; 518 case BPF_OR | BPF_FETCH: 519 emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) : 520 rv_amoor_w(rs, rs, rd, 1, 1), ctx); 521 if (!is64) 522 emit_zextw(rs, rs, ctx); 523 break; 524 case BPF_XOR | BPF_FETCH: 525 emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) : 526 rv_amoxor_w(rs, rs, rd, 1, 1), ctx); 527 if (!is64) 528 emit_zextw(rs, rs, ctx); 529 break; 530 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 531 case BPF_XCHG: 532 emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) : 533 rv_amoswap_w(rs, rs, rd, 1, 1), ctx); 534 if (!is64) 535 emit_zextw(rs, rs, ctx); 536 break; 537 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 538 case BPF_CMPXCHG: 539 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 540 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 541 rv_addiw(RV_REG_T2, r0, 0), ctx); 542 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 543 rv_lr_w(r0, 0, rd, 0, 0), ctx); 544 jmp_offset = ninsns_rvoff(8); 545 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 546 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 547 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 548 jmp_offset = ninsns_rvoff(-6); 549 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 550 emit(rv_fence(0x3, 0x3), ctx); 551 break; 552 } 553 } 554 555 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 556 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 557 #define REG_DONT_CLEAR_MARKER 0 /* RV_REG_ZERO unused in pt_regmap */ 558 559 bool ex_handler_bpf(const struct exception_table_entry *ex, 560 struct pt_regs *regs) 561 { 562 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 563 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 564 565 if (regs_offset != REG_DONT_CLEAR_MARKER) 566 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 567 regs->epc = (unsigned long)&ex->fixup - offset; 568 569 return true; 570 } 571 572 /* For accesses to BTF pointers, add an entry to the exception table */ 573 static int add_exception_handler(const struct bpf_insn *insn, 574 struct rv_jit_context *ctx, 575 int dst_reg, int insn_len) 576 { 577 struct exception_table_entry *ex; 578 unsigned long pc; 579 off_t ins_offset; 580 off_t fixup_offset; 581 582 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 583 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 584 BPF_MODE(insn->code) != BPF_PROBE_MEM32)) 585 return 0; 586 587 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 588 return -EINVAL; 589 590 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 591 return -EINVAL; 592 593 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 594 return -EINVAL; 595 596 ex = &ctx->prog->aux->extable[ctx->nexentries]; 597 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 598 599 /* 600 * This is the relative offset of the instruction that may fault from 601 * the exception table itself. This will be written to the exception 602 * table and if this instruction faults, the destination register will 603 * be set to '0' and the execution will jump to the next instruction. 604 */ 605 ins_offset = pc - (long)&ex->insn; 606 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 607 return -ERANGE; 608 609 /* 610 * Since the extable follows the program, the fixup offset is always 611 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 612 * to keep things simple, and put the destination register in the upper 613 * bits. We don't need to worry about buildtime or runtime sort 614 * modifying the upper bits because the table is already sorted, and 615 * isn't part of the main exception table. 616 * 617 * The fixup_offset is set to the next instruction from the instruction 618 * that may fault. The execution will jump to this after handling the 619 * fault. 620 */ 621 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 622 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 623 return -ERANGE; 624 625 /* 626 * The offsets above have been calculated using the RO buffer but we 627 * need to use the R/W buffer for writes. 628 * switch ex to rw buffer for writing. 629 */ 630 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); 631 632 ex->insn = ins_offset; 633 634 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 635 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 636 ex->type = EX_TYPE_BPF; 637 638 ctx->nexentries++; 639 return 0; 640 } 641 642 static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 643 { 644 s64 rvoff; 645 struct rv_jit_context ctx; 646 647 ctx.ninsns = 0; 648 ctx.insns = (u16 *)insns; 649 650 if (!target) { 651 emit(rv_nop(), &ctx); 652 emit(rv_nop(), &ctx); 653 return 0; 654 } 655 656 rvoff = (s64)(target - ip); 657 return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 658 } 659 660 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 661 void *old_addr, void *new_addr) 662 { 663 u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 664 bool is_call = poke_type == BPF_MOD_CALL; 665 int ret; 666 667 if (!is_kernel_text((unsigned long)ip) && 668 !is_bpf_text_address((unsigned long)ip)) 669 return -ENOTSUPP; 670 671 ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 672 if (ret) 673 return ret; 674 675 if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) 676 return -EFAULT; 677 678 ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 679 if (ret) 680 return ret; 681 682 cpus_read_lock(); 683 mutex_lock(&text_mutex); 684 if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) 685 ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); 686 mutex_unlock(&text_mutex); 687 cpus_read_unlock(); 688 689 return ret; 690 } 691 692 static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) 693 { 694 int i; 695 696 for (i = 0; i < nregs; i++) { 697 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 698 args_off -= 8; 699 } 700 } 701 702 static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) 703 { 704 int i; 705 706 for (i = 0; i < nregs; i++) { 707 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 708 args_off -= 8; 709 } 710 } 711 712 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 713 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 714 { 715 int ret, branch_off; 716 struct bpf_prog *p = l->link.prog; 717 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 718 719 if (l->cookie) { 720 emit_imm(RV_REG_T1, l->cookie, ctx); 721 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 722 } else { 723 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 724 } 725 726 /* arg1: prog */ 727 emit_imm(RV_REG_A0, (const s64)p, ctx); 728 /* arg2: &run_ctx */ 729 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 730 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 731 if (ret) 732 return ret; 733 734 /* store prog start time */ 735 emit_mv(RV_REG_S1, RV_REG_A0, ctx); 736 737 /* if (__bpf_prog_enter(prog) == 0) 738 * goto skip_exec_of_prog; 739 */ 740 branch_off = ctx->ninsns; 741 /* nop reserved for conditional jump */ 742 emit(rv_nop(), ctx); 743 744 /* arg1: &args_off */ 745 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 746 if (!p->jited) 747 /* arg2: progs[i]->insnsi for interpreter */ 748 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 749 ret = emit_call((const u64)p->bpf_func, true, ctx); 750 if (ret) 751 return ret; 752 753 if (save_ret) { 754 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 755 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 756 } 757 758 /* update branch with beqz */ 759 if (ctx->insns) { 760 int offset = ninsns_rvoff(ctx->ninsns - branch_off); 761 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 762 *(u32 *)(ctx->insns + branch_off) = insn; 763 } 764 765 /* arg1: prog */ 766 emit_imm(RV_REG_A0, (const s64)p, ctx); 767 /* arg2: prog start time */ 768 emit_mv(RV_REG_A1, RV_REG_S1, ctx); 769 /* arg3: &run_ctx */ 770 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 771 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 772 773 return ret; 774 } 775 776 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 777 const struct btf_func_model *m, 778 struct bpf_tramp_links *tlinks, 779 void *func_addr, u32 flags, 780 struct rv_jit_context *ctx) 781 { 782 int i, ret, offset; 783 int *branches_off = NULL; 784 int stack_size = 0, nregs = m->nr_args; 785 int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; 786 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 787 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 788 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 789 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 790 void *orig_call = func_addr; 791 bool save_ret; 792 u32 insn; 793 794 /* Two types of generated trampoline stack layout: 795 * 796 * 1. trampoline called from function entry 797 * -------------------------------------- 798 * FP + 8 [ RA to parent func ] return address to parent 799 * function 800 * FP + 0 [ FP of parent func ] frame pointer of parent 801 * function 802 * FP - 8 [ T0 to traced func ] return address of traced 803 * function 804 * FP - 16 [ FP of traced func ] frame pointer of traced 805 * function 806 * -------------------------------------- 807 * 808 * 2. trampoline called directly 809 * -------------------------------------- 810 * FP - 8 [ RA to caller func ] return address to caller 811 * function 812 * FP - 16 [ FP of caller func ] frame pointer of caller 813 * function 814 * -------------------------------------- 815 * 816 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 817 * BPF_TRAMP_F_RET_FENTRY_RET 818 * [ argN ] 819 * [ ... ] 820 * FP - args_off [ arg1 ] 821 * 822 * FP - nregs_off [ regs count ] 823 * 824 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 825 * 826 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 827 * 828 * FP - sreg_off [ callee saved reg ] 829 * 830 * [ pads ] pads for 16 bytes alignment 831 */ 832 833 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 834 return -ENOTSUPP; 835 836 /* extra regiters for struct arguments */ 837 for (i = 0; i < m->nr_args; i++) 838 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 839 nregs += round_up(m->arg_size[i], 8) / 8 - 1; 840 841 /* 8 arguments passed by registers */ 842 if (nregs > 8) 843 return -ENOTSUPP; 844 845 /* room of trampoline frame to store return address and frame pointer */ 846 stack_size += 16; 847 848 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 849 if (save_ret) { 850 stack_size += 16; /* Save both A5 (BPF R0) and A0 */ 851 retval_off = stack_size; 852 } 853 854 stack_size += nregs * 8; 855 args_off = stack_size; 856 857 stack_size += 8; 858 nregs_off = stack_size; 859 860 if (flags & BPF_TRAMP_F_IP_ARG) { 861 stack_size += 8; 862 ip_off = stack_size; 863 } 864 865 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 866 run_ctx_off = stack_size; 867 868 stack_size += 8; 869 sreg_off = stack_size; 870 871 stack_size = round_up(stack_size, 16); 872 873 if (!is_struct_ops) { 874 /* For the trampoline called from function entry, 875 * the frame of traced function and the frame of 876 * trampoline need to be considered. 877 */ 878 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 879 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 880 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 881 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 882 883 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 884 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 885 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 886 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 887 } else { 888 /* emit kcfi hash */ 889 emit_kcfi(cfi_get_func_hash(func_addr), ctx); 890 /* For the trampoline called directly, just handle 891 * the frame of trampoline. 892 */ 893 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 894 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 895 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 896 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 897 } 898 899 /* callee saved register S1 to pass start time */ 900 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 901 902 /* store ip address of the traced function */ 903 if (flags & BPF_TRAMP_F_IP_ARG) { 904 emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 905 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 906 } 907 908 emit_li(RV_REG_T1, nregs, ctx); 909 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 910 911 store_args(nregs, args_off, ctx); 912 913 /* skip to actual body of traced function */ 914 if (flags & BPF_TRAMP_F_SKIP_FRAME) 915 orig_call += RV_FENTRY_NINSNS * 4; 916 917 if (flags & BPF_TRAMP_F_CALL_ORIG) { 918 emit_imm(RV_REG_A0, (const s64)im, ctx); 919 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 920 if (ret) 921 return ret; 922 } 923 924 for (i = 0; i < fentry->nr_links; i++) { 925 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 926 flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 927 if (ret) 928 return ret; 929 } 930 931 if (fmod_ret->nr_links) { 932 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 933 if (!branches_off) 934 return -ENOMEM; 935 936 /* cleanup to avoid garbage return value confusion */ 937 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 938 for (i = 0; i < fmod_ret->nr_links; i++) { 939 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 940 run_ctx_off, true, ctx); 941 if (ret) 942 goto out; 943 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 944 branches_off[i] = ctx->ninsns; 945 /* nop reserved for conditional jump */ 946 emit(rv_nop(), ctx); 947 } 948 } 949 950 if (flags & BPF_TRAMP_F_CALL_ORIG) { 951 restore_args(nregs, args_off, ctx); 952 ret = emit_call((const u64)orig_call, true, ctx); 953 if (ret) 954 goto out; 955 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 956 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 957 im->ip_after_call = ctx->insns + ctx->ninsns; 958 /* 2 nops reserved for auipc+jalr pair */ 959 emit(rv_nop(), ctx); 960 emit(rv_nop(), ctx); 961 } 962 963 /* update branches saved in invoke_bpf_mod_ret with bnez */ 964 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 965 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 966 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 967 *(u32 *)(ctx->insns + branches_off[i]) = insn; 968 } 969 970 for (i = 0; i < fexit->nr_links; i++) { 971 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 972 run_ctx_off, false, ctx); 973 if (ret) 974 goto out; 975 } 976 977 if (flags & BPF_TRAMP_F_CALL_ORIG) { 978 im->ip_epilogue = ctx->insns + ctx->ninsns; 979 emit_imm(RV_REG_A0, (const s64)im, ctx); 980 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 981 if (ret) 982 goto out; 983 } 984 985 if (flags & BPF_TRAMP_F_RESTORE_REGS) 986 restore_args(nregs, args_off, ctx); 987 988 if (save_ret) { 989 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 990 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); 991 } 992 993 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 994 995 if (!is_struct_ops) { 996 /* trampoline called from function entry */ 997 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 998 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 999 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1000 1001 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 1002 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 1003 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 1004 1005 if (flags & BPF_TRAMP_F_SKIP_FRAME) 1006 /* return to parent function */ 1007 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1008 else 1009 /* return to traced function */ 1010 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 1011 } else { 1012 /* trampoline called directly */ 1013 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 1014 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1015 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1016 1017 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1018 } 1019 1020 ret = ctx->ninsns; 1021 out: 1022 kfree(branches_off); 1023 return ret; 1024 } 1025 1026 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1027 struct bpf_tramp_links *tlinks, void *func_addr) 1028 { 1029 struct bpf_tramp_image im; 1030 struct rv_jit_context ctx; 1031 int ret; 1032 1033 ctx.ninsns = 0; 1034 ctx.insns = NULL; 1035 ctx.ro_insns = NULL; 1036 ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); 1037 1038 return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); 1039 } 1040 1041 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 1042 void *image_end, const struct btf_func_model *m, 1043 u32 flags, struct bpf_tramp_links *tlinks, 1044 void *func_addr) 1045 { 1046 int ret; 1047 struct rv_jit_context ctx; 1048 1049 ctx.ninsns = 0; 1050 /* 1051 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the 1052 * JITed instructions and later copies it to a RX region (ctx.ro_insns). 1053 * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the 1054 * trampoline image uses the same memory area for writing and execution, 1055 * both ctx.insns and ctx.ro_insns can be set to image. 1056 */ 1057 ctx.insns = image; 1058 ctx.ro_insns = image; 1059 ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 1060 if (ret < 0) 1061 return ret; 1062 1063 bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); 1064 1065 return ninsns_rvoff(ret); 1066 } 1067 1068 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1069 bool extra_pass) 1070 { 1071 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1072 BPF_CLASS(insn->code) == BPF_JMP; 1073 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1074 struct bpf_prog_aux *aux = ctx->prog->aux; 1075 u8 rd = -1, rs = -1, code = insn->code; 1076 s16 off = insn->off; 1077 s32 imm = insn->imm; 1078 1079 init_regs(&rd, &rs, insn, ctx); 1080 1081 switch (code) { 1082 /* dst = src */ 1083 case BPF_ALU | BPF_MOV | BPF_X: 1084 case BPF_ALU64 | BPF_MOV | BPF_X: 1085 if (insn_is_cast_user(insn)) { 1086 emit_mv(RV_REG_T1, rs, ctx); 1087 emit_zextw(RV_REG_T1, RV_REG_T1, ctx); 1088 emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx); 1089 emit(rv_beq(RV_REG_T1, RV_REG_ZERO, 4), ctx); 1090 emit_or(RV_REG_T1, rd, RV_REG_T1, ctx); 1091 emit_mv(rd, RV_REG_T1, ctx); 1092 break; 1093 } else if (insn_is_mov_percpu_addr(insn)) { 1094 if (rd != rs) 1095 emit_mv(rd, rs, ctx); 1096 #ifdef CONFIG_SMP 1097 /* Load current CPU number in T1 */ 1098 emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu), 1099 RV_REG_TP, ctx); 1100 /* << 3 because offsets are 8 bytes */ 1101 emit_slli(RV_REG_T1, RV_REG_T1, 3, ctx); 1102 /* Load address of __per_cpu_offset array in T2 */ 1103 emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx); 1104 /* Add offset of current CPU to __per_cpu_offset */ 1105 emit_add(RV_REG_T1, RV_REG_T2, RV_REG_T1, ctx); 1106 /* Load __per_cpu_offset[cpu] in T1 */ 1107 emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx); 1108 /* Add the offset to Rd */ 1109 emit_add(rd, rd, RV_REG_T1, ctx); 1110 #endif 1111 } 1112 if (imm == 1) { 1113 /* Special mov32 for zext */ 1114 emit_zextw(rd, rd, ctx); 1115 break; 1116 } 1117 switch (insn->off) { 1118 case 0: 1119 emit_mv(rd, rs, ctx); 1120 break; 1121 case 8: 1122 emit_sextb(rd, rs, ctx); 1123 break; 1124 case 16: 1125 emit_sexth(rd, rs, ctx); 1126 break; 1127 case 32: 1128 emit_sextw(rd, rs, ctx); 1129 break; 1130 } 1131 if (!is64 && !aux->verifier_zext) 1132 emit_zextw(rd, rd, ctx); 1133 break; 1134 1135 /* dst = dst OP src */ 1136 case BPF_ALU | BPF_ADD | BPF_X: 1137 case BPF_ALU64 | BPF_ADD | BPF_X: 1138 emit_add(rd, rd, rs, ctx); 1139 if (!is64 && !aux->verifier_zext) 1140 emit_zextw(rd, rd, ctx); 1141 break; 1142 case BPF_ALU | BPF_SUB | BPF_X: 1143 case BPF_ALU64 | BPF_SUB | BPF_X: 1144 if (is64) 1145 emit_sub(rd, rd, rs, ctx); 1146 else 1147 emit_subw(rd, rd, rs, ctx); 1148 1149 if (!is64 && !aux->verifier_zext) 1150 emit_zextw(rd, rd, ctx); 1151 break; 1152 case BPF_ALU | BPF_AND | BPF_X: 1153 case BPF_ALU64 | BPF_AND | BPF_X: 1154 emit_and(rd, rd, rs, ctx); 1155 if (!is64 && !aux->verifier_zext) 1156 emit_zextw(rd, rd, ctx); 1157 break; 1158 case BPF_ALU | BPF_OR | BPF_X: 1159 case BPF_ALU64 | BPF_OR | BPF_X: 1160 emit_or(rd, rd, rs, ctx); 1161 if (!is64 && !aux->verifier_zext) 1162 emit_zextw(rd, rd, ctx); 1163 break; 1164 case BPF_ALU | BPF_XOR | BPF_X: 1165 case BPF_ALU64 | BPF_XOR | BPF_X: 1166 emit_xor(rd, rd, rs, ctx); 1167 if (!is64 && !aux->verifier_zext) 1168 emit_zextw(rd, rd, ctx); 1169 break; 1170 case BPF_ALU | BPF_MUL | BPF_X: 1171 case BPF_ALU64 | BPF_MUL | BPF_X: 1172 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1173 if (!is64 && !aux->verifier_zext) 1174 emit_zextw(rd, rd, ctx); 1175 break; 1176 case BPF_ALU | BPF_DIV | BPF_X: 1177 case BPF_ALU64 | BPF_DIV | BPF_X: 1178 if (off) 1179 emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1180 else 1181 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1182 if (!is64 && !aux->verifier_zext) 1183 emit_zextw(rd, rd, ctx); 1184 break; 1185 case BPF_ALU | BPF_MOD | BPF_X: 1186 case BPF_ALU64 | BPF_MOD | BPF_X: 1187 if (off) 1188 emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1189 else 1190 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1191 if (!is64 && !aux->verifier_zext) 1192 emit_zextw(rd, rd, ctx); 1193 break; 1194 case BPF_ALU | BPF_LSH | BPF_X: 1195 case BPF_ALU64 | BPF_LSH | BPF_X: 1196 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 1197 if (!is64 && !aux->verifier_zext) 1198 emit_zextw(rd, rd, ctx); 1199 break; 1200 case BPF_ALU | BPF_RSH | BPF_X: 1201 case BPF_ALU64 | BPF_RSH | BPF_X: 1202 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1203 if (!is64 && !aux->verifier_zext) 1204 emit_zextw(rd, rd, ctx); 1205 break; 1206 case BPF_ALU | BPF_ARSH | BPF_X: 1207 case BPF_ALU64 | BPF_ARSH | BPF_X: 1208 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1209 if (!is64 && !aux->verifier_zext) 1210 emit_zextw(rd, rd, ctx); 1211 break; 1212 1213 /* dst = -dst */ 1214 case BPF_ALU | BPF_NEG: 1215 case BPF_ALU64 | BPF_NEG: 1216 emit_sub(rd, RV_REG_ZERO, rd, ctx); 1217 if (!is64 && !aux->verifier_zext) 1218 emit_zextw(rd, rd, ctx); 1219 break; 1220 1221 /* dst = BSWAP##imm(dst) */ 1222 case BPF_ALU | BPF_END | BPF_FROM_LE: 1223 switch (imm) { 1224 case 16: 1225 emit_zexth(rd, rd, ctx); 1226 break; 1227 case 32: 1228 if (!aux->verifier_zext) 1229 emit_zextw(rd, rd, ctx); 1230 break; 1231 case 64: 1232 /* Do nothing */ 1233 break; 1234 } 1235 break; 1236 case BPF_ALU | BPF_END | BPF_FROM_BE: 1237 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1238 emit_bswap(rd, imm, ctx); 1239 break; 1240 1241 /* dst = imm */ 1242 case BPF_ALU | BPF_MOV | BPF_K: 1243 case BPF_ALU64 | BPF_MOV | BPF_K: 1244 emit_imm(rd, imm, ctx); 1245 if (!is64 && !aux->verifier_zext) 1246 emit_zextw(rd, rd, ctx); 1247 break; 1248 1249 /* dst = dst OP imm */ 1250 case BPF_ALU | BPF_ADD | BPF_K: 1251 case BPF_ALU64 | BPF_ADD | BPF_K: 1252 if (is_12b_int(imm)) { 1253 emit_addi(rd, rd, imm, ctx); 1254 } else { 1255 emit_imm(RV_REG_T1, imm, ctx); 1256 emit_add(rd, rd, RV_REG_T1, ctx); 1257 } 1258 if (!is64 && !aux->verifier_zext) 1259 emit_zextw(rd, rd, ctx); 1260 break; 1261 case BPF_ALU | BPF_SUB | BPF_K: 1262 case BPF_ALU64 | BPF_SUB | BPF_K: 1263 if (is_12b_int(-imm)) { 1264 emit_addi(rd, rd, -imm, ctx); 1265 } else { 1266 emit_imm(RV_REG_T1, imm, ctx); 1267 emit_sub(rd, rd, RV_REG_T1, ctx); 1268 } 1269 if (!is64 && !aux->verifier_zext) 1270 emit_zextw(rd, rd, ctx); 1271 break; 1272 case BPF_ALU | BPF_AND | BPF_K: 1273 case BPF_ALU64 | BPF_AND | BPF_K: 1274 if (is_12b_int(imm)) { 1275 emit_andi(rd, rd, imm, ctx); 1276 } else { 1277 emit_imm(RV_REG_T1, imm, ctx); 1278 emit_and(rd, rd, RV_REG_T1, ctx); 1279 } 1280 if (!is64 && !aux->verifier_zext) 1281 emit_zextw(rd, rd, ctx); 1282 break; 1283 case BPF_ALU | BPF_OR | BPF_K: 1284 case BPF_ALU64 | BPF_OR | BPF_K: 1285 if (is_12b_int(imm)) { 1286 emit(rv_ori(rd, rd, imm), ctx); 1287 } else { 1288 emit_imm(RV_REG_T1, imm, ctx); 1289 emit_or(rd, rd, RV_REG_T1, ctx); 1290 } 1291 if (!is64 && !aux->verifier_zext) 1292 emit_zextw(rd, rd, ctx); 1293 break; 1294 case BPF_ALU | BPF_XOR | BPF_K: 1295 case BPF_ALU64 | BPF_XOR | BPF_K: 1296 if (is_12b_int(imm)) { 1297 emit(rv_xori(rd, rd, imm), ctx); 1298 } else { 1299 emit_imm(RV_REG_T1, imm, ctx); 1300 emit_xor(rd, rd, RV_REG_T1, ctx); 1301 } 1302 if (!is64 && !aux->verifier_zext) 1303 emit_zextw(rd, rd, ctx); 1304 break; 1305 case BPF_ALU | BPF_MUL | BPF_K: 1306 case BPF_ALU64 | BPF_MUL | BPF_K: 1307 emit_imm(RV_REG_T1, imm, ctx); 1308 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1309 rv_mulw(rd, rd, RV_REG_T1), ctx); 1310 if (!is64 && !aux->verifier_zext) 1311 emit_zextw(rd, rd, ctx); 1312 break; 1313 case BPF_ALU | BPF_DIV | BPF_K: 1314 case BPF_ALU64 | BPF_DIV | BPF_K: 1315 emit_imm(RV_REG_T1, imm, ctx); 1316 if (off) 1317 emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1318 rv_divw(rd, rd, RV_REG_T1), ctx); 1319 else 1320 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1321 rv_divuw(rd, rd, RV_REG_T1), ctx); 1322 if (!is64 && !aux->verifier_zext) 1323 emit_zextw(rd, rd, ctx); 1324 break; 1325 case BPF_ALU | BPF_MOD | BPF_K: 1326 case BPF_ALU64 | BPF_MOD | BPF_K: 1327 emit_imm(RV_REG_T1, imm, ctx); 1328 if (off) 1329 emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1330 rv_remw(rd, rd, RV_REG_T1), ctx); 1331 else 1332 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1333 rv_remuw(rd, rd, RV_REG_T1), ctx); 1334 if (!is64 && !aux->verifier_zext) 1335 emit_zextw(rd, rd, ctx); 1336 break; 1337 case BPF_ALU | BPF_LSH | BPF_K: 1338 case BPF_ALU64 | BPF_LSH | BPF_K: 1339 emit_slli(rd, rd, imm, ctx); 1340 1341 if (!is64 && !aux->verifier_zext) 1342 emit_zextw(rd, rd, ctx); 1343 break; 1344 case BPF_ALU | BPF_RSH | BPF_K: 1345 case BPF_ALU64 | BPF_RSH | BPF_K: 1346 if (is64) 1347 emit_srli(rd, rd, imm, ctx); 1348 else 1349 emit(rv_srliw(rd, rd, imm), ctx); 1350 1351 if (!is64 && !aux->verifier_zext) 1352 emit_zextw(rd, rd, ctx); 1353 break; 1354 case BPF_ALU | BPF_ARSH | BPF_K: 1355 case BPF_ALU64 | BPF_ARSH | BPF_K: 1356 if (is64) 1357 emit_srai(rd, rd, imm, ctx); 1358 else 1359 emit(rv_sraiw(rd, rd, imm), ctx); 1360 1361 if (!is64 && !aux->verifier_zext) 1362 emit_zextw(rd, rd, ctx); 1363 break; 1364 1365 /* JUMP off */ 1366 case BPF_JMP | BPF_JA: 1367 case BPF_JMP32 | BPF_JA: 1368 if (BPF_CLASS(code) == BPF_JMP) 1369 rvoff = rv_offset(i, off, ctx); 1370 else 1371 rvoff = rv_offset(i, imm, ctx); 1372 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1373 if (ret) 1374 return ret; 1375 break; 1376 1377 /* IF (dst COND src) JUMP off */ 1378 case BPF_JMP | BPF_JEQ | BPF_X: 1379 case BPF_JMP32 | BPF_JEQ | BPF_X: 1380 case BPF_JMP | BPF_JGT | BPF_X: 1381 case BPF_JMP32 | BPF_JGT | BPF_X: 1382 case BPF_JMP | BPF_JLT | BPF_X: 1383 case BPF_JMP32 | BPF_JLT | BPF_X: 1384 case BPF_JMP | BPF_JGE | BPF_X: 1385 case BPF_JMP32 | BPF_JGE | BPF_X: 1386 case BPF_JMP | BPF_JLE | BPF_X: 1387 case BPF_JMP32 | BPF_JLE | BPF_X: 1388 case BPF_JMP | BPF_JNE | BPF_X: 1389 case BPF_JMP32 | BPF_JNE | BPF_X: 1390 case BPF_JMP | BPF_JSGT | BPF_X: 1391 case BPF_JMP32 | BPF_JSGT | BPF_X: 1392 case BPF_JMP | BPF_JSLT | BPF_X: 1393 case BPF_JMP32 | BPF_JSLT | BPF_X: 1394 case BPF_JMP | BPF_JSGE | BPF_X: 1395 case BPF_JMP32 | BPF_JSGE | BPF_X: 1396 case BPF_JMP | BPF_JSLE | BPF_X: 1397 case BPF_JMP32 | BPF_JSLE | BPF_X: 1398 case BPF_JMP | BPF_JSET | BPF_X: 1399 case BPF_JMP32 | BPF_JSET | BPF_X: 1400 rvoff = rv_offset(i, off, ctx); 1401 if (!is64) { 1402 s = ctx->ninsns; 1403 if (is_signed_bpf_cond(BPF_OP(code))) { 1404 emit_sextw_alt(&rs, RV_REG_T1, ctx); 1405 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1406 } else { 1407 emit_zextw_alt(&rs, RV_REG_T1, ctx); 1408 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1409 } 1410 e = ctx->ninsns; 1411 1412 /* Adjust for extra insns */ 1413 rvoff -= ninsns_rvoff(e - s); 1414 } 1415 1416 if (BPF_OP(code) == BPF_JSET) { 1417 /* Adjust for and */ 1418 rvoff -= 4; 1419 emit_and(RV_REG_T1, rd, rs, ctx); 1420 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1421 } else { 1422 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1423 } 1424 break; 1425 1426 /* IF (dst COND imm) JUMP off */ 1427 case BPF_JMP | BPF_JEQ | BPF_K: 1428 case BPF_JMP32 | BPF_JEQ | BPF_K: 1429 case BPF_JMP | BPF_JGT | BPF_K: 1430 case BPF_JMP32 | BPF_JGT | BPF_K: 1431 case BPF_JMP | BPF_JLT | BPF_K: 1432 case BPF_JMP32 | BPF_JLT | BPF_K: 1433 case BPF_JMP | BPF_JGE | BPF_K: 1434 case BPF_JMP32 | BPF_JGE | BPF_K: 1435 case BPF_JMP | BPF_JLE | BPF_K: 1436 case BPF_JMP32 | BPF_JLE | BPF_K: 1437 case BPF_JMP | BPF_JNE | BPF_K: 1438 case BPF_JMP32 | BPF_JNE | BPF_K: 1439 case BPF_JMP | BPF_JSGT | BPF_K: 1440 case BPF_JMP32 | BPF_JSGT | BPF_K: 1441 case BPF_JMP | BPF_JSLT | BPF_K: 1442 case BPF_JMP32 | BPF_JSLT | BPF_K: 1443 case BPF_JMP | BPF_JSGE | BPF_K: 1444 case BPF_JMP32 | BPF_JSGE | BPF_K: 1445 case BPF_JMP | BPF_JSLE | BPF_K: 1446 case BPF_JMP32 | BPF_JSLE | BPF_K: 1447 rvoff = rv_offset(i, off, ctx); 1448 s = ctx->ninsns; 1449 if (imm) 1450 emit_imm(RV_REG_T1, imm, ctx); 1451 rs = imm ? RV_REG_T1 : RV_REG_ZERO; 1452 if (!is64) { 1453 if (is_signed_bpf_cond(BPF_OP(code))) { 1454 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1455 /* rs has been sign extended */ 1456 } else { 1457 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1458 if (imm) 1459 emit_zextw(rs, rs, ctx); 1460 } 1461 } 1462 e = ctx->ninsns; 1463 1464 /* Adjust for extra insns */ 1465 rvoff -= ninsns_rvoff(e - s); 1466 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1467 break; 1468 1469 case BPF_JMP | BPF_JSET | BPF_K: 1470 case BPF_JMP32 | BPF_JSET | BPF_K: 1471 rvoff = rv_offset(i, off, ctx); 1472 s = ctx->ninsns; 1473 if (is_12b_int(imm)) { 1474 emit_andi(RV_REG_T1, rd, imm, ctx); 1475 } else { 1476 emit_imm(RV_REG_T1, imm, ctx); 1477 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1478 } 1479 /* For jset32, we should clear the upper 32 bits of t1, but 1480 * sign-extension is sufficient here and saves one instruction, 1481 * as t1 is used only in comparison against zero. 1482 */ 1483 if (!is64 && imm < 0) 1484 emit_sextw(RV_REG_T1, RV_REG_T1, ctx); 1485 e = ctx->ninsns; 1486 rvoff -= ninsns_rvoff(e - s); 1487 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1488 break; 1489 1490 /* function call */ 1491 case BPF_JMP | BPF_CALL: 1492 { 1493 bool fixed_addr; 1494 u64 addr; 1495 1496 /* Inline calls to bpf_get_smp_processor_id() 1497 * 1498 * RV_REG_TP holds the address of the current CPU's task_struct and thread_info is 1499 * at offset 0 in task_struct. 1500 * Load cpu from thread_info: 1501 * Set R0 to ((struct thread_info *)(RV_REG_TP))->cpu 1502 * 1503 * This replicates the implementation of raw_smp_processor_id() on RISCV 1504 */ 1505 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { 1506 /* Load current CPU number in R0 */ 1507 emit_ld(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu), 1508 RV_REG_TP, ctx); 1509 break; 1510 } 1511 1512 mark_call(ctx); 1513 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1514 &addr, &fixed_addr); 1515 if (ret < 0) 1516 return ret; 1517 1518 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1519 const struct btf_func_model *fm; 1520 int idx; 1521 1522 fm = bpf_jit_find_kfunc_model(ctx->prog, insn); 1523 if (!fm) 1524 return -EINVAL; 1525 1526 for (idx = 0; idx < fm->nr_args; idx++) { 1527 u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); 1528 1529 if (fm->arg_size[idx] == sizeof(int)) 1530 emit_sextw(reg, reg, ctx); 1531 } 1532 } 1533 1534 ret = emit_call(addr, fixed_addr, ctx); 1535 if (ret) 1536 return ret; 1537 1538 if (insn->src_reg != BPF_PSEUDO_CALL) 1539 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1540 break; 1541 } 1542 /* tail call */ 1543 case BPF_JMP | BPF_TAIL_CALL: 1544 if (emit_bpf_tail_call(i, ctx)) 1545 return -1; 1546 break; 1547 1548 /* function return */ 1549 case BPF_JMP | BPF_EXIT: 1550 if (i == ctx->prog->len - 1) 1551 break; 1552 1553 rvoff = epilogue_offset(ctx); 1554 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1555 if (ret) 1556 return ret; 1557 break; 1558 1559 /* dst = imm64 */ 1560 case BPF_LD | BPF_IMM | BPF_DW: 1561 { 1562 struct bpf_insn insn1 = insn[1]; 1563 u64 imm64; 1564 1565 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1566 if (bpf_pseudo_func(insn)) { 1567 /* fixed-length insns for extra jit pass */ 1568 ret = emit_addr(rd, imm64, extra_pass, ctx); 1569 if (ret) 1570 return ret; 1571 } else { 1572 emit_imm(rd, imm64, ctx); 1573 } 1574 1575 return 1; 1576 } 1577 1578 /* LDX: dst = *(unsigned size *)(src + off) */ 1579 case BPF_LDX | BPF_MEM | BPF_B: 1580 case BPF_LDX | BPF_MEM | BPF_H: 1581 case BPF_LDX | BPF_MEM | BPF_W: 1582 case BPF_LDX | BPF_MEM | BPF_DW: 1583 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1584 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1585 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1586 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1587 /* LDSX: dst = *(signed size *)(src + off) */ 1588 case BPF_LDX | BPF_MEMSX | BPF_B: 1589 case BPF_LDX | BPF_MEMSX | BPF_H: 1590 case BPF_LDX | BPF_MEMSX | BPF_W: 1591 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1592 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1593 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1594 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + RV_REG_ARENA + off) */ 1595 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1596 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1597 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1598 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1599 { 1600 int insn_len, insns_start; 1601 bool sign_ext; 1602 1603 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1604 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1605 1606 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1607 emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); 1608 rs = RV_REG_T2; 1609 } 1610 1611 switch (BPF_SIZE(code)) { 1612 case BPF_B: 1613 if (is_12b_int(off)) { 1614 insns_start = ctx->ninsns; 1615 if (sign_ext) 1616 emit(rv_lb(rd, off, rs), ctx); 1617 else 1618 emit(rv_lbu(rd, off, rs), ctx); 1619 insn_len = ctx->ninsns - insns_start; 1620 break; 1621 } 1622 1623 emit_imm(RV_REG_T1, off, ctx); 1624 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1625 insns_start = ctx->ninsns; 1626 if (sign_ext) 1627 emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1628 else 1629 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1630 insn_len = ctx->ninsns - insns_start; 1631 break; 1632 case BPF_H: 1633 if (is_12b_int(off)) { 1634 insns_start = ctx->ninsns; 1635 if (sign_ext) 1636 emit(rv_lh(rd, off, rs), ctx); 1637 else 1638 emit(rv_lhu(rd, off, rs), ctx); 1639 insn_len = ctx->ninsns - insns_start; 1640 break; 1641 } 1642 1643 emit_imm(RV_REG_T1, off, ctx); 1644 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1645 insns_start = ctx->ninsns; 1646 if (sign_ext) 1647 emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1648 else 1649 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1650 insn_len = ctx->ninsns - insns_start; 1651 break; 1652 case BPF_W: 1653 if (is_12b_int(off)) { 1654 insns_start = ctx->ninsns; 1655 if (sign_ext) 1656 emit(rv_lw(rd, off, rs), ctx); 1657 else 1658 emit(rv_lwu(rd, off, rs), ctx); 1659 insn_len = ctx->ninsns - insns_start; 1660 break; 1661 } 1662 1663 emit_imm(RV_REG_T1, off, ctx); 1664 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1665 insns_start = ctx->ninsns; 1666 if (sign_ext) 1667 emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1668 else 1669 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1670 insn_len = ctx->ninsns - insns_start; 1671 break; 1672 case BPF_DW: 1673 if (is_12b_int(off)) { 1674 insns_start = ctx->ninsns; 1675 emit_ld(rd, off, rs, ctx); 1676 insn_len = ctx->ninsns - insns_start; 1677 break; 1678 } 1679 1680 emit_imm(RV_REG_T1, off, ctx); 1681 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1682 insns_start = ctx->ninsns; 1683 emit_ld(rd, 0, RV_REG_T1, ctx); 1684 insn_len = ctx->ninsns - insns_start; 1685 break; 1686 } 1687 1688 ret = add_exception_handler(insn, ctx, rd, insn_len); 1689 if (ret) 1690 return ret; 1691 1692 if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1693 return 1; 1694 break; 1695 } 1696 /* speculation barrier */ 1697 case BPF_ST | BPF_NOSPEC: 1698 break; 1699 1700 /* ST: *(size *)(dst + off) = imm */ 1701 case BPF_ST | BPF_MEM | BPF_B: 1702 emit_imm(RV_REG_T1, imm, ctx); 1703 if (is_12b_int(off)) { 1704 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1705 break; 1706 } 1707 1708 emit_imm(RV_REG_T2, off, ctx); 1709 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1710 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1711 break; 1712 1713 case BPF_ST | BPF_MEM | BPF_H: 1714 emit_imm(RV_REG_T1, imm, ctx); 1715 if (is_12b_int(off)) { 1716 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1717 break; 1718 } 1719 1720 emit_imm(RV_REG_T2, off, ctx); 1721 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1722 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1723 break; 1724 case BPF_ST | BPF_MEM | BPF_W: 1725 emit_imm(RV_REG_T1, imm, ctx); 1726 if (is_12b_int(off)) { 1727 emit_sw(rd, off, RV_REG_T1, ctx); 1728 break; 1729 } 1730 1731 emit_imm(RV_REG_T2, off, ctx); 1732 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1733 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1734 break; 1735 case BPF_ST | BPF_MEM | BPF_DW: 1736 emit_imm(RV_REG_T1, imm, ctx); 1737 if (is_12b_int(off)) { 1738 emit_sd(rd, off, RV_REG_T1, ctx); 1739 break; 1740 } 1741 1742 emit_imm(RV_REG_T2, off, ctx); 1743 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1744 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1745 break; 1746 1747 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1748 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1749 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1750 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1751 { 1752 int insn_len, insns_start; 1753 1754 emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); 1755 rd = RV_REG_T3; 1756 1757 /* Load imm to a register then store it */ 1758 emit_imm(RV_REG_T1, imm, ctx); 1759 1760 switch (BPF_SIZE(code)) { 1761 case BPF_B: 1762 if (is_12b_int(off)) { 1763 insns_start = ctx->ninsns; 1764 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1765 insn_len = ctx->ninsns - insns_start; 1766 break; 1767 } 1768 1769 emit_imm(RV_REG_T2, off, ctx); 1770 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1771 insns_start = ctx->ninsns; 1772 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1773 insn_len = ctx->ninsns - insns_start; 1774 break; 1775 case BPF_H: 1776 if (is_12b_int(off)) { 1777 insns_start = ctx->ninsns; 1778 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1779 insn_len = ctx->ninsns - insns_start; 1780 break; 1781 } 1782 1783 emit_imm(RV_REG_T2, off, ctx); 1784 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1785 insns_start = ctx->ninsns; 1786 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1787 insn_len = ctx->ninsns - insns_start; 1788 break; 1789 case BPF_W: 1790 if (is_12b_int(off)) { 1791 insns_start = ctx->ninsns; 1792 emit_sw(rd, off, RV_REG_T1, ctx); 1793 insn_len = ctx->ninsns - insns_start; 1794 break; 1795 } 1796 1797 emit_imm(RV_REG_T2, off, ctx); 1798 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1799 insns_start = ctx->ninsns; 1800 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1801 insn_len = ctx->ninsns - insns_start; 1802 break; 1803 case BPF_DW: 1804 if (is_12b_int(off)) { 1805 insns_start = ctx->ninsns; 1806 emit_sd(rd, off, RV_REG_T1, ctx); 1807 insn_len = ctx->ninsns - insns_start; 1808 break; 1809 } 1810 1811 emit_imm(RV_REG_T2, off, ctx); 1812 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1813 insns_start = ctx->ninsns; 1814 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1815 insn_len = ctx->ninsns - insns_start; 1816 break; 1817 } 1818 1819 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1820 insn_len); 1821 if (ret) 1822 return ret; 1823 1824 break; 1825 } 1826 1827 /* STX: *(size *)(dst + off) = src */ 1828 case BPF_STX | BPF_MEM | BPF_B: 1829 if (is_12b_int(off)) { 1830 emit(rv_sb(rd, off, rs), ctx); 1831 break; 1832 } 1833 1834 emit_imm(RV_REG_T1, off, ctx); 1835 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1836 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1837 break; 1838 case BPF_STX | BPF_MEM | BPF_H: 1839 if (is_12b_int(off)) { 1840 emit(rv_sh(rd, off, rs), ctx); 1841 break; 1842 } 1843 1844 emit_imm(RV_REG_T1, off, ctx); 1845 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1846 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1847 break; 1848 case BPF_STX | BPF_MEM | BPF_W: 1849 if (is_12b_int(off)) { 1850 emit_sw(rd, off, rs, ctx); 1851 break; 1852 } 1853 1854 emit_imm(RV_REG_T1, off, ctx); 1855 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1856 emit_sw(RV_REG_T1, 0, rs, ctx); 1857 break; 1858 case BPF_STX | BPF_MEM | BPF_DW: 1859 if (is_12b_int(off)) { 1860 emit_sd(rd, off, rs, ctx); 1861 break; 1862 } 1863 1864 emit_imm(RV_REG_T1, off, ctx); 1865 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1866 emit_sd(RV_REG_T1, 0, rs, ctx); 1867 break; 1868 case BPF_STX | BPF_ATOMIC | BPF_W: 1869 case BPF_STX | BPF_ATOMIC | BPF_DW: 1870 emit_atomic(rd, rs, off, imm, 1871 BPF_SIZE(code) == BPF_DW, ctx); 1872 break; 1873 1874 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1875 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1876 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1877 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1878 { 1879 int insn_len, insns_start; 1880 1881 emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 1882 rd = RV_REG_T2; 1883 1884 switch (BPF_SIZE(code)) { 1885 case BPF_B: 1886 if (is_12b_int(off)) { 1887 insns_start = ctx->ninsns; 1888 emit(rv_sb(rd, off, rs), ctx); 1889 insn_len = ctx->ninsns - insns_start; 1890 break; 1891 } 1892 1893 emit_imm(RV_REG_T1, off, ctx); 1894 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1895 insns_start = ctx->ninsns; 1896 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1897 insn_len = ctx->ninsns - insns_start; 1898 break; 1899 case BPF_H: 1900 if (is_12b_int(off)) { 1901 insns_start = ctx->ninsns; 1902 emit(rv_sh(rd, off, rs), ctx); 1903 insn_len = ctx->ninsns - insns_start; 1904 break; 1905 } 1906 1907 emit_imm(RV_REG_T1, off, ctx); 1908 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1909 insns_start = ctx->ninsns; 1910 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1911 insn_len = ctx->ninsns - insns_start; 1912 break; 1913 case BPF_W: 1914 if (is_12b_int(off)) { 1915 insns_start = ctx->ninsns; 1916 emit_sw(rd, off, rs, ctx); 1917 insn_len = ctx->ninsns - insns_start; 1918 break; 1919 } 1920 1921 emit_imm(RV_REG_T1, off, ctx); 1922 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1923 insns_start = ctx->ninsns; 1924 emit_sw(RV_REG_T1, 0, rs, ctx); 1925 insn_len = ctx->ninsns - insns_start; 1926 break; 1927 case BPF_DW: 1928 if (is_12b_int(off)) { 1929 insns_start = ctx->ninsns; 1930 emit_sd(rd, off, rs, ctx); 1931 insn_len = ctx->ninsns - insns_start; 1932 break; 1933 } 1934 1935 emit_imm(RV_REG_T1, off, ctx); 1936 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1937 insns_start = ctx->ninsns; 1938 emit_sd(RV_REG_T1, 0, rs, ctx); 1939 insn_len = ctx->ninsns - insns_start; 1940 break; 1941 } 1942 1943 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1944 insn_len); 1945 if (ret) 1946 return ret; 1947 1948 break; 1949 } 1950 1951 default: 1952 pr_err("bpf-jit: unknown opcode %02x\n", code); 1953 return -EINVAL; 1954 } 1955 1956 return 0; 1957 } 1958 1959 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) 1960 { 1961 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1962 1963 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1964 if (bpf_stack_adjust) 1965 mark_fp(ctx); 1966 1967 if (seen_reg(RV_REG_RA, ctx)) 1968 stack_adjust += 8; 1969 stack_adjust += 8; /* RV_REG_FP */ 1970 if (seen_reg(RV_REG_S1, ctx)) 1971 stack_adjust += 8; 1972 if (seen_reg(RV_REG_S2, ctx)) 1973 stack_adjust += 8; 1974 if (seen_reg(RV_REG_S3, ctx)) 1975 stack_adjust += 8; 1976 if (seen_reg(RV_REG_S4, ctx)) 1977 stack_adjust += 8; 1978 if (seen_reg(RV_REG_S5, ctx)) 1979 stack_adjust += 8; 1980 if (seen_reg(RV_REG_S6, ctx)) 1981 stack_adjust += 8; 1982 if (ctx->arena_vm_start) 1983 stack_adjust += 8; 1984 1985 stack_adjust = round_up(stack_adjust, 16); 1986 stack_adjust += bpf_stack_adjust; 1987 1988 store_offset = stack_adjust - 8; 1989 1990 /* emit kcfi type preamble immediately before the first insn */ 1991 emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); 1992 1993 /* nops reserved for auipc+jalr pair */ 1994 for (i = 0; i < RV_FENTRY_NINSNS; i++) 1995 emit(rv_nop(), ctx); 1996 1997 /* First instruction is always setting the tail-call-counter 1998 * (TCC) register. This instruction is skipped for tail calls. 1999 * Force using a 4-byte (non-compressed) instruction. 2000 */ 2001 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 2002 2003 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 2004 2005 if (seen_reg(RV_REG_RA, ctx)) { 2006 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 2007 store_offset -= 8; 2008 } 2009 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 2010 store_offset -= 8; 2011 if (seen_reg(RV_REG_S1, ctx)) { 2012 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 2013 store_offset -= 8; 2014 } 2015 if (seen_reg(RV_REG_S2, ctx)) { 2016 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 2017 store_offset -= 8; 2018 } 2019 if (seen_reg(RV_REG_S3, ctx)) { 2020 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 2021 store_offset -= 8; 2022 } 2023 if (seen_reg(RV_REG_S4, ctx)) { 2024 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 2025 store_offset -= 8; 2026 } 2027 if (seen_reg(RV_REG_S5, ctx)) { 2028 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 2029 store_offset -= 8; 2030 } 2031 if (seen_reg(RV_REG_S6, ctx)) { 2032 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 2033 store_offset -= 8; 2034 } 2035 if (ctx->arena_vm_start) { 2036 emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); 2037 store_offset -= 8; 2038 } 2039 2040 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 2041 2042 if (bpf_stack_adjust) 2043 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 2044 2045 /* Program contains calls and tail calls, so RV_REG_TCC need 2046 * to be saved across calls. 2047 */ 2048 if (seen_tail_call(ctx) && seen_call(ctx)) 2049 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 2050 2051 ctx->stack_size = stack_adjust; 2052 2053 if (ctx->arena_vm_start) 2054 emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx); 2055 } 2056 2057 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 2058 { 2059 __build_epilogue(false, ctx); 2060 } 2061 2062 bool bpf_jit_supports_kfunc_call(void) 2063 { 2064 return true; 2065 } 2066 2067 bool bpf_jit_supports_ptr_xchg(void) 2068 { 2069 return true; 2070 } 2071 2072 bool bpf_jit_supports_arena(void) 2073 { 2074 return true; 2075 } 2076 2077 bool bpf_jit_supports_percpu_insn(void) 2078 { 2079 return true; 2080 } 2081 2082 bool bpf_jit_inlines_helper_call(s32 imm) 2083 { 2084 switch (imm) { 2085 case BPF_FUNC_get_smp_processor_id: 2086 return true; 2087 default: 2088 return false; 2089 } 2090 } 2091