1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <asm/patch.h> 14 #include <asm/cfi.h> 15 #include <asm/percpu.h> 16 #include "bpf_jit.h" 17 18 #define RV_MAX_REG_ARGS 8 19 #define RV_FENTRY_NINSNS 2 20 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) 21 /* imm that allows emit_imm to emit max count insns */ 22 #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF 23 24 #define RV_REG_TCC RV_REG_A6 25 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 26 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ 27 28 static const int regmap[] = { 29 [BPF_REG_0] = RV_REG_A5, 30 [BPF_REG_1] = RV_REG_A0, 31 [BPF_REG_2] = RV_REG_A1, 32 [BPF_REG_3] = RV_REG_A2, 33 [BPF_REG_4] = RV_REG_A3, 34 [BPF_REG_5] = RV_REG_A4, 35 [BPF_REG_6] = RV_REG_S1, 36 [BPF_REG_7] = RV_REG_S2, 37 [BPF_REG_8] = RV_REG_S3, 38 [BPF_REG_9] = RV_REG_S4, 39 [BPF_REG_FP] = RV_REG_S5, 40 [BPF_REG_AX] = RV_REG_T0, 41 }; 42 43 static const int pt_regmap[] = { 44 [RV_REG_A0] = offsetof(struct pt_regs, a0), 45 [RV_REG_A1] = offsetof(struct pt_regs, a1), 46 [RV_REG_A2] = offsetof(struct pt_regs, a2), 47 [RV_REG_A3] = offsetof(struct pt_regs, a3), 48 [RV_REG_A4] = offsetof(struct pt_regs, a4), 49 [RV_REG_A5] = offsetof(struct pt_regs, a5), 50 [RV_REG_S1] = offsetof(struct pt_regs, s1), 51 [RV_REG_S2] = offsetof(struct pt_regs, s2), 52 [RV_REG_S3] = offsetof(struct pt_regs, s3), 53 [RV_REG_S4] = offsetof(struct pt_regs, s4), 54 [RV_REG_S5] = offsetof(struct pt_regs, s5), 55 [RV_REG_T0] = offsetof(struct pt_regs, t0), 56 }; 57 58 enum { 59 RV_CTX_F_SEEN_TAIL_CALL = 0, 60 RV_CTX_F_SEEN_CALL = RV_REG_RA, 61 RV_CTX_F_SEEN_S1 = RV_REG_S1, 62 RV_CTX_F_SEEN_S2 = RV_REG_S2, 63 RV_CTX_F_SEEN_S3 = RV_REG_S3, 64 RV_CTX_F_SEEN_S4 = RV_REG_S4, 65 RV_CTX_F_SEEN_S5 = RV_REG_S5, 66 RV_CTX_F_SEEN_S6 = RV_REG_S6, 67 }; 68 69 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 70 { 71 u8 reg = regmap[bpf_reg]; 72 73 switch (reg) { 74 case RV_CTX_F_SEEN_S1: 75 case RV_CTX_F_SEEN_S2: 76 case RV_CTX_F_SEEN_S3: 77 case RV_CTX_F_SEEN_S4: 78 case RV_CTX_F_SEEN_S5: 79 case RV_CTX_F_SEEN_S6: 80 __set_bit(reg, &ctx->flags); 81 } 82 return reg; 83 }; 84 85 static bool seen_reg(int reg, struct rv_jit_context *ctx) 86 { 87 switch (reg) { 88 case RV_CTX_F_SEEN_CALL: 89 case RV_CTX_F_SEEN_S1: 90 case RV_CTX_F_SEEN_S2: 91 case RV_CTX_F_SEEN_S3: 92 case RV_CTX_F_SEEN_S4: 93 case RV_CTX_F_SEEN_S5: 94 case RV_CTX_F_SEEN_S6: 95 return test_bit(reg, &ctx->flags); 96 } 97 return false; 98 } 99 100 static void mark_fp(struct rv_jit_context *ctx) 101 { 102 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 103 } 104 105 static void mark_call(struct rv_jit_context *ctx) 106 { 107 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 108 } 109 110 static bool seen_call(struct rv_jit_context *ctx) 111 { 112 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 113 } 114 115 static void mark_tail_call(struct rv_jit_context *ctx) 116 { 117 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 118 } 119 120 static bool seen_tail_call(struct rv_jit_context *ctx) 121 { 122 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 123 } 124 125 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 126 { 127 mark_tail_call(ctx); 128 129 if (seen_call(ctx)) { 130 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 131 return RV_REG_S6; 132 } 133 return RV_REG_A6; 134 } 135 136 static bool is_32b_int(s64 val) 137 { 138 return -(1L << 31) <= val && val < (1L << 31); 139 } 140 141 static bool in_auipc_jalr_range(s64 val) 142 { 143 /* 144 * auipc+jalr can reach any signed PC-relative offset in the range 145 * [-2^31 - 2^11, 2^31 - 2^11). 146 */ 147 return (-(1L << 31) - (1L << 11)) <= val && 148 val < ((1L << 31) - (1L << 11)); 149 } 150 151 /* Modify rd pointer to alternate reg to avoid corrupting original reg */ 152 static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 153 { 154 emit_sextw(ra, *rd, ctx); 155 *rd = ra; 156 } 157 158 static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) 159 { 160 emit_zextw(ra, *rd, ctx); 161 *rd = ra; 162 } 163 164 /* Emit fixed-length instructions for address */ 165 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 166 { 167 /* 168 * Use the ro_insns(RX) to calculate the offset as the BPF program will 169 * finally run from this memory region. 170 */ 171 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); 172 s64 off = addr - ip; 173 s64 upper = (off + (1 << 11)) >> 12; 174 s64 lower = off & 0xfff; 175 176 if (extra_pass && !in_auipc_jalr_range(off)) { 177 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 178 return -ERANGE; 179 } 180 181 emit(rv_auipc(rd, upper), ctx); 182 emit(rv_addi(rd, rd, lower), ctx); 183 return 0; 184 } 185 186 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 187 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 188 { 189 /* Note that the immediate from the add is sign-extended, 190 * which means that we need to compensate this by adding 2^12, 191 * when the 12th bit is set. A simpler way of doing this, and 192 * getting rid of the check, is to just add 2**11 before the 193 * shift. The "Loading a 32-Bit constant" example from the 194 * "Computer Organization and Design, RISC-V edition" book by 195 * Patterson/Hennessy highlights this fact. 196 * 197 * This also means that we need to process LSB to MSB. 198 */ 199 s64 upper = (val + (1 << 11)) >> 12; 200 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 201 * and addi are signed and RVC checks will perform signed comparisons. 202 */ 203 s64 lower = ((val & 0xfff) << 52) >> 52; 204 int shift; 205 206 if (is_32b_int(val)) { 207 if (upper) 208 emit_lui(rd, upper, ctx); 209 210 if (!upper) { 211 emit_li(rd, lower, ctx); 212 return; 213 } 214 215 emit_addiw(rd, rd, lower, ctx); 216 return; 217 } 218 219 shift = __ffs(upper); 220 upper >>= shift; 221 shift += 12; 222 223 emit_imm(rd, upper, ctx); 224 225 emit_slli(rd, rd, shift, ctx); 226 if (lower) 227 emit_addi(rd, rd, lower, ctx); 228 } 229 230 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 231 { 232 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 233 234 if (seen_reg(RV_REG_RA, ctx)) { 235 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 236 store_offset -= 8; 237 } 238 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 239 store_offset -= 8; 240 if (seen_reg(RV_REG_S1, ctx)) { 241 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 242 store_offset -= 8; 243 } 244 if (seen_reg(RV_REG_S2, ctx)) { 245 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 246 store_offset -= 8; 247 } 248 if (seen_reg(RV_REG_S3, ctx)) { 249 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 250 store_offset -= 8; 251 } 252 if (seen_reg(RV_REG_S4, ctx)) { 253 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 254 store_offset -= 8; 255 } 256 if (seen_reg(RV_REG_S5, ctx)) { 257 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 258 store_offset -= 8; 259 } 260 if (seen_reg(RV_REG_S6, ctx)) { 261 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 262 store_offset -= 8; 263 } 264 if (ctx->arena_vm_start) { 265 emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); 266 store_offset -= 8; 267 } 268 269 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 270 /* Set return value. */ 271 if (!is_tail_call) 272 emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); 273 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 274 is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 275 ctx); 276 } 277 278 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 279 struct rv_jit_context *ctx) 280 { 281 switch (cond) { 282 case BPF_JEQ: 283 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 284 return; 285 case BPF_JGT: 286 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 287 return; 288 case BPF_JLT: 289 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 290 return; 291 case BPF_JGE: 292 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 293 return; 294 case BPF_JLE: 295 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 296 return; 297 case BPF_JNE: 298 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 299 return; 300 case BPF_JSGT: 301 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 302 return; 303 case BPF_JSLT: 304 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 305 return; 306 case BPF_JSGE: 307 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 308 return; 309 case BPF_JSLE: 310 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 311 } 312 } 313 314 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 315 struct rv_jit_context *ctx) 316 { 317 s64 upper, lower; 318 319 if (is_13b_int(rvoff)) { 320 emit_bcc(cond, rd, rs, rvoff, ctx); 321 return; 322 } 323 324 /* Adjust for jal */ 325 rvoff -= 4; 326 327 /* Transform, e.g.: 328 * bne rd,rs,foo 329 * to 330 * beq rd,rs,<.L1> 331 * (auipc foo) 332 * jal(r) foo 333 * .L1 334 */ 335 cond = invert_bpf_cond(cond); 336 if (is_21b_int(rvoff)) { 337 emit_bcc(cond, rd, rs, 8, ctx); 338 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 339 return; 340 } 341 342 /* 32b No need for an additional rvoff adjustment, since we 343 * get that from the auipc at PC', where PC = PC' + 4. 344 */ 345 upper = (rvoff + (1 << 11)) >> 12; 346 lower = rvoff & 0xfff; 347 348 emit_bcc(cond, rd, rs, 12, ctx); 349 emit(rv_auipc(RV_REG_T1, upper), ctx); 350 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 351 } 352 353 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 354 { 355 int tc_ninsn, off, start_insn = ctx->ninsns; 356 u8 tcc = rv_tail_call_reg(ctx); 357 358 /* a0: &ctx 359 * a1: &array 360 * a2: index 361 * 362 * if (index >= array->map.max_entries) 363 * goto out; 364 */ 365 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 366 ctx->offset[0]; 367 emit_zextw(RV_REG_A2, RV_REG_A2, ctx); 368 369 off = offsetof(struct bpf_array, map.max_entries); 370 if (is_12b_check(off, insn)) 371 return -1; 372 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 373 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 374 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 375 376 /* if (--TCC < 0) 377 * goto out; 378 */ 379 emit_addi(RV_REG_TCC, tcc, -1, ctx); 380 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 381 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 382 383 /* prog = array->ptrs[index]; 384 * if (!prog) 385 * goto out; 386 */ 387 emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx); 388 off = offsetof(struct bpf_array, ptrs); 389 if (is_12b_check(off, insn)) 390 return -1; 391 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 392 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 393 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 394 395 /* goto *(prog->bpf_func + 4); */ 396 off = offsetof(struct bpf_prog, bpf_func); 397 if (is_12b_check(off, insn)) 398 return -1; 399 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 400 __build_epilogue(true, ctx); 401 return 0; 402 } 403 404 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 405 struct rv_jit_context *ctx) 406 { 407 u8 code = insn->code; 408 409 switch (code) { 410 case BPF_JMP | BPF_JA: 411 case BPF_JMP | BPF_CALL: 412 case BPF_JMP | BPF_EXIT: 413 case BPF_JMP | BPF_TAIL_CALL: 414 break; 415 default: 416 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 417 } 418 419 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 420 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 421 code & BPF_LDX || code & BPF_STX) 422 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 423 } 424 425 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 426 struct rv_jit_context *ctx) 427 { 428 s64 upper, lower; 429 430 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 431 emit(rv_jal(rd, rvoff >> 1), ctx); 432 return 0; 433 } else if (in_auipc_jalr_range(rvoff)) { 434 upper = (rvoff + (1 << 11)) >> 12; 435 lower = rvoff & 0xfff; 436 emit(rv_auipc(RV_REG_T1, upper), ctx); 437 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 438 return 0; 439 } 440 441 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 442 return -ERANGE; 443 } 444 445 static bool is_signed_bpf_cond(u8 cond) 446 { 447 return cond == BPF_JSGT || cond == BPF_JSLT || 448 cond == BPF_JSGE || cond == BPF_JSLE; 449 } 450 451 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 452 { 453 s64 off = 0; 454 u64 ip; 455 456 if (addr && ctx->insns && ctx->ro_insns) { 457 /* 458 * Use the ro_insns(RX) to calculate the offset as the BPF 459 * program will finally run from this memory region. 460 */ 461 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); 462 off = addr - ip; 463 } 464 465 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 466 } 467 468 static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) 469 { 470 if (IS_ENABLED(CONFIG_CFI_CLANG)) 471 emit(hash, ctx); 472 } 473 474 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 475 struct rv_jit_context *ctx) 476 { 477 u8 r0; 478 int jmp_offset; 479 480 if (off) { 481 if (is_12b_int(off)) { 482 emit_addi(RV_REG_T1, rd, off, ctx); 483 } else { 484 emit_imm(RV_REG_T1, off, ctx); 485 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 486 } 487 rd = RV_REG_T1; 488 } 489 490 switch (imm) { 491 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 492 case BPF_ADD: 493 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 494 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 495 break; 496 case BPF_AND: 497 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 498 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 499 break; 500 case BPF_OR: 501 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 502 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 503 break; 504 case BPF_XOR: 505 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 506 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 507 break; 508 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 509 case BPF_ADD | BPF_FETCH: 510 emit(is64 ? rv_amoadd_d(rs, rs, rd, 1, 1) : 511 rv_amoadd_w(rs, rs, rd, 1, 1), ctx); 512 if (!is64) 513 emit_zextw(rs, rs, ctx); 514 break; 515 case BPF_AND | BPF_FETCH: 516 emit(is64 ? rv_amoand_d(rs, rs, rd, 1, 1) : 517 rv_amoand_w(rs, rs, rd, 1, 1), ctx); 518 if (!is64) 519 emit_zextw(rs, rs, ctx); 520 break; 521 case BPF_OR | BPF_FETCH: 522 emit(is64 ? rv_amoor_d(rs, rs, rd, 1, 1) : 523 rv_amoor_w(rs, rs, rd, 1, 1), ctx); 524 if (!is64) 525 emit_zextw(rs, rs, ctx); 526 break; 527 case BPF_XOR | BPF_FETCH: 528 emit(is64 ? rv_amoxor_d(rs, rs, rd, 1, 1) : 529 rv_amoxor_w(rs, rs, rd, 1, 1), ctx); 530 if (!is64) 531 emit_zextw(rs, rs, ctx); 532 break; 533 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 534 case BPF_XCHG: 535 emit(is64 ? rv_amoswap_d(rs, rs, rd, 1, 1) : 536 rv_amoswap_w(rs, rs, rd, 1, 1), ctx); 537 if (!is64) 538 emit_zextw(rs, rs, ctx); 539 break; 540 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 541 case BPF_CMPXCHG: 542 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 543 if (is64) 544 emit_mv(RV_REG_T2, r0, ctx); 545 else 546 emit_addiw(RV_REG_T2, r0, 0, ctx); 547 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 548 rv_lr_w(r0, 0, rd, 0, 0), ctx); 549 jmp_offset = ninsns_rvoff(8); 550 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 551 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 552 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 553 jmp_offset = ninsns_rvoff(-6); 554 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 555 emit(rv_fence(0x3, 0x3), ctx); 556 break; 557 } 558 } 559 560 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 561 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 562 #define REG_DONT_CLEAR_MARKER 0 /* RV_REG_ZERO unused in pt_regmap */ 563 564 bool ex_handler_bpf(const struct exception_table_entry *ex, 565 struct pt_regs *regs) 566 { 567 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 568 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 569 570 if (regs_offset != REG_DONT_CLEAR_MARKER) 571 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 572 regs->epc = (unsigned long)&ex->fixup - offset; 573 574 return true; 575 } 576 577 /* For accesses to BTF pointers, add an entry to the exception table */ 578 static int add_exception_handler(const struct bpf_insn *insn, 579 struct rv_jit_context *ctx, 580 int dst_reg, int insn_len) 581 { 582 struct exception_table_entry *ex; 583 unsigned long pc; 584 off_t ins_offset; 585 off_t fixup_offset; 586 587 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || 588 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 589 BPF_MODE(insn->code) != BPF_PROBE_MEM32)) 590 return 0; 591 592 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 593 return -EINVAL; 594 595 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 596 return -EINVAL; 597 598 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 599 return -EINVAL; 600 601 ex = &ctx->prog->aux->extable[ctx->nexentries]; 602 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; 603 604 /* 605 * This is the relative offset of the instruction that may fault from 606 * the exception table itself. This will be written to the exception 607 * table and if this instruction faults, the destination register will 608 * be set to '0' and the execution will jump to the next instruction. 609 */ 610 ins_offset = pc - (long)&ex->insn; 611 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 612 return -ERANGE; 613 614 /* 615 * Since the extable follows the program, the fixup offset is always 616 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 617 * to keep things simple, and put the destination register in the upper 618 * bits. We don't need to worry about buildtime or runtime sort 619 * modifying the upper bits because the table is already sorted, and 620 * isn't part of the main exception table. 621 * 622 * The fixup_offset is set to the next instruction from the instruction 623 * that may fault. The execution will jump to this after handling the 624 * fault. 625 */ 626 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 627 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 628 return -ERANGE; 629 630 /* 631 * The offsets above have been calculated using the RO buffer but we 632 * need to use the R/W buffer for writes. 633 * switch ex to rw buffer for writing. 634 */ 635 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); 636 637 ex->insn = ins_offset; 638 639 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 640 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 641 ex->type = EX_TYPE_BPF; 642 643 ctx->nexentries++; 644 return 0; 645 } 646 647 static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 648 { 649 s64 rvoff; 650 struct rv_jit_context ctx; 651 652 ctx.ninsns = 0; 653 ctx.insns = (u16 *)insns; 654 655 if (!target) { 656 emit(rv_nop(), &ctx); 657 emit(rv_nop(), &ctx); 658 return 0; 659 } 660 661 rvoff = (s64)(target - ip); 662 return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 663 } 664 665 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 666 void *old_addr, void *new_addr) 667 { 668 u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 669 bool is_call = poke_type == BPF_MOD_CALL; 670 int ret; 671 672 if (!is_kernel_text((unsigned long)ip) && 673 !is_bpf_text_address((unsigned long)ip)) 674 return -ENOTSUPP; 675 676 ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 677 if (ret) 678 return ret; 679 680 if (memcmp(ip, old_insns, RV_FENTRY_NBYTES)) 681 return -EFAULT; 682 683 ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 684 if (ret) 685 return ret; 686 687 cpus_read_lock(); 688 mutex_lock(&text_mutex); 689 if (memcmp(ip, new_insns, RV_FENTRY_NBYTES)) 690 ret = patch_text(ip, new_insns, RV_FENTRY_NBYTES); 691 mutex_unlock(&text_mutex); 692 cpus_read_unlock(); 693 694 return ret; 695 } 696 697 static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) 698 { 699 int i; 700 701 for (i = 0; i < nr_arg_slots; i++) { 702 if (i < RV_MAX_REG_ARGS) { 703 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 704 } else { 705 /* skip slots for T0 and FP of traced function */ 706 emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); 707 emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); 708 } 709 args_off -= 8; 710 } 711 } 712 713 static void restore_args(int nr_reg_args, int args_off, struct rv_jit_context *ctx) 714 { 715 int i; 716 717 for (i = 0; i < nr_reg_args; i++) { 718 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 719 args_off -= 8; 720 } 721 } 722 723 static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off, 724 struct rv_jit_context *ctx) 725 { 726 int i; 727 728 for (i = 0; i < nr_stack_args; i++) { 729 emit_ld(RV_REG_T1, -(args_off - RV_MAX_REG_ARGS * 8), RV_REG_FP, ctx); 730 emit_sd(RV_REG_FP, -stk_arg_off, RV_REG_T1, ctx); 731 args_off -= 8; 732 stk_arg_off -= 8; 733 } 734 } 735 736 static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 737 int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 738 { 739 int ret, branch_off; 740 struct bpf_prog *p = l->link.prog; 741 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 742 743 if (l->cookie) { 744 emit_imm(RV_REG_T1, l->cookie, ctx); 745 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 746 } else { 747 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 748 } 749 750 /* arg1: prog */ 751 emit_imm(RV_REG_A0, (const s64)p, ctx); 752 /* arg2: &run_ctx */ 753 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 754 ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 755 if (ret) 756 return ret; 757 758 /* store prog start time */ 759 emit_mv(RV_REG_S1, RV_REG_A0, ctx); 760 761 /* if (__bpf_prog_enter(prog) == 0) 762 * goto skip_exec_of_prog; 763 */ 764 branch_off = ctx->ninsns; 765 /* nop reserved for conditional jump */ 766 emit(rv_nop(), ctx); 767 768 /* arg1: &args_off */ 769 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 770 if (!p->jited) 771 /* arg2: progs[i]->insnsi for interpreter */ 772 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 773 ret = emit_call((const u64)p->bpf_func, true, ctx); 774 if (ret) 775 return ret; 776 777 if (save_ret) { 778 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 779 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 780 } 781 782 /* update branch with beqz */ 783 if (ctx->insns) { 784 int offset = ninsns_rvoff(ctx->ninsns - branch_off); 785 u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 786 *(u32 *)(ctx->insns + branch_off) = insn; 787 } 788 789 /* arg1: prog */ 790 emit_imm(RV_REG_A0, (const s64)p, ctx); 791 /* arg2: prog start time */ 792 emit_mv(RV_REG_A1, RV_REG_S1, ctx); 793 /* arg3: &run_ctx */ 794 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 795 ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 796 797 return ret; 798 } 799 800 static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 801 const struct btf_func_model *m, 802 struct bpf_tramp_links *tlinks, 803 void *func_addr, u32 flags, 804 struct rv_jit_context *ctx) 805 { 806 int i, ret, offset; 807 int *branches_off = NULL; 808 int stack_size = 0, nr_arg_slots = 0; 809 int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; 810 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 811 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 812 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 813 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 814 void *orig_call = func_addr; 815 bool save_ret; 816 u32 insn; 817 818 /* Two types of generated trampoline stack layout: 819 * 820 * 1. trampoline called from function entry 821 * -------------------------------------- 822 * FP + 8 [ RA to parent func ] return address to parent 823 * function 824 * FP + 0 [ FP of parent func ] frame pointer of parent 825 * function 826 * FP - 8 [ T0 to traced func ] return address of traced 827 * function 828 * FP - 16 [ FP of traced func ] frame pointer of traced 829 * function 830 * -------------------------------------- 831 * 832 * 2. trampoline called directly 833 * -------------------------------------- 834 * FP - 8 [ RA to caller func ] return address to caller 835 * function 836 * FP - 16 [ FP of caller func ] frame pointer of caller 837 * function 838 * -------------------------------------- 839 * 840 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 841 * BPF_TRAMP_F_RET_FENTRY_RET 842 * [ argN ] 843 * [ ... ] 844 * FP - args_off [ arg1 ] 845 * 846 * FP - nregs_off [ regs count ] 847 * 848 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 849 * 850 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 851 * 852 * FP - sreg_off [ callee saved reg ] 853 * 854 * [ pads ] pads for 16 bytes alignment 855 * 856 * [ stack_argN ] 857 * [ ... ] 858 * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG 859 */ 860 861 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 862 return -ENOTSUPP; 863 864 if (m->nr_args > MAX_BPF_FUNC_ARGS) 865 return -ENOTSUPP; 866 867 for (i = 0; i < m->nr_args; i++) 868 nr_arg_slots += round_up(m->arg_size[i], 8) / 8; 869 870 /* room of trampoline frame to store return address and frame pointer */ 871 stack_size += 16; 872 873 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 874 if (save_ret) { 875 stack_size += 16; /* Save both A5 (BPF R0) and A0 */ 876 retval_off = stack_size; 877 } 878 879 stack_size += nr_arg_slots * 8; 880 args_off = stack_size; 881 882 stack_size += 8; 883 nregs_off = stack_size; 884 885 if (flags & BPF_TRAMP_F_IP_ARG) { 886 stack_size += 8; 887 ip_off = stack_size; 888 } 889 890 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 891 run_ctx_off = stack_size; 892 893 stack_size += 8; 894 sreg_off = stack_size; 895 896 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) 897 stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; 898 899 stack_size = round_up(stack_size, STACK_ALIGN); 900 901 /* room for args on stack must be at the top of stack */ 902 stk_arg_off = stack_size; 903 904 if (!is_struct_ops) { 905 /* For the trampoline called from function entry, 906 * the frame of traced function and the frame of 907 * trampoline need to be considered. 908 */ 909 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 910 emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 911 emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 912 emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 913 914 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 915 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 916 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 917 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 918 } else { 919 /* emit kcfi hash */ 920 emit_kcfi(cfi_get_func_hash(func_addr), ctx); 921 /* For the trampoline called directly, just handle 922 * the frame of trampoline. 923 */ 924 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 925 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 926 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 927 emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 928 } 929 930 /* callee saved register S1 to pass start time */ 931 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 932 933 /* store ip address of the traced function */ 934 if (flags & BPF_TRAMP_F_IP_ARG) { 935 emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 936 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 937 } 938 939 emit_li(RV_REG_T1, nr_arg_slots, ctx); 940 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 941 942 store_args(nr_arg_slots, args_off, ctx); 943 944 /* skip to actual body of traced function */ 945 if (flags & BPF_TRAMP_F_SKIP_FRAME) 946 orig_call += RV_FENTRY_NINSNS * 4; 947 948 if (flags & BPF_TRAMP_F_CALL_ORIG) { 949 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); 950 ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 951 if (ret) 952 return ret; 953 } 954 955 for (i = 0; i < fentry->nr_links; i++) { 956 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 957 flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 958 if (ret) 959 return ret; 960 } 961 962 if (fmod_ret->nr_links) { 963 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 964 if (!branches_off) 965 return -ENOMEM; 966 967 /* cleanup to avoid garbage return value confusion */ 968 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 969 for (i = 0; i < fmod_ret->nr_links; i++) { 970 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 971 run_ctx_off, true, ctx); 972 if (ret) 973 goto out; 974 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 975 branches_off[i] = ctx->ninsns; 976 /* nop reserved for conditional jump */ 977 emit(rv_nop(), ctx); 978 } 979 } 980 981 if (flags & BPF_TRAMP_F_CALL_ORIG) { 982 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); 983 restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); 984 ret = emit_call((const u64)orig_call, true, ctx); 985 if (ret) 986 goto out; 987 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 988 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); 989 im->ip_after_call = ctx->ro_insns + ctx->ninsns; 990 /* 2 nops reserved for auipc+jalr pair */ 991 emit(rv_nop(), ctx); 992 emit(rv_nop(), ctx); 993 } 994 995 /* update branches saved in invoke_bpf_mod_ret with bnez */ 996 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 997 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 998 insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 999 *(u32 *)(ctx->insns + branches_off[i]) = insn; 1000 } 1001 1002 for (i = 0; i < fexit->nr_links; i++) { 1003 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 1004 run_ctx_off, false, ctx); 1005 if (ret) 1006 goto out; 1007 } 1008 1009 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1010 im->ip_epilogue = ctx->ro_insns + ctx->ninsns; 1011 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); 1012 ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 1013 if (ret) 1014 goto out; 1015 } 1016 1017 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1018 restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); 1019 1020 if (save_ret) { 1021 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 1022 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); 1023 } 1024 1025 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 1026 1027 if (!is_struct_ops) { 1028 /* trampoline called from function entry */ 1029 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 1030 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1031 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1032 1033 emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 1034 emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 1035 emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 1036 1037 if (flags & BPF_TRAMP_F_SKIP_FRAME) 1038 /* return to parent function */ 1039 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1040 else 1041 /* return to traced function */ 1042 emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 1043 } else { 1044 /* trampoline called directly */ 1045 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 1046 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 1047 emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 1048 1049 emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 1050 } 1051 1052 ret = ctx->ninsns; 1053 out: 1054 kfree(branches_off); 1055 return ret; 1056 } 1057 1058 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1059 struct bpf_tramp_links *tlinks, void *func_addr) 1060 { 1061 struct bpf_tramp_image im; 1062 struct rv_jit_context ctx; 1063 int ret; 1064 1065 ctx.ninsns = 0; 1066 ctx.insns = NULL; 1067 ctx.ro_insns = NULL; 1068 ret = __arch_prepare_bpf_trampoline(&im, m, tlinks, func_addr, flags, &ctx); 1069 1070 return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns); 1071 } 1072 1073 void *arch_alloc_bpf_trampoline(unsigned int size) 1074 { 1075 return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); 1076 } 1077 1078 void arch_free_bpf_trampoline(void *image, unsigned int size) 1079 { 1080 bpf_prog_pack_free(image, size); 1081 } 1082 1083 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, 1084 void *ro_image_end, const struct btf_func_model *m, 1085 u32 flags, struct bpf_tramp_links *tlinks, 1086 void *func_addr) 1087 { 1088 int ret; 1089 void *image, *res; 1090 struct rv_jit_context ctx; 1091 u32 size = ro_image_end - ro_image; 1092 1093 image = kvmalloc(size, GFP_KERNEL); 1094 if (!image) 1095 return -ENOMEM; 1096 1097 ctx.ninsns = 0; 1098 ctx.insns = image; 1099 ctx.ro_insns = ro_image; 1100 ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 1101 if (ret < 0) 1102 goto out; 1103 1104 if (WARN_ON(size < ninsns_rvoff(ctx.ninsns))) { 1105 ret = -E2BIG; 1106 goto out; 1107 } 1108 1109 res = bpf_arch_text_copy(ro_image, image, size); 1110 if (IS_ERR(res)) { 1111 ret = PTR_ERR(res); 1112 goto out; 1113 } 1114 1115 bpf_flush_icache(ro_image, ro_image_end); 1116 out: 1117 kvfree(image); 1118 return ret < 0 ? ret : size; 1119 } 1120 1121 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1122 bool extra_pass) 1123 { 1124 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1125 BPF_CLASS(insn->code) == BPF_JMP; 1126 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1127 struct bpf_prog_aux *aux = ctx->prog->aux; 1128 u8 rd = -1, rs = -1, code = insn->code; 1129 s16 off = insn->off; 1130 s32 imm = insn->imm; 1131 1132 init_regs(&rd, &rs, insn, ctx); 1133 1134 switch (code) { 1135 /* dst = src */ 1136 case BPF_ALU | BPF_MOV | BPF_X: 1137 case BPF_ALU64 | BPF_MOV | BPF_X: 1138 if (insn_is_cast_user(insn)) { 1139 emit_mv(RV_REG_T1, rs, ctx); 1140 emit_zextw(RV_REG_T1, RV_REG_T1, ctx); 1141 emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx); 1142 emit(rv_beq(RV_REG_T1, RV_REG_ZERO, 4), ctx); 1143 emit_or(RV_REG_T1, rd, RV_REG_T1, ctx); 1144 emit_mv(rd, RV_REG_T1, ctx); 1145 break; 1146 } else if (insn_is_mov_percpu_addr(insn)) { 1147 if (rd != rs) 1148 emit_mv(rd, rs, ctx); 1149 #ifdef CONFIG_SMP 1150 /* Load current CPU number in T1 */ 1151 emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu), 1152 RV_REG_TP, ctx); 1153 /* Load address of __per_cpu_offset array in T2 */ 1154 emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx); 1155 /* Get address of __per_cpu_offset[cpu] in T1 */ 1156 emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx); 1157 /* Load __per_cpu_offset[cpu] in T1 */ 1158 emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx); 1159 /* Add the offset to Rd */ 1160 emit_add(rd, rd, RV_REG_T1, ctx); 1161 #endif 1162 } 1163 if (imm == 1) { 1164 /* Special mov32 for zext */ 1165 emit_zextw(rd, rd, ctx); 1166 break; 1167 } 1168 switch (insn->off) { 1169 case 0: 1170 emit_mv(rd, rs, ctx); 1171 break; 1172 case 8: 1173 emit_sextb(rd, rs, ctx); 1174 break; 1175 case 16: 1176 emit_sexth(rd, rs, ctx); 1177 break; 1178 case 32: 1179 emit_sextw(rd, rs, ctx); 1180 break; 1181 } 1182 if (!is64 && !aux->verifier_zext) 1183 emit_zextw(rd, rd, ctx); 1184 break; 1185 1186 /* dst = dst OP src */ 1187 case BPF_ALU | BPF_ADD | BPF_X: 1188 case BPF_ALU64 | BPF_ADD | BPF_X: 1189 emit_add(rd, rd, rs, ctx); 1190 if (!is64 && !aux->verifier_zext) 1191 emit_zextw(rd, rd, ctx); 1192 break; 1193 case BPF_ALU | BPF_SUB | BPF_X: 1194 case BPF_ALU64 | BPF_SUB | BPF_X: 1195 if (is64) 1196 emit_sub(rd, rd, rs, ctx); 1197 else 1198 emit_subw(rd, rd, rs, ctx); 1199 1200 if (!is64 && !aux->verifier_zext) 1201 emit_zextw(rd, rd, ctx); 1202 break; 1203 case BPF_ALU | BPF_AND | BPF_X: 1204 case BPF_ALU64 | BPF_AND | BPF_X: 1205 emit_and(rd, rd, rs, ctx); 1206 if (!is64 && !aux->verifier_zext) 1207 emit_zextw(rd, rd, ctx); 1208 break; 1209 case BPF_ALU | BPF_OR | BPF_X: 1210 case BPF_ALU64 | BPF_OR | BPF_X: 1211 emit_or(rd, rd, rs, ctx); 1212 if (!is64 && !aux->verifier_zext) 1213 emit_zextw(rd, rd, ctx); 1214 break; 1215 case BPF_ALU | BPF_XOR | BPF_X: 1216 case BPF_ALU64 | BPF_XOR | BPF_X: 1217 emit_xor(rd, rd, rs, ctx); 1218 if (!is64 && !aux->verifier_zext) 1219 emit_zextw(rd, rd, ctx); 1220 break; 1221 case BPF_ALU | BPF_MUL | BPF_X: 1222 case BPF_ALU64 | BPF_MUL | BPF_X: 1223 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1224 if (!is64 && !aux->verifier_zext) 1225 emit_zextw(rd, rd, ctx); 1226 break; 1227 case BPF_ALU | BPF_DIV | BPF_X: 1228 case BPF_ALU64 | BPF_DIV | BPF_X: 1229 if (off) 1230 emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1231 else 1232 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1233 if (!is64 && !aux->verifier_zext) 1234 emit_zextw(rd, rd, ctx); 1235 break; 1236 case BPF_ALU | BPF_MOD | BPF_X: 1237 case BPF_ALU64 | BPF_MOD | BPF_X: 1238 if (off) 1239 emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1240 else 1241 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1242 if (!is64 && !aux->verifier_zext) 1243 emit_zextw(rd, rd, ctx); 1244 break; 1245 case BPF_ALU | BPF_LSH | BPF_X: 1246 case BPF_ALU64 | BPF_LSH | BPF_X: 1247 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 1248 if (!is64 && !aux->verifier_zext) 1249 emit_zextw(rd, rd, ctx); 1250 break; 1251 case BPF_ALU | BPF_RSH | BPF_X: 1252 case BPF_ALU64 | BPF_RSH | BPF_X: 1253 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1254 if (!is64 && !aux->verifier_zext) 1255 emit_zextw(rd, rd, ctx); 1256 break; 1257 case BPF_ALU | BPF_ARSH | BPF_X: 1258 case BPF_ALU64 | BPF_ARSH | BPF_X: 1259 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1260 if (!is64 && !aux->verifier_zext) 1261 emit_zextw(rd, rd, ctx); 1262 break; 1263 1264 /* dst = -dst */ 1265 case BPF_ALU | BPF_NEG: 1266 case BPF_ALU64 | BPF_NEG: 1267 emit_sub(rd, RV_REG_ZERO, rd, ctx); 1268 if (!is64 && !aux->verifier_zext) 1269 emit_zextw(rd, rd, ctx); 1270 break; 1271 1272 /* dst = BSWAP##imm(dst) */ 1273 case BPF_ALU | BPF_END | BPF_FROM_LE: 1274 switch (imm) { 1275 case 16: 1276 emit_zexth(rd, rd, ctx); 1277 break; 1278 case 32: 1279 if (!aux->verifier_zext) 1280 emit_zextw(rd, rd, ctx); 1281 break; 1282 case 64: 1283 /* Do nothing */ 1284 break; 1285 } 1286 break; 1287 case BPF_ALU | BPF_END | BPF_FROM_BE: 1288 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1289 emit_bswap(rd, imm, ctx); 1290 break; 1291 1292 /* dst = imm */ 1293 case BPF_ALU | BPF_MOV | BPF_K: 1294 case BPF_ALU64 | BPF_MOV | BPF_K: 1295 emit_imm(rd, imm, ctx); 1296 if (!is64 && !aux->verifier_zext) 1297 emit_zextw(rd, rd, ctx); 1298 break; 1299 1300 /* dst = dst OP imm */ 1301 case BPF_ALU | BPF_ADD | BPF_K: 1302 case BPF_ALU64 | BPF_ADD | BPF_K: 1303 if (is_12b_int(imm)) { 1304 emit_addi(rd, rd, imm, ctx); 1305 } else { 1306 emit_imm(RV_REG_T1, imm, ctx); 1307 emit_add(rd, rd, RV_REG_T1, ctx); 1308 } 1309 if (!is64 && !aux->verifier_zext) 1310 emit_zextw(rd, rd, ctx); 1311 break; 1312 case BPF_ALU | BPF_SUB | BPF_K: 1313 case BPF_ALU64 | BPF_SUB | BPF_K: 1314 if (is_12b_int(-imm)) { 1315 emit_addi(rd, rd, -imm, ctx); 1316 } else { 1317 emit_imm(RV_REG_T1, imm, ctx); 1318 emit_sub(rd, rd, RV_REG_T1, ctx); 1319 } 1320 if (!is64 && !aux->verifier_zext) 1321 emit_zextw(rd, rd, ctx); 1322 break; 1323 case BPF_ALU | BPF_AND | BPF_K: 1324 case BPF_ALU64 | BPF_AND | BPF_K: 1325 if (is_12b_int(imm)) { 1326 emit_andi(rd, rd, imm, ctx); 1327 } else { 1328 emit_imm(RV_REG_T1, imm, ctx); 1329 emit_and(rd, rd, RV_REG_T1, ctx); 1330 } 1331 if (!is64 && !aux->verifier_zext) 1332 emit_zextw(rd, rd, ctx); 1333 break; 1334 case BPF_ALU | BPF_OR | BPF_K: 1335 case BPF_ALU64 | BPF_OR | BPF_K: 1336 if (is_12b_int(imm)) { 1337 emit(rv_ori(rd, rd, imm), ctx); 1338 } else { 1339 emit_imm(RV_REG_T1, imm, ctx); 1340 emit_or(rd, rd, RV_REG_T1, ctx); 1341 } 1342 if (!is64 && !aux->verifier_zext) 1343 emit_zextw(rd, rd, ctx); 1344 break; 1345 case BPF_ALU | BPF_XOR | BPF_K: 1346 case BPF_ALU64 | BPF_XOR | BPF_K: 1347 if (is_12b_int(imm)) { 1348 emit(rv_xori(rd, rd, imm), ctx); 1349 } else { 1350 emit_imm(RV_REG_T1, imm, ctx); 1351 emit_xor(rd, rd, RV_REG_T1, ctx); 1352 } 1353 if (!is64 && !aux->verifier_zext) 1354 emit_zextw(rd, rd, ctx); 1355 break; 1356 case BPF_ALU | BPF_MUL | BPF_K: 1357 case BPF_ALU64 | BPF_MUL | BPF_K: 1358 emit_imm(RV_REG_T1, imm, ctx); 1359 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1360 rv_mulw(rd, rd, RV_REG_T1), ctx); 1361 if (!is64 && !aux->verifier_zext) 1362 emit_zextw(rd, rd, ctx); 1363 break; 1364 case BPF_ALU | BPF_DIV | BPF_K: 1365 case BPF_ALU64 | BPF_DIV | BPF_K: 1366 emit_imm(RV_REG_T1, imm, ctx); 1367 if (off) 1368 emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1369 rv_divw(rd, rd, RV_REG_T1), ctx); 1370 else 1371 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1372 rv_divuw(rd, rd, RV_REG_T1), ctx); 1373 if (!is64 && !aux->verifier_zext) 1374 emit_zextw(rd, rd, ctx); 1375 break; 1376 case BPF_ALU | BPF_MOD | BPF_K: 1377 case BPF_ALU64 | BPF_MOD | BPF_K: 1378 emit_imm(RV_REG_T1, imm, ctx); 1379 if (off) 1380 emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1381 rv_remw(rd, rd, RV_REG_T1), ctx); 1382 else 1383 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1384 rv_remuw(rd, rd, RV_REG_T1), ctx); 1385 if (!is64 && !aux->verifier_zext) 1386 emit_zextw(rd, rd, ctx); 1387 break; 1388 case BPF_ALU | BPF_LSH | BPF_K: 1389 case BPF_ALU64 | BPF_LSH | BPF_K: 1390 emit_slli(rd, rd, imm, ctx); 1391 1392 if (!is64 && !aux->verifier_zext) 1393 emit_zextw(rd, rd, ctx); 1394 break; 1395 case BPF_ALU | BPF_RSH | BPF_K: 1396 case BPF_ALU64 | BPF_RSH | BPF_K: 1397 if (is64) 1398 emit_srli(rd, rd, imm, ctx); 1399 else 1400 emit(rv_srliw(rd, rd, imm), ctx); 1401 1402 if (!is64 && !aux->verifier_zext) 1403 emit_zextw(rd, rd, ctx); 1404 break; 1405 case BPF_ALU | BPF_ARSH | BPF_K: 1406 case BPF_ALU64 | BPF_ARSH | BPF_K: 1407 if (is64) 1408 emit_srai(rd, rd, imm, ctx); 1409 else 1410 emit(rv_sraiw(rd, rd, imm), ctx); 1411 1412 if (!is64 && !aux->verifier_zext) 1413 emit_zextw(rd, rd, ctx); 1414 break; 1415 1416 /* JUMP off */ 1417 case BPF_JMP | BPF_JA: 1418 case BPF_JMP32 | BPF_JA: 1419 if (BPF_CLASS(code) == BPF_JMP) 1420 rvoff = rv_offset(i, off, ctx); 1421 else 1422 rvoff = rv_offset(i, imm, ctx); 1423 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1424 if (ret) 1425 return ret; 1426 break; 1427 1428 /* IF (dst COND src) JUMP off */ 1429 case BPF_JMP | BPF_JEQ | BPF_X: 1430 case BPF_JMP32 | BPF_JEQ | BPF_X: 1431 case BPF_JMP | BPF_JGT | BPF_X: 1432 case BPF_JMP32 | BPF_JGT | BPF_X: 1433 case BPF_JMP | BPF_JLT | BPF_X: 1434 case BPF_JMP32 | BPF_JLT | BPF_X: 1435 case BPF_JMP | BPF_JGE | BPF_X: 1436 case BPF_JMP32 | BPF_JGE | BPF_X: 1437 case BPF_JMP | BPF_JLE | BPF_X: 1438 case BPF_JMP32 | BPF_JLE | BPF_X: 1439 case BPF_JMP | BPF_JNE | BPF_X: 1440 case BPF_JMP32 | BPF_JNE | BPF_X: 1441 case BPF_JMP | BPF_JSGT | BPF_X: 1442 case BPF_JMP32 | BPF_JSGT | BPF_X: 1443 case BPF_JMP | BPF_JSLT | BPF_X: 1444 case BPF_JMP32 | BPF_JSLT | BPF_X: 1445 case BPF_JMP | BPF_JSGE | BPF_X: 1446 case BPF_JMP32 | BPF_JSGE | BPF_X: 1447 case BPF_JMP | BPF_JSLE | BPF_X: 1448 case BPF_JMP32 | BPF_JSLE | BPF_X: 1449 case BPF_JMP | BPF_JSET | BPF_X: 1450 case BPF_JMP32 | BPF_JSET | BPF_X: 1451 rvoff = rv_offset(i, off, ctx); 1452 if (!is64) { 1453 s = ctx->ninsns; 1454 if (is_signed_bpf_cond(BPF_OP(code))) { 1455 emit_sextw_alt(&rs, RV_REG_T1, ctx); 1456 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1457 } else { 1458 emit_zextw_alt(&rs, RV_REG_T1, ctx); 1459 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1460 } 1461 e = ctx->ninsns; 1462 1463 /* Adjust for extra insns */ 1464 rvoff -= ninsns_rvoff(e - s); 1465 } 1466 1467 if (BPF_OP(code) == BPF_JSET) { 1468 /* Adjust for and */ 1469 rvoff -= 4; 1470 emit_and(RV_REG_T1, rd, rs, ctx); 1471 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1472 } else { 1473 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1474 } 1475 break; 1476 1477 /* IF (dst COND imm) JUMP off */ 1478 case BPF_JMP | BPF_JEQ | BPF_K: 1479 case BPF_JMP32 | BPF_JEQ | BPF_K: 1480 case BPF_JMP | BPF_JGT | BPF_K: 1481 case BPF_JMP32 | BPF_JGT | BPF_K: 1482 case BPF_JMP | BPF_JLT | BPF_K: 1483 case BPF_JMP32 | BPF_JLT | BPF_K: 1484 case BPF_JMP | BPF_JGE | BPF_K: 1485 case BPF_JMP32 | BPF_JGE | BPF_K: 1486 case BPF_JMP | BPF_JLE | BPF_K: 1487 case BPF_JMP32 | BPF_JLE | BPF_K: 1488 case BPF_JMP | BPF_JNE | BPF_K: 1489 case BPF_JMP32 | BPF_JNE | BPF_K: 1490 case BPF_JMP | BPF_JSGT | BPF_K: 1491 case BPF_JMP32 | BPF_JSGT | BPF_K: 1492 case BPF_JMP | BPF_JSLT | BPF_K: 1493 case BPF_JMP32 | BPF_JSLT | BPF_K: 1494 case BPF_JMP | BPF_JSGE | BPF_K: 1495 case BPF_JMP32 | BPF_JSGE | BPF_K: 1496 case BPF_JMP | BPF_JSLE | BPF_K: 1497 case BPF_JMP32 | BPF_JSLE | BPF_K: 1498 rvoff = rv_offset(i, off, ctx); 1499 s = ctx->ninsns; 1500 if (imm) 1501 emit_imm(RV_REG_T1, imm, ctx); 1502 rs = imm ? RV_REG_T1 : RV_REG_ZERO; 1503 if (!is64) { 1504 if (is_signed_bpf_cond(BPF_OP(code))) { 1505 emit_sextw_alt(&rd, RV_REG_T2, ctx); 1506 /* rs has been sign extended */ 1507 } else { 1508 emit_zextw_alt(&rd, RV_REG_T2, ctx); 1509 if (imm) 1510 emit_zextw(rs, rs, ctx); 1511 } 1512 } 1513 e = ctx->ninsns; 1514 1515 /* Adjust for extra insns */ 1516 rvoff -= ninsns_rvoff(e - s); 1517 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1518 break; 1519 1520 case BPF_JMP | BPF_JSET | BPF_K: 1521 case BPF_JMP32 | BPF_JSET | BPF_K: 1522 rvoff = rv_offset(i, off, ctx); 1523 s = ctx->ninsns; 1524 if (is_12b_int(imm)) { 1525 emit_andi(RV_REG_T1, rd, imm, ctx); 1526 } else { 1527 emit_imm(RV_REG_T1, imm, ctx); 1528 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1529 } 1530 /* For jset32, we should clear the upper 32 bits of t1, but 1531 * sign-extension is sufficient here and saves one instruction, 1532 * as t1 is used only in comparison against zero. 1533 */ 1534 if (!is64 && imm < 0) 1535 emit_sextw(RV_REG_T1, RV_REG_T1, ctx); 1536 e = ctx->ninsns; 1537 rvoff -= ninsns_rvoff(e - s); 1538 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1539 break; 1540 1541 /* function call */ 1542 case BPF_JMP | BPF_CALL: 1543 { 1544 bool fixed_addr; 1545 u64 addr; 1546 1547 /* Inline calls to bpf_get_smp_processor_id() 1548 * 1549 * RV_REG_TP holds the address of the current CPU's task_struct and thread_info is 1550 * at offset 0 in task_struct. 1551 * Load cpu from thread_info: 1552 * Set R0 to ((struct thread_info *)(RV_REG_TP))->cpu 1553 * 1554 * This replicates the implementation of raw_smp_processor_id() on RISCV 1555 */ 1556 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { 1557 /* Load current CPU number in R0 */ 1558 emit_ld(bpf_to_rv_reg(BPF_REG_0, ctx), offsetof(struct thread_info, cpu), 1559 RV_REG_TP, ctx); 1560 break; 1561 } 1562 1563 mark_call(ctx); 1564 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1565 &addr, &fixed_addr); 1566 if (ret < 0) 1567 return ret; 1568 1569 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1570 const struct btf_func_model *fm; 1571 int idx; 1572 1573 fm = bpf_jit_find_kfunc_model(ctx->prog, insn); 1574 if (!fm) 1575 return -EINVAL; 1576 1577 for (idx = 0; idx < fm->nr_args; idx++) { 1578 u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); 1579 1580 if (fm->arg_size[idx] == sizeof(int)) 1581 emit_sextw(reg, reg, ctx); 1582 } 1583 } 1584 1585 ret = emit_call(addr, fixed_addr, ctx); 1586 if (ret) 1587 return ret; 1588 1589 if (insn->src_reg != BPF_PSEUDO_CALL) 1590 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1591 break; 1592 } 1593 /* tail call */ 1594 case BPF_JMP | BPF_TAIL_CALL: 1595 if (emit_bpf_tail_call(i, ctx)) 1596 return -1; 1597 break; 1598 1599 /* function return */ 1600 case BPF_JMP | BPF_EXIT: 1601 if (i == ctx->prog->len - 1) 1602 break; 1603 1604 rvoff = epilogue_offset(ctx); 1605 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1606 if (ret) 1607 return ret; 1608 break; 1609 1610 /* dst = imm64 */ 1611 case BPF_LD | BPF_IMM | BPF_DW: 1612 { 1613 struct bpf_insn insn1 = insn[1]; 1614 u64 imm64; 1615 1616 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1617 if (bpf_pseudo_func(insn)) { 1618 /* fixed-length insns for extra jit pass */ 1619 ret = emit_addr(rd, imm64, extra_pass, ctx); 1620 if (ret) 1621 return ret; 1622 } else { 1623 emit_imm(rd, imm64, ctx); 1624 } 1625 1626 return 1; 1627 } 1628 1629 /* LDX: dst = *(unsigned size *)(src + off) */ 1630 case BPF_LDX | BPF_MEM | BPF_B: 1631 case BPF_LDX | BPF_MEM | BPF_H: 1632 case BPF_LDX | BPF_MEM | BPF_W: 1633 case BPF_LDX | BPF_MEM | BPF_DW: 1634 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1635 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1636 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1637 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1638 /* LDSX: dst = *(signed size *)(src + off) */ 1639 case BPF_LDX | BPF_MEMSX | BPF_B: 1640 case BPF_LDX | BPF_MEMSX | BPF_H: 1641 case BPF_LDX | BPF_MEMSX | BPF_W: 1642 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1643 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1644 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1645 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + RV_REG_ARENA + off) */ 1646 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1647 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1648 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1649 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1650 { 1651 int insn_len, insns_start; 1652 bool sign_ext; 1653 1654 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 1655 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1656 1657 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { 1658 emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); 1659 rs = RV_REG_T2; 1660 } 1661 1662 switch (BPF_SIZE(code)) { 1663 case BPF_B: 1664 if (is_12b_int(off)) { 1665 insns_start = ctx->ninsns; 1666 if (sign_ext) 1667 emit(rv_lb(rd, off, rs), ctx); 1668 else 1669 emit(rv_lbu(rd, off, rs), ctx); 1670 insn_len = ctx->ninsns - insns_start; 1671 break; 1672 } 1673 1674 emit_imm(RV_REG_T1, off, ctx); 1675 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1676 insns_start = ctx->ninsns; 1677 if (sign_ext) 1678 emit(rv_lb(rd, 0, RV_REG_T1), ctx); 1679 else 1680 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1681 insn_len = ctx->ninsns - insns_start; 1682 break; 1683 case BPF_H: 1684 if (is_12b_int(off)) { 1685 insns_start = ctx->ninsns; 1686 if (sign_ext) 1687 emit(rv_lh(rd, off, rs), ctx); 1688 else 1689 emit(rv_lhu(rd, off, rs), ctx); 1690 insn_len = ctx->ninsns - insns_start; 1691 break; 1692 } 1693 1694 emit_imm(RV_REG_T1, off, ctx); 1695 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1696 insns_start = ctx->ninsns; 1697 if (sign_ext) 1698 emit(rv_lh(rd, 0, RV_REG_T1), ctx); 1699 else 1700 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1701 insn_len = ctx->ninsns - insns_start; 1702 break; 1703 case BPF_W: 1704 if (is_12b_int(off)) { 1705 insns_start = ctx->ninsns; 1706 if (sign_ext) 1707 emit(rv_lw(rd, off, rs), ctx); 1708 else 1709 emit(rv_lwu(rd, off, rs), ctx); 1710 insn_len = ctx->ninsns - insns_start; 1711 break; 1712 } 1713 1714 emit_imm(RV_REG_T1, off, ctx); 1715 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1716 insns_start = ctx->ninsns; 1717 if (sign_ext) 1718 emit(rv_lw(rd, 0, RV_REG_T1), ctx); 1719 else 1720 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1721 insn_len = ctx->ninsns - insns_start; 1722 break; 1723 case BPF_DW: 1724 if (is_12b_int(off)) { 1725 insns_start = ctx->ninsns; 1726 emit_ld(rd, off, rs, ctx); 1727 insn_len = ctx->ninsns - insns_start; 1728 break; 1729 } 1730 1731 emit_imm(RV_REG_T1, off, ctx); 1732 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1733 insns_start = ctx->ninsns; 1734 emit_ld(rd, 0, RV_REG_T1, ctx); 1735 insn_len = ctx->ninsns - insns_start; 1736 break; 1737 } 1738 1739 ret = add_exception_handler(insn, ctx, rd, insn_len); 1740 if (ret) 1741 return ret; 1742 1743 if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1744 return 1; 1745 break; 1746 } 1747 /* speculation barrier */ 1748 case BPF_ST | BPF_NOSPEC: 1749 break; 1750 1751 /* ST: *(size *)(dst + off) = imm */ 1752 case BPF_ST | BPF_MEM | BPF_B: 1753 emit_imm(RV_REG_T1, imm, ctx); 1754 if (is_12b_int(off)) { 1755 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1756 break; 1757 } 1758 1759 emit_imm(RV_REG_T2, off, ctx); 1760 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1761 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1762 break; 1763 1764 case BPF_ST | BPF_MEM | BPF_H: 1765 emit_imm(RV_REG_T1, imm, ctx); 1766 if (is_12b_int(off)) { 1767 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1768 break; 1769 } 1770 1771 emit_imm(RV_REG_T2, off, ctx); 1772 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1773 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1774 break; 1775 case BPF_ST | BPF_MEM | BPF_W: 1776 emit_imm(RV_REG_T1, imm, ctx); 1777 if (is_12b_int(off)) { 1778 emit_sw(rd, off, RV_REG_T1, ctx); 1779 break; 1780 } 1781 1782 emit_imm(RV_REG_T2, off, ctx); 1783 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1784 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1785 break; 1786 case BPF_ST | BPF_MEM | BPF_DW: 1787 emit_imm(RV_REG_T1, imm, ctx); 1788 if (is_12b_int(off)) { 1789 emit_sd(rd, off, RV_REG_T1, ctx); 1790 break; 1791 } 1792 1793 emit_imm(RV_REG_T2, off, ctx); 1794 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1795 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1796 break; 1797 1798 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1799 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1800 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1801 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1802 { 1803 int insn_len, insns_start; 1804 1805 emit_add(RV_REG_T3, rd, RV_REG_ARENA, ctx); 1806 rd = RV_REG_T3; 1807 1808 /* Load imm to a register then store it */ 1809 emit_imm(RV_REG_T1, imm, ctx); 1810 1811 switch (BPF_SIZE(code)) { 1812 case BPF_B: 1813 if (is_12b_int(off)) { 1814 insns_start = ctx->ninsns; 1815 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1816 insn_len = ctx->ninsns - insns_start; 1817 break; 1818 } 1819 1820 emit_imm(RV_REG_T2, off, ctx); 1821 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1822 insns_start = ctx->ninsns; 1823 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1824 insn_len = ctx->ninsns - insns_start; 1825 break; 1826 case BPF_H: 1827 if (is_12b_int(off)) { 1828 insns_start = ctx->ninsns; 1829 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1830 insn_len = ctx->ninsns - insns_start; 1831 break; 1832 } 1833 1834 emit_imm(RV_REG_T2, off, ctx); 1835 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1836 insns_start = ctx->ninsns; 1837 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1838 insn_len = ctx->ninsns - insns_start; 1839 break; 1840 case BPF_W: 1841 if (is_12b_int(off)) { 1842 insns_start = ctx->ninsns; 1843 emit_sw(rd, off, RV_REG_T1, ctx); 1844 insn_len = ctx->ninsns - insns_start; 1845 break; 1846 } 1847 1848 emit_imm(RV_REG_T2, off, ctx); 1849 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1850 insns_start = ctx->ninsns; 1851 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1852 insn_len = ctx->ninsns - insns_start; 1853 break; 1854 case BPF_DW: 1855 if (is_12b_int(off)) { 1856 insns_start = ctx->ninsns; 1857 emit_sd(rd, off, RV_REG_T1, ctx); 1858 insn_len = ctx->ninsns - insns_start; 1859 break; 1860 } 1861 1862 emit_imm(RV_REG_T2, off, ctx); 1863 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1864 insns_start = ctx->ninsns; 1865 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1866 insn_len = ctx->ninsns - insns_start; 1867 break; 1868 } 1869 1870 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1871 insn_len); 1872 if (ret) 1873 return ret; 1874 1875 break; 1876 } 1877 1878 /* STX: *(size *)(dst + off) = src */ 1879 case BPF_STX | BPF_MEM | BPF_B: 1880 if (is_12b_int(off)) { 1881 emit(rv_sb(rd, off, rs), ctx); 1882 break; 1883 } 1884 1885 emit_imm(RV_REG_T1, off, ctx); 1886 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1887 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1888 break; 1889 case BPF_STX | BPF_MEM | BPF_H: 1890 if (is_12b_int(off)) { 1891 emit(rv_sh(rd, off, rs), ctx); 1892 break; 1893 } 1894 1895 emit_imm(RV_REG_T1, off, ctx); 1896 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1897 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1898 break; 1899 case BPF_STX | BPF_MEM | BPF_W: 1900 if (is_12b_int(off)) { 1901 emit_sw(rd, off, rs, ctx); 1902 break; 1903 } 1904 1905 emit_imm(RV_REG_T1, off, ctx); 1906 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1907 emit_sw(RV_REG_T1, 0, rs, ctx); 1908 break; 1909 case BPF_STX | BPF_MEM | BPF_DW: 1910 if (is_12b_int(off)) { 1911 emit_sd(rd, off, rs, ctx); 1912 break; 1913 } 1914 1915 emit_imm(RV_REG_T1, off, ctx); 1916 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1917 emit_sd(RV_REG_T1, 0, rs, ctx); 1918 break; 1919 case BPF_STX | BPF_ATOMIC | BPF_W: 1920 case BPF_STX | BPF_ATOMIC | BPF_DW: 1921 emit_atomic(rd, rs, off, imm, 1922 BPF_SIZE(code) == BPF_DW, ctx); 1923 break; 1924 1925 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1926 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1927 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1928 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1929 { 1930 int insn_len, insns_start; 1931 1932 emit_add(RV_REG_T2, rd, RV_REG_ARENA, ctx); 1933 rd = RV_REG_T2; 1934 1935 switch (BPF_SIZE(code)) { 1936 case BPF_B: 1937 if (is_12b_int(off)) { 1938 insns_start = ctx->ninsns; 1939 emit(rv_sb(rd, off, rs), ctx); 1940 insn_len = ctx->ninsns - insns_start; 1941 break; 1942 } 1943 1944 emit_imm(RV_REG_T1, off, ctx); 1945 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1946 insns_start = ctx->ninsns; 1947 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1948 insn_len = ctx->ninsns - insns_start; 1949 break; 1950 case BPF_H: 1951 if (is_12b_int(off)) { 1952 insns_start = ctx->ninsns; 1953 emit(rv_sh(rd, off, rs), ctx); 1954 insn_len = ctx->ninsns - insns_start; 1955 break; 1956 } 1957 1958 emit_imm(RV_REG_T1, off, ctx); 1959 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1960 insns_start = ctx->ninsns; 1961 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1962 insn_len = ctx->ninsns - insns_start; 1963 break; 1964 case BPF_W: 1965 if (is_12b_int(off)) { 1966 insns_start = ctx->ninsns; 1967 emit_sw(rd, off, rs, ctx); 1968 insn_len = ctx->ninsns - insns_start; 1969 break; 1970 } 1971 1972 emit_imm(RV_REG_T1, off, ctx); 1973 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1974 insns_start = ctx->ninsns; 1975 emit_sw(RV_REG_T1, 0, rs, ctx); 1976 insn_len = ctx->ninsns - insns_start; 1977 break; 1978 case BPF_DW: 1979 if (is_12b_int(off)) { 1980 insns_start = ctx->ninsns; 1981 emit_sd(rd, off, rs, ctx); 1982 insn_len = ctx->ninsns - insns_start; 1983 break; 1984 } 1985 1986 emit_imm(RV_REG_T1, off, ctx); 1987 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1988 insns_start = ctx->ninsns; 1989 emit_sd(RV_REG_T1, 0, rs, ctx); 1990 insn_len = ctx->ninsns - insns_start; 1991 break; 1992 } 1993 1994 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER, 1995 insn_len); 1996 if (ret) 1997 return ret; 1998 1999 break; 2000 } 2001 2002 default: 2003 pr_err("bpf-jit: unknown opcode %02x\n", code); 2004 return -EINVAL; 2005 } 2006 2007 return 0; 2008 } 2009 2010 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) 2011 { 2012 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 2013 2014 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); 2015 if (bpf_stack_adjust) 2016 mark_fp(ctx); 2017 2018 if (seen_reg(RV_REG_RA, ctx)) 2019 stack_adjust += 8; 2020 stack_adjust += 8; /* RV_REG_FP */ 2021 if (seen_reg(RV_REG_S1, ctx)) 2022 stack_adjust += 8; 2023 if (seen_reg(RV_REG_S2, ctx)) 2024 stack_adjust += 8; 2025 if (seen_reg(RV_REG_S3, ctx)) 2026 stack_adjust += 8; 2027 if (seen_reg(RV_REG_S4, ctx)) 2028 stack_adjust += 8; 2029 if (seen_reg(RV_REG_S5, ctx)) 2030 stack_adjust += 8; 2031 if (seen_reg(RV_REG_S6, ctx)) 2032 stack_adjust += 8; 2033 if (ctx->arena_vm_start) 2034 stack_adjust += 8; 2035 2036 stack_adjust = round_up(stack_adjust, STACK_ALIGN); 2037 stack_adjust += bpf_stack_adjust; 2038 2039 store_offset = stack_adjust - 8; 2040 2041 /* emit kcfi type preamble immediately before the first insn */ 2042 emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); 2043 2044 /* nops reserved for auipc+jalr pair */ 2045 for (i = 0; i < RV_FENTRY_NINSNS; i++) 2046 emit(rv_nop(), ctx); 2047 2048 /* First instruction is always setting the tail-call-counter 2049 * (TCC) register. This instruction is skipped for tail calls. 2050 * Force using a 4-byte (non-compressed) instruction. 2051 */ 2052 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 2053 2054 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 2055 2056 if (seen_reg(RV_REG_RA, ctx)) { 2057 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 2058 store_offset -= 8; 2059 } 2060 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 2061 store_offset -= 8; 2062 if (seen_reg(RV_REG_S1, ctx)) { 2063 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 2064 store_offset -= 8; 2065 } 2066 if (seen_reg(RV_REG_S2, ctx)) { 2067 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 2068 store_offset -= 8; 2069 } 2070 if (seen_reg(RV_REG_S3, ctx)) { 2071 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 2072 store_offset -= 8; 2073 } 2074 if (seen_reg(RV_REG_S4, ctx)) { 2075 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 2076 store_offset -= 8; 2077 } 2078 if (seen_reg(RV_REG_S5, ctx)) { 2079 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 2080 store_offset -= 8; 2081 } 2082 if (seen_reg(RV_REG_S6, ctx)) { 2083 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 2084 store_offset -= 8; 2085 } 2086 if (ctx->arena_vm_start) { 2087 emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); 2088 store_offset -= 8; 2089 } 2090 2091 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 2092 2093 if (bpf_stack_adjust) 2094 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 2095 2096 /* Program contains calls and tail calls, so RV_REG_TCC need 2097 * to be saved across calls. 2098 */ 2099 if (seen_tail_call(ctx) && seen_call(ctx)) 2100 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 2101 2102 ctx->stack_size = stack_adjust; 2103 2104 if (ctx->arena_vm_start) 2105 emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx); 2106 } 2107 2108 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 2109 { 2110 __build_epilogue(false, ctx); 2111 } 2112 2113 bool bpf_jit_supports_kfunc_call(void) 2114 { 2115 return true; 2116 } 2117 2118 bool bpf_jit_supports_ptr_xchg(void) 2119 { 2120 return true; 2121 } 2122 2123 bool bpf_jit_supports_arena(void) 2124 { 2125 return true; 2126 } 2127 2128 bool bpf_jit_supports_percpu_insn(void) 2129 { 2130 return true; 2131 } 2132 2133 bool bpf_jit_inlines_helper_call(s32 imm) 2134 { 2135 switch (imm) { 2136 case BPF_FUNC_get_smp_processor_id: 2137 return true; 2138 default: 2139 return false; 2140 } 2141 } 2142