1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/memory.h> 8 #include "bpf_jit.h" 9 10 #define LOONGARCH_MAX_REG_ARGS 8 11 12 #define LOONGARCH_LONG_JUMP_NINSNS 5 13 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 14 15 #define LOONGARCH_FENTRY_NINSNS 2 16 #define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) 17 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 18 19 #define REG_TCC LOONGARCH_GPR_A6 20 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) 21 22 static const int regmap[] = { 23 /* return value from in-kernel function, and exit value for eBPF program */ 24 [BPF_REG_0] = LOONGARCH_GPR_A5, 25 /* arguments from eBPF program to in-kernel function */ 26 [BPF_REG_1] = LOONGARCH_GPR_A0, 27 [BPF_REG_2] = LOONGARCH_GPR_A1, 28 [BPF_REG_3] = LOONGARCH_GPR_A2, 29 [BPF_REG_4] = LOONGARCH_GPR_A3, 30 [BPF_REG_5] = LOONGARCH_GPR_A4, 31 /* callee saved registers that in-kernel function will preserve */ 32 [BPF_REG_6] = LOONGARCH_GPR_S0, 33 [BPF_REG_7] = LOONGARCH_GPR_S1, 34 [BPF_REG_8] = LOONGARCH_GPR_S2, 35 [BPF_REG_9] = LOONGARCH_GPR_S3, 36 /* read-only frame pointer to access stack */ 37 [BPF_REG_FP] = LOONGARCH_GPR_S4, 38 /* temporary register for blinding constants */ 39 [BPF_REG_AX] = LOONGARCH_GPR_T0, 40 }; 41 42 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) 43 { 44 const struct bpf_prog *prog = ctx->prog; 45 const bool is_main_prog = !bpf_is_subprog(prog); 46 47 if (is_main_prog) { 48 /* 49 * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT 50 * if (REG_TCC > T3 ) 51 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 52 * else 53 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 54 * REG_TCC = LOONGARCH_GPR_SP + store_offset 55 * 56 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 57 * 58 * The purpose of this code is to first push the TCC into stack, 59 * and then push the address of TCC into stack. 60 * In cases where bpf2bpf and tailcall are used in combination, 61 * the value in REG_TCC may be a count or an address, 62 * these two cases need to be judged and handled separately. 63 */ 64 emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 65 *store_offset -= sizeof(long); 66 67 emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); 68 69 /* 70 * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, 71 * push tcc into stack 72 */ 73 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 74 75 /* Push the address of TCC into the REG_TCC */ 76 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 77 78 emit_uncond_jmp(ctx, 2); 79 80 /* 81 * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, 82 * push tcc_ptr into stack 83 */ 84 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 85 } else { 86 *store_offset -= sizeof(long); 87 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 88 } 89 90 /* Push tcc_ptr into stack */ 91 *store_offset -= sizeof(long); 92 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 93 } 94 95 /* 96 * eBPF prog stack layout: 97 * 98 * high 99 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP 100 * | $ra | 101 * +-------------------------+ 102 * | $fp | 103 * +-------------------------+ 104 * | $s0 | 105 * +-------------------------+ 106 * | $s1 | 107 * +-------------------------+ 108 * | $s2 | 109 * +-------------------------+ 110 * | $s3 | 111 * +-------------------------+ 112 * | $s4 | 113 * +-------------------------+ 114 * | $s5 | 115 * +-------------------------+ 116 * | tcc | 117 * +-------------------------+ 118 * | tcc_ptr | 119 * +-------------------------+ <--BPF_REG_FP 120 * | prog->aux->stack_depth | 121 * | (optional) | 122 * current $sp -------------> +-------------------------+ 123 * low 124 */ 125 static void build_prologue(struct jit_ctx *ctx) 126 { 127 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 128 const struct bpf_prog *prog = ctx->prog; 129 const bool is_main_prog = !bpf_is_subprog(prog); 130 131 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 132 133 /* To store ra, fp, s0, s1, s2, s3, s4, s5 */ 134 stack_adjust += sizeof(long) * 8; 135 136 /* To store tcc and tcc_ptr */ 137 stack_adjust += sizeof(long) * 2; 138 139 stack_adjust = round_up(stack_adjust, 16); 140 stack_adjust += bpf_stack_adjust; 141 142 move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); 143 /* Reserve space for the move_imm + jirl instruction */ 144 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 145 emit_insn(ctx, nop); 146 147 /* 148 * First instruction initializes the tail call count (TCC) 149 * register to zero. On tail call we skip this instruction, 150 * and the TCC is passed in REG_TCC from the caller. 151 */ 152 if (is_main_prog) 153 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); 154 155 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); 156 157 store_offset = stack_adjust - sizeof(long); 158 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); 159 160 store_offset -= sizeof(long); 161 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); 162 163 store_offset -= sizeof(long); 164 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); 165 166 store_offset -= sizeof(long); 167 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); 168 169 store_offset -= sizeof(long); 170 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); 171 172 store_offset -= sizeof(long); 173 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); 174 175 store_offset -= sizeof(long); 176 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); 177 178 store_offset -= sizeof(long); 179 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 180 181 prepare_bpf_tail_call_cnt(ctx, &store_offset); 182 183 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); 184 185 if (bpf_stack_adjust) 186 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 187 188 ctx->stack_size = stack_adjust; 189 } 190 191 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) 192 { 193 int stack_adjust = ctx->stack_size; 194 int load_offset; 195 196 load_offset = stack_adjust - sizeof(long); 197 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); 198 199 load_offset -= sizeof(long); 200 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); 201 202 load_offset -= sizeof(long); 203 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); 204 205 load_offset -= sizeof(long); 206 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); 207 208 load_offset -= sizeof(long); 209 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); 210 211 load_offset -= sizeof(long); 212 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); 213 214 load_offset -= sizeof(long); 215 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); 216 217 load_offset -= sizeof(long); 218 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 219 220 /* 221 * When push into the stack, follow the order of tcc then tcc_ptr. 222 * When pop from the stack, first pop tcc_ptr then followed by tcc. 223 */ 224 load_offset -= 2 * sizeof(long); 225 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 226 227 load_offset += sizeof(long); 228 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 229 230 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); 231 232 if (!is_tail_call) { 233 /* Set return value */ 234 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); 235 /* Return to the caller */ 236 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 237 } else { 238 /* 239 * Call the next bpf prog and skip the first instruction 240 * of TCC initialization. 241 */ 242 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7); 243 } 244 } 245 246 static void build_epilogue(struct jit_ctx *ctx) 247 { 248 __build_epilogue(ctx, false); 249 } 250 251 bool bpf_jit_supports_kfunc_call(void) 252 { 253 return true; 254 } 255 256 bool bpf_jit_supports_far_kfunc_call(void) 257 { 258 return true; 259 } 260 261 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) 262 { 263 int off, tc_ninsn = 0; 264 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 265 u8 a1 = LOONGARCH_GPR_A1; 266 u8 a2 = LOONGARCH_GPR_A2; 267 u8 t1 = LOONGARCH_GPR_T1; 268 u8 t2 = LOONGARCH_GPR_T2; 269 u8 t3 = LOONGARCH_GPR_T3; 270 const int idx0 = ctx->idx; 271 272 #define cur_offset (ctx->idx - idx0) 273 #define jmp_offset (tc_ninsn - (cur_offset)) 274 275 /* 276 * a0: &ctx 277 * a1: &array 278 * a2: index 279 * 280 * if (index >= array->map.max_entries) 281 * goto out; 282 */ 283 tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; 284 emit_zext_32(ctx, a2, true); 285 286 off = offsetof(struct bpf_array, map.max_entries); 287 emit_insn(ctx, ldwu, t1, a1, off); 288 /* bgeu $a2, $t1, jmp_offset */ 289 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) 290 goto toofar; 291 292 /* 293 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) 294 * goto out; 295 */ 296 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 297 emit_insn(ctx, ldd, t3, REG_TCC, 0); 298 emit_insn(ctx, addid, t3, t3, 1); 299 emit_insn(ctx, std, t3, REG_TCC, 0); 300 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 301 if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0) 302 goto toofar; 303 304 /* 305 * prog = array->ptrs[index]; 306 * if (!prog) 307 * goto out; 308 */ 309 emit_insn(ctx, alsld, t2, a2, a1, 2); 310 off = offsetof(struct bpf_array, ptrs); 311 emit_insn(ctx, ldd, t2, t2, off); 312 /* beq $t2, $zero, jmp_offset */ 313 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 314 goto toofar; 315 316 /* goto *(prog->bpf_func + 4); */ 317 off = offsetof(struct bpf_prog, bpf_func); 318 emit_insn(ctx, ldd, t3, t2, off); 319 __build_epilogue(ctx, true); 320 321 return 0; 322 323 toofar: 324 pr_info_once("tail_call: jump too far\n"); 325 return -1; 326 #undef cur_offset 327 #undef jmp_offset 328 } 329 330 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) 331 { 332 const u8 t1 = LOONGARCH_GPR_T1; 333 const u8 t2 = LOONGARCH_GPR_T2; 334 const u8 t3 = LOONGARCH_GPR_T3; 335 const u8 r0 = regmap[BPF_REG_0]; 336 const u8 src = regmap[insn->src_reg]; 337 const u8 dst = regmap[insn->dst_reg]; 338 const s16 off = insn->off; 339 const s32 imm = insn->imm; 340 const bool isdw = BPF_SIZE(insn->code) == BPF_DW; 341 342 move_imm(ctx, t1, off, false); 343 emit_insn(ctx, addd, t1, dst, t1); 344 move_reg(ctx, t3, src); 345 346 switch (imm) { 347 /* lock *(size *)(dst + off) <op>= src */ 348 case BPF_ADD: 349 if (isdw) 350 emit_insn(ctx, amaddd, t2, t1, src); 351 else 352 emit_insn(ctx, amaddw, t2, t1, src); 353 break; 354 case BPF_AND: 355 if (isdw) 356 emit_insn(ctx, amandd, t2, t1, src); 357 else 358 emit_insn(ctx, amandw, t2, t1, src); 359 break; 360 case BPF_OR: 361 if (isdw) 362 emit_insn(ctx, amord, t2, t1, src); 363 else 364 emit_insn(ctx, amorw, t2, t1, src); 365 break; 366 case BPF_XOR: 367 if (isdw) 368 emit_insn(ctx, amxord, t2, t1, src); 369 else 370 emit_insn(ctx, amxorw, t2, t1, src); 371 break; 372 /* src = atomic_fetch_<op>(dst + off, src) */ 373 case BPF_ADD | BPF_FETCH: 374 if (isdw) { 375 emit_insn(ctx, amaddd, src, t1, t3); 376 } else { 377 emit_insn(ctx, amaddw, src, t1, t3); 378 emit_zext_32(ctx, src, true); 379 } 380 break; 381 case BPF_AND | BPF_FETCH: 382 if (isdw) { 383 emit_insn(ctx, amandd, src, t1, t3); 384 } else { 385 emit_insn(ctx, amandw, src, t1, t3); 386 emit_zext_32(ctx, src, true); 387 } 388 break; 389 case BPF_OR | BPF_FETCH: 390 if (isdw) { 391 emit_insn(ctx, amord, src, t1, t3); 392 } else { 393 emit_insn(ctx, amorw, src, t1, t3); 394 emit_zext_32(ctx, src, true); 395 } 396 break; 397 case BPF_XOR | BPF_FETCH: 398 if (isdw) { 399 emit_insn(ctx, amxord, src, t1, t3); 400 } else { 401 emit_insn(ctx, amxorw, src, t1, t3); 402 emit_zext_32(ctx, src, true); 403 } 404 break; 405 /* src = atomic_xchg(dst + off, src); */ 406 case BPF_XCHG: 407 if (isdw) { 408 emit_insn(ctx, amswapd, src, t1, t3); 409 } else { 410 emit_insn(ctx, amswapw, src, t1, t3); 411 emit_zext_32(ctx, src, true); 412 } 413 break; 414 /* r0 = atomic_cmpxchg(dst + off, r0, src); */ 415 case BPF_CMPXCHG: 416 move_reg(ctx, t2, r0); 417 if (isdw) { 418 emit_insn(ctx, lld, r0, t1, 0); 419 emit_insn(ctx, bne, t2, r0, 4); 420 move_reg(ctx, t3, src); 421 emit_insn(ctx, scd, t3, t1, 0); 422 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); 423 } else { 424 emit_insn(ctx, llw, r0, t1, 0); 425 emit_zext_32(ctx, t2, true); 426 emit_zext_32(ctx, r0, true); 427 emit_insn(ctx, bne, t2, r0, 4); 428 move_reg(ctx, t3, src); 429 emit_insn(ctx, scw, t3, t1, 0); 430 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); 431 emit_zext_32(ctx, r0, true); 432 } 433 break; 434 } 435 } 436 437 static bool is_signed_bpf_cond(u8 cond) 438 { 439 return cond == BPF_JSGT || cond == BPF_JSLT || 440 cond == BPF_JSGE || cond == BPF_JSLE; 441 } 442 443 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 444 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 445 446 bool ex_handler_bpf(const struct exception_table_entry *ex, 447 struct pt_regs *regs) 448 { 449 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 450 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 451 452 regs->regs[dst_reg] = 0; 453 regs->csr_era = (unsigned long)&ex->fixup - offset; 454 455 return true; 456 } 457 458 /* For accesses to BTF pointers, add an entry to the exception table */ 459 static int add_exception_handler(const struct bpf_insn *insn, 460 struct jit_ctx *ctx, 461 int dst_reg) 462 { 463 unsigned long pc; 464 off_t offset; 465 struct exception_table_entry *ex; 466 467 if (!ctx->image || !ctx->prog->aux->extable) 468 return 0; 469 470 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 471 BPF_MODE(insn->code) != BPF_PROBE_MEMSX) 472 return 0; 473 474 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) 475 return -EINVAL; 476 477 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 478 pc = (unsigned long)&ctx->image[ctx->idx - 1]; 479 480 offset = pc - (long)&ex->insn; 481 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 482 return -ERANGE; 483 484 ex->insn = offset; 485 486 /* 487 * Since the extable follows the program, the fixup offset is always 488 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 489 * to keep things simple, and put the destination register in the upper 490 * bits. We don't need to worry about buildtime or runtime sort 491 * modifying the upper bits because the table is already sorted, and 492 * isn't part of the main exception table. 493 */ 494 offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 495 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 496 return -ERANGE; 497 498 ex->type = EX_TYPE_BPF; 499 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 500 501 ctx->num_exentries++; 502 503 return 0; 504 } 505 506 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) 507 { 508 u8 tm = -1; 509 u64 func_addr; 510 bool func_addr_fixed, sign_extend; 511 int i = insn - ctx->prog->insnsi; 512 int ret, jmp_offset, tcc_ptr_off; 513 const u8 code = insn->code; 514 const u8 cond = BPF_OP(code); 515 const u8 t1 = LOONGARCH_GPR_T1; 516 const u8 t2 = LOONGARCH_GPR_T2; 517 const u8 src = regmap[insn->src_reg]; 518 const u8 dst = regmap[insn->dst_reg]; 519 const s16 off = insn->off; 520 const s32 imm = insn->imm; 521 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; 522 523 switch (code) { 524 /* dst = src */ 525 case BPF_ALU | BPF_MOV | BPF_X: 526 case BPF_ALU64 | BPF_MOV | BPF_X: 527 switch (off) { 528 case 0: 529 move_reg(ctx, dst, src); 530 emit_zext_32(ctx, dst, is32); 531 break; 532 case 8: 533 emit_insn(ctx, extwb, dst, src); 534 emit_zext_32(ctx, dst, is32); 535 break; 536 case 16: 537 emit_insn(ctx, extwh, dst, src); 538 emit_zext_32(ctx, dst, is32); 539 break; 540 case 32: 541 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); 542 break; 543 } 544 break; 545 546 /* dst = imm */ 547 case BPF_ALU | BPF_MOV | BPF_K: 548 case BPF_ALU64 | BPF_MOV | BPF_K: 549 move_imm(ctx, dst, imm, is32); 550 break; 551 552 /* dst = dst + src */ 553 case BPF_ALU | BPF_ADD | BPF_X: 554 case BPF_ALU64 | BPF_ADD | BPF_X: 555 emit_insn(ctx, addd, dst, dst, src); 556 emit_zext_32(ctx, dst, is32); 557 break; 558 559 /* dst = dst + imm */ 560 case BPF_ALU | BPF_ADD | BPF_K: 561 case BPF_ALU64 | BPF_ADD | BPF_K: 562 if (is_signed_imm12(imm)) { 563 emit_insn(ctx, addid, dst, dst, imm); 564 } else { 565 move_imm(ctx, t1, imm, is32); 566 emit_insn(ctx, addd, dst, dst, t1); 567 } 568 emit_zext_32(ctx, dst, is32); 569 break; 570 571 /* dst = dst - src */ 572 case BPF_ALU | BPF_SUB | BPF_X: 573 case BPF_ALU64 | BPF_SUB | BPF_X: 574 emit_insn(ctx, subd, dst, dst, src); 575 emit_zext_32(ctx, dst, is32); 576 break; 577 578 /* dst = dst - imm */ 579 case BPF_ALU | BPF_SUB | BPF_K: 580 case BPF_ALU64 | BPF_SUB | BPF_K: 581 if (is_signed_imm12(-imm)) { 582 emit_insn(ctx, addid, dst, dst, -imm); 583 } else { 584 move_imm(ctx, t1, imm, is32); 585 emit_insn(ctx, subd, dst, dst, t1); 586 } 587 emit_zext_32(ctx, dst, is32); 588 break; 589 590 /* dst = dst * src */ 591 case BPF_ALU | BPF_MUL | BPF_X: 592 case BPF_ALU64 | BPF_MUL | BPF_X: 593 emit_insn(ctx, muld, dst, dst, src); 594 emit_zext_32(ctx, dst, is32); 595 break; 596 597 /* dst = dst * imm */ 598 case BPF_ALU | BPF_MUL | BPF_K: 599 case BPF_ALU64 | BPF_MUL | BPF_K: 600 move_imm(ctx, t1, imm, is32); 601 emit_insn(ctx, muld, dst, dst, t1); 602 emit_zext_32(ctx, dst, is32); 603 break; 604 605 /* dst = dst / src */ 606 case BPF_ALU | BPF_DIV | BPF_X: 607 case BPF_ALU64 | BPF_DIV | BPF_X: 608 if (!off) { 609 emit_zext_32(ctx, dst, is32); 610 move_reg(ctx, t1, src); 611 emit_zext_32(ctx, t1, is32); 612 emit_insn(ctx, divdu, dst, dst, t1); 613 emit_zext_32(ctx, dst, is32); 614 } else { 615 emit_sext_32(ctx, dst, is32); 616 move_reg(ctx, t1, src); 617 emit_sext_32(ctx, t1, is32); 618 emit_insn(ctx, divd, dst, dst, t1); 619 emit_sext_32(ctx, dst, is32); 620 } 621 break; 622 623 /* dst = dst / imm */ 624 case BPF_ALU | BPF_DIV | BPF_K: 625 case BPF_ALU64 | BPF_DIV | BPF_K: 626 if (!off) { 627 move_imm(ctx, t1, imm, is32); 628 emit_zext_32(ctx, dst, is32); 629 emit_insn(ctx, divdu, dst, dst, t1); 630 emit_zext_32(ctx, dst, is32); 631 } else { 632 move_imm(ctx, t1, imm, false); 633 emit_sext_32(ctx, t1, is32); 634 emit_sext_32(ctx, dst, is32); 635 emit_insn(ctx, divd, dst, dst, t1); 636 emit_sext_32(ctx, dst, is32); 637 } 638 break; 639 640 /* dst = dst % src */ 641 case BPF_ALU | BPF_MOD | BPF_X: 642 case BPF_ALU64 | BPF_MOD | BPF_X: 643 if (!off) { 644 emit_zext_32(ctx, dst, is32); 645 move_reg(ctx, t1, src); 646 emit_zext_32(ctx, t1, is32); 647 emit_insn(ctx, moddu, dst, dst, t1); 648 emit_zext_32(ctx, dst, is32); 649 } else { 650 emit_sext_32(ctx, dst, is32); 651 move_reg(ctx, t1, src); 652 emit_sext_32(ctx, t1, is32); 653 emit_insn(ctx, modd, dst, dst, t1); 654 emit_sext_32(ctx, dst, is32); 655 } 656 break; 657 658 /* dst = dst % imm */ 659 case BPF_ALU | BPF_MOD | BPF_K: 660 case BPF_ALU64 | BPF_MOD | BPF_K: 661 if (!off) { 662 move_imm(ctx, t1, imm, is32); 663 emit_zext_32(ctx, dst, is32); 664 emit_insn(ctx, moddu, dst, dst, t1); 665 emit_zext_32(ctx, dst, is32); 666 } else { 667 move_imm(ctx, t1, imm, false); 668 emit_sext_32(ctx, t1, is32); 669 emit_sext_32(ctx, dst, is32); 670 emit_insn(ctx, modd, dst, dst, t1); 671 emit_sext_32(ctx, dst, is32); 672 } 673 break; 674 675 /* dst = -dst */ 676 case BPF_ALU | BPF_NEG: 677 case BPF_ALU64 | BPF_NEG: 678 move_imm(ctx, t1, imm, is32); 679 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); 680 emit_zext_32(ctx, dst, is32); 681 break; 682 683 /* dst = dst & src */ 684 case BPF_ALU | BPF_AND | BPF_X: 685 case BPF_ALU64 | BPF_AND | BPF_X: 686 emit_insn(ctx, and, dst, dst, src); 687 emit_zext_32(ctx, dst, is32); 688 break; 689 690 /* dst = dst & imm */ 691 case BPF_ALU | BPF_AND | BPF_K: 692 case BPF_ALU64 | BPF_AND | BPF_K: 693 if (is_unsigned_imm12(imm)) { 694 emit_insn(ctx, andi, dst, dst, imm); 695 } else { 696 move_imm(ctx, t1, imm, is32); 697 emit_insn(ctx, and, dst, dst, t1); 698 } 699 emit_zext_32(ctx, dst, is32); 700 break; 701 702 /* dst = dst | src */ 703 case BPF_ALU | BPF_OR | BPF_X: 704 case BPF_ALU64 | BPF_OR | BPF_X: 705 emit_insn(ctx, or, dst, dst, src); 706 emit_zext_32(ctx, dst, is32); 707 break; 708 709 /* dst = dst | imm */ 710 case BPF_ALU | BPF_OR | BPF_K: 711 case BPF_ALU64 | BPF_OR | BPF_K: 712 if (is_unsigned_imm12(imm)) { 713 emit_insn(ctx, ori, dst, dst, imm); 714 } else { 715 move_imm(ctx, t1, imm, is32); 716 emit_insn(ctx, or, dst, dst, t1); 717 } 718 emit_zext_32(ctx, dst, is32); 719 break; 720 721 /* dst = dst ^ src */ 722 case BPF_ALU | BPF_XOR | BPF_X: 723 case BPF_ALU64 | BPF_XOR | BPF_X: 724 emit_insn(ctx, xor, dst, dst, src); 725 emit_zext_32(ctx, dst, is32); 726 break; 727 728 /* dst = dst ^ imm */ 729 case BPF_ALU | BPF_XOR | BPF_K: 730 case BPF_ALU64 | BPF_XOR | BPF_K: 731 if (is_unsigned_imm12(imm)) { 732 emit_insn(ctx, xori, dst, dst, imm); 733 } else { 734 move_imm(ctx, t1, imm, is32); 735 emit_insn(ctx, xor, dst, dst, t1); 736 } 737 emit_zext_32(ctx, dst, is32); 738 break; 739 740 /* dst = dst << src (logical) */ 741 case BPF_ALU | BPF_LSH | BPF_X: 742 emit_insn(ctx, sllw, dst, dst, src); 743 emit_zext_32(ctx, dst, is32); 744 break; 745 746 case BPF_ALU64 | BPF_LSH | BPF_X: 747 emit_insn(ctx, slld, dst, dst, src); 748 break; 749 750 /* dst = dst << imm (logical) */ 751 case BPF_ALU | BPF_LSH | BPF_K: 752 emit_insn(ctx, slliw, dst, dst, imm); 753 emit_zext_32(ctx, dst, is32); 754 break; 755 756 case BPF_ALU64 | BPF_LSH | BPF_K: 757 emit_insn(ctx, sllid, dst, dst, imm); 758 break; 759 760 /* dst = dst >> src (logical) */ 761 case BPF_ALU | BPF_RSH | BPF_X: 762 emit_insn(ctx, srlw, dst, dst, src); 763 emit_zext_32(ctx, dst, is32); 764 break; 765 766 case BPF_ALU64 | BPF_RSH | BPF_X: 767 emit_insn(ctx, srld, dst, dst, src); 768 break; 769 770 /* dst = dst >> imm (logical) */ 771 case BPF_ALU | BPF_RSH | BPF_K: 772 emit_insn(ctx, srliw, dst, dst, imm); 773 emit_zext_32(ctx, dst, is32); 774 break; 775 776 case BPF_ALU64 | BPF_RSH | BPF_K: 777 emit_insn(ctx, srlid, dst, dst, imm); 778 break; 779 780 /* dst = dst >> src (arithmetic) */ 781 case BPF_ALU | BPF_ARSH | BPF_X: 782 emit_insn(ctx, sraw, dst, dst, src); 783 emit_zext_32(ctx, dst, is32); 784 break; 785 786 case BPF_ALU64 | BPF_ARSH | BPF_X: 787 emit_insn(ctx, srad, dst, dst, src); 788 break; 789 790 /* dst = dst >> imm (arithmetic) */ 791 case BPF_ALU | BPF_ARSH | BPF_K: 792 emit_insn(ctx, sraiw, dst, dst, imm); 793 emit_zext_32(ctx, dst, is32); 794 break; 795 796 case BPF_ALU64 | BPF_ARSH | BPF_K: 797 emit_insn(ctx, sraid, dst, dst, imm); 798 break; 799 800 /* dst = BSWAP##imm(dst) */ 801 case BPF_ALU | BPF_END | BPF_FROM_LE: 802 switch (imm) { 803 case 16: 804 /* zero-extend 16 bits into 64 bits */ 805 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 806 break; 807 case 32: 808 /* zero-extend 32 bits into 64 bits */ 809 emit_zext_32(ctx, dst, is32); 810 break; 811 case 64: 812 /* do nothing */ 813 break; 814 } 815 break; 816 817 case BPF_ALU | BPF_END | BPF_FROM_BE: 818 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 819 switch (imm) { 820 case 16: 821 emit_insn(ctx, revb2h, dst, dst); 822 /* zero-extend 16 bits into 64 bits */ 823 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 824 break; 825 case 32: 826 emit_insn(ctx, revb2w, dst, dst); 827 /* clear the upper 32 bits */ 828 emit_zext_32(ctx, dst, true); 829 break; 830 case 64: 831 emit_insn(ctx, revbd, dst, dst); 832 break; 833 } 834 break; 835 836 /* PC += off if dst cond src */ 837 case BPF_JMP | BPF_JEQ | BPF_X: 838 case BPF_JMP | BPF_JNE | BPF_X: 839 case BPF_JMP | BPF_JGT | BPF_X: 840 case BPF_JMP | BPF_JGE | BPF_X: 841 case BPF_JMP | BPF_JLT | BPF_X: 842 case BPF_JMP | BPF_JLE | BPF_X: 843 case BPF_JMP | BPF_JSGT | BPF_X: 844 case BPF_JMP | BPF_JSGE | BPF_X: 845 case BPF_JMP | BPF_JSLT | BPF_X: 846 case BPF_JMP | BPF_JSLE | BPF_X: 847 case BPF_JMP32 | BPF_JEQ | BPF_X: 848 case BPF_JMP32 | BPF_JNE | BPF_X: 849 case BPF_JMP32 | BPF_JGT | BPF_X: 850 case BPF_JMP32 | BPF_JGE | BPF_X: 851 case BPF_JMP32 | BPF_JLT | BPF_X: 852 case BPF_JMP32 | BPF_JLE | BPF_X: 853 case BPF_JMP32 | BPF_JSGT | BPF_X: 854 case BPF_JMP32 | BPF_JSGE | BPF_X: 855 case BPF_JMP32 | BPF_JSLT | BPF_X: 856 case BPF_JMP32 | BPF_JSLE | BPF_X: 857 jmp_offset = bpf2la_offset(i, off, ctx); 858 move_reg(ctx, t1, dst); 859 move_reg(ctx, t2, src); 860 if (is_signed_bpf_cond(BPF_OP(code))) { 861 emit_sext_32(ctx, t1, is32); 862 emit_sext_32(ctx, t2, is32); 863 } else { 864 emit_zext_32(ctx, t1, is32); 865 emit_zext_32(ctx, t2, is32); 866 } 867 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) 868 goto toofar; 869 break; 870 871 /* PC += off if dst cond imm */ 872 case BPF_JMP | BPF_JEQ | BPF_K: 873 case BPF_JMP | BPF_JNE | BPF_K: 874 case BPF_JMP | BPF_JGT | BPF_K: 875 case BPF_JMP | BPF_JGE | BPF_K: 876 case BPF_JMP | BPF_JLT | BPF_K: 877 case BPF_JMP | BPF_JLE | BPF_K: 878 case BPF_JMP | BPF_JSGT | BPF_K: 879 case BPF_JMP | BPF_JSGE | BPF_K: 880 case BPF_JMP | BPF_JSLT | BPF_K: 881 case BPF_JMP | BPF_JSLE | BPF_K: 882 case BPF_JMP32 | BPF_JEQ | BPF_K: 883 case BPF_JMP32 | BPF_JNE | BPF_K: 884 case BPF_JMP32 | BPF_JGT | BPF_K: 885 case BPF_JMP32 | BPF_JGE | BPF_K: 886 case BPF_JMP32 | BPF_JLT | BPF_K: 887 case BPF_JMP32 | BPF_JLE | BPF_K: 888 case BPF_JMP32 | BPF_JSGT | BPF_K: 889 case BPF_JMP32 | BPF_JSGE | BPF_K: 890 case BPF_JMP32 | BPF_JSLT | BPF_K: 891 case BPF_JMP32 | BPF_JSLE | BPF_K: 892 jmp_offset = bpf2la_offset(i, off, ctx); 893 if (imm) { 894 move_imm(ctx, t1, imm, false); 895 tm = t1; 896 } else { 897 /* If imm is 0, simply use zero register. */ 898 tm = LOONGARCH_GPR_ZERO; 899 } 900 move_reg(ctx, t2, dst); 901 if (is_signed_bpf_cond(BPF_OP(code))) { 902 emit_sext_32(ctx, tm, is32); 903 emit_sext_32(ctx, t2, is32); 904 } else { 905 emit_zext_32(ctx, tm, is32); 906 emit_zext_32(ctx, t2, is32); 907 } 908 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) 909 goto toofar; 910 break; 911 912 /* PC += off if dst & src */ 913 case BPF_JMP | BPF_JSET | BPF_X: 914 case BPF_JMP32 | BPF_JSET | BPF_X: 915 jmp_offset = bpf2la_offset(i, off, ctx); 916 emit_insn(ctx, and, t1, dst, src); 917 emit_zext_32(ctx, t1, is32); 918 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 919 goto toofar; 920 break; 921 922 /* PC += off if dst & imm */ 923 case BPF_JMP | BPF_JSET | BPF_K: 924 case BPF_JMP32 | BPF_JSET | BPF_K: 925 jmp_offset = bpf2la_offset(i, off, ctx); 926 move_imm(ctx, t1, imm, is32); 927 emit_insn(ctx, and, t1, dst, t1); 928 emit_zext_32(ctx, t1, is32); 929 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 930 goto toofar; 931 break; 932 933 /* PC += off */ 934 case BPF_JMP | BPF_JA: 935 case BPF_JMP32 | BPF_JA: 936 if (BPF_CLASS(code) == BPF_JMP) 937 jmp_offset = bpf2la_offset(i, off, ctx); 938 else 939 jmp_offset = bpf2la_offset(i, imm, ctx); 940 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 941 goto toofar; 942 break; 943 944 /* function call */ 945 case BPF_JMP | BPF_CALL: 946 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 947 &func_addr, &func_addr_fixed); 948 if (ret < 0) 949 return ret; 950 951 if (insn->src_reg == BPF_PSEUDO_CALL) { 952 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 953 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 954 } 955 956 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 957 const struct btf_func_model *m; 958 int i; 959 960 m = bpf_jit_find_kfunc_model(ctx->prog, insn); 961 if (!m) 962 return -EINVAL; 963 964 for (i = 0; i < m->nr_args; i++) { 965 u8 reg = regmap[BPF_REG_1 + i]; 966 bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; 967 968 emit_abi_ext(ctx, reg, m->arg_size[i], sign); 969 } 970 } 971 972 move_addr(ctx, t1, func_addr); 973 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); 974 975 if (insn->src_reg != BPF_PSEUDO_CALL) 976 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); 977 978 break; 979 980 /* tail call */ 981 case BPF_JMP | BPF_TAIL_CALL: 982 if (emit_bpf_tail_call(ctx, i) < 0) 983 return -EINVAL; 984 break; 985 986 /* function return */ 987 case BPF_JMP | BPF_EXIT: 988 if (i == ctx->prog->len - 1) 989 break; 990 991 jmp_offset = epilogue_offset(ctx); 992 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 993 goto toofar; 994 break; 995 996 /* dst = imm64 */ 997 case BPF_LD | BPF_IMM | BPF_DW: 998 { 999 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; 1000 1001 if (bpf_pseudo_func(insn)) 1002 move_addr(ctx, dst, imm64); 1003 else 1004 move_imm(ctx, dst, imm64, is32); 1005 return 1; 1006 } 1007 1008 /* dst = *(size *)(src + off) */ 1009 case BPF_LDX | BPF_MEM | BPF_B: 1010 case BPF_LDX | BPF_MEM | BPF_H: 1011 case BPF_LDX | BPF_MEM | BPF_W: 1012 case BPF_LDX | BPF_MEM | BPF_DW: 1013 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1014 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1015 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1016 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1017 /* dst_reg = (s64)*(signed size *)(src_reg + off) */ 1018 case BPF_LDX | BPF_MEMSX | BPF_B: 1019 case BPF_LDX | BPF_MEMSX | BPF_H: 1020 case BPF_LDX | BPF_MEMSX | BPF_W: 1021 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1022 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1023 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1024 sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || 1025 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1026 switch (BPF_SIZE(code)) { 1027 case BPF_B: 1028 if (is_signed_imm12(off)) { 1029 if (sign_extend) 1030 emit_insn(ctx, ldb, dst, src, off); 1031 else 1032 emit_insn(ctx, ldbu, dst, src, off); 1033 } else { 1034 move_imm(ctx, t1, off, is32); 1035 if (sign_extend) 1036 emit_insn(ctx, ldxb, dst, src, t1); 1037 else 1038 emit_insn(ctx, ldxbu, dst, src, t1); 1039 } 1040 break; 1041 case BPF_H: 1042 if (is_signed_imm12(off)) { 1043 if (sign_extend) 1044 emit_insn(ctx, ldh, dst, src, off); 1045 else 1046 emit_insn(ctx, ldhu, dst, src, off); 1047 } else { 1048 move_imm(ctx, t1, off, is32); 1049 if (sign_extend) 1050 emit_insn(ctx, ldxh, dst, src, t1); 1051 else 1052 emit_insn(ctx, ldxhu, dst, src, t1); 1053 } 1054 break; 1055 case BPF_W: 1056 if (is_signed_imm12(off)) { 1057 if (sign_extend) 1058 emit_insn(ctx, ldw, dst, src, off); 1059 else 1060 emit_insn(ctx, ldwu, dst, src, off); 1061 } else { 1062 move_imm(ctx, t1, off, is32); 1063 if (sign_extend) 1064 emit_insn(ctx, ldxw, dst, src, t1); 1065 else 1066 emit_insn(ctx, ldxwu, dst, src, t1); 1067 } 1068 break; 1069 case BPF_DW: 1070 move_imm(ctx, t1, off, is32); 1071 emit_insn(ctx, ldxd, dst, src, t1); 1072 break; 1073 } 1074 1075 ret = add_exception_handler(insn, ctx, dst); 1076 if (ret) 1077 return ret; 1078 break; 1079 1080 /* *(size *)(dst + off) = imm */ 1081 case BPF_ST | BPF_MEM | BPF_B: 1082 case BPF_ST | BPF_MEM | BPF_H: 1083 case BPF_ST | BPF_MEM | BPF_W: 1084 case BPF_ST | BPF_MEM | BPF_DW: 1085 switch (BPF_SIZE(code)) { 1086 case BPF_B: 1087 move_imm(ctx, t1, imm, is32); 1088 if (is_signed_imm12(off)) { 1089 emit_insn(ctx, stb, t1, dst, off); 1090 } else { 1091 move_imm(ctx, t2, off, is32); 1092 emit_insn(ctx, stxb, t1, dst, t2); 1093 } 1094 break; 1095 case BPF_H: 1096 move_imm(ctx, t1, imm, is32); 1097 if (is_signed_imm12(off)) { 1098 emit_insn(ctx, sth, t1, dst, off); 1099 } else { 1100 move_imm(ctx, t2, off, is32); 1101 emit_insn(ctx, stxh, t1, dst, t2); 1102 } 1103 break; 1104 case BPF_W: 1105 move_imm(ctx, t1, imm, is32); 1106 if (is_signed_imm12(off)) { 1107 emit_insn(ctx, stw, t1, dst, off); 1108 } else if (is_signed_imm14(off)) { 1109 emit_insn(ctx, stptrw, t1, dst, off); 1110 } else { 1111 move_imm(ctx, t2, off, is32); 1112 emit_insn(ctx, stxw, t1, dst, t2); 1113 } 1114 break; 1115 case BPF_DW: 1116 move_imm(ctx, t1, imm, is32); 1117 if (is_signed_imm12(off)) { 1118 emit_insn(ctx, std, t1, dst, off); 1119 } else if (is_signed_imm14(off)) { 1120 emit_insn(ctx, stptrd, t1, dst, off); 1121 } else { 1122 move_imm(ctx, t2, off, is32); 1123 emit_insn(ctx, stxd, t1, dst, t2); 1124 } 1125 break; 1126 } 1127 break; 1128 1129 /* *(size *)(dst + off) = src */ 1130 case BPF_STX | BPF_MEM | BPF_B: 1131 case BPF_STX | BPF_MEM | BPF_H: 1132 case BPF_STX | BPF_MEM | BPF_W: 1133 case BPF_STX | BPF_MEM | BPF_DW: 1134 switch (BPF_SIZE(code)) { 1135 case BPF_B: 1136 if (is_signed_imm12(off)) { 1137 emit_insn(ctx, stb, src, dst, off); 1138 } else { 1139 move_imm(ctx, t1, off, is32); 1140 emit_insn(ctx, stxb, src, dst, t1); 1141 } 1142 break; 1143 case BPF_H: 1144 if (is_signed_imm12(off)) { 1145 emit_insn(ctx, sth, src, dst, off); 1146 } else { 1147 move_imm(ctx, t1, off, is32); 1148 emit_insn(ctx, stxh, src, dst, t1); 1149 } 1150 break; 1151 case BPF_W: 1152 if (is_signed_imm12(off)) { 1153 emit_insn(ctx, stw, src, dst, off); 1154 } else if (is_signed_imm14(off)) { 1155 emit_insn(ctx, stptrw, src, dst, off); 1156 } else { 1157 move_imm(ctx, t1, off, is32); 1158 emit_insn(ctx, stxw, src, dst, t1); 1159 } 1160 break; 1161 case BPF_DW: 1162 if (is_signed_imm12(off)) { 1163 emit_insn(ctx, std, src, dst, off); 1164 } else if (is_signed_imm14(off)) { 1165 emit_insn(ctx, stptrd, src, dst, off); 1166 } else { 1167 move_imm(ctx, t1, off, is32); 1168 emit_insn(ctx, stxd, src, dst, t1); 1169 } 1170 break; 1171 } 1172 break; 1173 1174 case BPF_STX | BPF_ATOMIC | BPF_W: 1175 case BPF_STX | BPF_ATOMIC | BPF_DW: 1176 emit_atomic(insn, ctx); 1177 break; 1178 1179 /* Speculation barrier */ 1180 case BPF_ST | BPF_NOSPEC: 1181 break; 1182 1183 default: 1184 pr_err("bpf_jit: unknown opcode %02x\n", code); 1185 return -EINVAL; 1186 } 1187 1188 return 0; 1189 1190 toofar: 1191 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); 1192 return -E2BIG; 1193 } 1194 1195 static int build_body(struct jit_ctx *ctx, bool extra_pass) 1196 { 1197 int i; 1198 const struct bpf_prog *prog = ctx->prog; 1199 1200 for (i = 0; i < prog->len; i++) { 1201 const struct bpf_insn *insn = &prog->insnsi[i]; 1202 int ret; 1203 1204 if (ctx->image == NULL) 1205 ctx->offset[i] = ctx->idx; 1206 1207 ret = build_insn(insn, ctx, extra_pass); 1208 if (ret > 0) { 1209 i++; 1210 if (ctx->image == NULL) 1211 ctx->offset[i] = ctx->idx; 1212 continue; 1213 } 1214 if (ret) 1215 return ret; 1216 } 1217 1218 if (ctx->image == NULL) 1219 ctx->offset[i] = ctx->idx; 1220 1221 return 0; 1222 } 1223 1224 /* Fill space with break instructions */ 1225 static void jit_fill_hole(void *area, unsigned int size) 1226 { 1227 u32 *ptr; 1228 1229 /* We are guaranteed to have aligned memory */ 1230 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1231 *ptr++ = INSN_BREAK; 1232 } 1233 1234 static int validate_code(struct jit_ctx *ctx) 1235 { 1236 int i; 1237 union loongarch_instruction insn; 1238 1239 for (i = 0; i < ctx->idx; i++) { 1240 insn = ctx->image[i]; 1241 /* Check INSN_BREAK */ 1242 if (insn.word == INSN_BREAK) 1243 return -1; 1244 } 1245 1246 return 0; 1247 } 1248 1249 static int validate_ctx(struct jit_ctx *ctx) 1250 { 1251 if (validate_code(ctx)) 1252 return -1; 1253 1254 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) 1255 return -1; 1256 1257 return 0; 1258 } 1259 1260 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) 1261 { 1262 if (!target) { 1263 pr_err("bpf_jit: jump target address is error\n"); 1264 return -EFAULT; 1265 } 1266 1267 move_imm(ctx, LOONGARCH_GPR_T1, target, false); 1268 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); 1269 1270 return 0; 1271 } 1272 1273 static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 1274 { 1275 int i; 1276 struct jit_ctx ctx; 1277 1278 ctx.idx = 0; 1279 ctx.image = (union loongarch_instruction *)insns; 1280 1281 if (!target) { 1282 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1283 emit_insn((&ctx), nop); 1284 return 0; 1285 } 1286 1287 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target); 1288 } 1289 1290 static int emit_call(struct jit_ctx *ctx, u64 addr) 1291 { 1292 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); 1293 } 1294 1295 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 1296 { 1297 int ret; 1298 1299 mutex_lock(&text_mutex); 1300 ret = larch_insn_text_copy(dst, src, len); 1301 mutex_unlock(&text_mutex); 1302 1303 return ret ? ERR_PTR(-EINVAL) : dst; 1304 } 1305 1306 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1307 enum bpf_text_poke_type new_t, void *old_addr, 1308 void *new_addr) 1309 { 1310 int ret; 1311 bool is_call; 1312 unsigned long size = 0; 1313 unsigned long offset = 0; 1314 void *image = NULL; 1315 char namebuf[KSYM_NAME_LEN]; 1316 u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1317 u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1318 1319 /* Only poking bpf text is supported. Since kernel function entry 1320 * is set up by ftrace, we rely on ftrace to poke kernel functions. 1321 */ 1322 if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) 1323 return -ENOTSUPP; 1324 1325 image = ip - offset; 1326 1327 /* zero offset means we're poking bpf prog entry */ 1328 if (offset == 0) { 1329 /* skip to the nop instruction in bpf prog entry: 1330 * move t0, ra 1331 * nop 1332 */ 1333 ip = image + LOONGARCH_INSN_SIZE; 1334 } 1335 1336 is_call = old_t == BPF_MOD_CALL; 1337 ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); 1338 if (ret) 1339 return ret; 1340 1341 if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1342 return -EFAULT; 1343 1344 is_call = new_t == BPF_MOD_CALL; 1345 ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call); 1346 if (ret) 1347 return ret; 1348 1349 mutex_lock(&text_mutex); 1350 if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1351 ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); 1352 mutex_unlock(&text_mutex); 1353 1354 return ret; 1355 } 1356 1357 int bpf_arch_text_invalidate(void *dst, size_t len) 1358 { 1359 int i; 1360 int ret = 0; 1361 u32 *inst; 1362 1363 inst = kvmalloc(len, GFP_KERNEL); 1364 if (!inst) 1365 return -ENOMEM; 1366 1367 for (i = 0; i < (len / sizeof(u32)); i++) 1368 inst[i] = INSN_BREAK; 1369 1370 mutex_lock(&text_mutex); 1371 if (larch_insn_text_copy(dst, inst, len)) 1372 ret = -EINVAL; 1373 mutex_unlock(&text_mutex); 1374 1375 kvfree(inst); 1376 1377 return ret; 1378 } 1379 1380 static void store_args(struct jit_ctx *ctx, int nargs, int args_off) 1381 { 1382 int i; 1383 1384 for (i = 0; i < nargs; i++) { 1385 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1386 args_off -= 8; 1387 } 1388 } 1389 1390 static void restore_args(struct jit_ctx *ctx, int nargs, int args_off) 1391 { 1392 int i; 1393 1394 for (i = 0; i < nargs; i++) { 1395 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1396 args_off -= 8; 1397 } 1398 } 1399 1400 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, 1401 int args_off, int retval_off, int run_ctx_off, bool save_ret) 1402 { 1403 int ret; 1404 u32 *branch; 1405 struct bpf_prog *p = l->link.prog; 1406 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 1407 1408 if (l->cookie) { 1409 move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false); 1410 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1411 } else { 1412 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1413 } 1414 1415 /* arg1: prog */ 1416 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1417 /* arg2: &run_ctx */ 1418 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); 1419 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); 1420 if (ret) 1421 return ret; 1422 1423 /* store prog start time */ 1424 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); 1425 1426 /* 1427 * if (__bpf_prog_enter(prog) == 0) 1428 * goto skip_exec_of_prog; 1429 */ 1430 branch = (u32 *)ctx->image + ctx->idx; 1431 /* nop reserved for conditional jump */ 1432 emit_insn(ctx, nop); 1433 1434 /* arg1: &args_off */ 1435 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); 1436 if (!p->jited) 1437 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); 1438 ret = emit_call(ctx, (const u64)p->bpf_func); 1439 if (ret) 1440 return ret; 1441 1442 if (save_ret) { 1443 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1444 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1445 } 1446 1447 /* update branch with beqz */ 1448 if (ctx->image) { 1449 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; 1450 *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset); 1451 } 1452 1453 /* arg1: prog */ 1454 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1455 /* arg2: prog start time */ 1456 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); 1457 /* arg3: &run_ctx */ 1458 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); 1459 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); 1460 1461 return ret; 1462 } 1463 1464 static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, 1465 int args_off, int retval_off, int run_ctx_off, u32 **branches) 1466 { 1467 int i; 1468 1469 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 1470 for (i = 0; i < tl->nr_links; i++) { 1471 invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true); 1472 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); 1473 branches[i] = (u32 *)ctx->image + ctx->idx; 1474 emit_insn(ctx, nop); 1475 } 1476 } 1477 1478 void *arch_alloc_bpf_trampoline(unsigned int size) 1479 { 1480 return bpf_prog_pack_alloc(size, jit_fill_hole); 1481 } 1482 1483 void arch_free_bpf_trampoline(void *image, unsigned int size) 1484 { 1485 bpf_prog_pack_free(image, size); 1486 } 1487 1488 /* 1489 * Sign-extend the register if necessary 1490 */ 1491 static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign) 1492 { 1493 /* ABI requires unsigned char/short to be zero-extended */ 1494 if (!sign && (size == 1 || size == 2)) { 1495 if (rd != rj) 1496 move_reg(ctx, rd, rj); 1497 return; 1498 } 1499 1500 switch (size) { 1501 case 1: 1502 emit_insn(ctx, extwb, rd, rj); 1503 break; 1504 case 2: 1505 emit_insn(ctx, extwh, rd, rj); 1506 break; 1507 case 4: 1508 emit_insn(ctx, addiw, rd, rj, 0); 1509 break; 1510 case 8: 1511 if (rd != rj) 1512 move_reg(ctx, rd, rj); 1513 break; 1514 default: 1515 pr_warn("bpf_jit: invalid size %d for sign_extend\n", size); 1516 } 1517 } 1518 1519 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 1520 const struct btf_func_model *m, struct bpf_tramp_links *tlinks, 1521 void *func_addr, u32 flags) 1522 { 1523 int i, ret, save_ret; 1524 int stack_size, nargs; 1525 int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; 1526 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 1527 void *orig_call = func_addr; 1528 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 1529 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 1530 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 1531 u32 **branches = NULL; 1532 1533 /* 1534 * FP + 8 [ RA to parent func ] return address to parent 1535 * function 1536 * FP + 0 [ FP of parent func ] frame pointer of parent 1537 * function 1538 * FP - 8 [ T0 to traced func ] return address of traced 1539 * function 1540 * FP - 16 [ FP of traced func ] frame pointer of traced 1541 * function 1542 * 1543 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 1544 * BPF_TRAMP_F_RET_FENTRY_RET 1545 * [ argN ] 1546 * [ ... ] 1547 * FP - args_off [ arg1 ] 1548 * 1549 * FP - nargs_off [ regs count ] 1550 * 1551 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 1552 * 1553 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 1554 * 1555 * FP - sreg_off [ callee saved reg ] 1556 * 1557 * FP - tcc_ptr_off [ tail_call_cnt_ptr ] 1558 */ 1559 1560 if (m->nr_args > LOONGARCH_MAX_REG_ARGS) 1561 return -ENOTSUPP; 1562 1563 /* FIXME: No support of struct argument */ 1564 for (i = 0; i < m->nr_args; i++) { 1565 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 1566 return -ENOTSUPP; 1567 } 1568 1569 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 1570 return -ENOTSUPP; 1571 1572 /* Room of trampoline frame to store return address and frame pointer */ 1573 stack_size = 16; 1574 1575 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 1576 if (save_ret) 1577 stack_size += 16; /* Save BPF R0 and A0 */ 1578 1579 retval_off = stack_size; 1580 1581 /* Room of trampoline frame to store args */ 1582 nargs = m->nr_args; 1583 stack_size += nargs * 8; 1584 args_off = stack_size; 1585 1586 /* Room of trampoline frame to store args number */ 1587 stack_size += 8; 1588 nargs_off = stack_size; 1589 1590 /* Room of trampoline frame to store ip address */ 1591 if (flags & BPF_TRAMP_F_IP_ARG) { 1592 stack_size += 8; 1593 ip_off = stack_size; 1594 } 1595 1596 /* Room of trampoline frame to store struct bpf_tramp_run_ctx */ 1597 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 1598 run_ctx_off = stack_size; 1599 1600 stack_size += 8; 1601 sreg_off = stack_size; 1602 1603 /* Room of trampoline frame to store tail_call_cnt_ptr */ 1604 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { 1605 stack_size += 8; 1606 tcc_ptr_off = stack_size; 1607 } 1608 1609 stack_size = round_up(stack_size, 16); 1610 1611 if (is_struct_ops) { 1612 /* 1613 * For the trampoline called directly, just handle 1614 * the frame of trampoline. 1615 */ 1616 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1617 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1618 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1619 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1620 } else { 1621 /* 1622 * For the trampoline called from function entry, 1623 * the frame of traced function and the frame of 1624 * trampoline need to be considered. 1625 */ 1626 /* RA and FP for parent function */ 1627 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); 1628 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1629 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1630 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); 1631 1632 /* RA and FP for traced function */ 1633 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1634 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1635 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1636 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1637 } 1638 1639 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1640 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1641 1642 /* callee saved register S1 to pass start time */ 1643 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1644 1645 /* store ip address of the traced function */ 1646 if (flags & BPF_TRAMP_F_IP_ARG) { 1647 move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false); 1648 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off); 1649 } 1650 1651 /* store nargs number */ 1652 move_imm(ctx, LOONGARCH_GPR_T1, nargs, false); 1653 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off); 1654 1655 store_args(ctx, nargs, args_off); 1656 1657 /* To traced function */ 1658 /* Ftrace jump skips 2 NOP instructions */ 1659 if (is_kernel_text((unsigned long)orig_call) || 1660 is_module_text_address((unsigned long)orig_call)) 1661 orig_call += LOONGARCH_FENTRY_NBYTES; 1662 /* Direct jump skips 5 NOP instructions */ 1663 else if (is_bpf_text_address((unsigned long)orig_call)) 1664 orig_call += LOONGARCH_BPF_FENTRY_NBYTES; 1665 1666 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1667 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 1668 ret = emit_call(ctx, (const u64)__bpf_tramp_enter); 1669 if (ret) 1670 return ret; 1671 } 1672 1673 for (i = 0; i < fentry->nr_links; i++) { 1674 ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off, 1675 run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); 1676 if (ret) 1677 return ret; 1678 } 1679 if (fmod_ret->nr_links) { 1680 branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL); 1681 if (!branches) 1682 return -ENOMEM; 1683 1684 invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches); 1685 } 1686 1687 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1688 restore_args(ctx, m->nr_args, args_off); 1689 1690 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1691 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1692 1693 ret = emit_call(ctx, (const u64)orig_call); 1694 if (ret) 1695 goto out; 1696 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1697 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1698 im->ip_after_call = ctx->ro_image + ctx->idx; 1699 /* Reserve space for the move_imm + jirl instruction */ 1700 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1701 emit_insn(ctx, nop); 1702 } 1703 1704 for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) { 1705 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; 1706 *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset); 1707 } 1708 1709 for (i = 0; i < fexit->nr_links; i++) { 1710 ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false); 1711 if (ret) 1712 goto out; 1713 } 1714 1715 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1716 im->ip_epilogue = ctx->ro_image + ctx->idx; 1717 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 1718 ret = emit_call(ctx, (const u64)__bpf_tramp_exit); 1719 if (ret) 1720 goto out; 1721 } 1722 1723 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1724 restore_args(ctx, m->nr_args, args_off); 1725 1726 if (save_ret) { 1727 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1728 if (is_struct_ops) 1729 sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 1730 m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG); 1731 else 1732 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1733 } 1734 1735 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1736 1737 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1738 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1739 1740 if (is_struct_ops) { 1741 /* trampoline called directly */ 1742 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1743 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1744 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 1745 1746 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 1747 } else { 1748 /* trampoline called from function entry */ 1749 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1750 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1751 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 1752 1753 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1754 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1755 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); 1756 1757 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 1758 /* return to parent function */ 1759 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 1760 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); 1761 } else { 1762 /* return to traced function */ 1763 move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA); 1764 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 1765 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0); 1766 } 1767 } 1768 1769 ret = ctx->idx; 1770 out: 1771 kfree(branches); 1772 1773 return ret; 1774 } 1775 1776 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, 1777 void *ro_image_end, const struct btf_func_model *m, 1778 u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) 1779 { 1780 int ret, size; 1781 void *image, *tmp; 1782 struct jit_ctx ctx; 1783 1784 size = ro_image_end - ro_image; 1785 image = kvmalloc(size, GFP_KERNEL); 1786 if (!image) 1787 return -ENOMEM; 1788 1789 ctx.image = (union loongarch_instruction *)image; 1790 ctx.ro_image = (union loongarch_instruction *)ro_image; 1791 ctx.idx = 0; 1792 1793 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); 1794 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags); 1795 if (ret < 0) 1796 goto out; 1797 1798 if (validate_code(&ctx) < 0) { 1799 ret = -EINVAL; 1800 goto out; 1801 } 1802 1803 tmp = bpf_arch_text_copy(ro_image, image, size); 1804 if (IS_ERR(tmp)) { 1805 ret = PTR_ERR(tmp); 1806 goto out; 1807 } 1808 1809 out: 1810 kvfree(image); 1811 return ret < 0 ? ret : size; 1812 } 1813 1814 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1815 struct bpf_tramp_links *tlinks, void *func_addr) 1816 { 1817 int ret; 1818 struct jit_ctx ctx; 1819 struct bpf_tramp_image im; 1820 1821 ctx.image = NULL; 1822 ctx.idx = 0; 1823 1824 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags); 1825 1826 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; 1827 } 1828 1829 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1830 { 1831 bool tmp_blinded = false, extra_pass = false; 1832 u8 *image_ptr; 1833 int image_size, prog_size, extable_size; 1834 struct jit_ctx ctx; 1835 struct jit_data *jit_data; 1836 struct bpf_binary_header *header; 1837 struct bpf_prog *tmp, *orig_prog = prog; 1838 1839 /* 1840 * If BPF JIT was not enabled then we must fall back to 1841 * the interpreter. 1842 */ 1843 if (!prog->jit_requested) 1844 return orig_prog; 1845 1846 tmp = bpf_jit_blind_constants(prog); 1847 /* 1848 * If blinding was requested and we failed during blinding, 1849 * we must fall back to the interpreter. Otherwise, we save 1850 * the new JITed code. 1851 */ 1852 if (IS_ERR(tmp)) 1853 return orig_prog; 1854 1855 if (tmp != prog) { 1856 tmp_blinded = true; 1857 prog = tmp; 1858 } 1859 1860 jit_data = prog->aux->jit_data; 1861 if (!jit_data) { 1862 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1863 if (!jit_data) { 1864 prog = orig_prog; 1865 goto out; 1866 } 1867 prog->aux->jit_data = jit_data; 1868 } 1869 if (jit_data->ctx.offset) { 1870 ctx = jit_data->ctx; 1871 image_ptr = jit_data->image; 1872 header = jit_data->header; 1873 extra_pass = true; 1874 prog_size = sizeof(u32) * ctx.idx; 1875 goto skip_init_ctx; 1876 } 1877 1878 memset(&ctx, 0, sizeof(ctx)); 1879 ctx.prog = prog; 1880 1881 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 1882 if (ctx.offset == NULL) { 1883 prog = orig_prog; 1884 goto out_offset; 1885 } 1886 1887 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 1888 build_prologue(&ctx); 1889 if (build_body(&ctx, extra_pass)) { 1890 prog = orig_prog; 1891 goto out_offset; 1892 } 1893 ctx.epilogue_offset = ctx.idx; 1894 build_epilogue(&ctx); 1895 1896 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); 1897 1898 /* Now we know the actual image size. 1899 * As each LoongArch instruction is of length 32bit, 1900 * we are translating number of JITed intructions into 1901 * the size required to store these JITed code. 1902 */ 1903 prog_size = sizeof(u32) * ctx.idx; 1904 image_size = prog_size + extable_size; 1905 /* Now we know the size of the structure to make */ 1906 header = bpf_jit_binary_alloc(image_size, &image_ptr, 1907 sizeof(u32), jit_fill_hole); 1908 if (header == NULL) { 1909 prog = orig_prog; 1910 goto out_offset; 1911 } 1912 1913 /* 2. Now, the actual pass to generate final JIT code */ 1914 ctx.image = (union loongarch_instruction *)image_ptr; 1915 if (extable_size) 1916 prog->aux->extable = (void *)image_ptr + prog_size; 1917 1918 skip_init_ctx: 1919 ctx.idx = 0; 1920 ctx.num_exentries = 0; 1921 1922 build_prologue(&ctx); 1923 if (build_body(&ctx, extra_pass)) { 1924 bpf_jit_binary_free(header); 1925 prog = orig_prog; 1926 goto out_offset; 1927 } 1928 build_epilogue(&ctx); 1929 1930 /* 3. Extra pass to validate JITed code */ 1931 if (validate_ctx(&ctx)) { 1932 bpf_jit_binary_free(header); 1933 prog = orig_prog; 1934 goto out_offset; 1935 } 1936 1937 /* And we're done */ 1938 if (bpf_jit_enable > 1) 1939 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1940 1941 /* Update the icache */ 1942 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); 1943 1944 if (!prog->is_func || extra_pass) { 1945 int err; 1946 1947 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1948 pr_err_once("multi-func JIT bug %d != %d\n", 1949 ctx.idx, jit_data->ctx.idx); 1950 goto out_free; 1951 } 1952 err = bpf_jit_binary_lock_ro(header); 1953 if (err) { 1954 pr_err_once("bpf_jit_binary_lock_ro() returned %d\n", 1955 err); 1956 goto out_free; 1957 } 1958 } else { 1959 jit_data->ctx = ctx; 1960 jit_data->image = image_ptr; 1961 jit_data->header = header; 1962 } 1963 prog->jited = 1; 1964 prog->jited_len = prog_size; 1965 prog->bpf_func = (void *)ctx.image; 1966 1967 if (!prog->is_func || extra_pass) { 1968 int i; 1969 1970 /* offset[prog->len] is the size of program */ 1971 for (i = 0; i <= prog->len; i++) 1972 ctx.offset[i] *= LOONGARCH_INSN_SIZE; 1973 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 1974 1975 out_offset: 1976 kvfree(ctx.offset); 1977 kfree(jit_data); 1978 prog->aux->jit_data = NULL; 1979 } 1980 1981 out: 1982 if (tmp_blinded) 1983 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); 1984 1985 1986 return prog; 1987 1988 out_free: 1989 bpf_jit_binary_free(header); 1990 prog->bpf_func = NULL; 1991 prog->jited = 0; 1992 prog->jited_len = 0; 1993 goto out_offset; 1994 } 1995 1996 bool bpf_jit_bypass_spec_v1(void) 1997 { 1998 return true; 1999 } 2000 2001 bool bpf_jit_bypass_spec_v4(void) 2002 { 2003 return true; 2004 } 2005 2006 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ 2007 bool bpf_jit_supports_subprog_tailcalls(void) 2008 { 2009 return true; 2010 } 2011