1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/memory.h> 8 #include "bpf_jit.h" 9 10 #define LOONGARCH_MAX_REG_ARGS 8 11 12 #define LOONGARCH_LONG_JUMP_NINSNS 5 13 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 14 15 #define LOONGARCH_FENTRY_NINSNS 2 16 #define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) 17 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 18 19 #define REG_TCC LOONGARCH_GPR_A6 20 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) 21 22 static const int regmap[] = { 23 /* return value from in-kernel function, and exit value for eBPF program */ 24 [BPF_REG_0] = LOONGARCH_GPR_A5, 25 /* arguments from eBPF program to in-kernel function */ 26 [BPF_REG_1] = LOONGARCH_GPR_A0, 27 [BPF_REG_2] = LOONGARCH_GPR_A1, 28 [BPF_REG_3] = LOONGARCH_GPR_A2, 29 [BPF_REG_4] = LOONGARCH_GPR_A3, 30 [BPF_REG_5] = LOONGARCH_GPR_A4, 31 /* callee saved registers that in-kernel function will preserve */ 32 [BPF_REG_6] = LOONGARCH_GPR_S0, 33 [BPF_REG_7] = LOONGARCH_GPR_S1, 34 [BPF_REG_8] = LOONGARCH_GPR_S2, 35 [BPF_REG_9] = LOONGARCH_GPR_S3, 36 /* read-only frame pointer to access stack */ 37 [BPF_REG_FP] = LOONGARCH_GPR_S4, 38 /* temporary register for blinding constants */ 39 [BPF_REG_AX] = LOONGARCH_GPR_T0, 40 }; 41 42 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) 43 { 44 const struct bpf_prog *prog = ctx->prog; 45 const bool is_main_prog = !bpf_is_subprog(prog); 46 47 if (is_main_prog) { 48 /* 49 * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT 50 * if (REG_TCC > T3 ) 51 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 52 * else 53 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 54 * REG_TCC = LOONGARCH_GPR_SP + store_offset 55 * 56 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 57 * 58 * The purpose of this code is to first push the TCC into stack, 59 * and then push the address of TCC into stack. 60 * In cases where bpf2bpf and tailcall are used in combination, 61 * the value in REG_TCC may be a count or an address, 62 * these two cases need to be judged and handled separately. 63 */ 64 emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 65 *store_offset -= sizeof(long); 66 67 emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); 68 69 /* 70 * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, 71 * push tcc into stack 72 */ 73 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 74 75 /* Push the address of TCC into the REG_TCC */ 76 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 77 78 emit_uncond_jmp(ctx, 2); 79 80 /* 81 * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, 82 * push tcc_ptr into stack 83 */ 84 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 85 } else { 86 *store_offset -= sizeof(long); 87 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 88 } 89 90 /* Push tcc_ptr into stack */ 91 *store_offset -= sizeof(long); 92 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 93 } 94 95 /* 96 * eBPF prog stack layout: 97 * 98 * high 99 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP 100 * | $ra | 101 * +-------------------------+ 102 * | $fp | 103 * +-------------------------+ 104 * | $s0 | 105 * +-------------------------+ 106 * | $s1 | 107 * +-------------------------+ 108 * | $s2 | 109 * +-------------------------+ 110 * | $s3 | 111 * +-------------------------+ 112 * | $s4 | 113 * +-------------------------+ 114 * | $s5 | 115 * +-------------------------+ 116 * | tcc | 117 * +-------------------------+ 118 * | tcc_ptr | 119 * +-------------------------+ <--BPF_REG_FP 120 * | prog->aux->stack_depth | 121 * | (optional) | 122 * current $sp -------------> +-------------------------+ 123 * low 124 */ 125 static void build_prologue(struct jit_ctx *ctx) 126 { 127 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 128 const struct bpf_prog *prog = ctx->prog; 129 const bool is_main_prog = !bpf_is_subprog(prog); 130 131 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 132 133 /* To store ra, fp, s0, s1, s2, s3, s4, s5 */ 134 stack_adjust += sizeof(long) * 8; 135 136 /* To store tcc and tcc_ptr */ 137 stack_adjust += sizeof(long) * 2; 138 139 stack_adjust = round_up(stack_adjust, 16); 140 stack_adjust += bpf_stack_adjust; 141 142 /* Reserve space for the move_imm + jirl instruction */ 143 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 144 emit_insn(ctx, nop); 145 146 /* 147 * First instruction initializes the tail call count (TCC) 148 * register to zero. On tail call we skip this instruction, 149 * and the TCC is passed in REG_TCC from the caller. 150 */ 151 if (is_main_prog) 152 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); 153 154 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); 155 156 store_offset = stack_adjust - sizeof(long); 157 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); 158 159 store_offset -= sizeof(long); 160 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); 161 162 store_offset -= sizeof(long); 163 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); 164 165 store_offset -= sizeof(long); 166 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); 167 168 store_offset -= sizeof(long); 169 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); 170 171 store_offset -= sizeof(long); 172 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); 173 174 store_offset -= sizeof(long); 175 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); 176 177 store_offset -= sizeof(long); 178 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 179 180 prepare_bpf_tail_call_cnt(ctx, &store_offset); 181 182 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); 183 184 if (bpf_stack_adjust) 185 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 186 187 ctx->stack_size = stack_adjust; 188 } 189 190 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) 191 { 192 int stack_adjust = ctx->stack_size; 193 int load_offset; 194 195 load_offset = stack_adjust - sizeof(long); 196 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); 197 198 load_offset -= sizeof(long); 199 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); 200 201 load_offset -= sizeof(long); 202 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); 203 204 load_offset -= sizeof(long); 205 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); 206 207 load_offset -= sizeof(long); 208 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); 209 210 load_offset -= sizeof(long); 211 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); 212 213 load_offset -= sizeof(long); 214 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); 215 216 load_offset -= sizeof(long); 217 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 218 219 /* 220 * When push into the stack, follow the order of tcc then tcc_ptr. 221 * When pop from the stack, first pop tcc_ptr then followed by tcc. 222 */ 223 load_offset -= 2 * sizeof(long); 224 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 225 226 load_offset += sizeof(long); 227 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 228 229 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); 230 231 if (!is_tail_call) { 232 /* Set return value */ 233 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); 234 /* Return to the caller */ 235 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 236 } else { 237 /* 238 * Call the next bpf prog and skip the first instruction 239 * of TCC initialization. 240 */ 241 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6); 242 } 243 } 244 245 static void build_epilogue(struct jit_ctx *ctx) 246 { 247 __build_epilogue(ctx, false); 248 } 249 250 bool bpf_jit_supports_kfunc_call(void) 251 { 252 return true; 253 } 254 255 bool bpf_jit_supports_far_kfunc_call(void) 256 { 257 return true; 258 } 259 260 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) 261 { 262 int off, tc_ninsn = 0; 263 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 264 u8 a1 = LOONGARCH_GPR_A1; 265 u8 a2 = LOONGARCH_GPR_A2; 266 u8 t1 = LOONGARCH_GPR_T1; 267 u8 t2 = LOONGARCH_GPR_T2; 268 u8 t3 = LOONGARCH_GPR_T3; 269 const int idx0 = ctx->idx; 270 271 #define cur_offset (ctx->idx - idx0) 272 #define jmp_offset (tc_ninsn - (cur_offset)) 273 274 /* 275 * a0: &ctx 276 * a1: &array 277 * a2: index 278 * 279 * if (index >= array->map.max_entries) 280 * goto out; 281 */ 282 tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; 283 off = offsetof(struct bpf_array, map.max_entries); 284 emit_insn(ctx, ldwu, t1, a1, off); 285 /* bgeu $a2, $t1, jmp_offset */ 286 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) 287 goto toofar; 288 289 /* 290 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) 291 * goto out; 292 */ 293 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 294 emit_insn(ctx, ldd, t3, REG_TCC, 0); 295 emit_insn(ctx, addid, t3, t3, 1); 296 emit_insn(ctx, std, t3, REG_TCC, 0); 297 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 298 if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0) 299 goto toofar; 300 301 /* 302 * prog = array->ptrs[index]; 303 * if (!prog) 304 * goto out; 305 */ 306 emit_insn(ctx, alsld, t2, a2, a1, 2); 307 off = offsetof(struct bpf_array, ptrs); 308 emit_insn(ctx, ldd, t2, t2, off); 309 /* beq $t2, $zero, jmp_offset */ 310 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 311 goto toofar; 312 313 /* goto *(prog->bpf_func + 4); */ 314 off = offsetof(struct bpf_prog, bpf_func); 315 emit_insn(ctx, ldd, t3, t2, off); 316 __build_epilogue(ctx, true); 317 318 return 0; 319 320 toofar: 321 pr_info_once("tail_call: jump too far\n"); 322 return -1; 323 #undef cur_offset 324 #undef jmp_offset 325 } 326 327 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) 328 { 329 const u8 t1 = LOONGARCH_GPR_T1; 330 const u8 t2 = LOONGARCH_GPR_T2; 331 const u8 t3 = LOONGARCH_GPR_T3; 332 const u8 r0 = regmap[BPF_REG_0]; 333 const u8 src = regmap[insn->src_reg]; 334 const u8 dst = regmap[insn->dst_reg]; 335 const s16 off = insn->off; 336 const s32 imm = insn->imm; 337 const bool isdw = BPF_SIZE(insn->code) == BPF_DW; 338 339 move_imm(ctx, t1, off, false); 340 emit_insn(ctx, addd, t1, dst, t1); 341 move_reg(ctx, t3, src); 342 343 switch (imm) { 344 /* lock *(size *)(dst + off) <op>= src */ 345 case BPF_ADD: 346 if (isdw) 347 emit_insn(ctx, amaddd, t2, t1, src); 348 else 349 emit_insn(ctx, amaddw, t2, t1, src); 350 break; 351 case BPF_AND: 352 if (isdw) 353 emit_insn(ctx, amandd, t2, t1, src); 354 else 355 emit_insn(ctx, amandw, t2, t1, src); 356 break; 357 case BPF_OR: 358 if (isdw) 359 emit_insn(ctx, amord, t2, t1, src); 360 else 361 emit_insn(ctx, amorw, t2, t1, src); 362 break; 363 case BPF_XOR: 364 if (isdw) 365 emit_insn(ctx, amxord, t2, t1, src); 366 else 367 emit_insn(ctx, amxorw, t2, t1, src); 368 break; 369 /* src = atomic_fetch_<op>(dst + off, src) */ 370 case BPF_ADD | BPF_FETCH: 371 if (isdw) { 372 emit_insn(ctx, amaddd, src, t1, t3); 373 } else { 374 emit_insn(ctx, amaddw, src, t1, t3); 375 emit_zext_32(ctx, src, true); 376 } 377 break; 378 case BPF_AND | BPF_FETCH: 379 if (isdw) { 380 emit_insn(ctx, amandd, src, t1, t3); 381 } else { 382 emit_insn(ctx, amandw, src, t1, t3); 383 emit_zext_32(ctx, src, true); 384 } 385 break; 386 case BPF_OR | BPF_FETCH: 387 if (isdw) { 388 emit_insn(ctx, amord, src, t1, t3); 389 } else { 390 emit_insn(ctx, amorw, src, t1, t3); 391 emit_zext_32(ctx, src, true); 392 } 393 break; 394 case BPF_XOR | BPF_FETCH: 395 if (isdw) { 396 emit_insn(ctx, amxord, src, t1, t3); 397 } else { 398 emit_insn(ctx, amxorw, src, t1, t3); 399 emit_zext_32(ctx, src, true); 400 } 401 break; 402 /* src = atomic_xchg(dst + off, src); */ 403 case BPF_XCHG: 404 if (isdw) { 405 emit_insn(ctx, amswapd, src, t1, t3); 406 } else { 407 emit_insn(ctx, amswapw, src, t1, t3); 408 emit_zext_32(ctx, src, true); 409 } 410 break; 411 /* r0 = atomic_cmpxchg(dst + off, r0, src); */ 412 case BPF_CMPXCHG: 413 move_reg(ctx, t2, r0); 414 if (isdw) { 415 emit_insn(ctx, lld, r0, t1, 0); 416 emit_insn(ctx, bne, t2, r0, 4); 417 move_reg(ctx, t3, src); 418 emit_insn(ctx, scd, t3, t1, 0); 419 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); 420 } else { 421 emit_insn(ctx, llw, r0, t1, 0); 422 emit_zext_32(ctx, t2, true); 423 emit_zext_32(ctx, r0, true); 424 emit_insn(ctx, bne, t2, r0, 4); 425 move_reg(ctx, t3, src); 426 emit_insn(ctx, scw, t3, t1, 0); 427 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); 428 emit_zext_32(ctx, r0, true); 429 } 430 break; 431 } 432 } 433 434 static bool is_signed_bpf_cond(u8 cond) 435 { 436 return cond == BPF_JSGT || cond == BPF_JSLT || 437 cond == BPF_JSGE || cond == BPF_JSLE; 438 } 439 440 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 441 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 442 443 bool ex_handler_bpf(const struct exception_table_entry *ex, 444 struct pt_regs *regs) 445 { 446 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 447 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 448 449 regs->regs[dst_reg] = 0; 450 regs->csr_era = (unsigned long)&ex->fixup - offset; 451 452 return true; 453 } 454 455 /* For accesses to BTF pointers, add an entry to the exception table */ 456 static int add_exception_handler(const struct bpf_insn *insn, 457 struct jit_ctx *ctx, 458 int dst_reg) 459 { 460 unsigned long pc; 461 off_t offset; 462 struct exception_table_entry *ex; 463 464 if (!ctx->image || !ctx->prog->aux->extable) 465 return 0; 466 467 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 468 BPF_MODE(insn->code) != BPF_PROBE_MEMSX) 469 return 0; 470 471 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) 472 return -EINVAL; 473 474 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 475 pc = (unsigned long)&ctx->image[ctx->idx - 1]; 476 477 offset = pc - (long)&ex->insn; 478 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 479 return -ERANGE; 480 481 ex->insn = offset; 482 483 /* 484 * Since the extable follows the program, the fixup offset is always 485 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 486 * to keep things simple, and put the destination register in the upper 487 * bits. We don't need to worry about buildtime or runtime sort 488 * modifying the upper bits because the table is already sorted, and 489 * isn't part of the main exception table. 490 */ 491 offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 492 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 493 return -ERANGE; 494 495 ex->type = EX_TYPE_BPF; 496 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 497 498 ctx->num_exentries++; 499 500 return 0; 501 } 502 503 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) 504 { 505 u8 tm = -1; 506 u64 func_addr; 507 bool func_addr_fixed, sign_extend; 508 int i = insn - ctx->prog->insnsi; 509 int ret, jmp_offset, tcc_ptr_off; 510 const u8 code = insn->code; 511 const u8 cond = BPF_OP(code); 512 const u8 t1 = LOONGARCH_GPR_T1; 513 const u8 t2 = LOONGARCH_GPR_T2; 514 const u8 src = regmap[insn->src_reg]; 515 const u8 dst = regmap[insn->dst_reg]; 516 const s16 off = insn->off; 517 const s32 imm = insn->imm; 518 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; 519 520 switch (code) { 521 /* dst = src */ 522 case BPF_ALU | BPF_MOV | BPF_X: 523 case BPF_ALU64 | BPF_MOV | BPF_X: 524 switch (off) { 525 case 0: 526 move_reg(ctx, dst, src); 527 emit_zext_32(ctx, dst, is32); 528 break; 529 case 8: 530 emit_insn(ctx, extwb, dst, src); 531 emit_zext_32(ctx, dst, is32); 532 break; 533 case 16: 534 emit_insn(ctx, extwh, dst, src); 535 emit_zext_32(ctx, dst, is32); 536 break; 537 case 32: 538 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); 539 break; 540 } 541 break; 542 543 /* dst = imm */ 544 case BPF_ALU | BPF_MOV | BPF_K: 545 case BPF_ALU64 | BPF_MOV | BPF_K: 546 move_imm(ctx, dst, imm, is32); 547 break; 548 549 /* dst = dst + src */ 550 case BPF_ALU | BPF_ADD | BPF_X: 551 case BPF_ALU64 | BPF_ADD | BPF_X: 552 emit_insn(ctx, addd, dst, dst, src); 553 emit_zext_32(ctx, dst, is32); 554 break; 555 556 /* dst = dst + imm */ 557 case BPF_ALU | BPF_ADD | BPF_K: 558 case BPF_ALU64 | BPF_ADD | BPF_K: 559 if (is_signed_imm12(imm)) { 560 emit_insn(ctx, addid, dst, dst, imm); 561 } else { 562 move_imm(ctx, t1, imm, is32); 563 emit_insn(ctx, addd, dst, dst, t1); 564 } 565 emit_zext_32(ctx, dst, is32); 566 break; 567 568 /* dst = dst - src */ 569 case BPF_ALU | BPF_SUB | BPF_X: 570 case BPF_ALU64 | BPF_SUB | BPF_X: 571 emit_insn(ctx, subd, dst, dst, src); 572 emit_zext_32(ctx, dst, is32); 573 break; 574 575 /* dst = dst - imm */ 576 case BPF_ALU | BPF_SUB | BPF_K: 577 case BPF_ALU64 | BPF_SUB | BPF_K: 578 if (is_signed_imm12(-imm)) { 579 emit_insn(ctx, addid, dst, dst, -imm); 580 } else { 581 move_imm(ctx, t1, imm, is32); 582 emit_insn(ctx, subd, dst, dst, t1); 583 } 584 emit_zext_32(ctx, dst, is32); 585 break; 586 587 /* dst = dst * src */ 588 case BPF_ALU | BPF_MUL | BPF_X: 589 case BPF_ALU64 | BPF_MUL | BPF_X: 590 emit_insn(ctx, muld, dst, dst, src); 591 emit_zext_32(ctx, dst, is32); 592 break; 593 594 /* dst = dst * imm */ 595 case BPF_ALU | BPF_MUL | BPF_K: 596 case BPF_ALU64 | BPF_MUL | BPF_K: 597 move_imm(ctx, t1, imm, is32); 598 emit_insn(ctx, muld, dst, dst, t1); 599 emit_zext_32(ctx, dst, is32); 600 break; 601 602 /* dst = dst / src */ 603 case BPF_ALU | BPF_DIV | BPF_X: 604 case BPF_ALU64 | BPF_DIV | BPF_X: 605 if (!off) { 606 emit_zext_32(ctx, dst, is32); 607 move_reg(ctx, t1, src); 608 emit_zext_32(ctx, t1, is32); 609 emit_insn(ctx, divdu, dst, dst, t1); 610 emit_zext_32(ctx, dst, is32); 611 } else { 612 emit_sext_32(ctx, dst, is32); 613 move_reg(ctx, t1, src); 614 emit_sext_32(ctx, t1, is32); 615 emit_insn(ctx, divd, dst, dst, t1); 616 emit_sext_32(ctx, dst, is32); 617 } 618 break; 619 620 /* dst = dst / imm */ 621 case BPF_ALU | BPF_DIV | BPF_K: 622 case BPF_ALU64 | BPF_DIV | BPF_K: 623 if (!off) { 624 move_imm(ctx, t1, imm, is32); 625 emit_zext_32(ctx, dst, is32); 626 emit_insn(ctx, divdu, dst, dst, t1); 627 emit_zext_32(ctx, dst, is32); 628 } else { 629 move_imm(ctx, t1, imm, false); 630 emit_sext_32(ctx, t1, is32); 631 emit_sext_32(ctx, dst, is32); 632 emit_insn(ctx, divd, dst, dst, t1); 633 emit_sext_32(ctx, dst, is32); 634 } 635 break; 636 637 /* dst = dst % src */ 638 case BPF_ALU | BPF_MOD | BPF_X: 639 case BPF_ALU64 | BPF_MOD | BPF_X: 640 if (!off) { 641 emit_zext_32(ctx, dst, is32); 642 move_reg(ctx, t1, src); 643 emit_zext_32(ctx, t1, is32); 644 emit_insn(ctx, moddu, dst, dst, t1); 645 emit_zext_32(ctx, dst, is32); 646 } else { 647 emit_sext_32(ctx, dst, is32); 648 move_reg(ctx, t1, src); 649 emit_sext_32(ctx, t1, is32); 650 emit_insn(ctx, modd, dst, dst, t1); 651 emit_sext_32(ctx, dst, is32); 652 } 653 break; 654 655 /* dst = dst % imm */ 656 case BPF_ALU | BPF_MOD | BPF_K: 657 case BPF_ALU64 | BPF_MOD | BPF_K: 658 if (!off) { 659 move_imm(ctx, t1, imm, is32); 660 emit_zext_32(ctx, dst, is32); 661 emit_insn(ctx, moddu, dst, dst, t1); 662 emit_zext_32(ctx, dst, is32); 663 } else { 664 move_imm(ctx, t1, imm, false); 665 emit_sext_32(ctx, t1, is32); 666 emit_sext_32(ctx, dst, is32); 667 emit_insn(ctx, modd, dst, dst, t1); 668 emit_sext_32(ctx, dst, is32); 669 } 670 break; 671 672 /* dst = -dst */ 673 case BPF_ALU | BPF_NEG: 674 case BPF_ALU64 | BPF_NEG: 675 move_imm(ctx, t1, imm, is32); 676 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); 677 emit_zext_32(ctx, dst, is32); 678 break; 679 680 /* dst = dst & src */ 681 case BPF_ALU | BPF_AND | BPF_X: 682 case BPF_ALU64 | BPF_AND | BPF_X: 683 emit_insn(ctx, and, dst, dst, src); 684 emit_zext_32(ctx, dst, is32); 685 break; 686 687 /* dst = dst & imm */ 688 case BPF_ALU | BPF_AND | BPF_K: 689 case BPF_ALU64 | BPF_AND | BPF_K: 690 if (is_unsigned_imm12(imm)) { 691 emit_insn(ctx, andi, dst, dst, imm); 692 } else { 693 move_imm(ctx, t1, imm, is32); 694 emit_insn(ctx, and, dst, dst, t1); 695 } 696 emit_zext_32(ctx, dst, is32); 697 break; 698 699 /* dst = dst | src */ 700 case BPF_ALU | BPF_OR | BPF_X: 701 case BPF_ALU64 | BPF_OR | BPF_X: 702 emit_insn(ctx, or, dst, dst, src); 703 emit_zext_32(ctx, dst, is32); 704 break; 705 706 /* dst = dst | imm */ 707 case BPF_ALU | BPF_OR | BPF_K: 708 case BPF_ALU64 | BPF_OR | BPF_K: 709 if (is_unsigned_imm12(imm)) { 710 emit_insn(ctx, ori, dst, dst, imm); 711 } else { 712 move_imm(ctx, t1, imm, is32); 713 emit_insn(ctx, or, dst, dst, t1); 714 } 715 emit_zext_32(ctx, dst, is32); 716 break; 717 718 /* dst = dst ^ src */ 719 case BPF_ALU | BPF_XOR | BPF_X: 720 case BPF_ALU64 | BPF_XOR | BPF_X: 721 emit_insn(ctx, xor, dst, dst, src); 722 emit_zext_32(ctx, dst, is32); 723 break; 724 725 /* dst = dst ^ imm */ 726 case BPF_ALU | BPF_XOR | BPF_K: 727 case BPF_ALU64 | BPF_XOR | BPF_K: 728 if (is_unsigned_imm12(imm)) { 729 emit_insn(ctx, xori, dst, dst, imm); 730 } else { 731 move_imm(ctx, t1, imm, is32); 732 emit_insn(ctx, xor, dst, dst, t1); 733 } 734 emit_zext_32(ctx, dst, is32); 735 break; 736 737 /* dst = dst << src (logical) */ 738 case BPF_ALU | BPF_LSH | BPF_X: 739 emit_insn(ctx, sllw, dst, dst, src); 740 emit_zext_32(ctx, dst, is32); 741 break; 742 743 case BPF_ALU64 | BPF_LSH | BPF_X: 744 emit_insn(ctx, slld, dst, dst, src); 745 break; 746 747 /* dst = dst << imm (logical) */ 748 case BPF_ALU | BPF_LSH | BPF_K: 749 emit_insn(ctx, slliw, dst, dst, imm); 750 emit_zext_32(ctx, dst, is32); 751 break; 752 753 case BPF_ALU64 | BPF_LSH | BPF_K: 754 emit_insn(ctx, sllid, dst, dst, imm); 755 break; 756 757 /* dst = dst >> src (logical) */ 758 case BPF_ALU | BPF_RSH | BPF_X: 759 emit_insn(ctx, srlw, dst, dst, src); 760 emit_zext_32(ctx, dst, is32); 761 break; 762 763 case BPF_ALU64 | BPF_RSH | BPF_X: 764 emit_insn(ctx, srld, dst, dst, src); 765 break; 766 767 /* dst = dst >> imm (logical) */ 768 case BPF_ALU | BPF_RSH | BPF_K: 769 emit_insn(ctx, srliw, dst, dst, imm); 770 emit_zext_32(ctx, dst, is32); 771 break; 772 773 case BPF_ALU64 | BPF_RSH | BPF_K: 774 emit_insn(ctx, srlid, dst, dst, imm); 775 break; 776 777 /* dst = dst >> src (arithmetic) */ 778 case BPF_ALU | BPF_ARSH | BPF_X: 779 emit_insn(ctx, sraw, dst, dst, src); 780 emit_zext_32(ctx, dst, is32); 781 break; 782 783 case BPF_ALU64 | BPF_ARSH | BPF_X: 784 emit_insn(ctx, srad, dst, dst, src); 785 break; 786 787 /* dst = dst >> imm (arithmetic) */ 788 case BPF_ALU | BPF_ARSH | BPF_K: 789 emit_insn(ctx, sraiw, dst, dst, imm); 790 emit_zext_32(ctx, dst, is32); 791 break; 792 793 case BPF_ALU64 | BPF_ARSH | BPF_K: 794 emit_insn(ctx, sraid, dst, dst, imm); 795 break; 796 797 /* dst = BSWAP##imm(dst) */ 798 case BPF_ALU | BPF_END | BPF_FROM_LE: 799 switch (imm) { 800 case 16: 801 /* zero-extend 16 bits into 64 bits */ 802 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 803 break; 804 case 32: 805 /* zero-extend 32 bits into 64 bits */ 806 emit_zext_32(ctx, dst, is32); 807 break; 808 case 64: 809 /* do nothing */ 810 break; 811 } 812 break; 813 814 case BPF_ALU | BPF_END | BPF_FROM_BE: 815 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 816 switch (imm) { 817 case 16: 818 emit_insn(ctx, revb2h, dst, dst); 819 /* zero-extend 16 bits into 64 bits */ 820 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 821 break; 822 case 32: 823 emit_insn(ctx, revb2w, dst, dst); 824 /* clear the upper 32 bits */ 825 emit_zext_32(ctx, dst, true); 826 break; 827 case 64: 828 emit_insn(ctx, revbd, dst, dst); 829 break; 830 } 831 break; 832 833 /* PC += off if dst cond src */ 834 case BPF_JMP | BPF_JEQ | BPF_X: 835 case BPF_JMP | BPF_JNE | BPF_X: 836 case BPF_JMP | BPF_JGT | BPF_X: 837 case BPF_JMP | BPF_JGE | BPF_X: 838 case BPF_JMP | BPF_JLT | BPF_X: 839 case BPF_JMP | BPF_JLE | BPF_X: 840 case BPF_JMP | BPF_JSGT | BPF_X: 841 case BPF_JMP | BPF_JSGE | BPF_X: 842 case BPF_JMP | BPF_JSLT | BPF_X: 843 case BPF_JMP | BPF_JSLE | BPF_X: 844 case BPF_JMP32 | BPF_JEQ | BPF_X: 845 case BPF_JMP32 | BPF_JNE | BPF_X: 846 case BPF_JMP32 | BPF_JGT | BPF_X: 847 case BPF_JMP32 | BPF_JGE | BPF_X: 848 case BPF_JMP32 | BPF_JLT | BPF_X: 849 case BPF_JMP32 | BPF_JLE | BPF_X: 850 case BPF_JMP32 | BPF_JSGT | BPF_X: 851 case BPF_JMP32 | BPF_JSGE | BPF_X: 852 case BPF_JMP32 | BPF_JSLT | BPF_X: 853 case BPF_JMP32 | BPF_JSLE | BPF_X: 854 jmp_offset = bpf2la_offset(i, off, ctx); 855 move_reg(ctx, t1, dst); 856 move_reg(ctx, t2, src); 857 if (is_signed_bpf_cond(BPF_OP(code))) { 858 emit_sext_32(ctx, t1, is32); 859 emit_sext_32(ctx, t2, is32); 860 } else { 861 emit_zext_32(ctx, t1, is32); 862 emit_zext_32(ctx, t2, is32); 863 } 864 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) 865 goto toofar; 866 break; 867 868 /* PC += off if dst cond imm */ 869 case BPF_JMP | BPF_JEQ | BPF_K: 870 case BPF_JMP | BPF_JNE | BPF_K: 871 case BPF_JMP | BPF_JGT | BPF_K: 872 case BPF_JMP | BPF_JGE | BPF_K: 873 case BPF_JMP | BPF_JLT | BPF_K: 874 case BPF_JMP | BPF_JLE | BPF_K: 875 case BPF_JMP | BPF_JSGT | BPF_K: 876 case BPF_JMP | BPF_JSGE | BPF_K: 877 case BPF_JMP | BPF_JSLT | BPF_K: 878 case BPF_JMP | BPF_JSLE | BPF_K: 879 case BPF_JMP32 | BPF_JEQ | BPF_K: 880 case BPF_JMP32 | BPF_JNE | BPF_K: 881 case BPF_JMP32 | BPF_JGT | BPF_K: 882 case BPF_JMP32 | BPF_JGE | BPF_K: 883 case BPF_JMP32 | BPF_JLT | BPF_K: 884 case BPF_JMP32 | BPF_JLE | BPF_K: 885 case BPF_JMP32 | BPF_JSGT | BPF_K: 886 case BPF_JMP32 | BPF_JSGE | BPF_K: 887 case BPF_JMP32 | BPF_JSLT | BPF_K: 888 case BPF_JMP32 | BPF_JSLE | BPF_K: 889 jmp_offset = bpf2la_offset(i, off, ctx); 890 if (imm) { 891 move_imm(ctx, t1, imm, false); 892 tm = t1; 893 } else { 894 /* If imm is 0, simply use zero register. */ 895 tm = LOONGARCH_GPR_ZERO; 896 } 897 move_reg(ctx, t2, dst); 898 if (is_signed_bpf_cond(BPF_OP(code))) { 899 emit_sext_32(ctx, tm, is32); 900 emit_sext_32(ctx, t2, is32); 901 } else { 902 emit_zext_32(ctx, tm, is32); 903 emit_zext_32(ctx, t2, is32); 904 } 905 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) 906 goto toofar; 907 break; 908 909 /* PC += off if dst & src */ 910 case BPF_JMP | BPF_JSET | BPF_X: 911 case BPF_JMP32 | BPF_JSET | BPF_X: 912 jmp_offset = bpf2la_offset(i, off, ctx); 913 emit_insn(ctx, and, t1, dst, src); 914 emit_zext_32(ctx, t1, is32); 915 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 916 goto toofar; 917 break; 918 919 /* PC += off if dst & imm */ 920 case BPF_JMP | BPF_JSET | BPF_K: 921 case BPF_JMP32 | BPF_JSET | BPF_K: 922 jmp_offset = bpf2la_offset(i, off, ctx); 923 move_imm(ctx, t1, imm, is32); 924 emit_insn(ctx, and, t1, dst, t1); 925 emit_zext_32(ctx, t1, is32); 926 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 927 goto toofar; 928 break; 929 930 /* PC += off */ 931 case BPF_JMP | BPF_JA: 932 case BPF_JMP32 | BPF_JA: 933 if (BPF_CLASS(code) == BPF_JMP) 934 jmp_offset = bpf2la_offset(i, off, ctx); 935 else 936 jmp_offset = bpf2la_offset(i, imm, ctx); 937 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 938 goto toofar; 939 break; 940 941 /* function call */ 942 case BPF_JMP | BPF_CALL: 943 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 944 &func_addr, &func_addr_fixed); 945 if (ret < 0) 946 return ret; 947 948 if (insn->src_reg == BPF_PSEUDO_CALL) { 949 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 950 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 951 } 952 953 move_addr(ctx, t1, func_addr); 954 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); 955 956 if (insn->src_reg != BPF_PSEUDO_CALL) 957 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); 958 959 break; 960 961 /* tail call */ 962 case BPF_JMP | BPF_TAIL_CALL: 963 if (emit_bpf_tail_call(ctx, i) < 0) 964 return -EINVAL; 965 break; 966 967 /* function return */ 968 case BPF_JMP | BPF_EXIT: 969 if (i == ctx->prog->len - 1) 970 break; 971 972 jmp_offset = epilogue_offset(ctx); 973 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 974 goto toofar; 975 break; 976 977 /* dst = imm64 */ 978 case BPF_LD | BPF_IMM | BPF_DW: 979 { 980 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; 981 982 if (bpf_pseudo_func(insn)) 983 move_addr(ctx, dst, imm64); 984 else 985 move_imm(ctx, dst, imm64, is32); 986 return 1; 987 } 988 989 /* dst = *(size *)(src + off) */ 990 case BPF_LDX | BPF_MEM | BPF_B: 991 case BPF_LDX | BPF_MEM | BPF_H: 992 case BPF_LDX | BPF_MEM | BPF_W: 993 case BPF_LDX | BPF_MEM | BPF_DW: 994 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 995 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 996 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 997 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 998 /* dst_reg = (s64)*(signed size *)(src_reg + off) */ 999 case BPF_LDX | BPF_MEMSX | BPF_B: 1000 case BPF_LDX | BPF_MEMSX | BPF_H: 1001 case BPF_LDX | BPF_MEMSX | BPF_W: 1002 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1003 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1004 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1005 sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || 1006 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1007 switch (BPF_SIZE(code)) { 1008 case BPF_B: 1009 if (is_signed_imm12(off)) { 1010 if (sign_extend) 1011 emit_insn(ctx, ldb, dst, src, off); 1012 else 1013 emit_insn(ctx, ldbu, dst, src, off); 1014 } else { 1015 move_imm(ctx, t1, off, is32); 1016 if (sign_extend) 1017 emit_insn(ctx, ldxb, dst, src, t1); 1018 else 1019 emit_insn(ctx, ldxbu, dst, src, t1); 1020 } 1021 break; 1022 case BPF_H: 1023 if (is_signed_imm12(off)) { 1024 if (sign_extend) 1025 emit_insn(ctx, ldh, dst, src, off); 1026 else 1027 emit_insn(ctx, ldhu, dst, src, off); 1028 } else { 1029 move_imm(ctx, t1, off, is32); 1030 if (sign_extend) 1031 emit_insn(ctx, ldxh, dst, src, t1); 1032 else 1033 emit_insn(ctx, ldxhu, dst, src, t1); 1034 } 1035 break; 1036 case BPF_W: 1037 if (is_signed_imm12(off)) { 1038 if (sign_extend) 1039 emit_insn(ctx, ldw, dst, src, off); 1040 else 1041 emit_insn(ctx, ldwu, dst, src, off); 1042 } else { 1043 move_imm(ctx, t1, off, is32); 1044 if (sign_extend) 1045 emit_insn(ctx, ldxw, dst, src, t1); 1046 else 1047 emit_insn(ctx, ldxwu, dst, src, t1); 1048 } 1049 break; 1050 case BPF_DW: 1051 move_imm(ctx, t1, off, is32); 1052 emit_insn(ctx, ldxd, dst, src, t1); 1053 break; 1054 } 1055 1056 ret = add_exception_handler(insn, ctx, dst); 1057 if (ret) 1058 return ret; 1059 break; 1060 1061 /* *(size *)(dst + off) = imm */ 1062 case BPF_ST | BPF_MEM | BPF_B: 1063 case BPF_ST | BPF_MEM | BPF_H: 1064 case BPF_ST | BPF_MEM | BPF_W: 1065 case BPF_ST | BPF_MEM | BPF_DW: 1066 switch (BPF_SIZE(code)) { 1067 case BPF_B: 1068 move_imm(ctx, t1, imm, is32); 1069 if (is_signed_imm12(off)) { 1070 emit_insn(ctx, stb, t1, dst, off); 1071 } else { 1072 move_imm(ctx, t2, off, is32); 1073 emit_insn(ctx, stxb, t1, dst, t2); 1074 } 1075 break; 1076 case BPF_H: 1077 move_imm(ctx, t1, imm, is32); 1078 if (is_signed_imm12(off)) { 1079 emit_insn(ctx, sth, t1, dst, off); 1080 } else { 1081 move_imm(ctx, t2, off, is32); 1082 emit_insn(ctx, stxh, t1, dst, t2); 1083 } 1084 break; 1085 case BPF_W: 1086 move_imm(ctx, t1, imm, is32); 1087 if (is_signed_imm12(off)) { 1088 emit_insn(ctx, stw, t1, dst, off); 1089 } else if (is_signed_imm14(off)) { 1090 emit_insn(ctx, stptrw, t1, dst, off); 1091 } else { 1092 move_imm(ctx, t2, off, is32); 1093 emit_insn(ctx, stxw, t1, dst, t2); 1094 } 1095 break; 1096 case BPF_DW: 1097 move_imm(ctx, t1, imm, is32); 1098 if (is_signed_imm12(off)) { 1099 emit_insn(ctx, std, t1, dst, off); 1100 } else if (is_signed_imm14(off)) { 1101 emit_insn(ctx, stptrd, t1, dst, off); 1102 } else { 1103 move_imm(ctx, t2, off, is32); 1104 emit_insn(ctx, stxd, t1, dst, t2); 1105 } 1106 break; 1107 } 1108 break; 1109 1110 /* *(size *)(dst + off) = src */ 1111 case BPF_STX | BPF_MEM | BPF_B: 1112 case BPF_STX | BPF_MEM | BPF_H: 1113 case BPF_STX | BPF_MEM | BPF_W: 1114 case BPF_STX | BPF_MEM | BPF_DW: 1115 switch (BPF_SIZE(code)) { 1116 case BPF_B: 1117 if (is_signed_imm12(off)) { 1118 emit_insn(ctx, stb, src, dst, off); 1119 } else { 1120 move_imm(ctx, t1, off, is32); 1121 emit_insn(ctx, stxb, src, dst, t1); 1122 } 1123 break; 1124 case BPF_H: 1125 if (is_signed_imm12(off)) { 1126 emit_insn(ctx, sth, src, dst, off); 1127 } else { 1128 move_imm(ctx, t1, off, is32); 1129 emit_insn(ctx, stxh, src, dst, t1); 1130 } 1131 break; 1132 case BPF_W: 1133 if (is_signed_imm12(off)) { 1134 emit_insn(ctx, stw, src, dst, off); 1135 } else if (is_signed_imm14(off)) { 1136 emit_insn(ctx, stptrw, src, dst, off); 1137 } else { 1138 move_imm(ctx, t1, off, is32); 1139 emit_insn(ctx, stxw, src, dst, t1); 1140 } 1141 break; 1142 case BPF_DW: 1143 if (is_signed_imm12(off)) { 1144 emit_insn(ctx, std, src, dst, off); 1145 } else if (is_signed_imm14(off)) { 1146 emit_insn(ctx, stptrd, src, dst, off); 1147 } else { 1148 move_imm(ctx, t1, off, is32); 1149 emit_insn(ctx, stxd, src, dst, t1); 1150 } 1151 break; 1152 } 1153 break; 1154 1155 case BPF_STX | BPF_ATOMIC | BPF_W: 1156 case BPF_STX | BPF_ATOMIC | BPF_DW: 1157 emit_atomic(insn, ctx); 1158 break; 1159 1160 /* Speculation barrier */ 1161 case BPF_ST | BPF_NOSPEC: 1162 break; 1163 1164 default: 1165 pr_err("bpf_jit: unknown opcode %02x\n", code); 1166 return -EINVAL; 1167 } 1168 1169 return 0; 1170 1171 toofar: 1172 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); 1173 return -E2BIG; 1174 } 1175 1176 static int build_body(struct jit_ctx *ctx, bool extra_pass) 1177 { 1178 int i; 1179 const struct bpf_prog *prog = ctx->prog; 1180 1181 for (i = 0; i < prog->len; i++) { 1182 const struct bpf_insn *insn = &prog->insnsi[i]; 1183 int ret; 1184 1185 if (ctx->image == NULL) 1186 ctx->offset[i] = ctx->idx; 1187 1188 ret = build_insn(insn, ctx, extra_pass); 1189 if (ret > 0) { 1190 i++; 1191 if (ctx->image == NULL) 1192 ctx->offset[i] = ctx->idx; 1193 continue; 1194 } 1195 if (ret) 1196 return ret; 1197 } 1198 1199 if (ctx->image == NULL) 1200 ctx->offset[i] = ctx->idx; 1201 1202 return 0; 1203 } 1204 1205 /* Fill space with break instructions */ 1206 static void jit_fill_hole(void *area, unsigned int size) 1207 { 1208 u32 *ptr; 1209 1210 /* We are guaranteed to have aligned memory */ 1211 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1212 *ptr++ = INSN_BREAK; 1213 } 1214 1215 static int validate_code(struct jit_ctx *ctx) 1216 { 1217 int i; 1218 union loongarch_instruction insn; 1219 1220 for (i = 0; i < ctx->idx; i++) { 1221 insn = ctx->image[i]; 1222 /* Check INSN_BREAK */ 1223 if (insn.word == INSN_BREAK) 1224 return -1; 1225 } 1226 1227 return 0; 1228 } 1229 1230 static int validate_ctx(struct jit_ctx *ctx) 1231 { 1232 if (validate_code(ctx)) 1233 return -1; 1234 1235 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) 1236 return -1; 1237 1238 return 0; 1239 } 1240 1241 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) 1242 { 1243 if (!target) { 1244 pr_err("bpf_jit: jump target address is error\n"); 1245 return -EFAULT; 1246 } 1247 1248 move_imm(ctx, LOONGARCH_GPR_T1, target, false); 1249 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); 1250 1251 return 0; 1252 } 1253 1254 static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 1255 { 1256 int i; 1257 struct jit_ctx ctx; 1258 1259 ctx.idx = 0; 1260 ctx.image = (union loongarch_instruction *)insns; 1261 1262 if (!target) { 1263 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1264 emit_insn((&ctx), nop); 1265 return 0; 1266 } 1267 1268 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target); 1269 } 1270 1271 static int emit_call(struct jit_ctx *ctx, u64 addr) 1272 { 1273 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); 1274 } 1275 1276 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 1277 { 1278 int ret; 1279 1280 mutex_lock(&text_mutex); 1281 ret = larch_insn_text_copy(dst, src, len); 1282 mutex_unlock(&text_mutex); 1283 1284 return ret ? ERR_PTR(-EINVAL) : dst; 1285 } 1286 1287 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 1288 void *old_addr, void *new_addr) 1289 { 1290 int ret; 1291 bool is_call = (poke_type == BPF_MOD_CALL); 1292 u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1293 u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1294 1295 /* Only poking bpf text is supported. Since kernel function entry 1296 * is set up by ftrace, we rely on ftrace to poke kernel functions. 1297 */ 1298 if (!is_bpf_text_address((unsigned long)ip)) 1299 return -ENOTSUPP; 1300 1301 ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); 1302 if (ret) 1303 return ret; 1304 1305 if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1306 return -EFAULT; 1307 1308 ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call); 1309 if (ret) 1310 return ret; 1311 1312 mutex_lock(&text_mutex); 1313 if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1314 ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); 1315 mutex_unlock(&text_mutex); 1316 1317 return ret; 1318 } 1319 1320 int bpf_arch_text_invalidate(void *dst, size_t len) 1321 { 1322 int i; 1323 int ret = 0; 1324 u32 *inst; 1325 1326 inst = kvmalloc(len, GFP_KERNEL); 1327 if (!inst) 1328 return -ENOMEM; 1329 1330 for (i = 0; i < (len / sizeof(u32)); i++) 1331 inst[i] = INSN_BREAK; 1332 1333 mutex_lock(&text_mutex); 1334 if (larch_insn_text_copy(dst, inst, len)) 1335 ret = -EINVAL; 1336 mutex_unlock(&text_mutex); 1337 1338 kvfree(inst); 1339 1340 return ret; 1341 } 1342 1343 static void store_args(struct jit_ctx *ctx, int nargs, int args_off) 1344 { 1345 int i; 1346 1347 for (i = 0; i < nargs; i++) { 1348 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1349 args_off -= 8; 1350 } 1351 } 1352 1353 static void restore_args(struct jit_ctx *ctx, int nargs, int args_off) 1354 { 1355 int i; 1356 1357 for (i = 0; i < nargs; i++) { 1358 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1359 args_off -= 8; 1360 } 1361 } 1362 1363 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, 1364 int args_off, int retval_off, int run_ctx_off, bool save_ret) 1365 { 1366 int ret; 1367 u32 *branch; 1368 struct bpf_prog *p = l->link.prog; 1369 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 1370 1371 if (l->cookie) { 1372 move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false); 1373 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1374 } else { 1375 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1376 } 1377 1378 /* arg1: prog */ 1379 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1380 /* arg2: &run_ctx */ 1381 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); 1382 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); 1383 if (ret) 1384 return ret; 1385 1386 /* store prog start time */ 1387 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); 1388 1389 /* 1390 * if (__bpf_prog_enter(prog) == 0) 1391 * goto skip_exec_of_prog; 1392 */ 1393 branch = (u32 *)ctx->image + ctx->idx; 1394 /* nop reserved for conditional jump */ 1395 emit_insn(ctx, nop); 1396 1397 /* arg1: &args_off */ 1398 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); 1399 if (!p->jited) 1400 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); 1401 ret = emit_call(ctx, (const u64)p->bpf_func); 1402 if (ret) 1403 return ret; 1404 1405 if (save_ret) { 1406 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1407 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1408 } 1409 1410 /* update branch with beqz */ 1411 if (ctx->image) { 1412 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; 1413 *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset); 1414 } 1415 1416 /* arg1: prog */ 1417 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1418 /* arg2: prog start time */ 1419 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); 1420 /* arg3: &run_ctx */ 1421 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); 1422 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); 1423 1424 return ret; 1425 } 1426 1427 static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, 1428 int args_off, int retval_off, int run_ctx_off, u32 **branches) 1429 { 1430 int i; 1431 1432 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 1433 for (i = 0; i < tl->nr_links; i++) { 1434 invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true); 1435 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); 1436 branches[i] = (u32 *)ctx->image + ctx->idx; 1437 emit_insn(ctx, nop); 1438 } 1439 } 1440 1441 void *arch_alloc_bpf_trampoline(unsigned int size) 1442 { 1443 return bpf_prog_pack_alloc(size, jit_fill_hole); 1444 } 1445 1446 void arch_free_bpf_trampoline(void *image, unsigned int size) 1447 { 1448 bpf_prog_pack_free(image, size); 1449 } 1450 1451 /* 1452 * Sign-extend the register if necessary 1453 */ 1454 static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign) 1455 { 1456 /* ABI requires unsigned char/short to be zero-extended */ 1457 if (!sign && (size == 1 || size == 2)) { 1458 if (rd != rj) 1459 move_reg(ctx, rd, rj); 1460 return; 1461 } 1462 1463 switch (size) { 1464 case 1: 1465 emit_insn(ctx, extwb, rd, rj); 1466 break; 1467 case 2: 1468 emit_insn(ctx, extwh, rd, rj); 1469 break; 1470 case 4: 1471 emit_insn(ctx, addiw, rd, rj, 0); 1472 break; 1473 case 8: 1474 if (rd != rj) 1475 move_reg(ctx, rd, rj); 1476 break; 1477 default: 1478 pr_warn("bpf_jit: invalid size %d for sign_extend\n", size); 1479 } 1480 } 1481 1482 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 1483 const struct btf_func_model *m, struct bpf_tramp_links *tlinks, 1484 void *func_addr, u32 flags) 1485 { 1486 int i, ret, save_ret; 1487 int stack_size, nargs; 1488 int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; 1489 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 1490 void *orig_call = func_addr; 1491 struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 1492 struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 1493 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 1494 u32 **branches = NULL; 1495 1496 /* 1497 * FP + 8 [ RA to parent func ] return address to parent 1498 * function 1499 * FP + 0 [ FP of parent func ] frame pointer of parent 1500 * function 1501 * FP - 8 [ T0 to traced func ] return address of traced 1502 * function 1503 * FP - 16 [ FP of traced func ] frame pointer of traced 1504 * function 1505 * 1506 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 1507 * BPF_TRAMP_F_RET_FENTRY_RET 1508 * [ argN ] 1509 * [ ... ] 1510 * FP - args_off [ arg1 ] 1511 * 1512 * FP - nargs_off [ regs count ] 1513 * 1514 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 1515 * 1516 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 1517 * 1518 * FP - sreg_off [ callee saved reg ] 1519 * 1520 * FP - tcc_ptr_off [ tail_call_cnt_ptr ] 1521 */ 1522 1523 if (m->nr_args > LOONGARCH_MAX_REG_ARGS) 1524 return -ENOTSUPP; 1525 1526 /* FIXME: No support of struct argument */ 1527 for (i = 0; i < m->nr_args; i++) { 1528 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 1529 return -ENOTSUPP; 1530 } 1531 1532 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 1533 return -ENOTSUPP; 1534 1535 /* Room of trampoline frame to store return address and frame pointer */ 1536 stack_size = 16; 1537 1538 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 1539 if (save_ret) 1540 stack_size += 16; /* Save BPF R0 and A0 */ 1541 1542 retval_off = stack_size; 1543 1544 /* Room of trampoline frame to store args */ 1545 nargs = m->nr_args; 1546 stack_size += nargs * 8; 1547 args_off = stack_size; 1548 1549 /* Room of trampoline frame to store args number */ 1550 stack_size += 8; 1551 nargs_off = stack_size; 1552 1553 /* Room of trampoline frame to store ip address */ 1554 if (flags & BPF_TRAMP_F_IP_ARG) { 1555 stack_size += 8; 1556 ip_off = stack_size; 1557 } 1558 1559 /* Room of trampoline frame to store struct bpf_tramp_run_ctx */ 1560 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 1561 run_ctx_off = stack_size; 1562 1563 stack_size += 8; 1564 sreg_off = stack_size; 1565 1566 /* Room of trampoline frame to store tail_call_cnt_ptr */ 1567 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { 1568 stack_size += 8; 1569 tcc_ptr_off = stack_size; 1570 } 1571 1572 stack_size = round_up(stack_size, 16); 1573 1574 if (is_struct_ops) { 1575 /* 1576 * For the trampoline called directly, just handle 1577 * the frame of trampoline. 1578 */ 1579 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1580 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1581 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1582 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1583 } else { 1584 /* 1585 * For the trampoline called from function entry, 1586 * the frame of traced function and the frame of 1587 * trampoline need to be considered. 1588 */ 1589 /* RA and FP for parent function */ 1590 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); 1591 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1592 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1593 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); 1594 1595 /* RA and FP for traced function */ 1596 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1597 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1598 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1599 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1600 } 1601 1602 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1603 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1604 1605 /* callee saved register S1 to pass start time */ 1606 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1607 1608 /* store ip address of the traced function */ 1609 if (flags & BPF_TRAMP_F_IP_ARG) { 1610 move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false); 1611 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off); 1612 } 1613 1614 /* store nargs number */ 1615 move_imm(ctx, LOONGARCH_GPR_T1, nargs, false); 1616 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off); 1617 1618 store_args(ctx, nargs, args_off); 1619 1620 /* To traced function */ 1621 /* Ftrace jump skips 2 NOP instructions */ 1622 if (is_kernel_text((unsigned long)orig_call)) 1623 orig_call += LOONGARCH_FENTRY_NBYTES; 1624 /* Direct jump skips 5 NOP instructions */ 1625 else if (is_bpf_text_address((unsigned long)orig_call)) 1626 orig_call += LOONGARCH_BPF_FENTRY_NBYTES; 1627 1628 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1629 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 1630 ret = emit_call(ctx, (const u64)__bpf_tramp_enter); 1631 if (ret) 1632 return ret; 1633 } 1634 1635 for (i = 0; i < fentry->nr_links; i++) { 1636 ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off, 1637 run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); 1638 if (ret) 1639 return ret; 1640 } 1641 if (fmod_ret->nr_links) { 1642 branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL); 1643 if (!branches) 1644 return -ENOMEM; 1645 1646 invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches); 1647 } 1648 1649 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1650 restore_args(ctx, m->nr_args, args_off); 1651 1652 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1653 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1654 1655 ret = emit_call(ctx, (const u64)orig_call); 1656 if (ret) 1657 goto out; 1658 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1659 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1660 im->ip_after_call = ctx->ro_image + ctx->idx; 1661 /* Reserve space for the move_imm + jirl instruction */ 1662 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1663 emit_insn(ctx, nop); 1664 } 1665 1666 for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) { 1667 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; 1668 *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset); 1669 } 1670 1671 for (i = 0; i < fexit->nr_links; i++) { 1672 ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false); 1673 if (ret) 1674 goto out; 1675 } 1676 1677 if (flags & BPF_TRAMP_F_CALL_ORIG) { 1678 im->ip_epilogue = ctx->ro_image + ctx->idx; 1679 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 1680 ret = emit_call(ctx, (const u64)__bpf_tramp_exit); 1681 if (ret) 1682 goto out; 1683 } 1684 1685 if (flags & BPF_TRAMP_F_RESTORE_REGS) 1686 restore_args(ctx, m->nr_args, args_off); 1687 1688 if (save_ret) { 1689 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1690 if (is_struct_ops) 1691 sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 1692 m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG); 1693 else 1694 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1695 } 1696 1697 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1698 1699 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1700 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1701 1702 if (is_struct_ops) { 1703 /* trampoline called directly */ 1704 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1705 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1706 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 1707 1708 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 1709 } else { 1710 /* trampoline called from function entry */ 1711 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1712 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1713 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 1714 1715 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1716 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1717 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); 1718 1719 if (flags & BPF_TRAMP_F_SKIP_FRAME) 1720 /* return to parent function */ 1721 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 1722 else 1723 /* return to traced function */ 1724 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); 1725 } 1726 1727 ret = ctx->idx; 1728 out: 1729 kfree(branches); 1730 1731 return ret; 1732 } 1733 1734 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, 1735 void *ro_image_end, const struct btf_func_model *m, 1736 u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) 1737 { 1738 int ret, size; 1739 void *image, *tmp; 1740 struct jit_ctx ctx; 1741 1742 size = ro_image_end - ro_image; 1743 image = kvmalloc(size, GFP_KERNEL); 1744 if (!image) 1745 return -ENOMEM; 1746 1747 ctx.image = (union loongarch_instruction *)image; 1748 ctx.ro_image = (union loongarch_instruction *)ro_image; 1749 ctx.idx = 0; 1750 1751 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); 1752 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags); 1753 if (ret < 0) 1754 goto out; 1755 1756 if (validate_code(&ctx) < 0) { 1757 ret = -EINVAL; 1758 goto out; 1759 } 1760 1761 tmp = bpf_arch_text_copy(ro_image, image, size); 1762 if (IS_ERR(tmp)) { 1763 ret = PTR_ERR(tmp); 1764 goto out; 1765 } 1766 1767 out: 1768 kvfree(image); 1769 return ret < 0 ? ret : size; 1770 } 1771 1772 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 1773 struct bpf_tramp_links *tlinks, void *func_addr) 1774 { 1775 int ret; 1776 struct jit_ctx ctx; 1777 struct bpf_tramp_image im; 1778 1779 ctx.image = NULL; 1780 ctx.idx = 0; 1781 1782 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags); 1783 1784 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; 1785 } 1786 1787 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1788 { 1789 bool tmp_blinded = false, extra_pass = false; 1790 u8 *image_ptr; 1791 int image_size, prog_size, extable_size; 1792 struct jit_ctx ctx; 1793 struct jit_data *jit_data; 1794 struct bpf_binary_header *header; 1795 struct bpf_prog *tmp, *orig_prog = prog; 1796 1797 /* 1798 * If BPF JIT was not enabled then we must fall back to 1799 * the interpreter. 1800 */ 1801 if (!prog->jit_requested) 1802 return orig_prog; 1803 1804 tmp = bpf_jit_blind_constants(prog); 1805 /* 1806 * If blinding was requested and we failed during blinding, 1807 * we must fall back to the interpreter. Otherwise, we save 1808 * the new JITed code. 1809 */ 1810 if (IS_ERR(tmp)) 1811 return orig_prog; 1812 1813 if (tmp != prog) { 1814 tmp_blinded = true; 1815 prog = tmp; 1816 } 1817 1818 jit_data = prog->aux->jit_data; 1819 if (!jit_data) { 1820 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1821 if (!jit_data) { 1822 prog = orig_prog; 1823 goto out; 1824 } 1825 prog->aux->jit_data = jit_data; 1826 } 1827 if (jit_data->ctx.offset) { 1828 ctx = jit_data->ctx; 1829 image_ptr = jit_data->image; 1830 header = jit_data->header; 1831 extra_pass = true; 1832 prog_size = sizeof(u32) * ctx.idx; 1833 goto skip_init_ctx; 1834 } 1835 1836 memset(&ctx, 0, sizeof(ctx)); 1837 ctx.prog = prog; 1838 1839 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 1840 if (ctx.offset == NULL) { 1841 prog = orig_prog; 1842 goto out_offset; 1843 } 1844 1845 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 1846 build_prologue(&ctx); 1847 if (build_body(&ctx, extra_pass)) { 1848 prog = orig_prog; 1849 goto out_offset; 1850 } 1851 ctx.epilogue_offset = ctx.idx; 1852 build_epilogue(&ctx); 1853 1854 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); 1855 1856 /* Now we know the actual image size. 1857 * As each LoongArch instruction is of length 32bit, 1858 * we are translating number of JITed intructions into 1859 * the size required to store these JITed code. 1860 */ 1861 prog_size = sizeof(u32) * ctx.idx; 1862 image_size = prog_size + extable_size; 1863 /* Now we know the size of the structure to make */ 1864 header = bpf_jit_binary_alloc(image_size, &image_ptr, 1865 sizeof(u32), jit_fill_hole); 1866 if (header == NULL) { 1867 prog = orig_prog; 1868 goto out_offset; 1869 } 1870 1871 /* 2. Now, the actual pass to generate final JIT code */ 1872 ctx.image = (union loongarch_instruction *)image_ptr; 1873 if (extable_size) 1874 prog->aux->extable = (void *)image_ptr + prog_size; 1875 1876 skip_init_ctx: 1877 ctx.idx = 0; 1878 ctx.num_exentries = 0; 1879 1880 build_prologue(&ctx); 1881 if (build_body(&ctx, extra_pass)) { 1882 bpf_jit_binary_free(header); 1883 prog = orig_prog; 1884 goto out_offset; 1885 } 1886 build_epilogue(&ctx); 1887 1888 /* 3. Extra pass to validate JITed code */ 1889 if (validate_ctx(&ctx)) { 1890 bpf_jit_binary_free(header); 1891 prog = orig_prog; 1892 goto out_offset; 1893 } 1894 1895 /* And we're done */ 1896 if (bpf_jit_enable > 1) 1897 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1898 1899 /* Update the icache */ 1900 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); 1901 1902 if (!prog->is_func || extra_pass) { 1903 int err; 1904 1905 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1906 pr_err_once("multi-func JIT bug %d != %d\n", 1907 ctx.idx, jit_data->ctx.idx); 1908 goto out_free; 1909 } 1910 err = bpf_jit_binary_lock_ro(header); 1911 if (err) { 1912 pr_err_once("bpf_jit_binary_lock_ro() returned %d\n", 1913 err); 1914 goto out_free; 1915 } 1916 } else { 1917 jit_data->ctx = ctx; 1918 jit_data->image = image_ptr; 1919 jit_data->header = header; 1920 } 1921 prog->jited = 1; 1922 prog->jited_len = prog_size; 1923 prog->bpf_func = (void *)ctx.image; 1924 1925 if (!prog->is_func || extra_pass) { 1926 int i; 1927 1928 /* offset[prog->len] is the size of program */ 1929 for (i = 0; i <= prog->len; i++) 1930 ctx.offset[i] *= LOONGARCH_INSN_SIZE; 1931 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 1932 1933 out_offset: 1934 kvfree(ctx.offset); 1935 kfree(jit_data); 1936 prog->aux->jit_data = NULL; 1937 } 1938 1939 out: 1940 if (tmp_blinded) 1941 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); 1942 1943 1944 return prog; 1945 1946 out_free: 1947 bpf_jit_binary_free(header); 1948 prog->bpf_func = NULL; 1949 prog->jited = 0; 1950 prog->jited_len = 0; 1951 goto out_offset; 1952 } 1953 1954 bool bpf_jit_bypass_spec_v1(void) 1955 { 1956 return true; 1957 } 1958 1959 bool bpf_jit_bypass_spec_v4(void) 1960 { 1961 return true; 1962 } 1963 1964 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ 1965 bool bpf_jit_supports_subprog_tailcalls(void) 1966 { 1967 return true; 1968 } 1969