1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include "bpf_jit.h" 8 9 #define REG_TCC LOONGARCH_GPR_A6 10 #define TCC_SAVED LOONGARCH_GPR_S5 11 12 #define SAVE_RA BIT(0) 13 #define SAVE_TCC BIT(1) 14 15 static const int regmap[] = { 16 /* return value from in-kernel function, and exit value for eBPF program */ 17 [BPF_REG_0] = LOONGARCH_GPR_A5, 18 /* arguments from eBPF program to in-kernel function */ 19 [BPF_REG_1] = LOONGARCH_GPR_A0, 20 [BPF_REG_2] = LOONGARCH_GPR_A1, 21 [BPF_REG_3] = LOONGARCH_GPR_A2, 22 [BPF_REG_4] = LOONGARCH_GPR_A3, 23 [BPF_REG_5] = LOONGARCH_GPR_A4, 24 /* callee saved registers that in-kernel function will preserve */ 25 [BPF_REG_6] = LOONGARCH_GPR_S0, 26 [BPF_REG_7] = LOONGARCH_GPR_S1, 27 [BPF_REG_8] = LOONGARCH_GPR_S2, 28 [BPF_REG_9] = LOONGARCH_GPR_S3, 29 /* read-only frame pointer to access stack */ 30 [BPF_REG_FP] = LOONGARCH_GPR_S4, 31 /* temporary register for blinding constants */ 32 [BPF_REG_AX] = LOONGARCH_GPR_T0, 33 }; 34 35 static void mark_call(struct jit_ctx *ctx) 36 { 37 ctx->flags |= SAVE_RA; 38 } 39 40 static void mark_tail_call(struct jit_ctx *ctx) 41 { 42 ctx->flags |= SAVE_TCC; 43 } 44 45 static bool seen_call(struct jit_ctx *ctx) 46 { 47 return (ctx->flags & SAVE_RA); 48 } 49 50 static bool seen_tail_call(struct jit_ctx *ctx) 51 { 52 return (ctx->flags & SAVE_TCC); 53 } 54 55 static u8 tail_call_reg(struct jit_ctx *ctx) 56 { 57 if (seen_call(ctx)) 58 return TCC_SAVED; 59 60 return REG_TCC; 61 } 62 63 /* 64 * eBPF prog stack layout: 65 * 66 * high 67 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP 68 * | $ra | 69 * +-------------------------+ 70 * | $fp | 71 * +-------------------------+ 72 * | $s0 | 73 * +-------------------------+ 74 * | $s1 | 75 * +-------------------------+ 76 * | $s2 | 77 * +-------------------------+ 78 * | $s3 | 79 * +-------------------------+ 80 * | $s4 | 81 * +-------------------------+ 82 * | $s5 | 83 * +-------------------------+ <--BPF_REG_FP 84 * | prog->aux->stack_depth | 85 * | (optional) | 86 * current $sp -------------> +-------------------------+ 87 * low 88 */ 89 static void build_prologue(struct jit_ctx *ctx) 90 { 91 int stack_adjust = 0, store_offset, bpf_stack_adjust; 92 93 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 94 95 /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */ 96 stack_adjust += sizeof(long) * 8; 97 98 stack_adjust = round_up(stack_adjust, 16); 99 stack_adjust += bpf_stack_adjust; 100 101 /* 102 * First instruction initializes the tail call count (TCC). 103 * On tail call we skip this instruction, and the TCC is 104 * passed in REG_TCC from the caller. 105 */ 106 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 107 108 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); 109 110 store_offset = stack_adjust - sizeof(long); 111 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); 112 113 store_offset -= sizeof(long); 114 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); 115 116 store_offset -= sizeof(long); 117 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); 118 119 store_offset -= sizeof(long); 120 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); 121 122 store_offset -= sizeof(long); 123 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); 124 125 store_offset -= sizeof(long); 126 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); 127 128 store_offset -= sizeof(long); 129 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); 130 131 store_offset -= sizeof(long); 132 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 133 134 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); 135 136 if (bpf_stack_adjust) 137 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 138 139 /* 140 * Program contains calls and tail calls, so REG_TCC need 141 * to be saved across calls. 142 */ 143 if (seen_tail_call(ctx) && seen_call(ctx)) 144 move_reg(ctx, TCC_SAVED, REG_TCC); 145 else 146 emit_insn(ctx, nop); 147 148 ctx->stack_size = stack_adjust; 149 } 150 151 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) 152 { 153 int stack_adjust = ctx->stack_size; 154 int load_offset; 155 156 load_offset = stack_adjust - sizeof(long); 157 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); 158 159 load_offset -= sizeof(long); 160 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); 161 162 load_offset -= sizeof(long); 163 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); 164 165 load_offset -= sizeof(long); 166 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); 167 168 load_offset -= sizeof(long); 169 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); 170 171 load_offset -= sizeof(long); 172 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); 173 174 load_offset -= sizeof(long); 175 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); 176 177 load_offset -= sizeof(long); 178 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 179 180 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); 181 182 if (!is_tail_call) { 183 /* Set return value */ 184 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); 185 /* Return to the caller */ 186 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 187 } else { 188 /* 189 * Call the next bpf prog and skip the first instruction 190 * of TCC initialization. 191 */ 192 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 1); 193 } 194 } 195 196 static void build_epilogue(struct jit_ctx *ctx) 197 { 198 __build_epilogue(ctx, false); 199 } 200 201 bool bpf_jit_supports_kfunc_call(void) 202 { 203 return true; 204 } 205 206 bool bpf_jit_supports_far_kfunc_call(void) 207 { 208 return true; 209 } 210 211 /* initialized on the first pass of build_body() */ 212 static int out_offset = -1; 213 static int emit_bpf_tail_call(struct jit_ctx *ctx) 214 { 215 int off; 216 u8 tcc = tail_call_reg(ctx); 217 u8 a1 = LOONGARCH_GPR_A1; 218 u8 a2 = LOONGARCH_GPR_A2; 219 u8 t1 = LOONGARCH_GPR_T1; 220 u8 t2 = LOONGARCH_GPR_T2; 221 u8 t3 = LOONGARCH_GPR_T3; 222 const int idx0 = ctx->idx; 223 224 #define cur_offset (ctx->idx - idx0) 225 #define jmp_offset (out_offset - (cur_offset)) 226 227 /* 228 * a0: &ctx 229 * a1: &array 230 * a2: index 231 * 232 * if (index >= array->map.max_entries) 233 * goto out; 234 */ 235 off = offsetof(struct bpf_array, map.max_entries); 236 emit_insn(ctx, ldwu, t1, a1, off); 237 /* bgeu $a2, $t1, jmp_offset */ 238 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) 239 goto toofar; 240 241 /* 242 * if (--TCC < 0) 243 * goto out; 244 */ 245 emit_insn(ctx, addid, REG_TCC, tcc, -1); 246 if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 247 goto toofar; 248 249 /* 250 * prog = array->ptrs[index]; 251 * if (!prog) 252 * goto out; 253 */ 254 emit_insn(ctx, alsld, t2, a2, a1, 2); 255 off = offsetof(struct bpf_array, ptrs); 256 emit_insn(ctx, ldd, t2, t2, off); 257 /* beq $t2, $zero, jmp_offset */ 258 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 259 goto toofar; 260 261 /* goto *(prog->bpf_func + 4); */ 262 off = offsetof(struct bpf_prog, bpf_func); 263 emit_insn(ctx, ldd, t3, t2, off); 264 __build_epilogue(ctx, true); 265 266 /* out: */ 267 if (out_offset == -1) 268 out_offset = cur_offset; 269 if (cur_offset != out_offset) { 270 pr_err_once("tail_call out_offset = %d, expected %d!\n", 271 cur_offset, out_offset); 272 return -1; 273 } 274 275 return 0; 276 277 toofar: 278 pr_info_once("tail_call: jump too far\n"); 279 return -1; 280 #undef cur_offset 281 #undef jmp_offset 282 } 283 284 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) 285 { 286 const u8 t1 = LOONGARCH_GPR_T1; 287 const u8 t2 = LOONGARCH_GPR_T2; 288 const u8 t3 = LOONGARCH_GPR_T3; 289 const u8 r0 = regmap[BPF_REG_0]; 290 const u8 src = regmap[insn->src_reg]; 291 const u8 dst = regmap[insn->dst_reg]; 292 const s16 off = insn->off; 293 const s32 imm = insn->imm; 294 const bool isdw = BPF_SIZE(insn->code) == BPF_DW; 295 296 move_imm(ctx, t1, off, false); 297 emit_insn(ctx, addd, t1, dst, t1); 298 move_reg(ctx, t3, src); 299 300 switch (imm) { 301 /* lock *(size *)(dst + off) <op>= src */ 302 case BPF_ADD: 303 if (isdw) 304 emit_insn(ctx, amaddd, t2, t1, src); 305 else 306 emit_insn(ctx, amaddw, t2, t1, src); 307 break; 308 case BPF_AND: 309 if (isdw) 310 emit_insn(ctx, amandd, t2, t1, src); 311 else 312 emit_insn(ctx, amandw, t2, t1, src); 313 break; 314 case BPF_OR: 315 if (isdw) 316 emit_insn(ctx, amord, t2, t1, src); 317 else 318 emit_insn(ctx, amorw, t2, t1, src); 319 break; 320 case BPF_XOR: 321 if (isdw) 322 emit_insn(ctx, amxord, t2, t1, src); 323 else 324 emit_insn(ctx, amxorw, t2, t1, src); 325 break; 326 /* src = atomic_fetch_<op>(dst + off, src) */ 327 case BPF_ADD | BPF_FETCH: 328 if (isdw) { 329 emit_insn(ctx, amaddd, src, t1, t3); 330 } else { 331 emit_insn(ctx, amaddw, src, t1, t3); 332 emit_zext_32(ctx, src, true); 333 } 334 break; 335 case BPF_AND | BPF_FETCH: 336 if (isdw) { 337 emit_insn(ctx, amandd, src, t1, t3); 338 } else { 339 emit_insn(ctx, amandw, src, t1, t3); 340 emit_zext_32(ctx, src, true); 341 } 342 break; 343 case BPF_OR | BPF_FETCH: 344 if (isdw) { 345 emit_insn(ctx, amord, src, t1, t3); 346 } else { 347 emit_insn(ctx, amorw, src, t1, t3); 348 emit_zext_32(ctx, src, true); 349 } 350 break; 351 case BPF_XOR | BPF_FETCH: 352 if (isdw) { 353 emit_insn(ctx, amxord, src, t1, t3); 354 } else { 355 emit_insn(ctx, amxorw, src, t1, t3); 356 emit_zext_32(ctx, src, true); 357 } 358 break; 359 /* src = atomic_xchg(dst + off, src); */ 360 case BPF_XCHG: 361 if (isdw) { 362 emit_insn(ctx, amswapd, src, t1, t3); 363 } else { 364 emit_insn(ctx, amswapw, src, t1, t3); 365 emit_zext_32(ctx, src, true); 366 } 367 break; 368 /* r0 = atomic_cmpxchg(dst + off, r0, src); */ 369 case BPF_CMPXCHG: 370 move_reg(ctx, t2, r0); 371 if (isdw) { 372 emit_insn(ctx, lld, r0, t1, 0); 373 emit_insn(ctx, bne, t2, r0, 4); 374 move_reg(ctx, t3, src); 375 emit_insn(ctx, scd, t3, t1, 0); 376 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); 377 } else { 378 emit_insn(ctx, llw, r0, t1, 0); 379 emit_zext_32(ctx, t2, true); 380 emit_zext_32(ctx, r0, true); 381 emit_insn(ctx, bne, t2, r0, 4); 382 move_reg(ctx, t3, src); 383 emit_insn(ctx, scw, t3, t1, 0); 384 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); 385 emit_zext_32(ctx, r0, true); 386 } 387 break; 388 } 389 } 390 391 static bool is_signed_bpf_cond(u8 cond) 392 { 393 return cond == BPF_JSGT || cond == BPF_JSLT || 394 cond == BPF_JSGE || cond == BPF_JSLE; 395 } 396 397 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 398 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 399 400 bool ex_handler_bpf(const struct exception_table_entry *ex, 401 struct pt_regs *regs) 402 { 403 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 404 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 405 406 regs->regs[dst_reg] = 0; 407 regs->csr_era = (unsigned long)&ex->fixup - offset; 408 409 return true; 410 } 411 412 /* For accesses to BTF pointers, add an entry to the exception table */ 413 static int add_exception_handler(const struct bpf_insn *insn, 414 struct jit_ctx *ctx, 415 int dst_reg) 416 { 417 unsigned long pc; 418 off_t offset; 419 struct exception_table_entry *ex; 420 421 if (!ctx->image || !ctx->prog->aux->extable) 422 return 0; 423 424 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 425 BPF_MODE(insn->code) != BPF_PROBE_MEMSX) 426 return 0; 427 428 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) 429 return -EINVAL; 430 431 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 432 pc = (unsigned long)&ctx->image[ctx->idx - 1]; 433 434 offset = pc - (long)&ex->insn; 435 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 436 return -ERANGE; 437 438 ex->insn = offset; 439 440 /* 441 * Since the extable follows the program, the fixup offset is always 442 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 443 * to keep things simple, and put the destination register in the upper 444 * bits. We don't need to worry about buildtime or runtime sort 445 * modifying the upper bits because the table is already sorted, and 446 * isn't part of the main exception table. 447 */ 448 offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 449 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 450 return -ERANGE; 451 452 ex->type = EX_TYPE_BPF; 453 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 454 455 ctx->num_exentries++; 456 457 return 0; 458 } 459 460 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) 461 { 462 u8 tm = -1; 463 u64 func_addr; 464 bool func_addr_fixed, sign_extend; 465 int i = insn - ctx->prog->insnsi; 466 int ret, jmp_offset; 467 const u8 code = insn->code; 468 const u8 cond = BPF_OP(code); 469 const u8 t1 = LOONGARCH_GPR_T1; 470 const u8 t2 = LOONGARCH_GPR_T2; 471 const u8 src = regmap[insn->src_reg]; 472 const u8 dst = regmap[insn->dst_reg]; 473 const s16 off = insn->off; 474 const s32 imm = insn->imm; 475 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; 476 477 switch (code) { 478 /* dst = src */ 479 case BPF_ALU | BPF_MOV | BPF_X: 480 case BPF_ALU64 | BPF_MOV | BPF_X: 481 switch (off) { 482 case 0: 483 move_reg(ctx, dst, src); 484 emit_zext_32(ctx, dst, is32); 485 break; 486 case 8: 487 move_reg(ctx, t1, src); 488 emit_insn(ctx, extwb, dst, t1); 489 emit_zext_32(ctx, dst, is32); 490 break; 491 case 16: 492 move_reg(ctx, t1, src); 493 emit_insn(ctx, extwh, dst, t1); 494 emit_zext_32(ctx, dst, is32); 495 break; 496 case 32: 497 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); 498 break; 499 } 500 break; 501 502 /* dst = imm */ 503 case BPF_ALU | BPF_MOV | BPF_K: 504 case BPF_ALU64 | BPF_MOV | BPF_K: 505 move_imm(ctx, dst, imm, is32); 506 break; 507 508 /* dst = dst + src */ 509 case BPF_ALU | BPF_ADD | BPF_X: 510 case BPF_ALU64 | BPF_ADD | BPF_X: 511 emit_insn(ctx, addd, dst, dst, src); 512 emit_zext_32(ctx, dst, is32); 513 break; 514 515 /* dst = dst + imm */ 516 case BPF_ALU | BPF_ADD | BPF_K: 517 case BPF_ALU64 | BPF_ADD | BPF_K: 518 if (is_signed_imm12(imm)) { 519 emit_insn(ctx, addid, dst, dst, imm); 520 } else { 521 move_imm(ctx, t1, imm, is32); 522 emit_insn(ctx, addd, dst, dst, t1); 523 } 524 emit_zext_32(ctx, dst, is32); 525 break; 526 527 /* dst = dst - src */ 528 case BPF_ALU | BPF_SUB | BPF_X: 529 case BPF_ALU64 | BPF_SUB | BPF_X: 530 emit_insn(ctx, subd, dst, dst, src); 531 emit_zext_32(ctx, dst, is32); 532 break; 533 534 /* dst = dst - imm */ 535 case BPF_ALU | BPF_SUB | BPF_K: 536 case BPF_ALU64 | BPF_SUB | BPF_K: 537 if (is_signed_imm12(-imm)) { 538 emit_insn(ctx, addid, dst, dst, -imm); 539 } else { 540 move_imm(ctx, t1, imm, is32); 541 emit_insn(ctx, subd, dst, dst, t1); 542 } 543 emit_zext_32(ctx, dst, is32); 544 break; 545 546 /* dst = dst * src */ 547 case BPF_ALU | BPF_MUL | BPF_X: 548 case BPF_ALU64 | BPF_MUL | BPF_X: 549 emit_insn(ctx, muld, dst, dst, src); 550 emit_zext_32(ctx, dst, is32); 551 break; 552 553 /* dst = dst * imm */ 554 case BPF_ALU | BPF_MUL | BPF_K: 555 case BPF_ALU64 | BPF_MUL | BPF_K: 556 move_imm(ctx, t1, imm, is32); 557 emit_insn(ctx, muld, dst, dst, t1); 558 emit_zext_32(ctx, dst, is32); 559 break; 560 561 /* dst = dst / src */ 562 case BPF_ALU | BPF_DIV | BPF_X: 563 case BPF_ALU64 | BPF_DIV | BPF_X: 564 if (!off) { 565 emit_zext_32(ctx, dst, is32); 566 move_reg(ctx, t1, src); 567 emit_zext_32(ctx, t1, is32); 568 emit_insn(ctx, divdu, dst, dst, t1); 569 emit_zext_32(ctx, dst, is32); 570 } else { 571 emit_sext_32(ctx, dst, is32); 572 move_reg(ctx, t1, src); 573 emit_sext_32(ctx, t1, is32); 574 emit_insn(ctx, divd, dst, dst, t1); 575 emit_sext_32(ctx, dst, is32); 576 } 577 break; 578 579 /* dst = dst / imm */ 580 case BPF_ALU | BPF_DIV | BPF_K: 581 case BPF_ALU64 | BPF_DIV | BPF_K: 582 if (!off) { 583 move_imm(ctx, t1, imm, is32); 584 emit_zext_32(ctx, dst, is32); 585 emit_insn(ctx, divdu, dst, dst, t1); 586 emit_zext_32(ctx, dst, is32); 587 } else { 588 move_imm(ctx, t1, imm, false); 589 emit_sext_32(ctx, t1, is32); 590 emit_sext_32(ctx, dst, is32); 591 emit_insn(ctx, divd, dst, dst, t1); 592 emit_sext_32(ctx, dst, is32); 593 } 594 break; 595 596 /* dst = dst % src */ 597 case BPF_ALU | BPF_MOD | BPF_X: 598 case BPF_ALU64 | BPF_MOD | BPF_X: 599 if (!off) { 600 emit_zext_32(ctx, dst, is32); 601 move_reg(ctx, t1, src); 602 emit_zext_32(ctx, t1, is32); 603 emit_insn(ctx, moddu, dst, dst, t1); 604 emit_zext_32(ctx, dst, is32); 605 } else { 606 emit_sext_32(ctx, dst, is32); 607 move_reg(ctx, t1, src); 608 emit_sext_32(ctx, t1, is32); 609 emit_insn(ctx, modd, dst, dst, t1); 610 emit_sext_32(ctx, dst, is32); 611 } 612 break; 613 614 /* dst = dst % imm */ 615 case BPF_ALU | BPF_MOD | BPF_K: 616 case BPF_ALU64 | BPF_MOD | BPF_K: 617 if (!off) { 618 move_imm(ctx, t1, imm, is32); 619 emit_zext_32(ctx, dst, is32); 620 emit_insn(ctx, moddu, dst, dst, t1); 621 emit_zext_32(ctx, dst, is32); 622 } else { 623 move_imm(ctx, t1, imm, false); 624 emit_sext_32(ctx, t1, is32); 625 emit_sext_32(ctx, dst, is32); 626 emit_insn(ctx, modd, dst, dst, t1); 627 emit_sext_32(ctx, dst, is32); 628 } 629 break; 630 631 /* dst = -dst */ 632 case BPF_ALU | BPF_NEG: 633 case BPF_ALU64 | BPF_NEG: 634 move_imm(ctx, t1, imm, is32); 635 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); 636 emit_zext_32(ctx, dst, is32); 637 break; 638 639 /* dst = dst & src */ 640 case BPF_ALU | BPF_AND | BPF_X: 641 case BPF_ALU64 | BPF_AND | BPF_X: 642 emit_insn(ctx, and, dst, dst, src); 643 emit_zext_32(ctx, dst, is32); 644 break; 645 646 /* dst = dst & imm */ 647 case BPF_ALU | BPF_AND | BPF_K: 648 case BPF_ALU64 | BPF_AND | BPF_K: 649 if (is_unsigned_imm12(imm)) { 650 emit_insn(ctx, andi, dst, dst, imm); 651 } else { 652 move_imm(ctx, t1, imm, is32); 653 emit_insn(ctx, and, dst, dst, t1); 654 } 655 emit_zext_32(ctx, dst, is32); 656 break; 657 658 /* dst = dst | src */ 659 case BPF_ALU | BPF_OR | BPF_X: 660 case BPF_ALU64 | BPF_OR | BPF_X: 661 emit_insn(ctx, or, dst, dst, src); 662 emit_zext_32(ctx, dst, is32); 663 break; 664 665 /* dst = dst | imm */ 666 case BPF_ALU | BPF_OR | BPF_K: 667 case BPF_ALU64 | BPF_OR | BPF_K: 668 if (is_unsigned_imm12(imm)) { 669 emit_insn(ctx, ori, dst, dst, imm); 670 } else { 671 move_imm(ctx, t1, imm, is32); 672 emit_insn(ctx, or, dst, dst, t1); 673 } 674 emit_zext_32(ctx, dst, is32); 675 break; 676 677 /* dst = dst ^ src */ 678 case BPF_ALU | BPF_XOR | BPF_X: 679 case BPF_ALU64 | BPF_XOR | BPF_X: 680 emit_insn(ctx, xor, dst, dst, src); 681 emit_zext_32(ctx, dst, is32); 682 break; 683 684 /* dst = dst ^ imm */ 685 case BPF_ALU | BPF_XOR | BPF_K: 686 case BPF_ALU64 | BPF_XOR | BPF_K: 687 if (is_unsigned_imm12(imm)) { 688 emit_insn(ctx, xori, dst, dst, imm); 689 } else { 690 move_imm(ctx, t1, imm, is32); 691 emit_insn(ctx, xor, dst, dst, t1); 692 } 693 emit_zext_32(ctx, dst, is32); 694 break; 695 696 /* dst = dst << src (logical) */ 697 case BPF_ALU | BPF_LSH | BPF_X: 698 emit_insn(ctx, sllw, dst, dst, src); 699 emit_zext_32(ctx, dst, is32); 700 break; 701 702 case BPF_ALU64 | BPF_LSH | BPF_X: 703 emit_insn(ctx, slld, dst, dst, src); 704 break; 705 706 /* dst = dst << imm (logical) */ 707 case BPF_ALU | BPF_LSH | BPF_K: 708 emit_insn(ctx, slliw, dst, dst, imm); 709 emit_zext_32(ctx, dst, is32); 710 break; 711 712 case BPF_ALU64 | BPF_LSH | BPF_K: 713 emit_insn(ctx, sllid, dst, dst, imm); 714 break; 715 716 /* dst = dst >> src (logical) */ 717 case BPF_ALU | BPF_RSH | BPF_X: 718 emit_insn(ctx, srlw, dst, dst, src); 719 emit_zext_32(ctx, dst, is32); 720 break; 721 722 case BPF_ALU64 | BPF_RSH | BPF_X: 723 emit_insn(ctx, srld, dst, dst, src); 724 break; 725 726 /* dst = dst >> imm (logical) */ 727 case BPF_ALU | BPF_RSH | BPF_K: 728 emit_insn(ctx, srliw, dst, dst, imm); 729 emit_zext_32(ctx, dst, is32); 730 break; 731 732 case BPF_ALU64 | BPF_RSH | BPF_K: 733 emit_insn(ctx, srlid, dst, dst, imm); 734 break; 735 736 /* dst = dst >> src (arithmetic) */ 737 case BPF_ALU | BPF_ARSH | BPF_X: 738 emit_insn(ctx, sraw, dst, dst, src); 739 emit_zext_32(ctx, dst, is32); 740 break; 741 742 case BPF_ALU64 | BPF_ARSH | BPF_X: 743 emit_insn(ctx, srad, dst, dst, src); 744 break; 745 746 /* dst = dst >> imm (arithmetic) */ 747 case BPF_ALU | BPF_ARSH | BPF_K: 748 emit_insn(ctx, sraiw, dst, dst, imm); 749 emit_zext_32(ctx, dst, is32); 750 break; 751 752 case BPF_ALU64 | BPF_ARSH | BPF_K: 753 emit_insn(ctx, sraid, dst, dst, imm); 754 break; 755 756 /* dst = BSWAP##imm(dst) */ 757 case BPF_ALU | BPF_END | BPF_FROM_LE: 758 switch (imm) { 759 case 16: 760 /* zero-extend 16 bits into 64 bits */ 761 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 762 break; 763 case 32: 764 /* zero-extend 32 bits into 64 bits */ 765 emit_zext_32(ctx, dst, is32); 766 break; 767 case 64: 768 /* do nothing */ 769 break; 770 } 771 break; 772 773 case BPF_ALU | BPF_END | BPF_FROM_BE: 774 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 775 switch (imm) { 776 case 16: 777 emit_insn(ctx, revb2h, dst, dst); 778 /* zero-extend 16 bits into 64 bits */ 779 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 780 break; 781 case 32: 782 emit_insn(ctx, revb2w, dst, dst); 783 /* clear the upper 32 bits */ 784 emit_zext_32(ctx, dst, true); 785 break; 786 case 64: 787 emit_insn(ctx, revbd, dst, dst); 788 break; 789 } 790 break; 791 792 /* PC += off if dst cond src */ 793 case BPF_JMP | BPF_JEQ | BPF_X: 794 case BPF_JMP | BPF_JNE | BPF_X: 795 case BPF_JMP | BPF_JGT | BPF_X: 796 case BPF_JMP | BPF_JGE | BPF_X: 797 case BPF_JMP | BPF_JLT | BPF_X: 798 case BPF_JMP | BPF_JLE | BPF_X: 799 case BPF_JMP | BPF_JSGT | BPF_X: 800 case BPF_JMP | BPF_JSGE | BPF_X: 801 case BPF_JMP | BPF_JSLT | BPF_X: 802 case BPF_JMP | BPF_JSLE | BPF_X: 803 case BPF_JMP32 | BPF_JEQ | BPF_X: 804 case BPF_JMP32 | BPF_JNE | BPF_X: 805 case BPF_JMP32 | BPF_JGT | BPF_X: 806 case BPF_JMP32 | BPF_JGE | BPF_X: 807 case BPF_JMP32 | BPF_JLT | BPF_X: 808 case BPF_JMP32 | BPF_JLE | BPF_X: 809 case BPF_JMP32 | BPF_JSGT | BPF_X: 810 case BPF_JMP32 | BPF_JSGE | BPF_X: 811 case BPF_JMP32 | BPF_JSLT | BPF_X: 812 case BPF_JMP32 | BPF_JSLE | BPF_X: 813 jmp_offset = bpf2la_offset(i, off, ctx); 814 move_reg(ctx, t1, dst); 815 move_reg(ctx, t2, src); 816 if (is_signed_bpf_cond(BPF_OP(code))) { 817 emit_sext_32(ctx, t1, is32); 818 emit_sext_32(ctx, t2, is32); 819 } else { 820 emit_zext_32(ctx, t1, is32); 821 emit_zext_32(ctx, t2, is32); 822 } 823 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) 824 goto toofar; 825 break; 826 827 /* PC += off if dst cond imm */ 828 case BPF_JMP | BPF_JEQ | BPF_K: 829 case BPF_JMP | BPF_JNE | BPF_K: 830 case BPF_JMP | BPF_JGT | BPF_K: 831 case BPF_JMP | BPF_JGE | BPF_K: 832 case BPF_JMP | BPF_JLT | BPF_K: 833 case BPF_JMP | BPF_JLE | BPF_K: 834 case BPF_JMP | BPF_JSGT | BPF_K: 835 case BPF_JMP | BPF_JSGE | BPF_K: 836 case BPF_JMP | BPF_JSLT | BPF_K: 837 case BPF_JMP | BPF_JSLE | BPF_K: 838 case BPF_JMP32 | BPF_JEQ | BPF_K: 839 case BPF_JMP32 | BPF_JNE | BPF_K: 840 case BPF_JMP32 | BPF_JGT | BPF_K: 841 case BPF_JMP32 | BPF_JGE | BPF_K: 842 case BPF_JMP32 | BPF_JLT | BPF_K: 843 case BPF_JMP32 | BPF_JLE | BPF_K: 844 case BPF_JMP32 | BPF_JSGT | BPF_K: 845 case BPF_JMP32 | BPF_JSGE | BPF_K: 846 case BPF_JMP32 | BPF_JSLT | BPF_K: 847 case BPF_JMP32 | BPF_JSLE | BPF_K: 848 jmp_offset = bpf2la_offset(i, off, ctx); 849 if (imm) { 850 move_imm(ctx, t1, imm, false); 851 tm = t1; 852 } else { 853 /* If imm is 0, simply use zero register. */ 854 tm = LOONGARCH_GPR_ZERO; 855 } 856 move_reg(ctx, t2, dst); 857 if (is_signed_bpf_cond(BPF_OP(code))) { 858 emit_sext_32(ctx, tm, is32); 859 emit_sext_32(ctx, t2, is32); 860 } else { 861 emit_zext_32(ctx, tm, is32); 862 emit_zext_32(ctx, t2, is32); 863 } 864 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) 865 goto toofar; 866 break; 867 868 /* PC += off if dst & src */ 869 case BPF_JMP | BPF_JSET | BPF_X: 870 case BPF_JMP32 | BPF_JSET | BPF_X: 871 jmp_offset = bpf2la_offset(i, off, ctx); 872 emit_insn(ctx, and, t1, dst, src); 873 emit_zext_32(ctx, t1, is32); 874 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 875 goto toofar; 876 break; 877 878 /* PC += off if dst & imm */ 879 case BPF_JMP | BPF_JSET | BPF_K: 880 case BPF_JMP32 | BPF_JSET | BPF_K: 881 jmp_offset = bpf2la_offset(i, off, ctx); 882 move_imm(ctx, t1, imm, is32); 883 emit_insn(ctx, and, t1, dst, t1); 884 emit_zext_32(ctx, t1, is32); 885 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 886 goto toofar; 887 break; 888 889 /* PC += off */ 890 case BPF_JMP | BPF_JA: 891 case BPF_JMP32 | BPF_JA: 892 if (BPF_CLASS(code) == BPF_JMP) 893 jmp_offset = bpf2la_offset(i, off, ctx); 894 else 895 jmp_offset = bpf2la_offset(i, imm, ctx); 896 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 897 goto toofar; 898 break; 899 900 /* function call */ 901 case BPF_JMP | BPF_CALL: 902 mark_call(ctx); 903 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 904 &func_addr, &func_addr_fixed); 905 if (ret < 0) 906 return ret; 907 908 move_addr(ctx, t1, func_addr); 909 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); 910 911 if (insn->src_reg != BPF_PSEUDO_CALL) 912 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); 913 914 break; 915 916 /* tail call */ 917 case BPF_JMP | BPF_TAIL_CALL: 918 mark_tail_call(ctx); 919 if (emit_bpf_tail_call(ctx) < 0) 920 return -EINVAL; 921 break; 922 923 /* function return */ 924 case BPF_JMP | BPF_EXIT: 925 if (i == ctx->prog->len - 1) 926 break; 927 928 jmp_offset = epilogue_offset(ctx); 929 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 930 goto toofar; 931 break; 932 933 /* dst = imm64 */ 934 case BPF_LD | BPF_IMM | BPF_DW: 935 { 936 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; 937 938 if (bpf_pseudo_func(insn)) 939 move_addr(ctx, dst, imm64); 940 else 941 move_imm(ctx, dst, imm64, is32); 942 return 1; 943 } 944 945 /* dst = *(size *)(src + off) */ 946 case BPF_LDX | BPF_MEM | BPF_B: 947 case BPF_LDX | BPF_MEM | BPF_H: 948 case BPF_LDX | BPF_MEM | BPF_W: 949 case BPF_LDX | BPF_MEM | BPF_DW: 950 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 951 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 952 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 953 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 954 /* dst_reg = (s64)*(signed size *)(src_reg + off) */ 955 case BPF_LDX | BPF_MEMSX | BPF_B: 956 case BPF_LDX | BPF_MEMSX | BPF_H: 957 case BPF_LDX | BPF_MEMSX | BPF_W: 958 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 959 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 960 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 961 sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || 962 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 963 switch (BPF_SIZE(code)) { 964 case BPF_B: 965 if (is_signed_imm12(off)) { 966 if (sign_extend) 967 emit_insn(ctx, ldb, dst, src, off); 968 else 969 emit_insn(ctx, ldbu, dst, src, off); 970 } else { 971 move_imm(ctx, t1, off, is32); 972 if (sign_extend) 973 emit_insn(ctx, ldxb, dst, src, t1); 974 else 975 emit_insn(ctx, ldxbu, dst, src, t1); 976 } 977 break; 978 case BPF_H: 979 if (is_signed_imm12(off)) { 980 if (sign_extend) 981 emit_insn(ctx, ldh, dst, src, off); 982 else 983 emit_insn(ctx, ldhu, dst, src, off); 984 } else { 985 move_imm(ctx, t1, off, is32); 986 if (sign_extend) 987 emit_insn(ctx, ldxh, dst, src, t1); 988 else 989 emit_insn(ctx, ldxhu, dst, src, t1); 990 } 991 break; 992 case BPF_W: 993 if (is_signed_imm12(off)) { 994 if (sign_extend) 995 emit_insn(ctx, ldw, dst, src, off); 996 else 997 emit_insn(ctx, ldwu, dst, src, off); 998 } else { 999 move_imm(ctx, t1, off, is32); 1000 if (sign_extend) 1001 emit_insn(ctx, ldxw, dst, src, t1); 1002 else 1003 emit_insn(ctx, ldxwu, dst, src, t1); 1004 } 1005 break; 1006 case BPF_DW: 1007 move_imm(ctx, t1, off, is32); 1008 emit_insn(ctx, ldxd, dst, src, t1); 1009 break; 1010 } 1011 1012 ret = add_exception_handler(insn, ctx, dst); 1013 if (ret) 1014 return ret; 1015 break; 1016 1017 /* *(size *)(dst + off) = imm */ 1018 case BPF_ST | BPF_MEM | BPF_B: 1019 case BPF_ST | BPF_MEM | BPF_H: 1020 case BPF_ST | BPF_MEM | BPF_W: 1021 case BPF_ST | BPF_MEM | BPF_DW: 1022 switch (BPF_SIZE(code)) { 1023 case BPF_B: 1024 move_imm(ctx, t1, imm, is32); 1025 if (is_signed_imm12(off)) { 1026 emit_insn(ctx, stb, t1, dst, off); 1027 } else { 1028 move_imm(ctx, t2, off, is32); 1029 emit_insn(ctx, stxb, t1, dst, t2); 1030 } 1031 break; 1032 case BPF_H: 1033 move_imm(ctx, t1, imm, is32); 1034 if (is_signed_imm12(off)) { 1035 emit_insn(ctx, sth, t1, dst, off); 1036 } else { 1037 move_imm(ctx, t2, off, is32); 1038 emit_insn(ctx, stxh, t1, dst, t2); 1039 } 1040 break; 1041 case BPF_W: 1042 move_imm(ctx, t1, imm, is32); 1043 if (is_signed_imm12(off)) { 1044 emit_insn(ctx, stw, t1, dst, off); 1045 } else if (is_signed_imm14(off)) { 1046 emit_insn(ctx, stptrw, t1, dst, off); 1047 } else { 1048 move_imm(ctx, t2, off, is32); 1049 emit_insn(ctx, stxw, t1, dst, t2); 1050 } 1051 break; 1052 case BPF_DW: 1053 move_imm(ctx, t1, imm, is32); 1054 if (is_signed_imm12(off)) { 1055 emit_insn(ctx, std, t1, dst, off); 1056 } else if (is_signed_imm14(off)) { 1057 emit_insn(ctx, stptrd, t1, dst, off); 1058 } else { 1059 move_imm(ctx, t2, off, is32); 1060 emit_insn(ctx, stxd, t1, dst, t2); 1061 } 1062 break; 1063 } 1064 break; 1065 1066 /* *(size *)(dst + off) = src */ 1067 case BPF_STX | BPF_MEM | BPF_B: 1068 case BPF_STX | BPF_MEM | BPF_H: 1069 case BPF_STX | BPF_MEM | BPF_W: 1070 case BPF_STX | BPF_MEM | BPF_DW: 1071 switch (BPF_SIZE(code)) { 1072 case BPF_B: 1073 if (is_signed_imm12(off)) { 1074 emit_insn(ctx, stb, src, dst, off); 1075 } else { 1076 move_imm(ctx, t1, off, is32); 1077 emit_insn(ctx, stxb, src, dst, t1); 1078 } 1079 break; 1080 case BPF_H: 1081 if (is_signed_imm12(off)) { 1082 emit_insn(ctx, sth, src, dst, off); 1083 } else { 1084 move_imm(ctx, t1, off, is32); 1085 emit_insn(ctx, stxh, src, dst, t1); 1086 } 1087 break; 1088 case BPF_W: 1089 if (is_signed_imm12(off)) { 1090 emit_insn(ctx, stw, src, dst, off); 1091 } else if (is_signed_imm14(off)) { 1092 emit_insn(ctx, stptrw, src, dst, off); 1093 } else { 1094 move_imm(ctx, t1, off, is32); 1095 emit_insn(ctx, stxw, src, dst, t1); 1096 } 1097 break; 1098 case BPF_DW: 1099 if (is_signed_imm12(off)) { 1100 emit_insn(ctx, std, src, dst, off); 1101 } else if (is_signed_imm14(off)) { 1102 emit_insn(ctx, stptrd, src, dst, off); 1103 } else { 1104 move_imm(ctx, t1, off, is32); 1105 emit_insn(ctx, stxd, src, dst, t1); 1106 } 1107 break; 1108 } 1109 break; 1110 1111 case BPF_STX | BPF_ATOMIC | BPF_W: 1112 case BPF_STX | BPF_ATOMIC | BPF_DW: 1113 emit_atomic(insn, ctx); 1114 break; 1115 1116 /* Speculation barrier */ 1117 case BPF_ST | BPF_NOSPEC: 1118 break; 1119 1120 default: 1121 pr_err("bpf_jit: unknown opcode %02x\n", code); 1122 return -EINVAL; 1123 } 1124 1125 return 0; 1126 1127 toofar: 1128 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); 1129 return -E2BIG; 1130 } 1131 1132 static int build_body(struct jit_ctx *ctx, bool extra_pass) 1133 { 1134 int i; 1135 const struct bpf_prog *prog = ctx->prog; 1136 1137 for (i = 0; i < prog->len; i++) { 1138 const struct bpf_insn *insn = &prog->insnsi[i]; 1139 int ret; 1140 1141 if (ctx->image == NULL) 1142 ctx->offset[i] = ctx->idx; 1143 1144 ret = build_insn(insn, ctx, extra_pass); 1145 if (ret > 0) { 1146 i++; 1147 if (ctx->image == NULL) 1148 ctx->offset[i] = ctx->idx; 1149 continue; 1150 } 1151 if (ret) 1152 return ret; 1153 } 1154 1155 if (ctx->image == NULL) 1156 ctx->offset[i] = ctx->idx; 1157 1158 return 0; 1159 } 1160 1161 /* Fill space with break instructions */ 1162 static void jit_fill_hole(void *area, unsigned int size) 1163 { 1164 u32 *ptr; 1165 1166 /* We are guaranteed to have aligned memory */ 1167 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1168 *ptr++ = INSN_BREAK; 1169 } 1170 1171 static int validate_code(struct jit_ctx *ctx) 1172 { 1173 int i; 1174 union loongarch_instruction insn; 1175 1176 for (i = 0; i < ctx->idx; i++) { 1177 insn = ctx->image[i]; 1178 /* Check INSN_BREAK */ 1179 if (insn.word == INSN_BREAK) 1180 return -1; 1181 } 1182 1183 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) 1184 return -1; 1185 1186 return 0; 1187 } 1188 1189 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1190 { 1191 bool tmp_blinded = false, extra_pass = false; 1192 u8 *image_ptr; 1193 int image_size, prog_size, extable_size; 1194 struct jit_ctx ctx; 1195 struct jit_data *jit_data; 1196 struct bpf_binary_header *header; 1197 struct bpf_prog *tmp, *orig_prog = prog; 1198 1199 /* 1200 * If BPF JIT was not enabled then we must fall back to 1201 * the interpreter. 1202 */ 1203 if (!prog->jit_requested) 1204 return orig_prog; 1205 1206 tmp = bpf_jit_blind_constants(prog); 1207 /* 1208 * If blinding was requested and we failed during blinding, 1209 * we must fall back to the interpreter. Otherwise, we save 1210 * the new JITed code. 1211 */ 1212 if (IS_ERR(tmp)) 1213 return orig_prog; 1214 1215 if (tmp != prog) { 1216 tmp_blinded = true; 1217 prog = tmp; 1218 } 1219 1220 jit_data = prog->aux->jit_data; 1221 if (!jit_data) { 1222 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1223 if (!jit_data) { 1224 prog = orig_prog; 1225 goto out; 1226 } 1227 prog->aux->jit_data = jit_data; 1228 } 1229 if (jit_data->ctx.offset) { 1230 ctx = jit_data->ctx; 1231 image_ptr = jit_data->image; 1232 header = jit_data->header; 1233 extra_pass = true; 1234 prog_size = sizeof(u32) * ctx.idx; 1235 goto skip_init_ctx; 1236 } 1237 1238 memset(&ctx, 0, sizeof(ctx)); 1239 ctx.prog = prog; 1240 1241 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 1242 if (ctx.offset == NULL) { 1243 prog = orig_prog; 1244 goto out_offset; 1245 } 1246 1247 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 1248 build_prologue(&ctx); 1249 if (build_body(&ctx, extra_pass)) { 1250 prog = orig_prog; 1251 goto out_offset; 1252 } 1253 ctx.epilogue_offset = ctx.idx; 1254 build_epilogue(&ctx); 1255 1256 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); 1257 1258 /* Now we know the actual image size. 1259 * As each LoongArch instruction is of length 32bit, 1260 * we are translating number of JITed intructions into 1261 * the size required to store these JITed code. 1262 */ 1263 prog_size = sizeof(u32) * ctx.idx; 1264 image_size = prog_size + extable_size; 1265 /* Now we know the size of the structure to make */ 1266 header = bpf_jit_binary_alloc(image_size, &image_ptr, 1267 sizeof(u32), jit_fill_hole); 1268 if (header == NULL) { 1269 prog = orig_prog; 1270 goto out_offset; 1271 } 1272 1273 /* 2. Now, the actual pass to generate final JIT code */ 1274 ctx.image = (union loongarch_instruction *)image_ptr; 1275 if (extable_size) 1276 prog->aux->extable = (void *)image_ptr + prog_size; 1277 1278 skip_init_ctx: 1279 ctx.idx = 0; 1280 ctx.num_exentries = 0; 1281 1282 build_prologue(&ctx); 1283 if (build_body(&ctx, extra_pass)) { 1284 bpf_jit_binary_free(header); 1285 prog = orig_prog; 1286 goto out_offset; 1287 } 1288 build_epilogue(&ctx); 1289 1290 /* 3. Extra pass to validate JITed code */ 1291 if (validate_code(&ctx)) { 1292 bpf_jit_binary_free(header); 1293 prog = orig_prog; 1294 goto out_offset; 1295 } 1296 1297 /* And we're done */ 1298 if (bpf_jit_enable > 1) 1299 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 1300 1301 /* Update the icache */ 1302 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); 1303 1304 if (!prog->is_func || extra_pass) { 1305 int err; 1306 1307 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 1308 pr_err_once("multi-func JIT bug %d != %d\n", 1309 ctx.idx, jit_data->ctx.idx); 1310 goto out_free; 1311 } 1312 err = bpf_jit_binary_lock_ro(header); 1313 if (err) { 1314 pr_err_once("bpf_jit_binary_lock_ro() returned %d\n", 1315 err); 1316 goto out_free; 1317 } 1318 } else { 1319 jit_data->ctx = ctx; 1320 jit_data->image = image_ptr; 1321 jit_data->header = header; 1322 } 1323 prog->jited = 1; 1324 prog->jited_len = prog_size; 1325 prog->bpf_func = (void *)ctx.image; 1326 1327 if (!prog->is_func || extra_pass) { 1328 int i; 1329 1330 /* offset[prog->len] is the size of program */ 1331 for (i = 0; i <= prog->len; i++) 1332 ctx.offset[i] *= LOONGARCH_INSN_SIZE; 1333 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 1334 1335 out_offset: 1336 kvfree(ctx.offset); 1337 kfree(jit_data); 1338 prog->aux->jit_data = NULL; 1339 } 1340 1341 out: 1342 if (tmp_blinded) 1343 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); 1344 1345 out_offset = -1; 1346 1347 return prog; 1348 1349 out_free: 1350 bpf_jit_binary_free(header); 1351 prog->bpf_func = NULL; 1352 prog->jited = 0; 1353 prog->jited_len = 0; 1354 goto out_offset; 1355 } 1356 1357 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ 1358 bool bpf_jit_supports_subprog_tailcalls(void) 1359 { 1360 return true; 1361 } 1362