1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/memory.h> 8 #include <asm/asm-offsets.h> 9 #include "bpf_jit.h" 10 11 /* DBAR hint for LL/SC completion ordering, see __WEAK_LLSC_MB */ 12 #define DBAR_LLSC_MB 0x700 13 14 #define LOONGARCH_MAX_REG_ARGS 8 15 16 #define LOONGARCH_SAVE_RA_NINSNS 1 17 #define LOONGARCH_LONG_JUMP_NINSNS 5 18 #define LOONGARCH_TCC_SLOT_NINSNS 1 19 20 #define LOONGARCH_PROLOGUE_SKIP_INSNS \ 21 (LOONGARCH_SAVE_RA_NINSNS + LOONGARCH_LONG_JUMP_NINSNS + LOONGARCH_TCC_SLOT_NINSNS) 22 23 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 24 25 #define LOONGARCH_FENTRY_NINSNS 2 26 #define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) 27 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 28 29 #define REG_TCC LOONGARCH_GPR_A6 30 #define REG_ARENA LOONGARCH_GPR_S6 /* For storing arena_vm_start */ 31 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) 32 33 static const int regmap[] = { 34 /* return value from in-kernel function, and exit value for eBPF program */ 35 [BPF_REG_0] = LOONGARCH_GPR_A5, 36 /* arguments from eBPF program to in-kernel function */ 37 [BPF_REG_1] = LOONGARCH_GPR_A0, 38 [BPF_REG_2] = LOONGARCH_GPR_A1, 39 [BPF_REG_3] = LOONGARCH_GPR_A2, 40 [BPF_REG_4] = LOONGARCH_GPR_A3, 41 [BPF_REG_5] = LOONGARCH_GPR_A4, 42 /* callee saved registers that in-kernel function will preserve */ 43 [BPF_REG_6] = LOONGARCH_GPR_S0, 44 [BPF_REG_7] = LOONGARCH_GPR_S1, 45 [BPF_REG_8] = LOONGARCH_GPR_S2, 46 [BPF_REG_9] = LOONGARCH_GPR_S3, 47 /* read-only frame pointer to access stack */ 48 [BPF_REG_FP] = LOONGARCH_GPR_S4, 49 /* temporary register for blinding constants */ 50 [BPF_REG_AX] = LOONGARCH_GPR_T0, 51 }; 52 53 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) 54 { 55 const struct bpf_prog *prog = ctx->prog; 56 const bool is_main_prog = !bpf_is_subprog(prog); 57 58 *store_offset -= sizeof(long); 59 if (is_main_prog) { 60 /* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */ 61 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 62 63 /* Compute the absolute pointer to the local 'tcc' slot */ 64 emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset); 65 66 /* 67 * Branchless classification and blending: 68 * Combine interleaved inputs between a scalar count (0 to 33) 69 * and a kernel pointer address without runtime branching. 70 */ 71 emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1); 72 emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8); 73 emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8); 74 emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7); 75 } else { 76 /* Subprograms: backup the verified TCC pointer inherited via REG_TCC */ 77 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 78 } 79 80 /* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */ 81 *store_offset -= sizeof(long); 82 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 83 } 84 85 /* 86 * eBPF prog stack layout: 87 * 88 * high 89 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP 90 * | $ra | 91 * +-------------------------+ 92 * | $fp | 93 * +-------------------------+ 94 * | $s0 | 95 * +-------------------------+ 96 * | $s1 | 97 * +-------------------------+ 98 * | $s2 | 99 * +-------------------------+ 100 * | $s3 | 101 * +-------------------------+ 102 * | $s4 | 103 * +-------------------------+ 104 * | $s5 | 105 * +-------------------------+ 106 * | tcc | 107 * +-------------------------+ 108 * | tcc_ptr | 109 * +-------------------------+ 110 * | arena | 111 * | (optional) | 112 * +-------------------------+ <--BPF_REG_FP 113 * | prog->aux->stack_depth | 114 * | (optional) | 115 * current $sp -------------> +-------------------------+ 116 * low 117 */ 118 static void build_prologue(struct jit_ctx *ctx) 119 { 120 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 121 const struct bpf_prog *prog = ctx->prog; 122 const bool is_main_prog = !bpf_is_subprog(prog); 123 124 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 125 126 /* To store ra, fp, s0, s1, s2, s3, s4, s5 */ 127 stack_adjust += sizeof(long) * 8; 128 129 /* To store tcc and tcc_ptr */ 130 stack_adjust += sizeof(long) * 2; 131 132 if (ctx->arena_vm_start) 133 stack_adjust += sizeof(long); 134 135 stack_adjust = round_up(stack_adjust, 16); 136 stack_adjust += bpf_stack_adjust; 137 138 /* 139 * Save the original return address to a temporary register to prevent 140 * it from being overwritten, then reserve space for the long jump and 141 * fentry trampoline slot for dynamically patching by ftrace at runtime. 142 * These instructions are bypassed during a tail call invocation. 143 */ 144 move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); 145 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 146 emit_insn(ctx, nop); 147 148 /* 149 * First instruction initializes the tail call count (TCC) 150 * register to zero. On tail call we skip this instruction, 151 * and the TCC is passed in REG_TCC from the caller. 152 */ 153 if (is_main_prog) 154 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); 155 156 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); 157 158 store_offset = stack_adjust - sizeof(long); 159 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); 160 161 store_offset -= sizeof(long); 162 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); 163 164 store_offset -= sizeof(long); 165 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); 166 167 store_offset -= sizeof(long); 168 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); 169 170 store_offset -= sizeof(long); 171 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); 172 173 store_offset -= sizeof(long); 174 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); 175 176 store_offset -= sizeof(long); 177 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); 178 179 store_offset -= sizeof(long); 180 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 181 182 prepare_bpf_tail_call_cnt(ctx, &store_offset); 183 184 if (ctx->arena_vm_start) { 185 store_offset -= sizeof(long); 186 emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); 187 } 188 189 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); 190 191 if (bpf_stack_adjust) 192 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 193 194 ctx->stack_size = stack_adjust; 195 196 if (ctx->arena_vm_start) 197 move_imm(ctx, REG_ARENA, ctx->arena_vm_start, false); 198 } 199 200 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) 201 { 202 int stack_adjust = ctx->stack_size; 203 int load_offset; 204 205 load_offset = stack_adjust - sizeof(long); 206 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); 207 208 load_offset -= sizeof(long); 209 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); 210 211 load_offset -= sizeof(long); 212 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); 213 214 load_offset -= sizeof(long); 215 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); 216 217 load_offset -= sizeof(long); 218 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); 219 220 load_offset -= sizeof(long); 221 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); 222 223 load_offset -= sizeof(long); 224 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); 225 226 load_offset -= sizeof(long); 227 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 228 229 /* Only restore the TCC state into REG_TCC from the higher slot */ 230 load_offset -= sizeof(long); 231 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 232 233 /* Skip the unused local 'tcc_ptr' slot to align with arena */ 234 load_offset -= sizeof(long); 235 236 if (ctx->arena_vm_start) { 237 load_offset -= sizeof(long); 238 emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); 239 } 240 241 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); 242 243 if (!is_tail_call) { 244 /* Set return value */ 245 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); 246 /* Return to the caller */ 247 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 248 } else { 249 /* 250 * Tail call to the next BPF program, passing offset in number 251 * of instructions to jirl to bypass the initial setup slots. 252 */ 253 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, 254 LOONGARCH_GPR_T3, LOONGARCH_PROLOGUE_SKIP_INSNS); 255 } 256 } 257 258 static void build_epilogue(struct jit_ctx *ctx) 259 { 260 __build_epilogue(ctx, false); 261 } 262 263 bool bpf_jit_supports_kfunc_call(void) 264 { 265 return true; 266 } 267 268 bool bpf_jit_supports_far_kfunc_call(void) 269 { 270 return true; 271 } 272 273 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) 274 { 275 int off, jmp_offset; 276 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 277 u8 a1 = LOONGARCH_GPR_A1; 278 u8 a2 = LOONGARCH_GPR_A2; 279 u8 t1 = LOONGARCH_GPR_T1; 280 u8 t2 = LOONGARCH_GPR_T2; 281 u8 t3 = LOONGARCH_GPR_T3; 282 283 /* 284 * a0: &ctx 285 * a1: &array 286 * a2: index 287 * 288 * if (index >= array->map.max_entries) 289 * goto out; 290 */ 291 emit_zext_32(ctx, a2, true); 292 293 off = offsetof(struct bpf_array, map.max_entries); 294 emit_insn(ctx, ldwu, t1, a1, off); 295 /* bgeu $a2, $t1, jmp_offset */ 296 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; 297 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) 298 goto toofar; 299 300 /* 301 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) 302 * goto out; 303 */ 304 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 305 emit_insn(ctx, ldd, t3, REG_TCC, 0); 306 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 307 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; 308 if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) 309 goto toofar; 310 311 emit_insn(ctx, addid, t3, t3, 1); 312 313 /* 314 * prog = array->ptrs[index]; 315 * if (!prog) 316 * goto out; 317 */ 318 emit_insn(ctx, alsld, t2, a2, a1, 2); 319 off = offsetof(struct bpf_array, ptrs); 320 emit_insn(ctx, ldd, t2, t2, off); 321 /* beq $t2, $zero, jmp_offset */ 322 jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; 323 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 324 goto toofar; 325 326 emit_insn(ctx, std, t3, REG_TCC, 0); 327 328 /* goto *(prog->bpf_func + 4); */ 329 off = offsetof(struct bpf_prog, bpf_func); 330 emit_insn(ctx, ldd, t3, t2, off); 331 __build_epilogue(ctx, true); 332 333 return 0; 334 335 toofar: 336 pr_info_once("tail_call: jump too far\n"); 337 return -1; 338 } 339 340 static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64) 341 { 342 move_imm(ctx, reg, imm64, false); 343 emit_insn(ctx, std, reg, LOONGARCH_GPR_FP, stack_off); 344 } 345 346 static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) 347 { 348 const u8 t1 = LOONGARCH_GPR_T1; 349 const u8 t2 = LOONGARCH_GPR_T2; 350 const u8 t3 = LOONGARCH_GPR_T3; 351 const u8 r0 = regmap[BPF_REG_0]; 352 const u8 src = regmap[insn->src_reg]; 353 const u8 dst = regmap[insn->dst_reg]; 354 const s16 off = insn->off; 355 const s32 imm = insn->imm; 356 const bool isdw = BPF_SIZE(insn->code) == BPF_DW; 357 358 move_imm(ctx, t1, off, false); 359 emit_insn(ctx, addd, t1, dst, t1); 360 move_reg(ctx, t3, src); 361 362 switch (imm) { 363 /* lock *(size *)(dst + off) <op>= src */ 364 case BPF_ADD: 365 switch (BPF_SIZE(insn->code)) { 366 case BPF_B: 367 if (!cpu_has_lam_bh) { 368 pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); 369 return -EINVAL; 370 } 371 emit_insn(ctx, amaddb, t2, t1, src); 372 break; 373 case BPF_H: 374 if (!cpu_has_lam_bh) { 375 pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); 376 return -EINVAL; 377 } 378 emit_insn(ctx, amaddh, t2, t1, src); 379 break; 380 case BPF_W: 381 emit_insn(ctx, amaddw, t2, t1, src); 382 break; 383 case BPF_DW: 384 emit_insn(ctx, amaddd, t2, t1, src); 385 break; 386 } 387 break; 388 case BPF_AND: 389 if (isdw) 390 emit_insn(ctx, amandd, t2, t1, src); 391 else 392 emit_insn(ctx, amandw, t2, t1, src); 393 break; 394 case BPF_OR: 395 if (isdw) 396 emit_insn(ctx, amord, t2, t1, src); 397 else 398 emit_insn(ctx, amorw, t2, t1, src); 399 break; 400 case BPF_XOR: 401 if (isdw) 402 emit_insn(ctx, amxord, t2, t1, src); 403 else 404 emit_insn(ctx, amxorw, t2, t1, src); 405 break; 406 /* src = atomic_fetch_<op>(dst + off, src) */ 407 case BPF_ADD | BPF_FETCH: 408 switch (BPF_SIZE(insn->code)) { 409 case BPF_B: 410 if (!cpu_has_lam_bh) { 411 pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); 412 return -EINVAL; 413 } 414 emit_insn(ctx, amadddbb, src, t1, t3); 415 emit_zext_32(ctx, src, true); 416 break; 417 case BPF_H: 418 if (!cpu_has_lam_bh) { 419 pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); 420 return -EINVAL; 421 } 422 emit_insn(ctx, amadddbh, src, t1, t3); 423 emit_zext_32(ctx, src, true); 424 break; 425 case BPF_W: 426 emit_insn(ctx, amadddbw, src, t1, t3); 427 emit_zext_32(ctx, src, true); 428 break; 429 case BPF_DW: 430 emit_insn(ctx, amadddbd, src, t1, t3); 431 break; 432 } 433 break; 434 case BPF_AND | BPF_FETCH: 435 if (isdw) { 436 emit_insn(ctx, amanddbd, src, t1, t3); 437 } else { 438 emit_insn(ctx, amanddbw, src, t1, t3); 439 emit_zext_32(ctx, src, true); 440 } 441 break; 442 case BPF_OR | BPF_FETCH: 443 if (isdw) { 444 emit_insn(ctx, amordbd, src, t1, t3); 445 } else { 446 emit_insn(ctx, amordbw, src, t1, t3); 447 emit_zext_32(ctx, src, true); 448 } 449 break; 450 case BPF_XOR | BPF_FETCH: 451 if (isdw) { 452 emit_insn(ctx, amxordbd, src, t1, t3); 453 } else { 454 emit_insn(ctx, amxordbw, src, t1, t3); 455 emit_zext_32(ctx, src, true); 456 } 457 break; 458 /* src = atomic_xchg(dst + off, src); */ 459 case BPF_XCHG: 460 switch (BPF_SIZE(insn->code)) { 461 case BPF_B: 462 if (!cpu_has_lam_bh) { 463 pr_err_once("bpf-jit: amswap.b instruction is not supported\n"); 464 return -EINVAL; 465 } 466 emit_insn(ctx, amswapdbb, src, t1, t3); 467 emit_zext_32(ctx, src, true); 468 break; 469 case BPF_H: 470 if (!cpu_has_lam_bh) { 471 pr_err_once("bpf-jit: amswap.h instruction is not supported\n"); 472 return -EINVAL; 473 } 474 emit_insn(ctx, amswapdbh, src, t1, t3); 475 emit_zext_32(ctx, src, true); 476 break; 477 case BPF_W: 478 emit_insn(ctx, amswapdbw, src, t1, t3); 479 emit_zext_32(ctx, src, true); 480 break; 481 case BPF_DW: 482 emit_insn(ctx, amswapdbd, src, t1, t3); 483 break; 484 } 485 break; 486 /* r0 = atomic_cmpxchg(dst + off, r0, src); */ 487 case BPF_CMPXCHG: 488 move_reg(ctx, t2, r0); 489 if (isdw) { 490 emit_insn(ctx, lld, r0, t1, 0); 491 emit_insn(ctx, bne, t2, r0, 4); 492 move_reg(ctx, t3, src); 493 emit_insn(ctx, scd, t3, t1, 0); 494 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); 495 } else { 496 emit_insn(ctx, llw, r0, t1, 0); 497 emit_zext_32(ctx, t2, true); 498 emit_zext_32(ctx, r0, true); 499 emit_insn(ctx, bne, t2, r0, 4); 500 move_reg(ctx, t3, src); 501 emit_insn(ctx, scw, t3, t1, 0); 502 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); 503 emit_zext_32(ctx, r0, true); 504 } 505 emit_insn(ctx, dbar, DBAR_LLSC_MB); 506 break; 507 default: 508 pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm); 509 return -EINVAL; 510 } 511 512 return 0; 513 } 514 515 static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx) 516 { 517 const u8 t1 = LOONGARCH_GPR_T1; 518 const u8 src = regmap[insn->src_reg]; 519 const u8 dst = regmap[insn->dst_reg]; 520 const s16 off = insn->off; 521 const s32 imm = insn->imm; 522 523 switch (imm) { 524 /* dst_reg = load_acquire(src_reg + off16) */ 525 case BPF_LOAD_ACQ: 526 switch (BPF_SIZE(insn->code)) { 527 case BPF_B: 528 if (is_signed_imm12(off)) { 529 emit_insn(ctx, ldbu, dst, src, off); 530 } else { 531 move_imm(ctx, t1, off, false); 532 emit_insn(ctx, ldxbu, dst, src, t1); 533 } 534 break; 535 case BPF_H: 536 if (is_signed_imm12(off)) { 537 emit_insn(ctx, ldhu, dst, src, off); 538 } else { 539 move_imm(ctx, t1, off, false); 540 emit_insn(ctx, ldxhu, dst, src, t1); 541 } 542 break; 543 case BPF_W: 544 if (is_signed_imm12(off)) { 545 emit_insn(ctx, ldwu, dst, src, off); 546 } else { 547 move_imm(ctx, t1, off, false); 548 emit_insn(ctx, ldxwu, dst, src, t1); 549 } 550 break; 551 case BPF_DW: 552 if (is_signed_imm12(off)) { 553 emit_insn(ctx, ldd, dst, src, off); 554 } else { 555 move_imm(ctx, t1, off, false); 556 emit_insn(ctx, ldxd, dst, src, t1); 557 } 558 break; 559 } 560 emit_insn(ctx, dbar, 0b10100); 561 break; 562 /* store_release(dst_reg + off16, src_reg) */ 563 case BPF_STORE_REL: 564 emit_insn(ctx, dbar, 0b10010); 565 switch (BPF_SIZE(insn->code)) { 566 case BPF_B: 567 if (is_signed_imm12(off)) { 568 emit_insn(ctx, stb, src, dst, off); 569 } else { 570 move_imm(ctx, t1, off, false); 571 emit_insn(ctx, stxb, src, dst, t1); 572 } 573 break; 574 case BPF_H: 575 if (is_signed_imm12(off)) { 576 emit_insn(ctx, sth, src, dst, off); 577 } else { 578 move_imm(ctx, t1, off, false); 579 emit_insn(ctx, stxh, src, dst, t1); 580 } 581 break; 582 case BPF_W: 583 if (is_signed_imm12(off)) { 584 emit_insn(ctx, stw, src, dst, off); 585 } else { 586 move_imm(ctx, t1, off, false); 587 emit_insn(ctx, stxw, src, dst, t1); 588 } 589 break; 590 case BPF_DW: 591 if (is_signed_imm12(off)) { 592 emit_insn(ctx, std, src, dst, off); 593 } else { 594 move_imm(ctx, t1, off, false); 595 emit_insn(ctx, stxd, src, dst, t1); 596 } 597 break; 598 } 599 break; 600 default: 601 pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm); 602 return -EINVAL; 603 } 604 605 return 0; 606 } 607 608 static bool is_signed_bpf_cond(u8 cond) 609 { 610 return cond == BPF_JSGT || cond == BPF_JSLT || 611 cond == BPF_JSGE || cond == BPF_JSLE; 612 } 613 614 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 615 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 616 #define REG_DONT_CLEAR_MARKER 0 617 618 bool ex_handler_bpf(const struct exception_table_entry *ex, 619 struct pt_regs *regs) 620 { 621 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 622 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 623 624 if (dst_reg != REG_DONT_CLEAR_MARKER) 625 regs->regs[dst_reg] = 0; 626 regs->csr_era = (unsigned long)&ex->fixup - offset; 627 628 return true; 629 } 630 631 /* For accesses to BTF pointers, add an entry to the exception table */ 632 static int add_exception_handler(const struct bpf_insn *insn, 633 struct jit_ctx *ctx, 634 int dst_reg) 635 { 636 unsigned long pc; 637 off_t ins_offset, fixup_offset; 638 struct exception_table_entry *ex; 639 640 if (!ctx->image || !ctx->ro_image || !ctx->prog->aux->extable) 641 return 0; 642 643 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 644 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 645 BPF_MODE(insn->code) != BPF_PROBE_MEM32) 646 return 0; 647 648 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) 649 return -EINVAL; 650 651 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 652 pc = (unsigned long)&ctx->ro_image[ctx->idx - 1]; 653 654 /* 655 * This is the relative offset of the instruction that may fault from 656 * the exception table itself. This will be written to the exception 657 * table and if this instruction faults, the destination register will 658 * be set to '0' and the execution will jump to the next instruction. 659 */ 660 ins_offset = pc - (long)&ex->insn; 661 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 662 return -ERANGE; 663 664 /* 665 * Since the extable follows the program, the fixup offset is always 666 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 667 * to keep things simple, and put the destination register in the upper 668 * bits. We don't need to worry about buildtime or runtime sort 669 * modifying the upper bits because the table is already sorted, and 670 * isn't part of the main exception table. 671 * 672 * The fixup_offset is set to the next instruction from the instruction 673 * that may fault. The execution will jump to this after handling the fault. 674 */ 675 fixup_offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 676 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 677 return -ERANGE; 678 679 /* 680 * The offsets above have been calculated using the RO buffer but we 681 * need to use the R/W buffer for writes. Switch ex to rw buffer for writing. 682 */ 683 ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); 684 ex->insn = ins_offset; 685 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 686 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 687 ex->type = EX_TYPE_BPF; 688 689 ctx->num_exentries++; 690 691 return 0; 692 } 693 694 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) 695 { 696 u8 tm = -1; 697 u64 func_addr; 698 bool func_addr_fixed, sign_extend; 699 int i = insn - ctx->prog->insnsi; 700 int ret, jmp_offset, tcc_ptr_off; 701 const u8 code = insn->code; 702 const u8 cond = BPF_OP(code); 703 const u8 t1 = LOONGARCH_GPR_T1; 704 const u8 t2 = LOONGARCH_GPR_T2; 705 const u8 t3 = LOONGARCH_GPR_T3; 706 u8 src = regmap[insn->src_reg]; 707 u8 dst = regmap[insn->dst_reg]; 708 const s16 off = insn->off; 709 const s32 imm = insn->imm; 710 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; 711 712 switch (code) { 713 /* dst = src */ 714 case BPF_ALU | BPF_MOV | BPF_X: 715 case BPF_ALU64 | BPF_MOV | BPF_X: 716 if (insn_is_cast_user(insn)) { 717 move_reg(ctx, t1, src); 718 emit_zext_32(ctx, t1, true); 719 move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32, false); 720 emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 1); 721 emit_insn(ctx, or, t1, dst, t1); 722 move_reg(ctx, dst, t1); 723 break; 724 } 725 if (insn_is_mov_percpu_addr(insn)) { 726 if (dst != src) 727 move_reg(ctx, dst, src); 728 #ifdef CONFIG_SMP 729 /* dst += __my_cpu_offset, held in $r21 */ 730 emit_insn(ctx, addd, dst, dst, LOONGARCH_GPR_U0); 731 #endif 732 break; 733 } 734 switch (off) { 735 case 0: 736 move_reg(ctx, dst, src); 737 emit_zext_32(ctx, dst, is32); 738 break; 739 case 8: 740 emit_insn(ctx, extwb, dst, src); 741 emit_zext_32(ctx, dst, is32); 742 break; 743 case 16: 744 emit_insn(ctx, extwh, dst, src); 745 emit_zext_32(ctx, dst, is32); 746 break; 747 case 32: 748 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); 749 break; 750 } 751 break; 752 753 /* dst = imm */ 754 case BPF_ALU | BPF_MOV | BPF_K: 755 case BPF_ALU64 | BPF_MOV | BPF_K: 756 move_imm(ctx, dst, imm, is32); 757 break; 758 759 /* dst = dst + src */ 760 case BPF_ALU | BPF_ADD | BPF_X: 761 case BPF_ALU64 | BPF_ADD | BPF_X: 762 emit_insn(ctx, addd, dst, dst, src); 763 emit_zext_32(ctx, dst, is32); 764 break; 765 766 /* dst = dst + imm */ 767 case BPF_ALU | BPF_ADD | BPF_K: 768 case BPF_ALU64 | BPF_ADD | BPF_K: 769 if (is_signed_imm12(imm)) { 770 emit_insn(ctx, addid, dst, dst, imm); 771 } else { 772 move_imm(ctx, t1, imm, is32); 773 emit_insn(ctx, addd, dst, dst, t1); 774 } 775 emit_zext_32(ctx, dst, is32); 776 break; 777 778 /* dst = dst - src */ 779 case BPF_ALU | BPF_SUB | BPF_X: 780 case BPF_ALU64 | BPF_SUB | BPF_X: 781 emit_insn(ctx, subd, dst, dst, src); 782 emit_zext_32(ctx, dst, is32); 783 break; 784 785 /* dst = dst - imm */ 786 case BPF_ALU | BPF_SUB | BPF_K: 787 case BPF_ALU64 | BPF_SUB | BPF_K: 788 if (is_signed_imm12(-imm)) { 789 emit_insn(ctx, addid, dst, dst, -imm); 790 } else { 791 move_imm(ctx, t1, imm, is32); 792 emit_insn(ctx, subd, dst, dst, t1); 793 } 794 emit_zext_32(ctx, dst, is32); 795 break; 796 797 /* dst = dst * src */ 798 case BPF_ALU | BPF_MUL | BPF_X: 799 case BPF_ALU64 | BPF_MUL | BPF_X: 800 emit_insn(ctx, muld, dst, dst, src); 801 emit_zext_32(ctx, dst, is32); 802 break; 803 804 /* dst = dst * imm */ 805 case BPF_ALU | BPF_MUL | BPF_K: 806 case BPF_ALU64 | BPF_MUL | BPF_K: 807 move_imm(ctx, t1, imm, is32); 808 emit_insn(ctx, muld, dst, dst, t1); 809 emit_zext_32(ctx, dst, is32); 810 break; 811 812 /* dst = dst / src */ 813 case BPF_ALU | BPF_DIV | BPF_X: 814 case BPF_ALU64 | BPF_DIV | BPF_X: 815 if (!off) { 816 emit_zext_32(ctx, dst, is32); 817 move_reg(ctx, t1, src); 818 emit_zext_32(ctx, t1, is32); 819 emit_insn(ctx, divdu, dst, dst, t1); 820 emit_zext_32(ctx, dst, is32); 821 } else { 822 emit_sext_32(ctx, dst, is32); 823 move_reg(ctx, t1, src); 824 emit_sext_32(ctx, t1, is32); 825 emit_insn(ctx, divd, dst, dst, t1); 826 emit_zext_32(ctx, dst, is32); 827 } 828 break; 829 830 /* dst = dst / imm */ 831 case BPF_ALU | BPF_DIV | BPF_K: 832 case BPF_ALU64 | BPF_DIV | BPF_K: 833 if (!off) { 834 move_imm(ctx, t1, imm, is32); 835 emit_zext_32(ctx, dst, is32); 836 emit_insn(ctx, divdu, dst, dst, t1); 837 emit_zext_32(ctx, dst, is32); 838 } else { 839 move_imm(ctx, t1, imm, false); 840 emit_sext_32(ctx, t1, is32); 841 emit_sext_32(ctx, dst, is32); 842 emit_insn(ctx, divd, dst, dst, t1); 843 emit_zext_32(ctx, dst, is32); 844 } 845 break; 846 847 /* dst = dst % src */ 848 case BPF_ALU | BPF_MOD | BPF_X: 849 case BPF_ALU64 | BPF_MOD | BPF_X: 850 if (!off) { 851 emit_zext_32(ctx, dst, is32); 852 move_reg(ctx, t1, src); 853 emit_zext_32(ctx, t1, is32); 854 emit_insn(ctx, moddu, dst, dst, t1); 855 emit_zext_32(ctx, dst, is32); 856 } else { 857 emit_sext_32(ctx, dst, is32); 858 move_reg(ctx, t1, src); 859 emit_sext_32(ctx, t1, is32); 860 emit_insn(ctx, modd, dst, dst, t1); 861 emit_zext_32(ctx, dst, is32); 862 } 863 break; 864 865 /* dst = dst % imm */ 866 case BPF_ALU | BPF_MOD | BPF_K: 867 case BPF_ALU64 | BPF_MOD | BPF_K: 868 if (!off) { 869 move_imm(ctx, t1, imm, is32); 870 emit_zext_32(ctx, dst, is32); 871 emit_insn(ctx, moddu, dst, dst, t1); 872 emit_zext_32(ctx, dst, is32); 873 } else { 874 move_imm(ctx, t1, imm, false); 875 emit_sext_32(ctx, t1, is32); 876 emit_sext_32(ctx, dst, is32); 877 emit_insn(ctx, modd, dst, dst, t1); 878 emit_zext_32(ctx, dst, is32); 879 } 880 break; 881 882 /* dst = -dst */ 883 case BPF_ALU | BPF_NEG: 884 case BPF_ALU64 | BPF_NEG: 885 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); 886 emit_zext_32(ctx, dst, is32); 887 break; 888 889 /* dst = dst & src */ 890 case BPF_ALU | BPF_AND | BPF_X: 891 case BPF_ALU64 | BPF_AND | BPF_X: 892 emit_insn(ctx, and, dst, dst, src); 893 emit_zext_32(ctx, dst, is32); 894 break; 895 896 /* dst = dst & imm */ 897 case BPF_ALU | BPF_AND | BPF_K: 898 case BPF_ALU64 | BPF_AND | BPF_K: 899 if (is_unsigned_imm12(imm)) { 900 emit_insn(ctx, andi, dst, dst, imm); 901 } else { 902 move_imm(ctx, t1, imm, is32); 903 emit_insn(ctx, and, dst, dst, t1); 904 } 905 emit_zext_32(ctx, dst, is32); 906 break; 907 908 /* dst = dst | src */ 909 case BPF_ALU | BPF_OR | BPF_X: 910 case BPF_ALU64 | BPF_OR | BPF_X: 911 emit_insn(ctx, or, dst, dst, src); 912 emit_zext_32(ctx, dst, is32); 913 break; 914 915 /* dst = dst | imm */ 916 case BPF_ALU | BPF_OR | BPF_K: 917 case BPF_ALU64 | BPF_OR | BPF_K: 918 if (is_unsigned_imm12(imm)) { 919 emit_insn(ctx, ori, dst, dst, imm); 920 } else { 921 move_imm(ctx, t1, imm, is32); 922 emit_insn(ctx, or, dst, dst, t1); 923 } 924 emit_zext_32(ctx, dst, is32); 925 break; 926 927 /* dst = dst ^ src */ 928 case BPF_ALU | BPF_XOR | BPF_X: 929 case BPF_ALU64 | BPF_XOR | BPF_X: 930 emit_insn(ctx, xor, dst, dst, src); 931 emit_zext_32(ctx, dst, is32); 932 break; 933 934 /* dst = dst ^ imm */ 935 case BPF_ALU | BPF_XOR | BPF_K: 936 case BPF_ALU64 | BPF_XOR | BPF_K: 937 if (is_unsigned_imm12(imm)) { 938 emit_insn(ctx, xori, dst, dst, imm); 939 } else { 940 move_imm(ctx, t1, imm, is32); 941 emit_insn(ctx, xor, dst, dst, t1); 942 } 943 emit_zext_32(ctx, dst, is32); 944 break; 945 946 /* dst = dst << src (logical) */ 947 case BPF_ALU | BPF_LSH | BPF_X: 948 emit_insn(ctx, sllw, dst, dst, src); 949 emit_zext_32(ctx, dst, is32); 950 break; 951 952 case BPF_ALU64 | BPF_LSH | BPF_X: 953 emit_insn(ctx, slld, dst, dst, src); 954 break; 955 956 /* dst = dst << imm (logical) */ 957 case BPF_ALU | BPF_LSH | BPF_K: 958 emit_insn(ctx, slliw, dst, dst, imm); 959 emit_zext_32(ctx, dst, is32); 960 break; 961 962 case BPF_ALU64 | BPF_LSH | BPF_K: 963 emit_insn(ctx, sllid, dst, dst, imm); 964 break; 965 966 /* dst = dst >> src (logical) */ 967 case BPF_ALU | BPF_RSH | BPF_X: 968 emit_insn(ctx, srlw, dst, dst, src); 969 emit_zext_32(ctx, dst, is32); 970 break; 971 972 case BPF_ALU64 | BPF_RSH | BPF_X: 973 emit_insn(ctx, srld, dst, dst, src); 974 break; 975 976 /* dst = dst >> imm (logical) */ 977 case BPF_ALU | BPF_RSH | BPF_K: 978 emit_insn(ctx, srliw, dst, dst, imm); 979 emit_zext_32(ctx, dst, is32); 980 break; 981 982 case BPF_ALU64 | BPF_RSH | BPF_K: 983 emit_insn(ctx, srlid, dst, dst, imm); 984 break; 985 986 /* dst = dst >> src (arithmetic) */ 987 case BPF_ALU | BPF_ARSH | BPF_X: 988 emit_insn(ctx, sraw, dst, dst, src); 989 emit_zext_32(ctx, dst, is32); 990 break; 991 992 case BPF_ALU64 | BPF_ARSH | BPF_X: 993 emit_insn(ctx, srad, dst, dst, src); 994 break; 995 996 /* dst = dst >> imm (arithmetic) */ 997 case BPF_ALU | BPF_ARSH | BPF_K: 998 emit_insn(ctx, sraiw, dst, dst, imm); 999 emit_zext_32(ctx, dst, is32); 1000 break; 1001 1002 case BPF_ALU64 | BPF_ARSH | BPF_K: 1003 emit_insn(ctx, sraid, dst, dst, imm); 1004 break; 1005 1006 /* dst = BSWAP##imm(dst) */ 1007 case BPF_ALU | BPF_END | BPF_FROM_LE: 1008 switch (imm) { 1009 case 16: 1010 /* zero-extend 16 bits into 64 bits */ 1011 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 1012 break; 1013 case 32: 1014 /* zero-extend 32 bits into 64 bits */ 1015 emit_zext_32(ctx, dst, is32); 1016 break; 1017 case 64: 1018 /* do nothing */ 1019 break; 1020 } 1021 break; 1022 1023 case BPF_ALU | BPF_END | BPF_FROM_BE: 1024 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1025 switch (imm) { 1026 case 16: 1027 emit_insn(ctx, revb2h, dst, dst); 1028 /* zero-extend 16 bits into 64 bits */ 1029 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 1030 break; 1031 case 32: 1032 emit_insn(ctx, revb2w, dst, dst); 1033 /* clear the upper 32 bits */ 1034 emit_zext_32(ctx, dst, true); 1035 break; 1036 case 64: 1037 emit_insn(ctx, revbd, dst, dst); 1038 break; 1039 } 1040 break; 1041 1042 /* PC += off if dst cond src */ 1043 case BPF_JMP | BPF_JEQ | BPF_X: 1044 case BPF_JMP | BPF_JNE | BPF_X: 1045 case BPF_JMP | BPF_JGT | BPF_X: 1046 case BPF_JMP | BPF_JGE | BPF_X: 1047 case BPF_JMP | BPF_JLT | BPF_X: 1048 case BPF_JMP | BPF_JLE | BPF_X: 1049 case BPF_JMP | BPF_JSGT | BPF_X: 1050 case BPF_JMP | BPF_JSGE | BPF_X: 1051 case BPF_JMP | BPF_JSLT | BPF_X: 1052 case BPF_JMP | BPF_JSLE | BPF_X: 1053 case BPF_JMP32 | BPF_JEQ | BPF_X: 1054 case BPF_JMP32 | BPF_JNE | BPF_X: 1055 case BPF_JMP32 | BPF_JGT | BPF_X: 1056 case BPF_JMP32 | BPF_JGE | BPF_X: 1057 case BPF_JMP32 | BPF_JLT | BPF_X: 1058 case BPF_JMP32 | BPF_JLE | BPF_X: 1059 case BPF_JMP32 | BPF_JSGT | BPF_X: 1060 case BPF_JMP32 | BPF_JSGE | BPF_X: 1061 case BPF_JMP32 | BPF_JSLT | BPF_X: 1062 case BPF_JMP32 | BPF_JSLE | BPF_X: 1063 jmp_offset = bpf2la_offset(i, off, ctx); 1064 move_reg(ctx, t1, dst); 1065 move_reg(ctx, t2, src); 1066 if (is_signed_bpf_cond(BPF_OP(code))) { 1067 emit_sext_32(ctx, t1, is32); 1068 emit_sext_32(ctx, t2, is32); 1069 } else { 1070 emit_zext_32(ctx, t1, is32); 1071 emit_zext_32(ctx, t2, is32); 1072 } 1073 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) 1074 goto toofar; 1075 break; 1076 1077 /* PC += off if dst cond imm */ 1078 case BPF_JMP | BPF_JEQ | BPF_K: 1079 case BPF_JMP | BPF_JNE | BPF_K: 1080 case BPF_JMP | BPF_JGT | BPF_K: 1081 case BPF_JMP | BPF_JGE | BPF_K: 1082 case BPF_JMP | BPF_JLT | BPF_K: 1083 case BPF_JMP | BPF_JLE | BPF_K: 1084 case BPF_JMP | BPF_JSGT | BPF_K: 1085 case BPF_JMP | BPF_JSGE | BPF_K: 1086 case BPF_JMP | BPF_JSLT | BPF_K: 1087 case BPF_JMP | BPF_JSLE | BPF_K: 1088 case BPF_JMP32 | BPF_JEQ | BPF_K: 1089 case BPF_JMP32 | BPF_JNE | BPF_K: 1090 case BPF_JMP32 | BPF_JGT | BPF_K: 1091 case BPF_JMP32 | BPF_JGE | BPF_K: 1092 case BPF_JMP32 | BPF_JLT | BPF_K: 1093 case BPF_JMP32 | BPF_JLE | BPF_K: 1094 case BPF_JMP32 | BPF_JSGT | BPF_K: 1095 case BPF_JMP32 | BPF_JSGE | BPF_K: 1096 case BPF_JMP32 | BPF_JSLT | BPF_K: 1097 case BPF_JMP32 | BPF_JSLE | BPF_K: 1098 jmp_offset = bpf2la_offset(i, off, ctx); 1099 if (imm) { 1100 move_imm(ctx, t1, imm, false); 1101 tm = t1; 1102 } else { 1103 /* If imm is 0, simply use zero register. */ 1104 tm = LOONGARCH_GPR_ZERO; 1105 } 1106 move_reg(ctx, t2, dst); 1107 if (is_signed_bpf_cond(BPF_OP(code))) { 1108 emit_sext_32(ctx, tm, is32); 1109 emit_sext_32(ctx, t2, is32); 1110 } else { 1111 emit_zext_32(ctx, tm, is32); 1112 emit_zext_32(ctx, t2, is32); 1113 } 1114 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) 1115 goto toofar; 1116 break; 1117 1118 /* PC += off if dst & src */ 1119 case BPF_JMP | BPF_JSET | BPF_X: 1120 case BPF_JMP32 | BPF_JSET | BPF_X: 1121 jmp_offset = bpf2la_offset(i, off, ctx); 1122 emit_insn(ctx, and, t1, dst, src); 1123 emit_zext_32(ctx, t1, is32); 1124 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 1125 goto toofar; 1126 break; 1127 1128 /* PC += off if dst & imm */ 1129 case BPF_JMP | BPF_JSET | BPF_K: 1130 case BPF_JMP32 | BPF_JSET | BPF_K: 1131 jmp_offset = bpf2la_offset(i, off, ctx); 1132 move_imm(ctx, t1, imm, is32); 1133 emit_insn(ctx, and, t1, dst, t1); 1134 emit_zext_32(ctx, t1, is32); 1135 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 1136 goto toofar; 1137 break; 1138 1139 /* PC += off */ 1140 case BPF_JMP | BPF_JA: 1141 jmp_offset = bpf2la_offset(i, off, ctx); 1142 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 1143 goto toofar; 1144 break; 1145 case BPF_JMP32 | BPF_JA: 1146 jmp_offset = bpf2la_offset(i, imm, ctx); 1147 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 1148 goto toofar; 1149 break; 1150 1151 /* function call */ 1152 case BPF_JMP | BPF_CALL: 1153 /* Implement helper call to bpf_get_current_task/_btf() inline */ 1154 if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task || 1155 insn->imm == BPF_FUNC_get_current_task_btf)) { 1156 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_TP); 1157 break; 1158 } 1159 1160 /* Implement helper call to bpf_get_smp_processor_id() inline */ 1161 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { 1162 emit_insn(ctx, ldwu, regmap[BPF_REG_0], LOONGARCH_GPR_TP, TI_CPU); 1163 break; 1164 } 1165 1166 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1167 &func_addr, &func_addr_fixed); 1168 if (ret < 0) 1169 return ret; 1170 1171 if (insn->src_reg == BPF_PSEUDO_CALL) { 1172 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 1173 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 1174 } 1175 1176 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1177 const struct btf_func_model *m; 1178 int i; 1179 1180 m = bpf_jit_find_kfunc_model(ctx->prog, insn); 1181 if (!m) 1182 return -EINVAL; 1183 1184 for (i = 0; i < m->nr_args; i++) { 1185 u8 reg = regmap[BPF_REG_1 + i]; 1186 bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; 1187 1188 emit_abi_ext(ctx, reg, m->arg_size[i], sign); 1189 } 1190 } 1191 1192 move_addr(ctx, t1, func_addr); 1193 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); 1194 1195 /* 1196 * Call to arch_bpf_timed_may_goto() uses a custom calling 1197 * convention with the argument and return value in BPF_REG_AX, 1198 * so skip moving the C return value into BPF_REG_0. 1199 */ 1200 if (insn->src_reg != BPF_PSEUDO_CALL && 1201 func_addr != (u64)arch_bpf_timed_may_goto) 1202 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); 1203 1204 break; 1205 1206 /* tail call */ 1207 case BPF_JMP | BPF_TAIL_CALL: 1208 if (emit_bpf_tail_call(ctx, i) < 0) 1209 return -EINVAL; 1210 break; 1211 1212 /* function return */ 1213 case BPF_JMP | BPF_EXIT: 1214 if (i == ctx->prog->len - 1) 1215 break; 1216 1217 jmp_offset = epilogue_offset(ctx); 1218 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 1219 goto toofar; 1220 break; 1221 1222 /* dst = imm64 */ 1223 case BPF_LD | BPF_IMM | BPF_DW: 1224 { 1225 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; 1226 1227 if (bpf_pseudo_func(insn)) 1228 move_addr(ctx, dst, imm64); 1229 else 1230 move_imm(ctx, dst, imm64, is32); 1231 return 1; 1232 } 1233 1234 /* dst = *(size *)(src + off) */ 1235 case BPF_LDX | BPF_MEM | BPF_B: 1236 case BPF_LDX | BPF_MEM | BPF_H: 1237 case BPF_LDX | BPF_MEM | BPF_W: 1238 case BPF_LDX | BPF_MEM | BPF_DW: 1239 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1240 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1241 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1242 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1243 /* dst_reg = (s64)*(signed size *)(src_reg + off) */ 1244 case BPF_LDX | BPF_MEMSX | BPF_B: 1245 case BPF_LDX | BPF_MEMSX | BPF_H: 1246 case BPF_LDX | BPF_MEMSX | BPF_W: 1247 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1248 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1249 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1250 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + REG_ARENA + off) */ 1251 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1252 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1253 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1254 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1255 sign_extend = BPF_MODE(code) == BPF_MEMSX || 1256 BPF_MODE(code) == BPF_PROBE_MEMSX; 1257 1258 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1259 emit_insn(ctx, addd, t2, src, REG_ARENA); 1260 src = t2; 1261 } 1262 1263 switch (BPF_SIZE(code)) { 1264 case BPF_B: 1265 if (is_signed_imm12(off)) { 1266 if (sign_extend) 1267 emit_insn(ctx, ldb, dst, src, off); 1268 else 1269 emit_insn(ctx, ldbu, dst, src, off); 1270 } else { 1271 move_imm(ctx, t1, off, is32); 1272 if (sign_extend) 1273 emit_insn(ctx, ldxb, dst, src, t1); 1274 else 1275 emit_insn(ctx, ldxbu, dst, src, t1); 1276 } 1277 break; 1278 case BPF_H: 1279 if (is_signed_imm12(off)) { 1280 if (sign_extend) 1281 emit_insn(ctx, ldh, dst, src, off); 1282 else 1283 emit_insn(ctx, ldhu, dst, src, off); 1284 } else { 1285 move_imm(ctx, t1, off, is32); 1286 if (sign_extend) 1287 emit_insn(ctx, ldxh, dst, src, t1); 1288 else 1289 emit_insn(ctx, ldxhu, dst, src, t1); 1290 } 1291 break; 1292 case BPF_W: 1293 if (is_signed_imm12(off)) { 1294 if (sign_extend) 1295 emit_insn(ctx, ldw, dst, src, off); 1296 else 1297 emit_insn(ctx, ldwu, dst, src, off); 1298 } else { 1299 move_imm(ctx, t1, off, is32); 1300 if (sign_extend) 1301 emit_insn(ctx, ldxw, dst, src, t1); 1302 else 1303 emit_insn(ctx, ldxwu, dst, src, t1); 1304 } 1305 break; 1306 case BPF_DW: 1307 move_imm(ctx, t1, off, is32); 1308 emit_insn(ctx, ldxd, dst, src, t1); 1309 break; 1310 } 1311 1312 ret = add_exception_handler(insn, ctx, dst); 1313 if (ret) 1314 return ret; 1315 break; 1316 1317 /* *(size *)(dst + off) = imm */ 1318 case BPF_ST | BPF_MEM | BPF_B: 1319 case BPF_ST | BPF_MEM | BPF_H: 1320 case BPF_ST | BPF_MEM | BPF_W: 1321 case BPF_ST | BPF_MEM | BPF_DW: 1322 /* ST | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = imm */ 1323 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1324 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1325 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1326 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1327 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1328 emit_insn(ctx, addd, t3, dst, REG_ARENA); 1329 dst = t3; 1330 } 1331 1332 switch (BPF_SIZE(code)) { 1333 case BPF_B: 1334 move_imm(ctx, t1, imm, is32); 1335 if (is_signed_imm12(off)) { 1336 emit_insn(ctx, stb, t1, dst, off); 1337 } else { 1338 move_imm(ctx, t2, off, is32); 1339 emit_insn(ctx, stxb, t1, dst, t2); 1340 } 1341 break; 1342 case BPF_H: 1343 move_imm(ctx, t1, imm, is32); 1344 if (is_signed_imm12(off)) { 1345 emit_insn(ctx, sth, t1, dst, off); 1346 } else { 1347 move_imm(ctx, t2, off, is32); 1348 emit_insn(ctx, stxh, t1, dst, t2); 1349 } 1350 break; 1351 case BPF_W: 1352 move_imm(ctx, t1, imm, is32); 1353 if (is_signed_imm12(off)) { 1354 emit_insn(ctx, stw, t1, dst, off); 1355 } else if (is_signed_imm14(off)) { 1356 emit_insn(ctx, stptrw, t1, dst, off); 1357 } else { 1358 move_imm(ctx, t2, off, is32); 1359 emit_insn(ctx, stxw, t1, dst, t2); 1360 } 1361 break; 1362 case BPF_DW: 1363 move_imm(ctx, t1, imm, is32); 1364 if (is_signed_imm12(off)) { 1365 emit_insn(ctx, std, t1, dst, off); 1366 } else if (is_signed_imm14(off)) { 1367 emit_insn(ctx, stptrd, t1, dst, off); 1368 } else { 1369 move_imm(ctx, t2, off, is32); 1370 emit_insn(ctx, stxd, t1, dst, t2); 1371 } 1372 break; 1373 } 1374 1375 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1376 if (ret) 1377 return ret; 1378 break; 1379 1380 /* *(size *)(dst + off) = src */ 1381 case BPF_STX | BPF_MEM | BPF_B: 1382 case BPF_STX | BPF_MEM | BPF_H: 1383 case BPF_STX | BPF_MEM | BPF_W: 1384 case BPF_STX | BPF_MEM | BPF_DW: 1385 /* STX | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = src */ 1386 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1387 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1388 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1389 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1390 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1391 emit_insn(ctx, addd, t2, dst, REG_ARENA); 1392 dst = t2; 1393 } 1394 1395 switch (BPF_SIZE(code)) { 1396 case BPF_B: 1397 if (is_signed_imm12(off)) { 1398 emit_insn(ctx, stb, src, dst, off); 1399 } else { 1400 move_imm(ctx, t1, off, is32); 1401 emit_insn(ctx, stxb, src, dst, t1); 1402 } 1403 break; 1404 case BPF_H: 1405 if (is_signed_imm12(off)) { 1406 emit_insn(ctx, sth, src, dst, off); 1407 } else { 1408 move_imm(ctx, t1, off, is32); 1409 emit_insn(ctx, stxh, src, dst, t1); 1410 } 1411 break; 1412 case BPF_W: 1413 if (is_signed_imm12(off)) { 1414 emit_insn(ctx, stw, src, dst, off); 1415 } else if (is_signed_imm14(off)) { 1416 emit_insn(ctx, stptrw, src, dst, off); 1417 } else { 1418 move_imm(ctx, t1, off, is32); 1419 emit_insn(ctx, stxw, src, dst, t1); 1420 } 1421 break; 1422 case BPF_DW: 1423 if (is_signed_imm12(off)) { 1424 emit_insn(ctx, std, src, dst, off); 1425 } else if (is_signed_imm14(off)) { 1426 emit_insn(ctx, stptrd, src, dst, off); 1427 } else { 1428 move_imm(ctx, t1, off, is32); 1429 emit_insn(ctx, stxd, src, dst, t1); 1430 } 1431 break; 1432 } 1433 1434 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1435 if (ret) 1436 return ret; 1437 break; 1438 1439 /* Atomics */ 1440 case BPF_STX | BPF_ATOMIC | BPF_B: 1441 case BPF_STX | BPF_ATOMIC | BPF_H: 1442 case BPF_STX | BPF_ATOMIC | BPF_W: 1443 case BPF_STX | BPF_ATOMIC | BPF_DW: 1444 if (!bpf_atomic_is_load_store(insn)) 1445 ret = emit_atomic_rmw(insn, ctx); 1446 else 1447 ret = emit_atomic_ld_st(insn, ctx); 1448 if (ret) 1449 return ret; 1450 break; 1451 1452 /* Speculation barrier */ 1453 case BPF_ST | BPF_NOSPEC: 1454 break; 1455 1456 default: 1457 pr_err("bpf_jit: unknown opcode %02x\n", code); 1458 return -EINVAL; 1459 } 1460 1461 return 0; 1462 1463 toofar: 1464 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); 1465 return -E2BIG; 1466 } 1467 1468 static int build_body(struct jit_ctx *ctx, bool extra_pass) 1469 { 1470 int i; 1471 const struct bpf_prog *prog = ctx->prog; 1472 1473 for (i = 0; i < prog->len; i++) { 1474 const struct bpf_insn *insn = &prog->insnsi[i]; 1475 int ret; 1476 1477 if (ctx->image == NULL) 1478 ctx->offset[i] = ctx->idx; 1479 1480 ret = build_insn(insn, ctx, extra_pass); 1481 if (ret > 0) { 1482 i++; 1483 if (ctx->image == NULL) 1484 ctx->offset[i] = ctx->idx; 1485 continue; 1486 } 1487 if (ret) 1488 return ret; 1489 } 1490 1491 if (ctx->image == NULL) 1492 ctx->offset[i] = ctx->idx; 1493 1494 return 0; 1495 } 1496 1497 /* Fill space with break instructions */ 1498 static void jit_fill_hole(void *area, unsigned int size) 1499 { 1500 u32 *ptr; 1501 1502 /* We are guaranteed to have aligned memory */ 1503 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1504 *ptr++ = INSN_BREAK; 1505 } 1506 1507 static int validate_code(struct jit_ctx *ctx) 1508 { 1509 int i; 1510 union loongarch_instruction insn; 1511 1512 for (i = 0; i < ctx->idx; i++) { 1513 insn = ctx->image[i]; 1514 /* Check INSN_BREAK */ 1515 if (insn.word == INSN_BREAK) 1516 return -1; 1517 } 1518 1519 return 0; 1520 } 1521 1522 static int validate_ctx(struct jit_ctx *ctx) 1523 { 1524 if (validate_code(ctx)) 1525 return -1; 1526 1527 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) 1528 return -1; 1529 1530 return 0; 1531 } 1532 1533 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) 1534 { 1535 if (!target) { 1536 pr_err("bpf_jit: jump target address is error\n"); 1537 return -EFAULT; 1538 } 1539 1540 move_imm(ctx, LOONGARCH_GPR_T1, target, false); 1541 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); 1542 1543 return 0; 1544 } 1545 1546 static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 1547 { 1548 int i; 1549 struct jit_ctx ctx; 1550 1551 ctx.idx = 0; 1552 ctx.image = (union loongarch_instruction *)insns; 1553 1554 if (!target) { 1555 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1556 emit_insn((&ctx), nop); 1557 return 0; 1558 } 1559 1560 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target); 1561 } 1562 1563 static int emit_call(struct jit_ctx *ctx, u64 addr) 1564 { 1565 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); 1566 } 1567 1568 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 1569 { 1570 int ret; 1571 1572 cpus_read_lock(); 1573 mutex_lock(&text_mutex); 1574 ret = larch_insn_text_copy(dst, src, len); 1575 mutex_unlock(&text_mutex); 1576 cpus_read_unlock(); 1577 1578 return ret ? ERR_PTR(-EINVAL) : dst; 1579 } 1580 1581 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1582 enum bpf_text_poke_type new_t, void *old_addr, 1583 void *new_addr) 1584 { 1585 int ret; 1586 bool is_call; 1587 unsigned long size = 0; 1588 unsigned long offset = 0; 1589 void *image = NULL; 1590 char namebuf[KSYM_NAME_LEN]; 1591 u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1592 u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1593 1594 /* Only poking bpf text is supported. Since kernel function entry 1595 * is set up by ftrace, we rely on ftrace to poke kernel functions. 1596 */ 1597 if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) 1598 return -ENOTSUPP; 1599 1600 image = ip - offset; 1601 1602 /* zero offset means we're poking bpf prog entry */ 1603 if (offset == 0) { 1604 /* skip to the nop instruction in bpf prog entry: 1605 * move t0, ra 1606 * nop 1607 */ 1608 ip = image + LOONGARCH_INSN_SIZE; 1609 } 1610 1611 is_call = old_t == BPF_MOD_CALL; 1612 ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); 1613 if (ret) 1614 return ret; 1615 1616 if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1617 return -EFAULT; 1618 1619 is_call = new_t == BPF_MOD_CALL; 1620 ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call); 1621 if (ret) 1622 return ret; 1623 1624 cpus_read_lock(); 1625 mutex_lock(&text_mutex); 1626 if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1627 ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); 1628 mutex_unlock(&text_mutex); 1629 cpus_read_unlock(); 1630 1631 return ret; 1632 } 1633 1634 int bpf_arch_text_invalidate(void *dst, size_t len) 1635 { 1636 int i; 1637 int ret = 0; 1638 u32 *inst; 1639 1640 inst = kvmalloc(len, GFP_KERNEL); 1641 if (!inst) 1642 return -ENOMEM; 1643 1644 for (i = 0; i < (len / sizeof(u32)); i++) 1645 inst[i] = INSN_BREAK; 1646 1647 cpus_read_lock(); 1648 mutex_lock(&text_mutex); 1649 if (larch_insn_text_copy(dst, inst, len)) 1650 ret = -EINVAL; 1651 mutex_unlock(&text_mutex); 1652 cpus_read_unlock(); 1653 1654 kvfree(inst); 1655 1656 return ret; 1657 } 1658 1659 static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off) 1660 { 1661 int i; 1662 1663 for (i = 0; i < nr_arg_slots; i++) { 1664 if (i < LOONGARCH_MAX_REG_ARGS) 1665 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1666 else { 1667 /* Skip slots for T0 and FP of traced function */ 1668 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, 1669 16 + (i - LOONGARCH_MAX_REG_ARGS) * 8); 1670 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off); 1671 } 1672 args_off -= 8; 1673 } 1674 } 1675 1676 static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off) 1677 { 1678 int i; 1679 1680 for (i = 0; i < nr_reg_args; i++) { 1681 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1682 args_off -= 8; 1683 } 1684 } 1685 1686 static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, int stk_args_off) 1687 { 1688 int i; 1689 1690 for (i = 0; i < nr_stk_args; i++) { 1691 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, 1692 -(args_off - LOONGARCH_MAX_REG_ARGS * 8)); 1693 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -stk_args_off); 1694 args_off -= 8; 1695 stk_args_off -= 8; 1696 } 1697 } 1698 1699 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, 1700 int args_off, int retval_off, int run_ctx_off, bool save_ret) 1701 { 1702 int ret; 1703 u32 *branch; 1704 struct bpf_prog *p = n->link->prog; 1705 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 1706 1707 if (n->cookie) 1708 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, 1709 -run_ctx_off + cookie_off, n->cookie); 1710 else 1711 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1712 1713 /* arg1: prog */ 1714 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1715 /* arg2: &run_ctx */ 1716 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); 1717 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); 1718 if (ret) 1719 return ret; 1720 1721 /* store prog start time */ 1722 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); 1723 1724 /* 1725 * if (__bpf_prog_enter(prog) == 0) 1726 * goto skip_exec_of_prog; 1727 */ 1728 branch = (u32 *)ctx->image + ctx->idx; 1729 /* nop reserved for conditional jump */ 1730 emit_insn(ctx, nop); 1731 1732 /* arg1: &args_off */ 1733 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); 1734 if (!p->jited) 1735 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); 1736 ret = emit_call(ctx, (const u64)p->bpf_func); 1737 if (ret) 1738 return ret; 1739 1740 if (save_ret) { 1741 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1742 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1743 } 1744 1745 /* update branch with beqz */ 1746 if (ctx->image) { 1747 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; 1748 *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset); 1749 } 1750 1751 /* arg1: prog */ 1752 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1753 /* arg2: prog start time */ 1754 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); 1755 /* arg3: &run_ctx */ 1756 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); 1757 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); 1758 1759 return ret; 1760 } 1761 1762 static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, 1763 int args_off, int retval_off, int run_ctx_off, 1764 int func_meta_off, bool save_ret, u64 func_meta, int cookie_off) 1765 { 1766 int i, cur_cookie = (cookie_off - args_off) / 8; 1767 1768 for (i = 0; i < tn->nr_nodes; i++) { 1769 int err; 1770 1771 if (bpf_prog_calls_session_cookie(tn->nodes[i])) { 1772 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1773 1774 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta); 1775 cur_cookie--; 1776 } 1777 err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret); 1778 if (err) 1779 return err; 1780 } 1781 1782 return 0; 1783 } 1784 1785 void *arch_alloc_bpf_trampoline(unsigned int size) 1786 { 1787 return bpf_prog_pack_alloc(size, jit_fill_hole, false); 1788 } 1789 1790 void arch_free_bpf_trampoline(void *image, unsigned int size) 1791 { 1792 bpf_prog_pack_free(image, size); 1793 } 1794 1795 int arch_protect_bpf_trampoline(void *image, unsigned int size) 1796 { 1797 return 0; 1798 } 1799 1800 /* 1801 * Sign-extend the register if necessary 1802 */ 1803 static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign) 1804 { 1805 /* ABI requires unsigned char/short to be zero-extended */ 1806 if (!sign && (size == 1 || size == 2)) { 1807 if (rd != rj) 1808 move_reg(ctx, rd, rj); 1809 return; 1810 } 1811 1812 switch (size) { 1813 case 1: 1814 emit_insn(ctx, extwb, rd, rj); 1815 break; 1816 case 2: 1817 emit_insn(ctx, extwh, rd, rj); 1818 break; 1819 case 4: 1820 emit_insn(ctx, addiw, rd, rj, 0); 1821 break; 1822 case 8: 1823 if (rd != rj) 1824 move_reg(ctx, rd, rj); 1825 break; 1826 default: 1827 pr_warn("bpf_jit: invalid size %d for sign_extend\n", size); 1828 } 1829 } 1830 1831 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 1832 const struct btf_func_model *m, struct bpf_tramp_nodes *tnodes, 1833 void *func_addr, u32 flags) 1834 { 1835 int i, ret, save_ret; 1836 int cookie_cnt, cookie_off; 1837 int stack_size, args_off, stk_args_off, nr_arg_slots = 0; 1838 int retval_off, func_meta_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; 1839 unsigned long long func_meta; 1840 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 1841 void *orig_call = func_addr; 1842 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 1843 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 1844 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 1845 u32 **branches = NULL; 1846 1847 /* 1848 * FP + 8 [ RA to parent func ] return address to parent 1849 * function 1850 * FP + 0 [ FP of parent func ] frame pointer of parent 1851 * function 1852 * FP - 8 [ T0 to traced func ] return address of traced 1853 * function 1854 * FP - 16 [ FP of traced func ] frame pointer of traced 1855 * function 1856 * 1857 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 1858 * BPF_TRAMP_F_RET_FENTRY_RET 1859 * [ arg regN ] 1860 * [ ... ] 1861 * FP - args_off [ arg reg1 ] 1862 * 1863 * FP - func_meta_off [ regs count, etc ] 1864 * 1865 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 1866 * 1867 * [ stack cookie N ] 1868 * [ ... ] 1869 * FP - cookie_off [ stack cookie 1 ] 1870 * 1871 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 1872 * 1873 * FP - sreg_off [ callee saved reg ] 1874 * 1875 * FP - tcc_ptr_off [ tail_call_cnt_ptr ] 1876 * 1877 * [ stack_argN ] 1878 * [ ... ] 1879 * FP - stk_args_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG 1880 */ 1881 1882 if (m->nr_args > MAX_BPF_FUNC_ARGS) 1883 return -ENOTSUPP; 1884 1885 /* Extra registers for struct arguments */ 1886 for (i = 0; i < m->nr_args; i++) { 1887 /* 1888 * The struct argument size is at most 16 bytes, 1889 * enforced by the verifier. The struct argument 1890 * may be passed in a pair of registers if its 1891 * size is more than 8 bytes and no more than 16 1892 * bytes. 1893 */ 1894 nr_arg_slots += round_up(m->arg_size[i], 8) / 8; 1895 } 1896 1897 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 1898 return -ENOTSUPP; 1899 1900 /* Room of trampoline frame to store return address and frame pointer */ 1901 stack_size = 16; 1902 1903 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 1904 if (save_ret) 1905 stack_size += 16; /* Save BPF R0 and A0 */ 1906 1907 retval_off = stack_size; 1908 1909 /* Room of trampoline frame to store args */ 1910 stack_size += nr_arg_slots * 8; 1911 args_off = stack_size; 1912 1913 /* Room of function metadata, such as regs count */ 1914 stack_size += 8; 1915 func_meta_off = stack_size; 1916 1917 /* Room of trampoline frame to store ip address */ 1918 if (flags & BPF_TRAMP_F_IP_ARG) { 1919 stack_size += 8; 1920 ip_off = stack_size; 1921 } 1922 1923 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 1924 1925 /* Room for session cookies */ 1926 stack_size += cookie_cnt * 8; 1927 cookie_off = stack_size; 1928 1929 /* Room of trampoline frame to store struct bpf_tramp_run_ctx */ 1930 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 1931 run_ctx_off = stack_size; 1932 1933 stack_size += 8; 1934 sreg_off = stack_size; 1935 1936 /* Room of trampoline frame to store tail_call_cnt_ptr */ 1937 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { 1938 stack_size += 8; 1939 tcc_ptr_off = stack_size; 1940 } 1941 1942 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0)) 1943 stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8; 1944 1945 stack_size = round_up(stack_size, 16); 1946 1947 /* Room for args on stack must be at the top of stack */ 1948 stk_args_off = stack_size; 1949 1950 if (is_struct_ops) { 1951 /* 1952 * For the trampoline called directly, just handle 1953 * the frame of trampoline. 1954 */ 1955 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1956 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1957 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1958 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1959 } else { 1960 /* 1961 * For the trampoline called from function entry, 1962 * the frame of traced function and the frame of 1963 * trampoline need to be considered. 1964 */ 1965 /* RA and FP for parent function */ 1966 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); 1967 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1968 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1969 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); 1970 1971 /* RA and FP for traced function */ 1972 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1973 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1974 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1975 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1976 } 1977 1978 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1979 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1980 1981 /* callee saved register S1 to pass start time */ 1982 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1983 1984 /* store ip address of the traced function */ 1985 if (flags & BPF_TRAMP_F_IP_ARG) 1986 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -ip_off, (u64)func_addr); 1987 1988 /* store arg regs count */ 1989 func_meta = nr_arg_slots; 1990 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); 1991 1992 store_args(ctx, nr_arg_slots, args_off); 1993 1994 if (bpf_fsession_cnt(tnodes)) { 1995 /* clear all session cookies' value */ 1996 for (i = 0; i < cookie_cnt; i++) 1997 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -cookie_off + 8 * i); 1998 1999 /* clear return value to make sure fentry always get 0 */ 2000 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 2001 } 2002 2003 /* To traced function */ 2004 /* Ftrace jump skips 2 NOP instructions */ 2005 if (is_kernel_text((unsigned long)orig_call) || 2006 is_module_text_address((unsigned long)orig_call)) 2007 orig_call += LOONGARCH_FENTRY_NBYTES; 2008 /* Direct jump skips 5 NOP instructions */ 2009 else if (is_bpf_text_address((unsigned long)orig_call)) 2010 orig_call += LOONGARCH_BPF_FENTRY_NBYTES; 2011 2012 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2013 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 2014 ret = emit_call(ctx, (const u64)__bpf_tramp_enter); 2015 if (ret) 2016 return ret; 2017 } 2018 2019 if (fentry->nr_nodes) { 2020 ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off, 2021 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off); 2022 if (ret) 2023 return ret; 2024 } 2025 if (fmod_ret->nr_nodes) { 2026 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32 *), GFP_KERNEL); 2027 if (!branches) 2028 return -ENOMEM; 2029 2030 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 2031 for (i = 0; i < fmod_ret->nr_nodes; i++) { 2032 ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i], 2033 args_off, retval_off, run_ctx_off, true); 2034 if (ret) 2035 goto out; 2036 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); 2037 branches[i] = (u32 *)ctx->image + ctx->idx; 2038 emit_insn(ctx, nop); 2039 } 2040 } 2041 2042 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2043 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); 2044 restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_args_off); 2045 2046 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2047 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 2048 2049 ret = emit_call(ctx, (const u64)orig_call); 2050 if (ret) 2051 goto out; 2052 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 2053 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 2054 im->ip_after_call = ctx->ro_image + ctx->idx; 2055 /* Reserve space for the move_imm + jirl instruction */ 2056 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 2057 emit_insn(ctx, nop); 2058 } 2059 2060 for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) { 2061 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; 2062 *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset); 2063 } 2064 2065 /* Set "is_return" flag for fsession */ 2066 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2067 if (bpf_fsession_cnt(tnodes)) 2068 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); 2069 2070 if (fexit->nr_nodes) { 2071 ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off, 2072 func_meta_off, false, func_meta, cookie_off); 2073 if (ret) 2074 goto out; 2075 } 2076 2077 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2078 im->ip_epilogue = ctx->ro_image + ctx->idx; 2079 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 2080 ret = emit_call(ctx, (const u64)__bpf_tramp_exit); 2081 if (ret) 2082 goto out; 2083 } 2084 2085 if (flags & BPF_TRAMP_F_RESTORE_REGS) 2086 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); 2087 2088 if (save_ret) { 2089 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 2090 if (is_struct_ops) 2091 sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 2092 m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG); 2093 else 2094 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 2095 } 2096 2097 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 2098 2099 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2100 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 2101 2102 if (is_struct_ops) { 2103 /* trampoline called directly */ 2104 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 2105 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 2106 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 2107 2108 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 2109 } else { 2110 /* trampoline called from function entry */ 2111 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 2112 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 2113 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 2114 2115 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 2116 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 2117 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); 2118 2119 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 2120 /* return to parent function */ 2121 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 2122 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); 2123 } else { 2124 /* return to traced function */ 2125 move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA); 2126 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 2127 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0); 2128 } 2129 } 2130 2131 ret = ctx->idx; 2132 out: 2133 kfree(branches); 2134 2135 return ret; 2136 } 2137 2138 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, 2139 void *ro_image_end, const struct btf_func_model *m, 2140 u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) 2141 { 2142 int ret, size; 2143 void *image, *tmp; 2144 struct jit_ctx ctx; 2145 2146 size = ro_image_end - ro_image; 2147 image = kvmalloc(size, GFP_KERNEL); 2148 if (!image) 2149 return -ENOMEM; 2150 2151 ctx.image = (union loongarch_instruction *)image; 2152 ctx.ro_image = (union loongarch_instruction *)ro_image; 2153 ctx.idx = 0; 2154 2155 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); 2156 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tnodes, func_addr, flags); 2157 if (ret < 0) 2158 goto out; 2159 2160 if (validate_code(&ctx) < 0) { 2161 ret = -EINVAL; 2162 goto out; 2163 } 2164 2165 tmp = bpf_arch_text_copy(ro_image, image, size); 2166 if (IS_ERR(tmp)) { 2167 ret = PTR_ERR(tmp); 2168 goto out; 2169 } 2170 2171 out: 2172 kvfree(image); 2173 return ret < 0 ? ret : size; 2174 } 2175 2176 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 2177 struct bpf_tramp_nodes *tnodes, void *func_addr) 2178 { 2179 int ret; 2180 struct jit_ctx ctx; 2181 struct bpf_tramp_image im; 2182 2183 ctx.image = NULL; 2184 ctx.idx = 0; 2185 2186 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags); 2187 2188 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; 2189 } 2190 2191 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2192 { 2193 bool extra_pass = false; 2194 u8 *image_ptr, *ro_image_ptr; 2195 int image_size, prog_size, extable_size; 2196 struct jit_ctx ctx; 2197 struct jit_data *jit_data; 2198 struct bpf_binary_header *header; 2199 struct bpf_binary_header *ro_header; 2200 2201 /* 2202 * If BPF JIT was not enabled then we must fall back to 2203 * the interpreter. 2204 */ 2205 if (!prog->jit_requested) 2206 return prog; 2207 2208 jit_data = prog->aux->jit_data; 2209 if (!jit_data) { 2210 jit_data = kzalloc_obj(*jit_data); 2211 if (!jit_data) 2212 return prog; 2213 prog->aux->jit_data = jit_data; 2214 } 2215 if (jit_data->ctx.offset) { 2216 ctx = jit_data->ctx; 2217 ro_header = jit_data->ro_header; 2218 ro_image_ptr = (void *)ctx.ro_image; 2219 header = jit_data->header; 2220 image_ptr = (void *)header + ((void *)ro_image_ptr - (void *)ro_header); 2221 extra_pass = true; 2222 prog_size = sizeof(u32) * ctx.idx; 2223 goto skip_init_ctx; 2224 } 2225 2226 memset(&ctx, 0, sizeof(ctx)); 2227 ctx.prog = prog; 2228 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); 2229 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 2230 2231 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 2232 if (ctx.offset == NULL) 2233 goto out_offset; 2234 2235 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 2236 build_prologue(&ctx); 2237 if (build_body(&ctx, extra_pass)) 2238 goto out_offset; 2239 ctx.epilogue_offset = ctx.idx; 2240 build_epilogue(&ctx); 2241 2242 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); 2243 2244 /* Now we know the actual image size. 2245 * As each LoongArch instruction is of length 32bit, 2246 * we are translating number of JITed intructions into 2247 * the size required to store these JITed code. 2248 */ 2249 prog_size = sizeof(u32) * ctx.idx; 2250 image_size = prog_size + extable_size; 2251 /* Now we know the size of the structure to make */ 2252 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), 2253 &header, &image_ptr, jit_fill_hole, 2254 bpf_prog_was_classic(prog)); 2255 if (!ro_header) 2256 goto out_offset; 2257 2258 /* 2. Now, the actual pass to generate final JIT code */ 2259 /* 2260 * Use the image (RW) for writing the JITed instructions. But also save 2261 * the ro_image (RX) for calculating the offsets in the image. The RW 2262 * image will be later copied to the RX image from where the program will 2263 * run. The bpf_jit_binary_pack_finalize() will do this copy in the final 2264 * step. 2265 */ 2266 ctx.image = (union loongarch_instruction *)image_ptr; 2267 ctx.ro_image = (union loongarch_instruction *)ro_image_ptr; 2268 if (extable_size) 2269 prog->aux->extable = (void *)ro_image_ptr + prog_size; 2270 2271 skip_init_ctx: 2272 ctx.idx = 0; 2273 ctx.num_exentries = 0; 2274 2275 build_prologue(&ctx); 2276 if (build_body(&ctx, extra_pass)) 2277 goto out_free; 2278 build_epilogue(&ctx); 2279 2280 /* 3. Extra pass to validate JITed code */ 2281 if (validate_ctx(&ctx)) 2282 goto out_free; 2283 2284 /* And we're done */ 2285 if (bpf_jit_enable > 1) 2286 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 2287 2288 if (!prog->is_func || extra_pass) { 2289 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 2290 pr_err_once("multi-func JIT bug %d != %d\n", 2291 ctx.idx, jit_data->ctx.idx); 2292 goto out_free; 2293 } 2294 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { 2295 /* ro_header and header have been freed */ 2296 ro_header = NULL; 2297 header = NULL; 2298 goto out_free; 2299 } 2300 /* 2301 * The instructions have now been copied to the ROX region from 2302 * where they will execute. Now the data cache has to be cleaned 2303 * to the PoU and the I-cache has to be invalidated for the VAs. 2304 */ 2305 bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx); 2306 } else { 2307 jit_data->ctx = ctx; 2308 jit_data->header = header; 2309 jit_data->ro_header = ro_header; 2310 } 2311 prog->jited = 1; 2312 prog->jited_len = prog_size; 2313 prog->bpf_func = (void *)ctx.ro_image; 2314 2315 if (!prog->is_func || extra_pass) { 2316 int i; 2317 2318 /* offset[prog->len] is the size of program */ 2319 for (i = 0; i <= prog->len; i++) 2320 ctx.offset[i] *= LOONGARCH_INSN_SIZE; 2321 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 2322 2323 out_offset: 2324 kvfree(ctx.offset); 2325 kfree(jit_data); 2326 prog->aux->jit_data = NULL; 2327 } 2328 2329 return prog; 2330 2331 out_free: 2332 if (extra_pass) { 2333 prog->bpf_func = NULL; 2334 prog->jited = 0; 2335 prog->jited_len = 0; 2336 } 2337 2338 if (header) { 2339 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); 2340 bpf_jit_binary_pack_free(ro_header, header); 2341 } 2342 goto out_offset; 2343 } 2344 2345 void bpf_jit_free(struct bpf_prog *prog) 2346 { 2347 if (prog->jited) { 2348 struct jit_data *jit_data = prog->aux->jit_data; 2349 struct bpf_binary_header *hdr; 2350 2351 /* 2352 * If we fail the final pass of JIT (from jit_subprogs), the 2353 * program may not be finalized yet. Call finalize here before 2354 * freeing it. 2355 */ 2356 if (jit_data) { 2357 bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); 2358 kvfree(jit_data->ctx.offset); 2359 kfree(jit_data); 2360 } 2361 hdr = bpf_jit_binary_pack_hdr(prog); 2362 bpf_jit_binary_pack_free(hdr, NULL); 2363 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); 2364 } 2365 2366 bpf_prog_unlock_free(prog); 2367 } 2368 2369 #if defined(CONFIG_UNWINDER_ORC) 2370 #include <asm/unwind.h> 2371 2372 static noinline void walk_bpf_stackframe(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), 2373 void *cookie, unsigned long fp) 2374 { 2375 unsigned long addr; 2376 struct unwind_state state; 2377 struct pt_regs dummyregs; 2378 struct pt_regs *regs = &dummyregs; 2379 2380 regs->regs[1] = 0; 2381 regs->regs[22] = fp; 2382 regs->regs[3] = (unsigned long)__builtin_frame_address(0); 2383 regs->csr_era = (unsigned long)__builtin_return_address(0); 2384 2385 for (unwind_start(&state, current, regs); 2386 !unwind_done(&state); unwind_next_frame(&state)) { 2387 addr = unwind_get_return_address(&state); 2388 if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.fp)) 2389 break; 2390 } 2391 } 2392 2393 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) 2394 { 2395 unsigned long fp; 2396 2397 /* 2398 * Capture the live frame pointer ($r22) at the very front-line before 2399 * any kernel C code clobbers it. This must be a thin wrapper with no 2400 * large stack locals to prevent the compiler from reusing $r22 early. 2401 */ 2402 asm volatile("move %0, $r22" : "=r"(fp)); 2403 walk_bpf_stackframe(consume_fn, cookie, fp); 2404 } 2405 #endif /* CONFIG_UNWINDER_ORC */ 2406 2407 bool bpf_jit_bypass_spec_v1(void) 2408 { 2409 return true; 2410 } 2411 2412 bool bpf_jit_bypass_spec_v4(void) 2413 { 2414 return true; 2415 } 2416 2417 bool bpf_jit_supports_arena(void) 2418 { 2419 return true; 2420 } 2421 2422 bool bpf_jit_supports_fsession(void) 2423 { 2424 return true; 2425 } 2426 2427 bool bpf_jit_supports_percpu_insn(void) 2428 { 2429 return true; 2430 } 2431 2432 bool bpf_jit_supports_ptr_xchg(void) 2433 { 2434 return true; 2435 } 2436 2437 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ 2438 bool bpf_jit_supports_subprog_tailcalls(void) 2439 { 2440 return true; 2441 } 2442 2443 bool bpf_jit_supports_timed_may_goto(void) 2444 { 2445 return true; 2446 } 2447 2448 bool bpf_jit_inlines_helper_call(s32 imm) 2449 { 2450 switch (imm) { 2451 case BPF_FUNC_get_current_task: 2452 case BPF_FUNC_get_current_task_btf: 2453 case BPF_FUNC_get_smp_processor_id: 2454 return true; 2455 default: 2456 return false; 2457 } 2458 } 2459