1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/memory.h> 8 #include <asm/asm-offsets.h> 9 #include "bpf_jit.h" 10 11 #define LOONGARCH_MAX_REG_ARGS 8 12 13 #define LOONGARCH_SAVE_RA_NINSNS 1 14 #define LOONGARCH_LONG_JUMP_NINSNS 5 15 #define LOONGARCH_TCC_SLOT_NINSNS 1 16 17 #define LOONGARCH_PROLOGUE_SKIP_INSNS \ 18 (LOONGARCH_SAVE_RA_NINSNS + LOONGARCH_LONG_JUMP_NINSNS + LOONGARCH_TCC_SLOT_NINSNS) 19 20 #define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 21 22 #define LOONGARCH_FENTRY_NINSNS 2 23 #define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) 24 #define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) 25 26 #define REG_TCC LOONGARCH_GPR_A6 27 #define REG_ARENA LOONGARCH_GPR_S6 /* For storing arena_vm_start */ 28 #define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) 29 30 static const int regmap[] = { 31 /* return value from in-kernel function, and exit value for eBPF program */ 32 [BPF_REG_0] = LOONGARCH_GPR_A5, 33 /* arguments from eBPF program to in-kernel function */ 34 [BPF_REG_1] = LOONGARCH_GPR_A0, 35 [BPF_REG_2] = LOONGARCH_GPR_A1, 36 [BPF_REG_3] = LOONGARCH_GPR_A2, 37 [BPF_REG_4] = LOONGARCH_GPR_A3, 38 [BPF_REG_5] = LOONGARCH_GPR_A4, 39 /* callee saved registers that in-kernel function will preserve */ 40 [BPF_REG_6] = LOONGARCH_GPR_S0, 41 [BPF_REG_7] = LOONGARCH_GPR_S1, 42 [BPF_REG_8] = LOONGARCH_GPR_S2, 43 [BPF_REG_9] = LOONGARCH_GPR_S3, 44 /* read-only frame pointer to access stack */ 45 [BPF_REG_FP] = LOONGARCH_GPR_S4, 46 /* temporary register for blinding constants */ 47 [BPF_REG_AX] = LOONGARCH_GPR_T0, 48 }; 49 50 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) 51 { 52 const struct bpf_prog *prog = ctx->prog; 53 const bool is_main_prog = !bpf_is_subprog(prog); 54 55 if (is_main_prog) { 56 /* 57 * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT 58 * if (REG_TCC > T3 ) 59 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 60 * else 61 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 62 * REG_TCC = LOONGARCH_GPR_SP + store_offset 63 * 64 * std REG_TCC -> LOONGARCH_GPR_SP + store_offset 65 * 66 * The purpose of this code is to first push the TCC into stack, 67 * and then push the address of TCC into stack. 68 * In cases where bpf2bpf and tailcall are used in combination, 69 * the value in REG_TCC may be a count or an address, 70 * these two cases need to be judged and handled separately. 71 */ 72 emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 73 *store_offset -= sizeof(long); 74 75 emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); 76 77 /* 78 * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, 79 * push tcc into stack 80 */ 81 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 82 83 /* Push the address of TCC into the REG_TCC */ 84 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 85 86 emit_uncond_jmp(ctx, 2); 87 88 /* 89 * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, 90 * push tcc_ptr into stack 91 */ 92 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 93 } else { 94 *store_offset -= sizeof(long); 95 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 96 } 97 98 /* Push tcc_ptr into stack */ 99 *store_offset -= sizeof(long); 100 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); 101 } 102 103 /* 104 * eBPF prog stack layout: 105 * 106 * high 107 * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP 108 * | $ra | 109 * +-------------------------+ 110 * | $fp | 111 * +-------------------------+ 112 * | $s0 | 113 * +-------------------------+ 114 * | $s1 | 115 * +-------------------------+ 116 * | $s2 | 117 * +-------------------------+ 118 * | $s3 | 119 * +-------------------------+ 120 * | $s4 | 121 * +-------------------------+ 122 * | $s5 | 123 * +-------------------------+ 124 * | tcc | 125 * +-------------------------+ 126 * | tcc_ptr | 127 * +-------------------------+ <--BPF_REG_FP 128 * | prog->aux->stack_depth | 129 * | (optional) | 130 * current $sp -------------> +-------------------------+ 131 * low 132 */ 133 static void build_prologue(struct jit_ctx *ctx) 134 { 135 int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 136 const struct bpf_prog *prog = ctx->prog; 137 const bool is_main_prog = !bpf_is_subprog(prog); 138 139 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 140 141 /* To store ra, fp, s0, s1, s2, s3, s4, s5 */ 142 stack_adjust += sizeof(long) * 8; 143 144 /* To store tcc and tcc_ptr */ 145 stack_adjust += sizeof(long) * 2; 146 147 if (ctx->arena_vm_start) 148 stack_adjust += 8; 149 150 stack_adjust = round_up(stack_adjust, 16); 151 stack_adjust += bpf_stack_adjust; 152 153 /* 154 * Save the original return address to a temporary register to prevent 155 * it from being overwritten, then reserve space for the long jump and 156 * fentry trampoline slot for dynamically patching by ftrace at runtime. 157 * These instructions are bypassed during a tail call invocation. 158 */ 159 move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); 160 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 161 emit_insn(ctx, nop); 162 163 /* 164 * First instruction initializes the tail call count (TCC) 165 * register to zero. On tail call we skip this instruction, 166 * and the TCC is passed in REG_TCC from the caller. 167 */ 168 if (is_main_prog) 169 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); 170 171 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); 172 173 store_offset = stack_adjust - sizeof(long); 174 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); 175 176 store_offset -= sizeof(long); 177 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); 178 179 store_offset -= sizeof(long); 180 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); 181 182 store_offset -= sizeof(long); 183 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); 184 185 store_offset -= sizeof(long); 186 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); 187 188 store_offset -= sizeof(long); 189 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); 190 191 store_offset -= sizeof(long); 192 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); 193 194 store_offset -= sizeof(long); 195 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); 196 197 if (ctx->arena_vm_start) { 198 store_offset -= sizeof(long); 199 emit_insn(ctx, std, REG_ARENA, LOONGARCH_GPR_SP, store_offset); 200 } 201 202 prepare_bpf_tail_call_cnt(ctx, &store_offset); 203 204 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); 205 206 if (bpf_stack_adjust) 207 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); 208 209 ctx->stack_size = stack_adjust; 210 211 if (ctx->arena_vm_start) 212 move_imm(ctx, REG_ARENA, ctx->arena_vm_start, false); 213 } 214 215 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) 216 { 217 int stack_adjust = ctx->stack_size; 218 int load_offset; 219 220 load_offset = stack_adjust - sizeof(long); 221 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); 222 223 load_offset -= sizeof(long); 224 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); 225 226 load_offset -= sizeof(long); 227 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); 228 229 load_offset -= sizeof(long); 230 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); 231 232 load_offset -= sizeof(long); 233 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); 234 235 load_offset -= sizeof(long); 236 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); 237 238 load_offset -= sizeof(long); 239 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); 240 241 load_offset -= sizeof(long); 242 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); 243 244 if (ctx->arena_vm_start) { 245 load_offset -= sizeof(long); 246 emit_insn(ctx, ldd, REG_ARENA, LOONGARCH_GPR_SP, load_offset); 247 } 248 249 /* 250 * When push into the stack, follow the order of tcc then tcc_ptr. 251 * When pop from the stack, first pop tcc_ptr then followed by tcc. 252 */ 253 load_offset -= 2 * sizeof(long); 254 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 255 256 load_offset += sizeof(long); 257 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); 258 259 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); 260 261 if (!is_tail_call) { 262 /* Set return value */ 263 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); 264 /* Return to the caller */ 265 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 266 } else { 267 /* 268 * Tail call to the next BPF program, passing offset in number 269 * of instructions to jirl to bypass the initial setup slots. 270 */ 271 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, 272 LOONGARCH_GPR_T3, LOONGARCH_PROLOGUE_SKIP_INSNS); 273 } 274 } 275 276 static void build_epilogue(struct jit_ctx *ctx) 277 { 278 __build_epilogue(ctx, false); 279 } 280 281 bool bpf_jit_supports_kfunc_call(void) 282 { 283 return true; 284 } 285 286 bool bpf_jit_supports_far_kfunc_call(void) 287 { 288 return true; 289 } 290 291 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) 292 { 293 int off, tc_ninsn = 0; 294 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 295 u8 a1 = LOONGARCH_GPR_A1; 296 u8 a2 = LOONGARCH_GPR_A2; 297 u8 t1 = LOONGARCH_GPR_T1; 298 u8 t2 = LOONGARCH_GPR_T2; 299 u8 t3 = LOONGARCH_GPR_T3; 300 const int idx0 = ctx->idx; 301 302 #define cur_offset (ctx->idx - idx0) 303 #define jmp_offset (tc_ninsn - (cur_offset)) 304 305 /* 306 * a0: &ctx 307 * a1: &array 308 * a2: index 309 * 310 * if (index >= array->map.max_entries) 311 * goto out; 312 */ 313 tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; 314 emit_zext_32(ctx, a2, true); 315 316 off = offsetof(struct bpf_array, map.max_entries); 317 emit_insn(ctx, ldwu, t1, a1, off); 318 /* bgeu $a2, $t1, jmp_offset */ 319 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) 320 goto toofar; 321 322 /* 323 * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) 324 * goto out; 325 */ 326 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 327 emit_insn(ctx, ldd, t3, REG_TCC, 0); 328 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); 329 if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) 330 goto toofar; 331 332 emit_insn(ctx, addid, t3, t3, 1); 333 334 /* 335 * prog = array->ptrs[index]; 336 * if (!prog) 337 * goto out; 338 */ 339 emit_insn(ctx, alsld, t2, a2, a1, 2); 340 off = offsetof(struct bpf_array, ptrs); 341 emit_insn(ctx, ldd, t2, t2, off); 342 /* beq $t2, $zero, jmp_offset */ 343 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 344 goto toofar; 345 346 emit_insn(ctx, std, t3, REG_TCC, 0); 347 348 /* goto *(prog->bpf_func + 4); */ 349 off = offsetof(struct bpf_prog, bpf_func); 350 emit_insn(ctx, ldd, t3, t2, off); 351 __build_epilogue(ctx, true); 352 353 return 0; 354 355 toofar: 356 pr_info_once("tail_call: jump too far\n"); 357 return -1; 358 #undef cur_offset 359 #undef jmp_offset 360 } 361 362 static void emit_store_stack_imm64(struct jit_ctx *ctx, int reg, int stack_off, u64 imm64) 363 { 364 move_imm(ctx, reg, imm64, false); 365 emit_insn(ctx, std, reg, LOONGARCH_GPR_FP, stack_off); 366 } 367 368 static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx) 369 { 370 const u8 t1 = LOONGARCH_GPR_T1; 371 const u8 t2 = LOONGARCH_GPR_T2; 372 const u8 t3 = LOONGARCH_GPR_T3; 373 const u8 r0 = regmap[BPF_REG_0]; 374 const u8 src = regmap[insn->src_reg]; 375 const u8 dst = regmap[insn->dst_reg]; 376 const s16 off = insn->off; 377 const s32 imm = insn->imm; 378 const bool isdw = BPF_SIZE(insn->code) == BPF_DW; 379 380 move_imm(ctx, t1, off, false); 381 emit_insn(ctx, addd, t1, dst, t1); 382 move_reg(ctx, t3, src); 383 384 switch (imm) { 385 /* lock *(size *)(dst + off) <op>= src */ 386 case BPF_ADD: 387 switch (BPF_SIZE(insn->code)) { 388 case BPF_B: 389 if (!cpu_has_lam_bh) { 390 pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); 391 return -EINVAL; 392 } 393 emit_insn(ctx, amaddb, t2, t1, src); 394 break; 395 case BPF_H: 396 if (!cpu_has_lam_bh) { 397 pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); 398 return -EINVAL; 399 } 400 emit_insn(ctx, amaddh, t2, t1, src); 401 break; 402 case BPF_W: 403 emit_insn(ctx, amaddw, t2, t1, src); 404 break; 405 case BPF_DW: 406 emit_insn(ctx, amaddd, t2, t1, src); 407 break; 408 } 409 break; 410 case BPF_AND: 411 if (isdw) 412 emit_insn(ctx, amandd, t2, t1, src); 413 else 414 emit_insn(ctx, amandw, t2, t1, src); 415 break; 416 case BPF_OR: 417 if (isdw) 418 emit_insn(ctx, amord, t2, t1, src); 419 else 420 emit_insn(ctx, amorw, t2, t1, src); 421 break; 422 case BPF_XOR: 423 if (isdw) 424 emit_insn(ctx, amxord, t2, t1, src); 425 else 426 emit_insn(ctx, amxorw, t2, t1, src); 427 break; 428 /* src = atomic_fetch_<op>(dst + off, src) */ 429 case BPF_ADD | BPF_FETCH: 430 switch (BPF_SIZE(insn->code)) { 431 case BPF_B: 432 if (!cpu_has_lam_bh) { 433 pr_err_once("bpf-jit: amadd.b instruction is not supported\n"); 434 return -EINVAL; 435 } 436 emit_insn(ctx, amaddb, src, t1, t3); 437 emit_zext_32(ctx, src, true); 438 break; 439 case BPF_H: 440 if (!cpu_has_lam_bh) { 441 pr_err_once("bpf-jit: amadd.h instruction is not supported\n"); 442 return -EINVAL; 443 } 444 emit_insn(ctx, amaddh, src, t1, t3); 445 emit_zext_32(ctx, src, true); 446 break; 447 case BPF_W: 448 emit_insn(ctx, amaddw, src, t1, t3); 449 emit_zext_32(ctx, src, true); 450 break; 451 case BPF_DW: 452 emit_insn(ctx, amaddd, src, t1, t3); 453 break; 454 } 455 break; 456 case BPF_AND | BPF_FETCH: 457 if (isdw) { 458 emit_insn(ctx, amandd, src, t1, t3); 459 } else { 460 emit_insn(ctx, amandw, src, t1, t3); 461 emit_zext_32(ctx, src, true); 462 } 463 break; 464 case BPF_OR | BPF_FETCH: 465 if (isdw) { 466 emit_insn(ctx, amord, src, t1, t3); 467 } else { 468 emit_insn(ctx, amorw, src, t1, t3); 469 emit_zext_32(ctx, src, true); 470 } 471 break; 472 case BPF_XOR | BPF_FETCH: 473 if (isdw) { 474 emit_insn(ctx, amxord, src, t1, t3); 475 } else { 476 emit_insn(ctx, amxorw, src, t1, t3); 477 emit_zext_32(ctx, src, true); 478 } 479 break; 480 /* src = atomic_xchg(dst + off, src); */ 481 case BPF_XCHG: 482 switch (BPF_SIZE(insn->code)) { 483 case BPF_B: 484 if (!cpu_has_lam_bh) { 485 pr_err_once("bpf-jit: amswap.b instruction is not supported\n"); 486 return -EINVAL; 487 } 488 emit_insn(ctx, amswapb, src, t1, t3); 489 emit_zext_32(ctx, src, true); 490 break; 491 case BPF_H: 492 if (!cpu_has_lam_bh) { 493 pr_err_once("bpf-jit: amswap.h instruction is not supported\n"); 494 return -EINVAL; 495 } 496 emit_insn(ctx, amswaph, src, t1, t3); 497 emit_zext_32(ctx, src, true); 498 break; 499 case BPF_W: 500 emit_insn(ctx, amswapw, src, t1, t3); 501 emit_zext_32(ctx, src, true); 502 break; 503 case BPF_DW: 504 emit_insn(ctx, amswapd, src, t1, t3); 505 break; 506 } 507 break; 508 /* r0 = atomic_cmpxchg(dst + off, r0, src); */ 509 case BPF_CMPXCHG: 510 move_reg(ctx, t2, r0); 511 if (isdw) { 512 emit_insn(ctx, lld, r0, t1, 0); 513 emit_insn(ctx, bne, t2, r0, 4); 514 move_reg(ctx, t3, src); 515 emit_insn(ctx, scd, t3, t1, 0); 516 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); 517 } else { 518 emit_insn(ctx, llw, r0, t1, 0); 519 emit_zext_32(ctx, t2, true); 520 emit_zext_32(ctx, r0, true); 521 emit_insn(ctx, bne, t2, r0, 4); 522 move_reg(ctx, t3, src); 523 emit_insn(ctx, scw, t3, t1, 0); 524 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); 525 emit_zext_32(ctx, r0, true); 526 } 527 break; 528 default: 529 pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm); 530 return -EINVAL; 531 } 532 533 return 0; 534 } 535 536 static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx) 537 { 538 const u8 t1 = LOONGARCH_GPR_T1; 539 const u8 src = regmap[insn->src_reg]; 540 const u8 dst = regmap[insn->dst_reg]; 541 const s16 off = insn->off; 542 const s32 imm = insn->imm; 543 544 switch (imm) { 545 /* dst_reg = load_acquire(src_reg + off16) */ 546 case BPF_LOAD_ACQ: 547 switch (BPF_SIZE(insn->code)) { 548 case BPF_B: 549 if (is_signed_imm12(off)) { 550 emit_insn(ctx, ldbu, dst, src, off); 551 } else { 552 move_imm(ctx, t1, off, false); 553 emit_insn(ctx, ldxbu, dst, src, t1); 554 } 555 break; 556 case BPF_H: 557 if (is_signed_imm12(off)) { 558 emit_insn(ctx, ldhu, dst, src, off); 559 } else { 560 move_imm(ctx, t1, off, false); 561 emit_insn(ctx, ldxhu, dst, src, t1); 562 } 563 break; 564 case BPF_W: 565 if (is_signed_imm12(off)) { 566 emit_insn(ctx, ldwu, dst, src, off); 567 } else { 568 move_imm(ctx, t1, off, false); 569 emit_insn(ctx, ldxwu, dst, src, t1); 570 } 571 break; 572 case BPF_DW: 573 if (is_signed_imm12(off)) { 574 emit_insn(ctx, ldd, dst, src, off); 575 } else { 576 move_imm(ctx, t1, off, false); 577 emit_insn(ctx, ldxd, dst, src, t1); 578 } 579 break; 580 } 581 emit_insn(ctx, dbar, 0b10100); 582 break; 583 /* store_release(dst_reg + off16, src_reg) */ 584 case BPF_STORE_REL: 585 emit_insn(ctx, dbar, 0b10010); 586 switch (BPF_SIZE(insn->code)) { 587 case BPF_B: 588 if (is_signed_imm12(off)) { 589 emit_insn(ctx, stb, src, dst, off); 590 } else { 591 move_imm(ctx, t1, off, false); 592 emit_insn(ctx, stxb, src, dst, t1); 593 } 594 break; 595 case BPF_H: 596 if (is_signed_imm12(off)) { 597 emit_insn(ctx, sth, src, dst, off); 598 } else { 599 move_imm(ctx, t1, off, false); 600 emit_insn(ctx, stxh, src, dst, t1); 601 } 602 break; 603 case BPF_W: 604 if (is_signed_imm12(off)) { 605 emit_insn(ctx, stw, src, dst, off); 606 } else { 607 move_imm(ctx, t1, off, false); 608 emit_insn(ctx, stxw, src, dst, t1); 609 } 610 break; 611 case BPF_DW: 612 if (is_signed_imm12(off)) { 613 emit_insn(ctx, std, src, dst, off); 614 } else { 615 move_imm(ctx, t1, off, false); 616 emit_insn(ctx, stxd, src, dst, t1); 617 } 618 break; 619 } 620 break; 621 default: 622 pr_err_once("bpf-jit: invalid atomic load/store opcode %02x\n", imm); 623 return -EINVAL; 624 } 625 626 return 0; 627 } 628 629 static bool is_signed_bpf_cond(u8 cond) 630 { 631 return cond == BPF_JSGT || cond == BPF_JSLT || 632 cond == BPF_JSGE || cond == BPF_JSLE; 633 } 634 635 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 636 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 637 #define REG_DONT_CLEAR_MARKER 0 638 639 bool ex_handler_bpf(const struct exception_table_entry *ex, 640 struct pt_regs *regs) 641 { 642 int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 643 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 644 645 if (dst_reg != REG_DONT_CLEAR_MARKER) 646 regs->regs[dst_reg] = 0; 647 regs->csr_era = (unsigned long)&ex->fixup - offset; 648 649 return true; 650 } 651 652 /* For accesses to BTF pointers, add an entry to the exception table */ 653 static int add_exception_handler(const struct bpf_insn *insn, 654 struct jit_ctx *ctx, 655 int dst_reg) 656 { 657 unsigned long pc; 658 off_t ins_offset, fixup_offset; 659 struct exception_table_entry *ex; 660 661 if (!ctx->image || !ctx->ro_image || !ctx->prog->aux->extable) 662 return 0; 663 664 if (BPF_MODE(insn->code) != BPF_PROBE_MEM && 665 BPF_MODE(insn->code) != BPF_PROBE_MEMSX && 666 BPF_MODE(insn->code) != BPF_PROBE_MEM32) 667 return 0; 668 669 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) 670 return -EINVAL; 671 672 ex = &ctx->prog->aux->extable[ctx->num_exentries]; 673 pc = (unsigned long)&ctx->ro_image[ctx->idx - 1]; 674 675 /* 676 * This is the relative offset of the instruction that may fault from 677 * the exception table itself. This will be written to the exception 678 * table and if this instruction faults, the destination register will 679 * be set to '0' and the execution will jump to the next instruction. 680 */ 681 ins_offset = pc - (long)&ex->insn; 682 if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) 683 return -ERANGE; 684 685 /* 686 * Since the extable follows the program, the fixup offset is always 687 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 688 * to keep things simple, and put the destination register in the upper 689 * bits. We don't need to worry about buildtime or runtime sort 690 * modifying the upper bits because the table is already sorted, and 691 * isn't part of the main exception table. 692 * 693 * The fixup_offset is set to the next instruction from the instruction 694 * that may fault. The execution will jump to this after handling the fault. 695 */ 696 fixup_offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); 697 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset)) 698 return -ERANGE; 699 700 /* 701 * The offsets above have been calculated using the RO buffer but we 702 * need to use the R/W buffer for writes. Switch ex to rw buffer for writing. 703 */ 704 ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); 705 ex->insn = ins_offset; 706 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | 707 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 708 ex->type = EX_TYPE_BPF; 709 710 ctx->num_exentries++; 711 712 return 0; 713 } 714 715 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) 716 { 717 u8 tm = -1; 718 u64 func_addr; 719 bool func_addr_fixed, sign_extend; 720 int i = insn - ctx->prog->insnsi; 721 int ret, jmp_offset, tcc_ptr_off; 722 const u8 code = insn->code; 723 const u8 cond = BPF_OP(code); 724 const u8 t1 = LOONGARCH_GPR_T1; 725 const u8 t2 = LOONGARCH_GPR_T2; 726 const u8 t3 = LOONGARCH_GPR_T3; 727 u8 src = regmap[insn->src_reg]; 728 u8 dst = regmap[insn->dst_reg]; 729 const s16 off = insn->off; 730 const s32 imm = insn->imm; 731 const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; 732 733 switch (code) { 734 /* dst = src */ 735 case BPF_ALU | BPF_MOV | BPF_X: 736 case BPF_ALU64 | BPF_MOV | BPF_X: 737 if (insn_is_cast_user(insn)) { 738 move_reg(ctx, t1, src); 739 emit_zext_32(ctx, t1, true); 740 move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32, false); 741 emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 1); 742 emit_insn(ctx, or, t1, dst, t1); 743 move_reg(ctx, dst, t1); 744 break; 745 } 746 switch (off) { 747 case 0: 748 move_reg(ctx, dst, src); 749 emit_zext_32(ctx, dst, is32); 750 break; 751 case 8: 752 emit_insn(ctx, extwb, dst, src); 753 emit_zext_32(ctx, dst, is32); 754 break; 755 case 16: 756 emit_insn(ctx, extwh, dst, src); 757 emit_zext_32(ctx, dst, is32); 758 break; 759 case 32: 760 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); 761 break; 762 } 763 break; 764 765 /* dst = imm */ 766 case BPF_ALU | BPF_MOV | BPF_K: 767 case BPF_ALU64 | BPF_MOV | BPF_K: 768 move_imm(ctx, dst, imm, is32); 769 break; 770 771 /* dst = dst + src */ 772 case BPF_ALU | BPF_ADD | BPF_X: 773 case BPF_ALU64 | BPF_ADD | BPF_X: 774 emit_insn(ctx, addd, dst, dst, src); 775 emit_zext_32(ctx, dst, is32); 776 break; 777 778 /* dst = dst + imm */ 779 case BPF_ALU | BPF_ADD | BPF_K: 780 case BPF_ALU64 | BPF_ADD | BPF_K: 781 if (is_signed_imm12(imm)) { 782 emit_insn(ctx, addid, dst, dst, imm); 783 } else { 784 move_imm(ctx, t1, imm, is32); 785 emit_insn(ctx, addd, dst, dst, t1); 786 } 787 emit_zext_32(ctx, dst, is32); 788 break; 789 790 /* dst = dst - src */ 791 case BPF_ALU | BPF_SUB | BPF_X: 792 case BPF_ALU64 | BPF_SUB | BPF_X: 793 emit_insn(ctx, subd, dst, dst, src); 794 emit_zext_32(ctx, dst, is32); 795 break; 796 797 /* dst = dst - imm */ 798 case BPF_ALU | BPF_SUB | BPF_K: 799 case BPF_ALU64 | BPF_SUB | BPF_K: 800 if (is_signed_imm12(-imm)) { 801 emit_insn(ctx, addid, dst, dst, -imm); 802 } else { 803 move_imm(ctx, t1, imm, is32); 804 emit_insn(ctx, subd, dst, dst, t1); 805 } 806 emit_zext_32(ctx, dst, is32); 807 break; 808 809 /* dst = dst * src */ 810 case BPF_ALU | BPF_MUL | BPF_X: 811 case BPF_ALU64 | BPF_MUL | BPF_X: 812 emit_insn(ctx, muld, dst, dst, src); 813 emit_zext_32(ctx, dst, is32); 814 break; 815 816 /* dst = dst * imm */ 817 case BPF_ALU | BPF_MUL | BPF_K: 818 case BPF_ALU64 | BPF_MUL | BPF_K: 819 move_imm(ctx, t1, imm, is32); 820 emit_insn(ctx, muld, dst, dst, t1); 821 emit_zext_32(ctx, dst, is32); 822 break; 823 824 /* dst = dst / src */ 825 case BPF_ALU | BPF_DIV | BPF_X: 826 case BPF_ALU64 | BPF_DIV | BPF_X: 827 if (!off) { 828 emit_zext_32(ctx, dst, is32); 829 move_reg(ctx, t1, src); 830 emit_zext_32(ctx, t1, is32); 831 emit_insn(ctx, divdu, dst, dst, t1); 832 emit_zext_32(ctx, dst, is32); 833 } else { 834 emit_sext_32(ctx, dst, is32); 835 move_reg(ctx, t1, src); 836 emit_sext_32(ctx, t1, is32); 837 emit_insn(ctx, divd, dst, dst, t1); 838 emit_sext_32(ctx, dst, is32); 839 } 840 break; 841 842 /* dst = dst / imm */ 843 case BPF_ALU | BPF_DIV | BPF_K: 844 case BPF_ALU64 | BPF_DIV | BPF_K: 845 if (!off) { 846 move_imm(ctx, t1, imm, is32); 847 emit_zext_32(ctx, dst, is32); 848 emit_insn(ctx, divdu, dst, dst, t1); 849 emit_zext_32(ctx, dst, is32); 850 } else { 851 move_imm(ctx, t1, imm, false); 852 emit_sext_32(ctx, t1, is32); 853 emit_sext_32(ctx, dst, is32); 854 emit_insn(ctx, divd, dst, dst, t1); 855 emit_sext_32(ctx, dst, is32); 856 } 857 break; 858 859 /* dst = dst % src */ 860 case BPF_ALU | BPF_MOD | BPF_X: 861 case BPF_ALU64 | BPF_MOD | BPF_X: 862 if (!off) { 863 emit_zext_32(ctx, dst, is32); 864 move_reg(ctx, t1, src); 865 emit_zext_32(ctx, t1, is32); 866 emit_insn(ctx, moddu, dst, dst, t1); 867 emit_zext_32(ctx, dst, is32); 868 } else { 869 emit_sext_32(ctx, dst, is32); 870 move_reg(ctx, t1, src); 871 emit_sext_32(ctx, t1, is32); 872 emit_insn(ctx, modd, dst, dst, t1); 873 emit_sext_32(ctx, dst, is32); 874 } 875 break; 876 877 /* dst = dst % imm */ 878 case BPF_ALU | BPF_MOD | BPF_K: 879 case BPF_ALU64 | BPF_MOD | BPF_K: 880 if (!off) { 881 move_imm(ctx, t1, imm, is32); 882 emit_zext_32(ctx, dst, is32); 883 emit_insn(ctx, moddu, dst, dst, t1); 884 emit_zext_32(ctx, dst, is32); 885 } else { 886 move_imm(ctx, t1, imm, false); 887 emit_sext_32(ctx, t1, is32); 888 emit_sext_32(ctx, dst, is32); 889 emit_insn(ctx, modd, dst, dst, t1); 890 emit_sext_32(ctx, dst, is32); 891 } 892 break; 893 894 /* dst = -dst */ 895 case BPF_ALU | BPF_NEG: 896 case BPF_ALU64 | BPF_NEG: 897 move_imm(ctx, t1, imm, is32); 898 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); 899 emit_zext_32(ctx, dst, is32); 900 break; 901 902 /* dst = dst & src */ 903 case BPF_ALU | BPF_AND | BPF_X: 904 case BPF_ALU64 | BPF_AND | BPF_X: 905 emit_insn(ctx, and, dst, dst, src); 906 emit_zext_32(ctx, dst, is32); 907 break; 908 909 /* dst = dst & imm */ 910 case BPF_ALU | BPF_AND | BPF_K: 911 case BPF_ALU64 | BPF_AND | BPF_K: 912 if (is_unsigned_imm12(imm)) { 913 emit_insn(ctx, andi, dst, dst, imm); 914 } else { 915 move_imm(ctx, t1, imm, is32); 916 emit_insn(ctx, and, dst, dst, t1); 917 } 918 emit_zext_32(ctx, dst, is32); 919 break; 920 921 /* dst = dst | src */ 922 case BPF_ALU | BPF_OR | BPF_X: 923 case BPF_ALU64 | BPF_OR | BPF_X: 924 emit_insn(ctx, or, dst, dst, src); 925 emit_zext_32(ctx, dst, is32); 926 break; 927 928 /* dst = dst | imm */ 929 case BPF_ALU | BPF_OR | BPF_K: 930 case BPF_ALU64 | BPF_OR | BPF_K: 931 if (is_unsigned_imm12(imm)) { 932 emit_insn(ctx, ori, dst, dst, imm); 933 } else { 934 move_imm(ctx, t1, imm, is32); 935 emit_insn(ctx, or, dst, dst, t1); 936 } 937 emit_zext_32(ctx, dst, is32); 938 break; 939 940 /* dst = dst ^ src */ 941 case BPF_ALU | BPF_XOR | BPF_X: 942 case BPF_ALU64 | BPF_XOR | BPF_X: 943 emit_insn(ctx, xor, dst, dst, src); 944 emit_zext_32(ctx, dst, is32); 945 break; 946 947 /* dst = dst ^ imm */ 948 case BPF_ALU | BPF_XOR | BPF_K: 949 case BPF_ALU64 | BPF_XOR | BPF_K: 950 if (is_unsigned_imm12(imm)) { 951 emit_insn(ctx, xori, dst, dst, imm); 952 } else { 953 move_imm(ctx, t1, imm, is32); 954 emit_insn(ctx, xor, dst, dst, t1); 955 } 956 emit_zext_32(ctx, dst, is32); 957 break; 958 959 /* dst = dst << src (logical) */ 960 case BPF_ALU | BPF_LSH | BPF_X: 961 emit_insn(ctx, sllw, dst, dst, src); 962 emit_zext_32(ctx, dst, is32); 963 break; 964 965 case BPF_ALU64 | BPF_LSH | BPF_X: 966 emit_insn(ctx, slld, dst, dst, src); 967 break; 968 969 /* dst = dst << imm (logical) */ 970 case BPF_ALU | BPF_LSH | BPF_K: 971 emit_insn(ctx, slliw, dst, dst, imm); 972 emit_zext_32(ctx, dst, is32); 973 break; 974 975 case BPF_ALU64 | BPF_LSH | BPF_K: 976 emit_insn(ctx, sllid, dst, dst, imm); 977 break; 978 979 /* dst = dst >> src (logical) */ 980 case BPF_ALU | BPF_RSH | BPF_X: 981 emit_insn(ctx, srlw, dst, dst, src); 982 emit_zext_32(ctx, dst, is32); 983 break; 984 985 case BPF_ALU64 | BPF_RSH | BPF_X: 986 emit_insn(ctx, srld, dst, dst, src); 987 break; 988 989 /* dst = dst >> imm (logical) */ 990 case BPF_ALU | BPF_RSH | BPF_K: 991 emit_insn(ctx, srliw, dst, dst, imm); 992 emit_zext_32(ctx, dst, is32); 993 break; 994 995 case BPF_ALU64 | BPF_RSH | BPF_K: 996 emit_insn(ctx, srlid, dst, dst, imm); 997 break; 998 999 /* dst = dst >> src (arithmetic) */ 1000 case BPF_ALU | BPF_ARSH | BPF_X: 1001 emit_insn(ctx, sraw, dst, dst, src); 1002 emit_zext_32(ctx, dst, is32); 1003 break; 1004 1005 case BPF_ALU64 | BPF_ARSH | BPF_X: 1006 emit_insn(ctx, srad, dst, dst, src); 1007 break; 1008 1009 /* dst = dst >> imm (arithmetic) */ 1010 case BPF_ALU | BPF_ARSH | BPF_K: 1011 emit_insn(ctx, sraiw, dst, dst, imm); 1012 emit_zext_32(ctx, dst, is32); 1013 break; 1014 1015 case BPF_ALU64 | BPF_ARSH | BPF_K: 1016 emit_insn(ctx, sraid, dst, dst, imm); 1017 break; 1018 1019 /* dst = BSWAP##imm(dst) */ 1020 case BPF_ALU | BPF_END | BPF_FROM_LE: 1021 switch (imm) { 1022 case 16: 1023 /* zero-extend 16 bits into 64 bits */ 1024 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 1025 break; 1026 case 32: 1027 /* zero-extend 32 bits into 64 bits */ 1028 emit_zext_32(ctx, dst, is32); 1029 break; 1030 case 64: 1031 /* do nothing */ 1032 break; 1033 } 1034 break; 1035 1036 case BPF_ALU | BPF_END | BPF_FROM_BE: 1037 case BPF_ALU64 | BPF_END | BPF_FROM_LE: 1038 switch (imm) { 1039 case 16: 1040 emit_insn(ctx, revb2h, dst, dst); 1041 /* zero-extend 16 bits into 64 bits */ 1042 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); 1043 break; 1044 case 32: 1045 emit_insn(ctx, revb2w, dst, dst); 1046 /* clear the upper 32 bits */ 1047 emit_zext_32(ctx, dst, true); 1048 break; 1049 case 64: 1050 emit_insn(ctx, revbd, dst, dst); 1051 break; 1052 } 1053 break; 1054 1055 /* PC += off if dst cond src */ 1056 case BPF_JMP | BPF_JEQ | BPF_X: 1057 case BPF_JMP | BPF_JNE | BPF_X: 1058 case BPF_JMP | BPF_JGT | BPF_X: 1059 case BPF_JMP | BPF_JGE | BPF_X: 1060 case BPF_JMP | BPF_JLT | BPF_X: 1061 case BPF_JMP | BPF_JLE | BPF_X: 1062 case BPF_JMP | BPF_JSGT | BPF_X: 1063 case BPF_JMP | BPF_JSGE | BPF_X: 1064 case BPF_JMP | BPF_JSLT | BPF_X: 1065 case BPF_JMP | BPF_JSLE | BPF_X: 1066 case BPF_JMP32 | BPF_JEQ | BPF_X: 1067 case BPF_JMP32 | BPF_JNE | BPF_X: 1068 case BPF_JMP32 | BPF_JGT | BPF_X: 1069 case BPF_JMP32 | BPF_JGE | BPF_X: 1070 case BPF_JMP32 | BPF_JLT | BPF_X: 1071 case BPF_JMP32 | BPF_JLE | BPF_X: 1072 case BPF_JMP32 | BPF_JSGT | BPF_X: 1073 case BPF_JMP32 | BPF_JSGE | BPF_X: 1074 case BPF_JMP32 | BPF_JSLT | BPF_X: 1075 case BPF_JMP32 | BPF_JSLE | BPF_X: 1076 jmp_offset = bpf2la_offset(i, off, ctx); 1077 move_reg(ctx, t1, dst); 1078 move_reg(ctx, t2, src); 1079 if (is_signed_bpf_cond(BPF_OP(code))) { 1080 emit_sext_32(ctx, t1, is32); 1081 emit_sext_32(ctx, t2, is32); 1082 } else { 1083 emit_zext_32(ctx, t1, is32); 1084 emit_zext_32(ctx, t2, is32); 1085 } 1086 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) 1087 goto toofar; 1088 break; 1089 1090 /* PC += off if dst cond imm */ 1091 case BPF_JMP | BPF_JEQ | BPF_K: 1092 case BPF_JMP | BPF_JNE | BPF_K: 1093 case BPF_JMP | BPF_JGT | BPF_K: 1094 case BPF_JMP | BPF_JGE | BPF_K: 1095 case BPF_JMP | BPF_JLT | BPF_K: 1096 case BPF_JMP | BPF_JLE | BPF_K: 1097 case BPF_JMP | BPF_JSGT | BPF_K: 1098 case BPF_JMP | BPF_JSGE | BPF_K: 1099 case BPF_JMP | BPF_JSLT | BPF_K: 1100 case BPF_JMP | BPF_JSLE | BPF_K: 1101 case BPF_JMP32 | BPF_JEQ | BPF_K: 1102 case BPF_JMP32 | BPF_JNE | BPF_K: 1103 case BPF_JMP32 | BPF_JGT | BPF_K: 1104 case BPF_JMP32 | BPF_JGE | BPF_K: 1105 case BPF_JMP32 | BPF_JLT | BPF_K: 1106 case BPF_JMP32 | BPF_JLE | BPF_K: 1107 case BPF_JMP32 | BPF_JSGT | BPF_K: 1108 case BPF_JMP32 | BPF_JSGE | BPF_K: 1109 case BPF_JMP32 | BPF_JSLT | BPF_K: 1110 case BPF_JMP32 | BPF_JSLE | BPF_K: 1111 jmp_offset = bpf2la_offset(i, off, ctx); 1112 if (imm) { 1113 move_imm(ctx, t1, imm, false); 1114 tm = t1; 1115 } else { 1116 /* If imm is 0, simply use zero register. */ 1117 tm = LOONGARCH_GPR_ZERO; 1118 } 1119 move_reg(ctx, t2, dst); 1120 if (is_signed_bpf_cond(BPF_OP(code))) { 1121 emit_sext_32(ctx, tm, is32); 1122 emit_sext_32(ctx, t2, is32); 1123 } else { 1124 emit_zext_32(ctx, tm, is32); 1125 emit_zext_32(ctx, t2, is32); 1126 } 1127 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) 1128 goto toofar; 1129 break; 1130 1131 /* PC += off if dst & src */ 1132 case BPF_JMP | BPF_JSET | BPF_X: 1133 case BPF_JMP32 | BPF_JSET | BPF_X: 1134 jmp_offset = bpf2la_offset(i, off, ctx); 1135 emit_insn(ctx, and, t1, dst, src); 1136 emit_zext_32(ctx, t1, is32); 1137 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 1138 goto toofar; 1139 break; 1140 1141 /* PC += off if dst & imm */ 1142 case BPF_JMP | BPF_JSET | BPF_K: 1143 case BPF_JMP32 | BPF_JSET | BPF_K: 1144 jmp_offset = bpf2la_offset(i, off, ctx); 1145 move_imm(ctx, t1, imm, is32); 1146 emit_insn(ctx, and, t1, dst, t1); 1147 emit_zext_32(ctx, t1, is32); 1148 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) 1149 goto toofar; 1150 break; 1151 1152 /* PC += off */ 1153 case BPF_JMP | BPF_JA: 1154 case BPF_JMP32 | BPF_JA: 1155 if (BPF_CLASS(code) == BPF_JMP) 1156 jmp_offset = bpf2la_offset(i, off, ctx); 1157 else 1158 jmp_offset = bpf2la_offset(i, imm, ctx); 1159 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 1160 goto toofar; 1161 break; 1162 1163 /* function call */ 1164 case BPF_JMP | BPF_CALL: 1165 /* Implement helper call to bpf_get_current_task/_btf() inline */ 1166 if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task || 1167 insn->imm == BPF_FUNC_get_current_task_btf)) { 1168 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_TP); 1169 break; 1170 } 1171 1172 /* Implement helper call to bpf_get_smp_processor_id() inline */ 1173 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { 1174 emit_insn(ctx, ldwu, regmap[BPF_REG_0], LOONGARCH_GPR_TP, TI_CPU); 1175 break; 1176 } 1177 1178 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1179 &func_addr, &func_addr_fixed); 1180 if (ret < 0) 1181 return ret; 1182 1183 if (insn->src_reg == BPF_PSEUDO_CALL) { 1184 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); 1185 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); 1186 } 1187 1188 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 1189 const struct btf_func_model *m; 1190 int i; 1191 1192 m = bpf_jit_find_kfunc_model(ctx->prog, insn); 1193 if (!m) 1194 return -EINVAL; 1195 1196 for (i = 0; i < m->nr_args; i++) { 1197 u8 reg = regmap[BPF_REG_1 + i]; 1198 bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; 1199 1200 emit_abi_ext(ctx, reg, m->arg_size[i], sign); 1201 } 1202 } 1203 1204 move_addr(ctx, t1, func_addr); 1205 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); 1206 1207 if (insn->src_reg != BPF_PSEUDO_CALL) 1208 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); 1209 1210 break; 1211 1212 /* tail call */ 1213 case BPF_JMP | BPF_TAIL_CALL: 1214 if (emit_bpf_tail_call(ctx, i) < 0) 1215 return -EINVAL; 1216 break; 1217 1218 /* function return */ 1219 case BPF_JMP | BPF_EXIT: 1220 if (i == ctx->prog->len - 1) 1221 break; 1222 1223 jmp_offset = epilogue_offset(ctx); 1224 if (emit_uncond_jmp(ctx, jmp_offset) < 0) 1225 goto toofar; 1226 break; 1227 1228 /* dst = imm64 */ 1229 case BPF_LD | BPF_IMM | BPF_DW: 1230 { 1231 const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; 1232 1233 if (bpf_pseudo_func(insn)) 1234 move_addr(ctx, dst, imm64); 1235 else 1236 move_imm(ctx, dst, imm64, is32); 1237 return 1; 1238 } 1239 1240 /* dst = *(size *)(src + off) */ 1241 case BPF_LDX | BPF_MEM | BPF_B: 1242 case BPF_LDX | BPF_MEM | BPF_H: 1243 case BPF_LDX | BPF_MEM | BPF_W: 1244 case BPF_LDX | BPF_MEM | BPF_DW: 1245 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1246 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1247 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1248 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1249 /* dst_reg = (s64)*(signed size *)(src_reg + off) */ 1250 case BPF_LDX | BPF_MEMSX | BPF_B: 1251 case BPF_LDX | BPF_MEMSX | BPF_H: 1252 case BPF_LDX | BPF_MEMSX | BPF_W: 1253 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 1254 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 1255 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1256 /* LDX | PROBE_MEM32: dst = *(unsigned size *)(src + REG_ARENA + off) */ 1257 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: 1258 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: 1259 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: 1260 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: 1261 sign_extend = BPF_MODE(code) == BPF_MEMSX || 1262 BPF_MODE(code) == BPF_PROBE_MEMSX; 1263 1264 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1265 emit_insn(ctx, addd, t2, src, REG_ARENA); 1266 src = t2; 1267 } 1268 1269 switch (BPF_SIZE(code)) { 1270 case BPF_B: 1271 if (is_signed_imm12(off)) { 1272 if (sign_extend) 1273 emit_insn(ctx, ldb, dst, src, off); 1274 else 1275 emit_insn(ctx, ldbu, dst, src, off); 1276 } else { 1277 move_imm(ctx, t1, off, is32); 1278 if (sign_extend) 1279 emit_insn(ctx, ldxb, dst, src, t1); 1280 else 1281 emit_insn(ctx, ldxbu, dst, src, t1); 1282 } 1283 break; 1284 case BPF_H: 1285 if (is_signed_imm12(off)) { 1286 if (sign_extend) 1287 emit_insn(ctx, ldh, dst, src, off); 1288 else 1289 emit_insn(ctx, ldhu, dst, src, off); 1290 } else { 1291 move_imm(ctx, t1, off, is32); 1292 if (sign_extend) 1293 emit_insn(ctx, ldxh, dst, src, t1); 1294 else 1295 emit_insn(ctx, ldxhu, dst, src, t1); 1296 } 1297 break; 1298 case BPF_W: 1299 if (is_signed_imm12(off)) { 1300 if (sign_extend) 1301 emit_insn(ctx, ldw, dst, src, off); 1302 else 1303 emit_insn(ctx, ldwu, dst, src, off); 1304 } else { 1305 move_imm(ctx, t1, off, is32); 1306 if (sign_extend) 1307 emit_insn(ctx, ldxw, dst, src, t1); 1308 else 1309 emit_insn(ctx, ldxwu, dst, src, t1); 1310 } 1311 break; 1312 case BPF_DW: 1313 move_imm(ctx, t1, off, is32); 1314 emit_insn(ctx, ldxd, dst, src, t1); 1315 break; 1316 } 1317 1318 ret = add_exception_handler(insn, ctx, dst); 1319 if (ret) 1320 return ret; 1321 break; 1322 1323 /* *(size *)(dst + off) = imm */ 1324 case BPF_ST | BPF_MEM | BPF_B: 1325 case BPF_ST | BPF_MEM | BPF_H: 1326 case BPF_ST | BPF_MEM | BPF_W: 1327 case BPF_ST | BPF_MEM | BPF_DW: 1328 /* ST | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = imm */ 1329 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: 1330 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: 1331 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: 1332 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: 1333 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1334 emit_insn(ctx, addd, t3, dst, REG_ARENA); 1335 dst = t3; 1336 } 1337 1338 switch (BPF_SIZE(code)) { 1339 case BPF_B: 1340 move_imm(ctx, t1, imm, is32); 1341 if (is_signed_imm12(off)) { 1342 emit_insn(ctx, stb, t1, dst, off); 1343 } else { 1344 move_imm(ctx, t2, off, is32); 1345 emit_insn(ctx, stxb, t1, dst, t2); 1346 } 1347 break; 1348 case BPF_H: 1349 move_imm(ctx, t1, imm, is32); 1350 if (is_signed_imm12(off)) { 1351 emit_insn(ctx, sth, t1, dst, off); 1352 } else { 1353 move_imm(ctx, t2, off, is32); 1354 emit_insn(ctx, stxh, t1, dst, t2); 1355 } 1356 break; 1357 case BPF_W: 1358 move_imm(ctx, t1, imm, is32); 1359 if (is_signed_imm12(off)) { 1360 emit_insn(ctx, stw, t1, dst, off); 1361 } else if (is_signed_imm14(off)) { 1362 emit_insn(ctx, stptrw, t1, dst, off); 1363 } else { 1364 move_imm(ctx, t2, off, is32); 1365 emit_insn(ctx, stxw, t1, dst, t2); 1366 } 1367 break; 1368 case BPF_DW: 1369 move_imm(ctx, t1, imm, is32); 1370 if (is_signed_imm12(off)) { 1371 emit_insn(ctx, std, t1, dst, off); 1372 } else if (is_signed_imm14(off)) { 1373 emit_insn(ctx, stptrd, t1, dst, off); 1374 } else { 1375 move_imm(ctx, t2, off, is32); 1376 emit_insn(ctx, stxd, t1, dst, t2); 1377 } 1378 break; 1379 } 1380 1381 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1382 if (ret) 1383 return ret; 1384 break; 1385 1386 /* *(size *)(dst + off) = src */ 1387 case BPF_STX | BPF_MEM | BPF_B: 1388 case BPF_STX | BPF_MEM | BPF_H: 1389 case BPF_STX | BPF_MEM | BPF_W: 1390 case BPF_STX | BPF_MEM | BPF_DW: 1391 /* STX | PROBE_MEM32: *(size *)(dst + REG_ARENA + off) = src */ 1392 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: 1393 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: 1394 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: 1395 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: 1396 if (BPF_MODE(code) == BPF_PROBE_MEM32) { 1397 emit_insn(ctx, addd, t2, dst, REG_ARENA); 1398 dst = t2; 1399 } 1400 1401 switch (BPF_SIZE(code)) { 1402 case BPF_B: 1403 if (is_signed_imm12(off)) { 1404 emit_insn(ctx, stb, src, dst, off); 1405 } else { 1406 move_imm(ctx, t1, off, is32); 1407 emit_insn(ctx, stxb, src, dst, t1); 1408 } 1409 break; 1410 case BPF_H: 1411 if (is_signed_imm12(off)) { 1412 emit_insn(ctx, sth, src, dst, off); 1413 } else { 1414 move_imm(ctx, t1, off, is32); 1415 emit_insn(ctx, stxh, src, dst, t1); 1416 } 1417 break; 1418 case BPF_W: 1419 if (is_signed_imm12(off)) { 1420 emit_insn(ctx, stw, src, dst, off); 1421 } else if (is_signed_imm14(off)) { 1422 emit_insn(ctx, stptrw, src, dst, off); 1423 } else { 1424 move_imm(ctx, t1, off, is32); 1425 emit_insn(ctx, stxw, src, dst, t1); 1426 } 1427 break; 1428 case BPF_DW: 1429 if (is_signed_imm12(off)) { 1430 emit_insn(ctx, std, src, dst, off); 1431 } else if (is_signed_imm14(off)) { 1432 emit_insn(ctx, stptrd, src, dst, off); 1433 } else { 1434 move_imm(ctx, t1, off, is32); 1435 emit_insn(ctx, stxd, src, dst, t1); 1436 } 1437 break; 1438 } 1439 1440 ret = add_exception_handler(insn, ctx, REG_DONT_CLEAR_MARKER); 1441 if (ret) 1442 return ret; 1443 break; 1444 1445 /* Atomics */ 1446 case BPF_STX | BPF_ATOMIC | BPF_B: 1447 case BPF_STX | BPF_ATOMIC | BPF_H: 1448 case BPF_STX | BPF_ATOMIC | BPF_W: 1449 case BPF_STX | BPF_ATOMIC | BPF_DW: 1450 if (!bpf_atomic_is_load_store(insn)) 1451 ret = emit_atomic_rmw(insn, ctx); 1452 else 1453 ret = emit_atomic_ld_st(insn, ctx); 1454 if (ret) 1455 return ret; 1456 break; 1457 1458 /* Speculation barrier */ 1459 case BPF_ST | BPF_NOSPEC: 1460 break; 1461 1462 default: 1463 pr_err("bpf_jit: unknown opcode %02x\n", code); 1464 return -EINVAL; 1465 } 1466 1467 return 0; 1468 1469 toofar: 1470 pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); 1471 return -E2BIG; 1472 } 1473 1474 static int build_body(struct jit_ctx *ctx, bool extra_pass) 1475 { 1476 int i; 1477 const struct bpf_prog *prog = ctx->prog; 1478 1479 for (i = 0; i < prog->len; i++) { 1480 const struct bpf_insn *insn = &prog->insnsi[i]; 1481 int ret; 1482 1483 if (ctx->image == NULL) 1484 ctx->offset[i] = ctx->idx; 1485 1486 ret = build_insn(insn, ctx, extra_pass); 1487 if (ret > 0) { 1488 i++; 1489 if (ctx->image == NULL) 1490 ctx->offset[i] = ctx->idx; 1491 continue; 1492 } 1493 if (ret) 1494 return ret; 1495 } 1496 1497 if (ctx->image == NULL) 1498 ctx->offset[i] = ctx->idx; 1499 1500 return 0; 1501 } 1502 1503 /* Fill space with break instructions */ 1504 static void jit_fill_hole(void *area, unsigned int size) 1505 { 1506 u32 *ptr; 1507 1508 /* We are guaranteed to have aligned memory */ 1509 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1510 *ptr++ = INSN_BREAK; 1511 } 1512 1513 static int validate_code(struct jit_ctx *ctx) 1514 { 1515 int i; 1516 union loongarch_instruction insn; 1517 1518 for (i = 0; i < ctx->idx; i++) { 1519 insn = ctx->image[i]; 1520 /* Check INSN_BREAK */ 1521 if (insn.word == INSN_BREAK) 1522 return -1; 1523 } 1524 1525 return 0; 1526 } 1527 1528 static int validate_ctx(struct jit_ctx *ctx) 1529 { 1530 if (validate_code(ctx)) 1531 return -1; 1532 1533 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) 1534 return -1; 1535 1536 return 0; 1537 } 1538 1539 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) 1540 { 1541 if (!target) { 1542 pr_err("bpf_jit: jump target address is error\n"); 1543 return -EFAULT; 1544 } 1545 1546 move_imm(ctx, LOONGARCH_GPR_T1, target, false); 1547 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); 1548 1549 return 0; 1550 } 1551 1552 static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 1553 { 1554 int i; 1555 struct jit_ctx ctx; 1556 1557 ctx.idx = 0; 1558 ctx.image = (union loongarch_instruction *)insns; 1559 1560 if (!target) { 1561 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 1562 emit_insn((&ctx), nop); 1563 return 0; 1564 } 1565 1566 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target); 1567 } 1568 1569 static int emit_call(struct jit_ctx *ctx, u64 addr) 1570 { 1571 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); 1572 } 1573 1574 void *bpf_arch_text_copy(void *dst, void *src, size_t len) 1575 { 1576 int ret; 1577 1578 cpus_read_lock(); 1579 mutex_lock(&text_mutex); 1580 ret = larch_insn_text_copy(dst, src, len); 1581 mutex_unlock(&text_mutex); 1582 cpus_read_unlock(); 1583 1584 return ret ? ERR_PTR(-EINVAL) : dst; 1585 } 1586 1587 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, 1588 enum bpf_text_poke_type new_t, void *old_addr, 1589 void *new_addr) 1590 { 1591 int ret; 1592 bool is_call; 1593 unsigned long size = 0; 1594 unsigned long offset = 0; 1595 void *image = NULL; 1596 char namebuf[KSYM_NAME_LEN]; 1597 u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1598 u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; 1599 1600 /* Only poking bpf text is supported. Since kernel function entry 1601 * is set up by ftrace, we rely on ftrace to poke kernel functions. 1602 */ 1603 if (!bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) 1604 return -ENOTSUPP; 1605 1606 image = ip - offset; 1607 1608 /* zero offset means we're poking bpf prog entry */ 1609 if (offset == 0) { 1610 /* skip to the nop instruction in bpf prog entry: 1611 * move t0, ra 1612 * nop 1613 */ 1614 ip = image + LOONGARCH_INSN_SIZE; 1615 } 1616 1617 is_call = old_t == BPF_MOD_CALL; 1618 ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); 1619 if (ret) 1620 return ret; 1621 1622 if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1623 return -EFAULT; 1624 1625 is_call = new_t == BPF_MOD_CALL; 1626 ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call); 1627 if (ret) 1628 return ret; 1629 1630 cpus_read_lock(); 1631 mutex_lock(&text_mutex); 1632 if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) 1633 ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); 1634 mutex_unlock(&text_mutex); 1635 cpus_read_unlock(); 1636 1637 return ret; 1638 } 1639 1640 int bpf_arch_text_invalidate(void *dst, size_t len) 1641 { 1642 int i; 1643 int ret = 0; 1644 u32 *inst; 1645 1646 inst = kvmalloc(len, GFP_KERNEL); 1647 if (!inst) 1648 return -ENOMEM; 1649 1650 for (i = 0; i < (len / sizeof(u32)); i++) 1651 inst[i] = INSN_BREAK; 1652 1653 cpus_read_lock(); 1654 mutex_lock(&text_mutex); 1655 if (larch_insn_text_copy(dst, inst, len)) 1656 ret = -EINVAL; 1657 mutex_unlock(&text_mutex); 1658 cpus_read_unlock(); 1659 1660 kvfree(inst); 1661 1662 return ret; 1663 } 1664 1665 static void store_args(struct jit_ctx *ctx, int nr_arg_slots, int args_off) 1666 { 1667 int i; 1668 1669 for (i = 0; i < nr_arg_slots; i++) { 1670 if (i < LOONGARCH_MAX_REG_ARGS) 1671 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1672 else { 1673 /* Skip slots for T0 and FP of traced function */ 1674 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, 1675 16 + (i - LOONGARCH_MAX_REG_ARGS) * 8); 1676 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -args_off); 1677 } 1678 args_off -= 8; 1679 } 1680 } 1681 1682 static void restore_args(struct jit_ctx *ctx, int nr_reg_args, int args_off) 1683 { 1684 int i; 1685 1686 for (i = 0; i < nr_reg_args; i++) { 1687 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); 1688 args_off -= 8; 1689 } 1690 } 1691 1692 static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, int stk_args_off) 1693 { 1694 int i; 1695 1696 for (i = 0; i < nr_stk_args; i++) { 1697 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, 1698 -(args_off - LOONGARCH_MAX_REG_ARGS * 8)); 1699 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -stk_args_off); 1700 args_off -= 8; 1701 stk_args_off -= 8; 1702 } 1703 } 1704 1705 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, 1706 int args_off, int retval_off, int run_ctx_off, bool save_ret) 1707 { 1708 int ret; 1709 u32 *branch; 1710 struct bpf_prog *p = n->link->prog; 1711 int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 1712 1713 if (n->cookie) 1714 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, 1715 -run_ctx_off + cookie_off, n->cookie); 1716 else 1717 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); 1718 1719 /* arg1: prog */ 1720 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1721 /* arg2: &run_ctx */ 1722 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); 1723 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); 1724 if (ret) 1725 return ret; 1726 1727 /* store prog start time */ 1728 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); 1729 1730 /* 1731 * if (__bpf_prog_enter(prog) == 0) 1732 * goto skip_exec_of_prog; 1733 */ 1734 branch = (u32 *)ctx->image + ctx->idx; 1735 /* nop reserved for conditional jump */ 1736 emit_insn(ctx, nop); 1737 1738 /* arg1: &args_off */ 1739 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); 1740 if (!p->jited) 1741 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); 1742 ret = emit_call(ctx, (const u64)p->bpf_func); 1743 if (ret) 1744 return ret; 1745 1746 if (save_ret) { 1747 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 1748 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 1749 } 1750 1751 /* update branch with beqz */ 1752 if (ctx->image) { 1753 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; 1754 *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset); 1755 } 1756 1757 /* arg1: prog */ 1758 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); 1759 /* arg2: prog start time */ 1760 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); 1761 /* arg3: &run_ctx */ 1762 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); 1763 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); 1764 1765 return ret; 1766 } 1767 1768 static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, 1769 int args_off, int retval_off, int run_ctx_off, 1770 int func_meta_off, bool save_ret, u64 func_meta, int cookie_off) 1771 { 1772 int i, cur_cookie = (cookie_off - args_off) / 8; 1773 1774 for (i = 0; i < tn->nr_nodes; i++) { 1775 int err; 1776 1777 if (bpf_prog_calls_session_cookie(tn->nodes[i])) { 1778 u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT); 1779 1780 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta); 1781 cur_cookie--; 1782 } 1783 err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret); 1784 if (err) 1785 return err; 1786 } 1787 1788 return 0; 1789 } 1790 1791 void *arch_alloc_bpf_trampoline(unsigned int size) 1792 { 1793 return bpf_prog_pack_alloc(size, jit_fill_hole); 1794 } 1795 1796 void arch_free_bpf_trampoline(void *image, unsigned int size) 1797 { 1798 bpf_prog_pack_free(image, size); 1799 } 1800 1801 int arch_protect_bpf_trampoline(void *image, unsigned int size) 1802 { 1803 return 0; 1804 } 1805 1806 /* 1807 * Sign-extend the register if necessary 1808 */ 1809 static void sign_extend(struct jit_ctx *ctx, int rd, int rj, u8 size, bool sign) 1810 { 1811 /* ABI requires unsigned char/short to be zero-extended */ 1812 if (!sign && (size == 1 || size == 2)) { 1813 if (rd != rj) 1814 move_reg(ctx, rd, rj); 1815 return; 1816 } 1817 1818 switch (size) { 1819 case 1: 1820 emit_insn(ctx, extwb, rd, rj); 1821 break; 1822 case 2: 1823 emit_insn(ctx, extwh, rd, rj); 1824 break; 1825 case 4: 1826 emit_insn(ctx, addiw, rd, rj, 0); 1827 break; 1828 case 8: 1829 if (rd != rj) 1830 move_reg(ctx, rd, rj); 1831 break; 1832 default: 1833 pr_warn("bpf_jit: invalid size %d for sign_extend\n", size); 1834 } 1835 } 1836 1837 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, 1838 const struct btf_func_model *m, struct bpf_tramp_nodes *tnodes, 1839 void *func_addr, u32 flags) 1840 { 1841 int i, ret, save_ret; 1842 int cookie_cnt, cookie_off; 1843 int stack_size, args_off, stk_args_off, nr_arg_slots = 0; 1844 int retval_off, func_meta_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; 1845 unsigned long long func_meta; 1846 bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; 1847 void *orig_call = func_addr; 1848 struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; 1849 struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; 1850 struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; 1851 u32 **branches = NULL; 1852 1853 /* 1854 * FP + 8 [ RA to parent func ] return address to parent 1855 * function 1856 * FP + 0 [ FP of parent func ] frame pointer of parent 1857 * function 1858 * FP - 8 [ T0 to traced func ] return address of traced 1859 * function 1860 * FP - 16 [ FP of traced func ] frame pointer of traced 1861 * function 1862 * 1863 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 1864 * BPF_TRAMP_F_RET_FENTRY_RET 1865 * [ arg regN ] 1866 * [ ... ] 1867 * FP - args_off [ arg reg1 ] 1868 * 1869 * FP - func_meta_off [ regs count, etc ] 1870 * 1871 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 1872 * 1873 * [ stack cookie N ] 1874 * [ ... ] 1875 * FP - cookie_off [ stack cookie 1 ] 1876 * 1877 * FP - run_ctx_off [ bpf_tramp_run_ctx ] 1878 * 1879 * FP - sreg_off [ callee saved reg ] 1880 * 1881 * FP - tcc_ptr_off [ tail_call_cnt_ptr ] 1882 * 1883 * [ stack_argN ] 1884 * [ ... ] 1885 * FP - stk_args_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG 1886 */ 1887 1888 if (m->nr_args > MAX_BPF_FUNC_ARGS) 1889 return -ENOTSUPP; 1890 1891 /* Extra registers for struct arguments */ 1892 for (i = 0; i < m->nr_args; i++) { 1893 /* 1894 * The struct argument size is at most 16 bytes, 1895 * enforced by the verifier. The struct argument 1896 * may be passed in a pair of registers if its 1897 * size is more than 8 bytes and no more than 16 1898 * bytes. 1899 */ 1900 nr_arg_slots += round_up(m->arg_size[i], 8) / 8; 1901 } 1902 1903 if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 1904 return -ENOTSUPP; 1905 1906 /* Room of trampoline frame to store return address and frame pointer */ 1907 stack_size = 16; 1908 1909 save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 1910 if (save_ret) 1911 stack_size += 16; /* Save BPF R0 and A0 */ 1912 1913 retval_off = stack_size; 1914 1915 /* Room of trampoline frame to store args */ 1916 stack_size += nr_arg_slots * 8; 1917 args_off = stack_size; 1918 1919 /* Room of function metadata, such as regs count */ 1920 stack_size += 8; 1921 func_meta_off = stack_size; 1922 1923 /* Room of trampoline frame to store ip address */ 1924 if (flags & BPF_TRAMP_F_IP_ARG) { 1925 stack_size += 8; 1926 ip_off = stack_size; 1927 } 1928 1929 cookie_cnt = bpf_fsession_cookie_cnt(tnodes); 1930 1931 /* Room for session cookies */ 1932 stack_size += cookie_cnt * 8; 1933 cookie_off = stack_size; 1934 1935 /* Room of trampoline frame to store struct bpf_tramp_run_ctx */ 1936 stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 1937 run_ctx_off = stack_size; 1938 1939 stack_size += 8; 1940 sreg_off = stack_size; 1941 1942 /* Room of trampoline frame to store tail_call_cnt_ptr */ 1943 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { 1944 stack_size += 8; 1945 tcc_ptr_off = stack_size; 1946 } 1947 1948 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - LOONGARCH_MAX_REG_ARGS > 0)) 1949 stack_size += (nr_arg_slots - LOONGARCH_MAX_REG_ARGS) * 8; 1950 1951 stack_size = round_up(stack_size, 16); 1952 1953 /* Room for args on stack must be at the top of stack */ 1954 stk_args_off = stack_size; 1955 1956 if (is_struct_ops) { 1957 /* 1958 * For the trampoline called directly, just handle 1959 * the frame of trampoline. 1960 */ 1961 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1962 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 1963 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1964 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1965 } else { 1966 /* 1967 * For the trampoline called from function entry, 1968 * the frame of traced function and the frame of 1969 * trampoline need to be considered. 1970 */ 1971 /* RA and FP for parent function */ 1972 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); 1973 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 1974 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 1975 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); 1976 1977 /* RA and FP for traced function */ 1978 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); 1979 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 1980 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 1981 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); 1982 } 1983 1984 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 1985 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 1986 1987 /* callee saved register S1 to pass start time */ 1988 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 1989 1990 /* store ip address of the traced function */ 1991 if (flags & BPF_TRAMP_F_IP_ARG) 1992 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -ip_off, (u64)func_addr); 1993 1994 /* store arg regs count */ 1995 func_meta = nr_arg_slots; 1996 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); 1997 1998 store_args(ctx, nr_arg_slots, args_off); 1999 2000 if (bpf_fsession_cnt(tnodes)) { 2001 /* clear all session cookies' value */ 2002 for (i = 0; i < cookie_cnt; i++) 2003 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -cookie_off + 8 * i); 2004 2005 /* clear return value to make sure fentry always get 0 */ 2006 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 2007 } 2008 2009 /* To traced function */ 2010 /* Ftrace jump skips 2 NOP instructions */ 2011 if (is_kernel_text((unsigned long)orig_call) || 2012 is_module_text_address((unsigned long)orig_call)) 2013 orig_call += LOONGARCH_FENTRY_NBYTES; 2014 /* Direct jump skips 5 NOP instructions */ 2015 else if (is_bpf_text_address((unsigned long)orig_call)) 2016 orig_call += LOONGARCH_BPF_FENTRY_NBYTES; 2017 2018 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2019 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 2020 ret = emit_call(ctx, (const u64)__bpf_tramp_enter); 2021 if (ret) 2022 return ret; 2023 } 2024 2025 if (fentry->nr_nodes) { 2026 ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off, 2027 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off); 2028 if (ret) 2029 return ret; 2030 } 2031 if (fmod_ret->nr_nodes) { 2032 branches = kcalloc(fmod_ret->nr_nodes, sizeof(u32 *), GFP_KERNEL); 2033 if (!branches) 2034 return -ENOMEM; 2035 2036 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); 2037 for (i = 0; i < fmod_ret->nr_nodes; i++) { 2038 ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i], 2039 args_off, retval_off, run_ctx_off, true); 2040 if (ret) 2041 goto out; 2042 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); 2043 branches[i] = (u32 *)ctx->image + ctx->idx; 2044 emit_insn(ctx, nop); 2045 } 2046 } 2047 2048 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2049 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); 2050 restore_stk_args(ctx, nr_arg_slots - LOONGARCH_MAX_REG_ARGS, args_off, stk_args_off); 2051 2052 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2053 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 2054 2055 ret = emit_call(ctx, (const u64)orig_call); 2056 if (ret) 2057 goto out; 2058 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 2059 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 2060 im->ip_after_call = ctx->ro_image + ctx->idx; 2061 /* Reserve space for the move_imm + jirl instruction */ 2062 for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) 2063 emit_insn(ctx, nop); 2064 } 2065 2066 for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) { 2067 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; 2068 *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset); 2069 } 2070 2071 /* Set "is_return" flag for fsession */ 2072 func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); 2073 if (bpf_fsession_cnt(tnodes)) 2074 emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); 2075 2076 if (fexit->nr_nodes) { 2077 ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off, 2078 func_meta_off, false, func_meta, cookie_off); 2079 if (ret) 2080 goto out; 2081 } 2082 2083 if (flags & BPF_TRAMP_F_CALL_ORIG) { 2084 im->ip_epilogue = ctx->ro_image + ctx->idx; 2085 move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); 2086 ret = emit_call(ctx, (const u64)__bpf_tramp_exit); 2087 if (ret) 2088 goto out; 2089 } 2090 2091 if (flags & BPF_TRAMP_F_RESTORE_REGS) 2092 restore_args(ctx, min_t(int, nr_arg_slots, LOONGARCH_MAX_REG_ARGS), args_off); 2093 2094 if (save_ret) { 2095 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); 2096 if (is_struct_ops) 2097 sign_extend(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 2098 m->ret_size, m->ret_flags & BTF_FMODEL_SIGNED_ARG); 2099 else 2100 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); 2101 } 2102 2103 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); 2104 2105 if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) 2106 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); 2107 2108 if (is_struct_ops) { 2109 /* trampoline called directly */ 2110 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); 2111 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 2112 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 2113 2114 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); 2115 } else { 2116 /* trampoline called from function entry */ 2117 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); 2118 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); 2119 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); 2120 2121 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); 2122 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); 2123 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); 2124 2125 if (flags & BPF_TRAMP_F_SKIP_FRAME) { 2126 /* return to parent function */ 2127 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 2128 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); 2129 } else { 2130 /* return to traced function */ 2131 move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA); 2132 move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); 2133 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0); 2134 } 2135 } 2136 2137 ret = ctx->idx; 2138 out: 2139 kfree(branches); 2140 2141 return ret; 2142 } 2143 2144 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, 2145 void *ro_image_end, const struct btf_func_model *m, 2146 u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) 2147 { 2148 int ret, size; 2149 void *image, *tmp; 2150 struct jit_ctx ctx; 2151 2152 size = ro_image_end - ro_image; 2153 image = kvmalloc(size, GFP_KERNEL); 2154 if (!image) 2155 return -ENOMEM; 2156 2157 ctx.image = (union loongarch_instruction *)image; 2158 ctx.ro_image = (union loongarch_instruction *)ro_image; 2159 ctx.idx = 0; 2160 2161 jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); 2162 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tnodes, func_addr, flags); 2163 if (ret < 0) 2164 goto out; 2165 2166 if (validate_code(&ctx) < 0) { 2167 ret = -EINVAL; 2168 goto out; 2169 } 2170 2171 tmp = bpf_arch_text_copy(ro_image, image, size); 2172 if (IS_ERR(tmp)) { 2173 ret = PTR_ERR(tmp); 2174 goto out; 2175 } 2176 2177 out: 2178 kvfree(image); 2179 return ret < 0 ? ret : size; 2180 } 2181 2182 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, 2183 struct bpf_tramp_nodes *tnodes, void *func_addr) 2184 { 2185 int ret; 2186 struct jit_ctx ctx; 2187 struct bpf_tramp_image im; 2188 2189 ctx.image = NULL; 2190 ctx.idx = 0; 2191 2192 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags); 2193 2194 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE; 2195 } 2196 2197 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) 2198 { 2199 bool extra_pass = false; 2200 u8 *image_ptr, *ro_image_ptr; 2201 int image_size, prog_size, extable_size; 2202 struct jit_ctx ctx; 2203 struct jit_data *jit_data; 2204 struct bpf_binary_header *header; 2205 struct bpf_binary_header *ro_header; 2206 2207 /* 2208 * If BPF JIT was not enabled then we must fall back to 2209 * the interpreter. 2210 */ 2211 if (!prog->jit_requested) 2212 return prog; 2213 2214 jit_data = prog->aux->jit_data; 2215 if (!jit_data) { 2216 jit_data = kzalloc_obj(*jit_data); 2217 if (!jit_data) 2218 return prog; 2219 prog->aux->jit_data = jit_data; 2220 } 2221 if (jit_data->ctx.offset) { 2222 ctx = jit_data->ctx; 2223 ro_header = jit_data->ro_header; 2224 ro_image_ptr = (void *)ctx.ro_image; 2225 header = jit_data->header; 2226 image_ptr = (void *)header + ((void *)ro_image_ptr - (void *)ro_header); 2227 extra_pass = true; 2228 prog_size = sizeof(u32) * ctx.idx; 2229 goto skip_init_ctx; 2230 } 2231 2232 memset(&ctx, 0, sizeof(ctx)); 2233 ctx.prog = prog; 2234 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); 2235 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); 2236 2237 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); 2238 if (ctx.offset == NULL) 2239 goto out_offset; 2240 2241 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ 2242 build_prologue(&ctx); 2243 if (build_body(&ctx, extra_pass)) 2244 goto out_offset; 2245 ctx.epilogue_offset = ctx.idx; 2246 build_epilogue(&ctx); 2247 2248 extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); 2249 2250 /* Now we know the actual image size. 2251 * As each LoongArch instruction is of length 32bit, 2252 * we are translating number of JITed intructions into 2253 * the size required to store these JITed code. 2254 */ 2255 prog_size = sizeof(u32) * ctx.idx; 2256 image_size = prog_size + extable_size; 2257 /* Now we know the size of the structure to make */ 2258 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), 2259 &header, &image_ptr, jit_fill_hole); 2260 if (!ro_header) 2261 goto out_offset; 2262 2263 /* 2. Now, the actual pass to generate final JIT code */ 2264 /* 2265 * Use the image (RW) for writing the JITed instructions. But also save 2266 * the ro_image (RX) for calculating the offsets in the image. The RW 2267 * image will be later copied to the RX image from where the program will 2268 * run. The bpf_jit_binary_pack_finalize() will do this copy in the final 2269 * step. 2270 */ 2271 ctx.image = (union loongarch_instruction *)image_ptr; 2272 ctx.ro_image = (union loongarch_instruction *)ro_image_ptr; 2273 if (extable_size) 2274 prog->aux->extable = (void *)ro_image_ptr + prog_size; 2275 2276 skip_init_ctx: 2277 ctx.idx = 0; 2278 ctx.num_exentries = 0; 2279 2280 build_prologue(&ctx); 2281 if (build_body(&ctx, extra_pass)) 2282 goto out_free; 2283 build_epilogue(&ctx); 2284 2285 /* 3. Extra pass to validate JITed code */ 2286 if (validate_ctx(&ctx)) 2287 goto out_free; 2288 2289 /* And we're done */ 2290 if (bpf_jit_enable > 1) 2291 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); 2292 2293 if (!prog->is_func || extra_pass) { 2294 if (extra_pass && ctx.idx != jit_data->ctx.idx) { 2295 pr_err_once("multi-func JIT bug %d != %d\n", 2296 ctx.idx, jit_data->ctx.idx); 2297 goto out_free; 2298 } 2299 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { 2300 /* ro_header and header have been freed */ 2301 ro_header = NULL; 2302 header = NULL; 2303 goto out_free; 2304 } 2305 /* 2306 * The instructions have now been copied to the ROX region from 2307 * where they will execute. Now the data cache has to be cleaned 2308 * to the PoU and the I-cache has to be invalidated for the VAs. 2309 */ 2310 bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx); 2311 } else { 2312 jit_data->ctx = ctx; 2313 jit_data->header = header; 2314 jit_data->ro_header = ro_header; 2315 } 2316 prog->jited = 1; 2317 prog->jited_len = prog_size; 2318 prog->bpf_func = (void *)ctx.ro_image; 2319 2320 if (!prog->is_func || extra_pass) { 2321 int i; 2322 2323 /* offset[prog->len] is the size of program */ 2324 for (i = 0; i <= prog->len; i++) 2325 ctx.offset[i] *= LOONGARCH_INSN_SIZE; 2326 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); 2327 2328 out_offset: 2329 kvfree(ctx.offset); 2330 kfree(jit_data); 2331 prog->aux->jit_data = NULL; 2332 } 2333 2334 return prog; 2335 2336 out_free: 2337 if (extra_pass) { 2338 prog->bpf_func = NULL; 2339 prog->jited = 0; 2340 prog->jited_len = 0; 2341 } 2342 2343 if (header) { 2344 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); 2345 bpf_jit_binary_pack_free(ro_header, header); 2346 } 2347 goto out_offset; 2348 } 2349 2350 void bpf_jit_free(struct bpf_prog *prog) 2351 { 2352 if (prog->jited) { 2353 struct jit_data *jit_data = prog->aux->jit_data; 2354 struct bpf_binary_header *hdr; 2355 2356 /* 2357 * If we fail the final pass of JIT (from jit_subprogs), the 2358 * program may not be finalized yet. Call finalize here before 2359 * freeing it. 2360 */ 2361 if (jit_data) { 2362 bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); 2363 kfree(jit_data); 2364 } 2365 hdr = bpf_jit_binary_pack_hdr(prog); 2366 bpf_jit_binary_pack_free(hdr, NULL); 2367 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); 2368 } 2369 2370 bpf_prog_unlock_free(prog); 2371 } 2372 2373 bool bpf_jit_bypass_spec_v1(void) 2374 { 2375 return true; 2376 } 2377 2378 bool bpf_jit_bypass_spec_v4(void) 2379 { 2380 return true; 2381 } 2382 2383 bool bpf_jit_supports_arena(void) 2384 { 2385 return true; 2386 } 2387 2388 bool bpf_jit_supports_fsession(void) 2389 { 2390 return true; 2391 } 2392 2393 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ 2394 bool bpf_jit_supports_subprog_tailcalls(void) 2395 { 2396 return true; 2397 } 2398 2399 bool bpf_jit_inlines_helper_call(s32 imm) 2400 { 2401 switch (imm) { 2402 case BPF_FUNC_get_current_task: 2403 case BPF_FUNC_get_current_task_btf: 2404 case BPF_FUNC_get_smp_processor_id: 2405 return true; 2406 default: 2407 return false; 2408 } 2409 } 2410