Lines Matching +full:function +full:- +full:off
1 // SPDX-License-Identifier: GPL-2.0
3 * BPF JIT compiler for PA-RISC (64-bit)
10 * - check if bpf_jit_needs_zext() is needed (currently enabled)
11 * - implement arch_prepare_bpf_trampoline(), poke(), ...
39 * HPPA64 sp => +----------+ <= HPPA64 fp
42 * | ... | HPPA64 callee-saved registers
45 * +----------+ <= (BPF FP)
49 * | ... | Function call stack
51 * +----------+
58 #define EXIT_PTR_LOAD(reg) hppa64_ldd_im16(-FRAME_SIZE, HPPA_REG_SP, reg)
59 #define EXIT_PTR_STORE(reg) hppa64_std_im16(reg, -FRAME_SIZE, HPPA_REG_SP)
83 pos &= (BITS_PER_LONG - 1); in emit_hppa64_depd()
84 pos = 63 - pos; in emit_hppa64_depd()
85 len = 64 - len; in emit_hppa64_depd()
94 emit_hppa64_depd(src, 63-num, 64-num, target, 0, ctx); in emit_hppa64_shld()
101 pos &= (BITS_PER_LONG - 1); in emit_hppa64_extrd()
102 len = 64 - len; in emit_hppa64_extrd()
113 pos &= (32 - 1); in emit_hppa64_extrw()
114 len = 32 - len; in emit_hppa64_extrw()
126 emit_hppa64_extrd(src, 63-num, 64-num, target, signed_op, ctx); in emit_hppa64_shrd()
131 emit_hppa64_extrw(src, 31-num, 32-num, target, signed_op, ctx); in emit_hppa64_shrw()
134 /* Emit variable-length instructions for 32-bit imm */
159 /* Emit variable-length instructions for 64-bit imm */
164 /* get lower 32-bits into rd, sign extended */ in emit_imm()
167 /* do we have upper 32-bits too ? */ in emit_imm()
171 /* load upper 32-bits into lower tmpreg and deposit into rd */ in emit_imm()
187 if (relative_bits_ok(paoff - HPPA_BRANCH_DISPLACEMENT, 22)) { in emit_jump()
189 emit(hppa64_bl_long(paoff - HPPA_BRANCH_DISPLACEMENT), ctx); in emit_jump()
195 pc = (uintptr_t) &ctx->insns[ctx->ninsns]; in emit_jump()
197 /* even the 64-bit kernel runs in memory below 4GB */ in emit_jump()
199 return -E2BIG; in emit_jump()
224 /* load epilogue function pointer and jump to it. */ in __build_epilogue()
225 /* exit point is either at next instruction, or the outest TCC exit function */ in __build_epilogue()
229 /* NOTE: we are 64-bit and big-endian, so return lower sign-extended 32-bit value */ in __build_epilogue()
232 /* Restore callee-saved registers. */ in __build_epilogue()
236 emit(hppa64_ldd_im16(-REG_SIZE * i, HPPA_REG_SP, HPPA_R(i)), ctx); in __build_epilogue()
239 /* load original return pointer (stored by outest TCC function) */ in __build_epilogue()
240 emit(hppa64_ldd_im16(-2*REG_SIZE, HPPA_REG_SP, HPPA_REG_RP), ctx); in __build_epilogue()
243 emit(hppa64_ldd_im5(-REG_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); in __build_epilogue()
253 int off; in emit_branch() local
258 * to and() function and compare against zero in emit_branch()
261 paoff -= 1; /* reduce offset due to hppa_and() above */ in emit_branch()
268 s = ctx->ninsns; in emit_branch()
270 if (!relative_branch_ok(paoff - HPPA_BRANCH_DISPLACEMENT + 1, 12)) { in emit_branch()
280 off = far ? (2 - HPPA_BRANCH_DISPLACEMENT) : paoff - HPPA_BRANCH_DISPLACEMENT; in emit_branch()
283 /* IF (dst COND src) JUMP off */ in emit_branch()
285 emit(hppa_beq(rd, rs, off), ctx); in emit_branch()
288 emit(hppa_bgtu(rd, rs, off), ctx); in emit_branch()
291 emit(hppa_bltu(rd, rs, off), ctx); in emit_branch()
294 emit(hppa_bgeu(rd, rs, off), ctx); in emit_branch()
297 emit(hppa_bleu(rd, rs, off), ctx); in emit_branch()
300 emit(hppa_bne(rd, rs, off), ctx); in emit_branch()
303 emit(hppa_bgt(rd, rs, off), ctx); in emit_branch()
306 emit(hppa_blt(rd, rs, off), ctx); in emit_branch()
309 emit(hppa_bge(rd, rs, off), ctx); in emit_branch()
312 emit(hppa_ble(rd, rs, off), ctx); in emit_branch()
320 e = ctx->ninsns; in emit_branch()
322 paoff -= (e - s); in emit_branch()
345 * R1 -> &ctx in emit_bpf_tail_call()
346 * R2 -> &array in emit_bpf_tail_call()
347 * R3 -> index in emit_bpf_tail_call()
349 int off; in emit_bpf_tail_call() local
358 /* get address of TCC main exit function for error case into rp */ in emit_bpf_tail_call()
361 /* max_entries = array->map.max_entries; */ in emit_bpf_tail_call()
362 off = offsetof(struct bpf_array, map.max_entries); in emit_bpf_tail_call()
364 emit(hppa_ldw(off, arr_reg, HPPA_REG_T1), ctx); in emit_bpf_tail_call()
370 emit(hppa_bltu(idx_reg, HPPA_REG_T1, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); in emit_bpf_tail_call()
374 * if (--tcc < 0) in emit_bpf_tail_call()
378 emit(hppa_ldo(-1, HPPA_REG_TCC, HPPA_REG_TCC), ctx); in emit_bpf_tail_call()
379 emit(hppa_bge(HPPA_REG_TCC, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); in emit_bpf_tail_call()
383 * prog = array->ptrs[index]; in emit_bpf_tail_call()
389 off = offsetof(struct bpf_array, ptrs); in emit_bpf_tail_call()
390 BUILD_BUG_ON(off < 16); in emit_bpf_tail_call()
391 emit(hppa64_ldd_im16(off, HPPA_REG_T0, HPPA_REG_T0), ctx); in emit_bpf_tail_call()
392 emit(hppa_bne(HPPA_REG_T0, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); in emit_bpf_tail_call()
397 * goto *(prog->bpf_func + 4); in emit_bpf_tail_call()
399 off = offsetof(struct bpf_prog, bpf_func); in emit_bpf_tail_call()
400 BUILD_BUG_ON(off < 16); in emit_bpf_tail_call()
402 emit(hppa64_ldd_im16(off, HPPA_REG_T0, HPPA_REG_T0), ctx); in emit_bpf_tail_call()
410 u8 code = insn->code; in init_regs()
419 *rd = bpf_to_hppa_reg(insn->dst_reg, ctx); in init_regs()
425 *rs = bpf_to_hppa_reg(insn->src_reg, ctx); in init_regs()
486 /* load function address and gp from Elf64_Fdesc descriptor */ in emit_call()
499 emit(hppa_ldo(-offset_sp, HPPA_REG_SP, HPPA_REG_SP), ctx); in emit_call()
528 /* load function func_address and gp from Elf64_Fdesc descriptor */ in emit_call_libgcc_ll()
539 emit(hppa_beq(HPPA_REG_ARG1, HPPA_REG_ZERO, 2 - HPPA_BRANCH_DISPLACEMENT), ctx); in emit_call_libgcc_ll()
545 emit(hppa_ldo(-FRAME_SIZE, HPPA_REG_SP, HPPA_REG_SP), ctx); in emit_call_libgcc_ll()
554 static void emit_store(const s8 rd, const s8 rs, s16 off, in emit_store() argument
561 if (relative_bits_ok(off, 14)) in emit_store()
566 emit(hppa_addil(off, rd), ctx); in emit_store()
567 off = im11(off); in emit_store()
572 emit(hppa_stb(rs, off, dstreg), ctx); in emit_store()
575 emit(hppa_sth(rs, off, dstreg), ctx); in emit_store()
578 emit(hppa_stw(rs, off, dstreg), ctx); in emit_store()
581 if (off & 7) { in emit_store()
582 emit(hppa_ldo(off, dstreg, HPPA_REG_R1), ctx); in emit_store()
584 } else if (off >= -16 && off <= 15) in emit_store()
585 emit(hppa64_std_im5(rs, off, dstreg), ctx); in emit_store()
587 emit(hppa64_std_im16(rs, off, dstreg), ctx); in emit_store()
595 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
596 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
597 int s, e, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
599 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
600 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn()
601 s16 off = insn->off; in bpf_jit_emit_insn() local
602 s32 imm = insn->imm; in bpf_jit_emit_insn()
615 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
625 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
631 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
637 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
643 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
649 if (!is64 && !aux->verifier_zext && rs != rd) in bpf_jit_emit_insn()
660 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
671 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
682 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
694 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
704 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
715 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
719 /* dst = -dst */ in bpf_jit_emit_insn()
723 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
731 /* zero-extend 16 bits into 64 bits */ in bpf_jit_emit_insn()
732 emit_hppa64_depd(HPPA_REG_ZERO, 63-16, 64-16, rd, 1, ctx); in bpf_jit_emit_insn()
735 if (!aux->verifier_zext) in bpf_jit_emit_insn()
747 emit(hppa_extru(rd, 31 - 8, 8, HPPA_REG_T1), ctx); in bpf_jit_emit_insn()
754 emit_hppa64_depd(HPPA_REG_T1, 63-16, 8, HPPA_REG_T1, 1, ctx); in bpf_jit_emit_insn()
765 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm); in bpf_jit_emit_insn()
766 return -1; in bpf_jit_emit_insn()
774 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
787 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
792 if (relative_bits_ok(-imm, 14)) { in bpf_jit_emit_insn()
793 emit(hppa_ldo(-imm, rd, rd), ctx); in bpf_jit_emit_insn()
798 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
805 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
812 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
819 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
828 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
840 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
852 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
856 /* JUMP off */ in bpf_jit_emit_insn()
858 paoff = hppa_offset(i, off, ctx); in bpf_jit_emit_insn()
864 /* IF (dst COND src) JUMP off */ in bpf_jit_emit_insn()
887 paoff = hppa_offset(i, off, ctx); in bpf_jit_emit_insn()
889 s = ctx->ninsns; in bpf_jit_emit_insn()
894 e = ctx->ninsns; in bpf_jit_emit_insn()
897 paoff -= (e - s); in bpf_jit_emit_insn()
901 paoff -= 1; in bpf_jit_emit_insn()
910 /* IF (dst COND imm) JUMP off */ in bpf_jit_emit_insn()
931 paoff = hppa_offset(i, off, ctx); in bpf_jit_emit_insn()
932 s = ctx->ninsns; in bpf_jit_emit_insn()
945 e = ctx->ninsns; in bpf_jit_emit_insn()
948 paoff -= (e - s); in bpf_jit_emit_insn()
953 paoff = hppa_offset(i, off, ctx); in bpf_jit_emit_insn()
954 s = ctx->ninsns; in bpf_jit_emit_insn()
958 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
963 e = ctx->ninsns; in bpf_jit_emit_insn()
964 paoff -= (e - s); in bpf_jit_emit_insn()
967 /* function call */ in bpf_jit_emit_insn()
973 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, in bpf_jit_emit_insn()
987 /* function return */ in bpf_jit_emit_insn()
989 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
1010 /* LDX: dst = *(size *)(src + off) */ in bpf_jit_emit_insn()
1023 if (relative_bits_ok(off, 14)) in bpf_jit_emit_insn()
1030 emit(hppa_addil(off, rs), ctx); in bpf_jit_emit_insn()
1031 off = im11(off); in bpf_jit_emit_insn()
1036 emit(hppa_ldb(off, srcreg, rd), ctx); in bpf_jit_emit_insn()
1041 emit(hppa_ldh(off, srcreg, rd), ctx); in bpf_jit_emit_insn()
1046 emit(hppa_ldw(off, srcreg, rd), ctx); in bpf_jit_emit_insn()
1051 if (off & 7) { in bpf_jit_emit_insn()
1052 emit(hppa_ldo(off, srcreg, HPPA_REG_R1), ctx); in bpf_jit_emit_insn()
1054 } else if (off >= -16 && off <= 15) in bpf_jit_emit_insn()
1055 emit(hppa64_ldd_im5(off, srcreg, rd), ctx); in bpf_jit_emit_insn()
1057 emit(hppa64_ldd_im16(off, srcreg, rd), ctx); in bpf_jit_emit_insn()
1066 /* ST: *(size *)(dst + off) = imm */ in bpf_jit_emit_insn()
1067 /* STX: *(size *)(dst + off) = src */ in bpf_jit_emit_insn()
1082 emit_store(rd, rs, off, ctx, BPF_SIZE(code), BPF_MODE(code)); in bpf_jit_emit_insn()
1088 "bpf-jit: not supported: atomic operation %02x ***\n", in bpf_jit_emit_insn()
1089 insn->imm); in bpf_jit_emit_insn()
1090 return -EFAULT; in bpf_jit_emit_insn()
1093 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
1094 return -EINVAL; in bpf_jit_emit_insn()
1113 bpf_stack_adjust = ctx->prog->aux->stack_depth; in bpf_jit_build_prologue()
1122 * Then follows the virtual address of the eBPF function, in bpf_jit_build_prologue()
1123 * and the gp for this function. in bpf_jit_build_prologue()
1125 * The first instruction sets the tail-call-counter (TCC) register. in bpf_jit_build_prologue()
1127 * Use a temporary register instead of a caller-saved register initially. in bpf_jit_build_prologue()
1136 emit(hppa_beq(HPPA_REG_TCC_IN_INIT, HPPA_REG_R1, 6 - HPPA_BRANCH_DISPLACEMENT), ctx); in bpf_jit_build_prologue()
1137 emit(hppa64_bl_long(ctx->prologue_len - 3 - HPPA_BRANCH_DISPLACEMENT), ctx); in bpf_jit_build_prologue()
1139 /* store entry address of this eBPF function */ in bpf_jit_build_prologue()
1140 addr = (uintptr_t) &ctx->insns[0]; in bpf_jit_build_prologue()
1144 /* store gp of this eBPF function */ in bpf_jit_build_prologue()
1152 emit(hppa64_std_im5 (HPPA_REG_R1, -REG_SIZE, HPPA_REG_SP), ctx); in bpf_jit_build_prologue()
1153 emit(hppa64_std_im16(HPPA_REG_RP, -2*REG_SIZE, HPPA_REG_SP), ctx); in bpf_jit_build_prologue()
1155 /* Save callee-save registers. */ in bpf_jit_build_prologue()
1159 emit(hppa64_std_im16(HPPA_R(i), -REG_SIZE * i, HPPA_REG_SP), ctx); in bpf_jit_build_prologue()
1162 /* load function parameters; load all if we use tail functions */ in bpf_jit_build_prologue()
1185 * Save epilogue function pointer for outer TCC call chain. in bpf_jit_build_prologue()
1188 addr = (uintptr_t) &ctx->insns[ctx->epilogue_offset]; in bpf_jit_build_prologue()
1195 reg = regmap[BPF_REG_FP]; /* -> HPPA_REG_FP */ in bpf_jit_build_prologue()
1197 emit(hppa_ldo(-FRAME_SIZE, HPPA_REG_SP, reg), ctx); in bpf_jit_build_prologue()