Lines Matching +full:lower +full:- +full:case
1 // SPDX-License-Identifier: GPL-2.0
13 #include <asm/text-patching.h>
75 case RV_CTX_F_SEEN_S1: in bpf_to_rv_reg()
76 case RV_CTX_F_SEEN_S2: in bpf_to_rv_reg()
77 case RV_CTX_F_SEEN_S3: in bpf_to_rv_reg()
78 case RV_CTX_F_SEEN_S4: in bpf_to_rv_reg()
79 case RV_CTX_F_SEEN_S5: in bpf_to_rv_reg()
80 case RV_CTX_F_SEEN_S6: in bpf_to_rv_reg()
81 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
89 case RV_CTX_F_SEEN_CALL: in seen_reg()
90 case RV_CTX_F_SEEN_S1: in seen_reg()
91 case RV_CTX_F_SEEN_S2: in seen_reg()
92 case RV_CTX_F_SEEN_S3: in seen_reg()
93 case RV_CTX_F_SEEN_S4: in seen_reg()
94 case RV_CTX_F_SEEN_S5: in seen_reg()
95 case RV_CTX_F_SEEN_S6: in seen_reg()
96 return test_bit(reg, &ctx->flags); in seen_reg()
103 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
108 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
113 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
118 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
123 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
131 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
139 return -(1L << 31) <= val && val < (1L << 31); in is_32b_int()
145 * auipc+jalr can reach any signed PC-relative offset in the range in in_auipc_jalr_range()
146 * [-2^31 - 2^11, 2^31 - 2^11). in in_auipc_jalr_range()
148 return (-(1L << 31) - (1L << 11)) <= val && in in_auipc_jalr_range()
149 val < ((1L << 31) - (1L << 11)); in in_auipc_jalr_range()
165 /* Emit fixed-length instructions for address */
172 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); in emit_addr()
173 s64 off = addr - ip; in emit_addr()
175 s64 lower = off & 0xfff; in emit_addr() local
178 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); in emit_addr()
179 return -ERANGE; in emit_addr()
183 emit(rv_addi(rd, rd, lower), ctx); in emit_addr()
187 /* Emit variable-length instructions for 32-bit and 64-bit imm */
190 /* Note that the immediate from the add is sign-extended, in emit_imm()
194 * shift. The "Loading a 32-Bit constant" example from the in emit_imm()
195 * "Computer Organization and Design, RISC-V edition" book by in emit_imm()
201 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, in emit_imm()
204 s64 lower = ((val & 0xfff) << 52) >> 52; in emit_imm() local
212 emit_li(rd, lower, ctx); in emit_imm()
216 emit_addiw(rd, rd, lower, ctx); in emit_imm()
227 if (lower) in emit_imm()
228 emit_addi(rd, rd, lower, ctx); in emit_imm()
233 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
237 store_offset -= 8; in __build_epilogue()
240 store_offset -= 8; in __build_epilogue()
243 store_offset -= 8; in __build_epilogue()
247 store_offset -= 8; in __build_epilogue()
251 store_offset -= 8; in __build_epilogue()
255 store_offset -= 8; in __build_epilogue()
259 store_offset -= 8; in __build_epilogue()
263 store_offset -= 8; in __build_epilogue()
265 if (ctx->arena_vm_start) { in __build_epilogue()
267 store_offset -= 8; in __build_epilogue()
284 case BPF_JEQ: in emit_bcc()
287 case BPF_JGT: in emit_bcc()
290 case BPF_JLT: in emit_bcc()
293 case BPF_JGE: in emit_bcc()
296 case BPF_JLE: in emit_bcc()
299 case BPF_JNE: in emit_bcc()
302 case BPF_JSGT: in emit_bcc()
305 case BPF_JSLT: in emit_bcc()
308 case BPF_JSGE: in emit_bcc()
311 case BPF_JSLE: in emit_bcc()
319 s64 upper, lower; in emit_branch() local
327 rvoff -= 4; in emit_branch()
348 lower = rvoff & 0xfff; in emit_branch()
352 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); in emit_branch()
357 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
364 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
367 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
368 ctx->offset[0]; in emit_bpf_tail_call()
373 return -1; in emit_bpf_tail_call()
375 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
378 /* if (--TCC < 0) in emit_bpf_tail_call()
381 emit_addi(RV_REG_TCC, tcc, -1, ctx); in emit_bpf_tail_call()
382 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
385 /* prog = array->ptrs[index]; in emit_bpf_tail_call()
392 return -1; in emit_bpf_tail_call()
394 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
397 /* goto *(prog->bpf_func + 4); */ in emit_bpf_tail_call()
400 return -1; in emit_bpf_tail_call()
409 u8 code = insn->code; in init_regs()
412 case BPF_JMP | BPF_JA: in init_regs()
413 case BPF_JMP | BPF_CALL: in init_regs()
414 case BPF_JMP | BPF_EXIT: in init_regs()
415 case BPF_JMP | BPF_TAIL_CALL: in init_regs()
418 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
424 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
430 s64 upper, lower; in emit_jump_and_link() local
437 lower = rvoff & 0xfff; in emit_jump_and_link()
439 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); in emit_jump_and_link()
443 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); in emit_jump_and_link()
444 return -ERANGE; in emit_jump_and_link()
458 if (addr && ctx->insns && ctx->ro_insns) { in emit_call()
463 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); in emit_call()
464 off = addr - ip; in emit_call()
494 case BPF_ADD: in emit_atomic()
498 case BPF_AND: in emit_atomic()
502 case BPF_OR: in emit_atomic()
506 case BPF_XOR: in emit_atomic()
511 case BPF_ADD | BPF_FETCH: in emit_atomic()
517 case BPF_AND | BPF_FETCH: in emit_atomic()
523 case BPF_OR | BPF_FETCH: in emit_atomic()
529 case BPF_XOR | BPF_FETCH: in emit_atomic()
536 case BPF_XCHG: in emit_atomic()
543 case BPF_CMPXCHG: in emit_atomic()
555 jmp_offset = ninsns_rvoff(-6); in emit_atomic()
569 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); in ex_handler_bpf()
570 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); in ex_handler_bpf()
574 regs->epc = (unsigned long)&ex->fixup - offset; in ex_handler_bpf()
589 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || in add_exception_handler()
590 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && in add_exception_handler()
591 BPF_MODE(insn->code) != BPF_PROBE_MEM32)) in add_exception_handler()
594 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) in add_exception_handler()
595 return -EINVAL; in add_exception_handler()
597 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) in add_exception_handler()
598 return -EINVAL; in add_exception_handler()
601 return -EINVAL; in add_exception_handler()
603 ex = &ctx->prog->aux->extable[ctx->nexentries]; in add_exception_handler()
604 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; in add_exception_handler()
612 ins_offset = pc - (long)&ex->insn; in add_exception_handler()
614 return -ERANGE; in add_exception_handler()
628 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); in add_exception_handler()
630 return -ERANGE; in add_exception_handler()
637 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); in add_exception_handler()
639 ex->insn = ins_offset; in add_exception_handler()
641 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | in add_exception_handler()
643 ex->type = EX_TYPE_BPF; in add_exception_handler()
645 ctx->nexentries++; in add_exception_handler()
663 rvoff = (s64)(target - ip); in gen_jump_or_nops()
676 return -ENOTSUPP; in bpf_arch_text_poke()
683 return -EFAULT; in bpf_arch_text_poke()
705 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); in store_args()
708 emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); in store_args()
709 emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); in store_args()
711 args_off -= 8; in store_args()
720 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); in restore_args()
721 args_off -= 8; in restore_args()
731 emit_ld(RV_REG_T1, -(args_off - RV_MAX_REG_ARGS * 8), RV_REG_FP, ctx); in restore_stack_args()
732 emit_sd(RV_REG_FP, -stk_arg_off, RV_REG_T1, ctx); in restore_stack_args()
733 args_off -= 8; in restore_stack_args()
734 stk_arg_off -= 8; in restore_stack_args()
742 struct bpf_prog *p = l->link.prog; in invoke_bpf_prog()
745 if (l->cookie) { in invoke_bpf_prog()
746 emit_imm(RV_REG_T1, l->cookie, ctx); in invoke_bpf_prog()
747 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); in invoke_bpf_prog()
749 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); in invoke_bpf_prog()
755 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); in invoke_bpf_prog()
766 branch_off = ctx->ninsns; in invoke_bpf_prog()
771 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); in invoke_bpf_prog()
772 if (!p->jited) in invoke_bpf_prog()
773 /* arg2: progs[i]->insnsi for interpreter */ in invoke_bpf_prog()
774 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); in invoke_bpf_prog()
775 ret = emit_call((const u64)p->bpf_func, true, ctx); in invoke_bpf_prog()
780 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); in invoke_bpf_prog()
781 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); in invoke_bpf_prog()
785 if (ctx->insns) { in invoke_bpf_prog()
786 int offset = ninsns_rvoff(ctx->ninsns - branch_off); in invoke_bpf_prog()
788 *(u32 *)(ctx->insns + branch_off) = insn; in invoke_bpf_prog()
796 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); in invoke_bpf_prog()
823 * -------------------------------------- in __arch_prepare_bpf_trampoline()
828 * FP - 8 [ T0 to traced func ] return address of traced in __arch_prepare_bpf_trampoline()
830 * FP - 16 [ FP of traced func ] frame pointer of traced in __arch_prepare_bpf_trampoline()
832 * -------------------------------------- in __arch_prepare_bpf_trampoline()
835 * -------------------------------------- in __arch_prepare_bpf_trampoline()
836 * FP - 8 [ RA to caller func ] return address to caller in __arch_prepare_bpf_trampoline()
838 * FP - 16 [ FP of caller func ] frame pointer of caller in __arch_prepare_bpf_trampoline()
840 * -------------------------------------- in __arch_prepare_bpf_trampoline()
842 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or in __arch_prepare_bpf_trampoline()
846 * FP - args_off [ arg1 ] in __arch_prepare_bpf_trampoline()
848 * FP - nregs_off [ regs count ] in __arch_prepare_bpf_trampoline()
850 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG in __arch_prepare_bpf_trampoline()
852 * FP - run_ctx_off [ bpf_tramp_run_ctx ] in __arch_prepare_bpf_trampoline()
854 * FP - sreg_off [ callee saved reg ] in __arch_prepare_bpf_trampoline()
860 * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG in __arch_prepare_bpf_trampoline()
864 return -ENOTSUPP; in __arch_prepare_bpf_trampoline()
866 if (m->nr_args > MAX_BPF_FUNC_ARGS) in __arch_prepare_bpf_trampoline()
867 return -ENOTSUPP; in __arch_prepare_bpf_trampoline()
869 for (i = 0; i < m->nr_args; i++) in __arch_prepare_bpf_trampoline()
870 nr_arg_slots += round_up(m->arg_size[i], 8) / 8; in __arch_prepare_bpf_trampoline()
898 if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) in __arch_prepare_bpf_trampoline()
899 stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; in __arch_prepare_bpf_trampoline()
911 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); in __arch_prepare_bpf_trampoline()
916 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); in __arch_prepare_bpf_trampoline()
917 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); in __arch_prepare_bpf_trampoline()
918 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
926 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); in __arch_prepare_bpf_trampoline()
927 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); in __arch_prepare_bpf_trampoline()
928 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
933 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); in __arch_prepare_bpf_trampoline()
938 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); in __arch_prepare_bpf_trampoline()
942 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); in __arch_prepare_bpf_trampoline()
951 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); in __arch_prepare_bpf_trampoline()
957 for (i = 0; i < fentry->nr_links; i++) { in __arch_prepare_bpf_trampoline()
958 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, in __arch_prepare_bpf_trampoline()
964 if (fmod_ret->nr_links) { in __arch_prepare_bpf_trampoline()
965 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); in __arch_prepare_bpf_trampoline()
967 return -ENOMEM; in __arch_prepare_bpf_trampoline()
970 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); in __arch_prepare_bpf_trampoline()
971 for (i = 0; i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
972 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, in __arch_prepare_bpf_trampoline()
976 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
977 branches_off[i] = ctx->ninsns; in __arch_prepare_bpf_trampoline()
985 restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); in __arch_prepare_bpf_trampoline()
989 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); in __arch_prepare_bpf_trampoline()
990 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); in __arch_prepare_bpf_trampoline()
991 im->ip_after_call = ctx->ro_insns + ctx->ninsns; in __arch_prepare_bpf_trampoline()
998 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
999 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); in __arch_prepare_bpf_trampoline()
1001 *(u32 *)(ctx->insns + branches_off[i]) = insn; in __arch_prepare_bpf_trampoline()
1004 for (i = 0; i < fexit->nr_links; i++) { in __arch_prepare_bpf_trampoline()
1005 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, in __arch_prepare_bpf_trampoline()
1012 im->ip_epilogue = ctx->ro_insns + ctx->ninsns; in __arch_prepare_bpf_trampoline()
1013 emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); in __arch_prepare_bpf_trampoline()
1023 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
1024 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
1027 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
1031 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1032 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1047 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1048 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1054 ret = ctx->ninsns; in __arch_prepare_bpf_trampoline()
1093 u32 size = ro_image_end - ro_image; in arch_prepare_bpf_trampoline()
1097 return -ENOMEM; in arch_prepare_bpf_trampoline()
1107 ret = -E2BIG; in arch_prepare_bpf_trampoline()
1126 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
1127 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
1128 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
1129 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
1130 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn()
1131 s16 off = insn->off; in bpf_jit_emit_insn()
1132 s32 imm = insn->imm; in bpf_jit_emit_insn()
1138 case BPF_ALU | BPF_MOV | BPF_X: in bpf_jit_emit_insn()
1139 case BPF_ALU64 | BPF_MOV | BPF_X: in bpf_jit_emit_insn()
1143 emit_imm(rd, (ctx->user_vm_start >> 32) << 32, ctx); in bpf_jit_emit_insn()
1170 switch (insn->off) { in bpf_jit_emit_insn()
1171 case 0: in bpf_jit_emit_insn()
1174 case 8: in bpf_jit_emit_insn()
1177 case 16: in bpf_jit_emit_insn()
1180 case 32: in bpf_jit_emit_insn()
1184 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1189 case BPF_ALU | BPF_ADD | BPF_X: in bpf_jit_emit_insn()
1190 case BPF_ALU64 | BPF_ADD | BPF_X: in bpf_jit_emit_insn()
1192 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1195 case BPF_ALU | BPF_SUB | BPF_X: in bpf_jit_emit_insn()
1196 case BPF_ALU64 | BPF_SUB | BPF_X: in bpf_jit_emit_insn()
1202 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1205 case BPF_ALU | BPF_AND | BPF_X: in bpf_jit_emit_insn()
1206 case BPF_ALU64 | BPF_AND | BPF_X: in bpf_jit_emit_insn()
1208 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1211 case BPF_ALU | BPF_OR | BPF_X: in bpf_jit_emit_insn()
1212 case BPF_ALU64 | BPF_OR | BPF_X: in bpf_jit_emit_insn()
1214 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1217 case BPF_ALU | BPF_XOR | BPF_X: in bpf_jit_emit_insn()
1218 case BPF_ALU64 | BPF_XOR | BPF_X: in bpf_jit_emit_insn()
1220 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1223 case BPF_ALU | BPF_MUL | BPF_X: in bpf_jit_emit_insn()
1224 case BPF_ALU64 | BPF_MUL | BPF_X: in bpf_jit_emit_insn()
1226 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1229 case BPF_ALU | BPF_DIV | BPF_X: in bpf_jit_emit_insn()
1230 case BPF_ALU64 | BPF_DIV | BPF_X: in bpf_jit_emit_insn()
1235 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1238 case BPF_ALU | BPF_MOD | BPF_X: in bpf_jit_emit_insn()
1239 case BPF_ALU64 | BPF_MOD | BPF_X: in bpf_jit_emit_insn()
1244 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1247 case BPF_ALU | BPF_LSH | BPF_X: in bpf_jit_emit_insn()
1248 case BPF_ALU64 | BPF_LSH | BPF_X: in bpf_jit_emit_insn()
1250 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1253 case BPF_ALU | BPF_RSH | BPF_X: in bpf_jit_emit_insn()
1254 case BPF_ALU64 | BPF_RSH | BPF_X: in bpf_jit_emit_insn()
1256 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1259 case BPF_ALU | BPF_ARSH | BPF_X: in bpf_jit_emit_insn()
1260 case BPF_ALU64 | BPF_ARSH | BPF_X: in bpf_jit_emit_insn()
1262 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1266 /* dst = -dst */ in bpf_jit_emit_insn()
1267 case BPF_ALU | BPF_NEG: in bpf_jit_emit_insn()
1268 case BPF_ALU64 | BPF_NEG: in bpf_jit_emit_insn()
1270 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1275 case BPF_ALU | BPF_END | BPF_FROM_LE: in bpf_jit_emit_insn()
1277 case 16: in bpf_jit_emit_insn()
1280 case 32: in bpf_jit_emit_insn()
1281 if (!aux->verifier_zext) in bpf_jit_emit_insn()
1284 case 64: in bpf_jit_emit_insn()
1289 case BPF_ALU | BPF_END | BPF_FROM_BE: in bpf_jit_emit_insn()
1290 case BPF_ALU64 | BPF_END | BPF_FROM_LE: in bpf_jit_emit_insn()
1295 case BPF_ALU | BPF_MOV | BPF_K: in bpf_jit_emit_insn()
1296 case BPF_ALU64 | BPF_MOV | BPF_K: in bpf_jit_emit_insn()
1298 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1303 case BPF_ALU | BPF_ADD | BPF_K: in bpf_jit_emit_insn()
1304 case BPF_ALU64 | BPF_ADD | BPF_K: in bpf_jit_emit_insn()
1311 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1314 case BPF_ALU | BPF_SUB | BPF_K: in bpf_jit_emit_insn()
1315 case BPF_ALU64 | BPF_SUB | BPF_K: in bpf_jit_emit_insn()
1316 if (is_12b_int(-imm)) { in bpf_jit_emit_insn()
1317 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
1322 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1325 case BPF_ALU | BPF_AND | BPF_K: in bpf_jit_emit_insn()
1326 case BPF_ALU64 | BPF_AND | BPF_K: in bpf_jit_emit_insn()
1333 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1336 case BPF_ALU | BPF_OR | BPF_K: in bpf_jit_emit_insn()
1337 case BPF_ALU64 | BPF_OR | BPF_K: in bpf_jit_emit_insn()
1344 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1347 case BPF_ALU | BPF_XOR | BPF_K: in bpf_jit_emit_insn()
1348 case BPF_ALU64 | BPF_XOR | BPF_K: in bpf_jit_emit_insn()
1355 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1358 case BPF_ALU | BPF_MUL | BPF_K: in bpf_jit_emit_insn()
1359 case BPF_ALU64 | BPF_MUL | BPF_K: in bpf_jit_emit_insn()
1363 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1366 case BPF_ALU | BPF_DIV | BPF_K: in bpf_jit_emit_insn()
1367 case BPF_ALU64 | BPF_DIV | BPF_K: in bpf_jit_emit_insn()
1375 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1378 case BPF_ALU | BPF_MOD | BPF_K: in bpf_jit_emit_insn()
1379 case BPF_ALU64 | BPF_MOD | BPF_K: in bpf_jit_emit_insn()
1387 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1390 case BPF_ALU | BPF_LSH | BPF_K: in bpf_jit_emit_insn()
1391 case BPF_ALU64 | BPF_LSH | BPF_K: in bpf_jit_emit_insn()
1394 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1397 case BPF_ALU | BPF_RSH | BPF_K: in bpf_jit_emit_insn()
1398 case BPF_ALU64 | BPF_RSH | BPF_K: in bpf_jit_emit_insn()
1404 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1407 case BPF_ALU | BPF_ARSH | BPF_K: in bpf_jit_emit_insn()
1408 case BPF_ALU64 | BPF_ARSH | BPF_K: in bpf_jit_emit_insn()
1414 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1419 case BPF_JMP | BPF_JA: in bpf_jit_emit_insn()
1420 case BPF_JMP32 | BPF_JA: in bpf_jit_emit_insn()
1431 case BPF_JMP | BPF_JEQ | BPF_X: in bpf_jit_emit_insn()
1432 case BPF_JMP32 | BPF_JEQ | BPF_X: in bpf_jit_emit_insn()
1433 case BPF_JMP | BPF_JGT | BPF_X: in bpf_jit_emit_insn()
1434 case BPF_JMP32 | BPF_JGT | BPF_X: in bpf_jit_emit_insn()
1435 case BPF_JMP | BPF_JLT | BPF_X: in bpf_jit_emit_insn()
1436 case BPF_JMP32 | BPF_JLT | BPF_X: in bpf_jit_emit_insn()
1437 case BPF_JMP | BPF_JGE | BPF_X: in bpf_jit_emit_insn()
1438 case BPF_JMP32 | BPF_JGE | BPF_X: in bpf_jit_emit_insn()
1439 case BPF_JMP | BPF_JLE | BPF_X: in bpf_jit_emit_insn()
1440 case BPF_JMP32 | BPF_JLE | BPF_X: in bpf_jit_emit_insn()
1441 case BPF_JMP | BPF_JNE | BPF_X: in bpf_jit_emit_insn()
1442 case BPF_JMP32 | BPF_JNE | BPF_X: in bpf_jit_emit_insn()
1443 case BPF_JMP | BPF_JSGT | BPF_X: in bpf_jit_emit_insn()
1444 case BPF_JMP32 | BPF_JSGT | BPF_X: in bpf_jit_emit_insn()
1445 case BPF_JMP | BPF_JSLT | BPF_X: in bpf_jit_emit_insn()
1446 case BPF_JMP32 | BPF_JSLT | BPF_X: in bpf_jit_emit_insn()
1447 case BPF_JMP | BPF_JSGE | BPF_X: in bpf_jit_emit_insn()
1448 case BPF_JMP32 | BPF_JSGE | BPF_X: in bpf_jit_emit_insn()
1449 case BPF_JMP | BPF_JSLE | BPF_X: in bpf_jit_emit_insn()
1450 case BPF_JMP32 | BPF_JSLE | BPF_X: in bpf_jit_emit_insn()
1451 case BPF_JMP | BPF_JSET | BPF_X: in bpf_jit_emit_insn()
1452 case BPF_JMP32 | BPF_JSET | BPF_X: in bpf_jit_emit_insn()
1455 s = ctx->ninsns; in bpf_jit_emit_insn()
1463 e = ctx->ninsns; in bpf_jit_emit_insn()
1466 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1471 rvoff -= 4; in bpf_jit_emit_insn()
1480 case BPF_JMP | BPF_JEQ | BPF_K: in bpf_jit_emit_insn()
1481 case BPF_JMP32 | BPF_JEQ | BPF_K: in bpf_jit_emit_insn()
1482 case BPF_JMP | BPF_JGT | BPF_K: in bpf_jit_emit_insn()
1483 case BPF_JMP32 | BPF_JGT | BPF_K: in bpf_jit_emit_insn()
1484 case BPF_JMP | BPF_JLT | BPF_K: in bpf_jit_emit_insn()
1485 case BPF_JMP32 | BPF_JLT | BPF_K: in bpf_jit_emit_insn()
1486 case BPF_JMP | BPF_JGE | BPF_K: in bpf_jit_emit_insn()
1487 case BPF_JMP32 | BPF_JGE | BPF_K: in bpf_jit_emit_insn()
1488 case BPF_JMP | BPF_JLE | BPF_K: in bpf_jit_emit_insn()
1489 case BPF_JMP32 | BPF_JLE | BPF_K: in bpf_jit_emit_insn()
1490 case BPF_JMP | BPF_JNE | BPF_K: in bpf_jit_emit_insn()
1491 case BPF_JMP32 | BPF_JNE | BPF_K: in bpf_jit_emit_insn()
1492 case BPF_JMP | BPF_JSGT | BPF_K: in bpf_jit_emit_insn()
1493 case BPF_JMP32 | BPF_JSGT | BPF_K: in bpf_jit_emit_insn()
1494 case BPF_JMP | BPF_JSLT | BPF_K: in bpf_jit_emit_insn()
1495 case BPF_JMP32 | BPF_JSLT | BPF_K: in bpf_jit_emit_insn()
1496 case BPF_JMP | BPF_JSGE | BPF_K: in bpf_jit_emit_insn()
1497 case BPF_JMP32 | BPF_JSGE | BPF_K: in bpf_jit_emit_insn()
1498 case BPF_JMP | BPF_JSLE | BPF_K: in bpf_jit_emit_insn()
1499 case BPF_JMP32 | BPF_JSLE | BPF_K: in bpf_jit_emit_insn()
1501 s = ctx->ninsns; in bpf_jit_emit_insn()
1515 e = ctx->ninsns; in bpf_jit_emit_insn()
1518 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1522 case BPF_JMP | BPF_JSET | BPF_K: in bpf_jit_emit_insn()
1523 case BPF_JMP32 | BPF_JSET | BPF_K: in bpf_jit_emit_insn()
1525 s = ctx->ninsns; in bpf_jit_emit_insn()
1533 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
1538 e = ctx->ninsns; in bpf_jit_emit_insn()
1539 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1544 case BPF_JMP | BPF_CALL: in bpf_jit_emit_insn()
1554 * Set R0 to ((struct thread_info *)(RV_REG_TP))->cpu in bpf_jit_emit_insn()
1558 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) { in bpf_jit_emit_insn()
1566 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, in bpf_jit_emit_insn()
1571 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { in bpf_jit_emit_insn()
1575 fm = bpf_jit_find_kfunc_model(ctx->prog, insn); in bpf_jit_emit_insn()
1577 return -EINVAL; in bpf_jit_emit_insn()
1579 for (idx = 0; idx < fm->nr_args; idx++) { in bpf_jit_emit_insn()
1582 if (fm->arg_size[idx] == sizeof(int)) in bpf_jit_emit_insn()
1591 if (insn->src_reg != BPF_PSEUDO_CALL) in bpf_jit_emit_insn()
1596 case BPF_JMP | BPF_TAIL_CALL: in bpf_jit_emit_insn()
1598 return -1; in bpf_jit_emit_insn()
1602 case BPF_JMP | BPF_EXIT: in bpf_jit_emit_insn()
1603 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
1613 case BPF_LD | BPF_IMM | BPF_DW: in bpf_jit_emit_insn()
1620 /* fixed-length insns for extra jit pass */ in bpf_jit_emit_insn()
1632 case BPF_LDX | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
1633 case BPF_LDX | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
1634 case BPF_LDX | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
1635 case BPF_LDX | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
1636 case BPF_LDX | BPF_PROBE_MEM | BPF_B: in bpf_jit_emit_insn()
1637 case BPF_LDX | BPF_PROBE_MEM | BPF_H: in bpf_jit_emit_insn()
1638 case BPF_LDX | BPF_PROBE_MEM | BPF_W: in bpf_jit_emit_insn()
1639 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: in bpf_jit_emit_insn()
1641 case BPF_LDX | BPF_MEMSX | BPF_B: in bpf_jit_emit_insn()
1642 case BPF_LDX | BPF_MEMSX | BPF_H: in bpf_jit_emit_insn()
1643 case BPF_LDX | BPF_MEMSX | BPF_W: in bpf_jit_emit_insn()
1644 case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: in bpf_jit_emit_insn()
1645 case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: in bpf_jit_emit_insn()
1646 case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: in bpf_jit_emit_insn()
1648 case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: in bpf_jit_emit_insn()
1649 case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: in bpf_jit_emit_insn()
1650 case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: in bpf_jit_emit_insn()
1651 case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: in bpf_jit_emit_insn()
1656 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || in bpf_jit_emit_insn()
1657 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; in bpf_jit_emit_insn()
1659 if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { in bpf_jit_emit_insn()
1665 case BPF_B: in bpf_jit_emit_insn()
1667 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1672 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1678 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1683 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1685 case BPF_H: in bpf_jit_emit_insn()
1687 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1692 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1698 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1703 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1705 case BPF_W: in bpf_jit_emit_insn()
1707 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1712 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1718 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1723 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1725 case BPF_DW: in bpf_jit_emit_insn()
1727 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1729 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1735 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1737 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1750 case BPF_ST | BPF_NOSPEC: in bpf_jit_emit_insn()
1754 case BPF_ST | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
1766 case BPF_ST | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
1777 case BPF_ST | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
1788 case BPF_ST | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
1800 case BPF_ST | BPF_PROBE_MEM32 | BPF_B: in bpf_jit_emit_insn()
1801 case BPF_ST | BPF_PROBE_MEM32 | BPF_H: in bpf_jit_emit_insn()
1802 case BPF_ST | BPF_PROBE_MEM32 | BPF_W: in bpf_jit_emit_insn()
1803 case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: in bpf_jit_emit_insn()
1814 case BPF_B: in bpf_jit_emit_insn()
1816 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1818 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1824 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1826 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1828 case BPF_H: in bpf_jit_emit_insn()
1830 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1832 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1838 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1840 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1842 case BPF_W: in bpf_jit_emit_insn()
1844 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1846 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1852 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1854 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1856 case BPF_DW: in bpf_jit_emit_insn()
1858 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1860 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1866 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1868 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1881 case BPF_STX | BPF_MEM | BPF_B: in bpf_jit_emit_insn()
1891 case BPF_STX | BPF_MEM | BPF_H: in bpf_jit_emit_insn()
1901 case BPF_STX | BPF_MEM | BPF_W: in bpf_jit_emit_insn()
1911 case BPF_STX | BPF_MEM | BPF_DW: in bpf_jit_emit_insn()
1921 case BPF_STX | BPF_ATOMIC | BPF_W: in bpf_jit_emit_insn()
1922 case BPF_STX | BPF_ATOMIC | BPF_DW: in bpf_jit_emit_insn()
1927 case BPF_STX | BPF_PROBE_MEM32 | BPF_B: in bpf_jit_emit_insn()
1928 case BPF_STX | BPF_PROBE_MEM32 | BPF_H: in bpf_jit_emit_insn()
1929 case BPF_STX | BPF_PROBE_MEM32 | BPF_W: in bpf_jit_emit_insn()
1930 case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: in bpf_jit_emit_insn()
1938 case BPF_B: in bpf_jit_emit_insn()
1940 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1942 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1948 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1950 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1952 case BPF_H: in bpf_jit_emit_insn()
1954 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1956 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1962 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1964 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1966 case BPF_W: in bpf_jit_emit_insn()
1968 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1970 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1976 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1978 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1980 case BPF_DW: in bpf_jit_emit_insn()
1982 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1984 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1990 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1992 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
2005 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
2006 return -EINVAL; in bpf_jit_emit_insn()
2016 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); in bpf_jit_build_prologue()
2035 if (ctx->arena_vm_start) in bpf_jit_build_prologue()
2041 store_offset = stack_adjust - 8; in bpf_jit_build_prologue()
2050 /* First instruction is always setting the tail-call-counter in bpf_jit_build_prologue()
2052 * Force using a 4-byte (non-compressed) instruction. in bpf_jit_build_prologue()
2056 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
2060 store_offset -= 8; in bpf_jit_build_prologue()
2063 store_offset -= 8; in bpf_jit_build_prologue()
2066 store_offset -= 8; in bpf_jit_build_prologue()
2070 store_offset -= 8; in bpf_jit_build_prologue()
2074 store_offset -= 8; in bpf_jit_build_prologue()
2078 store_offset -= 8; in bpf_jit_build_prologue()
2082 store_offset -= 8; in bpf_jit_build_prologue()
2086 store_offset -= 8; in bpf_jit_build_prologue()
2088 if (ctx->arena_vm_start) { in bpf_jit_build_prologue()
2090 store_offset -= 8; in bpf_jit_build_prologue()
2104 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()
2106 if (ctx->arena_vm_start) in bpf_jit_build_prologue()
2107 emit_imm(RV_REG_ARENA, ctx->arena_vm_start, ctx); in bpf_jit_build_prologue()
2138 case BPF_FUNC_get_smp_processor_id: in bpf_jit_inlines_helper_call()