| H A D | bpf_jit_comp.c | 333 static void push_r9(u8 **pprog) in push_r9() argument
341 static void pop_r9(u8 **pprog) in pop_r9() argument
349 static void push_r12(u8 **pprog) in push_r12() argument
357 static void push_callee_regs(u8 **pprog, boo argument
372 pop_r12(u8 ** pprog) pop_r12() argument
380 pop_callee_regs(u8 ** pprog,bool * callee_regs_used) pop_callee_regs() argument
396 emit_add_rsp(u8 ** pprog,u16 depth) emit_add_rsp() argument
410 emit_sub_rsp(u8 ** pprog,u16 depth) emit_sub_rsp() argument
423 emit_nops(u8 ** pprog,int len) emit_nops() argument
448 emit_fineibt(u8 ** pprog,u8 * ip,u32 hash,int arity) emit_fineibt() argument
465 emit_kcfi(u8 ** pprog,u32 hash) emit_kcfi() argument
479 emit_cfi(u8 ** pprog,u8 * ip,u32 hash,int arity) emit_cfi() argument
500 emit_prologue_tail_call(u8 ** pprog,bool is_subprog) emit_prologue_tail_call() argument
536 emit_prologue(u8 ** pprog,u8 * ip,u32 stack_depth,bool ebpf_from_cbpf,bool tail_call_reachable,bool is_subprog,bool is_exception_cb) emit_prologue() argument
589 emit_patch(u8 ** pprog,void * func,void * ip,u8 opcode) emit_patch() argument
604 emit_call(u8 ** pprog,void * func,void * ip) emit_call() argument
609 emit_rsb_call(u8 ** pprog,void * func,void * ip) emit_rsb_call() argument
616 emit_jump(u8 ** pprog,void * func,void * ip) emit_jump() argument
686 __emit_indirect_jump(u8 ** pprog,int reg,bool ereg) __emit_indirect_jump() argument
698 emit_indirect_jump(u8 ** pprog,int bpf_reg,u8 * ip) emit_indirect_jump() argument
725 emit_return(u8 ** pprog,u8 * ip) emit_return() argument
757 emit_bpf_tail_call_indirect(struct bpf_prog * bpf_prog,u8 ** pprog,bool * callee_regs_used,u32 stack_depth,u8 * ip,struct jit_context * ctx) emit_bpf_tail_call_indirect() argument
846 emit_bpf_tail_call_direct(struct bpf_prog * bpf_prog,struct bpf_jit_poke_descriptor * poke,u8 ** pprog,u8 * ip,bool * callee_regs_used,u32 stack_depth,struct jit_context * ctx) emit_bpf_tail_call_direct() argument
939 emit_mov_imm32(u8 ** pprog,bool sign_propagate,u32 dst_reg,const u32 imm32) emit_mov_imm32() argument
979 emit_mov_imm64(u8 ** pprog,u32 dst_reg,const u32 imm32_hi,const u32 imm32_lo) emit_mov_imm64() argument
1005 emit_mov_reg(u8 ** pprog,bool is64,u32 dst_reg,u32 src_reg) emit_mov_reg() argument
1022 emit_movsx_reg(u8 ** pprog,int num_bits,bool is64,u32 dst_reg,u32 src_reg) emit_movsx_reg() argument
1055 emit_insn_suffix(u8 ** pprog,u32 ptr_reg,u32 val_reg,int off) emit_insn_suffix() argument
1074 emit_insn_suffix_SIB(u8 ** pprog,u32 ptr_reg,u32 val_reg,u32 index_reg,int off) emit_insn_suffix_SIB() argument
1089 maybe_emit_mod(u8 ** pprog,u32 dst_reg,u32 src_reg,bool is64) maybe_emit_mod() argument
1103 maybe_emit_1mod(u8 ** pprog,u32 reg,bool is64) maybe_emit_1mod() argument
1115 emit_ldx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldx() argument
1145 emit_ldsx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldsx() argument
1167 emit_ldx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_ldx_index() argument
1193 emit_ldsx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_ldsx_index() argument
1215 emit_ldx_r12(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldx_r12() argument
1226 emit_stx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_stx() argument
1260 emit_stx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_stx_index() argument
1286 emit_stx_r12(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_stx_r12() argument
1292 emit_st_index(u8 ** pprog,u32 size,u32 dst_reg,u32 index_reg,int off,int imm) emit_st_index() argument
1319 emit_st_r12(u8 ** pprog,u32 size,u32 dst_reg,int off,int imm) emit_st_r12() argument
1324 emit_store_stack_imm64(u8 ** pprog,int reg,int stack_off,u64 imm64) emit_store_stack_imm64() argument
1334 emit_atomic_rmw(u8 ** pprog,u32 atomic_op,u32 dst_reg,u32 src_reg,s16 off,u8 bpf_size) emit_atomic_rmw() argument
1376 emit_atomic_rmw_index(u8 ** pprog,u32 atomic_op,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_atomic_rmw_index() argument
1427 emit_atomic_ld_st(u8 ** pprog,u32 atomic_op,u32 dst_reg,u32 src_reg,s16 off,u8 bpf_size) emit_atomic_ld_st() argument
1448 emit_atomic_ld_st_index(u8 ** pprog,u32 atomic_op,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_atomic_ld_st_index() argument
1566 emit_3vex(u8 ** pprog,bool r,bool x,bool b,u8 m,bool w,u8 src_reg2,bool l,u8 pp) emit_3vex() argument
1603 emit_shiftx(u8 ** pprog,u32 dst_reg,u8 src_reg,bool is64,u8 op) emit_shiftx() argument
1614 emit_priv_frame_ptr(u8 ** pprog,void __percpu * priv_frame_ptr) emit_priv_frame_ptr() argument
1644 emit_spectre_bhb_barrier(u8 ** pprog,u8 * ip,struct bpf_prog * bpf_prog) emit_spectre_bhb_barrier() argument
1693 emit_kfunc_arena_args(struct bpf_prog * bpf_prog,const struct bpf_insn * insn,u8 ** pprog) emit_kfunc_arena_args() argument
3007 clean_stack_garbage(const struct btf_func_model * m,u8 ** pprog,int nr_stack_slots,int stack_size) clean_stack_garbage() argument
3078 emit_arena_arg_conv(u8 ** pprog,u32 src_reg,bool nullable,u32 base_lo) emit_arena_arg_conv() argument
3219 invoke_bpf_prog(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_node * node,int stack_size,int run_ctx_off,bool save_ret,void * image,void * rw_image) invoke_bpf_prog() argument
3308 emit_align(u8 ** pprog,u32 align) emit_align() argument
3319 emit_cond_near_jump(u8 ** pprog,void * func,void * ip,u8 jmp_cond) emit_cond_near_jump() argument
3334 invoke_bpf(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_nodes * tl,int stack_size,int run_ctx_off,int func_meta_off,bool save_ret,void * image,void * rw_image,u64 func_meta,int cookie_off) invoke_bpf() argument
3357 invoke_bpf_mod_ret(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_nodes * tl,int stack_size,int run_ctx_off,u8 ** branches,void * image,void * rw_image) invoke_bpf_mod_ret() argument
3835 emit_bpf_dispatcher(u8 ** pprog,int a,int b,s64 * progs,u8 * image,u8 * buf) emit_bpf_dispatcher() argument
[all...] |