| H A D | bpf_jit_comp.c | 331 static void push_r9(u8 **pprog) in push_r9() argument
333 u8 *prog = *pprog; in push_r9()
336 *pprog = prog; in push_r9()
339 static void pop_r9(u8 **pprog) in pop_r9() argument
341 u8 *prog = *pprog; in pop_r9()
344 *pprog = prog; in pop_r9()
347 static void push_r12(u8 **pprog) in push_r12() argument
349 u8 *prog = *pprog; in push_r12()
352 *pprog = prog; in push_r12()
355 static void push_callee_regs(u8 **pprog, boo argument
370 pop_r12(u8 ** pprog) pop_r12() argument
378 pop_callee_regs(u8 ** pprog,bool * callee_regs_used) pop_callee_regs() argument
393 emit_nops(u8 ** pprog,int len) emit_nops() argument
418 emit_fineibt(u8 ** pprog,u8 * ip,u32 hash,int arity) emit_fineibt() argument
435 emit_kcfi(u8 ** pprog,u32 hash) emit_kcfi() argument
449 emit_cfi(u8 ** pprog,u8 * ip,u32 hash,int arity) emit_cfi() argument
470 emit_prologue_tail_call(u8 ** pprog,bool is_subprog) emit_prologue_tail_call() argument
506 emit_prologue(u8 ** pprog,u8 * ip,u32 stack_depth,bool ebpf_from_cbpf,bool tail_call_reachable,bool is_subprog,bool is_exception_cb) emit_prologue() argument
559 emit_patch(u8 ** pprog,void * func,void * ip,u8 opcode) emit_patch() argument
574 emit_call(u8 ** pprog,void * func,void * ip) emit_call() argument
579 emit_rsb_call(u8 ** pprog,void * func,void * ip) emit_rsb_call() argument
586 emit_jump(u8 ** pprog,void * func,void * ip) emit_jump() argument
656 __emit_indirect_jump(u8 ** pprog,int reg,bool ereg) __emit_indirect_jump() argument
668 emit_indirect_jump(u8 ** pprog,int bpf_reg,u8 * ip) emit_indirect_jump() argument
695 emit_return(u8 ** pprog,u8 * ip) emit_return() argument
727 emit_bpf_tail_call_indirect(struct bpf_prog * bpf_prog,u8 ** pprog,bool * callee_regs_used,u32 stack_depth,u8 * ip,struct jit_context * ctx) emit_bpf_tail_call_indirect() argument
816 emit_bpf_tail_call_direct(struct bpf_prog * bpf_prog,struct bpf_jit_poke_descriptor * poke,u8 ** pprog,u8 * ip,bool * callee_regs_used,u32 stack_depth,struct jit_context * ctx) emit_bpf_tail_call_direct() argument
909 emit_mov_imm32(u8 ** pprog,bool sign_propagate,u32 dst_reg,const u32 imm32) emit_mov_imm32() argument
949 emit_mov_imm64(u8 ** pprog,u32 dst_reg,const u32 imm32_hi,const u32 imm32_lo) emit_mov_imm64() argument
975 emit_mov_reg(u8 ** pprog,bool is64,u32 dst_reg,u32 src_reg) emit_mov_reg() argument
992 emit_movsx_reg(u8 ** pprog,int num_bits,bool is64,u32 dst_reg,u32 src_reg) emit_movsx_reg() argument
1025 emit_insn_suffix(u8 ** pprog,u32 ptr_reg,u32 val_reg,int off) emit_insn_suffix() argument
1044 emit_insn_suffix_SIB(u8 ** pprog,u32 ptr_reg,u32 val_reg,u32 index_reg,int off) emit_insn_suffix_SIB() argument
1059 maybe_emit_mod(u8 ** pprog,u32 dst_reg,u32 src_reg,bool is64) maybe_emit_mod() argument
1073 maybe_emit_1mod(u8 ** pprog,u32 reg,bool is64) maybe_emit_1mod() argument
1085 emit_ldx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldx() argument
1115 emit_ldsx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldsx() argument
1137 emit_ldx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_ldx_index() argument
1163 emit_ldsx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_ldsx_index() argument
1185 emit_ldx_r12(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_ldx_r12() argument
1196 emit_stx(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_stx() argument
1230 emit_stx_index(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_stx_index() argument
1256 emit_stx_r12(u8 ** pprog,u32 size,u32 dst_reg,u32 src_reg,int off) emit_stx_r12() argument
1262 emit_st_index(u8 ** pprog,u32 size,u32 dst_reg,u32 index_reg,int off,int imm) emit_st_index() argument
1289 emit_st_r12(u8 ** pprog,u32 size,u32 dst_reg,int off,int imm) emit_st_r12() argument
1294 emit_store_stack_imm64(u8 ** pprog,int reg,int stack_off,u64 imm64) emit_store_stack_imm64() argument
1304 emit_atomic_rmw(u8 ** pprog,u32 atomic_op,u32 dst_reg,u32 src_reg,s16 off,u8 bpf_size) emit_atomic_rmw() argument
1346 emit_atomic_rmw_index(u8 ** pprog,u32 atomic_op,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_atomic_rmw_index() argument
1397 emit_atomic_ld_st(u8 ** pprog,u32 atomic_op,u32 dst_reg,u32 src_reg,s16 off,u8 bpf_size) emit_atomic_ld_st() argument
1418 emit_atomic_ld_st_index(u8 ** pprog,u32 atomic_op,u32 size,u32 dst_reg,u32 src_reg,u32 index_reg,int off) emit_atomic_ld_st_index() argument
1532 emit_3vex(u8 ** pprog,bool r,bool x,bool b,u8 m,bool w,u8 src_reg2,bool l,u8 pp) emit_3vex() argument
1569 emit_shiftx(u8 ** pprog,u32 dst_reg,u8 src_reg,bool is64,u8 op) emit_shiftx() argument
1580 emit_priv_frame_ptr(u8 ** pprog,void __percpu * priv_frame_ptr) emit_priv_frame_ptr() argument
1610 emit_spectre_bhb_barrier(u8 ** pprog,u8 * ip,struct bpf_prog * bpf_prog) emit_spectre_bhb_barrier() argument
2797 clean_stack_garbage(const struct btf_func_model * m,u8 ** pprog,int nr_stack_slots,int stack_size) clean_stack_garbage() argument
2971 invoke_bpf_prog(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_link * l,int stack_size,int run_ctx_off,bool save_ret,void * image,void * rw_image) invoke_bpf_prog() argument
3060 emit_align(u8 ** pprog,u32 align) emit_align() argument
3071 emit_cond_near_jump(u8 ** pprog,void * func,void * ip,u8 jmp_cond) emit_cond_near_jump() argument
3086 invoke_bpf(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_links * tl,int stack_size,int run_ctx_off,int func_meta_off,bool save_ret,void * image,void * rw_image,u64 func_meta,int cookie_off) invoke_bpf() argument
3109 invoke_bpf_mod_ret(const struct btf_func_model * m,u8 ** pprog,struct bpf_tramp_links * tl,int stack_size,int run_ctx_off,u8 ** branches,void * image,void * rw_image) invoke_bpf_mod_ret() argument
3587 emit_bpf_dispatcher(u8 ** pprog,int a,int b,s64 * progs,u8 * image,u8 * buf) emit_bpf_dispatcher() argument
[all...] |
| H A D | bpf_jit_comp32.c | 207 u8 **pprog) in emit_ia32_mov_i() argument
209 u8 *prog = *pprog; in emit_ia32_mov_i()
230 *pprog = prog; in emit_ia32_mov_i()
235 bool sstk, u8 **pprog) in emit_ia32_mov_r() argument
237 u8 *prog = *pprog; in emit_ia32_mov_r()
251 *pprog = prog; in emit_ia32_mov_r()
257 bool sstk, u8 **pprog, in emit_ia32_mov_r64() argument
260 emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog); in emit_ia32_mov_r64()
263 emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog); in emit_ia32_mov_r64()
266 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
271 emit_ia32_mov_i64(const bool is64,const u8 dst[],const u32 val,bool dstk,u8 ** pprog) emit_ia32_mov_i64() argument
286 emit_ia32_mul_r(const u8 dst,const u8 src,bool dstk,bool sstk,u8 ** pprog) emit_ia32_mul_r() argument
318 emit_ia32_to_le_r64(const u8 dst[],s32 val,bool dstk,u8 ** pprog,const struct bpf_prog_aux * aux) emit_ia32_to_le_r64() argument
366 emit_ia32_to_be_r64(const u8 dst[],s32 val,bool dstk,u8 ** pprog,const struct bpf_prog_aux * aux) emit_ia32_to_be_r64() argument
436 emit_ia32_div_mod_r(const u8 op,const u8 dst,const u8 src,bool dstk,bool sstk,u8 ** pprog) emit_ia32_div_mod_r() argument
483 emit_ia32_shift_r(const u8 op,const u8 dst,const u8 src,bool dstk,bool sstk,u8 ** pprog) emit_ia32_shift_r() argument
525 emit_ia32_alu_r(const bool is64,const bool hi,const u8 op,const u8 dst,const u8 src,bool dstk,bool sstk,u8 ** pprog) emit_ia32_alu_r() argument
580 emit_ia32_alu_r64(const bool is64,const u8 op,const u8 dst[],const u8 src[],bool dstk,bool sstk,u8 ** pprog,const struct bpf_prog_aux * aux) emit_ia32_alu_r64() argument
599 emit_ia32_alu_i(const bool is64,const bool hi,const u8 op,const u8 dst,const s32 val,bool dstk,u8 ** pprog) emit_ia32_alu_i() argument
679 emit_ia32_alu_i64(const bool is64,const u8 op,const u8 dst[],const u32 val,bool dstk,u8 ** pprog,const struct bpf_prog_aux * aux) emit_ia32_alu_i64() argument
698 emit_ia32_neg64(const u8 dst[],bool dstk,u8 ** pprog) emit_ia32_neg64() argument
732 emit_ia32_lsh_r64(const u8 dst[],const u8 src[],bool dstk,bool sstk,u8 ** pprog) emit_ia32_lsh_r64() argument
785 emit_ia32_arsh_r64(const u8 dst[],const u8 src[],bool dstk,bool sstk,u8 ** pprog) emit_ia32_arsh_r64() argument
838 emit_ia32_rsh_r64(const u8 dst[],const u8 src[],bool dstk,bool sstk,u8 ** pprog) emit_ia32_rsh_r64() argument
891 emit_ia32_lsh_i64(const u8 dst[],const u32 val,bool dstk,u8 ** pprog) emit_ia32_lsh_i64() argument
939 emit_ia32_rsh_i64(const u8 dst[],const u32 val,bool dstk,u8 ** pprog) emit_ia32_rsh_i64() argument
988 emit_ia32_arsh_i64(const u8 dst[],const u32 val,bool dstk,u8 ** pprog) emit_ia32_arsh_i64() argument
1036 emit_ia32_mul_r64(const u8 dst[],const u8 src[],bool dstk,bool sstk,u8 ** pprog) emit_ia32_mul_r64() argument
1113 emit_ia32_mul_i64(const u8 dst[],const u32 val,bool dstk,u8 ** pprog) emit_ia32_mul_i64() argument
1200 emit_prologue(u8 ** pprog,u32 stack_depth) emit_prologue() argument
1245 emit_epilogue(u8 ** pprog,u32 stack_depth) emit_epilogue() argument
1271 emit_jmp_edx(u8 ** pprog,u8 * ip) emit_jmp_edx() argument
1299 emit_bpf_tail_call(u8 ** pprog,u8 * ip) emit_bpf_tail_call() argument
1391 emit_push_r64(const u8 src[],u8 ** pprog) emit_push_r64() argument
1409 emit_push_r32(const u8 src[],u8 ** pprog) emit_push_r32() argument
1570 emit_kfunc_call(const struct bpf_prog * bpf_prog,u8 * end_addr,const struct bpf_insn * insn,u8 ** pprog) emit_kfunc_call() argument
[all...] |