331 static void push_r9(u8 **pprog)  in push_r9()  argument
333 	u8 *prog = *pprog;  in push_r9()
336 	*pprog = prog;  in push_r9()
339 static void pop_r9(u8 **pprog)  in pop_r9()  argument
341 	u8 *prog = *pprog;  in pop_r9()
344 	*pprog = prog;  in pop_r9()
347 static void push_r12(u8 **pprog)  in push_r12()  argument
349 	u8 *prog = *pprog;  in push_r12()
352 	*pprog = prog;  in push_r12()
355 static void push_callee_regs(u8 **pprog, bool *callee_regs_used)  in push_callee_regs()  argument
357 	u8 *prog = *pprog;  in push_callee_regs()
367 	*pprog = prog;  in push_callee_regs()
370 static void pop_r12(u8 **pprog)  in pop_r12()  argument
372 	u8 *prog = *pprog;  in pop_r12()
375 	*pprog = prog;  in pop_r12()
378 static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)  in pop_callee_regs()  argument
380 	u8 *prog = *pprog;  in pop_callee_regs()
390 	*pprog = prog;  in pop_callee_regs()
393 static void emit_nops(u8 **pprog, int len)  in emit_nops()  argument
395 	u8 *prog = *pprog;  in emit_nops()
409 	*pprog = prog;  in emit_nops()
418 static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity)  in emit_fineibt()  argument
420 	u8 *prog = *pprog;  in emit_fineibt()
432 	*pprog = prog;  in emit_fineibt()
435 static void emit_kcfi(u8 **pprog, u32 hash)  in emit_kcfi()  argument
437 	u8 *prog = *pprog;  in emit_kcfi()
455 	*pprog = prog;  in emit_kcfi()
458 static void emit_cfi(u8 **pprog, u8 *ip, u32 hash, int arity)  in emit_cfi()  argument
460 	u8 *prog = *pprog;  in emit_cfi()
476 	*pprog = prog;  in emit_cfi()
479 static void emit_prologue_tail_call(u8 **pprog, bool is_subprog)  in emit_prologue_tail_call()  argument
481 	u8 *prog = *pprog;  in emit_prologue_tail_call()
507 	*pprog = prog;  in emit_prologue_tail_call()
515 static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_from_cbpf,  in emit_prologue()  argument
519 	u8 *prog = *pprog;  in emit_prologue()
565 	*pprog = prog;  in emit_prologue()
568 static int emit_patch(u8 **pprog, void *func, void *ip, u8 opcode)  in emit_patch()  argument
570 	u8 *prog = *pprog;  in emit_patch()
579 	*pprog = prog;  in emit_patch()
583 static int emit_call(u8 **pprog, void *func, void *ip)  in emit_call()  argument
585 	return emit_patch(pprog, func, ip, 0xE8);  in emit_call()
588 static int emit_rsb_call(u8 **pprog, void *func, void *ip)  in emit_rsb_call()  argument
591 	ip += x86_call_depth_emit_accounting(pprog, func, ip);  in emit_rsb_call()
592 	return emit_patch(pprog, func, ip, 0xE8);  in emit_rsb_call()
595 static int emit_jump(u8 **pprog, void *func, void *ip)  in emit_jump()  argument
597 	return emit_patch(pprog, func, ip, 0xE9);  in emit_jump()
665 static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg)  in __emit_indirect_jump()  argument
667 	u8 *prog = *pprog;  in __emit_indirect_jump()
674 	*pprog = prog;  in __emit_indirect_jump()
677 static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)  in emit_indirect_jump()  argument
679 	u8 *prog = *pprog;  in emit_indirect_jump()
701 	*pprog = prog;  in emit_indirect_jump()
704 static void emit_return(u8 **pprog, u8 *ip)  in emit_return()  argument
706 	u8 *prog = *pprog;  in emit_return()
716 	*pprog = prog;  in emit_return()
736 					u8 **pprog, bool *callee_regs_used,  in emit_bpf_tail_call_indirect()  argument
741 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_indirect()
820 	*pprog = prog;  in emit_bpf_tail_call_indirect()
825 				      u8 **pprog, u8 *ip,  in emit_bpf_tail_call_direct()  argument
830 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_direct()
877 	*pprog = prog;  in emit_bpf_tail_call_direct()
918 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,  in emit_mov_imm32()  argument
921 	u8 *prog = *pprog;  in emit_mov_imm32()
955 	*pprog = prog;  in emit_mov_imm32()
958 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,  in emit_mov_imm64()  argument
962 	u8 *prog = *pprog;  in emit_mov_imm64()
981 	*pprog = prog;  in emit_mov_imm64()
984 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
986 	u8 *prog = *pprog;  in emit_mov_reg()
998 	*pprog = prog;  in emit_mov_reg()
1001 static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg,  in emit_movsx_reg()  argument
1004 	u8 *prog = *pprog;  in emit_movsx_reg()
1030 	*pprog = prog;  in emit_movsx_reg()
1034 static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)  in emit_insn_suffix()  argument
1036 	u8 *prog = *pprog;  in emit_insn_suffix()
1050 	*pprog = prog;  in emit_insn_suffix()
1053 static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off)  in emit_insn_suffix_SIB()  argument
1055 	u8 *prog = *pprog;  in emit_insn_suffix_SIB()
1062 	*pprog = prog;  in emit_insn_suffix_SIB()
1068 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
1070 	u8 *prog = *pprog;  in maybe_emit_mod()
1076 	*pprog = prog;  in maybe_emit_mod()
1082 static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)  in maybe_emit_1mod()  argument
1084 	u8 *prog = *pprog;  in maybe_emit_1mod()
1090 	*pprog = prog;  in maybe_emit_1mod()
1094 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
1096 	u8 *prog = *pprog;  in emit_ldx()
1120 	*pprog = prog;  in emit_ldx()
1124 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
1126 	u8 *prog = *pprog;  in emit_ldsx()
1143 	*pprog = prog;  in emit_ldsx()
1146 static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)  in emit_ldx_index()  argument
1148 	u8 *prog = *pprog;  in emit_ldx_index()
1169 	*pprog = prog;  in emit_ldx_index()
1172 static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)  in emit_ldsx_index()  argument
1174 	u8 *prog = *pprog;  in emit_ldsx_index()
1191 	*pprog = prog;  in emit_ldsx_index()
1194 static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx_r12()  argument
1196 	emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);  in emit_ldx_r12()
1205 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
1207 	u8 *prog = *pprog;  in emit_stx()
1235 	*pprog = prog;  in emit_stx()
1239 static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)  in emit_stx_index()  argument
1241 	u8 *prog = *pprog;  in emit_stx_index()
1262 	*pprog = prog;  in emit_stx_index()
1265 static void emit_stx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx_r12()  argument
1267 	emit_stx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);  in emit_stx_r12()
1271 static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int off, int imm)  in emit_st_index()  argument
1273 	u8 *prog = *pprog;  in emit_st_index()
1295 	*pprog = prog;  in emit_st_index()
1298 static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)  in emit_st_r12()  argument
1300 	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);  in emit_st_r12()
1303 static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,  in emit_atomic_rmw()  argument
1306 	u8 *prog = *pprog;  in emit_atomic_rmw()
1340 	*pprog = prog;  in emit_atomic_rmw()
1344 static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,  in emit_atomic_rmw_index()  argument
1348 	u8 *prog = *pprog;  in emit_atomic_rmw_index()
1389 	*pprog = prog;  in emit_atomic_rmw_index()
1393 static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,  in emit_atomic_ld_st()  argument
1399 		emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);  in emit_atomic_ld_st()
1403 		emit_stx(pprog, bpf_size, dst_reg, src_reg, off);  in emit_atomic_ld_st()
1414 static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,  in emit_atomic_ld_st_index()  argument
1421 		emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);  in emit_atomic_ld_st_index()
1425 		emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);  in emit_atomic_ld_st_index()
1528 static void emit_3vex(u8 **pprog, bool r, bool x, bool b, u8 m,  in emit_3vex()  argument
1531 	u8 *prog = *pprog;  in emit_3vex()
1561 	*pprog = prog;  in emit_3vex()
1565 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1567 	u8 *prog = *pprog;  in emit_shiftx()
1573 	*pprog = prog;  in emit_shiftx()
1576 static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr)  in emit_priv_frame_ptr()  argument
1578 	u8 *prog = *pprog;  in emit_priv_frame_ptr()
1591 	*pprog = prog;  in emit_priv_frame_ptr()
1606 static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,  in emit_spectre_bhb_barrier()  argument
1609 	u8 *prog = *pprog;  in emit_spectre_bhb_barrier()
1644 	*pprog = prog;  in emit_spectre_bhb_barrier()
2793 				u8 **pprog, int nr_stack_slots,  in clean_stack_garbage()  argument
2829 		prog = *pprog;  in clean_stack_garbage()
2836 		*pprog = prog;  in clean_stack_garbage()
2967 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_prog()  argument
2972 	u8 *prog = *pprog;  in invoke_bpf_prog()
3052 	*pprog = prog;  in invoke_bpf_prog()
3056 static void emit_align(u8 **pprog, u32 align)  in emit_align()  argument
3058 	u8 *target, *prog = *pprog;  in emit_align()
3064 	*pprog = prog;  in emit_align()
3067 static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)  in emit_cond_near_jump()  argument
3069 	u8 *prog = *pprog;  in emit_cond_near_jump()
3078 	*pprog = prog;  in emit_cond_near_jump()
3082 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf()  argument
3088 	u8 *prog = *pprog;  in invoke_bpf()
3095 	*pprog = prog;  in invoke_bpf()
3099 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_mod_ret()  argument
3104 	u8 *prog = *pprog;  in invoke_bpf_mod_ret()
3133 	*pprog = prog;  in invoke_bpf_mod_ret()
3563 static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)  in emit_bpf_dispatcher()  argument
3565 	u8 *jg_reloc, *prog = *pprog;  in emit_bpf_dispatcher()
3586 		*pprog = prog;  in emit_bpf_dispatcher()
3627 	*pprog = prog;  in emit_bpf_dispatcher()