1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * x86 implementation of rethook. Mostly copied from arch/x86/kernel/kprobes/core.c. 4 */ 5 #include <linux/bug.h> 6 #include <linux/rethook.h> 7 #include <linux/kprobes.h> 8 9 #include "kprobes/common.h" 10 11 __visible void arch_rethook_trampoline_callback(struct pt_regs *regs); 12 13 /* 14 * When a target function returns, this code saves registers and calls 15 * arch_rethook_trampoline_callback(), which calls the rethook handler. 16 */ 17 asm( 18 ".text\n" 19 ".global arch_rethook_trampoline\n" 20 ".type arch_rethook_trampoline, @function\n" 21 "arch_rethook_trampoline:\n" 22 #ifdef CONFIG_X86_64 23 /* Push a fake return address to tell the unwinder it's a kretprobe. */ 24 " pushq $arch_rethook_trampoline\n" 25 UNWIND_HINT_FUNC 26 /* Save the 'sp - 8', this will be fixed later. */ 27 " pushq %rsp\n" 28 " pushfq\n" 29 SAVE_REGS_STRING 30 " movq %rsp, %rdi\n" 31 " call arch_rethook_trampoline_callback\n" 32 RESTORE_REGS_STRING 33 /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ 34 " addq $8, %rsp\n" 35 " popfq\n" 36 #else 37 /* Push a fake return address to tell the unwinder it's a kretprobe. */ 38 " pushl $arch_rethook_trampoline\n" 39 UNWIND_HINT_FUNC 40 /* Save the 'sp - 4', this will be fixed later. */ 41 " pushl %esp\n" 42 " pushfl\n" 43 SAVE_REGS_STRING 44 " movl %esp, %eax\n" 45 " call arch_rethook_trampoline_callback\n" 46 RESTORE_REGS_STRING 47 /* In the callback function, 'regs->flags' is copied to 'regs->sp'. */ 48 " addl $4, %esp\n" 49 " popfl\n" 50 #endif 51 " ret\n" 52 ".size arch_rethook_trampoline, .-arch_rethook_trampoline\n" 53 ); 54 NOKPROBE_SYMBOL(arch_rethook_trampoline); 55 56 /* 57 * Called from arch_rethook_trampoline 58 */ 59 __used __visible void arch_rethook_trampoline_callback(struct pt_regs *regs) 60 { 61 unsigned long *frame_pointer; 62 63 /* fixup registers */ 64 regs->cs = __KERNEL_CS; 65 #ifdef CONFIG_X86_32 66 regs->gs = 0; 67 #endif 68 regs->ip = (unsigned long)&arch_rethook_trampoline; 69 regs->orig_ax = ~0UL; 70 regs->sp += sizeof(long); 71 frame_pointer = ®s->sp + 1; 72 73 /* 74 * The return address at 'frame_pointer' is recovered by the 75 * arch_rethook_fixup_return() which called from this 76 * rethook_trampoline_handler(). 77 */ 78 rethook_trampoline_handler(regs, (unsigned long)frame_pointer); 79 80 /* 81 * Copy FLAGS to 'pt_regs::sp' so that arch_rethook_trapmoline() 82 * can do RET right after POPF. 83 */ 84 regs->sp = regs->flags; 85 } 86 NOKPROBE_SYMBOL(arch_rethook_trampoline_callback); 87 88 /* 89 * arch_rethook_trampoline() skips updating frame pointer. The frame pointer 90 * saved in arch_rethook_trampoline_callback() points to the real caller 91 * function's frame pointer. Thus the arch_rethook_trampoline() doesn't have 92 * a standard stack frame with CONFIG_FRAME_POINTER=y. 93 * Let's mark it non-standard function. Anyway, FP unwinder can correctly 94 * unwind without the hint. 95 */ 96 STACK_FRAME_NON_STANDARD_FP(arch_rethook_trampoline); 97 98 /* This is called from rethook_trampoline_handler(). */ 99 void arch_rethook_fixup_return(struct pt_regs *regs, 100 unsigned long correct_ret_addr) 101 { 102 unsigned long *frame_pointer = ®s->sp + 1; 103 104 /* Replace fake return address with real one. */ 105 *frame_pointer = correct_ret_addr; 106 } 107 NOKPROBE_SYMBOL(arch_rethook_fixup_return); 108 109 void arch_rethook_prepare(struct rethook_node *rh, struct pt_regs *regs, bool mcount) 110 { 111 unsigned long *stack = (unsigned long *)regs->sp; 112 113 rh->ret_addr = stack[0]; 114 rh->frame = regs->sp; 115 116 /* Replace the return addr with trampoline addr */ 117 stack[0] = (unsigned long) arch_rethook_trampoline; 118 } 119 NOKPROBE_SYMBOL(arch_rethook_prepare); 120