1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2017 Steven Rostedt, VMware Inc. 4 */ 5 6#include <linux/linkage.h> 7#include <asm/page_types.h> 8#include <asm/segment.h> 9#include <asm/export.h> 10#include <asm/ftrace.h> 11#include <asm/nospec-branch.h> 12#include <asm/frame.h> 13#include <asm/asm-offsets.h> 14 15#ifdef CONFIG_FRAME_POINTER 16# define MCOUNT_FRAME 1 /* using frame = true */ 17#else 18# define MCOUNT_FRAME 0 /* using frame = false */ 19#endif 20 21SYM_FUNC_START(__fentry__) 22 RET 23SYM_FUNC_END(__fentry__) 24EXPORT_SYMBOL(__fentry__) 25 26SYM_CODE_START(ftrace_caller) 27 28#ifdef CONFIG_FRAME_POINTER 29 /* 30 * Frame pointers are of ip followed by bp. 31 * Since fentry is an immediate jump, we are left with 32 * parent-ip, function-ip. We need to add a frame with 33 * parent-ip followed by ebp. 34 */ 35 pushl 4(%esp) /* parent ip */ 36 pushl %ebp 37 movl %esp, %ebp 38 pushl 2*4(%esp) /* function ip */ 39 40 /* For mcount, the function ip is directly above */ 41 pushl %ebp 42 movl %esp, %ebp 43#endif 44 pushl %eax 45 pushl %ecx 46 pushl %edx 47 pushl $0 /* Pass NULL as regs pointer */ 48 49#ifdef CONFIG_FRAME_POINTER 50 /* Load parent ebp into edx */ 51 movl 4*4(%esp), %edx 52#else 53 /* There's no frame pointer, load the appropriate stack addr instead */ 54 lea 4*4(%esp), %edx 55#endif 56 57 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */ 58 /* Get the parent ip */ 59 movl 4(%edx), %edx /* edx has ebp */ 60 61 movl function_trace_op, %ecx 62 subl $MCOUNT_INSN_SIZE, %eax 63 64.globl ftrace_call 65ftrace_call: 66 call ftrace_stub 67 68 addl $4, %esp /* skip NULL pointer */ 69 popl %edx 70 popl %ecx 71 popl %eax 72#ifdef CONFIG_FRAME_POINTER 73 popl %ebp 74 addl $4,%esp /* skip function ip */ 75 popl %ebp /* this is the orig bp */ 76 addl $4, %esp /* skip parent ip */ 77#endif 78.Lftrace_ret: 79#ifdef CONFIG_FUNCTION_GRAPH_TRACER 80.globl ftrace_graph_call 81ftrace_graph_call: 82 jmp ftrace_stub 83#endif 84 85/* This is weak to keep gas from relaxing the jumps */ 86SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) 87 RET 88SYM_CODE_END(ftrace_caller) 89 90SYM_CODE_START(ftrace_regs_caller) 91 /* 92 * We're here from an mcount/fentry CALL, and the stack frame looks like: 93 * 94 * <previous context> 95 * RET-IP 96 * 97 * The purpose of this function is to call out in an emulated INT3 98 * environment with a stack frame like: 99 * 100 * <previous context> 101 * gap / RET-IP 102 * gap 103 * gap 104 * gap 105 * pt_regs 106 * 107 * We do _NOT_ restore: ss, flags, cs, gs, fs, es, ds 108 */ 109 subl $3*4, %esp # RET-IP + 3 gaps 110 pushl %ss # ss 111 pushl %esp # points at ss 112 addl $5*4, (%esp) # make it point at <previous context> 113 pushfl # flags 114 pushl $__KERNEL_CS # cs 115 pushl 7*4(%esp) # ip <- RET-IP 116 pushl $0 # orig_eax 117 118 pushl %gs 119 pushl %fs 120 pushl %es 121 pushl %ds 122 123 pushl %eax 124 pushl %ebp 125 pushl %edi 126 pushl %esi 127 pushl %edx 128 pushl %ecx 129 pushl %ebx 130 131 ENCODE_FRAME_POINTER 132 133 movl PT_EIP(%esp), %eax # 1st argument: IP 134 subl $MCOUNT_INSN_SIZE, %eax 135 movl 21*4(%esp), %edx # 2nd argument: parent ip 136 movl function_trace_op, %ecx # 3rd argument: ftrace_pos 137 pushl %esp # 4th argument: pt_regs 138 139SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) 140 call ftrace_stub 141 142 addl $4, %esp # skip 4th argument 143 144 /* place IP below the new SP */ 145 movl PT_OLDESP(%esp), %eax 146 movl PT_EIP(%esp), %ecx 147 movl %ecx, -4(%eax) 148 149 /* place EAX below that */ 150 movl PT_EAX(%esp), %ecx 151 movl %ecx, -8(%eax) 152 153 popl %ebx 154 popl %ecx 155 popl %edx 156 popl %esi 157 popl %edi 158 popl %ebp 159 160 lea -8(%eax), %esp 161 popl %eax 162 163 jmp .Lftrace_ret 164SYM_CODE_END(ftrace_regs_caller) 165 166#ifdef CONFIG_FUNCTION_GRAPH_TRACER 167SYM_CODE_START(ftrace_graph_caller) 168 pushl %eax 169 pushl %ecx 170 pushl %edx 171 movl 3*4(%esp), %eax 172 /* Even with frame pointers, fentry doesn't have one here */ 173 lea 4*4(%esp), %edx 174 movl $0, %ecx 175 subl $MCOUNT_INSN_SIZE, %eax 176 call prepare_ftrace_return 177 popl %edx 178 popl %ecx 179 popl %eax 180 RET 181SYM_CODE_END(ftrace_graph_caller) 182 183.globl return_to_handler 184return_to_handler: 185 pushl %eax 186 pushl %edx 187 movl $0, %eax 188 call ftrace_return_to_handler 189 movl %eax, %ecx 190 popl %edx 191 popl %eax 192 JMP_NOSPEC ecx 193#endif 194