1//===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of XRay, a dynamic runtime instrumentation system. 10// 11// This implements the X86-specific assembler for the trampolines. 12// 13//===----------------------------------------------------------------------===// 14 15#include "../builtins/assembly.h" 16#include "../sanitizer_common/sanitizer_asm.h" 17 18// XRay trampolines which are not produced by intrinsics are not System V AMD64 19// ABI compliant because they are called with a stack that is always misaligned 20// by 8 bytes with respect to a 16 bytes alignment. This is because they are 21// called immediately after the call to, or immediately before returning from, 22// the function being instrumented. This saves space in the patch point, but 23// misaligns the stack by 8 bytes. 24 25.macro ALIGN_STACK_16B 26#if defined(__APPLE__) 27 subq $$8, %rsp 28#else 29 subq $8, %rsp 30#endif 31 CFI_ADJUST_CFA_OFFSET(8) 32.endm 33 34.macro RESTORE_STACK_ALIGNMENT 35#if defined(__APPLE__) 36 addq $$8, %rsp 37#else 38 addq $8, %rsp 39#endif 40 CFI_ADJUST_CFA_OFFSET(-8) 41.endm 42 43// This macro should lower the stack pointer by an odd multiple of 8. 44.macro SAVE_REGISTERS 45 pushfq 46 CFI_ADJUST_CFA_OFFSET(8) 47 subq $240, %rsp 48 CFI_ADJUST_CFA_OFFSET(240) 49 movq %rbp, 232(%rsp) 50 movupd %xmm0, 216(%rsp) 51 movupd %xmm1, 200(%rsp) 52 movupd %xmm2, 184(%rsp) 53 movupd %xmm3, 168(%rsp) 54 movupd %xmm4, 152(%rsp) 55 movupd %xmm5, 136(%rsp) 56 movupd %xmm6, 120(%rsp) 57 movupd %xmm7, 104(%rsp) 58 movq %rdi, 96(%rsp) 59 movq %rax, 88(%rsp) 60 movq %rdx, 80(%rsp) 61 movq %rsi, 72(%rsp) 62 movq %rcx, 64(%rsp) 63 movq %r8, 56(%rsp) 64 movq %r9, 48(%rsp) 65 movq %r10, 40(%rsp) 66 movq %r11, 32(%rsp) 67 movq %r12, 24(%rsp) 68 movq %r13, 16(%rsp) 69 movq %r14, 8(%rsp) 70 movq %r15, 0(%rsp) 71.endm 72 73.macro RESTORE_REGISTERS 74 movq 232(%rsp), %rbp 75 movupd 216(%rsp), %xmm0 76 movupd 200(%rsp), %xmm1 77 movupd 184(%rsp), %xmm2 78 movupd 168(%rsp), %xmm3 79 movupd 152(%rsp), %xmm4 80 movupd 136(%rsp), %xmm5 81 movupd 120(%rsp) , %xmm6 82 movupd 104(%rsp) , %xmm7 83 movq 96(%rsp), %rdi 84 movq 88(%rsp), %rax 85 movq 80(%rsp), %rdx 86 movq 72(%rsp), %rsi 87 movq 64(%rsp), %rcx 88 movq 56(%rsp), %r8 89 movq 48(%rsp), %r9 90 movq 40(%rsp), %r10 91 movq 32(%rsp), %r11 92 movq 24(%rsp), %r12 93 movq 16(%rsp), %r13 94 movq 8(%rsp), %r14 95 movq 0(%rsp), %r15 96 addq $240, %rsp 97 CFI_ADJUST_CFA_OFFSET(-240) 98 popfq 99 CFI_ADJUST_CFA_OFFSET(-8) 100.endm 101 102 .text 103#if !defined(__APPLE__) 104 .section .text 105 .file "xray_trampoline_x86.S" 106#else 107 .section __TEXT,__text 108#endif 109 110//===----------------------------------------------------------------------===// 111 112 .globl ASM_SYMBOL(__xray_FunctionEntry) 113 ASM_HIDDEN(__xray_FunctionEntry) 114 .align 16, 0x90 115 ASM_TYPE_FUNCTION(__xray_FunctionEntry) 116# LLVM-MCA-BEGIN __xray_FunctionEntry 117ASM_SYMBOL(__xray_FunctionEntry): 118 CFI_STARTPROC 119 SAVE_REGISTERS 120 ALIGN_STACK_16B 121 122 // This load has to be atomic, it's concurrent with __xray_patch(). 123 // On x86/amd64, a simple (type-aligned) MOV instruction is enough. 124 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 125 testq %rax, %rax 126 je LOCAL_LABEL(tmp0) 127 128 // The patched function prologue puts its xray_instr_map index into %r10d. 129 movl %r10d, %edi 130 xor %esi,%esi 131 callq *%rax 132 133LOCAL_LABEL(tmp0): 134 RESTORE_STACK_ALIGNMENT 135 RESTORE_REGISTERS 136 retq 137# LLVM-MCA-END 138 ASM_SIZE(__xray_FunctionEntry) 139 CFI_ENDPROC 140 141//===----------------------------------------------------------------------===// 142 143 .globl ASM_SYMBOL(__xray_FunctionExit) 144 ASM_HIDDEN(__xray_FunctionExit) 145 .align 16, 0x90 146 ASM_TYPE_FUNCTION(__xray_FunctionExit) 147# LLVM-MCA-BEGIN __xray_FunctionExit 148ASM_SYMBOL(__xray_FunctionExit): 149 CFI_STARTPROC 150 ALIGN_STACK_16B 151 152 // Save the important registers first. Since we're assuming that this 153 // function is only jumped into, we only preserve the registers for 154 // returning. 155 subq $64, %rsp 156 CFI_ADJUST_CFA_OFFSET(64) 157 movq %rbp, 48(%rsp) 158 movupd %xmm0, 32(%rsp) 159 movupd %xmm1, 16(%rsp) 160 movq %rax, 8(%rsp) 161 movq %rdx, 0(%rsp) 162 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 163 testq %rax,%rax 164 je LOCAL_LABEL(tmp2) 165 166 movl %r10d, %edi 167 movl $1, %esi 168 callq *%rax 169 170LOCAL_LABEL(tmp2): 171 // Restore the important registers. 172 movq 48(%rsp), %rbp 173 movupd 32(%rsp), %xmm0 174 movupd 16(%rsp), %xmm1 175 movq 8(%rsp), %rax 176 movq 0(%rsp), %rdx 177 addq $64, %rsp 178 CFI_ADJUST_CFA_OFFSET(-64) 179 180 RESTORE_STACK_ALIGNMENT 181 retq 182# LLVM-MCA-END 183 ASM_SIZE(__xray_FunctionExit) 184 CFI_ENDPROC 185 186//===----------------------------------------------------------------------===// 187 188 .globl ASM_SYMBOL(__xray_FunctionTailExit) 189 ASM_HIDDEN(__xray_FunctionTailExit) 190 .align 16, 0x90 191 ASM_TYPE_FUNCTION(__xray_FunctionTailExit) 192# LLVM-MCA-BEGIN __xray_FunctionTailExit 193ASM_SYMBOL(__xray_FunctionTailExit): 194 CFI_STARTPROC 195 SAVE_REGISTERS 196 ALIGN_STACK_16B 197 198 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 199 testq %rax,%rax 200 je LOCAL_LABEL(tmp4) 201 202 movl %r10d, %edi 203 movl $2, %esi 204 callq *%rax 205 206LOCAL_LABEL(tmp4): 207 RESTORE_STACK_ALIGNMENT 208 RESTORE_REGISTERS 209 retq 210# LLVM-MCA-END 211 ASM_SIZE(__xray_FunctionTailExit) 212 CFI_ENDPROC 213 214//===----------------------------------------------------------------------===// 215 216 .globl ASM_SYMBOL(__xray_ArgLoggerEntry) 217 ASM_HIDDEN(__xray_ArgLoggerEntry) 218 .align 16, 0x90 219 ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry) 220# LLVM-MCA-BEGIN __xray_ArgLoggerEntry 221ASM_SYMBOL(__xray_ArgLoggerEntry): 222 CFI_STARTPROC 223 SAVE_REGISTERS 224 ALIGN_STACK_16B 225 226 // Again, these function pointer loads must be atomic; MOV is fine. 227 movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax 228 testq %rax, %rax 229 jne LOCAL_LABEL(arg1entryLog) 230 231 // If [arg1 logging handler] not set, defer to no-arg logging. 232 movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 233 testq %rax, %rax 234 je LOCAL_LABEL(arg1entryFail) 235 236LOCAL_LABEL(arg1entryLog): 237 238 // First argument will become the third 239 movq %rdi, %rdx 240 241 // XRayEntryType::LOG_ARGS_ENTRY into the second 242 mov $0x3, %esi 243 244 // 32-bit function ID becomes the first 245 movl %r10d, %edi 246 247 callq *%rax 248 249LOCAL_LABEL(arg1entryFail): 250 RESTORE_STACK_ALIGNMENT 251 RESTORE_REGISTERS 252 retq 253# LLVM-MCA-END 254 ASM_SIZE(__xray_ArgLoggerEntry) 255 CFI_ENDPROC 256 257//===----------------------------------------------------------------------===// 258 259// __xray_*Event have default visibility so that they can be referenced by user 260// DSOs that do not link against the runtime. 261 .global ASM_SYMBOL(__xray_CustomEvent) 262 .align 16, 0x90 263 ASM_TYPE_FUNCTION(__xray_CustomEvent) 264# LLVM-MCA-BEGIN __xray_CustomEvent 265ASM_SYMBOL(__xray_CustomEvent): 266 CFI_STARTPROC 267 SAVE_REGISTERS 268 269 // We take two arguments to this trampoline, which should be in rdi and rsi 270 // already. 271 movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax 272 testq %rax,%rax 273 je LOCAL_LABEL(customEventCleanup) 274 275 callq *%rax 276 277LOCAL_LABEL(customEventCleanup): 278 RESTORE_REGISTERS 279 retq 280# LLVM-MCA-END 281 ASM_SIZE(__xray_CustomEvent) 282 CFI_ENDPROC 283 284//===----------------------------------------------------------------------===// 285 286 .global ASM_SYMBOL(__xray_TypedEvent) 287 .align 16, 0x90 288 ASM_TYPE_FUNCTION(__xray_TypedEvent) 289# LLVM-MCA-BEGIN __xray_TypedEvent 290ASM_SYMBOL(__xray_TypedEvent): 291 CFI_STARTPROC 292 SAVE_REGISTERS 293 294 // We pass three arguments to this trampoline, which should be in rdi, rsi 295 // and rdx without our intervention. 296 movq ASM_SYMBOL(_ZN6__xray21XRayPatchedTypedEventE)(%rip), %rax 297 testq %rax,%rax 298 je LOCAL_LABEL(typedEventCleanup) 299 300 callq *%rax 301 302LOCAL_LABEL(typedEventCleanup): 303 RESTORE_REGISTERS 304 retq 305# LLVM-MCA-END 306 ASM_SIZE(__xray_TypedEvent) 307 CFI_ENDPROC 308 309//===----------------------------------------------------------------------===// 310 311NO_EXEC_STACK_DIRECTIVE 312