10b57cec5SDimitry Andric//===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===// 20b57cec5SDimitry Andric// 30b57cec5SDimitry Andric// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric// See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric// 70b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric// 90b57cec5SDimitry Andric// This file is a part of XRay, a dynamic runtime instrumentation system. 100b57cec5SDimitry Andric// 110b57cec5SDimitry Andric// This implements the X86-specific assembler for the trampolines. 120b57cec5SDimitry Andric// 130b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 140b57cec5SDimitry Andric 150b57cec5SDimitry Andric#include "../builtins/assembly.h" 160b57cec5SDimitry Andric#include "../sanitizer_common/sanitizer_asm.h" 170b57cec5SDimitry Andric 18fe6060f1SDimitry Andric// XRay trampolines which are not produced by intrinsics are not System V AMD64 19fe6060f1SDimitry Andric// ABI compliant because they are called with a stack that is always misaligned 20fe6060f1SDimitry Andric// by 8 bytes with respect to a 16 bytes alignment. This is because they are 21fe6060f1SDimitry Andric// called immediately after the call to, or immediately before returning from, 22fe6060f1SDimitry Andric// the function being instrumented. This saves space in the patch point, but 23fe6060f1SDimitry Andric// misaligns the stack by 8 bytes. 240b57cec5SDimitry Andric 25fe6060f1SDimitry Andric.macro ALIGN_STACK_16B 26fe6060f1SDimitry Andric#if defined(__APPLE__) 27fe6060f1SDimitry Andric subq $$8, %rsp 28fe6060f1SDimitry Andric#else 29fe6060f1SDimitry Andric subq $8, %rsp 30fe6060f1SDimitry Andric#endif 31fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(8) 32fe6060f1SDimitry Andric.endm 330b57cec5SDimitry Andric 34fe6060f1SDimitry Andric.macro RESTORE_STACK_ALIGNMENT 35fe6060f1SDimitry Andric#if defined(__APPLE__) 36fe6060f1SDimitry Andric addq $$8, %rsp 37fe6060f1SDimitry Andric#else 38fe6060f1SDimitry Andric addq $8, %rsp 39fe6060f1SDimitry Andric#endif 40fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(-8) 41fe6060f1SDimitry Andric.endm 42fe6060f1SDimitry Andric 43*0fca6ea1SDimitry Andric// This macro should lower the stack pointer by an odd multiple of 8. 440b57cec5SDimitry Andric.macro SAVE_REGISTERS 450b57cec5SDimitry Andric pushfq 46fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(8) 470b57cec5SDimitry Andric subq $240, %rsp 48fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(240) 490b57cec5SDimitry Andric movq %rbp, 232(%rsp) 500b57cec5SDimitry Andric movupd %xmm0, 216(%rsp) 510b57cec5SDimitry Andric movupd %xmm1, 200(%rsp) 520b57cec5SDimitry Andric movupd %xmm2, 184(%rsp) 530b57cec5SDimitry Andric movupd %xmm3, 168(%rsp) 540b57cec5SDimitry Andric movupd %xmm4, 152(%rsp) 550b57cec5SDimitry Andric movupd %xmm5, 136(%rsp) 560b57cec5SDimitry Andric movupd %xmm6, 120(%rsp) 570b57cec5SDimitry Andric movupd %xmm7, 104(%rsp) 580b57cec5SDimitry Andric movq %rdi, 96(%rsp) 590b57cec5SDimitry Andric movq %rax, 88(%rsp) 600b57cec5SDimitry Andric movq %rdx, 80(%rsp) 610b57cec5SDimitry Andric movq %rsi, 72(%rsp) 620b57cec5SDimitry Andric movq %rcx, 64(%rsp) 630b57cec5SDimitry Andric movq %r8, 56(%rsp) 640b57cec5SDimitry Andric movq %r9, 48(%rsp) 650b57cec5SDimitry Andric movq %r10, 40(%rsp) 660b57cec5SDimitry Andric movq %r11, 32(%rsp) 670b57cec5SDimitry Andric movq %r12, 24(%rsp) 680b57cec5SDimitry Andric movq %r13, 16(%rsp) 690b57cec5SDimitry Andric movq %r14, 8(%rsp) 700b57cec5SDimitry Andric movq %r15, 0(%rsp) 710b57cec5SDimitry Andric.endm 720b57cec5SDimitry Andric 730b57cec5SDimitry Andric.macro RESTORE_REGISTERS 740b57cec5SDimitry Andric movq 232(%rsp), %rbp 750b57cec5SDimitry Andric movupd 216(%rsp), %xmm0 760b57cec5SDimitry Andric movupd 200(%rsp), %xmm1 770b57cec5SDimitry Andric movupd 184(%rsp), %xmm2 780b57cec5SDimitry Andric movupd 168(%rsp), %xmm3 790b57cec5SDimitry Andric movupd 152(%rsp), %xmm4 800b57cec5SDimitry Andric movupd 136(%rsp), %xmm5 810b57cec5SDimitry Andric movupd 120(%rsp) , %xmm6 820b57cec5SDimitry Andric movupd 104(%rsp) , %xmm7 830b57cec5SDimitry Andric movq 96(%rsp), %rdi 840b57cec5SDimitry Andric movq 88(%rsp), %rax 850b57cec5SDimitry Andric movq 80(%rsp), %rdx 860b57cec5SDimitry Andric movq 72(%rsp), %rsi 870b57cec5SDimitry Andric movq 64(%rsp), %rcx 880b57cec5SDimitry Andric movq 56(%rsp), %r8 890b57cec5SDimitry Andric movq 48(%rsp), %r9 900b57cec5SDimitry Andric movq 40(%rsp), %r10 910b57cec5SDimitry Andric movq 32(%rsp), %r11 920b57cec5SDimitry Andric movq 24(%rsp), %r12 930b57cec5SDimitry Andric movq 16(%rsp), %r13 940b57cec5SDimitry Andric movq 8(%rsp), %r14 950b57cec5SDimitry Andric movq 0(%rsp), %r15 960b57cec5SDimitry Andric addq $240, %rsp 97fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(-240) 980b57cec5SDimitry Andric popfq 99fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(-8) 1000b57cec5SDimitry Andric.endm 1010b57cec5SDimitry Andric 1020b57cec5SDimitry Andric .text 1030b57cec5SDimitry Andric#if !defined(__APPLE__) 1040b57cec5SDimitry Andric .section .text 1050b57cec5SDimitry Andric .file "xray_trampoline_x86.S" 1060b57cec5SDimitry Andric#else 1070b57cec5SDimitry Andric .section __TEXT,__text 1080b57cec5SDimitry Andric#endif 1090b57cec5SDimitry Andric 1100b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1110b57cec5SDimitry Andric 1120b57cec5SDimitry Andric .globl ASM_SYMBOL(__xray_FunctionEntry) 1135ffd83dbSDimitry Andric ASM_HIDDEN(__xray_FunctionEntry) 1140b57cec5SDimitry Andric .align 16, 0x90 1150b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_FunctionEntry) 1160b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_FunctionEntry 1170b57cec5SDimitry AndricASM_SYMBOL(__xray_FunctionEntry): 1180b57cec5SDimitry Andric CFI_STARTPROC 1190b57cec5SDimitry Andric SAVE_REGISTERS 120*0fca6ea1SDimitry Andric ALIGN_STACK_16B 1210b57cec5SDimitry Andric 1220b57cec5SDimitry Andric // This load has to be atomic, it's concurrent with __xray_patch(). 1230b57cec5SDimitry Andric // On x86/amd64, a simple (type-aligned) MOV instruction is enough. 1240b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 1250b57cec5SDimitry Andric testq %rax, %rax 12606c3fb27SDimitry Andric je LOCAL_LABEL(tmp0) 1270b57cec5SDimitry Andric 1280b57cec5SDimitry Andric // The patched function prologue puts its xray_instr_map index into %r10d. 1290b57cec5SDimitry Andric movl %r10d, %edi 1300b57cec5SDimitry Andric xor %esi,%esi 131fe6060f1SDimitry Andric callq *%rax 1320b57cec5SDimitry Andric 13306c3fb27SDimitry AndricLOCAL_LABEL(tmp0): 134fe6060f1SDimitry Andric RESTORE_STACK_ALIGNMENT 135*0fca6ea1SDimitry Andric RESTORE_REGISTERS 1360b57cec5SDimitry Andric retq 1370b57cec5SDimitry Andric# LLVM-MCA-END 1380b57cec5SDimitry Andric ASM_SIZE(__xray_FunctionEntry) 1390b57cec5SDimitry Andric CFI_ENDPROC 1400b57cec5SDimitry Andric 1410b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric .globl ASM_SYMBOL(__xray_FunctionExit) 1445ffd83dbSDimitry Andric ASM_HIDDEN(__xray_FunctionExit) 1450b57cec5SDimitry Andric .align 16, 0x90 1460b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_FunctionExit) 1470b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_FunctionExit 1480b57cec5SDimitry AndricASM_SYMBOL(__xray_FunctionExit): 1490b57cec5SDimitry Andric CFI_STARTPROC 150fe6060f1SDimitry Andric ALIGN_STACK_16B 151fe6060f1SDimitry Andric 1520b57cec5SDimitry Andric // Save the important registers first. Since we're assuming that this 1530b57cec5SDimitry Andric // function is only jumped into, we only preserve the registers for 1540b57cec5SDimitry Andric // returning. 155fe6060f1SDimitry Andric subq $64, %rsp 156fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(64) 1570b57cec5SDimitry Andric movq %rbp, 48(%rsp) 1580b57cec5SDimitry Andric movupd %xmm0, 32(%rsp) 1590b57cec5SDimitry Andric movupd %xmm1, 16(%rsp) 1600b57cec5SDimitry Andric movq %rax, 8(%rsp) 1610b57cec5SDimitry Andric movq %rdx, 0(%rsp) 1620b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 1630b57cec5SDimitry Andric testq %rax,%rax 16406c3fb27SDimitry Andric je LOCAL_LABEL(tmp2) 1650b57cec5SDimitry Andric 1660b57cec5SDimitry Andric movl %r10d, %edi 1670b57cec5SDimitry Andric movl $1, %esi 168fe6060f1SDimitry Andric callq *%rax 1690b57cec5SDimitry Andric 17006c3fb27SDimitry AndricLOCAL_LABEL(tmp2): 1710b57cec5SDimitry Andric // Restore the important registers. 1720b57cec5SDimitry Andric movq 48(%rsp), %rbp 1730b57cec5SDimitry Andric movupd 32(%rsp), %xmm0 1740b57cec5SDimitry Andric movupd 16(%rsp), %xmm1 1750b57cec5SDimitry Andric movq 8(%rsp), %rax 1760b57cec5SDimitry Andric movq 0(%rsp), %rdx 177fe6060f1SDimitry Andric addq $64, %rsp 178fe6060f1SDimitry Andric CFI_ADJUST_CFA_OFFSET(-64) 179fe6060f1SDimitry Andric 180fe6060f1SDimitry Andric RESTORE_STACK_ALIGNMENT 1810b57cec5SDimitry Andric retq 1820b57cec5SDimitry Andric# LLVM-MCA-END 1830b57cec5SDimitry Andric ASM_SIZE(__xray_FunctionExit) 1840b57cec5SDimitry Andric CFI_ENDPROC 1850b57cec5SDimitry Andric 1860b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 1870b57cec5SDimitry Andric 1880b57cec5SDimitry Andric .globl ASM_SYMBOL(__xray_FunctionTailExit) 1895ffd83dbSDimitry Andric ASM_HIDDEN(__xray_FunctionTailExit) 1900b57cec5SDimitry Andric .align 16, 0x90 1910b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_FunctionTailExit) 1920b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_FunctionTailExit 1930b57cec5SDimitry AndricASM_SYMBOL(__xray_FunctionTailExit): 1940b57cec5SDimitry Andric CFI_STARTPROC 1950b57cec5SDimitry Andric SAVE_REGISTERS 196*0fca6ea1SDimitry Andric ALIGN_STACK_16B 1970b57cec5SDimitry Andric 1980b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 1990b57cec5SDimitry Andric testq %rax,%rax 20006c3fb27SDimitry Andric je LOCAL_LABEL(tmp4) 2010b57cec5SDimitry Andric 2020b57cec5SDimitry Andric movl %r10d, %edi 2030b57cec5SDimitry Andric movl $2, %esi 204fe6060f1SDimitry Andric callq *%rax 2050b57cec5SDimitry Andric 20606c3fb27SDimitry AndricLOCAL_LABEL(tmp4): 207fe6060f1SDimitry Andric RESTORE_STACK_ALIGNMENT 208*0fca6ea1SDimitry Andric RESTORE_REGISTERS 2090b57cec5SDimitry Andric retq 2100b57cec5SDimitry Andric# LLVM-MCA-END 2110b57cec5SDimitry Andric ASM_SIZE(__xray_FunctionTailExit) 2120b57cec5SDimitry Andric CFI_ENDPROC 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2150b57cec5SDimitry Andric 2160b57cec5SDimitry Andric .globl ASM_SYMBOL(__xray_ArgLoggerEntry) 2175ffd83dbSDimitry Andric ASM_HIDDEN(__xray_ArgLoggerEntry) 2180b57cec5SDimitry Andric .align 16, 0x90 2190b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry) 2200b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_ArgLoggerEntry 2210b57cec5SDimitry AndricASM_SYMBOL(__xray_ArgLoggerEntry): 2220b57cec5SDimitry Andric CFI_STARTPROC 2230b57cec5SDimitry Andric SAVE_REGISTERS 224*0fca6ea1SDimitry Andric ALIGN_STACK_16B 2250b57cec5SDimitry Andric 2260b57cec5SDimitry Andric // Again, these function pointer loads must be atomic; MOV is fine. 2270b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray13XRayArgLoggerE)(%rip), %rax 2280b57cec5SDimitry Andric testq %rax, %rax 22906c3fb27SDimitry Andric jne LOCAL_LABEL(arg1entryLog) 2300b57cec5SDimitry Andric 2310b57cec5SDimitry Andric // If [arg1 logging handler] not set, defer to no-arg logging. 2320b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax 2330b57cec5SDimitry Andric testq %rax, %rax 23406c3fb27SDimitry Andric je LOCAL_LABEL(arg1entryFail) 2350b57cec5SDimitry Andric 23606c3fb27SDimitry AndricLOCAL_LABEL(arg1entryLog): 2370b57cec5SDimitry Andric 2380b57cec5SDimitry Andric // First argument will become the third 2390b57cec5SDimitry Andric movq %rdi, %rdx 2400b57cec5SDimitry Andric 2410b57cec5SDimitry Andric // XRayEntryType::LOG_ARGS_ENTRY into the second 2420b57cec5SDimitry Andric mov $0x3, %esi 2430b57cec5SDimitry Andric 2440b57cec5SDimitry Andric // 32-bit function ID becomes the first 2450b57cec5SDimitry Andric movl %r10d, %edi 246fe6060f1SDimitry Andric 247fe6060f1SDimitry Andric callq *%rax 2480b57cec5SDimitry Andric 24906c3fb27SDimitry AndricLOCAL_LABEL(arg1entryFail): 250fe6060f1SDimitry Andric RESTORE_STACK_ALIGNMENT 251*0fca6ea1SDimitry Andric RESTORE_REGISTERS 2520b57cec5SDimitry Andric retq 2530b57cec5SDimitry Andric# LLVM-MCA-END 2540b57cec5SDimitry Andric ASM_SIZE(__xray_ArgLoggerEntry) 2550b57cec5SDimitry Andric CFI_ENDPROC 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2580b57cec5SDimitry Andric 259*0fca6ea1SDimitry Andric// __xray_*Event have default visibility so that they can be referenced by user 260*0fca6ea1SDimitry Andric// DSOs that do not link against the runtime. 2610b57cec5SDimitry Andric .global ASM_SYMBOL(__xray_CustomEvent) 2620b57cec5SDimitry Andric .align 16, 0x90 2630b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_CustomEvent) 2640b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_CustomEvent 2650b57cec5SDimitry AndricASM_SYMBOL(__xray_CustomEvent): 2660b57cec5SDimitry Andric CFI_STARTPROC 2670b57cec5SDimitry Andric SAVE_REGISTERS 2680b57cec5SDimitry Andric 2690b57cec5SDimitry Andric // We take two arguments to this trampoline, which should be in rdi and rsi 2700b57cec5SDimitry Andric // already. 2710b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray22XRayPatchedCustomEventE)(%rip), %rax 2720b57cec5SDimitry Andric testq %rax,%rax 27306c3fb27SDimitry Andric je LOCAL_LABEL(customEventCleanup) 2740b57cec5SDimitry Andric 275fe6060f1SDimitry Andric callq *%rax 2760b57cec5SDimitry Andric 27706c3fb27SDimitry AndricLOCAL_LABEL(customEventCleanup): 2780b57cec5SDimitry Andric RESTORE_REGISTERS 2790b57cec5SDimitry Andric retq 2800b57cec5SDimitry Andric# LLVM-MCA-END 2810b57cec5SDimitry Andric ASM_SIZE(__xray_CustomEvent) 2820b57cec5SDimitry Andric CFI_ENDPROC 2830b57cec5SDimitry Andric 2840b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 2850b57cec5SDimitry Andric 2860b57cec5SDimitry Andric .global ASM_SYMBOL(__xray_TypedEvent) 2870b57cec5SDimitry Andric .align 16, 0x90 2880b57cec5SDimitry Andric ASM_TYPE_FUNCTION(__xray_TypedEvent) 2890b57cec5SDimitry Andric# LLVM-MCA-BEGIN __xray_TypedEvent 2900b57cec5SDimitry AndricASM_SYMBOL(__xray_TypedEvent): 2910b57cec5SDimitry Andric CFI_STARTPROC 2920b57cec5SDimitry Andric SAVE_REGISTERS 2930b57cec5SDimitry Andric 2940b57cec5SDimitry Andric // We pass three arguments to this trampoline, which should be in rdi, rsi 2950b57cec5SDimitry Andric // and rdx without our intervention. 2960b57cec5SDimitry Andric movq ASM_SYMBOL(_ZN6__xray21XRayPatchedTypedEventE)(%rip), %rax 2970b57cec5SDimitry Andric testq %rax,%rax 29806c3fb27SDimitry Andric je LOCAL_LABEL(typedEventCleanup) 2990b57cec5SDimitry Andric 300fe6060f1SDimitry Andric callq *%rax 3010b57cec5SDimitry Andric 30206c3fb27SDimitry AndricLOCAL_LABEL(typedEventCleanup): 3030b57cec5SDimitry Andric RESTORE_REGISTERS 3040b57cec5SDimitry Andric retq 3050b57cec5SDimitry Andric# LLVM-MCA-END 3060b57cec5SDimitry Andric ASM_SIZE(__xray_TypedEvent) 3070b57cec5SDimitry Andric CFI_ENDPROC 3080b57cec5SDimitry Andric 3090b57cec5SDimitry Andric//===----------------------------------------------------------------------===// 3100b57cec5SDimitry Andric 3110b57cec5SDimitry AndricNO_EXEC_STACK_DIRECTIVE 312