xref: /freebsd/sys/cddl/dev/dtrace/amd64/dtrace_asm.S (revision 8ca79fbd4af2570968a92f08fb714d8c1bcb378d)
191eaf3e1SJohn Birrell/*
291eaf3e1SJohn Birrell * CDDL HEADER START
391eaf3e1SJohn Birrell *
491eaf3e1SJohn Birrell * The contents of this file are subject to the terms of the
591eaf3e1SJohn Birrell * Common Development and Distribution License (the "License").
691eaf3e1SJohn Birrell * You may not use this file except in compliance with the License.
791eaf3e1SJohn Birrell *
891eaf3e1SJohn Birrell * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
991eaf3e1SJohn Birrell * or http://www.opensolaris.org/os/licensing.
1091eaf3e1SJohn Birrell * See the License for the specific language governing permissions
1191eaf3e1SJohn Birrell * and limitations under the License.
1291eaf3e1SJohn Birrell *
1391eaf3e1SJohn Birrell * When distributing Covered Code, include this CDDL HEADER in each
1491eaf3e1SJohn Birrell * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1591eaf3e1SJohn Birrell * If applicable, add the following below this CDDL HEADER, with the
1691eaf3e1SJohn Birrell * fields enclosed by brackets "[]" replaced with your own identifying
1791eaf3e1SJohn Birrell * information: Portions Copyright [yyyy] [name of copyright owner]
1891eaf3e1SJohn Birrell *
1991eaf3e1SJohn Birrell * CDDL HEADER END
2091eaf3e1SJohn Birrell *
2191eaf3e1SJohn Birrell * Portions Copyright 2008 John Birrell <jb@freebsd.org>
2291eaf3e1SJohn Birrell *
2391eaf3e1SJohn Birrell * $FreeBSD$
2491eaf3e1SJohn Birrell *
2591eaf3e1SJohn Birrell */
2691eaf3e1SJohn Birrell/*
2791eaf3e1SJohn Birrell * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
2891eaf3e1SJohn Birrell * Use is subject to license terms.
2991eaf3e1SJohn Birrell */
3091eaf3e1SJohn Birrell
3191eaf3e1SJohn Birrell#define _ASM
3291eaf3e1SJohn Birrell
3391eaf3e1SJohn Birrell#include <machine/asmacros.h>
3491eaf3e1SJohn Birrell#include <sys/cpuvar_defs.h>
3591eaf3e1SJohn Birrell#include <sys/dtrace.h>
3691eaf3e1SJohn Birrell
37fc2a8776SEd Maste#include "assym.inc"
3891eaf3e1SJohn Birrell
3991eaf3e1SJohn Birrell#define INTR_POP				\
4091eaf3e1SJohn Birrell	MEXITCOUNT;				\
4191eaf3e1SJohn Birrell	movq	TF_RDI(%rsp),%rdi;		\
4291eaf3e1SJohn Birrell	movq	TF_RSI(%rsp),%rsi;		\
4391eaf3e1SJohn Birrell	movq	TF_RDX(%rsp),%rdx;		\
4491eaf3e1SJohn Birrell	movq	TF_RCX(%rsp),%rcx;		\
4591eaf3e1SJohn Birrell	movq	TF_R8(%rsp),%r8;		\
4691eaf3e1SJohn Birrell	movq	TF_R9(%rsp),%r9;		\
4791eaf3e1SJohn Birrell	movq	TF_RAX(%rsp),%rax;		\
4891eaf3e1SJohn Birrell	movq	TF_RBX(%rsp),%rbx;		\
4991eaf3e1SJohn Birrell	movq	TF_RBP(%rsp),%rbp;		\
5091eaf3e1SJohn Birrell	movq	TF_R10(%rsp),%r10;		\
5191eaf3e1SJohn Birrell	movq	TF_R11(%rsp),%r11;		\
5291eaf3e1SJohn Birrell	movq	TF_R12(%rsp),%r12;		\
5391eaf3e1SJohn Birrell	movq	TF_R13(%rsp),%r13;		\
5491eaf3e1SJohn Birrell	movq	TF_R14(%rsp),%r14;		\
5591eaf3e1SJohn Birrell	movq	TF_R15(%rsp),%r15;		\
5691eaf3e1SJohn Birrell	testb	$SEL_RPL_MASK,TF_CS(%rsp);	\
5791eaf3e1SJohn Birrell	jz	1f;				\
5891eaf3e1SJohn Birrell	cli;					\
5991eaf3e1SJohn Birrell	swapgs;					\
6091eaf3e1SJohn Birrell1:	addq	$TF_RIP,%rsp;
6191eaf3e1SJohn Birrell
6291eaf3e1SJohn Birrell
6391eaf3e1SJohn Birrell	ENTRY(dtrace_invop_start)
6491eaf3e1SJohn Birrell
6591eaf3e1SJohn Birrell	/*
6691eaf3e1SJohn Birrell	 * #BP traps with %rip set to the next address. We need to decrement
6791eaf3e1SJohn Birrell	 * the value to indicate the address of the int3 (0xcc) instruction
6891eaf3e1SJohn Birrell	 * that we substituted.
6991eaf3e1SJohn Birrell	 */
7091eaf3e1SJohn Birrell	movq	TF_RIP(%rsp), %rdi
7191eaf3e1SJohn Birrell	decq	%rdi
7291eaf3e1SJohn Birrell	movq	%rsp, %rsi
736c280659SMark Johnston	movq	TF_RAX(%rsp), %rdx
7491eaf3e1SJohn Birrell	call	dtrace_invop
7591eaf3e1SJohn Birrell	ALTENTRY(dtrace_invop_callsite)
7691eaf3e1SJohn Birrell	cmpl	$DTRACE_INVOP_PUSHL_EBP, %eax
7791eaf3e1SJohn Birrell	je	bp_push
7891eaf3e1SJohn Birrell	cmpl	$DTRACE_INVOP_LEAVE, %eax
7991eaf3e1SJohn Birrell	je	bp_leave
8091eaf3e1SJohn Birrell	cmpl	$DTRACE_INVOP_NOP, %eax
8191eaf3e1SJohn Birrell	je	bp_nop
8291eaf3e1SJohn Birrell	cmpl	$DTRACE_INVOP_RET, %eax
8391eaf3e1SJohn Birrell	je	bp_ret
8491eaf3e1SJohn Birrell
8591eaf3e1SJohn Birrell	/* When all else fails handle the trap in the usual way. */
8691eaf3e1SJohn Birrell	jmpq	*dtrace_invop_calltrap_addr
8791eaf3e1SJohn Birrell
8891eaf3e1SJohn Birrellbp_push:
8991eaf3e1SJohn Birrell	/*
9091eaf3e1SJohn Birrell	 * We must emulate a "pushq %rbp".  To do this, we pull the stack
9191eaf3e1SJohn Birrell	 * down 8 bytes, and then store the base pointer.
9291eaf3e1SJohn Birrell	 */
9391eaf3e1SJohn Birrell	INTR_POP
9491eaf3e1SJohn Birrell	subq	$16, %rsp		/* make room for %rbp */
9591eaf3e1SJohn Birrell	pushq	%rax			/* push temp */
9691eaf3e1SJohn Birrell	movq	24(%rsp), %rax		/* load calling RIP */
9791eaf3e1SJohn Birrell	movq	%rax, 8(%rsp)		/* store calling RIP */
9891eaf3e1SJohn Birrell	movq	32(%rsp), %rax		/* load calling CS */
9991eaf3e1SJohn Birrell	movq	%rax, 16(%rsp)		/* store calling CS */
10091eaf3e1SJohn Birrell	movq	40(%rsp), %rax		/* load calling RFLAGS */
10191eaf3e1SJohn Birrell	movq	%rax, 24(%rsp)		/* store calling RFLAGS */
10291eaf3e1SJohn Birrell	movq	48(%rsp), %rax		/* load calling RSP */
10391eaf3e1SJohn Birrell	subq	$8, %rax		/* make room for %rbp */
10491eaf3e1SJohn Birrell	movq	%rax, 32(%rsp)		/* store calling RSP */
10591eaf3e1SJohn Birrell	movq	56(%rsp), %rax		/* load calling SS */
10691eaf3e1SJohn Birrell	movq	%rax, 40(%rsp)		/* store calling SS */
10791eaf3e1SJohn Birrell	movq	32(%rsp), %rax		/* reload calling RSP */
10891eaf3e1SJohn Birrell	movq	%rbp, (%rax)		/* store %rbp there */
10991eaf3e1SJohn Birrell	popq	%rax			/* pop off temp */
11091eaf3e1SJohn Birrell	iretq				/* return from interrupt */
11191eaf3e1SJohn Birrell	/*NOTREACHED*/
11291eaf3e1SJohn Birrell
11391eaf3e1SJohn Birrellbp_leave:
11491eaf3e1SJohn Birrell	/*
11591eaf3e1SJohn Birrell	 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp"
11691eaf3e1SJohn Birrell	 * followed by a "popq %rbp".  This is quite a bit simpler on amd64
11791eaf3e1SJohn Birrell	 * than it is on i386 -- we can exploit the fact that the %rsp is
11891eaf3e1SJohn Birrell	 * explicitly saved to effect the pop without having to reshuffle
11991eaf3e1SJohn Birrell	 * the other data pushed for the trap.
12091eaf3e1SJohn Birrell	 */
12191eaf3e1SJohn Birrell	INTR_POP
12291eaf3e1SJohn Birrell	pushq	%rax			/* push temp */
12391eaf3e1SJohn Birrell	movq	8(%rsp), %rax		/* load calling RIP */
12491eaf3e1SJohn Birrell	movq	%rax, 8(%rsp)		/* store calling RIP */
12591eaf3e1SJohn Birrell	movq	(%rbp), %rax		/* get new %rbp */
12691eaf3e1SJohn Birrell	addq	$8, %rbp		/* adjust new %rsp */
12791eaf3e1SJohn Birrell	movq	%rbp, 32(%rsp)		/* store new %rsp */
12891eaf3e1SJohn Birrell	movq	%rax, %rbp		/* set new %rbp */
12991eaf3e1SJohn Birrell	popq	%rax			/* pop off temp */
13091eaf3e1SJohn Birrell	iretq				/* return from interrupt */
13191eaf3e1SJohn Birrell	/*NOTREACHED*/
13291eaf3e1SJohn Birrell
13391eaf3e1SJohn Birrellbp_nop:
13491eaf3e1SJohn Birrell	/* We must emulate a "nop". */
13591eaf3e1SJohn Birrell	INTR_POP
13691eaf3e1SJohn Birrell	iretq
13791eaf3e1SJohn Birrell	/*NOTREACHED*/
13891eaf3e1SJohn Birrell
13991eaf3e1SJohn Birrellbp_ret:
14091eaf3e1SJohn Birrell	INTR_POP
14191eaf3e1SJohn Birrell	pushq	%rax			/* push temp */
14291eaf3e1SJohn Birrell	movq	32(%rsp), %rax		/* load %rsp */
14391eaf3e1SJohn Birrell	movq	(%rax), %rax		/* load calling RIP */
14491eaf3e1SJohn Birrell	movq	%rax, 8(%rsp)		/* store calling RIP */
14591eaf3e1SJohn Birrell	addq	$8, 32(%rsp)		/* adjust new %rsp */
14691eaf3e1SJohn Birrell	popq	%rax			/* pop off temp */
14791eaf3e1SJohn Birrell	iretq				/* return from interrupt */
14891eaf3e1SJohn Birrell	/*NOTREACHED*/
14991eaf3e1SJohn Birrell
15091eaf3e1SJohn Birrell	END(dtrace_invop_start)
15191eaf3e1SJohn Birrell
15291eaf3e1SJohn Birrell/*
15391eaf3e1SJohn Birrellvoid dtrace_invop_init(void)
15491eaf3e1SJohn Birrell*/
15591eaf3e1SJohn Birrell	ENTRY(dtrace_invop_init)
15691eaf3e1SJohn Birrell	movq	$dtrace_invop_start, dtrace_invop_jump_addr(%rip)
15791eaf3e1SJohn Birrell	ret
15891eaf3e1SJohn Birrell	END(dtrace_invop_init)
15991eaf3e1SJohn Birrell
16091eaf3e1SJohn Birrell/*
16191eaf3e1SJohn Birrellvoid dtrace_invop_uninit(void)
16291eaf3e1SJohn Birrell*/
16391eaf3e1SJohn Birrell	ENTRY(dtrace_invop_uninit)
16491eaf3e1SJohn Birrell	movq	$0, dtrace_invop_jump_addr(%rip)
16591eaf3e1SJohn Birrell	ret
16691eaf3e1SJohn Birrell	END(dtrace_invop_uninit)
16791eaf3e1SJohn Birrell
16891eaf3e1SJohn Birrell/*
16991eaf3e1SJohn Birrellgreg_t dtrace_getfp(void)
17091eaf3e1SJohn Birrell*/
17191eaf3e1SJohn Birrell	ENTRY(dtrace_getfp)
17291eaf3e1SJohn Birrell	movq	%rbp, %rax
17391eaf3e1SJohn Birrell	ret
17491eaf3e1SJohn Birrell	END(dtrace_getfp)
17591eaf3e1SJohn Birrell
17691eaf3e1SJohn Birrell/*
17791eaf3e1SJohn Birrelluint32_t
17891eaf3e1SJohn Birrelldtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new)
17991eaf3e1SJohn Birrell*/
18091eaf3e1SJohn Birrell	ENTRY(dtrace_cas32)
18191eaf3e1SJohn Birrell	movl	%esi, %eax
18291eaf3e1SJohn Birrell	lock
18391eaf3e1SJohn Birrell	cmpxchgl %edx, (%rdi)
18491eaf3e1SJohn Birrell	ret
18591eaf3e1SJohn Birrell	END(dtrace_cas32)
18691eaf3e1SJohn Birrell
18791eaf3e1SJohn Birrell/*
18891eaf3e1SJohn Birrellvoid *
18991eaf3e1SJohn Birrelldtrace_casptr(void *target, void *cmp, void *new)
19091eaf3e1SJohn Birrell*/
19191eaf3e1SJohn Birrell	ENTRY(dtrace_casptr)
19291eaf3e1SJohn Birrell	movq	%rsi, %rax
19391eaf3e1SJohn Birrell	lock
19491eaf3e1SJohn Birrell	cmpxchgq %rdx, (%rdi)
19591eaf3e1SJohn Birrell	ret
19691eaf3e1SJohn Birrell	END(dtrace_casptr)
19791eaf3e1SJohn Birrell
19891eaf3e1SJohn Birrell/*
19991eaf3e1SJohn Birrelluintptr_t
20091eaf3e1SJohn Birrelldtrace_caller(int aframes)
20191eaf3e1SJohn Birrell*/
20291eaf3e1SJohn Birrell	ENTRY(dtrace_caller)
20391eaf3e1SJohn Birrell	movq	$-1, %rax
20491eaf3e1SJohn Birrell	ret
20591eaf3e1SJohn Birrell	END(dtrace_caller)
20691eaf3e1SJohn Birrell
20791eaf3e1SJohn Birrell/*
20891eaf3e1SJohn Birrellvoid
20991eaf3e1SJohn Birrelldtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
21091eaf3e1SJohn Birrell*/
211*8ca79fbdSMateusz Guzik	ENTRY(dtrace_copy_nosmap)
21291eaf3e1SJohn Birrell	pushq	%rbp
21391eaf3e1SJohn Birrell	movq	%rsp, %rbp
21491eaf3e1SJohn Birrell
21591eaf3e1SJohn Birrell	xchgq	%rdi, %rsi		/* make %rsi source, %rdi dest */
21691eaf3e1SJohn Birrell	movq	%rdx, %rcx		/* load count */
21791eaf3e1SJohn Birrell	repz				/* repeat for count ... */
21891eaf3e1SJohn Birrell	smovb				/*   move from %ds:rsi to %ed:rdi */
21991eaf3e1SJohn Birrell	leave
22091eaf3e1SJohn Birrell	ret
221*8ca79fbdSMateusz Guzik	END(dtrace_copy_nosmap)
222*8ca79fbdSMateusz Guzik
223*8ca79fbdSMateusz Guzik	ENTRY(dtrace_copy_smap)
224*8ca79fbdSMateusz Guzik	pushq	%rbp
225*8ca79fbdSMateusz Guzik	movq	%rsp, %rbp
226*8ca79fbdSMateusz Guzik
227*8ca79fbdSMateusz Guzik	xchgq	%rdi, %rsi		/* make %rsi source, %rdi dest */
228*8ca79fbdSMateusz Guzik	movq	%rdx, %rcx		/* load count */
229*8ca79fbdSMateusz Guzik	stac
230*8ca79fbdSMateusz Guzik	repz				/* repeat for count ... */
231*8ca79fbdSMateusz Guzik	smovb				/*   move from %ds:rsi to %ed:rdi */
232*8ca79fbdSMateusz Guzik	clac
233*8ca79fbdSMateusz Guzik	leave
234*8ca79fbdSMateusz Guzik	ret
235*8ca79fbdSMateusz Guzik	END(dtrace_copy_smap)
23691eaf3e1SJohn Birrell
23791eaf3e1SJohn Birrell/*
23891eaf3e1SJohn Birrellvoid
23991eaf3e1SJohn Birrelldtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
24091eaf3e1SJohn Birrell    volatile uint16_t *flags)
24191eaf3e1SJohn Birrell*/
242*8ca79fbdSMateusz Guzik	ENTRY(dtrace_copystr_nosmap)
24391eaf3e1SJohn Birrell	pushq	%rbp
24491eaf3e1SJohn Birrell	movq	%rsp, %rbp
24591eaf3e1SJohn Birrell
24691eaf3e1SJohn Birrell0:
24791eaf3e1SJohn Birrell	movb	(%rdi), %al		/* load from source */
24891eaf3e1SJohn Birrell	movb	%al, (%rsi)		/* store to destination */
24991eaf3e1SJohn Birrell	addq	$1, %rdi		/* increment source pointer */
25091eaf3e1SJohn Birrell	addq	$1, %rsi		/* increment destination pointer */
25191eaf3e1SJohn Birrell	subq	$1, %rdx		/* decrement remaining count */
25291eaf3e1SJohn Birrell	cmpb	$0, %al
25391eaf3e1SJohn Birrell	je	2f
25491eaf3e1SJohn Birrell	testq	$0xfff, %rdx		/* test if count is 4k-aligned */
25591eaf3e1SJohn Birrell	jnz	1f			/* if not, continue with copying */
25691eaf3e1SJohn Birrell	testq	$CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */
25791eaf3e1SJohn Birrell	jnz	2f
25891eaf3e1SJohn Birrell1:
25991eaf3e1SJohn Birrell	cmpq	$0, %rdx
26091eaf3e1SJohn Birrell	jne	0b
26191eaf3e1SJohn Birrell2:
26291eaf3e1SJohn Birrell	leave
26391eaf3e1SJohn Birrell	ret
26491eaf3e1SJohn Birrell
265*8ca79fbdSMateusz Guzik	END(dtrace_copystr_nosmap)
266*8ca79fbdSMateusz Guzik
267*8ca79fbdSMateusz Guzik	ENTRY(dtrace_copystr_smap)
268*8ca79fbdSMateusz Guzik	pushq	%rbp
269*8ca79fbdSMateusz Guzik	movq	%rsp, %rbp
270*8ca79fbdSMateusz Guzik
271*8ca79fbdSMateusz Guzik	stac
272*8ca79fbdSMateusz Guzik0:
273*8ca79fbdSMateusz Guzik	movb	(%rdi), %al		/* load from source */
274*8ca79fbdSMateusz Guzik	movb	%al, (%rsi)		/* store to destination */
275*8ca79fbdSMateusz Guzik	addq	$1, %rdi		/* increment source pointer */
276*8ca79fbdSMateusz Guzik	addq	$1, %rsi		/* increment destination pointer */
277*8ca79fbdSMateusz Guzik	subq	$1, %rdx		/* decrement remaining count */
278*8ca79fbdSMateusz Guzik	cmpb	$0, %al
279*8ca79fbdSMateusz Guzik	je	2f
280*8ca79fbdSMateusz Guzik	testq	$0xfff, %rdx		/* test if count is 4k-aligned */
281*8ca79fbdSMateusz Guzik	jnz	1f			/* if not, continue with copying */
282*8ca79fbdSMateusz Guzik	testq	$CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */
283*8ca79fbdSMateusz Guzik	jnz	2f
284*8ca79fbdSMateusz Guzik1:
285*8ca79fbdSMateusz Guzik	cmpq	$0, %rdx
286*8ca79fbdSMateusz Guzik	jne	0b
287*8ca79fbdSMateusz Guzik2:
288*8ca79fbdSMateusz Guzik	clac
289*8ca79fbdSMateusz Guzik	leave
290*8ca79fbdSMateusz Guzik	ret
291*8ca79fbdSMateusz Guzik
292*8ca79fbdSMateusz Guzik	END(dtrace_copystr_smap)
29391eaf3e1SJohn Birrell
29491eaf3e1SJohn Birrell/*
29591eaf3e1SJohn Birrelluintptr_t
29691eaf3e1SJohn Birrelldtrace_fulword(void *addr)
29791eaf3e1SJohn Birrell*/
298*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fulword_nosmap)
29991eaf3e1SJohn Birrell	movq	(%rdi), %rax
30091eaf3e1SJohn Birrell	ret
301*8ca79fbdSMateusz Guzik	END(dtrace_fulword_nosmap)
302*8ca79fbdSMateusz Guzik
303*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fulword_smap)
304*8ca79fbdSMateusz Guzik	stac
305*8ca79fbdSMateusz Guzik	movq	(%rdi), %rax
306*8ca79fbdSMateusz Guzik	clac
307*8ca79fbdSMateusz Guzik	ret
308*8ca79fbdSMateusz Guzik	END(dtrace_fulword_smap)
30991eaf3e1SJohn Birrell
31091eaf3e1SJohn Birrell/*
31191eaf3e1SJohn Birrelluint8_t
31291eaf3e1SJohn Birrelldtrace_fuword8_nocheck(void *addr)
31391eaf3e1SJohn Birrell*/
314*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword8_nocheck_nosmap)
31591eaf3e1SJohn Birrell	xorq	%rax, %rax
31691eaf3e1SJohn Birrell	movb	(%rdi), %al
31791eaf3e1SJohn Birrell	ret
318*8ca79fbdSMateusz Guzik	END(dtrace_fuword8_nocheck_nosmap)
319*8ca79fbdSMateusz Guzik
320*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword8_nocheck_smap)
321*8ca79fbdSMateusz Guzik	stac
322*8ca79fbdSMateusz Guzik	xorq	%rax, %rax
323*8ca79fbdSMateusz Guzik	movb	(%rdi), %al
324*8ca79fbdSMateusz Guzik	clac
325*8ca79fbdSMateusz Guzik	ret
326*8ca79fbdSMateusz Guzik	END(dtrace_fuword8_nocheck_smap)
32791eaf3e1SJohn Birrell
32891eaf3e1SJohn Birrell/*
32991eaf3e1SJohn Birrelluint16_t
33091eaf3e1SJohn Birrelldtrace_fuword16_nocheck(void *addr)
33191eaf3e1SJohn Birrell*/
332*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword16_nocheck_nosmap)
33391eaf3e1SJohn Birrell	xorq	%rax, %rax
33491eaf3e1SJohn Birrell	movw	(%rdi), %ax
33591eaf3e1SJohn Birrell	ret
336*8ca79fbdSMateusz Guzik	END(dtrace_fuword16_nocheck_nosmap)
337*8ca79fbdSMateusz Guzik
338*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword16_nocheck_smap)
339*8ca79fbdSMateusz Guzik	stac
340*8ca79fbdSMateusz Guzik	xorq	%rax, %rax
341*8ca79fbdSMateusz Guzik	movw	(%rdi), %ax
342*8ca79fbdSMateusz Guzik	clac
343*8ca79fbdSMateusz Guzik	ret
344*8ca79fbdSMateusz Guzik	END(dtrace_fuword16_nocheck_smap)
34591eaf3e1SJohn Birrell
34691eaf3e1SJohn Birrell/*
34791eaf3e1SJohn Birrelluint32_t
34891eaf3e1SJohn Birrelldtrace_fuword32_nocheck(void *addr)
34991eaf3e1SJohn Birrell*/
350*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword32_nocheck_nosmap)
35191eaf3e1SJohn Birrell	xorq	%rax, %rax
35291eaf3e1SJohn Birrell	movl	(%rdi), %eax
35391eaf3e1SJohn Birrell	ret
354*8ca79fbdSMateusz Guzik	END(dtrace_fuword32_nocheck_nosmap)
355*8ca79fbdSMateusz Guzik
356*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword32_nocheck_smap)
357*8ca79fbdSMateusz Guzik	stac
358*8ca79fbdSMateusz Guzik	xorq	%rax, %rax
359*8ca79fbdSMateusz Guzik	movl	(%rdi), %eax
360*8ca79fbdSMateusz Guzik	clac
361*8ca79fbdSMateusz Guzik	ret
362*8ca79fbdSMateusz Guzik	END(dtrace_fuword32_nocheck_smap)
36391eaf3e1SJohn Birrell
36491eaf3e1SJohn Birrell/*
36591eaf3e1SJohn Birrelluint64_t
36691eaf3e1SJohn Birrelldtrace_fuword64_nocheck(void *addr)
36791eaf3e1SJohn Birrell*/
368*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword64_nocheck_nosmap)
36991eaf3e1SJohn Birrell	movq	(%rdi), %rax
37091eaf3e1SJohn Birrell	ret
371*8ca79fbdSMateusz Guzik	END(dtrace_fuword64_nocheck_nosmap)
372*8ca79fbdSMateusz Guzik
373*8ca79fbdSMateusz Guzik	ENTRY(dtrace_fuword64_nocheck_smap)
374*8ca79fbdSMateusz Guzik	stac
375*8ca79fbdSMateusz Guzik	movq	(%rdi), %rax
376*8ca79fbdSMateusz Guzik	clac
377*8ca79fbdSMateusz Guzik	ret
378*8ca79fbdSMateusz Guzik	END(dtrace_fuword64_nocheck_smap)
37991eaf3e1SJohn Birrell
38091eaf3e1SJohn Birrell/*
38191eaf3e1SJohn Birrellvoid
38291eaf3e1SJohn Birrelldtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
38391eaf3e1SJohn Birrell    int fault, int fltoffs, uintptr_t illval)
38491eaf3e1SJohn Birrell*/
38591eaf3e1SJohn Birrell	ENTRY(dtrace_probe_error)
38691eaf3e1SJohn Birrell	pushq	%rbp
38791eaf3e1SJohn Birrell	movq	%rsp, %rbp
38891eaf3e1SJohn Birrell	subq	$0x8, %rsp
38991eaf3e1SJohn Birrell	movq	%r9, (%rsp)
39091eaf3e1SJohn Birrell	movq	%r8, %r9
39191eaf3e1SJohn Birrell	movq	%rcx, %r8
39291eaf3e1SJohn Birrell	movq	%rdx, %rcx
39391eaf3e1SJohn Birrell	movq	%rsi, %rdx
39491eaf3e1SJohn Birrell	movq	%rdi, %rsi
39591eaf3e1SJohn Birrell	movl	dtrace_probeid_error(%rip), %edi
39691eaf3e1SJohn Birrell	call	dtrace_probe
39791eaf3e1SJohn Birrell	addq	$0x8, %rsp
39891eaf3e1SJohn Birrell	leave
39991eaf3e1SJohn Birrell	ret
40091eaf3e1SJohn Birrell	END(dtrace_probe_error)
40191eaf3e1SJohn Birrell
40291eaf3e1SJohn Birrell/*
40391eaf3e1SJohn Birrellvoid
40491eaf3e1SJohn Birrelldtrace_membar_producer(void)
40591eaf3e1SJohn Birrell*/
40691eaf3e1SJohn Birrell	ENTRY(dtrace_membar_producer)
40791eaf3e1SJohn Birrell	rep;	ret	/* use 2 byte return instruction when branch target */
40891eaf3e1SJohn Birrell			/* AMD Software Optimization Guide - Section 6.2 */
40991eaf3e1SJohn Birrell	END(dtrace_membar_producer)
41091eaf3e1SJohn Birrell
41191eaf3e1SJohn Birrell/*
41291eaf3e1SJohn Birrellvoid
41391eaf3e1SJohn Birrelldtrace_membar_consumer(void)
41491eaf3e1SJohn Birrell*/
41591eaf3e1SJohn Birrell	ENTRY(dtrace_membar_consumer)
41691eaf3e1SJohn Birrell	rep;	ret	/* use 2 byte return instruction when branch target */
41791eaf3e1SJohn Birrell			/* AMD Software Optimization Guide - Section 6.2 */
41891eaf3e1SJohn Birrell	END(dtrace_membar_consumer)
41991eaf3e1SJohn Birrell
42091eaf3e1SJohn Birrell/*
42191eaf3e1SJohn Birrelldtrace_icookie_t
42291eaf3e1SJohn Birrelldtrace_interrupt_disable(void)
42391eaf3e1SJohn Birrell*/
42491eaf3e1SJohn Birrell	ENTRY(dtrace_interrupt_disable)
42591eaf3e1SJohn Birrell	pushfq
42691eaf3e1SJohn Birrell	popq	%rax
42791eaf3e1SJohn Birrell	cli
42891eaf3e1SJohn Birrell	ret
42991eaf3e1SJohn Birrell	END(dtrace_interrupt_disable)
43091eaf3e1SJohn Birrell
43191eaf3e1SJohn Birrell/*
43291eaf3e1SJohn Birrellvoid
43391eaf3e1SJohn Birrelldtrace_interrupt_enable(dtrace_icookie_t cookie)
43491eaf3e1SJohn Birrell*/
43591eaf3e1SJohn Birrell	ENTRY(dtrace_interrupt_enable)
43691eaf3e1SJohn Birrell	pushq	%rdi
43791eaf3e1SJohn Birrell	popfq
43891eaf3e1SJohn Birrell	ret
43991eaf3e1SJohn Birrell	END(dtrace_interrupt_enable)
440