xref: /freebsd/sys/amd64/include/asmacros.h (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1 /* -*- mode: asm -*- */
2 /*-
3  * SPDX-License-Identifier: BSD-3-Clause
4  *
5  * Copyright (c) 1993 The Regents of the University of California.
6  * All rights reserved.
7  *
8  * Copyright (c) 2018 The FreeBSD Foundation
9  * All rights reserved.
10  *
11  * Portions of this software were developed by
12  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
13  * the FreeBSD Foundation.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  * 3. Neither the name of the University nor the names of its contributors
24  *    may be used to endorse or promote products derived from this software
25  *    without specific prior written permission.
26  *
27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37  * SUCH DAMAGE.
38  *
39  * $FreeBSD$
40  */
41 
42 #ifndef _MACHINE_ASMACROS_H_
43 #define _MACHINE_ASMACROS_H_
44 
45 #include <sys/cdefs.h>
46 
47 /* XXX too much duplication in various asm*.h's. */
48 
49 /*
50  * CNAME is used to manage the relationship between symbol names in C
51  * and the equivalent assembly language names.  CNAME is given a name as
52  * it would be used in a C program.  It expands to the equivalent assembly
53  * language name.
54  */
55 #define CNAME(csym)		csym
56 
57 #define ALIGN_DATA	.p2align 3	/* 8 byte alignment, zero filled */
58 #ifdef GPROF
59 #define ALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
60 #else
61 #define ALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
62 #endif
63 #define SUPERALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
64 
65 #define GEN_ENTRY(name)		ALIGN_TEXT; .globl CNAME(name); \
66 				.type CNAME(name),@function; CNAME(name):
67 #define NON_GPROF_ENTRY(name)	GEN_ENTRY(name)
68 #define NON_GPROF_RET		.byte 0xc3	/* opcode for `ret' */
69 
70 #define	END(name)		.size name, . - name
71 
72 #ifdef GPROF
73 /*
74  * __mcount is like [.]mcount except that doesn't require its caller to set
75  * up a frame pointer.  It must be called before pushing anything onto the
76  * stack.  gcc should eventually generate code to call __mcount in most
77  * cases.  This would make -pg in combination with -fomit-frame-pointer
78  * useful.  gcc has a configuration variable PROFILE_BEFORE_PROLOGUE to
79  * allow profiling before setting up the frame pointer, but this is
80  * inadequate for good handling of special cases, e.g., -fpic works best
81  * with profiling after the prologue.
82  *
83  * [.]mexitcount is a new function to support non-statistical profiling if an
84  * accurate clock is available.  For C sources, calls to it are generated
85  * by the FreeBSD extension `-mprofiler-epilogue' to gcc.  It is best to
86  * call [.]mexitcount at the end of a function like the MEXITCOUNT macro does,
87  * but gcc currently generates calls to it at the start of the epilogue to
88  * avoid problems with -fpic.
89  *
90  * [.]mcount and __mcount may clobber the call-used registers and %ef.
91  * [.]mexitcount may clobber %ecx and %ef.
92  *
93  * Cross-jumping makes non-statistical profiling timing more complicated.
94  * It is handled in many cases by calling [.]mexitcount before jumping.  It
95  * is handled for conditional jumps using CROSSJUMP() and CROSSJUMP_LABEL().
96  * It is handled for some fault-handling jumps by not sharing the exit
97  * routine.
98  *
99  * ALTENTRY() must be before a corresponding ENTRY() so that it can jump to
100  * the main entry point.  Note that alt entries are counted twice.  They
101  * have to be counted as ordinary entries for gprof to get the call times
102  * right for the ordinary entries.
103  *
104  * High local labels are used in macros to avoid clashes with local labels
105  * in functions.
106  *
107  * Ordinary `ret' is used instead of a macro `RET' because there are a lot
108  * of `ret's.  0xc3 is the opcode for `ret' (`#define ret ... ret' can't
109  * be used because this file is sometimes preprocessed in traditional mode).
110  * `ret' clobbers eflags but this doesn't matter.
111  */
112 #define ALTENTRY(name)		GEN_ENTRY(name) ; MCOUNT ; MEXITCOUNT ; jmp 9f
113 #define	CROSSJUMP(jtrue, label, jfalse) \
114 	jfalse 8f; MEXITCOUNT; jmp __CONCAT(to,label); 8:
115 #define CROSSJUMPTARGET(label) \
116 	ALIGN_TEXT; __CONCAT(to,label): ; MCOUNT; jmp label
117 #define ENTRY(name)		GEN_ENTRY(name) ; 9: ; MCOUNT
118 #define FAKE_MCOUNT(caller)	pushq caller ; call __mcount ; popq %rcx
119 #define MCOUNT			call __mcount
120 #define MCOUNT_LABEL(name)	GEN_ENTRY(name) ; nop ; ALIGN_TEXT
121 #ifdef GUPROF
122 #define MEXITCOUNT		call .mexitcount
123 #define ret			MEXITCOUNT ; NON_GPROF_RET
124 #else
125 #define MEXITCOUNT
126 #endif
127 
128 #else /* !GPROF */
129 /*
130  * ALTENTRY() has to align because it is before a corresponding ENTRY().
131  * ENTRY() has to align to because there may be no ALTENTRY() before it.
132  * If there is a previous ALTENTRY() then the alignment code for ENTRY()
133  * is empty.
134  */
135 #define ALTENTRY(name)		GEN_ENTRY(name)
136 #define	CROSSJUMP(jtrue, label, jfalse)	jtrue label
137 #define	CROSSJUMPTARGET(label)
138 #define ENTRY(name)		GEN_ENTRY(name)
139 #define FAKE_MCOUNT(caller)
140 #define MCOUNT
141 #define MCOUNT_LABEL(name)
142 #define MEXITCOUNT
143 #endif /* GPROF */
144 
145 /*
146  * Convenience for adding frame pointers to hand-coded ASM.  Useful for
147  * DTrace, HWPMC, and KDB.
148  */
149 #define PUSH_FRAME_POINTER	\
150 	pushq	%rbp ;		\
151 	movq	%rsp, %rbp ;
152 #define POP_FRAME_POINTER	\
153 	popq	%rbp
154 
155 #ifdef LOCORE
156 /*
157  * Access per-CPU data.
158  */
159 #define	PCPU(member)	%gs:PC_ ## member
160 #define	PCPU_ADDR(member, reg)					\
161 	movq %gs:PC_PRVSPACE, reg ;				\
162 	addq $PC_ ## member, reg
163 
164 /*
165  * Convenience macro for declaring interrupt entry points.
166  */
167 #define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
168 			.type __CONCAT(X,name),@function; __CONCAT(X,name):
169 
170 	.macro	SAVE_SEGS
171 	movw	%fs,TF_FS(%rsp)
172 	movw	%gs,TF_GS(%rsp)
173 	movw	%es,TF_ES(%rsp)
174 	movw	%ds,TF_DS(%rsp)
175 	.endm
176 
177 	.macro	MOVE_STACKS qw
178 	.L.offset=0
179 	.rept	\qw
180 	movq	.L.offset(%rsp),%rdx
181 	movq	%rdx,.L.offset(%rax)
182 	.L.offset=.L.offset+8
183 	.endr
184 	.endm
185 
186 	.macro	PTI_UUENTRY has_err
187 	movq	PCPU(KCR3),%rax
188 	movq	%rax,%cr3
189 	movq	PCPU(RSP0),%rax
190 	subq	$PTI_SIZE,%rax
191 	MOVE_STACKS	((PTI_SIZE / 8) - 1 + \has_err)
192 	movq	%rax,%rsp
193 	popq	%rdx
194 	popq	%rax
195 	.endm
196 
197 	.macro	PTI_UENTRY has_err
198 	swapgs
199 	cmpq	$~0,PCPU(UCR3)
200 	je	1f
201 	pushq	%rax
202 	pushq	%rdx
203 	PTI_UUENTRY \has_err
204 1:
205 	.endm
206 
207 	.macro	PTI_ENTRY name, cont, has_err=0
208 	ALIGN_TEXT
209 	.globl	X\name\()_pti
210 	.type	X\name\()_pti,@function
211 X\name\()_pti:
212 	/* %rax, %rdx and possibly err not yet pushed */
213 	testb	$SEL_RPL_MASK,PTI_CS-(2+1-\has_err)*8(%rsp)
214 	jz	\cont
215 	PTI_UENTRY \has_err
216 	swapgs
217 	jmp	\cont
218 	.endm
219 
220 	.macro	PTI_INTRENTRY vec_name
221 	SUPERALIGN_TEXT
222 	.globl	X\vec_name\()_pti
223 	.type	X\vec_name\()_pti,@function
224 X\vec_name\()_pti:
225 	testb	$SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */
226 	jz	\vec_name\()_u
227 	PTI_UENTRY has_err=0
228 	jmp	\vec_name\()_u
229 	.endm
230 
231 	.macro	INTR_PUSH_FRAME vec_name
232 	SUPERALIGN_TEXT
233 	.globl	X\vec_name
234 	.type	X\vec_name,@function
235 X\vec_name:
236 	testb	$SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */
237 	jz	\vec_name\()_u		/* Yes, dont swapgs again */
238 	swapgs
239 \vec_name\()_u:
240 	subq	$TF_RIP,%rsp	/* skip dummy tf_err and tf_trapno */
241 	movq	%rdi,TF_RDI(%rsp)
242 	movq	%rsi,TF_RSI(%rsp)
243 	movq	%rdx,TF_RDX(%rsp)
244 	movq	%rcx,TF_RCX(%rsp)
245 	movq	%r8,TF_R8(%rsp)
246 	movq	%r9,TF_R9(%rsp)
247 	movq	%rax,TF_RAX(%rsp)
248 	movq	%rbx,TF_RBX(%rsp)
249 	movq	%rbp,TF_RBP(%rsp)
250 	movq	%r10,TF_R10(%rsp)
251 	movq	%r11,TF_R11(%rsp)
252 	movq	%r12,TF_R12(%rsp)
253 	movq	%r13,TF_R13(%rsp)
254 	movq	%r14,TF_R14(%rsp)
255 	movq	%r15,TF_R15(%rsp)
256 	SAVE_SEGS
257 	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
258 	cld
259 	testb	$SEL_RPL_MASK,TF_CS(%rsp)  /* come from kernel ? */
260 	jz	1f		/* yes, leave PCB_FULL_IRET alone */
261 	movq	PCPU(CURPCB),%r8
262 	andl	$~PCB_FULL_IRET,PCB_FLAGS(%r8)
263 1:
264 	.endm
265 
266 	.macro	INTR_HANDLER vec_name
267 	.text
268 	PTI_INTRENTRY	\vec_name
269 	INTR_PUSH_FRAME	\vec_name
270 	.endm
271 
272 	.macro	RESTORE_REGS
273 	movq	TF_RDI(%rsp),%rdi
274 	movq	TF_RSI(%rsp),%rsi
275 	movq	TF_RDX(%rsp),%rdx
276 	movq	TF_RCX(%rsp),%rcx
277 	movq	TF_R8(%rsp),%r8
278 	movq	TF_R9(%rsp),%r9
279 	movq	TF_RAX(%rsp),%rax
280 	movq	TF_RBX(%rsp),%rbx
281 	movq	TF_RBP(%rsp),%rbp
282 	movq	TF_R10(%rsp),%r10
283 	movq	TF_R11(%rsp),%r11
284 	movq	TF_R12(%rsp),%r12
285 	movq	TF_R13(%rsp),%r13
286 	movq	TF_R14(%rsp),%r14
287 	movq	TF_R15(%rsp),%r15
288 	.endm
289 
290 #endif /* LOCORE */
291 
292 #ifdef __STDC__
293 #define ELFNOTE(name, type, desctype, descdata...) \
294 .pushsection .note.name                 ;       \
295   .align 4                              ;       \
296   .long 2f - 1f         /* namesz */    ;       \
297   .long 4f - 3f         /* descsz */    ;       \
298   .long type                            ;       \
299 1:.asciz #name                          ;       \
300 2:.align 4                              ;       \
301 3:desctype descdata                     ;       \
302 4:.align 4                              ;       \
303 .popsection
304 #else /* !__STDC__, i.e. -traditional */
305 #define ELFNOTE(name, type, desctype, descdata) \
306 .pushsection .note.name                 ;       \
307   .align 4                              ;       \
308   .long 2f - 1f         /* namesz */    ;       \
309   .long 4f - 3f         /* descsz */    ;       \
310   .long type                            ;       \
311 1:.asciz "name"                         ;       \
312 2:.align 4                              ;       \
313 3:desctype descdata                     ;       \
314 4:.align 4                              ;       \
315 .popsection
316 #endif /* __STDC__ */
317 
318 #endif /* !_MACHINE_ASMACROS_H_ */
319