xref: /illumos-gate/usr/src/uts/i86pc/ml/syscall_asm_amd64.S (revision 5cd084edc8f03fe81d55c25eb29741c4081ae953)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 * Copyright 2024 MNX Cloud, Inc.
26 */
27
28#include <sys/asm_linkage.h>
29#include <sys/asm_misc.h>
30#include <sys/regset.h>
31#include <sys/privregs.h>
32#include <sys/psw.h>
33#include <sys/machbrand.h>
34
35#include <sys/segments.h>
36#include <sys/pcb.h>
37#include <sys/trap.h>
38#include <sys/ftrace.h>
39#include <sys/traptrace.h>
40#include <sys/clock.h>
41#include <sys/model.h>
42#include <sys/panic.h>
43
44#if defined(__xpv)
45#include <sys/hypervisor.h>
46#endif
47
48#include "assym.h"
49
50/*
51 * We implement five flavours of system call entry points
52 *
53 * -	syscall/sysretq		(amd64 generic)
54 * -	syscall/sysretl		(i386 plus SYSC bit)
55 * -	sysenter/sysexit	(i386 plus SEP bit)
56 * -	int/iret		(i386 generic)
57 * -	lcall/iret		(i386 generic)
58 *
59 * The current libc included in Solaris uses int/iret as the base unoptimized
60 * kernel entry method. Older libc implementations and legacy binaries may use
61 * the lcall call gate, so it must continue to be supported.
62 *
63 * System calls that use an lcall call gate are processed in trap() via a
64 * segment-not-present trap, i.e. lcalls are extremely slow(!).
65 *
66 * The basic pattern used in the 32-bit SYSC handler at this point in time is
67 * to have the bare minimum of assembler, and get to the C handlers as
68 * quickly as possible.
69 *
70 * The 64-bit handler is much closer to the sparcv9 handler; that's
71 * because of passing arguments in registers.  The 32-bit world still
72 * passes arguments on the stack -- that makes that handler substantially
73 * more complex.
74 *
75 * The two handlers share a few code fragments which are broken
76 * out into preprocessor macros below.
77 *
78 * XX64	come back and speed all this up later.  The 32-bit stuff looks
79 * especially easy to speed up the argument copying part ..
80 *
81 *
82 * Notes about segment register usage (c.f. the 32-bit kernel)
83 *
84 * In the 32-bit kernel, segment registers are dutifully saved and
85 * restored on all mode transitions because the kernel uses them directly.
86 * When the processor is running in 64-bit mode, segment registers are
87 * largely ignored.
88 *
89 * %cs and %ss
90 *	controlled by the hardware mechanisms that make mode transitions
91 *
92 * The remaining segment registers have to either be pointing at a valid
93 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
94 *
95 * %ds and %es
96 *	always ignored
97 *
98 * %fs and %gs
99 *	fsbase and gsbase are used to control the place they really point at.
100 *	The kernel only depends on %gs, and controls its own gsbase via swapgs
101 *
102 * Note that loading segment registers is still costly because the GDT
103 * lookup still happens (this is because the hardware can't know that we're
104 * not setting up these segment registers for a 32-bit program).  Thus we
105 * avoid doing this in the syscall path, and defer them to lwp context switch
106 * handlers, so the register values remain virtualized to the lwp.
107 */
108
109#if defined(SYSCALLTRACE)
110#define	ORL_SYSCALLTRACE(r32)		\
111	orl	syscalltrace(%rip), r32
112#else
113#define	ORL_SYSCALLTRACE(r32)
114#endif
115
116/*
117 * In the 32-bit kernel, we do absolutely nothing before getting into the
118 * brand callback checks.  In 64-bit land, we do swapgs and then come here.
119 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
120 * are still unused.
121 *
122 * Check if a brand_mach_ops callback is defined for the specified callback_id
123 * type.  If so invoke it with the kernel's %gs value loaded and the following
124 * data on the stack:
125 *
126 * stack:  --------------------------------------
127 *      32 | callback pointer			|
128 *    | 24 | user (or interrupt) stack pointer	|
129 *    | 16 | lwp pointer			|
130 *    v  8 | userland return address		|
131 *       0 | callback wrapper return addr	|
132 *         --------------------------------------
133 *
134 * Since we're pushing the userland return address onto the kernel stack
135 * we need to get that address without accessing the user's stack (since we
136 * can't trust that data).  There are different ways to get the userland
137 * return address depending on how the syscall trap was made:
138 *
139 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
140 * b) For sys_sysenter the return address is in %rdx.
141 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
142 *    the stack pointer points at the state saved when we took the interrupt:
143 *	 ------------------------
144 *    |  | user's %ss		|
145 *    |  | user's %esp		|
146 *    |  | EFLAGS register	|
147 *    v  | user's %cs		|
148 *       | user's %eip		|
149 *	 ------------------------
150 *
151 * The 2nd parameter to the BRAND_CALLBACK macro is either the
152 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
153 * used to generate the proper code to get the userland return address for
154 * each syscall entry point.
155 *
156 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
157 * is available as a scratch register within the callback.  If the callback
158 * returns within the kernel then this macro will restore %r15.  If the
159 * callback is going to return directly to userland then it should restore
160 * %r15 before returning to userland.
161 */
162#define BRAND_URET_FROM_REG(rip_reg)					\
163	pushq	rip_reg			/* push the return address	*/
164
165/*
166 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
167 * is currently pointing at the user return address (%eip).
168 */
169#define BRAND_URET_FROM_INTR_STACK()					\
170	movq	%gs:CPU_RTMP_RSP, %r15	/* grab the intr. stack pointer	*/ ;\
171	pushq	(%r15)			/* push the return address	*/
172
173#define	BRAND_CALLBACK(callback_id, push_userland_ret)			    \
174	movq	%rsp, %gs:CPU_RTMP_RSP	/* save the stack pointer	*/ ;\
175	movq	%r15, %gs:CPU_RTMP_R15	/* save %r15			*/ ;\
176	movq	%gs:CPU_THREAD, %r15	/* load the thread pointer	*/ ;\
177	movq	T_STACK(%r15), %rsp	/* switch to the kernel stack	*/ ;\
178	subq	$16, %rsp		/* save space for 2 pointers	*/ ;\
179	pushq	%r14			/* save %r14			*/ ;\
180	movq	%gs:CPU_RTMP_RSP, %r14					   ;\
181	movq	%r14, 8(%rsp)		/* stash the user stack pointer	*/ ;\
182	popq	%r14			/* restore %r14			*/ ;\
183	movq	T_LWP(%r15), %r15	/* load the lwp pointer		*/ ;\
184	pushq	%r15			/* push the lwp pointer		*/ ;\
185	movq	LWP_PROCP(%r15), %r15	/* load the proc pointer	*/ ;\
186	movq	P_BRAND(%r15), %r15	/* load the brand pointer	*/ ;\
187	movq	B_MACHOPS(%r15), %r15	/* load the machops pointer	*/ ;\
188	movq	_CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15		   ;\
189	cmpq	$0, %r15						   ;\
190	je	1f							   ;\
191	movq	%r15, 16(%rsp)		/* save the callback pointer	*/ ;\
192	push_userland_ret		/* push the return address	*/ ;\
193	movq	24(%rsp), %r15		/* load callback pointer	*/ ;\
194	INDIRECT_CALL_REG(r15)		/* call callback		*/ ;\
1951:	movq	%gs:CPU_RTMP_R15, %r15	/* restore %r15			*/ ;\
196	movq	%gs:CPU_RTMP_RSP, %rsp	/* restore the stack pointer	*/
197
198#define	MSTATE_TRANSITION(from, to)		\
199	movl	$from, %edi;			\
200	movl	$to, %esi;			\
201	call	syscall_mstate
202
203/*
204 * Check to see if a simple (direct) return is possible i.e.
205 *
206 *	if (t->t_post_sys_ast | syscalltrace |
207 *	    lwp->lwp_pcb.pcb_rupdate == 1)
208 *		do full version	;
209 *
210 * Preconditions:
211 * -	t is curthread
212 * Postconditions:
213 * -	condition code NE is set if post-sys is too complex
214 * -	rtmp is zeroed if it isn't (we rely on this!)
215 * -	ltmp is smashed
216 */
217#define	CHECK_POSTSYS_NE(t, ltmp, rtmp)			\
218	movq	T_LWP(t), ltmp;				\
219	movzbl	PCB_RUPDATE(ltmp), rtmp;		\
220	ORL_SYSCALLTRACE(rtmp);				\
221	orl	T_POST_SYS_AST(t), rtmp;		\
222	cmpl	$0, rtmp
223
224/*
225 * Fix up the lwp, thread, and eflags for a successful return
226 *
227 * Preconditions:
228 * -	zwreg contains zero
229 */
230#define	SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)		\
231	movb	$LWP_USER, LWP_STATE(lwp);		\
232	movw	zwreg, T_SYSNUM(t);			\
233	andb	$_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
234
235/*
236 * ASSERT(lwptoregs(lwp) == rp);
237 *
238 * This may seem obvious, but very odd things happen if this
239 * assertion is false
240 *
241 * Preconditions:
242 *	(%rsp is ready for normal call sequence)
243 * Postconditions (if assertion is true):
244 *	%r11 is smashed
245 *
246 * ASSERT(rp->r_cs == descnum)
247 *
248 * The code selector is written into the regs structure when the
249 * lwp stack is created.  We use this ASSERT to validate that
250 * the regs structure really matches how we came in.
251 *
252 * Preconditions:
253 *	(%rsp is ready for normal call sequence)
254 * Postconditions (if assertion is true):
255 *	-none-
256 *
257 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
258 *
259 * If this is false, it meant that we returned to userland without
260 * updating the segment registers as we were supposed to.
261 *
262 * Note that we must ensure no interrupts or other traps intervene
263 * between entering privileged mode and performing the assertion,
264 * otherwise we may perform a context switch on the thread, which
265 * will end up setting pcb_rupdate to 1 again.
266 *
267 * ASSERT(%cr0 & CR0_TS == 0);
268 * Preconditions:
269 *	(%rsp is ready for normal call sequence)
270 * Postconditions (if assertion is true):
271 *      (specified register is clobbered)
272 *
273 * Check to make sure that we are returning to user land and that CR0.TS
274 * is not set. This is required as part of the eager FPU (see
275 * uts/intel/os/fpu.c for more information).
276 */
277
278#if defined(DEBUG)
279
280__lwptoregs_msg:
281	.string	"syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
282
283__codesel_msg:
284	.string	"syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
285
286__no_rupdate_msg:
287	.string	"syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
288
289__bad_ts_msg:
290	.string "syscall_asm_amd64.s:%d CR0.TS set on user return"
291
292#define	ASSERT_LWPTOREGS(lwp, rp)			\
293	movq	LWP_REGS(lwp), %r11;			\
294	cmpq	rp, %r11;				\
295	je	7f;					\
296	leaq	__lwptoregs_msg(%rip), %rdi;		\
297	movl	$__LINE__, %esi;			\
298	movq	lwp, %rdx;				\
299	movq	%r11, %rcx;				\
300	movq	rp, %r8;				\
301	xorl	%eax, %eax;				\
302	call	panic;					\
3037:
304
305#define	ASSERT_NO_RUPDATE_PENDING(lwp)			\
306	testb	$0x1, PCB_RUPDATE(lwp);			\
307	je	8f;					\
308	movq	lwp, %rdx;				\
309	leaq	__no_rupdate_msg(%rip), %rdi;		\
310	movl	$__LINE__, %esi;			\
311	xorl	%eax, %eax;				\
312	call	panic;					\
3138:
314
315#define	ASSERT_CR0TS_ZERO(reg)				\
316	movq	%cr0, reg;				\
317	testq	$CR0_TS, reg;				\
318	jz	9f;					\
319	leaq	__bad_ts_msg(%rip), %rdi;		\
320	movl	$__LINE__, %esi;			\
321	xorl	%eax, %eax;				\
322	call	panic;					\
3239:
324
325#else
326#define	ASSERT_LWPTOREGS(lwp, rp)
327#define	ASSERT_NO_RUPDATE_PENDING(lwp)
328#define	ASSERT_CR0TS_ZERO(reg)
329#endif
330
331/*
332 * Do the traptrace thing and restore any registers we used
333 * in situ.  Assumes that %rsp is pointing at the base of
334 * the struct regs, obviously ..
335 */
336#ifdef TRAPTRACE
337#define	SYSCALL_TRAPTRACE(ttype)				\
338	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);		\
339	TRACE_REGS(%rdi, %rsp, %rbx, %rcx);			\
340	TRACE_STAMP(%rdi);	/* rdtsc clobbers %eax, %edx */	\
341	movq	REGOFF_RAX(%rsp), %rax;				\
342	movq	REGOFF_RBX(%rsp), %rbx;				\
343	movq	REGOFF_RCX(%rsp), %rcx;				\
344	movq	REGOFF_RDX(%rsp), %rdx;				\
345	movl	%eax, TTR_SYSNUM(%rdi);				\
346	movq	REGOFF_RDI(%rsp), %rdi
347
348#define	SYSCALL_TRAPTRACE32(ttype)				\
349	SYSCALL_TRAPTRACE(ttype);				\
350	/* paranoia: clean the top 32-bits of the registers */	\
351	orl	%eax, %eax;					\
352	orl	%ebx, %ebx;					\
353	orl	%ecx, %ecx;					\
354	orl	%edx, %edx;					\
355	orl	%edi, %edi
356#else	/* TRAPTRACE */
357#define	SYSCALL_TRAPTRACE(ttype)
358#define	SYSCALL_TRAPTRACE32(ttype)
359#endif	/* TRAPTRACE */
360
361/*
362 * The 64-bit libc syscall wrapper does this:
363 *
364 * fn(<args>)
365 * {
366 *	movq	%rcx, %r10	-- because syscall smashes %rcx
367 *	movl	$CODE, %eax
368 *	syscall
369 *	<error processing>
370 * }
371 *
372 * Thus when we come into the kernel:
373 *
374 *	%rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
375 *	%rax is the syscall number
376 *	%r12-%r15 contain caller state
377 *
378 * The syscall instruction arranges that:
379 *
380 *	%rcx contains the return %rip
381 *	%r11d contains bottom 32-bits of %rflags
382 *	%rflags is masked (as determined by the SFMASK msr)
383 *	%cs is set to UCS_SEL (as determined by the STAR msr)
384 *	%ss is set to UDS_SEL (as determined by the STAR msr)
385 *	%rip is set to sys_syscall (as determined by the LSTAR msr)
386 *
387 * Or in other words, we have no registers available at all.
388 * Only swapgs can save us!
389 *
390 * Under the hypervisor, the swapgs has happened already.  However, the
391 * state of the world is very different from that we're familiar with.
392 *
393 * In particular, we have a stack structure like that for interrupt
394 * gates, except that the %cs and %ss registers are modified for reasons
395 * that are not entirely clear.  Critically, the %rcx/%r11 values do
396 * *not* reflect the usage of those registers under a 'real' syscall[1];
397 * the stack, therefore, looks like this:
398 *
399 *	0x0(rsp)	potentially junk %rcx
400 *	0x8(rsp)	potentially junk %r11
401 *	0x10(rsp)	user %rip
402 *	0x18(rsp)	modified %cs
403 *	0x20(rsp)	user %rflags
404 *	0x28(rsp)	user %rsp
405 *	0x30(rsp)	modified %ss
406 *
407 *
408 * and before continuing on, we must load the %rip into %rcx and the
409 * %rflags into %r11.
410 *
411 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
412 * Sigh.
413 */
414#if defined(__xpv)
415#define	XPV_SYSCALL_PROD						\
416	movq	0x10(%rsp), %rcx;					\
417	movq	0x20(%rsp), %r11;					\
418	movq	0x28(%rsp), %rsp
419#else
420#define	XPV_SYSCALL_PROD /* nothing */
421#endif
422
423	ENTRY_NP2(brand_sys_syscall,_allsyscalls)
424	SWAPGS				/* kernel gsbase */
425	XPV_SYSCALL_PROD
426	BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
427	jmp	noprod_sys_syscall
428
429	ALTENTRY(sys_syscall)
430	SWAPGS				/* kernel gsbase */
431	XPV_SYSCALL_PROD
432
433noprod_sys_syscall:
434	movq	%r15, %gs:CPU_RTMP_R15
435	movq	%rsp, %gs:CPU_RTMP_RSP
436
437	movq	%gs:CPU_THREAD, %r15
438	movq	T_STACK(%r15), %rsp	/* switch from user to kernel stack */
439
440	ASSERT_UPCALL_MASK_IS_SET
441
442	movl	$UCS_SEL, REGOFF_CS(%rsp)
443	movq	%rcx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
444	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
445	movl	$UDS_SEL, REGOFF_SS(%rsp)
446
447	movl	%eax, %eax			/* wrapper: sysc# -> %eax */
448	movq	%rdi, REGOFF_RDI(%rsp)
449	movq	%rsi, REGOFF_RSI(%rsp)
450	movq	%rdx, REGOFF_RDX(%rsp)
451	movq	%r10, REGOFF_RCX(%rsp)		/* wrapper: %rcx -> %r10 */
452	movq	%r10, %rcx			/* arg[3] for direct calls */
453
454	movq	%r8, REGOFF_R8(%rsp)
455	movq	%r9, REGOFF_R9(%rsp)
456	movq	%rax, REGOFF_RAX(%rsp)
457	movq	%rbx, REGOFF_RBX(%rsp)
458
459	movq	%rbp, REGOFF_RBP(%rsp)
460	movq	%r10, REGOFF_R10(%rsp)
461	movq	%gs:CPU_RTMP_RSP, %r11
462	movq	%r11, REGOFF_RSP(%rsp)
463	movq	%r12, REGOFF_R12(%rsp)
464
465	movq	%r13, REGOFF_R13(%rsp)
466	movq	%r14, REGOFF_R14(%rsp)
467	movq	%gs:CPU_RTMP_R15, %r10
468	movq	%r10, REGOFF_R15(%rsp)
469	movq	$0, REGOFF_SAVFP(%rsp)
470	movq	$0, REGOFF_SAVPC(%rsp)
471
472	/*
473	 * Copy these registers here in case we end up stopped with
474	 * someone (like, say, /proc) messing with our register state.
475	 * We don't -restore- them unless we have to in update_sregs.
476	 *
477	 * Since userland -can't- change fsbase or gsbase directly,
478	 * and capturing them involves two serializing instructions,
479	 * we don't bother to capture them here.
480	 */
481	xorl	%ebx, %ebx
482	movw	%ds, %bx
483	movq	%rbx, REGOFF_DS(%rsp)
484	movw	%es, %bx
485	movq	%rbx, REGOFF_ES(%rsp)
486	movw	%fs, %bx
487	movq	%rbx, REGOFF_FS(%rsp)
488	movw	%gs, %bx
489	movq	%rbx, REGOFF_GS(%rsp)
490
491	/*
492	 * If we're trying to use TRAPTRACE though, I take that back: we're
493	 * probably debugging some problem in the SWAPGS logic and want to know
494	 * what the incoming gsbase was.
495	 *
496	 * Since we already did SWAPGS, record the KGSBASE.
497	 */
498#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
499	movl	$MSR_AMD_KGSBASE, %ecx
500	rdmsr
501	movl	%eax, REGOFF_GSBASE(%rsp)
502	movl	%edx, REGOFF_GSBASE+4(%rsp)
503#endif
504
505	/*
506	 * Machine state saved in the regs structure on the stack
507	 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
508	 * %eax is the syscall number
509	 * %rsp is the thread's stack, %r15 is curthread
510	 * REG_RSP(%rsp) is the user's stack
511	 */
512
513	SYSCALL_TRAPTRACE($TT_SYSC64)
514
515	movq	%rsp, %rbp
516
517	movq	T_LWP(%r15), %r14
518	ASSERT_NO_RUPDATE_PENDING(%r14)
519	ENABLE_INTR_FLAGS
520
521	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
522	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */
523
524	ASSERT_LWPTOREGS(%r14, %rsp)
525
526	movb	$LWP_SYS, LWP_STATE(%r14)
527	incq	LWP_RU_SYSC(%r14)
528	movb	$NORMALRETURN, LWP_EOSYS(%r14)
529
530	incq	%gs:CPU_STATS_SYS_SYSCALL
531
532	movw	%ax, T_SYSNUM(%r15)
533	movzbl	T_PRE_SYS(%r15), %ebx
534	ORL_SYSCALLTRACE(%ebx)
535	testl	%ebx, %ebx
536	jne	_syscall_pre
537
538_syscall_invoke:
539	movq	REGOFF_RDI(%rbp), %rdi
540	movq	REGOFF_RSI(%rbp), %rsi
541	movq	REGOFF_RDX(%rbp), %rdx
542	movq	REGOFF_RCX(%rbp), %rcx
543	movq	REGOFF_R8(%rbp), %r8
544	movq	REGOFF_R9(%rbp), %r9
545
546	cmpl	$NSYSCALL, %eax
547	jae	_syscall_ill
548	shll	$SYSENT_SIZE_SHIFT, %eax
549	leaq	sysent(%rax), %rbx
550
551	movq	SY_CALLC(%rbx), %rax
552	INDIRECT_CALL_REG(rax)
553
554	movq	%rax, %r12
555	movq	%rdx, %r13
556
557	/*
558	 * If the handler returns two ints, then we need to split the
559	 * 64-bit return value into two 32-bit values.
560	 */
561	testw	$SE_32RVAL2, SY_FLAGS(%rbx)
562	je	5f
563	movq	%r12, %r13
564	shrq	$32, %r13	/* upper 32-bits into %edx */
565	movl	%r12d, %r12d	/* lower 32-bits into %eax */
5665:
567	/*
568	 * Optimistically assume that there's no post-syscall
569	 * work to do.  (This is to avoid having to call syscall_mstate()
570	 * with interrupts disabled)
571	 */
572	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
573
574	/*
575	 * We must protect ourselves from being descheduled here;
576	 * If we were, and we ended up on another cpu, or another
577	 * lwp got in ahead of us, it could change the segment
578	 * registers without us noticing before we return to userland.
579	 */
580	CLI(%r14)
581	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
582	jne	_syscall_post
583
584	/*
585	 * We need to protect ourselves against non-canonical return values
586	 * because Intel doesn't check for them on sysret (AMD does).  Canonical
587	 * addresses on current amd64 processors only use 48-bits for VAs; an
588	 * address is canonical if all upper bits (47-63) are identical. If we
589	 * find a non-canonical %rip, we opt to go through the full
590	 * _syscall_post path which takes us into an iretq which is not
591	 * susceptible to the same problems sysret is.
592	 *
593	 * We're checking for a canonical address by first doing an arithmetic
594	 * shift. This will fill in the remaining bits with the value of bit 63.
595	 * If the address were canonical, the register would now have either all
596	 * zeroes or all ones in it. Therefore we add one (inducing overflow)
597	 * and compare against 1. A canonical address will either be zero or one
598	 * at this point, hence the use of ja.
599	 *
600	 * At this point, r12 and r13 have the return value so we can't use
601	 * those registers.
602	 */
603	movq	REGOFF_RIP(%rsp), %rcx
604	sarq	$47, %rcx
605	incq	%rcx
606	cmpq	$1, %rcx
607	ja	_syscall_post
608
609
610	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
611
612	movq	%r12, REGOFF_RAX(%rsp)
613	movq	%r13, REGOFF_RDX(%rsp)
614
615	/*
616	 * Clobber %r11 as we check CR0.TS.
617	 */
618	ASSERT_CR0TS_ZERO(%r11)
619
620	/*
621	 * To get back to userland, we need the return %rip in %rcx and
622	 * the return %rfl in %r11d.  The sysretq instruction also arranges
623	 * to fix up %cs and %ss; everything else is our responsibility.
624	 */
625	movq	REGOFF_RDI(%rsp), %rdi
626	movq	REGOFF_RSI(%rsp), %rsi
627	movq	REGOFF_RDX(%rsp), %rdx
628	/* %rcx used to restore %rip value */
629
630	movq	REGOFF_R8(%rsp), %r8
631	movq	REGOFF_R9(%rsp), %r9
632	movq	REGOFF_RAX(%rsp), %rax
633	movq	REGOFF_RBX(%rsp), %rbx
634
635	movq	REGOFF_RBP(%rsp), %rbp
636	movq	REGOFF_R10(%rsp), %r10
637	/* %r11 used to restore %rfl value */
638	movq	REGOFF_R12(%rsp), %r12
639
640	movq	REGOFF_R13(%rsp), %r13
641	movq	REGOFF_R14(%rsp), %r14
642	movq	REGOFF_R15(%rsp), %r15
643
644	movq	REGOFF_RIP(%rsp), %rcx
645	movl	REGOFF_RFL(%rsp), %r11d
646
647	/*
648	 * Unlike other cases, because we need to restore the user stack pointer
649	 * before exiting the kernel we must clear the microarch state before
650	 * getting here. This should be safe because it means that the only
651	 * values on the bus after this are based on the user's registers and
652	 * potentially the addresses where we stored them. Given the constraints
653	 * of sysret, that's how it has to be.
654	 */
655	call	x86_md_clear
656
657#if defined(__xpv)
658	addq	$REGOFF_RIP, %rsp
659#else
660	movq	REGOFF_RSP(%rsp), %rsp
661#endif
662
663        /*
664         * There can be no instructions between the ALTENTRY below and
665	 * SYSRET or we could end up breaking brand support. See label usage
666         * in sn1_brand_syscall_callback for an example.
667         */
668	ASSERT_UPCALL_MASK_IS_SET
669#if defined(__xpv)
670	SYSRETQ
671        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
672
673	/*
674	 * We can only get here after executing a brand syscall
675	 * interposition callback handler and simply need to
676	 * "sysretq" back to userland. On the hypervisor this
677	 * involves the iret hypercall which requires us to construct
678	 * just enough of the stack needed for the hypercall.
679	 * (rip, cs, rflags, rsp, ss).
680	 */
681	movq    %rsp, %gs:CPU_RTMP_RSP		/* save user's rsp */
682	movq	%gs:CPU_THREAD, %r11
683	movq	T_STACK(%r11), %rsp
684
685	movq	%rcx, REGOFF_RIP(%rsp)
686	movl	$UCS_SEL, REGOFF_CS(%rsp)
687	movq	%gs:CPU_RTMP_RSP, %r11
688	movq	%r11, REGOFF_RSP(%rsp)
689	pushfq
690	popq	%r11				/* hypercall enables ints */
691	movq	%r11, REGOFF_RFL(%rsp)
692	movl	$UDS_SEL, REGOFF_SS(%rsp)
693	addq	$REGOFF_RIP, %rsp
694	/*
695	 * XXPV: see comment in SYSRETQ definition for future optimization
696	 *       we could take.
697	 */
698	ASSERT_UPCALL_MASK_IS_SET
699	SYSRETQ
700#else
701        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
702	jmp	tr_sysretq
703#endif
704        /*NOTREACHED*/
705        SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
706
707_syscall_pre:
708	call	pre_syscall
709	movl	%eax, %r12d
710	testl	%eax, %eax
711	jne	_syscall_post_call
712	/*
713	 * Didn't abort, so reload the syscall args and invoke the handler.
714	 */
715	movzwl	T_SYSNUM(%r15), %eax
716	jmp	_syscall_invoke
717
718_syscall_ill:
719	call	nosys
720	movq	%rax, %r12
721	movq	%rdx, %r13
722	jmp	_syscall_post_call
723
724_syscall_post:
725	STI
726	/*
727	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
728	 * so that we can account for the extra work it takes us to finish.
729	 */
730	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
731_syscall_post_call:
732	movq	%r12, %rdi
733	movq	%r13, %rsi
734	call	post_syscall
735	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
736	jmp	_sys_rtt
737	SET_SIZE(sys_syscall)
738	SET_SIZE(brand_sys_syscall)
739
740	ENTRY_NP(brand_sys_syscall32)
741	SWAPGS				/* kernel gsbase */
742	XPV_TRAP_POP
743	BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
744	jmp	nopop_sys_syscall32
745
746	ALTENTRY(sys_syscall32)
747	SWAPGS				/* kernel gsbase */
748	XPV_TRAP_POP
749
750nopop_sys_syscall32:
751	movl	%esp, %r10d
752	movq	%gs:CPU_THREAD, %r15
753	movq	T_STACK(%r15), %rsp
754	movl	%eax, %eax
755
756	movl	$U32CS_SEL, REGOFF_CS(%rsp)
757	movl	%ecx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
758	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
759	movq	%r10, REGOFF_RSP(%rsp)
760	movl	$UDS_SEL, REGOFF_SS(%rsp)
761
762_syscall32_save:
763	movl	%edi, REGOFF_RDI(%rsp)
764	movl	%esi, REGOFF_RSI(%rsp)
765	movl	%ebp, REGOFF_RBP(%rsp)
766	movl	%ebx, REGOFF_RBX(%rsp)
767	movl	%edx, REGOFF_RDX(%rsp)
768	movl	%ecx, REGOFF_RCX(%rsp)
769	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
770	movq	$0, REGOFF_SAVFP(%rsp)
771	movq	$0, REGOFF_SAVPC(%rsp)
772
773	/*
774	 * Copy these registers here in case we end up stopped with
775	 * someone (like, say, /proc) messing with our register state.
776	 * We don't -restore- them unless we have to in update_sregs.
777	 *
778	 * Since userland -can't- change fsbase or gsbase directly,
779	 * we don't bother to capture them here.
780	 */
781	xorl	%ebx, %ebx
782	movw	%ds, %bx
783	movq	%rbx, REGOFF_DS(%rsp)
784	movw	%es, %bx
785	movq	%rbx, REGOFF_ES(%rsp)
786	movw	%fs, %bx
787	movq	%rbx, REGOFF_FS(%rsp)
788	movw	%gs, %bx
789	movq	%rbx, REGOFF_GS(%rsp)
790
791	/*
792	 * If we're trying to use TRAPTRACE though, I take that back: we're
793	 * probably debugging some problem in the SWAPGS logic and want to know
794	 * what the incoming gsbase was.
795	 *
796	 * Since we already did SWAPGS, record the KGSBASE.
797	 */
798#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
799	movl	$MSR_AMD_KGSBASE, %ecx
800	rdmsr
801	movl	%eax, REGOFF_GSBASE(%rsp)
802	movl	%edx, REGOFF_GSBASE+4(%rsp)
803#endif
804
805	/*
806	 * Application state saved in the regs structure on the stack
807	 * %eax is the syscall number
808	 * %rsp is the thread's stack, %r15 is curthread
809	 * REG_RSP(%rsp) is the user's stack
810	 */
811
812	SYSCALL_TRAPTRACE32($TT_SYSC)
813
814	movq	%rsp, %rbp
815
816	movq	T_LWP(%r15), %r14
817	ASSERT_NO_RUPDATE_PENDING(%r14)
818
819	ENABLE_INTR_FLAGS
820
821	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
822	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */
823
824	ASSERT_LWPTOREGS(%r14, %rsp)
825
826	incq	 %gs:CPU_STATS_SYS_SYSCALL
827
828	/*
829	 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
830	 * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
831	 * more succinctly:
832	 *
833	 *	SA(MAXSYSARGS * sizeof (long)) == 64
834	 */
835#define	SYS_DROP	64			/* drop for args */
836	subq	$SYS_DROP, %rsp
837	movb	$LWP_SYS, LWP_STATE(%r14)
838	movq	%r15, %rdi
839	movq	%rsp, %rsi
840	call	syscall_entry
841
842	/*
843	 * Fetch the arguments copied onto the kernel stack and put
844	 * them in the right registers to invoke a C-style syscall handler.
845	 * %rax contains the handler address.
846	 *
847	 * Ideas for making all this go faster of course include simply
848	 * forcibly fetching 6 arguments from the user stack under lofault
849	 * protection, reverting to copyin_args only when watchpoints
850	 * are in effect.
851	 *
852	 * (If we do this, make sure that exec and libthread leave
853	 * enough space at the top of the stack to ensure that we'll
854	 * never do a fetch from an invalid page.)
855	 *
856	 * Lots of ideas here, but they won't really help with bringup B-)
857	 * Correctness can't wait, performance can wait a little longer ..
858	 */
859
860	movq	%rax, %rbx
861	movl	0(%rsp), %edi
862	movl	8(%rsp), %esi
863	movl	0x10(%rsp), %edx
864	movl	0x18(%rsp), %ecx
865	movl	0x20(%rsp), %r8d
866	movl	0x28(%rsp), %r9d
867
868	movq	SY_CALLC(%rbx), %rax
869	INDIRECT_CALL_REG(rax)
870
871	movq	%rbp, %rsp	/* pop the args */
872
873	/*
874	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
875	 * On the 32-bit kernel, they always return that value in %eax:%edx
876	 * as required by the 32-bit ABI.
877	 *
878	 * Simulate the same behaviour by unconditionally splitting the
879	 * return value in the same way.
880	 */
881	movq	%rax, %r13
882	shrq	$32, %r13	/* upper 32-bits into %edx */
883	movl	%eax, %r12d	/* lower 32-bits into %eax */
884
885	/*
886	 * Optimistically assume that there's no post-syscall
887	 * work to do.  (This is to avoid having to call syscall_mstate()
888	 * with interrupts disabled)
889	 */
890	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
891
892	/*
893	 * We must protect ourselves from being descheduled here;
894	 * If we were, and we ended up on another cpu, or another
895	 * lwp got in ahead of us, it could change the segment
896	 * registers without us noticing before we return to userland.
897	 */
898	CLI(%r14)
899	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
900	jne	_full_syscall_postsys32
901	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
902
903	/*
904	 * Clobber %r11 as we check CR0.TS.
905	 */
906	ASSERT_CR0TS_ZERO(%r11)
907
908	/*
909	 * To get back to userland, we need to put the return %rip in %rcx and
910	 * the return %rfl in %r11d.  The sysret instruction also arranges
911	 * to fix up %cs and %ss; everything else is our responsibility.
912	 */
913
914	movl	%r12d, %eax			/* %eax: rval1 */
915	movl	REGOFF_RBX(%rsp), %ebx
916	/* %ecx used for return pointer */
917	movl	%r13d, %edx			/* %edx: rval2 */
918	movl	REGOFF_RBP(%rsp), %ebp
919	movl	REGOFF_RSI(%rsp), %esi
920	movl	REGOFF_RDI(%rsp), %edi
921
922	movl	REGOFF_RFL(%rsp), %r11d		/* %r11 -> eflags */
923	movl	REGOFF_RIP(%rsp), %ecx		/* %ecx -> %eip */
924	/*
925	 * Unlike other cases, because we need to restore the user stack pointer
926	 * before exiting the kernel we must clear the microarch state before
927	 * getting here. This should be safe because it means that the only
928	 * values on the bus after this are based on the user's registers and
929	 * potentially the addresses where we stored them. Given the constraints
930	 * of sysret, that's how it has to be.
931	 */
932	call	x86_md_clear
933
934	movl	REGOFF_RSP(%rsp), %esp
935
936	ASSERT_UPCALL_MASK_IS_SET
937        ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
938	jmp	tr_sysretl
939        SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
940	/*NOTREACHED*/
941
942_full_syscall_postsys32:
943	STI
944	/*
945	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
946	 * so that we can account for the extra work it takes us to finish.
947	 */
948	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
949	movq	%r15, %rdi
950	movq	%r12, %rsi			/* rval1 - %eax */
951	movq	%r13, %rdx			/* rval2 - %edx */
952	call	syscall_exit
953	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
954	jmp	_sys_rtt
955	SET_SIZE(sys_syscall32)
956	SET_SIZE(brand_sys_syscall32)
957
958/*
959 * System call handler via the sysenter instruction
960 * Used only for 32-bit system calls on the 64-bit kernel.
961 *
962 * The caller in userland has arranged that:
963 *
964 * -	%eax contains the syscall number
965 * -	%ecx contains the user %esp
966 * -	%edx contains the return %eip
967 * -	the user stack contains the args to the syscall
968 *
969 * Hardware and (privileged) initialization code have arranged that by
970 * the time the sysenter instructions completes:
971 *
972 * - %rip is pointing to sys_sysenter (below).
973 * - %cs and %ss are set to kernel text and stack (data) selectors.
974 * - %rsp is pointing at the lwp's stack
975 * - interrupts have been disabled.
976 *
977 * Note that we are unable to return both "rvals" to userland with
978 * this call, as %edx is used by the sysexit instruction.
979 *
980 * One final complication in this routine is its interaction with
981 * single-stepping in a debugger.  For most of the system call mechanisms, the
982 * CPU automatically clears the single-step flag before we enter the kernel.
983 * The sysenter mechanism does not clear the flag, so a user single-stepping
984 * through a libc routine may suddenly find themself single-stepping through the
985 * kernel.  To detect this, kmdb and trap() both compare the trap %pc to the
986 * [brand_]sys_enter addresses on each single-step trap.  If it finds that we
987 * have single-stepped to a sysenter entry point, it explicitly clears the flag
988 * and executes the sys_sysenter routine.
989 *
990 * One final complication in this final complication is the fact that we have
991 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
992 * If we enter at brand_sys_sysenter and start single-stepping through the
993 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
994 * kmdb cannot distinguish between that valid single-step and the undesirable
995 * one mentioned above.  To avoid this situation, we simply add a jump over the
996 * instruction at sys_sysenter to make it impossible to single-step to it.
997 */
998
999	ENTRY_NP(brand_sys_sysenter)
1000	SWAPGS				/* kernel gsbase */
1001	ALTENTRY(_brand_sys_sysenter_post_swapgs)
1002
1003	BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1004	/*
1005	 * Jump over sys_sysenter to allow single-stepping as described
1006	 * above.
1007	 */
1008	jmp	_sys_sysenter_post_swapgs
1009
1010	ALTENTRY(sys_sysenter)
1011	SWAPGS				/* kernel gsbase */
1012	ALTENTRY(_sys_sysenter_post_swapgs)
1013
1014	movq	%gs:CPU_THREAD, %r15
1015
1016	movl	$U32CS_SEL, REGOFF_CS(%rsp)
1017	movl	%ecx, REGOFF_RSP(%rsp)		/* wrapper: %esp -> %ecx */
1018	movl	%edx, REGOFF_RIP(%rsp)		/* wrapper: %eip -> %edx */
1019	/*
1020	 * NOTE: none of the instructions that run before we get here should
1021	 * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1022	 */
1023	pushfq
1024	popq	%r10
1025	movl	$UDS_SEL, REGOFF_SS(%rsp)
1026
1027	/*
1028	 * Set the interrupt flag before storing the flags to the
1029	 * flags image on the stack so we can return to user with
1030	 * interrupts enabled if we return via sys_rtt_syscall32
1031	 */
1032	orq	$PS_IE, %r10
1033	movq	%r10, REGOFF_RFL(%rsp)
1034
1035	movl	%edi, REGOFF_RDI(%rsp)
1036	movl	%esi, REGOFF_RSI(%rsp)
1037	movl	%ebp, REGOFF_RBP(%rsp)
1038	movl	%ebx, REGOFF_RBX(%rsp)
1039	movl	%edx, REGOFF_RDX(%rsp)
1040	movl	%ecx, REGOFF_RCX(%rsp)
1041	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
1042	movq	$0, REGOFF_SAVFP(%rsp)
1043	movq	$0, REGOFF_SAVPC(%rsp)
1044
1045	/*
1046	 * Copy these registers here in case we end up stopped with
1047	 * someone (like, say, /proc) messing with our register state.
1048	 * We don't -restore- them unless we have to in update_sregs.
1049	 *
1050	 * Since userland -can't- change fsbase or gsbase directly,
1051	 * we don't bother to capture them here.
1052	 */
1053	xorl	%ebx, %ebx
1054	movw	%ds, %bx
1055	movq	%rbx, REGOFF_DS(%rsp)
1056	movw	%es, %bx
1057	movq	%rbx, REGOFF_ES(%rsp)
1058	movw	%fs, %bx
1059	movq	%rbx, REGOFF_FS(%rsp)
1060	movw	%gs, %bx
1061	movq	%rbx, REGOFF_GS(%rsp)
1062
1063	/*
1064	 * If we're trying to use TRAPTRACE though, I take that back: we're
1065	 * probably debugging some problem in the SWAPGS logic and want to know
1066	 * what the incoming gsbase was.
1067	 *
1068	 * Since we already did SWAPGS, record the KGSBASE.
1069	 */
1070#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1071	movl	$MSR_AMD_KGSBASE, %ecx
1072	rdmsr
1073	movl	%eax, REGOFF_GSBASE(%rsp)
1074	movl	%edx, REGOFF_GSBASE+4(%rsp)
1075#endif
1076
1077	/*
1078	 * Application state saved in the regs structure on the stack
1079	 * %eax is the syscall number
1080	 * %rsp is the thread's stack, %r15 is curthread
1081	 * REG_RSP(%rsp) is the user's stack
1082	 */
1083
1084	SYSCALL_TRAPTRACE($TT_SYSENTER)
1085
1086	movq	%rsp, %rbp
1087
1088	movq	T_LWP(%r15), %r14
1089	ASSERT_NO_RUPDATE_PENDING(%r14)
1090
1091	ENABLE_INTR_FLAGS
1092
1093	/*
1094	 * Catch 64-bit process trying to issue sysenter instruction
1095	 * on Nocona based systems.
1096	 */
1097	movq	LWP_PROCP(%r14), %rax
1098	cmpq	$DATAMODEL_ILP32, P_MODEL(%rax)
1099	je	7f
1100
1101	/*
1102	 * For a non-32-bit process, simulate a #ud, since that's what
1103	 * native hardware does.  The traptrace entry (above) will
1104	 * let you know what really happened.
1105	 */
1106	movq	$T_ILLINST, REGOFF_TRAPNO(%rsp)
1107	movq	REGOFF_CS(%rsp), %rdi
1108	movq	%rdi, REGOFF_ERR(%rsp)
1109	movq	%rsp, %rdi
1110	movq	REGOFF_RIP(%rsp), %rsi
1111	movl	%gs:CPU_ID, %edx
1112	call	trap
1113	jmp	_sys_rtt
11147:
1115
1116	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1117	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate calls) */
1118
1119	ASSERT_LWPTOREGS(%r14, %rsp)
1120
1121	incq	%gs:CPU_STATS_SYS_SYSCALL
1122
1123	/*
1124	 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1125	 * placed into 64-bit (long) arg slots, plus one 64-bit
1126	 * (long) arg count, maintaining 16 byte alignment.
1127	 */
1128	subq	$SYS_DROP, %rsp
1129	movb	$LWP_SYS, LWP_STATE(%r14)
1130	movq	%r15, %rdi
1131	movq	%rsp, %rsi
1132	call	syscall_entry
1133
1134	/*
1135	 * Fetch the arguments copied onto the kernel stack and put
1136	 * them in the right registers to invoke a C-style syscall handler.
1137	 * %rax contains the handler address.
1138	 */
1139	movq	%rax, %rbx
1140	movl	0(%rsp), %edi
1141	movl	8(%rsp), %esi
1142	movl	0x10(%rsp), %edx
1143	movl	0x18(%rsp), %ecx
1144	movl	0x20(%rsp), %r8d
1145	movl	0x28(%rsp), %r9d
1146
1147	movq	SY_CALLC(%rbx), %rax
1148	INDIRECT_CALL_REG(rax)
1149
1150	movq	%rbp, %rsp	/* pop the args */
1151
1152	/*
1153	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1154	 * On the 32-bit kernel, the always return that value in %eax:%edx
1155	 * as required by the 32-bit ABI.
1156	 *
1157	 * Simulate the same behaviour by unconditionally splitting the
1158	 * return value in the same way.
1159	 */
1160	movq	%rax, %r13
1161	shrq	$32, %r13	/* upper 32-bits into %edx */
1162	movl	%eax, %r12d	/* lower 32-bits into %eax */
1163
1164	/*
1165	 * Optimistically assume that there's no post-syscall
1166	 * work to do.  (This is to avoid having to call syscall_mstate()
1167	 * with interrupts disabled)
1168	 */
1169	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1170
1171	/*
1172	 * We must protect ourselves from being descheduled here;
1173	 * If we were, and we ended up on another cpu, or another
1174	 * lwp got int ahead of us, it could change the segment
1175	 * registers without us noticing before we return to userland.
1176	 *
1177	 * This cli is undone in the tr_sysexit trampoline code.
1178	 */
1179	cli
1180	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1181	jne	_full_syscall_postsys32
1182	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1183
1184	/*
1185	 * To get back to userland, load up the 32-bit registers and
1186	 * sysexit back where we came from.
1187	 */
1188
1189	/*
1190	 * Interrupts will be turned on by the 'sti' executed just before
1191	 * sysexit.  The following ensures that restoring the user's rflags
1192	 * doesn't enable interrupts too soon.
1193	 */
1194	andq	$_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1195
1196	/*
1197	 * Clobber %r11 as we check CR0.TS.
1198	 */
1199	ASSERT_CR0TS_ZERO(%r11)
1200
1201	/*
1202	 * (There's no point in loading up %edx because the sysexit
1203	 * mechanism smashes it.)
1204	 */
1205	movl	%r12d, %eax
1206	movl	REGOFF_RBX(%rsp), %ebx
1207	movl	REGOFF_RBP(%rsp), %ebp
1208	movl	REGOFF_RSI(%rsp), %esi
1209	movl	REGOFF_RDI(%rsp), %edi
1210
1211	movl	REGOFF_RIP(%rsp), %edx	/* sysexit: %edx -> %eip */
1212	pushq	REGOFF_RFL(%rsp)
1213	popfq
1214	movl	REGOFF_RSP(%rsp), %ecx	/* sysexit: %ecx -> %esp */
1215        ALTENTRY(sys_sysenter_swapgs_sysexit)
1216	call	x86_md_clear
1217	jmp	tr_sysexit
1218	SET_SIZE(sys_sysenter_swapgs_sysexit)
1219	SET_SIZE(sys_sysenter)
1220	SET_SIZE(_sys_sysenter_post_swapgs)
1221	SET_SIZE(brand_sys_sysenter)
1222
1223/*
1224 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1225 * the generic i386 libc to do system calls. We do a small amount of setup
1226 * before jumping into the existing sys_syscall32 path.
1227 */
1228
1229	ENTRY_NP(brand_sys_syscall_int)
1230	SWAPGS				/* kernel gsbase */
1231	XPV_TRAP_POP
1232	call	smap_enable
1233	BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1234	jmp	nopop_syscall_int
1235
1236	ALTENTRY(sys_syscall_int)
1237	SWAPGS				/* kernel gsbase */
1238	XPV_TRAP_POP
1239	call	smap_enable
1240
1241nopop_syscall_int:
1242	movq	%gs:CPU_THREAD, %r15
1243	movq	T_STACK(%r15), %rsp
1244	movl	%eax, %eax
1245	/*
1246	 * Set t_post_sys on this thread to force ourselves out via the slow
1247	 * path. It might be possible at some later date to optimize this out
1248	 * and use a faster return mechanism.
1249	 */
1250	movb	$1, T_POST_SYS(%r15)
1251	CLEAN_CS
1252	jmp	_syscall32_save
1253	/*
1254	 * There should be no instructions between this label and SWAPGS/IRET
1255	 * or we could end up breaking branded zone support. See the usage of
1256	 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1257	 * for examples.
1258	 *
1259	 * We want to swapgs to maintain the invariant that all entries into
1260	 * tr_iret_user are done on the user gsbase.
1261	 */
1262	ALTENTRY(sys_sysint_swapgs_iret)
1263	call	x86_md_clear
1264	SWAPGS
1265	jmp	tr_iret_user
1266	/*NOTREACHED*/
1267	SET_SIZE(sys_sysint_swapgs_iret)
1268	SET_SIZE(sys_syscall_int)
1269	SET_SIZE(brand_sys_syscall_int)
1270
1271/*
1272 * Legacy 32-bit applications and old libc implementations do lcalls;
1273 * we should never get here because the LDT entry containing the syscall
1274 * segment descriptor has the "segment present" bit cleared, which means
1275 * we end up processing those system calls in trap() via a not-present trap.
1276 *
1277 * We do it this way because a call gate unhelpfully does -nothing- to the
1278 * interrupt flag bit, so an interrupt can run us just after the lcall
1279 * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
1280 * INTR_POP paths would have to be slightly more complex to dance around
1281 * this problem, and end up depending explicitly on the first
1282 * instruction of this handler being either swapgs or cli.
1283 */
1284
1285	ENTRY_NP(sys_lcall32)
1286	SWAPGS				/* kernel gsbase */
1287	pushq	$0
1288	pushq	%rbp
1289	movq	%rsp, %rbp
1290	leaq	__lcall_panic_str(%rip), %rdi
1291	xorl	%eax, %eax
1292	call	panic
1293	SET_SIZE(sys_lcall32)
1294
1295__lcall_panic_str:
1296	.string	"sys_lcall32: shouldn't be here!"
1297
1298/*
1299 * Declare a uintptr_t which covers the entire pc range of syscall
1300 * handlers for the stack walkers that need this.
1301 */
1302	.align	CPTRSIZE
1303	.globl	_allsyscalls_size
1304	.type	_allsyscalls_size, @object
1305_allsyscalls_size:
1306	.NWORD	. - _allsyscalls
1307	SET_SIZE(_allsyscalls_size)
1308
1309/*
1310 * These are the thread context handlers for lwps using sysenter/sysexit.
1311 */
1312
1313	/*
1314	 * setting this value to zero as we switch away causes the
1315	 * stack-pointer-on-sysenter to be NULL, ensuring that we
1316	 * don't silently corrupt another (preempted) thread stack
1317	 * when running an lwp that (somehow) didn't get sep_restore'd
1318	 */
1319	ENTRY_NP(sep_save)
1320	xorl	%edx, %edx
1321	xorl	%eax, %eax
1322	movl	$MSR_INTC_SEP_ESP, %ecx
1323	wrmsr
1324	ret
1325	SET_SIZE(sep_save)
1326
1327	/*
1328	 * Update the kernel stack pointer as we resume onto this cpu.
1329	 */
1330	ENTRY_NP(sep_restore)
1331	movq	%rdi, %rdx
1332	shrq	$32, %rdx
1333	movl	%edi, %eax
1334	movl	$MSR_INTC_SEP_ESP, %ecx
1335	wrmsr
1336	ret
1337	SET_SIZE(sep_restore)
1338