xref: /titanic_44/usr/src/uts/i86pc/ml/syscall_asm_amd64.s (revision a60349c89adffc0902b2353230891d8e7f2b24d9)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
24 */
25
26#include <sys/asm_linkage.h>
27#include <sys/asm_misc.h>
28#include <sys/regset.h>
29#include <sys/privregs.h>
30#include <sys/psw.h>
31#include <sys/machbrand.h>
32
33#if defined(__lint)
34
35#include <sys/types.h>
36#include <sys/thread.h>
37#include <sys/systm.h>
38
39#else	/* __lint */
40
41#include <sys/segments.h>
42#include <sys/pcb.h>
43#include <sys/trap.h>
44#include <sys/ftrace.h>
45#include <sys/traptrace.h>
46#include <sys/clock.h>
47#include <sys/model.h>
48#include <sys/panic.h>
49
50#if defined(__xpv)
51#include <sys/hypervisor.h>
52#endif
53
54#include "assym.h"
55
56#endif	/* __lint */
57
58/*
59 * We implement five flavours of system call entry points
60 *
61 * -	syscall/sysretq		(amd64 generic)
62 * -	syscall/sysretl		(i386 plus SYSC bit)
63 * -	sysenter/sysexit	(i386 plus SEP bit)
64 * -	int/iret		(i386 generic)
65 * -	lcall/iret		(i386 generic)
66 *
67 * The current libc included in Solaris uses int/iret as the base unoptimized
68 * kernel entry method. Older libc implementations and legacy binaries may use
69 * the lcall call gate, so it must continue to be supported.
70 *
71 * System calls that use an lcall call gate are processed in trap() via a
72 * segment-not-present trap, i.e. lcalls are extremely slow(!).
73 *
74 * The basic pattern used in the 32-bit SYSC handler at this point in time is
75 * to have the bare minimum of assembler, and get to the C handlers as
76 * quickly as possible.
77 *
78 * The 64-bit handler is much closer to the sparcv9 handler; that's
79 * because of passing arguments in registers.  The 32-bit world still
80 * passes arguments on the stack -- that makes that handler substantially
81 * more complex.
82 *
83 * The two handlers share a few code fragments which are broken
84 * out into preprocessor macros below.
85 *
86 * XX64	come back and speed all this up later.  The 32-bit stuff looks
87 * especially easy to speed up the argument copying part ..
88 *
89 *
90 * Notes about segment register usage (c.f. the 32-bit kernel)
91 *
92 * In the 32-bit kernel, segment registers are dutifully saved and
93 * restored on all mode transitions because the kernel uses them directly.
94 * When the processor is running in 64-bit mode, segment registers are
95 * largely ignored.
96 *
97 * %cs and %ss
98 *	controlled by the hardware mechanisms that make mode transitions
99 *
100 * The remaining segment registers have to either be pointing at a valid
101 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
102 *
103 * %ds and %es
104 *	always ignored
105 *
106 * %fs and %gs
107 *	fsbase and gsbase are used to control the place they really point at.
108 *	The kernel only depends on %gs, and controls its own gsbase via swapgs
109 *
110 * Note that loading segment registers is still costly because the GDT
111 * lookup still happens (this is because the hardware can't know that we're
112 * not setting up these segment registers for a 32-bit program).  Thus we
113 * avoid doing this in the syscall path, and defer them to lwp context switch
114 * handlers, so the register values remain virtualized to the lwp.
115 */
116
117#if defined(SYSCALLTRACE)
118#define	ORL_SYSCALLTRACE(r32)		\
119	orl	syscalltrace(%rip), r32
120#else
121#define	ORL_SYSCALLTRACE(r32)
122#endif
123
124/*
125 * In the 32-bit kernel, we do absolutely nothing before getting into the
126 * brand callback checks.  In 64-bit land, we do swapgs and then come here.
127 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
128 * are still unused.
129 *
130 * Check if a brand_mach_ops callback is defined for the specified callback_id
131 * type.  If so invoke it with the kernel's %gs value loaded and the following
132 * data on the stack:
133 *
134 * stack:  --------------------------------------
135 *      32 | callback pointer			|
136 *    | 24 | user (or interrupt) stack pointer	|
137 *    | 16 | lwp pointer			|
138 *    v  8 | userland return address		|
139 *       0 | callback wrapper return addr	|
140 *         --------------------------------------
141 *
142 * Since we're pushing the userland return address onto the kernel stack
143 * we need to get that address without accessing the user's stack (since we
144 * can't trust that data).  There are different ways to get the userland
145 * return address depending on how the syscall trap was made:
146 *
147 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
148 * b) For sys_sysenter the return address is in %rdx.
149 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
150 *    the stack pointer points at the state saved when we took the interrupt:
151 *	 ------------------------
152 *    |  | user's %ss		|
153 *    |  | user's %esp		|
154 *    |  | EFLAGS register	|
155 *    v  | user's %cs		|
156 *       | user's %eip		|
157 *	 ------------------------
158 *
159 * The 2nd parameter to the BRAND_CALLBACK macro is either the
160 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
161 * used to generate the proper code to get the userland return address for
162 * each syscall entry point.
163 *
164 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
165 * is available as a scratch register within the callback.  If the callback
166 * returns within the kernel then this macro will restore %r15.  If the
167 * callback is going to return directly to userland then it should restore
168 * %r15 before returning to userland.
169 */
170#define BRAND_URET_FROM_REG(rip_reg)					\
171	pushq	rip_reg			/* push the return address	*/
172
173/*
174 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
175 * is currently pointing at the user return address (%eip).
176 */
177#define BRAND_URET_FROM_INTR_STACK()					\
178	movq	%gs:CPU_RTMP_RSP, %r15	/* grab the intr. stack pointer	*/ ;\
179	pushq	(%r15)			/* push the return address	*/
180
181#define	BRAND_CALLBACK(callback_id, push_userland_ret)			    \
182	movq	%rsp, %gs:CPU_RTMP_RSP	/* save the stack pointer	*/ ;\
183	movq	%r15, %gs:CPU_RTMP_R15	/* save %r15			*/ ;\
184	movq	%gs:CPU_THREAD, %r15	/* load the thread pointer	*/ ;\
185	movq	T_STACK(%r15), %rsp	/* switch to the kernel stack	*/ ;\
186	subq	$16, %rsp		/* save space for 2 pointers	*/ ;\
187	pushq	%r14			/* save %r14			*/ ;\
188	movq	%gs:CPU_RTMP_RSP, %r14					   ;\
189	movq	%r14, 8(%rsp)		/* stash the user stack pointer	*/ ;\
190	popq	%r14			/* restore %r14			*/ ;\
191	movq	T_LWP(%r15), %r15	/* load the lwp pointer		*/ ;\
192	pushq	%r15			/* push the lwp pointer		*/ ;\
193	movq	LWP_PROCP(%r15), %r15	/* load the proc pointer	*/ ;\
194	movq	P_BRAND(%r15), %r15	/* load the brand pointer	*/ ;\
195	movq	B_MACHOPS(%r15), %r15	/* load the machops pointer	*/ ;\
196	movq	_CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15		   ;\
197	cmpq	$0, %r15						   ;\
198	je	1f							   ;\
199	movq	%r15, 16(%rsp)		/* save the callback pointer	*/ ;\
200	push_userland_ret		/* push the return address	*/ ;\
201	call	*24(%rsp)		/* call callback		*/ ;\
2021:	movq	%gs:CPU_RTMP_R15, %r15	/* restore %r15			*/ ;\
203	movq	%gs:CPU_RTMP_RSP, %rsp	/* restore the stack pointer	*/
204
205#define	MSTATE_TRANSITION(from, to)		\
206	movl	$from, %edi;			\
207	movl	$to, %esi;			\
208	call	syscall_mstate
209
210/*
211 * Check to see if a simple (direct) return is possible i.e.
212 *
213 *	if (t->t_post_sys_ast | syscalltrace |
214 *	    lwp->lwp_pcb.pcb_rupdate == 1)
215 *		do full version	;
216 *
217 * Preconditions:
218 * -	t is curthread
219 * Postconditions:
220 * -	condition code NE is set if post-sys is too complex
221 * -	rtmp is zeroed if it isn't (we rely on this!)
222 * -	ltmp is smashed
223 */
224#define	CHECK_POSTSYS_NE(t, ltmp, rtmp)			\
225	movq	T_LWP(t), ltmp;				\
226	movzbl	PCB_RUPDATE(ltmp), rtmp;		\
227	ORL_SYSCALLTRACE(rtmp);				\
228	orl	T_POST_SYS_AST(t), rtmp;		\
229	cmpl	$0, rtmp
230
231/*
232 * Fix up the lwp, thread, and eflags for a successful return
233 *
234 * Preconditions:
235 * -	zwreg contains zero
236 */
237#define	SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)		\
238	movb	$LWP_USER, LWP_STATE(lwp);		\
239	movw	zwreg, T_SYSNUM(t);			\
240	andb	$_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
241
242/*
243 * ASSERT(lwptoregs(lwp) == rp);
244 *
245 * This may seem obvious, but very odd things happen if this
246 * assertion is false
247 *
248 * Preconditions:
249 *	(%rsp is ready for normal call sequence)
250 * Postconditions (if assertion is true):
251 *	%r11 is smashed
252 *
253 * ASSERT(rp->r_cs == descnum)
254 *
255 * The code selector is written into the regs structure when the
256 * lwp stack is created.  We use this ASSERT to validate that
257 * the regs structure really matches how we came in.
258 *
259 * Preconditions:
260 *	(%rsp is ready for normal call sequence)
261 * Postconditions (if assertion is true):
262 *	-none-
263 *
264 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
265 *
266 * If this is false, it meant that we returned to userland without
267 * updating the segment registers as we were supposed to.
268 *
269 * Note that we must ensure no interrupts or other traps intervene
270 * between entering privileged mode and performing the assertion,
271 * otherwise we may perform a context switch on the thread, which
272 * will end up setting pcb_rupdate to 1 again.
273 */
274#if defined(DEBUG)
275
276#if !defined(__lint)
277
278__lwptoregs_msg:
279	.string	"syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
280
281__codesel_msg:
282	.string	"syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
283
284__no_rupdate_msg:
285	.string	"syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
286
287#endif	/* !__lint */
288
289#define	ASSERT_LWPTOREGS(lwp, rp)			\
290	movq	LWP_REGS(lwp), %r11;			\
291	cmpq	rp, %r11;				\
292	je	7f;					\
293	leaq	__lwptoregs_msg(%rip), %rdi;		\
294	movl	$__LINE__, %esi;			\
295	movq	lwp, %rdx;				\
296	movq	%r11, %rcx;				\
297	movq	rp, %r8;				\
298	xorl	%eax, %eax;				\
299	call	panic;					\
3007:
301
302#define	ASSERT_NO_RUPDATE_PENDING(lwp)			\
303	testb	$0x1, PCB_RUPDATE(lwp);			\
304	je	8f;					\
305	movq	lwp, %rdx;				\
306	leaq	__no_rupdate_msg(%rip), %rdi;		\
307	movl	$__LINE__, %esi;			\
308	xorl	%eax, %eax;				\
309	call	panic;					\
3108:
311
312#else
313#define	ASSERT_LWPTOREGS(lwp, rp)
314#define	ASSERT_NO_RUPDATE_PENDING(lwp)
315#endif
316
317/*
318 * Do the traptrace thing and restore any registers we used
319 * in situ.  Assumes that %rsp is pointing at the base of
320 * the struct regs, obviously ..
321 */
322#ifdef TRAPTRACE
323#define	SYSCALL_TRAPTRACE(ttype)				\
324	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);		\
325	TRACE_REGS(%rdi, %rsp, %rbx, %rcx);			\
326	TRACE_STAMP(%rdi);	/* rdtsc clobbers %eax, %edx */	\
327	movq	REGOFF_RAX(%rsp), %rax;				\
328	movq	REGOFF_RBX(%rsp), %rbx;				\
329	movq	REGOFF_RCX(%rsp), %rcx;				\
330	movq	REGOFF_RDX(%rsp), %rdx;				\
331	movl	%eax, TTR_SYSNUM(%rdi);				\
332	movq	REGOFF_RDI(%rsp), %rdi
333
334#define	SYSCALL_TRAPTRACE32(ttype)				\
335	SYSCALL_TRAPTRACE(ttype);				\
336	/* paranoia: clean the top 32-bits of the registers */	\
337	orl	%eax, %eax;					\
338	orl	%ebx, %ebx;					\
339	orl	%ecx, %ecx;					\
340	orl	%edx, %edx;					\
341	orl	%edi, %edi
342#else	/* TRAPTRACE */
343#define	SYSCALL_TRAPTRACE(ttype)
344#define	SYSCALL_TRAPTRACE32(ttype)
345#endif	/* TRAPTRACE */
346
347/*
348 * The 64-bit libc syscall wrapper does this:
349 *
350 * fn(<args>)
351 * {
352 *	movq	%rcx, %r10	-- because syscall smashes %rcx
353 *	movl	$CODE, %eax
354 *	syscall
355 *	<error processing>
356 * }
357 *
358 * Thus when we come into the kernel:
359 *
360 *	%rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
361 *	%rax is the syscall number
362 *	%r12-%r15 contain caller state
363 *
364 * The syscall instruction arranges that:
365 *
366 *	%rcx contains the return %rip
367 *	%r11d contains bottom 32-bits of %rflags
368 *	%rflags is masked (as determined by the SFMASK msr)
369 *	%cs is set to UCS_SEL (as determined by the STAR msr)
370 *	%ss is set to UDS_SEL (as determined by the STAR msr)
371 *	%rip is set to sys_syscall (as determined by the LSTAR msr)
372 *
373 * Or in other words, we have no registers available at all.
374 * Only swapgs can save us!
375 *
376 * Under the hypervisor, the swapgs has happened already.  However, the
377 * state of the world is very different from that we're familiar with.
378 *
379 * In particular, we have a stack structure like that for interrupt
380 * gates, except that the %cs and %ss registers are modified for reasons
381 * that are not entirely clear.  Critically, the %rcx/%r11 values do
382 * *not* reflect the usage of those registers under a 'real' syscall[1];
383 * the stack, therefore, looks like this:
384 *
385 *	0x0(rsp)	potentially junk %rcx
386 *	0x8(rsp)	potentially junk %r11
387 *	0x10(rsp)	user %rip
388 *	0x18(rsp)	modified %cs
389 *	0x20(rsp)	user %rflags
390 *	0x28(rsp)	user %rsp
391 *	0x30(rsp)	modified %ss
392 *
393 *
394 * and before continuing on, we must load the %rip into %rcx and the
395 * %rflags into %r11.
396 *
397 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
398 * Sigh.
399 */
400#if defined(__xpv)
401#define	XPV_SYSCALL_PROD						\
402	movq	0x10(%rsp), %rcx;					\
403	movq	0x20(%rsp), %r11;					\
404	movq	0x28(%rsp), %rsp
405#else
406#define	XPV_SYSCALL_PROD /* nothing */
407#endif
408
409#if defined(__lint)
410
411/*ARGSUSED*/
412void
413sys_syscall()
414{}
415
416void
417_allsyscalls()
418{}
419
420size_t _allsyscalls_size;
421
422#else	/* __lint */
423
424	ENTRY_NP2(brand_sys_syscall,_allsyscalls)
425	SWAPGS				/* kernel gsbase */
426	XPV_SYSCALL_PROD
427	BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
428	jmp	noprod_sys_syscall
429
430	ALTENTRY(sys_syscall)
431	SWAPGS				/* kernel gsbase */
432	XPV_SYSCALL_PROD
433
434noprod_sys_syscall:
435	movq	%r15, %gs:CPU_RTMP_R15
436	movq	%rsp, %gs:CPU_RTMP_RSP
437
438	movq	%gs:CPU_THREAD, %r15
439	movq	T_STACK(%r15), %rsp	/* switch from user to kernel stack */
440
441	ASSERT_UPCALL_MASK_IS_SET
442
443	movl	$UCS_SEL, REGOFF_CS(%rsp)
444	movq	%rcx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
445	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
446	movl	$UDS_SEL, REGOFF_SS(%rsp)
447
448	movl	%eax, %eax			/* wrapper: sysc# -> %eax */
449	movq	%rdi, REGOFF_RDI(%rsp)
450	movq	%rsi, REGOFF_RSI(%rsp)
451	movq	%rdx, REGOFF_RDX(%rsp)
452	movq	%r10, REGOFF_RCX(%rsp)		/* wrapper: %rcx -> %r10 */
453	movq	%r10, %rcx			/* arg[3] for direct calls */
454
455	movq	%r8, REGOFF_R8(%rsp)
456	movq	%r9, REGOFF_R9(%rsp)
457	movq	%rax, REGOFF_RAX(%rsp)
458	movq	%rbx, REGOFF_RBX(%rsp)
459
460	movq	%rbp, REGOFF_RBP(%rsp)
461	movq	%r10, REGOFF_R10(%rsp)
462	movq	%gs:CPU_RTMP_RSP, %r11
463	movq	%r11, REGOFF_RSP(%rsp)
464	movq	%r12, REGOFF_R12(%rsp)
465
466	movq	%r13, REGOFF_R13(%rsp)
467	movq	%r14, REGOFF_R14(%rsp)
468	movq	%gs:CPU_RTMP_R15, %r10
469	movq	%r10, REGOFF_R15(%rsp)
470	movq	$0, REGOFF_SAVFP(%rsp)
471	movq	$0, REGOFF_SAVPC(%rsp)
472
473	/*
474	 * Copy these registers here in case we end up stopped with
475	 * someone (like, say, /proc) messing with our register state.
476	 * We don't -restore- them unless we have to in update_sregs.
477	 *
478	 * Since userland -can't- change fsbase or gsbase directly,
479	 * and capturing them involves two serializing instructions,
480	 * we don't bother to capture them here.
481	 */
482	xorl	%ebx, %ebx
483	movw	%ds, %bx
484	movq	%rbx, REGOFF_DS(%rsp)
485	movw	%es, %bx
486	movq	%rbx, REGOFF_ES(%rsp)
487	movw	%fs, %bx
488	movq	%rbx, REGOFF_FS(%rsp)
489	movw	%gs, %bx
490	movq	%rbx, REGOFF_GS(%rsp)
491
492	/*
493	 * Machine state saved in the regs structure on the stack
494	 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
495	 * %eax is the syscall number
496	 * %rsp is the thread's stack, %r15 is curthread
497	 * REG_RSP(%rsp) is the user's stack
498	 */
499
500	SYSCALL_TRAPTRACE($TT_SYSC64)
501
502	movq	%rsp, %rbp
503
504	movq	T_LWP(%r15), %r14
505	ASSERT_NO_RUPDATE_PENDING(%r14)
506	ENABLE_INTR_FLAGS
507
508	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
509	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */
510
511	ASSERT_LWPTOREGS(%r14, %rsp)
512
513	movb	$LWP_SYS, LWP_STATE(%r14)
514	incq	LWP_RU_SYSC(%r14)
515	movb	$NORMALRETURN, LWP_EOSYS(%r14)
516
517	incq	%gs:CPU_STATS_SYS_SYSCALL
518
519	movw	%ax, T_SYSNUM(%r15)
520	movzbl	T_PRE_SYS(%r15), %ebx
521	ORL_SYSCALLTRACE(%ebx)
522	testl	%ebx, %ebx
523	jne	_syscall_pre
524
525_syscall_invoke:
526	movq	REGOFF_RDI(%rbp), %rdi
527	movq	REGOFF_RSI(%rbp), %rsi
528	movq	REGOFF_RDX(%rbp), %rdx
529	movq	REGOFF_RCX(%rbp), %rcx
530	movq	REGOFF_R8(%rbp), %r8
531	movq	REGOFF_R9(%rbp), %r9
532
533	cmpl	$NSYSCALL, %eax
534	jae	_syscall_ill
535	shll	$SYSENT_SIZE_SHIFT, %eax
536	leaq	sysent(%rax), %rbx
537
538	call	*SY_CALLC(%rbx)
539
540	movq	%rax, %r12
541	movq	%rdx, %r13
542
543	/*
544	 * If the handler returns two ints, then we need to split the
545	 * 64-bit return value into two 32-bit values.
546	 */
547	testw	$SE_32RVAL2, SY_FLAGS(%rbx)
548	je	5f
549	movq	%r12, %r13
550	shrq	$32, %r13	/* upper 32-bits into %edx */
551	movl	%r12d, %r12d	/* lower 32-bits into %eax */
5525:
553	/*
554	 * Optimistically assume that there's no post-syscall
555	 * work to do.  (This is to avoid having to call syscall_mstate()
556	 * with interrupts disabled)
557	 */
558	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
559
560	/*
561	 * We must protect ourselves from being descheduled here;
562	 * If we were, and we ended up on another cpu, or another
563	 * lwp got in ahead of us, it could change the segment
564	 * registers without us noticing before we return to userland.
565	 */
566	CLI(%r14)
567	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
568	jne	_syscall_post
569
570	/*
571	 * We need to protect ourselves against non-canonical return values
572	 * because Intel doesn't check for them on sysret (AMD does).  Canonical
573	 * addresses on current amd64 processors only use 48-bits for VAs; an
574	 * address is canonical if all upper bits (47-63) are identical. If we
575	 * find a non-canonical %rip, we opt to go through the full
576	 * _syscall_post path which takes us into an iretq which is not
577	 * susceptible to the same problems sysret is.
578	 *
579	 * We're checking for a canonical address by first doing an arithmetic
580	 * shift. This will fill in the remaining bits with the value of bit 63.
581	 * If the address were canonical, the register would now have either all
582	 * zeroes or all ones in it. Therefore we add one (inducing overflow)
583	 * and compare against 1. A canonical address will either be zero or one
584	 * at this point, hence the use of ja.
585	 *
586	 * At this point, r12 and r13 have the return value so we can't use
587	 * those registers.
588	 */
589	movq	REGOFF_RIP(%rsp), %rcx
590	sarq	$47, %rcx
591	incq	%rcx
592	cmpq	$1, %rcx
593	ja	_syscall_post
594
595
596	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
597
598	movq	%r12, REGOFF_RAX(%rsp)
599	movq	%r13, REGOFF_RDX(%rsp)
600
601	/*
602	 * To get back to userland, we need the return %rip in %rcx and
603	 * the return %rfl in %r11d.  The sysretq instruction also arranges
604	 * to fix up %cs and %ss; everything else is our responsibility.
605	 */
606	movq	REGOFF_RDI(%rsp), %rdi
607	movq	REGOFF_RSI(%rsp), %rsi
608	movq	REGOFF_RDX(%rsp), %rdx
609	/* %rcx used to restore %rip value */
610
611	movq	REGOFF_R8(%rsp), %r8
612	movq	REGOFF_R9(%rsp), %r9
613	movq	REGOFF_RAX(%rsp), %rax
614	movq	REGOFF_RBX(%rsp), %rbx
615
616	movq	REGOFF_RBP(%rsp), %rbp
617	movq	REGOFF_R10(%rsp), %r10
618	/* %r11 used to restore %rfl value */
619	movq	REGOFF_R12(%rsp), %r12
620
621	movq	REGOFF_R13(%rsp), %r13
622	movq	REGOFF_R14(%rsp), %r14
623	movq	REGOFF_R15(%rsp), %r15
624
625	movq	REGOFF_RIP(%rsp), %rcx
626	movl	REGOFF_RFL(%rsp), %r11d
627
628#if defined(__xpv)
629	addq	$REGOFF_RIP, %rsp
630#else
631	movq	REGOFF_RSP(%rsp), %rsp
632#endif
633
634        /*
635         * There can be no instructions between the ALTENTRY below and
636	 * SYSRET or we could end up breaking brand support. See label usage
637         * in sn1_brand_syscall_callback for an example.
638         */
639	ASSERT_UPCALL_MASK_IS_SET
640#if defined(__xpv)
641	SYSRETQ
642        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
643
644	/*
645	 * We can only get here after executing a brand syscall
646	 * interposition callback handler and simply need to
647	 * "sysretq" back to userland. On the hypervisor this
648	 * involves the iret hypercall which requires us to construct
649	 * just enough of the stack needed for the hypercall.
650	 * (rip, cs, rflags, rsp, ss).
651	 */
652	movq    %rsp, %gs:CPU_RTMP_RSP		/* save user's rsp */
653	movq	%gs:CPU_THREAD, %r11
654	movq	T_STACK(%r11), %rsp
655
656	movq	%rcx, REGOFF_RIP(%rsp)
657	movl	$UCS_SEL, REGOFF_CS(%rsp)
658	movq	%gs:CPU_RTMP_RSP, %r11
659	movq	%r11, REGOFF_RSP(%rsp)
660	pushfq
661	popq	%r11				/* hypercall enables ints */
662	movq	%r11, REGOFF_RFL(%rsp)
663	movl	$UDS_SEL, REGOFF_SS(%rsp)
664	addq	$REGOFF_RIP, %rsp
665	/*
666	 * XXPV: see comment in SYSRETQ definition for future optimization
667	 *       we could take.
668	 */
669	ASSERT_UPCALL_MASK_IS_SET
670	SYSRETQ
671#else
672        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
673	SWAPGS				/* user gsbase */
674	SYSRETQ
675#endif
676        /*NOTREACHED*/
677        SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
678
679_syscall_pre:
680	call	pre_syscall
681	movl	%eax, %r12d
682	testl	%eax, %eax
683	jne	_syscall_post_call
684	/*
685	 * Didn't abort, so reload the syscall args and invoke the handler.
686	 */
687	movzwl	T_SYSNUM(%r15), %eax
688	jmp	_syscall_invoke
689
690_syscall_ill:
691	call	nosys
692	movq	%rax, %r12
693	movq	%rdx, %r13
694	jmp	_syscall_post_call
695
696_syscall_post:
697	STI
698	/*
699	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
700	 * so that we can account for the extra work it takes us to finish.
701	 */
702	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
703_syscall_post_call:
704	movq	%r12, %rdi
705	movq	%r13, %rsi
706	call	post_syscall
707	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
708	jmp	_sys_rtt
709	SET_SIZE(sys_syscall)
710	SET_SIZE(brand_sys_syscall)
711
712#endif	/* __lint */
713
714#if defined(__lint)
715
716/*ARGSUSED*/
717void
718sys_syscall32()
719{}
720
721#else	/* __lint */
722
723	ENTRY_NP(brand_sys_syscall32)
724	SWAPGS				/* kernel gsbase */
725	XPV_TRAP_POP
726	BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
727	jmp	nopop_sys_syscall32
728
729	ALTENTRY(sys_syscall32)
730	SWAPGS				/* kernel gsbase */
731	XPV_TRAP_POP
732
733nopop_sys_syscall32:
734	movl	%esp, %r10d
735	movq	%gs:CPU_THREAD, %r15
736	movq	T_STACK(%r15), %rsp
737	movl	%eax, %eax
738
739	movl	$U32CS_SEL, REGOFF_CS(%rsp)
740	movl	%ecx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
741	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
742	movq	%r10, REGOFF_RSP(%rsp)
743	movl	$UDS_SEL, REGOFF_SS(%rsp)
744
745_syscall32_save:
746	movl	%edi, REGOFF_RDI(%rsp)
747	movl	%esi, REGOFF_RSI(%rsp)
748	movl	%ebp, REGOFF_RBP(%rsp)
749	movl	%ebx, REGOFF_RBX(%rsp)
750	movl	%edx, REGOFF_RDX(%rsp)
751	movl	%ecx, REGOFF_RCX(%rsp)
752	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
753	movq	$0, REGOFF_SAVFP(%rsp)
754	movq	$0, REGOFF_SAVPC(%rsp)
755
756	/*
757	 * Copy these registers here in case we end up stopped with
758	 * someone (like, say, /proc) messing with our register state.
759	 * We don't -restore- them unless we have to in update_sregs.
760	 *
761	 * Since userland -can't- change fsbase or gsbase directly,
762	 * we don't bother to capture them here.
763	 */
764	xorl	%ebx, %ebx
765	movw	%ds, %bx
766	movq	%rbx, REGOFF_DS(%rsp)
767	movw	%es, %bx
768	movq	%rbx, REGOFF_ES(%rsp)
769	movw	%fs, %bx
770	movq	%rbx, REGOFF_FS(%rsp)
771	movw	%gs, %bx
772	movq	%rbx, REGOFF_GS(%rsp)
773
774	/*
775	 * Application state saved in the regs structure on the stack
776	 * %eax is the syscall number
777	 * %rsp is the thread's stack, %r15 is curthread
778	 * REG_RSP(%rsp) is the user's stack
779	 */
780
781	SYSCALL_TRAPTRACE32($TT_SYSC)
782
783	movq	%rsp, %rbp
784
785	movq	T_LWP(%r15), %r14
786	ASSERT_NO_RUPDATE_PENDING(%r14)
787
788	ENABLE_INTR_FLAGS
789
790	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
791	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */
792
793	ASSERT_LWPTOREGS(%r14, %rsp)
794
795	incq	 %gs:CPU_STATS_SYS_SYSCALL
796
797	/*
798	 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
799	 * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
800	 * more succinctly:
801	 *
802	 *	SA(MAXSYSARGS * sizeof (long)) == 64
803	 */
804#define	SYS_DROP	64			/* drop for args */
805	subq	$SYS_DROP, %rsp
806	movb	$LWP_SYS, LWP_STATE(%r14)
807	movq	%r15, %rdi
808	movq	%rsp, %rsi
809	call	syscall_entry
810
811	/*
812	 * Fetch the arguments copied onto the kernel stack and put
813	 * them in the right registers to invoke a C-style syscall handler.
814	 * %rax contains the handler address.
815	 *
816	 * Ideas for making all this go faster of course include simply
817	 * forcibly fetching 6 arguments from the user stack under lofault
818	 * protection, reverting to copyin_args only when watchpoints
819	 * are in effect.
820	 *
821	 * (If we do this, make sure that exec and libthread leave
822	 * enough space at the top of the stack to ensure that we'll
823	 * never do a fetch from an invalid page.)
824	 *
825	 * Lots of ideas here, but they won't really help with bringup B-)
826	 * Correctness can't wait, performance can wait a little longer ..
827	 */
828
829	movq	%rax, %rbx
830	movl	0(%rsp), %edi
831	movl	8(%rsp), %esi
832	movl	0x10(%rsp), %edx
833	movl	0x18(%rsp), %ecx
834	movl	0x20(%rsp), %r8d
835	movl	0x28(%rsp), %r9d
836
837	call	*SY_CALLC(%rbx)
838
839	movq	%rbp, %rsp	/* pop the args */
840
841	/*
842	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
843	 * On the 32-bit kernel, they always return that value in %eax:%edx
844	 * as required by the 32-bit ABI.
845	 *
846	 * Simulate the same behaviour by unconditionally splitting the
847	 * return value in the same way.
848	 */
849	movq	%rax, %r13
850	shrq	$32, %r13	/* upper 32-bits into %edx */
851	movl	%eax, %r12d	/* lower 32-bits into %eax */
852
853	/*
854	 * Optimistically assume that there's no post-syscall
855	 * work to do.  (This is to avoid having to call syscall_mstate()
856	 * with interrupts disabled)
857	 */
858	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
859
860	/*
861	 * We must protect ourselves from being descheduled here;
862	 * If we were, and we ended up on another cpu, or another
863	 * lwp got in ahead of us, it could change the segment
864	 * registers without us noticing before we return to userland.
865	 */
866	CLI(%r14)
867	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
868	jne	_full_syscall_postsys32
869	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
870
871	/*
872	 * To get back to userland, we need to put the return %rip in %rcx and
873	 * the return %rfl in %r11d.  The sysret instruction also arranges
874	 * to fix up %cs and %ss; everything else is our responsibility.
875	 */
876
877	movl	%r12d, %eax			/* %eax: rval1 */
878	movl	REGOFF_RBX(%rsp), %ebx
879	/* %ecx used for return pointer */
880	movl	%r13d, %edx			/* %edx: rval2 */
881	movl	REGOFF_RBP(%rsp), %ebp
882	movl	REGOFF_RSI(%rsp), %esi
883	movl	REGOFF_RDI(%rsp), %edi
884
885	movl	REGOFF_RFL(%rsp), %r11d		/* %r11 -> eflags */
886	movl	REGOFF_RIP(%rsp), %ecx		/* %ecx -> %eip */
887	movl	REGOFF_RSP(%rsp), %esp
888
889	ASSERT_UPCALL_MASK_IS_SET
890        ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
891	SWAPGS				/* user gsbase */
892	SYSRETL
893        SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
894	/*NOTREACHED*/
895
896_full_syscall_postsys32:
897	STI
898	/*
899	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
900	 * so that we can account for the extra work it takes us to finish.
901	 */
902	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
903	movq	%r15, %rdi
904	movq	%r12, %rsi			/* rval1 - %eax */
905	movq	%r13, %rdx			/* rval2 - %edx */
906	call	syscall_exit
907	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
908	jmp	_sys_rtt
909	SET_SIZE(sys_syscall32)
910	SET_SIZE(brand_sys_syscall32)
911
912#endif	/* __lint */
913
914/*
915 * System call handler via the sysenter instruction
916 * Used only for 32-bit system calls on the 64-bit kernel.
917 *
918 * The caller in userland has arranged that:
919 *
920 * -	%eax contains the syscall number
921 * -	%ecx contains the user %esp
922 * -	%edx contains the return %eip
923 * -	the user stack contains the args to the syscall
924 *
925 * Hardware and (privileged) initialization code have arranged that by
926 * the time the sysenter instructions completes:
927 *
928 * - %rip is pointing to sys_sysenter (below).
929 * - %cs and %ss are set to kernel text and stack (data) selectors.
930 * - %rsp is pointing at the lwp's stack
931 * - interrupts have been disabled.
932 *
933 * Note that we are unable to return both "rvals" to userland with
934 * this call, as %edx is used by the sysexit instruction.
935 *
936 * One final complication in this routine is its interaction with
937 * single-stepping in a debugger.  For most of the system call mechanisms,
938 * the CPU automatically clears the single-step flag before we enter the
939 * kernel.  The sysenter mechanism does not clear the flag, so a user
940 * single-stepping through a libc routine may suddenly find him/herself
941 * single-stepping through the kernel.  To detect this, kmdb compares the
942 * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
943 * If it finds that we have single-stepped to a sysenter entry point, it
944 * explicitly clears the flag and executes the sys_sysenter routine.
945 *
946 * One final complication in this final complication is the fact that we
947 * have two different entry points for sysenter: brand_sys_sysenter and
948 * sys_sysenter.  If we enter at brand_sys_sysenter and start single-stepping
949 * through the kernel with kmdb, we will eventually hit the instruction at
950 * sys_sysenter.  kmdb cannot distinguish between that valid single-step
951 * and the undesirable one mentioned above.  To avoid this situation, we
952 * simply add a jump over the instruction at sys_sysenter to make it
953 * impossible to single-step to it.
954 */
955#if defined(__lint)
956
957void
958sys_sysenter()
959{}
960
961#else	/* __lint */
962
963	ENTRY_NP(brand_sys_sysenter)
964	SWAPGS				/* kernel gsbase */
965	ALTENTRY(_brand_sys_sysenter_post_swapgs)
966	BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
967	/*
968	 * Jump over sys_sysenter to allow single-stepping as described
969	 * above.
970	 */
971	jmp	_sys_sysenter_post_swapgs
972
973	ALTENTRY(sys_sysenter)
974	SWAPGS				/* kernel gsbase */
975
976	ALTENTRY(_sys_sysenter_post_swapgs)
977	movq	%gs:CPU_THREAD, %r15
978
979	movl	$U32CS_SEL, REGOFF_CS(%rsp)
980	movl	%ecx, REGOFF_RSP(%rsp)		/* wrapper: %esp -> %ecx */
981	movl	%edx, REGOFF_RIP(%rsp)		/* wrapper: %eip -> %edx */
982	pushfq
983	popq	%r10
984	movl	$UDS_SEL, REGOFF_SS(%rsp)
985
986	/*
987	 * Set the interrupt flag before storing the flags to the
988	 * flags image on the stack so we can return to user with
989	 * interrupts enabled if we return via sys_rtt_syscall32
990	 */
991	orq	$PS_IE, %r10
992	movq	%r10, REGOFF_RFL(%rsp)
993
994	movl	%edi, REGOFF_RDI(%rsp)
995	movl	%esi, REGOFF_RSI(%rsp)
996	movl	%ebp, REGOFF_RBP(%rsp)
997	movl	%ebx, REGOFF_RBX(%rsp)
998	movl	%edx, REGOFF_RDX(%rsp)
999	movl	%ecx, REGOFF_RCX(%rsp)
1000	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
1001	movq	$0, REGOFF_SAVFP(%rsp)
1002	movq	$0, REGOFF_SAVPC(%rsp)
1003
1004	/*
1005	 * Copy these registers here in case we end up stopped with
1006	 * someone (like, say, /proc) messing with our register state.
1007	 * We don't -restore- them unless we have to in update_sregs.
1008	 *
1009	 * Since userland -can't- change fsbase or gsbase directly,
1010	 * we don't bother to capture them here.
1011	 */
1012	xorl	%ebx, %ebx
1013	movw	%ds, %bx
1014	movq	%rbx, REGOFF_DS(%rsp)
1015	movw	%es, %bx
1016	movq	%rbx, REGOFF_ES(%rsp)
1017	movw	%fs, %bx
1018	movq	%rbx, REGOFF_FS(%rsp)
1019	movw	%gs, %bx
1020	movq	%rbx, REGOFF_GS(%rsp)
1021
1022	/*
1023	 * Application state saved in the regs structure on the stack
1024	 * %eax is the syscall number
1025	 * %rsp is the thread's stack, %r15 is curthread
1026	 * REG_RSP(%rsp) is the user's stack
1027	 */
1028
1029	SYSCALL_TRAPTRACE($TT_SYSENTER)
1030
1031	movq	%rsp, %rbp
1032
1033	movq	T_LWP(%r15), %r14
1034	ASSERT_NO_RUPDATE_PENDING(%r14)
1035
1036	ENABLE_INTR_FLAGS
1037
1038	/*
1039	 * Catch 64-bit process trying to issue sysenter instruction
1040	 * on Nocona based systems.
1041	 */
1042	movq	LWP_PROCP(%r14), %rax
1043	cmpq	$DATAMODEL_ILP32, P_MODEL(%rax)
1044	je	7f
1045
1046	/*
1047	 * For a non-32-bit process, simulate a #ud, since that's what
1048	 * native hardware does.  The traptrace entry (above) will
1049	 * let you know what really happened.
1050	 */
1051	movq	$T_ILLINST, REGOFF_TRAPNO(%rsp)
1052	movq	REGOFF_CS(%rsp), %rdi
1053	movq	%rdi, REGOFF_ERR(%rsp)
1054	movq	%rsp, %rdi
1055	movq	REGOFF_RIP(%rsp), %rsi
1056	movl	%gs:CPU_ID, %edx
1057	call	trap
1058	jmp	_sys_rtt
10597:
1060
1061	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1062	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate calls) */
1063
1064	ASSERT_LWPTOREGS(%r14, %rsp)
1065
1066	incq	%gs:CPU_STATS_SYS_SYSCALL
1067
1068	/*
1069	 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1070	 * placed into 64-bit (long) arg slots, plus one 64-bit
1071	 * (long) arg count, maintaining 16 byte alignment.
1072	 */
1073	subq	$SYS_DROP, %rsp
1074	movb	$LWP_SYS, LWP_STATE(%r14)
1075	movq	%r15, %rdi
1076	movq	%rsp, %rsi
1077	call	syscall_entry
1078
1079	/*
1080	 * Fetch the arguments copied onto the kernel stack and put
1081	 * them in the right registers to invoke a C-style syscall handler.
1082	 * %rax contains the handler address.
1083	 */
1084	movq	%rax, %rbx
1085	movl	0(%rsp), %edi
1086	movl	8(%rsp), %esi
1087	movl	0x10(%rsp), %edx
1088	movl	0x18(%rsp), %ecx
1089	movl	0x20(%rsp), %r8d
1090	movl	0x28(%rsp), %r9d
1091
1092	call	*SY_CALLC(%rbx)
1093
1094	movq	%rbp, %rsp	/* pop the args */
1095
1096	/*
1097	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1098	 * On the 32-bit kernel, the always return that value in %eax:%edx
1099	 * as required by the 32-bit ABI.
1100	 *
1101	 * Simulate the same behaviour by unconditionally splitting the
1102	 * return value in the same way.
1103	 */
1104	movq	%rax, %r13
1105	shrq	$32, %r13	/* upper 32-bits into %edx */
1106	movl	%eax, %r12d	/* lower 32-bits into %eax */
1107
1108	/*
1109	 * Optimistically assume that there's no post-syscall
1110	 * work to do.  (This is to avoid having to call syscall_mstate()
1111	 * with interrupts disabled)
1112	 */
1113	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1114
1115	/*
1116	 * We must protect ourselves from being descheduled here;
1117	 * If we were, and we ended up on another cpu, or another
1118	 * lwp got int ahead of us, it could change the segment
1119	 * registers without us noticing before we return to userland.
1120	 */
1121	cli
1122	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1123	jne	_full_syscall_postsys32
1124	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1125
1126	/*
1127	 * To get back to userland, load up the 32-bit registers and
1128	 * sysexit back where we came from.
1129	 */
1130
1131	/*
1132	 * Interrupts will be turned on by the 'sti' executed just before
1133	 * sysexit.  The following ensures that restoring the user's rflags
1134	 * doesn't enable interrupts too soon.
1135	 */
1136	andq	$_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1137
1138	/*
1139	 * (There's no point in loading up %edx because the sysexit
1140	 * mechanism smashes it.)
1141	 */
1142	movl	%r12d, %eax
1143	movl	REGOFF_RBX(%rsp), %ebx
1144	movl	REGOFF_RBP(%rsp), %ebp
1145	movl	REGOFF_RSI(%rsp), %esi
1146	movl	REGOFF_RDI(%rsp), %edi
1147
1148	movl	REGOFF_RIP(%rsp), %edx	/* sysexit: %edx -> %eip */
1149	pushq	REGOFF_RFL(%rsp)
1150	popfq
1151	movl	REGOFF_RSP(%rsp), %ecx	/* sysexit: %ecx -> %esp */
1152        ALTENTRY(sys_sysenter_swapgs_sysexit)
1153	swapgs
1154	sti
1155	sysexit
1156	SET_SIZE(sys_sysenter_swapgs_sysexit)
1157	SET_SIZE(sys_sysenter)
1158	SET_SIZE(_sys_sysenter_post_swapgs)
1159	SET_SIZE(brand_sys_sysenter)
1160
1161#endif	/* __lint */
1162
1163/*
1164 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1165 * the generic i386 libc to do system calls. We do a small amount of setup
1166 * before jumping into the existing sys_syscall32 path.
1167 */
1168#if defined(__lint)
1169
1170/*ARGSUSED*/
1171void
1172sys_syscall_int()
1173{}
1174
1175#else	/* __lint */
1176
1177	ENTRY_NP(brand_sys_syscall_int)
1178	SWAPGS				/* kernel gsbase */
1179	XPV_TRAP_POP
1180	BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1181	jmp	nopop_syscall_int
1182
1183	ALTENTRY(sys_syscall_int)
1184	SWAPGS				/* kernel gsbase */
1185	XPV_TRAP_POP
1186
1187nopop_syscall_int:
1188	movq	%gs:CPU_THREAD, %r15
1189	movq	T_STACK(%r15), %rsp
1190	movl	%eax, %eax
1191	/*
1192	 * Set t_post_sys on this thread to force ourselves out via the slow
1193	 * path. It might be possible at some later date to optimize this out
1194	 * and use a faster return mechanism.
1195	 */
1196	movb	$1, T_POST_SYS(%r15)
1197	CLEAN_CS
1198	jmp	_syscall32_save
1199	/*
1200	 * There should be no instructions between this label and SWAPGS/IRET
1201	 * or we could end up breaking branded zone support. See the usage of
1202	 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1203	 * for examples.
1204	 */
1205        ALTENTRY(sys_sysint_swapgs_iret)
1206	SWAPGS				/* user gsbase */
1207	IRET
1208	/*NOTREACHED*/
1209	SET_SIZE(sys_sysint_swapgs_iret)
1210	SET_SIZE(sys_syscall_int)
1211	SET_SIZE(brand_sys_syscall_int)
1212
1213#endif	/* __lint */
1214
1215/*
1216 * Legacy 32-bit applications and old libc implementations do lcalls;
1217 * we should never get here because the LDT entry containing the syscall
1218 * segment descriptor has the "segment present" bit cleared, which means
1219 * we end up processing those system calls in trap() via a not-present trap.
1220 *
1221 * We do it this way because a call gate unhelpfully does -nothing- to the
1222 * interrupt flag bit, so an interrupt can run us just after the lcall
1223 * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
1224 * INTR_POP paths would have to be slightly more complex to dance around
1225 * this problem, and end up depending explicitly on the first
1226 * instruction of this handler being either swapgs or cli.
1227 */
1228
1229#if defined(__lint)
1230
1231/*ARGSUSED*/
1232void
1233sys_lcall32()
1234{}
1235
1236#else	/* __lint */
1237
1238	ENTRY_NP(sys_lcall32)
1239	SWAPGS				/* kernel gsbase */
1240	pushq	$0
1241	pushq	%rbp
1242	movq	%rsp, %rbp
1243	leaq	__lcall_panic_str(%rip), %rdi
1244	xorl	%eax, %eax
1245	call	panic
1246	SET_SIZE(sys_lcall32)
1247
1248__lcall_panic_str:
1249	.string	"sys_lcall32: shouldn't be here!"
1250
1251/*
1252 * Declare a uintptr_t which covers the entire pc range of syscall
1253 * handlers for the stack walkers that need this.
1254 */
1255	.align	CPTRSIZE
1256	.globl	_allsyscalls_size
1257	.type	_allsyscalls_size, @object
1258_allsyscalls_size:
1259	.NWORD	. - _allsyscalls
1260	SET_SIZE(_allsyscalls_size)
1261
1262#endif	/* __lint */
1263
1264/*
1265 * These are the thread context handlers for lwps using sysenter/sysexit.
1266 */
1267
1268#if defined(__lint)
1269
1270/*ARGSUSED*/
1271void
1272sep_save(void *ksp)
1273{}
1274
1275/*ARGSUSED*/
1276void
1277sep_restore(void *ksp)
1278{}
1279
1280#else	/* __lint */
1281
1282	/*
1283	 * setting this value to zero as we switch away causes the
1284	 * stack-pointer-on-sysenter to be NULL, ensuring that we
1285	 * don't silently corrupt another (preempted) thread stack
1286	 * when running an lwp that (somehow) didn't get sep_restore'd
1287	 */
1288	ENTRY_NP(sep_save)
1289	xorl	%edx, %edx
1290	xorl	%eax, %eax
1291	movl	$MSR_INTC_SEP_ESP, %ecx
1292	wrmsr
1293	ret
1294	SET_SIZE(sep_save)
1295
1296	/*
1297	 * Update the kernel stack pointer as we resume onto this cpu.
1298	 */
1299	ENTRY_NP(sep_restore)
1300	movq	%rdi, %rdx
1301	shrq	$32, %rdx
1302	movl	%edi, %eax
1303	movl	$MSR_INTC_SEP_ESP, %ecx
1304	wrmsr
1305	ret
1306	SET_SIZE(sep_restore)
1307
1308#endif	/* __lint */
1309