xref: /titanic_50/usr/src/uts/i86pc/os/trap.c (revision 9aa23310b13612fb7817c242be9055ca53e8fe9b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
28 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T   */
29 /*		All Rights Reserved   				*/
30 /*								*/
31 /*	Copyright (c) 1987, 1988 Microsoft Corporation  	*/
32 /*		All Rights Reserved   				*/
33 /*								*/
34 
35 #pragma ident	"%Z%%M%	%I%	%E% SMI"
36 
37 #include <sys/types.h>
38 #include <sys/sysmacros.h>
39 #include <sys/param.h>
40 #include <sys/signal.h>
41 #include <sys/systm.h>
42 #include <sys/user.h>
43 #include <sys/proc.h>
44 #include <sys/disp.h>
45 #include <sys/class.h>
46 #include <sys/core.h>
47 #include <sys/syscall.h>
48 #include <sys/cpuvar.h>
49 #include <sys/vm.h>
50 #include <sys/sysinfo.h>
51 #include <sys/fault.h>
52 #include <sys/stack.h>
53 #include <sys/psw.h>
54 #include <sys/regset.h>
55 #include <sys/fp.h>
56 #include <sys/trap.h>
57 #include <sys/kmem.h>
58 #include <sys/vtrace.h>
59 #include <sys/cmn_err.h>
60 #include <sys/prsystm.h>
61 #include <sys/mutex_impl.h>
62 #include <sys/machsystm.h>
63 #include <sys/archsystm.h>
64 #include <sys/sdt.h>
65 #include <sys/avintr.h>
66 #include <sys/kobj.h>
67 
68 #include <vm/hat.h>
69 
70 #include <vm/seg_kmem.h>
71 #include <vm/as.h>
72 #include <vm/seg.h>
73 #include <vm/hat_pte.h>
74 #include <vm/hat_i86.h>
75 
76 #include <sys/procfs.h>
77 
78 #include <sys/reboot.h>
79 #include <sys/debug.h>
80 #include <sys/debugreg.h>
81 #include <sys/modctl.h>
82 #include <sys/aio_impl.h>
83 #include <sys/tnf.h>
84 #include <sys/tnf_probe.h>
85 #include <sys/cred.h>
86 #include <sys/mman.h>
87 #include <sys/x86_archext.h>
88 #include <sys/copyops.h>
89 #include <c2/audit.h>
90 #include <sys/ftrace.h>
91 #include <sys/panic.h>
92 #include <sys/traptrace.h>
93 #include <sys/ontrap.h>
94 #include <sys/cpc_impl.h>
95 #include <sys/bootconf.h>
96 #include <sys/bootinfo.h>
97 #include <sys/promif.h>
98 #include <sys/mach_mmu.h>
99 #if defined(__xpv)
100 #include <sys/hypervisor.h>
101 #endif
102 #include <sys/contract/process_impl.h>
103 
104 #define	USER	0x10000		/* user-mode flag added to trap type */
105 
106 static const char *trap_type_mnemonic[] = {
107 	"de",	"db",	"2",	"bp",
108 	"of",	"br",	"ud",	"nm",
109 	"df",	"9",	"ts",	"np",
110 	"ss",	"gp",	"pf",	"15",
111 	"mf",	"ac",	"mc",	"xf"
112 };
113 
114 static const char *trap_type[] = {
115 	"Divide error",				/* trap id 0 	*/
116 	"Debug",				/* trap id 1	*/
117 	"NMI interrupt",			/* trap id 2	*/
118 	"Breakpoint",				/* trap id 3 	*/
119 	"Overflow",				/* trap id 4 	*/
120 	"BOUND range exceeded",			/* trap id 5 	*/
121 	"Invalid opcode",			/* trap id 6 	*/
122 	"Device not available",			/* trap id 7 	*/
123 	"Double fault",				/* trap id 8 	*/
124 	"Coprocessor segment overrun",		/* trap id 9 	*/
125 	"Invalid TSS",				/* trap id 10 	*/
126 	"Segment not present",			/* trap id 11 	*/
127 	"Stack segment fault",			/* trap id 12 	*/
128 	"General protection",			/* trap id 13 	*/
129 	"Page fault",				/* trap id 14 	*/
130 	"Reserved",				/* trap id 15 	*/
131 	"x87 floating point error",		/* trap id 16 	*/
132 	"Alignment check",			/* trap id 17 	*/
133 	"Machine check",			/* trap id 18	*/
134 	"SIMD floating point exception",	/* trap id 19	*/
135 };
136 
137 #define	TRAP_TYPES	(sizeof (trap_type) / sizeof (trap_type[0]))
138 
139 #define	SLOW_SCALL_SIZE	2
140 #define	FAST_SCALL_SIZE	2
141 
142 int tudebug = 0;
143 int tudebugbpt = 0;
144 int tudebugfpe = 0;
145 int tudebugsse = 0;
146 
147 #if defined(TRAPDEBUG) || defined(lint)
148 int tdebug = 0;
149 int lodebug = 0;
150 int faultdebug = 0;
151 #else
152 #define	tdebug	0
153 #define	lodebug	0
154 #define	faultdebug	0
155 #endif /* defined(TRAPDEBUG) || defined(lint) */
156 
157 #if defined(TRAPTRACE)
158 /*
159  * trap trace record for cpu0 is allocated here.
160  * trap trace records for non-boot cpus are allocated in mp_startup_init().
161  */
162 static trap_trace_rec_t trap_tr0[TRAPTR_NENT];
163 trap_trace_ctl_t trap_trace_ctl[NCPU] = {
164 	{
165 	    (uintptr_t)trap_tr0,			/* next record */
166 	    (uintptr_t)trap_tr0,			/* first record */
167 	    (uintptr_t)(trap_tr0 + TRAPTR_NENT),	/* limit */
168 	    (uintptr_t)0				/* current */
169 	},
170 };
171 
172 /*
173  * default trap buffer size
174  */
175 size_t trap_trace_bufsize = TRAPTR_NENT * sizeof (trap_trace_rec_t);
176 int trap_trace_freeze = 0;
177 int trap_trace_off = 0;
178 
179 /*
180  * A dummy TRAPTRACE entry to use after death.
181  */
182 trap_trace_rec_t trap_trace_postmort;
183 
184 static void dump_ttrace(void);
185 #endif	/* TRAPTRACE */
186 static void dumpregs(struct regs *);
187 static void showregs(uint_t, struct regs *, caddr_t);
188 static int kern_gpfault(struct regs *);
189 
190 /*ARGSUSED*/
191 static int
192 die(uint_t type, struct regs *rp, caddr_t addr, processorid_t cpuid)
193 {
194 	struct panic_trap_info ti;
195 	const char *trap_name, *trap_mnemonic;
196 
197 	if (type < TRAP_TYPES) {
198 		trap_name = trap_type[type];
199 		trap_mnemonic = trap_type_mnemonic[type];
200 	} else {
201 		trap_name = "trap";
202 		trap_mnemonic = "-";
203 	}
204 
205 #ifdef TRAPTRACE
206 	TRAPTRACE_FREEZE;
207 #endif
208 
209 	ti.trap_regs = rp;
210 	ti.trap_type = type & ~USER;
211 	ti.trap_addr = addr;
212 
213 	curthread->t_panic_trap = &ti;
214 
215 	if (type == T_PGFLT && addr < (caddr_t)KERNELBASE) {
216 		panic("BAD TRAP: type=%x (#%s %s) rp=%p addr=%p "
217 		    "occurred in module \"%s\" due to %s",
218 		    type, trap_mnemonic, trap_name, (void *)rp, (void *)addr,
219 		    mod_containing_pc((caddr_t)rp->r_pc),
220 		    addr < (caddr_t)PAGESIZE ?
221 		    "a NULL pointer dereference" :
222 		    "an illegal access to a user address");
223 	} else
224 		panic("BAD TRAP: type=%x (#%s %s) rp=%p addr=%p",
225 		    type, trap_mnemonic, trap_name, (void *)rp, (void *)addr);
226 	return (0);
227 }
228 
229 /*
230  * Rewrite the instruction at pc to be an int $T_SYSCALLINT instruction.
231  *
232  * int <vector> is two bytes: 0xCD <vector>
233  */
234 
235 static int
236 rewrite_syscall(caddr_t pc)
237 {
238 	uchar_t instr[SLOW_SCALL_SIZE] = { 0xCD, T_SYSCALLINT };
239 
240 	if (uwrite(curthread->t_procp, instr, SLOW_SCALL_SIZE,
241 	    (uintptr_t)pc) != 0)
242 		return (1);
243 
244 	return (0);
245 }
246 
247 /*
248  * Test to see if the instruction at pc is sysenter or syscall. The second
249  * argument should be the x86 feature flag corresponding to the expected
250  * instruction.
251  *
252  * sysenter is two bytes: 0x0F 0x34
253  * syscall is two bytes:  0x0F 0x05
254  * int $T_SYSCALLINT is two bytes: 0xCD 0x91
255  */
256 
257 static int
258 instr_is_other_syscall(caddr_t pc, int which)
259 {
260 	uchar_t instr[FAST_SCALL_SIZE];
261 
262 	ASSERT(which == X86_SEP || which == X86_ASYSC || which == 0xCD);
263 
264 	if (copyin_nowatch(pc, (caddr_t)instr, FAST_SCALL_SIZE) != 0)
265 		return (0);
266 
267 	switch (which) {
268 	case X86_SEP:
269 		if (instr[0] == 0x0F && instr[1] == 0x34)
270 			return (1);
271 		break;
272 	case X86_ASYSC:
273 		if (instr[0] == 0x0F && instr[1] == 0x05)
274 			return (1);
275 		break;
276 	case 0xCD:
277 		if (instr[0] == 0xCD && instr[1] == T_SYSCALLINT)
278 			return (1);
279 		break;
280 	}
281 
282 	return (0);
283 }
284 
285 static const char *
286 syscall_insn_string(int syscall_insn)
287 {
288 	switch (syscall_insn) {
289 	case X86_SEP:
290 		return ("sysenter");
291 	case X86_ASYSC:
292 		return ("syscall");
293 	case 0xCD:
294 		return ("int");
295 	default:
296 		return ("Unknown");
297 	}
298 }
299 
300 static int
301 ldt_rewrite_syscall(struct regs *rp, proc_t *p, int syscall_insn)
302 {
303 	caddr_t	linearpc;
304 	int return_code = 0;
305 
306 	mutex_enter(&p->p_ldtlock);	/* Must be held across linear_pc() */
307 
308 	if (linear_pc(rp, p, &linearpc) == 0) {
309 
310 		/*
311 		 * If another thread beat us here, it already changed
312 		 * this site to the slower (int) syscall instruction.
313 		 */
314 		if (instr_is_other_syscall(linearpc, 0xCD)) {
315 			return_code = 1;
316 		} else if (instr_is_other_syscall(linearpc, syscall_insn)) {
317 
318 			if (rewrite_syscall(linearpc) == 0) {
319 				return_code = 1;
320 			}
321 #ifdef DEBUG
322 			else
323 				cmn_err(CE_WARN, "failed to rewrite %s "
324 				    "instruction in process %d",
325 				    syscall_insn_string(syscall_insn),
326 				    p->p_pid);
327 #endif /* DEBUG */
328 		}
329 	}
330 
331 	mutex_exit(&p->p_ldtlock);	/* Must be held across linear_pc() */
332 
333 	return (return_code);
334 }
335 
336 /*
337  * Test to see if the instruction at pc is a system call instruction.
338  *
339  * The bytes of an lcall instruction used for the syscall trap.
340  * static uchar_t lcall[7] = { 0x9a, 0, 0, 0, 0, 0x7, 0 };
341  * static uchar_t lcallalt[7] = { 0x9a, 0, 0, 0, 0, 0x27, 0 };
342  */
343 
344 #define	LCALLSIZE	7
345 
346 static int
347 instr_is_lcall_syscall(caddr_t pc)
348 {
349 	uchar_t instr[LCALLSIZE];
350 
351 	if (copyin_nowatch(pc, (caddr_t)instr, LCALLSIZE) == 0 &&
352 	    instr[0] == 0x9a &&
353 	    instr[1] == 0 &&
354 	    instr[2] == 0 &&
355 	    instr[3] == 0 &&
356 	    instr[4] == 0 &&
357 	    (instr[5] == 0x7 || instr[5] == 0x27) &&
358 	    instr[6] == 0)
359 		return (1);
360 
361 	return (0);
362 }
363 
364 #ifdef __amd64
365 
366 /*
367  * In the first revisions of amd64 CPUs produced by AMD, the LAHF and
368  * SAHF instructions were not implemented in 64-bit mode. Later revisions
369  * did implement these instructions. An extension to the cpuid instruction
370  * was added to check for the capability of executing these instructions
371  * in 64-bit mode.
372  *
373  * Intel originally did not implement these instructions in EM64T either,
374  * but added them in later revisions.
375  *
376  * So, there are different chip revisions by both vendors out there that
377  * may or may not implement these instructions. The easy solution is to
378  * just always emulate these instructions on demand.
379  *
380  * SAHF == store %ah in the lower 8 bits of %rflags (opcode 0x9e)
381  * LAHF == load the lower 8 bits of %rflags into %ah (opcode 0x9f)
382  */
383 
384 #define	LSAHFSIZE 1
385 
386 static int
387 instr_is_lsahf(caddr_t pc, uchar_t *instr)
388 {
389 	if (copyin_nowatch(pc, (caddr_t)instr, LSAHFSIZE) == 0 &&
390 	    (*instr == 0x9e || *instr == 0x9f))
391 		return (1);
392 	return (0);
393 }
394 
395 /*
396  * Emulate the LAHF and SAHF instructions. The reference manuals define
397  * these instructions to always load/store bit 1 as a 1, and bits 3 and 5
398  * as a 0. The other, defined, bits are copied (the PS_ICC bits and PS_P).
399  *
400  * Note that %ah is bits 8-15 of %rax.
401  */
402 static void
403 emulate_lsahf(struct regs *rp, uchar_t instr)
404 {
405 	if (instr == 0x9e) {
406 		/* sahf. Copy bits from %ah to flags. */
407 		rp->r_ps = (rp->r_ps & ~0xff) |
408 		    ((rp->r_rax >> 8) & PSL_LSAHFMASK) | PS_MB1;
409 	} else {
410 		/* lahf. Copy bits from flags to %ah. */
411 		rp->r_rax = (rp->r_rax & ~0xff00) |
412 		    (((rp->r_ps & PSL_LSAHFMASK) | PS_MB1) << 8);
413 	}
414 	rp->r_pc += LSAHFSIZE;
415 }
416 #endif /* __amd64 */
417 
418 #ifdef OPTERON_ERRATUM_91
419 
420 /*
421  * Test to see if the instruction at pc is a prefetch instruction.
422  *
423  * The first byte of prefetch instructions is always 0x0F.
424  * The second byte is 0x18 for regular prefetch or 0x0D for AMD 3dnow prefetch.
425  * The third byte is between 0 and 3 inclusive.
426  */
427 
428 #define	PREFETCHSIZE 3
429 
430 static int
431 cmp_to_prefetch(uchar_t *p)
432 {
433 	if (*p == 0x0F && (*(p+1) == 0x18 || *(p+1) == 0x0D) && *(p+2) <= 3)
434 		return (1);
435 	return (0);
436 }
437 
438 static int
439 instr_is_prefetch(caddr_t pc)
440 {
441 	uchar_t instr[PREFETCHSIZE];
442 	int	error;
443 
444 	error = copyin_nowatch(pc, (caddr_t)instr, PREFETCHSIZE);
445 
446 	if (error == 0 && cmp_to_prefetch(instr))
447 		return (1);
448 	return (0);
449 }
450 
451 #endif /* OPTERON_ERRATUM_91 */
452 
453 /*
454  * Called from the trap handler when a processor trap occurs.
455  *
456  * Note: All user-level traps that might call stop() must exit
457  * trap() by 'goto out' or by falling through.
458  * Note Also: trap() is usually called with interrupts enabled, (PS_IE == 1)
459  * however, there are paths that arrive here with PS_IE == 0 so special care
460  * must be taken in those cases.
461  */
462 void
463 trap(struct regs *rp, caddr_t addr, processorid_t cpuid)
464 {
465 	kthread_t *ct = curthread;
466 	enum seg_rw rw;
467 	unsigned type;
468 	proc_t *p = ttoproc(ct);
469 	klwp_t *lwp = ttolwp(ct);
470 	uintptr_t lofault;
471 	faultcode_t pagefault(), res, errcode;
472 	enum fault_type fault_type;
473 	k_siginfo_t siginfo;
474 	uint_t fault = 0;
475 	int mstate;
476 	int sicode = 0;
477 	int watchcode;
478 	int watchpage;
479 	caddr_t vaddr;
480 	int singlestep_twiddle;
481 	size_t sz;
482 	int ta;
483 #ifdef __amd64
484 	uchar_t instr;
485 #endif
486 
487 	ASSERT_STACK_ALIGNED();
488 
489 	type = rp->r_trapno;
490 	CPU_STATS_ADDQ(CPU, sys, trap, 1);
491 	ASSERT(ct->t_schedflag & TS_DONT_SWAP);
492 
493 	if (type == T_PGFLT) {
494 
495 		errcode = rp->r_err;
496 		if (errcode & PF_ERR_WRITE)
497 			rw = S_WRITE;
498 		else if ((caddr_t)rp->r_pc == addr ||
499 		    (mmu.pt_nx != 0 && (errcode & PF_ERR_EXEC)))
500 			rw = S_EXEC;
501 		else
502 			rw = S_READ;
503 
504 #if defined(__i386)
505 		/*
506 		 * Pentium Pro work-around
507 		 */
508 		if ((errcode & PF_ERR_PROT) && pentiumpro_bug4046376) {
509 			uint_t	attr;
510 			uint_t	priv_violation;
511 			uint_t	access_violation;
512 
513 			if (hat_getattr(addr < (caddr_t)kernelbase ?
514 			    curproc->p_as->a_hat : kas.a_hat, addr, &attr)
515 			    == -1) {
516 				errcode &= ~PF_ERR_PROT;
517 			} else {
518 				priv_violation = (errcode & PF_ERR_USER) &&
519 				    !(attr & PROT_USER);
520 				access_violation = (errcode & PF_ERR_WRITE) &&
521 				    !(attr & PROT_WRITE);
522 				if (!priv_violation && !access_violation)
523 					goto cleanup;
524 			}
525 		}
526 #endif /* __i386 */
527 
528 	} else if (type == T_SGLSTP && lwp != NULL)
529 		lwp->lwp_pcb.pcb_drstat = (uintptr_t)addr;
530 
531 	if (tdebug)
532 		showregs(type, rp, addr);
533 
534 	if (USERMODE(rp->r_cs)) {
535 		/*
536 		 * Set up the current cred to use during this trap. u_cred
537 		 * no longer exists.  t_cred is used instead.
538 		 * The current process credential applies to the thread for
539 		 * the entire trap.  If trapping from the kernel, this
540 		 * should already be set up.
541 		 */
542 		if (ct->t_cred != p->p_cred) {
543 			cred_t *oldcred = ct->t_cred;
544 			/*
545 			 * DTrace accesses t_cred in probe context.  t_cred
546 			 * must always be either NULL, or point to a valid,
547 			 * allocated cred structure.
548 			 */
549 			ct->t_cred = crgetcred();
550 			crfree(oldcred);
551 		}
552 		ASSERT(lwp != NULL);
553 		type |= USER;
554 		ASSERT(lwptoregs(lwp) == rp);
555 		lwp->lwp_state = LWP_SYS;
556 
557 		switch (type) {
558 		case T_PGFLT + USER:
559 			if ((caddr_t)rp->r_pc == addr)
560 				mstate = LMS_TFAULT;
561 			else
562 				mstate = LMS_DFAULT;
563 			break;
564 		default:
565 			mstate = LMS_TRAP;
566 			break;
567 		}
568 		/* Kernel probe */
569 		TNF_PROBE_1(thread_state, "thread", /* CSTYLED */,
570 		    tnf_microstate, state, mstate);
571 		mstate = new_mstate(ct, mstate);
572 
573 		bzero(&siginfo, sizeof (siginfo));
574 	}
575 
576 	switch (type) {
577 	case T_PGFLT + USER:
578 	case T_SGLSTP:
579 	case T_SGLSTP + USER:
580 	case T_BPTFLT + USER:
581 		break;
582 
583 	default:
584 		FTRACE_2("trap(): type=0x%lx, regs=0x%lx",
585 		    (ulong_t)type, (ulong_t)rp);
586 		break;
587 	}
588 
589 	switch (type) {
590 	case T_SIMDFPE:
591 		/* Make sure we enable interrupts before die()ing */
592 		sti();	/* The SIMD exception comes in via cmninttrap */
593 		/*FALLTHROUGH*/
594 	default:
595 		if (type & USER) {
596 			if (tudebug)
597 				showregs(type, rp, (caddr_t)0);
598 			printf("trap: Unknown trap type %d in user mode\n",
599 			    type & ~USER);
600 			siginfo.si_signo = SIGILL;
601 			siginfo.si_code  = ILL_ILLTRP;
602 			siginfo.si_addr  = (caddr_t)rp->r_pc;
603 			siginfo.si_trapno = type & ~USER;
604 			fault = FLTILL;
605 			break;
606 		} else {
607 			(void) die(type, rp, addr, cpuid);
608 			/*NOTREACHED*/
609 		}
610 
611 	case T_PGFLT:		/* system page fault */
612 		/*
613 		 * If we're under on_trap() protection (see <sys/ontrap.h>),
614 		 * set ot_trap and bounce back to the on_trap() call site
615 		 * via the installed trampoline.
616 		 */
617 		if ((ct->t_ontrap != NULL) &&
618 		    (ct->t_ontrap->ot_prot & OT_DATA_ACCESS)) {
619 			ct->t_ontrap->ot_trap |= OT_DATA_ACCESS;
620 			rp->r_pc = ct->t_ontrap->ot_trampoline;
621 			goto cleanup;
622 		}
623 
624 		/*
625 		 * See if we can handle as pagefault. Save lofault
626 		 * across this. Here we assume that an address
627 		 * less than KERNELBASE is a user fault.
628 		 * We can do this as copy.s routines verify that the
629 		 * starting address is less than KERNELBASE before
630 		 * starting and because we know that we always have
631 		 * KERNELBASE mapped as invalid to serve as a "barrier".
632 		 */
633 		lofault = ct->t_lofault;
634 		ct->t_lofault = 0;
635 
636 		mstate = new_mstate(ct, LMS_KFAULT);
637 
638 		if (addr < (caddr_t)kernelbase) {
639 			res = pagefault(addr,
640 			    (errcode & PF_ERR_PROT)? F_PROT: F_INVAL, rw, 0);
641 			if (res == FC_NOMAP &&
642 			    addr < p->p_usrstack &&
643 			    grow(addr))
644 				res = 0;
645 		} else {
646 			res = pagefault(addr,
647 			    (errcode & PF_ERR_PROT)? F_PROT: F_INVAL, rw, 1);
648 		}
649 		(void) new_mstate(ct, mstate);
650 
651 		/*
652 		 * Restore lofault. If we resolved the fault, exit.
653 		 * If we didn't and lofault wasn't set, die.
654 		 */
655 		ct->t_lofault = lofault;
656 		if (res == 0)
657 			goto cleanup;
658 
659 #if defined(OPTERON_ERRATUM_93) && defined(_LP64)
660 		if (lofault == 0 && opteron_erratum_93) {
661 			/*
662 			 * Workaround for Opteron Erratum 93. On return from
663 			 * a System Managment Interrupt at a HLT instruction
664 			 * the %rip might be truncated to a 32 bit value.
665 			 * BIOS is supposed to fix this, but some don't.
666 			 * If this occurs we simply restore the high order bits.
667 			 * The HLT instruction is 1 byte of 0xf4.
668 			 */
669 			uintptr_t	rip = rp->r_pc;
670 
671 			if ((rip & 0xfffffffful) == rip) {
672 				rip |= 0xfffffffful << 32;
673 				if (hat_getpfnum(kas.a_hat, (caddr_t)rip) !=
674 				    PFN_INVALID &&
675 				    (*(uchar_t *)rip == 0xf4 ||
676 				    *(uchar_t *)(rip - 1) == 0xf4)) {
677 					rp->r_pc = rip;
678 					goto cleanup;
679 				}
680 			}
681 		}
682 #endif /* OPTERON_ERRATUM_93 && _LP64 */
683 
684 #ifdef OPTERON_ERRATUM_91
685 		if (lofault == 0 && opteron_erratum_91) {
686 			/*
687 			 * Workaround for Opteron Erratum 91. Prefetches may
688 			 * generate a page fault (they're not supposed to do
689 			 * that!). If this occurs we simply return back to the
690 			 * instruction.
691 			 */
692 			caddr_t		pc = (caddr_t)rp->r_pc;
693 
694 			/*
695 			 * If the faulting PC is not mapped, this is a
696 			 * legitimate kernel page fault that must result in a
697 			 * panic. If the faulting PC is mapped, it could contain
698 			 * a prefetch instruction. Check for that here.
699 			 */
700 			if (hat_getpfnum(kas.a_hat, pc) != PFN_INVALID) {
701 				if (cmp_to_prefetch((uchar_t *)pc)) {
702 #ifdef DEBUG
703 					cmn_err(CE_WARN, "Opteron erratum 91 "
704 					    "occurred: kernel prefetch"
705 					    " at %p generated a page fault!",
706 					    (void *)rp->r_pc);
707 #endif /* DEBUG */
708 					goto cleanup;
709 				}
710 			}
711 			(void) die(type, rp, addr, cpuid);
712 		}
713 #endif /* OPTERON_ERRATUM_91 */
714 
715 		if (lofault == 0)
716 			(void) die(type, rp, addr, cpuid);
717 
718 		/*
719 		 * Cannot resolve fault.  Return to lofault.
720 		 */
721 		if (lodebug) {
722 			showregs(type, rp, addr);
723 			traceregs(rp);
724 		}
725 		if (FC_CODE(res) == FC_OBJERR)
726 			res = FC_ERRNO(res);
727 		else
728 			res = EFAULT;
729 		rp->r_r0 = res;
730 		rp->r_pc = ct->t_lofault;
731 		goto cleanup;
732 
733 	case T_PGFLT + USER:	/* user page fault */
734 		if (faultdebug) {
735 			char *fault_str;
736 
737 			switch (rw) {
738 			case S_READ:
739 				fault_str = "read";
740 				break;
741 			case S_WRITE:
742 				fault_str = "write";
743 				break;
744 			case S_EXEC:
745 				fault_str = "exec";
746 				break;
747 			default:
748 				fault_str = "";
749 				break;
750 			}
751 			printf("user %s fault:  addr=0x%lx errcode=0x%x\n",
752 			    fault_str, (uintptr_t)addr, errcode);
753 		}
754 
755 #if defined(OPTERON_ERRATUM_100) && defined(_LP64)
756 		/*
757 		 * Workaround for AMD erratum 100
758 		 *
759 		 * A 32-bit process may receive a page fault on a non
760 		 * 32-bit address by mistake. The range of the faulting
761 		 * address will be
762 		 *
763 		 *	0xffffffff80000000 .. 0xffffffffffffffff or
764 		 *	0x0000000100000000 .. 0x000000017fffffff
765 		 *
766 		 * The fault is always due to an instruction fetch, however
767 		 * the value of r_pc should be correct (in 32 bit range),
768 		 * so we ignore the page fault on the bogus address.
769 		 */
770 		if (p->p_model == DATAMODEL_ILP32 &&
771 		    (0xffffffff80000000 <= (uintptr_t)addr ||
772 		    (0x100000000 <= (uintptr_t)addr &&
773 		    (uintptr_t)addr <= 0x17fffffff))) {
774 			if (!opteron_erratum_100)
775 				panic("unexpected erratum #100");
776 			if (rp->r_pc <= 0xffffffff)
777 				goto out;
778 		}
779 #endif /* OPTERON_ERRATUM_100 && _LP64 */
780 
781 		ASSERT(!(curthread->t_flag & T_WATCHPT));
782 		watchpage = (pr_watch_active(p) && pr_is_watchpage(addr, rw));
783 #ifdef __i386
784 		/*
785 		 * In 32-bit mode, the lcall (system call) instruction fetches
786 		 * one word from the stack, at the stack pointer, because of the
787 		 * way the call gate is constructed.  This is a bogus
788 		 * read and should not be counted as a read watchpoint.
789 		 * We work around the problem here by testing to see if
790 		 * this situation applies and, if so, simply jumping to
791 		 * the code in locore.s that fields the system call trap.
792 		 * The registers on the stack are already set up properly
793 		 * due to the match between the call gate sequence and the
794 		 * trap gate sequence.  We just have to adjust the pc.
795 		 */
796 		if (watchpage && addr == (caddr_t)rp->r_sp &&
797 		    rw == S_READ && instr_is_lcall_syscall((caddr_t)rp->r_pc)) {
798 			extern void watch_syscall(void);
799 
800 			rp->r_pc += LCALLSIZE;
801 			watch_syscall();	/* never returns */
802 			/* NOTREACHED */
803 		}
804 #endif /* __i386 */
805 		vaddr = addr;
806 		if (!watchpage || (sz = instr_size(rp, &vaddr, rw)) <= 0)
807 			fault_type = (errcode & PF_ERR_PROT)? F_PROT: F_INVAL;
808 		else if ((watchcode = pr_is_watchpoint(&vaddr, &ta,
809 		    sz, NULL, rw)) != 0) {
810 			if (ta) {
811 				do_watch_step(vaddr, sz, rw,
812 				    watchcode, rp->r_pc);
813 				fault_type = F_INVAL;
814 			} else {
815 				bzero(&siginfo, sizeof (siginfo));
816 				siginfo.si_signo = SIGTRAP;
817 				siginfo.si_code = watchcode;
818 				siginfo.si_addr = vaddr;
819 				siginfo.si_trapafter = 0;
820 				siginfo.si_pc = (caddr_t)rp->r_pc;
821 				fault = FLTWATCH;
822 				break;
823 			}
824 		} else {
825 			/* XXX pr_watch_emul() never succeeds (for now) */
826 			if (rw != S_EXEC && pr_watch_emul(rp, vaddr, rw))
827 				goto out;
828 			do_watch_step(vaddr, sz, rw, 0, 0);
829 			fault_type = F_INVAL;
830 		}
831 
832 		res = pagefault(addr, fault_type, rw, 0);
833 
834 		/*
835 		 * If pagefault() succeeded, ok.
836 		 * Otherwise attempt to grow the stack.
837 		 */
838 		if (res == 0 ||
839 		    (res == FC_NOMAP &&
840 		    addr < p->p_usrstack &&
841 		    grow(addr))) {
842 			lwp->lwp_lastfault = FLTPAGE;
843 			lwp->lwp_lastfaddr = addr;
844 			if (prismember(&p->p_fltmask, FLTPAGE)) {
845 				bzero(&siginfo, sizeof (siginfo));
846 				siginfo.si_addr = addr;
847 				(void) stop_on_fault(FLTPAGE, &siginfo);
848 			}
849 			goto out;
850 		} else if (res == FC_PROT && addr < p->p_usrstack &&
851 		    (mmu.pt_nx != 0 && (errcode & PF_ERR_EXEC))) {
852 			report_stack_exec(p, addr);
853 		}
854 
855 #ifdef OPTERON_ERRATUM_91
856 		/*
857 		 * Workaround for Opteron Erratum 91. Prefetches may generate a
858 		 * page fault (they're not supposed to do that!). If this
859 		 * occurs we simply return back to the instruction.
860 		 *
861 		 * We rely on copyin to properly fault in the page with r_pc.
862 		 */
863 		if (opteron_erratum_91 &&
864 		    addr != (caddr_t)rp->r_pc &&
865 		    instr_is_prefetch((caddr_t)rp->r_pc)) {
866 #ifdef DEBUG
867 			cmn_err(CE_WARN, "Opteron erratum 91 occurred: "
868 			    "prefetch at %p in pid %d generated a trap!",
869 			    (void *)rp->r_pc, p->p_pid);
870 #endif /* DEBUG */
871 			goto out;
872 		}
873 #endif /* OPTERON_ERRATUM_91 */
874 
875 		if (tudebug)
876 			showregs(type, rp, addr);
877 		/*
878 		 * In the case where both pagefault and grow fail,
879 		 * set the code to the value provided by pagefault.
880 		 * We map all errors returned from pagefault() to SIGSEGV.
881 		 */
882 		bzero(&siginfo, sizeof (siginfo));
883 		siginfo.si_addr = addr;
884 		switch (FC_CODE(res)) {
885 		case FC_HWERR:
886 		case FC_NOSUPPORT:
887 			siginfo.si_signo = SIGBUS;
888 			siginfo.si_code = BUS_ADRERR;
889 			fault = FLTACCESS;
890 			break;
891 		case FC_ALIGN:
892 			siginfo.si_signo = SIGBUS;
893 			siginfo.si_code = BUS_ADRALN;
894 			fault = FLTACCESS;
895 			break;
896 		case FC_OBJERR:
897 			if ((siginfo.si_errno = FC_ERRNO(res)) != EINTR) {
898 				siginfo.si_signo = SIGBUS;
899 				siginfo.si_code = BUS_OBJERR;
900 				fault = FLTACCESS;
901 			}
902 			break;
903 		default:	/* FC_NOMAP or FC_PROT */
904 			siginfo.si_signo = SIGSEGV;
905 			siginfo.si_code =
906 			    (res == FC_NOMAP)? SEGV_MAPERR : SEGV_ACCERR;
907 			fault = FLTBOUNDS;
908 			break;
909 		}
910 		break;
911 
912 	case T_ILLINST + USER:	/* invalid opcode fault */
913 		/*
914 		 * If the syscall instruction is disabled due to LDT usage, a
915 		 * user program that attempts to execute it will trigger a #ud
916 		 * trap. Check for that case here. If this occurs on a CPU which
917 		 * doesn't even support syscall, the result of all of this will
918 		 * be to emulate that particular instruction.
919 		 */
920 		if (p->p_ldt != NULL &&
921 		    ldt_rewrite_syscall(rp, p, X86_ASYSC))
922 			goto out;
923 
924 #ifdef __amd64
925 		/*
926 		 * Emulate the LAHF and SAHF instructions if needed.
927 		 * See the instr_is_lsahf function for details.
928 		 */
929 		if (p->p_model == DATAMODEL_LP64 &&
930 		    instr_is_lsahf((caddr_t)rp->r_pc, &instr)) {
931 			emulate_lsahf(rp, instr);
932 			goto out;
933 		}
934 #endif
935 
936 		/*FALLTHROUGH*/
937 
938 		if (tudebug)
939 			showregs(type, rp, (caddr_t)0);
940 		siginfo.si_signo = SIGILL;
941 		siginfo.si_code  = ILL_ILLOPC;
942 		siginfo.si_addr  = (caddr_t)rp->r_pc;
943 		fault = FLTILL;
944 		break;
945 
946 	case T_ZERODIV + USER:		/* integer divide by zero */
947 		if (tudebug && tudebugfpe)
948 			showregs(type, rp, (caddr_t)0);
949 		siginfo.si_signo = SIGFPE;
950 		siginfo.si_code  = FPE_INTDIV;
951 		siginfo.si_addr  = (caddr_t)rp->r_pc;
952 		fault = FLTIZDIV;
953 		break;
954 
955 	case T_OVFLW + USER:	/* integer overflow */
956 		if (tudebug && tudebugfpe)
957 			showregs(type, rp, (caddr_t)0);
958 		siginfo.si_signo = SIGFPE;
959 		siginfo.si_code  = FPE_INTOVF;
960 		siginfo.si_addr  = (caddr_t)rp->r_pc;
961 		fault = FLTIOVF;
962 		break;
963 
964 	case T_NOEXTFLT + USER:	/* math coprocessor not available */
965 		if (tudebug && tudebugfpe)
966 			showregs(type, rp, addr);
967 		if (fpnoextflt(rp)) {
968 			siginfo.si_signo = SIGFPE;
969 			siginfo.si_code  = ILL_ILLOPC;
970 			siginfo.si_addr  = (caddr_t)rp->r_pc;
971 			fault = FLTFPE;
972 		}
973 		break;
974 
975 	case T_EXTOVRFLT:	/* extension overrun fault */
976 		/* check if we took a kernel trap on behalf of user */
977 		{
978 			extern  void ndptrap_frstor(void);
979 			if (rp->r_pc != (uintptr_t)ndptrap_frstor) {
980 				sti(); /* T_EXTOVRFLT comes in via cmninttrap */
981 				(void) die(type, rp, addr, cpuid);
982 			}
983 			type |= USER;
984 		}
985 		/*FALLTHROUGH*/
986 	case T_EXTOVRFLT + USER:	/* extension overrun fault */
987 		if (tudebug && tudebugfpe)
988 			showregs(type, rp, addr);
989 		if (fpextovrflt(rp)) {
990 			siginfo.si_signo = SIGSEGV;
991 			siginfo.si_code  = SEGV_MAPERR;
992 			siginfo.si_addr  = (caddr_t)rp->r_pc;
993 			fault = FLTBOUNDS;
994 		}
995 		break;
996 
997 	case T_EXTERRFLT:	/* x87 floating point exception pending */
998 		/* check if we took a kernel trap on behalf of user */
999 		{
1000 			extern  void ndptrap_frstor(void);
1001 			if (rp->r_pc != (uintptr_t)ndptrap_frstor) {
1002 				sti(); /* T_EXTERRFLT comes in via cmninttrap */
1003 				(void) die(type, rp, addr, cpuid);
1004 			}
1005 			type |= USER;
1006 		}
1007 		/*FALLTHROUGH*/
1008 
1009 	case T_EXTERRFLT + USER: /* x87 floating point exception pending */
1010 		if (tudebug && tudebugfpe)
1011 			showregs(type, rp, addr);
1012 		if (sicode = fpexterrflt(rp)) {
1013 			siginfo.si_signo = SIGFPE;
1014 			siginfo.si_code  = sicode;
1015 			siginfo.si_addr  = (caddr_t)rp->r_pc;
1016 			fault = FLTFPE;
1017 		}
1018 		break;
1019 
1020 	case T_SIMDFPE + USER:		/* SSE and SSE2 exceptions */
1021 		if (tudebug && tudebugsse)
1022 			showregs(type, rp, addr);
1023 		if ((x86_feature & (X86_SSE|X86_SSE2)) == 0) {
1024 			/*
1025 			 * There are rumours that some user instructions
1026 			 * on older CPUs can cause this trap to occur; in
1027 			 * which case send a SIGILL instead of a SIGFPE.
1028 			 */
1029 			siginfo.si_signo = SIGILL;
1030 			siginfo.si_code  = ILL_ILLTRP;
1031 			siginfo.si_addr  = (caddr_t)rp->r_pc;
1032 			siginfo.si_trapno = type & ~USER;
1033 			fault = FLTILL;
1034 		} else if ((sicode = fpsimderrflt(rp)) != 0) {
1035 			siginfo.si_signo = SIGFPE;
1036 			siginfo.si_code = sicode;
1037 			siginfo.si_addr = (caddr_t)rp->r_pc;
1038 			fault = FLTFPE;
1039 		}
1040 
1041 		sti();	/* The SIMD exception comes in via cmninttrap */
1042 		break;
1043 
1044 	case T_BPTFLT:	/* breakpoint trap */
1045 		/*
1046 		 * Kernel breakpoint traps should only happen when kmdb is
1047 		 * active, and even then, it'll have interposed on the IDT, so
1048 		 * control won't get here.  If it does, we've hit a breakpoint
1049 		 * without the debugger, which is very strange, and very
1050 		 * fatal.
1051 		 */
1052 		if (tudebug && tudebugbpt)
1053 			showregs(type, rp, (caddr_t)0);
1054 
1055 		(void) die(type, rp, addr, cpuid);
1056 		break;
1057 
1058 	case T_SGLSTP: /* single step/hw breakpoint exception */
1059 
1060 		/* Now evaluate how we got here */
1061 		if (lwp != NULL && (lwp->lwp_pcb.pcb_drstat & DR_SINGLESTEP)) {
1062 			/*
1063 			 * i386 single-steps even through lcalls which
1064 			 * change the privilege level. So we take a trap at
1065 			 * the first instruction in privileged mode.
1066 			 *
1067 			 * Set a flag to indicate that upon completion of
1068 			 * the system call, deal with the single-step trap.
1069 			 *
1070 			 * The same thing happens for sysenter, too.
1071 			 */
1072 			singlestep_twiddle = 0;
1073 			if (rp->r_pc == (uintptr_t)sys_sysenter ||
1074 			    rp->r_pc == (uintptr_t)brand_sys_sysenter) {
1075 				singlestep_twiddle = 1;
1076 #if defined(__amd64)
1077 				/*
1078 				 * Since we are already on the kernel's
1079 				 * %gs, on 64-bit systems the sysenter case
1080 				 * needs to adjust the pc to avoid
1081 				 * executing the swapgs instruction at the
1082 				 * top of the handler.
1083 				 */
1084 				if (rp->r_pc == (uintptr_t)sys_sysenter)
1085 					rp->r_pc = (uintptr_t)
1086 					    _sys_sysenter_post_swapgs;
1087 				else
1088 					rp->r_pc = (uintptr_t)
1089 					    _brand_sys_sysenter_post_swapgs;
1090 #endif
1091 			}
1092 #if defined(__i386)
1093 			else if (rp->r_pc == (uintptr_t)sys_call ||
1094 			    rp->r_pc == (uintptr_t)brand_sys_call) {
1095 				singlestep_twiddle = 1;
1096 			}
1097 #endif
1098 			else {
1099 				/* not on sysenter/syscall; uregs available */
1100 				if (tudebug && tudebugbpt)
1101 					showregs(type, rp, (caddr_t)0);
1102 			}
1103 			if (singlestep_twiddle) {
1104 				rp->r_ps &= ~PS_T; /* turn off trace */
1105 				lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
1106 				ct->t_post_sys = 1;
1107 				aston(curthread);
1108 				goto cleanup;
1109 			}
1110 		}
1111 		/* XXX - needs review on debugger interface? */
1112 		if (boothowto & RB_DEBUG)
1113 			debug_enter((char *)NULL);
1114 		else
1115 			(void) die(type, rp, addr, cpuid);
1116 		break;
1117 
1118 	case T_NMIFLT:	/* NMI interrupt */
1119 		printf("Unexpected NMI in system mode\n");
1120 		goto cleanup;
1121 
1122 	case T_NMIFLT + USER:	/* NMI interrupt */
1123 		printf("Unexpected NMI in user mode\n");
1124 		break;
1125 
1126 	case T_GPFLT:	/* general protection violation */
1127 		/*
1128 		 * Any #GP that occurs during an on_trap .. no_trap bracket
1129 		 * with OT_DATA_ACCESS or OT_SEGMENT_ACCESS protection,
1130 		 * or in a on_fault .. no_fault bracket, is forgiven
1131 		 * and we trampoline.  This protection is given regardless
1132 		 * of whether we are 32/64 bit etc - if a distinction is
1133 		 * required then define new on_trap protection types.
1134 		 *
1135 		 * On amd64, we can get a #gp from referencing addresses
1136 		 * in the virtual address hole e.g. from a copyin or in
1137 		 * update_sregs while updating user segment registers.
1138 		 *
1139 		 * On the 32-bit hypervisor we could also generate one in
1140 		 * mfn_to_pfn by reaching around or into where the hypervisor
1141 		 * lives which is protected by segmentation.
1142 		 */
1143 
1144 		/*
1145 		 * If we're under on_trap() protection (see <sys/ontrap.h>),
1146 		 * set ot_trap and trampoline back to the on_trap() call site
1147 		 * for OT_DATA_ACCESS or OT_SEGMENT_ACCESS.
1148 		 */
1149 		if (ct->t_ontrap != NULL) {
1150 			int ttype =  ct->t_ontrap->ot_prot &
1151 			    (OT_DATA_ACCESS | OT_SEGMENT_ACCESS);
1152 
1153 			if (ttype != 0) {
1154 				ct->t_ontrap->ot_trap |= ttype;
1155 				if (tudebug)
1156 					showregs(type, rp, (caddr_t)0);
1157 				rp->r_pc = ct->t_ontrap->ot_trampoline;
1158 				goto cleanup;
1159 			}
1160 		}
1161 
1162 		/*
1163 		 * If we're under lofault protection (copyin etc.),
1164 		 * longjmp back to lofault with an EFAULT.
1165 		 */
1166 		if (ct->t_lofault) {
1167 			/*
1168 			 * Fault is not resolvable, so just return to lofault
1169 			 */
1170 			if (lodebug) {
1171 				showregs(type, rp, addr);
1172 				traceregs(rp);
1173 			}
1174 			rp->r_r0 = EFAULT;
1175 			rp->r_pc = ct->t_lofault;
1176 			goto cleanup;
1177 		}
1178 
1179 		/*
1180 		 * We fall through to the next case, which repeats
1181 		 * the OT_SEGMENT_ACCESS check which we've already
1182 		 * done, so we'll always fall through to the
1183 		 * T_STKFLT case.
1184 		 */
1185 		/*FALLTHROUGH*/
1186 	case T_SEGFLT:	/* segment not present fault */
1187 		/*
1188 		 * One example of this is #NP in update_sregs while
1189 		 * attempting to update a user segment register
1190 		 * that points to a descriptor that is marked not
1191 		 * present.
1192 		 */
1193 		if (ct->t_ontrap != NULL &&
1194 		    ct->t_ontrap->ot_prot & OT_SEGMENT_ACCESS) {
1195 			ct->t_ontrap->ot_trap |= OT_SEGMENT_ACCESS;
1196 			if (tudebug)
1197 				showregs(type, rp, (caddr_t)0);
1198 			rp->r_pc = ct->t_ontrap->ot_trampoline;
1199 			goto cleanup;
1200 		}
1201 		/*FALLTHROUGH*/
1202 	case T_STKFLT:	/* stack fault */
1203 	case T_TSSFLT:	/* invalid TSS fault */
1204 		if (tudebug)
1205 			showregs(type, rp, (caddr_t)0);
1206 		if (kern_gpfault(rp))
1207 			(void) die(type, rp, addr, cpuid);
1208 		goto cleanup;
1209 
1210 	/*
1211 	 * ONLY 32-bit PROCESSES can USE a PRIVATE LDT! 64-bit apps
1212 	 * should have no need for them, so we put a stop to it here.
1213 	 *
1214 	 * So: not-present fault is ONLY valid for 32-bit processes with
1215 	 * a private LDT trying to do a system call. Emulate it.
1216 	 *
1217 	 * #gp fault is ONLY valid for 32-bit processes also, which DO NOT
1218 	 * have a private LDT, and are trying to do a system call. Emulate it.
1219 	 */
1220 
1221 	case T_SEGFLT + USER:	/* segment not present fault */
1222 	case T_GPFLT + USER:	/* general protection violation */
1223 #ifdef _SYSCALL32_IMPL
1224 		if (p->p_model != DATAMODEL_NATIVE) {
1225 #endif /* _SYSCALL32_IMPL */
1226 		if (instr_is_lcall_syscall((caddr_t)rp->r_pc)) {
1227 			if (type == T_SEGFLT + USER)
1228 				ASSERT(p->p_ldt != NULL);
1229 
1230 			if ((p->p_ldt == NULL && type == T_GPFLT + USER) ||
1231 			    type == T_SEGFLT + USER) {
1232 
1233 			/*
1234 			 * The user attempted a system call via the obsolete
1235 			 * call gate mechanism. Because the process doesn't have
1236 			 * an LDT (i.e. the ldtr contains 0), a #gp results.
1237 			 * Emulate the syscall here, just as we do above for a
1238 			 * #np trap.
1239 			 */
1240 
1241 			/*
1242 			 * Since this is a not-present trap, rp->r_pc points to
1243 			 * the trapping lcall instruction. We need to bump it
1244 			 * to the next insn so the app can continue on.
1245 			 */
1246 			rp->r_pc += LCALLSIZE;
1247 			lwp->lwp_regs = rp;
1248 
1249 			/*
1250 			 * Normally the microstate of the LWP is forced back to
1251 			 * LMS_USER by the syscall handlers. Emulate that
1252 			 * behavior here.
1253 			 */
1254 			mstate = LMS_USER;
1255 
1256 			dosyscall();
1257 			goto out;
1258 			}
1259 		}
1260 #ifdef _SYSCALL32_IMPL
1261 		}
1262 #endif /* _SYSCALL32_IMPL */
1263 		/*
1264 		 * If the current process is using a private LDT and the
1265 		 * trapping instruction is sysenter, the sysenter instruction
1266 		 * has been disabled on the CPU because it destroys segment
1267 		 * registers. If this is the case, rewrite the instruction to
1268 		 * be a safe system call and retry it. If this occurs on a CPU
1269 		 * which doesn't even support sysenter, the result of all of
1270 		 * this will be to emulate that particular instruction.
1271 		 */
1272 		if (p->p_ldt != NULL &&
1273 		    ldt_rewrite_syscall(rp, p, X86_SEP))
1274 			goto out;
1275 
1276 		/*FALLTHROUGH*/
1277 
1278 	case T_BOUNDFLT + USER:	/* bound fault */
1279 	case T_STKFLT + USER:	/* stack fault */
1280 	case T_TSSFLT + USER:	/* invalid TSS fault */
1281 		if (tudebug)
1282 			showregs(type, rp, (caddr_t)0);
1283 		siginfo.si_signo = SIGSEGV;
1284 		siginfo.si_code  = SEGV_MAPERR;
1285 		siginfo.si_addr  = (caddr_t)rp->r_pc;
1286 		fault = FLTBOUNDS;
1287 		break;
1288 
1289 	case T_ALIGNMENT + USER:	/* user alignment error (486) */
1290 		if (tudebug)
1291 			showregs(type, rp, (caddr_t)0);
1292 		bzero(&siginfo, sizeof (siginfo));
1293 		siginfo.si_signo = SIGBUS;
1294 		siginfo.si_code = BUS_ADRALN;
1295 		siginfo.si_addr = (caddr_t)rp->r_pc;
1296 		fault = FLTACCESS;
1297 		break;
1298 
1299 	case T_SGLSTP + USER: /* single step/hw breakpoint exception */
1300 		if (tudebug && tudebugbpt)
1301 			showregs(type, rp, (caddr_t)0);
1302 
1303 		/* Was it single-stepping? */
1304 		if (lwp->lwp_pcb.pcb_drstat & DR_SINGLESTEP) {
1305 			pcb_t *pcb = &lwp->lwp_pcb;
1306 
1307 			rp->r_ps &= ~PS_T;
1308 			/*
1309 			 * If both NORMAL_STEP and WATCH_STEP are in effect,
1310 			 * give precedence to WATCH_STEP.  If neither is set,
1311 			 * user must have set the PS_T bit in %efl; treat this
1312 			 * as NORMAL_STEP.
1313 			 */
1314 			if ((fault = undo_watch_step(&siginfo)) == 0 &&
1315 			    ((pcb->pcb_flags & NORMAL_STEP) ||
1316 			    !(pcb->pcb_flags & WATCH_STEP))) {
1317 				siginfo.si_signo = SIGTRAP;
1318 				siginfo.si_code = TRAP_TRACE;
1319 				siginfo.si_addr = (caddr_t)rp->r_pc;
1320 				fault = FLTTRACE;
1321 			}
1322 			pcb->pcb_flags &= ~(NORMAL_STEP|WATCH_STEP);
1323 		} else {
1324 			cmn_err(CE_WARN,
1325 			    "Unexpected INT 1 in user mode, dr6=%lx",
1326 			    lwp->lwp_pcb.pcb_drstat);
1327 		}
1328 		break;
1329 
1330 	case T_BPTFLT + USER:	/* breakpoint trap */
1331 		if (tudebug && tudebugbpt)
1332 			showregs(type, rp, (caddr_t)0);
1333 		/*
1334 		 * int 3 (the breakpoint instruction) leaves the pc referring
1335 		 * to the address one byte after the breakpointed address.
1336 		 * If the P_PR_BPTADJ flag has been set via /proc, We adjust
1337 		 * it back so it refers to the breakpointed address.
1338 		 */
1339 		if (p->p_proc_flag & P_PR_BPTADJ)
1340 			rp->r_pc--;
1341 		siginfo.si_signo = SIGTRAP;
1342 		siginfo.si_code  = TRAP_BRKPT;
1343 		siginfo.si_addr  = (caddr_t)rp->r_pc;
1344 		fault = FLTBPT;
1345 		break;
1346 
1347 	case T_AST:
1348 		/*
1349 		 * This occurs only after the cs register has been made to
1350 		 * look like a kernel selector, either through debugging or
1351 		 * possibly by functions like setcontext().  The thread is
1352 		 * about to cause a general protection fault at common_iret()
1353 		 * in locore.  We let that happen immediately instead of
1354 		 * doing the T_AST processing.
1355 		 */
1356 		goto cleanup;
1357 
1358 	case T_AST + USER:	/* profiling, resched, h/w error pseudo trap */
1359 		if (lwp->lwp_pcb.pcb_flags & ASYNC_HWERR) {
1360 			proc_t *p = ttoproc(curthread);
1361 
1362 			lwp->lwp_pcb.pcb_flags &= ~ASYNC_HWERR;
1363 			contract_process_hwerr(p->p_ct_process, p);
1364 			siginfo.si_signo = SIGKILL;
1365 			siginfo.si_code = SI_NOINFO;
1366 		} else if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW) {
1367 			lwp->lwp_pcb.pcb_flags &= ~CPC_OVERFLOW;
1368 			if (kcpc_overflow_ast()) {
1369 				/*
1370 				 * Signal performance counter overflow
1371 				 */
1372 				if (tudebug)
1373 					showregs(type, rp, (caddr_t)0);
1374 				bzero(&siginfo, sizeof (siginfo));
1375 				siginfo.si_signo = SIGEMT;
1376 				siginfo.si_code = EMT_CPCOVF;
1377 				siginfo.si_addr = (caddr_t)rp->r_pc;
1378 				fault = FLTCPCOVF;
1379 			}
1380 		}
1381 
1382 		break;
1383 	}
1384 
1385 	/*
1386 	 * We can't get here from a system trap
1387 	 */
1388 	ASSERT(type & USER);
1389 
1390 	if (fault) {
1391 		/* We took a fault so abort single step. */
1392 		lwp->lwp_pcb.pcb_flags &= ~(NORMAL_STEP|WATCH_STEP);
1393 		/*
1394 		 * Remember the fault and fault adddress
1395 		 * for real-time (SIGPROF) profiling.
1396 		 */
1397 		lwp->lwp_lastfault = fault;
1398 		lwp->lwp_lastfaddr = siginfo.si_addr;
1399 
1400 		DTRACE_PROC2(fault, int, fault, ksiginfo_t *, &siginfo);
1401 
1402 		/*
1403 		 * If a debugger has declared this fault to be an
1404 		 * event of interest, stop the lwp.  Otherwise just
1405 		 * deliver the associated signal.
1406 		 */
1407 		if (siginfo.si_signo != SIGKILL &&
1408 		    prismember(&p->p_fltmask, fault) &&
1409 		    stop_on_fault(fault, &siginfo) == 0)
1410 			siginfo.si_signo = 0;
1411 	}
1412 
1413 	if (siginfo.si_signo)
1414 		trapsig(&siginfo, (fault == FLTCPCOVF)? 0 : 1);
1415 
1416 	if (lwp->lwp_oweupc)
1417 		profil_tick(rp->r_pc);
1418 
1419 	if (ct->t_astflag | ct->t_sig_check) {
1420 		/*
1421 		 * Turn off the AST flag before checking all the conditions that
1422 		 * may have caused an AST.  This flag is on whenever a signal or
1423 		 * unusual condition should be handled after the next trap or
1424 		 * syscall.
1425 		 */
1426 		astoff(ct);
1427 		/*
1428 		 * If a single-step trap occurred on a syscall (see above)
1429 		 * recognize it now.  Do this before checking for signals
1430 		 * because deferred_singlestep_trap() may generate a SIGTRAP to
1431 		 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
1432 		 */
1433 		if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
1434 			deferred_singlestep_trap((caddr_t)rp->r_pc);
1435 
1436 		ct->t_sig_check = 0;
1437 
1438 		mutex_enter(&p->p_lock);
1439 		if (curthread->t_proc_flag & TP_CHANGEBIND) {
1440 			timer_lwpbind();
1441 			curthread->t_proc_flag &= ~TP_CHANGEBIND;
1442 		}
1443 		mutex_exit(&p->p_lock);
1444 
1445 		/*
1446 		 * for kaio requests that are on the per-process poll queue,
1447 		 * aiop->aio_pollq, they're AIO_POLL bit is set, the kernel
1448 		 * should copyout their result_t to user memory. by copying
1449 		 * out the result_t, the user can poll on memory waiting
1450 		 * for the kaio request to complete.
1451 		 */
1452 		if (p->p_aio)
1453 			aio_cleanup(0);
1454 		/*
1455 		 * If this LWP was asked to hold, call holdlwp(), which will
1456 		 * stop.  holdlwps() sets this up and calls pokelwps() which
1457 		 * sets the AST flag.
1458 		 *
1459 		 * Also check TP_EXITLWP, since this is used by fresh new LWPs
1460 		 * through lwp_rtt().  That flag is set if the lwp_create(2)
1461 		 * syscall failed after creating the LWP.
1462 		 */
1463 		if (ISHOLD(p))
1464 			holdlwp();
1465 
1466 		/*
1467 		 * All code that sets signals and makes ISSIG evaluate true must
1468 		 * set t_astflag afterwards.
1469 		 */
1470 		if (ISSIG_PENDING(ct, lwp, p)) {
1471 			if (issig(FORREAL))
1472 				psig();
1473 			ct->t_sig_check = 1;
1474 		}
1475 
1476 		if (ct->t_rprof != NULL) {
1477 			realsigprof(0, 0);
1478 			ct->t_sig_check = 1;
1479 		}
1480 
1481 		/*
1482 		 * /proc can't enable/disable the trace bit itself
1483 		 * because that could race with the call gate used by
1484 		 * system calls via "lcall". If that happened, an
1485 		 * invalid EFLAGS would result. prstep()/prnostep()
1486 		 * therefore schedule an AST for the purpose.
1487 		 */
1488 		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
1489 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
1490 			rp->r_ps |= PS_T;
1491 		}
1492 		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
1493 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
1494 			rp->r_ps &= ~PS_T;
1495 		}
1496 	}
1497 
1498 out:	/* We can't get here from a system trap */
1499 	ASSERT(type & USER);
1500 
1501 	if (ISHOLD(p))
1502 		holdlwp();
1503 
1504 	/*
1505 	 * Set state to LWP_USER here so preempt won't give us a kernel
1506 	 * priority if it occurs after this point.  Call CL_TRAPRET() to
1507 	 * restore the user-level priority.
1508 	 *
1509 	 * It is important that no locks (other than spinlocks) be entered
1510 	 * after this point before returning to user mode (unless lwp_state
1511 	 * is set back to LWP_SYS).
1512 	 */
1513 	lwp->lwp_state = LWP_USER;
1514 
1515 	if (ct->t_trapret) {
1516 		ct->t_trapret = 0;
1517 		thread_lock(ct);
1518 		CL_TRAPRET(ct);
1519 		thread_unlock(ct);
1520 	}
1521 	if (CPU->cpu_runrun || curthread->t_schedflag & TS_ANYWAITQ)
1522 		preempt();
1523 	(void) new_mstate(ct, mstate);
1524 
1525 	/* Kernel probe */
1526 	TNF_PROBE_1(thread_state, "thread", /* CSTYLED */,
1527 	    tnf_microstate, state, LMS_USER);
1528 
1529 	return;
1530 
1531 cleanup:	/* system traps end up here */
1532 	ASSERT(!(type & USER));
1533 }
1534 
1535 /*
1536  * Patch non-zero to disable preemption of threads in the kernel.
1537  */
1538 int IGNORE_KERNEL_PREEMPTION = 0;	/* XXX - delete this someday */
1539 
1540 struct kpreempt_cnts {		/* kernel preemption statistics */
1541 	int	kpc_idle;	/* executing idle thread */
1542 	int	kpc_intr;	/* executing interrupt thread */
1543 	int	kpc_clock;	/* executing clock thread */
1544 	int	kpc_blocked;	/* thread has blocked preemption (t_preempt) */
1545 	int	kpc_notonproc;	/* thread is surrendering processor */
1546 	int	kpc_inswtch;	/* thread has ratified scheduling decision */
1547 	int	kpc_prilevel;	/* processor interrupt level is too high */
1548 	int	kpc_apreempt;	/* asynchronous preemption */
1549 	int	kpc_spreempt;	/* synchronous preemption */
1550 } kpreempt_cnts;
1551 
1552 /*
1553  * kernel preemption: forced rescheduling, preempt the running kernel thread.
1554  *	the argument is old PIL for an interrupt,
1555  *	or the distingished value KPREEMPT_SYNC.
1556  */
1557 void
1558 kpreempt(int asyncspl)
1559 {
1560 	kthread_t *ct = curthread;
1561 
1562 	if (IGNORE_KERNEL_PREEMPTION) {
1563 		aston(CPU->cpu_dispthread);
1564 		return;
1565 	}
1566 
1567 	/*
1568 	 * Check that conditions are right for kernel preemption
1569 	 */
1570 	do {
1571 		if (ct->t_preempt) {
1572 			/*
1573 			 * either a privileged thread (idle, panic, interrupt)
1574 			 *	or will check when t_preempt is lowered
1575 			 */
1576 			if (ct->t_pri < 0)
1577 				kpreempt_cnts.kpc_idle++;
1578 			else if (ct->t_flag & T_INTR_THREAD) {
1579 				kpreempt_cnts.kpc_intr++;
1580 				if (ct->t_pil == CLOCK_LEVEL)
1581 					kpreempt_cnts.kpc_clock++;
1582 			} else
1583 				kpreempt_cnts.kpc_blocked++;
1584 			aston(CPU->cpu_dispthread);
1585 			return;
1586 		}
1587 		if (ct->t_state != TS_ONPROC ||
1588 		    ct->t_disp_queue != CPU->cpu_disp) {
1589 			/* this thread will be calling swtch() shortly */
1590 			kpreempt_cnts.kpc_notonproc++;
1591 			if (CPU->cpu_thread != CPU->cpu_dispthread) {
1592 				/* already in swtch(), force another */
1593 				kpreempt_cnts.kpc_inswtch++;
1594 				siron();
1595 			}
1596 			return;
1597 		}
1598 		if (getpil() >= DISP_LEVEL) {
1599 			/*
1600 			 * We can't preempt this thread if it is at
1601 			 * a PIL >= DISP_LEVEL since it may be holding
1602 			 * a spin lock (like sched_lock).
1603 			 */
1604 			siron();	/* check back later */
1605 			kpreempt_cnts.kpc_prilevel++;
1606 			return;
1607 		}
1608 		if (!interrupts_enabled()) {
1609 			/*
1610 			 * Can't preempt while running with ints disabled
1611 			 */
1612 			kpreempt_cnts.kpc_prilevel++;
1613 			return;
1614 		}
1615 		if (asyncspl != KPREEMPT_SYNC)
1616 			kpreempt_cnts.kpc_apreempt++;
1617 		else
1618 			kpreempt_cnts.kpc_spreempt++;
1619 
1620 		ct->t_preempt++;
1621 		preempt();
1622 		ct->t_preempt--;
1623 	} while (CPU->cpu_kprunrun);
1624 }
1625 
1626 /*
1627  * Print out debugging info.
1628  */
1629 static void
1630 showregs(uint_t type, struct regs *rp, caddr_t addr)
1631 {
1632 	int s;
1633 
1634 	s = spl7();
1635 	type &= ~USER;
1636 	if (PTOU(curproc)->u_comm[0])
1637 		printf("%s: ", PTOU(curproc)->u_comm);
1638 	if (type < TRAP_TYPES)
1639 		printf("#%s %s\n", trap_type_mnemonic[type], trap_type[type]);
1640 	else
1641 		switch (type) {
1642 		case T_SYSCALL:
1643 			printf("Syscall Trap:\n");
1644 			break;
1645 		case T_AST:
1646 			printf("AST\n");
1647 			break;
1648 		default:
1649 			printf("Bad Trap = %d\n", type);
1650 			break;
1651 		}
1652 	if (type == T_PGFLT) {
1653 		printf("Bad %s fault at addr=0x%lx\n",
1654 		    USERMODE(rp->r_cs) ? "user": "kernel", (uintptr_t)addr);
1655 	} else if (addr) {
1656 		printf("addr=0x%lx\n", (uintptr_t)addr);
1657 	}
1658 
1659 	printf("pid=%d, pc=0x%lx, sp=0x%lx, eflags=0x%lx\n",
1660 	    (ttoproc(curthread) && ttoproc(curthread)->p_pidp) ?
1661 	    ttoproc(curthread)->p_pid : 0, rp->r_pc, rp->r_sp, rp->r_ps);
1662 
1663 #if defined(__lint)
1664 	/*
1665 	 * this clause can be deleted when lint bug 4870403 is fixed
1666 	 * (lint thinks that bit 32 is illegal in a %b format string)
1667 	 */
1668 	printf("cr0: %x cr4: %b\n",
1669 	    (uint_t)getcr0(), (uint_t)getcr4(), FMT_CR4);
1670 #else
1671 	printf("cr0: %b cr4: %b\n",
1672 	    (uint_t)getcr0(), FMT_CR0, (uint_t)getcr4(), FMT_CR4);
1673 #endif	/* __lint */
1674 
1675 	printf("cr2: %lx", getcr2());
1676 #if !defined(__xpv)
1677 	printf("cr3: %lx", getcr3());
1678 #if defined(__amd64)
1679 	printf("cr8: %lx\n", getcr8());
1680 #endif
1681 #endif
1682 	printf("\n");
1683 
1684 	dumpregs(rp);
1685 	splx(s);
1686 }
1687 
1688 static void
1689 dumpregs(struct regs *rp)
1690 {
1691 #if defined(__amd64)
1692 	const char fmt[] = "\t%3s: %16lx %3s: %16lx %3s: %16lx\n";
1693 
1694 	printf(fmt, "rdi", rp->r_rdi, "rsi", rp->r_rsi, "rdx", rp->r_rdx);
1695 	printf(fmt, "rcx", rp->r_rcx, " r8", rp->r_r8, " r9", rp->r_r9);
1696 	printf(fmt, "rax", rp->r_rax, "rbx", rp->r_rbx, "rbp", rp->r_rbp);
1697 	printf(fmt, "r10", rp->r_r10, "r11", rp->r_r11, "r12", rp->r_r12);
1698 	printf(fmt, "r13", rp->r_r13, "r14", rp->r_r14, "r15", rp->r_r15);
1699 
1700 	printf(fmt, "fsb", rdmsr(MSR_AMD_FSBASE), "gsb", rdmsr(MSR_AMD_GSBASE),
1701 	    " ds", rp->r_ds);
1702 	printf(fmt, " es", rp->r_es, " fs", rp->r_fs, " gs", rp->r_gs);
1703 
1704 	printf(fmt, "trp", rp->r_trapno, "err", rp->r_err, "rip", rp->r_rip);
1705 	printf(fmt, " cs", rp->r_cs, "rfl", rp->r_rfl, "rsp", rp->r_rsp);
1706 
1707 	printf("\t%3s: %16lx\n", " ss", rp->r_ss);
1708 
1709 #elif defined(__i386)
1710 	const char fmt[] = "\t%3s: %8lx %3s: %8lx %3s: %8lx %3s: %8lx\n";
1711 
1712 	printf(fmt, " gs", rp->r_gs, " fs", rp->r_fs,
1713 	    " es", rp->r_es, " ds", rp->r_ds);
1714 	printf(fmt, "edi", rp->r_edi, "esi", rp->r_esi,
1715 	    "ebp", rp->r_ebp, "esp", rp->r_esp);
1716 	printf(fmt, "ebx", rp->r_ebx, "edx", rp->r_edx,
1717 	    "ecx", rp->r_ecx, "eax", rp->r_eax);
1718 	printf(fmt, "trp", rp->r_trapno, "err", rp->r_err,
1719 	    "eip", rp->r_eip, " cs", rp->r_cs);
1720 	printf("\t%3s: %8lx %3s: %8lx %3s: %8lx\n",
1721 	    "efl", rp->r_efl, "usp", rp->r_uesp, " ss", rp->r_ss);
1722 
1723 #endif	/* __i386 */
1724 }
1725 
1726 /*
1727  * Test to see if the instruction is iret on i386 or iretq on amd64.
1728  *
1729  * On the hypervisor we can only test for nopop_sys_rtt_syscall. If true
1730  * then we are in the context of hypervisor's failsafe handler because it
1731  * tried to iret and failed due to a bad selector. See xen_failsafe_callback.
1732  */
1733 static int
1734 instr_is_iret(caddr_t pc)
1735 {
1736 
1737 #if defined(__xpv)
1738 	extern void nopop_sys_rtt_syscall(void);
1739 	return ((pc == (caddr_t)nopop_sys_rtt_syscall) ? 1 : 0);
1740 
1741 #else
1742 
1743 #if defined(__amd64)
1744 	static const uint8_t iret_insn[2] = { 0x48, 0xcf };	/* iretq */
1745 
1746 #elif defined(__i386)
1747 	static const uint8_t iret_insn[1] = { 0xcf };		/* iret */
1748 #endif	/* __i386 */
1749 	return (bcmp(pc, iret_insn, sizeof (iret_insn)) == 0);
1750 
1751 #endif	/* __xpv */
1752 }
1753 
1754 #if defined(__i386)
1755 
1756 /*
1757  * Test to see if the instruction is part of __SEGREGS_POP
1758  *
1759  * Note carefully the appallingly awful dependency between
1760  * the instruction sequence used in __SEGREGS_POP and these
1761  * instructions encoded here.
1762  */
1763 static int
1764 instr_is_segregs_pop(caddr_t pc)
1765 {
1766 	static const uint8_t movw_0_esp_gs[4] = { 0x8e, 0x6c, 0x24, 0x0 };
1767 	static const uint8_t movw_4_esp_fs[4] = { 0x8e, 0x64, 0x24, 0x4 };
1768 	static const uint8_t movw_8_esp_es[4] = { 0x8e, 0x44, 0x24, 0x8 };
1769 	static const uint8_t movw_c_esp_ds[4] = { 0x8e, 0x5c, 0x24, 0xc };
1770 
1771 	if (bcmp(pc, movw_0_esp_gs, sizeof (movw_0_esp_gs)) == 0 ||
1772 	    bcmp(pc, movw_4_esp_fs, sizeof (movw_4_esp_fs)) == 0 ||
1773 	    bcmp(pc, movw_8_esp_es, sizeof (movw_8_esp_es)) == 0 ||
1774 	    bcmp(pc, movw_c_esp_ds, sizeof (movw_c_esp_ds)) == 0)
1775 		return (1);
1776 
1777 	return (0);
1778 }
1779 
1780 #endif	/* __i386 */
1781 
1782 /*
1783  * Test to see if the instruction is part of _sys_rtt.
1784  *
1785  * Again on the hypervisor if we try to IRET to user land with a bad code
1786  * or stack selector we will get vectored through xen_failsafe_callback.
1787  * In which case we assume we got here via _sys_rtt since we only allow
1788  * IRET to user land to take place in _sys_rtt.
1789  */
1790 static int
1791 instr_is_sys_rtt(caddr_t pc)
1792 {
1793 	extern void _sys_rtt(), _sys_rtt_end();
1794 
1795 	if ((uintptr_t)pc < (uintptr_t)_sys_rtt ||
1796 	    (uintptr_t)pc > (uintptr_t)_sys_rtt_end)
1797 		return (0);
1798 
1799 	return (1);
1800 }
1801 
1802 /*
1803  * Handle #gp faults in kernel mode.
1804  *
1805  * One legitimate way this can happen is if we attempt to update segment
1806  * registers to naughty values on the way out of the kernel.
1807  *
1808  * This can happen in a couple of ways: someone - either accidentally or
1809  * on purpose - creates (setcontext(2), lwp_create(2)) or modifies
1810  * (signal(2)) a ucontext that contains silly segment register values.
1811  * Or someone - either accidentally or on purpose - modifies the prgregset_t
1812  * of a subject process via /proc to contain silly segment register values.
1813  *
1814  * (The unfortunate part is that we can end up discovering the bad segment
1815  * register value in the middle of an 'iret' after we've popped most of the
1816  * stack.  So it becomes quite difficult to associate an accurate ucontext
1817  * with the lwp, because the act of taking the #gp trap overwrites most of
1818  * what we were going to send the lwp.)
1819  *
1820  * OTOH if it turns out that's -not- the problem, and we're -not- an lwp
1821  * trying to return to user mode and we get a #gp fault, then we need
1822  * to die() -- which will happen if we return non-zero from this routine.
1823  */
1824 static int
1825 kern_gpfault(struct regs *rp)
1826 {
1827 	kthread_t *t = curthread;
1828 	proc_t *p = ttoproc(t);
1829 	klwp_t *lwp = ttolwp(t);
1830 	struct regs tmpregs, *trp = NULL;
1831 	caddr_t pc = (caddr_t)rp->r_pc;
1832 	int v;
1833 
1834 	/*
1835 	 * if we're not an lwp, or in the case of running native the
1836 	 * pc range is outside _sys_rtt, then we should immediately
1837 	 * be die()ing horribly.
1838 	 */
1839 	if (lwp == NULL || !instr_is_sys_rtt(pc))
1840 		return (1);
1841 
1842 	/*
1843 	 * So at least we're in the right part of the kernel.
1844 	 *
1845 	 * Disassemble the instruction at the faulting pc.
1846 	 * Once we know what it is, we carefully reconstruct the stack
1847 	 * based on the order in which the stack is deconstructed in
1848 	 * _sys_rtt. Ew.
1849 	 */
1850 	if (instr_is_iret(pc)) {
1851 		/*
1852 		 * We took the #gp while trying to perform the IRET.
1853 		 * This means that either %cs or %ss are bad.
1854 		 * All we know for sure is that most of the general
1855 		 * registers have been restored, including the
1856 		 * segment registers, and all we have left on the
1857 		 * topmost part of the lwp's stack are the
1858 		 * registers that the iretq was unable to consume.
1859 		 *
1860 		 * All the rest of the state was crushed by the #gp
1861 		 * which pushed -its- registers atop our old save area
1862 		 * (because we had to decrement the stack pointer, sigh) so
1863 		 * all that we can try and do is to reconstruct the
1864 		 * crushed frame from the #gp trap frame itself.
1865 		 */
1866 		trp = &tmpregs;
1867 		trp->r_ss = lwptoregs(lwp)->r_ss;
1868 		trp->r_sp = lwptoregs(lwp)->r_sp;
1869 		trp->r_ps = lwptoregs(lwp)->r_ps;
1870 		trp->r_cs = lwptoregs(lwp)->r_cs;
1871 		trp->r_pc = lwptoregs(lwp)->r_pc;
1872 		bcopy(rp, trp, offsetof(struct regs, r_pc));
1873 
1874 		/*
1875 		 * Validate simple math
1876 		 */
1877 		ASSERT(trp->r_pc == lwptoregs(lwp)->r_pc);
1878 		ASSERT(trp->r_err == rp->r_err);
1879 
1880 
1881 
1882 	}
1883 
1884 #if defined(__amd64)
1885 	if (trp == NULL && lwp->lwp_pcb.pcb_rupdate != 0) {
1886 
1887 		/*
1888 		 * This is the common case -- we're trying to load
1889 		 * a bad segment register value in the only section
1890 		 * of kernel code that ever loads segment registers.
1891 		 *
1892 		 * We don't need to do anything at this point because
1893 		 * the pcb contains all the pending segment register
1894 		 * state, and the regs are still intact because we
1895 		 * didn't adjust the stack pointer yet.  Given the fidelity
1896 		 * of all this, we could conceivably send a signal
1897 		 * to the lwp, rather than core-ing.
1898 		 */
1899 		trp = lwptoregs(lwp);
1900 		ASSERT((caddr_t)trp == (caddr_t)rp->r_sp);
1901 	}
1902 
1903 #elif defined(__i386)
1904 
1905 	if (trp == NULL && instr_is_segregs_pop(pc))
1906 		trp = lwptoregs(lwp);
1907 
1908 #endif	/* __i386 */
1909 
1910 	if (trp == NULL)
1911 		return (1);
1912 
1913 	/*
1914 	 * If we get to here, we're reasonably confident that we've
1915 	 * correctly decoded what happened on the way out of the kernel.
1916 	 * Rewrite the lwp's registers so that we can create a core dump
1917 	 * the (at least vaguely) represents the mcontext we were
1918 	 * being asked to restore when things went so terribly wrong.
1919 	 */
1920 
1921 	/*
1922 	 * Make sure that we have a meaningful %trapno and %err.
1923 	 */
1924 	trp->r_trapno = rp->r_trapno;
1925 	trp->r_err = rp->r_err;
1926 
1927 	if ((caddr_t)trp != (caddr_t)lwptoregs(lwp))
1928 		bcopy(trp, lwptoregs(lwp), sizeof (*trp));
1929 
1930 
1931 	mutex_enter(&p->p_lock);
1932 	lwp->lwp_cursig = SIGSEGV;
1933 	mutex_exit(&p->p_lock);
1934 
1935 	/*
1936 	 * Terminate all LWPs but don't discard them.  If another lwp beat
1937 	 * us to the punch by calling exit(), evaporate now.
1938 	 */
1939 	proc_is_exiting(p);
1940 	if (exitlwps(1) != 0) {
1941 		mutex_enter(&p->p_lock);
1942 		lwp_exit();
1943 	}
1944 
1945 	if (audit_active)		/* audit core dump */
1946 		audit_core_start(SIGSEGV);
1947 	v = core(SIGSEGV, B_FALSE);
1948 	if (audit_active)		/* audit core dump */
1949 		audit_core_finish(v ? CLD_KILLED : CLD_DUMPED);
1950 	exit(v ? CLD_KILLED : CLD_DUMPED, SIGSEGV);
1951 	return (0);
1952 }
1953 
1954 /*
1955  * dump_tss() - Display the TSS structure
1956  */
1957 
1958 #if !defined(__xpv)
1959 #if defined(__amd64)
1960 
1961 static void
1962 dump_tss(void)
1963 {
1964 	const char tss_fmt[] = "tss.%s:\t0x%p\n";  /* Format string */
1965 	struct tss *tss = CPU->cpu_tss;
1966 
1967 	printf(tss_fmt, "tss_rsp0", (void *)tss->tss_rsp0);
1968 	printf(tss_fmt, "tss_rsp1", (void *)tss->tss_rsp1);
1969 	printf(tss_fmt, "tss_rsp2", (void *)tss->tss_rsp2);
1970 
1971 	printf(tss_fmt, "tss_ist1", (void *)tss->tss_ist1);
1972 	printf(tss_fmt, "tss_ist2", (void *)tss->tss_ist2);
1973 	printf(tss_fmt, "tss_ist3", (void *)tss->tss_ist3);
1974 	printf(tss_fmt, "tss_ist4", (void *)tss->tss_ist4);
1975 	printf(tss_fmt, "tss_ist5", (void *)tss->tss_ist5);
1976 	printf(tss_fmt, "tss_ist6", (void *)tss->tss_ist6);
1977 	printf(tss_fmt, "tss_ist7", (void *)tss->tss_ist7);
1978 }
1979 
1980 #elif defined(__i386)
1981 
1982 static void
1983 dump_tss(void)
1984 {
1985 	const char tss_fmt[] = "tss.%s:\t0x%p\n";  /* Format string */
1986 	struct tss *tss = CPU->cpu_tss;
1987 
1988 	printf(tss_fmt, "tss_link", (void *)(uintptr_t)tss->tss_link);
1989 	printf(tss_fmt, "tss_esp0", (void *)(uintptr_t)tss->tss_esp0);
1990 	printf(tss_fmt, "tss_ss0", (void *)(uintptr_t)tss->tss_ss0);
1991 	printf(tss_fmt, "tss_esp1", (void *)(uintptr_t)tss->tss_esp1);
1992 	printf(tss_fmt, "tss_ss1", (void *)(uintptr_t)tss->tss_ss1);
1993 	printf(tss_fmt, "tss_esp2", (void *)(uintptr_t)tss->tss_esp2);
1994 	printf(tss_fmt, "tss_ss2", (void *)(uintptr_t)tss->tss_ss2);
1995 	printf(tss_fmt, "tss_cr3", (void *)(uintptr_t)tss->tss_cr3);
1996 	printf(tss_fmt, "tss_eip", (void *)(uintptr_t)tss->tss_eip);
1997 	printf(tss_fmt, "tss_eflags", (void *)(uintptr_t)tss->tss_eflags);
1998 	printf(tss_fmt, "tss_eax", (void *)(uintptr_t)tss->tss_eax);
1999 	printf(tss_fmt, "tss_ebx", (void *)(uintptr_t)tss->tss_ebx);
2000 	printf(tss_fmt, "tss_ecx", (void *)(uintptr_t)tss->tss_ecx);
2001 	printf(tss_fmt, "tss_edx", (void *)(uintptr_t)tss->tss_edx);
2002 	printf(tss_fmt, "tss_esp", (void *)(uintptr_t)tss->tss_esp);
2003 }
2004 
2005 #endif	/* __amd64 */
2006 #endif	/* !__xpv */
2007 
2008 #if defined(TRAPTRACE)
2009 
2010 int ttrace_nrec = 10;		/* number of records to dump out */
2011 int ttrace_dump_nregs = 0;	/* dump out this many records with regs too */
2012 
2013 /*
2014  * Dump out the last ttrace_nrec traptrace records on each CPU
2015  */
2016 static void
2017 dump_ttrace(void)
2018 {
2019 	trap_trace_ctl_t *ttc;
2020 	trap_trace_rec_t *rec;
2021 	uintptr_t current;
2022 	int i, j, k;
2023 	int n = NCPU;
2024 #if defined(__amd64)
2025 	const char banner[] =
2026 	    "\ncpu          address    timestamp "
2027 	    "type  vc  handler   pc\n";
2028 	const char fmt1[] = "%3d %016lx %12llx ";
2029 #elif defined(__i386)
2030 	const char banner[] =
2031 	    "\ncpu  address     timestamp type  vc  handler   pc\n";
2032 	const char fmt1[] = "%3d %08lx %12llx ";
2033 #endif
2034 	const char fmt2[] = "%4s %3x ";
2035 	const char fmt3[] = "%8s ";
2036 
2037 	if (ttrace_nrec == 0)
2038 		return;
2039 
2040 	printf(banner);
2041 
2042 	for (i = 0; i < n; i++) {
2043 		ttc = &trap_trace_ctl[i];
2044 		if (ttc->ttc_first == NULL)
2045 			continue;
2046 
2047 		current = ttc->ttc_next - sizeof (trap_trace_rec_t);
2048 		for (j = 0; j < ttrace_nrec; j++) {
2049 			struct sysent	*sys;
2050 			struct autovec	*vec;
2051 			extern struct av_head autovect[];
2052 			int type;
2053 			ulong_t	off;
2054 			char *sym, *stype;
2055 
2056 			if (current < ttc->ttc_first)
2057 				current =
2058 				    ttc->ttc_limit - sizeof (trap_trace_rec_t);
2059 
2060 			if (current == NULL)
2061 				continue;
2062 
2063 			rec = (trap_trace_rec_t *)current;
2064 
2065 			if (rec->ttr_stamp == 0)
2066 				break;
2067 
2068 			printf(fmt1, i, (uintptr_t)rec, rec->ttr_stamp);
2069 
2070 			switch (rec->ttr_marker) {
2071 			case TT_SYSCALL:
2072 			case TT_SYSENTER:
2073 			case TT_SYSC:
2074 			case TT_SYSC64:
2075 #if defined(__amd64)
2076 				sys = &sysent32[rec->ttr_sysnum];
2077 				switch (rec->ttr_marker) {
2078 				case TT_SYSC64:
2079 					sys = &sysent[rec->ttr_sysnum];
2080 					/*FALLTHROUGH*/
2081 #elif defined(__i386)
2082 				sys = &sysent[rec->ttr_sysnum];
2083 				switch (rec->ttr_marker) {
2084 				case TT_SYSC64:
2085 #endif
2086 				case TT_SYSC:
2087 					stype = "sysc";	/* syscall */
2088 					break;
2089 				case TT_SYSCALL:
2090 					stype = "lcal";	/* lcall */
2091 					break;
2092 				case TT_SYSENTER:
2093 					stype = "syse";	/* sysenter */
2094 					break;
2095 				default:
2096 					break;
2097 				}
2098 				printf(fmt2, "sysc", rec->ttr_sysnum);
2099 				if (sys != NULL) {
2100 					sym = kobj_getsymname(
2101 					    (uintptr_t)sys->sy_callc,
2102 					    &off);
2103 					if (sym != NULL)
2104 						printf(fmt3, sym);
2105 					else
2106 						printf("%p ", sys->sy_callc);
2107 				} else {
2108 					printf(fmt3, "unknown");
2109 				}
2110 				break;
2111 
2112 			case TT_INTERRUPT:
2113 				printf(fmt2, "intr", rec->ttr_vector);
2114 				vec = (&autovect[rec->ttr_vector])->avh_link;
2115 				if (vec != NULL) {
2116 					sym = kobj_getsymname(
2117 					    (uintptr_t)vec->av_vector, &off);
2118 					if (sym != NULL)
2119 						printf(fmt3, sym);
2120 					else
2121 						printf("%p ", vec->av_vector);
2122 				} else {
2123 					printf(fmt3, "unknown ");
2124 				}
2125 				break;
2126 
2127 			case TT_TRAP:
2128 			case TT_EVENT:
2129 				type = rec->ttr_regs.r_trapno;
2130 				printf(fmt2, "trap", type);
2131 				if (type < TRAP_TYPES)
2132 					printf("     #%s ",
2133 					    trap_type_mnemonic[type]);
2134 				else
2135 					switch (type) {
2136 					case T_AST:
2137 						printf(fmt3, "ast");
2138 						break;
2139 					default:
2140 						printf(fmt3, "");
2141 						break;
2142 					}
2143 				break;
2144 
2145 			case TT_XCALL:
2146 				printf(fmt2, "xcal",
2147 				    rec->ttr_info.xc_entry.xce_marker);
2148 				printf(fmt3, "");
2149 				break;
2150 
2151 			default:
2152 				break;
2153 			}
2154 
2155 			sym = kobj_getsymname(rec->ttr_regs.r_pc, &off);
2156 			if (sym != NULL)
2157 				printf("%s+%lx\n", sym, off);
2158 			else
2159 				printf("%lx\n", rec->ttr_regs.r_pc);
2160 
2161 			if (ttrace_dump_nregs-- > 0) {
2162 				int s;
2163 
2164 				if (rec->ttr_marker == TT_INTERRUPT)
2165 					printf(
2166 					    "\t\tipl %x spl %x pri %x\n",
2167 					    rec->ttr_ipl,
2168 					    rec->ttr_spl,
2169 					    rec->ttr_pri);
2170 
2171 				dumpregs(&rec->ttr_regs);
2172 
2173 				printf("\t%3s: %p\n\n", " ct",
2174 				    (void *)rec->ttr_curthread);
2175 
2176 				/*
2177 				 * print out the pc stack that we recorded
2178 				 * at trap time (if any)
2179 				 */
2180 				for (s = 0; s < rec->ttr_sdepth; s++) {
2181 					uintptr_t fullpc;
2182 
2183 					if (s >= TTR_STACK_DEPTH) {
2184 						printf("ttr_sdepth corrupt\n");
2185 						break;
2186 					}
2187 
2188 					fullpc = (uintptr_t)rec->ttr_stack[s];
2189 
2190 					sym = kobj_getsymname(fullpc, &off);
2191 					if (sym != NULL)
2192 						printf("-> %s+0x%lx()\n",
2193 						    sym, off);
2194 					else
2195 						printf("-> 0x%lx()\n", fullpc);
2196 				}
2197 				printf("\n");
2198 			}
2199 			current -= sizeof (trap_trace_rec_t);
2200 		}
2201 	}
2202 }
2203 
2204 /*
2205  * Help with constructing traptrace records in C
2206  */
2207 trap_trace_rec_t *
2208 trap_trace_get_traceptr(uint8_t marker, ulong_t pc, ulong_t sp)
2209 {
2210 	trap_trace_rec_t *ttr;
2211 
2212 	if (trap_trace_freeze)
2213 		ttr = &trap_trace_postmort;
2214 	else {
2215 		trap_trace_ctl_t *ttc = &trap_trace_ctl[CPU->cpu_id];
2216 
2217 		ttr = (void *)ttc->ttc_next;
2218 
2219 		if (ttc->ttc_next >= ttc->ttc_limit)
2220 			ttc->ttc_next = ttc->ttc_first;
2221 		else
2222 			ttc->ttc_next += sizeof (trap_trace_rec_t);
2223 	}
2224 
2225 	ttr->ttr_regs.r_sp = sp;
2226 	ttr->ttr_regs.r_pc = pc;
2227 	ttr->ttr_cr2 = getcr2();
2228 	ttr->ttr_curthread = (uintptr_t)curthread;
2229 	ttr->ttr_stamp = tsc_read();
2230 	ttr->ttr_marker = marker;
2231 	return (ttr);
2232 }
2233 
2234 #endif	/* TRAPTRACE */
2235 
2236 void
2237 panic_showtrap(struct panic_trap_info *tip)
2238 {
2239 	showregs(tip->trap_type, tip->trap_regs, tip->trap_addr);
2240 
2241 #if defined(TRAPTRACE)
2242 	dump_ttrace();
2243 #endif
2244 
2245 #if !defined(__xpv)
2246 	if (tip->trap_type == T_DBLFLT)
2247 		dump_tss();
2248 #endif
2249 }
2250 
2251 void
2252 panic_savetrap(panic_data_t *pdp, struct panic_trap_info *tip)
2253 {
2254 	panic_saveregs(pdp, tip->trap_regs);
2255 }
2256