xref: /titanic_41/usr/src/uts/intel/ia32/os/archdep.c (revision e2c5185af3c50d9510e5df68aa37abdc6c0d3aac)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
26 /*	  All Rights Reserved  	*/
27 
28 #include <sys/param.h>
29 #include <sys/types.h>
30 #include <sys/vmparam.h>
31 #include <sys/systm.h>
32 #include <sys/signal.h>
33 #include <sys/stack.h>
34 #include <sys/regset.h>
35 #include <sys/privregs.h>
36 #include <sys/frame.h>
37 #include <sys/proc.h>
38 #include <sys/psw.h>
39 #include <sys/siginfo.h>
40 #include <sys/cpuvar.h>
41 #include <sys/asm_linkage.h>
42 #include <sys/kmem.h>
43 #include <sys/errno.h>
44 #include <sys/bootconf.h>
45 #include <sys/archsystm.h>
46 #include <sys/debug.h>
47 #include <sys/elf.h>
48 #include <sys/spl.h>
49 #include <sys/time.h>
50 #include <sys/atomic.h>
51 #include <sys/sysmacros.h>
52 #include <sys/cmn_err.h>
53 #include <sys/modctl.h>
54 #include <sys/kobj.h>
55 #include <sys/panic.h>
56 #include <sys/reboot.h>
57 #include <sys/time.h>
58 #include <sys/fp.h>
59 #include <sys/x86_archext.h>
60 #include <sys/auxv.h>
61 #include <sys/auxv_386.h>
62 #include <sys/dtrace.h>
63 #include <sys/brand.h>
64 #include <sys/machbrand.h>
65 
66 extern const struct fnsave_state x87_initial;
67 extern const struct fxsave_state sse_initial;
68 
69 /*
70  * Map an fnsave-formatted save area into an fxsave-formatted save area.
71  *
72  * Most fields are the same width, content and semantics.  However
73  * the tag word is compressed.
74  */
75 static void
76 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
77 {
78 	uint_t i, tagbits;
79 
80 	fx->fx_fcw = fn->f_fcw;
81 	fx->fx_fsw = fn->f_fsw;
82 
83 	/*
84 	 * copy element by element (because of holes)
85 	 */
86 	for (i = 0; i < 8; i++)
87 		bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
88 		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
89 
90 	/*
91 	 * synthesize compressed tag bits
92 	 */
93 	fx->fx_fctw = 0;
94 	for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
95 		if ((tagbits & 3) != 3)
96 			fx->fx_fctw |= (1 << i);
97 
98 	fx->fx_fop = fn->f_fop;
99 
100 #if defined(__amd64)
101 	fx->fx_rip = (uint64_t)fn->f_eip;
102 	fx->fx_rdp = (uint64_t)fn->f_dp;
103 #else
104 	fx->fx_eip = fn->f_eip;
105 	fx->fx_cs = fn->f_cs;
106 	fx->__fx_ign0 = 0;
107 	fx->fx_dp = fn->f_dp;
108 	fx->fx_ds = fn->f_ds;
109 	fx->__fx_ign1 = 0;
110 #endif
111 }
112 
113 /*
114  * Map from an fxsave-format save area to an fnsave-format save area.
115  */
116 static void
117 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
118 {
119 	uint_t i, top, tagbits;
120 
121 	fn->f_fcw = fx->fx_fcw;
122 	fn->__f_ign0 = 0;
123 	fn->f_fsw = fx->fx_fsw;
124 	fn->__f_ign1 = 0;
125 
126 	top = (fx->fx_fsw & FPS_TOP) >> 11;
127 
128 	/*
129 	 * copy element by element (because of holes)
130 	 */
131 	for (i = 0; i < 8; i++)
132 		bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
133 		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
134 
135 	/*
136 	 * synthesize uncompressed tag bits
137 	 */
138 	fn->f_ftw = 0;
139 	for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
140 		uint_t ibit, expo;
141 		const uint16_t *fpp;
142 		static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
143 
144 		if ((tagbits & 1) == 0) {
145 			fn->f_ftw |= 3 << (i << 1);	/* empty */
146 			continue;
147 		}
148 
149 		/*
150 		 * (tags refer to *physical* registers)
151 		 */
152 		fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
153 		ibit = fpp[3] >> 15;
154 		expo = fpp[4] & 0x7fff;
155 
156 		if (ibit && expo != 0 && expo != 0x7fff)
157 			continue;			/* valid fp number */
158 
159 		if (bcmp(fpp, &zero, sizeof (zero)))
160 			fn->f_ftw |= 2 << (i << 1);	/* NaN */
161 		else
162 			fn->f_ftw |= 1 << (i << 1);	/* fp zero */
163 	}
164 
165 	fn->f_fop = fx->fx_fop;
166 
167 	fn->__f_ign2 = 0;
168 #if defined(__amd64)
169 	fn->f_eip = (uint32_t)fx->fx_rip;
170 	fn->f_cs = U32CS_SEL;
171 	fn->f_dp = (uint32_t)fx->fx_rdp;
172 	fn->f_ds = UDS_SEL;
173 #else
174 	fn->f_eip = fx->fx_eip;
175 	fn->f_cs = fx->fx_cs;
176 	fn->f_dp = fx->fx_dp;
177 	fn->f_ds = fx->fx_ds;
178 #endif
179 	fn->__f_ign3 = 0;
180 }
181 
182 /*
183  * Map from an fpregset_t into an fxsave-format save area
184  */
185 static void
186 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
187 {
188 #if defined(__amd64)
189 	bcopy(fp, fx, sizeof (*fx));
190 #else
191 	const struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
192 
193 	fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
194 	fx->fx_mxcsr = fc->mxcsr;
195 	bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
196 #endif
197 	/*
198 	 * avoid useless #gp exceptions - mask reserved bits
199 	 */
200 	fx->fx_mxcsr &= sse_mxcsr_mask;
201 }
202 
203 /*
204  * Map from an fxsave-format save area into a fpregset_t
205  */
206 static void
207 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
208 {
209 #if defined(__amd64)
210 	bcopy(fx, fp, sizeof (*fx));
211 #else
212 	struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
213 
214 	fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
215 	fc->mxcsr = fx->fx_mxcsr;
216 	bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
217 #endif
218 }
219 
220 #if defined(_SYSCALL32_IMPL)
221 static void
222 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
223 {
224 	const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
225 
226 	fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
227 	/*
228 	 * avoid useless #gp exceptions - mask reserved bits
229 	 */
230 	fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
231 	bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
232 }
233 
234 static void
235 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
236 {
237 	struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
238 
239 	fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
240 	fc->mxcsr = fx->fx_mxcsr;
241 	bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
242 }
243 
244 static void
245 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
246 {
247 	fxsave_to_fpregset32((struct fxsave_state *)src, dst);
248 	dst->fp_reg_set.fpchip_state.status =
249 	    src->fp_reg_set.fpchip_state.status;
250 	dst->fp_reg_set.fpchip_state.xstatus =
251 	    src->fp_reg_set.fpchip_state.xstatus;
252 }
253 
254 static void
255 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
256 {
257 	fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
258 	dst->fp_reg_set.fpchip_state.status =
259 	    src->fp_reg_set.fpchip_state.status;
260 	dst->fp_reg_set.fpchip_state.xstatus =
261 	    src->fp_reg_set.fpchip_state.xstatus;
262 }
263 #endif
264 
265 /*
266  * Set floating-point registers from a native fpregset_t.
267  */
268 void
269 setfpregs(klwp_t *lwp, fpregset_t *fp)
270 {
271 	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
272 
273 	if (fpu->fpu_flags & FPU_EN) {
274 		if (!(fpu->fpu_flags & FPU_VALID)) {
275 			/*
276 			 * FPU context is still active, release the
277 			 * ownership.
278 			 */
279 			fp_free(fpu, 0);
280 		}
281 #if !defined(__amd64)
282 		if (fp_kind == __FP_SSE) {
283 #endif
284 			fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx);
285 			fpu->fpu_regs.kfpu_xstatus =
286 			    fp->fp_reg_set.fpchip_state.xstatus;
287 #if !defined(__amd64)
288 		} else
289 			bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn,
290 			    sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
291 #endif
292 		fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
293 		fpu->fpu_flags |= FPU_VALID;
294 	} else {
295 		/*
296 		 * If we are trying to change the FPU state of a thread which
297 		 * hasn't yet initialized floating point, store the state in
298 		 * the pcb and indicate that the state is valid.  When the
299 		 * thread enables floating point, it will use this state instead
300 		 * of the default state.
301 		 */
302 #if !defined(__amd64)
303 		if (fp_kind == __FP_SSE) {
304 #endif
305 			fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx);
306 			fpu->fpu_regs.kfpu_xstatus =
307 			    fp->fp_reg_set.fpchip_state.xstatus;
308 #if !defined(__amd64)
309 		} else
310 			bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn,
311 			    sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
312 #endif
313 		fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
314 		fpu->fpu_flags |= FPU_VALID;
315 	}
316 }
317 
318 /*
319  * Get floating-point registers into a native fpregset_t.
320  */
321 void
322 getfpregs(klwp_t *lwp, fpregset_t *fp)
323 {
324 	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
325 
326 	kpreempt_disable();
327 	if (fpu->fpu_flags & FPU_EN) {
328 		/*
329 		 * If we have FPU hw and the thread's pcb doesn't have
330 		 * a valid FPU state then get the state from the hw.
331 		 */
332 		if (fpu_exists && ttolwp(curthread) == lwp &&
333 		    !(fpu->fpu_flags & FPU_VALID))
334 			fp_save(fpu); /* get the current FPU state */
335 	}
336 
337 	/*
338 	 * There are 3 possible cases we have to be aware of here:
339 	 *
340 	 * 1. FPU is enabled.  FPU state is stored in the current LWP.
341 	 *
342 	 * 2. FPU is not enabled, and there have been no intervening /proc
343 	 *    modifications.  Return initial FPU state.
344 	 *
345 	 * 3. FPU is not enabled, but a /proc consumer has modified FPU state.
346 	 *    FPU state is stored in the current LWP.
347 	 */
348 	if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) {
349 		/*
350 		 * Cases 1 and 3.
351 		 */
352 #if !defined(__amd64)
353 		if (fp_kind == __FP_SSE) {
354 #endif
355 			fxsave_to_fpregset(&fpu->fpu_regs.kfpu_u.kfpu_fx, fp);
356 			fp->fp_reg_set.fpchip_state.xstatus =
357 			    fpu->fpu_regs.kfpu_xstatus;
358 #if !defined(__amd64)
359 		} else
360 			bcopy(&fpu->fpu_regs.kfpu_u.kfpu_fn, fp,
361 			    sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
362 #endif
363 		fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
364 	} else {
365 		/*
366 		 * Case 2.
367 		 */
368 #if !defined(__amd64)
369 		if (fp_kind == __FP_SSE) {
370 #endif
371 			fxsave_to_fpregset(&sse_initial, fp);
372 			fp->fp_reg_set.fpchip_state.xstatus =
373 			    fpu->fpu_regs.kfpu_xstatus;
374 #if !defined(__amd64)
375 		} else
376 			bcopy(&x87_initial, fp, sizeof (x87_initial));
377 #endif
378 		fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
379 	}
380 	kpreempt_enable();
381 }
382 
383 #if defined(_SYSCALL32_IMPL)
384 
385 /*
386  * Set floating-point registers from an fpregset32_t.
387  */
388 void
389 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
390 {
391 	fpregset_t fpregs;
392 
393 	fpregset_32ton(fp, &fpregs);
394 	setfpregs(lwp, &fpregs);
395 }
396 
397 /*
398  * Get floating-point registers into an fpregset32_t.
399  */
400 void
401 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
402 {
403 	fpregset_t fpregs;
404 
405 	getfpregs(lwp, &fpregs);
406 	fpregset_nto32(&fpregs, fp);
407 }
408 
409 #endif	/* _SYSCALL32_IMPL */
410 
411 /*
412  * Return the general registers
413  */
414 void
415 getgregs(klwp_t *lwp, gregset_t grp)
416 {
417 	struct regs *rp = lwptoregs(lwp);
418 #if defined(__amd64)
419 	struct pcb *pcb = &lwp->lwp_pcb;
420 	int thisthread = lwptot(lwp) == curthread;
421 
422 	grp[REG_RDI] = rp->r_rdi;
423 	grp[REG_RSI] = rp->r_rsi;
424 	grp[REG_RDX] = rp->r_rdx;
425 	grp[REG_RCX] = rp->r_rcx;
426 	grp[REG_R8] = rp->r_r8;
427 	grp[REG_R9] = rp->r_r9;
428 	grp[REG_RAX] = rp->r_rax;
429 	grp[REG_RBX] = rp->r_rbx;
430 	grp[REG_RBP] = rp->r_rbp;
431 	grp[REG_R10] = rp->r_r10;
432 	grp[REG_R11] = rp->r_r11;
433 	grp[REG_R12] = rp->r_r12;
434 	grp[REG_R13] = rp->r_r13;
435 	grp[REG_R14] = rp->r_r14;
436 	grp[REG_R15] = rp->r_r15;
437 	grp[REG_FSBASE] = pcb->pcb_fsbase;
438 	grp[REG_GSBASE] = pcb->pcb_gsbase;
439 	if (thisthread)
440 		kpreempt_disable();
441 	if (pcb->pcb_rupdate == 1) {
442 		grp[REG_DS] = pcb->pcb_ds;
443 		grp[REG_ES] = pcb->pcb_es;
444 		grp[REG_FS] = pcb->pcb_fs;
445 		grp[REG_GS] = pcb->pcb_gs;
446 	} else {
447 		grp[REG_DS] = rp->r_ds;
448 		grp[REG_ES] = rp->r_es;
449 		grp[REG_FS] = rp->r_fs;
450 		grp[REG_GS] = rp->r_gs;
451 	}
452 	if (thisthread)
453 		kpreempt_enable();
454 	grp[REG_TRAPNO] = rp->r_trapno;
455 	grp[REG_ERR] = rp->r_err;
456 	grp[REG_RIP] = rp->r_rip;
457 	grp[REG_CS] = rp->r_cs;
458 	grp[REG_SS] = rp->r_ss;
459 	grp[REG_RFL] = rp->r_rfl;
460 	grp[REG_RSP] = rp->r_rsp;
461 #else
462 	bcopy(&rp->r_gs, grp, sizeof (gregset_t));
463 #endif
464 }
465 
466 #if defined(_SYSCALL32_IMPL)
467 
468 void
469 getgregs32(klwp_t *lwp, gregset32_t grp)
470 {
471 	struct regs *rp = lwptoregs(lwp);
472 	struct pcb *pcb = &lwp->lwp_pcb;
473 	int thisthread = lwptot(lwp) == curthread;
474 
475 	if (thisthread)
476 		kpreempt_disable();
477 	if (pcb->pcb_rupdate == 1) {
478 		grp[GS] = (uint16_t)pcb->pcb_gs;
479 		grp[FS] = (uint16_t)pcb->pcb_fs;
480 		grp[DS] = (uint16_t)pcb->pcb_ds;
481 		grp[ES] = (uint16_t)pcb->pcb_es;
482 	} else {
483 		grp[GS] = (uint16_t)rp->r_gs;
484 		grp[FS] = (uint16_t)rp->r_fs;
485 		grp[DS] = (uint16_t)rp->r_ds;
486 		grp[ES] = (uint16_t)rp->r_es;
487 	}
488 	if (thisthread)
489 		kpreempt_enable();
490 	grp[EDI] = (greg32_t)rp->r_rdi;
491 	grp[ESI] = (greg32_t)rp->r_rsi;
492 	grp[EBP] = (greg32_t)rp->r_rbp;
493 	grp[ESP] = 0;
494 	grp[EBX] = (greg32_t)rp->r_rbx;
495 	grp[EDX] = (greg32_t)rp->r_rdx;
496 	grp[ECX] = (greg32_t)rp->r_rcx;
497 	grp[EAX] = (greg32_t)rp->r_rax;
498 	grp[TRAPNO] = (greg32_t)rp->r_trapno;
499 	grp[ERR] = (greg32_t)rp->r_err;
500 	grp[EIP] = (greg32_t)rp->r_rip;
501 	grp[CS] = (uint16_t)rp->r_cs;
502 	grp[EFL] = (greg32_t)rp->r_rfl;
503 	grp[UESP] = (greg32_t)rp->r_rsp;
504 	grp[SS] = (uint16_t)rp->r_ss;
505 }
506 
507 void
508 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
509 {
510 	mcontext_t *dmc = &dst->uc_mcontext;
511 	const mcontext32_t *smc = &src->uc_mcontext;
512 
513 	bzero(dst, sizeof (*dst));
514 	dst->uc_flags = src->uc_flags;
515 	dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
516 
517 	bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
518 
519 	dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
520 	dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
521 	dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
522 
523 	dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
524 	dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
525 	dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
526 	dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
527 	dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
528 	dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
529 	dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
530 	dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
531 	dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
532 	dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
533 	dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
534 	dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
535 	dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
536 	dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
537 	dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
538 	dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
539 	dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
540 	dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
541 
542 	/*
543 	 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
544 	 * otherwise there is no guarantee that anything in fpregs is valid.
545 	 */
546 	if (src->uc_flags & UC_FPU)
547 		fpregset_32ton(&src->uc_mcontext.fpregs,
548 		    &dst->uc_mcontext.fpregs);
549 }
550 
551 #endif	/* _SYSCALL32_IMPL */
552 
553 /*
554  * Return the user-level PC.
555  * If in a system call, return the address of the syscall trap.
556  */
557 greg_t
558 getuserpc()
559 {
560 	greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
561 	uint32_t insn;
562 
563 	if (curthread->t_sysnum == 0)
564 		return (upc);
565 
566 	/*
567 	 * We might've gotten here from sysenter (0xf 0x34),
568 	 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
569 	 *
570 	 * Go peek at the binary to figure it out..
571 	 */
572 	if (fuword32((void *)(upc - 2), &insn) != -1 &&
573 	    (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
574 		return (upc - 2);
575 	return (upc - 7);
576 }
577 
578 /*
579  * Protect segment registers from non-user privilege levels and GDT selectors
580  * other than USER_CS, USER_DS and lwp FS and GS values.  If the segment
581  * selector is non-null and not USER_CS/USER_DS, we make sure that the
582  * TI bit is set to point into the LDT and that the RPL is set to 3.
583  *
584  * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
585  * also explicitly zero the top 16 bits since they may be coming from the
586  * user's address space via setcontext(2) or /proc.
587  *
588  * Note about null selector. When running on the hypervisor if we allow a
589  * process to set its %cs to null selector with RPL of 0 the hypervisor will
590  * crash the domain. If running on bare metal we would get a #gp fault and
591  * be able to kill the process and continue on. Therefore we make sure to
592  * force RPL to SEL_UPL even for null selector when setting %cs.
593  */
594 
595 #if defined(IS_CS) || defined(IS_NOT_CS)
596 #error	"IS_CS and IS_NOT_CS already defined"
597 #endif
598 
599 #define	IS_CS		1
600 #define	IS_NOT_CS	0
601 
602 /*ARGSUSED*/
603 static greg_t
604 fix_segreg(greg_t sr, int iscs, model_t datamodel)
605 {
606 	switch (sr &= 0xffff) {
607 
608 	case 0:
609 		if (iscs == IS_CS)
610 			return (0 | SEL_UPL);
611 		else
612 			return (0);
613 
614 #if defined(__amd64)
615 	/*
616 	 * If lwp attempts to switch data model then force their
617 	 * code selector to be null selector.
618 	 */
619 	case U32CS_SEL:
620 		if (datamodel == DATAMODEL_NATIVE)
621 			return (0 | SEL_UPL);
622 		else
623 			return (sr);
624 
625 	case UCS_SEL:
626 		if (datamodel == DATAMODEL_ILP32)
627 			return (0 | SEL_UPL);
628 #elif defined(__i386)
629 	case UCS_SEL:
630 #endif
631 	/*FALLTHROUGH*/
632 	case UDS_SEL:
633 	case LWPFS_SEL:
634 	case LWPGS_SEL:
635 	case SEL_UPL:
636 		return (sr);
637 	default:
638 		break;
639 	}
640 
641 	/*
642 	 * Force it into the LDT in ring 3 for 32-bit processes, which by
643 	 * default do not have an LDT, so that any attempt to use an invalid
644 	 * selector will reference the (non-existant) LDT, and cause a #gp
645 	 * fault for the process.
646 	 *
647 	 * 64-bit processes get the null gdt selector since they
648 	 * are not allowed to have a private LDT.
649 	 */
650 #if defined(__amd64)
651 	if (datamodel == DATAMODEL_ILP32) {
652 		return (sr | SEL_TI_LDT | SEL_UPL);
653 	} else {
654 		if (iscs == IS_CS)
655 			return (0 | SEL_UPL);
656 		else
657 			return (0);
658 	}
659 
660 #elif defined(__i386)
661 	return (sr | SEL_TI_LDT | SEL_UPL);
662 #endif
663 }
664 
665 /*
666  * Set general registers.
667  */
668 void
669 setgregs(klwp_t *lwp, gregset_t grp)
670 {
671 	struct regs *rp = lwptoregs(lwp);
672 	model_t	datamodel = lwp_getdatamodel(lwp);
673 
674 #if defined(__amd64)
675 	struct pcb *pcb = &lwp->lwp_pcb;
676 	int thisthread = lwptot(lwp) == curthread;
677 
678 	if (datamodel == DATAMODEL_NATIVE) {
679 
680 		if (thisthread)
681 			(void) save_syscall_args();	/* copy the args */
682 
683 		rp->r_rdi = grp[REG_RDI];
684 		rp->r_rsi = grp[REG_RSI];
685 		rp->r_rdx = grp[REG_RDX];
686 		rp->r_rcx = grp[REG_RCX];
687 		rp->r_r8 = grp[REG_R8];
688 		rp->r_r9 = grp[REG_R9];
689 		rp->r_rax = grp[REG_RAX];
690 		rp->r_rbx = grp[REG_RBX];
691 		rp->r_rbp = grp[REG_RBP];
692 		rp->r_r10 = grp[REG_R10];
693 		rp->r_r11 = grp[REG_R11];
694 		rp->r_r12 = grp[REG_R12];
695 		rp->r_r13 = grp[REG_R13];
696 		rp->r_r14 = grp[REG_R14];
697 		rp->r_r15 = grp[REG_R15];
698 		rp->r_trapno = grp[REG_TRAPNO];
699 		rp->r_err = grp[REG_ERR];
700 		rp->r_rip = grp[REG_RIP];
701 		/*
702 		 * Setting %cs or %ss to anything else is quietly but
703 		 * quite definitely forbidden!
704 		 */
705 		rp->r_cs = UCS_SEL;
706 		rp->r_ss = UDS_SEL;
707 		rp->r_rsp = grp[REG_RSP];
708 
709 		if (thisthread)
710 			kpreempt_disable();
711 
712 		pcb->pcb_ds = UDS_SEL;
713 		pcb->pcb_es = UDS_SEL;
714 
715 		/*
716 		 * 64-bit processes -are- allowed to set their fsbase/gsbase
717 		 * values directly, but only if they're using the segment
718 		 * selectors that allow that semantic.
719 		 *
720 		 * (32-bit processes must use lwp_set_private().)
721 		 */
722 		pcb->pcb_fsbase = grp[REG_FSBASE];
723 		pcb->pcb_gsbase = grp[REG_GSBASE];
724 		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
725 		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
726 
727 		/*
728 		 * Ensure that we go out via update_sregs
729 		 */
730 		pcb->pcb_rupdate = 1;
731 		lwptot(lwp)->t_post_sys = 1;
732 		if (thisthread)
733 			kpreempt_enable();
734 #if defined(_SYSCALL32_IMPL)
735 	} else {
736 		rp->r_rdi = (uint32_t)grp[REG_RDI];
737 		rp->r_rsi = (uint32_t)grp[REG_RSI];
738 		rp->r_rdx = (uint32_t)grp[REG_RDX];
739 		rp->r_rcx = (uint32_t)grp[REG_RCX];
740 		rp->r_rax = (uint32_t)grp[REG_RAX];
741 		rp->r_rbx = (uint32_t)grp[REG_RBX];
742 		rp->r_rbp = (uint32_t)grp[REG_RBP];
743 		rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
744 		rp->r_err = (uint32_t)grp[REG_ERR];
745 		rp->r_rip = (uint32_t)grp[REG_RIP];
746 
747 		rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
748 		rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
749 
750 		rp->r_rsp = (uint32_t)grp[REG_RSP];
751 
752 		if (thisthread)
753 			kpreempt_disable();
754 
755 		pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
756 		pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
757 
758 		/*
759 		 * (See fsbase/gsbase commentary above)
760 		 */
761 		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
762 		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
763 
764 		/*
765 		 * Ensure that we go out via update_sregs
766 		 */
767 		pcb->pcb_rupdate = 1;
768 		lwptot(lwp)->t_post_sys = 1;
769 		if (thisthread)
770 			kpreempt_enable();
771 #endif
772 	}
773 
774 	/*
775 	 * Only certain bits of the flags register can be modified.
776 	 */
777 	rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
778 	    (grp[REG_RFL] & PSL_USERMASK);
779 
780 #elif defined(__i386)
781 
782 	/*
783 	 * Only certain bits of the flags register can be modified.
784 	 */
785 	grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK);
786 
787 	/*
788 	 * Copy saved registers from user stack.
789 	 */
790 	bcopy(grp, &rp->r_gs, sizeof (gregset_t));
791 
792 	rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel);
793 	rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel);
794 	rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel);
795 	rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel);
796 	rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel);
797 	rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel);
798 
799 #endif	/* __i386 */
800 }
801 
802 /*
803  * Determine whether eip is likely to have an interrupt frame
804  * on the stack.  We do this by comparing the address to the
805  * range of addresses spanned by several well-known routines.
806  */
807 extern void _interrupt();
808 extern void _allsyscalls();
809 extern void _cmntrap();
810 extern void fakesoftint();
811 
812 extern size_t _interrupt_size;
813 extern size_t _allsyscalls_size;
814 extern size_t _cmntrap_size;
815 extern size_t _fakesoftint_size;
816 
817 /*
818  * Get a pc-only stacktrace.  Used for kmem_alloc() buffer ownership tracking.
819  * Returns MIN(current stack depth, pcstack_limit).
820  */
821 int
822 getpcstack(pc_t *pcstack, int pcstack_limit)
823 {
824 	struct frame *fp = (struct frame *)getfp();
825 	struct frame *nextfp, *minfp, *stacktop;
826 	int depth = 0;
827 	int on_intr;
828 	uintptr_t pc;
829 
830 	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
831 		stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
832 	else
833 		stacktop = (struct frame *)curthread->t_stk;
834 	minfp = fp;
835 
836 	pc = ((struct regs *)fp)->r_pc;
837 
838 	while (depth < pcstack_limit) {
839 		nextfp = (struct frame *)fp->fr_savfp;
840 		pc = fp->fr_savpc;
841 		if (nextfp <= minfp || nextfp >= stacktop) {
842 			if (on_intr) {
843 				/*
844 				 * Hop from interrupt stack to thread stack.
845 				 */
846 				stacktop = (struct frame *)curthread->t_stk;
847 				minfp = (struct frame *)curthread->t_stkbase;
848 				on_intr = 0;
849 				continue;
850 			}
851 			break;
852 		}
853 		pcstack[depth++] = (pc_t)pc;
854 		fp = nextfp;
855 		minfp = fp;
856 	}
857 	return (depth);
858 }
859 
860 /*
861  * The following ELF header fields are defined as processor-specific
862  * in the V8 ABI:
863  *
864  *	e_ident[EI_DATA]	encoding of the processor-specific
865  *				data in the object file
866  *	e_machine		processor identification
867  *	e_flags			processor-specific flags associated
868  *				with the file
869  */
870 
871 /*
872  * The value of at_flags reflects a platform's cpu module support.
873  * at_flags is used to check for allowing a binary to execute and
874  * is passed as the value of the AT_FLAGS auxiliary vector.
875  */
876 int at_flags = 0;
877 
878 /*
879  * Check the processor-specific fields of an ELF header.
880  *
881  * returns 1 if the fields are valid, 0 otherwise
882  */
883 /*ARGSUSED2*/
884 int
885 elfheadcheck(
886 	unsigned char e_data,
887 	Elf32_Half e_machine,
888 	Elf32_Word e_flags)
889 {
890 	if (e_data != ELFDATA2LSB)
891 		return (0);
892 #if defined(__amd64)
893 	if (e_machine == EM_AMD64)
894 		return (1);
895 #endif
896 	return (e_machine == EM_386);
897 }
898 
899 uint_t auxv_hwcap_include = 0;	/* patch to enable unrecognized features */
900 uint_t auxv_hwcap_exclude = 0;	/* patch for broken cpus, debugging */
901 #if defined(_SYSCALL32_IMPL)
902 uint_t auxv_hwcap32_include = 0;	/* ditto for 32-bit apps */
903 uint_t auxv_hwcap32_exclude = 0;	/* ditto for 32-bit apps */
904 #endif
905 
906 /*
907  * Gather information about the processor and place it into auxv_hwcap
908  * so that it can be exported to the linker via the aux vector.
909  *
910  * We use this seemingly complicated mechanism so that we can ensure
911  * that /etc/system can be used to override what the system can or
912  * cannot discover for itself.
913  */
914 void
915 bind_hwcap(void)
916 {
917 	uint_t cpu_hwcap_flags = cpuid_pass4(NULL);
918 
919 	auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags) &
920 	    ~auxv_hwcap_exclude;
921 
922 #if defined(__amd64)
923 	/*
924 	 * On AMD processors, sysenter just doesn't work at all
925 	 * when the kernel is in long mode.  On IA-32e processors
926 	 * it does, but there's no real point in all the alternate
927 	 * mechanism when syscall works on both.
928 	 *
929 	 * Besides, the kernel's sysenter handler is expecting a
930 	 * 32-bit lwp ...
931 	 */
932 	auxv_hwcap &= ~AV_386_SEP;
933 #else
934 	/*
935 	 * 32-bit processes can -always- use the lahf/sahf instructions
936 	 */
937 	auxv_hwcap |= AV_386_AHF;
938 #endif
939 
940 	if (auxv_hwcap_include || auxv_hwcap_exclude)
941 		cmn_err(CE_CONT, "?user ABI extensions: %b\n",
942 		    auxv_hwcap, FMT_AV_386);
943 
944 #if defined(_SYSCALL32_IMPL)
945 	auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags) &
946 	    ~auxv_hwcap32_exclude;
947 
948 #if defined(__amd64)
949 	/*
950 	 * If this is an amd64 architecture machine from Intel, then
951 	 * syscall -doesn't- work in compatibility mode, only sysenter does.
952 	 *
953 	 * Sigh.
954 	 */
955 	if (!cpuid_syscall32_insn(NULL))
956 		auxv_hwcap32 &= ~AV_386_AMD_SYSC;
957 
958 	/*
959 	 * 32-bit processes can -always- use the lahf/sahf instructions
960 	 */
961 	auxv_hwcap32 |= AV_386_AHF;
962 #endif
963 
964 	if (auxv_hwcap32_include || auxv_hwcap32_exclude)
965 		cmn_err(CE_CONT, "?32-bit user ABI extensions: %b\n",
966 		    auxv_hwcap32, FMT_AV_386);
967 #endif
968 }
969 
970 /*
971  *	sync_icache() - this is called
972  *	in proc/fs/prusrio.c. x86 has an unified cache and therefore
973  *	this is a nop.
974  */
975 /* ARGSUSED */
976 void
977 sync_icache(caddr_t addr, uint_t len)
978 {
979 	/* Do nothing for now */
980 }
981 
982 /*ARGSUSED*/
983 void
984 sync_data_memory(caddr_t va, size_t len)
985 {
986 	/* Not implemented for this platform */
987 }
988 
989 int
990 __ipltospl(int ipl)
991 {
992 	return (ipltospl(ipl));
993 }
994 
995 /*
996  * The panic code invokes panic_saveregs() to record the contents of a
997  * regs structure into the specified panic_data structure for debuggers.
998  */
999 void
1000 panic_saveregs(panic_data_t *pdp, struct regs *rp)
1001 {
1002 	panic_nv_t *pnv = PANICNVGET(pdp);
1003 
1004 	struct cregs	creg;
1005 
1006 	getcregs(&creg);
1007 
1008 #if defined(__amd64)
1009 	PANICNVADD(pnv, "rdi", rp->r_rdi);
1010 	PANICNVADD(pnv, "rsi", rp->r_rsi);
1011 	PANICNVADD(pnv, "rdx", rp->r_rdx);
1012 	PANICNVADD(pnv, "rcx", rp->r_rcx);
1013 	PANICNVADD(pnv, "r8", rp->r_r8);
1014 	PANICNVADD(pnv, "r9", rp->r_r9);
1015 	PANICNVADD(pnv, "rax", rp->r_rax);
1016 	PANICNVADD(pnv, "rbx", rp->r_rbx);
1017 	PANICNVADD(pnv, "rbp", rp->r_rbp);
1018 	PANICNVADD(pnv, "r10", rp->r_r10);
1019 	PANICNVADD(pnv, "r10", rp->r_r10);
1020 	PANICNVADD(pnv, "r11", rp->r_r11);
1021 	PANICNVADD(pnv, "r12", rp->r_r12);
1022 	PANICNVADD(pnv, "r13", rp->r_r13);
1023 	PANICNVADD(pnv, "r14", rp->r_r14);
1024 	PANICNVADD(pnv, "r15", rp->r_r15);
1025 	PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
1026 	PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
1027 	PANICNVADD(pnv, "ds", rp->r_ds);
1028 	PANICNVADD(pnv, "es", rp->r_es);
1029 	PANICNVADD(pnv, "fs", rp->r_fs);
1030 	PANICNVADD(pnv, "gs", rp->r_gs);
1031 	PANICNVADD(pnv, "trapno", rp->r_trapno);
1032 	PANICNVADD(pnv, "err", rp->r_err);
1033 	PANICNVADD(pnv, "rip", rp->r_rip);
1034 	PANICNVADD(pnv, "cs", rp->r_cs);
1035 	PANICNVADD(pnv, "rflags", rp->r_rfl);
1036 	PANICNVADD(pnv, "rsp", rp->r_rsp);
1037 	PANICNVADD(pnv, "ss", rp->r_ss);
1038 	PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
1039 	PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
1040 	PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
1041 	PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
1042 #elif defined(__i386)
1043 	PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs);
1044 	PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs);
1045 	PANICNVADD(pnv, "es", (uint32_t)rp->r_es);
1046 	PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds);
1047 	PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi);
1048 	PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi);
1049 	PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp);
1050 	PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp);
1051 	PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx);
1052 	PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx);
1053 	PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx);
1054 	PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax);
1055 	PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno);
1056 	PANICNVADD(pnv, "err", (uint32_t)rp->r_err);
1057 	PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip);
1058 	PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs);
1059 	PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl);
1060 	PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp);
1061 	PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss);
1062 	PANICNVADD(pnv, "gdt", creg.cr_gdt);
1063 	PANICNVADD(pnv, "idt", creg.cr_idt);
1064 #endif	/* __i386 */
1065 
1066 	PANICNVADD(pnv, "ldt", creg.cr_ldt);
1067 	PANICNVADD(pnv, "task", creg.cr_task);
1068 	PANICNVADD(pnv, "cr0", creg.cr_cr0);
1069 	PANICNVADD(pnv, "cr2", creg.cr_cr2);
1070 	PANICNVADD(pnv, "cr3", creg.cr_cr3);
1071 	if (creg.cr_cr4)
1072 		PANICNVADD(pnv, "cr4", creg.cr_cr4);
1073 
1074 	PANICNVSET(pdp, pnv);
1075 }
1076 
1077 #define	TR_ARG_MAX 6	/* Max args to print, same as SPARC */
1078 
1079 #if !defined(__amd64)
1080 
1081 /*
1082  * Given a return address (%eip), determine the likely number of arguments
1083  * that were pushed on the stack prior to its execution.  We do this by
1084  * expecting that a typical call sequence consists of pushing arguments on
1085  * the stack, executing a call instruction, and then performing an add
1086  * on %esp to restore it to the value prior to pushing the arguments for
1087  * the call.  We attempt to detect such an add, and divide the addend
1088  * by the size of a word to determine the number of pushed arguments.
1089  *
1090  * If we do not find such an add, we punt and return TR_ARG_MAX. It is not
1091  * possible to reliably determine if a function took no arguments (i.e. was
1092  * void) because assembler routines do not reliably perform an add on %esp
1093  * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is
1094  * safer than returning 0.
1095  */
1096 static ulong_t
1097 argcount(uintptr_t eip)
1098 {
1099 	const uint8_t *ins = (const uint8_t *)eip;
1100 	ulong_t n;
1101 
1102 	enum {
1103 		M_MODRM_ESP = 0xc4,	/* Mod/RM byte indicates %esp */
1104 		M_ADD_IMM32 = 0x81,	/* ADD imm32 to r/m32 */
1105 		M_ADD_IMM8  = 0x83	/* ADD imm8 to r/m32 */
1106 	};
1107 
1108 	if (eip < KERNELBASE || ins[1] != M_MODRM_ESP)
1109 		return (TR_ARG_MAX);
1110 
1111 	switch (ins[0]) {
1112 	case M_ADD_IMM32:
1113 		n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24);
1114 		break;
1115 
1116 	case M_ADD_IMM8:
1117 		n = ins[2];
1118 		break;
1119 
1120 	default:
1121 		return (TR_ARG_MAX);
1122 	}
1123 
1124 	n /= sizeof (long);
1125 	return (MIN(n, TR_ARG_MAX));
1126 }
1127 
1128 #endif	/* !__amd64 */
1129 
1130 /*
1131  * Print a stack backtrace using the specified frame pointer.  We delay two
1132  * seconds before continuing, unless this is the panic traceback.
1133  * If we are in the process of panicking, we also attempt to write the
1134  * stack backtrace to a staticly assigned buffer, to allow the panic
1135  * code to find it and write it in to uncompressed pages within the
1136  * system crash dump.
1137  * Note that the frame for the starting stack pointer value is omitted because
1138  * the corresponding %eip is not known.
1139  */
1140 
1141 extern char *dump_stack_scratch;
1142 
1143 #if defined(__amd64)
1144 
1145 void
1146 traceback(caddr_t fpreg)
1147 {
1148 	struct frame	*fp = (struct frame *)fpreg;
1149 	struct frame	*nextfp;
1150 	uintptr_t	pc, nextpc;
1151 	ulong_t		off;
1152 	char		args[TR_ARG_MAX * 2 + 16], *sym;
1153 	uint_t	  offset = 0;
1154 	uint_t	  next_offset = 0;
1155 	char	    stack_buffer[1024];
1156 
1157 	if (!panicstr)
1158 		printf("traceback: %%fp = %p\n", (void *)fp);
1159 
1160 	if (panicstr && !dump_stack_scratch) {
1161 		printf("Warning - stack not written to the dump buffer\n");
1162 	}
1163 
1164 	fp = (struct frame *)plat_traceback(fpreg);
1165 	if ((uintptr_t)fp < KERNELBASE)
1166 		goto out;
1167 
1168 	pc = fp->fr_savpc;
1169 	fp = (struct frame *)fp->fr_savfp;
1170 
1171 	while ((uintptr_t)fp >= KERNELBASE) {
1172 		/*
1173 		 * XX64 Until port is complete tolerate 8-byte aligned
1174 		 * frame pointers but flag with a warning so they can
1175 		 * be fixed.
1176 		 */
1177 		if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1178 			if (((uintptr_t)fp & (8 - 1)) == 0) {
1179 				printf("  >> warning! 8-byte"
1180 				    " aligned %%fp = %p\n", (void *)fp);
1181 			} else {
1182 				printf(
1183 				    "  >> mis-aligned %%fp = %p\n", (void *)fp);
1184 				break;
1185 			}
1186 		}
1187 
1188 		args[0] = '\0';
1189 		nextpc = (uintptr_t)fp->fr_savpc;
1190 		nextfp = (struct frame *)fp->fr_savfp;
1191 		if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1192 			printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1193 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1194 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1195 			    "%s:%s+%lx (%s) | ",
1196 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1197 		} else {
1198 			printf("%016lx %lx (%s)\n",
1199 			    (uintptr_t)fp, pc, args);
1200 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1201 			    "%lx (%s) | ", pc, args);
1202 		}
1203 
1204 		if (panicstr && dump_stack_scratch) {
1205 			next_offset = offset + strlen(stack_buffer);
1206 			if (next_offset < STACK_BUF_SIZE) {
1207 				bcopy(stack_buffer, dump_stack_scratch + offset,
1208 				    strlen(stack_buffer));
1209 				offset = next_offset;
1210 			} else {
1211 				/*
1212 				 * In attempting to save the panic stack
1213 				 * to the dumpbuf we have overflowed that area.
1214 				 * Print a warning and continue to printf the
1215 				 * stack to the msgbuf
1216 				 */
1217 				printf("Warning: stack in the dump buffer"
1218 				    " may be incomplete\n");
1219 				offset = next_offset;
1220 			}
1221 		}
1222 
1223 		pc = nextpc;
1224 		fp = nextfp;
1225 	}
1226 out:
1227 	if (!panicstr) {
1228 		printf("end of traceback\n");
1229 		DELAY(2 * MICROSEC);
1230 	} else if (dump_stack_scratch) {
1231 		dump_stack_scratch[offset] = '\0';
1232 	}
1233 }
1234 
1235 #elif defined(__i386)
1236 
1237 void
1238 traceback(caddr_t fpreg)
1239 {
1240 	struct frame *fp = (struct frame *)fpreg;
1241 	struct frame *nextfp, *minfp, *stacktop;
1242 	uintptr_t pc, nextpc;
1243 	uint_t	  offset = 0;
1244 	uint_t	  next_offset = 0;
1245 	char	    stack_buffer[1024];
1246 
1247 	cpu_t *cpu;
1248 
1249 	/*
1250 	 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'.
1251 	 */
1252 	char args[TR_ARG_MAX * 2 + 8], *p;
1253 
1254 	int on_intr;
1255 	ulong_t off;
1256 	char *sym;
1257 
1258 	if (!panicstr)
1259 		printf("traceback: %%fp = %p\n", (void *)fp);
1260 
1261 	if (panicstr && !dump_stack_scratch) {
1262 		printf("Warning - stack not written to the dumpbuf\n");
1263 	}
1264 
1265 	/*
1266 	 * If we are panicking, all high-level interrupt information in
1267 	 * CPU was overwritten.  panic_cpu has the correct values.
1268 	 */
1269 	kpreempt_disable();			/* prevent migration */
1270 
1271 	cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU;
1272 
1273 	if ((on_intr = CPU_ON_INTR(cpu)) != 0)
1274 		stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME));
1275 	else
1276 		stacktop = (struct frame *)curthread->t_stk;
1277 
1278 	kpreempt_enable();
1279 
1280 	fp = (struct frame *)plat_traceback(fpreg);
1281 	if ((uintptr_t)fp < KERNELBASE)
1282 		goto out;
1283 
1284 	minfp = fp;	/* Baseline minimum frame pointer */
1285 	pc = fp->fr_savpc;
1286 	fp = (struct frame *)fp->fr_savfp;
1287 
1288 	while ((uintptr_t)fp >= KERNELBASE) {
1289 		ulong_t argc;
1290 		long *argv;
1291 
1292 		if (fp <= minfp || fp >= stacktop) {
1293 			if (on_intr) {
1294 				/*
1295 				 * Hop from interrupt stack to thread stack.
1296 				 */
1297 				stacktop = (struct frame *)curthread->t_stk;
1298 				minfp = (struct frame *)curthread->t_stkbase;
1299 				on_intr = 0;
1300 				continue;
1301 			}
1302 			break; /* we're outside of the expected range */
1303 		}
1304 
1305 		if ((uintptr_t)fp & (STACK_ALIGN - 1)) {
1306 			printf("  >> mis-aligned %%fp = %p\n", (void *)fp);
1307 			break;
1308 		}
1309 
1310 		nextpc = fp->fr_savpc;
1311 		nextfp = (struct frame *)fp->fr_savfp;
1312 		argc = argcount(nextpc);
1313 		argv = (long *)((char *)fp + sizeof (struct frame));
1314 
1315 		args[0] = '\0';
1316 		p = args;
1317 		while (argc-- > 0 && argv < (long *)stacktop) {
1318 			p += snprintf(p, args + sizeof (args) - p,
1319 			    "%s%lx", (p == args) ? "" : ", ", *argv++);
1320 		}
1321 
1322 		if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1323 			printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1324 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1325 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1326 			    "%s:%s+%lx (%s) | ",
1327 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1328 
1329 		} else {
1330 			printf("%08lx %lx (%s)\n",
1331 			    (uintptr_t)fp, pc, args);
1332 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1333 			    "%lx (%s) | ", pc, args);
1334 
1335 		}
1336 
1337 		if (panicstr && dump_stack_scratch) {
1338 			next_offset = offset + strlen(stack_buffer);
1339 			if (next_offset < STACK_BUF_SIZE) {
1340 				bcopy(stack_buffer, dump_stack_scratch + offset,
1341 				    strlen(stack_buffer));
1342 				offset = next_offset;
1343 			} else {
1344 				/*
1345 				 * In attempting to save the panic stack
1346 				 * to the dumpbuf we have overflowed that area.
1347 				 * Print a warning and continue to printf the
1348 				 * stack to the msgbuf
1349 				 */
1350 				printf("Warning: stack in the dumpbuf"
1351 				    " may be incomplete\n");
1352 				offset = next_offset;
1353 			}
1354 		}
1355 
1356 		minfp = fp;
1357 		pc = nextpc;
1358 		fp = nextfp;
1359 	}
1360 out:
1361 	if (!panicstr) {
1362 		printf("end of traceback\n");
1363 		DELAY(2 * MICROSEC);
1364 	} else if (dump_stack_scratch) {
1365 		dump_stack_scratch[offset] = '\0';
1366 	}
1367 
1368 }
1369 
1370 #endif	/* __i386 */
1371 
1372 /*
1373  * Generate a stack backtrace from a saved register set.
1374  */
1375 void
1376 traceregs(struct regs *rp)
1377 {
1378 	traceback((caddr_t)rp->r_fp);
1379 }
1380 
1381 void
1382 exec_set_sp(size_t stksize)
1383 {
1384 	klwp_t *lwp = ttolwp(curthread);
1385 
1386 	lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1387 }
1388 
1389 hrtime_t
1390 gethrtime_waitfree(void)
1391 {
1392 	return (dtrace_gethrtime());
1393 }
1394 
1395 hrtime_t
1396 gethrtime(void)
1397 {
1398 	return (gethrtimef());
1399 }
1400 
1401 hrtime_t
1402 gethrtime_unscaled(void)
1403 {
1404 	return (gethrtimeunscaledf());
1405 }
1406 
1407 void
1408 scalehrtime(hrtime_t *hrt)
1409 {
1410 	scalehrtimef(hrt);
1411 }
1412 
1413 uint64_t
1414 unscalehrtime(hrtime_t nsecs)
1415 {
1416 	return (unscalehrtimef(nsecs));
1417 }
1418 
1419 void
1420 gethrestime(timespec_t *tp)
1421 {
1422 	gethrestimef(tp);
1423 }
1424 
1425 #if defined(__amd64)
1426 /*
1427  * Part of the implementation of hres_tick(); this routine is
1428  * easier in C than assembler .. called with the hres_lock held.
1429  *
1430  * XX64	Many of these timekeeping variables need to be extern'ed in a header
1431  */
1432 
1433 #include <sys/time.h>
1434 #include <sys/machlock.h>
1435 
1436 extern int one_sec;
1437 extern int max_hres_adj;
1438 
1439 void
1440 __adj_hrestime(void)
1441 {
1442 	long long adj;
1443 
1444 	if (hrestime_adj == 0)
1445 		adj = 0;
1446 	else if (hrestime_adj > 0) {
1447 		if (hrestime_adj < max_hres_adj)
1448 			adj = hrestime_adj;
1449 		else
1450 			adj = max_hres_adj;
1451 	} else {
1452 		if (hrestime_adj < -max_hres_adj)
1453 			adj = -max_hres_adj;
1454 		else
1455 			adj = hrestime_adj;
1456 	}
1457 
1458 	timedelta -= adj;
1459 	hrestime_adj = timedelta;
1460 	hrestime.tv_nsec += adj;
1461 
1462 	while (hrestime.tv_nsec >= NANOSEC) {
1463 		one_sec++;
1464 		hrestime.tv_sec++;
1465 		hrestime.tv_nsec -= NANOSEC;
1466 	}
1467 }
1468 #endif
1469 
1470 /*
1471  * Wrapper functions to maintain backwards compability
1472  */
1473 int
1474 xcopyin(const void *uaddr, void *kaddr, size_t count)
1475 {
1476 	return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1477 }
1478 
1479 int
1480 xcopyout(const void *kaddr, void *uaddr, size_t count)
1481 {
1482 	return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
1483 }
1484