1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 #include <sys/param.h> 29 #include <sys/types.h> 30 #include <sys/vmparam.h> 31 #include <sys/systm.h> 32 #include <sys/signal.h> 33 #include <sys/stack.h> 34 #include <sys/regset.h> 35 #include <sys/privregs.h> 36 #include <sys/frame.h> 37 #include <sys/proc.h> 38 #include <sys/psw.h> 39 #include <sys/siginfo.h> 40 #include <sys/cpuvar.h> 41 #include <sys/asm_linkage.h> 42 #include <sys/kmem.h> 43 #include <sys/errno.h> 44 #include <sys/bootconf.h> 45 #include <sys/archsystm.h> 46 #include <sys/debug.h> 47 #include <sys/elf.h> 48 #include <sys/spl.h> 49 #include <sys/time.h> 50 #include <sys/atomic.h> 51 #include <sys/sysmacros.h> 52 #include <sys/cmn_err.h> 53 #include <sys/modctl.h> 54 #include <sys/kobj.h> 55 #include <sys/panic.h> 56 #include <sys/reboot.h> 57 #include <sys/time.h> 58 #include <sys/fp.h> 59 #include <sys/x86_archext.h> 60 #include <sys/auxv.h> 61 #include <sys/auxv_386.h> 62 #include <sys/dtrace.h> 63 #include <sys/brand.h> 64 #include <sys/machbrand.h> 65 66 extern const struct fnsave_state x87_initial; 67 extern const struct fxsave_state sse_initial; 68 69 /* 70 * Map an fnsave-formatted save area into an fxsave-formatted save area. 71 * 72 * Most fields are the same width, content and semantics. However 73 * the tag word is compressed. 74 */ 75 static void 76 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx) 77 { 78 uint_t i, tagbits; 79 80 fx->fx_fcw = fn->f_fcw; 81 fx->fx_fsw = fn->f_fsw; 82 83 /* 84 * copy element by element (because of holes) 85 */ 86 for (i = 0; i < 8; i++) 87 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0], 88 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */ 89 90 /* 91 * synthesize compressed tag bits 92 */ 93 fx->fx_fctw = 0; 94 for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2) 95 if ((tagbits & 3) != 3) 96 fx->fx_fctw |= (1 << i); 97 98 fx->fx_fop = fn->f_fop; 99 100 #if defined(__amd64) 101 fx->fx_rip = (uint64_t)fn->f_eip; 102 fx->fx_rdp = (uint64_t)fn->f_dp; 103 #else 104 fx->fx_eip = fn->f_eip; 105 fx->fx_cs = fn->f_cs; 106 fx->__fx_ign0 = 0; 107 fx->fx_dp = fn->f_dp; 108 fx->fx_ds = fn->f_ds; 109 fx->__fx_ign1 = 0; 110 #endif 111 } 112 113 /* 114 * Map from an fxsave-format save area to an fnsave-format save area. 115 */ 116 static void 117 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn) 118 { 119 uint_t i, top, tagbits; 120 121 fn->f_fcw = fx->fx_fcw; 122 fn->__f_ign0 = 0; 123 fn->f_fsw = fx->fx_fsw; 124 fn->__f_ign1 = 0; 125 126 top = (fx->fx_fsw & FPS_TOP) >> 11; 127 128 /* 129 * copy element by element (because of holes) 130 */ 131 for (i = 0; i < 8; i++) 132 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0], 133 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */ 134 135 /* 136 * synthesize uncompressed tag bits 137 */ 138 fn->f_ftw = 0; 139 for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) { 140 uint_t ibit, expo; 141 const uint16_t *fpp; 142 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 }; 143 144 if ((tagbits & 1) == 0) { 145 fn->f_ftw |= 3 << (i << 1); /* empty */ 146 continue; 147 } 148 149 /* 150 * (tags refer to *physical* registers) 151 */ 152 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0]; 153 ibit = fpp[3] >> 15; 154 expo = fpp[4] & 0x7fff; 155 156 if (ibit && expo != 0 && expo != 0x7fff) 157 continue; /* valid fp number */ 158 159 if (bcmp(fpp, &zero, sizeof (zero))) 160 fn->f_ftw |= 2 << (i << 1); /* NaN */ 161 else 162 fn->f_ftw |= 1 << (i << 1); /* fp zero */ 163 } 164 165 fn->f_fop = fx->fx_fop; 166 167 fn->__f_ign2 = 0; 168 #if defined(__amd64) 169 fn->f_eip = (uint32_t)fx->fx_rip; 170 fn->f_cs = U32CS_SEL; 171 fn->f_dp = (uint32_t)fx->fx_rdp; 172 fn->f_ds = UDS_SEL; 173 #else 174 fn->f_eip = fx->fx_eip; 175 fn->f_cs = fx->fx_cs; 176 fn->f_dp = fx->fx_dp; 177 fn->f_ds = fx->fx_ds; 178 #endif 179 fn->__f_ign3 = 0; 180 } 181 182 /* 183 * Map from an fpregset_t into an fxsave-format save area 184 */ 185 static void 186 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx) 187 { 188 #if defined(__amd64) 189 bcopy(fp, fx, sizeof (*fx)); 190 #else 191 const struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state; 192 193 fnsave_to_fxsave((const struct fnsave_state *)fc, fx); 194 fx->fx_mxcsr = fc->mxcsr; 195 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm)); 196 #endif 197 /* 198 * avoid useless #gp exceptions - mask reserved bits 199 */ 200 fx->fx_mxcsr &= sse_mxcsr_mask; 201 } 202 203 /* 204 * Map from an fxsave-format save area into a fpregset_t 205 */ 206 static void 207 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp) 208 { 209 #if defined(__amd64) 210 bcopy(fx, fp, sizeof (*fx)); 211 #else 212 struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state; 213 214 fxsave_to_fnsave(fx, (struct fnsave_state *)fc); 215 fc->mxcsr = fx->fx_mxcsr; 216 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm)); 217 #endif 218 } 219 220 #if defined(_SYSCALL32_IMPL) 221 static void 222 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx) 223 { 224 const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state; 225 226 fnsave_to_fxsave((const struct fnsave_state *)fc, fx); 227 /* 228 * avoid useless #gp exceptions - mask reserved bits 229 */ 230 fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr; 231 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm)); 232 } 233 234 static void 235 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp) 236 { 237 struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state; 238 239 fxsave_to_fnsave(fx, (struct fnsave_state *)fc); 240 fc->mxcsr = fx->fx_mxcsr; 241 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm)); 242 } 243 244 static void 245 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst) 246 { 247 fxsave_to_fpregset32((struct fxsave_state *)src, dst); 248 dst->fp_reg_set.fpchip_state.status = 249 src->fp_reg_set.fpchip_state.status; 250 dst->fp_reg_set.fpchip_state.xstatus = 251 src->fp_reg_set.fpchip_state.xstatus; 252 } 253 254 static void 255 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst) 256 { 257 fpregset32_to_fxsave(src, (struct fxsave_state *)dst); 258 dst->fp_reg_set.fpchip_state.status = 259 src->fp_reg_set.fpchip_state.status; 260 dst->fp_reg_set.fpchip_state.xstatus = 261 src->fp_reg_set.fpchip_state.xstatus; 262 } 263 #endif 264 265 /* 266 * Set floating-point registers from a native fpregset_t. 267 */ 268 void 269 setfpregs(klwp_t *lwp, fpregset_t *fp) 270 { 271 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu; 272 273 if (fpu->fpu_flags & FPU_EN) { 274 if (!(fpu->fpu_flags & FPU_VALID)) { 275 /* 276 * FPU context is still active, release the 277 * ownership. 278 */ 279 fp_free(fpu, 0); 280 } 281 #if !defined(__amd64) 282 if (fp_kind == __FP_SSE) { 283 #endif 284 fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx); 285 fpu->fpu_regs.kfpu_xstatus = 286 fp->fp_reg_set.fpchip_state.xstatus; 287 #if !defined(__amd64) 288 } else 289 bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn, 290 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn)); 291 #endif 292 fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status; 293 fpu->fpu_flags |= FPU_VALID; 294 } else { 295 /* 296 * If we are trying to change the FPU state of a thread which 297 * hasn't yet initialized floating point, store the state in 298 * the pcb and indicate that the state is valid. When the 299 * thread enables floating point, it will use this state instead 300 * of the default state. 301 */ 302 #if !defined(__amd64) 303 if (fp_kind == __FP_SSE) { 304 #endif 305 fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx); 306 fpu->fpu_regs.kfpu_xstatus = 307 fp->fp_reg_set.fpchip_state.xstatus; 308 #if !defined(__amd64) 309 } else 310 bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn, 311 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn)); 312 #endif 313 fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status; 314 fpu->fpu_flags |= FPU_VALID; 315 } 316 } 317 318 /* 319 * Get floating-point registers into a native fpregset_t. 320 */ 321 void 322 getfpregs(klwp_t *lwp, fpregset_t *fp) 323 { 324 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu; 325 326 kpreempt_disable(); 327 if (fpu->fpu_flags & FPU_EN) { 328 /* 329 * If we have FPU hw and the thread's pcb doesn't have 330 * a valid FPU state then get the state from the hw. 331 */ 332 if (fpu_exists && ttolwp(curthread) == lwp && 333 !(fpu->fpu_flags & FPU_VALID)) 334 fp_save(fpu); /* get the current FPU state */ 335 } 336 337 /* 338 * There are 3 possible cases we have to be aware of here: 339 * 340 * 1. FPU is enabled. FPU state is stored in the current LWP. 341 * 342 * 2. FPU is not enabled, and there have been no intervening /proc 343 * modifications. Return initial FPU state. 344 * 345 * 3. FPU is not enabled, but a /proc consumer has modified FPU state. 346 * FPU state is stored in the current LWP. 347 */ 348 if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) { 349 /* 350 * Cases 1 and 3. 351 */ 352 #if !defined(__amd64) 353 if (fp_kind == __FP_SSE) { 354 #endif 355 fxsave_to_fpregset(&fpu->fpu_regs.kfpu_u.kfpu_fx, fp); 356 fp->fp_reg_set.fpchip_state.xstatus = 357 fpu->fpu_regs.kfpu_xstatus; 358 #if !defined(__amd64) 359 } else 360 bcopy(&fpu->fpu_regs.kfpu_u.kfpu_fn, fp, 361 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn)); 362 #endif 363 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status; 364 } else { 365 /* 366 * Case 2. 367 */ 368 #if !defined(__amd64) 369 if (fp_kind == __FP_SSE) { 370 #endif 371 fxsave_to_fpregset(&sse_initial, fp); 372 fp->fp_reg_set.fpchip_state.xstatus = 373 fpu->fpu_regs.kfpu_xstatus; 374 #if !defined(__amd64) 375 } else 376 bcopy(&x87_initial, fp, sizeof (x87_initial)); 377 #endif 378 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status; 379 } 380 kpreempt_enable(); 381 } 382 383 #if defined(_SYSCALL32_IMPL) 384 385 /* 386 * Set floating-point registers from an fpregset32_t. 387 */ 388 void 389 setfpregs32(klwp_t *lwp, fpregset32_t *fp) 390 { 391 fpregset_t fpregs; 392 393 fpregset_32ton(fp, &fpregs); 394 setfpregs(lwp, &fpregs); 395 } 396 397 /* 398 * Get floating-point registers into an fpregset32_t. 399 */ 400 void 401 getfpregs32(klwp_t *lwp, fpregset32_t *fp) 402 { 403 fpregset_t fpregs; 404 405 getfpregs(lwp, &fpregs); 406 fpregset_nto32(&fpregs, fp); 407 } 408 409 #endif /* _SYSCALL32_IMPL */ 410 411 /* 412 * Return the general registers 413 */ 414 void 415 getgregs(klwp_t *lwp, gregset_t grp) 416 { 417 struct regs *rp = lwptoregs(lwp); 418 #if defined(__amd64) 419 struct pcb *pcb = &lwp->lwp_pcb; 420 int thisthread = lwptot(lwp) == curthread; 421 422 grp[REG_RDI] = rp->r_rdi; 423 grp[REG_RSI] = rp->r_rsi; 424 grp[REG_RDX] = rp->r_rdx; 425 grp[REG_RCX] = rp->r_rcx; 426 grp[REG_R8] = rp->r_r8; 427 grp[REG_R9] = rp->r_r9; 428 grp[REG_RAX] = rp->r_rax; 429 grp[REG_RBX] = rp->r_rbx; 430 grp[REG_RBP] = rp->r_rbp; 431 grp[REG_R10] = rp->r_r10; 432 grp[REG_R11] = rp->r_r11; 433 grp[REG_R12] = rp->r_r12; 434 grp[REG_R13] = rp->r_r13; 435 grp[REG_R14] = rp->r_r14; 436 grp[REG_R15] = rp->r_r15; 437 grp[REG_FSBASE] = pcb->pcb_fsbase; 438 grp[REG_GSBASE] = pcb->pcb_gsbase; 439 if (thisthread) 440 kpreempt_disable(); 441 if (pcb->pcb_rupdate == 1) { 442 grp[REG_DS] = pcb->pcb_ds; 443 grp[REG_ES] = pcb->pcb_es; 444 grp[REG_FS] = pcb->pcb_fs; 445 grp[REG_GS] = pcb->pcb_gs; 446 } else { 447 grp[REG_DS] = rp->r_ds; 448 grp[REG_ES] = rp->r_es; 449 grp[REG_FS] = rp->r_fs; 450 grp[REG_GS] = rp->r_gs; 451 } 452 if (thisthread) 453 kpreempt_enable(); 454 grp[REG_TRAPNO] = rp->r_trapno; 455 grp[REG_ERR] = rp->r_err; 456 grp[REG_RIP] = rp->r_rip; 457 grp[REG_CS] = rp->r_cs; 458 grp[REG_SS] = rp->r_ss; 459 grp[REG_RFL] = rp->r_rfl; 460 grp[REG_RSP] = rp->r_rsp; 461 #else 462 bcopy(&rp->r_gs, grp, sizeof (gregset_t)); 463 #endif 464 } 465 466 #if defined(_SYSCALL32_IMPL) 467 468 void 469 getgregs32(klwp_t *lwp, gregset32_t grp) 470 { 471 struct regs *rp = lwptoregs(lwp); 472 struct pcb *pcb = &lwp->lwp_pcb; 473 int thisthread = lwptot(lwp) == curthread; 474 475 if (thisthread) 476 kpreempt_disable(); 477 if (pcb->pcb_rupdate == 1) { 478 grp[GS] = (uint16_t)pcb->pcb_gs; 479 grp[FS] = (uint16_t)pcb->pcb_fs; 480 grp[DS] = (uint16_t)pcb->pcb_ds; 481 grp[ES] = (uint16_t)pcb->pcb_es; 482 } else { 483 grp[GS] = (uint16_t)rp->r_gs; 484 grp[FS] = (uint16_t)rp->r_fs; 485 grp[DS] = (uint16_t)rp->r_ds; 486 grp[ES] = (uint16_t)rp->r_es; 487 } 488 if (thisthread) 489 kpreempt_enable(); 490 grp[EDI] = (greg32_t)rp->r_rdi; 491 grp[ESI] = (greg32_t)rp->r_rsi; 492 grp[EBP] = (greg32_t)rp->r_rbp; 493 grp[ESP] = 0; 494 grp[EBX] = (greg32_t)rp->r_rbx; 495 grp[EDX] = (greg32_t)rp->r_rdx; 496 grp[ECX] = (greg32_t)rp->r_rcx; 497 grp[EAX] = (greg32_t)rp->r_rax; 498 grp[TRAPNO] = (greg32_t)rp->r_trapno; 499 grp[ERR] = (greg32_t)rp->r_err; 500 grp[EIP] = (greg32_t)rp->r_rip; 501 grp[CS] = (uint16_t)rp->r_cs; 502 grp[EFL] = (greg32_t)rp->r_rfl; 503 grp[UESP] = (greg32_t)rp->r_rsp; 504 grp[SS] = (uint16_t)rp->r_ss; 505 } 506 507 void 508 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst) 509 { 510 mcontext_t *dmc = &dst->uc_mcontext; 511 const mcontext32_t *smc = &src->uc_mcontext; 512 513 bzero(dst, sizeof (*dst)); 514 dst->uc_flags = src->uc_flags; 515 dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link; 516 517 bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask)); 518 519 dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp; 520 dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size; 521 dst->uc_stack.ss_flags = src->uc_stack.ss_flags; 522 523 dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS]; 524 dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS]; 525 dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES]; 526 dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS]; 527 dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI]; 528 dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI]; 529 dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP]; 530 dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX]; 531 dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX]; 532 dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX]; 533 dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX]; 534 dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO]; 535 dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR]; 536 dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP]; 537 dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS]; 538 dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL]; 539 dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP]; 540 dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS]; 541 542 /* 543 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set 544 * otherwise there is no guarantee that anything in fpregs is valid. 545 */ 546 if (src->uc_flags & UC_FPU) 547 fpregset_32ton(&src->uc_mcontext.fpregs, 548 &dst->uc_mcontext.fpregs); 549 } 550 551 #endif /* _SYSCALL32_IMPL */ 552 553 /* 554 * Return the user-level PC. 555 * If in a system call, return the address of the syscall trap. 556 */ 557 greg_t 558 getuserpc() 559 { 560 greg_t upc = lwptoregs(ttolwp(curthread))->r_pc; 561 uint32_t insn; 562 563 if (curthread->t_sysnum == 0) 564 return (upc); 565 566 /* 567 * We might've gotten here from sysenter (0xf 0x34), 568 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0). 569 * 570 * Go peek at the binary to figure it out.. 571 */ 572 if (fuword32((void *)(upc - 2), &insn) != -1 && 573 (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f) 574 return (upc - 2); 575 return (upc - 7); 576 } 577 578 /* 579 * Protect segment registers from non-user privilege levels and GDT selectors 580 * other than USER_CS, USER_DS and lwp FS and GS values. If the segment 581 * selector is non-null and not USER_CS/USER_DS, we make sure that the 582 * TI bit is set to point into the LDT and that the RPL is set to 3. 583 * 584 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we 585 * also explicitly zero the top 16 bits since they may be coming from the 586 * user's address space via setcontext(2) or /proc. 587 * 588 * Note about null selector. When running on the hypervisor if we allow a 589 * process to set its %cs to null selector with RPL of 0 the hypervisor will 590 * crash the domain. If running on bare metal we would get a #gp fault and 591 * be able to kill the process and continue on. Therefore we make sure to 592 * force RPL to SEL_UPL even for null selector when setting %cs. 593 */ 594 595 #if defined(IS_CS) || defined(IS_NOT_CS) 596 #error "IS_CS and IS_NOT_CS already defined" 597 #endif 598 599 #define IS_CS 1 600 #define IS_NOT_CS 0 601 602 /*ARGSUSED*/ 603 static greg_t 604 fix_segreg(greg_t sr, int iscs, model_t datamodel) 605 { 606 switch (sr &= 0xffff) { 607 608 case 0: 609 if (iscs == IS_CS) 610 return (0 | SEL_UPL); 611 else 612 return (0); 613 614 #if defined(__amd64) 615 /* 616 * If lwp attempts to switch data model then force their 617 * code selector to be null selector. 618 */ 619 case U32CS_SEL: 620 if (datamodel == DATAMODEL_NATIVE) 621 return (0 | SEL_UPL); 622 else 623 return (sr); 624 625 case UCS_SEL: 626 if (datamodel == DATAMODEL_ILP32) 627 return (0 | SEL_UPL); 628 #elif defined(__i386) 629 case UCS_SEL: 630 #endif 631 /*FALLTHROUGH*/ 632 case UDS_SEL: 633 case LWPFS_SEL: 634 case LWPGS_SEL: 635 case SEL_UPL: 636 return (sr); 637 default: 638 break; 639 } 640 641 /* 642 * Force it into the LDT in ring 3 for 32-bit processes, which by 643 * default do not have an LDT, so that any attempt to use an invalid 644 * selector will reference the (non-existant) LDT, and cause a #gp 645 * fault for the process. 646 * 647 * 64-bit processes get the null gdt selector since they 648 * are not allowed to have a private LDT. 649 */ 650 #if defined(__amd64) 651 if (datamodel == DATAMODEL_ILP32) { 652 return (sr | SEL_TI_LDT | SEL_UPL); 653 } else { 654 if (iscs == IS_CS) 655 return (0 | SEL_UPL); 656 else 657 return (0); 658 } 659 660 #elif defined(__i386) 661 return (sr | SEL_TI_LDT | SEL_UPL); 662 #endif 663 } 664 665 /* 666 * Set general registers. 667 */ 668 void 669 setgregs(klwp_t *lwp, gregset_t grp) 670 { 671 struct regs *rp = lwptoregs(lwp); 672 model_t datamodel = lwp_getdatamodel(lwp); 673 674 #if defined(__amd64) 675 struct pcb *pcb = &lwp->lwp_pcb; 676 int thisthread = lwptot(lwp) == curthread; 677 678 if (datamodel == DATAMODEL_NATIVE) { 679 680 if (thisthread) 681 (void) save_syscall_args(); /* copy the args */ 682 683 rp->r_rdi = grp[REG_RDI]; 684 rp->r_rsi = grp[REG_RSI]; 685 rp->r_rdx = grp[REG_RDX]; 686 rp->r_rcx = grp[REG_RCX]; 687 rp->r_r8 = grp[REG_R8]; 688 rp->r_r9 = grp[REG_R9]; 689 rp->r_rax = grp[REG_RAX]; 690 rp->r_rbx = grp[REG_RBX]; 691 rp->r_rbp = grp[REG_RBP]; 692 rp->r_r10 = grp[REG_R10]; 693 rp->r_r11 = grp[REG_R11]; 694 rp->r_r12 = grp[REG_R12]; 695 rp->r_r13 = grp[REG_R13]; 696 rp->r_r14 = grp[REG_R14]; 697 rp->r_r15 = grp[REG_R15]; 698 rp->r_trapno = grp[REG_TRAPNO]; 699 rp->r_err = grp[REG_ERR]; 700 rp->r_rip = grp[REG_RIP]; 701 /* 702 * Setting %cs or %ss to anything else is quietly but 703 * quite definitely forbidden! 704 */ 705 rp->r_cs = UCS_SEL; 706 rp->r_ss = UDS_SEL; 707 rp->r_rsp = grp[REG_RSP]; 708 709 if (thisthread) 710 kpreempt_disable(); 711 712 pcb->pcb_ds = UDS_SEL; 713 pcb->pcb_es = UDS_SEL; 714 715 /* 716 * 64-bit processes -are- allowed to set their fsbase/gsbase 717 * values directly, but only if they're using the segment 718 * selectors that allow that semantic. 719 * 720 * (32-bit processes must use lwp_set_private().) 721 */ 722 pcb->pcb_fsbase = grp[REG_FSBASE]; 723 pcb->pcb_gsbase = grp[REG_GSBASE]; 724 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel); 725 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel); 726 727 /* 728 * Ensure that we go out via update_sregs 729 */ 730 pcb->pcb_rupdate = 1; 731 lwptot(lwp)->t_post_sys = 1; 732 if (thisthread) 733 kpreempt_enable(); 734 #if defined(_SYSCALL32_IMPL) 735 } else { 736 rp->r_rdi = (uint32_t)grp[REG_RDI]; 737 rp->r_rsi = (uint32_t)grp[REG_RSI]; 738 rp->r_rdx = (uint32_t)grp[REG_RDX]; 739 rp->r_rcx = (uint32_t)grp[REG_RCX]; 740 rp->r_rax = (uint32_t)grp[REG_RAX]; 741 rp->r_rbx = (uint32_t)grp[REG_RBX]; 742 rp->r_rbp = (uint32_t)grp[REG_RBP]; 743 rp->r_trapno = (uint32_t)grp[REG_TRAPNO]; 744 rp->r_err = (uint32_t)grp[REG_ERR]; 745 rp->r_rip = (uint32_t)grp[REG_RIP]; 746 747 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel); 748 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel); 749 750 rp->r_rsp = (uint32_t)grp[REG_RSP]; 751 752 if (thisthread) 753 kpreempt_disable(); 754 755 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel); 756 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel); 757 758 /* 759 * (See fsbase/gsbase commentary above) 760 */ 761 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel); 762 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel); 763 764 /* 765 * Ensure that we go out via update_sregs 766 */ 767 pcb->pcb_rupdate = 1; 768 lwptot(lwp)->t_post_sys = 1; 769 if (thisthread) 770 kpreempt_enable(); 771 #endif 772 } 773 774 /* 775 * Only certain bits of the flags register can be modified. 776 */ 777 rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) | 778 (grp[REG_RFL] & PSL_USERMASK); 779 780 #elif defined(__i386) 781 782 /* 783 * Only certain bits of the flags register can be modified. 784 */ 785 grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK); 786 787 /* 788 * Copy saved registers from user stack. 789 */ 790 bcopy(grp, &rp->r_gs, sizeof (gregset_t)); 791 792 rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel); 793 rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel); 794 rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel); 795 rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel); 796 rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel); 797 rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel); 798 799 #endif /* __i386 */ 800 } 801 802 /* 803 * Determine whether eip is likely to have an interrupt frame 804 * on the stack. We do this by comparing the address to the 805 * range of addresses spanned by several well-known routines. 806 */ 807 extern void _interrupt(); 808 extern void _allsyscalls(); 809 extern void _cmntrap(); 810 extern void fakesoftint(); 811 812 extern size_t _interrupt_size; 813 extern size_t _allsyscalls_size; 814 extern size_t _cmntrap_size; 815 extern size_t _fakesoftint_size; 816 817 /* 818 * Get a pc-only stacktrace. Used for kmem_alloc() buffer ownership tracking. 819 * Returns MIN(current stack depth, pcstack_limit). 820 */ 821 int 822 getpcstack(pc_t *pcstack, int pcstack_limit) 823 { 824 struct frame *fp = (struct frame *)getfp(); 825 struct frame *nextfp, *minfp, *stacktop; 826 int depth = 0; 827 int on_intr; 828 uintptr_t pc; 829 830 if ((on_intr = CPU_ON_INTR(CPU)) != 0) 831 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME)); 832 else 833 stacktop = (struct frame *)curthread->t_stk; 834 minfp = fp; 835 836 pc = ((struct regs *)fp)->r_pc; 837 838 while (depth < pcstack_limit) { 839 nextfp = (struct frame *)fp->fr_savfp; 840 pc = fp->fr_savpc; 841 if (nextfp <= minfp || nextfp >= stacktop) { 842 if (on_intr) { 843 /* 844 * Hop from interrupt stack to thread stack. 845 */ 846 stacktop = (struct frame *)curthread->t_stk; 847 minfp = (struct frame *)curthread->t_stkbase; 848 on_intr = 0; 849 continue; 850 } 851 break; 852 } 853 pcstack[depth++] = (pc_t)pc; 854 fp = nextfp; 855 minfp = fp; 856 } 857 return (depth); 858 } 859 860 /* 861 * The following ELF header fields are defined as processor-specific 862 * in the V8 ABI: 863 * 864 * e_ident[EI_DATA] encoding of the processor-specific 865 * data in the object file 866 * e_machine processor identification 867 * e_flags processor-specific flags associated 868 * with the file 869 */ 870 871 /* 872 * The value of at_flags reflects a platform's cpu module support. 873 * at_flags is used to check for allowing a binary to execute and 874 * is passed as the value of the AT_FLAGS auxiliary vector. 875 */ 876 int at_flags = 0; 877 878 /* 879 * Check the processor-specific fields of an ELF header. 880 * 881 * returns 1 if the fields are valid, 0 otherwise 882 */ 883 /*ARGSUSED2*/ 884 int 885 elfheadcheck( 886 unsigned char e_data, 887 Elf32_Half e_machine, 888 Elf32_Word e_flags) 889 { 890 if (e_data != ELFDATA2LSB) 891 return (0); 892 #if defined(__amd64) 893 if (e_machine == EM_AMD64) 894 return (1); 895 #endif 896 return (e_machine == EM_386); 897 } 898 899 uint_t auxv_hwcap_include = 0; /* patch to enable unrecognized features */ 900 uint_t auxv_hwcap_exclude = 0; /* patch for broken cpus, debugging */ 901 #if defined(_SYSCALL32_IMPL) 902 uint_t auxv_hwcap32_include = 0; /* ditto for 32-bit apps */ 903 uint_t auxv_hwcap32_exclude = 0; /* ditto for 32-bit apps */ 904 #endif 905 906 /* 907 * Gather information about the processor and place it into auxv_hwcap 908 * so that it can be exported to the linker via the aux vector. 909 * 910 * We use this seemingly complicated mechanism so that we can ensure 911 * that /etc/system can be used to override what the system can or 912 * cannot discover for itself. 913 */ 914 void 915 bind_hwcap(void) 916 { 917 uint_t cpu_hwcap_flags = cpuid_pass4(NULL); 918 919 auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags) & 920 ~auxv_hwcap_exclude; 921 922 #if defined(__amd64) 923 /* 924 * On AMD processors, sysenter just doesn't work at all 925 * when the kernel is in long mode. On IA-32e processors 926 * it does, but there's no real point in all the alternate 927 * mechanism when syscall works on both. 928 * 929 * Besides, the kernel's sysenter handler is expecting a 930 * 32-bit lwp ... 931 */ 932 auxv_hwcap &= ~AV_386_SEP; 933 #else 934 /* 935 * 32-bit processes can -always- use the lahf/sahf instructions 936 */ 937 auxv_hwcap |= AV_386_AHF; 938 #endif 939 940 if (auxv_hwcap_include || auxv_hwcap_exclude) 941 cmn_err(CE_CONT, "?user ABI extensions: %b\n", 942 auxv_hwcap, FMT_AV_386); 943 944 #if defined(_SYSCALL32_IMPL) 945 auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags) & 946 ~auxv_hwcap32_exclude; 947 948 #if defined(__amd64) 949 /* 950 * If this is an amd64 architecture machine from Intel, then 951 * syscall -doesn't- work in compatibility mode, only sysenter does. 952 * 953 * Sigh. 954 */ 955 if (!cpuid_syscall32_insn(NULL)) 956 auxv_hwcap32 &= ~AV_386_AMD_SYSC; 957 958 /* 959 * 32-bit processes can -always- use the lahf/sahf instructions 960 */ 961 auxv_hwcap32 |= AV_386_AHF; 962 #endif 963 964 if (auxv_hwcap32_include || auxv_hwcap32_exclude) 965 cmn_err(CE_CONT, "?32-bit user ABI extensions: %b\n", 966 auxv_hwcap32, FMT_AV_386); 967 #endif 968 } 969 970 /* 971 * sync_icache() - this is called 972 * in proc/fs/prusrio.c. x86 has an unified cache and therefore 973 * this is a nop. 974 */ 975 /* ARGSUSED */ 976 void 977 sync_icache(caddr_t addr, uint_t len) 978 { 979 /* Do nothing for now */ 980 } 981 982 /*ARGSUSED*/ 983 void 984 sync_data_memory(caddr_t va, size_t len) 985 { 986 /* Not implemented for this platform */ 987 } 988 989 int 990 __ipltospl(int ipl) 991 { 992 return (ipltospl(ipl)); 993 } 994 995 /* 996 * The panic code invokes panic_saveregs() to record the contents of a 997 * regs structure into the specified panic_data structure for debuggers. 998 */ 999 void 1000 panic_saveregs(panic_data_t *pdp, struct regs *rp) 1001 { 1002 panic_nv_t *pnv = PANICNVGET(pdp); 1003 1004 struct cregs creg; 1005 1006 getcregs(&creg); 1007 1008 #if defined(__amd64) 1009 PANICNVADD(pnv, "rdi", rp->r_rdi); 1010 PANICNVADD(pnv, "rsi", rp->r_rsi); 1011 PANICNVADD(pnv, "rdx", rp->r_rdx); 1012 PANICNVADD(pnv, "rcx", rp->r_rcx); 1013 PANICNVADD(pnv, "r8", rp->r_r8); 1014 PANICNVADD(pnv, "r9", rp->r_r9); 1015 PANICNVADD(pnv, "rax", rp->r_rax); 1016 PANICNVADD(pnv, "rbx", rp->r_rbx); 1017 PANICNVADD(pnv, "rbp", rp->r_rbp); 1018 PANICNVADD(pnv, "r10", rp->r_r10); 1019 PANICNVADD(pnv, "r10", rp->r_r10); 1020 PANICNVADD(pnv, "r11", rp->r_r11); 1021 PANICNVADD(pnv, "r12", rp->r_r12); 1022 PANICNVADD(pnv, "r13", rp->r_r13); 1023 PANICNVADD(pnv, "r14", rp->r_r14); 1024 PANICNVADD(pnv, "r15", rp->r_r15); 1025 PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE)); 1026 PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE)); 1027 PANICNVADD(pnv, "ds", rp->r_ds); 1028 PANICNVADD(pnv, "es", rp->r_es); 1029 PANICNVADD(pnv, "fs", rp->r_fs); 1030 PANICNVADD(pnv, "gs", rp->r_gs); 1031 PANICNVADD(pnv, "trapno", rp->r_trapno); 1032 PANICNVADD(pnv, "err", rp->r_err); 1033 PANICNVADD(pnv, "rip", rp->r_rip); 1034 PANICNVADD(pnv, "cs", rp->r_cs); 1035 PANICNVADD(pnv, "rflags", rp->r_rfl); 1036 PANICNVADD(pnv, "rsp", rp->r_rsp); 1037 PANICNVADD(pnv, "ss", rp->r_ss); 1038 PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3])); 1039 PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0])); 1040 PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3])); 1041 PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0])); 1042 #elif defined(__i386) 1043 PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs); 1044 PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs); 1045 PANICNVADD(pnv, "es", (uint32_t)rp->r_es); 1046 PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds); 1047 PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi); 1048 PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi); 1049 PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp); 1050 PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp); 1051 PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx); 1052 PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx); 1053 PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx); 1054 PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax); 1055 PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno); 1056 PANICNVADD(pnv, "err", (uint32_t)rp->r_err); 1057 PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip); 1058 PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs); 1059 PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl); 1060 PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp); 1061 PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss); 1062 PANICNVADD(pnv, "gdt", creg.cr_gdt); 1063 PANICNVADD(pnv, "idt", creg.cr_idt); 1064 #endif /* __i386 */ 1065 1066 PANICNVADD(pnv, "ldt", creg.cr_ldt); 1067 PANICNVADD(pnv, "task", creg.cr_task); 1068 PANICNVADD(pnv, "cr0", creg.cr_cr0); 1069 PANICNVADD(pnv, "cr2", creg.cr_cr2); 1070 PANICNVADD(pnv, "cr3", creg.cr_cr3); 1071 if (creg.cr_cr4) 1072 PANICNVADD(pnv, "cr4", creg.cr_cr4); 1073 1074 PANICNVSET(pdp, pnv); 1075 } 1076 1077 #define TR_ARG_MAX 6 /* Max args to print, same as SPARC */ 1078 1079 #if !defined(__amd64) 1080 1081 /* 1082 * Given a return address (%eip), determine the likely number of arguments 1083 * that were pushed on the stack prior to its execution. We do this by 1084 * expecting that a typical call sequence consists of pushing arguments on 1085 * the stack, executing a call instruction, and then performing an add 1086 * on %esp to restore it to the value prior to pushing the arguments for 1087 * the call. We attempt to detect such an add, and divide the addend 1088 * by the size of a word to determine the number of pushed arguments. 1089 * 1090 * If we do not find such an add, we punt and return TR_ARG_MAX. It is not 1091 * possible to reliably determine if a function took no arguments (i.e. was 1092 * void) because assembler routines do not reliably perform an add on %esp 1093 * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is 1094 * safer than returning 0. 1095 */ 1096 static ulong_t 1097 argcount(uintptr_t eip) 1098 { 1099 const uint8_t *ins = (const uint8_t *)eip; 1100 ulong_t n; 1101 1102 enum { 1103 M_MODRM_ESP = 0xc4, /* Mod/RM byte indicates %esp */ 1104 M_ADD_IMM32 = 0x81, /* ADD imm32 to r/m32 */ 1105 M_ADD_IMM8 = 0x83 /* ADD imm8 to r/m32 */ 1106 }; 1107 1108 if (eip < KERNELBASE || ins[1] != M_MODRM_ESP) 1109 return (TR_ARG_MAX); 1110 1111 switch (ins[0]) { 1112 case M_ADD_IMM32: 1113 n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24); 1114 break; 1115 1116 case M_ADD_IMM8: 1117 n = ins[2]; 1118 break; 1119 1120 default: 1121 return (TR_ARG_MAX); 1122 } 1123 1124 n /= sizeof (long); 1125 return (MIN(n, TR_ARG_MAX)); 1126 } 1127 1128 #endif /* !__amd64 */ 1129 1130 /* 1131 * Print a stack backtrace using the specified frame pointer. We delay two 1132 * seconds before continuing, unless this is the panic traceback. Note 1133 * that the frame for the starting stack pointer value is omitted because 1134 * the corresponding %eip is not known. 1135 */ 1136 #if defined(__amd64) 1137 1138 void 1139 traceback(caddr_t fpreg) 1140 { 1141 struct frame *fp = (struct frame *)fpreg; 1142 struct frame *nextfp; 1143 uintptr_t pc, nextpc; 1144 ulong_t off; 1145 char args[TR_ARG_MAX * 2 + 16], *sym; 1146 1147 if (!panicstr) 1148 printf("traceback: %%fp = %p\n", (void *)fp); 1149 1150 fp = (struct frame *)plat_traceback(fpreg); 1151 if ((uintptr_t)fp < KERNELBASE) 1152 goto out; 1153 1154 pc = fp->fr_savpc; 1155 fp = (struct frame *)fp->fr_savfp; 1156 1157 while ((uintptr_t)fp >= KERNELBASE) { 1158 /* 1159 * XX64 Until port is complete tolerate 8-byte aligned 1160 * frame pointers but flag with a warning so they can 1161 * be fixed. 1162 */ 1163 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) { 1164 if (((uintptr_t)fp & (8 - 1)) == 0) { 1165 printf(" >> warning! 8-byte" 1166 " aligned %%fp = %p\n", (void *)fp); 1167 } else { 1168 printf( 1169 " >> mis-aligned %%fp = %p\n", (void *)fp); 1170 break; 1171 } 1172 } 1173 1174 args[0] = '\0'; 1175 nextpc = (uintptr_t)fp->fr_savpc; 1176 nextfp = (struct frame *)fp->fr_savfp; 1177 if ((sym = kobj_getsymname(pc, &off)) != NULL) { 1178 printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp, 1179 mod_containing_pc((caddr_t)pc), sym, off, args); 1180 } else { 1181 printf("%016lx %lx (%s)\n", 1182 (uintptr_t)fp, pc, args); 1183 } 1184 1185 pc = nextpc; 1186 fp = nextfp; 1187 } 1188 out: 1189 if (!panicstr) { 1190 printf("end of traceback\n"); 1191 DELAY(2 * MICROSEC); 1192 } 1193 } 1194 1195 #elif defined(__i386) 1196 1197 void 1198 traceback(caddr_t fpreg) 1199 { 1200 struct frame *fp = (struct frame *)fpreg; 1201 struct frame *nextfp, *minfp, *stacktop; 1202 uintptr_t pc, nextpc; 1203 1204 cpu_t *cpu; 1205 1206 /* 1207 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'. 1208 */ 1209 char args[TR_ARG_MAX * 2 + 8], *p; 1210 1211 int on_intr; 1212 ulong_t off; 1213 char *sym; 1214 1215 if (!panicstr) 1216 printf("traceback: %%fp = %p\n", (void *)fp); 1217 1218 /* 1219 * If we are panicking, all high-level interrupt information in 1220 * CPU was overwritten. panic_cpu has the correct values. 1221 */ 1222 kpreempt_disable(); /* prevent migration */ 1223 1224 cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU; 1225 1226 if ((on_intr = CPU_ON_INTR(cpu)) != 0) 1227 stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME)); 1228 else 1229 stacktop = (struct frame *)curthread->t_stk; 1230 1231 kpreempt_enable(); 1232 1233 fp = (struct frame *)plat_traceback(fpreg); 1234 if ((uintptr_t)fp < KERNELBASE) 1235 goto out; 1236 1237 minfp = fp; /* Baseline minimum frame pointer */ 1238 pc = fp->fr_savpc; 1239 fp = (struct frame *)fp->fr_savfp; 1240 1241 while ((uintptr_t)fp >= KERNELBASE) { 1242 ulong_t argc; 1243 long *argv; 1244 1245 if (fp <= minfp || fp >= stacktop) { 1246 if (on_intr) { 1247 /* 1248 * Hop from interrupt stack to thread stack. 1249 */ 1250 stacktop = (struct frame *)curthread->t_stk; 1251 minfp = (struct frame *)curthread->t_stkbase; 1252 on_intr = 0; 1253 continue; 1254 } 1255 break; /* we're outside of the expected range */ 1256 } 1257 1258 if ((uintptr_t)fp & (STACK_ALIGN - 1)) { 1259 printf(" >> mis-aligned %%fp = %p\n", (void *)fp); 1260 break; 1261 } 1262 1263 nextpc = fp->fr_savpc; 1264 nextfp = (struct frame *)fp->fr_savfp; 1265 argc = argcount(nextpc); 1266 argv = (long *)((char *)fp + sizeof (struct frame)); 1267 1268 args[0] = '\0'; 1269 p = args; 1270 while (argc-- > 0 && argv < (long *)stacktop) { 1271 p += snprintf(p, args + sizeof (args) - p, 1272 "%s%lx", (p == args) ? "" : ", ", *argv++); 1273 } 1274 1275 if ((sym = kobj_getsymname(pc, &off)) != NULL) { 1276 printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp, 1277 mod_containing_pc((caddr_t)pc), sym, off, args); 1278 } else { 1279 printf("%08lx %lx (%s)\n", 1280 (uintptr_t)fp, pc, args); 1281 } 1282 1283 minfp = fp; 1284 pc = nextpc; 1285 fp = nextfp; 1286 } 1287 out: 1288 if (!panicstr) { 1289 printf("end of traceback\n"); 1290 DELAY(2 * MICROSEC); 1291 } 1292 } 1293 1294 #endif /* __i386 */ 1295 1296 /* 1297 * Generate a stack backtrace from a saved register set. 1298 */ 1299 void 1300 traceregs(struct regs *rp) 1301 { 1302 traceback((caddr_t)rp->r_fp); 1303 } 1304 1305 void 1306 exec_set_sp(size_t stksize) 1307 { 1308 klwp_t *lwp = ttolwp(curthread); 1309 1310 lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize; 1311 } 1312 1313 hrtime_t 1314 gethrtime_waitfree(void) 1315 { 1316 return (dtrace_gethrtime()); 1317 } 1318 1319 hrtime_t 1320 gethrtime(void) 1321 { 1322 return (gethrtimef()); 1323 } 1324 1325 hrtime_t 1326 gethrtime_unscaled(void) 1327 { 1328 return (gethrtimeunscaledf()); 1329 } 1330 1331 void 1332 scalehrtime(hrtime_t *hrt) 1333 { 1334 scalehrtimef(hrt); 1335 } 1336 1337 uint64_t 1338 unscalehrtime(hrtime_t nsecs) 1339 { 1340 return (unscalehrtimef(nsecs)); 1341 } 1342 1343 void 1344 gethrestime(timespec_t *tp) 1345 { 1346 gethrestimef(tp); 1347 } 1348 1349 #if defined(__amd64) 1350 /* 1351 * Part of the implementation of hres_tick(); this routine is 1352 * easier in C than assembler .. called with the hres_lock held. 1353 * 1354 * XX64 Many of these timekeeping variables need to be extern'ed in a header 1355 */ 1356 1357 #include <sys/time.h> 1358 #include <sys/machlock.h> 1359 1360 extern int one_sec; 1361 extern int max_hres_adj; 1362 1363 void 1364 __adj_hrestime(void) 1365 { 1366 long long adj; 1367 1368 if (hrestime_adj == 0) 1369 adj = 0; 1370 else if (hrestime_adj > 0) { 1371 if (hrestime_adj < max_hres_adj) 1372 adj = hrestime_adj; 1373 else 1374 adj = max_hres_adj; 1375 } else { 1376 if (hrestime_adj < -max_hres_adj) 1377 adj = -max_hres_adj; 1378 else 1379 adj = hrestime_adj; 1380 } 1381 1382 timedelta -= adj; 1383 hrestime_adj = timedelta; 1384 hrestime.tv_nsec += adj; 1385 1386 while (hrestime.tv_nsec >= NANOSEC) { 1387 one_sec++; 1388 hrestime.tv_sec++; 1389 hrestime.tv_nsec -= NANOSEC; 1390 } 1391 } 1392 #endif 1393 1394 /* 1395 * Wrapper functions to maintain backwards compability 1396 */ 1397 int 1398 xcopyin(const void *uaddr, void *kaddr, size_t count) 1399 { 1400 return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED)); 1401 } 1402 1403 int 1404 xcopyout(const void *kaddr, void *uaddr, size_t count) 1405 { 1406 return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED)); 1407 } 1408