1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2017, Joyent, Inc. 25 */ 26 /* 27 * Copyright (c) 2010, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/segments.h> 35 #include <sys/sysmacros.h> 36 #include <sys/signal.h> 37 #include <sys/systm.h> 38 #include <sys/user.h> 39 #include <sys/mman.h> 40 #include <sys/vm.h> 41 42 #include <sys/disp.h> 43 #include <sys/class.h> 44 45 #include <sys/proc.h> 46 #include <sys/buf.h> 47 #include <sys/kmem.h> 48 49 #include <sys/reboot.h> 50 #include <sys/uadmin.h> 51 #include <sys/callb.h> 52 53 #include <sys/cred.h> 54 #include <sys/vnode.h> 55 #include <sys/file.h> 56 57 #include <sys/procfs.h> 58 #include <sys/acct.h> 59 60 #include <sys/vfs.h> 61 #include <sys/dnlc.h> 62 #include <sys/var.h> 63 #include <sys/cmn_err.h> 64 #include <sys/utsname.h> 65 #include <sys/debug.h> 66 67 #include <sys/dumphdr.h> 68 #include <sys/bootconf.h> 69 #include <sys/varargs.h> 70 #include <sys/promif.h> 71 #include <sys/modctl.h> 72 73 #include <sys/consdev.h> 74 #include <sys/frame.h> 75 76 #include <sys/sunddi.h> 77 #include <sys/ddidmareq.h> 78 #include <sys/psw.h> 79 #include <sys/regset.h> 80 #include <sys/privregs.h> 81 #include <sys/clock.h> 82 #include <sys/tss.h> 83 #include <sys/cpu.h> 84 #include <sys/stack.h> 85 #include <sys/trap.h> 86 #include <sys/pic.h> 87 #include <vm/hat.h> 88 #include <vm/anon.h> 89 #include <vm/as.h> 90 #include <vm/page.h> 91 #include <vm/seg.h> 92 #include <vm/seg_kmem.h> 93 #include <vm/seg_map.h> 94 #include <vm/seg_vn.h> 95 #include <vm/seg_kp.h> 96 #include <vm/hat_i86.h> 97 #include <sys/swap.h> 98 #include <sys/thread.h> 99 #include <sys/sysconf.h> 100 #include <sys/vm_machparam.h> 101 #include <sys/archsystm.h> 102 #include <sys/machsystm.h> 103 #include <sys/machlock.h> 104 #include <sys/x_call.h> 105 #include <sys/instance.h> 106 107 #include <sys/time.h> 108 #include <sys/smp_impldefs.h> 109 #include <sys/psm_types.h> 110 #include <sys/atomic.h> 111 #include <sys/panic.h> 112 #include <sys/cpuvar.h> 113 #include <sys/dtrace.h> 114 #include <sys/bl.h> 115 #include <sys/nvpair.h> 116 #include <sys/x86_archext.h> 117 #include <sys/pool_pset.h> 118 #include <sys/autoconf.h> 119 #include <sys/mem.h> 120 #include <sys/dumphdr.h> 121 #include <sys/compress.h> 122 #include <sys/cpu_module.h> 123 #if defined(__xpv) 124 #include <sys/hypervisor.h> 125 #include <sys/xpv_panic.h> 126 #endif 127 128 #include <sys/fastboot.h> 129 #include <sys/machelf.h> 130 #include <sys/kobj.h> 131 #include <sys/multiboot.h> 132 133 #ifdef TRAPTRACE 134 #include <sys/traptrace.h> 135 #endif /* TRAPTRACE */ 136 137 #include <c2/audit.h> 138 #include <sys/clock_impl.h> 139 140 extern void audit_enterprom(int); 141 extern void audit_exitprom(int); 142 143 /* 144 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used. 145 */ 146 int apix_enable = 1; 147 148 int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */ 149 150 /* 151 * Occassionally the kernel knows better whether to power-off or reboot. 152 */ 153 int force_shutdown_method = AD_UNKNOWN; 154 155 /* 156 * The panicbuf array is used to record messages and state: 157 */ 158 char panicbuf[PANICBUFSIZE]; 159 160 /* 161 * Flags to control Dynamic Reconfiguration features. 162 */ 163 uint64_t plat_dr_options; 164 165 /* 166 * Maximum physical address for memory DR operations. 167 */ 168 uint64_t plat_dr_physmax; 169 170 /* 171 * maxphys - used during physio 172 * klustsize - used for klustering by swapfs and specfs 173 */ 174 int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */ 175 int klustsize = 56 * 1024; 176 177 caddr_t p0_va; /* Virtual address for accessing physical page 0 */ 178 179 /* 180 * defined here, though unused on x86, 181 * to make kstat_fr.c happy. 182 */ 183 int vac; 184 185 void debug_enter(char *); 186 187 extern void pm_cfb_check_and_powerup(void); 188 extern void pm_cfb_rele(void); 189 190 extern fastboot_info_t newkernel; 191 192 /* 193 * Machine dependent code to reboot. 194 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer 195 * to a string to be used as the argument string when rebooting. 196 * 197 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely 198 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when 199 * we are in a normal shutdown sequence (interrupts are not blocked, the 200 * system is not panic'ing or being suspended). 201 */ 202 /*ARGSUSED*/ 203 void 204 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb) 205 { 206 processorid_t bootcpuid = 0; 207 static int is_first_quiesce = 1; 208 static int is_first_reset = 1; 209 int reset_status = 0; 210 static char fallback_str[] = "Falling back to regular reboot.\n"; 211 212 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid) 213 fcn = AD_BOOT; 214 215 if (!panicstr) { 216 kpreempt_disable(); 217 if (fcn == AD_FASTREBOOT) { 218 mutex_enter(&cpu_lock); 219 if (CPU_ACTIVE(cpu_get(bootcpuid))) { 220 affinity_set(bootcpuid); 221 } 222 mutex_exit(&cpu_lock); 223 } else { 224 affinity_set(CPU_CURRENT); 225 } 226 } 227 228 if (force_shutdown_method != AD_UNKNOWN) 229 fcn = force_shutdown_method; 230 231 /* 232 * XXX - rconsvp is set to NULL to ensure that output messages 233 * are sent to the underlying "hardware" device using the 234 * monitor's printf routine since we are in the process of 235 * either rebooting or halting the machine. 236 */ 237 rconsvp = NULL; 238 239 /* 240 * Print the reboot message now, before pausing other cpus. 241 * There is a race condition in the printing support that 242 * can deadlock multiprocessor machines. 243 */ 244 if (!(fcn == AD_HALT || fcn == AD_POWEROFF)) 245 prom_printf("rebooting...\n"); 246 247 if (IN_XPV_PANIC()) 248 reset(); 249 250 /* 251 * We can't bring up the console from above lock level, so do it now 252 */ 253 pm_cfb_check_and_powerup(); 254 255 /* make sure there are no more changes to the device tree */ 256 devtree_freeze(); 257 258 if (invoke_cb) 259 (void) callb_execute_class(CB_CL_MDBOOT, 0); 260 261 /* 262 * Clear any unresolved UEs from memory. 263 */ 264 page_retire_mdboot(); 265 266 #if defined(__xpv) 267 /* 268 * XXPV Should probably think some more about how we deal 269 * with panicing before it's really safe to panic. 270 * On hypervisors, we reboot very quickly.. Perhaps panic 271 * should only attempt to recover by rebooting if, 272 * say, we were able to mount the root filesystem, 273 * or if we successfully launched init(1m). 274 */ 275 if (panicstr && proc_init == NULL) 276 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff); 277 #endif 278 /* 279 * stop other cpus and raise our priority. since there is only 280 * one active cpu after this, and our priority will be too high 281 * for us to be preempted, we're essentially single threaded 282 * from here on out. 283 */ 284 (void) spl6(); 285 if (!panicstr) { 286 mutex_enter(&cpu_lock); 287 pause_cpus(NULL, NULL); 288 mutex_exit(&cpu_lock); 289 } 290 291 /* 292 * If the system is panicking, the preloaded kernel is valid, and 293 * fastreboot_onpanic has been set, and the system has been up for 294 * longer than fastreboot_onpanic_uptime (default to 10 minutes), 295 * choose Fast Reboot. 296 */ 297 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid && 298 fastreboot_onpanic && 299 (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) { 300 fcn = AD_FASTREBOOT; 301 } 302 303 /* 304 * Try to quiesce devices. 305 */ 306 if (is_first_quiesce) { 307 /* 308 * Clear is_first_quiesce before calling quiesce_devices() 309 * so that if quiesce_devices() causes panics, it will not 310 * be invoked again. 311 */ 312 is_first_quiesce = 0; 313 314 quiesce_active = 1; 315 quiesce_devices(ddi_root_node(), &reset_status); 316 if (reset_status == -1) { 317 if (fcn == AD_FASTREBOOT && !force_fastreboot) { 318 prom_printf("Driver(s) not capable of fast " 319 "reboot.\n"); 320 prom_printf(fallback_str); 321 fastreboot_capable = 0; 322 fcn = AD_BOOT; 323 } else if (fcn != AD_FASTREBOOT) 324 fastreboot_capable = 0; 325 } 326 quiesce_active = 0; 327 } 328 329 /* 330 * Try to reset devices. reset_leaves() should only be called 331 * a) when there are no other threads that could be accessing devices, 332 * and 333 * b) on a system that's not capable of fast reboot (fastreboot_capable 334 * being 0), or on a system where quiesce_devices() failed to 335 * complete (quiesce_active being 1). 336 */ 337 if (is_first_reset && (!fastreboot_capable || quiesce_active)) { 338 /* 339 * Clear is_first_reset before calling reset_devices() 340 * so that if reset_devices() causes panics, it will not 341 * be invoked again. 342 */ 343 is_first_reset = 0; 344 reset_leaves(); 345 } 346 347 /* Verify newkernel checksum */ 348 if (fastreboot_capable && fcn == AD_FASTREBOOT && 349 fastboot_cksum_verify(&newkernel) != 0) { 350 fastreboot_capable = 0; 351 prom_printf("Fast reboot: checksum failed for the new " 352 "kernel.\n"); 353 prom_printf(fallback_str); 354 } 355 356 (void) spl8(); 357 358 if (fastreboot_capable && fcn == AD_FASTREBOOT) { 359 /* 360 * psm_shutdown is called within fast_reboot() 361 */ 362 fast_reboot(); 363 } else { 364 (*psm_shutdownf)(cmd, fcn); 365 366 if (fcn == AD_HALT || fcn == AD_POWEROFF) 367 halt((char *)NULL); 368 else 369 prom_reboot(""); 370 } 371 /*NOTREACHED*/ 372 } 373 374 /* mdpreboot - may be called prior to mdboot while root fs still mounted */ 375 /*ARGSUSED*/ 376 void 377 mdpreboot(int cmd, int fcn, char *mdep) 378 { 379 if (fcn == AD_FASTREBOOT && !fastreboot_capable) { 380 fcn = AD_BOOT; 381 #ifdef __xpv 382 cmn_err(CE_WARN, "Fast reboot is not supported on xVM"); 383 #else 384 cmn_err(CE_WARN, 385 "Fast reboot is not supported on this platform%s", 386 fastreboot_nosup_message()); 387 #endif 388 } 389 390 if (fcn == AD_FASTREBOOT) { 391 fastboot_load_kernel(mdep); 392 if (!newkernel.fi_valid) 393 fcn = AD_BOOT; 394 } 395 396 (*psm_preshutdownf)(cmd, fcn); 397 } 398 399 static void 400 stop_other_cpus(void) 401 { 402 ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */ 403 cpuset_t xcset; 404 405 CPUSET_ALL_BUT(xcset, CPU->cpu_id); 406 xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt); 407 restore_int_flag(s); 408 } 409 410 /* 411 * Machine dependent abort sequence handling 412 */ 413 void 414 abort_sequence_enter(char *msg) 415 { 416 if (abort_enable == 0) { 417 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 418 audit_enterprom(0); 419 return; 420 } 421 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 422 audit_enterprom(1); 423 debug_enter(msg); 424 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 425 audit_exitprom(1); 426 } 427 428 /* 429 * Enter debugger. Called when the user types ctrl-alt-d or whenever 430 * code wants to enter the debugger and possibly resume later. 431 * 432 * msg: message to print, possibly NULL. 433 */ 434 void 435 debug_enter(char *msg) 436 { 437 if (dtrace_debugger_init != NULL) 438 (*dtrace_debugger_init)(); 439 440 if (msg != NULL || (boothowto & RB_DEBUG)) 441 prom_printf("\n"); 442 443 if (msg != NULL) 444 prom_printf("%s\n", msg); 445 446 if (boothowto & RB_DEBUG) 447 kmdb_enter(); 448 449 if (dtrace_debugger_fini != NULL) 450 (*dtrace_debugger_fini)(); 451 } 452 453 void 454 reset(void) 455 { 456 extern void acpi_reset_system(); 457 #if !defined(__xpv) 458 ushort_t *bios_memchk; 459 460 /* 461 * Can't use psm_map_phys or acpi_reset_system before the hat is 462 * initialized. 463 */ 464 if (khat_running) { 465 bios_memchk = (ushort_t *)psm_map_phys(0x472, 466 sizeof (ushort_t), PROT_READ | PROT_WRITE); 467 if (bios_memchk) 468 *bios_memchk = 0x1234; /* bios memory check disable */ 469 470 if (options_dip != NULL && 471 ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0, 472 "efi-systab")) { 473 if (bootops == NULL) 474 acpi_reset_system(); 475 efi_reset(); 476 } 477 478 /* 479 * The problem with using stubs is that we can call 480 * acpi_reset_system only after the kernel is up and running. 481 * 482 * We should create a global state to keep track of how far 483 * up the kernel is but for the time being we will depend on 484 * bootops. bootops cleared in startup_end(). 485 */ 486 if (bootops == NULL) 487 acpi_reset_system(); 488 } 489 490 pc_reset(); 491 #else 492 if (IN_XPV_PANIC()) { 493 if (khat_running && bootops == NULL) { 494 acpi_reset_system(); 495 } 496 497 pc_reset(); 498 } 499 500 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot); 501 panic("HYPERVISOR_shutdown() failed"); 502 #endif 503 /*NOTREACHED*/ 504 } 505 506 /* 507 * Halt the machine and return to the monitor 508 */ 509 void 510 halt(char *s) 511 { 512 stop_other_cpus(); /* send stop signal to other CPUs */ 513 if (s) 514 prom_printf("(%s) \n", s); 515 prom_exit_to_mon(); 516 /*NOTREACHED*/ 517 } 518 519 /* 520 * Initiate interrupt redistribution. 521 */ 522 void 523 i_ddi_intr_redist_all_cpus() 524 { 525 } 526 527 /* 528 * XXX These probably ought to live somewhere else 529 * XXX They are called from mem.c 530 */ 531 532 /* 533 * Convert page frame number to an OBMEM page frame number 534 * (i.e. put in the type bits -- zero for this implementation) 535 */ 536 pfn_t 537 impl_obmem_pfnum(pfn_t pf) 538 { 539 return (pf); 540 } 541 542 #ifdef NM_DEBUG 543 int nmi_test = 0; /* checked in intentry.s during clock int */ 544 int nmtest = -1; 545 nmfunc1(int arg, struct regs *rp) 546 { 547 printf("nmi called with arg = %x, regs = %x\n", arg, rp); 548 nmtest += 50; 549 if (arg == nmtest) { 550 printf("ip = %x\n", rp->r_pc); 551 return (1); 552 } 553 return (0); 554 } 555 556 #endif 557 558 #include <sys/bootsvcs.h> 559 560 /* Hacked up initialization for initial kernel check out is HERE. */ 561 /* The basic steps are: */ 562 /* kernel bootfuncs definition/initialization for KADB */ 563 /* kadb bootfuncs pointer initialization */ 564 /* putchar/getchar (interrupts disabled) */ 565 566 /* kadb bootfuncs pointer initialization */ 567 568 int 569 sysp_getchar() 570 { 571 int i; 572 ulong_t s; 573 574 if (cons_polledio == NULL) { 575 /* Uh oh */ 576 prom_printf("getchar called with no console\n"); 577 for (;;) 578 /* LOOP FOREVER */; 579 } 580 581 s = clear_int_flag(); 582 i = cons_polledio->cons_polledio_getchar( 583 cons_polledio->cons_polledio_argument); 584 restore_int_flag(s); 585 return (i); 586 } 587 588 void 589 sysp_putchar(int c) 590 { 591 ulong_t s; 592 593 /* 594 * We have no alternative but to drop the output on the floor. 595 */ 596 if (cons_polledio == NULL || 597 cons_polledio->cons_polledio_putchar == NULL) 598 return; 599 600 s = clear_int_flag(); 601 cons_polledio->cons_polledio_putchar( 602 cons_polledio->cons_polledio_argument, c); 603 restore_int_flag(s); 604 } 605 606 int 607 sysp_ischar() 608 { 609 int i; 610 ulong_t s; 611 612 if (cons_polledio == NULL || 613 cons_polledio->cons_polledio_ischar == NULL) 614 return (0); 615 616 s = clear_int_flag(); 617 i = cons_polledio->cons_polledio_ischar( 618 cons_polledio->cons_polledio_argument); 619 restore_int_flag(s); 620 return (i); 621 } 622 623 int 624 goany(void) 625 { 626 prom_printf("Type any key to continue "); 627 (void) prom_getchar(); 628 prom_printf("\n"); 629 return (1); 630 } 631 632 static struct boot_syscalls kern_sysp = { 633 sysp_getchar, /* unchar (*getchar)(); 7 */ 634 sysp_putchar, /* int (*putchar)(); 8 */ 635 sysp_ischar, /* int (*ischar)(); 9 */ 636 }; 637 638 #if defined(__xpv) 639 int using_kern_polledio; 640 #endif 641 642 void 643 kadb_uses_kernel() 644 { 645 /* 646 * This routine is now totally misnamed, since it does not in fact 647 * control kadb's I/O; it only controls the kernel's prom_* I/O. 648 */ 649 sysp = &kern_sysp; 650 #if defined(__xpv) 651 using_kern_polledio = 1; 652 #endif 653 } 654 655 /* 656 * the interface to the outside world 657 */ 658 659 /* 660 * poll_port -- wait for a register to achieve a 661 * specific state. Arguments are a mask of bits we care about, 662 * and two sub-masks. To return normally, all the bits in the 663 * first sub-mask must be ON, all the bits in the second sub- 664 * mask must be OFF. If about seconds pass without the register 665 * achieving the desired bit configuration, we return 1, else 666 * 0. 667 */ 668 int 669 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits) 670 { 671 int i; 672 ushort_t maskval; 673 674 for (i = 500000; i; i--) { 675 maskval = inb(port) & mask; 676 if (((maskval & onbits) == onbits) && 677 ((maskval & offbits) == 0)) 678 return (0); 679 drv_usecwait(10); 680 } 681 return (1); 682 } 683 684 /* 685 * set_idle_cpu is called from idle() when a CPU becomes idle. 686 */ 687 /*LINTED: static unused */ 688 static uint_t last_idle_cpu; 689 690 /*ARGSUSED*/ 691 void 692 set_idle_cpu(int cpun) 693 { 694 last_idle_cpu = cpun; 695 (*psm_set_idle_cpuf)(cpun); 696 } 697 698 /* 699 * unset_idle_cpu is called from idle() when a CPU is no longer idle. 700 */ 701 /*ARGSUSED*/ 702 void 703 unset_idle_cpu(int cpun) 704 { 705 (*psm_unset_idle_cpuf)(cpun); 706 } 707 708 /* 709 * This routine is almost correct now, but not quite. It still needs the 710 * equivalent concept of "hres_last_tick", just like on the sparc side. 711 * The idea is to take a snapshot of the hi-res timer while doing the 712 * hrestime_adj updates under hres_lock in locore, so that the small 713 * interval between interrupt assertion and interrupt processing is 714 * accounted for correctly. Once we have this, the code below should 715 * be modified to subtract off hres_last_tick rather than hrtime_base. 716 * 717 * I'd have done this myself, but I don't have source to all of the 718 * vendor-specific hi-res timer routines (grrr...). The generic hook I 719 * need is something like "gethrtime_unlocked()", which would be just like 720 * gethrtime() but would assume that you're already holding CLOCK_LOCK(). 721 * This is what the GET_HRTIME() macro is for on sparc (although it also 722 * serves the function of making time available without a function call 723 * so you don't take a register window overflow while traps are disabled). 724 */ 725 void 726 pc_gethrestime(timestruc_t *tp) 727 { 728 int lock_prev; 729 timestruc_t now; 730 int nslt; /* nsec since last tick */ 731 int adj; /* amount of adjustment to apply */ 732 733 loop: 734 lock_prev = hres_lock; 735 now = hrestime; 736 nslt = (int)(gethrtime() - hres_last_tick); 737 if (nslt < 0) { 738 /* 739 * nslt < 0 means a tick came between sampling 740 * gethrtime() and hres_last_tick; restart the loop 741 */ 742 743 goto loop; 744 } 745 now.tv_nsec += nslt; 746 if (hrestime_adj != 0) { 747 if (hrestime_adj > 0) { 748 adj = (nslt >> ADJ_SHIFT); 749 if (adj > hrestime_adj) 750 adj = (int)hrestime_adj; 751 } else { 752 adj = -(nslt >> ADJ_SHIFT); 753 if (adj < hrestime_adj) 754 adj = (int)hrestime_adj; 755 } 756 now.tv_nsec += adj; 757 } 758 while ((unsigned long)now.tv_nsec >= NANOSEC) { 759 760 /* 761 * We might have a large adjustment or have been in the 762 * debugger for a long time; take care of (at most) four 763 * of those missed seconds (tv_nsec is 32 bits, so 764 * anything >4s will be wrapping around). However, 765 * anything more than 2 seconds out of sync will trigger 766 * timedelta from clock() to go correct the time anyway, 767 * so do what we can, and let the big crowbar do the 768 * rest. A similar correction while loop exists inside 769 * hres_tick(); in all cases we'd like tv_nsec to 770 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing 771 * user processes, but if tv_sec's a little behind for a 772 * little while, that's OK; time still monotonically 773 * increases. 774 */ 775 776 now.tv_nsec -= NANOSEC; 777 now.tv_sec++; 778 } 779 if ((hres_lock & ~1) != lock_prev) 780 goto loop; 781 782 *tp = now; 783 } 784 785 void 786 gethrestime_lasttick(timespec_t *tp) 787 { 788 int s; 789 790 s = hr_clock_lock(); 791 *tp = hrestime; 792 hr_clock_unlock(s); 793 } 794 795 time_t 796 gethrestime_sec(void) 797 { 798 timestruc_t now; 799 800 gethrestime(&now); 801 return (now.tv_sec); 802 } 803 804 /* 805 * Initialize a kernel thread's stack 806 */ 807 808 caddr_t 809 thread_stk_init(caddr_t stk) 810 { 811 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0); 812 return (stk - SA(MINFRAME)); 813 } 814 815 /* 816 * Initialize lwp's kernel stack. 817 */ 818 819 #ifdef TRAPTRACE 820 /* 821 * There's a tricky interdependency here between use of sysenter and 822 * TRAPTRACE which needs recording to avoid future confusion (this is 823 * about the third time I've re-figured this out ..) 824 * 825 * Here's how debugging lcall works with TRAPTRACE. 826 * 827 * 1 We're in userland with a breakpoint on the lcall instruction. 828 * 2 We execute the instruction - the instruction pushes the userland 829 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel 830 * via the call gate. 831 * 3 The hardware raises a debug trap in kernel mode, the hardware 832 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt. 833 * 4 dbgtrap pushes the error code and trapno and calls cmntrap 834 * 5 cmntrap finishes building a trap frame 835 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk 836 * off the stack into the traptrace buffer. 837 * 838 * This means that the traptrace buffer contains the wrong values in 839 * %esp and %ss, but everything else in there is correct. 840 * 841 * Here's how debugging sysenter works with TRAPTRACE. 842 * 843 * a We're in userland with a breakpoint on the sysenter instruction. 844 * b We execute the instruction - the instruction pushes -nothing- 845 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged 846 * values to take us to sys_sysenter, at the top of the lwp's 847 * stack. 848 * c goto 3 849 * 850 * At this point, because we got into the kernel without the requisite 851 * five pushes on the stack, if we didn't make extra room, we'd 852 * end up with the TRACE_REGS macro fetching the saved %ss and %esp 853 * values from negative (unmapped) stack addresses -- which really bites. 854 * That's why we do the '-= 8' below. 855 * 856 * XXX Note that reading "up" lwp0's stack works because t0 is declared 857 * right next to t0stack in locore.s 858 */ 859 #endif 860 861 caddr_t 862 lwp_stk_init(klwp_t *lwp, caddr_t stk) 863 { 864 caddr_t oldstk; 865 struct pcb *pcb = &lwp->lwp_pcb; 866 867 oldstk = stk; 868 stk -= SA(sizeof (struct regs) + SA(MINFRAME)); 869 #ifdef TRAPTRACE 870 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */ 871 #endif 872 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul)); 873 bzero(stk, oldstk - stk); 874 lwp->lwp_regs = (void *)(stk + SA(MINFRAME)); 875 876 /* 877 * Arrange that the virtualized %fs and %gs GDT descriptors 878 * have a well-defined initial state (present, ring 3 879 * and of type data). 880 */ 881 #if defined(__amd64) 882 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) 883 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 884 else 885 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc; 886 #elif defined(__i386) 887 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 888 #endif /* __i386 */ 889 lwp_installctx(lwp); 890 return (stk); 891 } 892 893 /* 894 * Use this opportunity to free any dynamically allocated fp storage. 895 */ 896 void 897 lwp_stk_fini(klwp_t *lwp) 898 { 899 fp_lwp_cleanup(lwp); 900 } 901 902 void 903 lwp_fp_init(klwp_t *lwp) 904 { 905 fp_lwp_init(lwp); 906 } 907 908 /* 909 * If we're not the panic CPU, we wait in panic_idle for reboot. 910 */ 911 void 912 panic_idle(void) 913 { 914 splx(ipltospl(CLOCK_LEVEL)); 915 (void) setjmp(&curthread->t_pcb); 916 917 dumpsys_helper(); 918 919 #ifndef __xpv 920 for (;;) 921 i86_halt(); 922 #else 923 for (;;) 924 ; 925 #endif 926 } 927 928 /* 929 * Stop the other CPUs by cross-calling them and forcing them to enter 930 * the panic_idle() loop above. 931 */ 932 /*ARGSUSED*/ 933 void 934 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl) 935 { 936 processorid_t i; 937 cpuset_t xcset; 938 939 /* 940 * In the case of a Xen panic, the hypervisor has already stopped 941 * all of the CPUs. 942 */ 943 if (!IN_XPV_PANIC()) { 944 (void) splzs(); 945 946 CPUSET_ALL_BUT(xcset, cp->cpu_id); 947 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle); 948 } 949 950 for (i = 0; i < NCPU; i++) { 951 if (i != cp->cpu_id && cpu[i] != NULL && 952 (cpu[i]->cpu_flags & CPU_EXISTS)) 953 cpu[i]->cpu_flags |= CPU_QUIESCED; 954 } 955 } 956 957 /* 958 * Platform callback following each entry to panicsys(). 959 */ 960 /*ARGSUSED*/ 961 void 962 panic_enter_hw(int spl) 963 { 964 /* Nothing to do here */ 965 } 966 967 /* 968 * Platform-specific code to execute after panicstr is set: we invoke 969 * the PSM entry point to indicate that a panic has occurred. 970 */ 971 /*ARGSUSED*/ 972 void 973 panic_quiesce_hw(panic_data_t *pdp) 974 { 975 psm_notifyf(PSM_PANIC_ENTER); 976 977 cmi_panic_callback(); 978 979 #ifdef TRAPTRACE 980 /* 981 * Turn off TRAPTRACE 982 */ 983 TRAPTRACE_FREEZE; 984 #endif /* TRAPTRACE */ 985 } 986 987 /* 988 * Platform callback prior to writing crash dump. 989 */ 990 /*ARGSUSED*/ 991 void 992 panic_dump_hw(int spl) 993 { 994 /* Nothing to do here */ 995 } 996 997 void * 998 plat_traceback(void *fpreg) 999 { 1000 #ifdef __xpv 1001 if (IN_XPV_PANIC()) 1002 return (xpv_traceback(fpreg)); 1003 #endif 1004 return (fpreg); 1005 } 1006 1007 /*ARGSUSED*/ 1008 void 1009 plat_tod_fault(enum tod_fault_type tod_bad) 1010 {} 1011 1012 /*ARGSUSED*/ 1013 int 1014 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class) 1015 { 1016 return (ENOTSUP); 1017 } 1018 1019 /* 1020 * The underlying console output routines are protected by raising IPL in case 1021 * we are still calling into the early boot services. Once we start calling 1022 * the kernel console emulator, it will disable interrupts completely during 1023 * character rendering (see sysp_putchar, for example). Refer to the comments 1024 * and code in common/os/console.c for more information on these callbacks. 1025 */ 1026 /*ARGSUSED*/ 1027 int 1028 console_enter(int busy) 1029 { 1030 return (splzs()); 1031 } 1032 1033 /*ARGSUSED*/ 1034 void 1035 console_exit(int busy, int spl) 1036 { 1037 splx(spl); 1038 } 1039 1040 /* 1041 * Allocate a region of virtual address space, unmapped. 1042 * Stubbed out except on sparc, at least for now. 1043 */ 1044 /*ARGSUSED*/ 1045 void * 1046 boot_virt_alloc(void *addr, size_t size) 1047 { 1048 return (addr); 1049 } 1050 1051 volatile unsigned long tenmicrodata; 1052 1053 void 1054 tenmicrosec(void) 1055 { 1056 extern int gethrtime_hires; 1057 1058 if (gethrtime_hires) { 1059 hrtime_t start, end; 1060 start = end = gethrtime(); 1061 while ((end - start) < (10 * (NANOSEC / MICROSEC))) { 1062 SMT_PAUSE(); 1063 end = gethrtime(); 1064 } 1065 } else { 1066 #if defined(__xpv) 1067 hrtime_t newtime; 1068 1069 newtime = xpv_gethrtime() + 10000; /* now + 10 us */ 1070 while (xpv_gethrtime() < newtime) 1071 SMT_PAUSE(); 1072 #else /* __xpv */ 1073 int i; 1074 1075 /* 1076 * Artificial loop to induce delay. 1077 */ 1078 for (i = 0; i < microdata; i++) 1079 tenmicrodata = microdata; 1080 #endif /* __xpv */ 1081 } 1082 } 1083 1084 /* 1085 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES 1086 * long, and it fills in the array with the time spent on cpu in 1087 * each of the mstates, where time is returned in nsec. 1088 * 1089 * No guarantee is made that the returned values in times[] will 1090 * monotonically increase on sequential calls, although this will 1091 * be true in the long run. Any such guarantee must be handled by 1092 * the caller, if needed. This can happen if we fail to account 1093 * for elapsed time due to a generation counter conflict, yet we 1094 * did account for it on a prior call (see below). 1095 * 1096 * The complication is that the cpu in question may be updating 1097 * its microstate at the same time that we are reading it. 1098 * Because the microstate is only updated when the CPU's state 1099 * changes, the values in cpu_intracct[] can be indefinitely out 1100 * of date. To determine true current values, it is necessary to 1101 * compare the current time with cpu_mstate_start, and add the 1102 * difference to times[cpu_mstate]. 1103 * 1104 * This can be a problem if those values are changing out from 1105 * under us. Because the code path in new_cpu_mstate() is 1106 * performance critical, we have not added a lock to it. Instead, 1107 * we have added a generation counter. Before beginning 1108 * modifications, the counter is set to 0. After modifications, 1109 * it is set to the old value plus one. 1110 * 1111 * get_cpu_mstate() will not consider the values of cpu_mstate 1112 * and cpu_mstate_start to be usable unless the value of 1113 * cpu_mstate_gen is both non-zero and unchanged, both before and 1114 * after reading the mstate information. Note that we must 1115 * protect against out-of-order loads around accesses to the 1116 * generation counter. Also, this is a best effort approach in 1117 * that we do not retry should the counter be found to have 1118 * changed. 1119 * 1120 * cpu_intracct[] is used to identify time spent in each CPU 1121 * mstate while handling interrupts. Such time should be reported 1122 * against system time, and so is subtracted out from its 1123 * corresponding cpu_acct[] time and added to 1124 * cpu_acct[CMS_SYSTEM]. 1125 */ 1126 1127 void 1128 get_cpu_mstate(cpu_t *cpu, hrtime_t *times) 1129 { 1130 int i; 1131 hrtime_t now, start; 1132 uint16_t gen; 1133 uint16_t state; 1134 hrtime_t intracct[NCMSTATES]; 1135 1136 /* 1137 * Load all volatile state under the protection of membar. 1138 * cpu_acct[cpu_mstate] must be loaded to avoid double counting 1139 * of (now - cpu_mstate_start) by a change in CPU mstate that 1140 * arrives after we make our last check of cpu_mstate_gen. 1141 */ 1142 1143 now = gethrtime_unscaled(); 1144 gen = cpu->cpu_mstate_gen; 1145 1146 membar_consumer(); /* guarantee load ordering */ 1147 start = cpu->cpu_mstate_start; 1148 state = cpu->cpu_mstate; 1149 for (i = 0; i < NCMSTATES; i++) { 1150 intracct[i] = cpu->cpu_intracct[i]; 1151 times[i] = cpu->cpu_acct[i]; 1152 } 1153 membar_consumer(); /* guarantee load ordering */ 1154 1155 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start) 1156 times[state] += now - start; 1157 1158 for (i = 0; i < NCMSTATES; i++) { 1159 if (i == CMS_SYSTEM) 1160 continue; 1161 times[i] -= intracct[i]; 1162 if (times[i] < 0) { 1163 intracct[i] += times[i]; 1164 times[i] = 0; 1165 } 1166 times[CMS_SYSTEM] += intracct[i]; 1167 scalehrtime(×[i]); 1168 } 1169 scalehrtime(×[CMS_SYSTEM]); 1170 } 1171 1172 /* 1173 * This is a version of the rdmsr instruction that allows 1174 * an error code to be returned in the case of failure. 1175 */ 1176 int 1177 checked_rdmsr(uint_t msr, uint64_t *value) 1178 { 1179 if (!is_x86_feature(x86_featureset, X86FSET_MSR)) 1180 return (ENOTSUP); 1181 *value = rdmsr(msr); 1182 return (0); 1183 } 1184 1185 /* 1186 * This is a version of the wrmsr instruction that allows 1187 * an error code to be returned in the case of failure. 1188 */ 1189 int 1190 checked_wrmsr(uint_t msr, uint64_t value) 1191 { 1192 if (!is_x86_feature(x86_featureset, X86FSET_MSR)) 1193 return (ENOTSUP); 1194 wrmsr(msr, value); 1195 return (0); 1196 } 1197 1198 /* 1199 * The mem driver's usual method of using hat_devload() to establish a 1200 * temporary mapping will not work for foreign pages mapped into this 1201 * domain or for the special hypervisor-provided pages. For the foreign 1202 * pages, we often don't know which domain owns them, so we can't ask the 1203 * hypervisor to set up a new mapping. For the other pages, we don't have 1204 * a pfn, so we can't create a new PTE. For these special cases, we do a 1205 * direct uiomove() from the existing kernel virtual address. 1206 */ 1207 /*ARGSUSED*/ 1208 int 1209 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw) 1210 { 1211 #if defined(__xpv) 1212 void *va = (void *)(uintptr_t)uio->uio_loffset; 1213 off_t pageoff = uio->uio_loffset & PAGEOFFSET; 1214 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff), 1215 (size_t)uio->uio_iov->iov_len); 1216 1217 if ((rw == UIO_READ && 1218 (va == HYPERVISOR_shared_info || va == xen_info)) || 1219 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va)))) 1220 return (uiomove(va, nbytes, rw, uio)); 1221 #endif 1222 return (ENOTSUP); 1223 } 1224 1225 pgcnt_t 1226 num_phys_pages() 1227 { 1228 pgcnt_t npages = 0; 1229 struct memlist *mp; 1230 1231 #if defined(__xpv) 1232 if (DOMAIN_IS_INITDOMAIN(xen_info)) 1233 return (xpv_nr_phys_pages()); 1234 #endif /* __xpv */ 1235 1236 for (mp = phys_install; mp != NULL; mp = mp->ml_next) 1237 npages += mp->ml_size >> PAGESHIFT; 1238 1239 return (npages); 1240 } 1241 1242 /* cpu threshold for compressed dumps */ 1243 #ifdef _LP64 1244 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU; 1245 #else 1246 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU; 1247 #endif 1248 1249 int 1250 dump_plat_addr() 1251 { 1252 #ifdef __xpv 1253 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1254 mem_vtop_t mem_vtop; 1255 int cnt; 1256 1257 /* 1258 * On the hypervisor, we want to dump the page with shared_info on it. 1259 */ 1260 if (!IN_XPV_PANIC()) { 1261 mem_vtop.m_as = &kas; 1262 mem_vtop.m_va = HYPERVISOR_shared_info; 1263 mem_vtop.m_pfn = pfn; 1264 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1265 cnt = 1; 1266 } else { 1267 cnt = dump_xpv_addr(); 1268 } 1269 return (cnt); 1270 #else 1271 return (0); 1272 #endif 1273 } 1274 1275 void 1276 dump_plat_pfn() 1277 { 1278 #ifdef __xpv 1279 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1280 1281 if (!IN_XPV_PANIC()) 1282 dumpvp_write(&pfn, sizeof (pfn)); 1283 else 1284 dump_xpv_pfn(); 1285 #endif 1286 } 1287 1288 /*ARGSUSED*/ 1289 int 1290 dump_plat_data(void *dump_cbuf) 1291 { 1292 #ifdef __xpv 1293 uint32_t csize; 1294 int cnt; 1295 1296 if (!IN_XPV_PANIC()) { 1297 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf, 1298 PAGESIZE); 1299 dumpvp_write(&csize, sizeof (uint32_t)); 1300 dumpvp_write(dump_cbuf, csize); 1301 cnt = 1; 1302 } else { 1303 cnt = dump_xpv_data(dump_cbuf); 1304 } 1305 return (cnt); 1306 #else 1307 return (0); 1308 #endif 1309 } 1310 1311 /* 1312 * Calculates a linear address, given the CS selector and PC values 1313 * by looking up the %cs selector process's LDT or the CPU's GDT. 1314 * proc->p_ldtlock must be held across this call. 1315 */ 1316 int 1317 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1318 { 1319 user_desc_t *descrp; 1320 caddr_t baseaddr; 1321 uint16_t idx = SELTOIDX(rp->r_cs); 1322 1323 ASSERT(rp->r_cs <= 0xFFFF); 1324 ASSERT(MUTEX_HELD(&p->p_ldtlock)); 1325 1326 if (SELISLDT(rp->r_cs)) { 1327 /* 1328 * Currently 64 bit processes cannot have private LDTs. 1329 */ 1330 ASSERT(p->p_model != DATAMODEL_LP64); 1331 1332 if (p->p_ldt == NULL) 1333 return (-1); 1334 1335 descrp = &p->p_ldt[idx]; 1336 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1337 1338 /* 1339 * Calculate the linear address (wraparound is not only ok, 1340 * it's expected behavior). The cast to uint32_t is because 1341 * LDT selectors are only allowed in 32-bit processes. 1342 */ 1343 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1344 rp->r_pc); 1345 } else { 1346 #ifdef DEBUG 1347 descrp = &CPU->cpu_gdt[idx]; 1348 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1349 /* GDT-based descriptors' base addresses should always be 0 */ 1350 ASSERT(baseaddr == 0); 1351 #endif 1352 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1353 } 1354 1355 return (0); 1356 } 1357 1358 /* 1359 * The implementation of dtrace_linear_pc is similar to the that of 1360 * linear_pc, above, but here we acquire p_ldtlock before accessing 1361 * p_ldt. This implementation is used by the pid provider; we prefix 1362 * it with "dtrace_" to avoid inducing spurious tracing events. 1363 */ 1364 int 1365 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1366 { 1367 user_desc_t *descrp; 1368 caddr_t baseaddr; 1369 uint16_t idx = SELTOIDX(rp->r_cs); 1370 1371 ASSERT(rp->r_cs <= 0xFFFF); 1372 1373 if (SELISLDT(rp->r_cs)) { 1374 /* 1375 * Currently 64 bit processes cannot have private LDTs. 1376 */ 1377 ASSERT(p->p_model != DATAMODEL_LP64); 1378 1379 mutex_enter(&p->p_ldtlock); 1380 if (p->p_ldt == NULL) { 1381 mutex_exit(&p->p_ldtlock); 1382 return (-1); 1383 } 1384 descrp = &p->p_ldt[idx]; 1385 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1386 mutex_exit(&p->p_ldtlock); 1387 1388 /* 1389 * Calculate the linear address (wraparound is not only ok, 1390 * it's expected behavior). The cast to uint32_t is because 1391 * LDT selectors are only allowed in 32-bit processes. 1392 */ 1393 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1394 rp->r_pc); 1395 } else { 1396 #ifdef DEBUG 1397 descrp = &CPU->cpu_gdt[idx]; 1398 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1399 /* GDT-based descriptors' base addresses should always be 0 */ 1400 ASSERT(baseaddr == 0); 1401 #endif 1402 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1403 } 1404 1405 return (0); 1406 } 1407 1408 /* 1409 * We need to post a soft interrupt to reprogram the lbolt cyclic when 1410 * switching from event to cyclic driven lbolt. The following code adds 1411 * and posts the softint for x86. 1412 */ 1413 static ddi_softint_hdl_impl_t lbolt_softint_hdl = 1414 {0, 0, NULL, NULL, 0, NULL, NULL, NULL}; 1415 1416 void 1417 lbolt_softint_add(void) 1418 { 1419 (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL, 1420 (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL); 1421 } 1422 1423 void 1424 lbolt_softint_post(void) 1425 { 1426 (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending); 1427 } 1428 1429 boolean_t 1430 plat_dr_check_capability(uint64_t features) 1431 { 1432 return ((plat_dr_options & features) == features); 1433 } 1434 1435 boolean_t 1436 plat_dr_support_cpu(void) 1437 { 1438 return (plat_dr_options & PLAT_DR_FEATURE_CPU); 1439 } 1440 1441 boolean_t 1442 plat_dr_support_memory(void) 1443 { 1444 return (plat_dr_options & PLAT_DR_FEATURE_MEMORY); 1445 } 1446 1447 void 1448 plat_dr_enable_capability(uint64_t features) 1449 { 1450 atomic_or_64(&plat_dr_options, features); 1451 } 1452 1453 void 1454 plat_dr_disable_capability(uint64_t features) 1455 { 1456 atomic_and_64(&plat_dr_options, ~features); 1457 } 1458