1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2020 Joyent, Inc. 25 */ 26 /* 27 * Copyright (c) 2010, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/segments.h> 35 #include <sys/sysmacros.h> 36 #include <sys/signal.h> 37 #include <sys/systm.h> 38 #include <sys/user.h> 39 #include <sys/mman.h> 40 #include <sys/vm.h> 41 42 #include <sys/disp.h> 43 #include <sys/class.h> 44 45 #include <sys/proc.h> 46 #include <sys/buf.h> 47 #include <sys/kmem.h> 48 49 #include <sys/reboot.h> 50 #include <sys/uadmin.h> 51 #include <sys/callb.h> 52 53 #include <sys/cred.h> 54 #include <sys/vnode.h> 55 #include <sys/file.h> 56 57 #include <sys/procfs.h> 58 #include <sys/acct.h> 59 60 #include <sys/vfs.h> 61 #include <sys/dnlc.h> 62 #include <sys/var.h> 63 #include <sys/cmn_err.h> 64 #include <sys/utsname.h> 65 #include <sys/debug.h> 66 67 #include <sys/dumphdr.h> 68 #include <sys/bootconf.h> 69 #include <sys/varargs.h> 70 #include <sys/promif.h> 71 #include <sys/modctl.h> 72 73 #include <sys/consdev.h> 74 #include <sys/frame.h> 75 76 #include <sys/sunddi.h> 77 #include <sys/ddidmareq.h> 78 #include <sys/psw.h> 79 #include <sys/regset.h> 80 #include <sys/privregs.h> 81 #include <sys/clock.h> 82 #include <sys/tss.h> 83 #include <sys/cpu.h> 84 #include <sys/stack.h> 85 #include <sys/trap.h> 86 #include <sys/pic.h> 87 #include <vm/hat.h> 88 #include <vm/anon.h> 89 #include <vm/as.h> 90 #include <vm/page.h> 91 #include <vm/seg.h> 92 #include <vm/seg_kmem.h> 93 #include <vm/seg_map.h> 94 #include <vm/seg_vn.h> 95 #include <vm/seg_kp.h> 96 #include <vm/hat_i86.h> 97 #include <sys/swap.h> 98 #include <sys/thread.h> 99 #include <sys/sysconf.h> 100 #include <sys/vm_machparam.h> 101 #include <sys/archsystm.h> 102 #include <sys/machsystm.h> 103 #include <sys/machlock.h> 104 #include <sys/x_call.h> 105 #include <sys/instance.h> 106 107 #include <sys/time.h> 108 #include <sys/smp_impldefs.h> 109 #include <sys/psm_types.h> 110 #include <sys/atomic.h> 111 #include <sys/panic.h> 112 #include <sys/cpuvar.h> 113 #include <sys/dtrace.h> 114 #include <sys/bl.h> 115 #include <sys/nvpair.h> 116 #include <sys/x86_archext.h> 117 #include <sys/pool_pset.h> 118 #include <sys/autoconf.h> 119 #include <sys/mem.h> 120 #include <sys/dumphdr.h> 121 #include <sys/compress.h> 122 #include <sys/cpu_module.h> 123 #if defined(__xpv) 124 #include <sys/hypervisor.h> 125 #include <sys/xpv_panic.h> 126 #endif 127 128 #include <sys/fastboot.h> 129 #include <sys/machelf.h> 130 #include <sys/kobj.h> 131 #include <sys/multiboot.h> 132 133 #ifdef TRAPTRACE 134 #include <sys/traptrace.h> 135 #endif /* TRAPTRACE */ 136 137 #include <c2/audit.h> 138 #include <sys/clock_impl.h> 139 140 extern void audit_enterprom(int); 141 extern void audit_exitprom(int); 142 143 /* 144 * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used. 145 */ 146 int apix_enable = 1; 147 148 int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */ 149 150 /* 151 * Occassionally the kernel knows better whether to power-off or reboot. 152 */ 153 int force_shutdown_method = AD_UNKNOWN; 154 155 /* 156 * The panicbuf array is used to record messages and state: 157 */ 158 char panicbuf[PANICBUFSIZE]; 159 160 /* 161 * Flags to control Dynamic Reconfiguration features. 162 */ 163 uint64_t plat_dr_options; 164 165 /* 166 * Maximum physical address for memory DR operations. 167 */ 168 uint64_t plat_dr_physmax; 169 170 /* 171 * maxphys - used during physio 172 * klustsize - used for klustering by swapfs and specfs 173 */ 174 int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */ 175 int klustsize = 56 * 1024; 176 177 caddr_t p0_va; /* Virtual address for accessing physical page 0 */ 178 179 /* 180 * defined here, though unused on x86, 181 * to make kstat_fr.c happy. 182 */ 183 int vac; 184 185 void debug_enter(char *); 186 187 extern void pm_cfb_check_and_powerup(void); 188 extern void pm_cfb_rele(void); 189 190 extern fastboot_info_t newkernel; 191 192 /* 193 * Instructions to enable or disable SMAP, respectively. 194 */ 195 static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca }; 196 static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb }; 197 198 /* 199 * Machine dependent code to reboot. 200 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer 201 * to a string to be used as the argument string when rebooting. 202 * 203 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely 204 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when 205 * we are in a normal shutdown sequence (interrupts are not blocked, the 206 * system is not panic'ing or being suspended). 207 */ 208 /*ARGSUSED*/ 209 void 210 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb) 211 { 212 processorid_t bootcpuid = 0; 213 static int is_first_quiesce = 1; 214 static int is_first_reset = 1; 215 int reset_status = 0; 216 static char fallback_str[] = "Falling back to regular reboot.\n"; 217 218 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid) 219 fcn = AD_BOOT; 220 221 if (!panicstr) { 222 kpreempt_disable(); 223 if (fcn == AD_FASTREBOOT) { 224 mutex_enter(&cpu_lock); 225 if (CPU_ACTIVE(cpu_get(bootcpuid))) { 226 affinity_set(bootcpuid); 227 } 228 mutex_exit(&cpu_lock); 229 } else { 230 affinity_set(CPU_CURRENT); 231 } 232 } 233 234 if (force_shutdown_method != AD_UNKNOWN) 235 fcn = force_shutdown_method; 236 237 /* 238 * XXX - rconsvp is set to NULL to ensure that output messages 239 * are sent to the underlying "hardware" device using the 240 * monitor's printf routine since we are in the process of 241 * either rebooting or halting the machine. 242 */ 243 rconsvp = NULL; 244 245 /* 246 * Print the reboot message now, before pausing other cpus. 247 * There is a race condition in the printing support that 248 * can deadlock multiprocessor machines. 249 */ 250 if (!(fcn == AD_HALT || fcn == AD_POWEROFF)) 251 prom_printf("rebooting...\n"); 252 253 if (IN_XPV_PANIC()) 254 reset(); 255 256 /* 257 * We can't bring up the console from above lock level, so do it now 258 */ 259 pm_cfb_check_and_powerup(); 260 261 /* make sure there are no more changes to the device tree */ 262 devtree_freeze(); 263 264 if (invoke_cb) 265 (void) callb_execute_class(CB_CL_MDBOOT, 0); 266 267 /* 268 * Clear any unresolved UEs from memory. 269 */ 270 page_retire_mdboot(); 271 272 #if defined(__xpv) 273 /* 274 * XXPV Should probably think some more about how we deal 275 * with panicing before it's really safe to panic. 276 * On hypervisors, we reboot very quickly.. Perhaps panic 277 * should only attempt to recover by rebooting if, 278 * say, we were able to mount the root filesystem, 279 * or if we successfully launched init(8). 280 */ 281 if (panicstr && proc_init == NULL) 282 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff); 283 #endif 284 /* 285 * stop other cpus and raise our priority. since there is only 286 * one active cpu after this, and our priority will be too high 287 * for us to be preempted, we're essentially single threaded 288 * from here on out. 289 */ 290 (void) spl6(); 291 if (!panicstr) { 292 mutex_enter(&cpu_lock); 293 pause_cpus(NULL, NULL); 294 mutex_exit(&cpu_lock); 295 } 296 297 /* 298 * If the system is panicking, the preloaded kernel is valid, and 299 * fastreboot_onpanic has been set, and the system has been up for 300 * longer than fastreboot_onpanic_uptime (default to 10 minutes), 301 * choose Fast Reboot. 302 */ 303 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid && 304 fastreboot_onpanic && 305 (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) { 306 fcn = AD_FASTREBOOT; 307 } 308 309 /* 310 * Try to quiesce devices. 311 */ 312 if (is_first_quiesce) { 313 /* 314 * Clear is_first_quiesce before calling quiesce_devices() 315 * so that if quiesce_devices() causes panics, it will not 316 * be invoked again. 317 */ 318 is_first_quiesce = 0; 319 320 quiesce_active = 1; 321 quiesce_devices(ddi_root_node(), &reset_status); 322 if (reset_status == -1) { 323 if (fcn == AD_FASTREBOOT && !force_fastreboot) { 324 prom_printf("Driver(s) not capable of fast " 325 "reboot.\n"); 326 prom_printf(fallback_str); 327 fastreboot_capable = 0; 328 fcn = AD_BOOT; 329 } else if (fcn != AD_FASTREBOOT) 330 fastreboot_capable = 0; 331 } 332 quiesce_active = 0; 333 } 334 335 /* 336 * Try to reset devices. reset_leaves() should only be called 337 * a) when there are no other threads that could be accessing devices, 338 * and 339 * b) on a system that's not capable of fast reboot (fastreboot_capable 340 * being 0), or on a system where quiesce_devices() failed to 341 * complete (quiesce_active being 1). 342 */ 343 if (is_first_reset && (!fastreboot_capable || quiesce_active)) { 344 /* 345 * Clear is_first_reset before calling reset_devices() 346 * so that if reset_devices() causes panics, it will not 347 * be invoked again. 348 */ 349 is_first_reset = 0; 350 reset_leaves(); 351 } 352 353 /* Verify newkernel checksum */ 354 if (fastreboot_capable && fcn == AD_FASTREBOOT && 355 fastboot_cksum_verify(&newkernel) != 0) { 356 fastreboot_capable = 0; 357 prom_printf("Fast reboot: checksum failed for the new " 358 "kernel.\n"); 359 prom_printf(fallback_str); 360 } 361 362 (void) spl8(); 363 364 if (fastreboot_capable && fcn == AD_FASTREBOOT) { 365 /* 366 * psm_shutdown is called within fast_reboot() 367 */ 368 fast_reboot(); 369 } else { 370 (*psm_shutdownf)(cmd, fcn); 371 372 if (fcn == AD_HALT || fcn == AD_POWEROFF) 373 halt((char *)NULL); 374 else 375 prom_reboot(""); 376 } 377 /*NOTREACHED*/ 378 } 379 380 /* mdpreboot - may be called prior to mdboot while root fs still mounted */ 381 /*ARGSUSED*/ 382 void 383 mdpreboot(int cmd, int fcn, char *mdep) 384 { 385 if (fcn == AD_FASTREBOOT && !fastreboot_capable) { 386 fcn = AD_BOOT; 387 #ifdef __xpv 388 cmn_err(CE_WARN, "Fast reboot is not supported on xVM"); 389 #else 390 cmn_err(CE_WARN, 391 "Fast reboot is not supported on this platform%s", 392 fastreboot_nosup_message()); 393 #endif 394 } 395 396 if (fcn == AD_FASTREBOOT) { 397 fastboot_load_kernel(mdep); 398 if (!newkernel.fi_valid) 399 fcn = AD_BOOT; 400 } 401 402 (*psm_preshutdownf)(cmd, fcn); 403 } 404 405 static void 406 stop_other_cpus(void) 407 { 408 ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */ 409 cpuset_t xcset; 410 411 CPUSET_ALL_BUT(xcset, CPU->cpu_id); 412 xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt); 413 restore_int_flag(s); 414 } 415 416 /* 417 * Machine dependent abort sequence handling 418 */ 419 void 420 abort_sequence_enter(char *msg) 421 { 422 if (abort_enable == 0) { 423 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 424 audit_enterprom(0); 425 return; 426 } 427 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 428 audit_enterprom(1); 429 debug_enter(msg); 430 if (AU_ZONE_AUDITING(GET_KCTX_GZ)) 431 audit_exitprom(1); 432 } 433 434 /* 435 * Enter debugger. Called when the user types ctrl-alt-d or whenever 436 * code wants to enter the debugger and possibly resume later. 437 * 438 * msg: message to print, possibly NULL. 439 */ 440 void 441 debug_enter(char *msg) 442 { 443 if (dtrace_debugger_init != NULL) 444 (*dtrace_debugger_init)(); 445 446 if (msg != NULL || (boothowto & RB_DEBUG)) 447 prom_printf("\n"); 448 449 if (msg != NULL) 450 prom_printf("%s\n", msg); 451 452 if (boothowto & RB_DEBUG) 453 kmdb_enter(); 454 455 if (dtrace_debugger_fini != NULL) 456 (*dtrace_debugger_fini)(); 457 } 458 459 void 460 reset(void) 461 { 462 extern void acpi_reset_system(); 463 #if !defined(__xpv) 464 ushort_t *bios_memchk; 465 466 /* 467 * Can't use psm_map_phys or acpi_reset_system before the hat is 468 * initialized. 469 */ 470 if (khat_running) { 471 bios_memchk = (ushort_t *)psm_map_phys(0x472, 472 sizeof (ushort_t), PROT_READ | PROT_WRITE); 473 if (bios_memchk) 474 *bios_memchk = 0x1234; /* bios memory check disable */ 475 476 if (options_dip != NULL && 477 ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0, 478 "efi-systab")) { 479 if (bootops == NULL) 480 acpi_reset_system(); 481 efi_reset(); 482 } 483 484 /* 485 * The problem with using stubs is that we can call 486 * acpi_reset_system only after the kernel is up and running. 487 * 488 * We should create a global state to keep track of how far 489 * up the kernel is but for the time being we will depend on 490 * bootops. bootops cleared in startup_end(). 491 */ 492 if (bootops == NULL) 493 acpi_reset_system(); 494 } 495 496 pc_reset(); 497 #else 498 if (IN_XPV_PANIC()) { 499 if (khat_running && bootops == NULL) { 500 acpi_reset_system(); 501 } 502 503 pc_reset(); 504 } 505 506 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot); 507 panic("HYPERVISOR_shutdown() failed"); 508 #endif 509 /*NOTREACHED*/ 510 } 511 512 /* 513 * Halt the machine and return to the monitor 514 */ 515 void 516 halt(char *s) 517 { 518 stop_other_cpus(); /* send stop signal to other CPUs */ 519 if (s) 520 prom_printf("(%s) \n", s); 521 prom_exit_to_mon(); 522 /*NOTREACHED*/ 523 } 524 525 /* 526 * Initiate interrupt redistribution. 527 */ 528 void 529 i_ddi_intr_redist_all_cpus() 530 { 531 } 532 533 /* 534 * XXX These probably ought to live somewhere else 535 * XXX They are called from mem.c 536 */ 537 538 /* 539 * Convert page frame number to an OBMEM page frame number 540 * (i.e. put in the type bits -- zero for this implementation) 541 */ 542 pfn_t 543 impl_obmem_pfnum(pfn_t pf) 544 { 545 return (pf); 546 } 547 548 #ifdef NM_DEBUG 549 int nmi_test = 0; /* checked in intentry.s during clock int */ 550 int nmtest = -1; 551 nmfunc1(int arg, struct regs *rp) 552 { 553 printf("nmi called with arg = %x, regs = %x\n", arg, rp); 554 nmtest += 50; 555 if (arg == nmtest) { 556 printf("ip = %x\n", rp->r_pc); 557 return (1); 558 } 559 return (0); 560 } 561 562 #endif 563 564 #include <sys/bootsvcs.h> 565 566 /* Hacked up initialization for initial kernel check out is HERE. */ 567 /* The basic steps are: */ 568 /* kernel bootfuncs definition/initialization for KADB */ 569 /* kadb bootfuncs pointer initialization */ 570 /* putchar/getchar (interrupts disabled) */ 571 572 /* kadb bootfuncs pointer initialization */ 573 574 int 575 sysp_getchar() 576 { 577 int i; 578 ulong_t s; 579 580 if (cons_polledio == NULL) { 581 /* Uh oh */ 582 prom_printf("getchar called with no console\n"); 583 for (;;) 584 /* LOOP FOREVER */; 585 } 586 587 s = clear_int_flag(); 588 i = cons_polledio->cons_polledio_getchar( 589 cons_polledio->cons_polledio_argument); 590 restore_int_flag(s); 591 return (i); 592 } 593 594 void 595 sysp_putchar(int c) 596 { 597 ulong_t s; 598 599 /* 600 * We have no alternative but to drop the output on the floor. 601 */ 602 if (cons_polledio == NULL || 603 cons_polledio->cons_polledio_putchar == NULL) 604 return; 605 606 s = clear_int_flag(); 607 cons_polledio->cons_polledio_putchar( 608 cons_polledio->cons_polledio_argument, c); 609 restore_int_flag(s); 610 } 611 612 int 613 sysp_ischar() 614 { 615 int i; 616 ulong_t s; 617 618 if (cons_polledio == NULL || 619 cons_polledio->cons_polledio_ischar == NULL) 620 return (0); 621 622 s = clear_int_flag(); 623 i = cons_polledio->cons_polledio_ischar( 624 cons_polledio->cons_polledio_argument); 625 restore_int_flag(s); 626 return (i); 627 } 628 629 int 630 goany(void) 631 { 632 prom_printf("Type any key to continue "); 633 (void) prom_getchar(); 634 prom_printf("\n"); 635 return (1); 636 } 637 638 static struct boot_syscalls kern_sysp = { 639 sysp_getchar, /* unchar (*getchar)(); 7 */ 640 sysp_putchar, /* int (*putchar)(); 8 */ 641 sysp_ischar, /* int (*ischar)(); 9 */ 642 }; 643 644 #if defined(__xpv) 645 int using_kern_polledio; 646 #endif 647 648 void 649 kadb_uses_kernel() 650 { 651 /* 652 * This routine is now totally misnamed, since it does not in fact 653 * control kadb's I/O; it only controls the kernel's prom_* I/O. 654 */ 655 sysp = &kern_sysp; 656 #if defined(__xpv) 657 using_kern_polledio = 1; 658 #endif 659 } 660 661 /* 662 * the interface to the outside world 663 */ 664 665 /* 666 * poll_port -- wait for a register to achieve a 667 * specific state. Arguments are a mask of bits we care about, 668 * and two sub-masks. To return normally, all the bits in the 669 * first sub-mask must be ON, all the bits in the second sub- 670 * mask must be OFF. If about seconds pass without the register 671 * achieving the desired bit configuration, we return 1, else 672 * 0. 673 */ 674 int 675 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits) 676 { 677 int i; 678 ushort_t maskval; 679 680 for (i = 500000; i; i--) { 681 maskval = inb(port) & mask; 682 if (((maskval & onbits) == onbits) && 683 ((maskval & offbits) == 0)) 684 return (0); 685 drv_usecwait(10); 686 } 687 return (1); 688 } 689 690 /* 691 * set_idle_cpu is called from idle() when a CPU becomes idle. 692 */ 693 /*LINTED: static unused */ 694 static uint_t last_idle_cpu; 695 696 /*ARGSUSED*/ 697 void 698 set_idle_cpu(int cpun) 699 { 700 last_idle_cpu = cpun; 701 (*psm_set_idle_cpuf)(cpun); 702 } 703 704 /* 705 * unset_idle_cpu is called from idle() when a CPU is no longer idle. 706 */ 707 /*ARGSUSED*/ 708 void 709 unset_idle_cpu(int cpun) 710 { 711 (*psm_unset_idle_cpuf)(cpun); 712 } 713 714 /* 715 * This routine is almost correct now, but not quite. It still needs the 716 * equivalent concept of "hres_last_tick", just like on the sparc side. 717 * The idea is to take a snapshot of the hi-res timer while doing the 718 * hrestime_adj updates under hres_lock in locore, so that the small 719 * interval between interrupt assertion and interrupt processing is 720 * accounted for correctly. Once we have this, the code below should 721 * be modified to subtract off hres_last_tick rather than hrtime_base. 722 * 723 * I'd have done this myself, but I don't have source to all of the 724 * vendor-specific hi-res timer routines (grrr...). The generic hook I 725 * need is something like "gethrtime_unlocked()", which would be just like 726 * gethrtime() but would assume that you're already holding CLOCK_LOCK(). 727 * This is what the GET_HRTIME() macro is for on sparc (although it also 728 * serves the function of making time available without a function call 729 * so you don't take a register window overflow while traps are disabled). 730 */ 731 void 732 pc_gethrestime(timestruc_t *tp) 733 { 734 int lock_prev; 735 timestruc_t now; 736 int nslt; /* nsec since last tick */ 737 int adj; /* amount of adjustment to apply */ 738 739 loop: 740 lock_prev = hres_lock; 741 now = hrestime; 742 nslt = (int)(gethrtime() - hres_last_tick); 743 if (nslt < 0) { 744 /* 745 * nslt < 0 means a tick came between sampling 746 * gethrtime() and hres_last_tick; restart the loop 747 */ 748 749 goto loop; 750 } 751 now.tv_nsec += nslt; 752 if (hrestime_adj != 0) { 753 if (hrestime_adj > 0) { 754 adj = (nslt >> ADJ_SHIFT); 755 if (adj > hrestime_adj) 756 adj = (int)hrestime_adj; 757 } else { 758 adj = -(nslt >> ADJ_SHIFT); 759 if (adj < hrestime_adj) 760 adj = (int)hrestime_adj; 761 } 762 now.tv_nsec += adj; 763 } 764 while ((unsigned long)now.tv_nsec >= NANOSEC) { 765 766 /* 767 * We might have a large adjustment or have been in the 768 * debugger for a long time; take care of (at most) four 769 * of those missed seconds (tv_nsec is 32 bits, so 770 * anything >4s will be wrapping around). However, 771 * anything more than 2 seconds out of sync will trigger 772 * timedelta from clock() to go correct the time anyway, 773 * so do what we can, and let the big crowbar do the 774 * rest. A similar correction while loop exists inside 775 * hres_tick(); in all cases we'd like tv_nsec to 776 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing 777 * user processes, but if tv_sec's a little behind for a 778 * little while, that's OK; time still monotonically 779 * increases. 780 */ 781 782 now.tv_nsec -= NANOSEC; 783 now.tv_sec++; 784 } 785 if ((hres_lock & ~1) != lock_prev) 786 goto loop; 787 788 *tp = now; 789 } 790 791 void 792 gethrestime_lasttick(timespec_t *tp) 793 { 794 int s; 795 796 s = hr_clock_lock(); 797 *tp = hrestime; 798 hr_clock_unlock(s); 799 } 800 801 time_t 802 gethrestime_sec(void) 803 { 804 timestruc_t now; 805 806 gethrestime(&now); 807 return (now.tv_sec); 808 } 809 810 /* 811 * Initialize a kernel thread's stack 812 */ 813 814 caddr_t 815 thread_stk_init(caddr_t stk) 816 { 817 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0); 818 return (stk - SA(MINFRAME)); 819 } 820 821 /* 822 * Initialize lwp's kernel stack. 823 */ 824 825 #ifdef TRAPTRACE 826 /* 827 * There's a tricky interdependency here between use of sysenter and 828 * TRAPTRACE which needs recording to avoid future confusion (this is 829 * about the third time I've re-figured this out ..) 830 * 831 * Here's how debugging lcall works with TRAPTRACE. 832 * 833 * 1 We're in userland with a breakpoint on the lcall instruction. 834 * 2 We execute the instruction - the instruction pushes the userland 835 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel 836 * via the call gate. 837 * 3 The hardware raises a debug trap in kernel mode, the hardware 838 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt. 839 * 4 dbgtrap pushes the error code and trapno and calls cmntrap 840 * 5 cmntrap finishes building a trap frame 841 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk 842 * off the stack into the traptrace buffer. 843 * 844 * This means that the traptrace buffer contains the wrong values in 845 * %esp and %ss, but everything else in there is correct. 846 * 847 * Here's how debugging sysenter works with TRAPTRACE. 848 * 849 * a We're in userland with a breakpoint on the sysenter instruction. 850 * b We execute the instruction - the instruction pushes -nothing- 851 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged 852 * values to take us to sys_sysenter, at the top of the lwp's 853 * stack. 854 * c goto 3 855 * 856 * At this point, because we got into the kernel without the requisite 857 * five pushes on the stack, if we didn't make extra room, we'd 858 * end up with the TRACE_REGS macro fetching the saved %ss and %esp 859 * values from negative (unmapped) stack addresses -- which really bites. 860 * That's why we do the '-= 8' below. 861 * 862 * XXX Note that reading "up" lwp0's stack works because t0 is declared 863 * right next to t0stack in locore.s 864 */ 865 #endif 866 867 caddr_t 868 lwp_stk_init(klwp_t *lwp, caddr_t stk) 869 { 870 caddr_t oldstk; 871 struct pcb *pcb = &lwp->lwp_pcb; 872 873 oldstk = stk; 874 stk -= SA(sizeof (struct regs) + SA(MINFRAME)); 875 #ifdef TRAPTRACE 876 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */ 877 #endif 878 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul)); 879 bzero(stk, oldstk - stk); 880 lwp->lwp_regs = (void *)(stk + SA(MINFRAME)); 881 882 /* 883 * Arrange that the virtualized %fs and %gs GDT descriptors 884 * have a well-defined initial state (present, ring 3 885 * and of type data). 886 */ 887 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) 888 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 889 else 890 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc; 891 lwp_installctx(lwp); 892 return (stk); 893 } 894 895 /* 896 * Use this opportunity to free any dynamically allocated fp storage. 897 */ 898 void 899 lwp_stk_fini(klwp_t *lwp) 900 { 901 fp_lwp_cleanup(lwp); 902 } 903 904 void 905 lwp_fp_init(klwp_t *lwp) 906 { 907 fp_lwp_init(lwp); 908 } 909 910 /* 911 * If we're not the panic CPU, we wait in panic_idle for reboot. 912 */ 913 void 914 panic_idle(void) 915 { 916 splx(ipltospl(CLOCK_LEVEL)); 917 (void) setjmp(&curthread->t_pcb); 918 919 dumpsys_helper(); 920 921 #ifndef __xpv 922 for (;;) 923 i86_halt(); 924 #else 925 for (;;) 926 ; 927 #endif 928 } 929 930 /* 931 * Stop the other CPUs by cross-calling them and forcing them to enter 932 * the panic_idle() loop above. 933 */ 934 /*ARGSUSED*/ 935 void 936 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl) 937 { 938 processorid_t i; 939 cpuset_t xcset; 940 941 /* 942 * In the case of a Xen panic, the hypervisor has already stopped 943 * all of the CPUs. 944 */ 945 if (!IN_XPV_PANIC()) { 946 (void) splzs(); 947 948 CPUSET_ALL_BUT(xcset, cp->cpu_id); 949 xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle); 950 } 951 952 for (i = 0; i < NCPU; i++) { 953 if (i != cp->cpu_id && cpu[i] != NULL && 954 (cpu[i]->cpu_flags & CPU_EXISTS)) 955 cpu[i]->cpu_flags |= CPU_QUIESCED; 956 } 957 } 958 959 /* 960 * Platform callback following each entry to panicsys(). 961 */ 962 /*ARGSUSED*/ 963 void 964 panic_enter_hw(int spl) 965 { 966 /* Nothing to do here */ 967 } 968 969 /* 970 * Platform-specific code to execute after panicstr is set: we invoke 971 * the PSM entry point to indicate that a panic has occurred. 972 */ 973 /*ARGSUSED*/ 974 void 975 panic_quiesce_hw(panic_data_t *pdp) 976 { 977 psm_notifyf(PSM_PANIC_ENTER); 978 979 cmi_panic_callback(); 980 981 #ifdef TRAPTRACE 982 /* 983 * Turn off TRAPTRACE 984 */ 985 TRAPTRACE_FREEZE; 986 #endif /* TRAPTRACE */ 987 } 988 989 /* 990 * Platform callback prior to writing crash dump. 991 */ 992 /*ARGSUSED*/ 993 void 994 panic_dump_hw(int spl) 995 { 996 /* Nothing to do here */ 997 } 998 999 void * 1000 plat_traceback(void *fpreg) 1001 { 1002 #ifdef __xpv 1003 if (IN_XPV_PANIC()) 1004 return (xpv_traceback(fpreg)); 1005 #endif 1006 return (fpreg); 1007 } 1008 1009 /*ARGSUSED*/ 1010 void 1011 plat_tod_fault(enum tod_fault_type tod_bad) 1012 {} 1013 1014 /*ARGSUSED*/ 1015 int 1016 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class) 1017 { 1018 return (ENOTSUP); 1019 } 1020 1021 /* 1022 * The underlying console output routines are protected by raising IPL in case 1023 * we are still calling into the early boot services. Once we start calling 1024 * the kernel console emulator, it will disable interrupts completely during 1025 * character rendering (see sysp_putchar, for example). Refer to the comments 1026 * and code in common/os/console.c for more information on these callbacks. 1027 */ 1028 /*ARGSUSED*/ 1029 int 1030 console_enter(int busy) 1031 { 1032 return (splzs()); 1033 } 1034 1035 /*ARGSUSED*/ 1036 void 1037 console_exit(int busy, int spl) 1038 { 1039 splx(spl); 1040 } 1041 1042 /* 1043 * Allocate a region of virtual address space, unmapped. 1044 * Stubbed out except on sparc, at least for now. 1045 */ 1046 /*ARGSUSED*/ 1047 void * 1048 boot_virt_alloc(void *addr, size_t size) 1049 { 1050 return (addr); 1051 } 1052 1053 void 1054 tenmicrosec(void) 1055 { 1056 extern int gethrtime_hires; 1057 1058 if (gethrtime_hires) { 1059 hrtime_t start, end; 1060 start = end = gethrtime(); 1061 while ((end - start) < (10 * (NANOSEC / MICROSEC))) { 1062 SMT_PAUSE(); 1063 end = gethrtime(); 1064 } 1065 } else { 1066 #if defined(__xpv) 1067 hrtime_t newtime; 1068 1069 newtime = xpv_gethrtime() + 10000; /* now + 10 us */ 1070 while (xpv_gethrtime() < newtime) 1071 SMT_PAUSE(); 1072 #else /* __xpv */ 1073 panic("TSC was not calibrated!"); 1074 #endif /* __xpv */ 1075 } 1076 } 1077 1078 /* 1079 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES 1080 * long, and it fills in the array with the time spent on cpu in 1081 * each of the mstates, where time is returned in nsec. 1082 * 1083 * No guarantee is made that the returned values in times[] will 1084 * monotonically increase on sequential calls, although this will 1085 * be true in the long run. Any such guarantee must be handled by 1086 * the caller, if needed. This can happen if we fail to account 1087 * for elapsed time due to a generation counter conflict, yet we 1088 * did account for it on a prior call (see below). 1089 * 1090 * The complication is that the cpu in question may be updating 1091 * its microstate at the same time that we are reading it. 1092 * Because the microstate is only updated when the CPU's state 1093 * changes, the values in cpu_intracct[] can be indefinitely out 1094 * of date. To determine true current values, it is necessary to 1095 * compare the current time with cpu_mstate_start, and add the 1096 * difference to times[cpu_mstate]. 1097 * 1098 * This can be a problem if those values are changing out from 1099 * under us. Because the code path in new_cpu_mstate() is 1100 * performance critical, we have not added a lock to it. Instead, 1101 * we have added a generation counter. Before beginning 1102 * modifications, the counter is set to 0. After modifications, 1103 * it is set to the old value plus one. 1104 * 1105 * get_cpu_mstate() will not consider the values of cpu_mstate 1106 * and cpu_mstate_start to be usable unless the value of 1107 * cpu_mstate_gen is both non-zero and unchanged, both before and 1108 * after reading the mstate information. Note that we must 1109 * protect against out-of-order loads around accesses to the 1110 * generation counter. Also, this is a best effort approach in 1111 * that we do not retry should the counter be found to have 1112 * changed. 1113 * 1114 * cpu_intracct[] is used to identify time spent in each CPU 1115 * mstate while handling interrupts. Such time should be reported 1116 * against system time, and so is subtracted out from its 1117 * corresponding cpu_acct[] time and added to 1118 * cpu_acct[CMS_SYSTEM]. 1119 */ 1120 1121 void 1122 get_cpu_mstate(cpu_t *cpu, hrtime_t *times) 1123 { 1124 int i; 1125 hrtime_t now, start; 1126 uint16_t gen; 1127 uint16_t state; 1128 hrtime_t intracct[NCMSTATES]; 1129 1130 /* 1131 * Load all volatile state under the protection of membar. 1132 * cpu_acct[cpu_mstate] must be loaded to avoid double counting 1133 * of (now - cpu_mstate_start) by a change in CPU mstate that 1134 * arrives after we make our last check of cpu_mstate_gen. 1135 */ 1136 1137 now = gethrtime_unscaled(); 1138 gen = cpu->cpu_mstate_gen; 1139 1140 membar_consumer(); /* guarantee load ordering */ 1141 start = cpu->cpu_mstate_start; 1142 state = cpu->cpu_mstate; 1143 for (i = 0; i < NCMSTATES; i++) { 1144 intracct[i] = cpu->cpu_intracct[i]; 1145 times[i] = cpu->cpu_acct[i]; 1146 } 1147 membar_consumer(); /* guarantee load ordering */ 1148 1149 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start) 1150 times[state] += now - start; 1151 1152 for (i = 0; i < NCMSTATES; i++) { 1153 if (i == CMS_SYSTEM) 1154 continue; 1155 times[i] -= intracct[i]; 1156 if (times[i] < 0) { 1157 intracct[i] += times[i]; 1158 times[i] = 0; 1159 } 1160 times[CMS_SYSTEM] += intracct[i]; 1161 scalehrtime(×[i]); 1162 } 1163 scalehrtime(×[CMS_SYSTEM]); 1164 } 1165 1166 /* 1167 * This is a version of the rdmsr instruction that allows 1168 * an error code to be returned in the case of failure. 1169 */ 1170 int 1171 checked_rdmsr(uint_t msr, uint64_t *value) 1172 { 1173 if (!is_x86_feature(x86_featureset, X86FSET_MSR)) 1174 return (ENOTSUP); 1175 *value = rdmsr(msr); 1176 return (0); 1177 } 1178 1179 /* 1180 * This is a version of the wrmsr instruction that allows 1181 * an error code to be returned in the case of failure. 1182 */ 1183 int 1184 checked_wrmsr(uint_t msr, uint64_t value) 1185 { 1186 if (!is_x86_feature(x86_featureset, X86FSET_MSR)) 1187 return (ENOTSUP); 1188 wrmsr(msr, value); 1189 return (0); 1190 } 1191 1192 /* 1193 * The mem driver's usual method of using hat_devload() to establish a 1194 * temporary mapping will not work for foreign pages mapped into this 1195 * domain or for the special hypervisor-provided pages. For the foreign 1196 * pages, we often don't know which domain owns them, so we can't ask the 1197 * hypervisor to set up a new mapping. For the other pages, we don't have 1198 * a pfn, so we can't create a new PTE. For these special cases, we do a 1199 * direct uiomove() from the existing kernel virtual address. 1200 */ 1201 /*ARGSUSED*/ 1202 int 1203 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw) 1204 { 1205 #if defined(__xpv) 1206 void *va = (void *)(uintptr_t)uio->uio_loffset; 1207 off_t pageoff = uio->uio_loffset & PAGEOFFSET; 1208 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff), 1209 (size_t)uio->uio_iov->iov_len); 1210 1211 if ((rw == UIO_READ && 1212 (va == HYPERVISOR_shared_info || va == xen_info)) || 1213 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va)))) 1214 return (uiomove(va, nbytes, rw, uio)); 1215 #endif 1216 return (ENOTSUP); 1217 } 1218 1219 pgcnt_t 1220 num_phys_pages() 1221 { 1222 pgcnt_t npages = 0; 1223 struct memlist *mp; 1224 1225 #if defined(__xpv) 1226 if (DOMAIN_IS_INITDOMAIN(xen_info)) 1227 return (xpv_nr_phys_pages()); 1228 #endif /* __xpv */ 1229 1230 for (mp = phys_install; mp != NULL; mp = mp->ml_next) 1231 npages += mp->ml_size >> PAGESHIFT; 1232 1233 return (npages); 1234 } 1235 1236 /* cpu threshold for compressed dumps */ 1237 #ifdef _LP64 1238 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU; 1239 #else 1240 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU; 1241 #endif 1242 1243 int 1244 dump_plat_addr() 1245 { 1246 #ifdef __xpv 1247 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1248 mem_vtop_t mem_vtop; 1249 int cnt; 1250 1251 /* 1252 * On the hypervisor, we want to dump the page with shared_info on it. 1253 */ 1254 if (!IN_XPV_PANIC()) { 1255 mem_vtop.m_as = &kas; 1256 mem_vtop.m_va = HYPERVISOR_shared_info; 1257 mem_vtop.m_pfn = pfn; 1258 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1259 cnt = 1; 1260 } else { 1261 cnt = dump_xpv_addr(); 1262 } 1263 return (cnt); 1264 #else 1265 return (0); 1266 #endif 1267 } 1268 1269 void 1270 dump_plat_pfn() 1271 { 1272 #ifdef __xpv 1273 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1274 1275 if (!IN_XPV_PANIC()) 1276 dumpvp_write(&pfn, sizeof (pfn)); 1277 else 1278 dump_xpv_pfn(); 1279 #endif 1280 } 1281 1282 /*ARGSUSED*/ 1283 int 1284 dump_plat_data(void *dump_cbuf) 1285 { 1286 #ifdef __xpv 1287 uint32_t csize; 1288 int cnt; 1289 1290 if (!IN_XPV_PANIC()) { 1291 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf, 1292 PAGESIZE); 1293 dumpvp_write(&csize, sizeof (uint32_t)); 1294 dumpvp_write(dump_cbuf, csize); 1295 cnt = 1; 1296 } else { 1297 cnt = dump_xpv_data(dump_cbuf); 1298 } 1299 return (cnt); 1300 #else 1301 return (0); 1302 #endif 1303 } 1304 1305 /* 1306 * Calculates a linear address, given the CS selector and PC values 1307 * by looking up the %cs selector process's LDT or the CPU's GDT. 1308 * proc->p_ldtlock must be held across this call. 1309 */ 1310 int 1311 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1312 { 1313 user_desc_t *descrp; 1314 caddr_t baseaddr; 1315 uint16_t idx = SELTOIDX(rp->r_cs); 1316 1317 ASSERT(rp->r_cs <= 0xFFFF); 1318 ASSERT(MUTEX_HELD(&p->p_ldtlock)); 1319 1320 if (SELISLDT(rp->r_cs)) { 1321 /* 1322 * Currently 64 bit processes cannot have private LDTs. 1323 */ 1324 ASSERT(p->p_model != DATAMODEL_LP64); 1325 1326 if (p->p_ldt == NULL) 1327 return (-1); 1328 1329 descrp = &p->p_ldt[idx]; 1330 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1331 1332 /* 1333 * Calculate the linear address (wraparound is not only ok, 1334 * it's expected behavior). The cast to uint32_t is because 1335 * LDT selectors are only allowed in 32-bit processes. 1336 */ 1337 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1338 rp->r_pc); 1339 } else { 1340 #ifdef DEBUG 1341 descrp = &CPU->cpu_gdt[idx]; 1342 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1343 /* GDT-based descriptors' base addresses should always be 0 */ 1344 ASSERT(baseaddr == 0); 1345 #endif 1346 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1347 } 1348 1349 return (0); 1350 } 1351 1352 /* 1353 * The implementation of dtrace_linear_pc is similar to the that of 1354 * linear_pc, above, but here we acquire p_ldtlock before accessing 1355 * p_ldt. This implementation is used by the pid provider; we prefix 1356 * it with "dtrace_" to avoid inducing spurious tracing events. 1357 */ 1358 int 1359 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1360 { 1361 user_desc_t *descrp; 1362 caddr_t baseaddr; 1363 uint16_t idx = SELTOIDX(rp->r_cs); 1364 1365 ASSERT(rp->r_cs <= 0xFFFF); 1366 1367 if (SELISLDT(rp->r_cs)) { 1368 /* 1369 * Currently 64 bit processes cannot have private LDTs. 1370 */ 1371 ASSERT(p->p_model != DATAMODEL_LP64); 1372 1373 mutex_enter(&p->p_ldtlock); 1374 if (p->p_ldt == NULL) { 1375 mutex_exit(&p->p_ldtlock); 1376 return (-1); 1377 } 1378 descrp = &p->p_ldt[idx]; 1379 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1380 mutex_exit(&p->p_ldtlock); 1381 1382 /* 1383 * Calculate the linear address (wraparound is not only ok, 1384 * it's expected behavior). The cast to uint32_t is because 1385 * LDT selectors are only allowed in 32-bit processes. 1386 */ 1387 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1388 rp->r_pc); 1389 } else { 1390 #ifdef DEBUG 1391 descrp = &CPU->cpu_gdt[idx]; 1392 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1393 /* GDT-based descriptors' base addresses should always be 0 */ 1394 ASSERT(baseaddr == 0); 1395 #endif 1396 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1397 } 1398 1399 return (0); 1400 } 1401 1402 /* 1403 * We need to post a soft interrupt to reprogram the lbolt cyclic when 1404 * switching from event to cyclic driven lbolt. The following code adds 1405 * and posts the softint for x86. 1406 */ 1407 static ddi_softint_hdl_impl_t lbolt_softint_hdl = 1408 {0, 0, NULL, NULL, 0, NULL, NULL, NULL}; 1409 1410 void 1411 lbolt_softint_add(void) 1412 { 1413 (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL, 1414 (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL); 1415 } 1416 1417 void 1418 lbolt_softint_post(void) 1419 { 1420 (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending); 1421 } 1422 1423 boolean_t 1424 plat_dr_check_capability(uint64_t features) 1425 { 1426 return ((plat_dr_options & features) == features); 1427 } 1428 1429 boolean_t 1430 plat_dr_support_cpu(void) 1431 { 1432 return (plat_dr_options & PLAT_DR_FEATURE_CPU); 1433 } 1434 1435 boolean_t 1436 plat_dr_support_memory(void) 1437 { 1438 return (plat_dr_options & PLAT_DR_FEATURE_MEMORY); 1439 } 1440 1441 void 1442 plat_dr_enable_capability(uint64_t features) 1443 { 1444 atomic_or_64(&plat_dr_options, features); 1445 } 1446 1447 void 1448 plat_dr_disable_capability(uint64_t features) 1449 { 1450 atomic_and_64(&plat_dr_options, ~features); 1451 } 1452 1453 /* 1454 * If SMAP is supported, look through hi_calls and inline 1455 * calls to smap_enable() to clac and smap_disable() to stac. 1456 */ 1457 void 1458 hotinline_smap(hotinline_desc_t *hid) 1459 { 1460 if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE) 1461 return; 1462 1463 if (strcmp(hid->hid_symname, "smap_enable") == 0) { 1464 bcopy(clac_instr, (void *)hid->hid_instr_offset, 1465 sizeof (clac_instr)); 1466 } else if (strcmp(hid->hid_symname, "smap_disable") == 0) { 1467 bcopy(stac_instr, (void *)hid->hid_instr_offset, 1468 sizeof (stac_instr)); 1469 } 1470 } 1471 1472 /* 1473 * Loop through hi_calls and hand off the inlining to 1474 * the appropriate calls. 1475 */ 1476 void 1477 do_hotinlines(struct module *mp) 1478 { 1479 for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL; 1480 hid = hid->hid_next) { 1481 #if !defined(__xpv) 1482 hotinline_smap(hid); 1483 #endif /* __xpv */ 1484 } 1485 } 1486