1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/systm.h> 30 #include <sys/archsystm.h> 31 #include <sys/t_lock.h> 32 #include <sys/uadmin.h> 33 #include <sys/panic.h> 34 #include <sys/reboot.h> 35 #include <sys/autoconf.h> 36 #include <sys/machsystm.h> 37 #include <sys/promif.h> 38 #include <sys/membar.h> 39 #include <vm/hat_sfmmu.h> 40 #include <sys/cpu_module.h> 41 #include <sys/cpu_sgnblk_defs.h> 42 #include <sys/intreg.h> 43 #include <sys/consdev.h> 44 #include <sys/kdi_impl.h> 45 #include <sys/traptrace.h> 46 #include <sys/hypervisor_api.h> 47 #include <sys/vmsystm.h> 48 #include <sys/dtrace.h> 49 #include <sys/xc_impl.h> 50 #include <sys/callb.h> 51 #include <sys/mdesc.h> 52 #include <sys/mach_descrip.h> 53 #include <sys/wdt.h> 54 #include <sys/soft_state.h> 55 #include <sys/promimpl.h> 56 #include <sys/hsvc.h> 57 58 /* 59 * hvdump_buf_va is a pointer to the currently-configured hvdump_buf. 60 * A value of NULL indicates that this area is not configured. 61 * hvdump_buf_sz is tunable but will be clamped to HVDUMP_SIZE_MAX. 62 */ 63 64 caddr_t hvdump_buf_va; 65 uint64_t hvdump_buf_sz = HVDUMP_SIZE_DEFAULT; 66 static uint64_t hvdump_buf_pa; 67 68 u_longlong_t panic_tick; 69 70 extern u_longlong_t gettick(); 71 static void reboot_machine(char *); 72 static void update_hvdump_buffer(void); 73 74 /* 75 * For xt_sync synchronization. 76 */ 77 extern uint64_t xc_tick_limit; 78 extern uint64_t xc_tick_jump_limit; 79 80 /* 81 * We keep our own copies, used for cache flushing, because we can be called 82 * before cpu_fiximpl(). 83 */ 84 static int kdi_dcache_size; 85 static int kdi_dcache_linesize; 86 static int kdi_icache_size; 87 static int kdi_icache_linesize; 88 89 /* 90 * Assembly support for generic modules in sun4v/ml/mach_xc.s 91 */ 92 extern void init_mondo_nocheck(xcfunc_t *func, uint64_t arg1, uint64_t arg2); 93 extern void kdi_flush_idcache(int, int, int, int); 94 extern uint64_t get_cpuaddr(uint64_t, uint64_t); 95 96 /* 97 * Machine dependent code to reboot. 98 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer 99 * to a string to be used as the argument string when rebooting. 100 * 101 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely 102 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when 103 * we are in a normal shutdown sequence (interrupts are not blocked, the 104 * system is not panic'ing or being suspended). 105 */ 106 /*ARGSUSED*/ 107 void 108 mdboot(int cmd, int fcn, char *bootstr, boolean_t invoke_cb) 109 { 110 extern void pm_cfb_check_and_powerup(void); 111 112 /* 113 * XXX - rconsvp is set to NULL to ensure that output messages 114 * are sent to the underlying "hardware" device using the 115 * monitor's printf routine since we are in the process of 116 * either rebooting or halting the machine. 117 */ 118 rconsvp = NULL; 119 120 /* 121 * At a high interrupt level we can't: 122 * 1) bring up the console 123 * or 124 * 2) wait for pending interrupts prior to redistribution 125 * to the current CPU 126 * 127 * so we do them now. 128 */ 129 pm_cfb_check_and_powerup(); 130 131 /* make sure there are no more changes to the device tree */ 132 devtree_freeze(); 133 134 if (invoke_cb) 135 (void) callb_execute_class(CB_CL_MDBOOT, NULL); 136 137 /* 138 * Clear any unresolved UEs from memory. 139 */ 140 page_retire_mdboot(); 141 142 /* 143 * stop other cpus which also raise our priority. since there is only 144 * one active cpu after this, and our priority will be too high 145 * for us to be preempted, we're essentially single threaded 146 * from here on out. 147 */ 148 stop_other_cpus(); 149 150 /* 151 * try and reset leaf devices. reset_leaves() should only 152 * be called when there are no other threads that could be 153 * accessing devices 154 */ 155 reset_leaves(); 156 157 watchdog_clear(); 158 159 if (fcn == AD_HALT) { 160 mach_set_soft_state(SIS_TRANSITION, 161 &SOLARIS_SOFT_STATE_HALT_MSG); 162 halt((char *)NULL); 163 } else if (fcn == AD_POWEROFF) { 164 mach_set_soft_state(SIS_TRANSITION, 165 &SOLARIS_SOFT_STATE_POWER_MSG); 166 power_down(NULL); 167 } else { 168 if (bootstr == NULL) { 169 switch (fcn) { 170 171 case AD_BOOT: 172 bootstr = ""; 173 break; 174 175 case AD_IBOOT: 176 bootstr = "-a"; 177 break; 178 179 case AD_SBOOT: 180 bootstr = "-s"; 181 break; 182 183 case AD_SIBOOT: 184 bootstr = "-sa"; 185 break; 186 default: 187 cmn_err(CE_WARN, 188 "mdboot: invalid function %d", fcn); 189 bootstr = ""; 190 break; 191 } 192 } 193 mach_set_soft_state(SIS_TRANSITION, 194 &SOLARIS_SOFT_STATE_REBOOT_MSG); 195 reboot_machine(bootstr); 196 } 197 /* MAYBE REACHED */ 198 } 199 200 /* mdpreboot - may be called prior to mdboot while root fs still mounted */ 201 /*ARGSUSED*/ 202 void 203 mdpreboot(int cmd, int fcn, char *bootstr) 204 { 205 } 206 207 /* 208 * Halt the machine and then reboot with the device 209 * and arguments specified in bootstr. 210 */ 211 static void 212 reboot_machine(char *bootstr) 213 { 214 flush_windows(); 215 stop_other_cpus(); /* send stop signal to other CPUs */ 216 prom_printf("rebooting...\n"); 217 /* 218 * For platforms that use CPU signatures, we 219 * need to set the signature block to OS and 220 * the state to exiting for all the processors. 221 */ 222 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_REBOOT, -1); 223 prom_reboot(bootstr); 224 /*NOTREACHED*/ 225 } 226 227 /* 228 * We use the x-trap mechanism and idle_stop_xcall() to stop the other CPUs. 229 * Once in panic_idle() they raise spl, record their location, and spin. 230 */ 231 static void 232 panic_idle(void) 233 { 234 (void) spl7(); 235 236 debug_flush_windows(); 237 (void) setjmp(&curthread->t_pcb); 238 239 CPU->cpu_m.in_prom = 1; 240 membar_stld(); 241 242 for (;;); 243 } 244 245 /* 246 * Force the other CPUs to trap into panic_idle(), and then remove them 247 * from the cpu_ready_set so they will no longer receive cross-calls. 248 */ 249 /*ARGSUSED*/ 250 void 251 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl) 252 { 253 cpuset_t cps; 254 int i; 255 256 (void) splzs(); 257 CPUSET_ALL_BUT(cps, cp->cpu_id); 258 xt_some(cps, (xcfunc_t *)idle_stop_xcall, (uint64_t)&panic_idle, NULL); 259 260 for (i = 0; i < NCPU; i++) { 261 if (i != cp->cpu_id && CPU_XCALL_READY(i)) { 262 int ntries = 0x10000; 263 264 while (!cpu[i]->cpu_m.in_prom && ntries) { 265 DELAY(50); 266 ntries--; 267 } 268 269 if (!cpu[i]->cpu_m.in_prom) 270 printf("panic: failed to stop cpu%d\n", i); 271 272 cpu[i]->cpu_flags &= ~CPU_READY; 273 cpu[i]->cpu_flags |= CPU_QUIESCED; 274 CPUSET_DEL(cpu_ready_set, cpu[i]->cpu_id); 275 } 276 } 277 } 278 279 /* 280 * Platform callback following each entry to panicsys(). If we've panicked at 281 * level 14, we examine t_panic_trap to see if a fatal trap occurred. If so, 282 * we disable further %tick_cmpr interrupts. If not, an explicit call to panic 283 * was made and so we re-enqueue an interrupt request structure to allow 284 * further level 14 interrupts to be processed once we lower PIL. This allows 285 * us to handle panics from the deadman() CY_HIGH_LEVEL cyclic. 286 */ 287 void 288 panic_enter_hw(int spl) 289 { 290 if (!panic_tick) { 291 panic_tick = gettick(); 292 if (mach_htraptrace_enable) { 293 uint64_t prev_freeze; 294 295 /* there are no possible error codes for this hcall */ 296 (void) hv_ttrace_freeze((uint64_t)TRAP_TFREEZE_ALL, 297 &prev_freeze); 298 } 299 #ifdef TRAPTRACE 300 TRAPTRACE_FREEZE; 301 #endif 302 } 303 304 mach_set_soft_state(SIS_TRANSITION, &SOLARIS_SOFT_STATE_PANIC_MSG); 305 306 if (spl == ipltospl(PIL_14)) { 307 uint_t opstate = disable_vec_intr(); 308 309 if (curthread->t_panic_trap != NULL) { 310 tickcmpr_disable(); 311 intr_dequeue_req(PIL_14, cbe_level14_inum); 312 } else { 313 if (!tickcmpr_disabled()) 314 intr_enqueue_req(PIL_14, cbe_level14_inum); 315 /* 316 * Clear SOFTINT<14>, SOFTINT<0> (TICK_INT) 317 * and SOFTINT<16> (STICK_INT) to indicate 318 * that the current level 14 has been serviced. 319 */ 320 wr_clr_softint((1 << PIL_14) | 321 TICK_INT_MASK | STICK_INT_MASK); 322 } 323 324 enable_vec_intr(opstate); 325 } 326 } 327 328 /* 329 * Miscellaneous hardware-specific code to execute after panicstr is set 330 * by the panic code: we also print and record PTL1 panic information here. 331 */ 332 /*ARGSUSED*/ 333 void 334 panic_quiesce_hw(panic_data_t *pdp) 335 { 336 extern uint_t getpstate(void); 337 extern void setpstate(uint_t); 338 339 /* 340 * Turn off TRAPTRACE and save the current %tick value in panic_tick. 341 */ 342 if (!panic_tick) { 343 panic_tick = gettick(); 344 if (mach_htraptrace_enable) { 345 uint64_t prev_freeze; 346 347 /* there are no possible error codes for this hcall */ 348 (void) hv_ttrace_freeze((uint64_t)TRAP_TFREEZE_ALL, 349 &prev_freeze); 350 } 351 #ifdef TRAPTRACE 352 TRAPTRACE_FREEZE; 353 #endif 354 } 355 /* 356 * For Platforms that use CPU signatures, we 357 * need to set the signature block to OS, the state to 358 * exiting, and the substate to panic for all the processors. 359 */ 360 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_PANIC, -1); 361 362 update_hvdump_buffer(); 363 364 /* 365 * Disable further ECC errors from the bus nexus. 366 */ 367 (void) bus_func_invoke(BF_TYPE_ERRDIS); 368 369 /* 370 * Redirect all interrupts to the current CPU. 371 */ 372 intr_redist_all_cpus_shutdown(); 373 374 /* 375 * This call exists solely to support dumps to network 376 * devices after sync from OBP. 377 * 378 * If we came here via the sync callback, then on some 379 * platforms, interrupts may have arrived while we were 380 * stopped in OBP. OBP will arrange for those interrupts to 381 * be redelivered if you say "go", but not if you invoke a 382 * client callback like 'sync'. For some dump devices 383 * (network swap devices), we need interrupts to be 384 * delivered in order to dump, so we have to call the bus 385 * nexus driver to reset the interrupt state machines. 386 */ 387 (void) bus_func_invoke(BF_TYPE_RESINTR); 388 389 setpstate(getpstate() | PSTATE_IE); 390 } 391 392 /* 393 * Platforms that use CPU signatures need to set the signature block to OS and 394 * the state to exiting for all CPUs. PANIC_CONT indicates that we're about to 395 * write the crash dump, which tells the SSP/SMS to begin a timeout routine to 396 * reboot the machine if the dump never completes. 397 */ 398 /*ARGSUSED*/ 399 void 400 panic_dump_hw(int spl) 401 { 402 CPU_SIGNATURE(OS_SIG, SIGST_EXIT, SIGSUBST_DUMP, -1); 403 } 404 405 /* 406 * for ptl1_panic 407 */ 408 void 409 ptl1_init_cpu(struct cpu *cpu) 410 { 411 ptl1_state_t *pstate = &cpu->cpu_m.ptl1_state; 412 413 /*CONSTCOND*/ 414 if (sizeof (struct cpu) + PTL1_SSIZE > CPU_ALLOC_SIZE) { 415 panic("ptl1_init_cpu: not enough space left for ptl1_panic " 416 "stack, sizeof (struct cpu) = %lu", 417 (unsigned long)sizeof (struct cpu)); 418 } 419 420 pstate->ptl1_stktop = (uintptr_t)cpu + CPU_ALLOC_SIZE; 421 cpu_pa[cpu->cpu_id] = va_to_pa(cpu); 422 } 423 424 void 425 ptl1_panic_handler(ptl1_state_t *pstate) 426 { 427 static const char *ptl1_reasons[] = { 428 #ifdef PTL1_PANIC_DEBUG 429 "trap for debug purpose", /* PTL1_BAD_DEBUG */ 430 #else 431 "unknown trap", /* PTL1_BAD_DEBUG */ 432 #endif 433 "register window trap", /* PTL1_BAD_WTRAP */ 434 "kernel MMU miss", /* PTL1_BAD_KMISS */ 435 "kernel protection fault", /* PTL1_BAD_KPROT_FAULT */ 436 "ISM MMU miss", /* PTL1_BAD_ISM */ 437 "kernel MMU trap", /* PTL1_BAD_MMUTRAP */ 438 "kernel trap handler state", /* PTL1_BAD_TRAP */ 439 "floating point trap", /* PTL1_BAD_FPTRAP */ 440 #ifdef DEBUG 441 "pointer to intr_vec", /* PTL1_BAD_INTR_VEC */ 442 #else 443 "unknown trap", /* PTL1_BAD_INTR_VEC */ 444 #endif 445 #ifdef TRAPTRACE 446 "TRACE_PTR state", /* PTL1_BAD_TRACE_PTR */ 447 #else 448 "unknown trap", /* PTL1_BAD_TRACE_PTR */ 449 #endif 450 "stack overflow", /* PTL1_BAD_STACK */ 451 "DTrace flags", /* PTL1_BAD_DTRACE_FLAGS */ 452 "attempt to steal locked ctx", /* PTL1_BAD_CTX_STEAL */ 453 "CPU ECC error loop", /* PTL1_BAD_ECC */ 454 "unexpected error from hypervisor call", /* PTL1_BAD_HCALL */ 455 "unexpected global level(%gl)", /* PTL1_BAD_GL */ 456 "Watchdog Reset", /* PTL1_BAD_WATCHDOG */ 457 "unexpected RED mode trap", /* PTL1_BAD_RED */ 458 "return value EINVAL from hcall: "\ 459 "UNMAP_PERM_ADDR", /* PTL1_BAD_HCALL_UNMAP_PERM_EINVAL */ 460 "return value ENOMAP from hcall: "\ 461 "UNMAP_PERM_ADDR", /* PTL1_BAD_HCALL_UNMAP_PERM_ENOMAP */ 462 }; 463 464 uint_t reason = pstate->ptl1_regs.ptl1_gregs[0].ptl1_g1; 465 uint_t tl = pstate->ptl1_regs.ptl1_trap_regs[0].ptl1_tl; 466 struct trap_info ti = { 0 }; 467 468 /* 469 * Use trap_info for a place holder to call panic_savetrap() and 470 * panic_showtrap() to save and print out ptl1_panic information. 471 */ 472 if (curthread->t_panic_trap == NULL) 473 curthread->t_panic_trap = &ti; 474 475 if (reason < sizeof (ptl1_reasons) / sizeof (ptl1_reasons[0])) 476 panic("bad %s at TL %u", ptl1_reasons[reason], tl); 477 else 478 panic("ptl1_panic reason 0x%x at TL %u", reason, tl); 479 } 480 481 void 482 clear_watchdog_on_exit(void) 483 { 484 if (watchdog_enabled && watchdog_activated) { 485 prom_printf("Debugging requested; hardware watchdog " 486 "suspended.\n"); 487 (void) watchdog_suspend(); 488 } 489 } 490 491 /* 492 * Restore the watchdog timer when returning from a debugger 493 * after a panic or L1-A and resume watchdog pat. 494 */ 495 void 496 restore_watchdog_on_entry() 497 { 498 watchdog_resume(); 499 } 500 501 int 502 kdi_watchdog_disable(void) 503 { 504 watchdog_suspend(); 505 506 return (0); 507 } 508 509 void 510 kdi_watchdog_restore(void) 511 { 512 watchdog_resume(); 513 } 514 515 void 516 mach_dump_buffer_init(void) 517 { 518 uint64_t ret, minsize = 0; 519 520 if (hvdump_buf_sz > HVDUMP_SIZE_MAX) 521 hvdump_buf_sz = HVDUMP_SIZE_MAX; 522 523 hvdump_buf_va = contig_mem_alloc_align(hvdump_buf_sz, PAGESIZE); 524 if (hvdump_buf_va == NULL) 525 return; 526 527 hvdump_buf_pa = va_to_pa(hvdump_buf_va); 528 529 ret = hv_dump_buf_update(hvdump_buf_pa, hvdump_buf_sz, 530 &minsize); 531 532 if (ret != H_EOK) { 533 contig_mem_free(hvdump_buf_va, hvdump_buf_sz); 534 hvdump_buf_va = NULL; 535 cmn_err(CE_NOTE, "!Error in setting up hvstate" 536 "dump buffer. Error = 0x%lx, size = 0x%lx," 537 "buf_pa = 0x%lx", ret, hvdump_buf_sz, 538 hvdump_buf_pa); 539 540 if (ret == H_EINVAL) { 541 cmn_err(CE_NOTE, "!Buffer size too small." 542 "Available buffer size = 0x%lx," 543 "Minimum buffer size required = 0x%lx", 544 hvdump_buf_sz, minsize); 545 } 546 } 547 } 548 549 550 static void 551 update_hvdump_buffer(void) 552 { 553 uint64_t ret, dummy_val; 554 555 if (hvdump_buf_va == NULL) 556 return; 557 558 ret = hv_dump_buf_update(hvdump_buf_pa, hvdump_buf_sz, 559 &dummy_val); 560 if (ret != H_EOK) { 561 cmn_err(CE_NOTE, "!Cannot update hvstate dump" 562 "buffer. Error = 0x%lx", ret); 563 } 564 } 565 566 567 static int 568 getintprop(pnode_t node, char *name, int deflt) 569 { 570 int value; 571 572 switch (prom_getproplen(node, name)) { 573 case 0: 574 value = 1; /* boolean properties */ 575 break; 576 577 case sizeof (int): 578 (void) prom_getprop(node, name, (caddr_t)&value); 579 break; 580 581 default: 582 value = deflt; 583 break; 584 } 585 586 return (value); 587 } 588 589 /* 590 * Called by setcpudelay 591 */ 592 void 593 cpu_init_tick_freq(void) 594 { 595 md_t *mdp; 596 mde_cookie_t rootnode; 597 int listsz; 598 mde_cookie_t *listp = NULL; 599 int num_nodes; 600 uint64_t stick_prop; 601 602 if (broken_md_flag) { 603 sys_tick_freq = cpunodes[CPU->cpu_id].clock_freq; 604 return; 605 } 606 607 if ((mdp = md_get_handle()) == NULL) 608 panic("stick_frequency property not found in MD"); 609 610 rootnode = md_root_node(mdp); 611 ASSERT(rootnode != MDE_INVAL_ELEM_COOKIE); 612 613 num_nodes = md_node_count(mdp); 614 615 ASSERT(num_nodes > 0); 616 listsz = num_nodes * sizeof (mde_cookie_t); 617 listp = (mde_cookie_t *)prom_alloc((caddr_t)0, listsz, 0); 618 619 if (listp == NULL) 620 panic("cannot allocate list for MD properties"); 621 622 num_nodes = md_scan_dag(mdp, rootnode, md_find_name(mdp, "platform"), 623 md_find_name(mdp, "fwd"), listp); 624 625 ASSERT(num_nodes == 1); 626 627 if (md_get_prop_val(mdp, *listp, "stick-frequency", &stick_prop) != 0) 628 panic("stick_frequency property not found in MD"); 629 630 sys_tick_freq = stick_prop; 631 632 prom_free((caddr_t)listp, listsz); 633 (void) md_fini_handle(mdp); 634 } 635 636 int shipit(int n, uint64_t cpu_list_ra); 637 extern uint64_t xc_tick_limit; 638 extern uint64_t xc_tick_jump_limit; 639 640 #ifdef DEBUG 641 #define SEND_MONDO_STATS 1 642 #endif 643 644 #ifdef SEND_MONDO_STATS 645 uint32_t x_one_stimes[64]; 646 uint32_t x_one_ltimes[16]; 647 uint32_t x_set_stimes[64]; 648 uint32_t x_set_ltimes[16]; 649 uint32_t x_set_cpus[NCPU]; 650 #endif 651 652 void 653 send_one_mondo(int cpuid) 654 { 655 int retries, stat; 656 uint64_t starttick, endtick, tick, lasttick; 657 struct machcpu *mcpup = &(CPU->cpu_m); 658 659 CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 660 starttick = lasttick = gettick(); 661 mcpup->cpu_list[0] = (uint16_t)cpuid; 662 stat = shipit(1, mcpup->cpu_list_ra); 663 endtick = starttick + xc_tick_limit; 664 retries = 0; 665 while (stat != H_EOK) { 666 if (stat != H_EWOULDBLOCK) { 667 if (panic_quiesce) 668 return; 669 if (stat == H_ECPUERROR) 670 cmn_err(CE_PANIC, "send_one_mondo: " 671 "cpuid: 0x%x has been marked in " 672 "error", cpuid); 673 else 674 cmn_err(CE_PANIC, "send_one_mondo: " 675 "unexpected hypervisor error 0x%x " 676 "while sending a mondo to cpuid: " 677 "0x%x", stat, cpuid); 678 } 679 tick = gettick(); 680 /* 681 * If there is a big jump between the current tick 682 * count and lasttick, we have probably hit a break 683 * point. Adjust endtick accordingly to avoid panic. 684 */ 685 if (tick > (lasttick + xc_tick_jump_limit)) 686 endtick += (tick - lasttick); 687 lasttick = tick; 688 if (tick > endtick) { 689 if (panic_quiesce) 690 return; 691 cmn_err(CE_PANIC, "send mondo timeout " 692 "(target 0x%x) [retries: 0x%x hvstat: 0x%x]", 693 cpuid, retries, stat); 694 } 695 drv_usecwait(1); 696 stat = shipit(1, mcpup->cpu_list_ra); 697 retries++; 698 } 699 #ifdef SEND_MONDO_STATS 700 { 701 uint64_t n = gettick() - starttick; 702 if (n < 8192) 703 x_one_stimes[n >> 7]++; 704 else if (n < 15*8192) 705 x_one_ltimes[n >> 13]++; 706 else 707 x_one_ltimes[0xf]++; 708 } 709 #endif 710 } 711 712 void 713 send_mondo_set(cpuset_t set) 714 { 715 uint64_t starttick, endtick, tick, lasttick; 716 uint_t largestid, smallestid; 717 int i, j; 718 int ncpuids = 0; 719 int shipped = 0; 720 int retries = 0; 721 struct machcpu *mcpup = &(CPU->cpu_m); 722 723 ASSERT(!CPUSET_ISNULL(set)); 724 CPUSET_BOUNDS(set, smallestid, largestid); 725 if (smallestid == CPUSET_NOTINSET) { 726 return; 727 } 728 729 starttick = lasttick = gettick(); 730 endtick = starttick + xc_tick_limit; 731 732 /* 733 * Assemble CPU list for HV argument. We already know 734 * smallestid and largestid are members of set. 735 */ 736 mcpup->cpu_list[ncpuids++] = (uint16_t)smallestid; 737 if (largestid != smallestid) { 738 for (i = smallestid+1; i <= largestid-1; i++) { 739 if (CPU_IN_SET(set, i)) { 740 mcpup->cpu_list[ncpuids++] = (uint16_t)i; 741 } 742 } 743 mcpup->cpu_list[ncpuids++] = (uint16_t)largestid; 744 } 745 746 do { 747 int stat; 748 749 stat = shipit(ncpuids, mcpup->cpu_list_ra); 750 if (stat == H_EOK) { 751 shipped += ncpuids; 752 break; 753 } 754 755 /* 756 * Either not all CPU mondos were sent, or an 757 * error occurred. CPUs that were sent mondos 758 * have their CPU IDs overwritten in cpu_list. 759 * Reset cpu_list so that it only holds those 760 * CPU IDs that still need to be sent. 761 */ 762 for (i = 0, j = 0; i < ncpuids; i++) { 763 if (mcpup->cpu_list[i] == HV_SEND_MONDO_ENTRYDONE) { 764 shipped++; 765 } else { 766 mcpup->cpu_list[j++] = mcpup->cpu_list[i]; 767 } 768 } 769 ncpuids = j; 770 771 /* 772 * Now handle possible errors returned 773 * from hypervisor. 774 */ 775 if (stat == H_ECPUERROR) { 776 int errorcpus; 777 778 if (!panic_quiesce) 779 cmn_err(CE_CONT, "send_mondo_set: cpuid(s) "); 780 781 /* 782 * Remove any CPUs in the error state from 783 * cpu_list. At this point cpu_list only 784 * contains the CPU IDs for mondos not 785 * succesfully sent. 786 */ 787 for (i = 0, errorcpus = 0; i < ncpuids; i++) { 788 uint64_t state = CPU_STATE_INVALID; 789 uint16_t id = mcpup->cpu_list[i]; 790 791 (void) hv_cpu_state(id, &state); 792 if (state == CPU_STATE_ERROR) { 793 if (!panic_quiesce) 794 cmn_err(CE_CONT, "0x%x ", id); 795 errorcpus++; 796 } else if (errorcpus > 0) { 797 mcpup->cpu_list[i - errorcpus] = 798 mcpup->cpu_list[i]; 799 } 800 } 801 ncpuids -= errorcpus; 802 803 if (!panic_quiesce) { 804 if (errorcpus == 0) { 805 cmn_err(CE_CONT, "<none> have been " 806 "marked in error\n"); 807 cmn_err(CE_PANIC, "send_mondo_set: " 808 "hypervisor returned " 809 "H_ECPUERROR but no CPU in " 810 "cpu_list in error state"); 811 } else { 812 cmn_err(CE_CONT, "have been marked in " 813 "error\n"); 814 cmn_err(CE_PANIC, "send_mondo_set: " 815 "CPU(s) in error state"); 816 } 817 } 818 } else if (stat != H_EWOULDBLOCK) { 819 if (panic_quiesce) 820 return; 821 /* 822 * For all other errors, panic. 823 */ 824 cmn_err(CE_CONT, "send_mondo_set: unexpected " 825 "hypervisor error 0x%x while sending a " 826 "mondo to cpuid(s):", stat); 827 for (i = 0; i < ncpuids; i++) { 828 cmn_err(CE_CONT, " 0x%x", mcpup->cpu_list[i]); 829 } 830 cmn_err(CE_CONT, "\n"); 831 cmn_err(CE_PANIC, "send_mondo_set: unexpected " 832 "hypervisor error"); 833 } 834 835 tick = gettick(); 836 /* 837 * If there is a big jump between the current tick 838 * count and lasttick, we have probably hit a break 839 * point. Adjust endtick accordingly to avoid panic. 840 */ 841 if (tick > (lasttick + xc_tick_jump_limit)) 842 endtick += (tick - lasttick); 843 lasttick = tick; 844 if (tick > endtick) { 845 if (panic_quiesce) 846 return; 847 cmn_err(CE_CONT, "send mondo timeout " 848 "[retries: 0x%x] cpuids: ", retries); 849 for (i = 0; i < ncpuids; i++) 850 cmn_err(CE_CONT, " 0x%x", mcpup->cpu_list[i]); 851 cmn_err(CE_CONT, "\n"); 852 cmn_err(CE_PANIC, "send_mondo_set: timeout"); 853 } 854 855 while (gettick() < (tick + sys_clock_mhz)) 856 ; 857 retries++; 858 } while (ncpuids > 0); 859 860 CPU_STATS_ADDQ(CPU, sys, xcalls, shipped); 861 862 #ifdef SEND_MONDO_STATS 863 { 864 uint64_t n = gettick() - starttick; 865 if (n < 8192) 866 x_set_stimes[n >> 7]++; 867 else if (n < 15*8192) 868 x_set_ltimes[n >> 13]++; 869 else 870 x_set_ltimes[0xf]++; 871 } 872 x_set_cpus[shipped]++; 873 #endif 874 } 875 876 void 877 syncfpu(void) 878 { 879 } 880 881 void 882 cpu_flush_ecache(void) 883 { 884 } 885 886 void 887 sticksync_slave(void) 888 {} 889 890 void 891 sticksync_master(void) 892 {} 893 894 void 895 cpu_init_cache_scrub(void) 896 { 897 mach_set_soft_state(SIS_NORMAL, &SOLARIS_SOFT_STATE_RUN_MSG); 898 } 899 900 int 901 dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 902 { 903 int ret, watched; 904 905 watched = watch_disable_addr((void *)addr, 4, S_WRITE); 906 ret = dtrace_blksuword32(addr, data, 0); 907 if (watched) 908 watch_enable_addr((void *)addr, 4, S_WRITE); 909 910 return (ret); 911 } 912 913 int 914 dtrace_blksuword32(uintptr_t addr, uint32_t *data, int tryagain) 915 { 916 if (suword32((void *)addr, *data) == -1) 917 return (tryagain ? dtrace_blksuword32_err(addr, data) : -1); 918 dtrace_flush_sec(addr); 919 920 return (0); 921 } 922 923 /*ARGSUSED*/ 924 void 925 cpu_faulted_enter(struct cpu *cp) 926 { 927 } 928 929 /*ARGSUSED*/ 930 void 931 cpu_faulted_exit(struct cpu *cp) 932 { 933 } 934 935 static int 936 kdi_cpu_ready_iter(int (*cb)(int, void *), void *arg) 937 { 938 int rc, i; 939 940 for (rc = 0, i = 0; i < NCPU; i++) { 941 if (CPU_IN_SET(cpu_ready_set, i)) 942 rc += cb(i, arg); 943 } 944 945 return (rc); 946 } 947 948 /* 949 * Sends a cross-call to a specified processor. The caller assumes 950 * responsibility for repetition of cross-calls, as appropriate (MARSA for 951 * debugging). 952 */ 953 static int 954 kdi_xc_one(int cpuid, void (*func)(uintptr_t, uintptr_t), uintptr_t arg1, 955 uintptr_t arg2) 956 { 957 int stat; 958 struct machcpu *mcpup; 959 uint64_t cpuaddr_reg = 0, cpuaddr_scr = 0; 960 961 mcpup = &(((cpu_t *)get_cpuaddr(cpuaddr_reg, cpuaddr_scr))->cpu_m); 962 963 /* 964 * if (idsr_busy()) 965 * return (KDI_XC_RES_ERR); 966 */ 967 968 init_mondo_nocheck((xcfunc_t *)func, arg1, arg2); 969 970 mcpup->cpu_list[0] = (uint16_t)cpuid; 971 stat = shipit(1, mcpup->cpu_list_ra); 972 973 if (stat == 0) 974 return (KDI_XC_RES_OK); 975 else 976 return (KDI_XC_RES_NACK); 977 } 978 979 static void 980 kdi_tickwait(clock_t nticks) 981 { 982 clock_t endtick = gettick() + nticks; 983 984 while (gettick() < endtick); 985 } 986 987 static void 988 kdi_cpu_init(int dcache_size, int dcache_linesize, int icache_size, 989 int icache_linesize) 990 { 991 kdi_dcache_size = dcache_size; 992 kdi_dcache_linesize = dcache_linesize; 993 kdi_icache_size = icache_size; 994 kdi_icache_linesize = icache_linesize; 995 } 996 997 /* used directly by kdi_read/write_phys */ 998 void 999 kdi_flush_caches(void) 1000 { 1001 /* Not required on sun4v architecture. */ 1002 } 1003 1004 /*ARGSUSED*/ 1005 int 1006 kdi_get_stick(uint64_t *stickp) 1007 { 1008 return (-1); 1009 } 1010 1011 void 1012 cpu_kdi_init(kdi_t *kdi) 1013 { 1014 kdi->kdi_flush_caches = kdi_flush_caches; 1015 kdi->mkdi_cpu_init = kdi_cpu_init; 1016 kdi->mkdi_cpu_ready_iter = kdi_cpu_ready_iter; 1017 kdi->mkdi_xc_one = kdi_xc_one; 1018 kdi->mkdi_tickwait = kdi_tickwait; 1019 kdi->mkdi_get_stick = kdi_get_stick; 1020 } 1021 1022 uint64_t soft_state_message_ra[SOLARIS_SOFT_STATE_MSG_CNT]; 1023 static uint64_t soft_state_saved_state = (uint64_t)-1; 1024 static int soft_state_initialized = 0; 1025 static uint64_t soft_state_sup_minor; /* Supported minor number */ 1026 static hsvc_info_t soft_state_hsvc = { 1027 HSVC_REV_1, NULL, HSVC_GROUP_SOFT_STATE, 1, 0, NULL }; 1028 1029 1030 static void 1031 sun4v_system_claim(void) 1032 { 1033 watchdog_suspend(); 1034 /* 1035 * For "mdb -K", set soft state to debugging 1036 */ 1037 if (soft_state_saved_state == -1) { 1038 mach_get_soft_state(&soft_state_saved_state, 1039 &SOLARIS_SOFT_STATE_SAVED_MSG); 1040 } 1041 /* 1042 * check again as the read above may or may not have worked and if 1043 * it didn't then soft state will still be -1 1044 */ 1045 if (soft_state_saved_state != -1) { 1046 mach_set_soft_state(SIS_TRANSITION, 1047 &SOLARIS_SOFT_STATE_DEBUG_MSG); 1048 } 1049 } 1050 1051 static void 1052 sun4v_system_release(void) 1053 { 1054 watchdog_resume(); 1055 /* 1056 * For "mdb -K", set soft_state state back to original state on exit 1057 */ 1058 if (soft_state_saved_state != -1) { 1059 mach_set_soft_state(soft_state_saved_state, 1060 &SOLARIS_SOFT_STATE_SAVED_MSG); 1061 soft_state_saved_state = -1; 1062 } 1063 } 1064 1065 void 1066 plat_kdi_init(kdi_t *kdi) 1067 { 1068 kdi->pkdi_system_claim = sun4v_system_claim; 1069 kdi->pkdi_system_release = sun4v_system_release; 1070 } 1071 1072 /* 1073 * Routine to return memory information associated 1074 * with a physical address and syndrome. 1075 */ 1076 /* ARGSUSED */ 1077 int 1078 cpu_get_mem_info(uint64_t synd, uint64_t afar, 1079 uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1080 int *segsp, int *banksp, int *mcidp) 1081 { 1082 return (ENOTSUP); 1083 } 1084 1085 /* 1086 * This routine returns the size of the kernel's FRU name buffer. 1087 */ 1088 size_t 1089 cpu_get_name_bufsize() 1090 { 1091 return (UNUM_NAMLEN); 1092 } 1093 1094 /* 1095 * This routine is a more generic interface to cpu_get_mem_unum(), 1096 * that may be used by other modules (e.g. mm). 1097 */ 1098 /* ARGSUSED */ 1099 int 1100 cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 1101 char *buf, int buflen, int *lenp) 1102 { 1103 return (ENOTSUP); 1104 } 1105 1106 /* ARGSUSED */ 1107 int 1108 cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) 1109 { 1110 return (ENOTSUP); 1111 } 1112 1113 /* ARGSUSED */ 1114 int 1115 cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) 1116 { 1117 return (ENOTSUP); 1118 } 1119 1120 /* 1121 * xt_sync - wait for previous x-traps to finish 1122 */ 1123 void 1124 xt_sync(cpuset_t cpuset) 1125 { 1126 union { 1127 uint8_t volatile byte[NCPU]; 1128 uint64_t volatile xword[NCPU / 8]; 1129 } cpu_sync; 1130 uint64_t starttick, endtick, tick, lasttick; 1131 uint_t largestid, smallestid; 1132 int i, j; 1133 1134 kpreempt_disable(); 1135 CPUSET_DEL(cpuset, CPU->cpu_id); 1136 CPUSET_AND(cpuset, cpu_ready_set); 1137 1138 CPUSET_BOUNDS(cpuset, smallestid, largestid); 1139 if (smallestid == CPUSET_NOTINSET) 1140 goto out; 1141 1142 /* 1143 * Sun4v uses a queue for receiving mondos. Successful 1144 * transmission of a mondo only indicates that the mondo 1145 * has been written into the queue. 1146 * 1147 * We use an array of bytes to let each cpu to signal back 1148 * to the cross trap sender that the cross trap has been 1149 * executed. Set the byte to 1 before sending the cross trap 1150 * and wait until other cpus reset it to 0. 1151 */ 1152 bzero((void *)&cpu_sync, NCPU); 1153 cpu_sync.byte[smallestid] = 1; 1154 if (largestid != smallestid) { 1155 for (i = (smallestid + 1); i <= (largestid - 1); i++) 1156 if (CPU_IN_SET(cpuset, i)) 1157 cpu_sync.byte[i] = 1; 1158 cpu_sync.byte[largestid] = 1; 1159 } 1160 1161 xt_some(cpuset, (xcfunc_t *)xt_sync_tl1, 1162 (uint64_t)cpu_sync.byte, 0); 1163 1164 starttick = lasttick = gettick(); 1165 endtick = starttick + xc_tick_limit; 1166 1167 for (i = (smallestid / 8); i <= (largestid / 8); i++) { 1168 while (cpu_sync.xword[i] != 0) { 1169 tick = gettick(); 1170 /* 1171 * If there is a big jump between the current tick 1172 * count and lasttick, we have probably hit a break 1173 * point. Adjust endtick accordingly to avoid panic. 1174 */ 1175 if (tick > (lasttick + xc_tick_jump_limit)) { 1176 endtick += (tick - lasttick); 1177 } 1178 lasttick = tick; 1179 if (tick > endtick) { 1180 if (panic_quiesce) 1181 goto out; 1182 cmn_err(CE_CONT, "Cross trap sync timeout: " 1183 "cpuids:"); 1184 for (j = (i * 8); j <= largestid; j++) { 1185 if (cpu_sync.byte[j] != 0) 1186 cmn_err(CE_CONT, " 0x%x", j); 1187 } 1188 cmn_err(CE_CONT, "\n"); 1189 cmn_err(CE_PANIC, "xt_sync: timeout"); 1190 } 1191 } 1192 } 1193 1194 out: 1195 kpreempt_enable(); 1196 } 1197 1198 /* 1199 * Recalculate the values of the cross-call timeout variables based 1200 * on the value of the 'inter-cpu-latency' property of the platform node. 1201 * The property sets the number of nanosec to wait for a cross-call 1202 * to be acknowledged. Other timeout variables are derived from it. 1203 * 1204 * N.B. This implementation is aware of the internals of xc_init() 1205 * and updates many of the same variables. 1206 */ 1207 void 1208 recalc_xc_timeouts(void) 1209 { 1210 typedef union { 1211 uint64_t whole; 1212 struct { 1213 uint_t high; 1214 uint_t low; 1215 } half; 1216 } u_number; 1217 1218 /* See x_call.c for descriptions of these extern variables. */ 1219 extern uint64_t xc_tick_limit_scale; 1220 extern uint64_t xc_mondo_time_limit; 1221 extern uint64_t xc_func_time_limit; 1222 extern uint64_t xc_scale; 1223 extern uint64_t xc_mondo_multiplier; 1224 extern uint_t nsec_shift; 1225 1226 /* Temp versions of the target variables */ 1227 uint64_t tick_limit; 1228 uint64_t tick_jump_limit; 1229 uint64_t mondo_time_limit; 1230 uint64_t func_time_limit; 1231 uint64_t scale; 1232 1233 uint64_t latency; /* nanoseconds */ 1234 uint64_t maxfreq; 1235 uint64_t tick_limit_save = xc_tick_limit; 1236 uint_t tick_scale; 1237 uint64_t top; 1238 uint64_t bottom; 1239 u_number tk; 1240 1241 md_t *mdp; 1242 int nrnode; 1243 mde_cookie_t *platlist; 1244 1245 /* 1246 * Look up the 'inter-cpu-latency' (optional) property in the 1247 * platform node of the MD. The units are nanoseconds. 1248 */ 1249 if ((mdp = md_get_handle()) == NULL) { 1250 cmn_err(CE_WARN, "recalc_xc_timeouts: " 1251 "Unable to initialize machine description"); 1252 return; 1253 } 1254 1255 nrnode = md_alloc_scan_dag(mdp, 1256 md_root_node(mdp), "platform", "fwd", &platlist); 1257 1258 ASSERT(nrnode == 1); 1259 if (nrnode < 1) { 1260 cmn_err(CE_WARN, "recalc_xc_timeouts: platform node missing"); 1261 goto done; 1262 } 1263 1264 if (md_get_prop_val(mdp, platlist[0], 1265 "inter-cpu-latency", &latency) == -1) 1266 goto done; 1267 1268 /* 1269 * clock.h defines an assembly-language macro 1270 * (NATIVE_TIME_TO_NSEC_SCALE) to convert from %stick 1271 * units to nanoseconds. Since the inter-cpu-latency 1272 * units are nanoseconds and the xc_* variables require 1273 * %stick units, we need the inverse of that function. 1274 * The trick is to perform the calculation without 1275 * floating point, but also without integer truncation 1276 * or overflow. To understand the calculation below, 1277 * please read the discussion of the macro in clock.h. 1278 * Since this new code will be invoked infrequently, 1279 * we can afford to implement it in C. 1280 * 1281 * tick_scale is the reciprocal of nsec_scale which is 1282 * calculated at startup in setcpudelay(). The calc 1283 * of tick_limit parallels that of NATIVE_TIME_TO_NSEC_SCALE 1284 * except we use tick_scale instead of nsec_scale and 1285 * C instead of assembler. 1286 */ 1287 tick_scale = (uint_t)(((u_longlong_t)sys_tick_freq 1288 << (32 - nsec_shift)) / NANOSEC); 1289 1290 tk.whole = latency; 1291 top = ((uint64_t)tk.half.high << 4) * tick_scale; 1292 bottom = (((uint64_t)tk.half.low << 4) * (uint64_t)tick_scale) >> 32; 1293 tick_limit = top + bottom; 1294 1295 1296 /* 1297 * xc_init() calculated 'maxfreq' by looking at all the cpus, 1298 * and used it to derive some of the timeout variables that we 1299 * recalculate below. We can back into the original value by 1300 * using the inverse of one of those calculations. 1301 */ 1302 maxfreq = xc_mondo_time_limit / xc_scale; 1303 1304 /* 1305 * Don't allow the new timeout (xc_tick_limit) to fall below 1306 * the system tick frequency (stick). Allowing the timeout 1307 * to be set more tightly than this empirically determined 1308 * value may cause panics. 1309 */ 1310 tick_limit = tick_limit < sys_tick_freq ? sys_tick_freq : tick_limit; 1311 1312 tick_jump_limit = tick_limit / 32; 1313 tick_limit *= xc_tick_limit_scale; 1314 1315 /* 1316 * Recalculate xc_scale since it is used in a callback function 1317 * (xc_func_timeout_adj) to adjust two of the timeouts dynamically. 1318 * Make the change in xc_scale proportional to the change in 1319 * xc_tick_limit. 1320 */ 1321 scale = (xc_scale * tick_limit + sys_tick_freq / 2) / tick_limit_save; 1322 if (scale == 0) 1323 scale = 1; 1324 1325 mondo_time_limit = maxfreq * scale; 1326 func_time_limit = mondo_time_limit * xc_mondo_multiplier; 1327 1328 /* 1329 * Don't modify the timeouts if nothing has changed. Else, 1330 * stuff the variables with the freshly calculated (temp) 1331 * variables. This minimizes the window where the set of 1332 * values could be inconsistent. 1333 */ 1334 if (tick_limit != xc_tick_limit) { 1335 xc_tick_limit = tick_limit; 1336 xc_tick_jump_limit = tick_jump_limit; 1337 xc_scale = scale; 1338 xc_mondo_time_limit = mondo_time_limit; 1339 xc_func_time_limit = func_time_limit; 1340 /* 1341 * Force the new values to be used for future cross 1342 * calls. This is necessary only when we increase 1343 * the timeouts. 1344 */ 1345 if (tick_limit > tick_limit_save) { 1346 cpuset_t cpuset = cpu_ready_set; 1347 1348 xt_sync(cpuset); 1349 } 1350 } 1351 1352 done: 1353 if (nrnode > 0) 1354 md_free_scan_dag(mdp, &platlist); 1355 (void) md_fini_handle(mdp); 1356 } 1357 1358 void 1359 mach_soft_state_init(void) 1360 { 1361 int i; 1362 uint64_t ra; 1363 1364 /* 1365 * Try to register soft_state api. If it fails, soft_state api has not 1366 * been implemented in the firmware, so do not bother to setup 1367 * soft_state in the kernel. 1368 */ 1369 if ((i = hsvc_register(&soft_state_hsvc, &soft_state_sup_minor)) != 0) { 1370 return; 1371 } 1372 for (i = 0; i < SOLARIS_SOFT_STATE_MSG_CNT; i++) { 1373 ASSERT(strlen((const char *)(void *) 1374 soft_state_message_strings + i) < SSM_SIZE); 1375 if ((ra = va_to_pa((void *)(soft_state_message_strings + i))) == 1376 -1ll) { 1377 return; 1378 } 1379 soft_state_message_ra[i] = ra; 1380 } 1381 /* 1382 * Tell OBP that we are supporting Guest State 1383 */ 1384 prom_sun4v_soft_state_supported(); 1385 soft_state_initialized = 1; 1386 } 1387 1388 void 1389 mach_set_soft_state(uint64_t state, uint64_t *string_ra) 1390 { 1391 uint64_t rc; 1392 1393 if (soft_state_initialized && *string_ra) { 1394 rc = hv_soft_state_set(state, *string_ra); 1395 1396 if (rc != H_EOK) { 1397 cmn_err(CE_WARN, "hv_soft_state_set returned %ld\n", 1398 rc); 1399 } 1400 } 1401 } 1402 1403 void 1404 mach_get_soft_state(uint64_t *state, uint64_t *string_ra) 1405 { 1406 int rc; 1407 1408 if (soft_state_initialized && *string_ra) { 1409 rc = hv_soft_state_get(*string_ra, state); 1410 if (rc != H_EOK) { 1411 cmn_err(CE_WARN, "hv_soft_state_get returned %d\n", 1412 rc); 1413 *state = -1; 1414 } 1415 } 1416 } 1417