1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 27 * Copyright (c) 2016 by Delphix. All rights reserved. 28 * Copyright 2026 Oxide Computer Company 29 */ 30 31 /* 32 * When the operating system detects that it is in an invalid state, a panic 33 * is initiated in order to minimize potential damage to user data and to 34 * facilitate debugging. There are three major tasks to be performed in 35 * a system panic: recording information about the panic in memory (and thus 36 * making it part of the crash dump), synchronizing the file systems to 37 * preserve user file data, and generating the crash dump. We define the 38 * system to be in one of four states with respect to the panic code: 39 * 40 * CALM - the state of the system prior to any thread initiating a panic 41 * 42 * QUIESCE - the state of the system when the first thread to initiate 43 * a system panic records information about the cause of the panic 44 * and renders the system quiescent by stopping other processors 45 * 46 * SYNC - the state of the system when we synchronize the file systems 47 * DUMP - the state when we generate the crash dump. 48 * 49 * The transitions between these states are irreversible: once we begin 50 * panicking, we only make one attempt to perform the actions associated with 51 * each state. 52 * 53 * The panic code itself must be re-entrant because actions taken during any 54 * state may lead to another system panic. Additionally, any Solaris 55 * thread may initiate a panic at any time, and so we must have synchronization 56 * between threads which attempt to initiate a state transition simultaneously. 57 * The panic code makes use of a special locking primitive, a trigger, to 58 * perform this synchronization. A trigger is simply a word which is set 59 * atomically and can only be set once. We declare three triggers, one for 60 * each transition between the four states. When a thread enters the panic 61 * code it attempts to set each trigger; if it fails it moves on to the 62 * next trigger. A special case is the first trigger: if two threads race 63 * to perform the transition to QUIESCE, the losing thread may execute before 64 * the winner has a chance to stop its CPU. To solve this problem, we have 65 * the loser look ahead to see if any other triggers are set; if not, it 66 * presumes a panic is underway and simply spins. Unfortunately, since we 67 * are panicking, it is not possible to know this with absolute certainty. 68 * 69 * There are two common reasons for re-entering the panic code once a panic 70 * has been initiated: (1) after we debug_enter() at the end of QUIESCE, 71 * the operator may type "sync" instead of "go", and the PROM's sync callback 72 * routine will invoke panic(); (2) if the clock routine decides that sync 73 * or dump is not making progress, it will invoke panic() to force a timeout. 74 * The design assumes that a third possibility, another thread causing an 75 * unrelated panic while sync or dump is still underway, is extremely unlikely. 76 * If this situation occurs, we may end up triggering dump while sync is 77 * still in progress. This third case is considered extremely unlikely because 78 * all other CPUs are stopped and low-level interrupts have been blocked. 79 * 80 * The panic code is entered via a call directly to the vpanic() function, 81 * or its varargs wrappers panic() and cmn_err(9F). The vpanic routine 82 * is implemented in assembly language to record the current machine 83 * registers, attempt to set the trigger for the QUIESCE state, and 84 * if successful, switch stacks on to the panic_stack before calling into 85 * the common panicsys() routine. The first thread to initiate a panic 86 * is allowed to make use of the reserved panic_stack so that executing 87 * the panic code itself does not overwrite valuable data on that thread's 88 * stack *ahead* of the current stack pointer. This data will be preserved 89 * in the crash dump and may prove invaluable in determining what this 90 * thread has previously been doing. The first thread, saved in panic_thread, 91 * is also responsible for stopping the other CPUs as quickly as possible, 92 * and then setting the various panic_* variables. Most important among 93 * these is panicstr, which allows threads to subsequently bypass held 94 * locks so that we can proceed without ever blocking. We must stop the 95 * other CPUs *prior* to setting panicstr in case threads running there are 96 * currently spinning to acquire a lock; we want that state to be preserved. 97 * Every thread which initiates a panic has its T_PANIC flag set so we can 98 * identify all such threads in the crash dump. 99 * 100 * The panic_thread is also allowed to make use of the special memory buffer 101 * panicbuf, which on machines with appropriate hardware is preserved across 102 * reboots. We allow the panic_thread to store its register set and panic 103 * message in this buffer, so even if we fail to obtain a crash dump we will 104 * be able to examine the machine after reboot and determine some of the 105 * state at the time of the panic. If we do get a dump, the panic buffer 106 * data is structured so that a debugger can easily consume the information 107 * therein (see <sys/panic.h>). 108 * 109 * Each platform or architecture is required to implement the functions 110 * panic_savetrap() to record trap-specific information to panicbuf, 111 * panic_saveregs() to record a register set to panicbuf, panic_stopcpus() 112 * to halt all CPUs but the panicking CPU, panic_quiesce_hw() to perform 113 * miscellaneous platform-specific tasks *after* panicstr is set, 114 * panic_showtrap() to print trap-specific information to the console, 115 * and panic_dump_hw() to perform platform tasks prior to calling dumpsys(). 116 * 117 * A Note on Word Formation, courtesy of the Oxford Guide to English Usage: 118 * 119 * Words ending in -c interpose k before suffixes which otherwise would 120 * indicate a soft c, and thus the verb and adjective forms of 'panic' are 121 * spelled "panicked", "panicking", and "panicky" respectively. Use of 122 * the ill-conceived "panicing" and "panic'd" is discouraged. 123 */ 124 125 #include <sys/types.h> 126 #include <sys/varargs.h> 127 #include <sys/sysmacros.h> 128 #include <sys/cmn_err.h> 129 #include <sys/cpuvar.h> 130 #include <sys/thread.h> 131 #include <sys/t_lock.h> 132 #include <sys/cred.h> 133 #include <sys/systm.h> 134 #include <sys/archsystm.h> 135 #include <sys/uadmin.h> 136 #include <sys/callb.h> 137 #include <sys/vfs.h> 138 #include <sys/log.h> 139 #include <sys/disp.h> 140 #include <sys/param.h> 141 #include <sys/dumphdr.h> 142 #include <sys/ftrace.h> 143 #include <sys/reboot.h> 144 #include <sys/debug.h> 145 #include <sys/stack.h> 146 #include <sys/spl.h> 147 #include <sys/errorq.h> 148 #include <sys/panic.h> 149 #include <sys/fm/util.h> 150 #include <sys/clock_impl.h> 151 #include <sys/consdev.h> 152 #include <sys/sunddi.h> 153 154 /* 155 * Panic variables which are set once during the QUIESCE state by the 156 * first thread to initiate a panic. These are examined by post-mortem 157 * debugging tools; the inconsistent use of 'panic' versus 'panic_' in 158 * the variable naming is historical and allows legacy tools to work. 159 */ 160 #pragma align STACK_ALIGN(panic_stack) 161 char panic_stack[PANICSTKSIZE]; /* reserved stack for panic_thread */ 162 kthread_t *panic_thread; /* first thread to call panicsys() */ 163 cpu_t panic_cpu; /* cpu from first call to panicsys() */ 164 label_t panic_regs; /* setjmp label from panic_thread */ 165 label_t panic_pcb; /* t_pcb at time of panic */ 166 struct regs *panic_reg; /* regs struct from first panicsys() */ 167 char *volatile panicstr; /* format string to first panicsys() */ 168 va_list panicargs; /* arguments to first panicsys() */ 169 clock_t panic_lbolt; /* lbolt at time of panic */ 170 int64_t panic_lbolt64; /* lbolt64 at time of panic */ 171 hrtime_t panic_hrtime; /* hrtime at time of panic */ 172 timespec_t panic_hrestime; /* hrestime at time of panic */ 173 int panic_ipl; /* ipl on panic_cpu at time of panic */ 174 ushort_t panic_schedflag; /* t_schedflag for panic_thread */ 175 cpu_t *panic_bound_cpu; /* t_bound_cpu for panic_thread */ 176 char panic_preempt; /* t_preempt for panic_thread */ 177 dev_info_t *panic_dip; /* from dev_err(dip, CE_PANIC, ...) */ 178 179 /* 180 * Panic variables which can be set via /etc/system or patched while 181 * the system is in operation. Again, the stupid names are historic. 182 */ 183 char *panic_bootstr = NULL; /* mdboot string to use after panic */ 184 int panic_bootfcn = AD_BOOT; /* mdboot function to use after panic */ 185 int halt_on_panic = 0; /* halt after dump instead of reboot? */ 186 int nopanicdebug = 0; /* reboot instead of call debugger? */ 187 int in_sync = 0; /* skip vfs_syncall() and just dump? */ 188 189 /* 190 * The do_polled_io flag is set by the panic code to inform the SCSI subsystem 191 * to use polled mode instead of interrupt-driven i/o. 192 */ 193 int do_polled_io = 0; 194 195 /* 196 * The panic_forced flag is set by the uadmin A_DUMP code to inform the 197 * panic subsystem that it should not attempt an initial debug_enter. 198 */ 199 int panic_forced = 0; 200 201 /* 202 * Triggers for panic state transitions: 203 */ 204 int panic_quiesce; /* trigger for CALM -> QUIESCE */ 205 int panic_dump; /* trigger for QUIESCE -> DUMP */ 206 207 /* 208 * Variable signifying quiesce(9E) is in progress. 209 */ 210 volatile int quiesce_active = 0; 211 212 void 213 panicsys(const char *format, va_list alist, struct regs *rp, int on_panic_stack) 214 { 215 int s = spl8(); 216 kthread_t *t = curthread; 217 cpu_t *cp = CPU; 218 219 caddr_t intr_stack = NULL; 220 volatile uint_t intr_actv; 221 222 ushort_t schedflag = t->t_schedflag; 223 cpu_t *bound_cpu = t->t_bound_cpu; 224 char preempt = t->t_preempt; 225 label_t pcb = t->t_pcb; 226 227 (void) setjmp(&t->t_pcb); 228 t->t_flag |= T_PANIC; 229 230 t->t_schedflag |= TS_DONT_SWAP; 231 t->t_bound_cpu = cp; 232 t->t_preempt++; 233 234 panic_enter_hw(s); 235 236 /* 237 * If we're on the interrupt stack and an interrupt thread is available 238 * in this CPU's pool, preserve the interrupt stack by detaching an 239 * interrupt thread and making its stack the intr_stack. 240 */ 241 if (CPU_ON_INTR(cp) && cp->cpu_intr_thread != NULL) { 242 kthread_t *it = cp->cpu_intr_thread; 243 244 intr_stack = cp->cpu_intr_stack; 245 intr_actv = cp->cpu_intr_actv; 246 247 cp->cpu_intr_stack = thread_stk_init(it->t_stk); 248 cp->cpu_intr_thread = it->t_link; 249 250 /* 251 * Clear only the high level bits of cpu_intr_actv. 252 * We want to indicate that high-level interrupts are 253 * not active without destroying the low-level interrupt 254 * information stored there. 255 */ 256 cp->cpu_intr_actv &= ((1 << (LOCK_LEVEL + 1)) - 1); 257 } 258 259 /* 260 * Record one-time panic information and quiesce the other CPUs. 261 * Then print out the panic message and stack trace. 262 */ 263 if (on_panic_stack) { 264 panic_data_t *pdp = (panic_data_t *)panicbuf; 265 266 pdp->pd_version = PANICBUFVERS; 267 pdp->pd_msgoff = sizeof (panic_data_t) - sizeof (panic_nv_t); 268 269 (void) strncpy(pdp->pd_uuid, dump_get_uuid(), 270 sizeof (pdp->pd_uuid)); 271 272 if (t->t_panic_trap != NULL) 273 panic_savetrap(pdp, t->t_panic_trap); 274 else 275 panic_saveregs(pdp, rp); 276 277 (void) vsnprintf(&panicbuf[pdp->pd_msgoff], 278 PANICBUFSIZE - pdp->pd_msgoff, format, alist); 279 280 /* 281 * Call into the platform code to stop the other CPUs. 282 * We currently have all interrupts blocked, and expect that 283 * the platform code will lower ipl only as far as needed to 284 * perform cross-calls, and will acquire as *few* locks as is 285 * possible -- panicstr is not set so we can still deadlock. 286 */ 287 panic_stopcpus(cp, t, s); 288 289 panicstr = (char *)format; 290 va_copy(panicargs, alist); 291 panic_lbolt = LBOLT_NO_ACCOUNT; 292 panic_lbolt64 = LBOLT_NO_ACCOUNT64; 293 panic_hrestime = hrestime; 294 panic_hrtime = gethrtime_waitfree(); 295 panic_thread = t; 296 panic_regs = t->t_pcb; 297 panic_reg = rp; 298 panic_cpu = *cp; 299 panic_ipl = spltoipl(s); 300 panic_schedflag = schedflag; 301 panic_bound_cpu = bound_cpu; 302 panic_preempt = preempt; 303 panic_pcb = pcb; 304 305 if (intr_stack != NULL) { 306 panic_cpu.cpu_intr_stack = intr_stack; 307 panic_cpu.cpu_intr_actv = intr_actv; 308 } 309 310 /* 311 * From here on, the banner, the stack trace, and the dump all 312 * use polled console I/O, so give the console driver the 313 * chance to make itself ready. This is done only now that the 314 * other CPUs are stopped and panicstr is set, so the driver's 315 * hook can neither race the driver on another CPU nor block on 316 * a lock. There is no matching exit; we are not coming back. 317 */ 318 console_polled_enter(); 319 320 /* 321 * Lower ipl to 10 to keep clock() from running, but allow 322 * keyboard interrupts to enter the debugger. These callbacks 323 * are executed with panicstr set so they can bypass locks. 324 */ 325 splx(ipltospl(CLOCK_LEVEL)); 326 panic_quiesce_hw(pdp); 327 (void) FTRACE_STOP(); 328 (void) callb_execute_class(CB_CL_PANIC, 0); 329 330 if (log_intrq != NULL) 331 log_flushq(log_intrq); 332 333 /* 334 * If log_consq has been initialized and syslogd has started, 335 * print any messages in log_consq that haven't been consumed. 336 */ 337 if (log_consq != NULL && log_consq != log_backlogq) 338 log_printq(log_consq); 339 340 fm_banner(); 341 342 #if defined(__x86) 343 /* 344 * A hypervisor panic originates outside of Solaris, so we 345 * don't want to prepend the panic message with misleading 346 * pointers from within Solaris. 347 */ 348 if (!IN_XPV_PANIC()) 349 #endif 350 printf("\n\rpanic[cpu%d]/thread=%p: ", cp->cpu_id, 351 (void *)t); 352 if (panic_dip != NULL) { 353 printf("%s%d: ", ddi_driver_name(panic_dip), 354 ddi_get_instance(panic_dip)); 355 } 356 vprintf(format, alist); 357 printf("\n\n"); 358 359 if (t->t_panic_trap != NULL) { 360 panic_showtrap(t->t_panic_trap); 361 printf("\n"); 362 } 363 364 traceregs(rp); 365 printf("\n"); 366 367 if (((boothowto & RB_DEBUG) || obpdebug) && 368 !nopanicdebug && !panic_forced) { 369 if (dumpvp != NULL) { 370 debug_enter("panic: entering debugger " 371 "(continue to save dump)"); 372 } else { 373 debug_enter("panic: entering debugger " 374 "(no dump device, continue to reboot)"); 375 } 376 } 377 378 } else if (panic_dump != 0 || panicstr != NULL) { 379 printf("\n\rpanic[cpu%d]/thread=%p: ", cp->cpu_id, (void *)t); 380 vprintf(format, alist); 381 printf("\n"); 382 } else 383 goto spin; 384 385 /* 386 * Prior to performing dump, we make sure that do_polled_io is 387 * set, but we'll leave ipl at 10; deadman(), a CY_HIGH_LEVEL cyclic, 388 * will re-enter panic if we are not making progress with dump. 389 */ 390 /* 391 * Take the crash dump. If the dump trigger is already set, try to 392 * enter the debugger again before rebooting the system. 393 */ 394 if (panic_trigger(&panic_dump)) { 395 panic_dump_hw(s); 396 splx(ipltospl(CLOCK_LEVEL)); 397 errorq_panic(); 398 do_polled_io = 1; 399 dumpsys(); 400 } else if (((boothowto & RB_DEBUG) || obpdebug) && !nopanicdebug) { 401 debug_enter("panic: entering debugger (continue to reboot)"); 402 } else 403 printf("dump aborted: please record the above information!\n"); 404 405 if (halt_on_panic) 406 mdboot(A_REBOOT, AD_HALT, NULL, B_FALSE); 407 else 408 mdboot(A_REBOOT, panic_bootfcn, panic_bootstr, B_FALSE); 409 spin: 410 /* 411 * Restore ipl to at most CLOCK_LEVEL so we don't end up spinning 412 * and unable to jump into the debugger. 413 */ 414 splx(MIN(s, ipltospl(CLOCK_LEVEL))); 415 for (;;) 416 ; 417 } 418 419 void 420 panic(const char *format, ...) 421 { 422 va_list alist; 423 424 va_start(alist, format); 425 vpanic(format, alist); 426 va_end(alist); 427 } 428