1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2018 Joyent, Inc. 25 * Copyright 2018 Nexenta Systems, Inc. All rights reserved. 26 */ 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/vm.h> 32 #include <sys/proc.h> 33 #include <sys/file.h> 34 #include <sys/conf.h> 35 #include <sys/kmem.h> 36 #include <sys/mem.h> 37 #include <sys/mman.h> 38 #include <sys/vnode.h> 39 #include <sys/errno.h> 40 #include <sys/memlist.h> 41 #include <sys/dumphdr.h> 42 #include <sys/dumpadm.h> 43 #include <sys/ksyms.h> 44 #include <sys/compress.h> 45 #include <sys/stream.h> 46 #include <sys/strsun.h> 47 #include <sys/cmn_err.h> 48 #include <sys/bitmap.h> 49 #include <sys/modctl.h> 50 #include <sys/utsname.h> 51 #include <sys/systeminfo.h> 52 #include <sys/vmem.h> 53 #include <sys/log.h> 54 #include <sys/var.h> 55 #include <sys/debug.h> 56 #include <sys/sunddi.h> 57 #include <fs/fs_subr.h> 58 #include <sys/fs/snode.h> 59 #include <sys/ontrap.h> 60 #include <sys/panic.h> 61 #include <sys/dkio.h> 62 #include <sys/vtoc.h> 63 #include <sys/errorq.h> 64 #include <sys/fm/util.h> 65 #include <sys/fs/zfs.h> 66 67 #include <vm/hat.h> 68 #include <vm/as.h> 69 #include <vm/page.h> 70 #include <vm/pvn.h> 71 #include <vm/seg.h> 72 #include <vm/seg_kmem.h> 73 #include <sys/clock_impl.h> 74 #include <sys/hold_page.h> 75 #include <sys/cpu.h> 76 77 #include <bzip2/bzlib.h> 78 79 #define ONE_GIG (1024 * 1024 * 1024UL) 80 81 /* 82 * Crash dump time is dominated by disk write time. To reduce this, 83 * the stronger compression method bzip2 is applied to reduce the dump 84 * size and hence reduce I/O time. However, bzip2 is much more 85 * computationally expensive than the existing lzjb algorithm, so to 86 * avoid increasing compression time, CPUs that are otherwise idle 87 * during panic are employed to parallelize the compression task. 88 * Many helper CPUs are needed to prevent bzip2 from being a 89 * bottleneck, and on systems with too few CPUs, the lzjb algorithm is 90 * parallelized instead. Lastly, I/O and compression are performed by 91 * different CPUs, and are hence overlapped in time, unlike the older 92 * serial code. 93 * 94 * Another important consideration is the speed of the dump 95 * device. Faster disks need less CPUs in order to benefit from 96 * parallel lzjb versus parallel bzip2. Therefore, the CPU count 97 * threshold for switching from parallel lzjb to paralled bzip2 is 98 * elevated for faster disks. The dump device speed is adduced from 99 * the setting for dumpbuf.iosize, see dump_update_clevel. 100 */ 101 102 /* 103 * exported vars 104 */ 105 kmutex_t dump_lock; /* lock for dump configuration */ 106 dumphdr_t *dumphdr; /* dump header */ 107 int dump_conflags = DUMP_KERNEL; /* dump configuration flags */ 108 vnode_t *dumpvp; /* dump device vnode pointer */ 109 u_offset_t dumpvp_size; /* size of dump device, in bytes */ 110 char *dumppath; /* pathname of dump device */ 111 int dump_timeout = 120; /* timeout for dumping pages */ 112 int dump_timeleft; /* portion of dump_timeout remaining */ 113 int dump_ioerr; /* dump i/o error */ 114 int dump_check_used; /* enable check for used pages */ 115 char *dump_stack_scratch; /* scratch area for saving stack summary */ 116 117 /* 118 * Tunables for dump compression and parallelism. These can be set via 119 * /etc/system. 120 * 121 * dump_ncpu_low number of helpers for parallel lzjb 122 * This is also the minimum configuration. 123 * 124 * dump_bzip2_level bzip2 compression level: 1-9 125 * Higher numbers give greater compression, but take more memory 126 * and time. Memory used per helper is ~(dump_bzip2_level * 1MB). 127 * 128 * dump_plat_mincpu the cross-over limit for using bzip2 (per platform): 129 * if dump_plat_mincpu == 0, then always do single threaded dump 130 * if ncpu >= dump_plat_mincpu then try to use bzip2 131 * 132 * dump_metrics_on if set, metrics are collected in the kernel, passed 133 * to savecore via the dump file, and recorded by savecore in 134 * METRICS.txt. 135 */ 136 uint_t dump_ncpu_low = 4; /* minimum config for parallel lzjb */ 137 uint_t dump_bzip2_level = 1; /* bzip2 level (1-9) */ 138 139 /* Use dump_plat_mincpu_default unless this variable is set by /etc/system */ 140 #define MINCPU_NOT_SET ((uint_t)-1) 141 uint_t dump_plat_mincpu = MINCPU_NOT_SET; 142 143 /* tunables for pre-reserved heap */ 144 uint_t dump_kmem_permap = 1024; 145 uint_t dump_kmem_pages = 0; 146 147 /* Define multiple buffers per helper to avoid stalling */ 148 #define NCBUF_PER_HELPER 2 149 #define NCMAP_PER_HELPER 4 150 151 /* minimum number of helpers configured */ 152 #define MINHELPERS (dump_ncpu_low) 153 #define MINCBUFS (MINHELPERS * NCBUF_PER_HELPER) 154 155 /* 156 * Define constant parameters. 157 * 158 * CBUF_SIZE size of an output buffer 159 * 160 * CBUF_MAPSIZE size of virtual range for mapping pages 161 * 162 * CBUF_MAPNP size of virtual range in pages 163 * 164 */ 165 #define DUMP_1KB ((size_t)1 << 10) 166 #define DUMP_1MB ((size_t)1 << 20) 167 #define CBUF_SIZE ((size_t)1 << 17) 168 #define CBUF_MAPSHIFT (22) 169 #define CBUF_MAPSIZE ((size_t)1 << CBUF_MAPSHIFT) 170 #define CBUF_MAPNP ((size_t)1 << (CBUF_MAPSHIFT - PAGESHIFT)) 171 172 /* 173 * Compression metrics are accumulated nano-second subtotals. The 174 * results are normalized by the number of pages dumped. A report is 175 * generated when dumpsys() completes and is saved in the dump image 176 * after the trailing dump header. 177 * 178 * Metrics are always collected. Set the variable dump_metrics_on to 179 * cause metrics to be saved in the crash file, where savecore will 180 * save it in the file METRICS.txt. 181 */ 182 #define PERPAGES \ 183 PERPAGE(bitmap) PERPAGE(map) PERPAGE(unmap) \ 184 PERPAGE(copy) PERPAGE(compress) \ 185 PERPAGE(write) \ 186 PERPAGE(inwait) PERPAGE(outwait) 187 188 typedef struct perpage { 189 #define PERPAGE(x) hrtime_t x; 190 PERPAGES 191 #undef PERPAGE 192 } perpage_t; 193 194 /* 195 * This macro controls the code generation for collecting dump 196 * performance information. By default, the code is generated, but 197 * automatic saving of the information is disabled. If dump_metrics_on 198 * is set to 1, the timing information is passed to savecore via the 199 * crash file, where it is appended to the file dump-dir/METRICS.txt. 200 */ 201 #define COLLECT_METRICS 202 203 #ifdef COLLECT_METRICS 204 uint_t dump_metrics_on = 0; /* set to 1 to enable recording metrics */ 205 206 #define HRSTART(v, m) v##ts.m = gethrtime() 207 #define HRSTOP(v, m) v.m += gethrtime() - v##ts.m 208 #define HRBEGIN(v, m, s) v##ts.m = gethrtime(); v.size += s 209 #define HREND(v, m) v.m += gethrtime() - v##ts.m 210 #define HRNORM(v, m, n) v.m /= (n) 211 212 #else 213 #define HRSTART(v, m) 214 #define HRSTOP(v, m) 215 #define HRBEGIN(v, m, s) 216 #define HREND(v, m) 217 #define HRNORM(v, m, n) 218 #endif /* COLLECT_METRICS */ 219 220 /* 221 * Buffers for copying and compressing memory pages. 222 * 223 * cbuf_t buffer controllers: used for both input and output. 224 * 225 * The buffer state indicates how it is being used: 226 * 227 * CBUF_FREEMAP: CBUF_MAPSIZE virtual address range is available for 228 * mapping input pages. 229 * 230 * CBUF_INREADY: input pages are mapped and ready for compression by a 231 * helper. 232 * 233 * CBUF_USEDMAP: mapping has been consumed by a helper. Needs unmap. 234 * 235 * CBUF_FREEBUF: CBUF_SIZE output buffer, which is available. 236 * 237 * CBUF_WRITE: CBUF_SIZE block of compressed pages from a helper, 238 * ready to write out. 239 * 240 * CBUF_ERRMSG: CBUF_SIZE block of error messages from a helper 241 * (reports UE errors.) 242 */ 243 244 typedef enum cbufstate { 245 CBUF_FREEMAP, 246 CBUF_INREADY, 247 CBUF_USEDMAP, 248 CBUF_FREEBUF, 249 CBUF_WRITE, 250 CBUF_ERRMSG 251 } cbufstate_t; 252 253 typedef struct cbuf cbuf_t; 254 255 struct cbuf { 256 cbuf_t *next; /* next in list */ 257 cbufstate_t state; /* processing state */ 258 size_t used; /* amount used */ 259 size_t size; /* mem size */ 260 char *buf; /* kmem or vmem */ 261 pgcnt_t pagenum; /* index to pfn map */ 262 pgcnt_t bitnum; /* first set bitnum */ 263 pfn_t pfn; /* first pfn in mapped range */ 264 int off; /* byte offset to first pfn */ 265 }; 266 267 static char dump_osimage_uuid[36 + 1]; 268 269 #define isdigit(ch) ((ch) >= '0' && (ch) <= '9') 270 #define isxdigit(ch) (isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \ 271 ((ch) >= 'A' && (ch) <= 'F')) 272 273 /* 274 * cqueue_t queues: a uni-directional channel for communication 275 * from the master to helper tasks or vice-versa using put and 276 * get primitives. Both mappings and data buffers are passed via 277 * queues. Producers close a queue when done. The number of 278 * active producers is reference counted so the consumer can 279 * detect end of data. Concurrent access is mediated by atomic 280 * operations for panic dump, or mutex/cv for live dump. 281 * 282 * There a four queues, used as follows: 283 * 284 * Queue Dataflow NewState 285 * -------------------------------------------------- 286 * mainq master -> master FREEMAP 287 * master has initialized or unmapped an input buffer 288 * -------------------------------------------------- 289 * helperq master -> helper INREADY 290 * master has mapped input for use by helper 291 * -------------------------------------------------- 292 * mainq master <- helper USEDMAP 293 * helper is done with input 294 * -------------------------------------------------- 295 * freebufq master -> helper FREEBUF 296 * master has initialized or written an output buffer 297 * -------------------------------------------------- 298 * mainq master <- helper WRITE 299 * block of compressed pages from a helper 300 * -------------------------------------------------- 301 * mainq master <- helper ERRMSG 302 * error messages from a helper (memory error case) 303 * -------------------------------------------------- 304 * writerq master <- master WRITE 305 * non-blocking queue of blocks to write 306 * -------------------------------------------------- 307 */ 308 typedef struct cqueue { 309 cbuf_t *volatile first; /* first in list */ 310 cbuf_t *last; /* last in list */ 311 hrtime_t ts; /* timestamp */ 312 hrtime_t empty; /* total time empty */ 313 kmutex_t mutex; /* live state lock */ 314 kcondvar_t cv; /* live wait var */ 315 lock_t spinlock; /* panic mode spin lock */ 316 volatile uint_t open; /* producer ref count */ 317 } cqueue_t; 318 319 /* 320 * Convenience macros for using the cqueue functions 321 * Note that the caller must have defined "dumpsync_t *ds" 322 */ 323 #define CQ_IS_EMPTY(q) \ 324 (ds->q.first == NULL) 325 326 #define CQ_OPEN(q) \ 327 atomic_inc_uint(&ds->q.open) 328 329 #define CQ_CLOSE(q) \ 330 dumpsys_close_cq(&ds->q, ds->live) 331 332 #define CQ_PUT(q, cp, st) \ 333 dumpsys_put_cq(&ds->q, cp, st, ds->live) 334 335 #define CQ_GET(q) \ 336 dumpsys_get_cq(&ds->q, ds->live) 337 338 /* 339 * Dynamic state when dumpsys() is running. 340 */ 341 typedef struct dumpsync { 342 pgcnt_t npages; /* subtotal of pages dumped */ 343 pgcnt_t pages_mapped; /* subtotal of pages mapped */ 344 pgcnt_t pages_used; /* subtotal of pages used per map */ 345 size_t nwrite; /* subtotal of bytes written */ 346 uint_t live; /* running live dump */ 347 uint_t neednl; /* will need to print a newline */ 348 uint_t percent; /* dump progress */ 349 uint_t percent_done; /* dump progress reported */ 350 int sec_done; /* dump progress last report time */ 351 cqueue_t freebufq; /* free kmem bufs for writing */ 352 cqueue_t mainq; /* input for main task */ 353 cqueue_t helperq; /* input for helpers */ 354 cqueue_t writerq; /* input for writer */ 355 hrtime_t start; /* start time */ 356 hrtime_t elapsed; /* elapsed time when completed */ 357 hrtime_t iotime; /* time spent writing nwrite bytes */ 358 hrtime_t iowait; /* time spent waiting for output */ 359 hrtime_t iowaitts; /* iowait timestamp */ 360 perpage_t perpage; /* metrics */ 361 perpage_t perpagets; 362 int dumpcpu; /* master cpu */ 363 } dumpsync_t; 364 365 static dumpsync_t dumpsync; /* synchronization vars */ 366 367 /* 368 * helper_t helpers: contains the context for a stream. CPUs run in 369 * parallel at dump time; each CPU creates a single stream of 370 * compression data. Stream data is divided into CBUF_SIZE blocks. 371 * The blocks are written in order within a stream. But, blocks from 372 * multiple streams can be interleaved. Each stream is identified by a 373 * unique tag. 374 */ 375 typedef struct helper { 376 int helper; /* bound helper id */ 377 int tag; /* compression stream tag */ 378 perpage_t perpage; /* per page metrics */ 379 perpage_t perpagets; /* per page metrics (timestamps) */ 380 taskqid_t taskqid; /* live dump task ptr */ 381 int in, out; /* buffer offsets */ 382 cbuf_t *cpin, *cpout, *cperr; /* cbuf objects in process */ 383 dumpsync_t *ds; /* pointer to sync vars */ 384 size_t used; /* counts input consumed */ 385 char *page; /* buffer for page copy */ 386 char *lzbuf; /* lzjb output */ 387 bz_stream bzstream; /* bzip2 state */ 388 } helper_t; 389 390 #define MAINHELPER (-1) /* helper is also the main task */ 391 #define FREEHELPER (-2) /* unbound helper */ 392 #define DONEHELPER (-3) /* helper finished */ 393 394 /* 395 * configuration vars for dumpsys 396 */ 397 typedef struct dumpcfg { 398 int threshold; /* ncpu threshold for bzip2 */ 399 int nhelper; /* number of helpers */ 400 int nhelper_used; /* actual number of helpers used */ 401 int ncmap; /* number VA pages for compression */ 402 int ncbuf; /* number of bufs for compression */ 403 int ncbuf_used; /* number of bufs in use */ 404 uint_t clevel; /* dump compression level */ 405 helper_t *helper; /* array of helpers */ 406 cbuf_t *cmap; /* array of input (map) buffers */ 407 cbuf_t *cbuf; /* array of output buffers */ 408 ulong_t *helpermap; /* set of dumpsys helper CPU ids */ 409 ulong_t *bitmap; /* bitmap for marking pages to dump */ 410 ulong_t *rbitmap; /* bitmap for used CBUF_MAPSIZE ranges */ 411 pgcnt_t bitmapsize; /* size of bitmap */ 412 pgcnt_t rbitmapsize; /* size of bitmap for ranges */ 413 pgcnt_t found4m; /* number ranges allocated by dump */ 414 pgcnt_t foundsm; /* number small pages allocated by dump */ 415 pid_t *pids; /* list of process IDs at dump time */ 416 size_t maxsize; /* memory size needed at dump time */ 417 size_t maxvmsize; /* size of reserved VM */ 418 char *maxvm; /* reserved VM for spare pages */ 419 lock_t helper_lock; /* protect helper state */ 420 char helpers_wanted; /* flag to enable parallelism */ 421 } dumpcfg_t; 422 423 static dumpcfg_t dumpcfg; /* config vars */ 424 425 /* 426 * The dump I/O buffer. 427 * 428 * There is one I/O buffer used by dumpvp_write and dumvp_flush. It is 429 * sized according to the optimum device transfer speed. 430 */ 431 typedef struct dumpbuf { 432 vnode_t *cdev_vp; /* VCHR open of the dump device */ 433 len_t vp_limit; /* maximum write offset */ 434 offset_t vp_off; /* current dump device offset */ 435 char *cur; /* dump write pointer */ 436 char *start; /* dump buffer address */ 437 char *end; /* dump buffer end */ 438 size_t size; /* size of dumpbuf in bytes */ 439 size_t iosize; /* best transfer size for device */ 440 } dumpbuf_t; 441 442 dumpbuf_t dumpbuf; /* I/O buffer */ 443 444 /* 445 * For parallel dump, defines maximum time main task thread will wait 446 * for at least one helper to register in dumpcfg.helpermap, before 447 * assuming there are no helpers and falling back to serial mode. 448 * Value is chosen arbitrary and provides *really* long wait for any 449 * available helper to register. 450 */ 451 #define DUMP_HELPER_MAX_WAIT 1000 /* millisec */ 452 453 /* 454 * The dump I/O buffer must be at least one page, at most xfer_size 455 * bytes, and should scale with physmem in between. The transfer size 456 * passed in will either represent a global default (maxphys) or the 457 * best size for the device. The size of the dumpbuf I/O buffer is 458 * limited by dumpbuf_limit (8MB by default) because the dump 459 * performance saturates beyond a certain size. The default is to 460 * select 1/4096 of the memory. 461 */ 462 static int dumpbuf_fraction = 12; /* memory size scale factor */ 463 static size_t dumpbuf_limit = 8 * DUMP_1MB; /* max I/O buf size */ 464 465 static size_t 466 dumpbuf_iosize(size_t xfer_size) 467 { 468 size_t iosize = ptob(physmem >> dumpbuf_fraction); 469 470 if (iosize < PAGESIZE) 471 iosize = PAGESIZE; 472 else if (iosize > xfer_size) 473 iosize = xfer_size; 474 if (iosize > dumpbuf_limit) 475 iosize = dumpbuf_limit; 476 return (iosize & PAGEMASK); 477 } 478 479 /* 480 * resize the I/O buffer 481 */ 482 static void 483 dumpbuf_resize(void) 484 { 485 char *old_buf = dumpbuf.start; 486 size_t old_size = dumpbuf.size; 487 char *new_buf; 488 size_t new_size; 489 490 ASSERT(MUTEX_HELD(&dump_lock)); 491 492 new_size = dumpbuf_iosize(MAX(dumpbuf.iosize, maxphys)); 493 if (new_size <= old_size) 494 return; /* no need to reallocate buffer */ 495 496 new_buf = kmem_alloc(new_size, KM_SLEEP); 497 dumpbuf.size = new_size; 498 dumpbuf.start = new_buf; 499 dumpbuf.end = new_buf + new_size; 500 kmem_free(old_buf, old_size); 501 } 502 503 /* 504 * dump_update_clevel is called when dumpadm configures the dump device. 505 * Calculate number of helpers and buffers. 506 * Allocate the minimum configuration for now. 507 * 508 * When the dump file is configured we reserve a minimum amount of 509 * memory for use at crash time. But we reserve VA for all the memory 510 * we really want in order to do the fastest dump possible. The VA is 511 * backed by pages not being dumped, according to the bitmap. If 512 * there is insufficient spare memory, however, we fall back to the 513 * minimum. 514 * 515 * Live dump (savecore -L) always uses the minimum config. 516 * 517 * clevel 0 is single threaded lzjb 518 * clevel 1 is parallel lzjb 519 * clevel 2 is parallel bzip2 520 * 521 * The ncpu threshold is selected with dump_plat_mincpu. 522 * On OPL, set_platform_defaults() overrides the sun4u setting. 523 * The actual values are defined via DUMP_PLAT_*_MINCPU macros. 524 * 525 * Architecture Threshold Algorithm 526 * sun4u < 51 parallel lzjb 527 * sun4u >= 51 parallel bzip2(*) 528 * sun4u OPL < 8 parallel lzjb 529 * sun4u OPL >= 8 parallel bzip2(*) 530 * sun4v < 128 parallel lzjb 531 * sun4v >= 128 parallel bzip2(*) 532 * x86 < 11 parallel lzjb 533 * x86 >= 11 parallel bzip2(*) 534 * 32-bit N/A single-threaded lzjb 535 * 536 * (*) bzip2 is only chosen if there is sufficient available 537 * memory for buffers at dump time. See dumpsys_get_maxmem(). 538 * 539 * Faster dump devices have larger I/O buffers. The threshold value is 540 * increased according to the size of the dump I/O buffer, because 541 * parallel lzjb performs better with faster disks. For buffers >= 1MB 542 * the threshold is 3X; for buffers >= 256K threshold is 2X. 543 * 544 * For parallel dumps, the number of helpers is ncpu-1. The CPU 545 * running panic runs the main task. For single-threaded dumps, the 546 * panic CPU does lzjb compression (it is tagged as MAINHELPER.) 547 * 548 * Need multiple buffers per helper so that they do not block waiting 549 * for the main task. 550 * parallel single-threaded 551 * Number of output buffers: nhelper*2 1 552 * Number of mapping buffers: nhelper*4 1 553 * 554 */ 555 static void 556 dump_update_clevel() 557 { 558 int tag; 559 size_t bz2size; 560 helper_t *hp, *hpend; 561 cbuf_t *cp, *cpend; 562 dumpcfg_t *old = &dumpcfg; 563 dumpcfg_t newcfg = *old; 564 dumpcfg_t *new = &newcfg; 565 566 ASSERT(MUTEX_HELD(&dump_lock)); 567 568 /* 569 * Free the previously allocated bufs and VM. 570 */ 571 if (old->helper != NULL) { 572 573 /* helpers */ 574 hpend = &old->helper[old->nhelper]; 575 for (hp = old->helper; hp != hpend; hp++) { 576 if (hp->lzbuf != NULL) 577 kmem_free(hp->lzbuf, PAGESIZE); 578 if (hp->page != NULL) 579 kmem_free(hp->page, PAGESIZE); 580 } 581 kmem_free(old->helper, old->nhelper * sizeof (helper_t)); 582 583 /* VM space for mapping pages */ 584 cpend = &old->cmap[old->ncmap]; 585 for (cp = old->cmap; cp != cpend; cp++) 586 vmem_xfree(heap_arena, cp->buf, CBUF_MAPSIZE); 587 kmem_free(old->cmap, old->ncmap * sizeof (cbuf_t)); 588 589 /* output bufs */ 590 cpend = &old->cbuf[old->ncbuf]; 591 for (cp = old->cbuf; cp != cpend; cp++) 592 if (cp->buf != NULL) 593 kmem_free(cp->buf, cp->size); 594 kmem_free(old->cbuf, old->ncbuf * sizeof (cbuf_t)); 595 596 /* reserved VM for dumpsys_get_maxmem */ 597 if (old->maxvmsize > 0) 598 vmem_xfree(heap_arena, old->maxvm, old->maxvmsize); 599 } 600 601 /* 602 * Allocate memory and VM. 603 * One CPU runs dumpsys, the rest are helpers. 604 */ 605 new->nhelper = ncpus - 1; 606 if (new->nhelper < 1) 607 new->nhelper = 1; 608 609 if (new->nhelper > DUMP_MAX_NHELPER) 610 new->nhelper = DUMP_MAX_NHELPER; 611 612 /* use platform default, unless /etc/system overrides */ 613 if (dump_plat_mincpu == MINCPU_NOT_SET) 614 dump_plat_mincpu = dump_plat_mincpu_default; 615 616 /* increase threshold for faster disks */ 617 new->threshold = dump_plat_mincpu; 618 if (dumpbuf.iosize >= DUMP_1MB) 619 new->threshold *= 3; 620 else if (dumpbuf.iosize >= (256 * DUMP_1KB)) 621 new->threshold *= 2; 622 623 /* figure compression level based upon the computed threshold. */ 624 if (dump_plat_mincpu == 0 || new->nhelper < 2) { 625 new->clevel = 0; 626 new->nhelper = 1; 627 } else if ((new->nhelper + 1) >= new->threshold) { 628 new->clevel = DUMP_CLEVEL_BZIP2; 629 } else { 630 new->clevel = DUMP_CLEVEL_LZJB; 631 } 632 633 if (new->clevel == 0) { 634 new->ncbuf = 1; 635 new->ncmap = 1; 636 } else { 637 new->ncbuf = NCBUF_PER_HELPER * new->nhelper; 638 new->ncmap = NCMAP_PER_HELPER * new->nhelper; 639 } 640 641 /* 642 * Allocate new data structures and buffers for MINHELPERS, 643 * and also figure the max desired size. 644 */ 645 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 646 new->maxsize = 0; 647 new->maxvmsize = 0; 648 new->maxvm = NULL; 649 tag = 1; 650 new->helper = kmem_zalloc(new->nhelper * sizeof (helper_t), KM_SLEEP); 651 hpend = &new->helper[new->nhelper]; 652 for (hp = new->helper; hp != hpend; hp++) { 653 hp->tag = tag++; 654 if (hp < &new->helper[MINHELPERS]) { 655 hp->lzbuf = kmem_alloc(PAGESIZE, KM_SLEEP); 656 hp->page = kmem_alloc(PAGESIZE, KM_SLEEP); 657 } else if (new->clevel < DUMP_CLEVEL_BZIP2) { 658 new->maxsize += 2 * PAGESIZE; 659 } else { 660 new->maxsize += PAGESIZE; 661 } 662 if (new->clevel >= DUMP_CLEVEL_BZIP2) 663 new->maxsize += bz2size; 664 } 665 666 new->cbuf = kmem_zalloc(new->ncbuf * sizeof (cbuf_t), KM_SLEEP); 667 cpend = &new->cbuf[new->ncbuf]; 668 for (cp = new->cbuf; cp != cpend; cp++) { 669 cp->state = CBUF_FREEBUF; 670 cp->size = CBUF_SIZE; 671 if (cp < &new->cbuf[MINCBUFS]) 672 cp->buf = kmem_alloc(cp->size, KM_SLEEP); 673 else 674 new->maxsize += cp->size; 675 } 676 677 new->cmap = kmem_zalloc(new->ncmap * sizeof (cbuf_t), KM_SLEEP); 678 cpend = &new->cmap[new->ncmap]; 679 for (cp = new->cmap; cp != cpend; cp++) { 680 cp->state = CBUF_FREEMAP; 681 cp->size = CBUF_MAPSIZE; 682 cp->buf = vmem_xalloc(heap_arena, CBUF_MAPSIZE, CBUF_MAPSIZE, 683 0, 0, NULL, NULL, VM_SLEEP); 684 } 685 686 /* reserve VA to be backed with spare pages at crash time */ 687 if (new->maxsize > 0) { 688 new->maxsize = P2ROUNDUP(new->maxsize, PAGESIZE); 689 new->maxvmsize = P2ROUNDUP(new->maxsize, CBUF_MAPSIZE); 690 new->maxvm = vmem_xalloc(heap_arena, new->maxvmsize, 691 CBUF_MAPSIZE, 0, 0, NULL, NULL, VM_SLEEP); 692 } 693 694 /* 695 * Reserve memory for kmem allocation calls made during crash dump. The 696 * hat layer allocates memory for each mapping created, and the I/O path 697 * allocates buffers and data structs. 698 * 699 * On larger systems, we easily exceed the lower amount, so we need some 700 * more space; the cut-over point is relatively arbitrary. If we run 701 * out, the only impact is that kmem state in the dump becomes 702 * inconsistent. 703 */ 704 705 if (dump_kmem_pages == 0) { 706 if (physmem > (16 * ONE_GIG) / PAGESIZE) 707 dump_kmem_pages = 20; 708 else 709 dump_kmem_pages = 8; 710 } 711 712 kmem_dump_init((new->ncmap * dump_kmem_permap) + 713 (dump_kmem_pages * PAGESIZE)); 714 715 /* set new config pointers */ 716 *old = *new; 717 } 718 719 /* 720 * Define a struct memlist walker to optimize bitnum to pfn 721 * lookup. The walker maintains the state of the list traversal. 722 */ 723 typedef struct dumpmlw { 724 struct memlist *mp; /* current memlist */ 725 pgcnt_t basenum; /* bitnum base offset */ 726 pgcnt_t mppages; /* current memlist size */ 727 pgcnt_t mpleft; /* size to end of current memlist */ 728 pfn_t mpaddr; /* first pfn in memlist */ 729 } dumpmlw_t; 730 731 /* initialize the walker */ 732 static inline void 733 dump_init_memlist_walker(dumpmlw_t *pw) 734 { 735 pw->mp = phys_install; 736 pw->basenum = 0; 737 pw->mppages = pw->mp->ml_size >> PAGESHIFT; 738 pw->mpleft = pw->mppages; 739 pw->mpaddr = pw->mp->ml_address >> PAGESHIFT; 740 } 741 742 /* 743 * Lookup pfn given bitnum. The memlist can be quite long on some 744 * systems (e.g.: one per board). To optimize sequential lookups, the 745 * caller initializes and presents a memlist walker. 746 */ 747 static pfn_t 748 dump_bitnum_to_pfn(pgcnt_t bitnum, dumpmlw_t *pw) 749 { 750 bitnum -= pw->basenum; 751 while (pw->mp != NULL) { 752 if (bitnum < pw->mppages) { 753 pw->mpleft = pw->mppages - bitnum; 754 return (pw->mpaddr + bitnum); 755 } 756 bitnum -= pw->mppages; 757 pw->basenum += pw->mppages; 758 pw->mp = pw->mp->ml_next; 759 if (pw->mp != NULL) { 760 pw->mppages = pw->mp->ml_size >> PAGESHIFT; 761 pw->mpleft = pw->mppages; 762 pw->mpaddr = pw->mp->ml_address >> PAGESHIFT; 763 } 764 } 765 return (PFN_INVALID); 766 } 767 768 static pgcnt_t 769 dump_pfn_to_bitnum(pfn_t pfn) 770 { 771 struct memlist *mp; 772 pgcnt_t bitnum = 0; 773 774 for (mp = phys_install; mp != NULL; mp = mp->ml_next) { 775 if (pfn >= (mp->ml_address >> PAGESHIFT) && 776 pfn < ((mp->ml_address + mp->ml_size) >> PAGESHIFT)) 777 return (bitnum + pfn - (mp->ml_address >> PAGESHIFT)); 778 bitnum += mp->ml_size >> PAGESHIFT; 779 } 780 return ((pgcnt_t)-1); 781 } 782 783 /* 784 * Set/test bitmap for a CBUF_MAPSIZE range which includes pfn. The 785 * mapping of pfn to range index is imperfect because pfn and bitnum 786 * do not have the same phase. To make sure a CBUF_MAPSIZE range is 787 * covered, call this for both ends: 788 * dump_set_used(base) 789 * dump_set_used(base+CBUF_MAPNP-1) 790 * 791 * This is used during a panic dump to mark pages allocated by 792 * dumpsys_get_maxmem(). The macro IS_DUMP_PAGE(pp) is used by 793 * page_get_mnode_freelist() to make sure pages used by dump are never 794 * allocated. 795 */ 796 #define CBUF_MAPP2R(pfn) ((pfn) >> (CBUF_MAPSHIFT - PAGESHIFT)) 797 798 static void 799 dump_set_used(pfn_t pfn) 800 { 801 802 pgcnt_t bitnum, rbitnum; 803 804 bitnum = dump_pfn_to_bitnum(pfn); 805 ASSERT(bitnum != (pgcnt_t)-1); 806 807 rbitnum = CBUF_MAPP2R(bitnum); 808 ASSERT(rbitnum < dumpcfg.rbitmapsize); 809 810 BT_SET(dumpcfg.rbitmap, rbitnum); 811 } 812 813 int 814 dump_test_used(pfn_t pfn) 815 { 816 pgcnt_t bitnum, rbitnum; 817 818 bitnum = dump_pfn_to_bitnum(pfn); 819 ASSERT(bitnum != (pgcnt_t)-1); 820 821 rbitnum = CBUF_MAPP2R(bitnum); 822 ASSERT(rbitnum < dumpcfg.rbitmapsize); 823 824 return (BT_TEST(dumpcfg.rbitmap, rbitnum)); 825 } 826 827 /* 828 * dumpbzalloc and dumpbzfree are callbacks from the bzip2 library. 829 * dumpsys_get_maxmem() uses them for BZ2_bzCompressInit(). 830 */ 831 static void * 832 dumpbzalloc(void *opaque, int items, int size) 833 { 834 size_t *sz; 835 char *ret; 836 837 ASSERT(opaque != NULL); 838 sz = opaque; 839 ret = dumpcfg.maxvm + *sz; 840 *sz += items * size; 841 *sz = P2ROUNDUP(*sz, BZ2_BZALLOC_ALIGN); 842 ASSERT(*sz <= dumpcfg.maxvmsize); 843 return (ret); 844 } 845 846 /*ARGSUSED*/ 847 static void 848 dumpbzfree(void *opaque, void *addr) 849 { 850 } 851 852 /* 853 * Perform additional checks on the page to see if we can really use 854 * it. The kernel (kas) pages are always set in the bitmap. However, 855 * boot memory pages (prom_ppages or P_BOOTPAGES) are not in the 856 * bitmap. So we check for them. 857 */ 858 static inline int 859 dump_pfn_check(pfn_t pfn) 860 { 861 page_t *pp = page_numtopp_nolock(pfn); 862 if (pp == NULL || pp->p_pagenum != pfn || 863 #if defined(__sparc) 864 pp->p_vnode == &promvp || 865 #else 866 PP_ISBOOTPAGES(pp) || 867 #endif 868 pp->p_toxic != 0) 869 return (0); 870 return (1); 871 } 872 873 /* 874 * Check a range to see if all contained pages are available and 875 * return non-zero if the range can be used. 876 */ 877 static inline int 878 dump_range_check(pgcnt_t start, pgcnt_t end, pfn_t pfn) 879 { 880 for (; start < end; start++, pfn++) { 881 if (BT_TEST(dumpcfg.bitmap, start)) 882 return (0); 883 if (!dump_pfn_check(pfn)) 884 return (0); 885 } 886 return (1); 887 } 888 889 /* 890 * dumpsys_get_maxmem() is called during panic. Find unused ranges 891 * and use them for buffers. If we find enough memory switch to 892 * parallel bzip2, otherwise use parallel lzjb. 893 * 894 * It searches the dump bitmap in 2 passes. The first time it looks 895 * for CBUF_MAPSIZE ranges. On the second pass it uses small pages. 896 */ 897 static void 898 dumpsys_get_maxmem() 899 { 900 dumpcfg_t *cfg = &dumpcfg; 901 cbuf_t *endcp = &cfg->cbuf[cfg->ncbuf]; 902 helper_t *endhp = &cfg->helper[cfg->nhelper]; 903 pgcnt_t bitnum, end; 904 size_t sz, endsz, bz2size; 905 pfn_t pfn, off; 906 cbuf_t *cp; 907 helper_t *hp, *ohp; 908 dumpmlw_t mlw; 909 int k; 910 911 /* 912 * Setting dump_plat_mincpu to 0 at any time forces a serial 913 * dump. 914 */ 915 if (dump_plat_mincpu == 0) { 916 cfg->clevel = 0; 917 return; 918 } 919 920 /* 921 * There may be no point in looking for spare memory. If 922 * dumping all memory, then none is spare. If doing a serial 923 * dump, then already have buffers. 924 */ 925 if (cfg->maxsize == 0 || cfg->clevel < DUMP_CLEVEL_LZJB || 926 (dump_conflags & DUMP_ALL) != 0) { 927 if (cfg->clevel > DUMP_CLEVEL_LZJB) 928 cfg->clevel = DUMP_CLEVEL_LZJB; 929 return; 930 } 931 932 sz = 0; 933 cfg->found4m = 0; 934 cfg->foundsm = 0; 935 936 /* bitmap of ranges used to estimate which pfns are being used */ 937 bzero(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.rbitmapsize)); 938 939 /* find ranges that are not being dumped to use for buffers */ 940 dump_init_memlist_walker(&mlw); 941 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 942 dump_timeleft = dump_timeout; 943 end = bitnum + CBUF_MAPNP; 944 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 945 ASSERT(pfn != PFN_INVALID); 946 947 /* skip partial range at end of mem segment */ 948 if (mlw.mpleft < CBUF_MAPNP) { 949 end = bitnum + mlw.mpleft; 950 continue; 951 } 952 953 /* skip non aligned pages */ 954 off = P2PHASE(pfn, CBUF_MAPNP); 955 if (off != 0) { 956 end -= off; 957 continue; 958 } 959 960 if (!dump_range_check(bitnum, end, pfn)) 961 continue; 962 963 ASSERT((sz + CBUF_MAPSIZE) <= cfg->maxvmsize); 964 hat_devload(kas.a_hat, cfg->maxvm + sz, CBUF_MAPSIZE, pfn, 965 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 966 sz += CBUF_MAPSIZE; 967 cfg->found4m++; 968 969 /* set the bitmap for both ends to be sure to cover the range */ 970 dump_set_used(pfn); 971 dump_set_used(pfn + CBUF_MAPNP - 1); 972 973 if (sz >= cfg->maxsize) 974 goto foundmax; 975 } 976 977 /* Add small pages if we can't find enough large pages. */ 978 dump_init_memlist_walker(&mlw); 979 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 980 dump_timeleft = dump_timeout; 981 end = bitnum + CBUF_MAPNP; 982 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 983 ASSERT(pfn != PFN_INVALID); 984 985 /* Find any non-aligned pages at start and end of segment. */ 986 off = P2PHASE(pfn, CBUF_MAPNP); 987 if (mlw.mpleft < CBUF_MAPNP) { 988 end = bitnum + mlw.mpleft; 989 } else if (off != 0) { 990 end -= off; 991 } else if (cfg->found4m && dump_test_used(pfn)) { 992 continue; 993 } 994 995 for (; bitnum < end; bitnum++, pfn++) { 996 dump_timeleft = dump_timeout; 997 if (BT_TEST(dumpcfg.bitmap, bitnum)) 998 continue; 999 if (!dump_pfn_check(pfn)) 1000 continue; 1001 ASSERT((sz + PAGESIZE) <= cfg->maxvmsize); 1002 hat_devload(kas.a_hat, cfg->maxvm + sz, PAGESIZE, pfn, 1003 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 1004 sz += PAGESIZE; 1005 cfg->foundsm++; 1006 dump_set_used(pfn); 1007 if (sz >= cfg->maxsize) 1008 goto foundmax; 1009 } 1010 } 1011 1012 /* Fall back to lzjb if we did not get enough memory for bzip2. */ 1013 endsz = (cfg->maxsize * cfg->threshold) / cfg->nhelper; 1014 if (sz < endsz) { 1015 cfg->clevel = DUMP_CLEVEL_LZJB; 1016 } 1017 1018 /* Allocate memory for as many helpers as we can. */ 1019 foundmax: 1020 1021 /* Byte offsets into memory found and mapped above */ 1022 endsz = sz; 1023 sz = 0; 1024 1025 /* Set the size for bzip2 state. Only bzip2 needs it. */ 1026 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 1027 1028 /* Skip the preallocate output buffers. */ 1029 cp = &cfg->cbuf[MINCBUFS]; 1030 1031 /* Use this to move memory up from the preallocated helpers. */ 1032 ohp = cfg->helper; 1033 1034 /* Loop over all helpers and allocate memory. */ 1035 for (hp = cfg->helper; hp < endhp; hp++) { 1036 1037 /* Skip preallocated helpers by checking hp->page. */ 1038 if (hp->page == NULL) { 1039 if (cfg->clevel <= DUMP_CLEVEL_LZJB) { 1040 /* lzjb needs 2 1-page buffers */ 1041 if ((sz + (2 * PAGESIZE)) > endsz) 1042 break; 1043 hp->page = cfg->maxvm + sz; 1044 sz += PAGESIZE; 1045 hp->lzbuf = cfg->maxvm + sz; 1046 sz += PAGESIZE; 1047 1048 } else if (ohp->lzbuf != NULL) { 1049 /* re-use the preallocted lzjb page for bzip2 */ 1050 hp->page = ohp->lzbuf; 1051 ohp->lzbuf = NULL; 1052 ++ohp; 1053 1054 } else { 1055 /* bzip2 needs a 1-page buffer */ 1056 if ((sz + PAGESIZE) > endsz) 1057 break; 1058 hp->page = cfg->maxvm + sz; 1059 sz += PAGESIZE; 1060 } 1061 } 1062 1063 /* 1064 * Add output buffers per helper. The number of 1065 * buffers per helper is determined by the ratio of 1066 * ncbuf to nhelper. 1067 */ 1068 for (k = 0; cp < endcp && (sz + CBUF_SIZE) <= endsz && 1069 k < NCBUF_PER_HELPER; k++) { 1070 cp->state = CBUF_FREEBUF; 1071 cp->size = CBUF_SIZE; 1072 cp->buf = cfg->maxvm + sz; 1073 sz += CBUF_SIZE; 1074 ++cp; 1075 } 1076 1077 /* 1078 * bzip2 needs compression state. Use the dumpbzalloc 1079 * and dumpbzfree callbacks to allocate the memory. 1080 * bzip2 does allocation only at init time. 1081 */ 1082 if (cfg->clevel >= DUMP_CLEVEL_BZIP2) { 1083 if ((sz + bz2size) > endsz) { 1084 hp->page = NULL; 1085 break; 1086 } else { 1087 hp->bzstream.opaque = &sz; 1088 hp->bzstream.bzalloc = dumpbzalloc; 1089 hp->bzstream.bzfree = dumpbzfree; 1090 (void) BZ2_bzCompressInit(&hp->bzstream, 1091 dump_bzip2_level, 0, 0); 1092 hp->bzstream.opaque = NULL; 1093 } 1094 } 1095 } 1096 1097 /* Finish allocating output buffers */ 1098 for (; cp < endcp && (sz + CBUF_SIZE) <= endsz; cp++) { 1099 cp->state = CBUF_FREEBUF; 1100 cp->size = CBUF_SIZE; 1101 cp->buf = cfg->maxvm + sz; 1102 sz += CBUF_SIZE; 1103 } 1104 1105 /* Enable IS_DUMP_PAGE macro, which checks for pages we took. */ 1106 if (cfg->found4m || cfg->foundsm) 1107 dump_check_used = 1; 1108 1109 ASSERT(sz <= endsz); 1110 } 1111 1112 static void 1113 dumphdr_init(void) 1114 { 1115 pgcnt_t npages = 0; 1116 1117 ASSERT(MUTEX_HELD(&dump_lock)); 1118 1119 if (dumphdr == NULL) { 1120 dumphdr = kmem_zalloc(sizeof (dumphdr_t), KM_SLEEP); 1121 dumphdr->dump_magic = DUMP_MAGIC; 1122 dumphdr->dump_version = DUMP_VERSION; 1123 dumphdr->dump_wordsize = DUMP_WORDSIZE; 1124 dumphdr->dump_pageshift = PAGESHIFT; 1125 dumphdr->dump_pagesize = PAGESIZE; 1126 dumphdr->dump_utsname = utsname; 1127 (void) strcpy(dumphdr->dump_platform, platform); 1128 dumpbuf.size = dumpbuf_iosize(maxphys); 1129 dumpbuf.start = kmem_alloc(dumpbuf.size, KM_SLEEP); 1130 dumpbuf.end = dumpbuf.start + dumpbuf.size; 1131 dumpcfg.pids = kmem_alloc(v.v_proc * sizeof (pid_t), KM_SLEEP); 1132 dumpcfg.helpermap = kmem_zalloc(BT_SIZEOFMAP(NCPU), KM_SLEEP); 1133 LOCK_INIT_HELD(&dumpcfg.helper_lock); 1134 dump_stack_scratch = kmem_alloc(STACK_BUF_SIZE, KM_SLEEP); 1135 (void) strncpy(dumphdr->dump_uuid, dump_get_uuid(), 1136 sizeof (dumphdr->dump_uuid)); 1137 } 1138 1139 npages = num_phys_pages(); 1140 1141 if (dumpcfg.bitmapsize != npages) { 1142 size_t rlen = CBUF_MAPP2R(P2ROUNDUP(npages, CBUF_MAPNP)); 1143 void *map = kmem_alloc(BT_SIZEOFMAP(npages), KM_SLEEP); 1144 void *rmap = kmem_alloc(BT_SIZEOFMAP(rlen), KM_SLEEP); 1145 1146 if (dumpcfg.bitmap != NULL) 1147 kmem_free(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg. 1148 bitmapsize)); 1149 if (dumpcfg.rbitmap != NULL) 1150 kmem_free(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg. 1151 rbitmapsize)); 1152 dumpcfg.bitmap = map; 1153 dumpcfg.bitmapsize = npages; 1154 dumpcfg.rbitmap = rmap; 1155 dumpcfg.rbitmapsize = rlen; 1156 } 1157 } 1158 1159 /* 1160 * Establish a new dump device. 1161 */ 1162 int 1163 dumpinit(vnode_t *vp, char *name, int justchecking) 1164 { 1165 vnode_t *cvp; 1166 vattr_t vattr; 1167 vnode_t *cdev_vp; 1168 int error = 0; 1169 1170 ASSERT(MUTEX_HELD(&dump_lock)); 1171 1172 dumphdr_init(); 1173 1174 cvp = common_specvp(vp); 1175 if (cvp == dumpvp) 1176 return (0); 1177 1178 /* 1179 * Determine whether this is a plausible dump device. We want either: 1180 * (1) a real device that's not mounted and has a cb_dump routine, or 1181 * (2) a swapfile on some filesystem that has a vop_dump routine. 1182 */ 1183 if ((error = VOP_OPEN(&cvp, FREAD | FWRITE, kcred, NULL)) != 0) 1184 return (error); 1185 1186 vattr.va_mask = AT_SIZE | AT_TYPE | AT_RDEV; 1187 if ((error = VOP_GETATTR(cvp, &vattr, 0, kcred, NULL)) == 0) { 1188 if (vattr.va_type == VBLK || vattr.va_type == VCHR) { 1189 if (devopsp[getmajor(vattr.va_rdev)]-> 1190 devo_cb_ops->cb_dump == nodev) 1191 error = ENOTSUP; 1192 else if (vfs_devismounted(vattr.va_rdev)) 1193 error = EBUSY; 1194 if (strcmp(ddi_driver_name(VTOS(cvp)->s_dip), 1195 ZFS_DRIVER) == 0 && 1196 IS_SWAPVP(common_specvp(cvp))) 1197 error = EBUSY; 1198 } else { 1199 if (vn_matchopval(cvp, VOPNAME_DUMP, fs_nosys) || 1200 !IS_SWAPVP(cvp)) 1201 error = ENOTSUP; 1202 } 1203 } 1204 1205 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) 1206 error = ENOSPC; 1207 1208 if (error || justchecking) { 1209 (void) VOP_CLOSE(cvp, FREAD | FWRITE, 1, (offset_t)0, 1210 kcred, NULL); 1211 return (error); 1212 } 1213 1214 VN_HOLD(cvp); 1215 1216 if (dumpvp != NULL) 1217 dumpfini(); /* unconfigure the old dump device */ 1218 1219 dumpvp = cvp; 1220 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 1221 dumppath = kmem_alloc(strlen(name) + 1, KM_SLEEP); 1222 (void) strcpy(dumppath, name); 1223 dumpbuf.iosize = 0; 1224 1225 /* 1226 * If the dump device is a block device, attempt to open up the 1227 * corresponding character device and determine its maximum transfer 1228 * size. We use this information to potentially resize dumpbuf to a 1229 * larger and more optimal size for performing i/o to the dump device. 1230 */ 1231 if (cvp->v_type == VBLK && 1232 (cdev_vp = makespecvp(VTOS(cvp)->s_dev, VCHR)) != NULL) { 1233 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1234 size_t blk_size; 1235 struct dk_cinfo dki; 1236 struct dk_minfo minf; 1237 1238 if (VOP_IOCTL(cdev_vp, DKIOCGMEDIAINFO, 1239 (intptr_t)&minf, FKIOCTL, kcred, NULL, NULL) 1240 == 0 && minf.dki_lbsize != 0) 1241 blk_size = minf.dki_lbsize; 1242 else 1243 blk_size = DEV_BSIZE; 1244 1245 if (VOP_IOCTL(cdev_vp, DKIOCINFO, (intptr_t)&dki, 1246 FKIOCTL, kcred, NULL, NULL) == 0) { 1247 dumpbuf.iosize = dki.dki_maxtransfer * blk_size; 1248 dumpbuf_resize(); 1249 } 1250 /* 1251 * If we are working with a zvol then dumpify it 1252 * if it's not being used as swap. 1253 */ 1254 if (strcmp(dki.dki_dname, ZVOL_DRIVER) == 0) { 1255 if (IS_SWAPVP(common_specvp(cvp))) 1256 error = EBUSY; 1257 else if ((error = VOP_IOCTL(cdev_vp, 1258 DKIOCDUMPINIT, 0, FKIOCTL, kcred, 1259 NULL, NULL)) != 0) 1260 dumpfini(); 1261 } 1262 1263 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1264 kcred, NULL); 1265 } 1266 1267 VN_RELE(cdev_vp); 1268 } 1269 1270 cmn_err(CE_CONT, "?dump on %s size %llu MB\n", name, dumpvp_size >> 20); 1271 1272 dump_update_clevel(); 1273 1274 return (error); 1275 } 1276 1277 void 1278 dumpfini(void) 1279 { 1280 vattr_t vattr; 1281 boolean_t is_zfs = B_FALSE; 1282 vnode_t *cdev_vp; 1283 ASSERT(MUTEX_HELD(&dump_lock)); 1284 1285 kmem_free(dumppath, strlen(dumppath) + 1); 1286 1287 /* 1288 * Determine if we are using zvols for our dump device 1289 */ 1290 vattr.va_mask = AT_RDEV; 1291 if (VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL) == 0) { 1292 is_zfs = (getmajor(vattr.va_rdev) == 1293 ddi_name_to_major(ZFS_DRIVER)) ? B_TRUE : B_FALSE; 1294 } 1295 1296 /* 1297 * If we have a zvol dump device then we call into zfs so 1298 * that it may have a chance to cleanup. 1299 */ 1300 if (is_zfs && 1301 (cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR)) != NULL) { 1302 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1303 (void) VOP_IOCTL(cdev_vp, DKIOCDUMPFINI, 0, FKIOCTL, 1304 kcred, NULL, NULL); 1305 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1306 kcred, NULL); 1307 } 1308 VN_RELE(cdev_vp); 1309 } 1310 1311 (void) VOP_CLOSE(dumpvp, FREAD | FWRITE, 1, (offset_t)0, kcred, NULL); 1312 1313 VN_RELE(dumpvp); 1314 1315 dumpvp = NULL; 1316 dumpvp_size = 0; 1317 dumppath = NULL; 1318 } 1319 1320 static offset_t 1321 dumpvp_flush(void) 1322 { 1323 size_t size = P2ROUNDUP(dumpbuf.cur - dumpbuf.start, PAGESIZE); 1324 hrtime_t iotime; 1325 int err; 1326 1327 if (dumpbuf.vp_off + size > dumpbuf.vp_limit) { 1328 dump_ioerr = ENOSPC; 1329 dumpbuf.vp_off = dumpbuf.vp_limit; 1330 } else if (size != 0) { 1331 iotime = gethrtime(); 1332 dumpsync.iowait += iotime - dumpsync.iowaitts; 1333 if (panicstr) 1334 err = VOP_DUMP(dumpvp, dumpbuf.start, 1335 lbtodb(dumpbuf.vp_off), btod(size), NULL); 1336 else 1337 err = vn_rdwr(UIO_WRITE, dumpbuf.cdev_vp != NULL ? 1338 dumpbuf.cdev_vp : dumpvp, dumpbuf.start, size, 1339 dumpbuf.vp_off, UIO_SYSSPACE, 0, dumpbuf.vp_limit, 1340 kcred, 0); 1341 if (err && dump_ioerr == 0) 1342 dump_ioerr = err; 1343 dumpsync.iowaitts = gethrtime(); 1344 dumpsync.iotime += dumpsync.iowaitts - iotime; 1345 dumpsync.nwrite += size; 1346 dumpbuf.vp_off += size; 1347 } 1348 dumpbuf.cur = dumpbuf.start; 1349 dump_timeleft = dump_timeout; 1350 return (dumpbuf.vp_off); 1351 } 1352 1353 /* maximize write speed by keeping seek offset aligned with size */ 1354 void 1355 dumpvp_write(const void *va, size_t size) 1356 { 1357 size_t len, off, sz; 1358 1359 while (size != 0) { 1360 len = MIN(size, dumpbuf.end - dumpbuf.cur); 1361 if (len == 0) { 1362 off = P2PHASE(dumpbuf.vp_off, dumpbuf.size); 1363 if (off == 0 || !ISP2(dumpbuf.size)) { 1364 (void) dumpvp_flush(); 1365 } else { 1366 sz = dumpbuf.size - off; 1367 dumpbuf.cur = dumpbuf.start + sz; 1368 (void) dumpvp_flush(); 1369 ovbcopy(dumpbuf.start + sz, dumpbuf.start, off); 1370 dumpbuf.cur += off; 1371 } 1372 } else { 1373 bcopy(va, dumpbuf.cur, len); 1374 va = (char *)va + len; 1375 dumpbuf.cur += len; 1376 size -= len; 1377 } 1378 } 1379 } 1380 1381 /*ARGSUSED*/ 1382 static void 1383 dumpvp_ksyms_write(const void *src, void *dst, size_t size) 1384 { 1385 dumpvp_write(src, size); 1386 } 1387 1388 /* 1389 * Mark 'pfn' in the bitmap and dump its translation table entry. 1390 */ 1391 void 1392 dump_addpage(struct as *as, void *va, pfn_t pfn) 1393 { 1394 mem_vtop_t mem_vtop; 1395 pgcnt_t bitnum; 1396 1397 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1398 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1399 dumphdr->dump_npages++; 1400 BT_SET(dumpcfg.bitmap, bitnum); 1401 } 1402 dumphdr->dump_nvtop++; 1403 mem_vtop.m_as = as; 1404 mem_vtop.m_va = va; 1405 mem_vtop.m_pfn = pfn; 1406 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1407 } 1408 dump_timeleft = dump_timeout; 1409 } 1410 1411 /* 1412 * Mark 'pfn' in the bitmap 1413 */ 1414 void 1415 dump_page(pfn_t pfn) 1416 { 1417 pgcnt_t bitnum; 1418 1419 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1420 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1421 dumphdr->dump_npages++; 1422 BT_SET(dumpcfg.bitmap, bitnum); 1423 } 1424 } 1425 dump_timeleft = dump_timeout; 1426 } 1427 1428 /* 1429 * Dump the <as, va, pfn> information for a given address space. 1430 * SEGOP_DUMP() will call dump_addpage() for each page in the segment. 1431 */ 1432 static void 1433 dump_as(struct as *as) 1434 { 1435 struct seg *seg; 1436 1437 AS_LOCK_ENTER(as, RW_READER); 1438 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 1439 if (seg->s_as != as) 1440 break; 1441 if (seg->s_ops == NULL) 1442 continue; 1443 SEGOP_DUMP(seg); 1444 } 1445 AS_LOCK_EXIT(as); 1446 1447 if (seg != NULL) 1448 cmn_err(CE_WARN, "invalid segment %p in address space %p", 1449 (void *)seg, (void *)as); 1450 } 1451 1452 static int 1453 dump_process(pid_t pid) 1454 { 1455 proc_t *p = sprlock(pid); 1456 1457 if (p == NULL) 1458 return (-1); 1459 if (p->p_as != &kas) { 1460 mutex_exit(&p->p_lock); 1461 dump_as(p->p_as); 1462 mutex_enter(&p->p_lock); 1463 } 1464 1465 sprunlock(p); 1466 1467 return (0); 1468 } 1469 1470 /* 1471 * The following functions (dump_summary(), dump_ereports(), and 1472 * dump_messages()), write data to an uncompressed area within the 1473 * crashdump. The layout of these is 1474 * 1475 * +------------------------------------------------------------+ 1476 * | compressed pages | summary | ereports | messages | 1477 * +------------------------------------------------------------+ 1478 * 1479 * With the advent of saving a compressed crash dump by default, we 1480 * need to save a little more data to describe the failure mode in 1481 * an uncompressed buffer available before savecore uncompresses 1482 * the dump. Initially this is a copy of the stack trace. Additional 1483 * summary information should be added here. 1484 */ 1485 1486 void 1487 dump_summary(void) 1488 { 1489 u_offset_t dumpvp_start; 1490 summary_dump_t sd; 1491 1492 if (dumpvp == NULL || dumphdr == NULL) 1493 return; 1494 1495 dumpbuf.cur = dumpbuf.start; 1496 1497 dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE + 1498 DUMP_ERPTSIZE); 1499 dumpvp_start = dumpbuf.vp_limit - DUMP_SUMMARYSIZE; 1500 dumpbuf.vp_off = dumpvp_start; 1501 1502 sd.sd_magic = SUMMARY_MAGIC; 1503 sd.sd_ssum = checksum32(dump_stack_scratch, STACK_BUF_SIZE); 1504 dumpvp_write(&sd, sizeof (sd)); 1505 dumpvp_write(dump_stack_scratch, STACK_BUF_SIZE); 1506 1507 sd.sd_magic = 0; /* indicate end of summary */ 1508 dumpvp_write(&sd, sizeof (sd)); 1509 (void) dumpvp_flush(); 1510 } 1511 1512 void 1513 dump_ereports(void) 1514 { 1515 u_offset_t dumpvp_start; 1516 erpt_dump_t ed; 1517 1518 if (dumpvp == NULL || dumphdr == NULL) 1519 return; 1520 1521 dumpbuf.cur = dumpbuf.start; 1522 dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE); 1523 dumpvp_start = dumpbuf.vp_limit - DUMP_ERPTSIZE; 1524 dumpbuf.vp_off = dumpvp_start; 1525 1526 fm_ereport_dump(); 1527 if (panicstr) 1528 errorq_dump(); 1529 1530 bzero(&ed, sizeof (ed)); /* indicate end of ereports */ 1531 dumpvp_write(&ed, sizeof (ed)); 1532 (void) dumpvp_flush(); 1533 1534 if (!panicstr) { 1535 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1536 (size_t)(dumpbuf.vp_off - dumpvp_start), 1537 B_INVAL | B_FORCE, kcred, NULL); 1538 } 1539 } 1540 1541 void 1542 dump_messages(void) 1543 { 1544 log_dump_t ld; 1545 mblk_t *mctl, *mdata; 1546 queue_t *q, *qlast; 1547 u_offset_t dumpvp_start; 1548 1549 if (dumpvp == NULL || dumphdr == NULL || log_consq == NULL) 1550 return; 1551 1552 dumpbuf.cur = dumpbuf.start; 1553 dumpbuf.vp_limit = dumpvp_size - DUMP_OFFSET; 1554 dumpvp_start = dumpbuf.vp_limit - DUMP_LOGSIZE; 1555 dumpbuf.vp_off = dumpvp_start; 1556 1557 qlast = NULL; 1558 do { 1559 for (q = log_consq; q->q_next != qlast; q = q->q_next) 1560 continue; 1561 for (mctl = q->q_first; mctl != NULL; mctl = mctl->b_next) { 1562 dump_timeleft = dump_timeout; 1563 mdata = mctl->b_cont; 1564 ld.ld_magic = LOG_MAGIC; 1565 ld.ld_msgsize = MBLKL(mctl->b_cont); 1566 ld.ld_csum = checksum32(mctl->b_rptr, MBLKL(mctl)); 1567 ld.ld_msum = checksum32(mdata->b_rptr, MBLKL(mdata)); 1568 dumpvp_write(&ld, sizeof (ld)); 1569 dumpvp_write(mctl->b_rptr, MBLKL(mctl)); 1570 dumpvp_write(mdata->b_rptr, MBLKL(mdata)); 1571 } 1572 } while ((qlast = q) != log_consq); 1573 1574 ld.ld_magic = 0; /* indicate end of messages */ 1575 dumpvp_write(&ld, sizeof (ld)); 1576 (void) dumpvp_flush(); 1577 if (!panicstr) { 1578 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1579 (size_t)(dumpbuf.vp_off - dumpvp_start), 1580 B_INVAL | B_FORCE, kcred, NULL); 1581 } 1582 } 1583 1584 /* 1585 * The following functions are called on multiple CPUs during dump. 1586 * They must not use most kernel services, because all cross-calls are 1587 * disabled during panic. Therefore, blocking locks and cache flushes 1588 * will not work. 1589 */ 1590 1591 /* 1592 * Copy pages, trapping ECC errors. Also, for robustness, trap data 1593 * access in case something goes wrong in the hat layer and the 1594 * mapping is broken. 1595 */ 1596 static int 1597 dump_pagecopy(void *src, void *dst) 1598 { 1599 long *wsrc = (long *)src; 1600 long *wdst = (long *)dst; 1601 const ulong_t ncopies = PAGESIZE / sizeof (long); 1602 volatile int w = 0; 1603 volatile int ueoff = -1; 1604 on_trap_data_t otd; 1605 1606 if (on_trap(&otd, OT_DATA_EC | OT_DATA_ACCESS)) { 1607 if (ueoff == -1) 1608 ueoff = w * sizeof (long); 1609 /* report "bad ECC" or "bad address" */ 1610 #ifdef _LP64 1611 if (otd.ot_trap & OT_DATA_EC) 1612 wdst[w++] = 0x00badecc00badecc; 1613 else 1614 wdst[w++] = 0x00badadd00badadd; 1615 #else 1616 if (otd.ot_trap & OT_DATA_EC) 1617 wdst[w++] = 0x00badecc; 1618 else 1619 wdst[w++] = 0x00badadd; 1620 #endif 1621 } 1622 while (w < ncopies) { 1623 wdst[w] = wsrc[w]; 1624 w++; 1625 } 1626 no_trap(); 1627 return (ueoff); 1628 } 1629 1630 static void 1631 dumpsys_close_cq(cqueue_t *cq, int live) 1632 { 1633 if (live) { 1634 mutex_enter(&cq->mutex); 1635 atomic_dec_uint(&cq->open); 1636 cv_signal(&cq->cv); 1637 mutex_exit(&cq->mutex); 1638 } else { 1639 atomic_dec_uint(&cq->open); 1640 } 1641 } 1642 1643 static inline void 1644 dumpsys_spinlock(lock_t *lp) 1645 { 1646 uint_t backoff = 0; 1647 int loop_count = 0; 1648 1649 while (LOCK_HELD(lp) || !lock_spin_try(lp)) { 1650 if (++loop_count >= ncpus) { 1651 backoff = mutex_lock_backoff(0); 1652 loop_count = 0; 1653 } else { 1654 backoff = mutex_lock_backoff(backoff); 1655 } 1656 mutex_lock_delay(backoff); 1657 } 1658 } 1659 1660 static inline void 1661 dumpsys_spinunlock(lock_t *lp) 1662 { 1663 lock_clear(lp); 1664 } 1665 1666 static inline void 1667 dumpsys_lock(cqueue_t *cq, int live) 1668 { 1669 if (live) 1670 mutex_enter(&cq->mutex); 1671 else 1672 dumpsys_spinlock(&cq->spinlock); 1673 } 1674 1675 static inline void 1676 dumpsys_unlock(cqueue_t *cq, int live, int signal) 1677 { 1678 if (live) { 1679 if (signal) 1680 cv_signal(&cq->cv); 1681 mutex_exit(&cq->mutex); 1682 } else { 1683 dumpsys_spinunlock(&cq->spinlock); 1684 } 1685 } 1686 1687 static void 1688 dumpsys_wait_cq(cqueue_t *cq, int live) 1689 { 1690 if (live) { 1691 cv_wait(&cq->cv, &cq->mutex); 1692 } else { 1693 dumpsys_spinunlock(&cq->spinlock); 1694 while (cq->open) 1695 if (cq->first) 1696 break; 1697 dumpsys_spinlock(&cq->spinlock); 1698 } 1699 } 1700 1701 static void 1702 dumpsys_put_cq(cqueue_t *cq, cbuf_t *cp, int newstate, int live) 1703 { 1704 if (cp == NULL) 1705 return; 1706 1707 dumpsys_lock(cq, live); 1708 1709 if (cq->ts != 0) { 1710 cq->empty += gethrtime() - cq->ts; 1711 cq->ts = 0; 1712 } 1713 1714 cp->state = newstate; 1715 cp->next = NULL; 1716 if (cq->last == NULL) 1717 cq->first = cp; 1718 else 1719 cq->last->next = cp; 1720 cq->last = cp; 1721 1722 dumpsys_unlock(cq, live, 1); 1723 } 1724 1725 static cbuf_t * 1726 dumpsys_get_cq(cqueue_t *cq, int live) 1727 { 1728 cbuf_t *cp; 1729 hrtime_t now = gethrtime(); 1730 1731 dumpsys_lock(cq, live); 1732 1733 /* CONSTCOND */ 1734 while (1) { 1735 cp = (cbuf_t *)cq->first; 1736 if (cp == NULL) { 1737 if (cq->open == 0) 1738 break; 1739 dumpsys_wait_cq(cq, live); 1740 continue; 1741 } 1742 cq->first = cp->next; 1743 if (cq->first == NULL) { 1744 cq->last = NULL; 1745 cq->ts = now; 1746 } 1747 break; 1748 } 1749 1750 dumpsys_unlock(cq, live, cq->first != NULL || cq->open == 0); 1751 return (cp); 1752 } 1753 1754 /* 1755 * Send an error message to the console. If the main task is running 1756 * just write the message via uprintf. If a helper is running the 1757 * message has to be put on a queue for the main task. Setting fmt to 1758 * NULL means flush the error message buffer. If fmt is not NULL, just 1759 * add the text to the existing buffer. 1760 */ 1761 static void 1762 dumpsys_errmsg(helper_t *hp, const char *fmt, ...) 1763 { 1764 dumpsync_t *ds = hp->ds; 1765 cbuf_t *cp = hp->cperr; 1766 va_list adx; 1767 1768 if (hp->helper == MAINHELPER) { 1769 if (fmt != NULL) { 1770 if (ds->neednl) { 1771 uprintf("\n"); 1772 ds->neednl = 0; 1773 } 1774 va_start(adx, fmt); 1775 vuprintf(fmt, adx); 1776 va_end(adx); 1777 } 1778 } else if (fmt == NULL) { 1779 if (cp != NULL) { 1780 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1781 hp->cperr = NULL; 1782 } 1783 } else { 1784 if (hp->cperr == NULL) { 1785 cp = CQ_GET(freebufq); 1786 hp->cperr = cp; 1787 cp->used = 0; 1788 } 1789 va_start(adx, fmt); 1790 cp->used += vsnprintf(cp->buf + cp->used, cp->size - cp->used, 1791 fmt, adx); 1792 va_end(adx); 1793 if ((cp->used + LOG_MSGSIZE) > cp->size) { 1794 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1795 hp->cperr = NULL; 1796 } 1797 } 1798 } 1799 1800 /* 1801 * Write an output buffer to the dump file. If the main task is 1802 * running just write the data. If a helper is running the output is 1803 * placed on a queue for the main task. 1804 */ 1805 static void 1806 dumpsys_swrite(helper_t *hp, cbuf_t *cp, size_t used) 1807 { 1808 dumpsync_t *ds = hp->ds; 1809 1810 if (hp->helper == MAINHELPER) { 1811 HRSTART(ds->perpage, write); 1812 dumpvp_write(cp->buf, used); 1813 HRSTOP(ds->perpage, write); 1814 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 1815 } else { 1816 cp->used = used; 1817 CQ_PUT(mainq, cp, CBUF_WRITE); 1818 } 1819 } 1820 1821 /* 1822 * Copy one page within the mapped range. The offset starts at 0 and 1823 * is relative to the first pfn. cp->buf + cp->off is the address of 1824 * the first pfn. If dump_pagecopy returns a UE offset, create an 1825 * error message. Returns the offset to the next pfn in the range 1826 * selected by the bitmap. 1827 */ 1828 static int 1829 dumpsys_copy_page(helper_t *hp, int offset) 1830 { 1831 cbuf_t *cp = hp->cpin; 1832 int ueoff; 1833 1834 ASSERT(cp->off + offset + PAGESIZE <= cp->size); 1835 ASSERT(BT_TEST(dumpcfg.bitmap, cp->bitnum)); 1836 1837 ueoff = dump_pagecopy(cp->buf + cp->off + offset, hp->page); 1838 1839 /* ueoff is the offset in the page to a UE error */ 1840 if (ueoff != -1) { 1841 uint64_t pa = ptob(cp->pfn) + offset + ueoff; 1842 1843 dumpsys_errmsg(hp, "cpu %d: memory error at PA 0x%08x.%08x\n", 1844 CPU->cpu_id, (uint32_t)(pa >> 32), (uint32_t)pa); 1845 } 1846 1847 /* 1848 * Advance bitnum and offset to the next input page for the 1849 * next call to this function. 1850 */ 1851 offset += PAGESIZE; 1852 cp->bitnum++; 1853 while (cp->off + offset < cp->size) { 1854 if (BT_TEST(dumpcfg.bitmap, cp->bitnum)) 1855 break; 1856 offset += PAGESIZE; 1857 cp->bitnum++; 1858 } 1859 1860 return (offset); 1861 } 1862 1863 /* 1864 * Read the helper queue, and copy one mapped page. Return 0 when 1865 * done. Return 1 when a page has been copied into hp->page. 1866 */ 1867 static int 1868 dumpsys_sread(helper_t *hp) 1869 { 1870 dumpsync_t *ds = hp->ds; 1871 1872 /* CONSTCOND */ 1873 while (1) { 1874 1875 /* Find the next input buffer. */ 1876 if (hp->cpin == NULL) { 1877 HRSTART(hp->perpage, inwait); 1878 1879 /* CONSTCOND */ 1880 while (1) { 1881 hp->cpin = CQ_GET(helperq); 1882 dump_timeleft = dump_timeout; 1883 1884 /* 1885 * NULL return means the helper queue 1886 * is closed and empty. 1887 */ 1888 if (hp->cpin == NULL) 1889 break; 1890 1891 /* Have input, check for dump I/O error. */ 1892 if (!dump_ioerr) 1893 break; 1894 1895 /* 1896 * If an I/O error occurs, stay in the 1897 * loop in order to empty the helper 1898 * queue. Return the buffers to the 1899 * main task to unmap and free it. 1900 */ 1901 hp->cpin->used = 0; 1902 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1903 } 1904 HRSTOP(hp->perpage, inwait); 1905 1906 /* Stop here when the helper queue is closed. */ 1907 if (hp->cpin == NULL) 1908 break; 1909 1910 /* Set the offset=0 to get the first pfn. */ 1911 hp->in = 0; 1912 1913 /* Set the total processed to 0 */ 1914 hp->used = 0; 1915 } 1916 1917 /* Process the next page. */ 1918 if (hp->used < hp->cpin->used) { 1919 1920 /* 1921 * Get the next page from the input buffer and 1922 * return a copy. 1923 */ 1924 ASSERT(hp->in != -1); 1925 HRSTART(hp->perpage, copy); 1926 hp->in = dumpsys_copy_page(hp, hp->in); 1927 hp->used += PAGESIZE; 1928 HRSTOP(hp->perpage, copy); 1929 break; 1930 1931 } else { 1932 1933 /* 1934 * Done with the input. Flush the VM and 1935 * return the buffer to the main task. 1936 */ 1937 if (panicstr && hp->helper != MAINHELPER) 1938 hat_flush_range(kas.a_hat, 1939 hp->cpin->buf, hp->cpin->size); 1940 dumpsys_errmsg(hp, NULL); 1941 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1942 hp->cpin = NULL; 1943 } 1944 } 1945 1946 return (hp->cpin != NULL); 1947 } 1948 1949 /* 1950 * Compress size bytes starting at buf with bzip2 1951 * mode: 1952 * BZ_RUN add one more compressed page 1953 * BZ_FINISH no more input, flush the state 1954 */ 1955 static void 1956 dumpsys_bzrun(helper_t *hp, void *buf, size_t size, int mode) 1957 { 1958 dumpsync_t *ds = hp->ds; 1959 const int CSIZE = sizeof (dumpcsize_t); 1960 bz_stream *ps = &hp->bzstream; 1961 int rc = 0; 1962 uint32_t csize; 1963 dumpcsize_t cs; 1964 1965 /* Set input pointers to new input page */ 1966 if (size > 0) { 1967 ps->avail_in = size; 1968 ps->next_in = buf; 1969 } 1970 1971 /* CONSTCOND */ 1972 while (1) { 1973 1974 /* Quit when all input has been consumed */ 1975 if (ps->avail_in == 0 && mode == BZ_RUN) 1976 break; 1977 1978 /* Get a new output buffer */ 1979 if (hp->cpout == NULL) { 1980 HRSTART(hp->perpage, outwait); 1981 hp->cpout = CQ_GET(freebufq); 1982 HRSTOP(hp->perpage, outwait); 1983 ps->avail_out = hp->cpout->size - CSIZE; 1984 ps->next_out = hp->cpout->buf + CSIZE; 1985 } 1986 1987 /* Compress input, or finalize */ 1988 HRSTART(hp->perpage, compress); 1989 rc = BZ2_bzCompress(ps, mode); 1990 HRSTOP(hp->perpage, compress); 1991 1992 /* Check for error */ 1993 if (mode == BZ_RUN && rc != BZ_RUN_OK) { 1994 dumpsys_errmsg(hp, "%d: BZ_RUN error %s at page %lx\n", 1995 hp->helper, BZ2_bzErrorString(rc), 1996 hp->cpin->pagenum); 1997 break; 1998 } 1999 2000 /* Write the buffer if it is full, or we are flushing */ 2001 if (ps->avail_out == 0 || mode == BZ_FINISH) { 2002 csize = hp->cpout->size - CSIZE - ps->avail_out; 2003 cs = DUMP_SET_TAG(csize, hp->tag); 2004 if (csize > 0) { 2005 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 2006 dumpsys_swrite(hp, hp->cpout, csize + CSIZE); 2007 hp->cpout = NULL; 2008 } 2009 } 2010 2011 /* Check for final complete */ 2012 if (mode == BZ_FINISH) { 2013 if (rc == BZ_STREAM_END) 2014 break; 2015 if (rc != BZ_FINISH_OK) { 2016 dumpsys_errmsg(hp, "%d: BZ_FINISH error %s\n", 2017 hp->helper, BZ2_bzErrorString(rc)); 2018 break; 2019 } 2020 } 2021 } 2022 2023 /* Cleanup state and buffers */ 2024 if (mode == BZ_FINISH) { 2025 2026 /* Reset state so that it is re-usable. */ 2027 (void) BZ2_bzCompressReset(&hp->bzstream); 2028 2029 /* Give any unused outout buffer to the main task */ 2030 if (hp->cpout != NULL) { 2031 hp->cpout->used = 0; 2032 CQ_PUT(mainq, hp->cpout, CBUF_ERRMSG); 2033 hp->cpout = NULL; 2034 } 2035 } 2036 } 2037 2038 static void 2039 dumpsys_bz2compress(helper_t *hp) 2040 { 2041 dumpsync_t *ds = hp->ds; 2042 dumpstreamhdr_t sh; 2043 2044 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 2045 sh.stream_pagenum = (pgcnt_t)-1; 2046 sh.stream_npages = 0; 2047 hp->cpin = NULL; 2048 hp->cpout = NULL; 2049 hp->cperr = NULL; 2050 hp->in = 0; 2051 hp->out = 0; 2052 hp->bzstream.avail_in = 0; 2053 2054 /* Bump reference to mainq while we are running */ 2055 CQ_OPEN(mainq); 2056 2057 /* Get one page at a time */ 2058 while (dumpsys_sread(hp)) { 2059 if (sh.stream_pagenum != hp->cpin->pagenum) { 2060 sh.stream_pagenum = hp->cpin->pagenum; 2061 sh.stream_npages = btop(hp->cpin->used); 2062 dumpsys_bzrun(hp, &sh, sizeof (sh), BZ_RUN); 2063 } 2064 dumpsys_bzrun(hp, hp->page, PAGESIZE, 0); 2065 } 2066 2067 /* Done with input, flush any partial buffer */ 2068 if (sh.stream_pagenum != (pgcnt_t)-1) { 2069 dumpsys_bzrun(hp, NULL, 0, BZ_FINISH); 2070 dumpsys_errmsg(hp, NULL); 2071 } 2072 2073 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 2074 2075 /* Decrement main queue count, we are done */ 2076 CQ_CLOSE(mainq); 2077 } 2078 2079 /* 2080 * Compress with lzjb 2081 * write stream block if full or size==0 2082 * if csize==0 write stream header, else write <csize, data> 2083 * size==0 is a call to flush a buffer 2084 * hp->cpout is the buffer we are flushing or filling 2085 * hp->out is the next index to fill data 2086 * osize is either csize+data, or the size of a stream header 2087 */ 2088 static void 2089 dumpsys_lzjbrun(helper_t *hp, size_t csize, void *buf, size_t size) 2090 { 2091 dumpsync_t *ds = hp->ds; 2092 const int CSIZE = sizeof (dumpcsize_t); 2093 dumpcsize_t cs; 2094 size_t osize = csize > 0 ? CSIZE + size : size; 2095 2096 /* If flush, and there is no buffer, just return */ 2097 if (size == 0 && hp->cpout == NULL) 2098 return; 2099 2100 /* If flush, or cpout is full, write it out */ 2101 if (size == 0 || 2102 hp->cpout != NULL && hp->out + osize > hp->cpout->size) { 2103 2104 /* Set tag+size word at the front of the stream block. */ 2105 cs = DUMP_SET_TAG(hp->out - CSIZE, hp->tag); 2106 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 2107 2108 /* Write block to dump file. */ 2109 dumpsys_swrite(hp, hp->cpout, hp->out); 2110 2111 /* Clear pointer to indicate we need a new buffer */ 2112 hp->cpout = NULL; 2113 2114 /* flushing, we are done */ 2115 if (size == 0) 2116 return; 2117 } 2118 2119 /* Get an output buffer if we dont have one. */ 2120 if (hp->cpout == NULL) { 2121 HRSTART(hp->perpage, outwait); 2122 hp->cpout = CQ_GET(freebufq); 2123 HRSTOP(hp->perpage, outwait); 2124 hp->out = CSIZE; 2125 } 2126 2127 /* Store csize word. This is the size of compressed data. */ 2128 if (csize > 0) { 2129 cs = DUMP_SET_TAG(csize, 0); 2130 (void) memcpy(hp->cpout->buf + hp->out, &cs, CSIZE); 2131 hp->out += CSIZE; 2132 } 2133 2134 /* Store the data. */ 2135 (void) memcpy(hp->cpout->buf + hp->out, buf, size); 2136 hp->out += size; 2137 } 2138 2139 static void 2140 dumpsys_lzjbcompress(helper_t *hp) 2141 { 2142 dumpsync_t *ds = hp->ds; 2143 size_t csize; 2144 dumpstreamhdr_t sh; 2145 2146 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 2147 sh.stream_pagenum = (pfn_t)-1; 2148 sh.stream_npages = 0; 2149 hp->cpin = NULL; 2150 hp->cpout = NULL; 2151 hp->cperr = NULL; 2152 hp->in = 0; 2153 hp->out = 0; 2154 2155 /* Bump reference to mainq while we are running */ 2156 CQ_OPEN(mainq); 2157 2158 /* Get one page at a time */ 2159 while (dumpsys_sread(hp)) { 2160 2161 /* Create a stream header for each new input map */ 2162 if (sh.stream_pagenum != hp->cpin->pagenum) { 2163 sh.stream_pagenum = hp->cpin->pagenum; 2164 sh.stream_npages = btop(hp->cpin->used); 2165 dumpsys_lzjbrun(hp, 0, &sh, sizeof (sh)); 2166 } 2167 2168 /* Compress one page */ 2169 HRSTART(hp->perpage, compress); 2170 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2171 HRSTOP(hp->perpage, compress); 2172 2173 /* Add csize+data to output block */ 2174 ASSERT(csize > 0 && csize <= PAGESIZE); 2175 dumpsys_lzjbrun(hp, csize, hp->lzbuf, csize); 2176 } 2177 2178 /* Done with input, flush any partial buffer */ 2179 if (sh.stream_pagenum != (pfn_t)-1) { 2180 dumpsys_lzjbrun(hp, 0, NULL, 0); 2181 dumpsys_errmsg(hp, NULL); 2182 } 2183 2184 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 2185 2186 /* Decrement main queue count, we are done */ 2187 CQ_CLOSE(mainq); 2188 } 2189 2190 /* 2191 * Dump helper called from panic_idle() to compress pages. CPUs in 2192 * this path must not call most kernel services. 2193 * 2194 * During panic, all but one of the CPUs is idle. These CPUs are used 2195 * as helpers working in parallel to copy and compress memory 2196 * pages. During a panic, however, these processors cannot call any 2197 * kernel services. This is because mutexes become no-ops during 2198 * panic, and, cross-call interrupts are inhibited. Therefore, during 2199 * panic dump the helper CPUs communicate with the panic CPU using 2200 * memory variables. All memory mapping and I/O is performed by the 2201 * panic CPU. 2202 * 2203 * At dump configuration time, helper_lock is set and helpers_wanted 2204 * is 0. dumpsys() decides whether to set helpers_wanted before 2205 * clearing helper_lock. 2206 * 2207 * At panic time, idle CPUs spin-wait on helper_lock, then alternately 2208 * take the lock and become a helper, or return. 2209 */ 2210 void 2211 dumpsys_helper() 2212 { 2213 dumpsys_spinlock(&dumpcfg.helper_lock); 2214 if (dumpcfg.helpers_wanted) { 2215 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2216 2217 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2218 if (hp->helper == FREEHELPER) { 2219 hp->helper = CPU->cpu_id; 2220 BT_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2221 2222 dumpsys_spinunlock(&dumpcfg.helper_lock); 2223 2224 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2225 dumpsys_lzjbcompress(hp); 2226 else 2227 dumpsys_bz2compress(hp); 2228 2229 hp->helper = DONEHELPER; 2230 return; 2231 } 2232 } 2233 2234 /* No more helpers are needed. */ 2235 dumpcfg.helpers_wanted = 0; 2236 2237 } 2238 dumpsys_spinunlock(&dumpcfg.helper_lock); 2239 } 2240 2241 /* 2242 * No-wait helper callable in spin loops. 2243 * 2244 * Do not wait for helper_lock. Just check helpers_wanted. The caller 2245 * may decide to continue. This is the "c)ontinue, s)ync, r)eset? s" 2246 * case. 2247 */ 2248 void 2249 dumpsys_helper_nw() 2250 { 2251 if (dumpcfg.helpers_wanted) 2252 dumpsys_helper(); 2253 } 2254 2255 /* 2256 * Dump helper for live dumps. 2257 * These run as a system task. 2258 */ 2259 static void 2260 dumpsys_live_helper(void *arg) 2261 { 2262 helper_t *hp = arg; 2263 2264 BT_ATOMIC_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2265 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2266 dumpsys_lzjbcompress(hp); 2267 else 2268 dumpsys_bz2compress(hp); 2269 } 2270 2271 /* 2272 * Compress one page with lzjb (single threaded case) 2273 */ 2274 static void 2275 dumpsys_lzjb_page(helper_t *hp, cbuf_t *cp) 2276 { 2277 dumpsync_t *ds = hp->ds; 2278 uint32_t csize; 2279 2280 hp->helper = MAINHELPER; 2281 hp->in = 0; 2282 hp->used = 0; 2283 hp->cpin = cp; 2284 while (hp->used < cp->used) { 2285 HRSTART(hp->perpage, copy); 2286 hp->in = dumpsys_copy_page(hp, hp->in); 2287 hp->used += PAGESIZE; 2288 HRSTOP(hp->perpage, copy); 2289 2290 HRSTART(hp->perpage, compress); 2291 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2292 HRSTOP(hp->perpage, compress); 2293 2294 HRSTART(hp->perpage, write); 2295 dumpvp_write(&csize, sizeof (csize)); 2296 dumpvp_write(hp->lzbuf, csize); 2297 HRSTOP(hp->perpage, write); 2298 } 2299 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 2300 hp->cpin = NULL; 2301 } 2302 2303 /* 2304 * Main task to dump pages. This is called on the dump CPU. 2305 */ 2306 static void 2307 dumpsys_main_task(void *arg) 2308 { 2309 dumpsync_t *ds = arg; 2310 pgcnt_t pagenum = 0, bitnum = 0, hibitnum; 2311 dumpmlw_t mlw; 2312 cbuf_t *cp; 2313 pgcnt_t baseoff, pfnoff; 2314 pfn_t base, pfn; 2315 boolean_t dumpserial; 2316 int i; 2317 2318 /* 2319 * Fall back to serial mode if there are no helpers. 2320 * dump_plat_mincpu can be set to 0 at any time. 2321 * dumpcfg.helpermap must contain at least one member. 2322 * 2323 * It is possible that the helpers haven't registered 2324 * in helpermap yet; wait up to DUMP_HELPER_MAX_WAIT for 2325 * at least one helper to register. 2326 */ 2327 dumpserial = B_TRUE; 2328 if (dump_plat_mincpu != 0 && dumpcfg.clevel != 0) { 2329 hrtime_t hrtmax = MSEC2NSEC(DUMP_HELPER_MAX_WAIT); 2330 hrtime_t hrtstart = gethrtime(); 2331 2332 for (;;) { 2333 for (i = 0; i < BT_BITOUL(NCPU); ++i) { 2334 if (dumpcfg.helpermap[i] != 0) { 2335 dumpserial = B_FALSE; 2336 break; 2337 } 2338 } 2339 2340 if ((!dumpserial) || 2341 ((gethrtime() - hrtstart) >= hrtmax)) { 2342 break; 2343 } 2344 2345 SMT_PAUSE(); 2346 } 2347 2348 if (dumpserial) { 2349 dumpcfg.clevel = 0; 2350 if (dumpcfg.helper[0].lzbuf == NULL) { 2351 dumpcfg.helper[0].lzbuf = 2352 dumpcfg.helper[1].page; 2353 } 2354 } 2355 } 2356 2357 dump_init_memlist_walker(&mlw); 2358 2359 for (;;) { 2360 int sec = (gethrtime() - ds->start) / NANOSEC; 2361 2362 /* 2363 * Render a simple progress display on the system console to 2364 * make clear to the operator that the system has not hung. 2365 * Emit an update when dump progress has advanced by one 2366 * percent, or when no update has been drawn in the last 2367 * second. 2368 */ 2369 if (ds->percent > ds->percent_done || sec > ds->sec_done) { 2370 ds->sec_done = sec; 2371 ds->percent_done = ds->percent; 2372 uprintf("^\rdumping: %2d:%02d %3d%% done", 2373 sec / 60, sec % 60, ds->percent); 2374 ds->neednl = 1; 2375 } 2376 2377 while (CQ_IS_EMPTY(mainq) && !CQ_IS_EMPTY(writerq)) { 2378 2379 /* the writerq never blocks */ 2380 cp = CQ_GET(writerq); 2381 if (cp == NULL) 2382 break; 2383 2384 dump_timeleft = dump_timeout; 2385 2386 HRSTART(ds->perpage, write); 2387 dumpvp_write(cp->buf, cp->used); 2388 HRSTOP(ds->perpage, write); 2389 2390 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2391 } 2392 2393 /* 2394 * Wait here for some buffers to process. Returns NULL 2395 * when all helpers have terminated and all buffers 2396 * have been processed. 2397 */ 2398 cp = CQ_GET(mainq); 2399 2400 if (cp == NULL) { 2401 2402 /* Drain the write queue. */ 2403 if (!CQ_IS_EMPTY(writerq)) 2404 continue; 2405 2406 /* Main task exits here. */ 2407 break; 2408 } 2409 2410 dump_timeleft = dump_timeout; 2411 2412 switch (cp->state) { 2413 2414 case CBUF_FREEMAP: 2415 2416 /* 2417 * Note that we drop CBUF_FREEMAP buffers on 2418 * the floor (they will not be on any cqueue) 2419 * when we no longer need them. 2420 */ 2421 if (bitnum >= dumpcfg.bitmapsize) 2422 break; 2423 2424 if (dump_ioerr) { 2425 bitnum = dumpcfg.bitmapsize; 2426 CQ_CLOSE(helperq); 2427 break; 2428 } 2429 2430 HRSTART(ds->perpage, bitmap); 2431 for (; bitnum < dumpcfg.bitmapsize; bitnum++) 2432 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2433 break; 2434 HRSTOP(ds->perpage, bitmap); 2435 dump_timeleft = dump_timeout; 2436 2437 if (bitnum >= dumpcfg.bitmapsize) { 2438 CQ_CLOSE(helperq); 2439 break; 2440 } 2441 2442 /* 2443 * Try to map CBUF_MAPSIZE ranges. Can't 2444 * assume that memory segment size is a 2445 * multiple of CBUF_MAPSIZE. Can't assume that 2446 * the segment starts on a CBUF_MAPSIZE 2447 * boundary. 2448 */ 2449 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2450 ASSERT(pfn != PFN_INVALID); 2451 ASSERT(bitnum + mlw.mpleft <= dumpcfg.bitmapsize); 2452 2453 base = P2ALIGN(pfn, CBUF_MAPNP); 2454 if (base < mlw.mpaddr) { 2455 base = mlw.mpaddr; 2456 baseoff = P2PHASE(base, CBUF_MAPNP); 2457 } else { 2458 baseoff = 0; 2459 } 2460 2461 pfnoff = pfn - base; 2462 if (pfnoff + mlw.mpleft < CBUF_MAPNP) { 2463 hibitnum = bitnum + mlw.mpleft; 2464 cp->size = ptob(pfnoff + mlw.mpleft); 2465 } else { 2466 hibitnum = bitnum - pfnoff + CBUF_MAPNP - 2467 baseoff; 2468 cp->size = CBUF_MAPSIZE - ptob(baseoff); 2469 } 2470 2471 cp->pfn = pfn; 2472 cp->bitnum = bitnum++; 2473 cp->pagenum = pagenum++; 2474 cp->off = ptob(pfnoff); 2475 2476 for (; bitnum < hibitnum; bitnum++) 2477 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2478 pagenum++; 2479 2480 dump_timeleft = dump_timeout; 2481 cp->used = ptob(pagenum - cp->pagenum); 2482 2483 HRSTART(ds->perpage, map); 2484 hat_devload(kas.a_hat, cp->buf, cp->size, base, 2485 PROT_READ, HAT_LOAD_NOCONSIST); 2486 HRSTOP(ds->perpage, map); 2487 2488 ds->pages_mapped += btop(cp->size); 2489 ds->pages_used += pagenum - cp->pagenum; 2490 2491 CQ_OPEN(mainq); 2492 2493 /* 2494 * If there are no helpers the main task does 2495 * non-streams lzjb compress. 2496 */ 2497 if (dumpserial) { 2498 dumpsys_lzjb_page(dumpcfg.helper, cp); 2499 } else { 2500 /* pass mapped pages to a helper */ 2501 CQ_PUT(helperq, cp, CBUF_INREADY); 2502 } 2503 2504 /* the last page was done */ 2505 if (bitnum >= dumpcfg.bitmapsize) 2506 CQ_CLOSE(helperq); 2507 2508 break; 2509 2510 case CBUF_USEDMAP: 2511 2512 ds->npages += btop(cp->used); 2513 2514 HRSTART(ds->perpage, unmap); 2515 hat_unload(kas.a_hat, cp->buf, cp->size, HAT_UNLOAD); 2516 HRSTOP(ds->perpage, unmap); 2517 2518 if (bitnum < dumpcfg.bitmapsize) 2519 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2520 CQ_CLOSE(mainq); 2521 2522 ASSERT(ds->npages <= dumphdr->dump_npages); 2523 ds->percent = ds->npages * 100LL / dumphdr->dump_npages; 2524 break; 2525 2526 case CBUF_WRITE: 2527 2528 CQ_PUT(writerq, cp, CBUF_WRITE); 2529 break; 2530 2531 case CBUF_ERRMSG: 2532 2533 if (cp->used > 0) { 2534 cp->buf[cp->size - 2] = '\n'; 2535 cp->buf[cp->size - 1] = '\0'; 2536 if (ds->neednl) { 2537 uprintf("\n%s", cp->buf); 2538 ds->neednl = 0; 2539 } else { 2540 uprintf("%s", cp->buf); 2541 } 2542 /* wait for console output */ 2543 drv_usecwait(200000); 2544 dump_timeleft = dump_timeout; 2545 } 2546 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2547 break; 2548 2549 default: 2550 uprintf("dump: unexpected buffer state %d, " 2551 "buffer will be lost\n", cp->state); 2552 break; 2553 2554 } /* end switch */ 2555 } 2556 } 2557 2558 #ifdef COLLECT_METRICS 2559 size_t 2560 dumpsys_metrics(dumpsync_t *ds, char *buf, size_t size) 2561 { 2562 dumpcfg_t *cfg = &dumpcfg; 2563 int myid = CPU->cpu_seqid; 2564 int i, compress_ratio; 2565 int sec, iorate; 2566 helper_t *hp, *hpend = &cfg->helper[cfg->nhelper]; 2567 char *e = buf + size; 2568 char *p = buf; 2569 2570 sec = ds->elapsed / (1000 * 1000 * 1000ULL); 2571 if (sec < 1) 2572 sec = 1; 2573 2574 if (ds->iotime < 1) 2575 ds->iotime = 1; 2576 iorate = (ds->nwrite * 100000ULL) / ds->iotime; 2577 2578 compress_ratio = 100LL * ds->npages / btopr(ds->nwrite + 1); 2579 2580 #define P(...) (p += p < e ? snprintf(p, e - p, __VA_ARGS__) : 0) 2581 2582 P("Master cpu_seqid,%d\n", CPU->cpu_seqid); 2583 P("Master cpu_id,%d\n", CPU->cpu_id); 2584 P("dump_flags,0x%x\n", dumphdr->dump_flags); 2585 P("dump_ioerr,%d\n", dump_ioerr); 2586 2587 P("Helpers:\n"); 2588 for (i = 0; i < ncpus; i++) { 2589 if ((i & 15) == 0) 2590 P(",,%03d,", i); 2591 if (i == myid) 2592 P(" M"); 2593 else if (BT_TEST(cfg->helpermap, i)) 2594 P("%4d", cpu_seq[i]->cpu_id); 2595 else 2596 P(" *"); 2597 if ((i & 15) == 15) 2598 P("\n"); 2599 } 2600 2601 P("ncbuf_used,%d\n", cfg->ncbuf_used); 2602 P("ncmap,%d\n", cfg->ncmap); 2603 2604 P("Found %ldM ranges,%ld\n", (CBUF_MAPSIZE / DUMP_1MB), cfg->found4m); 2605 P("Found small pages,%ld\n", cfg->foundsm); 2606 2607 P("Compression level,%d\n", cfg->clevel); 2608 P("Compression type,%s %s", cfg->clevel == 0 ? "serial" : "parallel", 2609 cfg->clevel >= DUMP_CLEVEL_BZIP2 ? "bzip2" : "lzjb"); 2610 if (cfg->clevel >= DUMP_CLEVEL_BZIP2) 2611 P(" (level %d)\n", dump_bzip2_level); 2612 else 2613 P("\n"); 2614 P("Compression ratio,%d.%02d\n", compress_ratio / 100, compress_ratio % 2615 100); 2616 P("nhelper_used,%d\n", cfg->nhelper_used); 2617 2618 P("Dump I/O rate MBS,%d.%02d\n", iorate / 100, iorate % 100); 2619 P("..total bytes,%lld\n", (u_longlong_t)ds->nwrite); 2620 P("..total nsec,%lld\n", (u_longlong_t)ds->iotime); 2621 P("dumpbuf.iosize,%ld\n", dumpbuf.iosize); 2622 P("dumpbuf.size,%ld\n", dumpbuf.size); 2623 2624 P("Dump pages/sec,%llu\n", (u_longlong_t)ds->npages / sec); 2625 P("Dump pages,%llu\n", (u_longlong_t)ds->npages); 2626 P("Dump time,%d\n", sec); 2627 2628 if (ds->pages_mapped > 0) 2629 P("per-cent map utilization,%d\n", (int)((100 * ds->pages_used) 2630 / ds->pages_mapped)); 2631 2632 P("\nPer-page metrics:\n"); 2633 if (ds->npages > 0) { 2634 for (hp = cfg->helper; hp != hpend; hp++) { 2635 #define PERPAGE(x) ds->perpage.x += hp->perpage.x; 2636 PERPAGES; 2637 #undef PERPAGE 2638 } 2639 #define PERPAGE(x) \ 2640 P("%s nsec/page,%d\n", #x, (int)(ds->perpage.x / ds->npages)); 2641 PERPAGES; 2642 #undef PERPAGE 2643 P("freebufq.empty,%d\n", (int)(ds->freebufq.empty / 2644 ds->npages)); 2645 P("helperq.empty,%d\n", (int)(ds->helperq.empty / 2646 ds->npages)); 2647 P("writerq.empty,%d\n", (int)(ds->writerq.empty / 2648 ds->npages)); 2649 P("mainq.empty,%d\n", (int)(ds->mainq.empty / ds->npages)); 2650 2651 P("I/O wait nsec/page,%llu\n", (u_longlong_t)(ds->iowait / 2652 ds->npages)); 2653 } 2654 #undef P 2655 if (p < e) 2656 bzero(p, e - p); 2657 return (p - buf); 2658 } 2659 #endif /* COLLECT_METRICS */ 2660 2661 /* 2662 * Dump the system. 2663 */ 2664 void 2665 dumpsys(void) 2666 { 2667 dumpsync_t *ds = &dumpsync; 2668 taskq_t *livetaskq = NULL; 2669 pfn_t pfn; 2670 pgcnt_t bitnum; 2671 proc_t *p; 2672 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2673 cbuf_t *cp; 2674 pid_t npids, pidx; 2675 char *content; 2676 char *buf; 2677 size_t size; 2678 int save_dump_clevel; 2679 dumpmlw_t mlw; 2680 dumpcsize_t datatag; 2681 dumpdatahdr_t datahdr; 2682 2683 if (dumpvp == NULL || dumphdr == NULL) { 2684 uprintf("skipping system dump - no dump device configured\n"); 2685 if (panicstr) { 2686 dumpcfg.helpers_wanted = 0; 2687 dumpsys_spinunlock(&dumpcfg.helper_lock); 2688 } 2689 return; 2690 } 2691 dumpbuf.cur = dumpbuf.start; 2692 2693 /* clear the sync variables */ 2694 ASSERT(dumpcfg.nhelper > 0); 2695 bzero(ds, sizeof (*ds)); 2696 ds->dumpcpu = CPU->cpu_id; 2697 2698 /* 2699 * Calculate the starting block for dump. If we're dumping on a 2700 * swap device, start 1/5 of the way in; otherwise, start at the 2701 * beginning. And never use the first page -- it may be a disk label. 2702 */ 2703 if (dumpvp->v_flag & VISSWAP) 2704 dumphdr->dump_start = P2ROUNDUP(dumpvp_size / 5, DUMP_OFFSET); 2705 else 2706 dumphdr->dump_start = DUMP_OFFSET; 2707 2708 dumphdr->dump_flags = DF_VALID | DF_COMPLETE | DF_LIVE | DF_COMPRESSED; 2709 dumphdr->dump_crashtime = gethrestime_sec(); 2710 dumphdr->dump_npages = 0; 2711 dumphdr->dump_nvtop = 0; 2712 bzero(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.bitmapsize)); 2713 dump_timeleft = dump_timeout; 2714 2715 if (panicstr) { 2716 dumphdr->dump_flags &= ~DF_LIVE; 2717 (void) VOP_DUMPCTL(dumpvp, DUMP_FREE, NULL, NULL); 2718 (void) VOP_DUMPCTL(dumpvp, DUMP_ALLOC, NULL, NULL); 2719 (void) vsnprintf(dumphdr->dump_panicstring, DUMP_PANICSIZE, 2720 panicstr, panicargs); 2721 2722 } 2723 2724 if (dump_conflags & DUMP_ALL) 2725 content = "all"; 2726 else if (dump_conflags & DUMP_CURPROC) 2727 content = "kernel + curproc"; 2728 else 2729 content = "kernel"; 2730 uprintf("dumping to %s, offset %lld, content: %s\n", dumppath, 2731 dumphdr->dump_start, content); 2732 2733 /* Make sure nodename is current */ 2734 bcopy(utsname.nodename, dumphdr->dump_utsname.nodename, SYS_NMLN); 2735 2736 /* 2737 * If this is a live dump, try to open a VCHR vnode for better 2738 * performance. We must take care to flush the buffer cache 2739 * first. 2740 */ 2741 if (!panicstr) { 2742 vnode_t *cdev_vp, *cmn_cdev_vp; 2743 2744 ASSERT(dumpbuf.cdev_vp == NULL); 2745 cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR); 2746 if (cdev_vp != NULL) { 2747 cmn_cdev_vp = common_specvp(cdev_vp); 2748 if (VOP_OPEN(&cmn_cdev_vp, FREAD | FWRITE, kcred, NULL) 2749 == 0) { 2750 if (vn_has_cached_data(dumpvp)) 2751 (void) pvn_vplist_dirty(dumpvp, 0, NULL, 2752 B_INVAL | B_TRUNC, kcred); 2753 dumpbuf.cdev_vp = cmn_cdev_vp; 2754 } else { 2755 VN_RELE(cdev_vp); 2756 } 2757 } 2758 } 2759 2760 /* 2761 * Store a hires timestamp so we can look it up during debugging. 2762 */ 2763 lbolt_debug_entry(); 2764 2765 /* 2766 * Leave room for the message and ereport save areas and terminal dump 2767 * header. 2768 */ 2769 dumpbuf.vp_limit = dumpvp_size - DUMP_LOGSIZE - DUMP_OFFSET - 2770 DUMP_ERPTSIZE; 2771 2772 /* 2773 * Write out the symbol table. It's no longer compressed, 2774 * so its 'size' and 'csize' are equal. 2775 */ 2776 dumpbuf.vp_off = dumphdr->dump_ksyms = dumphdr->dump_start + PAGESIZE; 2777 dumphdr->dump_ksyms_size = dumphdr->dump_ksyms_csize = 2778 ksyms_snapshot(dumpvp_ksyms_write, NULL, LONG_MAX); 2779 2780 /* 2781 * Write out the translation map. 2782 */ 2783 dumphdr->dump_map = dumpvp_flush(); 2784 dump_as(&kas); 2785 dumphdr->dump_nvtop += dump_plat_addr(); 2786 2787 /* 2788 * call into hat, which may have unmapped pages that also need to 2789 * be in the dump 2790 */ 2791 hat_dump(); 2792 2793 if (dump_conflags & DUMP_ALL) { 2794 mutex_enter(&pidlock); 2795 2796 for (npids = 0, p = practive; p != NULL; p = p->p_next) 2797 dumpcfg.pids[npids++] = p->p_pid; 2798 2799 mutex_exit(&pidlock); 2800 2801 for (pidx = 0; pidx < npids; pidx++) 2802 (void) dump_process(dumpcfg.pids[pidx]); 2803 2804 dump_init_memlist_walker(&mlw); 2805 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2806 dump_timeleft = dump_timeout; 2807 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2808 /* 2809 * Some hypervisors do not have all pages available to 2810 * be accessed by the guest OS. Check for page 2811 * accessibility. 2812 */ 2813 if (plat_hold_page(pfn, PLAT_HOLD_NO_LOCK, NULL) != 2814 PLAT_HOLD_OK) 2815 continue; 2816 BT_SET(dumpcfg.bitmap, bitnum); 2817 } 2818 dumphdr->dump_npages = dumpcfg.bitmapsize; 2819 dumphdr->dump_flags |= DF_ALL; 2820 2821 } else if (dump_conflags & DUMP_CURPROC) { 2822 /* 2823 * Determine which pid is to be dumped. If we're panicking, we 2824 * dump the process associated with panic_thread (if any). If 2825 * this is a live dump, we dump the process associated with 2826 * curthread. 2827 */ 2828 npids = 0; 2829 if (panicstr) { 2830 if (panic_thread != NULL && 2831 panic_thread->t_procp != NULL && 2832 panic_thread->t_procp != &p0) { 2833 dumpcfg.pids[npids++] = 2834 panic_thread->t_procp->p_pid; 2835 } 2836 } else { 2837 dumpcfg.pids[npids++] = curthread->t_procp->p_pid; 2838 } 2839 2840 if (npids && dump_process(dumpcfg.pids[0]) == 0) 2841 dumphdr->dump_flags |= DF_CURPROC; 2842 else 2843 dumphdr->dump_flags |= DF_KERNEL; 2844 2845 } else { 2846 dumphdr->dump_flags |= DF_KERNEL; 2847 } 2848 2849 dumphdr->dump_hashmask = (1 << highbit(dumphdr->dump_nvtop - 1)) - 1; 2850 2851 /* 2852 * Write out the pfn table. 2853 */ 2854 dumphdr->dump_pfn = dumpvp_flush(); 2855 dump_init_memlist_walker(&mlw); 2856 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2857 dump_timeleft = dump_timeout; 2858 if (!BT_TEST(dumpcfg.bitmap, bitnum)) 2859 continue; 2860 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2861 ASSERT(pfn != PFN_INVALID); 2862 dumpvp_write(&pfn, sizeof (pfn_t)); 2863 } 2864 dump_plat_pfn(); 2865 2866 /* 2867 * Write out all the pages. 2868 * Map pages, copy them handling UEs, compress, and write them out. 2869 * Cooperate with any helpers running on CPUs in panic_idle(). 2870 */ 2871 dumphdr->dump_data = dumpvp_flush(); 2872 2873 bzero(dumpcfg.helpermap, BT_SIZEOFMAP(NCPU)); 2874 ds->live = dumpcfg.clevel > 0 && 2875 (dumphdr->dump_flags & DF_LIVE) != 0; 2876 2877 save_dump_clevel = dumpcfg.clevel; 2878 if (panicstr) 2879 dumpsys_get_maxmem(); 2880 else if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2881 dumpcfg.clevel = DUMP_CLEVEL_LZJB; 2882 2883 dumpcfg.nhelper_used = 0; 2884 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2885 if (hp->page == NULL) { 2886 hp->helper = DONEHELPER; 2887 continue; 2888 } 2889 ++dumpcfg.nhelper_used; 2890 hp->helper = FREEHELPER; 2891 hp->taskqid = 0; 2892 hp->ds = ds; 2893 bzero(&hp->perpage, sizeof (hp->perpage)); 2894 if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2895 (void) BZ2_bzCompressReset(&hp->bzstream); 2896 } 2897 2898 CQ_OPEN(freebufq); 2899 CQ_OPEN(helperq); 2900 2901 dumpcfg.ncbuf_used = 0; 2902 for (cp = dumpcfg.cbuf; cp != &dumpcfg.cbuf[dumpcfg.ncbuf]; cp++) { 2903 if (cp->buf != NULL) { 2904 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2905 ++dumpcfg.ncbuf_used; 2906 } 2907 } 2908 2909 for (cp = dumpcfg.cmap; cp != &dumpcfg.cmap[dumpcfg.ncmap]; cp++) 2910 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2911 2912 ds->start = gethrtime(); 2913 ds->iowaitts = ds->start; 2914 2915 /* start helpers */ 2916 if (ds->live) { 2917 int n = dumpcfg.nhelper_used; 2918 int pri = MINCLSYSPRI - 25; 2919 2920 livetaskq = taskq_create("LiveDump", n, pri, n, n, 2921 TASKQ_PREPOPULATE); 2922 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2923 if (hp->page == NULL) 2924 continue; 2925 hp->helper = hp - dumpcfg.helper; 2926 hp->taskqid = taskq_dispatch(livetaskq, 2927 dumpsys_live_helper, (void *)hp, TQ_NOSLEEP); 2928 } 2929 2930 } else { 2931 if (panicstr) 2932 kmem_dump_begin(); 2933 dumpcfg.helpers_wanted = dumpcfg.clevel > 0; 2934 dumpsys_spinunlock(&dumpcfg.helper_lock); 2935 } 2936 2937 /* run main task */ 2938 dumpsys_main_task(ds); 2939 2940 ds->elapsed = gethrtime() - ds->start; 2941 if (ds->elapsed < 1) 2942 ds->elapsed = 1; 2943 2944 if (livetaskq != NULL) 2945 taskq_destroy(livetaskq); 2946 2947 if (ds->neednl) { 2948 uprintf("\n"); 2949 ds->neednl = 0; 2950 } 2951 2952 /* record actual pages dumped */ 2953 dumphdr->dump_npages = ds->npages; 2954 2955 /* platform-specific data */ 2956 dumphdr->dump_npages += dump_plat_data(dumpcfg.cbuf[0].buf); 2957 2958 /* note any errors by clearing DF_COMPLETE */ 2959 if (dump_ioerr || ds->npages < dumphdr->dump_npages) 2960 dumphdr->dump_flags &= ~DF_COMPLETE; 2961 2962 /* end of stream blocks */ 2963 datatag = 0; 2964 dumpvp_write(&datatag, sizeof (datatag)); 2965 2966 bzero(&datahdr, sizeof (datahdr)); 2967 2968 /* buffer for metrics */ 2969 buf = dumpcfg.cbuf[0].buf; 2970 size = MIN(dumpcfg.cbuf[0].size, DUMP_OFFSET - sizeof (dumphdr_t) - 2971 sizeof (dumpdatahdr_t)); 2972 2973 /* finish the kmem intercepts, collect kmem verbose info */ 2974 if (panicstr) { 2975 datahdr.dump_metrics = kmem_dump_finish(buf, size); 2976 buf += datahdr.dump_metrics; 2977 size -= datahdr.dump_metrics; 2978 } 2979 2980 /* record in the header whether this is a fault-management panic */ 2981 if (panicstr) 2982 dumphdr->dump_fm_panic = is_fm_panic(); 2983 2984 /* compression info in data header */ 2985 datahdr.dump_datahdr_magic = DUMP_DATAHDR_MAGIC; 2986 datahdr.dump_datahdr_version = DUMP_DATAHDR_VERSION; 2987 datahdr.dump_maxcsize = CBUF_SIZE; 2988 datahdr.dump_maxrange = CBUF_MAPSIZE / PAGESIZE; 2989 datahdr.dump_nstreams = dumpcfg.nhelper_used; 2990 datahdr.dump_clevel = dumpcfg.clevel; 2991 #ifdef COLLECT_METRICS 2992 if (dump_metrics_on) 2993 datahdr.dump_metrics += dumpsys_metrics(ds, buf, size); 2994 #endif 2995 datahdr.dump_data_csize = dumpvp_flush() - dumphdr->dump_data; 2996 2997 /* 2998 * Write out the initial and terminal dump headers. 2999 */ 3000 dumpbuf.vp_off = dumphdr->dump_start; 3001 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 3002 (void) dumpvp_flush(); 3003 3004 dumpbuf.vp_limit = dumpvp_size; 3005 dumpbuf.vp_off = dumpbuf.vp_limit - DUMP_OFFSET; 3006 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 3007 dumpvp_write(&datahdr, sizeof (dumpdatahdr_t)); 3008 dumpvp_write(dumpcfg.cbuf[0].buf, datahdr.dump_metrics); 3009 3010 (void) dumpvp_flush(); 3011 3012 uprintf("\r%3d%% done: %llu pages dumped, ", 3013 ds->percent_done, (u_longlong_t)ds->npages); 3014 3015 if (dump_ioerr == 0) { 3016 uprintf("dump succeeded\n"); 3017 } else { 3018 uprintf("dump failed: error %d\n", dump_ioerr); 3019 #ifdef DEBUG 3020 if (panicstr) 3021 debug_enter("dump failed"); 3022 #endif 3023 } 3024 3025 /* 3026 * Write out all undelivered messages. This has to be the *last* 3027 * thing we do because the dump process itself emits messages. 3028 */ 3029 if (panicstr) { 3030 dump_summary(); 3031 dump_ereports(); 3032 dump_messages(); 3033 } 3034 3035 delay(2 * hz); /* let people see the 'done' message */ 3036 dump_timeleft = 0; 3037 dump_ioerr = 0; 3038 3039 /* restore settings after live dump completes */ 3040 if (!panicstr) { 3041 dumpcfg.clevel = save_dump_clevel; 3042 3043 /* release any VCHR open of the dump device */ 3044 if (dumpbuf.cdev_vp != NULL) { 3045 (void) VOP_CLOSE(dumpbuf.cdev_vp, FREAD | FWRITE, 1, 0, 3046 kcred, NULL); 3047 VN_RELE(dumpbuf.cdev_vp); 3048 dumpbuf.cdev_vp = NULL; 3049 } 3050 } 3051 } 3052 3053 /* 3054 * This function is called whenever the memory size, as represented 3055 * by the phys_install list, changes. 3056 */ 3057 void 3058 dump_resize() 3059 { 3060 mutex_enter(&dump_lock); 3061 dumphdr_init(); 3062 dumpbuf_resize(); 3063 dump_update_clevel(); 3064 mutex_exit(&dump_lock); 3065 } 3066 3067 /* 3068 * This function allows for dynamic resizing of a dump area. It assumes that 3069 * the underlying device has update its appropriate size(9P). 3070 */ 3071 int 3072 dumpvp_resize() 3073 { 3074 int error; 3075 vattr_t vattr; 3076 3077 mutex_enter(&dump_lock); 3078 vattr.va_mask = AT_SIZE; 3079 if ((error = VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL)) != 0) { 3080 mutex_exit(&dump_lock); 3081 return (error); 3082 } 3083 3084 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) { 3085 mutex_exit(&dump_lock); 3086 return (ENOSPC); 3087 } 3088 3089 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 3090 mutex_exit(&dump_lock); 3091 return (0); 3092 } 3093 3094 int 3095 dump_set_uuid(const char *uuidstr) 3096 { 3097 const char *ptr; 3098 int i; 3099 3100 if (uuidstr == NULL || strnlen(uuidstr, 36 + 1) != 36) 3101 return (EINVAL); 3102 3103 /* uuid_parse is not common code so check manually */ 3104 for (i = 0, ptr = uuidstr; i < 36; i++, ptr++) { 3105 switch (i) { 3106 case 8: 3107 case 13: 3108 case 18: 3109 case 23: 3110 if (*ptr != '-') 3111 return (EINVAL); 3112 break; 3113 3114 default: 3115 if (!isxdigit(*ptr)) 3116 return (EINVAL); 3117 break; 3118 } 3119 } 3120 3121 if (dump_osimage_uuid[0] != '\0') 3122 return (EALREADY); 3123 3124 (void) strncpy(dump_osimage_uuid, uuidstr, 36 + 1); 3125 3126 cmn_err(CE_CONT, "?This illumos instance has UUID %s\n", 3127 dump_osimage_uuid); 3128 3129 return (0); 3130 } 3131 3132 const char * 3133 dump_get_uuid(void) 3134 { 3135 return (dump_osimage_uuid[0] != '\0' ? dump_osimage_uuid : ""); 3136 } 3137