1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/param.h> 28 #include <sys/systm.h> 29 #include <sys/vm.h> 30 #include <sys/proc.h> 31 #include <sys/file.h> 32 #include <sys/conf.h> 33 #include <sys/kmem.h> 34 #include <sys/mem.h> 35 #include <sys/mman.h> 36 #include <sys/vnode.h> 37 #include <sys/errno.h> 38 #include <sys/memlist.h> 39 #include <sys/dumphdr.h> 40 #include <sys/dumpadm.h> 41 #include <sys/ksyms.h> 42 #include <sys/compress.h> 43 #include <sys/stream.h> 44 #include <sys/strsun.h> 45 #include <sys/cmn_err.h> 46 #include <sys/bitmap.h> 47 #include <sys/modctl.h> 48 #include <sys/utsname.h> 49 #include <sys/systeminfo.h> 50 #include <sys/vmem.h> 51 #include <sys/log.h> 52 #include <sys/var.h> 53 #include <sys/debug.h> 54 #include <sys/sunddi.h> 55 #include <fs/fs_subr.h> 56 #include <sys/fs/snode.h> 57 #include <sys/ontrap.h> 58 #include <sys/panic.h> 59 #include <sys/dkio.h> 60 #include <sys/vtoc.h> 61 #include <sys/errorq.h> 62 #include <sys/fm/util.h> 63 #include <sys/fs/zfs.h> 64 65 #include <vm/hat.h> 66 #include <vm/as.h> 67 #include <vm/page.h> 68 #include <vm/pvn.h> 69 #include <vm/seg.h> 70 #include <vm/seg_kmem.h> 71 #include <sys/clock_impl.h> 72 #include <sys/hold_page.h> 73 74 #include <bzip2/bzlib.h> 75 76 /* 77 * Crash dump time is dominated by disk write time. To reduce this, 78 * the stronger compression method bzip2 is applied to reduce the dump 79 * size and hence reduce I/O time. However, bzip2 is much more 80 * computationally expensive than the existing lzjb algorithm, so to 81 * avoid increasing compression time, CPUs that are otherwise idle 82 * during panic are employed to parallelize the compression task. 83 * Many helper CPUs are needed to prevent bzip2 from being a 84 * bottleneck, and on systems with too few CPUs, the lzjb algorithm is 85 * parallelized instead. Lastly, I/O and compression are performed by 86 * different CPUs, and are hence overlapped in time, unlike the older 87 * serial code. 88 * 89 * Another important consideration is the speed of the dump 90 * device. Faster disks need less CPUs in order to benefit from 91 * parallel lzjb versus parallel bzip2. Therefore, the CPU count 92 * threshold for switching from parallel lzjb to paralled bzip2 is 93 * elevated for faster disks. The dump device speed is adduced from 94 * the setting for dumpbuf.iosize, see dump_update_clevel. 95 */ 96 97 /* 98 * exported vars 99 */ 100 kmutex_t dump_lock; /* lock for dump configuration */ 101 dumphdr_t *dumphdr; /* dump header */ 102 int dump_conflags = DUMP_KERNEL; /* dump configuration flags */ 103 vnode_t *dumpvp; /* dump device vnode pointer */ 104 u_offset_t dumpvp_size; /* size of dump device, in bytes */ 105 char *dumppath; /* pathname of dump device */ 106 int dump_timeout = 120; /* timeout for dumping pages */ 107 int dump_timeleft; /* portion of dump_timeout remaining */ 108 int dump_ioerr; /* dump i/o error */ 109 int dump_check_used; /* enable check for used pages */ 110 char *dump_stack_scratch; /* scratch area for saving stack summary */ 111 112 /* 113 * Tunables for dump compression and parallelism. These can be set via 114 * /etc/system. 115 * 116 * dump_ncpu_low number of helpers for parallel lzjb 117 * This is also the minimum configuration. 118 * 119 * dump_bzip2_level bzip2 compression level: 1-9 120 * Higher numbers give greater compression, but take more memory 121 * and time. Memory used per helper is ~(dump_bzip2_level * 1MB). 122 * 123 * dump_plat_mincpu the cross-over limit for using bzip2 (per platform): 124 * if dump_plat_mincpu == 0, then always do single threaded dump 125 * if ncpu >= dump_plat_mincpu then try to use bzip2 126 * 127 * dump_metrics_on if set, metrics are collected in the kernel, passed 128 * to savecore via the dump file, and recorded by savecore in 129 * METRICS.txt. 130 */ 131 uint_t dump_ncpu_low = 4; /* minimum config for parallel lzjb */ 132 uint_t dump_bzip2_level = 1; /* bzip2 level (1-9) */ 133 134 /* Use dump_plat_mincpu_default unless this variable is set by /etc/system */ 135 #define MINCPU_NOT_SET ((uint_t)-1) 136 uint_t dump_plat_mincpu = MINCPU_NOT_SET; 137 138 /* tunables for pre-reserved heap */ 139 uint_t dump_kmem_permap = 1024; 140 uint_t dump_kmem_pages = 8; 141 142 /* Define multiple buffers per helper to avoid stalling */ 143 #define NCBUF_PER_HELPER 2 144 #define NCMAP_PER_HELPER 4 145 146 /* minimum number of helpers configured */ 147 #define MINHELPERS (dump_ncpu_low) 148 #define MINCBUFS (MINHELPERS * NCBUF_PER_HELPER) 149 150 /* 151 * Define constant parameters. 152 * 153 * CBUF_SIZE size of an output buffer 154 * 155 * CBUF_MAPSIZE size of virtual range for mapping pages 156 * 157 * CBUF_MAPNP size of virtual range in pages 158 * 159 */ 160 #define DUMP_1KB ((size_t)1 << 10) 161 #define DUMP_1MB ((size_t)1 << 20) 162 #define CBUF_SIZE ((size_t)1 << 17) 163 #define CBUF_MAPSHIFT (22) 164 #define CBUF_MAPSIZE ((size_t)1 << CBUF_MAPSHIFT) 165 #define CBUF_MAPNP ((size_t)1 << (CBUF_MAPSHIFT - PAGESHIFT)) 166 167 /* 168 * Compression metrics are accumulated nano-second subtotals. The 169 * results are normalized by the number of pages dumped. A report is 170 * generated when dumpsys() completes and is saved in the dump image 171 * after the trailing dump header. 172 * 173 * Metrics are always collected. Set the variable dump_metrics_on to 174 * cause metrics to be saved in the crash file, where savecore will 175 * save it in the file METRICS.txt. 176 */ 177 #define PERPAGES \ 178 PERPAGE(bitmap) PERPAGE(map) PERPAGE(unmap) \ 179 PERPAGE(copy) PERPAGE(compress) \ 180 PERPAGE(write) \ 181 PERPAGE(inwait) PERPAGE(outwait) 182 183 typedef struct perpage { 184 #define PERPAGE(x) hrtime_t x; 185 PERPAGES 186 #undef PERPAGE 187 } perpage_t; 188 189 /* 190 * This macro controls the code generation for collecting dump 191 * performance information. By default, the code is generated, but 192 * automatic saving of the information is disabled. If dump_metrics_on 193 * is set to 1, the timing information is passed to savecore via the 194 * crash file, where it is appended to the file dump-dir/METRICS.txt. 195 */ 196 #define COLLECT_METRICS 197 198 #ifdef COLLECT_METRICS 199 uint_t dump_metrics_on = 0; /* set to 1 to enable recording metrics */ 200 201 #define HRSTART(v, m) v##ts.m = gethrtime() 202 #define HRSTOP(v, m) v.m += gethrtime() - v##ts.m 203 #define HRBEGIN(v, m, s) v##ts.m = gethrtime(); v.size += s 204 #define HREND(v, m) v.m += gethrtime() - v##ts.m 205 #define HRNORM(v, m, n) v.m /= (n) 206 207 #else 208 #define HRSTART(v, m) 209 #define HRSTOP(v, m) 210 #define HRBEGIN(v, m, s) 211 #define HREND(v, m) 212 #define HRNORM(v, m, n) 213 #endif /* COLLECT_METRICS */ 214 215 /* 216 * Buffers for copying and compressing memory pages. 217 * 218 * cbuf_t buffer controllers: used for both input and output. 219 * 220 * The buffer state indicates how it is being used: 221 * 222 * CBUF_FREEMAP: CBUF_MAPSIZE virtual address range is available for 223 * mapping input pages. 224 * 225 * CBUF_INREADY: input pages are mapped and ready for compression by a 226 * helper. 227 * 228 * CBUF_USEDMAP: mapping has been consumed by a helper. Needs unmap. 229 * 230 * CBUF_FREEBUF: CBUF_SIZE output buffer, which is available. 231 * 232 * CBUF_WRITE: CBUF_SIZE block of compressed pages from a helper, 233 * ready to write out. 234 * 235 * CBUF_ERRMSG: CBUF_SIZE block of error messages from a helper 236 * (reports UE errors.) 237 */ 238 239 typedef enum cbufstate { 240 CBUF_FREEMAP, 241 CBUF_INREADY, 242 CBUF_USEDMAP, 243 CBUF_FREEBUF, 244 CBUF_WRITE, 245 CBUF_ERRMSG 246 } cbufstate_t; 247 248 typedef struct cbuf cbuf_t; 249 250 struct cbuf { 251 cbuf_t *next; /* next in list */ 252 cbufstate_t state; /* processing state */ 253 size_t used; /* amount used */ 254 size_t size; /* mem size */ 255 char *buf; /* kmem or vmem */ 256 pgcnt_t pagenum; /* index to pfn map */ 257 pgcnt_t bitnum; /* first set bitnum */ 258 pfn_t pfn; /* first pfn in mapped range */ 259 int off; /* byte offset to first pfn */ 260 }; 261 262 static char dump_osimage_uuid[36 + 1]; 263 264 #define isdigit(ch) ((ch) >= '0' && (ch) <= '9') 265 #define isxdigit(ch) (isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \ 266 ((ch) >= 'A' && (ch) <= 'F')) 267 268 /* 269 * cqueue_t queues: a uni-directional channel for communication 270 * from the master to helper tasks or vice-versa using put and 271 * get primitives. Both mappings and data buffers are passed via 272 * queues. Producers close a queue when done. The number of 273 * active producers is reference counted so the consumer can 274 * detect end of data. Concurrent access is mediated by atomic 275 * operations for panic dump, or mutex/cv for live dump. 276 * 277 * There a four queues, used as follows: 278 * 279 * Queue Dataflow NewState 280 * -------------------------------------------------- 281 * mainq master -> master FREEMAP 282 * master has initialized or unmapped an input buffer 283 * -------------------------------------------------- 284 * helperq master -> helper INREADY 285 * master has mapped input for use by helper 286 * -------------------------------------------------- 287 * mainq master <- helper USEDMAP 288 * helper is done with input 289 * -------------------------------------------------- 290 * freebufq master -> helper FREEBUF 291 * master has initialized or written an output buffer 292 * -------------------------------------------------- 293 * mainq master <- helper WRITE 294 * block of compressed pages from a helper 295 * -------------------------------------------------- 296 * mainq master <- helper ERRMSG 297 * error messages from a helper (memory error case) 298 * -------------------------------------------------- 299 * writerq master <- master WRITE 300 * non-blocking queue of blocks to write 301 * -------------------------------------------------- 302 */ 303 typedef struct cqueue { 304 cbuf_t *volatile first; /* first in list */ 305 cbuf_t *last; /* last in list */ 306 hrtime_t ts; /* timestamp */ 307 hrtime_t empty; /* total time empty */ 308 kmutex_t mutex; /* live state lock */ 309 kcondvar_t cv; /* live wait var */ 310 lock_t spinlock; /* panic mode spin lock */ 311 volatile uint_t open; /* producer ref count */ 312 } cqueue_t; 313 314 /* 315 * Convenience macros for using the cqueue functions 316 * Note that the caller must have defined "dumpsync_t *ds" 317 */ 318 #define CQ_IS_EMPTY(q) \ 319 (ds->q.first == NULL) 320 321 #define CQ_OPEN(q) \ 322 atomic_inc_uint(&ds->q.open) 323 324 #define CQ_CLOSE(q) \ 325 dumpsys_close_cq(&ds->q, ds->live) 326 327 #define CQ_PUT(q, cp, st) \ 328 dumpsys_put_cq(&ds->q, cp, st, ds->live) 329 330 #define CQ_GET(q) \ 331 dumpsys_get_cq(&ds->q, ds->live) 332 333 /* 334 * Dynamic state when dumpsys() is running. 335 */ 336 typedef struct dumpsync { 337 pgcnt_t npages; /* subtotal of pages dumped */ 338 pgcnt_t pages_mapped; /* subtotal of pages mapped */ 339 pgcnt_t pages_used; /* subtotal of pages used per map */ 340 size_t nwrite; /* subtotal of bytes written */ 341 uint_t live; /* running live dump */ 342 uint_t neednl; /* will need to print a newline */ 343 uint_t percent; /* dump progress */ 344 uint_t percent_done; /* dump progress reported */ 345 cqueue_t freebufq; /* free kmem bufs for writing */ 346 cqueue_t mainq; /* input for main task */ 347 cqueue_t helperq; /* input for helpers */ 348 cqueue_t writerq; /* input for writer */ 349 hrtime_t start; /* start time */ 350 hrtime_t elapsed; /* elapsed time when completed */ 351 hrtime_t iotime; /* time spent writing nwrite bytes */ 352 hrtime_t iowait; /* time spent waiting for output */ 353 hrtime_t iowaitts; /* iowait timestamp */ 354 perpage_t perpage; /* metrics */ 355 perpage_t perpagets; 356 int dumpcpu; /* master cpu */ 357 } dumpsync_t; 358 359 static dumpsync_t dumpsync; /* synchronization vars */ 360 361 /* 362 * helper_t helpers: contains the context for a stream. CPUs run in 363 * parallel at dump time; each CPU creates a single stream of 364 * compression data. Stream data is divided into CBUF_SIZE blocks. 365 * The blocks are written in order within a stream. But, blocks from 366 * multiple streams can be interleaved. Each stream is identified by a 367 * unique tag. 368 */ 369 typedef struct helper { 370 int helper; /* bound helper id */ 371 int tag; /* compression stream tag */ 372 perpage_t perpage; /* per page metrics */ 373 perpage_t perpagets; /* per page metrics (timestamps) */ 374 taskqid_t taskqid; /* live dump task ptr */ 375 int in, out; /* buffer offsets */ 376 cbuf_t *cpin, *cpout, *cperr; /* cbuf objects in process */ 377 dumpsync_t *ds; /* pointer to sync vars */ 378 size_t used; /* counts input consumed */ 379 char *page; /* buffer for page copy */ 380 char *lzbuf; /* lzjb output */ 381 bz_stream bzstream; /* bzip2 state */ 382 } helper_t; 383 384 #define MAINHELPER (-1) /* helper is also the main task */ 385 #define FREEHELPER (-2) /* unbound helper */ 386 #define DONEHELPER (-3) /* helper finished */ 387 388 /* 389 * configuration vars for dumpsys 390 */ 391 typedef struct dumpcfg { 392 int threshold; /* ncpu threshold for bzip2 */ 393 int nhelper; /* number of helpers */ 394 int nhelper_used; /* actual number of helpers used */ 395 int ncmap; /* number VA pages for compression */ 396 int ncbuf; /* number of bufs for compression */ 397 int ncbuf_used; /* number of bufs in use */ 398 uint_t clevel; /* dump compression level */ 399 helper_t *helper; /* array of helpers */ 400 cbuf_t *cmap; /* array of input (map) buffers */ 401 cbuf_t *cbuf; /* array of output buffers */ 402 ulong_t *helpermap; /* set of dumpsys helper CPU ids */ 403 ulong_t *bitmap; /* bitmap for marking pages to dump */ 404 ulong_t *rbitmap; /* bitmap for used CBUF_MAPSIZE ranges */ 405 pgcnt_t bitmapsize; /* size of bitmap */ 406 pgcnt_t rbitmapsize; /* size of bitmap for ranges */ 407 pgcnt_t found4m; /* number ranges allocated by dump */ 408 pgcnt_t foundsm; /* number small pages allocated by dump */ 409 pid_t *pids; /* list of process IDs at dump time */ 410 size_t maxsize; /* memory size needed at dump time */ 411 size_t maxvmsize; /* size of reserved VM */ 412 char *maxvm; /* reserved VM for spare pages */ 413 lock_t helper_lock; /* protect helper state */ 414 char helpers_wanted; /* flag to enable parallelism */ 415 } dumpcfg_t; 416 417 static dumpcfg_t dumpcfg; /* config vars */ 418 419 /* 420 * The dump I/O buffer. 421 * 422 * There is one I/O buffer used by dumpvp_write and dumvp_flush. It is 423 * sized according to the optimum device transfer speed. 424 */ 425 typedef struct dumpbuf { 426 vnode_t *cdev_vp; /* VCHR open of the dump device */ 427 len_t vp_limit; /* maximum write offset */ 428 offset_t vp_off; /* current dump device offset */ 429 char *cur; /* dump write pointer */ 430 char *start; /* dump buffer address */ 431 char *end; /* dump buffer end */ 432 size_t size; /* size of dumpbuf in bytes */ 433 size_t iosize; /* best transfer size for device */ 434 } dumpbuf_t; 435 436 dumpbuf_t dumpbuf; /* I/O buffer */ 437 438 /* 439 * The dump I/O buffer must be at least one page, at most xfer_size 440 * bytes, and should scale with physmem in between. The transfer size 441 * passed in will either represent a global default (maxphys) or the 442 * best size for the device. The size of the dumpbuf I/O buffer is 443 * limited by dumpbuf_limit (8MB by default) because the dump 444 * performance saturates beyond a certain size. The default is to 445 * select 1/4096 of the memory. 446 */ 447 static int dumpbuf_fraction = 12; /* memory size scale factor */ 448 static size_t dumpbuf_limit = 8 * DUMP_1MB; /* max I/O buf size */ 449 450 static size_t 451 dumpbuf_iosize(size_t xfer_size) 452 { 453 size_t iosize = ptob(physmem >> dumpbuf_fraction); 454 455 if (iosize < PAGESIZE) 456 iosize = PAGESIZE; 457 else if (iosize > xfer_size) 458 iosize = xfer_size; 459 if (iosize > dumpbuf_limit) 460 iosize = dumpbuf_limit; 461 return (iosize & PAGEMASK); 462 } 463 464 /* 465 * resize the I/O buffer 466 */ 467 static void 468 dumpbuf_resize(void) 469 { 470 char *old_buf = dumpbuf.start; 471 size_t old_size = dumpbuf.size; 472 char *new_buf; 473 size_t new_size; 474 475 ASSERT(MUTEX_HELD(&dump_lock)); 476 477 new_size = dumpbuf_iosize(MAX(dumpbuf.iosize, maxphys)); 478 if (new_size <= old_size) 479 return; /* no need to reallocate buffer */ 480 481 new_buf = kmem_alloc(new_size, KM_SLEEP); 482 dumpbuf.size = new_size; 483 dumpbuf.start = new_buf; 484 dumpbuf.end = new_buf + new_size; 485 kmem_free(old_buf, old_size); 486 } 487 488 /* 489 * dump_update_clevel is called when dumpadm configures the dump device. 490 * Calculate number of helpers and buffers. 491 * Allocate the minimum configuration for now. 492 * 493 * When the dump file is configured we reserve a minimum amount of 494 * memory for use at crash time. But we reserve VA for all the memory 495 * we really want in order to do the fastest dump possible. The VA is 496 * backed by pages not being dumped, according to the bitmap. If 497 * there is insufficient spare memory, however, we fall back to the 498 * minimum. 499 * 500 * Live dump (savecore -L) always uses the minimum config. 501 * 502 * clevel 0 is single threaded lzjb 503 * clevel 1 is parallel lzjb 504 * clevel 2 is parallel bzip2 505 * 506 * The ncpu threshold is selected with dump_plat_mincpu. 507 * On OPL, set_platform_defaults() overrides the sun4u setting. 508 * The actual values are defined via DUMP_PLAT_*_MINCPU macros. 509 * 510 * Architecture Threshold Algorithm 511 * sun4u < 51 parallel lzjb 512 * sun4u >= 51 parallel bzip2(*) 513 * sun4u OPL < 8 parallel lzjb 514 * sun4u OPL >= 8 parallel bzip2(*) 515 * sun4v < 128 parallel lzjb 516 * sun4v >= 128 parallel bzip2(*) 517 * x86 < 11 parallel lzjb 518 * x86 >= 11 parallel bzip2(*) 519 * 32-bit N/A single-threaded lzjb 520 * 521 * (*) bzip2 is only chosen if there is sufficient available 522 * memory for buffers at dump time. See dumpsys_get_maxmem(). 523 * 524 * Faster dump devices have larger I/O buffers. The threshold value is 525 * increased according to the size of the dump I/O buffer, because 526 * parallel lzjb performs better with faster disks. For buffers >= 1MB 527 * the threshold is 3X; for buffers >= 256K threshold is 2X. 528 * 529 * For parallel dumps, the number of helpers is ncpu-1. The CPU 530 * running panic runs the main task. For single-threaded dumps, the 531 * panic CPU does lzjb compression (it is tagged as MAINHELPER.) 532 * 533 * Need multiple buffers per helper so that they do not block waiting 534 * for the main task. 535 * parallel single-threaded 536 * Number of output buffers: nhelper*2 1 537 * Number of mapping buffers: nhelper*4 1 538 * 539 */ 540 static void 541 dump_update_clevel() 542 { 543 int tag; 544 size_t bz2size; 545 helper_t *hp, *hpend; 546 cbuf_t *cp, *cpend; 547 dumpcfg_t *old = &dumpcfg; 548 dumpcfg_t newcfg = *old; 549 dumpcfg_t *new = &newcfg; 550 551 ASSERT(MUTEX_HELD(&dump_lock)); 552 553 /* 554 * Free the previously allocated bufs and VM. 555 */ 556 if (old->helper != NULL) { 557 558 /* helpers */ 559 hpend = &old->helper[old->nhelper]; 560 for (hp = old->helper; hp != hpend; hp++) { 561 if (hp->lzbuf != NULL) 562 kmem_free(hp->lzbuf, PAGESIZE); 563 if (hp->page != NULL) 564 kmem_free(hp->page, PAGESIZE); 565 } 566 kmem_free(old->helper, old->nhelper * sizeof (helper_t)); 567 568 /* VM space for mapping pages */ 569 cpend = &old->cmap[old->ncmap]; 570 for (cp = old->cmap; cp != cpend; cp++) 571 vmem_xfree(heap_arena, cp->buf, CBUF_MAPSIZE); 572 kmem_free(old->cmap, old->ncmap * sizeof (cbuf_t)); 573 574 /* output bufs */ 575 cpend = &old->cbuf[old->ncbuf]; 576 for (cp = old->cbuf; cp != cpend; cp++) 577 if (cp->buf != NULL) 578 kmem_free(cp->buf, cp->size); 579 kmem_free(old->cbuf, old->ncbuf * sizeof (cbuf_t)); 580 581 /* reserved VM for dumpsys_get_maxmem */ 582 if (old->maxvmsize > 0) 583 vmem_xfree(heap_arena, old->maxvm, old->maxvmsize); 584 } 585 586 /* 587 * Allocate memory and VM. 588 * One CPU runs dumpsys, the rest are helpers. 589 */ 590 new->nhelper = ncpus - 1; 591 if (new->nhelper < 1) 592 new->nhelper = 1; 593 594 if (new->nhelper > DUMP_MAX_NHELPER) 595 new->nhelper = DUMP_MAX_NHELPER; 596 597 /* use platform default, unless /etc/system overrides */ 598 if (dump_plat_mincpu == MINCPU_NOT_SET) 599 dump_plat_mincpu = dump_plat_mincpu_default; 600 601 /* increase threshold for faster disks */ 602 new->threshold = dump_plat_mincpu; 603 if (dumpbuf.iosize >= DUMP_1MB) 604 new->threshold *= 3; 605 else if (dumpbuf.iosize >= (256 * DUMP_1KB)) 606 new->threshold *= 2; 607 608 /* figure compression level based upon the computed threshold. */ 609 if (dump_plat_mincpu == 0 || new->nhelper < 2) { 610 new->clevel = 0; 611 new->nhelper = 1; 612 } else if ((new->nhelper + 1) >= new->threshold) { 613 new->clevel = DUMP_CLEVEL_BZIP2; 614 } else { 615 new->clevel = DUMP_CLEVEL_LZJB; 616 } 617 618 if (new->clevel == 0) { 619 new->ncbuf = 1; 620 new->ncmap = 1; 621 } else { 622 new->ncbuf = NCBUF_PER_HELPER * new->nhelper; 623 new->ncmap = NCMAP_PER_HELPER * new->nhelper; 624 } 625 626 /* 627 * Allocate new data structures and buffers for MINHELPERS, 628 * and also figure the max desired size. 629 */ 630 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 631 new->maxsize = 0; 632 new->maxvmsize = 0; 633 new->maxvm = NULL; 634 tag = 1; 635 new->helper = kmem_zalloc(new->nhelper * sizeof (helper_t), KM_SLEEP); 636 hpend = &new->helper[new->nhelper]; 637 for (hp = new->helper; hp != hpend; hp++) { 638 hp->tag = tag++; 639 if (hp < &new->helper[MINHELPERS]) { 640 hp->lzbuf = kmem_alloc(PAGESIZE, KM_SLEEP); 641 hp->page = kmem_alloc(PAGESIZE, KM_SLEEP); 642 } else if (new->clevel < DUMP_CLEVEL_BZIP2) { 643 new->maxsize += 2 * PAGESIZE; 644 } else { 645 new->maxsize += PAGESIZE; 646 } 647 if (new->clevel >= DUMP_CLEVEL_BZIP2) 648 new->maxsize += bz2size; 649 } 650 651 new->cbuf = kmem_zalloc(new->ncbuf * sizeof (cbuf_t), KM_SLEEP); 652 cpend = &new->cbuf[new->ncbuf]; 653 for (cp = new->cbuf; cp != cpend; cp++) { 654 cp->state = CBUF_FREEBUF; 655 cp->size = CBUF_SIZE; 656 if (cp < &new->cbuf[MINCBUFS]) 657 cp->buf = kmem_alloc(cp->size, KM_SLEEP); 658 else 659 new->maxsize += cp->size; 660 } 661 662 new->cmap = kmem_zalloc(new->ncmap * sizeof (cbuf_t), KM_SLEEP); 663 cpend = &new->cmap[new->ncmap]; 664 for (cp = new->cmap; cp != cpend; cp++) { 665 cp->state = CBUF_FREEMAP; 666 cp->size = CBUF_MAPSIZE; 667 cp->buf = vmem_xalloc(heap_arena, CBUF_MAPSIZE, CBUF_MAPSIZE, 668 0, 0, NULL, NULL, VM_SLEEP); 669 } 670 671 /* reserve VA to be backed with spare pages at crash time */ 672 if (new->maxsize > 0) { 673 new->maxsize = P2ROUNDUP(new->maxsize, PAGESIZE); 674 new->maxvmsize = P2ROUNDUP(new->maxsize, CBUF_MAPSIZE); 675 new->maxvm = vmem_xalloc(heap_arena, new->maxvmsize, 676 CBUF_MAPSIZE, 0, 0, NULL, NULL, VM_SLEEP); 677 } 678 679 /* 680 * Reserve memory for kmem allocation calls made during crash 681 * dump. The hat layer allocates memory for each mapping 682 * created, and the I/O path allocates buffers and data structs. 683 * Add a few pages for safety. 684 */ 685 kmem_dump_init((new->ncmap * dump_kmem_permap) + 686 (dump_kmem_pages * PAGESIZE)); 687 688 /* set new config pointers */ 689 *old = *new; 690 } 691 692 /* 693 * Define a struct memlist walker to optimize bitnum to pfn 694 * lookup. The walker maintains the state of the list traversal. 695 */ 696 typedef struct dumpmlw { 697 struct memlist *mp; /* current memlist */ 698 pgcnt_t basenum; /* bitnum base offset */ 699 pgcnt_t mppages; /* current memlist size */ 700 pgcnt_t mpleft; /* size to end of current memlist */ 701 pfn_t mpaddr; /* first pfn in memlist */ 702 } dumpmlw_t; 703 704 /* initialize the walker */ 705 static inline void 706 dump_init_memlist_walker(dumpmlw_t *pw) 707 { 708 pw->mp = phys_install; 709 pw->basenum = 0; 710 pw->mppages = pw->mp->ml_size >> PAGESHIFT; 711 pw->mpleft = pw->mppages; 712 pw->mpaddr = pw->mp->ml_address >> PAGESHIFT; 713 } 714 715 /* 716 * Lookup pfn given bitnum. The memlist can be quite long on some 717 * systems (e.g.: one per board). To optimize sequential lookups, the 718 * caller initializes and presents a memlist walker. 719 */ 720 static pfn_t 721 dump_bitnum_to_pfn(pgcnt_t bitnum, dumpmlw_t *pw) 722 { 723 bitnum -= pw->basenum; 724 while (pw->mp != NULL) { 725 if (bitnum < pw->mppages) { 726 pw->mpleft = pw->mppages - bitnum; 727 return (pw->mpaddr + bitnum); 728 } 729 bitnum -= pw->mppages; 730 pw->basenum += pw->mppages; 731 pw->mp = pw->mp->ml_next; 732 if (pw->mp != NULL) { 733 pw->mppages = pw->mp->ml_size >> PAGESHIFT; 734 pw->mpleft = pw->mppages; 735 pw->mpaddr = pw->mp->ml_address >> PAGESHIFT; 736 } 737 } 738 return (PFN_INVALID); 739 } 740 741 static pgcnt_t 742 dump_pfn_to_bitnum(pfn_t pfn) 743 { 744 struct memlist *mp; 745 pgcnt_t bitnum = 0; 746 747 for (mp = phys_install; mp != NULL; mp = mp->ml_next) { 748 if (pfn >= (mp->ml_address >> PAGESHIFT) && 749 pfn < ((mp->ml_address + mp->ml_size) >> PAGESHIFT)) 750 return (bitnum + pfn - (mp->ml_address >> PAGESHIFT)); 751 bitnum += mp->ml_size >> PAGESHIFT; 752 } 753 return ((pgcnt_t)-1); 754 } 755 756 /* 757 * Set/test bitmap for a CBUF_MAPSIZE range which includes pfn. The 758 * mapping of pfn to range index is imperfect because pfn and bitnum 759 * do not have the same phase. To make sure a CBUF_MAPSIZE range is 760 * covered, call this for both ends: 761 * dump_set_used(base) 762 * dump_set_used(base+CBUF_MAPNP-1) 763 * 764 * This is used during a panic dump to mark pages allocated by 765 * dumpsys_get_maxmem(). The macro IS_DUMP_PAGE(pp) is used by 766 * page_get_mnode_freelist() to make sure pages used by dump are never 767 * allocated. 768 */ 769 #define CBUF_MAPP2R(pfn) ((pfn) >> (CBUF_MAPSHIFT - PAGESHIFT)) 770 771 static void 772 dump_set_used(pfn_t pfn) 773 { 774 775 pgcnt_t bitnum, rbitnum; 776 777 bitnum = dump_pfn_to_bitnum(pfn); 778 ASSERT(bitnum != (pgcnt_t)-1); 779 780 rbitnum = CBUF_MAPP2R(bitnum); 781 ASSERT(rbitnum < dumpcfg.rbitmapsize); 782 783 BT_SET(dumpcfg.rbitmap, rbitnum); 784 } 785 786 int 787 dump_test_used(pfn_t pfn) 788 { 789 pgcnt_t bitnum, rbitnum; 790 791 bitnum = dump_pfn_to_bitnum(pfn); 792 ASSERT(bitnum != (pgcnt_t)-1); 793 794 rbitnum = CBUF_MAPP2R(bitnum); 795 ASSERT(rbitnum < dumpcfg.rbitmapsize); 796 797 return (BT_TEST(dumpcfg.rbitmap, rbitnum)); 798 } 799 800 /* 801 * dumpbzalloc and dumpbzfree are callbacks from the bzip2 library. 802 * dumpsys_get_maxmem() uses them for BZ2_bzCompressInit(). 803 */ 804 static void * 805 dumpbzalloc(void *opaque, int items, int size) 806 { 807 size_t *sz; 808 char *ret; 809 810 ASSERT(opaque != NULL); 811 sz = opaque; 812 ret = dumpcfg.maxvm + *sz; 813 *sz += items * size; 814 *sz = P2ROUNDUP(*sz, BZ2_BZALLOC_ALIGN); 815 ASSERT(*sz <= dumpcfg.maxvmsize); 816 return (ret); 817 } 818 819 /*ARGSUSED*/ 820 static void 821 dumpbzfree(void *opaque, void *addr) 822 { 823 } 824 825 /* 826 * Perform additional checks on the page to see if we can really use 827 * it. The kernel (kas) pages are always set in the bitmap. However, 828 * boot memory pages (prom_ppages or P_BOOTPAGES) are not in the 829 * bitmap. So we check for them. 830 */ 831 static inline int 832 dump_pfn_check(pfn_t pfn) 833 { 834 page_t *pp = page_numtopp_nolock(pfn); 835 if (pp == NULL || pp->p_pagenum != pfn || 836 #if defined(__sparc) 837 pp->p_vnode == &promvp || 838 #else 839 PP_ISBOOTPAGES(pp) || 840 #endif 841 pp->p_toxic != 0) 842 return (0); 843 return (1); 844 } 845 846 /* 847 * Check a range to see if all contained pages are available and 848 * return non-zero if the range can be used. 849 */ 850 static inline int 851 dump_range_check(pgcnt_t start, pgcnt_t end, pfn_t pfn) 852 { 853 for (; start < end; start++, pfn++) { 854 if (BT_TEST(dumpcfg.bitmap, start)) 855 return (0); 856 if (!dump_pfn_check(pfn)) 857 return (0); 858 } 859 return (1); 860 } 861 862 /* 863 * dumpsys_get_maxmem() is called during panic. Find unused ranges 864 * and use them for buffers. If we find enough memory switch to 865 * parallel bzip2, otherwise use parallel lzjb. 866 * 867 * It searches the dump bitmap in 2 passes. The first time it looks 868 * for CBUF_MAPSIZE ranges. On the second pass it uses small pages. 869 */ 870 static void 871 dumpsys_get_maxmem() 872 { 873 dumpcfg_t *cfg = &dumpcfg; 874 cbuf_t *endcp = &cfg->cbuf[cfg->ncbuf]; 875 helper_t *endhp = &cfg->helper[cfg->nhelper]; 876 pgcnt_t bitnum, end; 877 size_t sz, endsz, bz2size; 878 pfn_t pfn, off; 879 cbuf_t *cp; 880 helper_t *hp, *ohp; 881 dumpmlw_t mlw; 882 int k; 883 884 /* 885 * Setting dump_plat_mincpu to 0 at any time forces a serial 886 * dump. 887 */ 888 if (dump_plat_mincpu == 0) { 889 cfg->clevel = 0; 890 return; 891 } 892 893 /* 894 * There may be no point in looking for spare memory. If 895 * dumping all memory, then none is spare. If doing a serial 896 * dump, then already have buffers. 897 */ 898 if (cfg->maxsize == 0 || cfg->clevel < DUMP_CLEVEL_LZJB || 899 (dump_conflags & DUMP_ALL) != 0) { 900 if (cfg->clevel > DUMP_CLEVEL_LZJB) 901 cfg->clevel = DUMP_CLEVEL_LZJB; 902 return; 903 } 904 905 sz = 0; 906 cfg->found4m = 0; 907 cfg->foundsm = 0; 908 909 /* bitmap of ranges used to estimate which pfns are being used */ 910 bzero(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.rbitmapsize)); 911 912 /* find ranges that are not being dumped to use for buffers */ 913 dump_init_memlist_walker(&mlw); 914 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 915 dump_timeleft = dump_timeout; 916 end = bitnum + CBUF_MAPNP; 917 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 918 ASSERT(pfn != PFN_INVALID); 919 920 /* skip partial range at end of mem segment */ 921 if (mlw.mpleft < CBUF_MAPNP) { 922 end = bitnum + mlw.mpleft; 923 continue; 924 } 925 926 /* skip non aligned pages */ 927 off = P2PHASE(pfn, CBUF_MAPNP); 928 if (off != 0) { 929 end -= off; 930 continue; 931 } 932 933 if (!dump_range_check(bitnum, end, pfn)) 934 continue; 935 936 ASSERT((sz + CBUF_MAPSIZE) <= cfg->maxvmsize); 937 hat_devload(kas.a_hat, cfg->maxvm + sz, CBUF_MAPSIZE, pfn, 938 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 939 sz += CBUF_MAPSIZE; 940 cfg->found4m++; 941 942 /* set the bitmap for both ends to be sure to cover the range */ 943 dump_set_used(pfn); 944 dump_set_used(pfn + CBUF_MAPNP - 1); 945 946 if (sz >= cfg->maxsize) 947 goto foundmax; 948 } 949 950 /* Add small pages if we can't find enough large pages. */ 951 dump_init_memlist_walker(&mlw); 952 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 953 dump_timeleft = dump_timeout; 954 end = bitnum + CBUF_MAPNP; 955 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 956 ASSERT(pfn != PFN_INVALID); 957 958 /* Find any non-aligned pages at start and end of segment. */ 959 off = P2PHASE(pfn, CBUF_MAPNP); 960 if (mlw.mpleft < CBUF_MAPNP) { 961 end = bitnum + mlw.mpleft; 962 } else if (off != 0) { 963 end -= off; 964 } else if (cfg->found4m && dump_test_used(pfn)) { 965 continue; 966 } 967 968 for (; bitnum < end; bitnum++, pfn++) { 969 dump_timeleft = dump_timeout; 970 if (BT_TEST(dumpcfg.bitmap, bitnum)) 971 continue; 972 if (!dump_pfn_check(pfn)) 973 continue; 974 ASSERT((sz + PAGESIZE) <= cfg->maxvmsize); 975 hat_devload(kas.a_hat, cfg->maxvm + sz, PAGESIZE, pfn, 976 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 977 sz += PAGESIZE; 978 cfg->foundsm++; 979 dump_set_used(pfn); 980 if (sz >= cfg->maxsize) 981 goto foundmax; 982 } 983 } 984 985 /* Fall back to lzjb if we did not get enough memory for bzip2. */ 986 endsz = (cfg->maxsize * cfg->threshold) / cfg->nhelper; 987 if (sz < endsz) { 988 cfg->clevel = DUMP_CLEVEL_LZJB; 989 } 990 991 /* Allocate memory for as many helpers as we can. */ 992 foundmax: 993 994 /* Byte offsets into memory found and mapped above */ 995 endsz = sz; 996 sz = 0; 997 998 /* Set the size for bzip2 state. Only bzip2 needs it. */ 999 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 1000 1001 /* Skip the preallocate output buffers. */ 1002 cp = &cfg->cbuf[MINCBUFS]; 1003 1004 /* Use this to move memory up from the preallocated helpers. */ 1005 ohp = cfg->helper; 1006 1007 /* Loop over all helpers and allocate memory. */ 1008 for (hp = cfg->helper; hp < endhp; hp++) { 1009 1010 /* Skip preallocated helpers by checking hp->page. */ 1011 if (hp->page == NULL) { 1012 if (cfg->clevel <= DUMP_CLEVEL_LZJB) { 1013 /* lzjb needs 2 1-page buffers */ 1014 if ((sz + (2 * PAGESIZE)) > endsz) 1015 break; 1016 hp->page = cfg->maxvm + sz; 1017 sz += PAGESIZE; 1018 hp->lzbuf = cfg->maxvm + sz; 1019 sz += PAGESIZE; 1020 1021 } else if (ohp->lzbuf != NULL) { 1022 /* re-use the preallocted lzjb page for bzip2 */ 1023 hp->page = ohp->lzbuf; 1024 ohp->lzbuf = NULL; 1025 ++ohp; 1026 1027 } else { 1028 /* bzip2 needs a 1-page buffer */ 1029 if ((sz + PAGESIZE) > endsz) 1030 break; 1031 hp->page = cfg->maxvm + sz; 1032 sz += PAGESIZE; 1033 } 1034 } 1035 1036 /* 1037 * Add output buffers per helper. The number of 1038 * buffers per helper is determined by the ratio of 1039 * ncbuf to nhelper. 1040 */ 1041 for (k = 0; cp < endcp && (sz + CBUF_SIZE) <= endsz && 1042 k < NCBUF_PER_HELPER; k++) { 1043 cp->state = CBUF_FREEBUF; 1044 cp->size = CBUF_SIZE; 1045 cp->buf = cfg->maxvm + sz; 1046 sz += CBUF_SIZE; 1047 ++cp; 1048 } 1049 1050 /* 1051 * bzip2 needs compression state. Use the dumpbzalloc 1052 * and dumpbzfree callbacks to allocate the memory. 1053 * bzip2 does allocation only at init time. 1054 */ 1055 if (cfg->clevel >= DUMP_CLEVEL_BZIP2) { 1056 if ((sz + bz2size) > endsz) { 1057 hp->page = NULL; 1058 break; 1059 } else { 1060 hp->bzstream.opaque = &sz; 1061 hp->bzstream.bzalloc = dumpbzalloc; 1062 hp->bzstream.bzfree = dumpbzfree; 1063 (void) BZ2_bzCompressInit(&hp->bzstream, 1064 dump_bzip2_level, 0, 0); 1065 hp->bzstream.opaque = NULL; 1066 } 1067 } 1068 } 1069 1070 /* Finish allocating output buffers */ 1071 for (; cp < endcp && (sz + CBUF_SIZE) <= endsz; cp++) { 1072 cp->state = CBUF_FREEBUF; 1073 cp->size = CBUF_SIZE; 1074 cp->buf = cfg->maxvm + sz; 1075 sz += CBUF_SIZE; 1076 } 1077 1078 /* Enable IS_DUMP_PAGE macro, which checks for pages we took. */ 1079 if (cfg->found4m || cfg->foundsm) 1080 dump_check_used = 1; 1081 1082 ASSERT(sz <= endsz); 1083 } 1084 1085 static void 1086 dumphdr_init(void) 1087 { 1088 pgcnt_t npages = 0; 1089 1090 ASSERT(MUTEX_HELD(&dump_lock)); 1091 1092 if (dumphdr == NULL) { 1093 dumphdr = kmem_zalloc(sizeof (dumphdr_t), KM_SLEEP); 1094 dumphdr->dump_magic = DUMP_MAGIC; 1095 dumphdr->dump_version = DUMP_VERSION; 1096 dumphdr->dump_wordsize = DUMP_WORDSIZE; 1097 dumphdr->dump_pageshift = PAGESHIFT; 1098 dumphdr->dump_pagesize = PAGESIZE; 1099 dumphdr->dump_utsname = utsname; 1100 (void) strcpy(dumphdr->dump_platform, platform); 1101 dumpbuf.size = dumpbuf_iosize(maxphys); 1102 dumpbuf.start = kmem_alloc(dumpbuf.size, KM_SLEEP); 1103 dumpbuf.end = dumpbuf.start + dumpbuf.size; 1104 dumpcfg.pids = kmem_alloc(v.v_proc * sizeof (pid_t), KM_SLEEP); 1105 dumpcfg.helpermap = kmem_zalloc(BT_SIZEOFMAP(NCPU), KM_SLEEP); 1106 LOCK_INIT_HELD(&dumpcfg.helper_lock); 1107 dump_stack_scratch = kmem_alloc(STACK_BUF_SIZE, KM_SLEEP); 1108 (void) strncpy(dumphdr->dump_uuid, dump_get_uuid(), 1109 sizeof (dumphdr->dump_uuid)); 1110 } 1111 1112 npages = num_phys_pages(); 1113 1114 if (dumpcfg.bitmapsize != npages) { 1115 size_t rlen = CBUF_MAPP2R(P2ROUNDUP(npages, CBUF_MAPNP)); 1116 void *map = kmem_alloc(BT_SIZEOFMAP(npages), KM_SLEEP); 1117 void *rmap = kmem_alloc(BT_SIZEOFMAP(rlen), KM_SLEEP); 1118 1119 if (dumpcfg.bitmap != NULL) 1120 kmem_free(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg. 1121 bitmapsize)); 1122 if (dumpcfg.rbitmap != NULL) 1123 kmem_free(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg. 1124 rbitmapsize)); 1125 dumpcfg.bitmap = map; 1126 dumpcfg.bitmapsize = npages; 1127 dumpcfg.rbitmap = rmap; 1128 dumpcfg.rbitmapsize = rlen; 1129 } 1130 } 1131 1132 /* 1133 * Establish a new dump device. 1134 */ 1135 int 1136 dumpinit(vnode_t *vp, char *name, int justchecking) 1137 { 1138 vnode_t *cvp; 1139 vattr_t vattr; 1140 vnode_t *cdev_vp; 1141 int error = 0; 1142 1143 ASSERT(MUTEX_HELD(&dump_lock)); 1144 1145 dumphdr_init(); 1146 1147 cvp = common_specvp(vp); 1148 if (cvp == dumpvp) 1149 return (0); 1150 1151 /* 1152 * Determine whether this is a plausible dump device. We want either: 1153 * (1) a real device that's not mounted and has a cb_dump routine, or 1154 * (2) a swapfile on some filesystem that has a vop_dump routine. 1155 */ 1156 if ((error = VOP_OPEN(&cvp, FREAD | FWRITE, kcred, NULL)) != 0) 1157 return (error); 1158 1159 vattr.va_mask = AT_SIZE | AT_TYPE | AT_RDEV; 1160 if ((error = VOP_GETATTR(cvp, &vattr, 0, kcred, NULL)) == 0) { 1161 if (vattr.va_type == VBLK || vattr.va_type == VCHR) { 1162 if (devopsp[getmajor(vattr.va_rdev)]-> 1163 devo_cb_ops->cb_dump == nodev) 1164 error = ENOTSUP; 1165 else if (vfs_devismounted(vattr.va_rdev)) 1166 error = EBUSY; 1167 if (strcmp(ddi_driver_name(VTOS(cvp)->s_dip), 1168 ZFS_DRIVER) == 0 && 1169 IS_SWAPVP(common_specvp(cvp))) 1170 error = EBUSY; 1171 } else { 1172 if (vn_matchopval(cvp, VOPNAME_DUMP, fs_nosys) || 1173 !IS_SWAPVP(cvp)) 1174 error = ENOTSUP; 1175 } 1176 } 1177 1178 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) 1179 error = ENOSPC; 1180 1181 if (error || justchecking) { 1182 (void) VOP_CLOSE(cvp, FREAD | FWRITE, 1, (offset_t)0, 1183 kcred, NULL); 1184 return (error); 1185 } 1186 1187 VN_HOLD(cvp); 1188 1189 if (dumpvp != NULL) 1190 dumpfini(); /* unconfigure the old dump device */ 1191 1192 dumpvp = cvp; 1193 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 1194 dumppath = kmem_alloc(strlen(name) + 1, KM_SLEEP); 1195 (void) strcpy(dumppath, name); 1196 dumpbuf.iosize = 0; 1197 1198 /* 1199 * If the dump device is a block device, attempt to open up the 1200 * corresponding character device and determine its maximum transfer 1201 * size. We use this information to potentially resize dumpbuf to a 1202 * larger and more optimal size for performing i/o to the dump device. 1203 */ 1204 if (cvp->v_type == VBLK && 1205 (cdev_vp = makespecvp(VTOS(cvp)->s_dev, VCHR)) != NULL) { 1206 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1207 size_t blk_size; 1208 struct dk_cinfo dki; 1209 struct dk_minfo minf; 1210 1211 if (VOP_IOCTL(cdev_vp, DKIOCGMEDIAINFO, 1212 (intptr_t)&minf, FKIOCTL, kcred, NULL, NULL) 1213 == 0 && minf.dki_lbsize != 0) 1214 blk_size = minf.dki_lbsize; 1215 else 1216 blk_size = DEV_BSIZE; 1217 1218 if (VOP_IOCTL(cdev_vp, DKIOCINFO, (intptr_t)&dki, 1219 FKIOCTL, kcred, NULL, NULL) == 0) { 1220 dumpbuf.iosize = dki.dki_maxtransfer * blk_size; 1221 dumpbuf_resize(); 1222 } 1223 /* 1224 * If we are working with a zvol then dumpify it 1225 * if it's not being used as swap. 1226 */ 1227 if (strcmp(dki.dki_dname, ZVOL_DRIVER) == 0) { 1228 if (IS_SWAPVP(common_specvp(cvp))) 1229 error = EBUSY; 1230 else if ((error = VOP_IOCTL(cdev_vp, 1231 DKIOCDUMPINIT, NULL, FKIOCTL, kcred, 1232 NULL, NULL)) != 0) 1233 dumpfini(); 1234 } 1235 1236 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1237 kcred, NULL); 1238 } 1239 1240 VN_RELE(cdev_vp); 1241 } 1242 1243 cmn_err(CE_CONT, "?dump on %s size %llu MB\n", name, dumpvp_size >> 20); 1244 1245 dump_update_clevel(); 1246 1247 return (error); 1248 } 1249 1250 void 1251 dumpfini(void) 1252 { 1253 vattr_t vattr; 1254 boolean_t is_zfs = B_FALSE; 1255 vnode_t *cdev_vp; 1256 ASSERT(MUTEX_HELD(&dump_lock)); 1257 1258 kmem_free(dumppath, strlen(dumppath) + 1); 1259 1260 /* 1261 * Determine if we are using zvols for our dump device 1262 */ 1263 vattr.va_mask = AT_RDEV; 1264 if (VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL) == 0) { 1265 is_zfs = (getmajor(vattr.va_rdev) == 1266 ddi_name_to_major(ZFS_DRIVER)) ? B_TRUE : B_FALSE; 1267 } 1268 1269 /* 1270 * If we have a zvol dump device then we call into zfs so 1271 * that it may have a chance to cleanup. 1272 */ 1273 if (is_zfs && 1274 (cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR)) != NULL) { 1275 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1276 (void) VOP_IOCTL(cdev_vp, DKIOCDUMPFINI, NULL, FKIOCTL, 1277 kcred, NULL, NULL); 1278 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1279 kcred, NULL); 1280 } 1281 VN_RELE(cdev_vp); 1282 } 1283 1284 (void) VOP_CLOSE(dumpvp, FREAD | FWRITE, 1, (offset_t)0, kcred, NULL); 1285 1286 VN_RELE(dumpvp); 1287 1288 dumpvp = NULL; 1289 dumpvp_size = 0; 1290 dumppath = NULL; 1291 } 1292 1293 static offset_t 1294 dumpvp_flush(void) 1295 { 1296 size_t size = P2ROUNDUP(dumpbuf.cur - dumpbuf.start, PAGESIZE); 1297 hrtime_t iotime; 1298 int err; 1299 1300 if (dumpbuf.vp_off + size > dumpbuf.vp_limit) { 1301 dump_ioerr = ENOSPC; 1302 dumpbuf.vp_off = dumpbuf.vp_limit; 1303 } else if (size != 0) { 1304 iotime = gethrtime(); 1305 dumpsync.iowait += iotime - dumpsync.iowaitts; 1306 if (panicstr) 1307 err = VOP_DUMP(dumpvp, dumpbuf.start, 1308 lbtodb(dumpbuf.vp_off), btod(size), NULL); 1309 else 1310 err = vn_rdwr(UIO_WRITE, dumpbuf.cdev_vp != NULL ? 1311 dumpbuf.cdev_vp : dumpvp, dumpbuf.start, size, 1312 dumpbuf.vp_off, UIO_SYSSPACE, 0, dumpbuf.vp_limit, 1313 kcred, 0); 1314 if (err && dump_ioerr == 0) 1315 dump_ioerr = err; 1316 dumpsync.iowaitts = gethrtime(); 1317 dumpsync.iotime += dumpsync.iowaitts - iotime; 1318 dumpsync.nwrite += size; 1319 dumpbuf.vp_off += size; 1320 } 1321 dumpbuf.cur = dumpbuf.start; 1322 dump_timeleft = dump_timeout; 1323 return (dumpbuf.vp_off); 1324 } 1325 1326 /* maximize write speed by keeping seek offset aligned with size */ 1327 void 1328 dumpvp_write(const void *va, size_t size) 1329 { 1330 size_t len, off, sz; 1331 1332 while (size != 0) { 1333 len = MIN(size, dumpbuf.end - dumpbuf.cur); 1334 if (len == 0) { 1335 off = P2PHASE(dumpbuf.vp_off, dumpbuf.size); 1336 if (off == 0 || !ISP2(dumpbuf.size)) { 1337 (void) dumpvp_flush(); 1338 } else { 1339 sz = dumpbuf.size - off; 1340 dumpbuf.cur = dumpbuf.start + sz; 1341 (void) dumpvp_flush(); 1342 ovbcopy(dumpbuf.start + sz, dumpbuf.start, off); 1343 dumpbuf.cur += off; 1344 } 1345 } else { 1346 bcopy(va, dumpbuf.cur, len); 1347 va = (char *)va + len; 1348 dumpbuf.cur += len; 1349 size -= len; 1350 } 1351 } 1352 } 1353 1354 /*ARGSUSED*/ 1355 static void 1356 dumpvp_ksyms_write(const void *src, void *dst, size_t size) 1357 { 1358 dumpvp_write(src, size); 1359 } 1360 1361 /* 1362 * Mark 'pfn' in the bitmap and dump its translation table entry. 1363 */ 1364 void 1365 dump_addpage(struct as *as, void *va, pfn_t pfn) 1366 { 1367 mem_vtop_t mem_vtop; 1368 pgcnt_t bitnum; 1369 1370 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1371 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1372 dumphdr->dump_npages++; 1373 BT_SET(dumpcfg.bitmap, bitnum); 1374 } 1375 dumphdr->dump_nvtop++; 1376 mem_vtop.m_as = as; 1377 mem_vtop.m_va = va; 1378 mem_vtop.m_pfn = pfn; 1379 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1380 } 1381 dump_timeleft = dump_timeout; 1382 } 1383 1384 /* 1385 * Mark 'pfn' in the bitmap 1386 */ 1387 void 1388 dump_page(pfn_t pfn) 1389 { 1390 pgcnt_t bitnum; 1391 1392 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1393 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1394 dumphdr->dump_npages++; 1395 BT_SET(dumpcfg.bitmap, bitnum); 1396 } 1397 } 1398 dump_timeleft = dump_timeout; 1399 } 1400 1401 /* 1402 * Dump the <as, va, pfn> information for a given address space. 1403 * SEGOP_DUMP() will call dump_addpage() for each page in the segment. 1404 */ 1405 static void 1406 dump_as(struct as *as) 1407 { 1408 struct seg *seg; 1409 1410 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1411 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 1412 if (seg->s_as != as) 1413 break; 1414 if (seg->s_ops == NULL) 1415 continue; 1416 SEGOP_DUMP(seg); 1417 } 1418 AS_LOCK_EXIT(as, &as->a_lock); 1419 1420 if (seg != NULL) 1421 cmn_err(CE_WARN, "invalid segment %p in address space %p", 1422 (void *)seg, (void *)as); 1423 } 1424 1425 static int 1426 dump_process(pid_t pid) 1427 { 1428 proc_t *p = sprlock(pid); 1429 1430 if (p == NULL) 1431 return (-1); 1432 if (p->p_as != &kas) { 1433 mutex_exit(&p->p_lock); 1434 dump_as(p->p_as); 1435 mutex_enter(&p->p_lock); 1436 } 1437 1438 sprunlock(p); 1439 1440 return (0); 1441 } 1442 1443 /* 1444 * The following functions (dump_summary(), dump_ereports(), and 1445 * dump_messages()), write data to an uncompressed area within the 1446 * crashdump. The layout of these is 1447 * 1448 * +------------------------------------------------------------+ 1449 * | compressed pages | summary | ereports | messages | 1450 * +------------------------------------------------------------+ 1451 * 1452 * With the advent of saving a compressed crash dump by default, we 1453 * need to save a little more data to describe the failure mode in 1454 * an uncompressed buffer available before savecore uncompresses 1455 * the dump. Initially this is a copy of the stack trace. Additional 1456 * summary information should be added here. 1457 */ 1458 1459 void 1460 dump_summary(void) 1461 { 1462 u_offset_t dumpvp_start; 1463 summary_dump_t sd; 1464 1465 if (dumpvp == NULL || dumphdr == NULL) 1466 return; 1467 1468 dumpbuf.cur = dumpbuf.start; 1469 1470 dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE + 1471 DUMP_ERPTSIZE); 1472 dumpvp_start = dumpbuf.vp_limit - DUMP_SUMMARYSIZE; 1473 dumpbuf.vp_off = dumpvp_start; 1474 1475 sd.sd_magic = SUMMARY_MAGIC; 1476 sd.sd_ssum = checksum32(dump_stack_scratch, STACK_BUF_SIZE); 1477 dumpvp_write(&sd, sizeof (sd)); 1478 dumpvp_write(dump_stack_scratch, STACK_BUF_SIZE); 1479 1480 sd.sd_magic = 0; /* indicate end of summary */ 1481 dumpvp_write(&sd, sizeof (sd)); 1482 (void) dumpvp_flush(); 1483 } 1484 1485 void 1486 dump_ereports(void) 1487 { 1488 u_offset_t dumpvp_start; 1489 erpt_dump_t ed; 1490 1491 if (dumpvp == NULL || dumphdr == NULL) 1492 return; 1493 1494 dumpbuf.cur = dumpbuf.start; 1495 dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE); 1496 dumpvp_start = dumpbuf.vp_limit - DUMP_ERPTSIZE; 1497 dumpbuf.vp_off = dumpvp_start; 1498 1499 fm_ereport_dump(); 1500 if (panicstr) 1501 errorq_dump(); 1502 1503 bzero(&ed, sizeof (ed)); /* indicate end of ereports */ 1504 dumpvp_write(&ed, sizeof (ed)); 1505 (void) dumpvp_flush(); 1506 1507 if (!panicstr) { 1508 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1509 (size_t)(dumpbuf.vp_off - dumpvp_start), 1510 B_INVAL | B_FORCE, kcred, NULL); 1511 } 1512 } 1513 1514 void 1515 dump_messages(void) 1516 { 1517 log_dump_t ld; 1518 mblk_t *mctl, *mdata; 1519 queue_t *q, *qlast; 1520 u_offset_t dumpvp_start; 1521 1522 if (dumpvp == NULL || dumphdr == NULL || log_consq == NULL) 1523 return; 1524 1525 dumpbuf.cur = dumpbuf.start; 1526 dumpbuf.vp_limit = dumpvp_size - DUMP_OFFSET; 1527 dumpvp_start = dumpbuf.vp_limit - DUMP_LOGSIZE; 1528 dumpbuf.vp_off = dumpvp_start; 1529 1530 qlast = NULL; 1531 do { 1532 for (q = log_consq; q->q_next != qlast; q = q->q_next) 1533 continue; 1534 for (mctl = q->q_first; mctl != NULL; mctl = mctl->b_next) { 1535 dump_timeleft = dump_timeout; 1536 mdata = mctl->b_cont; 1537 ld.ld_magic = LOG_MAGIC; 1538 ld.ld_msgsize = MBLKL(mctl->b_cont); 1539 ld.ld_csum = checksum32(mctl->b_rptr, MBLKL(mctl)); 1540 ld.ld_msum = checksum32(mdata->b_rptr, MBLKL(mdata)); 1541 dumpvp_write(&ld, sizeof (ld)); 1542 dumpvp_write(mctl->b_rptr, MBLKL(mctl)); 1543 dumpvp_write(mdata->b_rptr, MBLKL(mdata)); 1544 } 1545 } while ((qlast = q) != log_consq); 1546 1547 ld.ld_magic = 0; /* indicate end of messages */ 1548 dumpvp_write(&ld, sizeof (ld)); 1549 (void) dumpvp_flush(); 1550 if (!panicstr) { 1551 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1552 (size_t)(dumpbuf.vp_off - dumpvp_start), 1553 B_INVAL | B_FORCE, kcred, NULL); 1554 } 1555 } 1556 1557 /* 1558 * The following functions are called on multiple CPUs during dump. 1559 * They must not use most kernel services, because all cross-calls are 1560 * disabled during panic. Therefore, blocking locks and cache flushes 1561 * will not work. 1562 */ 1563 1564 /* 1565 * Copy pages, trapping ECC errors. Also, for robustness, trap data 1566 * access in case something goes wrong in the hat layer and the 1567 * mapping is broken. 1568 */ 1569 static int 1570 dump_pagecopy(void *src, void *dst) 1571 { 1572 long *wsrc = (long *)src; 1573 long *wdst = (long *)dst; 1574 const ulong_t ncopies = PAGESIZE / sizeof (long); 1575 volatile int w = 0; 1576 volatile int ueoff = -1; 1577 on_trap_data_t otd; 1578 1579 if (on_trap(&otd, OT_DATA_EC | OT_DATA_ACCESS)) { 1580 if (ueoff == -1) 1581 ueoff = w * sizeof (long); 1582 /* report "bad ECC" or "bad address" */ 1583 #ifdef _LP64 1584 if (otd.ot_trap & OT_DATA_EC) 1585 wdst[w++] = 0x00badecc00badecc; 1586 else 1587 wdst[w++] = 0x00badadd00badadd; 1588 #else 1589 if (otd.ot_trap & OT_DATA_EC) 1590 wdst[w++] = 0x00badecc; 1591 else 1592 wdst[w++] = 0x00badadd; 1593 #endif 1594 } 1595 while (w < ncopies) { 1596 wdst[w] = wsrc[w]; 1597 w++; 1598 } 1599 no_trap(); 1600 return (ueoff); 1601 } 1602 1603 static void 1604 dumpsys_close_cq(cqueue_t *cq, int live) 1605 { 1606 if (live) { 1607 mutex_enter(&cq->mutex); 1608 atomic_dec_uint(&cq->open); 1609 cv_signal(&cq->cv); 1610 mutex_exit(&cq->mutex); 1611 } else { 1612 atomic_dec_uint(&cq->open); 1613 } 1614 } 1615 1616 static inline void 1617 dumpsys_spinlock(lock_t *lp) 1618 { 1619 uint_t backoff = 0; 1620 int loop_count = 0; 1621 1622 while (LOCK_HELD(lp) || !lock_spin_try(lp)) { 1623 if (++loop_count >= ncpus) { 1624 backoff = mutex_lock_backoff(0); 1625 loop_count = 0; 1626 } else { 1627 backoff = mutex_lock_backoff(backoff); 1628 } 1629 mutex_lock_delay(backoff); 1630 } 1631 } 1632 1633 static inline void 1634 dumpsys_spinunlock(lock_t *lp) 1635 { 1636 lock_clear(lp); 1637 } 1638 1639 static inline void 1640 dumpsys_lock(cqueue_t *cq, int live) 1641 { 1642 if (live) 1643 mutex_enter(&cq->mutex); 1644 else 1645 dumpsys_spinlock(&cq->spinlock); 1646 } 1647 1648 static inline void 1649 dumpsys_unlock(cqueue_t *cq, int live, int signal) 1650 { 1651 if (live) { 1652 if (signal) 1653 cv_signal(&cq->cv); 1654 mutex_exit(&cq->mutex); 1655 } else { 1656 dumpsys_spinunlock(&cq->spinlock); 1657 } 1658 } 1659 1660 static void 1661 dumpsys_wait_cq(cqueue_t *cq, int live) 1662 { 1663 if (live) { 1664 cv_wait(&cq->cv, &cq->mutex); 1665 } else { 1666 dumpsys_spinunlock(&cq->spinlock); 1667 while (cq->open) 1668 if (cq->first) 1669 break; 1670 dumpsys_spinlock(&cq->spinlock); 1671 } 1672 } 1673 1674 static void 1675 dumpsys_put_cq(cqueue_t *cq, cbuf_t *cp, int newstate, int live) 1676 { 1677 if (cp == NULL) 1678 return; 1679 1680 dumpsys_lock(cq, live); 1681 1682 if (cq->ts != 0) { 1683 cq->empty += gethrtime() - cq->ts; 1684 cq->ts = 0; 1685 } 1686 1687 cp->state = newstate; 1688 cp->next = NULL; 1689 if (cq->last == NULL) 1690 cq->first = cp; 1691 else 1692 cq->last->next = cp; 1693 cq->last = cp; 1694 1695 dumpsys_unlock(cq, live, 1); 1696 } 1697 1698 static cbuf_t * 1699 dumpsys_get_cq(cqueue_t *cq, int live) 1700 { 1701 cbuf_t *cp; 1702 hrtime_t now = gethrtime(); 1703 1704 dumpsys_lock(cq, live); 1705 1706 /* CONSTCOND */ 1707 while (1) { 1708 cp = (cbuf_t *)cq->first; 1709 if (cp == NULL) { 1710 if (cq->open == 0) 1711 break; 1712 dumpsys_wait_cq(cq, live); 1713 continue; 1714 } 1715 cq->first = cp->next; 1716 if (cq->first == NULL) { 1717 cq->last = NULL; 1718 cq->ts = now; 1719 } 1720 break; 1721 } 1722 1723 dumpsys_unlock(cq, live, cq->first != NULL || cq->open == 0); 1724 return (cp); 1725 } 1726 1727 /* 1728 * Send an error message to the console. If the main task is running 1729 * just write the message via uprintf. If a helper is running the 1730 * message has to be put on a queue for the main task. Setting fmt to 1731 * NULL means flush the error message buffer. If fmt is not NULL, just 1732 * add the text to the existing buffer. 1733 */ 1734 static void 1735 dumpsys_errmsg(helper_t *hp, const char *fmt, ...) 1736 { 1737 dumpsync_t *ds = hp->ds; 1738 cbuf_t *cp = hp->cperr; 1739 va_list adx; 1740 1741 if (hp->helper == MAINHELPER) { 1742 if (fmt != NULL) { 1743 if (ds->neednl) { 1744 uprintf("\n"); 1745 ds->neednl = 0; 1746 } 1747 va_start(adx, fmt); 1748 vuprintf(fmt, adx); 1749 va_end(adx); 1750 } 1751 } else if (fmt == NULL) { 1752 if (cp != NULL) { 1753 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1754 hp->cperr = NULL; 1755 } 1756 } else { 1757 if (hp->cperr == NULL) { 1758 cp = CQ_GET(freebufq); 1759 hp->cperr = cp; 1760 cp->used = 0; 1761 } 1762 va_start(adx, fmt); 1763 cp->used += vsnprintf(cp->buf + cp->used, cp->size - cp->used, 1764 fmt, adx); 1765 va_end(adx); 1766 if ((cp->used + LOG_MSGSIZE) > cp->size) { 1767 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1768 hp->cperr = NULL; 1769 } 1770 } 1771 } 1772 1773 /* 1774 * Write an output buffer to the dump file. If the main task is 1775 * running just write the data. If a helper is running the output is 1776 * placed on a queue for the main task. 1777 */ 1778 static void 1779 dumpsys_swrite(helper_t *hp, cbuf_t *cp, size_t used) 1780 { 1781 dumpsync_t *ds = hp->ds; 1782 1783 if (hp->helper == MAINHELPER) { 1784 HRSTART(ds->perpage, write); 1785 dumpvp_write(cp->buf, used); 1786 HRSTOP(ds->perpage, write); 1787 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 1788 } else { 1789 cp->used = used; 1790 CQ_PUT(mainq, cp, CBUF_WRITE); 1791 } 1792 } 1793 1794 /* 1795 * Copy one page within the mapped range. The offset starts at 0 and 1796 * is relative to the first pfn. cp->buf + cp->off is the address of 1797 * the first pfn. If dump_pagecopy returns a UE offset, create an 1798 * error message. Returns the offset to the next pfn in the range 1799 * selected by the bitmap. 1800 */ 1801 static int 1802 dumpsys_copy_page(helper_t *hp, int offset) 1803 { 1804 cbuf_t *cp = hp->cpin; 1805 int ueoff; 1806 1807 ASSERT(cp->off + offset + PAGESIZE <= cp->size); 1808 ASSERT(BT_TEST(dumpcfg.bitmap, cp->bitnum)); 1809 1810 ueoff = dump_pagecopy(cp->buf + cp->off + offset, hp->page); 1811 1812 /* ueoff is the offset in the page to a UE error */ 1813 if (ueoff != -1) { 1814 uint64_t pa = ptob(cp->pfn) + offset + ueoff; 1815 1816 dumpsys_errmsg(hp, "cpu %d: memory error at PA 0x%08x.%08x\n", 1817 CPU->cpu_id, (uint32_t)(pa >> 32), (uint32_t)pa); 1818 } 1819 1820 /* 1821 * Advance bitnum and offset to the next input page for the 1822 * next call to this function. 1823 */ 1824 offset += PAGESIZE; 1825 cp->bitnum++; 1826 while (cp->off + offset < cp->size) { 1827 if (BT_TEST(dumpcfg.bitmap, cp->bitnum)) 1828 break; 1829 offset += PAGESIZE; 1830 cp->bitnum++; 1831 } 1832 1833 return (offset); 1834 } 1835 1836 /* 1837 * Read the helper queue, and copy one mapped page. Return 0 when 1838 * done. Return 1 when a page has been copied into hp->page. 1839 */ 1840 static int 1841 dumpsys_sread(helper_t *hp) 1842 { 1843 dumpsync_t *ds = hp->ds; 1844 1845 /* CONSTCOND */ 1846 while (1) { 1847 1848 /* Find the next input buffer. */ 1849 if (hp->cpin == NULL) { 1850 HRSTART(hp->perpage, inwait); 1851 1852 /* CONSTCOND */ 1853 while (1) { 1854 hp->cpin = CQ_GET(helperq); 1855 dump_timeleft = dump_timeout; 1856 1857 /* 1858 * NULL return means the helper queue 1859 * is closed and empty. 1860 */ 1861 if (hp->cpin == NULL) 1862 break; 1863 1864 /* Have input, check for dump I/O error. */ 1865 if (!dump_ioerr) 1866 break; 1867 1868 /* 1869 * If an I/O error occurs, stay in the 1870 * loop in order to empty the helper 1871 * queue. Return the buffers to the 1872 * main task to unmap and free it. 1873 */ 1874 hp->cpin->used = 0; 1875 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1876 } 1877 HRSTOP(hp->perpage, inwait); 1878 1879 /* Stop here when the helper queue is closed. */ 1880 if (hp->cpin == NULL) 1881 break; 1882 1883 /* Set the offset=0 to get the first pfn. */ 1884 hp->in = 0; 1885 1886 /* Set the total processed to 0 */ 1887 hp->used = 0; 1888 } 1889 1890 /* Process the next page. */ 1891 if (hp->used < hp->cpin->used) { 1892 1893 /* 1894 * Get the next page from the input buffer and 1895 * return a copy. 1896 */ 1897 ASSERT(hp->in != -1); 1898 HRSTART(hp->perpage, copy); 1899 hp->in = dumpsys_copy_page(hp, hp->in); 1900 hp->used += PAGESIZE; 1901 HRSTOP(hp->perpage, copy); 1902 break; 1903 1904 } else { 1905 1906 /* 1907 * Done with the input. Flush the VM and 1908 * return the buffer to the main task. 1909 */ 1910 if (panicstr && hp->helper != MAINHELPER) 1911 hat_flush_range(kas.a_hat, 1912 hp->cpin->buf, hp->cpin->size); 1913 dumpsys_errmsg(hp, NULL); 1914 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1915 hp->cpin = NULL; 1916 } 1917 } 1918 1919 return (hp->cpin != NULL); 1920 } 1921 1922 /* 1923 * Compress size bytes starting at buf with bzip2 1924 * mode: 1925 * BZ_RUN add one more compressed page 1926 * BZ_FINISH no more input, flush the state 1927 */ 1928 static void 1929 dumpsys_bzrun(helper_t *hp, void *buf, size_t size, int mode) 1930 { 1931 dumpsync_t *ds = hp->ds; 1932 const int CSIZE = sizeof (dumpcsize_t); 1933 bz_stream *ps = &hp->bzstream; 1934 int rc = 0; 1935 uint32_t csize; 1936 dumpcsize_t cs; 1937 1938 /* Set input pointers to new input page */ 1939 if (size > 0) { 1940 ps->avail_in = size; 1941 ps->next_in = buf; 1942 } 1943 1944 /* CONSTCOND */ 1945 while (1) { 1946 1947 /* Quit when all input has been consumed */ 1948 if (ps->avail_in == 0 && mode == BZ_RUN) 1949 break; 1950 1951 /* Get a new output buffer */ 1952 if (hp->cpout == NULL) { 1953 HRSTART(hp->perpage, outwait); 1954 hp->cpout = CQ_GET(freebufq); 1955 HRSTOP(hp->perpage, outwait); 1956 ps->avail_out = hp->cpout->size - CSIZE; 1957 ps->next_out = hp->cpout->buf + CSIZE; 1958 } 1959 1960 /* Compress input, or finalize */ 1961 HRSTART(hp->perpage, compress); 1962 rc = BZ2_bzCompress(ps, mode); 1963 HRSTOP(hp->perpage, compress); 1964 1965 /* Check for error */ 1966 if (mode == BZ_RUN && rc != BZ_RUN_OK) { 1967 dumpsys_errmsg(hp, "%d: BZ_RUN error %s at page %lx\n", 1968 hp->helper, BZ2_bzErrorString(rc), 1969 hp->cpin->pagenum); 1970 break; 1971 } 1972 1973 /* Write the buffer if it is full, or we are flushing */ 1974 if (ps->avail_out == 0 || mode == BZ_FINISH) { 1975 csize = hp->cpout->size - CSIZE - ps->avail_out; 1976 cs = DUMP_SET_TAG(csize, hp->tag); 1977 if (csize > 0) { 1978 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 1979 dumpsys_swrite(hp, hp->cpout, csize + CSIZE); 1980 hp->cpout = NULL; 1981 } 1982 } 1983 1984 /* Check for final complete */ 1985 if (mode == BZ_FINISH) { 1986 if (rc == BZ_STREAM_END) 1987 break; 1988 if (rc != BZ_FINISH_OK) { 1989 dumpsys_errmsg(hp, "%d: BZ_FINISH error %s\n", 1990 hp->helper, BZ2_bzErrorString(rc)); 1991 break; 1992 } 1993 } 1994 } 1995 1996 /* Cleanup state and buffers */ 1997 if (mode == BZ_FINISH) { 1998 1999 /* Reset state so that it is re-usable. */ 2000 (void) BZ2_bzCompressReset(&hp->bzstream); 2001 2002 /* Give any unused outout buffer to the main task */ 2003 if (hp->cpout != NULL) { 2004 hp->cpout->used = 0; 2005 CQ_PUT(mainq, hp->cpout, CBUF_ERRMSG); 2006 hp->cpout = NULL; 2007 } 2008 } 2009 } 2010 2011 static void 2012 dumpsys_bz2compress(helper_t *hp) 2013 { 2014 dumpsync_t *ds = hp->ds; 2015 dumpstreamhdr_t sh; 2016 2017 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 2018 sh.stream_pagenum = (pgcnt_t)-1; 2019 sh.stream_npages = 0; 2020 hp->cpin = NULL; 2021 hp->cpout = NULL; 2022 hp->cperr = NULL; 2023 hp->in = 0; 2024 hp->out = 0; 2025 hp->bzstream.avail_in = 0; 2026 2027 /* Bump reference to mainq while we are running */ 2028 CQ_OPEN(mainq); 2029 2030 /* Get one page at a time */ 2031 while (dumpsys_sread(hp)) { 2032 if (sh.stream_pagenum != hp->cpin->pagenum) { 2033 sh.stream_pagenum = hp->cpin->pagenum; 2034 sh.stream_npages = btop(hp->cpin->used); 2035 dumpsys_bzrun(hp, &sh, sizeof (sh), BZ_RUN); 2036 } 2037 dumpsys_bzrun(hp, hp->page, PAGESIZE, 0); 2038 } 2039 2040 /* Done with input, flush any partial buffer */ 2041 if (sh.stream_pagenum != (pgcnt_t)-1) { 2042 dumpsys_bzrun(hp, NULL, 0, BZ_FINISH); 2043 dumpsys_errmsg(hp, NULL); 2044 } 2045 2046 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 2047 2048 /* Decrement main queue count, we are done */ 2049 CQ_CLOSE(mainq); 2050 } 2051 2052 /* 2053 * Compress with lzjb 2054 * write stream block if full or size==0 2055 * if csize==0 write stream header, else write <csize, data> 2056 * size==0 is a call to flush a buffer 2057 * hp->cpout is the buffer we are flushing or filling 2058 * hp->out is the next index to fill data 2059 * osize is either csize+data, or the size of a stream header 2060 */ 2061 static void 2062 dumpsys_lzjbrun(helper_t *hp, size_t csize, void *buf, size_t size) 2063 { 2064 dumpsync_t *ds = hp->ds; 2065 const int CSIZE = sizeof (dumpcsize_t); 2066 dumpcsize_t cs; 2067 size_t osize = csize > 0 ? CSIZE + size : size; 2068 2069 /* If flush, and there is no buffer, just return */ 2070 if (size == 0 && hp->cpout == NULL) 2071 return; 2072 2073 /* If flush, or cpout is full, write it out */ 2074 if (size == 0 || 2075 hp->cpout != NULL && hp->out + osize > hp->cpout->size) { 2076 2077 /* Set tag+size word at the front of the stream block. */ 2078 cs = DUMP_SET_TAG(hp->out - CSIZE, hp->tag); 2079 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 2080 2081 /* Write block to dump file. */ 2082 dumpsys_swrite(hp, hp->cpout, hp->out); 2083 2084 /* Clear pointer to indicate we need a new buffer */ 2085 hp->cpout = NULL; 2086 2087 /* flushing, we are done */ 2088 if (size == 0) 2089 return; 2090 } 2091 2092 /* Get an output buffer if we dont have one. */ 2093 if (hp->cpout == NULL) { 2094 HRSTART(hp->perpage, outwait); 2095 hp->cpout = CQ_GET(freebufq); 2096 HRSTOP(hp->perpage, outwait); 2097 hp->out = CSIZE; 2098 } 2099 2100 /* Store csize word. This is the size of compressed data. */ 2101 if (csize > 0) { 2102 cs = DUMP_SET_TAG(csize, 0); 2103 (void) memcpy(hp->cpout->buf + hp->out, &cs, CSIZE); 2104 hp->out += CSIZE; 2105 } 2106 2107 /* Store the data. */ 2108 (void) memcpy(hp->cpout->buf + hp->out, buf, size); 2109 hp->out += size; 2110 } 2111 2112 static void 2113 dumpsys_lzjbcompress(helper_t *hp) 2114 { 2115 dumpsync_t *ds = hp->ds; 2116 size_t csize; 2117 dumpstreamhdr_t sh; 2118 2119 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 2120 sh.stream_pagenum = (pfn_t)-1; 2121 sh.stream_npages = 0; 2122 hp->cpin = NULL; 2123 hp->cpout = NULL; 2124 hp->cperr = NULL; 2125 hp->in = 0; 2126 hp->out = 0; 2127 2128 /* Bump reference to mainq while we are running */ 2129 CQ_OPEN(mainq); 2130 2131 /* Get one page at a time */ 2132 while (dumpsys_sread(hp)) { 2133 2134 /* Create a stream header for each new input map */ 2135 if (sh.stream_pagenum != hp->cpin->pagenum) { 2136 sh.stream_pagenum = hp->cpin->pagenum; 2137 sh.stream_npages = btop(hp->cpin->used); 2138 dumpsys_lzjbrun(hp, 0, &sh, sizeof (sh)); 2139 } 2140 2141 /* Compress one page */ 2142 HRSTART(hp->perpage, compress); 2143 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2144 HRSTOP(hp->perpage, compress); 2145 2146 /* Add csize+data to output block */ 2147 ASSERT(csize > 0 && csize <= PAGESIZE); 2148 dumpsys_lzjbrun(hp, csize, hp->lzbuf, csize); 2149 } 2150 2151 /* Done with input, flush any partial buffer */ 2152 if (sh.stream_pagenum != (pfn_t)-1) { 2153 dumpsys_lzjbrun(hp, 0, NULL, 0); 2154 dumpsys_errmsg(hp, NULL); 2155 } 2156 2157 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 2158 2159 /* Decrement main queue count, we are done */ 2160 CQ_CLOSE(mainq); 2161 } 2162 2163 /* 2164 * Dump helper called from panic_idle() to compress pages. CPUs in 2165 * this path must not call most kernel services. 2166 * 2167 * During panic, all but one of the CPUs is idle. These CPUs are used 2168 * as helpers working in parallel to copy and compress memory 2169 * pages. During a panic, however, these processors cannot call any 2170 * kernel services. This is because mutexes become no-ops during 2171 * panic, and, cross-call interrupts are inhibited. Therefore, during 2172 * panic dump the helper CPUs communicate with the panic CPU using 2173 * memory variables. All memory mapping and I/O is performed by the 2174 * panic CPU. 2175 * 2176 * At dump configuration time, helper_lock is set and helpers_wanted 2177 * is 0. dumpsys() decides whether to set helpers_wanted before 2178 * clearing helper_lock. 2179 * 2180 * At panic time, idle CPUs spin-wait on helper_lock, then alternately 2181 * take the lock and become a helper, or return. 2182 */ 2183 void 2184 dumpsys_helper() 2185 { 2186 dumpsys_spinlock(&dumpcfg.helper_lock); 2187 if (dumpcfg.helpers_wanted) { 2188 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2189 2190 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2191 if (hp->helper == FREEHELPER) { 2192 hp->helper = CPU->cpu_id; 2193 BT_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2194 2195 dumpsys_spinunlock(&dumpcfg.helper_lock); 2196 2197 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2198 dumpsys_lzjbcompress(hp); 2199 else 2200 dumpsys_bz2compress(hp); 2201 2202 hp->helper = DONEHELPER; 2203 return; 2204 } 2205 } 2206 2207 /* No more helpers are needed. */ 2208 dumpcfg.helpers_wanted = 0; 2209 2210 } 2211 dumpsys_spinunlock(&dumpcfg.helper_lock); 2212 } 2213 2214 /* 2215 * No-wait helper callable in spin loops. 2216 * 2217 * Do not wait for helper_lock. Just check helpers_wanted. The caller 2218 * may decide to continue. This is the "c)ontinue, s)ync, r)eset? s" 2219 * case. 2220 */ 2221 void 2222 dumpsys_helper_nw() 2223 { 2224 if (dumpcfg.helpers_wanted) 2225 dumpsys_helper(); 2226 } 2227 2228 /* 2229 * Dump helper for live dumps. 2230 * These run as a system task. 2231 */ 2232 static void 2233 dumpsys_live_helper(void *arg) 2234 { 2235 helper_t *hp = arg; 2236 2237 BT_ATOMIC_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2238 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2239 dumpsys_lzjbcompress(hp); 2240 else 2241 dumpsys_bz2compress(hp); 2242 } 2243 2244 /* 2245 * Compress one page with lzjb (single threaded case) 2246 */ 2247 static void 2248 dumpsys_lzjb_page(helper_t *hp, cbuf_t *cp) 2249 { 2250 dumpsync_t *ds = hp->ds; 2251 uint32_t csize; 2252 2253 hp->helper = MAINHELPER; 2254 hp->in = 0; 2255 hp->used = 0; 2256 hp->cpin = cp; 2257 while (hp->used < cp->used) { 2258 HRSTART(hp->perpage, copy); 2259 hp->in = dumpsys_copy_page(hp, hp->in); 2260 hp->used += PAGESIZE; 2261 HRSTOP(hp->perpage, copy); 2262 2263 HRSTART(hp->perpage, compress); 2264 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2265 HRSTOP(hp->perpage, compress); 2266 2267 HRSTART(hp->perpage, write); 2268 dumpvp_write(&csize, sizeof (csize)); 2269 dumpvp_write(hp->lzbuf, csize); 2270 HRSTOP(hp->perpage, write); 2271 } 2272 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 2273 hp->cpin = NULL; 2274 } 2275 2276 /* 2277 * Main task to dump pages. This is called on the dump CPU. 2278 */ 2279 static void 2280 dumpsys_main_task(void *arg) 2281 { 2282 dumpsync_t *ds = arg; 2283 pgcnt_t pagenum = 0, bitnum = 0, hibitnum; 2284 dumpmlw_t mlw; 2285 cbuf_t *cp; 2286 pgcnt_t baseoff, pfnoff; 2287 pfn_t base, pfn; 2288 int sec, i, dumpserial; 2289 2290 /* 2291 * Fall back to serial mode if there are no helpers. 2292 * dump_plat_mincpu can be set to 0 at any time. 2293 * dumpcfg.helpermap must contain at least one member. 2294 */ 2295 dumpserial = 1; 2296 2297 if (dump_plat_mincpu != 0 && dumpcfg.clevel != 0) { 2298 for (i = 0; i < BT_BITOUL(NCPU); ++i) { 2299 if (dumpcfg.helpermap[i] != 0) { 2300 dumpserial = 0; 2301 break; 2302 } 2303 } 2304 } 2305 2306 if (dumpserial) { 2307 dumpcfg.clevel = 0; 2308 if (dumpcfg.helper[0].lzbuf == NULL) 2309 dumpcfg.helper[0].lzbuf = dumpcfg.helper[1].page; 2310 } 2311 2312 dump_init_memlist_walker(&mlw); 2313 2314 /* CONSTCOND */ 2315 while (1) { 2316 2317 if (ds->percent > ds->percent_done) { 2318 ds->percent_done = ds->percent; 2319 sec = (gethrtime() - ds->start) / 1000 / 1000 / 1000; 2320 uprintf("^\r%2d:%02d %3d%% done", 2321 sec / 60, sec % 60, ds->percent); 2322 ds->neednl = 1; 2323 } 2324 2325 while (CQ_IS_EMPTY(mainq) && !CQ_IS_EMPTY(writerq)) { 2326 2327 /* the writerq never blocks */ 2328 cp = CQ_GET(writerq); 2329 if (cp == NULL) 2330 break; 2331 2332 dump_timeleft = dump_timeout; 2333 2334 HRSTART(ds->perpage, write); 2335 dumpvp_write(cp->buf, cp->used); 2336 HRSTOP(ds->perpage, write); 2337 2338 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2339 } 2340 2341 /* 2342 * Wait here for some buffers to process. Returns NULL 2343 * when all helpers have terminated and all buffers 2344 * have been processed. 2345 */ 2346 cp = CQ_GET(mainq); 2347 2348 if (cp == NULL) { 2349 2350 /* Drain the write queue. */ 2351 if (!CQ_IS_EMPTY(writerq)) 2352 continue; 2353 2354 /* Main task exits here. */ 2355 break; 2356 } 2357 2358 dump_timeleft = dump_timeout; 2359 2360 switch (cp->state) { 2361 2362 case CBUF_FREEMAP: 2363 2364 /* 2365 * Note that we drop CBUF_FREEMAP buffers on 2366 * the floor (they will not be on any cqueue) 2367 * when we no longer need them. 2368 */ 2369 if (bitnum >= dumpcfg.bitmapsize) 2370 break; 2371 2372 if (dump_ioerr) { 2373 bitnum = dumpcfg.bitmapsize; 2374 CQ_CLOSE(helperq); 2375 break; 2376 } 2377 2378 HRSTART(ds->perpage, bitmap); 2379 for (; bitnum < dumpcfg.bitmapsize; bitnum++) 2380 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2381 break; 2382 HRSTOP(ds->perpage, bitmap); 2383 dump_timeleft = dump_timeout; 2384 2385 if (bitnum >= dumpcfg.bitmapsize) { 2386 CQ_CLOSE(helperq); 2387 break; 2388 } 2389 2390 /* 2391 * Try to map CBUF_MAPSIZE ranges. Can't 2392 * assume that memory segment size is a 2393 * multiple of CBUF_MAPSIZE. Can't assume that 2394 * the segment starts on a CBUF_MAPSIZE 2395 * boundary. 2396 */ 2397 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2398 ASSERT(pfn != PFN_INVALID); 2399 ASSERT(bitnum + mlw.mpleft <= dumpcfg.bitmapsize); 2400 2401 base = P2ALIGN(pfn, CBUF_MAPNP); 2402 if (base < mlw.mpaddr) { 2403 base = mlw.mpaddr; 2404 baseoff = P2PHASE(base, CBUF_MAPNP); 2405 } else { 2406 baseoff = 0; 2407 } 2408 2409 pfnoff = pfn - base; 2410 if (pfnoff + mlw.mpleft < CBUF_MAPNP) { 2411 hibitnum = bitnum + mlw.mpleft; 2412 cp->size = ptob(pfnoff + mlw.mpleft); 2413 } else { 2414 hibitnum = bitnum - pfnoff + CBUF_MAPNP - 2415 baseoff; 2416 cp->size = CBUF_MAPSIZE - ptob(baseoff); 2417 } 2418 2419 cp->pfn = pfn; 2420 cp->bitnum = bitnum++; 2421 cp->pagenum = pagenum++; 2422 cp->off = ptob(pfnoff); 2423 2424 for (; bitnum < hibitnum; bitnum++) 2425 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2426 pagenum++; 2427 2428 dump_timeleft = dump_timeout; 2429 cp->used = ptob(pagenum - cp->pagenum); 2430 2431 HRSTART(ds->perpage, map); 2432 hat_devload(kas.a_hat, cp->buf, cp->size, base, 2433 PROT_READ, HAT_LOAD_NOCONSIST); 2434 HRSTOP(ds->perpage, map); 2435 2436 ds->pages_mapped += btop(cp->size); 2437 ds->pages_used += pagenum - cp->pagenum; 2438 2439 CQ_OPEN(mainq); 2440 2441 /* 2442 * If there are no helpers the main task does 2443 * non-streams lzjb compress. 2444 */ 2445 if (dumpserial) { 2446 dumpsys_lzjb_page(dumpcfg.helper, cp); 2447 break; 2448 } 2449 2450 /* pass mapped pages to a helper */ 2451 CQ_PUT(helperq, cp, CBUF_INREADY); 2452 2453 /* the last page was done */ 2454 if (bitnum >= dumpcfg.bitmapsize) 2455 CQ_CLOSE(helperq); 2456 2457 break; 2458 2459 case CBUF_USEDMAP: 2460 2461 ds->npages += btop(cp->used); 2462 2463 HRSTART(ds->perpage, unmap); 2464 hat_unload(kas.a_hat, cp->buf, cp->size, HAT_UNLOAD); 2465 HRSTOP(ds->perpage, unmap); 2466 2467 if (bitnum < dumpcfg.bitmapsize) 2468 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2469 CQ_CLOSE(mainq); 2470 2471 ASSERT(ds->npages <= dumphdr->dump_npages); 2472 ds->percent = ds->npages * 100LL / dumphdr->dump_npages; 2473 break; 2474 2475 case CBUF_WRITE: 2476 2477 CQ_PUT(writerq, cp, CBUF_WRITE); 2478 break; 2479 2480 case CBUF_ERRMSG: 2481 2482 if (cp->used > 0) { 2483 cp->buf[cp->size - 2] = '\n'; 2484 cp->buf[cp->size - 1] = '\0'; 2485 if (ds->neednl) { 2486 uprintf("\n%s", cp->buf); 2487 ds->neednl = 0; 2488 } else { 2489 uprintf("%s", cp->buf); 2490 } 2491 /* wait for console output */ 2492 drv_usecwait(200000); 2493 dump_timeleft = dump_timeout; 2494 } 2495 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2496 break; 2497 2498 default: 2499 uprintf("dump: unexpected buffer state %d, " 2500 "buffer will be lost\n", cp->state); 2501 break; 2502 2503 } /* end switch */ 2504 2505 } /* end while(1) */ 2506 } 2507 2508 #ifdef COLLECT_METRICS 2509 size_t 2510 dumpsys_metrics(dumpsync_t *ds, char *buf, size_t size) 2511 { 2512 dumpcfg_t *cfg = &dumpcfg; 2513 int myid = CPU->cpu_seqid; 2514 int i, compress_ratio; 2515 int sec, iorate; 2516 helper_t *hp, *hpend = &cfg->helper[cfg->nhelper]; 2517 char *e = buf + size; 2518 char *p = buf; 2519 2520 sec = ds->elapsed / (1000 * 1000 * 1000ULL); 2521 if (sec < 1) 2522 sec = 1; 2523 2524 if (ds->iotime < 1) 2525 ds->iotime = 1; 2526 iorate = (ds->nwrite * 100000ULL) / ds->iotime; 2527 2528 compress_ratio = 100LL * ds->npages / btopr(ds->nwrite + 1); 2529 2530 #define P(...) (p += p < e ? snprintf(p, e - p, __VA_ARGS__) : 0) 2531 2532 P("Master cpu_seqid,%d\n", CPU->cpu_seqid); 2533 P("Master cpu_id,%d\n", CPU->cpu_id); 2534 P("dump_flags,0x%x\n", dumphdr->dump_flags); 2535 P("dump_ioerr,%d\n", dump_ioerr); 2536 2537 P("Helpers:\n"); 2538 for (i = 0; i < ncpus; i++) { 2539 if ((i & 15) == 0) 2540 P(",,%03d,", i); 2541 if (i == myid) 2542 P(" M"); 2543 else if (BT_TEST(cfg->helpermap, i)) 2544 P("%4d", cpu_seq[i]->cpu_id); 2545 else 2546 P(" *"); 2547 if ((i & 15) == 15) 2548 P("\n"); 2549 } 2550 2551 P("ncbuf_used,%d\n", cfg->ncbuf_used); 2552 P("ncmap,%d\n", cfg->ncmap); 2553 2554 P("Found %ldM ranges,%ld\n", (CBUF_MAPSIZE / DUMP_1MB), cfg->found4m); 2555 P("Found small pages,%ld\n", cfg->foundsm); 2556 2557 P("Compression level,%d\n", cfg->clevel); 2558 P("Compression type,%s %s\n", cfg->clevel == 0 ? "serial" : "parallel", 2559 cfg->clevel >= DUMP_CLEVEL_BZIP2 ? "bzip2" : "lzjb"); 2560 P("Compression ratio,%d.%02d\n", compress_ratio / 100, compress_ratio % 2561 100); 2562 P("nhelper_used,%d\n", cfg->nhelper_used); 2563 2564 P("Dump I/O rate MBS,%d.%02d\n", iorate / 100, iorate % 100); 2565 P("..total bytes,%lld\n", (u_longlong_t)ds->nwrite); 2566 P("..total nsec,%lld\n", (u_longlong_t)ds->iotime); 2567 P("dumpbuf.iosize,%ld\n", dumpbuf.iosize); 2568 P("dumpbuf.size,%ld\n", dumpbuf.size); 2569 2570 P("Dump pages/sec,%llu\n", (u_longlong_t)ds->npages / sec); 2571 P("Dump pages,%llu\n", (u_longlong_t)ds->npages); 2572 P("Dump time,%d\n", sec); 2573 2574 if (ds->pages_mapped > 0) 2575 P("per-cent map utilization,%d\n", (int)((100 * ds->pages_used) 2576 / ds->pages_mapped)); 2577 2578 P("\nPer-page metrics:\n"); 2579 if (ds->npages > 0) { 2580 for (hp = cfg->helper; hp != hpend; hp++) { 2581 #define PERPAGE(x) ds->perpage.x += hp->perpage.x; 2582 PERPAGES; 2583 #undef PERPAGE 2584 } 2585 #define PERPAGE(x) \ 2586 P("%s nsec/page,%d\n", #x, (int)(ds->perpage.x / ds->npages)); 2587 PERPAGES; 2588 #undef PERPAGE 2589 P("freebufq.empty,%d\n", (int)(ds->freebufq.empty / 2590 ds->npages)); 2591 P("helperq.empty,%d\n", (int)(ds->helperq.empty / 2592 ds->npages)); 2593 P("writerq.empty,%d\n", (int)(ds->writerq.empty / 2594 ds->npages)); 2595 P("mainq.empty,%d\n", (int)(ds->mainq.empty / ds->npages)); 2596 2597 P("I/O wait nsec/page,%llu\n", (u_longlong_t)(ds->iowait / 2598 ds->npages)); 2599 } 2600 #undef P 2601 if (p < e) 2602 bzero(p, e - p); 2603 return (p - buf); 2604 } 2605 #endif /* COLLECT_METRICS */ 2606 2607 /* 2608 * Dump the system. 2609 */ 2610 void 2611 dumpsys(void) 2612 { 2613 dumpsync_t *ds = &dumpsync; 2614 taskq_t *livetaskq = NULL; 2615 pfn_t pfn; 2616 pgcnt_t bitnum; 2617 proc_t *p; 2618 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2619 cbuf_t *cp; 2620 pid_t npids, pidx; 2621 char *content; 2622 char *buf; 2623 size_t size; 2624 int save_dump_clevel; 2625 dumpmlw_t mlw; 2626 dumpcsize_t datatag; 2627 dumpdatahdr_t datahdr; 2628 2629 if (dumpvp == NULL || dumphdr == NULL) { 2630 uprintf("skipping system dump - no dump device configured\n"); 2631 if (panicstr) { 2632 dumpcfg.helpers_wanted = 0; 2633 dumpsys_spinunlock(&dumpcfg.helper_lock); 2634 } 2635 return; 2636 } 2637 dumpbuf.cur = dumpbuf.start; 2638 2639 /* clear the sync variables */ 2640 ASSERT(dumpcfg.nhelper > 0); 2641 bzero(ds, sizeof (*ds)); 2642 ds->dumpcpu = CPU->cpu_id; 2643 2644 /* 2645 * Calculate the starting block for dump. If we're dumping on a 2646 * swap device, start 1/5 of the way in; otherwise, start at the 2647 * beginning. And never use the first page -- it may be a disk label. 2648 */ 2649 if (dumpvp->v_flag & VISSWAP) 2650 dumphdr->dump_start = P2ROUNDUP(dumpvp_size / 5, DUMP_OFFSET); 2651 else 2652 dumphdr->dump_start = DUMP_OFFSET; 2653 2654 dumphdr->dump_flags = DF_VALID | DF_COMPLETE | DF_LIVE | DF_COMPRESSED; 2655 dumphdr->dump_crashtime = gethrestime_sec(); 2656 dumphdr->dump_npages = 0; 2657 dumphdr->dump_nvtop = 0; 2658 bzero(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.bitmapsize)); 2659 dump_timeleft = dump_timeout; 2660 2661 if (panicstr) { 2662 dumphdr->dump_flags &= ~DF_LIVE; 2663 (void) VOP_DUMPCTL(dumpvp, DUMP_FREE, NULL, NULL); 2664 (void) VOP_DUMPCTL(dumpvp, DUMP_ALLOC, NULL, NULL); 2665 (void) vsnprintf(dumphdr->dump_panicstring, DUMP_PANICSIZE, 2666 panicstr, panicargs); 2667 2668 } 2669 2670 if (dump_conflags & DUMP_ALL) 2671 content = "all"; 2672 else if (dump_conflags & DUMP_CURPROC) 2673 content = "kernel + curproc"; 2674 else 2675 content = "kernel"; 2676 uprintf("dumping to %s, offset %lld, content: %s\n", dumppath, 2677 dumphdr->dump_start, content); 2678 2679 /* Make sure nodename is current */ 2680 bcopy(utsname.nodename, dumphdr->dump_utsname.nodename, SYS_NMLN); 2681 2682 /* 2683 * If this is a live dump, try to open a VCHR vnode for better 2684 * performance. We must take care to flush the buffer cache 2685 * first. 2686 */ 2687 if (!panicstr) { 2688 vnode_t *cdev_vp, *cmn_cdev_vp; 2689 2690 ASSERT(dumpbuf.cdev_vp == NULL); 2691 cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR); 2692 if (cdev_vp != NULL) { 2693 cmn_cdev_vp = common_specvp(cdev_vp); 2694 if (VOP_OPEN(&cmn_cdev_vp, FREAD | FWRITE, kcred, NULL) 2695 == 0) { 2696 if (vn_has_cached_data(dumpvp)) 2697 (void) pvn_vplist_dirty(dumpvp, 0, NULL, 2698 B_INVAL | B_TRUNC, kcred); 2699 dumpbuf.cdev_vp = cmn_cdev_vp; 2700 } else { 2701 VN_RELE(cdev_vp); 2702 } 2703 } 2704 } 2705 2706 /* 2707 * Store a hires timestamp so we can look it up during debugging. 2708 */ 2709 lbolt_debug_entry(); 2710 2711 /* 2712 * Leave room for the message and ereport save areas and terminal dump 2713 * header. 2714 */ 2715 dumpbuf.vp_limit = dumpvp_size - DUMP_LOGSIZE - DUMP_OFFSET - 2716 DUMP_ERPTSIZE; 2717 2718 /* 2719 * Write out the symbol table. It's no longer compressed, 2720 * so its 'size' and 'csize' are equal. 2721 */ 2722 dumpbuf.vp_off = dumphdr->dump_ksyms = dumphdr->dump_start + PAGESIZE; 2723 dumphdr->dump_ksyms_size = dumphdr->dump_ksyms_csize = 2724 ksyms_snapshot(dumpvp_ksyms_write, NULL, LONG_MAX); 2725 2726 /* 2727 * Write out the translation map. 2728 */ 2729 dumphdr->dump_map = dumpvp_flush(); 2730 dump_as(&kas); 2731 dumphdr->dump_nvtop += dump_plat_addr(); 2732 2733 /* 2734 * call into hat, which may have unmapped pages that also need to 2735 * be in the dump 2736 */ 2737 hat_dump(); 2738 2739 if (dump_conflags & DUMP_ALL) { 2740 mutex_enter(&pidlock); 2741 2742 for (npids = 0, p = practive; p != NULL; p = p->p_next) 2743 dumpcfg.pids[npids++] = p->p_pid; 2744 2745 mutex_exit(&pidlock); 2746 2747 for (pidx = 0; pidx < npids; pidx++) 2748 (void) dump_process(dumpcfg.pids[pidx]); 2749 2750 dump_init_memlist_walker(&mlw); 2751 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2752 dump_timeleft = dump_timeout; 2753 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2754 /* 2755 * Some hypervisors do not have all pages available to 2756 * be accessed by the guest OS. Check for page 2757 * accessibility. 2758 */ 2759 if (plat_hold_page(pfn, PLAT_HOLD_NO_LOCK, NULL) != 2760 PLAT_HOLD_OK) 2761 continue; 2762 BT_SET(dumpcfg.bitmap, bitnum); 2763 } 2764 dumphdr->dump_npages = dumpcfg.bitmapsize; 2765 dumphdr->dump_flags |= DF_ALL; 2766 2767 } else if (dump_conflags & DUMP_CURPROC) { 2768 /* 2769 * Determine which pid is to be dumped. If we're panicking, we 2770 * dump the process associated with panic_thread (if any). If 2771 * this is a live dump, we dump the process associated with 2772 * curthread. 2773 */ 2774 npids = 0; 2775 if (panicstr) { 2776 if (panic_thread != NULL && 2777 panic_thread->t_procp != NULL && 2778 panic_thread->t_procp != &p0) { 2779 dumpcfg.pids[npids++] = 2780 panic_thread->t_procp->p_pid; 2781 } 2782 } else { 2783 dumpcfg.pids[npids++] = curthread->t_procp->p_pid; 2784 } 2785 2786 if (npids && dump_process(dumpcfg.pids[0]) == 0) 2787 dumphdr->dump_flags |= DF_CURPROC; 2788 else 2789 dumphdr->dump_flags |= DF_KERNEL; 2790 2791 } else { 2792 dumphdr->dump_flags |= DF_KERNEL; 2793 } 2794 2795 dumphdr->dump_hashmask = (1 << highbit(dumphdr->dump_nvtop - 1)) - 1; 2796 2797 /* 2798 * Write out the pfn table. 2799 */ 2800 dumphdr->dump_pfn = dumpvp_flush(); 2801 dump_init_memlist_walker(&mlw); 2802 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2803 dump_timeleft = dump_timeout; 2804 if (!BT_TEST(dumpcfg.bitmap, bitnum)) 2805 continue; 2806 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2807 ASSERT(pfn != PFN_INVALID); 2808 dumpvp_write(&pfn, sizeof (pfn_t)); 2809 } 2810 dump_plat_pfn(); 2811 2812 /* 2813 * Write out all the pages. 2814 * Map pages, copy them handling UEs, compress, and write them out. 2815 * Cooperate with any helpers running on CPUs in panic_idle(). 2816 */ 2817 dumphdr->dump_data = dumpvp_flush(); 2818 2819 bzero(dumpcfg.helpermap, BT_SIZEOFMAP(NCPU)); 2820 ds->live = dumpcfg.clevel > 0 && 2821 (dumphdr->dump_flags & DF_LIVE) != 0; 2822 2823 save_dump_clevel = dumpcfg.clevel; 2824 if (panicstr) 2825 dumpsys_get_maxmem(); 2826 else if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2827 dumpcfg.clevel = DUMP_CLEVEL_LZJB; 2828 2829 dumpcfg.nhelper_used = 0; 2830 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2831 if (hp->page == NULL) { 2832 hp->helper = DONEHELPER; 2833 continue; 2834 } 2835 ++dumpcfg.nhelper_used; 2836 hp->helper = FREEHELPER; 2837 hp->taskqid = NULL; 2838 hp->ds = ds; 2839 bzero(&hp->perpage, sizeof (hp->perpage)); 2840 if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2841 (void) BZ2_bzCompressReset(&hp->bzstream); 2842 } 2843 2844 CQ_OPEN(freebufq); 2845 CQ_OPEN(helperq); 2846 2847 dumpcfg.ncbuf_used = 0; 2848 for (cp = dumpcfg.cbuf; cp != &dumpcfg.cbuf[dumpcfg.ncbuf]; cp++) { 2849 if (cp->buf != NULL) { 2850 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2851 ++dumpcfg.ncbuf_used; 2852 } 2853 } 2854 2855 for (cp = dumpcfg.cmap; cp != &dumpcfg.cmap[dumpcfg.ncmap]; cp++) 2856 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2857 2858 ds->start = gethrtime(); 2859 ds->iowaitts = ds->start; 2860 2861 /* start helpers */ 2862 if (ds->live) { 2863 int n = dumpcfg.nhelper_used; 2864 int pri = MINCLSYSPRI - 25; 2865 2866 livetaskq = taskq_create("LiveDump", n, pri, n, n, 2867 TASKQ_PREPOPULATE); 2868 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2869 if (hp->page == NULL) 2870 continue; 2871 hp->helper = hp - dumpcfg.helper; 2872 hp->taskqid = taskq_dispatch(livetaskq, 2873 dumpsys_live_helper, (void *)hp, TQ_NOSLEEP); 2874 } 2875 2876 } else { 2877 if (panicstr) 2878 kmem_dump_begin(); 2879 dumpcfg.helpers_wanted = dumpcfg.clevel > 0; 2880 dumpsys_spinunlock(&dumpcfg.helper_lock); 2881 } 2882 2883 /* run main task */ 2884 dumpsys_main_task(ds); 2885 2886 ds->elapsed = gethrtime() - ds->start; 2887 if (ds->elapsed < 1) 2888 ds->elapsed = 1; 2889 2890 if (livetaskq != NULL) 2891 taskq_destroy(livetaskq); 2892 2893 if (ds->neednl) { 2894 uprintf("\n"); 2895 ds->neednl = 0; 2896 } 2897 2898 /* record actual pages dumped */ 2899 dumphdr->dump_npages = ds->npages; 2900 2901 /* platform-specific data */ 2902 dumphdr->dump_npages += dump_plat_data(dumpcfg.cbuf[0].buf); 2903 2904 /* note any errors by clearing DF_COMPLETE */ 2905 if (dump_ioerr || ds->npages < dumphdr->dump_npages) 2906 dumphdr->dump_flags &= ~DF_COMPLETE; 2907 2908 /* end of stream blocks */ 2909 datatag = 0; 2910 dumpvp_write(&datatag, sizeof (datatag)); 2911 2912 bzero(&datahdr, sizeof (datahdr)); 2913 2914 /* buffer for metrics */ 2915 buf = dumpcfg.cbuf[0].buf; 2916 size = MIN(dumpcfg.cbuf[0].size, DUMP_OFFSET - sizeof (dumphdr_t) - 2917 sizeof (dumpdatahdr_t)); 2918 2919 /* finish the kmem intercepts, collect kmem verbose info */ 2920 if (panicstr) { 2921 datahdr.dump_metrics = kmem_dump_finish(buf, size); 2922 buf += datahdr.dump_metrics; 2923 size -= datahdr.dump_metrics; 2924 } 2925 2926 /* record in the header whether this is a fault-management panic */ 2927 if (panicstr) 2928 dumphdr->dump_fm_panic = is_fm_panic(); 2929 2930 /* compression info in data header */ 2931 datahdr.dump_datahdr_magic = DUMP_DATAHDR_MAGIC; 2932 datahdr.dump_datahdr_version = DUMP_DATAHDR_VERSION; 2933 datahdr.dump_maxcsize = CBUF_SIZE; 2934 datahdr.dump_maxrange = CBUF_MAPSIZE / PAGESIZE; 2935 datahdr.dump_nstreams = dumpcfg.nhelper_used; 2936 datahdr.dump_clevel = dumpcfg.clevel; 2937 #ifdef COLLECT_METRICS 2938 if (dump_metrics_on) 2939 datahdr.dump_metrics += dumpsys_metrics(ds, buf, size); 2940 #endif 2941 datahdr.dump_data_csize = dumpvp_flush() - dumphdr->dump_data; 2942 2943 /* 2944 * Write out the initial and terminal dump headers. 2945 */ 2946 dumpbuf.vp_off = dumphdr->dump_start; 2947 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 2948 (void) dumpvp_flush(); 2949 2950 dumpbuf.vp_limit = dumpvp_size; 2951 dumpbuf.vp_off = dumpbuf.vp_limit - DUMP_OFFSET; 2952 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 2953 dumpvp_write(&datahdr, sizeof (dumpdatahdr_t)); 2954 dumpvp_write(dumpcfg.cbuf[0].buf, datahdr.dump_metrics); 2955 2956 (void) dumpvp_flush(); 2957 2958 uprintf("\r%3d%% done: %llu pages dumped, ", 2959 ds->percent_done, (u_longlong_t)ds->npages); 2960 2961 if (dump_ioerr == 0) { 2962 uprintf("dump succeeded\n"); 2963 } else { 2964 uprintf("dump failed: error %d\n", dump_ioerr); 2965 #ifdef DEBUG 2966 if (panicstr) 2967 debug_enter("dump failed"); 2968 #endif 2969 } 2970 2971 /* 2972 * Write out all undelivered messages. This has to be the *last* 2973 * thing we do because the dump process itself emits messages. 2974 */ 2975 if (panicstr) { 2976 dump_summary(); 2977 dump_ereports(); 2978 dump_messages(); 2979 } 2980 2981 delay(2 * hz); /* let people see the 'done' message */ 2982 dump_timeleft = 0; 2983 dump_ioerr = 0; 2984 2985 /* restore settings after live dump completes */ 2986 if (!panicstr) { 2987 dumpcfg.clevel = save_dump_clevel; 2988 2989 /* release any VCHR open of the dump device */ 2990 if (dumpbuf.cdev_vp != NULL) { 2991 (void) VOP_CLOSE(dumpbuf.cdev_vp, FREAD | FWRITE, 1, 0, 2992 kcred, NULL); 2993 VN_RELE(dumpbuf.cdev_vp); 2994 dumpbuf.cdev_vp = NULL; 2995 } 2996 } 2997 } 2998 2999 /* 3000 * This function is called whenever the memory size, as represented 3001 * by the phys_install list, changes. 3002 */ 3003 void 3004 dump_resize() 3005 { 3006 mutex_enter(&dump_lock); 3007 dumphdr_init(); 3008 dumpbuf_resize(); 3009 dump_update_clevel(); 3010 mutex_exit(&dump_lock); 3011 } 3012 3013 /* 3014 * This function allows for dynamic resizing of a dump area. It assumes that 3015 * the underlying device has update its appropriate size(9P). 3016 */ 3017 int 3018 dumpvp_resize() 3019 { 3020 int error; 3021 vattr_t vattr; 3022 3023 mutex_enter(&dump_lock); 3024 vattr.va_mask = AT_SIZE; 3025 if ((error = VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL)) != 0) { 3026 mutex_exit(&dump_lock); 3027 return (error); 3028 } 3029 3030 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) { 3031 mutex_exit(&dump_lock); 3032 return (ENOSPC); 3033 } 3034 3035 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 3036 mutex_exit(&dump_lock); 3037 return (0); 3038 } 3039 3040 int 3041 dump_set_uuid(const char *uuidstr) 3042 { 3043 const char *ptr; 3044 int i; 3045 3046 if (uuidstr == NULL || strnlen(uuidstr, 36 + 1) != 36) 3047 return (EINVAL); 3048 3049 /* uuid_parse is not common code so check manually */ 3050 for (i = 0, ptr = uuidstr; i < 36; i++, ptr++) { 3051 switch (i) { 3052 case 8: 3053 case 13: 3054 case 18: 3055 case 23: 3056 if (*ptr != '-') 3057 return (EINVAL); 3058 break; 3059 3060 default: 3061 if (!isxdigit(*ptr)) 3062 return (EINVAL); 3063 break; 3064 } 3065 } 3066 3067 if (dump_osimage_uuid[0] != '\0') 3068 return (EALREADY); 3069 3070 (void) strncpy(dump_osimage_uuid, uuidstr, 36 + 1); 3071 3072 cmn_err(CE_CONT, "?This Solaris instance has UUID %s\n", 3073 dump_osimage_uuid); 3074 3075 return (0); 3076 } 3077 3078 const char * 3079 dump_get_uuid(void) 3080 { 3081 return (dump_osimage_uuid[0] != '\0' ? dump_osimage_uuid : ""); 3082 } 3083