1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/vm.h> 31 #include <sys/proc.h> 32 #include <sys/file.h> 33 #include <sys/conf.h> 34 #include <sys/kmem.h> 35 #include <sys/mem.h> 36 #include <sys/mman.h> 37 #include <sys/vnode.h> 38 #include <sys/errno.h> 39 #include <sys/memlist.h> 40 #include <sys/dumphdr.h> 41 #include <sys/dumpadm.h> 42 #include <sys/ksyms.h> 43 #include <sys/compress.h> 44 #include <sys/stream.h> 45 #include <sys/strsun.h> 46 #include <sys/cmn_err.h> 47 #include <sys/bitmap.h> 48 #include <sys/modctl.h> 49 #include <sys/utsname.h> 50 #include <sys/systeminfo.h> 51 #include <sys/vmem.h> 52 #include <sys/log.h> 53 #include <sys/var.h> 54 #include <sys/debug.h> 55 #include <sys/sunddi.h> 56 #include <fs/fs_subr.h> 57 #include <sys/fs/snode.h> 58 #include <sys/ontrap.h> 59 #include <sys/panic.h> 60 #include <sys/dkio.h> 61 #include <sys/vtoc.h> 62 #include <sys/errorq.h> 63 #include <sys/fm/util.h> 64 #include <sys/fs/zfs.h> 65 66 #include <vm/hat.h> 67 #include <vm/as.h> 68 #include <vm/page.h> 69 #include <vm/pvn.h> 70 #include <vm/seg.h> 71 #include <vm/seg_kmem.h> 72 73 #include <bzip2/bzlib.h> 74 75 /* 76 * Crash dump time is dominated by disk write time. To reduce this, 77 * the stronger compression method bzip2 is applied to reduce the dump 78 * size and hence reduce I/O time. However, bzip2 is much more 79 * computationally expensive than the existing lzjb algorithm, so to 80 * avoid increasing compression time, CPUs that are otherwise idle 81 * during panic are employed to parallelize the compression task. 82 * Many helper CPUs are needed to prevent bzip2 from being a 83 * bottleneck, and on systems with too few CPUs, the lzjb algorithm is 84 * parallelized instead. Lastly, I/O and compression are performed by 85 * different CPUs, and are hence overlapped in time, unlike the older 86 * serial code. 87 * 88 * Another important consideration is the speed of the dump 89 * device. Faster disks need less CPUs in order to benefit from 90 * parallel lzjb versus parallel bzip2. Therefore, the CPU count 91 * threshold for switching from parallel lzjb to paralled bzip2 is 92 * elevated for faster disks. The dump device speed is adduced from 93 * the setting for dumpbuf.iosize, see dump_update_clevel. 94 */ 95 96 /* 97 * exported vars 98 */ 99 kmutex_t dump_lock; /* lock for dump configuration */ 100 dumphdr_t *dumphdr; /* dump header */ 101 int dump_conflags = DUMP_KERNEL; /* dump configuration flags */ 102 vnode_t *dumpvp; /* dump device vnode pointer */ 103 u_offset_t dumpvp_size; /* size of dump device, in bytes */ 104 char *dumppath; /* pathname of dump device */ 105 int dump_timeout = 120; /* timeout for dumping pages */ 106 int dump_timeleft; /* portion of dump_timeout remaining */ 107 int dump_ioerr; /* dump i/o error */ 108 int dump_check_used; /* enable check for used pages */ 109 110 /* 111 * Tunables for dump compression and parallelism. These can be set via 112 * /etc/system. 113 * 114 * dump_ncpu_low number of helpers for parallel lzjb 115 * This is also the minimum configuration. 116 * 117 * dump_bzip2_level bzip2 compression level: 1-9 118 * Higher numbers give greater compression, but take more memory 119 * and time. Memory used per helper is ~(dump_bzip2_level * 1MB). 120 * 121 * dump_plat_mincpu the cross-over limit for using bzip2 (per platform): 122 * if dump_plat_mincpu == 0, then always do single threaded dump 123 * if ncpu >= dump_plat_mincpu then try to use bzip2 124 * 125 * dump_metrics_on if set, metrics are collected in the kernel, passed 126 * to savecore via the dump file, and recorded by savecore in 127 * METRICS.txt. 128 */ 129 uint_t dump_ncpu_low = 4; /* minimum config for parallel lzjb */ 130 uint_t dump_bzip2_level = 1; /* bzip2 level (1-9) */ 131 132 /* Define multiple buffers per helper to avoid stalling */ 133 #define NCBUF_PER_HELPER 2 134 #define NCMAP_PER_HELPER 4 135 136 /* minimum number of helpers configured */ 137 #define MINHELPERS (dump_ncpu_low) 138 #define MINCBUFS (MINHELPERS * NCBUF_PER_HELPER) 139 140 /* 141 * Define constant parameters. 142 * 143 * CBUF_SIZE size of an output buffer 144 * 145 * CBUF_MAPSIZE size of virtual range for mapping pages 146 * 147 * CBUF_MAPNP size of virtual range in pages 148 * 149 */ 150 #define DUMP_1KB ((size_t)1 << 10) 151 #define DUMP_1MB ((size_t)1 << 20) 152 #define CBUF_SIZE ((size_t)1 << 17) 153 #define CBUF_MAPSHIFT (22) 154 #define CBUF_MAPSIZE ((size_t)1 << CBUF_MAPSHIFT) 155 #define CBUF_MAPNP ((size_t)1 << (CBUF_MAPSHIFT - PAGESHIFT)) 156 157 /* 158 * Compression metrics are accumulated nano-second subtotals. The 159 * results are normalized by the number of pages dumped. A report is 160 * generated when dumpsys() completes and is saved in the dump image 161 * after the trailing dump header. 162 * 163 * Metrics are always collected. Set the variable dump_metrics_on to 164 * cause metrics to be saved in the crash file, where savecore will 165 * save it in the file METRICS.txt. 166 */ 167 #define PERPAGES \ 168 PERPAGE(bitmap) PERPAGE(map) PERPAGE(unmap) \ 169 PERPAGE(copy) PERPAGE(compress) \ 170 PERPAGE(write) \ 171 PERPAGE(inwait) PERPAGE(outwait) 172 173 typedef struct perpage { 174 #define PERPAGE(x) hrtime_t x; 175 PERPAGES 176 #undef PERPAGE 177 } perpage_t; 178 179 /* 180 * This macro controls the code generation for collecting dump 181 * performance information. By default, the code is generated, but 182 * automatic saving of the information is disabled. If dump_metrics_on 183 * is set to 1, the timing information is passed to savecore via the 184 * crash file, where it is appended to the file dump-dir/METRICS.txt. 185 */ 186 #define COLLECT_METRICS 187 188 #ifdef COLLECT_METRICS 189 uint_t dump_metrics_on = 0; /* set to 1 to enable recording metrics */ 190 191 #define HRSTART(v, m) v##ts.m = gethrtime() 192 #define HRSTOP(v, m) v.m += gethrtime() - v##ts.m 193 #define HRBEGIN(v, m, s) v##ts.m = gethrtime(); v.size += s 194 #define HREND(v, m) v.m += gethrtime() - v##ts.m 195 #define HRNORM(v, m, n) v.m /= (n) 196 197 #else 198 #define HRSTART(v, m) 199 #define HRSTOP(v, m) 200 #define HRBEGIN(v, m, s) 201 #define HREND(v, m) 202 #define HRNORM(v, m, n) 203 #endif /* COLLECT_METRICS */ 204 205 /* 206 * Buffers for copying and compressing memory pages. 207 * 208 * cbuf_t buffer controllers: used for both input and output. 209 * 210 * The buffer state indicates how it is being used: 211 * 212 * CBUF_FREEMAP: CBUF_MAPSIZE virtual address range is available for 213 * mapping input pages. 214 * 215 * CBUF_INREADY: input pages are mapped and ready for compression by a 216 * helper. 217 * 218 * CBUF_USEDMAP: mapping has been consumed by a helper. Needs unmap. 219 * 220 * CBUF_FREEBUF: CBUF_SIZE output buffer, which is available. 221 * 222 * CBUF_WRITE: CBUF_SIZE block of compressed pages from a helper, 223 * ready to write out. 224 * 225 * CBUF_ERRMSG: CBUF_SIZE block of error messages from a helper 226 * (reports UE errors.) 227 */ 228 229 typedef enum cbufstate { 230 CBUF_FREEMAP, 231 CBUF_INREADY, 232 CBUF_USEDMAP, 233 CBUF_FREEBUF, 234 CBUF_WRITE, 235 CBUF_ERRMSG 236 } cbufstate_t; 237 238 typedef struct cbuf cbuf_t; 239 240 struct cbuf { 241 cbuf_t *next; /* next in list */ 242 cbufstate_t state; /* processing state */ 243 size_t used; /* amount used */ 244 size_t size; /* mem size */ 245 char *buf; /* kmem or vmem */ 246 pgcnt_t pagenum; /* index to pfn map */ 247 pgcnt_t bitnum; /* first set bitnum */ 248 pfn_t pfn; /* first pfn in mapped range */ 249 int off; /* byte offset to first pfn */ 250 }; 251 252 /* 253 * cqueue_t queues: a uni-directional channel for communication 254 * from the master to helper tasks or vice-versa using put and 255 * get primitives. Both mappings and data buffers are passed via 256 * queues. Producers close a queue when done. The number of 257 * active producers is reference counted so the consumer can 258 * detect end of data. Concurrent access is mediated by atomic 259 * operations for panic dump, or mutex/cv for live dump. 260 * 261 * There a four queues, used as follows: 262 * 263 * Queue Dataflow NewState 264 * -------------------------------------------------- 265 * mainq master -> master FREEMAP 266 * master has initialized or unmapped an input buffer 267 * -------------------------------------------------- 268 * helperq master -> helper INREADY 269 * master has mapped input for use by helper 270 * -------------------------------------------------- 271 * mainq master <- helper USEDMAP 272 * helper is done with input 273 * -------------------------------------------------- 274 * freebufq master -> helper FREEBUF 275 * master has initialized or written an output buffer 276 * -------------------------------------------------- 277 * mainq master <- helper WRITE 278 * block of compressed pages from a helper 279 * -------------------------------------------------- 280 * mainq master <- helper ERRMSG 281 * error messages from a helper (memory error case) 282 * -------------------------------------------------- 283 * writerq master <- master WRITE 284 * non-blocking queue of blocks to write 285 * -------------------------------------------------- 286 */ 287 typedef struct cqueue { 288 cbuf_t *volatile first; /* first in list */ 289 cbuf_t *last; /* last in list */ 290 hrtime_t ts; /* timestamp */ 291 hrtime_t empty; /* total time empty */ 292 kmutex_t mutex; /* live state lock */ 293 kcondvar_t cv; /* live wait var */ 294 lock_t spinlock; /* panic mode spin lock */ 295 volatile uint_t open; /* producer ref count */ 296 } cqueue_t; 297 298 /* 299 * Convenience macros for using the cqueue functions 300 * Note that the caller must have defined "dumpsync_t *ds" 301 */ 302 #define CQ_IS_EMPTY(q) \ 303 (ds->q.first == NULL) 304 305 #define CQ_OPEN(q) \ 306 atomic_inc_uint(&ds->q.open) 307 308 #define CQ_CLOSE(q) \ 309 dumpsys_close_cq(&ds->q, ds->live) 310 311 #define CQ_PUT(q, cp, st) \ 312 dumpsys_put_cq(&ds->q, cp, st, ds->live) 313 314 #define CQ_GET(q) \ 315 dumpsys_get_cq(&ds->q, ds->live) 316 317 /* 318 * Dynamic state when dumpsys() is running. 319 */ 320 typedef struct dumpsync { 321 pgcnt_t npages; /* subtotal of pages dumped */ 322 pgcnt_t pages_mapped; /* subtotal of pages mapped */ 323 pgcnt_t pages_used; /* subtotal of pages used per map */ 324 size_t nwrite; /* subtotal of bytes written */ 325 uint_t live; /* running live dump */ 326 uint_t neednl; /* will need to print a newline */ 327 uint_t percent; /* dump progress */ 328 uint_t percent_done; /* dump progress reported */ 329 cqueue_t freebufq; /* free kmem bufs for writing */ 330 cqueue_t mainq; /* input for main task */ 331 cqueue_t helperq; /* input for helpers */ 332 cqueue_t writerq; /* input for writer */ 333 hrtime_t start; /* start time */ 334 hrtime_t elapsed; /* elapsed time when completed */ 335 hrtime_t iotime; /* time spent writing nwrite bytes */ 336 hrtime_t iowait; /* time spent waiting for output */ 337 hrtime_t iowaitts; /* iowait timestamp */ 338 perpage_t perpage; /* metrics */ 339 perpage_t perpagets; 340 int dumpcpu; /* master cpu */ 341 } dumpsync_t; 342 343 static dumpsync_t dumpsync; /* synchronization vars */ 344 345 /* 346 * helper_t helpers: contains the context for a stream. CPUs run in 347 * parallel at dump time; each CPU creates a single stream of 348 * compression data. Stream data is divided into CBUF_SIZE blocks. 349 * The blocks are written in order within a stream. But, blocks from 350 * multiple streams can be interleaved. Each stream is identified by a 351 * unique tag. 352 */ 353 typedef struct helper { 354 int helper; /* bound helper id */ 355 int tag; /* compression stream tag */ 356 perpage_t perpage; /* per page metrics */ 357 perpage_t perpagets; /* per page metrics (timestamps) */ 358 taskqid_t taskqid; /* live dump task ptr */ 359 int in, out; /* buffer offsets */ 360 cbuf_t *cpin, *cpout, *cperr; /* cbuf objects in process */ 361 dumpsync_t *ds; /* pointer to sync vars */ 362 size_t used; /* counts input consumed */ 363 char *page; /* buffer for page copy */ 364 char *lzbuf; /* lzjb output */ 365 bz_stream bzstream; /* bzip2 state */ 366 } helper_t; 367 368 #define MAINHELPER (-1) /* helper is also the main task */ 369 #define FREEHELPER (-2) /* unbound helper */ 370 #define DONEHELPER (-3) /* helper finished */ 371 372 /* 373 * configuration vars for dumpsys 374 */ 375 typedef struct dumpcfg { 376 int threshold; /* ncpu threshold for bzip2 */ 377 int nhelper; /* number of helpers */ 378 int nhelper_used; /* actual number of helpers used */ 379 int ncmap; /* number VA pages for compression */ 380 int ncbuf; /* number of bufs for compression */ 381 int ncbuf_used; /* number of bufs in use */ 382 uint_t clevel; /* dump compression level */ 383 helper_t *helper; /* array of helpers */ 384 cbuf_t *cmap; /* array of input (map) buffers */ 385 cbuf_t *cbuf; /* array of output buffers */ 386 ulong_t *helpermap; /* set of dumpsys helper CPU ids */ 387 ulong_t *bitmap; /* bitmap for marking pages to dump */ 388 ulong_t *rbitmap; /* bitmap for used CBUF_MAPSIZE ranges */ 389 pgcnt_t bitmapsize; /* size of bitmap */ 390 pgcnt_t rbitmapsize; /* size of bitmap for ranges */ 391 pgcnt_t found4m; /* number ranges allocated by dump */ 392 pgcnt_t foundsm; /* number small pages allocated by dump */ 393 pid_t *pids; /* list of process IDs at dump time */ 394 size_t maxsize; /* memory size needed at dump time */ 395 size_t maxvmsize; /* size of reserved VM */ 396 char *maxvm; /* reserved VM for spare pages */ 397 lock_t helper_lock; /* protect helper state */ 398 char helpers_wanted; /* flag to enable parallelism */ 399 } dumpcfg_t; 400 401 static dumpcfg_t dumpcfg; /* config vars */ 402 403 /* 404 * The dump I/O buffer. 405 * 406 * There is one I/O buffer used by dumpvp_write and dumvp_flush. It is 407 * sized according to the optimum device transfer speed. 408 */ 409 typedef struct dumpbuf { 410 vnode_t *cdev_vp; /* VCHR open of the dump device */ 411 len_t vp_limit; /* maximum write offset */ 412 offset_t vp_off; /* current dump device offset */ 413 char *cur; /* dump write pointer */ 414 char *start; /* dump buffer address */ 415 char *end; /* dump buffer end */ 416 size_t size; /* size of dumpbuf in bytes */ 417 size_t iosize; /* best transfer size for device */ 418 } dumpbuf_t; 419 420 dumpbuf_t dumpbuf; /* I/O buffer */ 421 422 /* 423 * The dump I/O buffer must be at least one page, at most xfer_size 424 * bytes, and should scale with physmem in between. The transfer size 425 * passed in will either represent a global default (maxphys) or the 426 * best size for the device. The size of the dumpbuf I/O buffer is 427 * limited by dumpbuf_limit (8MB by default) because the dump 428 * performance saturates beyond a certain size. The default is to 429 * select 1/4096 of the memory. 430 */ 431 static int dumpbuf_fraction = 12; /* memory size scale factor */ 432 static size_t dumpbuf_limit = 8 * DUMP_1MB; /* max I/O buf size */ 433 434 static size_t 435 dumpbuf_iosize(size_t xfer_size) 436 { 437 size_t iosize = ptob(physmem >> dumpbuf_fraction); 438 439 if (iosize < PAGESIZE) 440 iosize = PAGESIZE; 441 else if (iosize > xfer_size) 442 iosize = xfer_size; 443 if (iosize > dumpbuf_limit) 444 iosize = dumpbuf_limit; 445 return (iosize & PAGEMASK); 446 } 447 448 /* 449 * resize the I/O buffer 450 */ 451 static void 452 dumpbuf_resize(void) 453 { 454 char *old_buf = dumpbuf.start; 455 size_t old_size = dumpbuf.size; 456 char *new_buf; 457 size_t new_size; 458 459 ASSERT(MUTEX_HELD(&dump_lock)); 460 461 new_size = dumpbuf_iosize(MAX(dumpbuf.iosize, maxphys)); 462 if (new_size <= old_size) 463 return; /* no need to reallocate buffer */ 464 465 new_buf = kmem_alloc(new_size, KM_SLEEP); 466 dumpbuf.size = new_size; 467 dumpbuf.start = new_buf; 468 dumpbuf.end = new_buf + new_size; 469 kmem_free(old_buf, old_size); 470 } 471 472 /* 473 * dump_update_clevel is called when dumpadm configures the dump device. 474 * Calculate number of helpers and buffers. 475 * Allocate the minimum configuration for now. 476 * 477 * When the dump file is configured we reserve a minimum amount of 478 * memory for use at crash time. But we reserve VA for all the memory 479 * we really want in order to do the fastest dump possible. The VA is 480 * backed by pages not being dumped, according to the bitmap. If 481 * there is insufficient spare memory, however, we fall back to the 482 * minimum. 483 * 484 * Live dump (savecore -L) always uses the minimum config. 485 * 486 * clevel 0 is single threaded lzjb 487 * clevel 1 is parallel lzjb 488 * clevel 2 is parallel bzip2 489 * 490 * The ncpu threshold is selected with dump_plat_mincpu. 491 * On OPL, set_platform_defaults() overrides the sun4u setting. 492 * The actual values are defined via DUMP_PLAT_*_MINCPU macros. 493 * 494 * Architecture Threshold Algorithm 495 * sun4u < 51 parallel lzjb 496 * sun4u >= 51 parallel bzip2(*) 497 * sun4u OPL < 8 parallel lzjb 498 * sun4u OPL >= 8 parallel bzip2(*) 499 * sun4v < 128 parallel lzjb 500 * sun4v >= 128 parallel bzip2(*) 501 * x86 < 11 parallel lzjb 502 * x86 >= 11 parallel bzip2(*) 503 * 32-bit N/A single-threaded lzjb 504 * 505 * (*) bzip2 is only chosen if there is sufficient available 506 * memory for buffers at dump time. See dumpsys_get_maxmem(). 507 * 508 * Faster dump devices have larger I/O buffers. The threshold value is 509 * increased according to the size of the dump I/O buffer, because 510 * parallel lzjb performs better with faster disks. For buffers >= 1MB 511 * the threshold is 3X; for buffers >= 256K threshold is 2X. 512 * 513 * For parallel dumps, the number of helpers is ncpu-1. The CPU 514 * running panic runs the main task. For single-threaded dumps, the 515 * panic CPU does lzjb compression (it is tagged as MAINHELPER.) 516 * 517 * Need multiple buffers per helper so that they do not block waiting 518 * for the main task. 519 * parallel single-threaded 520 * Number of output buffers: nhelper*2 1 521 * Number of mapping buffers: nhelper*4 1 522 * 523 */ 524 static void 525 dump_update_clevel() 526 { 527 int tag; 528 size_t bz2size; 529 helper_t *hp, *hpend; 530 cbuf_t *cp, *cpend; 531 dumpcfg_t *old = &dumpcfg; 532 dumpcfg_t newcfg = *old; 533 dumpcfg_t *new = &newcfg; 534 535 ASSERT(MUTEX_HELD(&dump_lock)); 536 537 /* 538 * Free the previously allocated bufs and VM. 539 */ 540 if (old->helper != NULL) { 541 542 /* helpers */ 543 hpend = &old->helper[old->nhelper]; 544 for (hp = old->helper; hp != hpend; hp++) { 545 if (hp->lzbuf != NULL) 546 kmem_free(hp->lzbuf, PAGESIZE); 547 if (hp->page != NULL) 548 kmem_free(hp->page, PAGESIZE); 549 } 550 kmem_free(old->helper, old->nhelper * sizeof (helper_t)); 551 552 /* VM space for mapping pages */ 553 cpend = &old->cmap[old->ncmap]; 554 for (cp = old->cmap; cp != cpend; cp++) 555 vmem_xfree(heap_arena, cp->buf, CBUF_MAPSIZE); 556 kmem_free(old->cmap, old->ncmap * sizeof (cbuf_t)); 557 558 /* output bufs */ 559 cpend = &old->cbuf[old->ncbuf]; 560 for (cp = old->cbuf; cp != cpend; cp++) 561 if (cp->buf != NULL) 562 kmem_free(cp->buf, cp->size); 563 kmem_free(old->cbuf, old->ncbuf * sizeof (cbuf_t)); 564 565 /* reserved VM for dumpsys_get_maxmem */ 566 if (old->maxvmsize > 0) 567 vmem_xfree(heap_arena, old->maxvm, old->maxvmsize); 568 } 569 570 /* 571 * Allocate memory and VM. 572 * One CPU runs dumpsys, the rest are helpers. 573 */ 574 new->nhelper = ncpus - 1; 575 if (new->nhelper < 1) 576 new->nhelper = 1; 577 578 if (new->nhelper > DUMP_MAX_NHELPER) 579 new->nhelper = DUMP_MAX_NHELPER; 580 581 /* increase threshold for faster disks */ 582 new->threshold = dump_plat_mincpu; 583 if (dumpbuf.iosize >= DUMP_1MB) 584 new->threshold *= 3; 585 else if (dumpbuf.iosize >= (256 * DUMP_1KB)) 586 new->threshold *= 2; 587 588 /* figure compression level based upon the computed threshold. */ 589 if (dump_plat_mincpu == 0 || new->nhelper < 2) { 590 new->clevel = 0; 591 new->nhelper = 1; 592 } else if ((new->nhelper + 1) >= new->threshold) { 593 new->clevel = DUMP_CLEVEL_BZIP2; 594 } else { 595 new->clevel = DUMP_CLEVEL_LZJB; 596 } 597 598 if (new->clevel == 0) { 599 new->ncbuf = 1; 600 new->ncmap = 1; 601 } else { 602 new->ncbuf = NCBUF_PER_HELPER * new->nhelper; 603 new->ncmap = NCMAP_PER_HELPER * new->nhelper; 604 } 605 606 /* 607 * Allocate new data structures and buffers for MINHELPERS, 608 * and also figure the max desired size. 609 */ 610 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 611 new->maxsize = 0; 612 new->maxvmsize = 0; 613 new->maxvm = NULL; 614 tag = 1; 615 new->helper = kmem_zalloc(new->nhelper * sizeof (helper_t), KM_SLEEP); 616 hpend = &new->helper[new->nhelper]; 617 for (hp = new->helper; hp != hpend; hp++) { 618 hp->tag = tag++; 619 if (hp < &new->helper[MINHELPERS]) { 620 hp->lzbuf = kmem_alloc(PAGESIZE, KM_SLEEP); 621 hp->page = kmem_alloc(PAGESIZE, KM_SLEEP); 622 } else if (new->clevel < DUMP_CLEVEL_BZIP2) { 623 new->maxsize += 2 * PAGESIZE; 624 } else { 625 new->maxsize += PAGESIZE; 626 } 627 if (new->clevel >= DUMP_CLEVEL_BZIP2) 628 new->maxsize += bz2size; 629 } 630 631 new->cbuf = kmem_zalloc(new->ncbuf * sizeof (cbuf_t), KM_SLEEP); 632 cpend = &new->cbuf[new->ncbuf]; 633 for (cp = new->cbuf; cp != cpend; cp++) { 634 cp->state = CBUF_FREEBUF; 635 cp->size = CBUF_SIZE; 636 if (cp < &new->cbuf[MINCBUFS]) 637 cp->buf = kmem_alloc(cp->size, KM_SLEEP); 638 else 639 new->maxsize += cp->size; 640 } 641 642 new->cmap = kmem_zalloc(new->ncmap * sizeof (cbuf_t), KM_SLEEP); 643 cpend = &new->cmap[new->ncmap]; 644 for (cp = new->cmap; cp != cpend; cp++) { 645 cp->state = CBUF_FREEMAP; 646 cp->size = CBUF_MAPSIZE; 647 cp->buf = vmem_xalloc(heap_arena, CBUF_MAPSIZE, CBUF_MAPSIZE, 648 0, 0, NULL, NULL, VM_SLEEP); 649 } 650 651 /* reserve VA to be backed with spare pages at crash time */ 652 if (new->maxsize > 0) { 653 new->maxsize = P2ROUNDUP(new->maxsize, PAGESIZE); 654 new->maxvmsize = P2ROUNDUP(new->maxsize, CBUF_MAPSIZE); 655 new->maxvm = vmem_xalloc(heap_arena, new->maxvmsize, 656 CBUF_MAPSIZE, 0, 0, NULL, NULL, VM_SLEEP); 657 } 658 659 /* set new config pointers */ 660 *old = *new; 661 } 662 663 /* 664 * Define a struct memlist walker to optimize bitnum to pfn 665 * lookup. The walker maintains the state of the list traversal. 666 */ 667 typedef struct dumpmlw { 668 struct memlist *mp; /* current memlist */ 669 pgcnt_t basenum; /* bitnum base offset */ 670 pgcnt_t mppages; /* current memlist size */ 671 pgcnt_t mpleft; /* size to end of current memlist */ 672 pfn_t mpaddr; /* first pfn in memlist */ 673 } dumpmlw_t; 674 675 /* initialize the walker */ 676 static inline void 677 dump_init_memlist_walker(dumpmlw_t *pw) 678 { 679 pw->mp = phys_install; 680 pw->basenum = 0; 681 pw->mppages = pw->mp->size >> PAGESHIFT; 682 pw->mpleft = pw->mppages; 683 pw->mpaddr = pw->mp->address >> PAGESHIFT; 684 } 685 686 /* 687 * Lookup pfn given bitnum. The memlist can be quite long on some 688 * systems (e.g.: one per board). To optimize sequential lookups, the 689 * caller initializes and presents a memlist walker. 690 */ 691 static pfn_t 692 dump_bitnum_to_pfn(pgcnt_t bitnum, dumpmlw_t *pw) 693 { 694 bitnum -= pw->basenum; 695 while (pw->mp != NULL) { 696 if (bitnum < pw->mppages) { 697 pw->mpleft = pw->mppages - bitnum; 698 return (pw->mpaddr + bitnum); 699 } 700 bitnum -= pw->mppages; 701 pw->basenum += pw->mppages; 702 pw->mp = pw->mp->next; 703 if (pw->mp != NULL) { 704 pw->mppages = pw->mp->size >> PAGESHIFT; 705 pw->mpleft = pw->mppages; 706 pw->mpaddr = pw->mp->address >> PAGESHIFT; 707 } 708 } 709 return (PFN_INVALID); 710 } 711 712 static pgcnt_t 713 dump_pfn_to_bitnum(pfn_t pfn) 714 { 715 struct memlist *mp; 716 pgcnt_t bitnum = 0; 717 718 for (mp = phys_install; mp != NULL; mp = mp->next) { 719 if (pfn >= (mp->address >> PAGESHIFT) && 720 pfn < ((mp->address + mp->size) >> PAGESHIFT)) 721 return (bitnum + pfn - (mp->address >> PAGESHIFT)); 722 bitnum += mp->size >> PAGESHIFT; 723 } 724 return ((pgcnt_t)-1); 725 } 726 727 /* 728 * Set/test bitmap for a CBUF_MAPSIZE range which includes pfn. The 729 * mapping of pfn to range index is imperfect because pfn and bitnum 730 * do not have the same phase. To make sure a CBUF_MAPSIZE range is 731 * covered, call this for both ends: 732 * dump_set_used(base) 733 * dump_set_used(base+CBUF_MAPNP-1) 734 * 735 * This is used during a panic dump to mark pages allocated by 736 * dumpsys_get_maxmem(). The macro IS_DUMP_PAGE(pp) is used by 737 * page_get_mnode_freelist() to make sure pages used by dump are never 738 * allocated. 739 */ 740 #define CBUF_MAPP2R(pfn) ((pfn) >> (CBUF_MAPSHIFT - PAGESHIFT)) 741 742 static void 743 dump_set_used(pfn_t pfn) 744 { 745 746 pgcnt_t bitnum, rbitnum; 747 748 bitnum = dump_pfn_to_bitnum(pfn); 749 ASSERT(bitnum != (pgcnt_t)-1); 750 751 rbitnum = CBUF_MAPP2R(bitnum); 752 ASSERT(rbitnum < dumpcfg.rbitmapsize); 753 754 BT_SET(dumpcfg.rbitmap, rbitnum); 755 } 756 757 int 758 dump_test_used(pfn_t pfn) 759 { 760 pgcnt_t bitnum, rbitnum; 761 762 bitnum = dump_pfn_to_bitnum(pfn); 763 ASSERT(bitnum != (pgcnt_t)-1); 764 765 rbitnum = CBUF_MAPP2R(bitnum); 766 ASSERT(rbitnum < dumpcfg.rbitmapsize); 767 768 return (BT_TEST(dumpcfg.rbitmap, rbitnum)); 769 } 770 771 /* 772 * dumpbzalloc and dumpbzfree are callbacks from the bzip2 library. 773 * dumpsys_get_maxmem() uses them for BZ2_bzCompressInit(). 774 */ 775 static void * 776 dumpbzalloc(void *opaque, int items, int size) 777 { 778 size_t *sz; 779 char *ret; 780 781 ASSERT(opaque != NULL); 782 sz = opaque; 783 ret = dumpcfg.maxvm + *sz; 784 *sz += items * size; 785 *sz = P2ROUNDUP(*sz, BZ2_BZALLOC_ALIGN); 786 ASSERT(*sz <= dumpcfg.maxvmsize); 787 return (ret); 788 } 789 790 /*ARGSUSED*/ 791 static void 792 dumpbzfree(void *opaque, void *addr) 793 { 794 } 795 796 /* 797 * Perform additional checks on the page to see if we can really use 798 * it. The kernel (kas) pages are always set in the bitmap. However, 799 * boot memory pages (prom_ppages or P_BOOTPAGES) are not in the 800 * bitmap. So we check for them. 801 */ 802 static inline int 803 dump_pfn_check(pfn_t pfn) 804 { 805 page_t *pp = page_numtopp_nolock(pfn); 806 #if defined(__sparc) 807 extern struct vnode prom_ppages; 808 #endif 809 810 if (pp == NULL || pp->p_pagenum != pfn || 811 #if defined(__sparc) 812 pp->p_vnode == &prom_ppages || 813 #else 814 PP_ISBOOTPAGES(pp) || 815 #endif 816 pp->p_toxic != 0) 817 return (0); 818 return (1); 819 } 820 821 /* 822 * Check a range to see if all contained pages are available and 823 * return non-zero if the range can be used. 824 */ 825 static inline int 826 dump_range_check(pgcnt_t start, pgcnt_t end, pfn_t pfn) 827 { 828 for (; start < end; start++, pfn++) { 829 if (BT_TEST(dumpcfg.bitmap, start)) 830 return (0); 831 if (!dump_pfn_check(pfn)) 832 return (0); 833 } 834 return (1); 835 } 836 837 /* 838 * dumpsys_get_maxmem() is called during panic. Find unused ranges 839 * and use them for buffers. If we find enough memory switch to 840 * parallel bzip2, otherwise use parallel lzjb. 841 * 842 * It searches the dump bitmap in 2 passes. The first time it looks 843 * for CBUF_MAPSIZE ranges. On the second pass it uses small pages. 844 */ 845 static void 846 dumpsys_get_maxmem() 847 { 848 dumpcfg_t *cfg = &dumpcfg; 849 cbuf_t *endcp = &cfg->cbuf[cfg->ncbuf]; 850 helper_t *endhp = &cfg->helper[cfg->nhelper]; 851 pgcnt_t bitnum, end; 852 size_t sz, endsz, bz2size; 853 pfn_t pfn, off; 854 cbuf_t *cp; 855 helper_t *hp, *ohp; 856 dumpmlw_t mlw; 857 int k; 858 859 if (cfg->maxsize == 0 || cfg->clevel < DUMP_CLEVEL_LZJB || 860 (dump_conflags & DUMP_ALL) != 0) 861 return; 862 863 sz = 0; 864 cfg->found4m = 0; 865 cfg->foundsm = 0; 866 867 /* bitmap of ranges used to estimate which pfns are being used */ 868 bzero(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.rbitmapsize)); 869 870 /* find ranges that are not being dumped to use for buffers */ 871 dump_init_memlist_walker(&mlw); 872 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 873 dump_timeleft = dump_timeout; 874 end = bitnum + CBUF_MAPNP; 875 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 876 ASSERT(pfn != PFN_INVALID); 877 878 /* skip partial range at end of mem segment */ 879 if (mlw.mpleft < CBUF_MAPNP) { 880 end = bitnum + mlw.mpleft; 881 continue; 882 } 883 884 /* skip non aligned pages */ 885 off = P2PHASE(pfn, CBUF_MAPNP); 886 if (off != 0) { 887 end -= off; 888 continue; 889 } 890 891 if (!dump_range_check(bitnum, end, pfn)) 892 continue; 893 894 ASSERT((sz + CBUF_MAPSIZE) <= cfg->maxvmsize); 895 hat_devload(kas.a_hat, cfg->maxvm + sz, CBUF_MAPSIZE, pfn, 896 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 897 sz += CBUF_MAPSIZE; 898 cfg->found4m++; 899 900 /* set the bitmap for both ends to be sure to cover the range */ 901 dump_set_used(pfn); 902 dump_set_used(pfn + CBUF_MAPNP - 1); 903 904 if (sz >= cfg->maxsize) 905 goto foundmax; 906 } 907 908 /* Add small pages if we can't find enough large pages. */ 909 dump_init_memlist_walker(&mlw); 910 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) { 911 dump_timeleft = dump_timeout; 912 end = bitnum + CBUF_MAPNP; 913 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 914 ASSERT(pfn != PFN_INVALID); 915 916 /* Find any non-aligned pages at start and end of segment. */ 917 off = P2PHASE(pfn, CBUF_MAPNP); 918 if (mlw.mpleft < CBUF_MAPNP) { 919 end = bitnum + mlw.mpleft; 920 } else if (off != 0) { 921 end -= off; 922 } else if (cfg->found4m && dump_test_used(pfn)) { 923 continue; 924 } 925 926 for (; bitnum < end; bitnum++, pfn++) { 927 dump_timeleft = dump_timeout; 928 if (BT_TEST(dumpcfg.bitmap, bitnum)) 929 continue; 930 if (!dump_pfn_check(pfn)) 931 continue; 932 ASSERT((sz + PAGESIZE) <= cfg->maxvmsize); 933 hat_devload(kas.a_hat, cfg->maxvm + sz, PAGESIZE, pfn, 934 PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST); 935 sz += PAGESIZE; 936 cfg->foundsm++; 937 dump_set_used(pfn); 938 if (sz >= cfg->maxsize) 939 goto foundmax; 940 } 941 } 942 943 /* Fall back to lzjb if we did not get enough memory for bzip2. */ 944 endsz = (cfg->maxsize * cfg->threshold) / cfg->nhelper; 945 if (sz < endsz) { 946 cfg->clevel = DUMP_CLEVEL_LZJB; 947 } 948 949 /* Allocate memory for as many helpers as we can. */ 950 foundmax: 951 952 /* Byte offsets into memory found and mapped above */ 953 endsz = sz; 954 sz = 0; 955 956 /* Set the size for bzip2 state. Only bzip2 needs it. */ 957 bz2size = BZ2_bzCompressInitSize(dump_bzip2_level); 958 959 /* Skip the preallocate output buffers. */ 960 cp = &cfg->cbuf[MINCBUFS]; 961 962 /* Use this to move memory up from the preallocated helpers. */ 963 ohp = cfg->helper; 964 965 /* Loop over all helpers and allocate memory. */ 966 for (hp = cfg->helper; hp < endhp; hp++) { 967 968 /* Skip preallocated helpers by checking hp->page. */ 969 if (hp->page == NULL) { 970 if (cfg->clevel <= DUMP_CLEVEL_LZJB) { 971 /* lzjb needs 2 1-page buffers */ 972 if ((sz + (2 * PAGESIZE)) > endsz) 973 break; 974 hp->page = cfg->maxvm + sz; 975 sz += PAGESIZE; 976 hp->lzbuf = cfg->maxvm + sz; 977 sz += PAGESIZE; 978 979 } else if (ohp->lzbuf != NULL) { 980 /* re-use the preallocted lzjb page for bzip2 */ 981 hp->page = ohp->lzbuf; 982 ohp->lzbuf = NULL; 983 ++ohp; 984 985 } else { 986 /* bzip2 needs a 1-page buffer */ 987 if ((sz + PAGESIZE) > endsz) 988 break; 989 hp->page = cfg->maxvm + sz; 990 sz += PAGESIZE; 991 } 992 } 993 994 /* 995 * Add output buffers per helper. The number of 996 * buffers per helper is determined by the ratio of 997 * ncbuf to nhelper. 998 */ 999 for (k = 0; cp < endcp && (sz + CBUF_SIZE) <= endsz && 1000 k < NCBUF_PER_HELPER; k++) { 1001 cp->state = CBUF_FREEBUF; 1002 cp->size = CBUF_SIZE; 1003 cp->buf = cfg->maxvm + sz; 1004 sz += CBUF_SIZE; 1005 ++cp; 1006 } 1007 1008 /* 1009 * bzip2 needs compression state. Use the dumpbzalloc 1010 * and dumpbzfree callbacks to allocate the memory. 1011 * bzip2 does allocation only at init time. 1012 */ 1013 if (cfg->clevel >= DUMP_CLEVEL_BZIP2) { 1014 if ((sz + bz2size) > endsz) { 1015 hp->page = NULL; 1016 break; 1017 } else { 1018 hp->bzstream.opaque = &sz; 1019 hp->bzstream.bzalloc = dumpbzalloc; 1020 hp->bzstream.bzfree = dumpbzfree; 1021 (void) BZ2_bzCompressInit(&hp->bzstream, 1022 dump_bzip2_level, 0, 0); 1023 hp->bzstream.opaque = NULL; 1024 } 1025 } 1026 } 1027 1028 /* Finish allocating output buffers */ 1029 for (; cp < endcp && (sz + CBUF_SIZE) <= endsz; cp++) { 1030 cp->state = CBUF_FREEBUF; 1031 cp->size = CBUF_SIZE; 1032 cp->buf = cfg->maxvm + sz; 1033 sz += CBUF_SIZE; 1034 } 1035 1036 /* Enable IS_DUMP_PAGE macro, which checks for pages we took. */ 1037 if (cfg->found4m || cfg->foundsm) 1038 dump_check_used = 1; 1039 1040 ASSERT(sz <= endsz); 1041 } 1042 1043 static void 1044 dumphdr_init(void) 1045 { 1046 pgcnt_t npages = 0; 1047 1048 ASSERT(MUTEX_HELD(&dump_lock)); 1049 1050 if (dumphdr == NULL) { 1051 dumphdr = kmem_zalloc(sizeof (dumphdr_t), KM_SLEEP); 1052 dumphdr->dump_magic = DUMP_MAGIC; 1053 dumphdr->dump_version = DUMP_VERSION; 1054 dumphdr->dump_wordsize = DUMP_WORDSIZE; 1055 dumphdr->dump_pageshift = PAGESHIFT; 1056 dumphdr->dump_pagesize = PAGESIZE; 1057 dumphdr->dump_utsname = utsname; 1058 (void) strcpy(dumphdr->dump_platform, platform); 1059 dumpbuf.size = dumpbuf_iosize(maxphys); 1060 dumpbuf.start = kmem_alloc(dumpbuf.size, KM_SLEEP); 1061 dumpbuf.end = dumpbuf.start + dumpbuf.size; 1062 dumpcfg.pids = kmem_alloc(v.v_proc * sizeof (pid_t), KM_SLEEP); 1063 dumpcfg.helpermap = kmem_zalloc(BT_SIZEOFMAP(NCPU), KM_SLEEP); 1064 LOCK_INIT_HELD(&dumpcfg.helper_lock); 1065 } 1066 1067 npages = num_phys_pages(); 1068 1069 if (dumpcfg.bitmapsize != npages) { 1070 size_t rlen = CBUF_MAPP2R(P2ROUNDUP(npages, CBUF_MAPNP)); 1071 void *map = kmem_alloc(BT_SIZEOFMAP(npages), KM_SLEEP); 1072 void *rmap = kmem_alloc(BT_SIZEOFMAP(rlen), KM_SLEEP); 1073 1074 if (dumpcfg.bitmap != NULL) 1075 kmem_free(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg. 1076 bitmapsize)); 1077 if (dumpcfg.rbitmap != NULL) 1078 kmem_free(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg. 1079 rbitmapsize)); 1080 dumpcfg.bitmap = map; 1081 dumpcfg.bitmapsize = npages; 1082 dumpcfg.rbitmap = rmap; 1083 dumpcfg.rbitmapsize = rlen; 1084 } 1085 } 1086 1087 /* 1088 * Establish a new dump device. 1089 */ 1090 int 1091 dumpinit(vnode_t *vp, char *name, int justchecking) 1092 { 1093 vnode_t *cvp; 1094 vattr_t vattr; 1095 vnode_t *cdev_vp; 1096 int error = 0; 1097 1098 ASSERT(MUTEX_HELD(&dump_lock)); 1099 1100 dumphdr_init(); 1101 1102 cvp = common_specvp(vp); 1103 if (cvp == dumpvp) 1104 return (0); 1105 1106 /* 1107 * Determine whether this is a plausible dump device. We want either: 1108 * (1) a real device that's not mounted and has a cb_dump routine, or 1109 * (2) a swapfile on some filesystem that has a vop_dump routine. 1110 */ 1111 if ((error = VOP_OPEN(&cvp, FREAD | FWRITE, kcred, NULL)) != 0) 1112 return (error); 1113 1114 vattr.va_mask = AT_SIZE | AT_TYPE | AT_RDEV; 1115 if ((error = VOP_GETATTR(cvp, &vattr, 0, kcred, NULL)) == 0) { 1116 if (vattr.va_type == VBLK || vattr.va_type == VCHR) { 1117 if (devopsp[getmajor(vattr.va_rdev)]-> 1118 devo_cb_ops->cb_dump == nodev) 1119 error = ENOTSUP; 1120 else if (vfs_devismounted(vattr.va_rdev)) 1121 error = EBUSY; 1122 if (strcmp(ddi_driver_name(VTOS(cvp)->s_dip), 1123 ZFS_DRIVER) == 0 && 1124 IS_SWAPVP(common_specvp(cvp))) 1125 error = EBUSY; 1126 } else { 1127 if (vn_matchopval(cvp, VOPNAME_DUMP, fs_nosys) || 1128 !IS_SWAPVP(cvp)) 1129 error = ENOTSUP; 1130 } 1131 } 1132 1133 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) 1134 error = ENOSPC; 1135 1136 if (error || justchecking) { 1137 (void) VOP_CLOSE(cvp, FREAD | FWRITE, 1, (offset_t)0, 1138 kcred, NULL); 1139 return (error); 1140 } 1141 1142 VN_HOLD(cvp); 1143 1144 if (dumpvp != NULL) 1145 dumpfini(); /* unconfigure the old dump device */ 1146 1147 dumpvp = cvp; 1148 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 1149 dumppath = kmem_alloc(strlen(name) + 1, KM_SLEEP); 1150 (void) strcpy(dumppath, name); 1151 dumpbuf.iosize = 0; 1152 1153 /* 1154 * If the dump device is a block device, attempt to open up the 1155 * corresponding character device and determine its maximum transfer 1156 * size. We use this information to potentially resize dumpbuf to a 1157 * larger and more optimal size for performing i/o to the dump device. 1158 */ 1159 if (cvp->v_type == VBLK && 1160 (cdev_vp = makespecvp(VTOS(cvp)->s_dev, VCHR)) != NULL) { 1161 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1162 size_t blk_size; 1163 struct dk_cinfo dki; 1164 struct dk_minfo minf; 1165 1166 if (VOP_IOCTL(cdev_vp, DKIOCGMEDIAINFO, 1167 (intptr_t)&minf, FKIOCTL, kcred, NULL, NULL) 1168 == 0 && minf.dki_lbsize != 0) 1169 blk_size = minf.dki_lbsize; 1170 else 1171 blk_size = DEV_BSIZE; 1172 1173 if (VOP_IOCTL(cdev_vp, DKIOCINFO, (intptr_t)&dki, 1174 FKIOCTL, kcred, NULL, NULL) == 0) { 1175 dumpbuf.iosize = dki.dki_maxtransfer * blk_size; 1176 dumpbuf_resize(); 1177 } 1178 /* 1179 * If we are working with a zvol then dumpify it 1180 * if it's not being used as swap. 1181 */ 1182 if (strcmp(dki.dki_dname, ZVOL_DRIVER) == 0) { 1183 if (IS_SWAPVP(common_specvp(cvp))) 1184 error = EBUSY; 1185 else if ((error = VOP_IOCTL(cdev_vp, 1186 DKIOCDUMPINIT, NULL, FKIOCTL, kcred, 1187 NULL, NULL)) != 0) 1188 dumpfini(); 1189 } 1190 1191 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1192 kcred, NULL); 1193 } 1194 1195 VN_RELE(cdev_vp); 1196 } 1197 1198 cmn_err(CE_CONT, "?dump on %s size %llu MB\n", name, dumpvp_size >> 20); 1199 1200 dump_update_clevel(); 1201 1202 return (error); 1203 } 1204 1205 void 1206 dumpfini(void) 1207 { 1208 vattr_t vattr; 1209 boolean_t is_zfs = B_FALSE; 1210 vnode_t *cdev_vp; 1211 ASSERT(MUTEX_HELD(&dump_lock)); 1212 1213 kmem_free(dumppath, strlen(dumppath) + 1); 1214 1215 /* 1216 * Determine if we are using zvols for our dump device 1217 */ 1218 vattr.va_mask = AT_RDEV; 1219 if (VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL) == 0) { 1220 is_zfs = (getmajor(vattr.va_rdev) == 1221 ddi_name_to_major(ZFS_DRIVER)) ? B_TRUE : B_FALSE; 1222 } 1223 1224 /* 1225 * If we have a zvol dump device then we call into zfs so 1226 * that it may have a chance to cleanup. 1227 */ 1228 if (is_zfs && 1229 (cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR)) != NULL) { 1230 if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) { 1231 (void) VOP_IOCTL(cdev_vp, DKIOCDUMPFINI, NULL, FKIOCTL, 1232 kcred, NULL, NULL); 1233 (void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0, 1234 kcred, NULL); 1235 } 1236 VN_RELE(cdev_vp); 1237 } 1238 1239 (void) VOP_CLOSE(dumpvp, FREAD | FWRITE, 1, (offset_t)0, kcred, NULL); 1240 1241 VN_RELE(dumpvp); 1242 1243 dumpvp = NULL; 1244 dumpvp_size = 0; 1245 dumppath = NULL; 1246 } 1247 1248 static offset_t 1249 dumpvp_flush(void) 1250 { 1251 size_t size = P2ROUNDUP(dumpbuf.cur - dumpbuf.start, PAGESIZE); 1252 hrtime_t iotime; 1253 int err; 1254 1255 if (dumpbuf.vp_off + size > dumpbuf.vp_limit) { 1256 dump_ioerr = ENOSPC; 1257 dumpbuf.vp_off = dumpbuf.vp_limit; 1258 } else if (size != 0) { 1259 iotime = gethrtime(); 1260 dumpsync.iowait += iotime - dumpsync.iowaitts; 1261 if (panicstr) 1262 err = VOP_DUMP(dumpvp, dumpbuf.start, 1263 lbtodb(dumpbuf.vp_off), btod(size), NULL); 1264 else 1265 err = vn_rdwr(UIO_WRITE, dumpbuf.cdev_vp != NULL ? 1266 dumpbuf.cdev_vp : dumpvp, dumpbuf.start, size, 1267 dumpbuf.vp_off, UIO_SYSSPACE, 0, dumpbuf.vp_limit, 1268 kcred, 0); 1269 if (err && dump_ioerr == 0) 1270 dump_ioerr = err; 1271 dumpsync.iowaitts = gethrtime(); 1272 dumpsync.iotime += dumpsync.iowaitts - iotime; 1273 dumpsync.nwrite += size; 1274 dumpbuf.vp_off += size; 1275 } 1276 dumpbuf.cur = dumpbuf.start; 1277 dump_timeleft = dump_timeout; 1278 return (dumpbuf.vp_off); 1279 } 1280 1281 /* maximize write speed by keeping seek offset aligned with size */ 1282 void 1283 dumpvp_write(const void *va, size_t size) 1284 { 1285 size_t len, off, sz; 1286 1287 while (size != 0) { 1288 len = MIN(size, dumpbuf.end - dumpbuf.cur); 1289 if (len == 0) { 1290 off = P2PHASE(dumpbuf.vp_off, dumpbuf.size); 1291 if (off == 0 || !ISP2(dumpbuf.size)) { 1292 (void) dumpvp_flush(); 1293 } else { 1294 sz = dumpbuf.size - off; 1295 dumpbuf.cur = dumpbuf.start + sz; 1296 (void) dumpvp_flush(); 1297 ovbcopy(dumpbuf.start + sz, dumpbuf.start, off); 1298 dumpbuf.cur += off; 1299 } 1300 } else { 1301 bcopy(va, dumpbuf.cur, len); 1302 va = (char *)va + len; 1303 dumpbuf.cur += len; 1304 size -= len; 1305 } 1306 } 1307 } 1308 1309 /*ARGSUSED*/ 1310 static void 1311 dumpvp_ksyms_write(const void *src, void *dst, size_t size) 1312 { 1313 dumpvp_write(src, size); 1314 } 1315 1316 /* 1317 * Mark 'pfn' in the bitmap and dump its translation table entry. 1318 */ 1319 void 1320 dump_addpage(struct as *as, void *va, pfn_t pfn) 1321 { 1322 mem_vtop_t mem_vtop; 1323 pgcnt_t bitnum; 1324 1325 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1326 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1327 dumphdr->dump_npages++; 1328 BT_SET(dumpcfg.bitmap, bitnum); 1329 } 1330 dumphdr->dump_nvtop++; 1331 mem_vtop.m_as = as; 1332 mem_vtop.m_va = va; 1333 mem_vtop.m_pfn = pfn; 1334 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1335 } 1336 dump_timeleft = dump_timeout; 1337 } 1338 1339 /* 1340 * Mark 'pfn' in the bitmap 1341 */ 1342 void 1343 dump_page(pfn_t pfn) 1344 { 1345 pgcnt_t bitnum; 1346 1347 if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) { 1348 if (!BT_TEST(dumpcfg.bitmap, bitnum)) { 1349 dumphdr->dump_npages++; 1350 BT_SET(dumpcfg.bitmap, bitnum); 1351 } 1352 } 1353 dump_timeleft = dump_timeout; 1354 } 1355 1356 /* 1357 * Dump the <as, va, pfn> information for a given address space. 1358 * SEGOP_DUMP() will call dump_addpage() for each page in the segment. 1359 */ 1360 static void 1361 dump_as(struct as *as) 1362 { 1363 struct seg *seg; 1364 1365 AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 1366 for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 1367 if (seg->s_as != as) 1368 break; 1369 if (seg->s_ops == NULL) 1370 continue; 1371 SEGOP_DUMP(seg); 1372 } 1373 AS_LOCK_EXIT(as, &as->a_lock); 1374 1375 if (seg != NULL) 1376 cmn_err(CE_WARN, "invalid segment %p in address space %p", 1377 (void *)seg, (void *)as); 1378 } 1379 1380 static int 1381 dump_process(pid_t pid) 1382 { 1383 proc_t *p = sprlock(pid); 1384 1385 if (p == NULL) 1386 return (-1); 1387 if (p->p_as != &kas) { 1388 mutex_exit(&p->p_lock); 1389 dump_as(p->p_as); 1390 mutex_enter(&p->p_lock); 1391 } 1392 1393 sprunlock(p); 1394 1395 return (0); 1396 } 1397 1398 void 1399 dump_ereports(void) 1400 { 1401 u_offset_t dumpvp_start; 1402 erpt_dump_t ed; 1403 1404 if (dumpvp == NULL || dumphdr == NULL) 1405 return; 1406 1407 dumpbuf.cur = dumpbuf.start; 1408 dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE); 1409 dumpvp_start = dumpbuf.vp_limit - DUMP_ERPTSIZE; 1410 dumpbuf.vp_off = dumpvp_start; 1411 1412 fm_ereport_dump(); 1413 if (panicstr) 1414 errorq_dump(); 1415 1416 bzero(&ed, sizeof (ed)); /* indicate end of ereports */ 1417 dumpvp_write(&ed, sizeof (ed)); 1418 (void) dumpvp_flush(); 1419 1420 if (!panicstr) { 1421 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1422 (size_t)(dumpbuf.vp_off - dumpvp_start), 1423 B_INVAL | B_FORCE, kcred, NULL); 1424 } 1425 } 1426 1427 void 1428 dump_messages(void) 1429 { 1430 log_dump_t ld; 1431 mblk_t *mctl, *mdata; 1432 queue_t *q, *qlast; 1433 u_offset_t dumpvp_start; 1434 1435 if (dumpvp == NULL || dumphdr == NULL || log_consq == NULL) 1436 return; 1437 1438 dumpbuf.cur = dumpbuf.start; 1439 dumpbuf.vp_limit = dumpvp_size - DUMP_OFFSET; 1440 dumpvp_start = dumpbuf.vp_limit - DUMP_LOGSIZE; 1441 dumpbuf.vp_off = dumpvp_start; 1442 1443 qlast = NULL; 1444 do { 1445 for (q = log_consq; q->q_next != qlast; q = q->q_next) 1446 continue; 1447 for (mctl = q->q_first; mctl != NULL; mctl = mctl->b_next) { 1448 dump_timeleft = dump_timeout; 1449 mdata = mctl->b_cont; 1450 ld.ld_magic = LOG_MAGIC; 1451 ld.ld_msgsize = MBLKL(mctl->b_cont); 1452 ld.ld_csum = checksum32(mctl->b_rptr, MBLKL(mctl)); 1453 ld.ld_msum = checksum32(mdata->b_rptr, MBLKL(mdata)); 1454 dumpvp_write(&ld, sizeof (ld)); 1455 dumpvp_write(mctl->b_rptr, MBLKL(mctl)); 1456 dumpvp_write(mdata->b_rptr, MBLKL(mdata)); 1457 } 1458 } while ((qlast = q) != log_consq); 1459 1460 ld.ld_magic = 0; /* indicate end of messages */ 1461 dumpvp_write(&ld, sizeof (ld)); 1462 (void) dumpvp_flush(); 1463 if (!panicstr) { 1464 (void) VOP_PUTPAGE(dumpvp, dumpvp_start, 1465 (size_t)(dumpbuf.vp_off - dumpvp_start), 1466 B_INVAL | B_FORCE, kcred, NULL); 1467 } 1468 } 1469 1470 /* 1471 * The following functions are called on multiple CPUs during dump. 1472 * They must not use most kernel services, because all cross-calls are 1473 * disabled during panic. Therefore, blocking locks and cache flushes 1474 * will not work. 1475 */ 1476 1477 static int 1478 dump_pagecopy(void *src, void *dst) 1479 { 1480 long *wsrc = (long *)src; 1481 long *wdst = (long *)dst; 1482 const ulong_t ncopies = PAGESIZE / sizeof (long); 1483 volatile int w = 0; 1484 volatile int ueoff = -1; 1485 on_trap_data_t otd; 1486 1487 if (on_trap(&otd, OT_DATA_EC)) { 1488 if (ueoff == -1) 1489 ueoff = w * sizeof (long); 1490 #ifdef _LP64 1491 wdst[w++] = 0xbadecc00badecc; 1492 #else 1493 wdst[w++] = 0xbadecc; 1494 #endif 1495 } 1496 while (w < ncopies) { 1497 wdst[w] = wsrc[w]; 1498 w++; 1499 } 1500 no_trap(); 1501 return (ueoff); 1502 } 1503 1504 static void 1505 dumpsys_close_cq(cqueue_t *cq, int live) 1506 { 1507 if (live) { 1508 mutex_enter(&cq->mutex); 1509 atomic_dec_uint(&cq->open); 1510 cv_signal(&cq->cv); 1511 mutex_exit(&cq->mutex); 1512 } else { 1513 atomic_dec_uint(&cq->open); 1514 } 1515 } 1516 1517 static inline void 1518 dumpsys_spinlock(lock_t *lp) 1519 { 1520 uint_t backoff = 0; 1521 int loop_count = 0; 1522 1523 while (LOCK_HELD(lp) || !lock_spin_try(lp)) { 1524 if (++loop_count >= ncpus) { 1525 backoff = mutex_lock_backoff(0); 1526 loop_count = 0; 1527 } else { 1528 backoff = mutex_lock_backoff(backoff); 1529 } 1530 mutex_lock_delay(backoff); 1531 } 1532 } 1533 1534 static inline void 1535 dumpsys_spinunlock(lock_t *lp) 1536 { 1537 lock_clear(lp); 1538 } 1539 1540 static inline void 1541 dumpsys_lock(cqueue_t *cq, int live) 1542 { 1543 if (live) 1544 mutex_enter(&cq->mutex); 1545 else 1546 dumpsys_spinlock(&cq->spinlock); 1547 } 1548 1549 static inline void 1550 dumpsys_unlock(cqueue_t *cq, int live, int signal) 1551 { 1552 if (live) { 1553 if (signal) 1554 cv_signal(&cq->cv); 1555 mutex_exit(&cq->mutex); 1556 } else { 1557 dumpsys_spinunlock(&cq->spinlock); 1558 } 1559 } 1560 1561 static void 1562 dumpsys_wait_cq(cqueue_t *cq, int live) 1563 { 1564 if (live) { 1565 cv_wait(&cq->cv, &cq->mutex); 1566 } else { 1567 dumpsys_spinunlock(&cq->spinlock); 1568 while (cq->open) 1569 if (cq->first) 1570 break; 1571 dumpsys_spinlock(&cq->spinlock); 1572 } 1573 } 1574 1575 static void 1576 dumpsys_put_cq(cqueue_t *cq, cbuf_t *cp, int newstate, int live) 1577 { 1578 if (cp == NULL) 1579 return; 1580 1581 dumpsys_lock(cq, live); 1582 1583 if (cq->ts != 0) { 1584 cq->empty += gethrtime() - cq->ts; 1585 cq->ts = 0; 1586 } 1587 1588 cp->state = newstate; 1589 cp->next = NULL; 1590 if (cq->last == NULL) 1591 cq->first = cp; 1592 else 1593 cq->last->next = cp; 1594 cq->last = cp; 1595 1596 dumpsys_unlock(cq, live, 1); 1597 } 1598 1599 static cbuf_t * 1600 dumpsys_get_cq(cqueue_t *cq, int live) 1601 { 1602 cbuf_t *cp; 1603 hrtime_t now = gethrtime(); 1604 1605 dumpsys_lock(cq, live); 1606 1607 /* CONSTCOND */ 1608 while (1) { 1609 cp = (cbuf_t *)cq->first; 1610 if (cp == NULL) { 1611 if (cq->open == 0) 1612 break; 1613 dumpsys_wait_cq(cq, live); 1614 continue; 1615 } 1616 cq->first = cp->next; 1617 if (cq->first == NULL) { 1618 cq->last = NULL; 1619 cq->ts = now; 1620 } 1621 break; 1622 } 1623 1624 dumpsys_unlock(cq, live, cq->first != NULL || cq->open == 0); 1625 return (cp); 1626 } 1627 1628 /* 1629 * Send an error message to the console. If the main task is running 1630 * just write the message via uprintf. If a helper is running the 1631 * message has to be put on a queue for the main task. Setting fmt to 1632 * NULL means flush the error message buffer. If fmt is not NULL, just 1633 * add the text to the existing buffer. 1634 */ 1635 static void 1636 dumpsys_errmsg(helper_t *hp, const char *fmt, ...) 1637 { 1638 dumpsync_t *ds = hp->ds; 1639 cbuf_t *cp = hp->cperr; 1640 va_list adx; 1641 1642 if (hp->helper == MAINHELPER) { 1643 if (fmt != NULL) { 1644 if (ds->neednl) { 1645 uprintf("\n"); 1646 ds->neednl = 0; 1647 } 1648 va_start(adx, fmt); 1649 vuprintf(fmt, adx); 1650 va_end(adx); 1651 } 1652 } else if (fmt == NULL) { 1653 if (cp != NULL) { 1654 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1655 hp->cperr = NULL; 1656 } 1657 } else { 1658 if (hp->cperr == NULL) { 1659 cp = CQ_GET(freebufq); 1660 hp->cperr = cp; 1661 cp->used = 0; 1662 } 1663 va_start(adx, fmt); 1664 cp->used += vsnprintf(cp->buf + cp->used, cp->size - cp->used, 1665 fmt, adx); 1666 va_end(adx); 1667 if ((cp->used + LOG_MSGSIZE) > cp->size) { 1668 CQ_PUT(mainq, cp, CBUF_ERRMSG); 1669 hp->cperr = NULL; 1670 } 1671 } 1672 } 1673 1674 /* 1675 * Write an output buffer to the dump file. If the main task is 1676 * running just write the data. If a helper is running the output is 1677 * placed on a queue for the main task. 1678 */ 1679 static void 1680 dumpsys_swrite(helper_t *hp, cbuf_t *cp, size_t used) 1681 { 1682 dumpsync_t *ds = hp->ds; 1683 1684 if (hp->helper == MAINHELPER) { 1685 HRSTART(ds->perpage, write); 1686 dumpvp_write(cp->buf, used); 1687 HRSTOP(ds->perpage, write); 1688 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 1689 } else { 1690 cp->used = used; 1691 CQ_PUT(mainq, cp, CBUF_WRITE); 1692 } 1693 } 1694 1695 /* 1696 * Copy one page within the mapped range. The offset starts at 0 and 1697 * is relative to the first pfn. cp->buf + cp->off is the address of 1698 * the first pfn. If dump_pagecopy returns a UE offset, create an 1699 * error message. Returns the offset to the next pfn in the range 1700 * selected by the bitmap. 1701 */ 1702 static int 1703 dumpsys_copy_page(helper_t *hp, int offset) 1704 { 1705 cbuf_t *cp = hp->cpin; 1706 int ueoff; 1707 1708 ASSERT(cp->off + offset + PAGESIZE <= cp->size); 1709 ASSERT(BT_TEST(dumpcfg.bitmap, cp->bitnum)); 1710 1711 ueoff = dump_pagecopy(cp->buf + cp->off + offset, hp->page); 1712 1713 /* ueoff is the offset in the page to a UE error */ 1714 if (ueoff != -1) { 1715 uint64_t pa = ptob(cp->pfn) + offset + ueoff; 1716 1717 dumpsys_errmsg(hp, "memory error at PA 0x%08x.%08x\n", 1718 (uint32_t)(pa >> 32), (uint32_t)pa); 1719 } 1720 1721 /* 1722 * Advance bitnum and offset to the next input page for the 1723 * next call to this function. 1724 */ 1725 offset += PAGESIZE; 1726 cp->bitnum++; 1727 while (cp->off + offset < cp->size) { 1728 if (BT_TEST(dumpcfg.bitmap, cp->bitnum)) 1729 break; 1730 offset += PAGESIZE; 1731 cp->bitnum++; 1732 } 1733 1734 return (offset); 1735 } 1736 1737 /* 1738 * Read the helper queue, and copy one mapped page. Return 0 when 1739 * done. Return 1 when a page has been copied into hp->page. 1740 */ 1741 static int 1742 dumpsys_sread(helper_t *hp) 1743 { 1744 dumpsync_t *ds = hp->ds; 1745 1746 /* CONSTCOND */ 1747 while (1) { 1748 1749 /* Find the next input buffer. */ 1750 if (hp->cpin == NULL) { 1751 HRSTART(hp->perpage, inwait); 1752 1753 /* CONSTCOND */ 1754 while (1) { 1755 hp->cpin = CQ_GET(helperq); 1756 dump_timeleft = dump_timeout; 1757 1758 /* 1759 * NULL return means the helper queue 1760 * is closed and empty. 1761 */ 1762 if (hp->cpin == NULL) 1763 break; 1764 1765 /* Have input, check for dump I/O error. */ 1766 if (!dump_ioerr) 1767 break; 1768 1769 /* 1770 * If an I/O error occurs, stay in the 1771 * loop in order to empty the helper 1772 * queue. Return the buffers to the 1773 * main task to unmap and free it. 1774 */ 1775 hp->cpin->used = 0; 1776 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1777 } 1778 HRSTOP(hp->perpage, inwait); 1779 1780 /* Stop here when the helper queue is closed. */ 1781 if (hp->cpin == NULL) 1782 break; 1783 1784 /* Set the offset=0 to get the first pfn. */ 1785 hp->in = 0; 1786 1787 /* Set the total processed to 0 */ 1788 hp->used = 0; 1789 } 1790 1791 /* Process the next page. */ 1792 if (hp->used < hp->cpin->used) { 1793 1794 /* 1795 * Get the next page from the input buffer and 1796 * return a copy. 1797 */ 1798 ASSERT(hp->in != -1); 1799 HRSTART(hp->perpage, copy); 1800 hp->in = dumpsys_copy_page(hp, hp->in); 1801 hp->used += PAGESIZE; 1802 HRSTOP(hp->perpage, copy); 1803 break; 1804 1805 } else { 1806 1807 /* 1808 * Done with the input. Flush the VM and 1809 * return the buffer to the main task. 1810 */ 1811 if (panicstr && hp->helper != MAINHELPER) 1812 hat_flush_range(kas.a_hat, 1813 hp->cpin->buf, hp->cpin->size); 1814 dumpsys_errmsg(hp, NULL); 1815 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 1816 hp->cpin = NULL; 1817 } 1818 } 1819 1820 return (hp->cpin != NULL); 1821 } 1822 1823 /* 1824 * Compress size bytes starting at buf with bzip2 1825 * mode: 1826 * BZ_RUN add one more compressed page 1827 * BZ_FINISH no more input, flush the state 1828 */ 1829 static void 1830 dumpsys_bzrun(helper_t *hp, void *buf, size_t size, int mode) 1831 { 1832 dumpsync_t *ds = hp->ds; 1833 const int CSIZE = sizeof (dumpcsize_t); 1834 bz_stream *ps = &hp->bzstream; 1835 int rc = 0; 1836 uint32_t csize; 1837 dumpcsize_t cs; 1838 1839 /* Set input pointers to new input page */ 1840 if (size > 0) { 1841 ps->avail_in = size; 1842 ps->next_in = buf; 1843 } 1844 1845 /* CONSTCOND */ 1846 while (1) { 1847 1848 /* Quit when all input has been consumed */ 1849 if (ps->avail_in == 0 && mode == BZ_RUN) 1850 break; 1851 1852 /* Get a new output buffer */ 1853 if (hp->cpout == NULL) { 1854 HRSTART(hp->perpage, outwait); 1855 hp->cpout = CQ_GET(freebufq); 1856 HRSTOP(hp->perpage, outwait); 1857 ps->avail_out = hp->cpout->size - CSIZE; 1858 ps->next_out = hp->cpout->buf + CSIZE; 1859 } 1860 1861 /* Compress input, or finalize */ 1862 HRSTART(hp->perpage, compress); 1863 rc = BZ2_bzCompress(ps, mode); 1864 HRSTOP(hp->perpage, compress); 1865 1866 /* Check for error */ 1867 if (mode == BZ_RUN && rc != BZ_RUN_OK) { 1868 dumpsys_errmsg(hp, "%d: BZ_RUN error %s at page %lx\n", 1869 hp->helper, BZ2_bzErrorString(rc), 1870 hp->cpin->pagenum); 1871 break; 1872 } 1873 1874 /* Write the buffer if it is full, or we are flushing */ 1875 if (ps->avail_out == 0 || mode == BZ_FINISH) { 1876 csize = hp->cpout->size - CSIZE - ps->avail_out; 1877 cs = DUMP_SET_TAG(csize, hp->tag); 1878 if (csize > 0) { 1879 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 1880 dumpsys_swrite(hp, hp->cpout, csize + CSIZE); 1881 hp->cpout = NULL; 1882 } 1883 } 1884 1885 /* Check for final complete */ 1886 if (mode == BZ_FINISH) { 1887 if (rc == BZ_STREAM_END) 1888 break; 1889 if (rc != BZ_FINISH_OK) { 1890 dumpsys_errmsg(hp, "%d: BZ_FINISH error %s\n", 1891 hp->helper, BZ2_bzErrorString(rc)); 1892 break; 1893 } 1894 } 1895 } 1896 1897 /* Cleanup state and buffers */ 1898 if (mode == BZ_FINISH) { 1899 1900 /* Reset state so that it is re-usable. */ 1901 (void) BZ2_bzCompressReset(&hp->bzstream); 1902 1903 /* Give any unused outout buffer to the main task */ 1904 if (hp->cpout != NULL) { 1905 hp->cpout->used = 0; 1906 CQ_PUT(mainq, hp->cpout, CBUF_ERRMSG); 1907 hp->cpout = NULL; 1908 } 1909 } 1910 } 1911 1912 static void 1913 dumpsys_bz2compress(helper_t *hp) 1914 { 1915 dumpsync_t *ds = hp->ds; 1916 dumpstreamhdr_t sh; 1917 1918 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 1919 sh.stream_pagenum = (pgcnt_t)-1; 1920 sh.stream_npages = 0; 1921 hp->cpin = NULL; 1922 hp->cpout = NULL; 1923 hp->cperr = NULL; 1924 hp->in = 0; 1925 hp->out = 0; 1926 hp->bzstream.avail_in = 0; 1927 1928 /* Bump reference to mainq while we are running */ 1929 CQ_OPEN(mainq); 1930 1931 /* Get one page at a time */ 1932 while (dumpsys_sread(hp)) { 1933 if (sh.stream_pagenum != hp->cpin->pagenum) { 1934 sh.stream_pagenum = hp->cpin->pagenum; 1935 sh.stream_npages = btop(hp->cpin->used); 1936 dumpsys_bzrun(hp, &sh, sizeof (sh), BZ_RUN); 1937 } 1938 dumpsys_bzrun(hp, hp->page, PAGESIZE, 0); 1939 } 1940 1941 /* Done with input, flush any partial buffer */ 1942 if (sh.stream_pagenum != (pgcnt_t)-1) { 1943 dumpsys_bzrun(hp, NULL, 0, BZ_FINISH); 1944 dumpsys_errmsg(hp, NULL); 1945 } 1946 1947 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 1948 1949 /* Decrement main queue count, we are done */ 1950 CQ_CLOSE(mainq); 1951 } 1952 1953 /* 1954 * Compress with lzjb 1955 * write stream block if full or size==0 1956 * if csize==0 write stream header, else write <csize, data> 1957 * size==0 is a call to flush a buffer 1958 * hp->cpout is the buffer we are flushing or filling 1959 * hp->out is the next index to fill data 1960 * osize is either csize+data, or the size of a stream header 1961 */ 1962 static void 1963 dumpsys_lzjbrun(helper_t *hp, size_t csize, void *buf, size_t size) 1964 { 1965 dumpsync_t *ds = hp->ds; 1966 const int CSIZE = sizeof (dumpcsize_t); 1967 dumpcsize_t cs; 1968 size_t osize = csize > 0 ? CSIZE + size : size; 1969 1970 /* If flush, and there is no buffer, just return */ 1971 if (size == 0 && hp->cpout == NULL) 1972 return; 1973 1974 /* If flush, or cpout is full, write it out */ 1975 if (size == 0 || 1976 hp->cpout != NULL && hp->out + osize > hp->cpout->size) { 1977 1978 /* Set tag+size word at the front of the stream block. */ 1979 cs = DUMP_SET_TAG(hp->out - CSIZE, hp->tag); 1980 (void) memcpy(hp->cpout->buf, &cs, CSIZE); 1981 1982 /* Write block to dump file. */ 1983 dumpsys_swrite(hp, hp->cpout, hp->out); 1984 1985 /* Clear pointer to indicate we need a new buffer */ 1986 hp->cpout = NULL; 1987 1988 /* flushing, we are done */ 1989 if (size == 0) 1990 return; 1991 } 1992 1993 /* Get an output buffer if we dont have one. */ 1994 if (hp->cpout == NULL) { 1995 HRSTART(hp->perpage, outwait); 1996 hp->cpout = CQ_GET(freebufq); 1997 HRSTOP(hp->perpage, outwait); 1998 hp->out = CSIZE; 1999 } 2000 2001 /* Store csize word. This is the size of compressed data. */ 2002 if (csize > 0) { 2003 cs = DUMP_SET_TAG(csize, 0); 2004 (void) memcpy(hp->cpout->buf + hp->out, &cs, CSIZE); 2005 hp->out += CSIZE; 2006 } 2007 2008 /* Store the data. */ 2009 (void) memcpy(hp->cpout->buf + hp->out, buf, size); 2010 hp->out += size; 2011 } 2012 2013 static void 2014 dumpsys_lzjbcompress(helper_t *hp) 2015 { 2016 dumpsync_t *ds = hp->ds; 2017 size_t csize; 2018 dumpstreamhdr_t sh; 2019 2020 (void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC); 2021 sh.stream_pagenum = (pfn_t)-1; 2022 sh.stream_npages = 0; 2023 hp->cpin = NULL; 2024 hp->cpout = NULL; 2025 hp->cperr = NULL; 2026 hp->in = 0; 2027 hp->out = 0; 2028 2029 /* Bump reference to mainq while we are running */ 2030 CQ_OPEN(mainq); 2031 2032 /* Get one page at a time */ 2033 while (dumpsys_sread(hp)) { 2034 2035 /* Create a stream header for each new input map */ 2036 if (sh.stream_pagenum != hp->cpin->pagenum) { 2037 sh.stream_pagenum = hp->cpin->pagenum; 2038 sh.stream_npages = btop(hp->cpin->used); 2039 dumpsys_lzjbrun(hp, 0, &sh, sizeof (sh)); 2040 } 2041 2042 /* Compress one page */ 2043 HRSTART(hp->perpage, compress); 2044 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2045 HRSTOP(hp->perpage, compress); 2046 2047 /* Add csize+data to output block */ 2048 ASSERT(csize > 0 && csize <= PAGESIZE); 2049 dumpsys_lzjbrun(hp, csize, hp->lzbuf, csize); 2050 } 2051 2052 /* Done with input, flush any partial buffer */ 2053 if (sh.stream_pagenum != (pfn_t)-1) { 2054 dumpsys_lzjbrun(hp, 0, NULL, 0); 2055 dumpsys_errmsg(hp, NULL); 2056 } 2057 2058 ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL); 2059 2060 /* Decrement main queue count, we are done */ 2061 CQ_CLOSE(mainq); 2062 } 2063 2064 /* 2065 * Dump helper called from panic_idle() to compress pages. CPUs in 2066 * this path must not call most kernel services. 2067 * 2068 * During panic, all but one of the CPUs is idle. These CPUs are used 2069 * as helpers working in parallel to copy and compress memory 2070 * pages. During a panic, however, these processors cannot call any 2071 * kernel services. This is because mutexes become no-ops during 2072 * panic, and, cross-call interrupts are inhibited. Therefore, during 2073 * panic dump the helper CPUs communicate with the panic CPU using 2074 * memory variables. All memory mapping and I/O is performed by the 2075 * panic CPU. 2076 */ 2077 void 2078 dumpsys_helper() 2079 { 2080 dumpsys_spinlock(&dumpcfg.helper_lock); 2081 if (dumpcfg.helpers_wanted) { 2082 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2083 2084 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2085 if (hp->helper == FREEHELPER) { 2086 hp->helper = CPU->cpu_id; 2087 BT_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2088 2089 dumpsys_spinunlock(&dumpcfg.helper_lock); 2090 2091 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2092 dumpsys_lzjbcompress(hp); 2093 else 2094 dumpsys_bz2compress(hp); 2095 2096 hp->helper = DONEHELPER; 2097 return; 2098 } 2099 } 2100 } 2101 dumpsys_spinunlock(&dumpcfg.helper_lock); 2102 } 2103 2104 /* 2105 * Dump helper for live dumps. 2106 * These run as a system task. 2107 */ 2108 static void 2109 dumpsys_live_helper(void *arg) 2110 { 2111 helper_t *hp = arg; 2112 2113 BT_ATOMIC_SET(dumpcfg.helpermap, CPU->cpu_seqid); 2114 if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2) 2115 dumpsys_lzjbcompress(hp); 2116 else 2117 dumpsys_bz2compress(hp); 2118 } 2119 2120 /* 2121 * Compress one page with lzjb (single threaded case) 2122 */ 2123 static void 2124 dumpsys_lzjb_page(helper_t *hp, cbuf_t *cp) 2125 { 2126 dumpsync_t *ds = hp->ds; 2127 uint32_t csize; 2128 2129 hp->helper = MAINHELPER; 2130 hp->in = 0; 2131 hp->used = 0; 2132 hp->cpin = cp; 2133 while (hp->used < cp->used) { 2134 HRSTART(hp->perpage, copy); 2135 hp->in = dumpsys_copy_page(hp, hp->in); 2136 hp->used += PAGESIZE; 2137 HRSTOP(hp->perpage, copy); 2138 2139 HRSTART(hp->perpage, compress); 2140 csize = compress(hp->page, hp->lzbuf, PAGESIZE); 2141 HRSTOP(hp->perpage, compress); 2142 2143 HRSTART(hp->perpage, write); 2144 dumpvp_write(&csize, sizeof (csize)); 2145 dumpvp_write(hp->lzbuf, csize); 2146 HRSTOP(hp->perpage, write); 2147 } 2148 CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP); 2149 hp->cpin = NULL; 2150 } 2151 2152 /* 2153 * Main task to dump pages. This is called on the dump CPU. 2154 */ 2155 static void 2156 dumpsys_main_task(void *arg) 2157 { 2158 dumpsync_t *ds = arg; 2159 pgcnt_t pagenum = 0, bitnum = 0, hibitnum; 2160 dumpmlw_t mlw; 2161 cbuf_t *cp; 2162 pgcnt_t baseoff, pfnoff; 2163 pfn_t base, pfn; 2164 int sec; 2165 2166 dump_init_memlist_walker(&mlw); 2167 2168 /* CONSTCOND */ 2169 while (1) { 2170 2171 if (ds->percent > ds->percent_done) { 2172 ds->percent_done = ds->percent; 2173 sec = (gethrtime() - ds->start) / 1000 / 1000 / 1000; 2174 uprintf("^\r%2d:%02d %3d%% done", 2175 sec / 60, sec % 60, ds->percent); 2176 ds->neednl = 1; 2177 } 2178 2179 while (CQ_IS_EMPTY(mainq) && !CQ_IS_EMPTY(writerq)) { 2180 2181 /* the writerq never blocks */ 2182 cp = CQ_GET(writerq); 2183 if (cp == NULL) 2184 break; 2185 2186 dump_timeleft = dump_timeout; 2187 2188 HRSTART(ds->perpage, write); 2189 dumpvp_write(cp->buf, cp->used); 2190 HRSTOP(ds->perpage, write); 2191 2192 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2193 } 2194 2195 /* 2196 * Wait here for some buffers to process. Returns NULL 2197 * when all helpers have terminated and all buffers 2198 * have been processed. 2199 */ 2200 cp = CQ_GET(mainq); 2201 2202 if (cp == NULL) { 2203 2204 /* Drain the write queue. */ 2205 if (!CQ_IS_EMPTY(writerq)) 2206 continue; 2207 2208 /* Main task exits here. */ 2209 break; 2210 } 2211 2212 dump_timeleft = dump_timeout; 2213 2214 switch (cp->state) { 2215 2216 case CBUF_FREEMAP: 2217 2218 /* 2219 * Note that we drop CBUF_FREEMAP buffers on 2220 * the floor (they will not be on any cqueue) 2221 * when we no longer need them. 2222 */ 2223 if (bitnum >= dumpcfg.bitmapsize) 2224 break; 2225 2226 if (dump_ioerr) { 2227 bitnum = dumpcfg.bitmapsize; 2228 CQ_CLOSE(helperq); 2229 break; 2230 } 2231 2232 HRSTART(ds->perpage, bitmap); 2233 for (; bitnum < dumpcfg.bitmapsize; bitnum++) 2234 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2235 break; 2236 HRSTOP(ds->perpage, bitmap); 2237 dump_timeleft = dump_timeout; 2238 2239 if (bitnum >= dumpcfg.bitmapsize) { 2240 CQ_CLOSE(helperq); 2241 break; 2242 } 2243 2244 /* 2245 * Try to map CBUF_MAPSIZE ranges. Can't 2246 * assume that memory segment size is a 2247 * multiple of CBUF_MAPSIZE. Can't assume that 2248 * the segment starts on a CBUF_MAPSIZE 2249 * boundary. 2250 */ 2251 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2252 ASSERT(pfn != PFN_INVALID); 2253 ASSERT(bitnum + mlw.mpleft <= dumpcfg.bitmapsize); 2254 2255 base = P2ALIGN(pfn, CBUF_MAPNP); 2256 if (base < mlw.mpaddr) { 2257 base = mlw.mpaddr; 2258 baseoff = P2PHASE(base, CBUF_MAPNP); 2259 } else { 2260 baseoff = 0; 2261 } 2262 2263 pfnoff = pfn - base; 2264 if (pfnoff + mlw.mpleft < CBUF_MAPNP) { 2265 hibitnum = bitnum + mlw.mpleft; 2266 cp->size = ptob(pfnoff + mlw.mpleft); 2267 } else { 2268 hibitnum = bitnum - pfnoff + CBUF_MAPNP - 2269 baseoff; 2270 cp->size = CBUF_MAPSIZE - ptob(baseoff); 2271 } 2272 2273 cp->pfn = pfn; 2274 cp->bitnum = bitnum++; 2275 cp->pagenum = pagenum++; 2276 cp->off = ptob(pfnoff); 2277 2278 for (; bitnum < hibitnum; bitnum++) 2279 if (BT_TEST(dumpcfg.bitmap, bitnum)) 2280 pagenum++; 2281 2282 dump_timeleft = dump_timeout; 2283 cp->used = ptob(pagenum - cp->pagenum); 2284 2285 HRSTART(ds->perpage, map); 2286 hat_devload(kas.a_hat, cp->buf, cp->size, base, 2287 PROT_READ, HAT_LOAD_NOCONSIST); 2288 HRSTOP(ds->perpage, map); 2289 2290 ds->pages_mapped += btop(cp->size); 2291 ds->pages_used += pagenum - cp->pagenum; 2292 2293 CQ_OPEN(mainq); 2294 2295 /* 2296 * If there are no helpers the main task does 2297 * non-streams lzjb compress. 2298 */ 2299 if (dumpcfg.clevel == 0) { 2300 dumpsys_lzjb_page(dumpcfg.helper, cp); 2301 break; 2302 } 2303 2304 /* pass mapped pages to a helper */ 2305 CQ_PUT(helperq, cp, CBUF_INREADY); 2306 2307 /* the last page was done */ 2308 if (bitnum >= dumpcfg.bitmapsize) 2309 CQ_CLOSE(helperq); 2310 2311 break; 2312 2313 case CBUF_USEDMAP: 2314 2315 ds->npages += btop(cp->used); 2316 2317 HRSTART(ds->perpage, unmap); 2318 hat_unload(kas.a_hat, cp->buf, cp->size, HAT_UNLOAD); 2319 HRSTOP(ds->perpage, unmap); 2320 2321 if (bitnum < dumpcfg.bitmapsize) 2322 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2323 CQ_CLOSE(mainq); 2324 2325 ASSERT(ds->npages <= dumphdr->dump_npages); 2326 ds->percent = ds->npages * 100LL / dumphdr->dump_npages; 2327 break; 2328 2329 case CBUF_WRITE: 2330 2331 CQ_PUT(writerq, cp, CBUF_WRITE); 2332 break; 2333 2334 case CBUF_ERRMSG: 2335 2336 if (cp->used > 0) { 2337 cp->buf[cp->size - 2] = '\n'; 2338 cp->buf[cp->size - 1] = '\0'; 2339 if (ds->neednl) { 2340 uprintf("\n%s", cp->buf); 2341 ds->neednl = 0; 2342 } else { 2343 uprintf("%s", cp->buf); 2344 } 2345 } 2346 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2347 break; 2348 2349 default: 2350 uprintf("dump: unexpected buffer state %d, " 2351 "buffer will be lost\n", cp->state); 2352 break; 2353 2354 } /* end switch */ 2355 2356 } /* end while(1) */ 2357 } 2358 2359 #ifdef COLLECT_METRICS 2360 size_t 2361 dumpsys_metrics(dumpsync_t *ds, char *buf, size_t size) 2362 { 2363 dumpcfg_t *cfg = &dumpcfg; 2364 int myid = CPU->cpu_seqid; 2365 int i, compress_ratio; 2366 int sec, iorate; 2367 helper_t *hp, *hpend = &cfg->helper[cfg->nhelper]; 2368 char *e = buf + size; 2369 char *p = buf; 2370 2371 sec = ds->elapsed / (1000 * 1000 * 1000ULL); 2372 if (sec < 1) 2373 sec = 1; 2374 2375 if (ds->iotime < 1) 2376 ds->iotime = 1; 2377 iorate = (ds->nwrite * 100000ULL) / ds->iotime; 2378 2379 compress_ratio = 100LL * ds->npages / btopr(ds->nwrite + 1); 2380 2381 #define P(...) (p += p < e ? snprintf(p, e - p, __VA_ARGS__) : 0) 2382 2383 P("Master cpu_seqid,%d\n", CPU->cpu_seqid); 2384 P("Master cpu_id,%d\n", CPU->cpu_id); 2385 P("dump_flags,0x%x\n", dumphdr->dump_flags); 2386 P("dump_ioerr,%d\n", dump_ioerr); 2387 2388 P("Helpers:\n"); 2389 for (i = 0; i < ncpus; i++) { 2390 if ((i & 15) == 0) 2391 P(",,%03d,", i); 2392 if (i == myid) 2393 P(" M"); 2394 else if (BT_TEST(cfg->helpermap, i)) 2395 P("%4d", cpu_seq[i]->cpu_id); 2396 else 2397 P(" *"); 2398 if ((i & 15) == 15) 2399 P("\n"); 2400 } 2401 2402 P("ncbuf_used,%d\n", cfg->ncbuf_used); 2403 P("ncmap,%d\n", cfg->ncmap); 2404 2405 P("Found %ldM ranges,%ld\n", (CBUF_MAPSIZE / DUMP_1MB), cfg->found4m); 2406 P("Found small pages,%ld\n", cfg->foundsm); 2407 2408 P("Compression level,%d\n", cfg->clevel); 2409 P("Compression type,%s %s\n", cfg->clevel == 0 ? "serial" : "parallel", 2410 cfg->clevel >= DUMP_CLEVEL_BZIP2 ? "bzip2" : "lzjb"); 2411 P("Compression ratio,%d.%02d\n", compress_ratio / 100, compress_ratio % 2412 100); 2413 P("nhelper_used,%d\n", cfg->nhelper_used); 2414 2415 P("Dump I/O rate MBS,%d.%02d\n", iorate / 100, iorate % 100); 2416 P("..total bytes,%lld\n", (u_longlong_t)ds->nwrite); 2417 P("..total nsec,%lld\n", (u_longlong_t)ds->iotime); 2418 P("dumpbuf.iosize,%ld\n", dumpbuf.iosize); 2419 P("dumpbuf.size,%ld\n", dumpbuf.size); 2420 2421 P("Dump pages/sec,%llu\n", (u_longlong_t)ds->npages / sec); 2422 P("Dump pages,%llu\n", (u_longlong_t)ds->npages); 2423 P("Dump time,%d\n", sec); 2424 2425 if (ds->pages_mapped > 0) 2426 P("per-cent map utilization,%d\n", (int)((100 * ds->pages_used) 2427 / ds->pages_mapped)); 2428 2429 P("\nPer-page metrics:\n"); 2430 if (ds->npages > 0) { 2431 for (hp = cfg->helper; hp != hpend; hp++) { 2432 #define PERPAGE(x) ds->perpage.x += hp->perpage.x; 2433 PERPAGES; 2434 #undef PERPAGE 2435 } 2436 #define PERPAGE(x) \ 2437 P("%s nsec/page,%d\n", #x, (int)(ds->perpage.x / ds->npages)); 2438 PERPAGES; 2439 #undef PERPAGE 2440 P("freebufq.empty,%d\n", (int)(ds->freebufq.empty / 2441 ds->npages)); 2442 P("helperq.empty,%d\n", (int)(ds->helperq.empty / 2443 ds->npages)); 2444 P("writerq.empty,%d\n", (int)(ds->writerq.empty / 2445 ds->npages)); 2446 P("mainq.empty,%d\n", (int)(ds->mainq.empty / ds->npages)); 2447 2448 P("I/O wait nsec/page,%llu\n", (u_longlong_t)(ds->iowait / 2449 ds->npages)); 2450 } 2451 #undef P 2452 if (p < e) 2453 bzero(p, e - p); 2454 return (p - buf); 2455 } 2456 #endif /* COLLECT_METRICS */ 2457 2458 /* 2459 * Dump the system. 2460 */ 2461 void 2462 dumpsys(void) 2463 { 2464 dumpsync_t *ds = &dumpsync; 2465 taskq_t *livetaskq = NULL; 2466 pfn_t pfn; 2467 pgcnt_t bitnum; 2468 proc_t *p; 2469 helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper]; 2470 cbuf_t *cp; 2471 pid_t npids, pidx; 2472 char *content; 2473 int save_dump_clevel; 2474 dumpmlw_t mlw; 2475 dumpcsize_t datatag; 2476 dumpdatahdr_t datahdr; 2477 2478 if (dumpvp == NULL || dumphdr == NULL) { 2479 uprintf("skipping system dump - no dump device configured\n"); 2480 if (panicstr) { 2481 dumpcfg.helpers_wanted = 0; 2482 dumpsys_spinunlock(&dumpcfg.helper_lock); 2483 } 2484 return; 2485 } 2486 dumpbuf.cur = dumpbuf.start; 2487 2488 /* clear the sync variables */ 2489 ASSERT(dumpcfg.nhelper > 0); 2490 bzero(ds, sizeof (*ds)); 2491 ds->dumpcpu = CPU->cpu_id; 2492 2493 /* 2494 * Calculate the starting block for dump. If we're dumping on a 2495 * swap device, start 1/5 of the way in; otherwise, start at the 2496 * beginning. And never use the first page -- it may be a disk label. 2497 */ 2498 if (dumpvp->v_flag & VISSWAP) 2499 dumphdr->dump_start = P2ROUNDUP(dumpvp_size / 5, DUMP_OFFSET); 2500 else 2501 dumphdr->dump_start = DUMP_OFFSET; 2502 2503 dumphdr->dump_flags = DF_VALID | DF_COMPLETE | DF_LIVE | DF_COMPRESSED; 2504 dumphdr->dump_crashtime = gethrestime_sec(); 2505 dumphdr->dump_npages = 0; 2506 dumphdr->dump_nvtop = 0; 2507 bzero(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.bitmapsize)); 2508 dump_timeleft = dump_timeout; 2509 2510 if (panicstr) { 2511 dumphdr->dump_flags &= ~DF_LIVE; 2512 (void) VOP_DUMPCTL(dumpvp, DUMP_FREE, NULL, NULL); 2513 (void) VOP_DUMPCTL(dumpvp, DUMP_ALLOC, NULL, NULL); 2514 (void) vsnprintf(dumphdr->dump_panicstring, DUMP_PANICSIZE, 2515 panicstr, panicargs); 2516 2517 } 2518 2519 if (dump_conflags & DUMP_ALL) 2520 content = "all"; 2521 else if (dump_conflags & DUMP_CURPROC) 2522 content = "kernel + curproc"; 2523 else 2524 content = "kernel"; 2525 uprintf("dumping to %s, offset %lld, content: %s\n", dumppath, 2526 dumphdr->dump_start, content); 2527 2528 /* Make sure nodename is current */ 2529 bcopy(utsname.nodename, dumphdr->dump_utsname.nodename, SYS_NMLN); 2530 2531 /* 2532 * If this is a live dump, try to open a VCHR vnode for better 2533 * performance. We must take care to flush the buffer cache 2534 * first. 2535 */ 2536 if (!panicstr) { 2537 vnode_t *cdev_vp, *cmn_cdev_vp; 2538 2539 ASSERT(dumpbuf.cdev_vp == NULL); 2540 cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR); 2541 if (cdev_vp != NULL) { 2542 cmn_cdev_vp = common_specvp(cdev_vp); 2543 if (VOP_OPEN(&cmn_cdev_vp, FREAD | FWRITE, kcred, NULL) 2544 == 0) { 2545 if (vn_has_cached_data(dumpvp)) 2546 (void) pvn_vplist_dirty(dumpvp, 0, NULL, 2547 B_INVAL | B_TRUNC, kcred); 2548 dumpbuf.cdev_vp = cmn_cdev_vp; 2549 } else { 2550 VN_RELE(cdev_vp); 2551 } 2552 } 2553 } 2554 2555 /* 2556 * Leave room for the message and ereport save areas and terminal dump 2557 * header. 2558 */ 2559 dumpbuf.vp_limit = dumpvp_size - DUMP_LOGSIZE - DUMP_OFFSET - 2560 DUMP_ERPTSIZE; 2561 2562 /* 2563 * Write out the symbol table. It's no longer compressed, 2564 * so its 'size' and 'csize' are equal. 2565 */ 2566 dumpbuf.vp_off = dumphdr->dump_ksyms = dumphdr->dump_start + PAGESIZE; 2567 dumphdr->dump_ksyms_size = dumphdr->dump_ksyms_csize = 2568 ksyms_snapshot(dumpvp_ksyms_write, NULL, LONG_MAX); 2569 2570 /* 2571 * Write out the translation map. 2572 */ 2573 dumphdr->dump_map = dumpvp_flush(); 2574 dump_as(&kas); 2575 dumphdr->dump_nvtop += dump_plat_addr(); 2576 2577 /* 2578 * call into hat, which may have unmapped pages that also need to 2579 * be in the dump 2580 */ 2581 hat_dump(); 2582 2583 if (dump_conflags & DUMP_ALL) { 2584 mutex_enter(&pidlock); 2585 2586 for (npids = 0, p = practive; p != NULL; p = p->p_next) 2587 dumpcfg.pids[npids++] = p->p_pid; 2588 2589 mutex_exit(&pidlock); 2590 2591 for (pidx = 0; pidx < npids; pidx++) 2592 (void) dump_process(dumpcfg.pids[pidx]); 2593 2594 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2595 dump_timeleft = dump_timeout; 2596 BT_SET(dumpcfg.bitmap, bitnum); 2597 } 2598 dumphdr->dump_npages = dumpcfg.bitmapsize; 2599 dumphdr->dump_flags |= DF_ALL; 2600 2601 } else if (dump_conflags & DUMP_CURPROC) { 2602 /* 2603 * Determine which pid is to be dumped. If we're panicking, we 2604 * dump the process associated with panic_thread (if any). If 2605 * this is a live dump, we dump the process associated with 2606 * curthread. 2607 */ 2608 npids = 0; 2609 if (panicstr) { 2610 if (panic_thread != NULL && 2611 panic_thread->t_procp != NULL && 2612 panic_thread->t_procp != &p0) { 2613 dumpcfg.pids[npids++] = 2614 panic_thread->t_procp->p_pid; 2615 } 2616 } else { 2617 dumpcfg.pids[npids++] = curthread->t_procp->p_pid; 2618 } 2619 2620 if (npids && dump_process(dumpcfg.pids[0]) == 0) 2621 dumphdr->dump_flags |= DF_CURPROC; 2622 else 2623 dumphdr->dump_flags |= DF_KERNEL; 2624 2625 } else { 2626 dumphdr->dump_flags |= DF_KERNEL; 2627 } 2628 2629 dumphdr->dump_hashmask = (1 << highbit(dumphdr->dump_nvtop - 1)) - 1; 2630 2631 /* 2632 * Write out the pfn table. 2633 */ 2634 dumphdr->dump_pfn = dumpvp_flush(); 2635 dump_init_memlist_walker(&mlw); 2636 for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) { 2637 dump_timeleft = dump_timeout; 2638 if (!BT_TEST(dumpcfg.bitmap, bitnum)) 2639 continue; 2640 pfn = dump_bitnum_to_pfn(bitnum, &mlw); 2641 ASSERT(pfn != PFN_INVALID); 2642 dumpvp_write(&pfn, sizeof (pfn_t)); 2643 } 2644 dump_plat_pfn(); 2645 2646 /* 2647 * Write out all the pages. 2648 * Map pages, copy them handling UEs, compress, and write them out. 2649 * Cooperate with any helpers running on CPUs in panic_idle(). 2650 */ 2651 dumphdr->dump_data = dumpvp_flush(); 2652 2653 bzero(dumpcfg.helpermap, BT_SIZEOFMAP(NCPU)); 2654 ds->live = dumpcfg.clevel > 0 && 2655 (dumphdr->dump_flags & DF_LIVE) != 0; 2656 2657 save_dump_clevel = dumpcfg.clevel; 2658 if (panicstr) 2659 dumpsys_get_maxmem(); 2660 else if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2661 dumpcfg.clevel = DUMP_CLEVEL_LZJB; 2662 2663 dumpcfg.nhelper_used = 0; 2664 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2665 if (hp->page == NULL) { 2666 hp->helper = DONEHELPER; 2667 continue; 2668 } 2669 ++dumpcfg.nhelper_used; 2670 hp->helper = FREEHELPER; 2671 hp->taskqid = NULL; 2672 hp->ds = ds; 2673 bzero(&hp->perpage, sizeof (hp->perpage)); 2674 if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2) 2675 (void) BZ2_bzCompressReset(&hp->bzstream); 2676 } 2677 2678 CQ_OPEN(freebufq); 2679 CQ_OPEN(helperq); 2680 2681 dumpcfg.ncbuf_used = 0; 2682 for (cp = dumpcfg.cbuf; cp != &dumpcfg.cbuf[dumpcfg.ncbuf]; cp++) { 2683 if (cp->buf != NULL) { 2684 CQ_PUT(freebufq, cp, CBUF_FREEBUF); 2685 ++dumpcfg.ncbuf_used; 2686 } 2687 } 2688 2689 for (cp = dumpcfg.cmap; cp != &dumpcfg.cmap[dumpcfg.ncmap]; cp++) 2690 CQ_PUT(mainq, cp, CBUF_FREEMAP); 2691 2692 ds->start = gethrtime(); 2693 ds->iowaitts = ds->start; 2694 2695 /* start helpers */ 2696 if (ds->live) { 2697 int n = dumpcfg.nhelper_used; 2698 int pri = MINCLSYSPRI - 25; 2699 2700 livetaskq = taskq_create("LiveDump", n, pri, n, n, 2701 TASKQ_PREPOPULATE); 2702 for (hp = dumpcfg.helper; hp != hpend; hp++) { 2703 if (hp->page == NULL) 2704 continue; 2705 hp->helper = hp - dumpcfg.helper; 2706 hp->taskqid = taskq_dispatch(livetaskq, 2707 dumpsys_live_helper, (void *)hp, TQ_NOSLEEP); 2708 } 2709 2710 } else { 2711 dumpcfg.helpers_wanted = dumpcfg.clevel > 0; 2712 dumpsys_spinunlock(&dumpcfg.helper_lock); 2713 } 2714 2715 /* run main task */ 2716 dumpsys_main_task(ds); 2717 2718 ds->elapsed = gethrtime() - ds->start; 2719 if (ds->elapsed < 1) 2720 ds->elapsed = 1; 2721 2722 if (livetaskq != NULL) 2723 taskq_destroy(livetaskq); 2724 2725 if (ds->neednl) { 2726 uprintf("\n"); 2727 ds->neednl = 0; 2728 } 2729 2730 /* record actual pages dumped */ 2731 dumphdr->dump_npages = ds->npages; 2732 2733 /* platform-specific data */ 2734 dumphdr->dump_npages += dump_plat_data(dumpcfg.cbuf[0].buf); 2735 2736 /* note any errors by clearing DF_COMPLETE */ 2737 if (dump_ioerr || ds->npages < dumphdr->dump_npages) 2738 dumphdr->dump_flags &= ~DF_COMPLETE; 2739 2740 /* end of stream blocks */ 2741 datatag = 0; 2742 dumpvp_write(&datatag, sizeof (datatag)); 2743 2744 /* compression info in data header */ 2745 bzero(&datahdr, sizeof (datahdr)); 2746 datahdr.dump_datahdr_magic = DUMP_DATAHDR_MAGIC; 2747 datahdr.dump_datahdr_version = DUMP_DATAHDR_VERSION; 2748 datahdr.dump_maxcsize = CBUF_SIZE; 2749 datahdr.dump_maxrange = CBUF_MAPSIZE / PAGESIZE; 2750 datahdr.dump_nstreams = dumpcfg.nhelper_used; 2751 datahdr.dump_clevel = dumpcfg.clevel; 2752 #ifdef COLLECT_METRICS 2753 if (dump_metrics_on) 2754 datahdr.dump_metrics = dumpsys_metrics(ds, dumpcfg.cbuf[0].buf, 2755 MIN(dumpcfg.cbuf[0].size, DUMP_OFFSET - sizeof (dumphdr_t) - 2756 sizeof (dumpdatahdr_t))); 2757 #endif 2758 datahdr.dump_data_csize = dumpvp_flush() - dumphdr->dump_data; 2759 2760 /* 2761 * Write out the initial and terminal dump headers. 2762 */ 2763 dumpbuf.vp_off = dumphdr->dump_start; 2764 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 2765 (void) dumpvp_flush(); 2766 2767 dumpbuf.vp_limit = dumpvp_size; 2768 dumpbuf.vp_off = dumpbuf.vp_limit - DUMP_OFFSET; 2769 dumpvp_write(dumphdr, sizeof (dumphdr_t)); 2770 dumpvp_write(&datahdr, sizeof (dumpdatahdr_t)); 2771 dumpvp_write(dumpcfg.cbuf[0].buf, datahdr.dump_metrics); 2772 2773 (void) dumpvp_flush(); 2774 2775 uprintf("\r%3d%% done: %llu pages dumped, ", 2776 ds->percent_done, (u_longlong_t)ds->npages); 2777 2778 if (dump_ioerr == 0) { 2779 uprintf("dump succeeded\n"); 2780 } else { 2781 uprintf("dump failed: error %d\n", dump_ioerr); 2782 #ifdef DEBUG 2783 if (panicstr) 2784 debug_enter("dump failed"); 2785 #endif 2786 } 2787 2788 /* 2789 * Write out all undelivered messages. This has to be the *last* 2790 * thing we do because the dump process itself emits messages. 2791 */ 2792 if (panicstr) { 2793 dump_ereports(); 2794 dump_messages(); 2795 } 2796 2797 delay(2 * hz); /* let people see the 'done' message */ 2798 dump_timeleft = 0; 2799 dump_ioerr = 0; 2800 2801 /* restore settings after live dump completes */ 2802 if (!panicstr) { 2803 dumpcfg.clevel = save_dump_clevel; 2804 2805 /* release any VCHR open of the dump device */ 2806 if (dumpbuf.cdev_vp != NULL) { 2807 (void) VOP_CLOSE(dumpbuf.cdev_vp, FREAD | FWRITE, 1, 0, 2808 kcred, NULL); 2809 VN_RELE(dumpbuf.cdev_vp); 2810 dumpbuf.cdev_vp = NULL; 2811 } 2812 } 2813 } 2814 2815 /* 2816 * This function is called whenever the memory size, as represented 2817 * by the phys_install list, changes. 2818 */ 2819 void 2820 dump_resize() 2821 { 2822 mutex_enter(&dump_lock); 2823 dumphdr_init(); 2824 dumpbuf_resize(); 2825 dump_update_clevel(); 2826 mutex_exit(&dump_lock); 2827 } 2828 2829 /* 2830 * This function allows for dynamic resizing of a dump area. It assumes that 2831 * the underlying device has update its appropriate size(9P). 2832 */ 2833 int 2834 dumpvp_resize() 2835 { 2836 int error; 2837 vattr_t vattr; 2838 2839 mutex_enter(&dump_lock); 2840 vattr.va_mask = AT_SIZE; 2841 if ((error = VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL)) != 0) { 2842 mutex_exit(&dump_lock); 2843 return (error); 2844 } 2845 2846 if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) { 2847 mutex_exit(&dump_lock); 2848 return (ENOSPC); 2849 } 2850 2851 dumpvp_size = vattr.va_size & -DUMP_OFFSET; 2852 mutex_exit(&dump_lock); 2853 return (0); 2854 } 2855