1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/zfs_context.h> 30 #include <sys/spa.h> 31 #include <sys/txg.h> 32 #include <sys/spa_impl.h> 33 #include <sys/vdev_impl.h> 34 #include <sys/zio_impl.h> 35 #include <sys/zio_compress.h> 36 #include <sys/zio_checksum.h> 37 38 static void zio_vdev_io_enter(zio_t *zio); 39 static void zio_vdev_io_exit(zio_t *zio); 40 41 /* 42 * ========================================================================== 43 * I/O priority table 44 * ========================================================================== 45 */ 46 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 47 0, /* ZIO_PRIORITY_NOW */ 48 0, /* ZIO_PRIORITY_SYNC_READ */ 49 0, /* ZIO_PRIORITY_SYNC_WRITE */ 50 6, /* ZIO_PRIORITY_ASYNC_READ */ 51 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 52 4, /* ZIO_PRIORITY_FREE */ 53 0, /* ZIO_PRIORITY_CACHE_FILL */ 54 0, /* ZIO_PRIORITY_LOG_WRITE */ 55 10, /* ZIO_PRIORITY_RESILVER */ 56 20, /* ZIO_PRIORITY_SCRUB */ 57 }; 58 59 /* 60 * ========================================================================== 61 * I/O type descriptions 62 * ========================================================================== 63 */ 64 char *zio_type_name[ZIO_TYPES] = { 65 "null", "read", "write", "free", "claim", "ioctl" }; 66 67 /* At or above this size, force gang blocking - for testing */ 68 uint64_t zio_gang_bang = SPA_MAXBLOCKSIZE + 1; 69 70 typedef struct zio_sync_pass { 71 int zp_defer_free; /* defer frees after this pass */ 72 int zp_dontcompress; /* don't compress after this pass */ 73 int zp_rewrite; /* rewrite new bps after this pass */ 74 } zio_sync_pass_t; 75 76 zio_sync_pass_t zio_sync_pass = { 77 1, /* zp_defer_free */ 78 4, /* zp_dontcompress */ 79 1, /* zp_rewrite */ 80 }; 81 82 /* 83 * ========================================================================== 84 * I/O kmem caches 85 * ========================================================================== 86 */ 87 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 88 89 void 90 zio_init(void) 91 { 92 size_t c; 93 94 /* 95 * For small buffers, we want a cache for each multiple of 96 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 97 * for each quarter-power of 2. For large buffers, we want 98 * a cache for each multiple of PAGESIZE. 99 */ 100 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 101 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 102 size_t p2 = size; 103 size_t align = 0; 104 105 while (p2 & (p2 - 1)) 106 p2 &= p2 - 1; 107 108 if (size <= 4 * SPA_MINBLOCKSIZE) { 109 align = SPA_MINBLOCKSIZE; 110 } else if (P2PHASE(size, PAGESIZE) == 0) { 111 align = PAGESIZE; 112 } else if (P2PHASE(size, p2 >> 2) == 0) { 113 align = p2 >> 2; 114 } 115 116 if (align != 0) { 117 char name[30]; 118 (void) sprintf(name, "zio_buf_%lu", size); 119 zio_buf_cache[c] = kmem_cache_create(name, size, 120 align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); 121 dprintf("creating cache for size %5lx align %5lx\n", 122 size, align); 123 } 124 } 125 126 while (--c != 0) { 127 ASSERT(zio_buf_cache[c] != NULL); 128 if (zio_buf_cache[c - 1] == NULL) 129 zio_buf_cache[c - 1] = zio_buf_cache[c]; 130 } 131 } 132 133 void 134 zio_fini(void) 135 { 136 size_t c; 137 kmem_cache_t *last_cache = NULL; 138 139 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 140 if (zio_buf_cache[c] != last_cache) { 141 last_cache = zio_buf_cache[c]; 142 kmem_cache_destroy(zio_buf_cache[c]); 143 } 144 zio_buf_cache[c] = NULL; 145 } 146 } 147 148 /* 149 * ========================================================================== 150 * Allocate and free I/O buffers 151 * ========================================================================== 152 */ 153 void * 154 zio_buf_alloc(size_t size) 155 { 156 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 157 158 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 159 160 return (kmem_cache_alloc(zio_buf_cache[c], KM_SLEEP)); 161 } 162 163 void 164 zio_buf_free(void *buf, size_t size) 165 { 166 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 167 168 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 169 170 kmem_cache_free(zio_buf_cache[c], buf); 171 } 172 173 /* 174 * ========================================================================== 175 * Push and pop I/O transform buffers 176 * ========================================================================== 177 */ 178 static void 179 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) 180 { 181 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 182 183 zt->zt_data = data; 184 zt->zt_size = size; 185 zt->zt_bufsize = bufsize; 186 187 zt->zt_next = zio->io_transform_stack; 188 zio->io_transform_stack = zt; 189 190 zio->io_data = data; 191 zio->io_size = size; 192 } 193 194 static void 195 zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) 196 { 197 zio_transform_t *zt = zio->io_transform_stack; 198 199 *data = zt->zt_data; 200 *size = zt->zt_size; 201 *bufsize = zt->zt_bufsize; 202 203 zio->io_transform_stack = zt->zt_next; 204 kmem_free(zt, sizeof (zio_transform_t)); 205 206 if ((zt = zio->io_transform_stack) != NULL) { 207 zio->io_data = zt->zt_data; 208 zio->io_size = zt->zt_size; 209 } 210 } 211 212 static void 213 zio_clear_transform_stack(zio_t *zio) 214 { 215 void *data; 216 uint64_t size, bufsize; 217 218 ASSERT(zio->io_transform_stack != NULL); 219 220 zio_pop_transform(zio, &data, &size, &bufsize); 221 while (zio->io_transform_stack != NULL) { 222 zio_buf_free(data, bufsize); 223 zio_pop_transform(zio, &data, &size, &bufsize); 224 } 225 } 226 227 /* 228 * ========================================================================== 229 * Create the various types of I/O (read, write, free) 230 * ========================================================================== 231 */ 232 static zio_t * 233 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 234 void *data, uint64_t size, zio_done_func_t *done, void *private, 235 zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) 236 { 237 zio_t *zio; 238 239 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 240 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 241 242 zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); 243 zio->io_parent = pio; 244 zio->io_spa = spa; 245 zio->io_txg = txg; 246 if (bp != NULL) { 247 zio->io_bp = bp; 248 zio->io_bp_copy = *bp; 249 zio->io_bp_orig = *bp; 250 /* XXBP - Need to inherit this when it matters */ 251 zio->io_dva_index = 0; 252 } 253 zio->io_done = done; 254 zio->io_private = private; 255 zio->io_type = type; 256 zio->io_priority = priority; 257 zio->io_stage = stage; 258 zio->io_pipeline = pipeline; 259 zio->io_async_stages = ZIO_ASYNC_PIPELINE_STAGES; 260 zio->io_timestamp = lbolt64; 261 zio->io_flags = flags; 262 zio_push_transform(zio, data, size, size); 263 264 if (pio == NULL) { 265 if (!(flags & ZIO_FLAG_CONFIG_HELD)) 266 spa_config_enter(zio->io_spa, RW_READER); 267 zio->io_root = zio; 268 } else { 269 zio->io_root = pio->io_root; 270 271 mutex_enter(&pio->io_lock); 272 if (stage < ZIO_STAGE_READY) 273 pio->io_children_notready++; 274 pio->io_children_notdone++; 275 zio->io_sibling_next = pio->io_child; 276 zio->io_sibling_prev = NULL; 277 if (pio->io_child != NULL) 278 pio->io_child->io_sibling_prev = zio; 279 pio->io_child = zio; 280 mutex_exit(&pio->io_lock); 281 } 282 283 return (zio); 284 } 285 286 zio_t * 287 zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, 288 int flags) 289 { 290 zio_t *zio; 291 292 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 293 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, 294 ZIO_WAIT_FOR_CHILDREN_PIPELINE); 295 296 return (zio); 297 } 298 299 zio_t * 300 zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) 301 { 302 return (zio_null(NULL, spa, done, private, flags)); 303 } 304 305 zio_t * 306 zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, 307 uint64_t size, zio_done_func_t *done, void *private, 308 int priority, int flags) 309 { 310 zio_t *zio; 311 dva_t *dva; 312 313 ASSERT3U(size, ==, BP_GET_LSIZE(bp)); 314 315 zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, 316 ZIO_TYPE_READ, priority, flags, ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); 317 318 /* 319 * Work off our copy of the bp so the caller can free it. 320 */ 321 zio->io_bp = &zio->io_bp_copy; 322 323 bp = zio->io_bp; 324 dva = ZIO_GET_DVA(zio); 325 326 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 327 uint64_t csize = BP_GET_PSIZE(bp); 328 void *cbuf = zio_buf_alloc(csize); 329 330 zio_push_transform(zio, cbuf, csize, csize); 331 zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; 332 } 333 334 if (DVA_GET_GANG(dva)) { 335 uint64_t gsize = SPA_GANGBLOCKSIZE; 336 void *gbuf = zio_buf_alloc(gsize); 337 338 zio_push_transform(zio, gbuf, gsize, gsize); 339 zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; 340 } 341 342 return (zio); 343 } 344 345 zio_t * 346 zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, 347 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 348 zio_done_func_t *done, void *private, int priority, int flags) 349 { 350 zio_t *zio; 351 352 ASSERT(checksum >= ZIO_CHECKSUM_OFF && 353 checksum < ZIO_CHECKSUM_FUNCTIONS); 354 355 ASSERT(compress >= ZIO_COMPRESS_OFF && 356 compress < ZIO_COMPRESS_FUNCTIONS); 357 358 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 359 ZIO_TYPE_WRITE, priority, flags, 360 ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); 361 362 zio->io_checksum = checksum; 363 zio->io_compress = compress; 364 365 if (compress != ZIO_COMPRESS_OFF) 366 zio->io_async_stages |= 1U << ZIO_STAGE_WRITE_COMPRESS; 367 368 if (bp->blk_birth != txg) { 369 /* XXX the bp usually (always?) gets re-zeroed later */ 370 BP_ZERO(bp); 371 BP_SET_LSIZE(bp, size); 372 BP_SET_PSIZE(bp, size); 373 } 374 375 return (zio); 376 } 377 378 zio_t * 379 zio_rewrite(zio_t *pio, spa_t *spa, int checksum, 380 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 381 zio_done_func_t *done, void *private, int priority, int flags) 382 { 383 zio_t *zio; 384 385 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 386 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 387 ZIO_TYPE_WRITE, priority, flags, 388 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 389 390 zio->io_checksum = checksum; 391 zio->io_compress = ZIO_COMPRESS_OFF; 392 393 return (zio); 394 } 395 396 static zio_t * 397 zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, 398 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 399 zio_done_func_t *done, void *private, int priority, int flags) 400 { 401 zio_t *zio; 402 403 BP_ZERO(bp); 404 BP_SET_LSIZE(bp, size); 405 BP_SET_PSIZE(bp, size); 406 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 407 408 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 409 ZIO_TYPE_WRITE, priority, flags, 410 ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); 411 412 zio->io_checksum = checksum; 413 zio->io_compress = ZIO_COMPRESS_OFF; 414 415 return (zio); 416 } 417 418 zio_t * 419 zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 420 zio_done_func_t *done, void *private) 421 { 422 zio_t *zio; 423 424 ASSERT(!BP_IS_HOLE(bp)); 425 426 if (txg == spa->spa_syncing_txg && 427 spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { 428 bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); 429 return (zio_null(pio, spa, NULL, NULL, 0)); 430 } 431 432 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 433 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 434 ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, 0, 435 ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 436 437 zio->io_bp = &zio->io_bp_copy; 438 439 return (zio); 440 } 441 442 zio_t * 443 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 444 zio_done_func_t *done, void *private) 445 { 446 zio_t *zio; 447 448 /* 449 * A claim is an allocation of a specific block. Claims are needed 450 * to support immediate writes in the intent log. The issue is that 451 * immediate writes contain committed data, but in a txg that was 452 * *not* committed. Upon opening the pool after an unclean shutdown, 453 * the intent log claims all blocks that contain immediate write data 454 * so that the SPA knows they're in use. 455 * 456 * All claims *must* be resolved in the first txg -- before the SPA 457 * starts allocating blocks -- so that nothing is allocated twice. 458 */ 459 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 460 ASSERT3U(spa_first_txg(spa), <=, txg); 461 462 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 463 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 464 ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, 465 ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 466 467 zio->io_bp = &zio->io_bp_copy; 468 469 return (zio); 470 } 471 472 zio_t * 473 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 474 zio_done_func_t *done, void *private, int priority, int flags) 475 { 476 zio_t *zio; 477 int c; 478 479 if (vd->vdev_children == 0) { 480 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 481 ZIO_TYPE_IOCTL, priority, flags, 482 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 483 484 zio->io_vd = vd; 485 zio->io_cmd = cmd; 486 } else { 487 zio = zio_null(pio, spa, NULL, NULL, flags); 488 489 for (c = 0; c < vd->vdev_children; c++) 490 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 491 done, private, priority, flags)); 492 } 493 494 return (zio); 495 } 496 497 static void 498 zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, 499 int checksum) 500 { 501 ASSERT(vd->vdev_children == 0); 502 503 ASSERT(size <= SPA_MAXBLOCKSIZE); 504 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 505 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 506 507 ASSERT(offset + size <= VDEV_LABEL_START_SIZE || 508 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 509 ASSERT3U(offset + size, <=, vd->vdev_psize); 510 511 BP_ZERO(bp); 512 513 BP_SET_LSIZE(bp, size); 514 BP_SET_PSIZE(bp, size); 515 516 BP_SET_CHECKSUM(bp, checksum); 517 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 518 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 519 520 if (checksum != ZIO_CHECKSUM_OFF) 521 ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); 522 } 523 524 zio_t * 525 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 526 void *data, int checksum, zio_done_func_t *done, void *private, 527 int priority, int flags) 528 { 529 zio_t *zio; 530 blkptr_t blk; 531 532 zio_phys_bp_init(vd, &blk, offset, size, checksum); 533 534 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 535 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, 536 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 537 538 zio->io_vd = vd; 539 zio->io_offset = offset; 540 541 /* 542 * Work off our copy of the bp so the caller can free it. 543 */ 544 zio->io_bp = &zio->io_bp_copy; 545 546 return (zio); 547 } 548 549 zio_t * 550 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 551 void *data, int checksum, zio_done_func_t *done, void *private, 552 int priority, int flags) 553 { 554 zio_block_tail_t *zbt; 555 void *wbuf; 556 zio_t *zio; 557 blkptr_t blk; 558 559 zio_phys_bp_init(vd, &blk, offset, size, checksum); 560 561 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 562 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, 563 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 564 565 zio->io_vd = vd; 566 zio->io_offset = offset; 567 568 zio->io_bp = &zio->io_bp_copy; 569 zio->io_checksum = checksum; 570 571 if (zio_checksum_table[checksum].ci_zbt) { 572 /* 573 * zbt checksums are necessarily destructive -- they modify 574 * one word of the write buffer to hold the verifier/checksum. 575 * Therefore, we must make a local copy in case the data is 576 * being written to multiple places. 577 */ 578 wbuf = zio_buf_alloc(size); 579 bcopy(data, wbuf, size); 580 zio_push_transform(zio, wbuf, size, size); 581 582 zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; 583 zbt->zbt_cksum = blk.blk_cksum; 584 } 585 586 return (zio); 587 } 588 589 /* 590 * Create a child I/O to do some work for us. It has no associated bp. 591 */ 592 zio_t * 593 zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 594 void *data, uint64_t size, int type, int priority, int flags, 595 zio_done_func_t *done, void *private) 596 { 597 uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; 598 zio_t *cio; 599 600 if (type == ZIO_TYPE_READ && bp != NULL) { 601 /* 602 * If we have the bp, then the child should perform the 603 * checksum and the parent need not. This pushes error 604 * detection as close to the leaves as possible and 605 * eliminates redundant checksums in the interior nodes. 606 */ 607 pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; 608 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 609 } 610 611 cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, 612 done, private, type, priority, 613 (zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, 614 ZIO_STAGE_VDEV_IO_SETUP - 1, pipeline); 615 616 cio->io_vd = vd; 617 cio->io_offset = offset; 618 619 return (cio); 620 } 621 622 /* 623 * ========================================================================== 624 * Initiate I/O, either sync or async 625 * ========================================================================== 626 */ 627 int 628 zio_wait(zio_t *zio) 629 { 630 int error; 631 632 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 633 634 zio->io_waiter = curthread; 635 636 zio_next_stage_async(zio); 637 638 mutex_enter(&zio->io_lock); 639 while (zio->io_stalled != ZIO_STAGE_DONE) 640 cv_wait(&zio->io_cv, &zio->io_lock); 641 mutex_exit(&zio->io_lock); 642 643 error = zio->io_error; 644 645 kmem_free(zio, sizeof (zio_t)); 646 647 return (error); 648 } 649 650 void 651 zio_nowait(zio_t *zio) 652 { 653 zio_next_stage_async(zio); 654 } 655 656 /* 657 * ========================================================================== 658 * I/O pipeline interlocks: parent/child dependency scoreboarding 659 * ========================================================================== 660 */ 661 static void 662 zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) 663 { 664 mutex_enter(&zio->io_lock); 665 if (*countp == 0) { 666 ASSERT(zio->io_stalled == 0); 667 mutex_exit(&zio->io_lock); 668 zio_next_stage(zio); 669 } else { 670 if (zio->io_stage == ZIO_STAGE_VDEV_IO_START) 671 zio_vdev_io_exit(zio); 672 zio->io_stalled = stage; 673 mutex_exit(&zio->io_lock); 674 } 675 } 676 677 static void 678 zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) 679 { 680 zio_t *pio = zio->io_parent; 681 682 mutex_enter(&pio->io_lock); 683 if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 684 pio->io_error = zio->io_error; 685 if (--*countp == 0 && pio->io_stalled == stage) { 686 if (pio->io_stage == ZIO_STAGE_VDEV_IO_START) 687 zio_vdev_io_enter(pio); 688 pio->io_stalled = 0; 689 mutex_exit(&pio->io_lock); 690 zio_next_stage_async(pio); 691 } else { 692 mutex_exit(&pio->io_lock); 693 } 694 } 695 696 static void 697 zio_wait_children_ready(zio_t *zio) 698 { 699 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 700 &zio->io_children_notready); 701 } 702 703 void 704 zio_wait_children_done(zio_t *zio) 705 { 706 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 707 &zio->io_children_notdone); 708 } 709 710 static void 711 zio_ready(zio_t *zio) 712 { 713 zio_t *pio = zio->io_parent; 714 715 if (pio != NULL) 716 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 717 &pio->io_children_notready); 718 719 if (zio->io_bp) 720 zio->io_bp_copy = *zio->io_bp; 721 722 zio_next_stage(zio); 723 } 724 725 static void 726 zio_done(zio_t *zio) 727 { 728 zio_t *pio = zio->io_parent; 729 spa_t *spa = zio->io_spa; 730 blkptr_t *bp = zio->io_bp; 731 vdev_t *vd = zio->io_vd; 732 char blkbuf[BP_SPRINTF_LEN]; 733 734 ASSERT(zio->io_children_notready == 0); 735 ASSERT(zio->io_children_notdone == 0); 736 737 if (bp != NULL) { 738 ASSERT(bp->blk_pad[0] == 0); 739 ASSERT(bp->blk_pad[1] == 0); 740 ASSERT(bp->blk_pad[2] == 0); 741 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); 742 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 743 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) 744 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 745 } 746 747 if (vd != NULL) 748 vdev_stat_update(zio); 749 750 if (zio->io_error) { 751 sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 752 bp ? bp : &zio->io_bp_copy); 753 dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): error %d\n", 754 zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", 755 zio_type_name[zio->io_type], 756 vdev_description(vd), 757 (u_longlong_t)zio->io_offset, 758 zio, blkbuf, zio->io_error); 759 } 760 761 if (zio->io_numerrors != 0 && zio->io_type == ZIO_TYPE_WRITE) { 762 sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 763 bp ? bp : &zio->io_bp_copy); 764 dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): %d errors\n", 765 "partial write", 766 zio_type_name[zio->io_type], 767 vdev_description(vd), 768 (u_longlong_t)zio->io_offset, 769 zio, blkbuf, zio->io_numerrors); 770 } 771 772 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 773 sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 774 bp ? bp : &zio->io_bp_copy); 775 panic("ZFS: %s (%s on %s off %llx: zio %p %s): error %d", 776 zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", 777 zio_type_name[zio->io_type], 778 vdev_description(vd), 779 (u_longlong_t)zio->io_offset, 780 zio, blkbuf, zio->io_error); 781 } 782 783 zio_clear_transform_stack(zio); 784 785 if (zio->io_done) 786 zio->io_done(zio); 787 788 ASSERT(zio->io_delegate_list == NULL); 789 ASSERT(zio->io_delegate_next == NULL); 790 791 if (pio != NULL) { 792 zio_t *next, *prev; 793 794 mutex_enter(&pio->io_lock); 795 next = zio->io_sibling_next; 796 prev = zio->io_sibling_prev; 797 if (next != NULL) 798 next->io_sibling_prev = prev; 799 if (prev != NULL) 800 prev->io_sibling_next = next; 801 if (pio->io_child == zio) 802 pio->io_child = next; 803 mutex_exit(&pio->io_lock); 804 805 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 806 &pio->io_children_notdone); 807 } 808 809 if (pio == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_HELD)) 810 spa_config_exit(spa); 811 812 if (zio->io_waiter != NULL) { 813 mutex_enter(&zio->io_lock); 814 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 815 zio->io_stalled = zio->io_stage; 816 cv_broadcast(&zio->io_cv); 817 mutex_exit(&zio->io_lock); 818 } else { 819 kmem_free(zio, sizeof (zio_t)); 820 } 821 } 822 823 /* 824 * ========================================================================== 825 * Compression support 826 * ========================================================================== 827 */ 828 static void 829 zio_write_compress(zio_t *zio) 830 { 831 int compress = zio->io_compress; 832 blkptr_t *bp = zio->io_bp; 833 void *cbuf; 834 uint64_t lsize = zio->io_size; 835 uint64_t csize = lsize; 836 uint64_t cbufsize = 0; 837 int pass; 838 839 if (bp->blk_birth == zio->io_txg) { 840 /* 841 * We're rewriting an existing block, which means we're 842 * working on behalf of spa_sync(). For spa_sync() to 843 * converge, it must eventually be the case that we don't 844 * have to allocate new blocks. But compression changes 845 * the blocksize, which forces a reallocate, and makes 846 * convergence take longer. Therefore, after the first 847 * few passes, stop compressing to ensure convergence. 848 */ 849 pass = spa_sync_pass(zio->io_spa); 850 if (pass > zio_sync_pass.zp_dontcompress) 851 compress = ZIO_COMPRESS_OFF; 852 } else { 853 ASSERT(BP_IS_HOLE(bp)); 854 pass = 1; 855 } 856 857 if (compress != ZIO_COMPRESS_OFF) 858 if (!zio_compress_data(compress, zio->io_data, zio->io_size, 859 &cbuf, &csize, &cbufsize)) 860 compress = ZIO_COMPRESS_OFF; 861 862 if (compress != ZIO_COMPRESS_OFF && csize != 0) 863 zio_push_transform(zio, cbuf, csize, cbufsize); 864 865 /* 866 * The final pass of spa_sync() must be all rewrites, but the first 867 * few passes offer a trade-off: allocating blocks defers convergence, 868 * but newly allocated blocks are sequential, so they can be written 869 * to disk faster. Therefore, we allow the first few passes of 870 * spa_sync() to reallocate new blocks, but force rewrites after that. 871 * There should only be a handful of blocks after pass 1 in any case. 872 */ 873 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && 874 pass > zio_sync_pass.zp_rewrite) { 875 ASSERT(csize != 0); 876 ASSERT3U(BP_GET_COMPRESS(bp), ==, compress); 877 ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); 878 879 zio->io_pipeline = ZIO_REWRITE_PIPELINE; 880 } else { 881 if (bp->blk_birth == zio->io_txg) { 882 ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); 883 bzero(bp, sizeof (blkptr_t)); 884 } 885 if (csize == 0) { 886 BP_ZERO(bp); 887 zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; 888 } else { 889 BP_SET_LSIZE(bp, lsize); 890 BP_SET_PSIZE(bp, csize); 891 BP_SET_COMPRESS(bp, compress); 892 zio->io_pipeline = ZIO_WRITE_ALLOCATE_PIPELINE; 893 } 894 } 895 896 zio_next_stage(zio); 897 } 898 899 static void 900 zio_read_decompress(zio_t *zio) 901 { 902 blkptr_t *bp = zio->io_bp; 903 void *data; 904 uint64_t size; 905 uint64_t bufsize; 906 int compress = BP_GET_COMPRESS(bp); 907 908 ASSERT(compress != ZIO_COMPRESS_OFF); 909 910 zio_pop_transform(zio, &data, &size, &bufsize); 911 912 if (zio_decompress_data(compress, data, size, 913 zio->io_data, zio->io_size)) 914 zio->io_error = EIO; 915 916 zio_buf_free(data, bufsize); 917 918 zio_next_stage(zio); 919 } 920 921 /* 922 * ========================================================================== 923 * Gang block support 924 * ========================================================================== 925 */ 926 static void 927 zio_gang_pipeline(zio_t *zio) 928 { 929 /* 930 * By default, the pipeline assumes that we're dealing with a gang 931 * block. If we're not, strip out any gang-specific stages. 932 */ 933 if (!DVA_GET_GANG(ZIO_GET_DVA(zio))) 934 zio->io_pipeline &= ~ZIO_GANG_STAGES; 935 936 zio_next_stage(zio); 937 } 938 939 static void 940 zio_gang_byteswap(zio_t *zio) 941 { 942 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 943 944 if (BP_SHOULD_BYTESWAP(zio->io_bp)) 945 byteswap_uint64_array(zio->io_data, zio->io_size); 946 } 947 948 static void 949 zio_get_gang_header(zio_t *zio) 950 { 951 blkptr_t *bp = zio->io_bp; 952 uint64_t gsize = SPA_GANGBLOCKSIZE; 953 void *gbuf = zio_buf_alloc(gsize); 954 955 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 956 957 zio_push_transform(zio, gbuf, gsize, gsize); 958 959 zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, 960 NULL, NULL, ZIO_TYPE_READ, zio->io_priority, 961 zio->io_flags & ZIO_FLAG_GANG_INHERIT, 962 ZIO_STAGE_OPEN, ZIO_READ_PIPELINE)); 963 964 zio_wait_children_done(zio); 965 } 966 967 static void 968 zio_read_gang_members(zio_t *zio) 969 { 970 zio_gbh_phys_t *gbh; 971 uint64_t gsize, gbufsize, loff, lsize; 972 int i; 973 974 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 975 976 zio_gang_byteswap(zio); 977 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 978 979 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 980 blkptr_t *gbp = &gbh->zg_blkptr[i]; 981 lsize = BP_GET_PSIZE(gbp); 982 983 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 984 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 985 ASSERT3U(loff + lsize, <=, zio->io_size); 986 ASSERT(i < SPA_GBH_NBLKPTRS); 987 ASSERT(!BP_IS_HOLE(gbp)); 988 989 zio_nowait(zio_read(zio, zio->io_spa, gbp, 990 (char *)zio->io_data + loff, lsize, NULL, NULL, 991 zio->io_priority, zio->io_flags & ZIO_FLAG_GANG_INHERIT)); 992 } 993 994 zio_buf_free(gbh, gbufsize); 995 zio_wait_children_done(zio); 996 } 997 998 static void 999 zio_rewrite_gang_members(zio_t *zio) 1000 { 1001 zio_gbh_phys_t *gbh; 1002 uint64_t gsize, gbufsize, loff, lsize; 1003 int i; 1004 1005 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1006 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1007 1008 zio_gang_byteswap(zio); 1009 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1010 1011 ASSERT(gsize == gbufsize); 1012 1013 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1014 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1015 lsize = BP_GET_PSIZE(gbp); 1016 1017 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1018 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1019 ASSERT3U(loff + lsize, <=, zio->io_size); 1020 ASSERT(i < SPA_GBH_NBLKPTRS); 1021 ASSERT(!BP_IS_HOLE(gbp)); 1022 1023 zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, 1024 zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, 1025 NULL, NULL, zio->io_priority, zio->io_flags)); 1026 } 1027 1028 zio_push_transform(zio, gbh, gsize, gbufsize); 1029 zio_wait_children_ready(zio); 1030 } 1031 1032 static void 1033 zio_free_gang_members(zio_t *zio) 1034 { 1035 zio_gbh_phys_t *gbh; 1036 uint64_t gsize, gbufsize; 1037 int i; 1038 1039 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1040 1041 zio_gang_byteswap(zio); 1042 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1043 1044 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1045 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1046 1047 if (BP_IS_HOLE(gbp)) 1048 continue; 1049 zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 1050 gbp, NULL, NULL)); 1051 } 1052 1053 zio_buf_free(gbh, gbufsize); 1054 zio_next_stage(zio); 1055 } 1056 1057 static void 1058 zio_claim_gang_members(zio_t *zio) 1059 { 1060 zio_gbh_phys_t *gbh; 1061 uint64_t gsize, gbufsize; 1062 int i; 1063 1064 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1065 1066 zio_gang_byteswap(zio); 1067 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1068 1069 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1070 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1071 if (BP_IS_HOLE(gbp)) 1072 continue; 1073 zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, 1074 gbp, NULL, NULL)); 1075 } 1076 1077 zio_buf_free(gbh, gbufsize); 1078 zio_next_stage(zio); 1079 } 1080 1081 static void 1082 zio_write_allocate_gang_member_done(zio_t *zio) 1083 { 1084 zio_t *pio = zio->io_parent; 1085 dva_t *cdva = ZIO_GET_DVA(zio); 1086 dva_t *pdva = ZIO_GET_DVA(pio); 1087 uint64_t asize; 1088 1089 ASSERT(DVA_GET_GANG(pdva)); 1090 1091 /* XXBP - Need to be careful here with multiple DVAs */ 1092 mutex_enter(&pio->io_lock); 1093 asize = DVA_GET_ASIZE(pdva); 1094 asize += DVA_GET_ASIZE(cdva); 1095 DVA_SET_ASIZE(pdva, asize); 1096 mutex_exit(&pio->io_lock); 1097 } 1098 1099 static void 1100 zio_write_allocate_gang_members(zio_t *zio) 1101 { 1102 blkptr_t *bp = zio->io_bp; 1103 dva_t *dva = ZIO_GET_DVA(zio); 1104 zio_gbh_phys_t *gbh; 1105 uint64_t resid = zio->io_size; 1106 uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); 1107 uint64_t gsize, loff, lsize; 1108 uint32_t gbps_left; 1109 int error; 1110 int i; 1111 1112 gsize = SPA_GANGBLOCKSIZE; 1113 gbps_left = SPA_GBH_NBLKPTRS; 1114 1115 error = metaslab_alloc(zio->io_spa, gsize, dva, zio->io_txg); 1116 if (error == ENOSPC) 1117 panic("can't allocate gang block header"); 1118 ASSERT(error == 0); 1119 1120 DVA_SET_GANG(dva, 1); 1121 1122 bp->blk_birth = zio->io_txg; 1123 1124 gbh = zio_buf_alloc(gsize); 1125 bzero(gbh, gsize); 1126 1127 for (loff = 0, i = 0; loff != zio->io_size; 1128 loff += lsize, resid -= lsize, gbps_left--, i++) { 1129 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1130 dva = &gbp->blk_dva[0]; 1131 1132 ASSERT(gbps_left != 0); 1133 maxalloc = MIN(maxalloc, resid); 1134 1135 while (resid <= maxalloc * gbps_left) { 1136 error = metaslab_alloc(zio->io_spa, maxalloc, dva, 1137 zio->io_txg); 1138 if (error == 0) 1139 break; 1140 ASSERT3U(error, ==, ENOSPC); 1141 if (maxalloc == SPA_MINBLOCKSIZE) 1142 panic("really out of space"); 1143 maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); 1144 } 1145 1146 if (resid <= maxalloc * gbps_left) { 1147 lsize = maxalloc; 1148 BP_SET_LSIZE(gbp, lsize); 1149 BP_SET_PSIZE(gbp, lsize); 1150 BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); 1151 gbp->blk_birth = zio->io_txg; 1152 zio_nowait(zio_rewrite(zio, zio->io_spa, 1153 zio->io_checksum, zio->io_txg, gbp, 1154 (char *)zio->io_data + loff, lsize, 1155 zio_write_allocate_gang_member_done, NULL, 1156 zio->io_priority, zio->io_flags)); 1157 } else { 1158 lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); 1159 ASSERT(lsize != SPA_MINBLOCKSIZE); 1160 zio_nowait(zio_write_allocate(zio, zio->io_spa, 1161 zio->io_checksum, zio->io_txg, gbp, 1162 (char *)zio->io_data + loff, lsize, 1163 zio_write_allocate_gang_member_done, NULL, 1164 zio->io_priority, zio->io_flags)); 1165 } 1166 } 1167 1168 ASSERT(resid == 0 && loff == zio->io_size); 1169 1170 zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; 1171 1172 zio_push_transform(zio, gbh, gsize, gsize); 1173 zio_wait_children_done(zio); 1174 } 1175 1176 /* 1177 * ========================================================================== 1178 * Allocate and free blocks 1179 * ========================================================================== 1180 */ 1181 static void 1182 zio_dva_allocate(zio_t *zio) 1183 { 1184 blkptr_t *bp = zio->io_bp; 1185 dva_t *dva = ZIO_GET_DVA(zio); 1186 int error; 1187 1188 ASSERT(BP_IS_HOLE(bp)); 1189 1190 /* For testing, make some blocks above a certain size be gang blocks */ 1191 if (zio->io_size >= zio_gang_bang && (lbolt & 0x3) == 0) { 1192 zio_write_allocate_gang_members(zio); 1193 return; 1194 } 1195 1196 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1197 1198 error = metaslab_alloc(zio->io_spa, zio->io_size, dva, zio->io_txg); 1199 1200 if (error == 0) { 1201 bp->blk_birth = zio->io_txg; 1202 } else if (error == ENOSPC) { 1203 if (zio->io_size == SPA_MINBLOCKSIZE) 1204 panic("really, truly out of space"); 1205 zio_write_allocate_gang_members(zio); 1206 return; 1207 } else { 1208 zio->io_error = error; 1209 } 1210 zio_next_stage(zio); 1211 } 1212 1213 static void 1214 zio_dva_free(zio_t *zio) 1215 { 1216 blkptr_t *bp = zio->io_bp; 1217 dva_t *dva = ZIO_GET_DVA(zio); 1218 1219 ASSERT(!BP_IS_HOLE(bp)); 1220 1221 metaslab_free(zio->io_spa, dva, zio->io_txg); 1222 1223 BP_ZERO(bp); 1224 1225 zio_next_stage(zio); 1226 } 1227 1228 static void 1229 zio_dva_claim(zio_t *zio) 1230 { 1231 blkptr_t *bp = zio->io_bp; 1232 dva_t *dva = ZIO_GET_DVA(zio); 1233 1234 ASSERT(!BP_IS_HOLE(bp)); 1235 1236 zio->io_error = metaslab_claim(zio->io_spa, dva, zio->io_txg); 1237 1238 zio_next_stage(zio); 1239 } 1240 1241 static void 1242 zio_dva_translate(zio_t *zio) 1243 { 1244 spa_t *spa = zio->io_spa; 1245 dva_t *dva = ZIO_GET_DVA(zio); 1246 uint64_t vdev = DVA_GET_VDEV(dva); 1247 uint64_t offset = DVA_GET_OFFSET(dva); 1248 1249 ASSERT3U(zio->io_size, ==, ZIO_GET_IOSIZE(zio)); 1250 1251 zio->io_offset = offset; 1252 1253 if ((zio->io_vd = vdev_lookup_top(spa, vdev)) == NULL) 1254 zio->io_error = ENXIO; 1255 else if (offset + zio->io_size > zio->io_vd->vdev_asize) 1256 zio->io_error = EOVERFLOW; 1257 1258 zio_next_stage(zio); 1259 } 1260 1261 /* 1262 * ========================================================================== 1263 * Read and write to physical devices 1264 * ========================================================================== 1265 */ 1266 static void 1267 zio_vdev_io_enter(zio_t *zio) 1268 { 1269 vdev_t *tvd = zio->io_vd->vdev_top; 1270 1271 mutex_enter(&tvd->vdev_io_lock); 1272 ASSERT(zio->io_pending.list_next == NULL); 1273 list_insert_tail(&tvd->vdev_io_pending, zio); 1274 mutex_exit(&tvd->vdev_io_lock); 1275 } 1276 1277 static void 1278 zio_vdev_io_exit(zio_t *zio) 1279 { 1280 vdev_t *tvd = zio->io_vd->vdev_top; 1281 1282 mutex_enter(&tvd->vdev_io_lock); 1283 ASSERT(zio->io_pending.list_next != NULL); 1284 list_remove(&tvd->vdev_io_pending, zio); 1285 if (list_head(&tvd->vdev_io_pending) == NULL) 1286 cv_broadcast(&tvd->vdev_io_cv); 1287 mutex_exit(&tvd->vdev_io_lock); 1288 } 1289 1290 static void 1291 zio_vdev_io_retry(void *vdarg) 1292 { 1293 vdev_t *vd = vdarg; 1294 zio_t *zio, *zq; 1295 1296 ASSERT(vd == vd->vdev_top); 1297 1298 /* XXPOLICY */ 1299 delay(hz); 1300 1301 vdev_reopen(vd, &zq); 1302 1303 while ((zio = zq) != NULL) { 1304 zq = zio->io_retry_next; 1305 zio->io_retry_next = NULL; 1306 dprintf("async retry #%d for I/O to %s offset %llx\n", 1307 zio->io_retries, vdev_description(vd), zio->io_offset); 1308 zio_next_stage_async(zio); 1309 } 1310 } 1311 1312 static void 1313 zio_vdev_io_setup(zio_t *zio) 1314 { 1315 vdev_t *vd = zio->io_vd; 1316 1317 /* XXPOLICY */ 1318 if (zio->io_retries == 0 && vd == vd->vdev_top) 1319 zio->io_flags |= ZIO_FLAG_FAILFAST; 1320 1321 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && vd->vdev_children == 0) { 1322 zio->io_flags |= ZIO_FLAG_PHYSICAL; 1323 zio->io_offset += VDEV_LABEL_START_SIZE; 1324 } 1325 1326 zio_vdev_io_enter(zio); 1327 1328 zio_next_stage(zio); 1329 } 1330 1331 static void 1332 zio_vdev_io_start(zio_t *zio) 1333 { 1334 blkptr_t *bp = zio->io_bp; 1335 1336 ASSERT(P2PHASE(zio->io_offset, 1ULL << zio->io_vd->vdev_ashift) == 0); 1337 ASSERT(P2PHASE(zio->io_size, 1ULL << zio->io_vd->vdev_ashift) == 0); 1338 ASSERT(bp == NULL || ZIO_GET_IOSIZE(zio) == zio->io_size); 1339 ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); 1340 1341 vdev_io_start(zio); 1342 1343 /* zio_next_stage_async() gets called from io completion interrupt */ 1344 } 1345 1346 static void 1347 zio_vdev_io_done(zio_t *zio) 1348 { 1349 vdev_io_done(zio); 1350 } 1351 1352 /* XXPOLICY */ 1353 static boolean_t 1354 zio_should_retry(zio_t *zio) 1355 { 1356 vdev_t *vd = zio->io_vd; 1357 1358 if (zio->io_error == 0) 1359 return (B_FALSE); 1360 if (zio->io_delegate_list != NULL) 1361 return (B_FALSE); 1362 if (vd != vd->vdev_top) 1363 return (B_FALSE); 1364 if (zio->io_flags & ZIO_FLAG_DONT_RETRY) 1365 return (B_FALSE); 1366 if (zio->io_retries > 300 && 1367 (zio->io_flags & (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL))) 1368 return (B_FALSE); 1369 if (zio->io_retries > 1 && 1370 (zio->io_error == ECKSUM || zio->io_error == ENXIO)) 1371 return (B_FALSE); 1372 1373 return (B_TRUE); 1374 } 1375 1376 static void 1377 zio_vdev_io_assess(zio_t *zio) 1378 { 1379 vdev_t *vd = zio->io_vd; 1380 vdev_t *tvd = vd->vdev_top; 1381 1382 zio_vdev_io_exit(zio); 1383 1384 ASSERT(zio->io_vsd == NULL); 1385 1386 /* 1387 * If the I/O failed, determine whether we should attempt to retry it. 1388 */ 1389 /* XXPOLICY */ 1390 if (zio_should_retry(zio)) { 1391 zio_t *zq; 1392 1393 ASSERT(tvd == vd); 1394 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)); 1395 1396 zio->io_retries++; 1397 zio->io_error = 0; 1398 zio->io_flags &= ZIO_FLAG_VDEV_INHERIT; 1399 /* XXPOLICY */ 1400 zio->io_flags &= ~ZIO_FLAG_FAILFAST; 1401 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1402 zio->io_stage = ZIO_STAGE_VDEV_IO_SETUP - 1; 1403 1404 dprintf("retry #%d for %s to %s offset %llx\n", 1405 zio->io_retries, zio_type_name[zio->io_type], 1406 vdev_description(vd), zio->io_offset); 1407 1408 /* 1409 * If this is the first retry, do it immediately. 1410 */ 1411 /* XXPOLICY */ 1412 if (zio->io_retries == 1) { 1413 zio_next_stage_async(zio); 1414 return; 1415 } 1416 1417 /* 1418 * This was not the first retry, so go through the 1419 * longer enqueue/delay/vdev_reopen() process. 1420 */ 1421 mutex_enter(&tvd->vdev_io_lock); 1422 ASSERT(zio->io_retry_next == NULL); 1423 zio->io_retry_next = zq = tvd->vdev_io_retry; 1424 tvd->vdev_io_retry = zio; 1425 mutex_exit(&tvd->vdev_io_lock); 1426 if (zq == NULL) 1427 (void) taskq_dispatch( 1428 tvd->vdev_spa->spa_vdev_retry_taskq, 1429 zio_vdev_io_retry, tvd, TQ_SLEEP); 1430 return; 1431 } 1432 1433 zio_next_stage(zio); 1434 } 1435 1436 void 1437 zio_vdev_io_reissue(zio_t *zio) 1438 { 1439 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1440 ASSERT(zio->io_error == 0); 1441 1442 zio->io_stage--; 1443 } 1444 1445 void 1446 zio_vdev_io_redone(zio_t *zio) 1447 { 1448 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 1449 1450 zio->io_stage--; 1451 } 1452 1453 void 1454 zio_vdev_io_bypass(zio_t *zio) 1455 { 1456 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1457 ASSERT(zio->io_error == 0); 1458 1459 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 1460 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; 1461 } 1462 1463 /* 1464 * ========================================================================== 1465 * Generate and verify checksums 1466 * ========================================================================== 1467 */ 1468 static void 1469 zio_checksum_generate(zio_t *zio) 1470 { 1471 int checksum = zio->io_checksum; 1472 blkptr_t *bp = zio->io_bp; 1473 1474 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1475 1476 BP_SET_CHECKSUM(bp, checksum); 1477 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1478 1479 zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); 1480 1481 zio_next_stage(zio); 1482 } 1483 1484 static void 1485 zio_gang_checksum_generate(zio_t *zio) 1486 { 1487 zio_cksum_t zc; 1488 zio_gbh_phys_t *gbh = zio->io_data; 1489 1490 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1491 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1492 1493 zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); 1494 1495 zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); 1496 1497 zio_next_stage(zio); 1498 } 1499 1500 static void 1501 zio_checksum_verify(zio_t *zio) 1502 { 1503 if (zio->io_bp != NULL) { 1504 zio->io_error = zio_checksum_error(zio); 1505 if (zio->io_error) { 1506 dprintf("bad checksum on vdev %s\n", 1507 vdev_description(zio->io_vd)); 1508 } 1509 } 1510 1511 zio_next_stage(zio); 1512 } 1513 1514 /* 1515 * Called by RAID-Z to ensure we don't compute the checksum twice. 1516 */ 1517 void 1518 zio_checksum_verified(zio_t *zio) 1519 { 1520 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 1521 } 1522 1523 /* 1524 * Set the external verifier for a gang block based on stuff in the bp 1525 */ 1526 void 1527 zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) 1528 { 1529 zcp->zc_word[0] = DVA_GET_VDEV(ZIO_GET_DVA(zio)); 1530 zcp->zc_word[1] = DVA_GET_OFFSET(ZIO_GET_DVA(zio)); 1531 zcp->zc_word[2] = zio->io_bp->blk_birth; 1532 zcp->zc_word[3] = 0; 1533 } 1534 1535 /* 1536 * ========================================================================== 1537 * Define the pipeline 1538 * ========================================================================== 1539 */ 1540 typedef void zio_pipe_stage_t(zio_t *zio); 1541 1542 static void 1543 zio_badop(zio_t *zio) 1544 { 1545 panic("Invalid I/O pipeline stage %u for zio %p", zio->io_stage, zio); 1546 } 1547 1548 zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { 1549 zio_badop, 1550 zio_wait_children_ready, 1551 zio_write_compress, 1552 zio_checksum_generate, 1553 zio_gang_pipeline, 1554 zio_get_gang_header, 1555 zio_rewrite_gang_members, 1556 zio_free_gang_members, 1557 zio_claim_gang_members, 1558 zio_dva_allocate, 1559 zio_dva_free, 1560 zio_dva_claim, 1561 zio_gang_checksum_generate, 1562 zio_ready, 1563 zio_dva_translate, 1564 zio_vdev_io_setup, 1565 zio_vdev_io_start, 1566 zio_vdev_io_done, 1567 zio_vdev_io_assess, 1568 zio_wait_children_done, 1569 zio_checksum_verify, 1570 zio_read_gang_members, 1571 zio_read_decompress, 1572 zio_done, 1573 zio_badop 1574 }; 1575 1576 /* 1577 * Move an I/O to the next stage of the pipeline and execute that stage. 1578 * There's no locking on io_stage because there's no legitimate way for 1579 * multiple threads to be attempting to process the same I/O. 1580 */ 1581 void 1582 zio_next_stage(zio_t *zio) 1583 { 1584 uint32_t pipeline = zio->io_pipeline; 1585 1586 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1587 1588 if (zio->io_error) { 1589 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1590 zio, vdev_description(zio->io_vd), 1591 zio->io_offset, zio->io_stage, zio->io_error); 1592 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1593 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1594 } 1595 1596 while (((1U << ++zio->io_stage) & pipeline) == 0) 1597 continue; 1598 1599 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1600 ASSERT(zio->io_stalled == 0); 1601 1602 zio_pipeline[zio->io_stage](zio); 1603 } 1604 1605 void 1606 zio_next_stage_async(zio_t *zio) 1607 { 1608 taskq_t *tq; 1609 uint32_t pipeline = zio->io_pipeline; 1610 1611 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1612 1613 if (zio->io_error) { 1614 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1615 zio, vdev_description(zio->io_vd), 1616 zio->io_offset, zio->io_stage, zio->io_error); 1617 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1618 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1619 } 1620 1621 while (((1U << ++zio->io_stage) & pipeline) == 0) 1622 continue; 1623 1624 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1625 ASSERT(zio->io_stalled == 0); 1626 1627 /* 1628 * For performance, we'll probably want two sets of task queues: 1629 * per-CPU issue taskqs and per-CPU completion taskqs. The per-CPU 1630 * part is for read performance: since we have to make a pass over 1631 * the data to checksum it anyway, we want to do this on the same CPU 1632 * that issued the read, because (assuming CPU scheduling affinity) 1633 * that thread is probably still there. Getting this optimization 1634 * right avoids performance-hostile cache-to-cache transfers. 1635 * 1636 * Note that having two sets of task queues is also necessary for 1637 * correctness: if all of the issue threads get bogged down waiting 1638 * for dependent reads (e.g. metaslab freelist) to complete, then 1639 * there won't be any threads available to service I/O completion 1640 * interrupts. 1641 */ 1642 if ((1U << zio->io_stage) & zio->io_async_stages) { 1643 if (zio->io_stage < ZIO_STAGE_VDEV_IO_DONE) 1644 tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; 1645 else 1646 tq = zio->io_spa->spa_zio_intr_taskq[zio->io_type]; 1647 (void) taskq_dispatch(tq, 1648 (task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); 1649 } else { 1650 zio_pipeline[zio->io_stage](zio); 1651 } 1652 } 1653 1654 /* 1655 * Try to allocate an intent log block. Return 0 on success, errno on failure. 1656 */ 1657 int 1658 zio_alloc_blk(spa_t *spa, int checksum, uint64_t size, blkptr_t *bp, 1659 uint64_t txg) 1660 { 1661 int error; 1662 1663 spa_config_enter(spa, RW_READER); 1664 1665 BP_ZERO(bp); 1666 1667 error = metaslab_alloc(spa, size, BP_IDENTITY(bp), txg); 1668 1669 if (error == 0) { 1670 BP_SET_CHECKSUM(bp, checksum); 1671 BP_SET_LSIZE(bp, size); 1672 BP_SET_PSIZE(bp, size); 1673 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 1674 BP_SET_TYPE(bp, DMU_OT_INTENT_LOG); 1675 BP_SET_LEVEL(bp, 0); 1676 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1677 bp->blk_birth = txg; 1678 } 1679 1680 spa_config_exit(spa); 1681 1682 return (error); 1683 } 1684 1685 /* 1686 * Free an intent log block. We know it can't be a gang block, so there's 1687 * nothing to do except metaslab_free() it. 1688 */ 1689 void 1690 zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) 1691 { 1692 ASSERT(DVA_GET_GANG(BP_IDENTITY(bp)) == 0); 1693 1694 dprintf_bp(bp, "txg %llu: ", txg); 1695 1696 spa_config_enter(spa, RW_READER); 1697 1698 metaslab_free(spa, BP_IDENTITY(bp), txg); 1699 1700 spa_config_exit(spa); 1701 } 1702