1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/zfs_context.h> 30 #include <sys/spa.h> 31 #include <sys/txg.h> 32 #include <sys/spa_impl.h> 33 #include <sys/vdev_impl.h> 34 #include <sys/zio_impl.h> 35 #include <sys/zio_compress.h> 36 #include <sys/zio_checksum.h> 37 38 static void zio_vdev_io_enter(zio_t *zio); 39 static void zio_vdev_io_exit(zio_t *zio); 40 41 /* 42 * ========================================================================== 43 * I/O priority table 44 * ========================================================================== 45 */ 46 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 47 0, /* ZIO_PRIORITY_NOW */ 48 0, /* ZIO_PRIORITY_SYNC_READ */ 49 0, /* ZIO_PRIORITY_SYNC_WRITE */ 50 6, /* ZIO_PRIORITY_ASYNC_READ */ 51 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 52 4, /* ZIO_PRIORITY_FREE */ 53 0, /* ZIO_PRIORITY_CACHE_FILL */ 54 0, /* ZIO_PRIORITY_LOG_WRITE */ 55 10, /* ZIO_PRIORITY_RESILVER */ 56 20, /* ZIO_PRIORITY_SCRUB */ 57 }; 58 59 /* 60 * ========================================================================== 61 * I/O type descriptions 62 * ========================================================================== 63 */ 64 char *zio_type_name[ZIO_TYPES] = { 65 "null", "read", "write", "free", "claim", "ioctl" }; 66 67 /* At or above this size, force gang blocking - for testing */ 68 uint64_t zio_gang_bang = SPA_MAXBLOCKSIZE + 1; 69 70 typedef struct zio_sync_pass { 71 int zp_defer_free; /* defer frees after this pass */ 72 int zp_dontcompress; /* don't compress after this pass */ 73 int zp_rewrite; /* rewrite new bps after this pass */ 74 } zio_sync_pass_t; 75 76 zio_sync_pass_t zio_sync_pass = { 77 1, /* zp_defer_free */ 78 4, /* zp_dontcompress */ 79 1, /* zp_rewrite */ 80 }; 81 82 /* 83 * ========================================================================== 84 * I/O kmem caches 85 * ========================================================================== 86 */ 87 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 88 89 void 90 zio_init(void) 91 { 92 size_t c; 93 94 /* 95 * For small buffers, we want a cache for each multiple of 96 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 97 * for each quarter-power of 2. For large buffers, we want 98 * a cache for each multiple of PAGESIZE. 99 */ 100 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 101 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 102 size_t p2 = size; 103 size_t align = 0; 104 105 while (p2 & (p2 - 1)) 106 p2 &= p2 - 1; 107 108 if (size <= 4 * SPA_MINBLOCKSIZE) { 109 align = SPA_MINBLOCKSIZE; 110 } else if (P2PHASE(size, PAGESIZE) == 0) { 111 align = PAGESIZE; 112 } else if (P2PHASE(size, p2 >> 2) == 0) { 113 align = p2 >> 2; 114 } 115 116 if (align != 0) { 117 char name[30]; 118 (void) sprintf(name, "zio_buf_%lu", size); 119 zio_buf_cache[c] = kmem_cache_create(name, size, 120 align, NULL, NULL, NULL, NULL, NULL, 0); 121 dprintf("creating cache for size %5lx align %5lx\n", 122 size, align); 123 } 124 } 125 126 while (--c != 0) { 127 ASSERT(zio_buf_cache[c] != NULL); 128 if (zio_buf_cache[c - 1] == NULL) 129 zio_buf_cache[c - 1] = zio_buf_cache[c]; 130 } 131 } 132 133 void 134 zio_fini(void) 135 { 136 size_t c; 137 kmem_cache_t *last_cache = NULL; 138 139 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 140 if (zio_buf_cache[c] != last_cache) { 141 last_cache = zio_buf_cache[c]; 142 kmem_cache_destroy(zio_buf_cache[c]); 143 } 144 zio_buf_cache[c] = NULL; 145 } 146 } 147 148 /* 149 * ========================================================================== 150 * Allocate and free I/O buffers 151 * ========================================================================== 152 */ 153 void * 154 zio_buf_alloc(size_t size) 155 { 156 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 157 158 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 159 160 return (kmem_cache_alloc(zio_buf_cache[c], KM_SLEEP)); 161 } 162 163 void 164 zio_buf_free(void *buf, size_t size) 165 { 166 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 167 168 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 169 170 kmem_cache_free(zio_buf_cache[c], buf); 171 } 172 173 /* 174 * ========================================================================== 175 * Push and pop I/O transform buffers 176 * ========================================================================== 177 */ 178 static void 179 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) 180 { 181 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 182 183 zt->zt_data = data; 184 zt->zt_size = size; 185 zt->zt_bufsize = bufsize; 186 187 zt->zt_next = zio->io_transform_stack; 188 zio->io_transform_stack = zt; 189 190 zio->io_data = data; 191 zio->io_size = size; 192 } 193 194 static void 195 zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) 196 { 197 zio_transform_t *zt = zio->io_transform_stack; 198 199 *data = zt->zt_data; 200 *size = zt->zt_size; 201 *bufsize = zt->zt_bufsize; 202 203 zio->io_transform_stack = zt->zt_next; 204 kmem_free(zt, sizeof (zio_transform_t)); 205 206 if ((zt = zio->io_transform_stack) != NULL) { 207 zio->io_data = zt->zt_data; 208 zio->io_size = zt->zt_size; 209 } 210 } 211 212 static void 213 zio_clear_transform_stack(zio_t *zio) 214 { 215 void *data; 216 uint64_t size, bufsize; 217 218 ASSERT(zio->io_transform_stack != NULL); 219 220 zio_pop_transform(zio, &data, &size, &bufsize); 221 while (zio->io_transform_stack != NULL) { 222 zio_buf_free(data, bufsize); 223 zio_pop_transform(zio, &data, &size, &bufsize); 224 } 225 } 226 227 /* 228 * ========================================================================== 229 * Create the various types of I/O (read, write, free) 230 * ========================================================================== 231 */ 232 static zio_t * 233 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 234 void *data, uint64_t size, zio_done_func_t *done, void *private, 235 zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) 236 { 237 zio_t *zio; 238 239 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 240 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 241 242 zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); 243 zio->io_parent = pio; 244 zio->io_spa = spa; 245 zio->io_txg = txg; 246 if (bp != NULL) { 247 zio->io_bp = bp; 248 zio->io_bp_copy = *bp; 249 zio->io_bp_orig = *bp; 250 /* XXBP - Need to inherit this when it matters */ 251 zio->io_dva_index = 0; 252 } 253 zio->io_done = done; 254 zio->io_private = private; 255 zio->io_type = type; 256 zio->io_priority = priority; 257 zio->io_stage = stage; 258 zio->io_pipeline = pipeline; 259 zio->io_async_stages = ZIO_ASYNC_PIPELINE_STAGES; 260 zio->io_timestamp = lbolt64; 261 zio->io_flags = flags; 262 zio_push_transform(zio, data, size, size); 263 264 if (pio == NULL) { 265 if (!(flags & ZIO_FLAG_CONFIG_HELD)) 266 spa_config_enter(zio->io_spa, RW_READER); 267 zio->io_root = zio; 268 } else { 269 zio->io_root = pio->io_root; 270 271 mutex_enter(&pio->io_lock); 272 if (stage < ZIO_STAGE_READY) 273 pio->io_children_notready++; 274 pio->io_children_notdone++; 275 zio->io_sibling_next = pio->io_child; 276 zio->io_sibling_prev = NULL; 277 if (pio->io_child != NULL) 278 pio->io_child->io_sibling_prev = zio; 279 pio->io_child = zio; 280 mutex_exit(&pio->io_lock); 281 } 282 283 return (zio); 284 } 285 286 zio_t * 287 zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, 288 int flags) 289 { 290 zio_t *zio; 291 292 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 293 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, 294 ZIO_WAIT_FOR_CHILDREN_PIPELINE); 295 296 return (zio); 297 } 298 299 zio_t * 300 zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) 301 { 302 return (zio_null(NULL, spa, done, private, flags)); 303 } 304 305 zio_t * 306 zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, 307 uint64_t size, zio_done_func_t *done, void *private, 308 int priority, int flags) 309 { 310 zio_t *zio; 311 dva_t *dva; 312 313 ASSERT3U(size, ==, BP_GET_LSIZE(bp)); 314 315 zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, 316 ZIO_TYPE_READ, priority, flags, ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); 317 318 /* 319 * Work off our copy of the bp so the caller can free it. 320 */ 321 zio->io_bp = &zio->io_bp_copy; 322 323 bp = zio->io_bp; 324 dva = ZIO_GET_DVA(zio); 325 326 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 327 uint64_t csize = BP_GET_PSIZE(bp); 328 void *cbuf = zio_buf_alloc(csize); 329 330 zio_push_transform(zio, cbuf, csize, csize); 331 zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; 332 } 333 334 if (DVA_GET_GANG(dva)) { 335 uint64_t gsize = SPA_GANGBLOCKSIZE; 336 void *gbuf = zio_buf_alloc(gsize); 337 338 zio_push_transform(zio, gbuf, gsize, gsize); 339 zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; 340 } 341 342 return (zio); 343 } 344 345 zio_t * 346 zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, 347 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 348 zio_done_func_t *done, void *private, int priority, int flags) 349 { 350 zio_t *zio; 351 352 ASSERT(checksum >= ZIO_CHECKSUM_OFF && 353 checksum < ZIO_CHECKSUM_FUNCTIONS); 354 355 ASSERT(compress >= ZIO_COMPRESS_OFF && 356 compress < ZIO_COMPRESS_FUNCTIONS); 357 358 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 359 ZIO_TYPE_WRITE, priority, flags, 360 ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); 361 362 zio->io_checksum = checksum; 363 zio->io_compress = compress; 364 365 if (compress != ZIO_COMPRESS_OFF) 366 zio->io_async_stages |= 1U << ZIO_STAGE_WRITE_COMPRESS; 367 368 if (bp->blk_birth != txg) { 369 /* XXX the bp usually (always?) gets re-zeroed later */ 370 BP_ZERO(bp); 371 BP_SET_LSIZE(bp, size); 372 BP_SET_PSIZE(bp, size); 373 } 374 375 return (zio); 376 } 377 378 zio_t * 379 zio_rewrite(zio_t *pio, spa_t *spa, int checksum, 380 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 381 zio_done_func_t *done, void *private, int priority, int flags) 382 { 383 zio_t *zio; 384 385 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 386 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 387 ZIO_TYPE_WRITE, priority, flags, 388 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 389 390 zio->io_checksum = checksum; 391 zio->io_compress = ZIO_COMPRESS_OFF; 392 393 return (zio); 394 } 395 396 static zio_t * 397 zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, 398 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 399 zio_done_func_t *done, void *private, int priority, int flags) 400 { 401 zio_t *zio; 402 403 BP_ZERO(bp); 404 BP_SET_LSIZE(bp, size); 405 BP_SET_PSIZE(bp, size); 406 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 407 408 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 409 ZIO_TYPE_WRITE, priority, flags, 410 ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); 411 412 zio->io_checksum = checksum; 413 zio->io_compress = ZIO_COMPRESS_OFF; 414 415 return (zio); 416 } 417 418 zio_t * 419 zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 420 zio_done_func_t *done, void *private) 421 { 422 zio_t *zio; 423 424 ASSERT(!BP_IS_HOLE(bp)); 425 426 if (txg == spa->spa_syncing_txg && 427 spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { 428 bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); 429 return (zio_null(pio, spa, NULL, NULL, 0)); 430 } 431 432 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 433 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 434 ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, 0, 435 ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 436 437 zio->io_bp = &zio->io_bp_copy; 438 439 return (zio); 440 } 441 442 zio_t * 443 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 444 zio_done_func_t *done, void *private) 445 { 446 zio_t *zio; 447 448 /* 449 * A claim is an allocation of a specific block. Claims are needed 450 * to support immediate writes in the intent log. The issue is that 451 * immediate writes contain committed data, but in a txg that was 452 * *not* committed. Upon opening the pool after an unclean shutdown, 453 * the intent log claims all blocks that contain immediate write data 454 * so that the SPA knows they're in use. 455 * 456 * All claims *must* be resolved in the first txg -- before the SPA 457 * starts allocating blocks -- so that nothing is allocated twice. 458 */ 459 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 460 ASSERT3U(spa_first_txg(spa), <=, txg); 461 462 /* XXBP - We need to re-evaluate when to insert pipeline stages */ 463 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 464 ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, 465 ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 466 467 zio->io_bp = &zio->io_bp_copy; 468 469 return (zio); 470 } 471 472 zio_t * 473 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 474 zio_done_func_t *done, void *private, int priority, int flags) 475 { 476 zio_t *zio; 477 int c; 478 479 if (vd->vdev_children == 0) { 480 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 481 ZIO_TYPE_IOCTL, priority, flags, 482 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 483 484 zio->io_vd = vd; 485 zio->io_cmd = cmd; 486 } else { 487 zio = zio_null(pio, spa, NULL, NULL, flags); 488 489 for (c = 0; c < vd->vdev_children; c++) 490 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 491 done, private, priority, flags)); 492 } 493 494 return (zio); 495 } 496 497 static void 498 zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, 499 int checksum) 500 { 501 ASSERT(vd->vdev_children == 0); 502 503 ASSERT(size <= SPA_MAXBLOCKSIZE); 504 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 505 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 506 507 ASSERT(offset + size <= VDEV_LABEL_START_SIZE || 508 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 509 ASSERT3U(offset + size, <=, vd->vdev_psize); 510 511 BP_ZERO(bp); 512 513 BP_SET_LSIZE(bp, size); 514 BP_SET_PSIZE(bp, size); 515 516 BP_SET_CHECKSUM(bp, checksum); 517 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 518 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 519 520 if (checksum != ZIO_CHECKSUM_OFF) 521 ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); 522 } 523 524 zio_t * 525 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 526 void *data, int checksum, zio_done_func_t *done, void *private, 527 int priority, int flags) 528 { 529 zio_t *zio; 530 blkptr_t blk; 531 532 zio_phys_bp_init(vd, &blk, offset, size, checksum); 533 534 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 535 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, 536 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 537 538 zio->io_vd = vd; 539 zio->io_offset = offset; 540 541 /* 542 * Work off our copy of the bp so the caller can free it. 543 */ 544 zio->io_bp = &zio->io_bp_copy; 545 546 return (zio); 547 } 548 549 zio_t * 550 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 551 void *data, int checksum, zio_done_func_t *done, void *private, 552 int priority, int flags) 553 { 554 zio_block_tail_t *zbt; 555 void *wbuf; 556 zio_t *zio; 557 blkptr_t blk; 558 559 zio_phys_bp_init(vd, &blk, offset, size, checksum); 560 561 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 562 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, 563 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 564 565 zio->io_vd = vd; 566 zio->io_offset = offset; 567 568 zio->io_bp = &zio->io_bp_copy; 569 zio->io_checksum = checksum; 570 571 if (zio_checksum_table[checksum].ci_zbt) { 572 /* 573 * zbt checksums are necessarily destructive -- they modify 574 * one word of the write buffer to hold the verifier/checksum. 575 * Therefore, we must make a local copy in case the data is 576 * being written to multiple places. 577 */ 578 wbuf = zio_buf_alloc(size); 579 bcopy(data, wbuf, size); 580 zio_push_transform(zio, wbuf, size, size); 581 582 zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; 583 zbt->zbt_cksum = blk.blk_cksum; 584 } 585 586 return (zio); 587 } 588 589 /* 590 * Create a child I/O to do some work for us. It has no associated bp. 591 */ 592 zio_t * 593 zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 594 void *data, uint64_t size, int type, int priority, int flags, 595 zio_done_func_t *done, void *private) 596 { 597 uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; 598 zio_t *cio; 599 600 if (type == ZIO_TYPE_READ && bp != NULL) { 601 /* 602 * If we have the bp, then the child should perform the 603 * checksum and the parent need not. This pushes error 604 * detection as close to the leaves as possible and 605 * eliminates redundant checksums in the interior nodes. 606 */ 607 pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; 608 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 609 } 610 611 cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, 612 done, private, type, priority, 613 (zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, 614 ZIO_STAGE_VDEV_IO_SETUP - 1, pipeline); 615 616 cio->io_vd = vd; 617 cio->io_offset = offset; 618 619 return (cio); 620 } 621 622 /* 623 * ========================================================================== 624 * Initiate I/O, either sync or async 625 * ========================================================================== 626 */ 627 int 628 zio_wait(zio_t *zio) 629 { 630 int error; 631 632 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 633 634 zio->io_waiter = curthread; 635 636 zio_next_stage_async(zio); 637 638 mutex_enter(&zio->io_lock); 639 while (zio->io_stalled != ZIO_STAGE_DONE) 640 cv_wait(&zio->io_cv, &zio->io_lock); 641 mutex_exit(&zio->io_lock); 642 643 error = zio->io_error; 644 645 kmem_free(zio, sizeof (zio_t)); 646 647 return (error); 648 } 649 650 void 651 zio_nowait(zio_t *zio) 652 { 653 zio_next_stage_async(zio); 654 } 655 656 /* 657 * ========================================================================== 658 * I/O pipeline interlocks: parent/child dependency scoreboarding 659 * ========================================================================== 660 */ 661 static void 662 zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) 663 { 664 mutex_enter(&zio->io_lock); 665 if (*countp == 0) { 666 ASSERT(zio->io_stalled == 0); 667 mutex_exit(&zio->io_lock); 668 zio_next_stage(zio); 669 } else { 670 if (zio->io_stage == ZIO_STAGE_VDEV_IO_START) 671 zio_vdev_io_exit(zio); 672 zio->io_stalled = stage; 673 mutex_exit(&zio->io_lock); 674 } 675 } 676 677 static void 678 zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) 679 { 680 zio_t *pio = zio->io_parent; 681 682 mutex_enter(&pio->io_lock); 683 if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 684 pio->io_error = zio->io_error; 685 if (--*countp == 0 && pio->io_stalled == stage) { 686 if (pio->io_stage == ZIO_STAGE_VDEV_IO_START) 687 zio_vdev_io_enter(pio); 688 pio->io_stalled = 0; 689 mutex_exit(&pio->io_lock); 690 zio_next_stage_async(pio); 691 } else { 692 mutex_exit(&pio->io_lock); 693 } 694 } 695 696 static void 697 zio_wait_children_ready(zio_t *zio) 698 { 699 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 700 &zio->io_children_notready); 701 } 702 703 void 704 zio_wait_children_done(zio_t *zio) 705 { 706 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 707 &zio->io_children_notdone); 708 } 709 710 static void 711 zio_ready(zio_t *zio) 712 { 713 zio_t *pio = zio->io_parent; 714 715 if (pio != NULL) 716 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 717 &pio->io_children_notready); 718 719 if (zio->io_bp) 720 zio->io_bp_copy = *zio->io_bp; 721 722 zio_next_stage(zio); 723 } 724 725 static void 726 zio_done(zio_t *zio) 727 { 728 zio_t *pio = zio->io_parent; 729 spa_t *spa = zio->io_spa; 730 blkptr_t *bp = zio->io_bp; 731 vdev_t *vd = zio->io_vd; 732 char blkbuf[300]; 733 734 ASSERT(zio->io_children_notready == 0); 735 ASSERT(zio->io_children_notdone == 0); 736 737 if (bp != NULL) { 738 ASSERT(bp->blk_pad[0] == 0); 739 ASSERT(bp->blk_pad[1] == 0); 740 ASSERT(bp->blk_pad[2] == 0); 741 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); 742 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 743 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) 744 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 745 } 746 747 if (vd != NULL) 748 vdev_stat_update(zio); 749 750 if (zio->io_error) { 751 sprintf_blkptr(blkbuf, bp ? bp : &zio->io_bp_copy); 752 dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): error %d\n", 753 zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", 754 zio_type_name[zio->io_type], 755 vdev_description(vd), 756 (u_longlong_t)zio->io_offset, 757 zio, blkbuf, zio->io_error); 758 } 759 760 if (zio->io_numerrors != 0 && zio->io_type == ZIO_TYPE_WRITE) { 761 sprintf_blkptr(blkbuf, bp ? bp : &zio->io_bp_copy); 762 dprintf("ZFS: %s (%s on %s off %llx: zio %p %s): %d errors\n", 763 "partial write", 764 zio_type_name[zio->io_type], 765 vdev_description(vd), 766 (u_longlong_t)zio->io_offset, 767 zio, blkbuf, zio->io_numerrors); 768 } 769 770 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 771 sprintf_blkptr(blkbuf, bp ? bp : &zio->io_bp_copy); 772 panic("ZFS: %s (%s on %s off %llx: zio %p %s): error %d", 773 zio->io_error == ECKSUM ? "bad checksum" : "I/O failure", 774 zio_type_name[zio->io_type], 775 vdev_description(vd), 776 (u_longlong_t)zio->io_offset, 777 zio, blkbuf, zio->io_error); 778 } 779 780 zio_clear_transform_stack(zio); 781 782 if (zio->io_done) 783 zio->io_done(zio); 784 785 ASSERT(zio->io_delegate_list == NULL); 786 ASSERT(zio->io_delegate_next == NULL); 787 788 if (pio != NULL) { 789 zio_t *next, *prev; 790 791 mutex_enter(&pio->io_lock); 792 next = zio->io_sibling_next; 793 prev = zio->io_sibling_prev; 794 if (next != NULL) 795 next->io_sibling_prev = prev; 796 if (prev != NULL) 797 prev->io_sibling_next = next; 798 if (pio->io_child == zio) 799 pio->io_child = next; 800 mutex_exit(&pio->io_lock); 801 802 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 803 &pio->io_children_notdone); 804 } 805 806 if (pio == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_HELD)) 807 spa_config_exit(spa); 808 809 if (zio->io_waiter != NULL) { 810 mutex_enter(&zio->io_lock); 811 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 812 zio->io_stalled = zio->io_stage; 813 cv_broadcast(&zio->io_cv); 814 mutex_exit(&zio->io_lock); 815 } else { 816 kmem_free(zio, sizeof (zio_t)); 817 } 818 } 819 820 /* 821 * ========================================================================== 822 * Compression support 823 * ========================================================================== 824 */ 825 static void 826 zio_write_compress(zio_t *zio) 827 { 828 int compress = zio->io_compress; 829 blkptr_t *bp = zio->io_bp; 830 void *cbuf; 831 uint64_t lsize = zio->io_size; 832 uint64_t csize = lsize; 833 uint64_t cbufsize = 0; 834 int pass; 835 836 if (bp->blk_birth == zio->io_txg) { 837 /* 838 * We're rewriting an existing block, which means we're 839 * working on behalf of spa_sync(). For spa_sync() to 840 * converge, it must eventually be the case that we don't 841 * have to allocate new blocks. But compression changes 842 * the blocksize, which forces a reallocate, and makes 843 * convergence take longer. Therefore, after the first 844 * few passes, stop compressing to ensure convergence. 845 */ 846 pass = spa_sync_pass(zio->io_spa); 847 if (pass > zio_sync_pass.zp_dontcompress) 848 compress = ZIO_COMPRESS_OFF; 849 } else { 850 ASSERT(BP_IS_HOLE(bp)); 851 pass = 1; 852 } 853 854 if (compress != ZIO_COMPRESS_OFF) 855 if (!zio_compress_data(compress, zio->io_data, zio->io_size, 856 &cbuf, &csize, &cbufsize)) 857 compress = ZIO_COMPRESS_OFF; 858 859 if (compress != ZIO_COMPRESS_OFF && csize != 0) 860 zio_push_transform(zio, cbuf, csize, cbufsize); 861 862 /* 863 * The final pass of spa_sync() must be all rewrites, but the first 864 * few passes offer a trade-off: allocating blocks defers convergence, 865 * but newly allocated blocks are sequential, so they can be written 866 * to disk faster. Therefore, we allow the first few passes of 867 * spa_sync() to reallocate new blocks, but force rewrites after that. 868 * There should only be a handful of blocks after pass 1 in any case. 869 */ 870 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && 871 pass > zio_sync_pass.zp_rewrite) { 872 ASSERT(csize != 0); 873 ASSERT3U(BP_GET_COMPRESS(bp), ==, compress); 874 ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); 875 876 zio->io_pipeline = ZIO_REWRITE_PIPELINE; 877 } else { 878 if (bp->blk_birth == zio->io_txg) { 879 ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); 880 bzero(bp, sizeof (blkptr_t)); 881 } 882 if (csize == 0) { 883 BP_ZERO(bp); 884 zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; 885 } else { 886 BP_SET_LSIZE(bp, lsize); 887 BP_SET_PSIZE(bp, csize); 888 BP_SET_COMPRESS(bp, compress); 889 zio->io_pipeline = ZIO_WRITE_ALLOCATE_PIPELINE; 890 } 891 } 892 893 zio_next_stage(zio); 894 } 895 896 static void 897 zio_read_decompress(zio_t *zio) 898 { 899 blkptr_t *bp = zio->io_bp; 900 void *data; 901 uint64_t size; 902 uint64_t bufsize; 903 int compress = BP_GET_COMPRESS(bp); 904 905 ASSERT(compress != ZIO_COMPRESS_OFF); 906 907 zio_pop_transform(zio, &data, &size, &bufsize); 908 909 if (zio_decompress_data(compress, data, size, 910 zio->io_data, zio->io_size)) 911 zio->io_error = EIO; 912 913 zio_buf_free(data, bufsize); 914 915 zio_next_stage(zio); 916 } 917 918 /* 919 * ========================================================================== 920 * Gang block support 921 * ========================================================================== 922 */ 923 static void 924 zio_gang_pipeline(zio_t *zio) 925 { 926 /* 927 * By default, the pipeline assumes that we're dealing with a gang 928 * block. If we're not, strip out any gang-specific stages. 929 */ 930 if (!DVA_GET_GANG(ZIO_GET_DVA(zio))) 931 zio->io_pipeline &= ~ZIO_GANG_STAGES; 932 933 zio_next_stage(zio); 934 } 935 936 static void 937 zio_gang_byteswap(zio_t *zio) 938 { 939 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 940 941 if (BP_SHOULD_BYTESWAP(zio->io_bp)) 942 byteswap_uint64_array(zio->io_data, zio->io_size); 943 } 944 945 static void 946 zio_get_gang_header(zio_t *zio) 947 { 948 blkptr_t *bp = zio->io_bp; 949 uint64_t gsize = SPA_GANGBLOCKSIZE; 950 void *gbuf = zio_buf_alloc(gsize); 951 952 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 953 954 zio_push_transform(zio, gbuf, gsize, gsize); 955 956 zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, 957 NULL, NULL, ZIO_TYPE_READ, zio->io_priority, 958 zio->io_flags & ZIO_FLAG_GANG_INHERIT, 959 ZIO_STAGE_OPEN, ZIO_READ_PIPELINE)); 960 961 zio_wait_children_done(zio); 962 } 963 964 static void 965 zio_read_gang_members(zio_t *zio) 966 { 967 zio_gbh_phys_t *gbh; 968 uint64_t gsize, gbufsize, loff, lsize; 969 int i; 970 971 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 972 973 zio_gang_byteswap(zio); 974 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 975 976 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 977 blkptr_t *gbp = &gbh->zg_blkptr[i]; 978 lsize = BP_GET_PSIZE(gbp); 979 980 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 981 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 982 ASSERT3U(loff + lsize, <=, zio->io_size); 983 ASSERT(i < SPA_GBH_NBLKPTRS); 984 ASSERT(!BP_IS_HOLE(gbp)); 985 986 zio_nowait(zio_read(zio, zio->io_spa, gbp, 987 (char *)zio->io_data + loff, lsize, NULL, NULL, 988 zio->io_priority, zio->io_flags & ZIO_FLAG_GANG_INHERIT)); 989 } 990 991 zio_buf_free(gbh, gbufsize); 992 zio_wait_children_done(zio); 993 } 994 995 static void 996 zio_rewrite_gang_members(zio_t *zio) 997 { 998 zio_gbh_phys_t *gbh; 999 uint64_t gsize, gbufsize, loff, lsize; 1000 int i; 1001 1002 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1003 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1004 1005 zio_gang_byteswap(zio); 1006 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1007 1008 ASSERT(gsize == gbufsize); 1009 1010 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1011 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1012 lsize = BP_GET_PSIZE(gbp); 1013 1014 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1015 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1016 ASSERT3U(loff + lsize, <=, zio->io_size); 1017 ASSERT(i < SPA_GBH_NBLKPTRS); 1018 ASSERT(!BP_IS_HOLE(gbp)); 1019 1020 zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, 1021 zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, 1022 NULL, NULL, zio->io_priority, zio->io_flags)); 1023 } 1024 1025 zio_push_transform(zio, gbh, gsize, gbufsize); 1026 zio_wait_children_ready(zio); 1027 } 1028 1029 static void 1030 zio_free_gang_members(zio_t *zio) 1031 { 1032 zio_gbh_phys_t *gbh; 1033 uint64_t gsize, gbufsize; 1034 int i; 1035 1036 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1037 1038 zio_gang_byteswap(zio); 1039 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1040 1041 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1042 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1043 1044 if (BP_IS_HOLE(gbp)) 1045 continue; 1046 zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 1047 gbp, NULL, NULL)); 1048 } 1049 1050 zio_buf_free(gbh, gbufsize); 1051 zio_next_stage(zio); 1052 } 1053 1054 static void 1055 zio_claim_gang_members(zio_t *zio) 1056 { 1057 zio_gbh_phys_t *gbh; 1058 uint64_t gsize, gbufsize; 1059 int i; 1060 1061 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1062 1063 zio_gang_byteswap(zio); 1064 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1065 1066 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1067 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1068 if (BP_IS_HOLE(gbp)) 1069 continue; 1070 zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, 1071 gbp, NULL, NULL)); 1072 } 1073 1074 zio_buf_free(gbh, gbufsize); 1075 zio_next_stage(zio); 1076 } 1077 1078 static void 1079 zio_write_allocate_gang_member_done(zio_t *zio) 1080 { 1081 zio_t *pio = zio->io_parent; 1082 dva_t *cdva = ZIO_GET_DVA(zio); 1083 dva_t *pdva = ZIO_GET_DVA(pio); 1084 uint64_t asize; 1085 1086 ASSERT(DVA_GET_GANG(pdva)); 1087 1088 /* XXBP - Need to be careful here with multiple DVAs */ 1089 mutex_enter(&pio->io_lock); 1090 asize = DVA_GET_ASIZE(pdva); 1091 asize += DVA_GET_ASIZE(cdva); 1092 DVA_SET_ASIZE(pdva, asize); 1093 mutex_exit(&pio->io_lock); 1094 } 1095 1096 static void 1097 zio_write_allocate_gang_members(zio_t *zio) 1098 { 1099 blkptr_t *bp = zio->io_bp; 1100 dva_t *dva = ZIO_GET_DVA(zio); 1101 zio_gbh_phys_t *gbh; 1102 uint64_t resid = zio->io_size; 1103 uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); 1104 uint64_t gsize, loff, lsize; 1105 uint32_t gbps_left; 1106 int error; 1107 int i; 1108 1109 gsize = SPA_GANGBLOCKSIZE; 1110 gbps_left = SPA_GBH_NBLKPTRS; 1111 1112 error = metaslab_alloc(zio->io_spa, gsize, dva, zio->io_txg); 1113 if (error == ENOSPC) 1114 panic("can't allocate gang block header"); 1115 ASSERT(error == 0); 1116 1117 DVA_SET_GANG(dva, 1); 1118 1119 bp->blk_birth = zio->io_txg; 1120 1121 gbh = zio_buf_alloc(gsize); 1122 bzero(gbh, gsize); 1123 1124 for (loff = 0, i = 0; loff != zio->io_size; 1125 loff += lsize, resid -= lsize, gbps_left--, i++) { 1126 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1127 dva = &gbp->blk_dva[0]; 1128 1129 ASSERT(gbps_left != 0); 1130 maxalloc = MIN(maxalloc, resid); 1131 1132 while (resid <= maxalloc * gbps_left) { 1133 error = metaslab_alloc(zio->io_spa, maxalloc, dva, 1134 zio->io_txg); 1135 if (error == 0) 1136 break; 1137 ASSERT3U(error, ==, ENOSPC); 1138 if (maxalloc == SPA_MINBLOCKSIZE) 1139 panic("really out of space"); 1140 maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); 1141 } 1142 1143 if (resid <= maxalloc * gbps_left) { 1144 lsize = maxalloc; 1145 BP_SET_LSIZE(gbp, lsize); 1146 BP_SET_PSIZE(gbp, lsize); 1147 BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); 1148 gbp->blk_birth = zio->io_txg; 1149 zio_nowait(zio_rewrite(zio, zio->io_spa, 1150 zio->io_checksum, zio->io_txg, gbp, 1151 (char *)zio->io_data + loff, lsize, 1152 zio_write_allocate_gang_member_done, NULL, 1153 zio->io_priority, zio->io_flags)); 1154 } else { 1155 lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); 1156 ASSERT(lsize != SPA_MINBLOCKSIZE); 1157 zio_nowait(zio_write_allocate(zio, zio->io_spa, 1158 zio->io_checksum, zio->io_txg, gbp, 1159 (char *)zio->io_data + loff, lsize, 1160 zio_write_allocate_gang_member_done, NULL, 1161 zio->io_priority, zio->io_flags)); 1162 } 1163 } 1164 1165 ASSERT(resid == 0 && loff == zio->io_size); 1166 1167 zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; 1168 1169 zio_push_transform(zio, gbh, gsize, gsize); 1170 zio_wait_children_done(zio); 1171 } 1172 1173 /* 1174 * ========================================================================== 1175 * Allocate and free blocks 1176 * ========================================================================== 1177 */ 1178 static void 1179 zio_dva_allocate(zio_t *zio) 1180 { 1181 blkptr_t *bp = zio->io_bp; 1182 dva_t *dva = ZIO_GET_DVA(zio); 1183 int error; 1184 1185 ASSERT(BP_IS_HOLE(bp)); 1186 1187 /* For testing, make some blocks above a certain size be gang blocks */ 1188 if (zio->io_size >= zio_gang_bang && (lbolt & 0x3) == 0) { 1189 zio_write_allocate_gang_members(zio); 1190 return; 1191 } 1192 1193 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1194 1195 error = metaslab_alloc(zio->io_spa, zio->io_size, dva, zio->io_txg); 1196 1197 if (error == 0) { 1198 bp->blk_birth = zio->io_txg; 1199 } else if (error == ENOSPC) { 1200 if (zio->io_size == SPA_MINBLOCKSIZE) 1201 panic("really, truly out of space"); 1202 zio_write_allocate_gang_members(zio); 1203 return; 1204 } else { 1205 zio->io_error = error; 1206 } 1207 zio_next_stage(zio); 1208 } 1209 1210 static void 1211 zio_dva_free(zio_t *zio) 1212 { 1213 blkptr_t *bp = zio->io_bp; 1214 dva_t *dva = ZIO_GET_DVA(zio); 1215 1216 ASSERT(!BP_IS_HOLE(bp)); 1217 1218 metaslab_free(zio->io_spa, dva, zio->io_txg); 1219 1220 BP_ZERO(bp); 1221 1222 zio_next_stage(zio); 1223 } 1224 1225 static void 1226 zio_dva_claim(zio_t *zio) 1227 { 1228 blkptr_t *bp = zio->io_bp; 1229 dva_t *dva = ZIO_GET_DVA(zio); 1230 1231 ASSERT(!BP_IS_HOLE(bp)); 1232 1233 zio->io_error = metaslab_claim(zio->io_spa, dva, zio->io_txg); 1234 1235 zio_next_stage(zio); 1236 } 1237 1238 static void 1239 zio_dva_translate(zio_t *zio) 1240 { 1241 spa_t *spa = zio->io_spa; 1242 dva_t *dva = ZIO_GET_DVA(zio); 1243 uint64_t vdev = DVA_GET_VDEV(dva); 1244 uint64_t offset = DVA_GET_OFFSET(dva); 1245 1246 ASSERT3U(zio->io_size, ==, ZIO_GET_IOSIZE(zio)); 1247 1248 zio->io_offset = offset; 1249 1250 if ((zio->io_vd = vdev_lookup_top(spa, vdev)) == NULL) 1251 zio->io_error = ENXIO; 1252 else if (offset + zio->io_size > zio->io_vd->vdev_asize) 1253 zio->io_error = EOVERFLOW; 1254 1255 zio_next_stage(zio); 1256 } 1257 1258 /* 1259 * ========================================================================== 1260 * Read and write to physical devices 1261 * ========================================================================== 1262 */ 1263 static void 1264 zio_vdev_io_enter(zio_t *zio) 1265 { 1266 vdev_t *tvd = zio->io_vd->vdev_top; 1267 1268 mutex_enter(&tvd->vdev_io_lock); 1269 ASSERT(zio->io_pending.list_next == NULL); 1270 list_insert_tail(&tvd->vdev_io_pending, zio); 1271 mutex_exit(&tvd->vdev_io_lock); 1272 } 1273 1274 static void 1275 zio_vdev_io_exit(zio_t *zio) 1276 { 1277 vdev_t *tvd = zio->io_vd->vdev_top; 1278 1279 mutex_enter(&tvd->vdev_io_lock); 1280 ASSERT(zio->io_pending.list_next != NULL); 1281 list_remove(&tvd->vdev_io_pending, zio); 1282 if (list_head(&tvd->vdev_io_pending) == NULL) 1283 cv_broadcast(&tvd->vdev_io_cv); 1284 mutex_exit(&tvd->vdev_io_lock); 1285 } 1286 1287 static void 1288 zio_vdev_io_retry(void *vdarg) 1289 { 1290 vdev_t *vd = vdarg; 1291 zio_t *zio, *zq; 1292 1293 ASSERT(vd == vd->vdev_top); 1294 1295 /* XXPOLICY */ 1296 delay(hz); 1297 1298 vdev_reopen(vd, &zq); 1299 1300 while ((zio = zq) != NULL) { 1301 zq = zio->io_retry_next; 1302 zio->io_retry_next = NULL; 1303 dprintf("async retry #%d for I/O to %s offset %llx\n", 1304 zio->io_retries, vdev_description(vd), zio->io_offset); 1305 zio_next_stage_async(zio); 1306 } 1307 } 1308 1309 static void 1310 zio_vdev_io_setup(zio_t *zio) 1311 { 1312 vdev_t *vd = zio->io_vd; 1313 1314 /* XXPOLICY */ 1315 if (zio->io_retries == 0 && vd == vd->vdev_top) 1316 zio->io_flags |= ZIO_FLAG_FAILFAST; 1317 1318 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && vd->vdev_children == 0) { 1319 zio->io_flags |= ZIO_FLAG_PHYSICAL; 1320 zio->io_offset += VDEV_LABEL_START_SIZE; 1321 } 1322 1323 zio_vdev_io_enter(zio); 1324 1325 zio_next_stage(zio); 1326 } 1327 1328 static void 1329 zio_vdev_io_start(zio_t *zio) 1330 { 1331 blkptr_t *bp = zio->io_bp; 1332 1333 ASSERT(P2PHASE(zio->io_offset, 1ULL << zio->io_vd->vdev_ashift) == 0); 1334 ASSERT(P2PHASE(zio->io_size, 1ULL << zio->io_vd->vdev_ashift) == 0); 1335 ASSERT(bp == NULL || ZIO_GET_IOSIZE(zio) == zio->io_size); 1336 ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); 1337 1338 vdev_io_start(zio); 1339 1340 /* zio_next_stage_async() gets called from io completion interrupt */ 1341 } 1342 1343 static void 1344 zio_vdev_io_done(zio_t *zio) 1345 { 1346 vdev_io_done(zio); 1347 } 1348 1349 /* XXPOLICY */ 1350 static boolean_t 1351 zio_should_retry(zio_t *zio) 1352 { 1353 vdev_t *vd = zio->io_vd; 1354 1355 if (zio->io_error == 0) 1356 return (B_FALSE); 1357 if (zio->io_delegate_list != NULL) 1358 return (B_FALSE); 1359 if (vd != vd->vdev_top) 1360 return (B_FALSE); 1361 if (zio->io_flags & ZIO_FLAG_DONT_RETRY) 1362 return (B_FALSE); 1363 if (zio->io_retries > 300 && 1364 (zio->io_flags & (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL))) 1365 return (B_FALSE); 1366 if (zio->io_retries > 1 && 1367 (zio->io_error == ECKSUM || zio->io_error == ENXIO)) 1368 return (B_FALSE); 1369 1370 return (B_TRUE); 1371 } 1372 1373 static void 1374 zio_vdev_io_assess(zio_t *zio) 1375 { 1376 vdev_t *vd = zio->io_vd; 1377 vdev_t *tvd = vd->vdev_top; 1378 1379 zio_vdev_io_exit(zio); 1380 1381 ASSERT(zio->io_vsd == NULL); 1382 1383 /* 1384 * If the I/O failed, determine whether we should attempt to retry it. 1385 */ 1386 /* XXPOLICY */ 1387 if (zio_should_retry(zio)) { 1388 zio_t *zq; 1389 1390 ASSERT(tvd == vd); 1391 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)); 1392 1393 zio->io_retries++; 1394 zio->io_error = 0; 1395 zio->io_flags &= ZIO_FLAG_VDEV_INHERIT; 1396 /* XXPOLICY */ 1397 zio->io_flags &= ~ZIO_FLAG_FAILFAST; 1398 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1399 zio->io_stage = ZIO_STAGE_VDEV_IO_SETUP - 1; 1400 1401 dprintf("retry #%d for %s to %s offset %llx\n", 1402 zio->io_retries, zio_type_name[zio->io_type], 1403 vdev_description(vd), zio->io_offset); 1404 1405 /* 1406 * If this is the first retry, do it immediately. 1407 */ 1408 /* XXPOLICY */ 1409 if (zio->io_retries == 1) { 1410 zio_next_stage_async(zio); 1411 return; 1412 } 1413 1414 /* 1415 * This was not the first retry, so go through the 1416 * longer enqueue/delay/vdev_reopen() process. 1417 */ 1418 mutex_enter(&tvd->vdev_io_lock); 1419 ASSERT(zio->io_retry_next == NULL); 1420 zio->io_retry_next = zq = tvd->vdev_io_retry; 1421 tvd->vdev_io_retry = zio; 1422 mutex_exit(&tvd->vdev_io_lock); 1423 if (zq == NULL) 1424 (void) taskq_dispatch( 1425 tvd->vdev_spa->spa_vdev_retry_taskq, 1426 zio_vdev_io_retry, tvd, TQ_SLEEP); 1427 return; 1428 } 1429 1430 zio_next_stage(zio); 1431 } 1432 1433 void 1434 zio_vdev_io_reissue(zio_t *zio) 1435 { 1436 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1437 ASSERT(zio->io_error == 0); 1438 1439 zio->io_stage--; 1440 } 1441 1442 void 1443 zio_vdev_io_redone(zio_t *zio) 1444 { 1445 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 1446 1447 zio->io_stage--; 1448 } 1449 1450 void 1451 zio_vdev_io_bypass(zio_t *zio) 1452 { 1453 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1454 ASSERT(zio->io_error == 0); 1455 1456 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 1457 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; 1458 } 1459 1460 /* 1461 * ========================================================================== 1462 * Generate and verify checksums 1463 * ========================================================================== 1464 */ 1465 static void 1466 zio_checksum_generate(zio_t *zio) 1467 { 1468 int checksum = zio->io_checksum; 1469 blkptr_t *bp = zio->io_bp; 1470 1471 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1472 1473 BP_SET_CHECKSUM(bp, checksum); 1474 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1475 1476 zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); 1477 1478 zio_next_stage(zio); 1479 } 1480 1481 static void 1482 zio_gang_checksum_generate(zio_t *zio) 1483 { 1484 zio_cksum_t zc; 1485 zio_gbh_phys_t *gbh = zio->io_data; 1486 1487 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1488 ASSERT(DVA_GET_GANG(ZIO_GET_DVA(zio))); 1489 1490 zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); 1491 1492 zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); 1493 1494 zio_next_stage(zio); 1495 } 1496 1497 static void 1498 zio_checksum_verify(zio_t *zio) 1499 { 1500 if (zio->io_bp != NULL) { 1501 zio->io_error = zio_checksum_error(zio); 1502 if (zio->io_error) { 1503 dprintf("bad checksum on vdev %s\n", 1504 vdev_description(zio->io_vd)); 1505 } 1506 } 1507 1508 zio_next_stage(zio); 1509 } 1510 1511 /* 1512 * Called by RAID-Z to ensure we don't compute the checksum twice. 1513 */ 1514 void 1515 zio_checksum_verified(zio_t *zio) 1516 { 1517 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 1518 } 1519 1520 /* 1521 * Set the external verifier for a gang block based on stuff in the bp 1522 */ 1523 void 1524 zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) 1525 { 1526 zcp->zc_word[0] = DVA_GET_VDEV(ZIO_GET_DVA(zio)); 1527 zcp->zc_word[1] = DVA_GET_OFFSET(ZIO_GET_DVA(zio)); 1528 zcp->zc_word[2] = zio->io_bp->blk_birth; 1529 zcp->zc_word[3] = 0; 1530 } 1531 1532 /* 1533 * ========================================================================== 1534 * Define the pipeline 1535 * ========================================================================== 1536 */ 1537 typedef void zio_pipe_stage_t(zio_t *zio); 1538 1539 static void 1540 zio_badop(zio_t *zio) 1541 { 1542 panic("Invalid I/O pipeline stage %u for zio %p", zio->io_stage, zio); 1543 } 1544 1545 zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { 1546 zio_badop, 1547 zio_wait_children_ready, 1548 zio_write_compress, 1549 zio_checksum_generate, 1550 zio_gang_pipeline, 1551 zio_get_gang_header, 1552 zio_rewrite_gang_members, 1553 zio_free_gang_members, 1554 zio_claim_gang_members, 1555 zio_dva_allocate, 1556 zio_dva_free, 1557 zio_dva_claim, 1558 zio_gang_checksum_generate, 1559 zio_ready, 1560 zio_dva_translate, 1561 zio_vdev_io_setup, 1562 zio_vdev_io_start, 1563 zio_vdev_io_done, 1564 zio_vdev_io_assess, 1565 zio_wait_children_done, 1566 zio_checksum_verify, 1567 zio_read_gang_members, 1568 zio_read_decompress, 1569 zio_done, 1570 zio_badop 1571 }; 1572 1573 /* 1574 * Move an I/O to the next stage of the pipeline and execute that stage. 1575 * There's no locking on io_stage because there's no legitimate way for 1576 * multiple threads to be attempting to process the same I/O. 1577 */ 1578 void 1579 zio_next_stage(zio_t *zio) 1580 { 1581 uint32_t pipeline = zio->io_pipeline; 1582 1583 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1584 1585 if (zio->io_error) { 1586 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1587 zio, vdev_description(zio->io_vd), 1588 zio->io_offset, zio->io_stage, zio->io_error); 1589 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1590 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1591 } 1592 1593 while (((1U << ++zio->io_stage) & pipeline) == 0) 1594 continue; 1595 1596 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1597 ASSERT(zio->io_stalled == 0); 1598 1599 zio_pipeline[zio->io_stage](zio); 1600 } 1601 1602 void 1603 zio_next_stage_async(zio_t *zio) 1604 { 1605 taskq_t *tq; 1606 uint32_t pipeline = zio->io_pipeline; 1607 1608 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1609 1610 if (zio->io_error) { 1611 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1612 zio, vdev_description(zio->io_vd), 1613 zio->io_offset, zio->io_stage, zio->io_error); 1614 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1615 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1616 } 1617 1618 while (((1U << ++zio->io_stage) & pipeline) == 0) 1619 continue; 1620 1621 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1622 ASSERT(zio->io_stalled == 0); 1623 1624 /* 1625 * For performance, we'll probably want two sets of task queues: 1626 * per-CPU issue taskqs and per-CPU completion taskqs. The per-CPU 1627 * part is for read performance: since we have to make a pass over 1628 * the data to checksum it anyway, we want to do this on the same CPU 1629 * that issued the read, because (assuming CPU scheduling affinity) 1630 * that thread is probably still there. Getting this optimization 1631 * right avoids performance-hostile cache-to-cache transfers. 1632 * 1633 * Note that having two sets of task queues is also necessary for 1634 * correctness: if all of the issue threads get bogged down waiting 1635 * for dependent reads (e.g. metaslab freelist) to complete, then 1636 * there won't be any threads available to service I/O completion 1637 * interrupts. 1638 */ 1639 if ((1U << zio->io_stage) & zio->io_async_stages) { 1640 if (zio->io_stage < ZIO_STAGE_VDEV_IO_DONE) 1641 tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; 1642 else 1643 tq = zio->io_spa->spa_zio_intr_taskq[zio->io_type]; 1644 (void) taskq_dispatch(tq, 1645 (task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); 1646 } else { 1647 zio_pipeline[zio->io_stage](zio); 1648 } 1649 } 1650 1651 /* 1652 * Try to allocate an intent log block. Return 0 on success, errno on failure. 1653 */ 1654 int 1655 zio_alloc_blk(spa_t *spa, int checksum, uint64_t size, blkptr_t *bp, 1656 uint64_t txg) 1657 { 1658 int error; 1659 1660 spa_config_enter(spa, RW_READER); 1661 1662 BP_ZERO(bp); 1663 1664 error = metaslab_alloc(spa, size, BP_IDENTITY(bp), txg); 1665 1666 if (error == 0) { 1667 BP_SET_CHECKSUM(bp, checksum); 1668 BP_SET_LSIZE(bp, size); 1669 BP_SET_PSIZE(bp, size); 1670 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 1671 BP_SET_TYPE(bp, DMU_OT_INTENT_LOG); 1672 BP_SET_LEVEL(bp, 0); 1673 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1674 bp->blk_birth = txg; 1675 } 1676 1677 spa_config_exit(spa); 1678 1679 return (error); 1680 } 1681 1682 /* 1683 * Free an intent log block. We know it can't be a gang block, so there's 1684 * nothing to do except metaslab_free() it. 1685 */ 1686 void 1687 zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) 1688 { 1689 ASSERT(DVA_GET_GANG(BP_IDENTITY(bp)) == 0); 1690 1691 dprintf_bp(bp, "txg %llu: ", txg); 1692 1693 spa_config_enter(spa, RW_READER); 1694 1695 metaslab_free(spa, BP_IDENTITY(bp), txg); 1696 1697 spa_config_exit(spa); 1698 } 1699