1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/fm/fs/zfs.h> 30 #include <sys/spa.h> 31 #include <sys/txg.h> 32 #include <sys/spa_impl.h> 33 #include <sys/vdev_impl.h> 34 #include <sys/zio_impl.h> 35 #include <sys/zio_compress.h> 36 #include <sys/zio_checksum.h> 37 38 /* 39 * ========================================================================== 40 * I/O priority table 41 * ========================================================================== 42 */ 43 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 44 0, /* ZIO_PRIORITY_NOW */ 45 0, /* ZIO_PRIORITY_SYNC_READ */ 46 0, /* ZIO_PRIORITY_SYNC_WRITE */ 47 6, /* ZIO_PRIORITY_ASYNC_READ */ 48 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 49 4, /* ZIO_PRIORITY_FREE */ 50 0, /* ZIO_PRIORITY_CACHE_FILL */ 51 0, /* ZIO_PRIORITY_LOG_WRITE */ 52 10, /* ZIO_PRIORITY_RESILVER */ 53 20, /* ZIO_PRIORITY_SCRUB */ 54 }; 55 56 /* 57 * ========================================================================== 58 * I/O type descriptions 59 * ========================================================================== 60 */ 61 char *zio_type_name[ZIO_TYPES] = { 62 "null", "read", "write", "free", "claim", "ioctl" }; 63 64 /* At or above this size, force gang blocking - for testing */ 65 uint64_t zio_gang_bang = SPA_MAXBLOCKSIZE + 1; 66 67 typedef struct zio_sync_pass { 68 int zp_defer_free; /* defer frees after this pass */ 69 int zp_dontcompress; /* don't compress after this pass */ 70 int zp_rewrite; /* rewrite new bps after this pass */ 71 } zio_sync_pass_t; 72 73 zio_sync_pass_t zio_sync_pass = { 74 1, /* zp_defer_free */ 75 4, /* zp_dontcompress */ 76 1, /* zp_rewrite */ 77 }; 78 79 /* 80 * ========================================================================== 81 * I/O kmem caches 82 * ========================================================================== 83 */ 84 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 85 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 86 87 #ifdef _KERNEL 88 extern vmem_t *zio_alloc_arena; 89 #endif 90 91 void 92 zio_init(void) 93 { 94 size_t c; 95 vmem_t *data_alloc_arena = NULL; 96 97 #ifdef _KERNEL 98 data_alloc_arena = zio_alloc_arena; 99 #endif 100 101 /* 102 * For small buffers, we want a cache for each multiple of 103 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 104 * for each quarter-power of 2. For large buffers, we want 105 * a cache for each multiple of PAGESIZE. 106 */ 107 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 108 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 109 size_t p2 = size; 110 size_t align = 0; 111 112 while (p2 & (p2 - 1)) 113 p2 &= p2 - 1; 114 115 if (size <= 4 * SPA_MINBLOCKSIZE) { 116 align = SPA_MINBLOCKSIZE; 117 } else if (P2PHASE(size, PAGESIZE) == 0) { 118 align = PAGESIZE; 119 } else if (P2PHASE(size, p2 >> 2) == 0) { 120 align = p2 >> 2; 121 } 122 123 if (align != 0) { 124 char name[36]; 125 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 126 zio_buf_cache[c] = kmem_cache_create(name, size, 127 align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG); 128 129 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 130 zio_data_buf_cache[c] = kmem_cache_create(name, size, 131 align, NULL, NULL, NULL, NULL, data_alloc_arena, 132 KMC_NODEBUG); 133 134 dprintf("creating cache for size %5lx align %5lx\n", 135 size, align); 136 } 137 } 138 139 while (--c != 0) { 140 ASSERT(zio_buf_cache[c] != NULL); 141 if (zio_buf_cache[c - 1] == NULL) 142 zio_buf_cache[c - 1] = zio_buf_cache[c]; 143 144 ASSERT(zio_data_buf_cache[c] != NULL); 145 if (zio_data_buf_cache[c - 1] == NULL) 146 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 147 } 148 149 zio_inject_init(); 150 } 151 152 void 153 zio_fini(void) 154 { 155 size_t c; 156 kmem_cache_t *last_cache = NULL; 157 kmem_cache_t *last_data_cache = NULL; 158 159 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 160 if (zio_buf_cache[c] != last_cache) { 161 last_cache = zio_buf_cache[c]; 162 kmem_cache_destroy(zio_buf_cache[c]); 163 } 164 zio_buf_cache[c] = NULL; 165 166 if (zio_data_buf_cache[c] != last_data_cache) { 167 last_data_cache = zio_data_buf_cache[c]; 168 kmem_cache_destroy(zio_data_buf_cache[c]); 169 } 170 zio_data_buf_cache[c] = NULL; 171 } 172 173 zio_inject_fini(); 174 } 175 176 /* 177 * ========================================================================== 178 * Allocate and free I/O buffers 179 * ========================================================================== 180 */ 181 182 /* 183 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 184 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 185 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 186 * excess / transient data in-core during a crashdump. 187 */ 188 void * 189 zio_buf_alloc(size_t size) 190 { 191 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 192 193 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 194 195 return (kmem_cache_alloc(zio_buf_cache[c], KM_SLEEP)); 196 } 197 198 /* 199 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 200 * crashdump if the kernel panics. This exists so that we will limit the amount 201 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 202 * of kernel heap dumped to disk when the kernel panics) 203 */ 204 void * 205 zio_data_buf_alloc(size_t size) 206 { 207 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 208 209 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 210 211 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_SLEEP)); 212 } 213 214 void 215 zio_buf_free(void *buf, size_t size) 216 { 217 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 218 219 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 220 221 kmem_cache_free(zio_buf_cache[c], buf); 222 } 223 224 void 225 zio_data_buf_free(void *buf, size_t size) 226 { 227 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 228 229 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 230 231 kmem_cache_free(zio_data_buf_cache[c], buf); 232 } 233 /* 234 * ========================================================================== 235 * Push and pop I/O transform buffers 236 * ========================================================================== 237 */ 238 static void 239 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize) 240 { 241 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 242 243 zt->zt_data = data; 244 zt->zt_size = size; 245 zt->zt_bufsize = bufsize; 246 247 zt->zt_next = zio->io_transform_stack; 248 zio->io_transform_stack = zt; 249 250 zio->io_data = data; 251 zio->io_size = size; 252 } 253 254 static void 255 zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize) 256 { 257 zio_transform_t *zt = zio->io_transform_stack; 258 259 *data = zt->zt_data; 260 *size = zt->zt_size; 261 *bufsize = zt->zt_bufsize; 262 263 zio->io_transform_stack = zt->zt_next; 264 kmem_free(zt, sizeof (zio_transform_t)); 265 266 if ((zt = zio->io_transform_stack) != NULL) { 267 zio->io_data = zt->zt_data; 268 zio->io_size = zt->zt_size; 269 } 270 } 271 272 static void 273 zio_clear_transform_stack(zio_t *zio) 274 { 275 void *data; 276 uint64_t size, bufsize; 277 278 ASSERT(zio->io_transform_stack != NULL); 279 280 zio_pop_transform(zio, &data, &size, &bufsize); 281 while (zio->io_transform_stack != NULL) { 282 zio_buf_free(data, bufsize); 283 zio_pop_transform(zio, &data, &size, &bufsize); 284 } 285 } 286 287 /* 288 * ========================================================================== 289 * Create the various types of I/O (read, write, free) 290 * ========================================================================== 291 */ 292 static zio_t * 293 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 294 void *data, uint64_t size, zio_done_func_t *done, void *private, 295 zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline) 296 { 297 zio_t *zio; 298 299 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 300 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 301 302 zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); 303 zio->io_parent = pio; 304 zio->io_spa = spa; 305 zio->io_txg = txg; 306 if (bp != NULL) { 307 zio->io_bp = bp; 308 zio->io_bp_copy = *bp; 309 zio->io_bp_orig = *bp; 310 } 311 zio->io_done = done; 312 zio->io_private = private; 313 zio->io_type = type; 314 zio->io_priority = priority; 315 zio->io_stage = stage; 316 zio->io_pipeline = pipeline; 317 zio->io_async_stages = ZIO_ASYNC_PIPELINE_STAGES; 318 zio->io_timestamp = lbolt64; 319 zio->io_flags = flags; 320 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 321 zio_push_transform(zio, data, size, size); 322 323 if (pio == NULL) { 324 if (!(flags & ZIO_FLAG_CONFIG_HELD)) 325 spa_config_enter(zio->io_spa, RW_READER, zio); 326 zio->io_root = zio; 327 } else { 328 zio->io_root = pio->io_root; 329 if (!(flags & ZIO_FLAG_NOBOOKMARK)) 330 zio->io_logical = pio->io_logical; 331 mutex_enter(&pio->io_lock); 332 if (stage < ZIO_STAGE_READY) 333 pio->io_children_notready++; 334 pio->io_children_notdone++; 335 zio->io_sibling_next = pio->io_child; 336 zio->io_sibling_prev = NULL; 337 if (pio->io_child != NULL) 338 pio->io_child->io_sibling_prev = zio; 339 pio->io_child = zio; 340 zio->io_ndvas = pio->io_ndvas; 341 mutex_exit(&pio->io_lock); 342 } 343 344 return (zio); 345 } 346 347 zio_t * 348 zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private, 349 int flags) 350 { 351 zio_t *zio; 352 353 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 354 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN, 355 ZIO_WAIT_FOR_CHILDREN_PIPELINE); 356 357 return (zio); 358 } 359 360 zio_t * 361 zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags) 362 { 363 return (zio_null(NULL, spa, done, private, flags)); 364 } 365 366 zio_t * 367 zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data, 368 uint64_t size, zio_done_func_t *done, void *private, 369 int priority, int flags, zbookmark_t *zb) 370 { 371 zio_t *zio; 372 373 ASSERT3U(size, ==, BP_GET_LSIZE(bp)); 374 375 zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private, 376 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_USER, 377 ZIO_STAGE_OPEN, ZIO_READ_PIPELINE); 378 zio->io_bookmark = *zb; 379 380 zio->io_logical = zio; 381 382 /* 383 * Work off our copy of the bp so the caller can free it. 384 */ 385 zio->io_bp = &zio->io_bp_copy; 386 387 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 388 uint64_t csize = BP_GET_PSIZE(bp); 389 void *cbuf = zio_buf_alloc(csize); 390 391 zio_push_transform(zio, cbuf, csize, csize); 392 zio->io_pipeline |= 1U << ZIO_STAGE_READ_DECOMPRESS; 393 } 394 395 if (BP_IS_GANG(bp)) { 396 uint64_t gsize = SPA_GANGBLOCKSIZE; 397 void *gbuf = zio_buf_alloc(gsize); 398 399 zio_push_transform(zio, gbuf, gsize, gsize); 400 zio->io_pipeline |= 1U << ZIO_STAGE_READ_GANG_MEMBERS; 401 } 402 403 return (zio); 404 } 405 406 zio_t * 407 zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 408 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 409 zio_done_func_t *done, void *private, int priority, int flags, 410 zbookmark_t *zb) 411 { 412 zio_t *zio; 413 414 ASSERT(checksum >= ZIO_CHECKSUM_OFF && 415 checksum < ZIO_CHECKSUM_FUNCTIONS); 416 417 ASSERT(compress >= ZIO_COMPRESS_OFF && 418 compress < ZIO_COMPRESS_FUNCTIONS); 419 420 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 421 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 422 ZIO_STAGE_OPEN, ZIO_WRITE_PIPELINE); 423 424 zio->io_bookmark = *zb; 425 426 zio->io_logical = zio; 427 428 zio->io_checksum = checksum; 429 zio->io_compress = compress; 430 zio->io_ndvas = ncopies; 431 432 if (compress != ZIO_COMPRESS_OFF) 433 zio->io_async_stages |= 1U << ZIO_STAGE_WRITE_COMPRESS; 434 435 if (bp->blk_birth != txg) { 436 /* XXX the bp usually (always?) gets re-zeroed later */ 437 BP_ZERO(bp); 438 BP_SET_LSIZE(bp, size); 439 BP_SET_PSIZE(bp, size); 440 } else { 441 /* Make sure someone doesn't change their mind on overwrites */ 442 ASSERT(MIN(zio->io_ndvas + BP_IS_GANG(bp), 443 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 444 } 445 446 return (zio); 447 } 448 449 zio_t * 450 zio_rewrite(zio_t *pio, spa_t *spa, int checksum, 451 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 452 zio_done_func_t *done, void *private, int priority, int flags, 453 zbookmark_t *zb) 454 { 455 zio_t *zio; 456 457 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 458 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER, 459 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 460 461 zio->io_bookmark = *zb; 462 zio->io_checksum = checksum; 463 zio->io_compress = ZIO_COMPRESS_OFF; 464 465 if (pio != NULL) 466 ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 467 468 return (zio); 469 } 470 471 static zio_t * 472 zio_write_allocate(zio_t *pio, spa_t *spa, int checksum, 473 uint64_t txg, blkptr_t *bp, void *data, uint64_t size, 474 zio_done_func_t *done, void *private, int priority, int flags) 475 { 476 zio_t *zio; 477 478 BP_ZERO(bp); 479 BP_SET_LSIZE(bp, size); 480 BP_SET_PSIZE(bp, size); 481 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 482 483 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 484 ZIO_TYPE_WRITE, priority, flags, 485 ZIO_STAGE_OPEN, ZIO_WRITE_ALLOCATE_PIPELINE); 486 487 zio->io_checksum = checksum; 488 zio->io_compress = ZIO_COMPRESS_OFF; 489 490 return (zio); 491 } 492 493 zio_t * 494 zio_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 495 zio_done_func_t *done, void *private) 496 { 497 zio_t *zio; 498 499 ASSERT(!BP_IS_HOLE(bp)); 500 501 if (txg == spa->spa_syncing_txg && 502 spa->spa_sync_pass > zio_sync_pass.zp_defer_free) { 503 bplist_enqueue_deferred(&spa->spa_sync_bplist, bp); 504 return (zio_null(pio, spa, NULL, NULL, 0)); 505 } 506 507 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 508 ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, ZIO_FLAG_USER, 509 ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 510 511 zio->io_bp = &zio->io_bp_copy; 512 513 return (zio); 514 } 515 516 zio_t * 517 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 518 zio_done_func_t *done, void *private) 519 { 520 zio_t *zio; 521 522 /* 523 * A claim is an allocation of a specific block. Claims are needed 524 * to support immediate writes in the intent log. The issue is that 525 * immediate writes contain committed data, but in a txg that was 526 * *not* committed. Upon opening the pool after an unclean shutdown, 527 * the intent log claims all blocks that contain immediate write data 528 * so that the SPA knows they're in use. 529 * 530 * All claims *must* be resolved in the first txg -- before the SPA 531 * starts allocating blocks -- so that nothing is allocated twice. 532 */ 533 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 534 ASSERT3U(spa_first_txg(spa), <=, txg); 535 536 zio = zio_create(pio, spa, txg, bp, NULL, 0, done, private, 537 ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 0, 538 ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 539 540 zio->io_bp = &zio->io_bp_copy; 541 542 return (zio); 543 } 544 545 zio_t * 546 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 547 zio_done_func_t *done, void *private, int priority, int flags) 548 { 549 zio_t *zio; 550 int c; 551 552 if (vd->vdev_children == 0) { 553 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 554 ZIO_TYPE_IOCTL, priority, flags, 555 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 556 557 zio->io_vd = vd; 558 zio->io_cmd = cmd; 559 } else { 560 zio = zio_null(pio, spa, NULL, NULL, flags); 561 562 for (c = 0; c < vd->vdev_children; c++) 563 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 564 done, private, priority, flags)); 565 } 566 567 return (zio); 568 } 569 570 static void 571 zio_phys_bp_init(vdev_t *vd, blkptr_t *bp, uint64_t offset, uint64_t size, 572 int checksum) 573 { 574 ASSERT(vd->vdev_children == 0); 575 576 ASSERT(size <= SPA_MAXBLOCKSIZE); 577 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 578 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 579 580 ASSERT(offset + size <= VDEV_LABEL_START_SIZE || 581 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 582 ASSERT3U(offset + size, <=, vd->vdev_psize); 583 584 BP_ZERO(bp); 585 586 BP_SET_LSIZE(bp, size); 587 BP_SET_PSIZE(bp, size); 588 589 BP_SET_CHECKSUM(bp, checksum); 590 BP_SET_COMPRESS(bp, ZIO_COMPRESS_OFF); 591 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 592 593 if (checksum != ZIO_CHECKSUM_OFF) 594 ZIO_SET_CHECKSUM(&bp->blk_cksum, offset, 0, 0, 0); 595 } 596 597 zio_t * 598 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 599 void *data, int checksum, zio_done_func_t *done, void *private, 600 int priority, int flags) 601 { 602 zio_t *zio; 603 blkptr_t blk; 604 605 zio_phys_bp_init(vd, &blk, offset, size, checksum); 606 607 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 608 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, 609 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 610 611 zio->io_vd = vd; 612 zio->io_offset = offset; 613 614 /* 615 * Work off our copy of the bp so the caller can free it. 616 */ 617 zio->io_bp = &zio->io_bp_copy; 618 619 return (zio); 620 } 621 622 zio_t * 623 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 624 void *data, int checksum, zio_done_func_t *done, void *private, 625 int priority, int flags) 626 { 627 zio_block_tail_t *zbt; 628 void *wbuf; 629 zio_t *zio; 630 blkptr_t blk; 631 632 zio_phys_bp_init(vd, &blk, offset, size, checksum); 633 634 zio = zio_create(pio, vd->vdev_spa, 0, &blk, data, size, done, private, 635 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, 636 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 637 638 zio->io_vd = vd; 639 zio->io_offset = offset; 640 641 zio->io_bp = &zio->io_bp_copy; 642 zio->io_checksum = checksum; 643 644 if (zio_checksum_table[checksum].ci_zbt) { 645 /* 646 * zbt checksums are necessarily destructive -- they modify 647 * one word of the write buffer to hold the verifier/checksum. 648 * Therefore, we must make a local copy in case the data is 649 * being written to multiple places. 650 */ 651 wbuf = zio_buf_alloc(size); 652 bcopy(data, wbuf, size); 653 zio_push_transform(zio, wbuf, size, size); 654 655 zbt = (zio_block_tail_t *)((char *)wbuf + size) - 1; 656 zbt->zbt_cksum = blk.blk_cksum; 657 } 658 659 return (zio); 660 } 661 662 /* 663 * Create a child I/O to do some work for us. It has no associated bp. 664 */ 665 zio_t * 666 zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 667 void *data, uint64_t size, int type, int priority, int flags, 668 zio_done_func_t *done, void *private) 669 { 670 uint32_t pipeline = ZIO_VDEV_CHILD_PIPELINE; 671 zio_t *cio; 672 673 if (type == ZIO_TYPE_READ && bp != NULL) { 674 /* 675 * If we have the bp, then the child should perform the 676 * checksum and the parent need not. This pushes error 677 * detection as close to the leaves as possible and 678 * eliminates redundant checksums in the interior nodes. 679 */ 680 pipeline |= 1U << ZIO_STAGE_CHECKSUM_VERIFY; 681 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 682 } 683 684 cio = zio_create(zio, zio->io_spa, zio->io_txg, bp, data, size, 685 done, private, type, priority, 686 (zio->io_flags & ZIO_FLAG_VDEV_INHERIT) | ZIO_FLAG_CANFAIL | flags, 687 ZIO_STAGE_VDEV_IO_START - 1, pipeline); 688 689 cio->io_vd = vd; 690 cio->io_offset = offset; 691 692 return (cio); 693 } 694 695 /* 696 * ========================================================================== 697 * Initiate I/O, either sync or async 698 * ========================================================================== 699 */ 700 int 701 zio_wait(zio_t *zio) 702 { 703 int error; 704 705 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 706 707 zio->io_waiter = curthread; 708 709 zio_next_stage_async(zio); 710 711 mutex_enter(&zio->io_lock); 712 while (zio->io_stalled != ZIO_STAGE_DONE) 713 cv_wait(&zio->io_cv, &zio->io_lock); 714 mutex_exit(&zio->io_lock); 715 716 error = zio->io_error; 717 mutex_destroy(&zio->io_lock); 718 kmem_free(zio, sizeof (zio_t)); 719 720 return (error); 721 } 722 723 void 724 zio_nowait(zio_t *zio) 725 { 726 zio_next_stage_async(zio); 727 } 728 729 /* 730 * ========================================================================== 731 * I/O pipeline interlocks: parent/child dependency scoreboarding 732 * ========================================================================== 733 */ 734 static void 735 zio_wait_for_children(zio_t *zio, uint32_t stage, uint64_t *countp) 736 { 737 mutex_enter(&zio->io_lock); 738 if (*countp == 0) { 739 ASSERT(zio->io_stalled == 0); 740 mutex_exit(&zio->io_lock); 741 zio_next_stage(zio); 742 } else { 743 zio->io_stalled = stage; 744 mutex_exit(&zio->io_lock); 745 } 746 } 747 748 static void 749 zio_notify_parent(zio_t *zio, uint32_t stage, uint64_t *countp) 750 { 751 zio_t *pio = zio->io_parent; 752 753 mutex_enter(&pio->io_lock); 754 if (pio->io_error == 0 && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 755 pio->io_error = zio->io_error; 756 if (--*countp == 0 && pio->io_stalled == stage) { 757 pio->io_stalled = 0; 758 mutex_exit(&pio->io_lock); 759 zio_next_stage_async(pio); 760 } else { 761 mutex_exit(&pio->io_lock); 762 } 763 } 764 765 static void 766 zio_wait_children_ready(zio_t *zio) 767 { 768 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 769 &zio->io_children_notready); 770 } 771 772 void 773 zio_wait_children_done(zio_t *zio) 774 { 775 zio_wait_for_children(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 776 &zio->io_children_notdone); 777 } 778 779 static void 780 zio_ready(zio_t *zio) 781 { 782 zio_t *pio = zio->io_parent; 783 784 if (pio != NULL) 785 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_READY, 786 &pio->io_children_notready); 787 788 if (zio->io_bp) 789 zio->io_bp_copy = *zio->io_bp; 790 791 zio_next_stage(zio); 792 } 793 794 static void 795 zio_done(zio_t *zio) 796 { 797 zio_t *pio = zio->io_parent; 798 spa_t *spa = zio->io_spa; 799 blkptr_t *bp = zio->io_bp; 800 vdev_t *vd = zio->io_vd; 801 char blkbuf[BP_SPRINTF_LEN]; 802 803 ASSERT(zio->io_children_notready == 0); 804 ASSERT(zio->io_children_notdone == 0); 805 806 if (bp != NULL) { 807 ASSERT(bp->blk_pad[0] == 0); 808 ASSERT(bp->blk_pad[1] == 0); 809 ASSERT(bp->blk_pad[2] == 0); 810 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0); 811 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 812 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 813 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 814 if (zio->io_ndvas != 0) 815 ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(bp)); 816 ASSERT(BP_COUNT_GANG(bp) == 0 || 817 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 818 } 819 } 820 821 if (vd != NULL) 822 vdev_stat_update(zio); 823 824 if (zio->io_error) { 825 /* 826 * If this I/O is attached to a particular vdev, 827 * generate an error message describing the I/O failure 828 * at the block level. We ignore these errors if the 829 * device is currently unavailable. 830 */ 831 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 832 zfs_ereport_post(FM_EREPORT_ZFS_IO, 833 zio->io_spa, vd, zio, 0, 0); 834 835 if ((zio->io_error == EIO || 836 !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) && 837 zio->io_logical == zio) { 838 /* 839 * For root I/O requests, tell the SPA to log the error 840 * appropriately. Also, generate a logical data 841 * ereport. 842 */ 843 spa_log_error(zio->io_spa, zio); 844 845 zfs_ereport_post(FM_EREPORT_ZFS_DATA, 846 zio->io_spa, NULL, zio, 0, 0); 847 } 848 849 /* 850 * For I/O requests that cannot fail, panic appropriately. 851 */ 852 if (!(zio->io_flags & ZIO_FLAG_CANFAIL)) { 853 sprintf_blkptr(blkbuf, BP_SPRINTF_LEN, 854 bp ? bp : &zio->io_bp_copy); 855 panic("ZFS: %s (%s on %s off %llx: zio %p %s): error " 856 "%d", zio->io_error == ECKSUM ? 857 "bad checksum" : "I/O failure", 858 zio_type_name[zio->io_type], 859 vdev_description(vd), 860 (u_longlong_t)zio->io_offset, 861 zio, blkbuf, zio->io_error); 862 } 863 } 864 865 zio_clear_transform_stack(zio); 866 867 if (zio->io_done) 868 zio->io_done(zio); 869 870 ASSERT(zio->io_delegate_list == NULL); 871 ASSERT(zio->io_delegate_next == NULL); 872 873 if (pio != NULL) { 874 zio_t *next, *prev; 875 876 mutex_enter(&pio->io_lock); 877 next = zio->io_sibling_next; 878 prev = zio->io_sibling_prev; 879 if (next != NULL) 880 next->io_sibling_prev = prev; 881 if (prev != NULL) 882 prev->io_sibling_next = next; 883 if (pio->io_child == zio) 884 pio->io_child = next; 885 mutex_exit(&pio->io_lock); 886 887 zio_notify_parent(zio, ZIO_STAGE_WAIT_CHILDREN_DONE, 888 &pio->io_children_notdone); 889 } 890 891 if (pio == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_HELD)) 892 spa_config_exit(spa, zio); 893 894 if (zio->io_waiter != NULL) { 895 mutex_enter(&zio->io_lock); 896 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 897 zio->io_stalled = zio->io_stage; 898 cv_broadcast(&zio->io_cv); 899 mutex_exit(&zio->io_lock); 900 } else { 901 kmem_free(zio, sizeof (zio_t)); 902 } 903 } 904 905 /* 906 * ========================================================================== 907 * Compression support 908 * ========================================================================== 909 */ 910 static void 911 zio_write_compress(zio_t *zio) 912 { 913 int compress = zio->io_compress; 914 blkptr_t *bp = zio->io_bp; 915 void *cbuf; 916 uint64_t lsize = zio->io_size; 917 uint64_t csize = lsize; 918 uint64_t cbufsize = 0; 919 int pass; 920 921 if (bp->blk_birth == zio->io_txg) { 922 /* 923 * We're rewriting an existing block, which means we're 924 * working on behalf of spa_sync(). For spa_sync() to 925 * converge, it must eventually be the case that we don't 926 * have to allocate new blocks. But compression changes 927 * the blocksize, which forces a reallocate, and makes 928 * convergence take longer. Therefore, after the first 929 * few passes, stop compressing to ensure convergence. 930 */ 931 pass = spa_sync_pass(zio->io_spa); 932 if (pass > zio_sync_pass.zp_dontcompress) 933 compress = ZIO_COMPRESS_OFF; 934 } else { 935 ASSERT(BP_IS_HOLE(bp)); 936 pass = 1; 937 } 938 939 if (compress != ZIO_COMPRESS_OFF) 940 if (!zio_compress_data(compress, zio->io_data, zio->io_size, 941 &cbuf, &csize, &cbufsize)) 942 compress = ZIO_COMPRESS_OFF; 943 944 if (compress != ZIO_COMPRESS_OFF && csize != 0) 945 zio_push_transform(zio, cbuf, csize, cbufsize); 946 947 /* 948 * The final pass of spa_sync() must be all rewrites, but the first 949 * few passes offer a trade-off: allocating blocks defers convergence, 950 * but newly allocated blocks are sequential, so they can be written 951 * to disk faster. Therefore, we allow the first few passes of 952 * spa_sync() to reallocate new blocks, but force rewrites after that. 953 * There should only be a handful of blocks after pass 1 in any case. 954 */ 955 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == csize && 956 pass > zio_sync_pass.zp_rewrite) { 957 ASSERT(csize != 0); 958 BP_SET_LSIZE(bp, lsize); 959 BP_SET_COMPRESS(bp, compress); 960 zio->io_pipeline = ZIO_REWRITE_PIPELINE; 961 } else { 962 if (bp->blk_birth == zio->io_txg) { 963 ASSERT3U(BP_GET_LSIZE(bp), ==, lsize); 964 bzero(bp, sizeof (blkptr_t)); 965 } 966 if (csize == 0) { 967 BP_ZERO(bp); 968 zio->io_pipeline = ZIO_WAIT_FOR_CHILDREN_PIPELINE; 969 } else { 970 ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 971 BP_SET_LSIZE(bp, lsize); 972 BP_SET_PSIZE(bp, csize); 973 BP_SET_COMPRESS(bp, compress); 974 zio->io_pipeline = ZIO_WRITE_ALLOCATE_PIPELINE; 975 } 976 } 977 978 zio_next_stage(zio); 979 } 980 981 static void 982 zio_read_decompress(zio_t *zio) 983 { 984 blkptr_t *bp = zio->io_bp; 985 void *data; 986 uint64_t size; 987 uint64_t bufsize; 988 int compress = BP_GET_COMPRESS(bp); 989 990 ASSERT(compress != ZIO_COMPRESS_OFF); 991 992 zio_pop_transform(zio, &data, &size, &bufsize); 993 994 if (zio_decompress_data(compress, data, size, 995 zio->io_data, zio->io_size)) 996 zio->io_error = EIO; 997 998 zio_buf_free(data, bufsize); 999 1000 zio_next_stage(zio); 1001 } 1002 1003 /* 1004 * ========================================================================== 1005 * Gang block support 1006 * ========================================================================== 1007 */ 1008 static void 1009 zio_gang_pipeline(zio_t *zio) 1010 { 1011 /* 1012 * By default, the pipeline assumes that we're dealing with a gang 1013 * block. If we're not, strip out any gang-specific stages. 1014 */ 1015 if (!BP_IS_GANG(zio->io_bp)) 1016 zio->io_pipeline &= ~ZIO_GANG_STAGES; 1017 1018 zio_next_stage(zio); 1019 } 1020 1021 static void 1022 zio_gang_byteswap(zio_t *zio) 1023 { 1024 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1025 1026 if (BP_SHOULD_BYTESWAP(zio->io_bp)) 1027 byteswap_uint64_array(zio->io_data, zio->io_size); 1028 } 1029 1030 static void 1031 zio_get_gang_header(zio_t *zio) 1032 { 1033 blkptr_t *bp = zio->io_bp; 1034 uint64_t gsize = SPA_GANGBLOCKSIZE; 1035 void *gbuf = zio_buf_alloc(gsize); 1036 1037 ASSERT(BP_IS_GANG(bp)); 1038 1039 zio_push_transform(zio, gbuf, gsize, gsize); 1040 1041 zio_nowait(zio_create(zio, zio->io_spa, bp->blk_birth, bp, gbuf, gsize, 1042 NULL, NULL, ZIO_TYPE_READ, zio->io_priority, 1043 zio->io_flags & ZIO_FLAG_GANG_INHERIT, 1044 ZIO_STAGE_OPEN, ZIO_READ_PIPELINE)); 1045 1046 zio_wait_children_done(zio); 1047 } 1048 1049 static void 1050 zio_read_gang_members(zio_t *zio) 1051 { 1052 zio_gbh_phys_t *gbh; 1053 uint64_t gsize, gbufsize, loff, lsize; 1054 int i; 1055 1056 ASSERT(BP_IS_GANG(zio->io_bp)); 1057 1058 zio_gang_byteswap(zio); 1059 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1060 1061 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1062 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1063 lsize = BP_GET_PSIZE(gbp); 1064 1065 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1066 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1067 ASSERT3U(loff + lsize, <=, zio->io_size); 1068 ASSERT(i < SPA_GBH_NBLKPTRS); 1069 ASSERT(!BP_IS_HOLE(gbp)); 1070 1071 zio_nowait(zio_read(zio, zio->io_spa, gbp, 1072 (char *)zio->io_data + loff, lsize, NULL, NULL, 1073 zio->io_priority, zio->io_flags & ZIO_FLAG_GANG_INHERIT, 1074 &zio->io_bookmark)); 1075 } 1076 1077 zio_buf_free(gbh, gbufsize); 1078 zio_wait_children_done(zio); 1079 } 1080 1081 static void 1082 zio_rewrite_gang_members(zio_t *zio) 1083 { 1084 zio_gbh_phys_t *gbh; 1085 uint64_t gsize, gbufsize, loff, lsize; 1086 int i; 1087 1088 ASSERT(BP_IS_GANG(zio->io_bp)); 1089 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1090 1091 zio_gang_byteswap(zio); 1092 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1093 1094 ASSERT(gsize == gbufsize); 1095 1096 for (loff = 0, i = 0; loff != zio->io_size; loff += lsize, i++) { 1097 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1098 lsize = BP_GET_PSIZE(gbp); 1099 1100 ASSERT(BP_GET_COMPRESS(gbp) == ZIO_COMPRESS_OFF); 1101 ASSERT3U(lsize, ==, BP_GET_LSIZE(gbp)); 1102 ASSERT3U(loff + lsize, <=, zio->io_size); 1103 ASSERT(i < SPA_GBH_NBLKPTRS); 1104 ASSERT(!BP_IS_HOLE(gbp)); 1105 1106 zio_nowait(zio_rewrite(zio, zio->io_spa, zio->io_checksum, 1107 zio->io_txg, gbp, (char *)zio->io_data + loff, lsize, 1108 NULL, NULL, zio->io_priority, zio->io_flags, 1109 &zio->io_bookmark)); 1110 } 1111 1112 zio_push_transform(zio, gbh, gsize, gbufsize); 1113 zio_wait_children_ready(zio); 1114 } 1115 1116 static void 1117 zio_free_gang_members(zio_t *zio) 1118 { 1119 zio_gbh_phys_t *gbh; 1120 uint64_t gsize, gbufsize; 1121 int i; 1122 1123 ASSERT(BP_IS_GANG(zio->io_bp)); 1124 1125 zio_gang_byteswap(zio); 1126 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1127 1128 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1129 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1130 1131 if (BP_IS_HOLE(gbp)) 1132 continue; 1133 zio_nowait(zio_free(zio, zio->io_spa, zio->io_txg, 1134 gbp, NULL, NULL)); 1135 } 1136 1137 zio_buf_free(gbh, gbufsize); 1138 zio_next_stage(zio); 1139 } 1140 1141 static void 1142 zio_claim_gang_members(zio_t *zio) 1143 { 1144 zio_gbh_phys_t *gbh; 1145 uint64_t gsize, gbufsize; 1146 int i; 1147 1148 ASSERT(BP_IS_GANG(zio->io_bp)); 1149 1150 zio_gang_byteswap(zio); 1151 zio_pop_transform(zio, (void **)&gbh, &gsize, &gbufsize); 1152 1153 for (i = 0; i < SPA_GBH_NBLKPTRS; i++) { 1154 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1155 if (BP_IS_HOLE(gbp)) 1156 continue; 1157 zio_nowait(zio_claim(zio, zio->io_spa, zio->io_txg, 1158 gbp, NULL, NULL)); 1159 } 1160 1161 zio_buf_free(gbh, gbufsize); 1162 zio_next_stage(zio); 1163 } 1164 1165 static void 1166 zio_write_allocate_gang_member_done(zio_t *zio) 1167 { 1168 zio_t *pio = zio->io_parent; 1169 dva_t *cdva = zio->io_bp->blk_dva; 1170 dva_t *pdva = pio->io_bp->blk_dva; 1171 uint64_t asize; 1172 int d; 1173 1174 ASSERT3U(pio->io_ndvas, ==, zio->io_ndvas); 1175 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1176 ASSERT3U(zio->io_ndvas, <=, BP_GET_NDVAS(zio->io_bp)); 1177 ASSERT3U(pio->io_ndvas, <=, BP_GET_NDVAS(pio->io_bp)); 1178 1179 mutex_enter(&pio->io_lock); 1180 for (d = 0; d < BP_GET_NDVAS(pio->io_bp); d++) { 1181 ASSERT(DVA_GET_GANG(&pdva[d])); 1182 asize = DVA_GET_ASIZE(&pdva[d]); 1183 asize += DVA_GET_ASIZE(&cdva[d]); 1184 DVA_SET_ASIZE(&pdva[d], asize); 1185 } 1186 mutex_exit(&pio->io_lock); 1187 } 1188 1189 static void 1190 zio_write_allocate_gang_members(zio_t *zio) 1191 { 1192 blkptr_t *bp = zio->io_bp; 1193 dva_t *dva = bp->blk_dva; 1194 spa_t *spa = zio->io_spa; 1195 zio_gbh_phys_t *gbh; 1196 uint64_t txg = zio->io_txg; 1197 uint64_t resid = zio->io_size; 1198 uint64_t maxalloc = P2ROUNDUP(zio->io_size >> 1, SPA_MINBLOCKSIZE); 1199 uint64_t gsize, loff, lsize; 1200 uint32_t gbps_left; 1201 int ndvas = zio->io_ndvas; 1202 int gbh_ndvas = MIN(ndvas + 1, spa_max_replication(spa)); 1203 int error; 1204 int i, d; 1205 1206 gsize = SPA_GANGBLOCKSIZE; 1207 gbps_left = SPA_GBH_NBLKPTRS; 1208 1209 error = metaslab_alloc(spa, gsize, bp, gbh_ndvas, txg, NULL, B_FALSE); 1210 if (error == ENOSPC) 1211 panic("can't allocate gang block header"); 1212 ASSERT(error == 0); 1213 1214 for (d = 0; d < gbh_ndvas; d++) 1215 DVA_SET_GANG(&dva[d], 1); 1216 1217 bp->blk_birth = txg; 1218 1219 gbh = zio_buf_alloc(gsize); 1220 bzero(gbh, gsize); 1221 1222 /* We need to test multi-level gang blocks */ 1223 if (maxalloc >= zio_gang_bang && (lbolt & 0x1) == 0) 1224 maxalloc = MAX(maxalloc >> 2, SPA_MINBLOCKSIZE); 1225 1226 for (loff = 0, i = 0; loff != zio->io_size; 1227 loff += lsize, resid -= lsize, gbps_left--, i++) { 1228 blkptr_t *gbp = &gbh->zg_blkptr[i]; 1229 dva = gbp->blk_dva; 1230 1231 ASSERT(gbps_left != 0); 1232 maxalloc = MIN(maxalloc, resid); 1233 1234 while (resid <= maxalloc * gbps_left) { 1235 error = metaslab_alloc(spa, maxalloc, gbp, ndvas, 1236 txg, bp, B_FALSE); 1237 if (error == 0) 1238 break; 1239 ASSERT3U(error, ==, ENOSPC); 1240 if (maxalloc == SPA_MINBLOCKSIZE) 1241 panic("really out of space"); 1242 maxalloc = P2ROUNDUP(maxalloc >> 1, SPA_MINBLOCKSIZE); 1243 } 1244 1245 if (resid <= maxalloc * gbps_left) { 1246 lsize = maxalloc; 1247 BP_SET_LSIZE(gbp, lsize); 1248 BP_SET_PSIZE(gbp, lsize); 1249 BP_SET_COMPRESS(gbp, ZIO_COMPRESS_OFF); 1250 gbp->blk_birth = txg; 1251 zio_nowait(zio_rewrite(zio, spa, 1252 zio->io_checksum, txg, gbp, 1253 (char *)zio->io_data + loff, lsize, 1254 zio_write_allocate_gang_member_done, NULL, 1255 zio->io_priority, zio->io_flags, 1256 &zio->io_bookmark)); 1257 } else { 1258 lsize = P2ROUNDUP(resid / gbps_left, SPA_MINBLOCKSIZE); 1259 ASSERT(lsize != SPA_MINBLOCKSIZE); 1260 zio_nowait(zio_write_allocate(zio, spa, 1261 zio->io_checksum, txg, gbp, 1262 (char *)zio->io_data + loff, lsize, 1263 zio_write_allocate_gang_member_done, NULL, 1264 zio->io_priority, zio->io_flags)); 1265 } 1266 } 1267 1268 ASSERT(resid == 0 && loff == zio->io_size); 1269 1270 zio->io_pipeline |= 1U << ZIO_STAGE_GANG_CHECKSUM_GENERATE; 1271 1272 zio_push_transform(zio, gbh, gsize, gsize); 1273 /* 1274 * As much as we'd like this to be zio_wait_children_ready(), 1275 * updating our ASIZE doesn't happen until the io_done callback, 1276 * so we have to wait for that to finish in order for our BP 1277 * to be stable. 1278 */ 1279 zio_wait_children_done(zio); 1280 } 1281 1282 /* 1283 * ========================================================================== 1284 * Allocate and free blocks 1285 * ========================================================================== 1286 */ 1287 static void 1288 zio_dva_allocate(zio_t *zio) 1289 { 1290 blkptr_t *bp = zio->io_bp; 1291 int error; 1292 1293 ASSERT(BP_IS_HOLE(bp)); 1294 ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 1295 ASSERT3U(zio->io_ndvas, >, 0); 1296 ASSERT3U(zio->io_ndvas, <=, spa_max_replication(zio->io_spa)); 1297 1298 /* For testing, make some blocks above a certain size be gang blocks */ 1299 if (zio->io_size >= zio_gang_bang && (lbolt & 0x3) == 0) { 1300 zio_write_allocate_gang_members(zio); 1301 return; 1302 } 1303 1304 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1305 1306 error = metaslab_alloc(zio->io_spa, zio->io_size, bp, zio->io_ndvas, 1307 zio->io_txg, NULL, B_FALSE); 1308 1309 if (error == 0) { 1310 bp->blk_birth = zio->io_txg; 1311 } else if (error == ENOSPC) { 1312 if (zio->io_size == SPA_MINBLOCKSIZE) 1313 panic("really, truly out of space"); 1314 zio_write_allocate_gang_members(zio); 1315 return; 1316 } else { 1317 zio->io_error = error; 1318 } 1319 zio_next_stage(zio); 1320 } 1321 1322 static void 1323 zio_dva_free(zio_t *zio) 1324 { 1325 blkptr_t *bp = zio->io_bp; 1326 1327 metaslab_free(zio->io_spa, bp, zio->io_txg, B_FALSE); 1328 1329 BP_ZERO(bp); 1330 1331 zio_next_stage(zio); 1332 } 1333 1334 static void 1335 zio_dva_claim(zio_t *zio) 1336 { 1337 zio->io_error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 1338 1339 zio_next_stage(zio); 1340 } 1341 1342 /* 1343 * ========================================================================== 1344 * Read and write to physical devices 1345 * ========================================================================== 1346 */ 1347 1348 static void 1349 zio_vdev_io_start(zio_t *zio) 1350 { 1351 vdev_t *vd = zio->io_vd; 1352 vdev_t *tvd = vd ? vd->vdev_top : NULL; 1353 blkptr_t *bp = zio->io_bp; 1354 uint64_t align; 1355 1356 if (vd == NULL) { 1357 /* The mirror_ops handle multiple DVAs in a single BP */ 1358 vdev_mirror_ops.vdev_op_io_start(zio); 1359 return; 1360 } 1361 1362 align = 1ULL << tvd->vdev_ashift; 1363 1364 if (zio->io_retries == 0 && vd == tvd) 1365 zio->io_flags |= ZIO_FLAG_FAILFAST; 1366 1367 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 1368 vd->vdev_children == 0) { 1369 zio->io_flags |= ZIO_FLAG_PHYSICAL; 1370 zio->io_offset += VDEV_LABEL_START_SIZE; 1371 } 1372 1373 if (P2PHASE(zio->io_size, align) != 0) { 1374 uint64_t asize = P2ROUNDUP(zio->io_size, align); 1375 char *abuf = zio_buf_alloc(asize); 1376 ASSERT(vd == tvd); 1377 if (zio->io_type == ZIO_TYPE_WRITE) { 1378 bcopy(zio->io_data, abuf, zio->io_size); 1379 bzero(abuf + zio->io_size, asize - zio->io_size); 1380 } 1381 zio_push_transform(zio, abuf, asize, asize); 1382 ASSERT(!(zio->io_flags & ZIO_FLAG_SUBBLOCK)); 1383 zio->io_flags |= ZIO_FLAG_SUBBLOCK; 1384 } 1385 1386 ASSERT(P2PHASE(zio->io_offset, align) == 0); 1387 ASSERT(P2PHASE(zio->io_size, align) == 0); 1388 ASSERT(bp == NULL || 1389 P2ROUNDUP(ZIO_GET_IOSIZE(zio), align) == zio->io_size); 1390 ASSERT(zio->io_type != ZIO_TYPE_WRITE || (spa_mode & FWRITE)); 1391 1392 vdev_io_start(zio); 1393 1394 /* zio_next_stage_async() gets called from io completion interrupt */ 1395 } 1396 1397 static void 1398 zio_vdev_io_done(zio_t *zio) 1399 { 1400 if (zio->io_vd == NULL) 1401 /* The mirror_ops handle multiple DVAs in a single BP */ 1402 vdev_mirror_ops.vdev_op_io_done(zio); 1403 else 1404 vdev_io_done(zio); 1405 } 1406 1407 /* XXPOLICY */ 1408 boolean_t 1409 zio_should_retry(zio_t *zio) 1410 { 1411 vdev_t *vd = zio->io_vd; 1412 1413 if (zio->io_error == 0) 1414 return (B_FALSE); 1415 if (zio->io_delegate_list != NULL) 1416 return (B_FALSE); 1417 if (vd && vd != vd->vdev_top) 1418 return (B_FALSE); 1419 if (zio->io_flags & ZIO_FLAG_DONT_RETRY) 1420 return (B_FALSE); 1421 if (zio->io_retries > 0) 1422 return (B_FALSE); 1423 1424 return (B_TRUE); 1425 } 1426 1427 static void 1428 zio_vdev_io_assess(zio_t *zio) 1429 { 1430 vdev_t *vd = zio->io_vd; 1431 vdev_t *tvd = vd ? vd->vdev_top : NULL; 1432 1433 ASSERT(zio->io_vsd == NULL); 1434 1435 if (zio->io_flags & ZIO_FLAG_SUBBLOCK) { 1436 void *abuf; 1437 uint64_t asize; 1438 ASSERT(vd == tvd); 1439 zio_pop_transform(zio, &abuf, &asize, &asize); 1440 if (zio->io_type == ZIO_TYPE_READ) 1441 bcopy(abuf, zio->io_data, zio->io_size); 1442 zio_buf_free(abuf, asize); 1443 zio->io_flags &= ~ZIO_FLAG_SUBBLOCK; 1444 } 1445 1446 if (zio_injection_enabled && !zio->io_error) 1447 zio->io_error = zio_handle_fault_injection(zio, EIO); 1448 1449 /* 1450 * If the I/O failed, determine whether we should attempt to retry it. 1451 */ 1452 /* XXPOLICY */ 1453 if (zio_should_retry(zio)) { 1454 ASSERT(tvd == vd); 1455 1456 zio->io_retries++; 1457 zio->io_error = 0; 1458 zio->io_flags &= ZIO_FLAG_VDEV_INHERIT; 1459 /* XXPOLICY */ 1460 zio->io_flags &= ~ZIO_FLAG_FAILFAST; 1461 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1462 zio->io_stage = ZIO_STAGE_VDEV_IO_START - 1; 1463 1464 dprintf("retry #%d for %s to %s offset %llx\n", 1465 zio->io_retries, zio_type_name[zio->io_type], 1466 vdev_description(vd), zio->io_offset); 1467 1468 zio_next_stage_async(zio); 1469 return; 1470 } 1471 1472 if (zio->io_error != 0 && zio->io_error != ECKSUM && 1473 !(zio->io_flags & ZIO_FLAG_SPECULATIVE) && vd) { 1474 /* 1475 * Poor man's hotplug support. Even if we're done retrying this 1476 * I/O, try to reopen the vdev to see if it's still attached. 1477 * To avoid excessive thrashing, we only try it once a minute. 1478 * This also has the effect of detecting when missing devices 1479 * have come back, by polling the device once a minute. 1480 * 1481 * We need to do this asynchronously because we can't grab 1482 * all the necessary locks way down here. 1483 */ 1484 if (gethrtime() - vd->vdev_last_try > 60ULL * NANOSEC) { 1485 vd->vdev_last_try = gethrtime(); 1486 tvd->vdev_reopen_wanted = 1; 1487 spa_async_request(vd->vdev_spa, SPA_ASYNC_REOPEN); 1488 } 1489 } 1490 1491 zio_next_stage(zio); 1492 } 1493 1494 void 1495 zio_vdev_io_reissue(zio_t *zio) 1496 { 1497 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1498 ASSERT(zio->io_error == 0); 1499 1500 zio->io_stage--; 1501 } 1502 1503 void 1504 zio_vdev_io_redone(zio_t *zio) 1505 { 1506 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 1507 1508 zio->io_stage--; 1509 } 1510 1511 void 1512 zio_vdev_io_bypass(zio_t *zio) 1513 { 1514 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 1515 ASSERT(zio->io_error == 0); 1516 1517 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 1518 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS - 1; 1519 } 1520 1521 /* 1522 * ========================================================================== 1523 * Generate and verify checksums 1524 * ========================================================================== 1525 */ 1526 static void 1527 zio_checksum_generate(zio_t *zio) 1528 { 1529 int checksum = zio->io_checksum; 1530 blkptr_t *bp = zio->io_bp; 1531 1532 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 1533 1534 BP_SET_CHECKSUM(bp, checksum); 1535 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1536 1537 zio_checksum(checksum, &bp->blk_cksum, zio->io_data, zio->io_size); 1538 1539 zio_next_stage(zio); 1540 } 1541 1542 static void 1543 zio_gang_checksum_generate(zio_t *zio) 1544 { 1545 zio_cksum_t zc; 1546 zio_gbh_phys_t *gbh = zio->io_data; 1547 1548 ASSERT(BP_IS_GANG(zio->io_bp)); 1549 ASSERT3U(zio->io_size, ==, SPA_GANGBLOCKSIZE); 1550 1551 zio_set_gang_verifier(zio, &gbh->zg_tail.zbt_cksum); 1552 1553 zio_checksum(ZIO_CHECKSUM_GANG_HEADER, &zc, zio->io_data, zio->io_size); 1554 1555 zio_next_stage(zio); 1556 } 1557 1558 static void 1559 zio_checksum_verify(zio_t *zio) 1560 { 1561 if (zio->io_bp != NULL) { 1562 zio->io_error = zio_checksum_error(zio); 1563 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) 1564 zfs_ereport_post(FM_EREPORT_ZFS_CHECKSUM, 1565 zio->io_spa, zio->io_vd, zio, 0, 0); 1566 } 1567 1568 zio_next_stage(zio); 1569 } 1570 1571 /* 1572 * Called by RAID-Z to ensure we don't compute the checksum twice. 1573 */ 1574 void 1575 zio_checksum_verified(zio_t *zio) 1576 { 1577 zio->io_pipeline &= ~(1U << ZIO_STAGE_CHECKSUM_VERIFY); 1578 } 1579 1580 /* 1581 * Set the external verifier for a gang block based on stuff in the bp 1582 */ 1583 void 1584 zio_set_gang_verifier(zio_t *zio, zio_cksum_t *zcp) 1585 { 1586 blkptr_t *bp = zio->io_bp; 1587 1588 zcp->zc_word[0] = DVA_GET_VDEV(BP_IDENTITY(bp)); 1589 zcp->zc_word[1] = DVA_GET_OFFSET(BP_IDENTITY(bp)); 1590 zcp->zc_word[2] = bp->blk_birth; 1591 zcp->zc_word[3] = 0; 1592 } 1593 1594 /* 1595 * ========================================================================== 1596 * Define the pipeline 1597 * ========================================================================== 1598 */ 1599 typedef void zio_pipe_stage_t(zio_t *zio); 1600 1601 static void 1602 zio_badop(zio_t *zio) 1603 { 1604 panic("Invalid I/O pipeline stage %u for zio %p", zio->io_stage, zio); 1605 } 1606 1607 zio_pipe_stage_t *zio_pipeline[ZIO_STAGE_DONE + 2] = { 1608 zio_badop, 1609 zio_wait_children_ready, 1610 zio_write_compress, 1611 zio_checksum_generate, 1612 zio_gang_pipeline, 1613 zio_get_gang_header, 1614 zio_rewrite_gang_members, 1615 zio_free_gang_members, 1616 zio_claim_gang_members, 1617 zio_dva_allocate, 1618 zio_dva_free, 1619 zio_dva_claim, 1620 zio_gang_checksum_generate, 1621 zio_ready, 1622 zio_vdev_io_start, 1623 zio_vdev_io_done, 1624 zio_vdev_io_assess, 1625 zio_wait_children_done, 1626 zio_checksum_verify, 1627 zio_read_gang_members, 1628 zio_read_decompress, 1629 zio_done, 1630 zio_badop 1631 }; 1632 1633 /* 1634 * Move an I/O to the next stage of the pipeline and execute that stage. 1635 * There's no locking on io_stage because there's no legitimate way for 1636 * multiple threads to be attempting to process the same I/O. 1637 */ 1638 void 1639 zio_next_stage(zio_t *zio) 1640 { 1641 uint32_t pipeline = zio->io_pipeline; 1642 1643 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1644 1645 if (zio->io_error) { 1646 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1647 zio, vdev_description(zio->io_vd), 1648 zio->io_offset, zio->io_stage, zio->io_error); 1649 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1650 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1651 } 1652 1653 while (((1U << ++zio->io_stage) & pipeline) == 0) 1654 continue; 1655 1656 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1657 ASSERT(zio->io_stalled == 0); 1658 1659 zio_pipeline[zio->io_stage](zio); 1660 } 1661 1662 void 1663 zio_next_stage_async(zio_t *zio) 1664 { 1665 taskq_t *tq; 1666 uint32_t pipeline = zio->io_pipeline; 1667 1668 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1669 1670 if (zio->io_error) { 1671 dprintf("zio %p vdev %s offset %llx stage %d error %d\n", 1672 zio, vdev_description(zio->io_vd), 1673 zio->io_offset, zio->io_stage, zio->io_error); 1674 if (((1U << zio->io_stage) & ZIO_VDEV_IO_PIPELINE) == 0) 1675 pipeline &= ZIO_ERROR_PIPELINE_MASK; 1676 } 1677 1678 while (((1U << ++zio->io_stage) & pipeline) == 0) 1679 continue; 1680 1681 ASSERT(zio->io_stage <= ZIO_STAGE_DONE); 1682 ASSERT(zio->io_stalled == 0); 1683 1684 /* 1685 * For performance, we'll probably want two sets of task queues: 1686 * per-CPU issue taskqs and per-CPU completion taskqs. The per-CPU 1687 * part is for read performance: since we have to make a pass over 1688 * the data to checksum it anyway, we want to do this on the same CPU 1689 * that issued the read, because (assuming CPU scheduling affinity) 1690 * that thread is probably still there. Getting this optimization 1691 * right avoids performance-hostile cache-to-cache transfers. 1692 * 1693 * Note that having two sets of task queues is also necessary for 1694 * correctness: if all of the issue threads get bogged down waiting 1695 * for dependent reads (e.g. metaslab freelist) to complete, then 1696 * there won't be any threads available to service I/O completion 1697 * interrupts. 1698 */ 1699 if ((1U << zio->io_stage) & zio->io_async_stages) { 1700 if (zio->io_stage < ZIO_STAGE_VDEV_IO_DONE) 1701 tq = zio->io_spa->spa_zio_issue_taskq[zio->io_type]; 1702 else 1703 tq = zio->io_spa->spa_zio_intr_taskq[zio->io_type]; 1704 (void) taskq_dispatch(tq, 1705 (task_func_t *)zio_pipeline[zio->io_stage], zio, TQ_SLEEP); 1706 } else { 1707 zio_pipeline[zio->io_stage](zio); 1708 } 1709 } 1710 1711 /* 1712 * Try to allocate an intent log block. Return 0 on success, errno on failure. 1713 */ 1714 int 1715 zio_alloc_blk(spa_t *spa, uint64_t size, blkptr_t *new_bp, blkptr_t *old_bp, 1716 uint64_t txg) 1717 { 1718 int error; 1719 1720 spa_config_enter(spa, RW_READER, FTAG); 1721 1722 /* 1723 * We were passed the previous log blocks dva_t in bp->blk_dva[0]. 1724 */ 1725 error = metaslab_alloc(spa, size, new_bp, 1, txg, old_bp, B_TRUE); 1726 1727 if (error == 0) { 1728 BP_SET_LSIZE(new_bp, size); 1729 BP_SET_PSIZE(new_bp, size); 1730 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 1731 BP_SET_CHECKSUM(new_bp, ZIO_CHECKSUM_ZILOG); 1732 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 1733 BP_SET_LEVEL(new_bp, 0); 1734 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 1735 new_bp->blk_birth = txg; 1736 } 1737 1738 spa_config_exit(spa, FTAG); 1739 1740 return (error); 1741 } 1742 1743 /* 1744 * Free an intent log block. We know it can't be a gang block, so there's 1745 * nothing to do except metaslab_free() it. 1746 */ 1747 void 1748 zio_free_blk(spa_t *spa, blkptr_t *bp, uint64_t txg) 1749 { 1750 ASSERT(!BP_IS_GANG(bp)); 1751 1752 spa_config_enter(spa, RW_READER, FTAG); 1753 1754 metaslab_free(spa, bp, txg, B_FALSE); 1755 1756 spa_config_exit(spa, FTAG); 1757 } 1758