1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011 by Delphix. All rights reserved. 24 */ 25 26 #include <sys/zfs_context.h> 27 #include <sys/fm/fs/zfs.h> 28 #include <sys/spa.h> 29 #include <sys/txg.h> 30 #include <sys/spa_impl.h> 31 #include <sys/vdev_impl.h> 32 #include <sys/zio_impl.h> 33 #include <sys/zio_compress.h> 34 #include <sys/zio_checksum.h> 35 #include <sys/dmu_objset.h> 36 #include <sys/arc.h> 37 #include <sys/ddt.h> 38 39 /* 40 * ========================================================================== 41 * I/O priority table 42 * ========================================================================== 43 */ 44 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 45 0, /* ZIO_PRIORITY_NOW */ 46 0, /* ZIO_PRIORITY_SYNC_READ */ 47 0, /* ZIO_PRIORITY_SYNC_WRITE */ 48 0, /* ZIO_PRIORITY_LOG_WRITE */ 49 1, /* ZIO_PRIORITY_CACHE_FILL */ 50 1, /* ZIO_PRIORITY_AGG */ 51 4, /* ZIO_PRIORITY_FREE */ 52 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 53 6, /* ZIO_PRIORITY_ASYNC_READ */ 54 10, /* ZIO_PRIORITY_RESILVER */ 55 20, /* ZIO_PRIORITY_SCRUB */ 56 2, /* ZIO_PRIORITY_DDT_PREFETCH */ 57 }; 58 59 /* 60 * ========================================================================== 61 * I/O type descriptions 62 * ========================================================================== 63 */ 64 char *zio_type_name[ZIO_TYPES] = { 65 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 66 "zio_ioctl" 67 }; 68 69 /* 70 * ========================================================================== 71 * I/O kmem caches 72 * ========================================================================== 73 */ 74 kmem_cache_t *zio_cache; 75 kmem_cache_t *zio_link_cache; 76 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 77 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 78 79 #ifdef _KERNEL 80 extern vmem_t *zio_alloc_arena; 81 #endif 82 extern int zfs_mg_alloc_failures; 83 84 /* 85 * An allocating zio is one that either currently has the DVA allocate 86 * stage set or will have it later in its lifetime. 87 */ 88 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 89 90 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 91 92 #ifdef ZFS_DEBUG 93 int zio_buf_debug_limit = 16384; 94 #else 95 int zio_buf_debug_limit = 0; 96 #endif 97 98 void 99 zio_init(void) 100 { 101 size_t c; 102 vmem_t *data_alloc_arena = NULL; 103 104 #ifdef _KERNEL 105 data_alloc_arena = zio_alloc_arena; 106 #endif 107 zio_cache = kmem_cache_create("zio_cache", 108 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 109 zio_link_cache = kmem_cache_create("zio_link_cache", 110 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 111 112 /* 113 * For small buffers, we want a cache for each multiple of 114 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 115 * for each quarter-power of 2. For large buffers, we want 116 * a cache for each multiple of PAGESIZE. 117 */ 118 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 119 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 120 size_t p2 = size; 121 size_t align = 0; 122 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 123 124 while (p2 & (p2 - 1)) 125 p2 &= p2 - 1; 126 127 if (size <= 4 * SPA_MINBLOCKSIZE) { 128 align = SPA_MINBLOCKSIZE; 129 } else if (P2PHASE(size, PAGESIZE) == 0) { 130 align = PAGESIZE; 131 } else if (P2PHASE(size, p2 >> 2) == 0) { 132 align = p2 >> 2; 133 } 134 135 if (align != 0) { 136 char name[36]; 137 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 138 zio_buf_cache[c] = kmem_cache_create(name, size, 139 align, NULL, NULL, NULL, NULL, NULL, cflags); 140 141 /* 142 * Since zio_data bufs do not appear in crash dumps, we 143 * pass KMC_NOTOUCH so that no allocator metadata is 144 * stored with the buffers. 145 */ 146 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 147 zio_data_buf_cache[c] = kmem_cache_create(name, size, 148 align, NULL, NULL, NULL, NULL, data_alloc_arena, 149 cflags | KMC_NOTOUCH); 150 } 151 } 152 153 while (--c != 0) { 154 ASSERT(zio_buf_cache[c] != NULL); 155 if (zio_buf_cache[c - 1] == NULL) 156 zio_buf_cache[c - 1] = zio_buf_cache[c]; 157 158 ASSERT(zio_data_buf_cache[c] != NULL); 159 if (zio_data_buf_cache[c - 1] == NULL) 160 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 161 } 162 163 /* 164 * The zio write taskqs have 1 thread per cpu, allow 1/2 of the taskqs 165 * to fail 3 times per txg or 8 failures, whichever is greater. 166 */ 167 zfs_mg_alloc_failures = MAX((3 * max_ncpus / 2), 8); 168 169 zio_inject_init(); 170 } 171 172 void 173 zio_fini(void) 174 { 175 size_t c; 176 kmem_cache_t *last_cache = NULL; 177 kmem_cache_t *last_data_cache = NULL; 178 179 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 180 if (zio_buf_cache[c] != last_cache) { 181 last_cache = zio_buf_cache[c]; 182 kmem_cache_destroy(zio_buf_cache[c]); 183 } 184 zio_buf_cache[c] = NULL; 185 186 if (zio_data_buf_cache[c] != last_data_cache) { 187 last_data_cache = zio_data_buf_cache[c]; 188 kmem_cache_destroy(zio_data_buf_cache[c]); 189 } 190 zio_data_buf_cache[c] = NULL; 191 } 192 193 kmem_cache_destroy(zio_link_cache); 194 kmem_cache_destroy(zio_cache); 195 196 zio_inject_fini(); 197 } 198 199 /* 200 * ========================================================================== 201 * Allocate and free I/O buffers 202 * ========================================================================== 203 */ 204 205 /* 206 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 207 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 208 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 209 * excess / transient data in-core during a crashdump. 210 */ 211 void * 212 zio_buf_alloc(size_t size) 213 { 214 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 215 216 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 217 218 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 219 } 220 221 /* 222 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 223 * crashdump if the kernel panics. This exists so that we will limit the amount 224 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 225 * of kernel heap dumped to disk when the kernel panics) 226 */ 227 void * 228 zio_data_buf_alloc(size_t size) 229 { 230 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 231 232 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 233 234 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 235 } 236 237 void 238 zio_buf_free(void *buf, size_t size) 239 { 240 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 241 242 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 243 244 kmem_cache_free(zio_buf_cache[c], buf); 245 } 246 247 void 248 zio_data_buf_free(void *buf, size_t size) 249 { 250 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 251 252 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 253 254 kmem_cache_free(zio_data_buf_cache[c], buf); 255 } 256 257 /* 258 * ========================================================================== 259 * Push and pop I/O transform buffers 260 * ========================================================================== 261 */ 262 static void 263 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 264 zio_transform_func_t *transform) 265 { 266 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 267 268 zt->zt_orig_data = zio->io_data; 269 zt->zt_orig_size = zio->io_size; 270 zt->zt_bufsize = bufsize; 271 zt->zt_transform = transform; 272 273 zt->zt_next = zio->io_transform_stack; 274 zio->io_transform_stack = zt; 275 276 zio->io_data = data; 277 zio->io_size = size; 278 } 279 280 static void 281 zio_pop_transforms(zio_t *zio) 282 { 283 zio_transform_t *zt; 284 285 while ((zt = zio->io_transform_stack) != NULL) { 286 if (zt->zt_transform != NULL) 287 zt->zt_transform(zio, 288 zt->zt_orig_data, zt->zt_orig_size); 289 290 if (zt->zt_bufsize != 0) 291 zio_buf_free(zio->io_data, zt->zt_bufsize); 292 293 zio->io_data = zt->zt_orig_data; 294 zio->io_size = zt->zt_orig_size; 295 zio->io_transform_stack = zt->zt_next; 296 297 kmem_free(zt, sizeof (zio_transform_t)); 298 } 299 } 300 301 /* 302 * ========================================================================== 303 * I/O transform callbacks for subblocks and decompression 304 * ========================================================================== 305 */ 306 static void 307 zio_subblock(zio_t *zio, void *data, uint64_t size) 308 { 309 ASSERT(zio->io_size > size); 310 311 if (zio->io_type == ZIO_TYPE_READ) 312 bcopy(zio->io_data, data, size); 313 } 314 315 static void 316 zio_decompress(zio_t *zio, void *data, uint64_t size) 317 { 318 if (zio->io_error == 0 && 319 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 320 zio->io_data, data, zio->io_size, size) != 0) 321 zio->io_error = EIO; 322 } 323 324 /* 325 * ========================================================================== 326 * I/O parent/child relationships and pipeline interlocks 327 * ========================================================================== 328 */ 329 /* 330 * NOTE - Callers to zio_walk_parents() and zio_walk_children must 331 * continue calling these functions until they return NULL. 332 * Otherwise, the next caller will pick up the list walk in 333 * some indeterminate state. (Otherwise every caller would 334 * have to pass in a cookie to keep the state represented by 335 * io_walk_link, which gets annoying.) 336 */ 337 zio_t * 338 zio_walk_parents(zio_t *cio) 339 { 340 zio_link_t *zl = cio->io_walk_link; 341 list_t *pl = &cio->io_parent_list; 342 343 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl); 344 cio->io_walk_link = zl; 345 346 if (zl == NULL) 347 return (NULL); 348 349 ASSERT(zl->zl_child == cio); 350 return (zl->zl_parent); 351 } 352 353 zio_t * 354 zio_walk_children(zio_t *pio) 355 { 356 zio_link_t *zl = pio->io_walk_link; 357 list_t *cl = &pio->io_child_list; 358 359 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl); 360 pio->io_walk_link = zl; 361 362 if (zl == NULL) 363 return (NULL); 364 365 ASSERT(zl->zl_parent == pio); 366 return (zl->zl_child); 367 } 368 369 zio_t * 370 zio_unique_parent(zio_t *cio) 371 { 372 zio_t *pio = zio_walk_parents(cio); 373 374 VERIFY(zio_walk_parents(cio) == NULL); 375 return (pio); 376 } 377 378 void 379 zio_add_child(zio_t *pio, zio_t *cio) 380 { 381 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 382 383 /* 384 * Logical I/Os can have logical, gang, or vdev children. 385 * Gang I/Os can have gang or vdev children. 386 * Vdev I/Os can only have vdev children. 387 * The following ASSERT captures all of these constraints. 388 */ 389 ASSERT(cio->io_child_type <= pio->io_child_type); 390 391 zl->zl_parent = pio; 392 zl->zl_child = cio; 393 394 mutex_enter(&cio->io_lock); 395 mutex_enter(&pio->io_lock); 396 397 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 398 399 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 400 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 401 402 list_insert_head(&pio->io_child_list, zl); 403 list_insert_head(&cio->io_parent_list, zl); 404 405 pio->io_child_count++; 406 cio->io_parent_count++; 407 408 mutex_exit(&pio->io_lock); 409 mutex_exit(&cio->io_lock); 410 } 411 412 static void 413 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 414 { 415 ASSERT(zl->zl_parent == pio); 416 ASSERT(zl->zl_child == cio); 417 418 mutex_enter(&cio->io_lock); 419 mutex_enter(&pio->io_lock); 420 421 list_remove(&pio->io_child_list, zl); 422 list_remove(&cio->io_parent_list, zl); 423 424 pio->io_child_count--; 425 cio->io_parent_count--; 426 427 mutex_exit(&pio->io_lock); 428 mutex_exit(&cio->io_lock); 429 430 kmem_cache_free(zio_link_cache, zl); 431 } 432 433 static boolean_t 434 zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 435 { 436 uint64_t *countp = &zio->io_children[child][wait]; 437 boolean_t waiting = B_FALSE; 438 439 mutex_enter(&zio->io_lock); 440 ASSERT(zio->io_stall == NULL); 441 if (*countp != 0) { 442 zio->io_stage >>= 1; 443 zio->io_stall = countp; 444 waiting = B_TRUE; 445 } 446 mutex_exit(&zio->io_lock); 447 448 return (waiting); 449 } 450 451 static void 452 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 453 { 454 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 455 int *errorp = &pio->io_child_error[zio->io_child_type]; 456 457 mutex_enter(&pio->io_lock); 458 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 459 *errorp = zio_worst_error(*errorp, zio->io_error); 460 pio->io_reexecute |= zio->io_reexecute; 461 ASSERT3U(*countp, >, 0); 462 if (--*countp == 0 && pio->io_stall == countp) { 463 pio->io_stall = NULL; 464 mutex_exit(&pio->io_lock); 465 zio_execute(pio); 466 } else { 467 mutex_exit(&pio->io_lock); 468 } 469 } 470 471 static void 472 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 473 { 474 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 475 zio->io_error = zio->io_child_error[c]; 476 } 477 478 /* 479 * ========================================================================== 480 * Create the various types of I/O (read, write, free, etc) 481 * ========================================================================== 482 */ 483 static zio_t * 484 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 485 void *data, uint64_t size, zio_done_func_t *done, void *private, 486 zio_type_t type, int priority, enum zio_flag flags, 487 vdev_t *vd, uint64_t offset, const zbookmark_t *zb, 488 enum zio_stage stage, enum zio_stage pipeline) 489 { 490 zio_t *zio; 491 492 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 493 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 494 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 495 496 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 497 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 498 ASSERT(vd || stage == ZIO_STAGE_OPEN); 499 500 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 501 bzero(zio, sizeof (zio_t)); 502 503 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 504 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 505 506 list_create(&zio->io_parent_list, sizeof (zio_link_t), 507 offsetof(zio_link_t, zl_parent_node)); 508 list_create(&zio->io_child_list, sizeof (zio_link_t), 509 offsetof(zio_link_t, zl_child_node)); 510 511 if (vd != NULL) 512 zio->io_child_type = ZIO_CHILD_VDEV; 513 else if (flags & ZIO_FLAG_GANG_CHILD) 514 zio->io_child_type = ZIO_CHILD_GANG; 515 else if (flags & ZIO_FLAG_DDT_CHILD) 516 zio->io_child_type = ZIO_CHILD_DDT; 517 else 518 zio->io_child_type = ZIO_CHILD_LOGICAL; 519 520 if (bp != NULL) { 521 zio->io_bp = (blkptr_t *)bp; 522 zio->io_bp_copy = *bp; 523 zio->io_bp_orig = *bp; 524 if (type != ZIO_TYPE_WRITE || 525 zio->io_child_type == ZIO_CHILD_DDT) 526 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 527 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 528 zio->io_logical = zio; 529 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 530 pipeline |= ZIO_GANG_STAGES; 531 } 532 533 zio->io_spa = spa; 534 zio->io_txg = txg; 535 zio->io_done = done; 536 zio->io_private = private; 537 zio->io_type = type; 538 zio->io_priority = priority; 539 zio->io_vd = vd; 540 zio->io_offset = offset; 541 zio->io_orig_data = zio->io_data = data; 542 zio->io_orig_size = zio->io_size = size; 543 zio->io_orig_flags = zio->io_flags = flags; 544 zio->io_orig_stage = zio->io_stage = stage; 545 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 546 547 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 548 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 549 550 if (zb != NULL) 551 zio->io_bookmark = *zb; 552 553 if (pio != NULL) { 554 if (zio->io_logical == NULL) 555 zio->io_logical = pio->io_logical; 556 if (zio->io_child_type == ZIO_CHILD_GANG) 557 zio->io_gang_leader = pio->io_gang_leader; 558 zio_add_child(pio, zio); 559 } 560 561 return (zio); 562 } 563 564 static void 565 zio_destroy(zio_t *zio) 566 { 567 list_destroy(&zio->io_parent_list); 568 list_destroy(&zio->io_child_list); 569 mutex_destroy(&zio->io_lock); 570 cv_destroy(&zio->io_cv); 571 kmem_cache_free(zio_cache, zio); 572 } 573 574 zio_t * 575 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 576 void *private, enum zio_flag flags) 577 { 578 zio_t *zio; 579 580 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 581 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 582 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 583 584 return (zio); 585 } 586 587 zio_t * 588 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 589 { 590 return (zio_null(NULL, spa, NULL, done, private, flags)); 591 } 592 593 zio_t * 594 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 595 void *data, uint64_t size, zio_done_func_t *done, void *private, 596 int priority, enum zio_flag flags, const zbookmark_t *zb) 597 { 598 zio_t *zio; 599 600 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 601 data, size, done, private, 602 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 603 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 604 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 605 606 return (zio); 607 } 608 609 zio_t * 610 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 611 void *data, uint64_t size, const zio_prop_t *zp, 612 zio_done_func_t *ready, zio_done_func_t *done, void *private, 613 int priority, enum zio_flag flags, const zbookmark_t *zb) 614 { 615 zio_t *zio; 616 617 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 618 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 619 zp->zp_compress >= ZIO_COMPRESS_OFF && 620 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 621 zp->zp_type < DMU_OT_NUMTYPES && 622 zp->zp_level < 32 && 623 zp->zp_copies > 0 && 624 zp->zp_copies <= spa_max_replication(spa) && 625 zp->zp_dedup <= 1 && 626 zp->zp_dedup_verify <= 1); 627 628 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 629 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 630 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 631 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 632 633 zio->io_ready = ready; 634 zio->io_prop = *zp; 635 636 return (zio); 637 } 638 639 zio_t * 640 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 641 uint64_t size, zio_done_func_t *done, void *private, int priority, 642 enum zio_flag flags, zbookmark_t *zb) 643 { 644 zio_t *zio; 645 646 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 647 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 648 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 649 650 return (zio); 651 } 652 653 void 654 zio_write_override(zio_t *zio, blkptr_t *bp, int copies) 655 { 656 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 657 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 658 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 659 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 660 661 zio->io_prop.zp_copies = copies; 662 zio->io_bp_override = bp; 663 } 664 665 void 666 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 667 { 668 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 669 } 670 671 zio_t * 672 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 673 enum zio_flag flags) 674 { 675 zio_t *zio; 676 677 dprintf_bp(bp, "freeing in txg %llu, pass %u", 678 (longlong_t)txg, spa->spa_sync_pass); 679 680 ASSERT(!BP_IS_HOLE(bp)); 681 ASSERT(spa_syncing_txg(spa) == txg); 682 ASSERT(spa_sync_pass(spa) <= SYNC_PASS_DEFERRED_FREE); 683 684 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 685 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, flags, 686 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 687 688 return (zio); 689 } 690 691 zio_t * 692 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 693 zio_done_func_t *done, void *private, enum zio_flag flags) 694 { 695 zio_t *zio; 696 697 /* 698 * A claim is an allocation of a specific block. Claims are needed 699 * to support immediate writes in the intent log. The issue is that 700 * immediate writes contain committed data, but in a txg that was 701 * *not* committed. Upon opening the pool after an unclean shutdown, 702 * the intent log claims all blocks that contain immediate write data 703 * so that the SPA knows they're in use. 704 * 705 * All claims *must* be resolved in the first txg -- before the SPA 706 * starts allocating blocks -- so that nothing is allocated twice. 707 * If txg == 0 we just verify that the block is claimable. 708 */ 709 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 710 ASSERT(txg == spa_first_txg(spa) || txg == 0); 711 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 712 713 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 714 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 715 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 716 717 return (zio); 718 } 719 720 zio_t * 721 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 722 zio_done_func_t *done, void *private, int priority, enum zio_flag flags) 723 { 724 zio_t *zio; 725 int c; 726 727 if (vd->vdev_children == 0) { 728 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 729 ZIO_TYPE_IOCTL, priority, flags, vd, 0, NULL, 730 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 731 732 zio->io_cmd = cmd; 733 } else { 734 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 735 736 for (c = 0; c < vd->vdev_children; c++) 737 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 738 done, private, priority, flags)); 739 } 740 741 return (zio); 742 } 743 744 zio_t * 745 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 746 void *data, int checksum, zio_done_func_t *done, void *private, 747 int priority, enum zio_flag flags, boolean_t labels) 748 { 749 zio_t *zio; 750 751 ASSERT(vd->vdev_children == 0); 752 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 753 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 754 ASSERT3U(offset + size, <=, vd->vdev_psize); 755 756 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 757 ZIO_TYPE_READ, priority, flags, vd, offset, NULL, 758 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 759 760 zio->io_prop.zp_checksum = checksum; 761 762 return (zio); 763 } 764 765 zio_t * 766 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 767 void *data, int checksum, zio_done_func_t *done, void *private, 768 int priority, enum zio_flag flags, boolean_t labels) 769 { 770 zio_t *zio; 771 772 ASSERT(vd->vdev_children == 0); 773 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 774 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 775 ASSERT3U(offset + size, <=, vd->vdev_psize); 776 777 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 778 ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL, 779 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 780 781 zio->io_prop.zp_checksum = checksum; 782 783 if (zio_checksum_table[checksum].ci_eck) { 784 /* 785 * zec checksums are necessarily destructive -- they modify 786 * the end of the write buffer to hold the verifier/checksum. 787 * Therefore, we must make a local copy in case the data is 788 * being written to multiple places in parallel. 789 */ 790 void *wbuf = zio_buf_alloc(size); 791 bcopy(data, wbuf, size); 792 zio_push_transform(zio, wbuf, size, size, NULL); 793 } 794 795 return (zio); 796 } 797 798 /* 799 * Create a child I/O to do some work for us. 800 */ 801 zio_t * 802 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 803 void *data, uint64_t size, int type, int priority, enum zio_flag flags, 804 zio_done_func_t *done, void *private) 805 { 806 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 807 zio_t *zio; 808 809 ASSERT(vd->vdev_parent == 810 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 811 812 if (type == ZIO_TYPE_READ && bp != NULL) { 813 /* 814 * If we have the bp, then the child should perform the 815 * checksum and the parent need not. This pushes error 816 * detection as close to the leaves as possible and 817 * eliminates redundant checksums in the interior nodes. 818 */ 819 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 820 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 821 } 822 823 if (vd->vdev_children == 0) 824 offset += VDEV_LABEL_START_SIZE; 825 826 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 827 828 /* 829 * If we've decided to do a repair, the write is not speculative -- 830 * even if the original read was. 831 */ 832 if (flags & ZIO_FLAG_IO_REPAIR) 833 flags &= ~ZIO_FLAG_SPECULATIVE; 834 835 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 836 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 837 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 838 839 return (zio); 840 } 841 842 zio_t * 843 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 844 int type, int priority, enum zio_flag flags, 845 zio_done_func_t *done, void *private) 846 { 847 zio_t *zio; 848 849 ASSERT(vd->vdev_ops->vdev_op_leaf); 850 851 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 852 data, size, done, private, type, priority, 853 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY, 854 vd, offset, NULL, 855 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 856 857 return (zio); 858 } 859 860 void 861 zio_flush(zio_t *zio, vdev_t *vd) 862 { 863 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 864 NULL, NULL, ZIO_PRIORITY_NOW, 865 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 866 } 867 868 void 869 zio_shrink(zio_t *zio, uint64_t size) 870 { 871 ASSERT(zio->io_executor == NULL); 872 ASSERT(zio->io_orig_size == zio->io_size); 873 ASSERT(size <= zio->io_size); 874 875 /* 876 * We don't shrink for raidz because of problems with the 877 * reconstruction when reading back less than the block size. 878 * Note, BP_IS_RAIDZ() assumes no compression. 879 */ 880 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 881 if (!BP_IS_RAIDZ(zio->io_bp)) 882 zio->io_orig_size = zio->io_size = size; 883 } 884 885 /* 886 * ========================================================================== 887 * Prepare to read and write logical blocks 888 * ========================================================================== 889 */ 890 891 static int 892 zio_read_bp_init(zio_t *zio) 893 { 894 blkptr_t *bp = zio->io_bp; 895 896 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 897 zio->io_child_type == ZIO_CHILD_LOGICAL && 898 !(zio->io_flags & ZIO_FLAG_RAW)) { 899 uint64_t psize = BP_GET_PSIZE(bp); 900 void *cbuf = zio_buf_alloc(psize); 901 902 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 903 } 904 905 if (!dmu_ot[BP_GET_TYPE(bp)].ot_metadata && BP_GET_LEVEL(bp) == 0) 906 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 907 908 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 909 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 910 911 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 912 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 913 914 return (ZIO_PIPELINE_CONTINUE); 915 } 916 917 static int 918 zio_write_bp_init(zio_t *zio) 919 { 920 spa_t *spa = zio->io_spa; 921 zio_prop_t *zp = &zio->io_prop; 922 enum zio_compress compress = zp->zp_compress; 923 blkptr_t *bp = zio->io_bp; 924 uint64_t lsize = zio->io_size; 925 uint64_t psize = lsize; 926 int pass = 1; 927 928 /* 929 * If our children haven't all reached the ready stage, 930 * wait for them and then repeat this pipeline stage. 931 */ 932 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 933 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 934 return (ZIO_PIPELINE_STOP); 935 936 if (!IO_IS_ALLOCATING(zio)) 937 return (ZIO_PIPELINE_CONTINUE); 938 939 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 940 941 if (zio->io_bp_override) { 942 ASSERT(bp->blk_birth != zio->io_txg); 943 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 944 945 *bp = *zio->io_bp_override; 946 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 947 948 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 949 return (ZIO_PIPELINE_CONTINUE); 950 951 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup || 952 zp->zp_dedup_verify); 953 954 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 955 BP_SET_DEDUP(bp, 1); 956 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 957 return (ZIO_PIPELINE_CONTINUE); 958 } 959 zio->io_bp_override = NULL; 960 BP_ZERO(bp); 961 } 962 963 if (bp->blk_birth == zio->io_txg) { 964 /* 965 * We're rewriting an existing block, which means we're 966 * working on behalf of spa_sync(). For spa_sync() to 967 * converge, it must eventually be the case that we don't 968 * have to allocate new blocks. But compression changes 969 * the blocksize, which forces a reallocate, and makes 970 * convergence take longer. Therefore, after the first 971 * few passes, stop compressing to ensure convergence. 972 */ 973 pass = spa_sync_pass(spa); 974 975 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 976 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 977 ASSERT(!BP_GET_DEDUP(bp)); 978 979 if (pass > SYNC_PASS_DONT_COMPRESS) 980 compress = ZIO_COMPRESS_OFF; 981 982 /* Make sure someone doesn't change their mind on overwrites */ 983 ASSERT(MIN(zp->zp_copies + BP_IS_GANG(bp), 984 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 985 } 986 987 if (compress != ZIO_COMPRESS_OFF) { 988 void *cbuf = zio_buf_alloc(lsize); 989 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize); 990 if (psize == 0 || psize == lsize) { 991 compress = ZIO_COMPRESS_OFF; 992 zio_buf_free(cbuf, lsize); 993 } else { 994 ASSERT(psize < lsize); 995 zio_push_transform(zio, cbuf, psize, lsize, NULL); 996 } 997 } 998 999 /* 1000 * The final pass of spa_sync() must be all rewrites, but the first 1001 * few passes offer a trade-off: allocating blocks defers convergence, 1002 * but newly allocated blocks are sequential, so they can be written 1003 * to disk faster. Therefore, we allow the first few passes of 1004 * spa_sync() to allocate new blocks, but force rewrites after that. 1005 * There should only be a handful of blocks after pass 1 in any case. 1006 */ 1007 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == psize && 1008 pass > SYNC_PASS_REWRITE) { 1009 ASSERT(psize != 0); 1010 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1011 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1012 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1013 } else { 1014 BP_ZERO(bp); 1015 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1016 } 1017 1018 if (psize == 0) { 1019 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1020 } else { 1021 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1022 BP_SET_LSIZE(bp, lsize); 1023 BP_SET_PSIZE(bp, psize); 1024 BP_SET_COMPRESS(bp, compress); 1025 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1026 BP_SET_TYPE(bp, zp->zp_type); 1027 BP_SET_LEVEL(bp, zp->zp_level); 1028 BP_SET_DEDUP(bp, zp->zp_dedup); 1029 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1030 if (zp->zp_dedup) { 1031 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1032 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1033 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1034 } 1035 } 1036 1037 return (ZIO_PIPELINE_CONTINUE); 1038 } 1039 1040 static int 1041 zio_free_bp_init(zio_t *zio) 1042 { 1043 blkptr_t *bp = zio->io_bp; 1044 1045 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1046 if (BP_GET_DEDUP(bp)) 1047 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1048 } 1049 1050 return (ZIO_PIPELINE_CONTINUE); 1051 } 1052 1053 /* 1054 * ========================================================================== 1055 * Execute the I/O pipeline 1056 * ========================================================================== 1057 */ 1058 1059 static void 1060 zio_taskq_dispatch(zio_t *zio, enum zio_taskq_type q, boolean_t cutinline) 1061 { 1062 spa_t *spa = zio->io_spa; 1063 zio_type_t t = zio->io_type; 1064 int flags = TQ_SLEEP | (cutinline ? TQ_FRONT : 0); 1065 1066 /* 1067 * If we're a config writer or a probe, the normal issue and 1068 * interrupt threads may all be blocked waiting for the config lock. 1069 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1070 */ 1071 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1072 t = ZIO_TYPE_NULL; 1073 1074 /* 1075 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1076 */ 1077 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1078 t = ZIO_TYPE_NULL; 1079 1080 /* 1081 * If this is a high priority I/O, then use the high priority taskq. 1082 */ 1083 if (zio->io_priority == ZIO_PRIORITY_NOW && 1084 spa->spa_zio_taskq[t][q + 1] != NULL) 1085 q++; 1086 1087 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1088 (void) taskq_dispatch(spa->spa_zio_taskq[t][q], 1089 (task_func_t *)zio_execute, zio, flags); 1090 } 1091 1092 static boolean_t 1093 zio_taskq_member(zio_t *zio, enum zio_taskq_type q) 1094 { 1095 kthread_t *executor = zio->io_executor; 1096 spa_t *spa = zio->io_spa; 1097 1098 for (zio_type_t t = 0; t < ZIO_TYPES; t++) 1099 if (taskq_member(spa->spa_zio_taskq[t][q], executor)) 1100 return (B_TRUE); 1101 1102 return (B_FALSE); 1103 } 1104 1105 static int 1106 zio_issue_async(zio_t *zio) 1107 { 1108 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1109 1110 return (ZIO_PIPELINE_STOP); 1111 } 1112 1113 void 1114 zio_interrupt(zio_t *zio) 1115 { 1116 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1117 } 1118 1119 /* 1120 * Execute the I/O pipeline until one of the following occurs: 1121 * (1) the I/O completes; (2) the pipeline stalls waiting for 1122 * dependent child I/Os; (3) the I/O issues, so we're waiting 1123 * for an I/O completion interrupt; (4) the I/O is delegated by 1124 * vdev-level caching or aggregation; (5) the I/O is deferred 1125 * due to vdev-level queueing; (6) the I/O is handed off to 1126 * another thread. In all cases, the pipeline stops whenever 1127 * there's no CPU work; it never burns a thread in cv_wait(). 1128 * 1129 * There's no locking on io_stage because there's no legitimate way 1130 * for multiple threads to be attempting to process the same I/O. 1131 */ 1132 static zio_pipe_stage_t *zio_pipeline[]; 1133 1134 void 1135 zio_execute(zio_t *zio) 1136 { 1137 zio->io_executor = curthread; 1138 1139 while (zio->io_stage < ZIO_STAGE_DONE) { 1140 enum zio_stage pipeline = zio->io_pipeline; 1141 enum zio_stage stage = zio->io_stage; 1142 int rv; 1143 1144 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1145 ASSERT(ISP2(stage)); 1146 ASSERT(zio->io_stall == NULL); 1147 1148 do { 1149 stage <<= 1; 1150 } while ((stage & pipeline) == 0); 1151 1152 ASSERT(stage <= ZIO_STAGE_DONE); 1153 1154 /* 1155 * If we are in interrupt context and this pipeline stage 1156 * will grab a config lock that is held across I/O, 1157 * or may wait for an I/O that needs an interrupt thread 1158 * to complete, issue async to avoid deadlock. 1159 * 1160 * For VDEV_IO_START, we cut in line so that the io will 1161 * be sent to disk promptly. 1162 */ 1163 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1164 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1165 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1166 zio_requeue_io_start_cut_in_line : B_FALSE; 1167 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1168 return; 1169 } 1170 1171 zio->io_stage = stage; 1172 rv = zio_pipeline[highbit(stage) - 1](zio); 1173 1174 if (rv == ZIO_PIPELINE_STOP) 1175 return; 1176 1177 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1178 } 1179 } 1180 1181 /* 1182 * ========================================================================== 1183 * Initiate I/O, either sync or async 1184 * ========================================================================== 1185 */ 1186 int 1187 zio_wait(zio_t *zio) 1188 { 1189 int error; 1190 1191 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1192 ASSERT(zio->io_executor == NULL); 1193 1194 zio->io_waiter = curthread; 1195 1196 zio_execute(zio); 1197 1198 mutex_enter(&zio->io_lock); 1199 while (zio->io_executor != NULL) 1200 cv_wait(&zio->io_cv, &zio->io_lock); 1201 mutex_exit(&zio->io_lock); 1202 1203 error = zio->io_error; 1204 zio_destroy(zio); 1205 1206 return (error); 1207 } 1208 1209 void 1210 zio_nowait(zio_t *zio) 1211 { 1212 ASSERT(zio->io_executor == NULL); 1213 1214 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1215 zio_unique_parent(zio) == NULL) { 1216 /* 1217 * This is a logical async I/O with no parent to wait for it. 1218 * We add it to the spa_async_root_zio "Godfather" I/O which 1219 * will ensure they complete prior to unloading the pool. 1220 */ 1221 spa_t *spa = zio->io_spa; 1222 1223 zio_add_child(spa->spa_async_zio_root, zio); 1224 } 1225 1226 zio_execute(zio); 1227 } 1228 1229 /* 1230 * ========================================================================== 1231 * Reexecute or suspend/resume failed I/O 1232 * ========================================================================== 1233 */ 1234 1235 static void 1236 zio_reexecute(zio_t *pio) 1237 { 1238 zio_t *cio, *cio_next; 1239 1240 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1241 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1242 ASSERT(pio->io_gang_leader == NULL); 1243 ASSERT(pio->io_gang_tree == NULL); 1244 1245 pio->io_flags = pio->io_orig_flags; 1246 pio->io_stage = pio->io_orig_stage; 1247 pio->io_pipeline = pio->io_orig_pipeline; 1248 pio->io_reexecute = 0; 1249 pio->io_error = 0; 1250 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1251 pio->io_state[w] = 0; 1252 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1253 pio->io_child_error[c] = 0; 1254 1255 if (IO_IS_ALLOCATING(pio)) 1256 BP_ZERO(pio->io_bp); 1257 1258 /* 1259 * As we reexecute pio's children, new children could be created. 1260 * New children go to the head of pio's io_child_list, however, 1261 * so we will (correctly) not reexecute them. The key is that 1262 * the remainder of pio's io_child_list, from 'cio_next' onward, 1263 * cannot be affected by any side effects of reexecuting 'cio'. 1264 */ 1265 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) { 1266 cio_next = zio_walk_children(pio); 1267 mutex_enter(&pio->io_lock); 1268 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1269 pio->io_children[cio->io_child_type][w]++; 1270 mutex_exit(&pio->io_lock); 1271 zio_reexecute(cio); 1272 } 1273 1274 /* 1275 * Now that all children have been reexecuted, execute the parent. 1276 * We don't reexecute "The Godfather" I/O here as it's the 1277 * responsibility of the caller to wait on him. 1278 */ 1279 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) 1280 zio_execute(pio); 1281 } 1282 1283 void 1284 zio_suspend(spa_t *spa, zio_t *zio) 1285 { 1286 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1287 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1288 "failure and the failure mode property for this pool " 1289 "is set to panic.", spa_name(spa)); 1290 1291 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1292 1293 mutex_enter(&spa->spa_suspend_lock); 1294 1295 if (spa->spa_suspend_zio_root == NULL) 1296 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1297 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1298 ZIO_FLAG_GODFATHER); 1299 1300 spa->spa_suspended = B_TRUE; 1301 1302 if (zio != NULL) { 1303 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1304 ASSERT(zio != spa->spa_suspend_zio_root); 1305 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1306 ASSERT(zio_unique_parent(zio) == NULL); 1307 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1308 zio_add_child(spa->spa_suspend_zio_root, zio); 1309 } 1310 1311 mutex_exit(&spa->spa_suspend_lock); 1312 } 1313 1314 int 1315 zio_resume(spa_t *spa) 1316 { 1317 zio_t *pio; 1318 1319 /* 1320 * Reexecute all previously suspended i/o. 1321 */ 1322 mutex_enter(&spa->spa_suspend_lock); 1323 spa->spa_suspended = B_FALSE; 1324 cv_broadcast(&spa->spa_suspend_cv); 1325 pio = spa->spa_suspend_zio_root; 1326 spa->spa_suspend_zio_root = NULL; 1327 mutex_exit(&spa->spa_suspend_lock); 1328 1329 if (pio == NULL) 1330 return (0); 1331 1332 zio_reexecute(pio); 1333 return (zio_wait(pio)); 1334 } 1335 1336 void 1337 zio_resume_wait(spa_t *spa) 1338 { 1339 mutex_enter(&spa->spa_suspend_lock); 1340 while (spa_suspended(spa)) 1341 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1342 mutex_exit(&spa->spa_suspend_lock); 1343 } 1344 1345 /* 1346 * ========================================================================== 1347 * Gang blocks. 1348 * 1349 * A gang block is a collection of small blocks that looks to the DMU 1350 * like one large block. When zio_dva_allocate() cannot find a block 1351 * of the requested size, due to either severe fragmentation or the pool 1352 * being nearly full, it calls zio_write_gang_block() to construct the 1353 * block from smaller fragments. 1354 * 1355 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1356 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1357 * an indirect block: it's an array of block pointers. It consumes 1358 * only one sector and hence is allocatable regardless of fragmentation. 1359 * The gang header's bps point to its gang members, which hold the data. 1360 * 1361 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1362 * as the verifier to ensure uniqueness of the SHA256 checksum. 1363 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1364 * not the gang header. This ensures that data block signatures (needed for 1365 * deduplication) are independent of how the block is physically stored. 1366 * 1367 * Gang blocks can be nested: a gang member may itself be a gang block. 1368 * Thus every gang block is a tree in which root and all interior nodes are 1369 * gang headers, and the leaves are normal blocks that contain user data. 1370 * The root of the gang tree is called the gang leader. 1371 * 1372 * To perform any operation (read, rewrite, free, claim) on a gang block, 1373 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1374 * in the io_gang_tree field of the original logical i/o by recursively 1375 * reading the gang leader and all gang headers below it. This yields 1376 * an in-core tree containing the contents of every gang header and the 1377 * bps for every constituent of the gang block. 1378 * 1379 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1380 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1381 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1382 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1383 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1384 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1385 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1386 * of the gang header plus zio_checksum_compute() of the data to update the 1387 * gang header's blk_cksum as described above. 1388 * 1389 * The two-phase assemble/issue model solves the problem of partial failure -- 1390 * what if you'd freed part of a gang block but then couldn't read the 1391 * gang header for another part? Assembling the entire gang tree first 1392 * ensures that all the necessary gang header I/O has succeeded before 1393 * starting the actual work of free, claim, or write. Once the gang tree 1394 * is assembled, free and claim are in-memory operations that cannot fail. 1395 * 1396 * In the event that a gang write fails, zio_dva_unallocate() walks the 1397 * gang tree to immediately free (i.e. insert back into the space map) 1398 * everything we've allocated. This ensures that we don't get ENOSPC 1399 * errors during repeated suspend/resume cycles due to a flaky device. 1400 * 1401 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1402 * the gang tree, we won't modify the block, so we can safely defer the free 1403 * (knowing that the block is still intact). If we *can* assemble the gang 1404 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1405 * each constituent bp and we can allocate a new block on the next sync pass. 1406 * 1407 * In all cases, the gang tree allows complete recovery from partial failure. 1408 * ========================================================================== 1409 */ 1410 1411 static zio_t * 1412 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1413 { 1414 if (gn != NULL) 1415 return (pio); 1416 1417 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1418 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1419 &pio->io_bookmark)); 1420 } 1421 1422 zio_t * 1423 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1424 { 1425 zio_t *zio; 1426 1427 if (gn != NULL) { 1428 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1429 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1430 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1431 /* 1432 * As we rewrite each gang header, the pipeline will compute 1433 * a new gang block header checksum for it; but no one will 1434 * compute a new data checksum, so we do that here. The one 1435 * exception is the gang leader: the pipeline already computed 1436 * its data checksum because that stage precedes gang assembly. 1437 * (Presently, nothing actually uses interior data checksums; 1438 * this is just good hygiene.) 1439 */ 1440 if (gn != pio->io_gang_leader->io_gang_tree) { 1441 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1442 data, BP_GET_PSIZE(bp)); 1443 } 1444 /* 1445 * If we are here to damage data for testing purposes, 1446 * leave the GBH alone so that we can detect the damage. 1447 */ 1448 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1449 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1450 } else { 1451 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1452 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1453 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1454 } 1455 1456 return (zio); 1457 } 1458 1459 /* ARGSUSED */ 1460 zio_t * 1461 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1462 { 1463 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1464 ZIO_GANG_CHILD_FLAGS(pio))); 1465 } 1466 1467 /* ARGSUSED */ 1468 zio_t * 1469 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1470 { 1471 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1472 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1473 } 1474 1475 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1476 NULL, 1477 zio_read_gang, 1478 zio_rewrite_gang, 1479 zio_free_gang, 1480 zio_claim_gang, 1481 NULL 1482 }; 1483 1484 static void zio_gang_tree_assemble_done(zio_t *zio); 1485 1486 static zio_gang_node_t * 1487 zio_gang_node_alloc(zio_gang_node_t **gnpp) 1488 { 1489 zio_gang_node_t *gn; 1490 1491 ASSERT(*gnpp == NULL); 1492 1493 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1494 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1495 *gnpp = gn; 1496 1497 return (gn); 1498 } 1499 1500 static void 1501 zio_gang_node_free(zio_gang_node_t **gnpp) 1502 { 1503 zio_gang_node_t *gn = *gnpp; 1504 1505 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1506 ASSERT(gn->gn_child[g] == NULL); 1507 1508 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1509 kmem_free(gn, sizeof (*gn)); 1510 *gnpp = NULL; 1511 } 1512 1513 static void 1514 zio_gang_tree_free(zio_gang_node_t **gnpp) 1515 { 1516 zio_gang_node_t *gn = *gnpp; 1517 1518 if (gn == NULL) 1519 return; 1520 1521 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1522 zio_gang_tree_free(&gn->gn_child[g]); 1523 1524 zio_gang_node_free(gnpp); 1525 } 1526 1527 static void 1528 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1529 { 1530 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1531 1532 ASSERT(gio->io_gang_leader == gio); 1533 ASSERT(BP_IS_GANG(bp)); 1534 1535 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1536 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1537 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1538 } 1539 1540 static void 1541 zio_gang_tree_assemble_done(zio_t *zio) 1542 { 1543 zio_t *gio = zio->io_gang_leader; 1544 zio_gang_node_t *gn = zio->io_private; 1545 blkptr_t *bp = zio->io_bp; 1546 1547 ASSERT(gio == zio_unique_parent(zio)); 1548 ASSERT(zio->io_child_count == 0); 1549 1550 if (zio->io_error) 1551 return; 1552 1553 if (BP_SHOULD_BYTESWAP(bp)) 1554 byteswap_uint64_array(zio->io_data, zio->io_size); 1555 1556 ASSERT(zio->io_data == gn->gn_gbh); 1557 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1558 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1559 1560 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1561 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1562 if (!BP_IS_GANG(gbp)) 1563 continue; 1564 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1565 } 1566 } 1567 1568 static void 1569 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1570 { 1571 zio_t *gio = pio->io_gang_leader; 1572 zio_t *zio; 1573 1574 ASSERT(BP_IS_GANG(bp) == !!gn); 1575 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1576 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1577 1578 /* 1579 * If you're a gang header, your data is in gn->gn_gbh. 1580 * If you're a gang member, your data is in 'data' and gn == NULL. 1581 */ 1582 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1583 1584 if (gn != NULL) { 1585 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1586 1587 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1588 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1589 if (BP_IS_HOLE(gbp)) 1590 continue; 1591 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1592 data = (char *)data + BP_GET_PSIZE(gbp); 1593 } 1594 } 1595 1596 if (gn == gio->io_gang_tree) 1597 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1598 1599 if (zio != pio) 1600 zio_nowait(zio); 1601 } 1602 1603 static int 1604 zio_gang_assemble(zio_t *zio) 1605 { 1606 blkptr_t *bp = zio->io_bp; 1607 1608 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 1609 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1610 1611 zio->io_gang_leader = zio; 1612 1613 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 1614 1615 return (ZIO_PIPELINE_CONTINUE); 1616 } 1617 1618 static int 1619 zio_gang_issue(zio_t *zio) 1620 { 1621 blkptr_t *bp = zio->io_bp; 1622 1623 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 1624 return (ZIO_PIPELINE_STOP); 1625 1626 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 1627 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1628 1629 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 1630 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 1631 else 1632 zio_gang_tree_free(&zio->io_gang_tree); 1633 1634 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1635 1636 return (ZIO_PIPELINE_CONTINUE); 1637 } 1638 1639 static void 1640 zio_write_gang_member_ready(zio_t *zio) 1641 { 1642 zio_t *pio = zio_unique_parent(zio); 1643 zio_t *gio = zio->io_gang_leader; 1644 dva_t *cdva = zio->io_bp->blk_dva; 1645 dva_t *pdva = pio->io_bp->blk_dva; 1646 uint64_t asize; 1647 1648 if (BP_IS_HOLE(zio->io_bp)) 1649 return; 1650 1651 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 1652 1653 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 1654 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 1655 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 1656 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 1657 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1658 1659 mutex_enter(&pio->io_lock); 1660 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 1661 ASSERT(DVA_GET_GANG(&pdva[d])); 1662 asize = DVA_GET_ASIZE(&pdva[d]); 1663 asize += DVA_GET_ASIZE(&cdva[d]); 1664 DVA_SET_ASIZE(&pdva[d], asize); 1665 } 1666 mutex_exit(&pio->io_lock); 1667 } 1668 1669 static int 1670 zio_write_gang_block(zio_t *pio) 1671 { 1672 spa_t *spa = pio->io_spa; 1673 blkptr_t *bp = pio->io_bp; 1674 zio_t *gio = pio->io_gang_leader; 1675 zio_t *zio; 1676 zio_gang_node_t *gn, **gnpp; 1677 zio_gbh_phys_t *gbh; 1678 uint64_t txg = pio->io_txg; 1679 uint64_t resid = pio->io_size; 1680 uint64_t lsize; 1681 int copies = gio->io_prop.zp_copies; 1682 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 1683 zio_prop_t zp; 1684 int error; 1685 1686 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE, 1687 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, 1688 METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER); 1689 if (error) { 1690 pio->io_error = error; 1691 return (ZIO_PIPELINE_CONTINUE); 1692 } 1693 1694 if (pio == gio) { 1695 gnpp = &gio->io_gang_tree; 1696 } else { 1697 gnpp = pio->io_private; 1698 ASSERT(pio->io_ready == zio_write_gang_member_ready); 1699 } 1700 1701 gn = zio_gang_node_alloc(gnpp); 1702 gbh = gn->gn_gbh; 1703 bzero(gbh, SPA_GANGBLOCKSIZE); 1704 1705 /* 1706 * Create the gang header. 1707 */ 1708 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 1709 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1710 1711 /* 1712 * Create and nowait the gang children. 1713 */ 1714 for (int g = 0; resid != 0; resid -= lsize, g++) { 1715 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 1716 SPA_MINBLOCKSIZE); 1717 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 1718 1719 zp.zp_checksum = gio->io_prop.zp_checksum; 1720 zp.zp_compress = ZIO_COMPRESS_OFF; 1721 zp.zp_type = DMU_OT_NONE; 1722 zp.zp_level = 0; 1723 zp.zp_copies = gio->io_prop.zp_copies; 1724 zp.zp_dedup = 0; 1725 zp.zp_dedup_verify = 0; 1726 1727 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 1728 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 1729 zio_write_gang_member_ready, NULL, &gn->gn_child[g], 1730 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1731 &pio->io_bookmark)); 1732 } 1733 1734 /* 1735 * Set pio's pipeline to just wait for zio to finish. 1736 */ 1737 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1738 1739 zio_nowait(zio); 1740 1741 return (ZIO_PIPELINE_CONTINUE); 1742 } 1743 1744 /* 1745 * ========================================================================== 1746 * Dedup 1747 * ========================================================================== 1748 */ 1749 static void 1750 zio_ddt_child_read_done(zio_t *zio) 1751 { 1752 blkptr_t *bp = zio->io_bp; 1753 ddt_entry_t *dde = zio->io_private; 1754 ddt_phys_t *ddp; 1755 zio_t *pio = zio_unique_parent(zio); 1756 1757 mutex_enter(&pio->io_lock); 1758 ddp = ddt_phys_select(dde, bp); 1759 if (zio->io_error == 0) 1760 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 1761 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 1762 dde->dde_repair_data = zio->io_data; 1763 else 1764 zio_buf_free(zio->io_data, zio->io_size); 1765 mutex_exit(&pio->io_lock); 1766 } 1767 1768 static int 1769 zio_ddt_read_start(zio_t *zio) 1770 { 1771 blkptr_t *bp = zio->io_bp; 1772 1773 ASSERT(BP_GET_DEDUP(bp)); 1774 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 1775 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1776 1777 if (zio->io_child_error[ZIO_CHILD_DDT]) { 1778 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1779 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 1780 ddt_phys_t *ddp = dde->dde_phys; 1781 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 1782 blkptr_t blk; 1783 1784 ASSERT(zio->io_vsd == NULL); 1785 zio->io_vsd = dde; 1786 1787 if (ddp_self == NULL) 1788 return (ZIO_PIPELINE_CONTINUE); 1789 1790 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1791 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 1792 continue; 1793 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 1794 &blk); 1795 zio_nowait(zio_read(zio, zio->io_spa, &blk, 1796 zio_buf_alloc(zio->io_size), zio->io_size, 1797 zio_ddt_child_read_done, dde, zio->io_priority, 1798 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 1799 &zio->io_bookmark)); 1800 } 1801 return (ZIO_PIPELINE_CONTINUE); 1802 } 1803 1804 zio_nowait(zio_read(zio, zio->io_spa, bp, 1805 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 1806 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 1807 1808 return (ZIO_PIPELINE_CONTINUE); 1809 } 1810 1811 static int 1812 zio_ddt_read_done(zio_t *zio) 1813 { 1814 blkptr_t *bp = zio->io_bp; 1815 1816 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 1817 return (ZIO_PIPELINE_STOP); 1818 1819 ASSERT(BP_GET_DEDUP(bp)); 1820 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 1821 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1822 1823 if (zio->io_child_error[ZIO_CHILD_DDT]) { 1824 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1825 ddt_entry_t *dde = zio->io_vsd; 1826 if (ddt == NULL) { 1827 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 1828 return (ZIO_PIPELINE_CONTINUE); 1829 } 1830 if (dde == NULL) { 1831 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 1832 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1833 return (ZIO_PIPELINE_STOP); 1834 } 1835 if (dde->dde_repair_data != NULL) { 1836 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 1837 zio->io_child_error[ZIO_CHILD_DDT] = 0; 1838 } 1839 ddt_repair_done(ddt, dde); 1840 zio->io_vsd = NULL; 1841 } 1842 1843 ASSERT(zio->io_vsd == NULL); 1844 1845 return (ZIO_PIPELINE_CONTINUE); 1846 } 1847 1848 static boolean_t 1849 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 1850 { 1851 spa_t *spa = zio->io_spa; 1852 1853 /* 1854 * Note: we compare the original data, not the transformed data, 1855 * because when zio->io_bp is an override bp, we will not have 1856 * pushed the I/O transforms. That's an important optimization 1857 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 1858 */ 1859 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 1860 zio_t *lio = dde->dde_lead_zio[p]; 1861 1862 if (lio != NULL) { 1863 return (lio->io_orig_size != zio->io_orig_size || 1864 bcmp(zio->io_orig_data, lio->io_orig_data, 1865 zio->io_orig_size) != 0); 1866 } 1867 } 1868 1869 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 1870 ddt_phys_t *ddp = &dde->dde_phys[p]; 1871 1872 if (ddp->ddp_phys_birth != 0) { 1873 arc_buf_t *abuf = NULL; 1874 uint32_t aflags = ARC_WAIT; 1875 blkptr_t blk = *zio->io_bp; 1876 int error; 1877 1878 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 1879 1880 ddt_exit(ddt); 1881 1882 error = arc_read_nolock(NULL, spa, &blk, 1883 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 1884 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1885 &aflags, &zio->io_bookmark); 1886 1887 if (error == 0) { 1888 if (arc_buf_size(abuf) != zio->io_orig_size || 1889 bcmp(abuf->b_data, zio->io_orig_data, 1890 zio->io_orig_size) != 0) 1891 error = EEXIST; 1892 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 1893 } 1894 1895 ddt_enter(ddt); 1896 return (error != 0); 1897 } 1898 } 1899 1900 return (B_FALSE); 1901 } 1902 1903 static void 1904 zio_ddt_child_write_ready(zio_t *zio) 1905 { 1906 int p = zio->io_prop.zp_copies; 1907 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 1908 ddt_entry_t *dde = zio->io_private; 1909 ddt_phys_t *ddp = &dde->dde_phys[p]; 1910 zio_t *pio; 1911 1912 if (zio->io_error) 1913 return; 1914 1915 ddt_enter(ddt); 1916 1917 ASSERT(dde->dde_lead_zio[p] == zio); 1918 1919 ddt_phys_fill(ddp, zio->io_bp); 1920 1921 while ((pio = zio_walk_parents(zio)) != NULL) 1922 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 1923 1924 ddt_exit(ddt); 1925 } 1926 1927 static void 1928 zio_ddt_child_write_done(zio_t *zio) 1929 { 1930 int p = zio->io_prop.zp_copies; 1931 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 1932 ddt_entry_t *dde = zio->io_private; 1933 ddt_phys_t *ddp = &dde->dde_phys[p]; 1934 1935 ddt_enter(ddt); 1936 1937 ASSERT(ddp->ddp_refcnt == 0); 1938 ASSERT(dde->dde_lead_zio[p] == zio); 1939 dde->dde_lead_zio[p] = NULL; 1940 1941 if (zio->io_error == 0) { 1942 while (zio_walk_parents(zio) != NULL) 1943 ddt_phys_addref(ddp); 1944 } else { 1945 ddt_phys_clear(ddp); 1946 } 1947 1948 ddt_exit(ddt); 1949 } 1950 1951 static void 1952 zio_ddt_ditto_write_done(zio_t *zio) 1953 { 1954 int p = DDT_PHYS_DITTO; 1955 zio_prop_t *zp = &zio->io_prop; 1956 blkptr_t *bp = zio->io_bp; 1957 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1958 ddt_entry_t *dde = zio->io_private; 1959 ddt_phys_t *ddp = &dde->dde_phys[p]; 1960 ddt_key_t *ddk = &dde->dde_key; 1961 1962 ddt_enter(ddt); 1963 1964 ASSERT(ddp->ddp_refcnt == 0); 1965 ASSERT(dde->dde_lead_zio[p] == zio); 1966 dde->dde_lead_zio[p] = NULL; 1967 1968 if (zio->io_error == 0) { 1969 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 1970 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 1971 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 1972 if (ddp->ddp_phys_birth != 0) 1973 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 1974 ddt_phys_fill(ddp, bp); 1975 } 1976 1977 ddt_exit(ddt); 1978 } 1979 1980 static int 1981 zio_ddt_write(zio_t *zio) 1982 { 1983 spa_t *spa = zio->io_spa; 1984 blkptr_t *bp = zio->io_bp; 1985 uint64_t txg = zio->io_txg; 1986 zio_prop_t *zp = &zio->io_prop; 1987 int p = zp->zp_copies; 1988 int ditto_copies; 1989 zio_t *cio = NULL; 1990 zio_t *dio = NULL; 1991 ddt_t *ddt = ddt_select(spa, bp); 1992 ddt_entry_t *dde; 1993 ddt_phys_t *ddp; 1994 1995 ASSERT(BP_GET_DEDUP(bp)); 1996 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 1997 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 1998 1999 ddt_enter(ddt); 2000 dde = ddt_lookup(ddt, bp, B_TRUE); 2001 ddp = &dde->dde_phys[p]; 2002 2003 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2004 /* 2005 * If we're using a weak checksum, upgrade to a strong checksum 2006 * and try again. If we're already using a strong checksum, 2007 * we can't resolve it, so just convert to an ordinary write. 2008 * (And automatically e-mail a paper to Nature?) 2009 */ 2010 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) { 2011 zp->zp_checksum = spa_dedup_checksum(spa); 2012 zio_pop_transforms(zio); 2013 zio->io_stage = ZIO_STAGE_OPEN; 2014 BP_ZERO(bp); 2015 } else { 2016 zp->zp_dedup = 0; 2017 } 2018 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2019 ddt_exit(ddt); 2020 return (ZIO_PIPELINE_CONTINUE); 2021 } 2022 2023 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2024 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2025 2026 if (ditto_copies > ddt_ditto_copies_present(dde) && 2027 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2028 zio_prop_t czp = *zp; 2029 2030 czp.zp_copies = ditto_copies; 2031 2032 /* 2033 * If we arrived here with an override bp, we won't have run 2034 * the transform stack, so we won't have the data we need to 2035 * generate a child i/o. So, toss the override bp and restart. 2036 * This is safe, because using the override bp is just an 2037 * optimization; and it's rare, so the cost doesn't matter. 2038 */ 2039 if (zio->io_bp_override) { 2040 zio_pop_transforms(zio); 2041 zio->io_stage = ZIO_STAGE_OPEN; 2042 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2043 zio->io_bp_override = NULL; 2044 BP_ZERO(bp); 2045 ddt_exit(ddt); 2046 return (ZIO_PIPELINE_CONTINUE); 2047 } 2048 2049 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2050 zio->io_orig_size, &czp, NULL, 2051 zio_ddt_ditto_write_done, dde, zio->io_priority, 2052 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2053 2054 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2055 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2056 } 2057 2058 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2059 if (ddp->ddp_phys_birth != 0) 2060 ddt_bp_fill(ddp, bp, txg); 2061 if (dde->dde_lead_zio[p] != NULL) 2062 zio_add_child(zio, dde->dde_lead_zio[p]); 2063 else 2064 ddt_phys_addref(ddp); 2065 } else if (zio->io_bp_override) { 2066 ASSERT(bp->blk_birth == txg); 2067 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2068 ddt_phys_fill(ddp, bp); 2069 ddt_phys_addref(ddp); 2070 } else { 2071 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2072 zio->io_orig_size, zp, zio_ddt_child_write_ready, 2073 zio_ddt_child_write_done, dde, zio->io_priority, 2074 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2075 2076 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2077 dde->dde_lead_zio[p] = cio; 2078 } 2079 2080 ddt_exit(ddt); 2081 2082 if (cio) 2083 zio_nowait(cio); 2084 if (dio) 2085 zio_nowait(dio); 2086 2087 return (ZIO_PIPELINE_CONTINUE); 2088 } 2089 2090 ddt_entry_t *freedde; /* for debugging */ 2091 2092 static int 2093 zio_ddt_free(zio_t *zio) 2094 { 2095 spa_t *spa = zio->io_spa; 2096 blkptr_t *bp = zio->io_bp; 2097 ddt_t *ddt = ddt_select(spa, bp); 2098 ddt_entry_t *dde; 2099 ddt_phys_t *ddp; 2100 2101 ASSERT(BP_GET_DEDUP(bp)); 2102 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2103 2104 ddt_enter(ddt); 2105 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2106 ddp = ddt_phys_select(dde, bp); 2107 ddt_phys_decref(ddp); 2108 ddt_exit(ddt); 2109 2110 return (ZIO_PIPELINE_CONTINUE); 2111 } 2112 2113 /* 2114 * ========================================================================== 2115 * Allocate and free blocks 2116 * ========================================================================== 2117 */ 2118 static int 2119 zio_dva_allocate(zio_t *zio) 2120 { 2121 spa_t *spa = zio->io_spa; 2122 metaslab_class_t *mc = spa_normal_class(spa); 2123 blkptr_t *bp = zio->io_bp; 2124 int error; 2125 int flags = 0; 2126 2127 if (zio->io_gang_leader == NULL) { 2128 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2129 zio->io_gang_leader = zio; 2130 } 2131 2132 ASSERT(BP_IS_HOLE(bp)); 2133 ASSERT3U(BP_GET_NDVAS(bp), ==, 0); 2134 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2135 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2136 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2137 2138 /* 2139 * The dump device does not support gang blocks so allocation on 2140 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid 2141 * the "fast" gang feature. 2142 */ 2143 flags |= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0; 2144 flags |= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ? 2145 METASLAB_GANG_CHILD : 0; 2146 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2147 zio->io_prop.zp_copies, zio->io_txg, NULL, flags); 2148 2149 if (error) { 2150 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2151 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2152 error); 2153 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2154 return (zio_write_gang_block(zio)); 2155 zio->io_error = error; 2156 } 2157 2158 return (ZIO_PIPELINE_CONTINUE); 2159 } 2160 2161 static int 2162 zio_dva_free(zio_t *zio) 2163 { 2164 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2165 2166 return (ZIO_PIPELINE_CONTINUE); 2167 } 2168 2169 static int 2170 zio_dva_claim(zio_t *zio) 2171 { 2172 int error; 2173 2174 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2175 if (error) 2176 zio->io_error = error; 2177 2178 return (ZIO_PIPELINE_CONTINUE); 2179 } 2180 2181 /* 2182 * Undo an allocation. This is used by zio_done() when an I/O fails 2183 * and we want to give back the block we just allocated. 2184 * This handles both normal blocks and gang blocks. 2185 */ 2186 static void 2187 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2188 { 2189 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2190 ASSERT(zio->io_bp_override == NULL); 2191 2192 if (!BP_IS_HOLE(bp)) 2193 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2194 2195 if (gn != NULL) { 2196 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2197 zio_dva_unallocate(zio, gn->gn_child[g], 2198 &gn->gn_gbh->zg_blkptr[g]); 2199 } 2200 } 2201 } 2202 2203 /* 2204 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2205 */ 2206 int 2207 zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2208 uint64_t size, boolean_t use_slog) 2209 { 2210 int error = 1; 2211 2212 ASSERT(txg > spa_syncing_txg(spa)); 2213 2214 if (use_slog) 2215 error = metaslab_alloc(spa, spa_log_class(spa), size, 2216 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID); 2217 2218 if (error) 2219 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2220 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID); 2221 2222 if (error == 0) { 2223 BP_SET_LSIZE(new_bp, size); 2224 BP_SET_PSIZE(new_bp, size); 2225 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2226 BP_SET_CHECKSUM(new_bp, 2227 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2228 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2229 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2230 BP_SET_LEVEL(new_bp, 0); 2231 BP_SET_DEDUP(new_bp, 0); 2232 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2233 } 2234 2235 return (error); 2236 } 2237 2238 /* 2239 * Free an intent log block. 2240 */ 2241 void 2242 zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2243 { 2244 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2245 ASSERT(!BP_IS_GANG(bp)); 2246 2247 zio_free(spa, txg, bp); 2248 } 2249 2250 /* 2251 * ========================================================================== 2252 * Read and write to physical devices 2253 * ========================================================================== 2254 */ 2255 static int 2256 zio_vdev_io_start(zio_t *zio) 2257 { 2258 vdev_t *vd = zio->io_vd; 2259 uint64_t align; 2260 spa_t *spa = zio->io_spa; 2261 2262 ASSERT(zio->io_error == 0); 2263 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2264 2265 if (vd == NULL) { 2266 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2267 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2268 2269 /* 2270 * The mirror_ops handle multiple DVAs in a single BP. 2271 */ 2272 return (vdev_mirror_ops.vdev_op_io_start(zio)); 2273 } 2274 2275 /* 2276 * We keep track of time-sensitive I/Os so that the scan thread 2277 * can quickly react to certain workloads. In particular, we care 2278 * about non-scrubbing, top-level reads and writes with the following 2279 * characteristics: 2280 * - synchronous writes of user data to non-slog devices 2281 * - any reads of user data 2282 * When these conditions are met, adjust the timestamp of spa_last_io 2283 * which allows the scan thread to adjust its workload accordingly. 2284 */ 2285 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2286 vd == vd->vdev_top && !vd->vdev_islog && 2287 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2288 zio->io_txg != spa_syncing_txg(spa)) { 2289 uint64_t old = spa->spa_last_io; 2290 uint64_t new = ddi_get_lbolt64(); 2291 if (old != new) 2292 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2293 } 2294 2295 align = 1ULL << vd->vdev_top->vdev_ashift; 2296 2297 if (P2PHASE(zio->io_size, align) != 0) { 2298 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2299 char *abuf = zio_buf_alloc(asize); 2300 ASSERT(vd == vd->vdev_top); 2301 if (zio->io_type == ZIO_TYPE_WRITE) { 2302 bcopy(zio->io_data, abuf, zio->io_size); 2303 bzero(abuf + zio->io_size, asize - zio->io_size); 2304 } 2305 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 2306 } 2307 2308 ASSERT(P2PHASE(zio->io_offset, align) == 0); 2309 ASSERT(P2PHASE(zio->io_size, align) == 0); 2310 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 2311 2312 /* 2313 * If this is a repair I/O, and there's no self-healing involved -- 2314 * that is, we're just resilvering what we expect to resilver -- 2315 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2316 * This prevents spurious resilvering with nested replication. 2317 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2318 * A is out of date, we'll read from C+D, then use the data to 2319 * resilver A+B -- but we don't actually want to resilver B, just A. 2320 * The top-level mirror has no way to know this, so instead we just 2321 * discard unnecessary repairs as we work our way down the vdev tree. 2322 * The same logic applies to any form of nested replication: 2323 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2324 */ 2325 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2326 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2327 zio->io_txg != 0 && /* not a delegated i/o */ 2328 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2329 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2330 zio_vdev_io_bypass(zio); 2331 return (ZIO_PIPELINE_CONTINUE); 2332 } 2333 2334 if (vd->vdev_ops->vdev_op_leaf && 2335 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2336 2337 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio) == 0) 2338 return (ZIO_PIPELINE_CONTINUE); 2339 2340 if ((zio = vdev_queue_io(zio)) == NULL) 2341 return (ZIO_PIPELINE_STOP); 2342 2343 if (!vdev_accessible(vd, zio)) { 2344 zio->io_error = ENXIO; 2345 zio_interrupt(zio); 2346 return (ZIO_PIPELINE_STOP); 2347 } 2348 } 2349 2350 return (vd->vdev_ops->vdev_op_io_start(zio)); 2351 } 2352 2353 static int 2354 zio_vdev_io_done(zio_t *zio) 2355 { 2356 vdev_t *vd = zio->io_vd; 2357 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 2358 boolean_t unexpected_error = B_FALSE; 2359 2360 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2361 return (ZIO_PIPELINE_STOP); 2362 2363 ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); 2364 2365 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) { 2366 2367 vdev_queue_io_done(zio); 2368 2369 if (zio->io_type == ZIO_TYPE_WRITE) 2370 vdev_cache_write(zio); 2371 2372 if (zio_injection_enabled && zio->io_error == 0) 2373 zio->io_error = zio_handle_device_injection(vd, 2374 zio, EIO); 2375 2376 if (zio_injection_enabled && zio->io_error == 0) 2377 zio->io_error = zio_handle_label_injection(zio, EIO); 2378 2379 if (zio->io_error) { 2380 if (!vdev_accessible(vd, zio)) { 2381 zio->io_error = ENXIO; 2382 } else { 2383 unexpected_error = B_TRUE; 2384 } 2385 } 2386 } 2387 2388 ops->vdev_op_io_done(zio); 2389 2390 if (unexpected_error) 2391 VERIFY(vdev_probe(vd, zio) == NULL); 2392 2393 return (ZIO_PIPELINE_CONTINUE); 2394 } 2395 2396 /* 2397 * For non-raidz ZIOs, we can just copy aside the bad data read from the 2398 * disk, and use that to finish the checksum ereport later. 2399 */ 2400 static void 2401 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 2402 const void *good_buf) 2403 { 2404 /* no processing needed */ 2405 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 2406 } 2407 2408 /*ARGSUSED*/ 2409 void 2410 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 2411 { 2412 void *buf = zio_buf_alloc(zio->io_size); 2413 2414 bcopy(zio->io_data, buf, zio->io_size); 2415 2416 zcr->zcr_cbinfo = zio->io_size; 2417 zcr->zcr_cbdata = buf; 2418 zcr->zcr_finish = zio_vsd_default_cksum_finish; 2419 zcr->zcr_free = zio_buf_free; 2420 } 2421 2422 static int 2423 zio_vdev_io_assess(zio_t *zio) 2424 { 2425 vdev_t *vd = zio->io_vd; 2426 2427 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2428 return (ZIO_PIPELINE_STOP); 2429 2430 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2431 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 2432 2433 if (zio->io_vsd != NULL) { 2434 zio->io_vsd_ops->vsd_free(zio); 2435 zio->io_vsd = NULL; 2436 } 2437 2438 if (zio_injection_enabled && zio->io_error == 0) 2439 zio->io_error = zio_handle_fault_injection(zio, EIO); 2440 2441 /* 2442 * If the I/O failed, determine whether we should attempt to retry it. 2443 * 2444 * On retry, we cut in line in the issue queue, since we don't want 2445 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 2446 */ 2447 if (zio->io_error && vd == NULL && 2448 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 2449 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 2450 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 2451 zio->io_error = 0; 2452 zio->io_flags |= ZIO_FLAG_IO_RETRY | 2453 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 2454 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 2455 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 2456 zio_requeue_io_start_cut_in_line); 2457 return (ZIO_PIPELINE_STOP); 2458 } 2459 2460 /* 2461 * If we got an error on a leaf device, convert it to ENXIO 2462 * if the device is not accessible at all. 2463 */ 2464 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 2465 !vdev_accessible(vd, zio)) 2466 zio->io_error = ENXIO; 2467 2468 /* 2469 * If we can't write to an interior vdev (mirror or RAID-Z), 2470 * set vdev_cant_write so that we stop trying to allocate from it. 2471 */ 2472 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 2473 vd != NULL && !vd->vdev_ops->vdev_op_leaf) 2474 vd->vdev_cant_write = B_TRUE; 2475 2476 if (zio->io_error) 2477 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2478 2479 return (ZIO_PIPELINE_CONTINUE); 2480 } 2481 2482 void 2483 zio_vdev_io_reissue(zio_t *zio) 2484 { 2485 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2486 ASSERT(zio->io_error == 0); 2487 2488 zio->io_stage >>= 1; 2489 } 2490 2491 void 2492 zio_vdev_io_redone(zio_t *zio) 2493 { 2494 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 2495 2496 zio->io_stage >>= 1; 2497 } 2498 2499 void 2500 zio_vdev_io_bypass(zio_t *zio) 2501 { 2502 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2503 ASSERT(zio->io_error == 0); 2504 2505 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 2506 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 2507 } 2508 2509 /* 2510 * ========================================================================== 2511 * Generate and verify checksums 2512 * ========================================================================== 2513 */ 2514 static int 2515 zio_checksum_generate(zio_t *zio) 2516 { 2517 blkptr_t *bp = zio->io_bp; 2518 enum zio_checksum checksum; 2519 2520 if (bp == NULL) { 2521 /* 2522 * This is zio_write_phys(). 2523 * We're either generating a label checksum, or none at all. 2524 */ 2525 checksum = zio->io_prop.zp_checksum; 2526 2527 if (checksum == ZIO_CHECKSUM_OFF) 2528 return (ZIO_PIPELINE_CONTINUE); 2529 2530 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 2531 } else { 2532 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 2533 ASSERT(!IO_IS_ALLOCATING(zio)); 2534 checksum = ZIO_CHECKSUM_GANG_HEADER; 2535 } else { 2536 checksum = BP_GET_CHECKSUM(bp); 2537 } 2538 } 2539 2540 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 2541 2542 return (ZIO_PIPELINE_CONTINUE); 2543 } 2544 2545 static int 2546 zio_checksum_verify(zio_t *zio) 2547 { 2548 zio_bad_cksum_t info; 2549 blkptr_t *bp = zio->io_bp; 2550 int error; 2551 2552 ASSERT(zio->io_vd != NULL); 2553 2554 if (bp == NULL) { 2555 /* 2556 * This is zio_read_phys(). 2557 * We're either verifying a label checksum, or nothing at all. 2558 */ 2559 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 2560 return (ZIO_PIPELINE_CONTINUE); 2561 2562 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 2563 } 2564 2565 if ((error = zio_checksum_error(zio, &info)) != 0) { 2566 zio->io_error = error; 2567 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 2568 zfs_ereport_start_checksum(zio->io_spa, 2569 zio->io_vd, zio, zio->io_offset, 2570 zio->io_size, NULL, &info); 2571 } 2572 } 2573 2574 return (ZIO_PIPELINE_CONTINUE); 2575 } 2576 2577 /* 2578 * Called by RAID-Z to ensure we don't compute the checksum twice. 2579 */ 2580 void 2581 zio_checksum_verified(zio_t *zio) 2582 { 2583 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 2584 } 2585 2586 /* 2587 * ========================================================================== 2588 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 2589 * An error of 0 indictes success. ENXIO indicates whole-device failure, 2590 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 2591 * indicate errors that are specific to one I/O, and most likely permanent. 2592 * Any other error is presumed to be worse because we weren't expecting it. 2593 * ========================================================================== 2594 */ 2595 int 2596 zio_worst_error(int e1, int e2) 2597 { 2598 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 2599 int r1, r2; 2600 2601 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 2602 if (e1 == zio_error_rank[r1]) 2603 break; 2604 2605 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 2606 if (e2 == zio_error_rank[r2]) 2607 break; 2608 2609 return (r1 > r2 ? e1 : e2); 2610 } 2611 2612 /* 2613 * ========================================================================== 2614 * I/O completion 2615 * ========================================================================== 2616 */ 2617 static int 2618 zio_ready(zio_t *zio) 2619 { 2620 blkptr_t *bp = zio->io_bp; 2621 zio_t *pio, *pio_next; 2622 2623 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 2624 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 2625 return (ZIO_PIPELINE_STOP); 2626 2627 if (zio->io_ready) { 2628 ASSERT(IO_IS_ALLOCATING(zio)); 2629 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2630 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 2631 2632 zio->io_ready(zio); 2633 } 2634 2635 if (bp != NULL && bp != &zio->io_bp_copy) 2636 zio->io_bp_copy = *bp; 2637 2638 if (zio->io_error) 2639 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2640 2641 mutex_enter(&zio->io_lock); 2642 zio->io_state[ZIO_WAIT_READY] = 1; 2643 pio = zio_walk_parents(zio); 2644 mutex_exit(&zio->io_lock); 2645 2646 /* 2647 * As we notify zio's parents, new parents could be added. 2648 * New parents go to the head of zio's io_parent_list, however, 2649 * so we will (correctly) not notify them. The remainder of zio's 2650 * io_parent_list, from 'pio_next' onward, cannot change because 2651 * all parents must wait for us to be done before they can be done. 2652 */ 2653 for (; pio != NULL; pio = pio_next) { 2654 pio_next = zio_walk_parents(zio); 2655 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 2656 } 2657 2658 if (zio->io_flags & ZIO_FLAG_NODATA) { 2659 if (BP_IS_GANG(bp)) { 2660 zio->io_flags &= ~ZIO_FLAG_NODATA; 2661 } else { 2662 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 2663 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2664 } 2665 } 2666 2667 if (zio_injection_enabled && 2668 zio->io_spa->spa_syncing_txg == zio->io_txg) 2669 zio_handle_ignored_writes(zio); 2670 2671 return (ZIO_PIPELINE_CONTINUE); 2672 } 2673 2674 static int 2675 zio_done(zio_t *zio) 2676 { 2677 spa_t *spa = zio->io_spa; 2678 zio_t *lio = zio->io_logical; 2679 blkptr_t *bp = zio->io_bp; 2680 vdev_t *vd = zio->io_vd; 2681 uint64_t psize = zio->io_size; 2682 zio_t *pio, *pio_next; 2683 2684 /* 2685 * If our children haven't all completed, 2686 * wait for them and then repeat this pipeline stage. 2687 */ 2688 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 2689 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 2690 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 2691 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 2692 return (ZIO_PIPELINE_STOP); 2693 2694 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 2695 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 2696 ASSERT(zio->io_children[c][w] == 0); 2697 2698 if (bp != NULL) { 2699 ASSERT(bp->blk_pad[0] == 0); 2700 ASSERT(bp->blk_pad[1] == 0); 2701 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 2702 (bp == zio_unique_parent(zio)->io_bp)); 2703 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 2704 zio->io_bp_override == NULL && 2705 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2706 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 2707 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 2708 ASSERT(BP_COUNT_GANG(bp) == 0 || 2709 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 2710 } 2711 } 2712 2713 /* 2714 * If there were child vdev/gang/ddt errors, they apply to us now. 2715 */ 2716 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 2717 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 2718 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 2719 2720 /* 2721 * If the I/O on the transformed data was successful, generate any 2722 * checksum reports now while we still have the transformed data. 2723 */ 2724 if (zio->io_error == 0) { 2725 while (zio->io_cksum_report != NULL) { 2726 zio_cksum_report_t *zcr = zio->io_cksum_report; 2727 uint64_t align = zcr->zcr_align; 2728 uint64_t asize = P2ROUNDUP(psize, align); 2729 char *abuf = zio->io_data; 2730 2731 if (asize != psize) { 2732 abuf = zio_buf_alloc(asize); 2733 bcopy(zio->io_data, abuf, psize); 2734 bzero(abuf + psize, asize - psize); 2735 } 2736 2737 zio->io_cksum_report = zcr->zcr_next; 2738 zcr->zcr_next = NULL; 2739 zcr->zcr_finish(zcr, abuf); 2740 zfs_ereport_free_checksum(zcr); 2741 2742 if (asize != psize) 2743 zio_buf_free(abuf, asize); 2744 } 2745 } 2746 2747 zio_pop_transforms(zio); /* note: may set zio->io_error */ 2748 2749 vdev_stat_update(zio, psize); 2750 2751 if (zio->io_error) { 2752 /* 2753 * If this I/O is attached to a particular vdev, 2754 * generate an error message describing the I/O failure 2755 * at the block level. We ignore these errors if the 2756 * device is currently unavailable. 2757 */ 2758 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 2759 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 2760 2761 if ((zio->io_error == EIO || !(zio->io_flags & 2762 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 2763 zio == lio) { 2764 /* 2765 * For logical I/O requests, tell the SPA to log the 2766 * error and generate a logical data ereport. 2767 */ 2768 spa_log_error(spa, zio); 2769 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 2770 0, 0); 2771 } 2772 } 2773 2774 if (zio->io_error && zio == lio) { 2775 /* 2776 * Determine whether zio should be reexecuted. This will 2777 * propagate all the way to the root via zio_notify_parent(). 2778 */ 2779 ASSERT(vd == NULL && bp != NULL); 2780 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2781 2782 if (IO_IS_ALLOCATING(zio) && 2783 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 2784 if (zio->io_error != ENOSPC) 2785 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 2786 else 2787 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2788 } 2789 2790 if ((zio->io_type == ZIO_TYPE_READ || 2791 zio->io_type == ZIO_TYPE_FREE) && 2792 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 2793 zio->io_error == ENXIO && 2794 spa_load_state(spa) == SPA_LOAD_NONE && 2795 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 2796 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2797 2798 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 2799 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2800 2801 /* 2802 * Here is a possibly good place to attempt to do 2803 * either combinatorial reconstruction or error correction 2804 * based on checksums. It also might be a good place 2805 * to send out preliminary ereports before we suspend 2806 * processing. 2807 */ 2808 } 2809 2810 /* 2811 * If there were logical child errors, they apply to us now. 2812 * We defer this until now to avoid conflating logical child 2813 * errors with errors that happened to the zio itself when 2814 * updating vdev stats and reporting FMA events above. 2815 */ 2816 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 2817 2818 if ((zio->io_error || zio->io_reexecute) && 2819 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 2820 !(zio->io_flags & ZIO_FLAG_IO_REWRITE)) 2821 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 2822 2823 zio_gang_tree_free(&zio->io_gang_tree); 2824 2825 /* 2826 * Godfather I/Os should never suspend. 2827 */ 2828 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 2829 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 2830 zio->io_reexecute = 0; 2831 2832 if (zio->io_reexecute) { 2833 /* 2834 * This is a logical I/O that wants to reexecute. 2835 * 2836 * Reexecute is top-down. When an i/o fails, if it's not 2837 * the root, it simply notifies its parent and sticks around. 2838 * The parent, seeing that it still has children in zio_done(), 2839 * does the same. This percolates all the way up to the root. 2840 * The root i/o will reexecute or suspend the entire tree. 2841 * 2842 * This approach ensures that zio_reexecute() honors 2843 * all the original i/o dependency relationships, e.g. 2844 * parents not executing until children are ready. 2845 */ 2846 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2847 2848 zio->io_gang_leader = NULL; 2849 2850 mutex_enter(&zio->io_lock); 2851 zio->io_state[ZIO_WAIT_DONE] = 1; 2852 mutex_exit(&zio->io_lock); 2853 2854 /* 2855 * "The Godfather" I/O monitors its children but is 2856 * not a true parent to them. It will track them through 2857 * the pipeline but severs its ties whenever they get into 2858 * trouble (e.g. suspended). This allows "The Godfather" 2859 * I/O to return status without blocking. 2860 */ 2861 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 2862 zio_link_t *zl = zio->io_walk_link; 2863 pio_next = zio_walk_parents(zio); 2864 2865 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 2866 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 2867 zio_remove_child(pio, zio, zl); 2868 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 2869 } 2870 } 2871 2872 if ((pio = zio_unique_parent(zio)) != NULL) { 2873 /* 2874 * We're not a root i/o, so there's nothing to do 2875 * but notify our parent. Don't propagate errors 2876 * upward since we haven't permanently failed yet. 2877 */ 2878 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 2879 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 2880 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 2881 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 2882 /* 2883 * We'd fail again if we reexecuted now, so suspend 2884 * until conditions improve (e.g. device comes online). 2885 */ 2886 zio_suspend(spa, zio); 2887 } else { 2888 /* 2889 * Reexecution is potentially a huge amount of work. 2890 * Hand it off to the otherwise-unused claim taskq. 2891 */ 2892 (void) taskq_dispatch( 2893 spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE], 2894 (task_func_t *)zio_reexecute, zio, TQ_SLEEP); 2895 } 2896 return (ZIO_PIPELINE_STOP); 2897 } 2898 2899 ASSERT(zio->io_child_count == 0); 2900 ASSERT(zio->io_reexecute == 0); 2901 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 2902 2903 /* 2904 * Report any checksum errors, since the I/O is complete. 2905 */ 2906 while (zio->io_cksum_report != NULL) { 2907 zio_cksum_report_t *zcr = zio->io_cksum_report; 2908 zio->io_cksum_report = zcr->zcr_next; 2909 zcr->zcr_next = NULL; 2910 zcr->zcr_finish(zcr, NULL); 2911 zfs_ereport_free_checksum(zcr); 2912 } 2913 2914 /* 2915 * It is the responsibility of the done callback to ensure that this 2916 * particular zio is no longer discoverable for adoption, and as 2917 * such, cannot acquire any new parents. 2918 */ 2919 if (zio->io_done) 2920 zio->io_done(zio); 2921 2922 mutex_enter(&zio->io_lock); 2923 zio->io_state[ZIO_WAIT_DONE] = 1; 2924 mutex_exit(&zio->io_lock); 2925 2926 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 2927 zio_link_t *zl = zio->io_walk_link; 2928 pio_next = zio_walk_parents(zio); 2929 zio_remove_child(pio, zio, zl); 2930 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 2931 } 2932 2933 if (zio->io_waiter != NULL) { 2934 mutex_enter(&zio->io_lock); 2935 zio->io_executor = NULL; 2936 cv_broadcast(&zio->io_cv); 2937 mutex_exit(&zio->io_lock); 2938 } else { 2939 zio_destroy(zio); 2940 } 2941 2942 return (ZIO_PIPELINE_STOP); 2943 } 2944 2945 /* 2946 * ========================================================================== 2947 * I/O pipeline definition 2948 * ========================================================================== 2949 */ 2950 static zio_pipe_stage_t *zio_pipeline[] = { 2951 NULL, 2952 zio_read_bp_init, 2953 zio_free_bp_init, 2954 zio_issue_async, 2955 zio_write_bp_init, 2956 zio_checksum_generate, 2957 zio_ddt_read_start, 2958 zio_ddt_read_done, 2959 zio_ddt_write, 2960 zio_ddt_free, 2961 zio_gang_assemble, 2962 zio_gang_issue, 2963 zio_dva_allocate, 2964 zio_dva_free, 2965 zio_dva_claim, 2966 zio_ready, 2967 zio_vdev_io_start, 2968 zio_vdev_io_done, 2969 zio_vdev_io_assess, 2970 zio_checksum_verify, 2971 zio_done 2972 }; 2973