1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2012 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 */ 26 27 #include <sys/zfs_context.h> 28 #include <sys/fm/fs/zfs.h> 29 #include <sys/spa.h> 30 #include <sys/txg.h> 31 #include <sys/spa_impl.h> 32 #include <sys/vdev_impl.h> 33 #include <sys/zio_impl.h> 34 #include <sys/zio_compress.h> 35 #include <sys/zio_checksum.h> 36 #include <sys/dmu_objset.h> 37 #include <sys/arc.h> 38 #include <sys/ddt.h> 39 40 /* 41 * ========================================================================== 42 * I/O priority table 43 * ========================================================================== 44 */ 45 uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = { 46 0, /* ZIO_PRIORITY_NOW */ 47 0, /* ZIO_PRIORITY_SYNC_READ */ 48 0, /* ZIO_PRIORITY_SYNC_WRITE */ 49 0, /* ZIO_PRIORITY_LOG_WRITE */ 50 1, /* ZIO_PRIORITY_CACHE_FILL */ 51 1, /* ZIO_PRIORITY_AGG */ 52 4, /* ZIO_PRIORITY_FREE */ 53 4, /* ZIO_PRIORITY_ASYNC_WRITE */ 54 6, /* ZIO_PRIORITY_ASYNC_READ */ 55 10, /* ZIO_PRIORITY_RESILVER */ 56 20, /* ZIO_PRIORITY_SCRUB */ 57 2, /* ZIO_PRIORITY_DDT_PREFETCH */ 58 }; 59 60 /* 61 * ========================================================================== 62 * I/O type descriptions 63 * ========================================================================== 64 */ 65 char *zio_type_name[ZIO_TYPES] = { 66 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 67 "zio_ioctl" 68 }; 69 70 /* 71 * ========================================================================== 72 * I/O kmem caches 73 * ========================================================================== 74 */ 75 kmem_cache_t *zio_cache; 76 kmem_cache_t *zio_link_cache; 77 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 78 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 79 80 #ifdef _KERNEL 81 extern vmem_t *zio_alloc_arena; 82 #endif 83 extern int zfs_mg_alloc_failures; 84 85 /* 86 * The following actions directly effect the spa's sync-to-convergence logic. 87 * The values below define the sync pass when we start performing the action. 88 * Care should be taken when changing these values as they directly impact 89 * spa_sync() performance. Tuning these values may introduce subtle performance 90 * pathologies and should only be done in the context of performance analysis. 91 * These tunables will eventually be removed and replaced with #defines once 92 * enough analysis has been done to determine optimal values. 93 * 94 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 95 * regular blocks are not deferred. 96 */ 97 int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 98 int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 99 int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 100 101 /* 102 * An allocating zio is one that either currently has the DVA allocate 103 * stage set or will have it later in its lifetime. 104 */ 105 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 106 107 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 108 109 #ifdef ZFS_DEBUG 110 int zio_buf_debug_limit = 16384; 111 #else 112 int zio_buf_debug_limit = 0; 113 #endif 114 115 void 116 zio_init(void) 117 { 118 size_t c; 119 vmem_t *data_alloc_arena = NULL; 120 121 #ifdef _KERNEL 122 data_alloc_arena = zio_alloc_arena; 123 #endif 124 zio_cache = kmem_cache_create("zio_cache", 125 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 126 zio_link_cache = kmem_cache_create("zio_link_cache", 127 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 128 129 /* 130 * For small buffers, we want a cache for each multiple of 131 * SPA_MINBLOCKSIZE. For medium-size buffers, we want a cache 132 * for each quarter-power of 2. For large buffers, we want 133 * a cache for each multiple of PAGESIZE. 134 */ 135 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 136 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 137 size_t p2 = size; 138 size_t align = 0; 139 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 140 141 while (p2 & (p2 - 1)) 142 p2 &= p2 - 1; 143 144 #ifndef _KERNEL 145 /* 146 * If we are using watchpoints, put each buffer on its own page, 147 * to eliminate the performance overhead of trapping to the 148 * kernel when modifying a non-watched buffer that shares the 149 * page with a watched buffer. 150 */ 151 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 152 continue; 153 #endif 154 if (size <= 4 * SPA_MINBLOCKSIZE) { 155 align = SPA_MINBLOCKSIZE; 156 } else if (IS_P2ALIGNED(size, PAGESIZE)) { 157 align = PAGESIZE; 158 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 159 align = p2 >> 2; 160 } 161 162 if (align != 0) { 163 char name[36]; 164 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 165 zio_buf_cache[c] = kmem_cache_create(name, size, 166 align, NULL, NULL, NULL, NULL, NULL, cflags); 167 168 /* 169 * Since zio_data bufs do not appear in crash dumps, we 170 * pass KMC_NOTOUCH so that no allocator metadata is 171 * stored with the buffers. 172 */ 173 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 174 zio_data_buf_cache[c] = kmem_cache_create(name, size, 175 align, NULL, NULL, NULL, NULL, data_alloc_arena, 176 cflags | KMC_NOTOUCH); 177 } 178 } 179 180 while (--c != 0) { 181 ASSERT(zio_buf_cache[c] != NULL); 182 if (zio_buf_cache[c - 1] == NULL) 183 zio_buf_cache[c - 1] = zio_buf_cache[c]; 184 185 ASSERT(zio_data_buf_cache[c] != NULL); 186 if (zio_data_buf_cache[c - 1] == NULL) 187 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 188 } 189 190 /* 191 * The zio write taskqs have 1 thread per cpu, allow 1/2 of the taskqs 192 * to fail 3 times per txg or 8 failures, whichever is greater. 193 */ 194 zfs_mg_alloc_failures = MAX((3 * max_ncpus / 2), 8); 195 196 zio_inject_init(); 197 } 198 199 void 200 zio_fini(void) 201 { 202 size_t c; 203 kmem_cache_t *last_cache = NULL; 204 kmem_cache_t *last_data_cache = NULL; 205 206 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 207 if (zio_buf_cache[c] != last_cache) { 208 last_cache = zio_buf_cache[c]; 209 kmem_cache_destroy(zio_buf_cache[c]); 210 } 211 zio_buf_cache[c] = NULL; 212 213 if (zio_data_buf_cache[c] != last_data_cache) { 214 last_data_cache = zio_data_buf_cache[c]; 215 kmem_cache_destroy(zio_data_buf_cache[c]); 216 } 217 zio_data_buf_cache[c] = NULL; 218 } 219 220 kmem_cache_destroy(zio_link_cache); 221 kmem_cache_destroy(zio_cache); 222 223 zio_inject_fini(); 224 } 225 226 /* 227 * ========================================================================== 228 * Allocate and free I/O buffers 229 * ========================================================================== 230 */ 231 232 /* 233 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 234 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 235 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 236 * excess / transient data in-core during a crashdump. 237 */ 238 void * 239 zio_buf_alloc(size_t size) 240 { 241 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 242 243 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 244 245 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 246 } 247 248 /* 249 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 250 * crashdump if the kernel panics. This exists so that we will limit the amount 251 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 252 * of kernel heap dumped to disk when the kernel panics) 253 */ 254 void * 255 zio_data_buf_alloc(size_t size) 256 { 257 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 258 259 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 260 261 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 262 } 263 264 void 265 zio_buf_free(void *buf, size_t size) 266 { 267 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 268 269 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 270 271 kmem_cache_free(zio_buf_cache[c], buf); 272 } 273 274 void 275 zio_data_buf_free(void *buf, size_t size) 276 { 277 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 278 279 ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 280 281 kmem_cache_free(zio_data_buf_cache[c], buf); 282 } 283 284 /* 285 * ========================================================================== 286 * Push and pop I/O transform buffers 287 * ========================================================================== 288 */ 289 static void 290 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 291 zio_transform_func_t *transform) 292 { 293 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 294 295 zt->zt_orig_data = zio->io_data; 296 zt->zt_orig_size = zio->io_size; 297 zt->zt_bufsize = bufsize; 298 zt->zt_transform = transform; 299 300 zt->zt_next = zio->io_transform_stack; 301 zio->io_transform_stack = zt; 302 303 zio->io_data = data; 304 zio->io_size = size; 305 } 306 307 static void 308 zio_pop_transforms(zio_t *zio) 309 { 310 zio_transform_t *zt; 311 312 while ((zt = zio->io_transform_stack) != NULL) { 313 if (zt->zt_transform != NULL) 314 zt->zt_transform(zio, 315 zt->zt_orig_data, zt->zt_orig_size); 316 317 if (zt->zt_bufsize != 0) 318 zio_buf_free(zio->io_data, zt->zt_bufsize); 319 320 zio->io_data = zt->zt_orig_data; 321 zio->io_size = zt->zt_orig_size; 322 zio->io_transform_stack = zt->zt_next; 323 324 kmem_free(zt, sizeof (zio_transform_t)); 325 } 326 } 327 328 /* 329 * ========================================================================== 330 * I/O transform callbacks for subblocks and decompression 331 * ========================================================================== 332 */ 333 static void 334 zio_subblock(zio_t *zio, void *data, uint64_t size) 335 { 336 ASSERT(zio->io_size > size); 337 338 if (zio->io_type == ZIO_TYPE_READ) 339 bcopy(zio->io_data, data, size); 340 } 341 342 static void 343 zio_decompress(zio_t *zio, void *data, uint64_t size) 344 { 345 if (zio->io_error == 0 && 346 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 347 zio->io_data, data, zio->io_size, size) != 0) 348 zio->io_error = EIO; 349 } 350 351 /* 352 * ========================================================================== 353 * I/O parent/child relationships and pipeline interlocks 354 * ========================================================================== 355 */ 356 /* 357 * NOTE - Callers to zio_walk_parents() and zio_walk_children must 358 * continue calling these functions until they return NULL. 359 * Otherwise, the next caller will pick up the list walk in 360 * some indeterminate state. (Otherwise every caller would 361 * have to pass in a cookie to keep the state represented by 362 * io_walk_link, which gets annoying.) 363 */ 364 zio_t * 365 zio_walk_parents(zio_t *cio) 366 { 367 zio_link_t *zl = cio->io_walk_link; 368 list_t *pl = &cio->io_parent_list; 369 370 zl = (zl == NULL) ? list_head(pl) : list_next(pl, zl); 371 cio->io_walk_link = zl; 372 373 if (zl == NULL) 374 return (NULL); 375 376 ASSERT(zl->zl_child == cio); 377 return (zl->zl_parent); 378 } 379 380 zio_t * 381 zio_walk_children(zio_t *pio) 382 { 383 zio_link_t *zl = pio->io_walk_link; 384 list_t *cl = &pio->io_child_list; 385 386 zl = (zl == NULL) ? list_head(cl) : list_next(cl, zl); 387 pio->io_walk_link = zl; 388 389 if (zl == NULL) 390 return (NULL); 391 392 ASSERT(zl->zl_parent == pio); 393 return (zl->zl_child); 394 } 395 396 zio_t * 397 zio_unique_parent(zio_t *cio) 398 { 399 zio_t *pio = zio_walk_parents(cio); 400 401 VERIFY(zio_walk_parents(cio) == NULL); 402 return (pio); 403 } 404 405 void 406 zio_add_child(zio_t *pio, zio_t *cio) 407 { 408 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 409 410 /* 411 * Logical I/Os can have logical, gang, or vdev children. 412 * Gang I/Os can have gang or vdev children. 413 * Vdev I/Os can only have vdev children. 414 * The following ASSERT captures all of these constraints. 415 */ 416 ASSERT(cio->io_child_type <= pio->io_child_type); 417 418 zl->zl_parent = pio; 419 zl->zl_child = cio; 420 421 mutex_enter(&cio->io_lock); 422 mutex_enter(&pio->io_lock); 423 424 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 425 426 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 427 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 428 429 list_insert_head(&pio->io_child_list, zl); 430 list_insert_head(&cio->io_parent_list, zl); 431 432 pio->io_child_count++; 433 cio->io_parent_count++; 434 435 mutex_exit(&pio->io_lock); 436 mutex_exit(&cio->io_lock); 437 } 438 439 static void 440 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 441 { 442 ASSERT(zl->zl_parent == pio); 443 ASSERT(zl->zl_child == cio); 444 445 mutex_enter(&cio->io_lock); 446 mutex_enter(&pio->io_lock); 447 448 list_remove(&pio->io_child_list, zl); 449 list_remove(&cio->io_parent_list, zl); 450 451 pio->io_child_count--; 452 cio->io_parent_count--; 453 454 mutex_exit(&pio->io_lock); 455 mutex_exit(&cio->io_lock); 456 457 kmem_cache_free(zio_link_cache, zl); 458 } 459 460 static boolean_t 461 zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 462 { 463 uint64_t *countp = &zio->io_children[child][wait]; 464 boolean_t waiting = B_FALSE; 465 466 mutex_enter(&zio->io_lock); 467 ASSERT(zio->io_stall == NULL); 468 if (*countp != 0) { 469 zio->io_stage >>= 1; 470 zio->io_stall = countp; 471 waiting = B_TRUE; 472 } 473 mutex_exit(&zio->io_lock); 474 475 return (waiting); 476 } 477 478 static void 479 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 480 { 481 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 482 int *errorp = &pio->io_child_error[zio->io_child_type]; 483 484 mutex_enter(&pio->io_lock); 485 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 486 *errorp = zio_worst_error(*errorp, zio->io_error); 487 pio->io_reexecute |= zio->io_reexecute; 488 ASSERT3U(*countp, >, 0); 489 if (--*countp == 0 && pio->io_stall == countp) { 490 pio->io_stall = NULL; 491 mutex_exit(&pio->io_lock); 492 zio_execute(pio); 493 } else { 494 mutex_exit(&pio->io_lock); 495 } 496 } 497 498 static void 499 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 500 { 501 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 502 zio->io_error = zio->io_child_error[c]; 503 } 504 505 /* 506 * ========================================================================== 507 * Create the various types of I/O (read, write, free, etc) 508 * ========================================================================== 509 */ 510 static zio_t * 511 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 512 void *data, uint64_t size, zio_done_func_t *done, void *private, 513 zio_type_t type, int priority, enum zio_flag flags, 514 vdev_t *vd, uint64_t offset, const zbookmark_t *zb, 515 enum zio_stage stage, enum zio_stage pipeline) 516 { 517 zio_t *zio; 518 519 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 520 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 521 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 522 523 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 524 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 525 ASSERT(vd || stage == ZIO_STAGE_OPEN); 526 527 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 528 bzero(zio, sizeof (zio_t)); 529 530 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 531 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 532 533 list_create(&zio->io_parent_list, sizeof (zio_link_t), 534 offsetof(zio_link_t, zl_parent_node)); 535 list_create(&zio->io_child_list, sizeof (zio_link_t), 536 offsetof(zio_link_t, zl_child_node)); 537 538 if (vd != NULL) 539 zio->io_child_type = ZIO_CHILD_VDEV; 540 else if (flags & ZIO_FLAG_GANG_CHILD) 541 zio->io_child_type = ZIO_CHILD_GANG; 542 else if (flags & ZIO_FLAG_DDT_CHILD) 543 zio->io_child_type = ZIO_CHILD_DDT; 544 else 545 zio->io_child_type = ZIO_CHILD_LOGICAL; 546 547 if (bp != NULL) { 548 zio->io_bp = (blkptr_t *)bp; 549 zio->io_bp_copy = *bp; 550 zio->io_bp_orig = *bp; 551 if (type != ZIO_TYPE_WRITE || 552 zio->io_child_type == ZIO_CHILD_DDT) 553 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 554 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 555 zio->io_logical = zio; 556 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 557 pipeline |= ZIO_GANG_STAGES; 558 } 559 560 zio->io_spa = spa; 561 zio->io_txg = txg; 562 zio->io_done = done; 563 zio->io_private = private; 564 zio->io_type = type; 565 zio->io_priority = priority; 566 zio->io_vd = vd; 567 zio->io_offset = offset; 568 zio->io_orig_data = zio->io_data = data; 569 zio->io_orig_size = zio->io_size = size; 570 zio->io_orig_flags = zio->io_flags = flags; 571 zio->io_orig_stage = zio->io_stage = stage; 572 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 573 574 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 575 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 576 577 if (zb != NULL) 578 zio->io_bookmark = *zb; 579 580 if (pio != NULL) { 581 if (zio->io_logical == NULL) 582 zio->io_logical = pio->io_logical; 583 if (zio->io_child_type == ZIO_CHILD_GANG) 584 zio->io_gang_leader = pio->io_gang_leader; 585 zio_add_child(pio, zio); 586 } 587 588 return (zio); 589 } 590 591 static void 592 zio_destroy(zio_t *zio) 593 { 594 list_destroy(&zio->io_parent_list); 595 list_destroy(&zio->io_child_list); 596 mutex_destroy(&zio->io_lock); 597 cv_destroy(&zio->io_cv); 598 kmem_cache_free(zio_cache, zio); 599 } 600 601 zio_t * 602 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 603 void *private, enum zio_flag flags) 604 { 605 zio_t *zio; 606 607 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 608 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 609 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 610 611 return (zio); 612 } 613 614 zio_t * 615 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 616 { 617 return (zio_null(NULL, spa, NULL, done, private, flags)); 618 } 619 620 zio_t * 621 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 622 void *data, uint64_t size, zio_done_func_t *done, void *private, 623 int priority, enum zio_flag flags, const zbookmark_t *zb) 624 { 625 zio_t *zio; 626 627 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 628 data, size, done, private, 629 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 630 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 631 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 632 633 return (zio); 634 } 635 636 zio_t * 637 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 638 void *data, uint64_t size, const zio_prop_t *zp, 639 zio_done_func_t *ready, zio_done_func_t *done, void *private, 640 int priority, enum zio_flag flags, const zbookmark_t *zb) 641 { 642 zio_t *zio; 643 644 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 645 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 646 zp->zp_compress >= ZIO_COMPRESS_OFF && 647 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 648 DMU_OT_IS_VALID(zp->zp_type) && 649 zp->zp_level < 32 && 650 zp->zp_copies > 0 && 651 zp->zp_copies <= spa_max_replication(spa)); 652 653 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 654 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 655 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 656 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 657 658 zio->io_ready = ready; 659 zio->io_prop = *zp; 660 661 return (zio); 662 } 663 664 zio_t * 665 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 666 uint64_t size, zio_done_func_t *done, void *private, int priority, 667 enum zio_flag flags, zbookmark_t *zb) 668 { 669 zio_t *zio; 670 671 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 672 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 673 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 674 675 return (zio); 676 } 677 678 void 679 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 680 { 681 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 682 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 683 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 684 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 685 686 /* 687 * We must reset the io_prop to match the values that existed 688 * when the bp was first written by dmu_sync() keeping in mind 689 * that nopwrite and dedup are mutually exclusive. 690 */ 691 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 692 zio->io_prop.zp_nopwrite = nopwrite; 693 zio->io_prop.zp_copies = copies; 694 zio->io_bp_override = bp; 695 } 696 697 void 698 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 699 { 700 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 701 } 702 703 zio_t * 704 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 705 enum zio_flag flags) 706 { 707 zio_t *zio; 708 709 dprintf_bp(bp, "freeing in txg %llu, pass %u", 710 (longlong_t)txg, spa->spa_sync_pass); 711 712 ASSERT(!BP_IS_HOLE(bp)); 713 ASSERT(spa_syncing_txg(spa) == txg); 714 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free); 715 716 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 717 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_FREE, flags, 718 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_FREE_PIPELINE); 719 720 return (zio); 721 } 722 723 zio_t * 724 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 725 zio_done_func_t *done, void *private, enum zio_flag flags) 726 { 727 zio_t *zio; 728 729 /* 730 * A claim is an allocation of a specific block. Claims are needed 731 * to support immediate writes in the intent log. The issue is that 732 * immediate writes contain committed data, but in a txg that was 733 * *not* committed. Upon opening the pool after an unclean shutdown, 734 * the intent log claims all blocks that contain immediate write data 735 * so that the SPA knows they're in use. 736 * 737 * All claims *must* be resolved in the first txg -- before the SPA 738 * starts allocating blocks -- so that nothing is allocated twice. 739 * If txg == 0 we just verify that the block is claimable. 740 */ 741 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 742 ASSERT(txg == spa_first_txg(spa) || txg == 0); 743 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 744 745 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 746 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 747 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 748 749 return (zio); 750 } 751 752 zio_t * 753 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 754 zio_done_func_t *done, void *private, int priority, enum zio_flag flags) 755 { 756 zio_t *zio; 757 int c; 758 759 if (vd->vdev_children == 0) { 760 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 761 ZIO_TYPE_IOCTL, priority, flags, vd, 0, NULL, 762 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 763 764 zio->io_cmd = cmd; 765 } else { 766 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 767 768 for (c = 0; c < vd->vdev_children; c++) 769 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 770 done, private, priority, flags)); 771 } 772 773 return (zio); 774 } 775 776 zio_t * 777 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 778 void *data, int checksum, zio_done_func_t *done, void *private, 779 int priority, enum zio_flag flags, boolean_t labels) 780 { 781 zio_t *zio; 782 783 ASSERT(vd->vdev_children == 0); 784 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 785 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 786 ASSERT3U(offset + size, <=, vd->vdev_psize); 787 788 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 789 ZIO_TYPE_READ, priority, flags, vd, offset, NULL, 790 ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 791 792 zio->io_prop.zp_checksum = checksum; 793 794 return (zio); 795 } 796 797 zio_t * 798 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 799 void *data, int checksum, zio_done_func_t *done, void *private, 800 int priority, enum zio_flag flags, boolean_t labels) 801 { 802 zio_t *zio; 803 804 ASSERT(vd->vdev_children == 0); 805 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 806 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 807 ASSERT3U(offset + size, <=, vd->vdev_psize); 808 809 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 810 ZIO_TYPE_WRITE, priority, flags, vd, offset, NULL, 811 ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 812 813 zio->io_prop.zp_checksum = checksum; 814 815 if (zio_checksum_table[checksum].ci_eck) { 816 /* 817 * zec checksums are necessarily destructive -- they modify 818 * the end of the write buffer to hold the verifier/checksum. 819 * Therefore, we must make a local copy in case the data is 820 * being written to multiple places in parallel. 821 */ 822 void *wbuf = zio_buf_alloc(size); 823 bcopy(data, wbuf, size); 824 zio_push_transform(zio, wbuf, size, size, NULL); 825 } 826 827 return (zio); 828 } 829 830 /* 831 * Create a child I/O to do some work for us. 832 */ 833 zio_t * 834 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 835 void *data, uint64_t size, int type, int priority, enum zio_flag flags, 836 zio_done_func_t *done, void *private) 837 { 838 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 839 zio_t *zio; 840 841 ASSERT(vd->vdev_parent == 842 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 843 844 if (type == ZIO_TYPE_READ && bp != NULL) { 845 /* 846 * If we have the bp, then the child should perform the 847 * checksum and the parent need not. This pushes error 848 * detection as close to the leaves as possible and 849 * eliminates redundant checksums in the interior nodes. 850 */ 851 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 852 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 853 } 854 855 if (vd->vdev_children == 0) 856 offset += VDEV_LABEL_START_SIZE; 857 858 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 859 860 /* 861 * If we've decided to do a repair, the write is not speculative -- 862 * even if the original read was. 863 */ 864 if (flags & ZIO_FLAG_IO_REPAIR) 865 flags &= ~ZIO_FLAG_SPECULATIVE; 866 867 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 868 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 869 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 870 871 return (zio); 872 } 873 874 zio_t * 875 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 876 int type, int priority, enum zio_flag flags, 877 zio_done_func_t *done, void *private) 878 { 879 zio_t *zio; 880 881 ASSERT(vd->vdev_ops->vdev_op_leaf); 882 883 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 884 data, size, done, private, type, priority, 885 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY, 886 vd, offset, NULL, 887 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 888 889 return (zio); 890 } 891 892 void 893 zio_flush(zio_t *zio, vdev_t *vd) 894 { 895 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 896 NULL, NULL, ZIO_PRIORITY_NOW, 897 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 898 } 899 900 void 901 zio_shrink(zio_t *zio, uint64_t size) 902 { 903 ASSERT(zio->io_executor == NULL); 904 ASSERT(zio->io_orig_size == zio->io_size); 905 ASSERT(size <= zio->io_size); 906 907 /* 908 * We don't shrink for raidz because of problems with the 909 * reconstruction when reading back less than the block size. 910 * Note, BP_IS_RAIDZ() assumes no compression. 911 */ 912 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 913 if (!BP_IS_RAIDZ(zio->io_bp)) 914 zio->io_orig_size = zio->io_size = size; 915 } 916 917 /* 918 * ========================================================================== 919 * Prepare to read and write logical blocks 920 * ========================================================================== 921 */ 922 923 static int 924 zio_read_bp_init(zio_t *zio) 925 { 926 blkptr_t *bp = zio->io_bp; 927 928 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 929 zio->io_child_type == ZIO_CHILD_LOGICAL && 930 !(zio->io_flags & ZIO_FLAG_RAW)) { 931 uint64_t psize = BP_GET_PSIZE(bp); 932 void *cbuf = zio_buf_alloc(psize); 933 934 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 935 } 936 937 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 938 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 939 940 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 941 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 942 943 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 944 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 945 946 return (ZIO_PIPELINE_CONTINUE); 947 } 948 949 static int 950 zio_write_bp_init(zio_t *zio) 951 { 952 spa_t *spa = zio->io_spa; 953 zio_prop_t *zp = &zio->io_prop; 954 enum zio_compress compress = zp->zp_compress; 955 blkptr_t *bp = zio->io_bp; 956 uint64_t lsize = zio->io_size; 957 uint64_t psize = lsize; 958 int pass = 1; 959 960 /* 961 * If our children haven't all reached the ready stage, 962 * wait for them and then repeat this pipeline stage. 963 */ 964 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 965 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 966 return (ZIO_PIPELINE_STOP); 967 968 if (!IO_IS_ALLOCATING(zio)) 969 return (ZIO_PIPELINE_CONTINUE); 970 971 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 972 973 if (zio->io_bp_override) { 974 ASSERT(bp->blk_birth != zio->io_txg); 975 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 976 977 *bp = *zio->io_bp_override; 978 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 979 980 /* 981 * If we've been overridden and nopwrite is set then 982 * set the flag accordingly to indicate that a nopwrite 983 * has already occurred. 984 */ 985 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 986 ASSERT(!zp->zp_dedup); 987 zio->io_flags |= ZIO_FLAG_NOPWRITE; 988 return (ZIO_PIPELINE_CONTINUE); 989 } 990 991 ASSERT(!zp->zp_nopwrite); 992 993 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 994 return (ZIO_PIPELINE_CONTINUE); 995 996 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup || 997 zp->zp_dedup_verify); 998 999 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 1000 BP_SET_DEDUP(bp, 1); 1001 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1002 return (ZIO_PIPELINE_CONTINUE); 1003 } 1004 zio->io_bp_override = NULL; 1005 BP_ZERO(bp); 1006 } 1007 1008 if (bp->blk_birth == zio->io_txg) { 1009 /* 1010 * We're rewriting an existing block, which means we're 1011 * working on behalf of spa_sync(). For spa_sync() to 1012 * converge, it must eventually be the case that we don't 1013 * have to allocate new blocks. But compression changes 1014 * the blocksize, which forces a reallocate, and makes 1015 * convergence take longer. Therefore, after the first 1016 * few passes, stop compressing to ensure convergence. 1017 */ 1018 pass = spa_sync_pass(spa); 1019 1020 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1021 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1022 ASSERT(!BP_GET_DEDUP(bp)); 1023 1024 if (pass >= zfs_sync_pass_dont_compress) 1025 compress = ZIO_COMPRESS_OFF; 1026 1027 /* Make sure someone doesn't change their mind on overwrites */ 1028 ASSERT(MIN(zp->zp_copies + BP_IS_GANG(bp), 1029 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1030 } 1031 1032 if (compress != ZIO_COMPRESS_OFF) { 1033 void *cbuf = zio_buf_alloc(lsize); 1034 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize); 1035 if (psize == 0 || psize == lsize) { 1036 compress = ZIO_COMPRESS_OFF; 1037 zio_buf_free(cbuf, lsize); 1038 } else { 1039 ASSERT(psize < lsize); 1040 zio_push_transform(zio, cbuf, psize, lsize, NULL); 1041 } 1042 } 1043 1044 /* 1045 * The final pass of spa_sync() must be all rewrites, but the first 1046 * few passes offer a trade-off: allocating blocks defers convergence, 1047 * but newly allocated blocks are sequential, so they can be written 1048 * to disk faster. Therefore, we allow the first few passes of 1049 * spa_sync() to allocate new blocks, but force rewrites after that. 1050 * There should only be a handful of blocks after pass 1 in any case. 1051 */ 1052 if (bp->blk_birth == zio->io_txg && BP_GET_PSIZE(bp) == psize && 1053 pass >= zfs_sync_pass_rewrite) { 1054 ASSERT(psize != 0); 1055 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1056 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1057 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1058 } else { 1059 BP_ZERO(bp); 1060 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1061 } 1062 1063 if (psize == 0) { 1064 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1065 } else { 1066 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1067 BP_SET_LSIZE(bp, lsize); 1068 BP_SET_PSIZE(bp, psize); 1069 BP_SET_COMPRESS(bp, compress); 1070 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1071 BP_SET_TYPE(bp, zp->zp_type); 1072 BP_SET_LEVEL(bp, zp->zp_level); 1073 BP_SET_DEDUP(bp, zp->zp_dedup); 1074 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1075 if (zp->zp_dedup) { 1076 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1077 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1078 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1079 } 1080 if (zp->zp_nopwrite) { 1081 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1082 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1083 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1084 } 1085 } 1086 1087 return (ZIO_PIPELINE_CONTINUE); 1088 } 1089 1090 static int 1091 zio_free_bp_init(zio_t *zio) 1092 { 1093 blkptr_t *bp = zio->io_bp; 1094 1095 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1096 if (BP_GET_DEDUP(bp)) 1097 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1098 } 1099 1100 return (ZIO_PIPELINE_CONTINUE); 1101 } 1102 1103 /* 1104 * ========================================================================== 1105 * Execute the I/O pipeline 1106 * ========================================================================== 1107 */ 1108 1109 static void 1110 zio_taskq_dispatch(zio_t *zio, enum zio_taskq_type q, boolean_t cutinline) 1111 { 1112 spa_t *spa = zio->io_spa; 1113 zio_type_t t = zio->io_type; 1114 int flags = (cutinline ? TQ_FRONT : 0); 1115 1116 /* 1117 * If we're a config writer or a probe, the normal issue and 1118 * interrupt threads may all be blocked waiting for the config lock. 1119 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1120 */ 1121 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1122 t = ZIO_TYPE_NULL; 1123 1124 /* 1125 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1126 */ 1127 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1128 t = ZIO_TYPE_NULL; 1129 1130 /* 1131 * If this is a high priority I/O, then use the high priority taskq. 1132 */ 1133 if (zio->io_priority == ZIO_PRIORITY_NOW && 1134 spa->spa_zio_taskq[t][q + 1] != NULL) 1135 q++; 1136 1137 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1138 1139 /* 1140 * NB: We are assuming that the zio can only be dispatched 1141 * to a single taskq at a time. It would be a grievous error 1142 * to dispatch the zio to another taskq at the same time. 1143 */ 1144 ASSERT(zio->io_tqent.tqent_next == NULL); 1145 taskq_dispatch_ent(spa->spa_zio_taskq[t][q], 1146 (task_func_t *)zio_execute, zio, flags, &zio->io_tqent); 1147 } 1148 1149 static boolean_t 1150 zio_taskq_member(zio_t *zio, enum zio_taskq_type q) 1151 { 1152 kthread_t *executor = zio->io_executor; 1153 spa_t *spa = zio->io_spa; 1154 1155 for (zio_type_t t = 0; t < ZIO_TYPES; t++) 1156 if (taskq_member(spa->spa_zio_taskq[t][q], executor)) 1157 return (B_TRUE); 1158 1159 return (B_FALSE); 1160 } 1161 1162 static int 1163 zio_issue_async(zio_t *zio) 1164 { 1165 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1166 1167 return (ZIO_PIPELINE_STOP); 1168 } 1169 1170 void 1171 zio_interrupt(zio_t *zio) 1172 { 1173 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1174 } 1175 1176 /* 1177 * Execute the I/O pipeline until one of the following occurs: 1178 * (1) the I/O completes; (2) the pipeline stalls waiting for 1179 * dependent child I/Os; (3) the I/O issues, so we're waiting 1180 * for an I/O completion interrupt; (4) the I/O is delegated by 1181 * vdev-level caching or aggregation; (5) the I/O is deferred 1182 * due to vdev-level queueing; (6) the I/O is handed off to 1183 * another thread. In all cases, the pipeline stops whenever 1184 * there's no CPU work; it never burns a thread in cv_wait(). 1185 * 1186 * There's no locking on io_stage because there's no legitimate way 1187 * for multiple threads to be attempting to process the same I/O. 1188 */ 1189 static zio_pipe_stage_t *zio_pipeline[]; 1190 1191 void 1192 zio_execute(zio_t *zio) 1193 { 1194 zio->io_executor = curthread; 1195 1196 while (zio->io_stage < ZIO_STAGE_DONE) { 1197 enum zio_stage pipeline = zio->io_pipeline; 1198 enum zio_stage stage = zio->io_stage; 1199 int rv; 1200 1201 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1202 ASSERT(ISP2(stage)); 1203 ASSERT(zio->io_stall == NULL); 1204 1205 do { 1206 stage <<= 1; 1207 } while ((stage & pipeline) == 0); 1208 1209 ASSERT(stage <= ZIO_STAGE_DONE); 1210 1211 /* 1212 * If we are in interrupt context and this pipeline stage 1213 * will grab a config lock that is held across I/O, 1214 * or may wait for an I/O that needs an interrupt thread 1215 * to complete, issue async to avoid deadlock. 1216 * 1217 * For VDEV_IO_START, we cut in line so that the io will 1218 * be sent to disk promptly. 1219 */ 1220 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1221 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1222 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1223 zio_requeue_io_start_cut_in_line : B_FALSE; 1224 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1225 return; 1226 } 1227 1228 zio->io_stage = stage; 1229 rv = zio_pipeline[highbit(stage) - 1](zio); 1230 1231 if (rv == ZIO_PIPELINE_STOP) 1232 return; 1233 1234 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1235 } 1236 } 1237 1238 /* 1239 * ========================================================================== 1240 * Initiate I/O, either sync or async 1241 * ========================================================================== 1242 */ 1243 int 1244 zio_wait(zio_t *zio) 1245 { 1246 int error; 1247 1248 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1249 ASSERT(zio->io_executor == NULL); 1250 1251 zio->io_waiter = curthread; 1252 1253 zio_execute(zio); 1254 1255 mutex_enter(&zio->io_lock); 1256 while (zio->io_executor != NULL) 1257 cv_wait(&zio->io_cv, &zio->io_lock); 1258 mutex_exit(&zio->io_lock); 1259 1260 error = zio->io_error; 1261 zio_destroy(zio); 1262 1263 return (error); 1264 } 1265 1266 void 1267 zio_nowait(zio_t *zio) 1268 { 1269 ASSERT(zio->io_executor == NULL); 1270 1271 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1272 zio_unique_parent(zio) == NULL) { 1273 /* 1274 * This is a logical async I/O with no parent to wait for it. 1275 * We add it to the spa_async_root_zio "Godfather" I/O which 1276 * will ensure they complete prior to unloading the pool. 1277 */ 1278 spa_t *spa = zio->io_spa; 1279 1280 zio_add_child(spa->spa_async_zio_root, zio); 1281 } 1282 1283 zio_execute(zio); 1284 } 1285 1286 /* 1287 * ========================================================================== 1288 * Reexecute or suspend/resume failed I/O 1289 * ========================================================================== 1290 */ 1291 1292 static void 1293 zio_reexecute(zio_t *pio) 1294 { 1295 zio_t *cio, *cio_next; 1296 1297 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1298 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1299 ASSERT(pio->io_gang_leader == NULL); 1300 ASSERT(pio->io_gang_tree == NULL); 1301 1302 pio->io_flags = pio->io_orig_flags; 1303 pio->io_stage = pio->io_orig_stage; 1304 pio->io_pipeline = pio->io_orig_pipeline; 1305 pio->io_reexecute = 0; 1306 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1307 pio->io_error = 0; 1308 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1309 pio->io_state[w] = 0; 1310 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1311 pio->io_child_error[c] = 0; 1312 1313 if (IO_IS_ALLOCATING(pio)) 1314 BP_ZERO(pio->io_bp); 1315 1316 /* 1317 * As we reexecute pio's children, new children could be created. 1318 * New children go to the head of pio's io_child_list, however, 1319 * so we will (correctly) not reexecute them. The key is that 1320 * the remainder of pio's io_child_list, from 'cio_next' onward, 1321 * cannot be affected by any side effects of reexecuting 'cio'. 1322 */ 1323 for (cio = zio_walk_children(pio); cio != NULL; cio = cio_next) { 1324 cio_next = zio_walk_children(pio); 1325 mutex_enter(&pio->io_lock); 1326 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1327 pio->io_children[cio->io_child_type][w]++; 1328 mutex_exit(&pio->io_lock); 1329 zio_reexecute(cio); 1330 } 1331 1332 /* 1333 * Now that all children have been reexecuted, execute the parent. 1334 * We don't reexecute "The Godfather" I/O here as it's the 1335 * responsibility of the caller to wait on him. 1336 */ 1337 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) 1338 zio_execute(pio); 1339 } 1340 1341 void 1342 zio_suspend(spa_t *spa, zio_t *zio) 1343 { 1344 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1345 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1346 "failure and the failure mode property for this pool " 1347 "is set to panic.", spa_name(spa)); 1348 1349 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1350 1351 mutex_enter(&spa->spa_suspend_lock); 1352 1353 if (spa->spa_suspend_zio_root == NULL) 1354 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1355 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1356 ZIO_FLAG_GODFATHER); 1357 1358 spa->spa_suspended = B_TRUE; 1359 1360 if (zio != NULL) { 1361 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1362 ASSERT(zio != spa->spa_suspend_zio_root); 1363 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1364 ASSERT(zio_unique_parent(zio) == NULL); 1365 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1366 zio_add_child(spa->spa_suspend_zio_root, zio); 1367 } 1368 1369 mutex_exit(&spa->spa_suspend_lock); 1370 } 1371 1372 int 1373 zio_resume(spa_t *spa) 1374 { 1375 zio_t *pio; 1376 1377 /* 1378 * Reexecute all previously suspended i/o. 1379 */ 1380 mutex_enter(&spa->spa_suspend_lock); 1381 spa->spa_suspended = B_FALSE; 1382 cv_broadcast(&spa->spa_suspend_cv); 1383 pio = spa->spa_suspend_zio_root; 1384 spa->spa_suspend_zio_root = NULL; 1385 mutex_exit(&spa->spa_suspend_lock); 1386 1387 if (pio == NULL) 1388 return (0); 1389 1390 zio_reexecute(pio); 1391 return (zio_wait(pio)); 1392 } 1393 1394 void 1395 zio_resume_wait(spa_t *spa) 1396 { 1397 mutex_enter(&spa->spa_suspend_lock); 1398 while (spa_suspended(spa)) 1399 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1400 mutex_exit(&spa->spa_suspend_lock); 1401 } 1402 1403 /* 1404 * ========================================================================== 1405 * Gang blocks. 1406 * 1407 * A gang block is a collection of small blocks that looks to the DMU 1408 * like one large block. When zio_dva_allocate() cannot find a block 1409 * of the requested size, due to either severe fragmentation or the pool 1410 * being nearly full, it calls zio_write_gang_block() to construct the 1411 * block from smaller fragments. 1412 * 1413 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1414 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1415 * an indirect block: it's an array of block pointers. It consumes 1416 * only one sector and hence is allocatable regardless of fragmentation. 1417 * The gang header's bps point to its gang members, which hold the data. 1418 * 1419 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1420 * as the verifier to ensure uniqueness of the SHA256 checksum. 1421 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1422 * not the gang header. This ensures that data block signatures (needed for 1423 * deduplication) are independent of how the block is physically stored. 1424 * 1425 * Gang blocks can be nested: a gang member may itself be a gang block. 1426 * Thus every gang block is a tree in which root and all interior nodes are 1427 * gang headers, and the leaves are normal blocks that contain user data. 1428 * The root of the gang tree is called the gang leader. 1429 * 1430 * To perform any operation (read, rewrite, free, claim) on a gang block, 1431 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1432 * in the io_gang_tree field of the original logical i/o by recursively 1433 * reading the gang leader and all gang headers below it. This yields 1434 * an in-core tree containing the contents of every gang header and the 1435 * bps for every constituent of the gang block. 1436 * 1437 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1438 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1439 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1440 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1441 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1442 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1443 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1444 * of the gang header plus zio_checksum_compute() of the data to update the 1445 * gang header's blk_cksum as described above. 1446 * 1447 * The two-phase assemble/issue model solves the problem of partial failure -- 1448 * what if you'd freed part of a gang block but then couldn't read the 1449 * gang header for another part? Assembling the entire gang tree first 1450 * ensures that all the necessary gang header I/O has succeeded before 1451 * starting the actual work of free, claim, or write. Once the gang tree 1452 * is assembled, free and claim are in-memory operations that cannot fail. 1453 * 1454 * In the event that a gang write fails, zio_dva_unallocate() walks the 1455 * gang tree to immediately free (i.e. insert back into the space map) 1456 * everything we've allocated. This ensures that we don't get ENOSPC 1457 * errors during repeated suspend/resume cycles due to a flaky device. 1458 * 1459 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1460 * the gang tree, we won't modify the block, so we can safely defer the free 1461 * (knowing that the block is still intact). If we *can* assemble the gang 1462 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1463 * each constituent bp and we can allocate a new block on the next sync pass. 1464 * 1465 * In all cases, the gang tree allows complete recovery from partial failure. 1466 * ========================================================================== 1467 */ 1468 1469 static zio_t * 1470 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1471 { 1472 if (gn != NULL) 1473 return (pio); 1474 1475 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1476 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1477 &pio->io_bookmark)); 1478 } 1479 1480 zio_t * 1481 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1482 { 1483 zio_t *zio; 1484 1485 if (gn != NULL) { 1486 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1487 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1488 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1489 /* 1490 * As we rewrite each gang header, the pipeline will compute 1491 * a new gang block header checksum for it; but no one will 1492 * compute a new data checksum, so we do that here. The one 1493 * exception is the gang leader: the pipeline already computed 1494 * its data checksum because that stage precedes gang assembly. 1495 * (Presently, nothing actually uses interior data checksums; 1496 * this is just good hygiene.) 1497 */ 1498 if (gn != pio->io_gang_leader->io_gang_tree) { 1499 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1500 data, BP_GET_PSIZE(bp)); 1501 } 1502 /* 1503 * If we are here to damage data for testing purposes, 1504 * leave the GBH alone so that we can detect the damage. 1505 */ 1506 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1507 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1508 } else { 1509 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1510 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1511 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1512 } 1513 1514 return (zio); 1515 } 1516 1517 /* ARGSUSED */ 1518 zio_t * 1519 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1520 { 1521 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1522 ZIO_GANG_CHILD_FLAGS(pio))); 1523 } 1524 1525 /* ARGSUSED */ 1526 zio_t * 1527 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1528 { 1529 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1530 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1531 } 1532 1533 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1534 NULL, 1535 zio_read_gang, 1536 zio_rewrite_gang, 1537 zio_free_gang, 1538 zio_claim_gang, 1539 NULL 1540 }; 1541 1542 static void zio_gang_tree_assemble_done(zio_t *zio); 1543 1544 static zio_gang_node_t * 1545 zio_gang_node_alloc(zio_gang_node_t **gnpp) 1546 { 1547 zio_gang_node_t *gn; 1548 1549 ASSERT(*gnpp == NULL); 1550 1551 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1552 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1553 *gnpp = gn; 1554 1555 return (gn); 1556 } 1557 1558 static void 1559 zio_gang_node_free(zio_gang_node_t **gnpp) 1560 { 1561 zio_gang_node_t *gn = *gnpp; 1562 1563 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1564 ASSERT(gn->gn_child[g] == NULL); 1565 1566 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1567 kmem_free(gn, sizeof (*gn)); 1568 *gnpp = NULL; 1569 } 1570 1571 static void 1572 zio_gang_tree_free(zio_gang_node_t **gnpp) 1573 { 1574 zio_gang_node_t *gn = *gnpp; 1575 1576 if (gn == NULL) 1577 return; 1578 1579 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1580 zio_gang_tree_free(&gn->gn_child[g]); 1581 1582 zio_gang_node_free(gnpp); 1583 } 1584 1585 static void 1586 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1587 { 1588 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1589 1590 ASSERT(gio->io_gang_leader == gio); 1591 ASSERT(BP_IS_GANG(bp)); 1592 1593 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1594 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1595 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1596 } 1597 1598 static void 1599 zio_gang_tree_assemble_done(zio_t *zio) 1600 { 1601 zio_t *gio = zio->io_gang_leader; 1602 zio_gang_node_t *gn = zio->io_private; 1603 blkptr_t *bp = zio->io_bp; 1604 1605 ASSERT(gio == zio_unique_parent(zio)); 1606 ASSERT(zio->io_child_count == 0); 1607 1608 if (zio->io_error) 1609 return; 1610 1611 if (BP_SHOULD_BYTESWAP(bp)) 1612 byteswap_uint64_array(zio->io_data, zio->io_size); 1613 1614 ASSERT(zio->io_data == gn->gn_gbh); 1615 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1616 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1617 1618 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1619 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1620 if (!BP_IS_GANG(gbp)) 1621 continue; 1622 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1623 } 1624 } 1625 1626 static void 1627 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1628 { 1629 zio_t *gio = pio->io_gang_leader; 1630 zio_t *zio; 1631 1632 ASSERT(BP_IS_GANG(bp) == !!gn); 1633 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1634 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1635 1636 /* 1637 * If you're a gang header, your data is in gn->gn_gbh. 1638 * If you're a gang member, your data is in 'data' and gn == NULL. 1639 */ 1640 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1641 1642 if (gn != NULL) { 1643 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1644 1645 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1646 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1647 if (BP_IS_HOLE(gbp)) 1648 continue; 1649 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1650 data = (char *)data + BP_GET_PSIZE(gbp); 1651 } 1652 } 1653 1654 if (gn == gio->io_gang_tree) 1655 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1656 1657 if (zio != pio) 1658 zio_nowait(zio); 1659 } 1660 1661 static int 1662 zio_gang_assemble(zio_t *zio) 1663 { 1664 blkptr_t *bp = zio->io_bp; 1665 1666 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 1667 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1668 1669 zio->io_gang_leader = zio; 1670 1671 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 1672 1673 return (ZIO_PIPELINE_CONTINUE); 1674 } 1675 1676 static int 1677 zio_gang_issue(zio_t *zio) 1678 { 1679 blkptr_t *bp = zio->io_bp; 1680 1681 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 1682 return (ZIO_PIPELINE_STOP); 1683 1684 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 1685 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 1686 1687 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 1688 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 1689 else 1690 zio_gang_tree_free(&zio->io_gang_tree); 1691 1692 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1693 1694 return (ZIO_PIPELINE_CONTINUE); 1695 } 1696 1697 static void 1698 zio_write_gang_member_ready(zio_t *zio) 1699 { 1700 zio_t *pio = zio_unique_parent(zio); 1701 zio_t *gio = zio->io_gang_leader; 1702 dva_t *cdva = zio->io_bp->blk_dva; 1703 dva_t *pdva = pio->io_bp->blk_dva; 1704 uint64_t asize; 1705 1706 if (BP_IS_HOLE(zio->io_bp)) 1707 return; 1708 1709 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 1710 1711 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 1712 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 1713 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 1714 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 1715 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 1716 1717 mutex_enter(&pio->io_lock); 1718 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 1719 ASSERT(DVA_GET_GANG(&pdva[d])); 1720 asize = DVA_GET_ASIZE(&pdva[d]); 1721 asize += DVA_GET_ASIZE(&cdva[d]); 1722 DVA_SET_ASIZE(&pdva[d], asize); 1723 } 1724 mutex_exit(&pio->io_lock); 1725 } 1726 1727 static int 1728 zio_write_gang_block(zio_t *pio) 1729 { 1730 spa_t *spa = pio->io_spa; 1731 blkptr_t *bp = pio->io_bp; 1732 zio_t *gio = pio->io_gang_leader; 1733 zio_t *zio; 1734 zio_gang_node_t *gn, **gnpp; 1735 zio_gbh_phys_t *gbh; 1736 uint64_t txg = pio->io_txg; 1737 uint64_t resid = pio->io_size; 1738 uint64_t lsize; 1739 int copies = gio->io_prop.zp_copies; 1740 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 1741 zio_prop_t zp; 1742 int error; 1743 1744 error = metaslab_alloc(spa, spa_normal_class(spa), SPA_GANGBLOCKSIZE, 1745 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, 1746 METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER); 1747 if (error) { 1748 pio->io_error = error; 1749 return (ZIO_PIPELINE_CONTINUE); 1750 } 1751 1752 if (pio == gio) { 1753 gnpp = &gio->io_gang_tree; 1754 } else { 1755 gnpp = pio->io_private; 1756 ASSERT(pio->io_ready == zio_write_gang_member_ready); 1757 } 1758 1759 gn = zio_gang_node_alloc(gnpp); 1760 gbh = gn->gn_gbh; 1761 bzero(gbh, SPA_GANGBLOCKSIZE); 1762 1763 /* 1764 * Create the gang header. 1765 */ 1766 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 1767 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1768 1769 /* 1770 * Create and nowait the gang children. 1771 */ 1772 for (int g = 0; resid != 0; resid -= lsize, g++) { 1773 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 1774 SPA_MINBLOCKSIZE); 1775 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 1776 1777 zp.zp_checksum = gio->io_prop.zp_checksum; 1778 zp.zp_compress = ZIO_COMPRESS_OFF; 1779 zp.zp_type = DMU_OT_NONE; 1780 zp.zp_level = 0; 1781 zp.zp_copies = gio->io_prop.zp_copies; 1782 zp.zp_dedup = B_FALSE; 1783 zp.zp_dedup_verify = B_FALSE; 1784 zp.zp_nopwrite = B_FALSE; 1785 1786 zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 1787 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 1788 zio_write_gang_member_ready, NULL, &gn->gn_child[g], 1789 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1790 &pio->io_bookmark)); 1791 } 1792 1793 /* 1794 * Set pio's pipeline to just wait for zio to finish. 1795 */ 1796 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1797 1798 zio_nowait(zio); 1799 1800 return (ZIO_PIPELINE_CONTINUE); 1801 } 1802 1803 /* 1804 * The zio_nop_write stage in the pipeline determines if allocating 1805 * a new bp is necessary. By leveraging a cryptographically secure checksum, 1806 * such as SHA256, we can compare the checksums of the new data and the old 1807 * to determine if allocating a new block is required. The nopwrite 1808 * feature can handle writes in either syncing or open context (i.e. zil 1809 * writes) and as a result is mutually exclusive with dedup. 1810 */ 1811 static int 1812 zio_nop_write(zio_t *zio) 1813 { 1814 blkptr_t *bp = zio->io_bp; 1815 blkptr_t *bp_orig = &zio->io_bp_orig; 1816 zio_prop_t *zp = &zio->io_prop; 1817 1818 ASSERT(BP_GET_LEVEL(bp) == 0); 1819 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1820 ASSERT(zp->zp_nopwrite); 1821 ASSERT(!zp->zp_dedup); 1822 ASSERT(zio->io_bp_override == NULL); 1823 ASSERT(IO_IS_ALLOCATING(zio)); 1824 1825 /* 1826 * Check to see if the original bp and the new bp have matching 1827 * characteristics (i.e. same checksum, compression algorithms, etc). 1828 * If they don't then just continue with the pipeline which will 1829 * allocate a new bp. 1830 */ 1831 if (BP_IS_HOLE(bp_orig) || 1832 !zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_dedup || 1833 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 1834 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 1835 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 1836 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 1837 return (ZIO_PIPELINE_CONTINUE); 1838 1839 /* 1840 * If the checksums match then reset the pipeline so that we 1841 * avoid allocating a new bp and issuing any I/O. 1842 */ 1843 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 1844 ASSERT(zio_checksum_table[zp->zp_checksum].ci_dedup); 1845 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 1846 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 1847 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 1848 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 1849 sizeof (uint64_t)) == 0); 1850 1851 *bp = *bp_orig; 1852 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1853 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1854 } 1855 1856 return (ZIO_PIPELINE_CONTINUE); 1857 } 1858 1859 /* 1860 * ========================================================================== 1861 * Dedup 1862 * ========================================================================== 1863 */ 1864 static void 1865 zio_ddt_child_read_done(zio_t *zio) 1866 { 1867 blkptr_t *bp = zio->io_bp; 1868 ddt_entry_t *dde = zio->io_private; 1869 ddt_phys_t *ddp; 1870 zio_t *pio = zio_unique_parent(zio); 1871 1872 mutex_enter(&pio->io_lock); 1873 ddp = ddt_phys_select(dde, bp); 1874 if (zio->io_error == 0) 1875 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 1876 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 1877 dde->dde_repair_data = zio->io_data; 1878 else 1879 zio_buf_free(zio->io_data, zio->io_size); 1880 mutex_exit(&pio->io_lock); 1881 } 1882 1883 static int 1884 zio_ddt_read_start(zio_t *zio) 1885 { 1886 blkptr_t *bp = zio->io_bp; 1887 1888 ASSERT(BP_GET_DEDUP(bp)); 1889 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 1890 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1891 1892 if (zio->io_child_error[ZIO_CHILD_DDT]) { 1893 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1894 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 1895 ddt_phys_t *ddp = dde->dde_phys; 1896 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 1897 blkptr_t blk; 1898 1899 ASSERT(zio->io_vsd == NULL); 1900 zio->io_vsd = dde; 1901 1902 if (ddp_self == NULL) 1903 return (ZIO_PIPELINE_CONTINUE); 1904 1905 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1906 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 1907 continue; 1908 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 1909 &blk); 1910 zio_nowait(zio_read(zio, zio->io_spa, &blk, 1911 zio_buf_alloc(zio->io_size), zio->io_size, 1912 zio_ddt_child_read_done, dde, zio->io_priority, 1913 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 1914 &zio->io_bookmark)); 1915 } 1916 return (ZIO_PIPELINE_CONTINUE); 1917 } 1918 1919 zio_nowait(zio_read(zio, zio->io_spa, bp, 1920 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 1921 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 1922 1923 return (ZIO_PIPELINE_CONTINUE); 1924 } 1925 1926 static int 1927 zio_ddt_read_done(zio_t *zio) 1928 { 1929 blkptr_t *bp = zio->io_bp; 1930 1931 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 1932 return (ZIO_PIPELINE_STOP); 1933 1934 ASSERT(BP_GET_DEDUP(bp)); 1935 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 1936 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1937 1938 if (zio->io_child_error[ZIO_CHILD_DDT]) { 1939 ddt_t *ddt = ddt_select(zio->io_spa, bp); 1940 ddt_entry_t *dde = zio->io_vsd; 1941 if (ddt == NULL) { 1942 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 1943 return (ZIO_PIPELINE_CONTINUE); 1944 } 1945 if (dde == NULL) { 1946 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 1947 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1948 return (ZIO_PIPELINE_STOP); 1949 } 1950 if (dde->dde_repair_data != NULL) { 1951 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 1952 zio->io_child_error[ZIO_CHILD_DDT] = 0; 1953 } 1954 ddt_repair_done(ddt, dde); 1955 zio->io_vsd = NULL; 1956 } 1957 1958 ASSERT(zio->io_vsd == NULL); 1959 1960 return (ZIO_PIPELINE_CONTINUE); 1961 } 1962 1963 static boolean_t 1964 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 1965 { 1966 spa_t *spa = zio->io_spa; 1967 1968 /* 1969 * Note: we compare the original data, not the transformed data, 1970 * because when zio->io_bp is an override bp, we will not have 1971 * pushed the I/O transforms. That's an important optimization 1972 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 1973 */ 1974 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 1975 zio_t *lio = dde->dde_lead_zio[p]; 1976 1977 if (lio != NULL) { 1978 return (lio->io_orig_size != zio->io_orig_size || 1979 bcmp(zio->io_orig_data, lio->io_orig_data, 1980 zio->io_orig_size) != 0); 1981 } 1982 } 1983 1984 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 1985 ddt_phys_t *ddp = &dde->dde_phys[p]; 1986 1987 if (ddp->ddp_phys_birth != 0) { 1988 arc_buf_t *abuf = NULL; 1989 uint32_t aflags = ARC_WAIT; 1990 blkptr_t blk = *zio->io_bp; 1991 int error; 1992 1993 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 1994 1995 ddt_exit(ddt); 1996 1997 error = arc_read_nolock(NULL, spa, &blk, 1998 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 1999 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2000 &aflags, &zio->io_bookmark); 2001 2002 if (error == 0) { 2003 if (arc_buf_size(abuf) != zio->io_orig_size || 2004 bcmp(abuf->b_data, zio->io_orig_data, 2005 zio->io_orig_size) != 0) 2006 error = EEXIST; 2007 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 2008 } 2009 2010 ddt_enter(ddt); 2011 return (error != 0); 2012 } 2013 } 2014 2015 return (B_FALSE); 2016 } 2017 2018 static void 2019 zio_ddt_child_write_ready(zio_t *zio) 2020 { 2021 int p = zio->io_prop.zp_copies; 2022 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2023 ddt_entry_t *dde = zio->io_private; 2024 ddt_phys_t *ddp = &dde->dde_phys[p]; 2025 zio_t *pio; 2026 2027 if (zio->io_error) 2028 return; 2029 2030 ddt_enter(ddt); 2031 2032 ASSERT(dde->dde_lead_zio[p] == zio); 2033 2034 ddt_phys_fill(ddp, zio->io_bp); 2035 2036 while ((pio = zio_walk_parents(zio)) != NULL) 2037 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2038 2039 ddt_exit(ddt); 2040 } 2041 2042 static void 2043 zio_ddt_child_write_done(zio_t *zio) 2044 { 2045 int p = zio->io_prop.zp_copies; 2046 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2047 ddt_entry_t *dde = zio->io_private; 2048 ddt_phys_t *ddp = &dde->dde_phys[p]; 2049 2050 ddt_enter(ddt); 2051 2052 ASSERT(ddp->ddp_refcnt == 0); 2053 ASSERT(dde->dde_lead_zio[p] == zio); 2054 dde->dde_lead_zio[p] = NULL; 2055 2056 if (zio->io_error == 0) { 2057 while (zio_walk_parents(zio) != NULL) 2058 ddt_phys_addref(ddp); 2059 } else { 2060 ddt_phys_clear(ddp); 2061 } 2062 2063 ddt_exit(ddt); 2064 } 2065 2066 static void 2067 zio_ddt_ditto_write_done(zio_t *zio) 2068 { 2069 int p = DDT_PHYS_DITTO; 2070 zio_prop_t *zp = &zio->io_prop; 2071 blkptr_t *bp = zio->io_bp; 2072 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2073 ddt_entry_t *dde = zio->io_private; 2074 ddt_phys_t *ddp = &dde->dde_phys[p]; 2075 ddt_key_t *ddk = &dde->dde_key; 2076 2077 ddt_enter(ddt); 2078 2079 ASSERT(ddp->ddp_refcnt == 0); 2080 ASSERT(dde->dde_lead_zio[p] == zio); 2081 dde->dde_lead_zio[p] = NULL; 2082 2083 if (zio->io_error == 0) { 2084 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2085 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2086 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2087 if (ddp->ddp_phys_birth != 0) 2088 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2089 ddt_phys_fill(ddp, bp); 2090 } 2091 2092 ddt_exit(ddt); 2093 } 2094 2095 static int 2096 zio_ddt_write(zio_t *zio) 2097 { 2098 spa_t *spa = zio->io_spa; 2099 blkptr_t *bp = zio->io_bp; 2100 uint64_t txg = zio->io_txg; 2101 zio_prop_t *zp = &zio->io_prop; 2102 int p = zp->zp_copies; 2103 int ditto_copies; 2104 zio_t *cio = NULL; 2105 zio_t *dio = NULL; 2106 ddt_t *ddt = ddt_select(spa, bp); 2107 ddt_entry_t *dde; 2108 ddt_phys_t *ddp; 2109 2110 ASSERT(BP_GET_DEDUP(bp)); 2111 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2112 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2113 2114 ddt_enter(ddt); 2115 dde = ddt_lookup(ddt, bp, B_TRUE); 2116 ddp = &dde->dde_phys[p]; 2117 2118 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2119 /* 2120 * If we're using a weak checksum, upgrade to a strong checksum 2121 * and try again. If we're already using a strong checksum, 2122 * we can't resolve it, so just convert to an ordinary write. 2123 * (And automatically e-mail a paper to Nature?) 2124 */ 2125 if (!zio_checksum_table[zp->zp_checksum].ci_dedup) { 2126 zp->zp_checksum = spa_dedup_checksum(spa); 2127 zio_pop_transforms(zio); 2128 zio->io_stage = ZIO_STAGE_OPEN; 2129 BP_ZERO(bp); 2130 } else { 2131 zp->zp_dedup = B_FALSE; 2132 } 2133 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2134 ddt_exit(ddt); 2135 return (ZIO_PIPELINE_CONTINUE); 2136 } 2137 2138 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2139 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2140 2141 if (ditto_copies > ddt_ditto_copies_present(dde) && 2142 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2143 zio_prop_t czp = *zp; 2144 2145 czp.zp_copies = ditto_copies; 2146 2147 /* 2148 * If we arrived here with an override bp, we won't have run 2149 * the transform stack, so we won't have the data we need to 2150 * generate a child i/o. So, toss the override bp and restart. 2151 * This is safe, because using the override bp is just an 2152 * optimization; and it's rare, so the cost doesn't matter. 2153 */ 2154 if (zio->io_bp_override) { 2155 zio_pop_transforms(zio); 2156 zio->io_stage = ZIO_STAGE_OPEN; 2157 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2158 zio->io_bp_override = NULL; 2159 BP_ZERO(bp); 2160 ddt_exit(ddt); 2161 return (ZIO_PIPELINE_CONTINUE); 2162 } 2163 2164 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2165 zio->io_orig_size, &czp, NULL, 2166 zio_ddt_ditto_write_done, dde, zio->io_priority, 2167 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2168 2169 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2170 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2171 } 2172 2173 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2174 if (ddp->ddp_phys_birth != 0) 2175 ddt_bp_fill(ddp, bp, txg); 2176 if (dde->dde_lead_zio[p] != NULL) 2177 zio_add_child(zio, dde->dde_lead_zio[p]); 2178 else 2179 ddt_phys_addref(ddp); 2180 } else if (zio->io_bp_override) { 2181 ASSERT(bp->blk_birth == txg); 2182 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2183 ddt_phys_fill(ddp, bp); 2184 ddt_phys_addref(ddp); 2185 } else { 2186 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2187 zio->io_orig_size, zp, zio_ddt_child_write_ready, 2188 zio_ddt_child_write_done, dde, zio->io_priority, 2189 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2190 2191 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2192 dde->dde_lead_zio[p] = cio; 2193 } 2194 2195 ddt_exit(ddt); 2196 2197 if (cio) 2198 zio_nowait(cio); 2199 if (dio) 2200 zio_nowait(dio); 2201 2202 return (ZIO_PIPELINE_CONTINUE); 2203 } 2204 2205 ddt_entry_t *freedde; /* for debugging */ 2206 2207 static int 2208 zio_ddt_free(zio_t *zio) 2209 { 2210 spa_t *spa = zio->io_spa; 2211 blkptr_t *bp = zio->io_bp; 2212 ddt_t *ddt = ddt_select(spa, bp); 2213 ddt_entry_t *dde; 2214 ddt_phys_t *ddp; 2215 2216 ASSERT(BP_GET_DEDUP(bp)); 2217 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2218 2219 ddt_enter(ddt); 2220 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2221 ddp = ddt_phys_select(dde, bp); 2222 ddt_phys_decref(ddp); 2223 ddt_exit(ddt); 2224 2225 return (ZIO_PIPELINE_CONTINUE); 2226 } 2227 2228 /* 2229 * ========================================================================== 2230 * Allocate and free blocks 2231 * ========================================================================== 2232 */ 2233 static int 2234 zio_dva_allocate(zio_t *zio) 2235 { 2236 spa_t *spa = zio->io_spa; 2237 metaslab_class_t *mc = spa_normal_class(spa); 2238 blkptr_t *bp = zio->io_bp; 2239 int error; 2240 int flags = 0; 2241 2242 if (zio->io_gang_leader == NULL) { 2243 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2244 zio->io_gang_leader = zio; 2245 } 2246 2247 ASSERT(BP_IS_HOLE(bp)); 2248 ASSERT0(BP_GET_NDVAS(bp)); 2249 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2250 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2251 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2252 2253 /* 2254 * The dump device does not support gang blocks so allocation on 2255 * behalf of the dump device (i.e. ZIO_FLAG_NODATA) must avoid 2256 * the "fast" gang feature. 2257 */ 2258 flags |= (zio->io_flags & ZIO_FLAG_NODATA) ? METASLAB_GANG_AVOID : 0; 2259 flags |= (zio->io_flags & ZIO_FLAG_GANG_CHILD) ? 2260 METASLAB_GANG_CHILD : 0; 2261 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2262 zio->io_prop.zp_copies, zio->io_txg, NULL, flags); 2263 2264 if (error) { 2265 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2266 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2267 error); 2268 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2269 return (zio_write_gang_block(zio)); 2270 zio->io_error = error; 2271 } 2272 2273 return (ZIO_PIPELINE_CONTINUE); 2274 } 2275 2276 static int 2277 zio_dva_free(zio_t *zio) 2278 { 2279 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2280 2281 return (ZIO_PIPELINE_CONTINUE); 2282 } 2283 2284 static int 2285 zio_dva_claim(zio_t *zio) 2286 { 2287 int error; 2288 2289 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2290 if (error) 2291 zio->io_error = error; 2292 2293 return (ZIO_PIPELINE_CONTINUE); 2294 } 2295 2296 /* 2297 * Undo an allocation. This is used by zio_done() when an I/O fails 2298 * and we want to give back the block we just allocated. 2299 * This handles both normal blocks and gang blocks. 2300 */ 2301 static void 2302 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2303 { 2304 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2305 ASSERT(zio->io_bp_override == NULL); 2306 2307 if (!BP_IS_HOLE(bp)) 2308 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2309 2310 if (gn != NULL) { 2311 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2312 zio_dva_unallocate(zio, gn->gn_child[g], 2313 &gn->gn_gbh->zg_blkptr[g]); 2314 } 2315 } 2316 } 2317 2318 /* 2319 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2320 */ 2321 int 2322 zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2323 uint64_t size, boolean_t use_slog) 2324 { 2325 int error = 1; 2326 2327 ASSERT(txg > spa_syncing_txg(spa)); 2328 2329 /* 2330 * ZIL blocks are always contiguous (i.e. not gang blocks) so we 2331 * set the METASLAB_GANG_AVOID flag so that they don't "fast gang" 2332 * when allocating them. 2333 */ 2334 if (use_slog) { 2335 error = metaslab_alloc(spa, spa_log_class(spa), size, 2336 new_bp, 1, txg, old_bp, 2337 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2338 } 2339 2340 if (error) { 2341 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2342 new_bp, 1, txg, old_bp, 2343 METASLAB_HINTBP_AVOID | METASLAB_GANG_AVOID); 2344 } 2345 2346 if (error == 0) { 2347 BP_SET_LSIZE(new_bp, size); 2348 BP_SET_PSIZE(new_bp, size); 2349 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2350 BP_SET_CHECKSUM(new_bp, 2351 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2352 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2353 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2354 BP_SET_LEVEL(new_bp, 0); 2355 BP_SET_DEDUP(new_bp, 0); 2356 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2357 } 2358 2359 return (error); 2360 } 2361 2362 /* 2363 * Free an intent log block. 2364 */ 2365 void 2366 zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2367 { 2368 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2369 ASSERT(!BP_IS_GANG(bp)); 2370 2371 zio_free(spa, txg, bp); 2372 } 2373 2374 /* 2375 * ========================================================================== 2376 * Read and write to physical devices 2377 * ========================================================================== 2378 */ 2379 static int 2380 zio_vdev_io_start(zio_t *zio) 2381 { 2382 vdev_t *vd = zio->io_vd; 2383 uint64_t align; 2384 spa_t *spa = zio->io_spa; 2385 2386 ASSERT(zio->io_error == 0); 2387 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2388 2389 if (vd == NULL) { 2390 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2391 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2392 2393 /* 2394 * The mirror_ops handle multiple DVAs in a single BP. 2395 */ 2396 return (vdev_mirror_ops.vdev_op_io_start(zio)); 2397 } 2398 2399 /* 2400 * We keep track of time-sensitive I/Os so that the scan thread 2401 * can quickly react to certain workloads. In particular, we care 2402 * about non-scrubbing, top-level reads and writes with the following 2403 * characteristics: 2404 * - synchronous writes of user data to non-slog devices 2405 * - any reads of user data 2406 * When these conditions are met, adjust the timestamp of spa_last_io 2407 * which allows the scan thread to adjust its workload accordingly. 2408 */ 2409 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2410 vd == vd->vdev_top && !vd->vdev_islog && 2411 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2412 zio->io_txg != spa_syncing_txg(spa)) { 2413 uint64_t old = spa->spa_last_io; 2414 uint64_t new = ddi_get_lbolt64(); 2415 if (old != new) 2416 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2417 } 2418 2419 align = 1ULL << vd->vdev_top->vdev_ashift; 2420 2421 if (P2PHASE(zio->io_size, align) != 0) { 2422 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2423 char *abuf = zio_buf_alloc(asize); 2424 ASSERT(vd == vd->vdev_top); 2425 if (zio->io_type == ZIO_TYPE_WRITE) { 2426 bcopy(zio->io_data, abuf, zio->io_size); 2427 bzero(abuf + zio->io_size, asize - zio->io_size); 2428 } 2429 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 2430 } 2431 2432 ASSERT(P2PHASE(zio->io_offset, align) == 0); 2433 ASSERT(P2PHASE(zio->io_size, align) == 0); 2434 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 2435 2436 /* 2437 * If this is a repair I/O, and there's no self-healing involved -- 2438 * that is, we're just resilvering what we expect to resilver -- 2439 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2440 * This prevents spurious resilvering with nested replication. 2441 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2442 * A is out of date, we'll read from C+D, then use the data to 2443 * resilver A+B -- but we don't actually want to resilver B, just A. 2444 * The top-level mirror has no way to know this, so instead we just 2445 * discard unnecessary repairs as we work our way down the vdev tree. 2446 * The same logic applies to any form of nested replication: 2447 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2448 */ 2449 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2450 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2451 zio->io_txg != 0 && /* not a delegated i/o */ 2452 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2453 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2454 zio_vdev_io_bypass(zio); 2455 return (ZIO_PIPELINE_CONTINUE); 2456 } 2457 2458 if (vd->vdev_ops->vdev_op_leaf && 2459 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2460 2461 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio) == 0) 2462 return (ZIO_PIPELINE_CONTINUE); 2463 2464 if ((zio = vdev_queue_io(zio)) == NULL) 2465 return (ZIO_PIPELINE_STOP); 2466 2467 if (!vdev_accessible(vd, zio)) { 2468 zio->io_error = ENXIO; 2469 zio_interrupt(zio); 2470 return (ZIO_PIPELINE_STOP); 2471 } 2472 } 2473 2474 return (vd->vdev_ops->vdev_op_io_start(zio)); 2475 } 2476 2477 static int 2478 zio_vdev_io_done(zio_t *zio) 2479 { 2480 vdev_t *vd = zio->io_vd; 2481 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 2482 boolean_t unexpected_error = B_FALSE; 2483 2484 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2485 return (ZIO_PIPELINE_STOP); 2486 2487 ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); 2488 2489 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) { 2490 2491 vdev_queue_io_done(zio); 2492 2493 if (zio->io_type == ZIO_TYPE_WRITE) 2494 vdev_cache_write(zio); 2495 2496 if (zio_injection_enabled && zio->io_error == 0) 2497 zio->io_error = zio_handle_device_injection(vd, 2498 zio, EIO); 2499 2500 if (zio_injection_enabled && zio->io_error == 0) 2501 zio->io_error = zio_handle_label_injection(zio, EIO); 2502 2503 if (zio->io_error) { 2504 if (!vdev_accessible(vd, zio)) { 2505 zio->io_error = ENXIO; 2506 } else { 2507 unexpected_error = B_TRUE; 2508 } 2509 } 2510 } 2511 2512 ops->vdev_op_io_done(zio); 2513 2514 if (unexpected_error) 2515 VERIFY(vdev_probe(vd, zio) == NULL); 2516 2517 return (ZIO_PIPELINE_CONTINUE); 2518 } 2519 2520 /* 2521 * For non-raidz ZIOs, we can just copy aside the bad data read from the 2522 * disk, and use that to finish the checksum ereport later. 2523 */ 2524 static void 2525 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 2526 const void *good_buf) 2527 { 2528 /* no processing needed */ 2529 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 2530 } 2531 2532 /*ARGSUSED*/ 2533 void 2534 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 2535 { 2536 void *buf = zio_buf_alloc(zio->io_size); 2537 2538 bcopy(zio->io_data, buf, zio->io_size); 2539 2540 zcr->zcr_cbinfo = zio->io_size; 2541 zcr->zcr_cbdata = buf; 2542 zcr->zcr_finish = zio_vsd_default_cksum_finish; 2543 zcr->zcr_free = zio_buf_free; 2544 } 2545 2546 static int 2547 zio_vdev_io_assess(zio_t *zio) 2548 { 2549 vdev_t *vd = zio->io_vd; 2550 2551 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 2552 return (ZIO_PIPELINE_STOP); 2553 2554 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2555 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 2556 2557 if (zio->io_vsd != NULL) { 2558 zio->io_vsd_ops->vsd_free(zio); 2559 zio->io_vsd = NULL; 2560 } 2561 2562 if (zio_injection_enabled && zio->io_error == 0) 2563 zio->io_error = zio_handle_fault_injection(zio, EIO); 2564 2565 /* 2566 * If the I/O failed, determine whether we should attempt to retry it. 2567 * 2568 * On retry, we cut in line in the issue queue, since we don't want 2569 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 2570 */ 2571 if (zio->io_error && vd == NULL && 2572 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 2573 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 2574 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 2575 zio->io_error = 0; 2576 zio->io_flags |= ZIO_FLAG_IO_RETRY | 2577 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 2578 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 2579 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 2580 zio_requeue_io_start_cut_in_line); 2581 return (ZIO_PIPELINE_STOP); 2582 } 2583 2584 /* 2585 * If we got an error on a leaf device, convert it to ENXIO 2586 * if the device is not accessible at all. 2587 */ 2588 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 2589 !vdev_accessible(vd, zio)) 2590 zio->io_error = ENXIO; 2591 2592 /* 2593 * If we can't write to an interior vdev (mirror or RAID-Z), 2594 * set vdev_cant_write so that we stop trying to allocate from it. 2595 */ 2596 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 2597 vd != NULL && !vd->vdev_ops->vdev_op_leaf) 2598 vd->vdev_cant_write = B_TRUE; 2599 2600 if (zio->io_error) 2601 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2602 2603 return (ZIO_PIPELINE_CONTINUE); 2604 } 2605 2606 void 2607 zio_vdev_io_reissue(zio_t *zio) 2608 { 2609 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2610 ASSERT(zio->io_error == 0); 2611 2612 zio->io_stage >>= 1; 2613 } 2614 2615 void 2616 zio_vdev_io_redone(zio_t *zio) 2617 { 2618 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 2619 2620 zio->io_stage >>= 1; 2621 } 2622 2623 void 2624 zio_vdev_io_bypass(zio_t *zio) 2625 { 2626 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 2627 ASSERT(zio->io_error == 0); 2628 2629 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 2630 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 2631 } 2632 2633 /* 2634 * ========================================================================== 2635 * Generate and verify checksums 2636 * ========================================================================== 2637 */ 2638 static int 2639 zio_checksum_generate(zio_t *zio) 2640 { 2641 blkptr_t *bp = zio->io_bp; 2642 enum zio_checksum checksum; 2643 2644 if (bp == NULL) { 2645 /* 2646 * This is zio_write_phys(). 2647 * We're either generating a label checksum, or none at all. 2648 */ 2649 checksum = zio->io_prop.zp_checksum; 2650 2651 if (checksum == ZIO_CHECKSUM_OFF) 2652 return (ZIO_PIPELINE_CONTINUE); 2653 2654 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 2655 } else { 2656 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 2657 ASSERT(!IO_IS_ALLOCATING(zio)); 2658 checksum = ZIO_CHECKSUM_GANG_HEADER; 2659 } else { 2660 checksum = BP_GET_CHECKSUM(bp); 2661 } 2662 } 2663 2664 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 2665 2666 return (ZIO_PIPELINE_CONTINUE); 2667 } 2668 2669 static int 2670 zio_checksum_verify(zio_t *zio) 2671 { 2672 zio_bad_cksum_t info; 2673 blkptr_t *bp = zio->io_bp; 2674 int error; 2675 2676 ASSERT(zio->io_vd != NULL); 2677 2678 if (bp == NULL) { 2679 /* 2680 * This is zio_read_phys(). 2681 * We're either verifying a label checksum, or nothing at all. 2682 */ 2683 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 2684 return (ZIO_PIPELINE_CONTINUE); 2685 2686 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 2687 } 2688 2689 if ((error = zio_checksum_error(zio, &info)) != 0) { 2690 zio->io_error = error; 2691 if (!(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 2692 zfs_ereport_start_checksum(zio->io_spa, 2693 zio->io_vd, zio, zio->io_offset, 2694 zio->io_size, NULL, &info); 2695 } 2696 } 2697 2698 return (ZIO_PIPELINE_CONTINUE); 2699 } 2700 2701 /* 2702 * Called by RAID-Z to ensure we don't compute the checksum twice. 2703 */ 2704 void 2705 zio_checksum_verified(zio_t *zio) 2706 { 2707 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 2708 } 2709 2710 /* 2711 * ========================================================================== 2712 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 2713 * An error of 0 indictes success. ENXIO indicates whole-device failure, 2714 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 2715 * indicate errors that are specific to one I/O, and most likely permanent. 2716 * Any other error is presumed to be worse because we weren't expecting it. 2717 * ========================================================================== 2718 */ 2719 int 2720 zio_worst_error(int e1, int e2) 2721 { 2722 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 2723 int r1, r2; 2724 2725 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 2726 if (e1 == zio_error_rank[r1]) 2727 break; 2728 2729 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 2730 if (e2 == zio_error_rank[r2]) 2731 break; 2732 2733 return (r1 > r2 ? e1 : e2); 2734 } 2735 2736 /* 2737 * ========================================================================== 2738 * I/O completion 2739 * ========================================================================== 2740 */ 2741 static int 2742 zio_ready(zio_t *zio) 2743 { 2744 blkptr_t *bp = zio->io_bp; 2745 zio_t *pio, *pio_next; 2746 2747 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 2748 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 2749 return (ZIO_PIPELINE_STOP); 2750 2751 if (zio->io_ready) { 2752 ASSERT(IO_IS_ALLOCATING(zio)); 2753 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 2754 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 2755 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 2756 2757 zio->io_ready(zio); 2758 } 2759 2760 if (bp != NULL && bp != &zio->io_bp_copy) 2761 zio->io_bp_copy = *bp; 2762 2763 if (zio->io_error) 2764 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2765 2766 mutex_enter(&zio->io_lock); 2767 zio->io_state[ZIO_WAIT_READY] = 1; 2768 pio = zio_walk_parents(zio); 2769 mutex_exit(&zio->io_lock); 2770 2771 /* 2772 * As we notify zio's parents, new parents could be added. 2773 * New parents go to the head of zio's io_parent_list, however, 2774 * so we will (correctly) not notify them. The remainder of zio's 2775 * io_parent_list, from 'pio_next' onward, cannot change because 2776 * all parents must wait for us to be done before they can be done. 2777 */ 2778 for (; pio != NULL; pio = pio_next) { 2779 pio_next = zio_walk_parents(zio); 2780 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 2781 } 2782 2783 if (zio->io_flags & ZIO_FLAG_NODATA) { 2784 if (BP_IS_GANG(bp)) { 2785 zio->io_flags &= ~ZIO_FLAG_NODATA; 2786 } else { 2787 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 2788 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2789 } 2790 } 2791 2792 if (zio_injection_enabled && 2793 zio->io_spa->spa_syncing_txg == zio->io_txg) 2794 zio_handle_ignored_writes(zio); 2795 2796 return (ZIO_PIPELINE_CONTINUE); 2797 } 2798 2799 static int 2800 zio_done(zio_t *zio) 2801 { 2802 spa_t *spa = zio->io_spa; 2803 zio_t *lio = zio->io_logical; 2804 blkptr_t *bp = zio->io_bp; 2805 vdev_t *vd = zio->io_vd; 2806 uint64_t psize = zio->io_size; 2807 zio_t *pio, *pio_next; 2808 2809 /* 2810 * If our children haven't all completed, 2811 * wait for them and then repeat this pipeline stage. 2812 */ 2813 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 2814 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 2815 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 2816 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 2817 return (ZIO_PIPELINE_STOP); 2818 2819 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 2820 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 2821 ASSERT(zio->io_children[c][w] == 0); 2822 2823 if (bp != NULL) { 2824 ASSERT(bp->blk_pad[0] == 0); 2825 ASSERT(bp->blk_pad[1] == 0); 2826 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 2827 (bp == zio_unique_parent(zio)->io_bp)); 2828 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 2829 zio->io_bp_override == NULL && 2830 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 2831 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 2832 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 2833 ASSERT(BP_COUNT_GANG(bp) == 0 || 2834 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 2835 } 2836 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 2837 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 2838 } 2839 2840 /* 2841 * If there were child vdev/gang/ddt errors, they apply to us now. 2842 */ 2843 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 2844 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 2845 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 2846 2847 /* 2848 * If the I/O on the transformed data was successful, generate any 2849 * checksum reports now while we still have the transformed data. 2850 */ 2851 if (zio->io_error == 0) { 2852 while (zio->io_cksum_report != NULL) { 2853 zio_cksum_report_t *zcr = zio->io_cksum_report; 2854 uint64_t align = zcr->zcr_align; 2855 uint64_t asize = P2ROUNDUP(psize, align); 2856 char *abuf = zio->io_data; 2857 2858 if (asize != psize) { 2859 abuf = zio_buf_alloc(asize); 2860 bcopy(zio->io_data, abuf, psize); 2861 bzero(abuf + psize, asize - psize); 2862 } 2863 2864 zio->io_cksum_report = zcr->zcr_next; 2865 zcr->zcr_next = NULL; 2866 zcr->zcr_finish(zcr, abuf); 2867 zfs_ereport_free_checksum(zcr); 2868 2869 if (asize != psize) 2870 zio_buf_free(abuf, asize); 2871 } 2872 } 2873 2874 zio_pop_transforms(zio); /* note: may set zio->io_error */ 2875 2876 vdev_stat_update(zio, psize); 2877 2878 if (zio->io_error) { 2879 /* 2880 * If this I/O is attached to a particular vdev, 2881 * generate an error message describing the I/O failure 2882 * at the block level. We ignore these errors if the 2883 * device is currently unavailable. 2884 */ 2885 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 2886 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 2887 2888 if ((zio->io_error == EIO || !(zio->io_flags & 2889 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 2890 zio == lio) { 2891 /* 2892 * For logical I/O requests, tell the SPA to log the 2893 * error and generate a logical data ereport. 2894 */ 2895 spa_log_error(spa, zio); 2896 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 2897 0, 0); 2898 } 2899 } 2900 2901 if (zio->io_error && zio == lio) { 2902 /* 2903 * Determine whether zio should be reexecuted. This will 2904 * propagate all the way to the root via zio_notify_parent(). 2905 */ 2906 ASSERT(vd == NULL && bp != NULL); 2907 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2908 2909 if (IO_IS_ALLOCATING(zio) && 2910 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 2911 if (zio->io_error != ENOSPC) 2912 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 2913 else 2914 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2915 } 2916 2917 if ((zio->io_type == ZIO_TYPE_READ || 2918 zio->io_type == ZIO_TYPE_FREE) && 2919 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 2920 zio->io_error == ENXIO && 2921 spa_load_state(spa) == SPA_LOAD_NONE && 2922 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 2923 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2924 2925 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 2926 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 2927 2928 /* 2929 * Here is a possibly good place to attempt to do 2930 * either combinatorial reconstruction or error correction 2931 * based on checksums. It also might be a good place 2932 * to send out preliminary ereports before we suspend 2933 * processing. 2934 */ 2935 } 2936 2937 /* 2938 * If there were logical child errors, they apply to us now. 2939 * We defer this until now to avoid conflating logical child 2940 * errors with errors that happened to the zio itself when 2941 * updating vdev stats and reporting FMA events above. 2942 */ 2943 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 2944 2945 if ((zio->io_error || zio->io_reexecute) && 2946 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 2947 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 2948 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 2949 2950 zio_gang_tree_free(&zio->io_gang_tree); 2951 2952 /* 2953 * Godfather I/Os should never suspend. 2954 */ 2955 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 2956 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 2957 zio->io_reexecute = 0; 2958 2959 if (zio->io_reexecute) { 2960 /* 2961 * This is a logical I/O that wants to reexecute. 2962 * 2963 * Reexecute is top-down. When an i/o fails, if it's not 2964 * the root, it simply notifies its parent and sticks around. 2965 * The parent, seeing that it still has children in zio_done(), 2966 * does the same. This percolates all the way up to the root. 2967 * The root i/o will reexecute or suspend the entire tree. 2968 * 2969 * This approach ensures that zio_reexecute() honors 2970 * all the original i/o dependency relationships, e.g. 2971 * parents not executing until children are ready. 2972 */ 2973 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2974 2975 zio->io_gang_leader = NULL; 2976 2977 mutex_enter(&zio->io_lock); 2978 zio->io_state[ZIO_WAIT_DONE] = 1; 2979 mutex_exit(&zio->io_lock); 2980 2981 /* 2982 * "The Godfather" I/O monitors its children but is 2983 * not a true parent to them. It will track them through 2984 * the pipeline but severs its ties whenever they get into 2985 * trouble (e.g. suspended). This allows "The Godfather" 2986 * I/O to return status without blocking. 2987 */ 2988 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 2989 zio_link_t *zl = zio->io_walk_link; 2990 pio_next = zio_walk_parents(zio); 2991 2992 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 2993 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 2994 zio_remove_child(pio, zio, zl); 2995 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 2996 } 2997 } 2998 2999 if ((pio = zio_unique_parent(zio)) != NULL) { 3000 /* 3001 * We're not a root i/o, so there's nothing to do 3002 * but notify our parent. Don't propagate errors 3003 * upward since we haven't permanently failed yet. 3004 */ 3005 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 3006 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 3007 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3008 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 3009 /* 3010 * We'd fail again if we reexecuted now, so suspend 3011 * until conditions improve (e.g. device comes online). 3012 */ 3013 zio_suspend(spa, zio); 3014 } else { 3015 /* 3016 * Reexecution is potentially a huge amount of work. 3017 * Hand it off to the otherwise-unused claim taskq. 3018 */ 3019 ASSERT(zio->io_tqent.tqent_next == NULL); 3020 taskq_dispatch_ent( 3021 spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE], 3022 (task_func_t *)zio_reexecute, zio, 0, 3023 &zio->io_tqent); 3024 } 3025 return (ZIO_PIPELINE_STOP); 3026 } 3027 3028 ASSERT(zio->io_child_count == 0); 3029 ASSERT(zio->io_reexecute == 0); 3030 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 3031 3032 /* 3033 * Report any checksum errors, since the I/O is complete. 3034 */ 3035 while (zio->io_cksum_report != NULL) { 3036 zio_cksum_report_t *zcr = zio->io_cksum_report; 3037 zio->io_cksum_report = zcr->zcr_next; 3038 zcr->zcr_next = NULL; 3039 zcr->zcr_finish(zcr, NULL); 3040 zfs_ereport_free_checksum(zcr); 3041 } 3042 3043 /* 3044 * It is the responsibility of the done callback to ensure that this 3045 * particular zio is no longer discoverable for adoption, and as 3046 * such, cannot acquire any new parents. 3047 */ 3048 if (zio->io_done) 3049 zio->io_done(zio); 3050 3051 mutex_enter(&zio->io_lock); 3052 zio->io_state[ZIO_WAIT_DONE] = 1; 3053 mutex_exit(&zio->io_lock); 3054 3055 for (pio = zio_walk_parents(zio); pio != NULL; pio = pio_next) { 3056 zio_link_t *zl = zio->io_walk_link; 3057 pio_next = zio_walk_parents(zio); 3058 zio_remove_child(pio, zio, zl); 3059 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3060 } 3061 3062 if (zio->io_waiter != NULL) { 3063 mutex_enter(&zio->io_lock); 3064 zio->io_executor = NULL; 3065 cv_broadcast(&zio->io_cv); 3066 mutex_exit(&zio->io_lock); 3067 } else { 3068 zio_destroy(zio); 3069 } 3070 3071 return (ZIO_PIPELINE_STOP); 3072 } 3073 3074 /* 3075 * ========================================================================== 3076 * I/O pipeline definition 3077 * ========================================================================== 3078 */ 3079 static zio_pipe_stage_t *zio_pipeline[] = { 3080 NULL, 3081 zio_read_bp_init, 3082 zio_free_bp_init, 3083 zio_issue_async, 3084 zio_write_bp_init, 3085 zio_checksum_generate, 3086 zio_nop_write, 3087 zio_ddt_read_start, 3088 zio_ddt_read_done, 3089 zio_ddt_write, 3090 zio_ddt_free, 3091 zio_gang_assemble, 3092 zio_gang_issue, 3093 zio_dva_allocate, 3094 zio_dva_free, 3095 zio_dva_claim, 3096 zio_ready, 3097 zio_vdev_io_start, 3098 zio_vdev_io_done, 3099 zio_vdev_io_assess, 3100 zio_checksum_verify, 3101 zio_done 3102 }; 3103 3104 /* dnp is the dnode for zb1->zb_object */ 3105 boolean_t 3106 zbookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1, 3107 const zbookmark_t *zb2) 3108 { 3109 uint64_t zb1nextL0, zb2thisobj; 3110 3111 ASSERT(zb1->zb_objset == zb2->zb_objset); 3112 ASSERT(zb2->zb_level == 0); 3113 3114 /* 3115 * A bookmark in the deadlist is considered to be after 3116 * everything else. 3117 */ 3118 if (zb2->zb_object == DMU_DEADLIST_OBJECT) 3119 return (B_TRUE); 3120 3121 /* The objset_phys_t isn't before anything. */ 3122 if (dnp == NULL) 3123 return (B_FALSE); 3124 3125 zb1nextL0 = (zb1->zb_blkid + 1) << 3126 ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)); 3127 3128 zb2thisobj = zb2->zb_object ? zb2->zb_object : 3129 zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT); 3130 3131 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 3132 uint64_t nextobj = zb1nextL0 * 3133 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT; 3134 return (nextobj <= zb2thisobj); 3135 } 3136 3137 if (zb1->zb_object < zb2thisobj) 3138 return (B_TRUE); 3139 if (zb1->zb_object > zb2thisobj) 3140 return (B_FALSE); 3141 if (zb2->zb_object == DMU_META_DNODE_OBJECT) 3142 return (B_FALSE); 3143 return (zb1nextL0 <= zb2->zb_blkid); 3144 } 3145