1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2014 Integros [integros.com] 26 * Copyright (c) 2017, Intel Corporation. 27 */ 28 29 #include <sys/sysmacros.h> 30 #include <sys/zfs_context.h> 31 #include <sys/fm/fs/zfs.h> 32 #include <sys/spa.h> 33 #include <sys/txg.h> 34 #include <sys/spa_impl.h> 35 #include <sys/vdev_impl.h> 36 #include <sys/vdev_trim.h> 37 #include <sys/zio_impl.h> 38 #include <sys/zio_compress.h> 39 #include <sys/zio_checksum.h> 40 #include <sys/dmu_objset.h> 41 #include <sys/arc.h> 42 #include <sys/ddt.h> 43 #include <sys/blkptr.h> 44 #include <sys/zfeature.h> 45 #include <sys/dsl_scan.h> 46 #include <sys/metaslab_impl.h> 47 #include <sys/abd.h> 48 #include <sys/cityhash.h> 49 #include <sys/dsl_crypt.h> 50 51 /* 52 * ========================================================================== 53 * I/O type descriptions 54 * ========================================================================== 55 */ 56 const char *zio_type_name[ZIO_TYPES] = { 57 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 58 "zio_ioctl", "z_trim" 59 }; 60 61 boolean_t zio_dva_throttle_enabled = B_TRUE; 62 63 /* 64 * ========================================================================== 65 * I/O kmem caches 66 * ========================================================================== 67 */ 68 kmem_cache_t *zio_cache; 69 kmem_cache_t *zio_link_cache; 70 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 71 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 72 73 #ifdef _KERNEL 74 extern vmem_t *zio_alloc_arena; 75 #endif 76 77 #define ZIO_PIPELINE_CONTINUE 0x100 78 #define ZIO_PIPELINE_STOP 0x101 79 80 #define BP_SPANB(indblkshift, level) \ 81 (((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT))) 82 #define COMPARE_META_LEVEL 0x80000000ul 83 /* 84 * The following actions directly effect the spa's sync-to-convergence logic. 85 * The values below define the sync pass when we start performing the action. 86 * Care should be taken when changing these values as they directly impact 87 * spa_sync() performance. Tuning these values may introduce subtle performance 88 * pathologies and should only be done in the context of performance analysis. 89 * These tunables will eventually be removed and replaced with #defines once 90 * enough analysis has been done to determine optimal values. 91 * 92 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 93 * regular blocks are not deferred. 94 */ 95 int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 96 int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 97 int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 98 99 /* 100 * An allocating zio is one that either currently has the DVA allocate 101 * stage set or will have it later in its lifetime. 102 */ 103 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 104 105 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 106 107 #ifdef ZFS_DEBUG 108 int zio_buf_debug_limit = 16384; 109 #else 110 int zio_buf_debug_limit = 0; 111 #endif 112 113 static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t); 114 115 void 116 zio_init(void) 117 { 118 size_t c; 119 vmem_t *data_alloc_arena = NULL; 120 121 #ifdef _KERNEL 122 data_alloc_arena = zio_alloc_arena; 123 #endif 124 zio_cache = kmem_cache_create("zio_cache", 125 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 126 zio_link_cache = kmem_cache_create("zio_link_cache", 127 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 128 129 /* 130 * For small buffers, we want a cache for each multiple of 131 * SPA_MINBLOCKSIZE. For larger buffers, we want a cache 132 * for each quarter-power of 2. 133 */ 134 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 135 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 136 size_t p2 = size; 137 size_t align = 0; 138 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 139 140 while (!ISP2(p2)) 141 p2 &= p2 - 1; 142 143 #ifndef _KERNEL 144 /* 145 * If we are using watchpoints, put each buffer on its own page, 146 * to eliminate the performance overhead of trapping to the 147 * kernel when modifying a non-watched buffer that shares the 148 * page with a watched buffer. 149 */ 150 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 151 continue; 152 #endif 153 if (size <= 4 * SPA_MINBLOCKSIZE) { 154 align = SPA_MINBLOCKSIZE; 155 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 156 align = MIN(p2 >> 2, PAGESIZE); 157 } 158 159 if (align != 0) { 160 char name[36]; 161 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 162 zio_buf_cache[c] = kmem_cache_create(name, size, 163 align, NULL, NULL, NULL, NULL, NULL, cflags); 164 165 /* 166 * Since zio_data bufs do not appear in crash dumps, we 167 * pass KMC_NOTOUCH so that no allocator metadata is 168 * stored with the buffers. 169 */ 170 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 171 zio_data_buf_cache[c] = kmem_cache_create(name, size, 172 align, NULL, NULL, NULL, NULL, data_alloc_arena, 173 cflags | KMC_NOTOUCH); 174 } 175 } 176 177 while (--c != 0) { 178 ASSERT(zio_buf_cache[c] != NULL); 179 if (zio_buf_cache[c - 1] == NULL) 180 zio_buf_cache[c - 1] = zio_buf_cache[c]; 181 182 ASSERT(zio_data_buf_cache[c] != NULL); 183 if (zio_data_buf_cache[c - 1] == NULL) 184 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 185 } 186 187 zio_inject_init(); 188 } 189 190 void 191 zio_fini(void) 192 { 193 size_t c; 194 kmem_cache_t *last_cache = NULL; 195 kmem_cache_t *last_data_cache = NULL; 196 197 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 198 if (zio_buf_cache[c] != last_cache) { 199 last_cache = zio_buf_cache[c]; 200 kmem_cache_destroy(zio_buf_cache[c]); 201 } 202 zio_buf_cache[c] = NULL; 203 204 if (zio_data_buf_cache[c] != last_data_cache) { 205 last_data_cache = zio_data_buf_cache[c]; 206 kmem_cache_destroy(zio_data_buf_cache[c]); 207 } 208 zio_data_buf_cache[c] = NULL; 209 } 210 211 kmem_cache_destroy(zio_link_cache); 212 kmem_cache_destroy(zio_cache); 213 214 zio_inject_fini(); 215 } 216 217 /* 218 * ========================================================================== 219 * Allocate and free I/O buffers 220 * ========================================================================== 221 */ 222 223 /* 224 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 225 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 226 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 227 * excess / transient data in-core during a crashdump. 228 */ 229 void * 230 zio_buf_alloc(size_t size) 231 { 232 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 233 234 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 235 236 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 237 } 238 239 /* 240 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 241 * crashdump if the kernel panics. This exists so that we will limit the amount 242 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 243 * of kernel heap dumped to disk when the kernel panics) 244 */ 245 void * 246 zio_data_buf_alloc(size_t size) 247 { 248 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 249 250 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 251 252 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 253 } 254 255 void 256 zio_buf_free(void *buf, size_t size) 257 { 258 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 259 260 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 261 262 kmem_cache_free(zio_buf_cache[c], buf); 263 } 264 265 void 266 zio_data_buf_free(void *buf, size_t size) 267 { 268 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 269 270 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 271 272 kmem_cache_free(zio_data_buf_cache[c], buf); 273 } 274 275 /* ARGSUSED */ 276 static void 277 zio_abd_free(void *abd, size_t size) 278 { 279 abd_free((abd_t *)abd); 280 } 281 282 /* 283 * ========================================================================== 284 * Push and pop I/O transform buffers 285 * ========================================================================== 286 */ 287 void 288 zio_push_transform(zio_t *zio, abd_t *data, uint64_t size, uint64_t bufsize, 289 zio_transform_func_t *transform) 290 { 291 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 292 293 /* 294 * Ensure that anyone expecting this zio to contain a linear ABD isn't 295 * going to get a nasty surprise when they try to access the data. 296 */ 297 IMPLY(abd_is_linear(zio->io_abd), abd_is_linear(data)); 298 299 zt->zt_orig_abd = zio->io_abd; 300 zt->zt_orig_size = zio->io_size; 301 zt->zt_bufsize = bufsize; 302 zt->zt_transform = transform; 303 304 zt->zt_next = zio->io_transform_stack; 305 zio->io_transform_stack = zt; 306 307 zio->io_abd = data; 308 zio->io_size = size; 309 } 310 311 void 312 zio_pop_transforms(zio_t *zio) 313 { 314 zio_transform_t *zt; 315 316 while ((zt = zio->io_transform_stack) != NULL) { 317 if (zt->zt_transform != NULL) 318 zt->zt_transform(zio, 319 zt->zt_orig_abd, zt->zt_orig_size); 320 321 if (zt->zt_bufsize != 0) 322 abd_free(zio->io_abd); 323 324 zio->io_abd = zt->zt_orig_abd; 325 zio->io_size = zt->zt_orig_size; 326 zio->io_transform_stack = zt->zt_next; 327 328 kmem_free(zt, sizeof (zio_transform_t)); 329 } 330 } 331 332 /* 333 * ========================================================================== 334 * I/O transform callbacks for subblocks, decompression, and decryption 335 * ========================================================================== 336 */ 337 static void 338 zio_subblock(zio_t *zio, abd_t *data, uint64_t size) 339 { 340 ASSERT(zio->io_size > size); 341 342 if (zio->io_type == ZIO_TYPE_READ) 343 abd_copy(data, zio->io_abd, size); 344 } 345 346 static void 347 zio_decompress(zio_t *zio, abd_t *data, uint64_t size) 348 { 349 if (zio->io_error == 0) { 350 void *tmp = abd_borrow_buf(data, size); 351 int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 352 zio->io_abd, tmp, zio->io_size, size); 353 abd_return_buf_copy(data, tmp, size); 354 355 if (ret != 0) 356 zio->io_error = SET_ERROR(EIO); 357 } 358 } 359 360 static void 361 zio_decrypt(zio_t *zio, abd_t *data, uint64_t size) 362 { 363 int ret; 364 void *tmp; 365 blkptr_t *bp = zio->io_bp; 366 spa_t *spa = zio->io_spa; 367 uint64_t dsobj = zio->io_bookmark.zb_objset; 368 uint64_t lsize = BP_GET_LSIZE(bp); 369 dmu_object_type_t ot = BP_GET_TYPE(bp); 370 uint8_t salt[ZIO_DATA_SALT_LEN]; 371 uint8_t iv[ZIO_DATA_IV_LEN]; 372 uint8_t mac[ZIO_DATA_MAC_LEN]; 373 boolean_t no_crypt = B_FALSE; 374 375 ASSERT(BP_USES_CRYPT(bp)); 376 ASSERT3U(size, !=, 0); 377 378 if (zio->io_error != 0) 379 return; 380 381 /* 382 * Verify the cksum of MACs stored in an indirect bp. It will always 383 * be possible to verify this since it does not require an encryption 384 * key. 385 */ 386 if (BP_HAS_INDIRECT_MAC_CKSUM(bp)) { 387 zio_crypt_decode_mac_bp(bp, mac); 388 389 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 390 /* 391 * We haven't decompressed the data yet, but 392 * zio_crypt_do_indirect_mac_checksum() requires 393 * decompressed data to be able to parse out the MACs 394 * from the indirect block. We decompress it now and 395 * throw away the result after we are finished. 396 */ 397 tmp = zio_buf_alloc(lsize); 398 ret = zio_decompress_data(BP_GET_COMPRESS(bp), 399 zio->io_abd, tmp, zio->io_size, lsize); 400 if (ret != 0) { 401 ret = SET_ERROR(EIO); 402 goto error; 403 } 404 ret = zio_crypt_do_indirect_mac_checksum(B_FALSE, 405 tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac); 406 zio_buf_free(tmp, lsize); 407 } else { 408 ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE, 409 zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac); 410 } 411 abd_copy(data, zio->io_abd, size); 412 413 if (ret != 0) 414 goto error; 415 416 return; 417 } 418 419 /* 420 * If this is an authenticated block, just check the MAC. It would be 421 * nice to separate this out into its own flag, but for the moment 422 * enum zio_flag is out of bits. 423 */ 424 if (BP_IS_AUTHENTICATED(bp)) { 425 if (ot == DMU_OT_OBJSET) { 426 ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, 427 dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp)); 428 } else { 429 zio_crypt_decode_mac_bp(bp, mac); 430 ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, 431 zio->io_abd, size, mac); 432 } 433 abd_copy(data, zio->io_abd, size); 434 435 if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) { 436 ret = zio_handle_decrypt_injection(spa, 437 &zio->io_bookmark, ot, ECKSUM); 438 } 439 if (ret != 0) 440 goto error; 441 442 return; 443 } 444 445 zio_crypt_decode_params_bp(bp, salt, iv); 446 447 if (ot == DMU_OT_INTENT_LOG) { 448 tmp = abd_borrow_buf_copy(zio->io_abd, sizeof (zil_chain_t)); 449 zio_crypt_decode_mac_zil(tmp, mac); 450 abd_return_buf(zio->io_abd, tmp, sizeof (zil_chain_t)); 451 } else { 452 zio_crypt_decode_mac_bp(bp, mac); 453 } 454 455 ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp), 456 BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data, 457 zio->io_abd, &no_crypt); 458 if (no_crypt) 459 abd_copy(data, zio->io_abd, size); 460 461 if (ret != 0) 462 goto error; 463 464 return; 465 466 error: 467 /* assert that the key was found unless this was speculative */ 468 ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE)); 469 470 /* 471 * If there was a decryption / authentication error return EIO as 472 * the io_error. If this was not a speculative zio, create an ereport. 473 */ 474 if (ret == ECKSUM) { 475 zio->io_error = SET_ERROR(EIO); 476 if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) { 477 spa_log_error(spa, &zio->io_bookmark); 478 zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION, 479 spa, NULL, &zio->io_bookmark, zio, 0, 0); 480 } 481 } else { 482 zio->io_error = ret; 483 } 484 } 485 486 /* 487 * ========================================================================== 488 * I/O parent/child relationships and pipeline interlocks 489 * ========================================================================== 490 */ 491 zio_t * 492 zio_walk_parents(zio_t *cio, zio_link_t **zl) 493 { 494 list_t *pl = &cio->io_parent_list; 495 496 *zl = (*zl == NULL) ? list_head(pl) : list_next(pl, *zl); 497 if (*zl == NULL) 498 return (NULL); 499 500 ASSERT((*zl)->zl_child == cio); 501 return ((*zl)->zl_parent); 502 } 503 504 zio_t * 505 zio_walk_children(zio_t *pio, zio_link_t **zl) 506 { 507 list_t *cl = &pio->io_child_list; 508 509 ASSERT(MUTEX_HELD(&pio->io_lock)); 510 511 *zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl); 512 if (*zl == NULL) 513 return (NULL); 514 515 ASSERT((*zl)->zl_parent == pio); 516 return ((*zl)->zl_child); 517 } 518 519 zio_t * 520 zio_unique_parent(zio_t *cio) 521 { 522 zio_link_t *zl = NULL; 523 zio_t *pio = zio_walk_parents(cio, &zl); 524 525 VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL); 526 return (pio); 527 } 528 529 void 530 zio_add_child(zio_t *pio, zio_t *cio) 531 { 532 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 533 534 /* 535 * Logical I/Os can have logical, gang, or vdev children. 536 * Gang I/Os can have gang or vdev children. 537 * Vdev I/Os can only have vdev children. 538 * The following ASSERT captures all of these constraints. 539 */ 540 ASSERT3S(cio->io_child_type, <=, pio->io_child_type); 541 542 zl->zl_parent = pio; 543 zl->zl_child = cio; 544 545 mutex_enter(&pio->io_lock); 546 mutex_enter(&cio->io_lock); 547 548 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 549 550 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 551 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 552 553 list_insert_head(&pio->io_child_list, zl); 554 list_insert_head(&cio->io_parent_list, zl); 555 556 pio->io_child_count++; 557 cio->io_parent_count++; 558 559 mutex_exit(&cio->io_lock); 560 mutex_exit(&pio->io_lock); 561 } 562 563 static void 564 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 565 { 566 ASSERT(zl->zl_parent == pio); 567 ASSERT(zl->zl_child == cio); 568 569 mutex_enter(&pio->io_lock); 570 mutex_enter(&cio->io_lock); 571 572 list_remove(&pio->io_child_list, zl); 573 list_remove(&cio->io_parent_list, zl); 574 575 pio->io_child_count--; 576 cio->io_parent_count--; 577 578 mutex_exit(&cio->io_lock); 579 mutex_exit(&pio->io_lock); 580 581 kmem_cache_free(zio_link_cache, zl); 582 } 583 584 static boolean_t 585 zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait) 586 { 587 boolean_t waiting = B_FALSE; 588 589 mutex_enter(&zio->io_lock); 590 ASSERT(zio->io_stall == NULL); 591 for (int c = 0; c < ZIO_CHILD_TYPES; c++) { 592 if (!(ZIO_CHILD_BIT_IS_SET(childbits, c))) 593 continue; 594 595 uint64_t *countp = &zio->io_children[c][wait]; 596 if (*countp != 0) { 597 zio->io_stage >>= 1; 598 ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN); 599 zio->io_stall = countp; 600 waiting = B_TRUE; 601 break; 602 } 603 } 604 mutex_exit(&zio->io_lock); 605 return (waiting); 606 } 607 608 static void 609 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 610 { 611 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 612 int *errorp = &pio->io_child_error[zio->io_child_type]; 613 614 mutex_enter(&pio->io_lock); 615 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 616 *errorp = zio_worst_error(*errorp, zio->io_error); 617 pio->io_reexecute |= zio->io_reexecute; 618 ASSERT3U(*countp, >, 0); 619 620 (*countp)--; 621 622 if (*countp == 0 && pio->io_stall == countp) { 623 zio_taskq_type_t type = 624 pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE : 625 ZIO_TASKQ_INTERRUPT; 626 pio->io_stall = NULL; 627 mutex_exit(&pio->io_lock); 628 /* 629 * Dispatch the parent zio in its own taskq so that 630 * the child can continue to make progress. This also 631 * prevents overflowing the stack when we have deeply nested 632 * parent-child relationships. 633 */ 634 zio_taskq_dispatch(pio, type, B_FALSE); 635 } else { 636 mutex_exit(&pio->io_lock); 637 } 638 } 639 640 static void 641 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 642 { 643 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 644 zio->io_error = zio->io_child_error[c]; 645 } 646 647 int 648 zio_bookmark_compare(const void *x1, const void *x2) 649 { 650 const zio_t *z1 = x1; 651 const zio_t *z2 = x2; 652 653 if (z1->io_bookmark.zb_objset < z2->io_bookmark.zb_objset) 654 return (-1); 655 if (z1->io_bookmark.zb_objset > z2->io_bookmark.zb_objset) 656 return (1); 657 658 if (z1->io_bookmark.zb_object < z2->io_bookmark.zb_object) 659 return (-1); 660 if (z1->io_bookmark.zb_object > z2->io_bookmark.zb_object) 661 return (1); 662 663 if (z1->io_bookmark.zb_level < z2->io_bookmark.zb_level) 664 return (-1); 665 if (z1->io_bookmark.zb_level > z2->io_bookmark.zb_level) 666 return (1); 667 668 if (z1->io_bookmark.zb_blkid < z2->io_bookmark.zb_blkid) 669 return (-1); 670 if (z1->io_bookmark.zb_blkid > z2->io_bookmark.zb_blkid) 671 return (1); 672 673 if (z1 < z2) 674 return (-1); 675 if (z1 > z2) 676 return (1); 677 678 return (0); 679 } 680 681 /* 682 * ========================================================================== 683 * Create the various types of I/O (read, write, free, etc) 684 * ========================================================================== 685 */ 686 static zio_t * 687 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 688 abd_t *data, uint64_t lsize, uint64_t psize, zio_done_func_t *done, 689 void *private, zio_type_t type, zio_priority_t priority, 690 enum zio_flag flags, vdev_t *vd, uint64_t offset, 691 const zbookmark_phys_t *zb, enum zio_stage stage, enum zio_stage pipeline) 692 { 693 zio_t *zio; 694 695 IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE); 696 ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0); 697 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 698 699 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 700 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 701 ASSERT(vd || stage == ZIO_STAGE_OPEN); 702 703 IMPLY(lsize != psize, (flags & ZIO_FLAG_RAW_COMPRESS) != 0); 704 705 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 706 bzero(zio, sizeof (zio_t)); 707 708 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 709 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 710 711 list_create(&zio->io_parent_list, sizeof (zio_link_t), 712 offsetof(zio_link_t, zl_parent_node)); 713 list_create(&zio->io_child_list, sizeof (zio_link_t), 714 offsetof(zio_link_t, zl_child_node)); 715 metaslab_trace_init(&zio->io_alloc_list); 716 717 if (vd != NULL) 718 zio->io_child_type = ZIO_CHILD_VDEV; 719 else if (flags & ZIO_FLAG_GANG_CHILD) 720 zio->io_child_type = ZIO_CHILD_GANG; 721 else if (flags & ZIO_FLAG_DDT_CHILD) 722 zio->io_child_type = ZIO_CHILD_DDT; 723 else 724 zio->io_child_type = ZIO_CHILD_LOGICAL; 725 726 if (bp != NULL) { 727 zio->io_bp = (blkptr_t *)bp; 728 zio->io_bp_copy = *bp; 729 zio->io_bp_orig = *bp; 730 if (type != ZIO_TYPE_WRITE || 731 zio->io_child_type == ZIO_CHILD_DDT) 732 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 733 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 734 zio->io_logical = zio; 735 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 736 pipeline |= ZIO_GANG_STAGES; 737 } 738 739 zio->io_spa = spa; 740 zio->io_txg = txg; 741 zio->io_done = done; 742 zio->io_private = private; 743 zio->io_type = type; 744 zio->io_priority = priority; 745 zio->io_vd = vd; 746 zio->io_offset = offset; 747 zio->io_orig_abd = zio->io_abd = data; 748 zio->io_orig_size = zio->io_size = psize; 749 zio->io_lsize = lsize; 750 zio->io_orig_flags = zio->io_flags = flags; 751 zio->io_orig_stage = zio->io_stage = stage; 752 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 753 zio->io_pipeline_trace = ZIO_STAGE_OPEN; 754 755 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 756 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 757 758 if (zb != NULL) 759 zio->io_bookmark = *zb; 760 761 if (pio != NULL) { 762 if (zio->io_metaslab_class == NULL) 763 zio->io_metaslab_class = pio->io_metaslab_class; 764 if (zio->io_logical == NULL) 765 zio->io_logical = pio->io_logical; 766 if (zio->io_child_type == ZIO_CHILD_GANG) 767 zio->io_gang_leader = pio->io_gang_leader; 768 zio_add_child(pio, zio); 769 } 770 771 return (zio); 772 } 773 774 static void 775 zio_destroy(zio_t *zio) 776 { 777 metaslab_trace_fini(&zio->io_alloc_list); 778 list_destroy(&zio->io_parent_list); 779 list_destroy(&zio->io_child_list); 780 mutex_destroy(&zio->io_lock); 781 cv_destroy(&zio->io_cv); 782 kmem_cache_free(zio_cache, zio); 783 } 784 785 zio_t * 786 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 787 void *private, enum zio_flag flags) 788 { 789 zio_t *zio; 790 791 zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private, 792 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 793 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 794 795 return (zio); 796 } 797 798 zio_t * 799 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 800 { 801 return (zio_null(NULL, spa, NULL, done, private, flags)); 802 } 803 804 void 805 zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp) 806 { 807 if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) { 808 zfs_panic_recover("blkptr at %p has invalid TYPE %llu", 809 bp, (longlong_t)BP_GET_TYPE(bp)); 810 } 811 if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS || 812 BP_GET_CHECKSUM(bp) <= ZIO_CHECKSUM_ON) { 813 zfs_panic_recover("blkptr at %p has invalid CHECKSUM %llu", 814 bp, (longlong_t)BP_GET_CHECKSUM(bp)); 815 } 816 if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS || 817 BP_GET_COMPRESS(bp) <= ZIO_COMPRESS_ON) { 818 zfs_panic_recover("blkptr at %p has invalid COMPRESS %llu", 819 bp, (longlong_t)BP_GET_COMPRESS(bp)); 820 } 821 if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) { 822 zfs_panic_recover("blkptr at %p has invalid LSIZE %llu", 823 bp, (longlong_t)BP_GET_LSIZE(bp)); 824 } 825 if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) { 826 zfs_panic_recover("blkptr at %p has invalid PSIZE %llu", 827 bp, (longlong_t)BP_GET_PSIZE(bp)); 828 } 829 830 if (BP_IS_EMBEDDED(bp)) { 831 if (BPE_GET_ETYPE(bp) > NUM_BP_EMBEDDED_TYPES) { 832 zfs_panic_recover("blkptr at %p has invalid ETYPE %llu", 833 bp, (longlong_t)BPE_GET_ETYPE(bp)); 834 } 835 } 836 837 /* 838 * Do not verify individual DVAs if the config is not trusted. This 839 * will be done once the zio is executed in vdev_mirror_map_alloc. 840 */ 841 if (!spa->spa_trust_config) 842 return; 843 844 /* 845 * Pool-specific checks. 846 * 847 * Note: it would be nice to verify that the blk_birth and 848 * BP_PHYSICAL_BIRTH() are not too large. However, spa_freeze() 849 * allows the birth time of log blocks (and dmu_sync()-ed blocks 850 * that are in the log) to be arbitrarily large. 851 */ 852 for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 853 uint64_t vdevid = DVA_GET_VDEV(&bp->blk_dva[i]); 854 if (vdevid >= spa->spa_root_vdev->vdev_children) { 855 zfs_panic_recover("blkptr at %p DVA %u has invalid " 856 "VDEV %llu", 857 bp, i, (longlong_t)vdevid); 858 continue; 859 } 860 vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid]; 861 if (vd == NULL) { 862 zfs_panic_recover("blkptr at %p DVA %u has invalid " 863 "VDEV %llu", 864 bp, i, (longlong_t)vdevid); 865 continue; 866 } 867 if (vd->vdev_ops == &vdev_hole_ops) { 868 zfs_panic_recover("blkptr at %p DVA %u has hole " 869 "VDEV %llu", 870 bp, i, (longlong_t)vdevid); 871 continue; 872 } 873 if (vd->vdev_ops == &vdev_missing_ops) { 874 /* 875 * "missing" vdevs are valid during import, but we 876 * don't have their detailed info (e.g. asize), so 877 * we can't perform any more checks on them. 878 */ 879 continue; 880 } 881 uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]); 882 uint64_t asize = DVA_GET_ASIZE(&bp->blk_dva[i]); 883 if (BP_IS_GANG(bp)) 884 asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE); 885 if (offset + asize > vd->vdev_asize) { 886 zfs_panic_recover("blkptr at %p DVA %u has invalid " 887 "OFFSET %llu", 888 bp, i, (longlong_t)offset); 889 } 890 } 891 } 892 893 boolean_t 894 zfs_dva_valid(spa_t *spa, const dva_t *dva, const blkptr_t *bp) 895 { 896 uint64_t vdevid = DVA_GET_VDEV(dva); 897 898 if (vdevid >= spa->spa_root_vdev->vdev_children) 899 return (B_FALSE); 900 901 vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid]; 902 if (vd == NULL) 903 return (B_FALSE); 904 905 if (vd->vdev_ops == &vdev_hole_ops) 906 return (B_FALSE); 907 908 if (vd->vdev_ops == &vdev_missing_ops) { 909 return (B_FALSE); 910 } 911 912 uint64_t offset = DVA_GET_OFFSET(dva); 913 uint64_t asize = DVA_GET_ASIZE(dva); 914 915 if (BP_IS_GANG(bp)) 916 asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE); 917 if (offset + asize > vd->vdev_asize) 918 return (B_FALSE); 919 920 return (B_TRUE); 921 } 922 923 zio_t * 924 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 925 abd_t *data, uint64_t size, zio_done_func_t *done, void *private, 926 zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb) 927 { 928 zio_t *zio; 929 930 zfs_blkptr_verify(spa, bp); 931 932 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 933 data, size, size, done, private, 934 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 935 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 936 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 937 938 return (zio); 939 } 940 941 zio_t * 942 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 943 abd_t *data, uint64_t lsize, uint64_t psize, const zio_prop_t *zp, 944 zio_done_func_t *ready, zio_done_func_t *children_ready, 945 zio_done_func_t *physdone, zio_done_func_t *done, 946 void *private, zio_priority_t priority, enum zio_flag flags, 947 const zbookmark_phys_t *zb) 948 { 949 zio_t *zio; 950 951 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 952 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 953 zp->zp_compress >= ZIO_COMPRESS_OFF && 954 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 955 DMU_OT_IS_VALID(zp->zp_type) && 956 zp->zp_level < 32 && 957 zp->zp_copies > 0 && 958 zp->zp_copies <= spa_max_replication(spa)); 959 960 zio = zio_create(pio, spa, txg, bp, data, lsize, psize, done, private, 961 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 962 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 963 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 964 965 zio->io_ready = ready; 966 zio->io_children_ready = children_ready; 967 zio->io_physdone = physdone; 968 zio->io_prop = *zp; 969 970 /* 971 * Data can be NULL if we are going to call zio_write_override() to 972 * provide the already-allocated BP. But we may need the data to 973 * verify a dedup hit (if requested). In this case, don't try to 974 * dedup (just take the already-allocated BP verbatim). Encrypted 975 * dedup blocks need data as well so we also disable dedup in this 976 * case. 977 */ 978 if (data == NULL && 979 (zio->io_prop.zp_dedup_verify || zio->io_prop.zp_encrypt)) { 980 zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE; 981 } 982 983 return (zio); 984 } 985 986 zio_t * 987 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, abd_t *data, 988 uint64_t size, zio_done_func_t *done, void *private, 989 zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb) 990 { 991 zio_t *zio; 992 993 zio = zio_create(pio, spa, txg, bp, data, size, size, done, private, 994 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_IO_REWRITE, NULL, 0, zb, 995 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 996 997 return (zio); 998 } 999 1000 void 1001 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 1002 { 1003 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 1004 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1005 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1006 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 1007 1008 /* 1009 * We must reset the io_prop to match the values that existed 1010 * when the bp was first written by dmu_sync() keeping in mind 1011 * that nopwrite and dedup are mutually exclusive. 1012 */ 1013 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 1014 zio->io_prop.zp_nopwrite = nopwrite; 1015 zio->io_prop.zp_copies = copies; 1016 zio->io_bp_override = bp; 1017 } 1018 1019 void 1020 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 1021 { 1022 1023 zfs_blkptr_verify(spa, bp); 1024 1025 /* 1026 * The check for EMBEDDED is a performance optimization. We 1027 * process the free here (by ignoring it) rather than 1028 * putting it on the list and then processing it in zio_free_sync(). 1029 */ 1030 if (BP_IS_EMBEDDED(bp)) 1031 return; 1032 metaslab_check_free(spa, bp); 1033 1034 /* 1035 * Frees that are for the currently-syncing txg, are not going to be 1036 * deferred, and which will not need to do a read (i.e. not GANG or 1037 * DEDUP), can be processed immediately. Otherwise, put them on the 1038 * in-memory list for later processing. 1039 * 1040 * Note that we only defer frees after zfs_sync_pass_deferred_free 1041 * when the log space map feature is disabled. [see relevant comment 1042 * in spa_sync_iterate_to_convergence()] 1043 */ 1044 if (BP_IS_GANG(bp) || 1045 BP_GET_DEDUP(bp) || 1046 txg != spa->spa_syncing_txg || 1047 (spa_sync_pass(spa) >= zfs_sync_pass_deferred_free && 1048 !spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP))) { 1049 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 1050 } else { 1051 VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 0))); 1052 } 1053 } 1054 1055 zio_t * 1056 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 1057 enum zio_flag flags) 1058 { 1059 zio_t *zio; 1060 enum zio_stage stage = ZIO_FREE_PIPELINE; 1061 1062 ASSERT(!BP_IS_HOLE(bp)); 1063 ASSERT(spa_syncing_txg(spa) == txg); 1064 1065 if (BP_IS_EMBEDDED(bp)) 1066 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 1067 1068 metaslab_check_free(spa, bp); 1069 arc_freed(spa, bp); 1070 dsl_scan_freed(spa, bp); 1071 1072 /* 1073 * GANG and DEDUP blocks can induce a read (for the gang block header, 1074 * or the DDT), so issue them asynchronously so that this thread is 1075 * not tied up. 1076 */ 1077 if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) 1078 stage |= ZIO_STAGE_ISSUE_ASYNC; 1079 1080 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 1081 BP_GET_PSIZE(bp), NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, 1082 flags, NULL, 0, NULL, ZIO_STAGE_OPEN, stage); 1083 1084 return (zio); 1085 } 1086 1087 zio_t * 1088 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 1089 zio_done_func_t *done, void *private, enum zio_flag flags) 1090 { 1091 zio_t *zio; 1092 1093 zfs_blkptr_verify(spa, bp); 1094 1095 if (BP_IS_EMBEDDED(bp)) 1096 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 1097 1098 /* 1099 * A claim is an allocation of a specific block. Claims are needed 1100 * to support immediate writes in the intent log. The issue is that 1101 * immediate writes contain committed data, but in a txg that was 1102 * *not* committed. Upon opening the pool after an unclean shutdown, 1103 * the intent log claims all blocks that contain immediate write data 1104 * so that the SPA knows they're in use. 1105 * 1106 * All claims *must* be resolved in the first txg -- before the SPA 1107 * starts allocating blocks -- so that nothing is allocated twice. 1108 * If txg == 0 we just verify that the block is claimable. 1109 */ 1110 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, 1111 spa_min_claim_txg(spa)); 1112 ASSERT(txg == spa_min_claim_txg(spa) || txg == 0); 1113 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 1114 1115 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 1116 BP_GET_PSIZE(bp), done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 1117 flags, NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 1118 ASSERT0(zio->io_queued_timestamp); 1119 1120 return (zio); 1121 } 1122 1123 zio_t * 1124 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 1125 zio_done_func_t *done, void *private, enum zio_flag flags) 1126 { 1127 zio_t *zio; 1128 int c; 1129 1130 if (vd->vdev_children == 0) { 1131 zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private, 1132 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 1133 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 1134 1135 zio->io_cmd = cmd; 1136 } else { 1137 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 1138 1139 for (c = 0; c < vd->vdev_children; c++) 1140 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 1141 done, private, flags)); 1142 } 1143 1144 return (zio); 1145 } 1146 1147 zio_t * 1148 zio_trim(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1149 zio_done_func_t *done, void *private, zio_priority_t priority, 1150 enum zio_flag flags, enum trim_flag trim_flags) 1151 { 1152 zio_t *zio; 1153 1154 ASSERT0(vd->vdev_children); 1155 ASSERT0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 1156 ASSERT0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 1157 ASSERT3U(size, !=, 0); 1158 1159 zio = zio_create(pio, vd->vdev_spa, 0, NULL, NULL, size, size, done, 1160 private, ZIO_TYPE_TRIM, priority, flags | ZIO_FLAG_PHYSICAL, 1161 vd, offset, NULL, ZIO_STAGE_OPEN, ZIO_TRIM_PIPELINE); 1162 zio->io_trim_flags = trim_flags; 1163 1164 return (zio); 1165 } 1166 1167 zio_t * 1168 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1169 abd_t *data, int checksum, zio_done_func_t *done, void *private, 1170 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 1171 { 1172 zio_t *zio; 1173 1174 ASSERT(vd->vdev_children == 0); 1175 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 1176 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 1177 ASSERT3U(offset + size, <=, vd->vdev_psize); 1178 1179 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done, 1180 private, ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd, 1181 offset, NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 1182 1183 zio->io_prop.zp_checksum = checksum; 1184 1185 return (zio); 1186 } 1187 1188 zio_t * 1189 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1190 abd_t *data, int checksum, zio_done_func_t *done, void *private, 1191 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 1192 { 1193 zio_t *zio; 1194 1195 ASSERT(vd->vdev_children == 0); 1196 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 1197 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 1198 ASSERT3U(offset + size, <=, vd->vdev_psize); 1199 1200 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done, 1201 private, ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd, 1202 offset, NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 1203 1204 zio->io_prop.zp_checksum = checksum; 1205 1206 if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) { 1207 /* 1208 * zec checksums are necessarily destructive -- they modify 1209 * the end of the write buffer to hold the verifier/checksum. 1210 * Therefore, we must make a local copy in case the data is 1211 * being written to multiple places in parallel. 1212 */ 1213 abd_t *wbuf = abd_alloc_sametype(data, size); 1214 abd_copy(wbuf, data, size); 1215 1216 zio_push_transform(zio, wbuf, size, size, NULL); 1217 } 1218 1219 return (zio); 1220 } 1221 1222 /* 1223 * Create a child I/O to do some work for us. 1224 */ 1225 zio_t * 1226 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 1227 abd_t *data, uint64_t size, int type, zio_priority_t priority, 1228 enum zio_flag flags, zio_done_func_t *done, void *private) 1229 { 1230 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 1231 zio_t *zio; 1232 1233 /* 1234 * vdev child I/Os do not propagate their error to the parent. 1235 * Therefore, for correct operation the caller *must* check for 1236 * and handle the error in the child i/o's done callback. 1237 * The only exceptions are i/os that we don't care about 1238 * (OPTIONAL or REPAIR). 1239 */ 1240 ASSERT((flags & ZIO_FLAG_OPTIONAL) || (flags & ZIO_FLAG_IO_REPAIR) || 1241 done != NULL); 1242 1243 if (type == ZIO_TYPE_READ && bp != NULL) { 1244 /* 1245 * If we have the bp, then the child should perform the 1246 * checksum and the parent need not. This pushes error 1247 * detection as close to the leaves as possible and 1248 * eliminates redundant checksums in the interior nodes. 1249 */ 1250 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 1251 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 1252 } 1253 1254 if (vd->vdev_ops->vdev_op_leaf) { 1255 ASSERT0(vd->vdev_children); 1256 offset += VDEV_LABEL_START_SIZE; 1257 } 1258 1259 flags |= ZIO_VDEV_CHILD_FLAGS(pio); 1260 1261 /* 1262 * If we've decided to do a repair, the write is not speculative -- 1263 * even if the original read was. 1264 */ 1265 if (flags & ZIO_FLAG_IO_REPAIR) 1266 flags &= ~ZIO_FLAG_SPECULATIVE; 1267 1268 /* 1269 * If we're creating a child I/O that is not associated with a 1270 * top-level vdev, then the child zio is not an allocating I/O. 1271 * If this is a retried I/O then we ignore it since we will 1272 * have already processed the original allocating I/O. 1273 */ 1274 if (flags & ZIO_FLAG_IO_ALLOCATING && 1275 (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) { 1276 ASSERT(pio->io_metaslab_class != NULL); 1277 ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled); 1278 ASSERT(type == ZIO_TYPE_WRITE); 1279 ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE); 1280 ASSERT(!(flags & ZIO_FLAG_IO_REPAIR)); 1281 ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) || 1282 pio->io_child_type == ZIO_CHILD_GANG); 1283 1284 flags &= ~ZIO_FLAG_IO_ALLOCATING; 1285 } 1286 1287 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size, 1288 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 1289 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 1290 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 1291 1292 zio->io_physdone = pio->io_physdone; 1293 if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL) 1294 zio->io_logical->io_phys_children++; 1295 1296 return (zio); 1297 } 1298 1299 zio_t * 1300 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, abd_t *data, uint64_t size, 1301 zio_type_t type, zio_priority_t priority, enum zio_flag flags, 1302 zio_done_func_t *done, void *private) 1303 { 1304 zio_t *zio; 1305 1306 ASSERT(vd->vdev_ops->vdev_op_leaf); 1307 1308 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 1309 data, size, size, done, private, type, priority, 1310 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 1311 vd, offset, NULL, 1312 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 1313 1314 return (zio); 1315 } 1316 1317 void 1318 zio_flush(zio_t *zio, vdev_t *vd) 1319 { 1320 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 1321 NULL, NULL, 1322 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 1323 } 1324 1325 void 1326 zio_shrink(zio_t *zio, uint64_t size) 1327 { 1328 ASSERT3P(zio->io_executor, ==, NULL); 1329 ASSERT3P(zio->io_orig_size, ==, zio->io_size); 1330 ASSERT3U(size, <=, zio->io_size); 1331 1332 /* 1333 * We don't shrink for raidz because of problems with the 1334 * reconstruction when reading back less than the block size. 1335 * Note, BP_IS_RAIDZ() assumes no compression. 1336 */ 1337 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 1338 if (!BP_IS_RAIDZ(zio->io_bp)) { 1339 /* we are not doing a raw write */ 1340 ASSERT3U(zio->io_size, ==, zio->io_lsize); 1341 zio->io_orig_size = zio->io_size = zio->io_lsize = size; 1342 } 1343 } 1344 1345 /* 1346 * ========================================================================== 1347 * Prepare to read and write logical blocks 1348 * ========================================================================== 1349 */ 1350 1351 static int 1352 zio_read_bp_init(zio_t *zio) 1353 { 1354 blkptr_t *bp = zio->io_bp; 1355 uint64_t psize = 1356 BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp); 1357 1358 ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy); 1359 1360 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 1361 zio->io_child_type == ZIO_CHILD_LOGICAL && 1362 !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { 1363 zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize), 1364 psize, psize, zio_decompress); 1365 } 1366 1367 if (((BP_IS_PROTECTED(bp) && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) || 1368 BP_HAS_INDIRECT_MAC_CKSUM(bp)) && 1369 zio->io_child_type == ZIO_CHILD_LOGICAL) { 1370 zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize), 1371 psize, psize, zio_decrypt); 1372 } 1373 1374 if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) { 1375 int psize = BPE_GET_PSIZE(bp); 1376 void *data = abd_borrow_buf(zio->io_abd, psize); 1377 1378 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1379 decode_embedded_bp_compressed(bp, data); 1380 abd_return_buf_copy(zio->io_abd, data, psize); 1381 } else { 1382 ASSERT(!BP_IS_EMBEDDED(bp)); 1383 ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy); 1384 } 1385 1386 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 1387 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1388 1389 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 1390 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1391 1392 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 1393 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 1394 1395 return (ZIO_PIPELINE_CONTINUE); 1396 } 1397 1398 static int 1399 zio_write_bp_init(zio_t *zio) 1400 { 1401 if (!IO_IS_ALLOCATING(zio)) 1402 return (ZIO_PIPELINE_CONTINUE); 1403 1404 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1405 1406 if (zio->io_bp_override) { 1407 blkptr_t *bp = zio->io_bp; 1408 zio_prop_t *zp = &zio->io_prop; 1409 1410 ASSERT(bp->blk_birth != zio->io_txg); 1411 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 1412 1413 *bp = *zio->io_bp_override; 1414 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1415 1416 if (BP_IS_EMBEDDED(bp)) 1417 return (ZIO_PIPELINE_CONTINUE); 1418 1419 /* 1420 * If we've been overridden and nopwrite is set then 1421 * set the flag accordingly to indicate that a nopwrite 1422 * has already occurred. 1423 */ 1424 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1425 ASSERT(!zp->zp_dedup); 1426 ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum); 1427 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1428 return (ZIO_PIPELINE_CONTINUE); 1429 } 1430 1431 ASSERT(!zp->zp_nopwrite); 1432 1433 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1434 return (ZIO_PIPELINE_CONTINUE); 1435 1436 ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags & 1437 ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify); 1438 1439 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum && 1440 !zp->zp_encrypt) { 1441 BP_SET_DEDUP(bp, 1); 1442 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1443 return (ZIO_PIPELINE_CONTINUE); 1444 } 1445 1446 /* 1447 * We were unable to handle this as an override bp, treat 1448 * it as a regular write I/O. 1449 */ 1450 zio->io_bp_override = NULL; 1451 *bp = zio->io_bp_orig; 1452 zio->io_pipeline = zio->io_orig_pipeline; 1453 } 1454 1455 return (ZIO_PIPELINE_CONTINUE); 1456 } 1457 1458 static int 1459 zio_write_compress(zio_t *zio) 1460 { 1461 spa_t *spa = zio->io_spa; 1462 zio_prop_t *zp = &zio->io_prop; 1463 enum zio_compress compress = zp->zp_compress; 1464 blkptr_t *bp = zio->io_bp; 1465 uint64_t lsize = zio->io_lsize; 1466 uint64_t psize = zio->io_size; 1467 int pass = 1; 1468 1469 /* 1470 * If our children haven't all reached the ready stage, 1471 * wait for them and then repeat this pipeline stage. 1472 */ 1473 if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT | 1474 ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) { 1475 return (ZIO_PIPELINE_STOP); 1476 } 1477 1478 if (!IO_IS_ALLOCATING(zio)) 1479 return (ZIO_PIPELINE_CONTINUE); 1480 1481 if (zio->io_children_ready != NULL) { 1482 /* 1483 * Now that all our children are ready, run the callback 1484 * associated with this zio in case it wants to modify the 1485 * data to be written. 1486 */ 1487 ASSERT3U(zp->zp_level, >, 0); 1488 zio->io_children_ready(zio); 1489 } 1490 1491 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1492 ASSERT(zio->io_bp_override == NULL); 1493 1494 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) { 1495 /* 1496 * We're rewriting an existing block, which means we're 1497 * working on behalf of spa_sync(). For spa_sync() to 1498 * converge, it must eventually be the case that we don't 1499 * have to allocate new blocks. But compression changes 1500 * the blocksize, which forces a reallocate, and makes 1501 * convergence take longer. Therefore, after the first 1502 * few passes, stop compressing to ensure convergence. 1503 */ 1504 pass = spa_sync_pass(spa); 1505 1506 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1507 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1508 ASSERT(!BP_GET_DEDUP(bp)); 1509 1510 if (pass >= zfs_sync_pass_dont_compress) 1511 compress = ZIO_COMPRESS_OFF; 1512 1513 /* Make sure someone doesn't change their mind on overwrites */ 1514 ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp), 1515 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1516 } 1517 1518 /* If it's a compressed write that is not raw, compress the buffer. */ 1519 if (compress != ZIO_COMPRESS_OFF && 1520 !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { 1521 void *cbuf = zio_buf_alloc(lsize); 1522 psize = zio_compress_data(compress, zio->io_abd, cbuf, lsize); 1523 if (psize == 0 || psize == lsize) { 1524 compress = ZIO_COMPRESS_OFF; 1525 zio_buf_free(cbuf, lsize); 1526 } else if (!zp->zp_dedup && !zp->zp_encrypt && 1527 psize <= BPE_PAYLOAD_SIZE && 1528 zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && 1529 spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { 1530 encode_embedded_bp_compressed(bp, 1531 cbuf, compress, lsize, psize); 1532 BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); 1533 BP_SET_TYPE(bp, zio->io_prop.zp_type); 1534 BP_SET_LEVEL(bp, zio->io_prop.zp_level); 1535 zio_buf_free(cbuf, lsize); 1536 bp->blk_birth = zio->io_txg; 1537 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1538 ASSERT(spa_feature_is_active(spa, 1539 SPA_FEATURE_EMBEDDED_DATA)); 1540 return (ZIO_PIPELINE_CONTINUE); 1541 } else { 1542 /* 1543 * Round up compressed size up to the ashift 1544 * of the smallest-ashift device, and zero the tail. 1545 * This ensures that the compressed size of the BP 1546 * (and thus compressratio property) are correct, 1547 * in that we charge for the padding used to fill out 1548 * the last sector. 1549 */ 1550 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 1551 size_t rounded = (size_t)P2ROUNDUP(psize, 1552 1ULL << spa->spa_min_ashift); 1553 if (rounded >= lsize) { 1554 compress = ZIO_COMPRESS_OFF; 1555 zio_buf_free(cbuf, lsize); 1556 psize = lsize; 1557 } else { 1558 abd_t *cdata = abd_get_from_buf(cbuf, lsize); 1559 abd_take_ownership_of_buf(cdata, B_TRUE); 1560 abd_zero_off(cdata, psize, rounded - psize); 1561 psize = rounded; 1562 zio_push_transform(zio, cdata, 1563 psize, lsize, NULL); 1564 } 1565 } 1566 1567 /* 1568 * We were unable to handle this as an override bp, treat 1569 * it as a regular write I/O. 1570 */ 1571 zio->io_bp_override = NULL; 1572 *bp = zio->io_bp_orig; 1573 zio->io_pipeline = zio->io_orig_pipeline; 1574 1575 } else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 && 1576 zp->zp_type == DMU_OT_DNODE) { 1577 /* 1578 * The DMU actually relies on the zio layer's compression 1579 * to free metadnode blocks that have had all contained 1580 * dnodes freed. As a result, even when doing a raw 1581 * receive, we must check whether the block can be compressed 1582 * to a hole. 1583 */ 1584 psize = zio_compress_data(ZIO_COMPRESS_EMPTY, 1585 zio->io_abd, NULL, lsize); 1586 if (psize == 0) 1587 compress = ZIO_COMPRESS_OFF; 1588 } else { 1589 ASSERT3U(psize, !=, 0); 1590 } 1591 1592 /* 1593 * The final pass of spa_sync() must be all rewrites, but the first 1594 * few passes offer a trade-off: allocating blocks defers convergence, 1595 * but newly allocated blocks are sequential, so they can be written 1596 * to disk faster. Therefore, we allow the first few passes of 1597 * spa_sync() to allocate new blocks, but force rewrites after that. 1598 * There should only be a handful of blocks after pass 1 in any case. 1599 */ 1600 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg && 1601 BP_GET_PSIZE(bp) == psize && 1602 pass >= zfs_sync_pass_rewrite) { 1603 VERIFY3U(psize, !=, 0); 1604 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1605 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1606 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1607 } else { 1608 BP_ZERO(bp); 1609 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1610 } 1611 1612 if (psize == 0) { 1613 if (zio->io_bp_orig.blk_birth != 0 && 1614 spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) { 1615 BP_SET_LSIZE(bp, lsize); 1616 BP_SET_TYPE(bp, zp->zp_type); 1617 BP_SET_LEVEL(bp, zp->zp_level); 1618 BP_SET_BIRTH(bp, zio->io_txg, 0); 1619 } 1620 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1621 } else { 1622 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1623 BP_SET_LSIZE(bp, lsize); 1624 BP_SET_TYPE(bp, zp->zp_type); 1625 BP_SET_LEVEL(bp, zp->zp_level); 1626 BP_SET_PSIZE(bp, psize); 1627 BP_SET_COMPRESS(bp, compress); 1628 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1629 BP_SET_DEDUP(bp, zp->zp_dedup); 1630 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1631 if (zp->zp_dedup) { 1632 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1633 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1634 ASSERT(!zp->zp_encrypt || 1635 DMU_OT_IS_ENCRYPTED(zp->zp_type)); 1636 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1637 } 1638 if (zp->zp_nopwrite) { 1639 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1640 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1641 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1642 } 1643 } 1644 return (ZIO_PIPELINE_CONTINUE); 1645 } 1646 1647 static int 1648 zio_free_bp_init(zio_t *zio) 1649 { 1650 blkptr_t *bp = zio->io_bp; 1651 1652 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1653 if (BP_GET_DEDUP(bp)) 1654 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1655 } 1656 1657 ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy); 1658 1659 return (ZIO_PIPELINE_CONTINUE); 1660 } 1661 1662 /* 1663 * ========================================================================== 1664 * Execute the I/O pipeline 1665 * ========================================================================== 1666 */ 1667 1668 static void 1669 zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1670 { 1671 spa_t *spa = zio->io_spa; 1672 zio_type_t t = zio->io_type; 1673 int flags = (cutinline ? TQ_FRONT : 0); 1674 1675 /* 1676 * If we're a config writer or a probe, the normal issue and 1677 * interrupt threads may all be blocked waiting for the config lock. 1678 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1679 */ 1680 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1681 t = ZIO_TYPE_NULL; 1682 1683 /* 1684 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1685 */ 1686 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1687 t = ZIO_TYPE_NULL; 1688 1689 /* 1690 * If this is a high priority I/O, then use the high priority taskq if 1691 * available. 1692 */ 1693 if ((zio->io_priority == ZIO_PRIORITY_NOW || 1694 zio->io_priority == ZIO_PRIORITY_SYNC_WRITE) && 1695 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 1696 q++; 1697 1698 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1699 1700 /* 1701 * NB: We are assuming that the zio can only be dispatched 1702 * to a single taskq at a time. It would be a grievous error 1703 * to dispatch the zio to another taskq at the same time. 1704 */ 1705 ASSERT(zio->io_tqent.tqent_next == NULL); 1706 spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio, 1707 flags, &zio->io_tqent); 1708 } 1709 1710 static boolean_t 1711 zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 1712 { 1713 kthread_t *executor = zio->io_executor; 1714 spa_t *spa = zio->io_spa; 1715 1716 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 1717 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1718 uint_t i; 1719 for (i = 0; i < tqs->stqs_count; i++) { 1720 if (taskq_member(tqs->stqs_taskq[i], executor)) 1721 return (B_TRUE); 1722 } 1723 } 1724 1725 return (B_FALSE); 1726 } 1727 1728 static int 1729 zio_issue_async(zio_t *zio) 1730 { 1731 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1732 1733 return (ZIO_PIPELINE_STOP); 1734 } 1735 1736 void 1737 zio_interrupt(zio_t *zio) 1738 { 1739 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1740 } 1741 1742 void 1743 zio_delay_interrupt(zio_t *zio) 1744 { 1745 /* 1746 * The timeout_generic() function isn't defined in userspace, so 1747 * rather than trying to implement the function, the zio delay 1748 * functionality has been disabled for userspace builds. 1749 */ 1750 1751 #ifdef _KERNEL 1752 /* 1753 * If io_target_timestamp is zero, then no delay has been registered 1754 * for this IO, thus jump to the end of this function and "skip" the 1755 * delay; issuing it directly to the zio layer. 1756 */ 1757 if (zio->io_target_timestamp != 0) { 1758 hrtime_t now = gethrtime(); 1759 1760 if (now >= zio->io_target_timestamp) { 1761 /* 1762 * This IO has already taken longer than the target 1763 * delay to complete, so we don't want to delay it 1764 * any longer; we "miss" the delay and issue it 1765 * directly to the zio layer. This is likely due to 1766 * the target latency being set to a value less than 1767 * the underlying hardware can satisfy (e.g. delay 1768 * set to 1ms, but the disks take 10ms to complete an 1769 * IO request). 1770 */ 1771 1772 DTRACE_PROBE2(zio__delay__miss, zio_t *, zio, 1773 hrtime_t, now); 1774 1775 zio_interrupt(zio); 1776 } else { 1777 hrtime_t diff = zio->io_target_timestamp - now; 1778 1779 DTRACE_PROBE3(zio__delay__hit, zio_t *, zio, 1780 hrtime_t, now, hrtime_t, diff); 1781 1782 (void) timeout_generic(CALLOUT_NORMAL, 1783 (void (*)(void *))zio_interrupt, zio, diff, 1, 0); 1784 } 1785 1786 return; 1787 } 1788 #endif 1789 1790 DTRACE_PROBE1(zio__delay__skip, zio_t *, zio); 1791 zio_interrupt(zio); 1792 } 1793 1794 /* 1795 * Execute the I/O pipeline until one of the following occurs: 1796 * 1797 * (1) the I/O completes 1798 * (2) the pipeline stalls waiting for dependent child I/Os 1799 * (3) the I/O issues, so we're waiting for an I/O completion interrupt 1800 * (4) the I/O is delegated by vdev-level caching or aggregation 1801 * (5) the I/O is deferred due to vdev-level queueing 1802 * (6) the I/O is handed off to another thread. 1803 * 1804 * In all cases, the pipeline stops whenever there's no CPU work; it never 1805 * burns a thread in cv_wait(). 1806 * 1807 * There's no locking on io_stage because there's no legitimate way 1808 * for multiple threads to be attempting to process the same I/O. 1809 */ 1810 static zio_pipe_stage_t *zio_pipeline[]; 1811 1812 void 1813 zio_execute(zio_t *zio) 1814 { 1815 zio->io_executor = curthread; 1816 1817 ASSERT3U(zio->io_queued_timestamp, >, 0); 1818 1819 while (zio->io_stage < ZIO_STAGE_DONE) { 1820 enum zio_stage pipeline = zio->io_pipeline; 1821 enum zio_stage stage = zio->io_stage; 1822 int rv; 1823 1824 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1825 ASSERT(ISP2(stage)); 1826 ASSERT(zio->io_stall == NULL); 1827 1828 do { 1829 stage <<= 1; 1830 } while ((stage & pipeline) == 0); 1831 1832 ASSERT(stage <= ZIO_STAGE_DONE); 1833 1834 /* 1835 * If we are in interrupt context and this pipeline stage 1836 * will grab a config lock that is held across I/O, 1837 * or may wait for an I/O that needs an interrupt thread 1838 * to complete, issue async to avoid deadlock. 1839 * 1840 * For VDEV_IO_START, we cut in line so that the io will 1841 * be sent to disk promptly. 1842 */ 1843 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1844 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1845 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1846 zio_requeue_io_start_cut_in_line : B_FALSE; 1847 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1848 return; 1849 } 1850 1851 zio->io_stage = stage; 1852 zio->io_pipeline_trace |= zio->io_stage; 1853 rv = zio_pipeline[highbit64(stage) - 1](zio); 1854 1855 if (rv == ZIO_PIPELINE_STOP) 1856 return; 1857 1858 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1859 } 1860 } 1861 1862 /* 1863 * ========================================================================== 1864 * Initiate I/O, either sync or async 1865 * ========================================================================== 1866 */ 1867 int 1868 zio_wait(zio_t *zio) 1869 { 1870 int error; 1871 1872 ASSERT3P(zio->io_stage, ==, ZIO_STAGE_OPEN); 1873 ASSERT3P(zio->io_executor, ==, NULL); 1874 1875 zio->io_waiter = curthread; 1876 ASSERT0(zio->io_queued_timestamp); 1877 zio->io_queued_timestamp = gethrtime(); 1878 1879 zio_execute(zio); 1880 1881 mutex_enter(&zio->io_lock); 1882 while (zio->io_executor != NULL) 1883 cv_wait(&zio->io_cv, &zio->io_lock); 1884 mutex_exit(&zio->io_lock); 1885 1886 error = zio->io_error; 1887 zio_destroy(zio); 1888 1889 return (error); 1890 } 1891 1892 void 1893 zio_nowait(zio_t *zio) 1894 { 1895 ASSERT3P(zio->io_executor, ==, NULL); 1896 1897 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1898 zio_unique_parent(zio) == NULL) { 1899 /* 1900 * This is a logical async I/O with no parent to wait for it. 1901 * We add it to the spa_async_root_zio "Godfather" I/O which 1902 * will ensure they complete prior to unloading the pool. 1903 */ 1904 spa_t *spa = zio->io_spa; 1905 1906 zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio); 1907 } 1908 1909 ASSERT0(zio->io_queued_timestamp); 1910 zio->io_queued_timestamp = gethrtime(); 1911 zio_execute(zio); 1912 } 1913 1914 /* 1915 * ========================================================================== 1916 * Reexecute, cancel, or suspend/resume failed I/O 1917 * ========================================================================== 1918 */ 1919 1920 static void 1921 zio_reexecute(zio_t *pio) 1922 { 1923 zio_t *cio, *cio_next; 1924 1925 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1926 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1927 ASSERT(pio->io_gang_leader == NULL); 1928 ASSERT(pio->io_gang_tree == NULL); 1929 1930 pio->io_flags = pio->io_orig_flags; 1931 pio->io_stage = pio->io_orig_stage; 1932 pio->io_pipeline = pio->io_orig_pipeline; 1933 pio->io_reexecute = 0; 1934 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1935 pio->io_pipeline_trace = 0; 1936 pio->io_error = 0; 1937 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1938 pio->io_state[w] = 0; 1939 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1940 pio->io_child_error[c] = 0; 1941 1942 if (IO_IS_ALLOCATING(pio)) 1943 BP_ZERO(pio->io_bp); 1944 1945 /* 1946 * As we reexecute pio's children, new children could be created. 1947 * New children go to the head of pio's io_child_list, however, 1948 * so we will (correctly) not reexecute them. The key is that 1949 * the remainder of pio's io_child_list, from 'cio_next' onward, 1950 * cannot be affected by any side effects of reexecuting 'cio'. 1951 */ 1952 zio_link_t *zl = NULL; 1953 mutex_enter(&pio->io_lock); 1954 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 1955 cio_next = zio_walk_children(pio, &zl); 1956 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1957 pio->io_children[cio->io_child_type][w]++; 1958 mutex_exit(&pio->io_lock); 1959 zio_reexecute(cio); 1960 mutex_enter(&pio->io_lock); 1961 } 1962 mutex_exit(&pio->io_lock); 1963 1964 /* 1965 * Now that all children have been reexecuted, execute the parent. 1966 * We don't reexecute "The Godfather" I/O here as it's the 1967 * responsibility of the caller to wait on it. 1968 */ 1969 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) { 1970 pio->io_queued_timestamp = gethrtime(); 1971 zio_execute(pio); 1972 } 1973 } 1974 1975 void 1976 zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason) 1977 { 1978 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1979 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1980 "failure and the failure mode property for this pool " 1981 "is set to panic.", spa_name(spa)); 1982 1983 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, 1984 NULL, NULL, 0, 0); 1985 1986 mutex_enter(&spa->spa_suspend_lock); 1987 1988 if (spa->spa_suspend_zio_root == NULL) 1989 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1990 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1991 ZIO_FLAG_GODFATHER); 1992 1993 spa->spa_suspended = reason; 1994 1995 if (zio != NULL) { 1996 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1997 ASSERT(zio != spa->spa_suspend_zio_root); 1998 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1999 ASSERT(zio_unique_parent(zio) == NULL); 2000 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 2001 zio_add_child(spa->spa_suspend_zio_root, zio); 2002 } 2003 2004 mutex_exit(&spa->spa_suspend_lock); 2005 } 2006 2007 int 2008 zio_resume(spa_t *spa) 2009 { 2010 zio_t *pio; 2011 2012 /* 2013 * Reexecute all previously suspended i/o. 2014 */ 2015 mutex_enter(&spa->spa_suspend_lock); 2016 spa->spa_suspended = ZIO_SUSPEND_NONE; 2017 cv_broadcast(&spa->spa_suspend_cv); 2018 pio = spa->spa_suspend_zio_root; 2019 spa->spa_suspend_zio_root = NULL; 2020 mutex_exit(&spa->spa_suspend_lock); 2021 2022 if (pio == NULL) 2023 return (0); 2024 2025 zio_reexecute(pio); 2026 return (zio_wait(pio)); 2027 } 2028 2029 void 2030 zio_resume_wait(spa_t *spa) 2031 { 2032 mutex_enter(&spa->spa_suspend_lock); 2033 while (spa_suspended(spa)) 2034 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 2035 mutex_exit(&spa->spa_suspend_lock); 2036 } 2037 2038 /* 2039 * ========================================================================== 2040 * Gang blocks. 2041 * 2042 * A gang block is a collection of small blocks that looks to the DMU 2043 * like one large block. When zio_dva_allocate() cannot find a block 2044 * of the requested size, due to either severe fragmentation or the pool 2045 * being nearly full, it calls zio_write_gang_block() to construct the 2046 * block from smaller fragments. 2047 * 2048 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 2049 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 2050 * an indirect block: it's an array of block pointers. It consumes 2051 * only one sector and hence is allocatable regardless of fragmentation. 2052 * The gang header's bps point to its gang members, which hold the data. 2053 * 2054 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 2055 * as the verifier to ensure uniqueness of the SHA256 checksum. 2056 * Critically, the gang block bp's blk_cksum is the checksum of the data, 2057 * not the gang header. This ensures that data block signatures (needed for 2058 * deduplication) are independent of how the block is physically stored. 2059 * 2060 * Gang blocks can be nested: a gang member may itself be a gang block. 2061 * Thus every gang block is a tree in which root and all interior nodes are 2062 * gang headers, and the leaves are normal blocks that contain user data. 2063 * The root of the gang tree is called the gang leader. 2064 * 2065 * To perform any operation (read, rewrite, free, claim) on a gang block, 2066 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 2067 * in the io_gang_tree field of the original logical i/o by recursively 2068 * reading the gang leader and all gang headers below it. This yields 2069 * an in-core tree containing the contents of every gang header and the 2070 * bps for every constituent of the gang block. 2071 * 2072 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 2073 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 2074 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 2075 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 2076 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 2077 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 2078 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 2079 * of the gang header plus zio_checksum_compute() of the data to update the 2080 * gang header's blk_cksum as described above. 2081 * 2082 * The two-phase assemble/issue model solves the problem of partial failure -- 2083 * what if you'd freed part of a gang block but then couldn't read the 2084 * gang header for another part? Assembling the entire gang tree first 2085 * ensures that all the necessary gang header I/O has succeeded before 2086 * starting the actual work of free, claim, or write. Once the gang tree 2087 * is assembled, free and claim are in-memory operations that cannot fail. 2088 * 2089 * In the event that a gang write fails, zio_dva_unallocate() walks the 2090 * gang tree to immediately free (i.e. insert back into the space map) 2091 * everything we've allocated. This ensures that we don't get ENOSPC 2092 * errors during repeated suspend/resume cycles due to a flaky device. 2093 * 2094 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 2095 * the gang tree, we won't modify the block, so we can safely defer the free 2096 * (knowing that the block is still intact). If we *can* assemble the gang 2097 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 2098 * each constituent bp and we can allocate a new block on the next sync pass. 2099 * 2100 * In all cases, the gang tree allows complete recovery from partial failure. 2101 * ========================================================================== 2102 */ 2103 2104 static void 2105 zio_gang_issue_func_done(zio_t *zio) 2106 { 2107 abd_put(zio->io_abd); 2108 } 2109 2110 static zio_t * 2111 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2112 uint64_t offset) 2113 { 2114 if (gn != NULL) 2115 return (pio); 2116 2117 return (zio_read(pio, pio->io_spa, bp, abd_get_offset(data, offset), 2118 BP_GET_PSIZE(bp), zio_gang_issue_func_done, 2119 NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 2120 &pio->io_bookmark)); 2121 } 2122 2123 static zio_t * 2124 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2125 uint64_t offset) 2126 { 2127 zio_t *zio; 2128 2129 if (gn != NULL) { 2130 abd_t *gbh_abd = 2131 abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2132 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 2133 gbh_abd, SPA_GANGBLOCKSIZE, zio_gang_issue_func_done, NULL, 2134 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 2135 &pio->io_bookmark); 2136 /* 2137 * As we rewrite each gang header, the pipeline will compute 2138 * a new gang block header checksum for it; but no one will 2139 * compute a new data checksum, so we do that here. The one 2140 * exception is the gang leader: the pipeline already computed 2141 * its data checksum because that stage precedes gang assembly. 2142 * (Presently, nothing actually uses interior data checksums; 2143 * this is just good hygiene.) 2144 */ 2145 if (gn != pio->io_gang_leader->io_gang_tree) { 2146 abd_t *buf = abd_get_offset(data, offset); 2147 2148 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 2149 buf, BP_GET_PSIZE(bp)); 2150 2151 abd_put(buf); 2152 } 2153 /* 2154 * If we are here to damage data for testing purposes, 2155 * leave the GBH alone so that we can detect the damage. 2156 */ 2157 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 2158 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2159 } else { 2160 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 2161 abd_get_offset(data, offset), BP_GET_PSIZE(bp), 2162 zio_gang_issue_func_done, NULL, pio->io_priority, 2163 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2164 } 2165 2166 return (zio); 2167 } 2168 2169 /* ARGSUSED */ 2170 static zio_t * 2171 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2172 uint64_t offset) 2173 { 2174 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 2175 ZIO_GANG_CHILD_FLAGS(pio))); 2176 } 2177 2178 /* ARGSUSED */ 2179 static zio_t * 2180 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2181 uint64_t offset) 2182 { 2183 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 2184 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 2185 } 2186 2187 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 2188 NULL, 2189 zio_read_gang, 2190 zio_rewrite_gang, 2191 zio_free_gang, 2192 zio_claim_gang, 2193 NULL 2194 }; 2195 2196 static void zio_gang_tree_assemble_done(zio_t *zio); 2197 2198 static zio_gang_node_t * 2199 zio_gang_node_alloc(zio_gang_node_t **gnpp) 2200 { 2201 zio_gang_node_t *gn; 2202 2203 ASSERT(*gnpp == NULL); 2204 2205 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 2206 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 2207 *gnpp = gn; 2208 2209 return (gn); 2210 } 2211 2212 static void 2213 zio_gang_node_free(zio_gang_node_t **gnpp) 2214 { 2215 zio_gang_node_t *gn = *gnpp; 2216 2217 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 2218 ASSERT(gn->gn_child[g] == NULL); 2219 2220 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2221 kmem_free(gn, sizeof (*gn)); 2222 *gnpp = NULL; 2223 } 2224 2225 static void 2226 zio_gang_tree_free(zio_gang_node_t **gnpp) 2227 { 2228 zio_gang_node_t *gn = *gnpp; 2229 2230 if (gn == NULL) 2231 return; 2232 2233 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 2234 zio_gang_tree_free(&gn->gn_child[g]); 2235 2236 zio_gang_node_free(gnpp); 2237 } 2238 2239 static void 2240 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 2241 { 2242 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 2243 abd_t *gbh_abd = abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2244 2245 ASSERT(gio->io_gang_leader == gio); 2246 ASSERT(BP_IS_GANG(bp)); 2247 2248 zio_nowait(zio_read(gio, gio->io_spa, bp, gbh_abd, SPA_GANGBLOCKSIZE, 2249 zio_gang_tree_assemble_done, gn, gio->io_priority, 2250 ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 2251 } 2252 2253 static void 2254 zio_gang_tree_assemble_done(zio_t *zio) 2255 { 2256 zio_t *gio = zio->io_gang_leader; 2257 zio_gang_node_t *gn = zio->io_private; 2258 blkptr_t *bp = zio->io_bp; 2259 2260 ASSERT(gio == zio_unique_parent(zio)); 2261 ASSERT(zio->io_child_count == 0); 2262 2263 if (zio->io_error) 2264 return; 2265 2266 /* this ABD was created from a linear buf in zio_gang_tree_assemble */ 2267 if (BP_SHOULD_BYTESWAP(bp)) 2268 byteswap_uint64_array(abd_to_buf(zio->io_abd), zio->io_size); 2269 2270 ASSERT3P(abd_to_buf(zio->io_abd), ==, gn->gn_gbh); 2271 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 2272 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 2273 2274 abd_put(zio->io_abd); 2275 2276 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2277 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 2278 if (!BP_IS_GANG(gbp)) 2279 continue; 2280 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 2281 } 2282 } 2283 2284 static void 2285 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, abd_t *data, 2286 uint64_t offset) 2287 { 2288 zio_t *gio = pio->io_gang_leader; 2289 zio_t *zio; 2290 2291 ASSERT(BP_IS_GANG(bp) == !!gn); 2292 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 2293 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 2294 2295 /* 2296 * If you're a gang header, your data is in gn->gn_gbh. 2297 * If you're a gang member, your data is in 'data' and gn == NULL. 2298 */ 2299 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data, offset); 2300 2301 if (gn != NULL) { 2302 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 2303 2304 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2305 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 2306 if (BP_IS_HOLE(gbp)) 2307 continue; 2308 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data, 2309 offset); 2310 offset += BP_GET_PSIZE(gbp); 2311 } 2312 } 2313 2314 if (gn == gio->io_gang_tree) 2315 ASSERT3U(gio->io_size, ==, offset); 2316 2317 if (zio != pio) 2318 zio_nowait(zio); 2319 } 2320 2321 static int 2322 zio_gang_assemble(zio_t *zio) 2323 { 2324 blkptr_t *bp = zio->io_bp; 2325 2326 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 2327 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2328 2329 zio->io_gang_leader = zio; 2330 2331 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 2332 2333 return (ZIO_PIPELINE_CONTINUE); 2334 } 2335 2336 static int 2337 zio_gang_issue(zio_t *zio) 2338 { 2339 blkptr_t *bp = zio->io_bp; 2340 2341 if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) { 2342 return (ZIO_PIPELINE_STOP); 2343 } 2344 2345 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 2346 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2347 2348 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 2349 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_abd, 2350 0); 2351 else 2352 zio_gang_tree_free(&zio->io_gang_tree); 2353 2354 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2355 2356 return (ZIO_PIPELINE_CONTINUE); 2357 } 2358 2359 static void 2360 zio_write_gang_member_ready(zio_t *zio) 2361 { 2362 zio_t *pio = zio_unique_parent(zio); 2363 zio_t *gio = zio->io_gang_leader; 2364 dva_t *cdva = zio->io_bp->blk_dva; 2365 dva_t *pdva = pio->io_bp->blk_dva; 2366 uint64_t asize; 2367 2368 if (BP_IS_HOLE(zio->io_bp)) 2369 return; 2370 2371 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 2372 2373 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 2374 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 2375 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 2376 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 2377 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 2378 2379 mutex_enter(&pio->io_lock); 2380 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 2381 ASSERT(DVA_GET_GANG(&pdva[d])); 2382 asize = DVA_GET_ASIZE(&pdva[d]); 2383 asize += DVA_GET_ASIZE(&cdva[d]); 2384 DVA_SET_ASIZE(&pdva[d], asize); 2385 } 2386 mutex_exit(&pio->io_lock); 2387 } 2388 2389 static void 2390 zio_write_gang_done(zio_t *zio) 2391 { 2392 /* 2393 * The io_abd field will be NULL for a zio with no data. The io_flags 2394 * will initially have the ZIO_FLAG_NODATA bit flag set, but we can't 2395 * check for it here as it is cleared in zio_ready. 2396 */ 2397 if (zio->io_abd != NULL) 2398 abd_put(zio->io_abd); 2399 } 2400 2401 static int 2402 zio_write_gang_block(zio_t *pio) 2403 { 2404 spa_t *spa = pio->io_spa; 2405 metaslab_class_t *mc = spa_normal_class(spa); 2406 blkptr_t *bp = pio->io_bp; 2407 zio_t *gio = pio->io_gang_leader; 2408 zio_t *zio; 2409 zio_gang_node_t *gn, **gnpp; 2410 zio_gbh_phys_t *gbh; 2411 abd_t *gbh_abd; 2412 uint64_t txg = pio->io_txg; 2413 uint64_t resid = pio->io_size; 2414 uint64_t lsize; 2415 int copies = gio->io_prop.zp_copies; 2416 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 2417 zio_prop_t zp; 2418 int error; 2419 boolean_t has_data = !(pio->io_flags & ZIO_FLAG_NODATA); 2420 2421 /* 2422 * encrypted blocks need DVA[2] free so encrypted gang headers can't 2423 * have a third copy. 2424 */ 2425 if (gio->io_prop.zp_encrypt && gbh_copies >= SPA_DVAS_PER_BP) 2426 gbh_copies = SPA_DVAS_PER_BP - 1; 2427 2428 int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER; 2429 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2430 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2431 ASSERT(has_data); 2432 2433 flags |= METASLAB_ASYNC_ALLOC; 2434 VERIFY(zfs_refcount_held(&mc->mc_alloc_slots[pio->io_allocator], 2435 pio)); 2436 2437 /* 2438 * The logical zio has already placed a reservation for 2439 * 'copies' allocation slots but gang blocks may require 2440 * additional copies. These additional copies 2441 * (i.e. gbh_copies - copies) are guaranteed to succeed 2442 * since metaslab_class_throttle_reserve() always allows 2443 * additional reservations for gang blocks. 2444 */ 2445 VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies, 2446 pio->io_allocator, pio, flags)); 2447 } 2448 2449 error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE, 2450 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags, 2451 &pio->io_alloc_list, pio, pio->io_allocator); 2452 if (error) { 2453 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2454 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2455 ASSERT(has_data); 2456 2457 /* 2458 * If we failed to allocate the gang block header then 2459 * we remove any additional allocation reservations that 2460 * we placed here. The original reservation will 2461 * be removed when the logical I/O goes to the ready 2462 * stage. 2463 */ 2464 metaslab_class_throttle_unreserve(mc, 2465 gbh_copies - copies, pio->io_allocator, pio); 2466 } 2467 pio->io_error = error; 2468 return (ZIO_PIPELINE_CONTINUE); 2469 } 2470 2471 if (pio == gio) { 2472 gnpp = &gio->io_gang_tree; 2473 } else { 2474 gnpp = pio->io_private; 2475 ASSERT(pio->io_ready == zio_write_gang_member_ready); 2476 } 2477 2478 gn = zio_gang_node_alloc(gnpp); 2479 gbh = gn->gn_gbh; 2480 bzero(gbh, SPA_GANGBLOCKSIZE); 2481 gbh_abd = abd_get_from_buf(gbh, SPA_GANGBLOCKSIZE); 2482 2483 /* 2484 * Create the gang header. 2485 */ 2486 zio = zio_rewrite(pio, spa, txg, bp, gbh_abd, SPA_GANGBLOCKSIZE, 2487 zio_write_gang_done, NULL, pio->io_priority, 2488 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2489 2490 /* 2491 * Create and nowait the gang children. 2492 */ 2493 for (int g = 0; resid != 0; resid -= lsize, g++) { 2494 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 2495 SPA_MINBLOCKSIZE); 2496 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 2497 2498 zp.zp_checksum = gio->io_prop.zp_checksum; 2499 zp.zp_compress = ZIO_COMPRESS_OFF; 2500 zp.zp_type = DMU_OT_NONE; 2501 zp.zp_level = 0; 2502 zp.zp_copies = gio->io_prop.zp_copies; 2503 zp.zp_dedup = B_FALSE; 2504 zp.zp_dedup_verify = B_FALSE; 2505 zp.zp_nopwrite = B_FALSE; 2506 zp.zp_encrypt = gio->io_prop.zp_encrypt; 2507 zp.zp_byteorder = gio->io_prop.zp_byteorder; 2508 bzero(zp.zp_salt, ZIO_DATA_SALT_LEN); 2509 bzero(zp.zp_iv, ZIO_DATA_IV_LEN); 2510 bzero(zp.zp_mac, ZIO_DATA_MAC_LEN); 2511 2512 zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 2513 has_data ? abd_get_offset(pio->io_abd, pio->io_size - 2514 resid) : NULL, lsize, lsize, &zp, 2515 zio_write_gang_member_ready, NULL, NULL, 2516 zio_write_gang_done, &gn->gn_child[g], pio->io_priority, 2517 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2518 2519 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2520 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2521 ASSERT(has_data); 2522 2523 /* 2524 * Gang children won't throttle but we should 2525 * account for their work, so reserve an allocation 2526 * slot for them here. 2527 */ 2528 VERIFY(metaslab_class_throttle_reserve(mc, 2529 zp.zp_copies, cio->io_allocator, cio, flags)); 2530 } 2531 zio_nowait(cio); 2532 } 2533 2534 /* 2535 * Set pio's pipeline to just wait for zio to finish. 2536 */ 2537 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2538 2539 zio_nowait(zio); 2540 2541 return (ZIO_PIPELINE_CONTINUE); 2542 } 2543 2544 /* 2545 * The zio_nop_write stage in the pipeline determines if allocating a 2546 * new bp is necessary. The nopwrite feature can handle writes in 2547 * either syncing or open context (i.e. zil writes) and as a result is 2548 * mutually exclusive with dedup. 2549 * 2550 * By leveraging a cryptographically secure checksum, such as SHA256, we 2551 * can compare the checksums of the new data and the old to determine if 2552 * allocating a new block is required. Note that our requirements for 2553 * cryptographic strength are fairly weak: there can't be any accidental 2554 * hash collisions, but we don't need to be secure against intentional 2555 * (malicious) collisions. To trigger a nopwrite, you have to be able 2556 * to write the file to begin with, and triggering an incorrect (hash 2557 * collision) nopwrite is no worse than simply writing to the file. 2558 * That said, there are no known attacks against the checksum algorithms 2559 * used for nopwrite, assuming that the salt and the checksums 2560 * themselves remain secret. 2561 */ 2562 static int 2563 zio_nop_write(zio_t *zio) 2564 { 2565 blkptr_t *bp = zio->io_bp; 2566 blkptr_t *bp_orig = &zio->io_bp_orig; 2567 zio_prop_t *zp = &zio->io_prop; 2568 2569 ASSERT(BP_GET_LEVEL(bp) == 0); 2570 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 2571 ASSERT(zp->zp_nopwrite); 2572 ASSERT(!zp->zp_dedup); 2573 ASSERT(zio->io_bp_override == NULL); 2574 ASSERT(IO_IS_ALLOCATING(zio)); 2575 2576 /* 2577 * Check to see if the original bp and the new bp have matching 2578 * characteristics (i.e. same checksum, compression algorithms, etc). 2579 * If they don't then just continue with the pipeline which will 2580 * allocate a new bp. 2581 */ 2582 if (BP_IS_HOLE(bp_orig) || 2583 !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags & 2584 ZCHECKSUM_FLAG_NOPWRITE) || 2585 BP_IS_ENCRYPTED(bp) || BP_IS_ENCRYPTED(bp_orig) || 2586 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 2587 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 2588 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 2589 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 2590 return (ZIO_PIPELINE_CONTINUE); 2591 2592 /* 2593 * If the checksums match then reset the pipeline so that we 2594 * avoid allocating a new bp and issuing any I/O. 2595 */ 2596 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 2597 ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags & 2598 ZCHECKSUM_FLAG_NOPWRITE); 2599 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 2600 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 2601 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 2602 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 2603 sizeof (uint64_t)) == 0); 2604 2605 *bp = *bp_orig; 2606 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2607 zio->io_flags |= ZIO_FLAG_NOPWRITE; 2608 } 2609 2610 return (ZIO_PIPELINE_CONTINUE); 2611 } 2612 2613 /* 2614 * ========================================================================== 2615 * Dedup 2616 * ========================================================================== 2617 */ 2618 static void 2619 zio_ddt_child_read_done(zio_t *zio) 2620 { 2621 blkptr_t *bp = zio->io_bp; 2622 ddt_entry_t *dde = zio->io_private; 2623 ddt_phys_t *ddp; 2624 zio_t *pio = zio_unique_parent(zio); 2625 2626 mutex_enter(&pio->io_lock); 2627 ddp = ddt_phys_select(dde, bp); 2628 if (zio->io_error == 0) 2629 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 2630 2631 if (zio->io_error == 0 && dde->dde_repair_abd == NULL) 2632 dde->dde_repair_abd = zio->io_abd; 2633 else 2634 abd_free(zio->io_abd); 2635 mutex_exit(&pio->io_lock); 2636 } 2637 2638 static int 2639 zio_ddt_read_start(zio_t *zio) 2640 { 2641 blkptr_t *bp = zio->io_bp; 2642 2643 ASSERT(BP_GET_DEDUP(bp)); 2644 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2645 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2646 2647 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2648 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2649 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 2650 ddt_phys_t *ddp = dde->dde_phys; 2651 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 2652 blkptr_t blk; 2653 2654 ASSERT(zio->io_vsd == NULL); 2655 zio->io_vsd = dde; 2656 2657 if (ddp_self == NULL) 2658 return (ZIO_PIPELINE_CONTINUE); 2659 2660 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2661 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 2662 continue; 2663 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 2664 &blk); 2665 zio_nowait(zio_read(zio, zio->io_spa, &blk, 2666 abd_alloc_for_io(zio->io_size, B_TRUE), 2667 zio->io_size, zio_ddt_child_read_done, dde, 2668 zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) | 2669 ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark)); 2670 } 2671 return (ZIO_PIPELINE_CONTINUE); 2672 } 2673 2674 zio_nowait(zio_read(zio, zio->io_spa, bp, 2675 zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority, 2676 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 2677 2678 return (ZIO_PIPELINE_CONTINUE); 2679 } 2680 2681 static int 2682 zio_ddt_read_done(zio_t *zio) 2683 { 2684 blkptr_t *bp = zio->io_bp; 2685 2686 if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) { 2687 return (ZIO_PIPELINE_STOP); 2688 } 2689 2690 ASSERT(BP_GET_DEDUP(bp)); 2691 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2692 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2693 2694 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2695 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2696 ddt_entry_t *dde = zio->io_vsd; 2697 if (ddt == NULL) { 2698 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 2699 return (ZIO_PIPELINE_CONTINUE); 2700 } 2701 if (dde == NULL) { 2702 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 2703 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2704 return (ZIO_PIPELINE_STOP); 2705 } 2706 if (dde->dde_repair_abd != NULL) { 2707 abd_copy(zio->io_abd, dde->dde_repair_abd, 2708 zio->io_size); 2709 zio->io_child_error[ZIO_CHILD_DDT] = 0; 2710 } 2711 ddt_repair_done(ddt, dde); 2712 zio->io_vsd = NULL; 2713 } 2714 2715 ASSERT(zio->io_vsd == NULL); 2716 2717 return (ZIO_PIPELINE_CONTINUE); 2718 } 2719 2720 static boolean_t 2721 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 2722 { 2723 spa_t *spa = zio->io_spa; 2724 boolean_t do_raw = !!(zio->io_flags & ZIO_FLAG_RAW); 2725 2726 /* We should never get a raw, override zio */ 2727 ASSERT(!(zio->io_bp_override && do_raw)); 2728 2729 /* 2730 * Note: we compare the original data, not the transformed data, 2731 * because when zio->io_bp is an override bp, we will not have 2732 * pushed the I/O transforms. That's an important optimization 2733 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 2734 * However, we should never get a raw, override zio so in these 2735 * cases we can compare the io_data directly. This is useful because 2736 * it allows us to do dedup verification even if we don't have access 2737 * to the original data (for instance, if the encryption keys aren't 2738 * loaded). 2739 */ 2740 2741 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2742 zio_t *lio = dde->dde_lead_zio[p]; 2743 2744 if (lio != NULL && do_raw) { 2745 return (lio->io_size != zio->io_size || 2746 abd_cmp(zio->io_abd, lio->io_abd, 2747 zio->io_size) != 0); 2748 } else if (lio != NULL) { 2749 return (lio->io_orig_size != zio->io_orig_size || 2750 abd_cmp(zio->io_orig_abd, lio->io_orig_abd, 2751 zio->io_orig_size) != 0); 2752 } 2753 } 2754 2755 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2756 ddt_phys_t *ddp = &dde->dde_phys[p]; 2757 2758 if (ddp->ddp_phys_birth != 0 && do_raw) { 2759 blkptr_t blk = *zio->io_bp; 2760 uint64_t psize; 2761 abd_t *tmpabd; 2762 int error; 2763 2764 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2765 psize = BP_GET_PSIZE(&blk); 2766 2767 if (psize != zio->io_size) 2768 return (B_TRUE); 2769 2770 ddt_exit(ddt); 2771 2772 tmpabd = abd_alloc_for_io(psize, B_TRUE); 2773 2774 error = zio_wait(zio_read(NULL, spa, &blk, tmpabd, 2775 psize, NULL, NULL, ZIO_PRIORITY_SYNC_READ, 2776 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 2777 ZIO_FLAG_RAW, &zio->io_bookmark)); 2778 2779 if (error == 0) { 2780 if (abd_cmp(tmpabd, zio->io_abd, psize) != 0) 2781 error = SET_ERROR(ENOENT); 2782 } 2783 2784 abd_free(tmpabd); 2785 ddt_enter(ddt); 2786 return (error != 0); 2787 } else if (ddp->ddp_phys_birth != 0) { 2788 arc_buf_t *abuf = NULL; 2789 arc_flags_t aflags = ARC_FLAG_WAIT; 2790 int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE; 2791 blkptr_t blk = *zio->io_bp; 2792 int error; 2793 2794 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2795 2796 if (BP_GET_LSIZE(&blk) != zio->io_orig_size) 2797 return (B_TRUE); 2798 2799 ddt_exit(ddt); 2800 2801 /* 2802 * Intuitively, it would make more sense to compare 2803 * io_abd than io_orig_abd in the raw case since you 2804 * don't want to look at any transformations that have 2805 * happened to the data. However, for raw I/Os the 2806 * data will actually be the same in io_abd and 2807 * io_orig_abd, so all we have to do is issue this as 2808 * a raw ARC read. 2809 */ 2810 if (do_raw) { 2811 zio_flags |= ZIO_FLAG_RAW; 2812 ASSERT3U(zio->io_size, ==, zio->io_orig_size); 2813 ASSERT0(abd_cmp(zio->io_abd, zio->io_orig_abd, 2814 zio->io_size)); 2815 ASSERT3P(zio->io_transform_stack, ==, NULL); 2816 } 2817 2818 error = arc_read(NULL, spa, &blk, 2819 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 2820 zio_flags, &aflags, &zio->io_bookmark); 2821 2822 if (error == 0) { 2823 if (abd_cmp_buf(zio->io_orig_abd, abuf->b_data, 2824 zio->io_orig_size) != 0) 2825 error = SET_ERROR(ENOENT); 2826 arc_buf_destroy(abuf, &abuf); 2827 } 2828 2829 ddt_enter(ddt); 2830 return (error != 0); 2831 } 2832 } 2833 2834 return (B_FALSE); 2835 } 2836 2837 static void 2838 zio_ddt_child_write_ready(zio_t *zio) 2839 { 2840 int p = zio->io_prop.zp_copies; 2841 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2842 ddt_entry_t *dde = zio->io_private; 2843 ddt_phys_t *ddp = &dde->dde_phys[p]; 2844 zio_t *pio; 2845 2846 if (zio->io_error) 2847 return; 2848 2849 ddt_enter(ddt); 2850 2851 ASSERT(dde->dde_lead_zio[p] == zio); 2852 2853 ddt_phys_fill(ddp, zio->io_bp); 2854 2855 zio_link_t *zl = NULL; 2856 while ((pio = zio_walk_parents(zio, &zl)) != NULL) 2857 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2858 2859 ddt_exit(ddt); 2860 } 2861 2862 static void 2863 zio_ddt_child_write_done(zio_t *zio) 2864 { 2865 int p = zio->io_prop.zp_copies; 2866 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2867 ddt_entry_t *dde = zio->io_private; 2868 ddt_phys_t *ddp = &dde->dde_phys[p]; 2869 2870 ddt_enter(ddt); 2871 2872 ASSERT(ddp->ddp_refcnt == 0); 2873 ASSERT(dde->dde_lead_zio[p] == zio); 2874 dde->dde_lead_zio[p] = NULL; 2875 2876 if (zio->io_error == 0) { 2877 zio_link_t *zl = NULL; 2878 while (zio_walk_parents(zio, &zl) != NULL) 2879 ddt_phys_addref(ddp); 2880 } else { 2881 ddt_phys_clear(ddp); 2882 } 2883 2884 ddt_exit(ddt); 2885 } 2886 2887 static void 2888 zio_ddt_ditto_write_done(zio_t *zio) 2889 { 2890 int p = DDT_PHYS_DITTO; 2891 zio_prop_t *zp = &zio->io_prop; 2892 blkptr_t *bp = zio->io_bp; 2893 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2894 ddt_entry_t *dde = zio->io_private; 2895 ddt_phys_t *ddp = &dde->dde_phys[p]; 2896 ddt_key_t *ddk = &dde->dde_key; 2897 2898 ddt_enter(ddt); 2899 2900 ASSERT(ddp->ddp_refcnt == 0); 2901 ASSERT(dde->dde_lead_zio[p] == zio); 2902 dde->dde_lead_zio[p] = NULL; 2903 2904 if (zio->io_error == 0) { 2905 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2906 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2907 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2908 if (ddp->ddp_phys_birth != 0) 2909 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2910 ddt_phys_fill(ddp, bp); 2911 } 2912 2913 ddt_exit(ddt); 2914 } 2915 2916 static int 2917 zio_ddt_write(zio_t *zio) 2918 { 2919 spa_t *spa = zio->io_spa; 2920 blkptr_t *bp = zio->io_bp; 2921 uint64_t txg = zio->io_txg; 2922 zio_prop_t *zp = &zio->io_prop; 2923 int p = zp->zp_copies; 2924 int ditto_copies; 2925 zio_t *cio = NULL; 2926 zio_t *dio = NULL; 2927 ddt_t *ddt = ddt_select(spa, bp); 2928 ddt_entry_t *dde; 2929 ddt_phys_t *ddp; 2930 2931 ASSERT(BP_GET_DEDUP(bp)); 2932 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2933 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2934 ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW))); 2935 2936 ddt_enter(ddt); 2937 dde = ddt_lookup(ddt, bp, B_TRUE); 2938 ddp = &dde->dde_phys[p]; 2939 2940 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2941 /* 2942 * If we're using a weak checksum, upgrade to a strong checksum 2943 * and try again. If we're already using a strong checksum, 2944 * we can't resolve it, so just convert to an ordinary write. 2945 * (And automatically e-mail a paper to Nature?) 2946 */ 2947 if (!(zio_checksum_table[zp->zp_checksum].ci_flags & 2948 ZCHECKSUM_FLAG_DEDUP)) { 2949 zp->zp_checksum = spa_dedup_checksum(spa); 2950 zio_pop_transforms(zio); 2951 zio->io_stage = ZIO_STAGE_OPEN; 2952 BP_ZERO(bp); 2953 } else { 2954 zp->zp_dedup = B_FALSE; 2955 BP_SET_DEDUP(bp, B_FALSE); 2956 } 2957 ASSERT(!BP_GET_DEDUP(bp)); 2958 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2959 ddt_exit(ddt); 2960 return (ZIO_PIPELINE_CONTINUE); 2961 } 2962 2963 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2964 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2965 2966 if (ditto_copies > ddt_ditto_copies_present(dde) && 2967 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2968 zio_prop_t czp = *zp; 2969 2970 czp.zp_copies = ditto_copies; 2971 2972 /* 2973 * If we arrived here with an override bp, we won't have run 2974 * the transform stack, so we won't have the data we need to 2975 * generate a child i/o. So, toss the override bp and restart. 2976 * This is safe, because using the override bp is just an 2977 * optimization; and it's rare, so the cost doesn't matter. 2978 */ 2979 if (zio->io_bp_override) { 2980 zio_pop_transforms(zio); 2981 zio->io_stage = ZIO_STAGE_OPEN; 2982 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2983 zio->io_bp_override = NULL; 2984 BP_ZERO(bp); 2985 ddt_exit(ddt); 2986 return (ZIO_PIPELINE_CONTINUE); 2987 } 2988 2989 dio = zio_write(zio, spa, txg, bp, zio->io_orig_abd, 2990 zio->io_orig_size, zio->io_orig_size, &czp, NULL, NULL, 2991 NULL, zio_ddt_ditto_write_done, dde, zio->io_priority, 2992 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2993 2994 zio_push_transform(dio, zio->io_abd, zio->io_size, 0, NULL); 2995 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2996 } 2997 2998 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2999 if (ddp->ddp_phys_birth != 0) 3000 ddt_bp_fill(ddp, bp, txg); 3001 if (dde->dde_lead_zio[p] != NULL) 3002 zio_add_child(zio, dde->dde_lead_zio[p]); 3003 else 3004 ddt_phys_addref(ddp); 3005 } else if (zio->io_bp_override) { 3006 ASSERT(bp->blk_birth == txg); 3007 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 3008 ddt_phys_fill(ddp, bp); 3009 ddt_phys_addref(ddp); 3010 } else { 3011 cio = zio_write(zio, spa, txg, bp, zio->io_orig_abd, 3012 zio->io_orig_size, zio->io_orig_size, zp, 3013 zio_ddt_child_write_ready, NULL, NULL, 3014 zio_ddt_child_write_done, dde, zio->io_priority, 3015 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 3016 3017 zio_push_transform(cio, zio->io_abd, zio->io_size, 0, NULL); 3018 dde->dde_lead_zio[p] = cio; 3019 } 3020 3021 ddt_exit(ddt); 3022 3023 if (cio) 3024 zio_nowait(cio); 3025 if (dio) 3026 zio_nowait(dio); 3027 3028 return (ZIO_PIPELINE_CONTINUE); 3029 } 3030 3031 ddt_entry_t *freedde; /* for debugging */ 3032 3033 static int 3034 zio_ddt_free(zio_t *zio) 3035 { 3036 spa_t *spa = zio->io_spa; 3037 blkptr_t *bp = zio->io_bp; 3038 ddt_t *ddt = ddt_select(spa, bp); 3039 ddt_entry_t *dde; 3040 ddt_phys_t *ddp; 3041 3042 ASSERT(BP_GET_DEDUP(bp)); 3043 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3044 3045 ddt_enter(ddt); 3046 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 3047 ddp = ddt_phys_select(dde, bp); 3048 ddt_phys_decref(ddp); 3049 ddt_exit(ddt); 3050 3051 return (ZIO_PIPELINE_CONTINUE); 3052 } 3053 3054 /* 3055 * ========================================================================== 3056 * Allocate and free blocks 3057 * ========================================================================== 3058 */ 3059 3060 static zio_t * 3061 zio_io_to_allocate(spa_t *spa, int allocator) 3062 { 3063 zio_t *zio; 3064 3065 ASSERT(MUTEX_HELD(&spa->spa_alloc_locks[allocator])); 3066 3067 zio = avl_first(&spa->spa_alloc_trees[allocator]); 3068 if (zio == NULL) 3069 return (NULL); 3070 3071 ASSERT(IO_IS_ALLOCATING(zio)); 3072 3073 /* 3074 * Try to place a reservation for this zio. If we're unable to 3075 * reserve then we throttle. 3076 */ 3077 ASSERT3U(zio->io_allocator, ==, allocator); 3078 if (!metaslab_class_throttle_reserve(zio->io_metaslab_class, 3079 zio->io_prop.zp_copies, zio->io_allocator, zio, 0)) { 3080 return (NULL); 3081 } 3082 3083 avl_remove(&spa->spa_alloc_trees[allocator], zio); 3084 ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE); 3085 3086 return (zio); 3087 } 3088 3089 static int 3090 zio_dva_throttle(zio_t *zio) 3091 { 3092 spa_t *spa = zio->io_spa; 3093 zio_t *nio; 3094 metaslab_class_t *mc; 3095 3096 /* locate an appropriate allocation class */ 3097 mc = spa_preferred_class(spa, zio->io_size, zio->io_prop.zp_type, 3098 zio->io_prop.zp_level, zio->io_prop.zp_zpl_smallblk); 3099 3100 if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE || 3101 !mc->mc_alloc_throttle_enabled || 3102 zio->io_child_type == ZIO_CHILD_GANG || 3103 zio->io_flags & ZIO_FLAG_NODATA) { 3104 return (ZIO_PIPELINE_CONTINUE); 3105 } 3106 3107 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 3108 3109 ASSERT3U(zio->io_queued_timestamp, >, 0); 3110 ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE); 3111 3112 zbookmark_phys_t *bm = &zio->io_bookmark; 3113 /* 3114 * We want to try to use as many allocators as possible to help improve 3115 * performance, but we also want logically adjacent IOs to be physically 3116 * adjacent to improve sequential read performance. We chunk each object 3117 * into 2^20 block regions, and then hash based on the objset, object, 3118 * level, and region to accomplish both of these goals. 3119 */ 3120 zio->io_allocator = cityhash4(bm->zb_objset, bm->zb_object, 3121 bm->zb_level, bm->zb_blkid >> 20) % spa->spa_alloc_count; 3122 mutex_enter(&spa->spa_alloc_locks[zio->io_allocator]); 3123 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 3124 zio->io_metaslab_class = mc; 3125 avl_add(&spa->spa_alloc_trees[zio->io_allocator], zio); 3126 nio = zio_io_to_allocate(spa, zio->io_allocator); 3127 mutex_exit(&spa->spa_alloc_locks[zio->io_allocator]); 3128 3129 if (nio == zio) 3130 return (ZIO_PIPELINE_CONTINUE); 3131 3132 if (nio != NULL) { 3133 ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE); 3134 /* 3135 * We are passing control to a new zio so make sure that 3136 * it is processed by a different thread. We do this to 3137 * avoid stack overflows that can occur when parents are 3138 * throttled and children are making progress. We allow 3139 * it to go to the head of the taskq since it's already 3140 * been waiting. 3141 */ 3142 zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE); 3143 } 3144 return (ZIO_PIPELINE_STOP); 3145 } 3146 3147 static void 3148 zio_allocate_dispatch(spa_t *spa, int allocator) 3149 { 3150 zio_t *zio; 3151 3152 mutex_enter(&spa->spa_alloc_locks[allocator]); 3153 zio = zio_io_to_allocate(spa, allocator); 3154 mutex_exit(&spa->spa_alloc_locks[allocator]); 3155 if (zio == NULL) 3156 return; 3157 3158 ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE); 3159 ASSERT0(zio->io_error); 3160 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE); 3161 } 3162 3163 static int 3164 zio_dva_allocate(zio_t *zio) 3165 { 3166 spa_t *spa = zio->io_spa; 3167 metaslab_class_t *mc; 3168 blkptr_t *bp = zio->io_bp; 3169 int error; 3170 int flags = 0; 3171 3172 if (zio->io_gang_leader == NULL) { 3173 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 3174 zio->io_gang_leader = zio; 3175 } 3176 3177 ASSERT(BP_IS_HOLE(bp)); 3178 ASSERT0(BP_GET_NDVAS(bp)); 3179 ASSERT3U(zio->io_prop.zp_copies, >, 0); 3180 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 3181 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 3182 3183 if (zio->io_flags & ZIO_FLAG_NODATA) 3184 flags |= METASLAB_DONT_THROTTLE; 3185 if (zio->io_flags & ZIO_FLAG_GANG_CHILD) 3186 flags |= METASLAB_GANG_CHILD; 3187 if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) 3188 flags |= METASLAB_ASYNC_ALLOC; 3189 3190 /* 3191 * if not already chosen, locate an appropriate allocation class 3192 */ 3193 mc = zio->io_metaslab_class; 3194 if (mc == NULL) { 3195 mc = spa_preferred_class(spa, zio->io_size, 3196 zio->io_prop.zp_type, zio->io_prop.zp_level, 3197 zio->io_prop.zp_zpl_smallblk); 3198 zio->io_metaslab_class = mc; 3199 } 3200 3201 error = metaslab_alloc(spa, mc, zio->io_size, bp, 3202 zio->io_prop.zp_copies, zio->io_txg, NULL, flags, 3203 &zio->io_alloc_list, zio, zio->io_allocator); 3204 3205 /* 3206 * Fallback to normal class when an alloc class is full 3207 */ 3208 if (error == ENOSPC && mc != spa_normal_class(spa)) { 3209 /* 3210 * If throttling, transfer reservation over to normal class. 3211 * The io_allocator slot can remain the same even though we 3212 * are switching classes. 3213 */ 3214 if (mc->mc_alloc_throttle_enabled && 3215 (zio->io_flags & ZIO_FLAG_IO_ALLOCATING)) { 3216 metaslab_class_throttle_unreserve(mc, 3217 zio->io_prop.zp_copies, zio->io_allocator, zio); 3218 zio->io_flags &= ~ZIO_FLAG_IO_ALLOCATING; 3219 3220 mc = spa_normal_class(spa); 3221 VERIFY(metaslab_class_throttle_reserve(mc, 3222 zio->io_prop.zp_copies, zio->io_allocator, zio, 3223 flags | METASLAB_MUST_RESERVE)); 3224 } else { 3225 mc = spa_normal_class(spa); 3226 } 3227 zio->io_metaslab_class = mc; 3228 3229 error = metaslab_alloc(spa, mc, zio->io_size, bp, 3230 zio->io_prop.zp_copies, zio->io_txg, NULL, flags, 3231 &zio->io_alloc_list, zio, zio->io_allocator); 3232 } 3233 3234 if (error != 0) { 3235 zfs_dbgmsg("%s: metaslab allocation failure: zio %p, " 3236 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 3237 error); 3238 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 3239 return (zio_write_gang_block(zio)); 3240 zio->io_error = error; 3241 } 3242 3243 return (ZIO_PIPELINE_CONTINUE); 3244 } 3245 3246 static int 3247 zio_dva_free(zio_t *zio) 3248 { 3249 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 3250 3251 return (ZIO_PIPELINE_CONTINUE); 3252 } 3253 3254 static int 3255 zio_dva_claim(zio_t *zio) 3256 { 3257 int error; 3258 3259 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 3260 if (error) 3261 zio->io_error = error; 3262 3263 return (ZIO_PIPELINE_CONTINUE); 3264 } 3265 3266 /* 3267 * Undo an allocation. This is used by zio_done() when an I/O fails 3268 * and we want to give back the block we just allocated. 3269 * This handles both normal blocks and gang blocks. 3270 */ 3271 static void 3272 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 3273 { 3274 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 3275 ASSERT(zio->io_bp_override == NULL); 3276 3277 if (!BP_IS_HOLE(bp)) 3278 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 3279 3280 if (gn != NULL) { 3281 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 3282 zio_dva_unallocate(zio, gn->gn_child[g], 3283 &gn->gn_gbh->zg_blkptr[g]); 3284 } 3285 } 3286 } 3287 3288 /* 3289 * Try to allocate an intent log block. Return 0 on success, errno on failure. 3290 */ 3291 int 3292 zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, 3293 blkptr_t *old_bp, uint64_t size, boolean_t *slog) 3294 { 3295 int error = 1; 3296 zio_alloc_list_t io_alloc_list; 3297 3298 ASSERT(txg > spa_syncing_txg(spa)); 3299 3300 metaslab_trace_init(&io_alloc_list); 3301 3302 /* 3303 * Block pointer fields are useful to metaslabs for stats and debugging. 3304 * Fill in the obvious ones before calling into metaslab_alloc(). 3305 */ 3306 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 3307 BP_SET_PSIZE(new_bp, size); 3308 BP_SET_LEVEL(new_bp, 0); 3309 3310 /* 3311 * When allocating a zil block, we don't have information about 3312 * the final destination of the block except the objset it's part 3313 * of, so we just hash the objset ID to pick the allocator to get 3314 * some parallelism. 3315 */ 3316 error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1, 3317 txg, old_bp, METASLAB_HINTBP_AVOID, &io_alloc_list, NULL, 3318 cityhash4(0, 0, 0, 3319 os->os_dsl_dataset->ds_object) % spa->spa_alloc_count); 3320 if (error == 0) { 3321 *slog = TRUE; 3322 } else { 3323 error = metaslab_alloc(spa, spa_normal_class(spa), size, 3324 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID, 3325 &io_alloc_list, NULL, cityhash4(0, 0, 0, 3326 os->os_dsl_dataset->ds_object) % spa->spa_alloc_count); 3327 if (error == 0) 3328 *slog = FALSE; 3329 } 3330 metaslab_trace_fini(&io_alloc_list); 3331 3332 if (error == 0) { 3333 BP_SET_LSIZE(new_bp, size); 3334 BP_SET_PSIZE(new_bp, size); 3335 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 3336 BP_SET_CHECKSUM(new_bp, 3337 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 3338 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 3339 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 3340 BP_SET_LEVEL(new_bp, 0); 3341 BP_SET_DEDUP(new_bp, 0); 3342 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 3343 3344 /* 3345 * encrypted blocks will require an IV and salt. We generate 3346 * these now since we will not be rewriting the bp at 3347 * rewrite time. 3348 */ 3349 if (os->os_encrypted) { 3350 uint8_t iv[ZIO_DATA_IV_LEN]; 3351 uint8_t salt[ZIO_DATA_SALT_LEN]; 3352 3353 BP_SET_CRYPT(new_bp, B_TRUE); 3354 VERIFY0(spa_crypt_get_salt(spa, 3355 dmu_objset_id(os), salt)); 3356 VERIFY0(zio_crypt_generate_iv(iv)); 3357 3358 zio_crypt_encode_params_bp(new_bp, salt, iv); 3359 } 3360 } else { 3361 zfs_dbgmsg("%s: zil block allocation failure: " 3362 "size %llu, error %d", spa_name(spa), size, error); 3363 } 3364 3365 return (error); 3366 } 3367 3368 /* 3369 * ========================================================================== 3370 * Read and write to physical devices 3371 * ========================================================================== 3372 */ 3373 3374 /* 3375 * Issue an I/O to the underlying vdev. Typically the issue pipeline 3376 * stops after this stage and will resume upon I/O completion. 3377 * However, there are instances where the vdev layer may need to 3378 * continue the pipeline when an I/O was not issued. Since the I/O 3379 * that was sent to the vdev layer might be different than the one 3380 * currently active in the pipeline (see vdev_queue_io()), we explicitly 3381 * force the underlying vdev layers to call either zio_execute() or 3382 * zio_interrupt() to ensure that the pipeline continues with the correct I/O. 3383 */ 3384 static int 3385 zio_vdev_io_start(zio_t *zio) 3386 { 3387 vdev_t *vd = zio->io_vd; 3388 uint64_t align; 3389 spa_t *spa = zio->io_spa; 3390 3391 ASSERT(zio->io_error == 0); 3392 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 3393 3394 if (vd == NULL) { 3395 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 3396 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 3397 3398 /* 3399 * The mirror_ops handle multiple DVAs in a single BP. 3400 */ 3401 vdev_mirror_ops.vdev_op_io_start(zio); 3402 return (ZIO_PIPELINE_STOP); 3403 } 3404 3405 ASSERT3P(zio->io_logical, !=, zio); 3406 if (zio->io_type == ZIO_TYPE_WRITE) { 3407 ASSERT(spa->spa_trust_config); 3408 3409 /* 3410 * Note: the code can handle other kinds of writes, 3411 * but we don't expect them. 3412 */ 3413 if (zio->io_vd->vdev_removing) { 3414 ASSERT(zio->io_flags & 3415 (ZIO_FLAG_PHYSICAL | ZIO_FLAG_SELF_HEAL | 3416 ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE)); 3417 } 3418 } 3419 3420 align = 1ULL << vd->vdev_top->vdev_ashift; 3421 3422 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 3423 P2PHASE(zio->io_size, align) != 0) { 3424 /* Transform logical writes to be a full physical block size. */ 3425 uint64_t asize = P2ROUNDUP(zio->io_size, align); 3426 abd_t *abuf = abd_alloc_sametype(zio->io_abd, asize); 3427 ASSERT(vd == vd->vdev_top); 3428 if (zio->io_type == ZIO_TYPE_WRITE) { 3429 abd_copy(abuf, zio->io_abd, zio->io_size); 3430 abd_zero_off(abuf, zio->io_size, asize - zio->io_size); 3431 } 3432 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 3433 } 3434 3435 /* 3436 * If this is not a physical io, make sure that it is properly aligned 3437 * before proceeding. 3438 */ 3439 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) { 3440 ASSERT0(P2PHASE(zio->io_offset, align)); 3441 ASSERT0(P2PHASE(zio->io_size, align)); 3442 } else { 3443 /* 3444 * For physical writes, we allow 512b aligned writes and assume 3445 * the device will perform a read-modify-write as necessary. 3446 */ 3447 ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE)); 3448 ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE)); 3449 } 3450 3451 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 3452 3453 /* 3454 * If this is a repair I/O, and there's no self-healing involved -- 3455 * that is, we're just resilvering what we expect to resilver -- 3456 * then don't do the I/O unless zio's txg is actually in vd's DTL. 3457 * This prevents spurious resilvering. 3458 * 3459 * There are a few ways that we can end up creating these spurious 3460 * resilver i/os: 3461 * 3462 * 1. A resilver i/o will be issued if any DVA in the BP has a 3463 * dirty DTL. The mirror code will issue resilver writes to 3464 * each DVA, including the one(s) that are not on vdevs with dirty 3465 * DTLs. 3466 * 3467 * 2. With nested replication, which happens when we have a 3468 * "replacing" or "spare" vdev that's a child of a mirror or raidz. 3469 * For example, given mirror(replacing(A+B), C), it's likely that 3470 * only A is out of date (it's the new device). In this case, we'll 3471 * read from C, then use the data to resilver A+B -- but we don't 3472 * actually want to resilver B, just A. The top-level mirror has no 3473 * way to know this, so instead we just discard unnecessary repairs 3474 * as we work our way down the vdev tree. 3475 * 3476 * 3. ZTEST also creates mirrors of mirrors, mirrors of raidz, etc. 3477 * The same logic applies to any form of nested replication: ditto 3478 * + mirror, RAID-Z + replacing, etc. 3479 * 3480 * However, indirect vdevs point off to other vdevs which may have 3481 * DTL's, so we never bypass them. The child i/os on concrete vdevs 3482 * will be properly bypassed instead. 3483 */ 3484 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 3485 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 3486 zio->io_txg != 0 && /* not a delegated i/o */ 3487 vd->vdev_ops != &vdev_indirect_ops && 3488 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 3489 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 3490 zio_vdev_io_bypass(zio); 3491 return (ZIO_PIPELINE_CONTINUE); 3492 } 3493 3494 if (vd->vdev_ops->vdev_op_leaf && (zio->io_type == ZIO_TYPE_READ || 3495 zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM)) { 3496 3497 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio)) 3498 return (ZIO_PIPELINE_CONTINUE); 3499 3500 if ((zio = vdev_queue_io(zio)) == NULL) 3501 return (ZIO_PIPELINE_STOP); 3502 3503 if (!vdev_accessible(vd, zio)) { 3504 zio->io_error = SET_ERROR(ENXIO); 3505 zio_interrupt(zio); 3506 return (ZIO_PIPELINE_STOP); 3507 } 3508 } 3509 3510 vd->vdev_ops->vdev_op_io_start(zio); 3511 return (ZIO_PIPELINE_STOP); 3512 } 3513 3514 static int 3515 zio_vdev_io_done(zio_t *zio) 3516 { 3517 vdev_t *vd = zio->io_vd; 3518 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 3519 boolean_t unexpected_error = B_FALSE; 3520 3521 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { 3522 return (ZIO_PIPELINE_STOP); 3523 } 3524 3525 ASSERT(zio->io_type == ZIO_TYPE_READ || 3526 zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM); 3527 3528 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) { 3529 3530 vdev_queue_io_done(zio); 3531 3532 if (zio->io_type == ZIO_TYPE_WRITE) 3533 vdev_cache_write(zio); 3534 3535 if (zio_injection_enabled && zio->io_error == 0) 3536 zio->io_error = zio_handle_device_injection(vd, 3537 zio, EIO); 3538 3539 if (zio_injection_enabled && zio->io_error == 0) 3540 zio->io_error = zio_handle_label_injection(zio, EIO); 3541 3542 if (zio->io_error && zio->io_type != ZIO_TYPE_TRIM) { 3543 if (!vdev_accessible(vd, zio)) { 3544 zio->io_error = SET_ERROR(ENXIO); 3545 } else { 3546 unexpected_error = B_TRUE; 3547 } 3548 } 3549 } 3550 3551 ops->vdev_op_io_done(zio); 3552 3553 if (unexpected_error) 3554 VERIFY(vdev_probe(vd, zio) == NULL); 3555 3556 return (ZIO_PIPELINE_CONTINUE); 3557 } 3558 3559 /* 3560 * This function is used to change the priority of an existing zio that is 3561 * currently in-flight. This is used by the arc to upgrade priority in the 3562 * event that a demand read is made for a block that is currently queued 3563 * as a scrub or async read IO. Otherwise, the high priority read request 3564 * would end up having to wait for the lower priority IO. 3565 */ 3566 void 3567 zio_change_priority(zio_t *pio, zio_priority_t priority) 3568 { 3569 zio_t *cio, *cio_next; 3570 zio_link_t *zl = NULL; 3571 3572 ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 3573 3574 if (pio->io_vd != NULL && pio->io_vd->vdev_ops->vdev_op_leaf) { 3575 vdev_queue_change_io_priority(pio, priority); 3576 } else { 3577 pio->io_priority = priority; 3578 } 3579 3580 mutex_enter(&pio->io_lock); 3581 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 3582 cio_next = zio_walk_children(pio, &zl); 3583 zio_change_priority(cio, priority); 3584 } 3585 mutex_exit(&pio->io_lock); 3586 } 3587 3588 /* 3589 * For non-raidz ZIOs, we can just copy aside the bad data read from the 3590 * disk, and use that to finish the checksum ereport later. 3591 */ 3592 static void 3593 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 3594 const abd_t *good_buf) 3595 { 3596 /* no processing needed */ 3597 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 3598 } 3599 3600 /*ARGSUSED*/ 3601 void 3602 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 3603 { 3604 void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size); 3605 3606 abd_copy(abd, zio->io_abd, zio->io_size); 3607 3608 zcr->zcr_cbinfo = zio->io_size; 3609 zcr->zcr_cbdata = abd; 3610 zcr->zcr_finish = zio_vsd_default_cksum_finish; 3611 zcr->zcr_free = zio_abd_free; 3612 } 3613 3614 static int 3615 zio_vdev_io_assess(zio_t *zio) 3616 { 3617 vdev_t *vd = zio->io_vd; 3618 3619 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { 3620 return (ZIO_PIPELINE_STOP); 3621 } 3622 3623 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 3624 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 3625 3626 if (zio->io_vsd != NULL) { 3627 zio->io_vsd_ops->vsd_free(zio); 3628 zio->io_vsd = NULL; 3629 } 3630 3631 if (zio_injection_enabled && zio->io_error == 0) 3632 zio->io_error = zio_handle_fault_injection(zio, EIO); 3633 3634 /* 3635 * If the I/O failed, determine whether we should attempt to retry it. 3636 * 3637 * On retry, we cut in line in the issue queue, since we don't want 3638 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 3639 */ 3640 if (zio->io_error && vd == NULL && 3641 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 3642 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 3643 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 3644 zio->io_error = 0; 3645 zio->io_flags |= ZIO_FLAG_IO_RETRY | 3646 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 3647 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 3648 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 3649 zio_requeue_io_start_cut_in_line); 3650 return (ZIO_PIPELINE_STOP); 3651 } 3652 3653 /* 3654 * If we got an error on a leaf device, convert it to ENXIO 3655 * if the device is not accessible at all. 3656 */ 3657 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 3658 !vdev_accessible(vd, zio)) 3659 zio->io_error = SET_ERROR(ENXIO); 3660 3661 /* 3662 * If we can't write to an interior vdev (mirror or RAID-Z), 3663 * set vdev_cant_write so that we stop trying to allocate from it. 3664 */ 3665 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 3666 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 3667 vd->vdev_cant_write = B_TRUE; 3668 } 3669 3670 /* 3671 * If a cache flush returns ENOTSUP or ENOTTY, we know that no future 3672 * attempts will ever succeed. In this case we set a persistent 3673 * boolean flag so that we don't bother with it in the future. 3674 */ 3675 if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) && 3676 zio->io_type == ZIO_TYPE_IOCTL && 3677 zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL) 3678 vd->vdev_nowritecache = B_TRUE; 3679 3680 if (zio->io_error) 3681 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3682 3683 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 3684 zio->io_physdone != NULL) { 3685 ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED)); 3686 ASSERT(zio->io_child_type == ZIO_CHILD_VDEV); 3687 zio->io_physdone(zio->io_logical); 3688 } 3689 3690 return (ZIO_PIPELINE_CONTINUE); 3691 } 3692 3693 void 3694 zio_vdev_io_reissue(zio_t *zio) 3695 { 3696 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 3697 ASSERT(zio->io_error == 0); 3698 3699 zio->io_stage >>= 1; 3700 } 3701 3702 void 3703 zio_vdev_io_redone(zio_t *zio) 3704 { 3705 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 3706 3707 zio->io_stage >>= 1; 3708 } 3709 3710 void 3711 zio_vdev_io_bypass(zio_t *zio) 3712 { 3713 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 3714 ASSERT(zio->io_error == 0); 3715 3716 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 3717 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 3718 } 3719 3720 /* 3721 * ========================================================================== 3722 * Encrypt and store encryption parameters 3723 * ========================================================================== 3724 */ 3725 3726 3727 /* 3728 * This function is used for ZIO_STAGE_ENCRYPT. It is responsible for 3729 * managing the storage of encryption parameters and passing them to the 3730 * lower-level encryption functions. 3731 */ 3732 static int 3733 zio_encrypt(zio_t *zio) 3734 { 3735 zio_prop_t *zp = &zio->io_prop; 3736 spa_t *spa = zio->io_spa; 3737 blkptr_t *bp = zio->io_bp; 3738 uint64_t psize = BP_GET_PSIZE(bp); 3739 uint64_t dsobj = zio->io_bookmark.zb_objset; 3740 dmu_object_type_t ot = BP_GET_TYPE(bp); 3741 void *enc_buf = NULL; 3742 abd_t *eabd = NULL; 3743 uint8_t salt[ZIO_DATA_SALT_LEN]; 3744 uint8_t iv[ZIO_DATA_IV_LEN]; 3745 uint8_t mac[ZIO_DATA_MAC_LEN]; 3746 boolean_t no_crypt = B_FALSE; 3747 3748 /* the root zio already encrypted the data */ 3749 if (zio->io_child_type == ZIO_CHILD_GANG) 3750 return (ZIO_PIPELINE_CONTINUE); 3751 3752 /* only ZIL blocks are re-encrypted on rewrite */ 3753 if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG) 3754 return (ZIO_PIPELINE_CONTINUE); 3755 3756 if (!(zp->zp_encrypt || BP_IS_ENCRYPTED(bp))) { 3757 BP_SET_CRYPT(bp, B_FALSE); 3758 return (ZIO_PIPELINE_CONTINUE); 3759 } 3760 3761 /* if we are doing raw encryption set the provided encryption params */ 3762 if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) { 3763 ASSERT0(BP_GET_LEVEL(bp)); 3764 BP_SET_CRYPT(bp, B_TRUE); 3765 BP_SET_BYTEORDER(bp, zp->zp_byteorder); 3766 if (ot != DMU_OT_OBJSET) 3767 zio_crypt_encode_mac_bp(bp, zp->zp_mac); 3768 3769 /* dnode blocks must be written out in the provided byteorder */ 3770 if (zp->zp_byteorder != ZFS_HOST_BYTEORDER && 3771 ot == DMU_OT_DNODE) { 3772 void *bswap_buf = zio_buf_alloc(psize); 3773 abd_t *babd = abd_get_from_buf(bswap_buf, psize); 3774 3775 ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 3776 abd_copy_to_buf(bswap_buf, zio->io_abd, psize); 3777 dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf, 3778 psize); 3779 3780 abd_take_ownership_of_buf(babd, B_TRUE); 3781 zio_push_transform(zio, babd, psize, psize, NULL); 3782 } 3783 3784 if (DMU_OT_IS_ENCRYPTED(ot)) 3785 zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv); 3786 return (ZIO_PIPELINE_CONTINUE); 3787 } 3788 3789 /* indirect blocks only maintain a cksum of the lower level MACs */ 3790 if (BP_GET_LEVEL(bp) > 0) { 3791 BP_SET_CRYPT(bp, B_TRUE); 3792 VERIFY0(zio_crypt_do_indirect_mac_checksum_abd(B_TRUE, 3793 zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp), 3794 mac)); 3795 zio_crypt_encode_mac_bp(bp, mac); 3796 return (ZIO_PIPELINE_CONTINUE); 3797 } 3798 3799 /* 3800 * Objset blocks are a special case since they have 2 256-bit MACs 3801 * embedded within them. 3802 */ 3803 if (ot == DMU_OT_OBJSET) { 3804 ASSERT0(DMU_OT_IS_ENCRYPTED(ot)); 3805 ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 3806 BP_SET_CRYPT(bp, B_TRUE); 3807 VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj, 3808 zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp))); 3809 return (ZIO_PIPELINE_CONTINUE); 3810 } 3811 3812 /* unencrypted object types are only authenticated with a MAC */ 3813 if (!DMU_OT_IS_ENCRYPTED(ot)) { 3814 BP_SET_CRYPT(bp, B_TRUE); 3815 VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj, 3816 zio->io_abd, psize, mac)); 3817 zio_crypt_encode_mac_bp(bp, mac); 3818 return (ZIO_PIPELINE_CONTINUE); 3819 } 3820 3821 /* 3822 * Later passes of sync-to-convergence may decide to rewrite data 3823 * in place to avoid more disk reallocations. This presents a problem 3824 * for encryption because this consitutes rewriting the new data with 3825 * the same encryption key and IV. However, this only applies to blocks 3826 * in the MOS (particularly the spacemaps) and we do not encrypt the 3827 * MOS. We assert that the zio is allocating or an intent log write 3828 * to enforce this. 3829 */ 3830 ASSERT(IO_IS_ALLOCATING(zio) || ot == DMU_OT_INTENT_LOG); 3831 ASSERT(BP_GET_LEVEL(bp) == 0 || ot == DMU_OT_INTENT_LOG); 3832 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION)); 3833 ASSERT3U(psize, !=, 0); 3834 3835 enc_buf = zio_buf_alloc(psize); 3836 eabd = abd_get_from_buf(enc_buf, psize); 3837 abd_take_ownership_of_buf(eabd, B_TRUE); 3838 3839 /* 3840 * For an explanation of what encryption parameters are stored 3841 * where, see the block comment in zio_crypt.c. 3842 */ 3843 if (ot == DMU_OT_INTENT_LOG) { 3844 zio_crypt_decode_params_bp(bp, salt, iv); 3845 } else { 3846 BP_SET_CRYPT(bp, B_TRUE); 3847 } 3848 3849 /* Perform the encryption. This should not fail */ 3850 VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark, 3851 BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), 3852 salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt)); 3853 3854 /* encode encryption metadata into the bp */ 3855 if (ot == DMU_OT_INTENT_LOG) { 3856 /* 3857 * ZIL blocks store the MAC in the embedded checksum, so the 3858 * transform must always be applied. 3859 */ 3860 zio_crypt_encode_mac_zil(enc_buf, mac); 3861 zio_push_transform(zio, eabd, psize, psize, NULL); 3862 } else { 3863 BP_SET_CRYPT(bp, B_TRUE); 3864 zio_crypt_encode_params_bp(bp, salt, iv); 3865 zio_crypt_encode_mac_bp(bp, mac); 3866 3867 if (no_crypt) { 3868 ASSERT3U(ot, ==, DMU_OT_DNODE); 3869 abd_free(eabd); 3870 } else { 3871 zio_push_transform(zio, eabd, psize, psize, NULL); 3872 } 3873 } 3874 3875 return (ZIO_PIPELINE_CONTINUE); 3876 } 3877 3878 /* 3879 * ========================================================================== 3880 * Generate and verify checksums 3881 * ========================================================================== 3882 */ 3883 static int 3884 zio_checksum_generate(zio_t *zio) 3885 { 3886 blkptr_t *bp = zio->io_bp; 3887 enum zio_checksum checksum; 3888 3889 if (bp == NULL) { 3890 /* 3891 * This is zio_write_phys(). 3892 * We're either generating a label checksum, or none at all. 3893 */ 3894 checksum = zio->io_prop.zp_checksum; 3895 3896 if (checksum == ZIO_CHECKSUM_OFF) 3897 return (ZIO_PIPELINE_CONTINUE); 3898 3899 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 3900 } else { 3901 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 3902 ASSERT(!IO_IS_ALLOCATING(zio)); 3903 checksum = ZIO_CHECKSUM_GANG_HEADER; 3904 } else { 3905 checksum = BP_GET_CHECKSUM(bp); 3906 } 3907 } 3908 3909 zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size); 3910 3911 return (ZIO_PIPELINE_CONTINUE); 3912 } 3913 3914 static int 3915 zio_checksum_verify(zio_t *zio) 3916 { 3917 zio_bad_cksum_t info; 3918 blkptr_t *bp = zio->io_bp; 3919 int error; 3920 3921 ASSERT(zio->io_vd != NULL); 3922 3923 if (bp == NULL) { 3924 /* 3925 * This is zio_read_phys(). 3926 * We're either verifying a label checksum, or nothing at all. 3927 */ 3928 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 3929 return (ZIO_PIPELINE_CONTINUE); 3930 3931 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 3932 } 3933 3934 if ((error = zio_checksum_error(zio, &info)) != 0) { 3935 zio->io_error = error; 3936 if (error == ECKSUM && 3937 !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 3938 zfs_ereport_start_checksum(zio->io_spa, 3939 zio->io_vd, &zio->io_bookmark, zio, 3940 zio->io_offset, zio->io_size, NULL, &info); 3941 } 3942 } 3943 3944 return (ZIO_PIPELINE_CONTINUE); 3945 } 3946 3947 /* 3948 * Called by RAID-Z to ensure we don't compute the checksum twice. 3949 */ 3950 void 3951 zio_checksum_verified(zio_t *zio) 3952 { 3953 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 3954 } 3955 3956 /* 3957 * ========================================================================== 3958 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 3959 * An error of 0 indicates success. ENXIO indicates whole-device failure, 3960 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 3961 * indicate errors that are specific to one I/O, and most likely permanent. 3962 * Any other error is presumed to be worse because we weren't expecting it. 3963 * ========================================================================== 3964 */ 3965 int 3966 zio_worst_error(int e1, int e2) 3967 { 3968 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 3969 int r1, r2; 3970 3971 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 3972 if (e1 == zio_error_rank[r1]) 3973 break; 3974 3975 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 3976 if (e2 == zio_error_rank[r2]) 3977 break; 3978 3979 return (r1 > r2 ? e1 : e2); 3980 } 3981 3982 /* 3983 * ========================================================================== 3984 * I/O completion 3985 * ========================================================================== 3986 */ 3987 static int 3988 zio_ready(zio_t *zio) 3989 { 3990 blkptr_t *bp = zio->io_bp; 3991 zio_t *pio, *pio_next; 3992 zio_link_t *zl = NULL; 3993 3994 if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT, 3995 ZIO_WAIT_READY)) { 3996 return (ZIO_PIPELINE_STOP); 3997 } 3998 3999 if (zio->io_ready) { 4000 ASSERT(IO_IS_ALLOCATING(zio)); 4001 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 4002 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 4003 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 4004 4005 zio->io_ready(zio); 4006 } 4007 4008 if (bp != NULL && bp != &zio->io_bp_copy) 4009 zio->io_bp_copy = *bp; 4010 4011 if (zio->io_error != 0) { 4012 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 4013 4014 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 4015 ASSERT(IO_IS_ALLOCATING(zio)); 4016 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 4017 ASSERT(zio->io_metaslab_class != NULL); 4018 4019 /* 4020 * We were unable to allocate anything, unreserve and 4021 * issue the next I/O to allocate. 4022 */ 4023 metaslab_class_throttle_unreserve( 4024 zio->io_metaslab_class, zio->io_prop.zp_copies, 4025 zio->io_allocator, zio); 4026 zio_allocate_dispatch(zio->io_spa, zio->io_allocator); 4027 } 4028 } 4029 4030 mutex_enter(&zio->io_lock); 4031 zio->io_state[ZIO_WAIT_READY] = 1; 4032 pio = zio_walk_parents(zio, &zl); 4033 mutex_exit(&zio->io_lock); 4034 4035 /* 4036 * As we notify zio's parents, new parents could be added. 4037 * New parents go to the head of zio's io_parent_list, however, 4038 * so we will (correctly) not notify them. The remainder of zio's 4039 * io_parent_list, from 'pio_next' onward, cannot change because 4040 * all parents must wait for us to be done before they can be done. 4041 */ 4042 for (; pio != NULL; pio = pio_next) { 4043 pio_next = zio_walk_parents(zio, &zl); 4044 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 4045 } 4046 4047 if (zio->io_flags & ZIO_FLAG_NODATA) { 4048 if (BP_IS_GANG(bp)) { 4049 zio->io_flags &= ~ZIO_FLAG_NODATA; 4050 } else { 4051 ASSERT((uintptr_t)zio->io_abd < SPA_MAXBLOCKSIZE); 4052 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 4053 } 4054 } 4055 4056 if (zio_injection_enabled && 4057 zio->io_spa->spa_syncing_txg == zio->io_txg) 4058 zio_handle_ignored_writes(zio); 4059 4060 return (ZIO_PIPELINE_CONTINUE); 4061 } 4062 4063 /* 4064 * Update the allocation throttle accounting. 4065 */ 4066 static void 4067 zio_dva_throttle_done(zio_t *zio) 4068 { 4069 zio_t *lio = zio->io_logical; 4070 zio_t *pio = zio_unique_parent(zio); 4071 vdev_t *vd = zio->io_vd; 4072 int flags = METASLAB_ASYNC_ALLOC; 4073 4074 ASSERT3P(zio->io_bp, !=, NULL); 4075 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 4076 ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE); 4077 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 4078 ASSERT(vd != NULL); 4079 ASSERT3P(vd, ==, vd->vdev_top); 4080 ASSERT(!(zio->io_flags & (ZIO_FLAG_IO_REPAIR | ZIO_FLAG_IO_RETRY))); 4081 ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING); 4082 ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE)); 4083 ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA)); 4084 4085 /* 4086 * Parents of gang children can have two flavors -- ones that 4087 * allocated the gang header (will have ZIO_FLAG_IO_REWRITE set) 4088 * and ones that allocated the constituent blocks. The allocation 4089 * throttle needs to know the allocating parent zio so we must find 4090 * it here. 4091 */ 4092 if (pio->io_child_type == ZIO_CHILD_GANG) { 4093 /* 4094 * If our parent is a rewrite gang child then our grandparent 4095 * would have been the one that performed the allocation. 4096 */ 4097 if (pio->io_flags & ZIO_FLAG_IO_REWRITE) 4098 pio = zio_unique_parent(pio); 4099 flags |= METASLAB_GANG_CHILD; 4100 } 4101 4102 ASSERT(IO_IS_ALLOCATING(pio)); 4103 ASSERT3P(zio, !=, zio->io_logical); 4104 ASSERT(zio->io_logical != NULL); 4105 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR)); 4106 ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE); 4107 ASSERT(zio->io_metaslab_class != NULL); 4108 4109 mutex_enter(&pio->io_lock); 4110 metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags, 4111 pio->io_allocator, B_TRUE); 4112 mutex_exit(&pio->io_lock); 4113 4114 metaslab_class_throttle_unreserve(zio->io_metaslab_class, 1, 4115 pio->io_allocator, pio); 4116 4117 /* 4118 * Call into the pipeline to see if there is more work that 4119 * needs to be done. If there is work to be done it will be 4120 * dispatched to another taskq thread. 4121 */ 4122 zio_allocate_dispatch(zio->io_spa, pio->io_allocator); 4123 } 4124 4125 static int 4126 zio_done(zio_t *zio) 4127 { 4128 spa_t *spa = zio->io_spa; 4129 zio_t *lio = zio->io_logical; 4130 blkptr_t *bp = zio->io_bp; 4131 vdev_t *vd = zio->io_vd; 4132 uint64_t psize = zio->io_size; 4133 zio_t *pio, *pio_next; 4134 zio_link_t *zl = NULL; 4135 4136 /* 4137 * If our children haven't all completed, 4138 * wait for them and then repeat this pipeline stage. 4139 */ 4140 if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) { 4141 return (ZIO_PIPELINE_STOP); 4142 } 4143 4144 /* 4145 * If the allocation throttle is enabled, then update the accounting. 4146 * We only track child I/Os that are part of an allocating async 4147 * write. We must do this since the allocation is performed 4148 * by the logical I/O but the actual write is done by child I/Os. 4149 */ 4150 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING && 4151 zio->io_child_type == ZIO_CHILD_VDEV) { 4152 ASSERT(zio->io_metaslab_class != NULL); 4153 ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled); 4154 zio_dva_throttle_done(zio); 4155 } 4156 4157 /* 4158 * If the allocation throttle is enabled, verify that 4159 * we have decremented the refcounts for every I/O that was throttled. 4160 */ 4161 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 4162 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 4163 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 4164 ASSERT(bp != NULL); 4165 4166 metaslab_group_alloc_verify(spa, zio->io_bp, zio, 4167 zio->io_allocator); 4168 VERIFY(zfs_refcount_not_held( 4169 &zio->io_metaslab_class->mc_alloc_slots[zio->io_allocator], 4170 zio)); 4171 } 4172 4173 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 4174 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 4175 ASSERT(zio->io_children[c][w] == 0); 4176 4177 if (bp != NULL && !BP_IS_EMBEDDED(bp)) { 4178 ASSERT(bp->blk_pad[0] == 0); 4179 ASSERT(bp->blk_pad[1] == 0); 4180 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 4181 (bp == zio_unique_parent(zio)->io_bp)); 4182 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 4183 zio->io_bp_override == NULL && 4184 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 4185 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 4186 ASSERT(BP_COUNT_GANG(bp) == 0 || 4187 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 4188 } 4189 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 4190 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 4191 } 4192 4193 /* 4194 * If there were child vdev/gang/ddt errors, they apply to us now. 4195 */ 4196 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 4197 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 4198 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 4199 4200 /* 4201 * If the I/O on the transformed data was successful, generate any 4202 * checksum reports now while we still have the transformed data. 4203 */ 4204 if (zio->io_error == 0) { 4205 while (zio->io_cksum_report != NULL) { 4206 zio_cksum_report_t *zcr = zio->io_cksum_report; 4207 uint64_t align = zcr->zcr_align; 4208 uint64_t asize = P2ROUNDUP(psize, align); 4209 abd_t *adata = zio->io_abd; 4210 4211 if (asize != psize) { 4212 adata = abd_alloc(asize, B_TRUE); 4213 abd_copy(adata, zio->io_abd, psize); 4214 abd_zero_off(adata, psize, asize - psize); 4215 } 4216 4217 zio->io_cksum_report = zcr->zcr_next; 4218 zcr->zcr_next = NULL; 4219 zcr->zcr_finish(zcr, adata); 4220 zfs_ereport_free_checksum(zcr); 4221 4222 if (asize != psize) 4223 abd_free(adata); 4224 } 4225 } 4226 4227 zio_pop_transforms(zio); /* note: may set zio->io_error */ 4228 4229 vdev_stat_update(zio, psize); 4230 4231 if (zio->io_error) { 4232 /* 4233 * If this I/O is attached to a particular vdev, 4234 * generate an error message describing the I/O failure 4235 * at the block level. We ignore these errors if the 4236 * device is currently unavailable. 4237 */ 4238 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 4239 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, 4240 &zio->io_bookmark, zio, 0, 0); 4241 4242 if ((zio->io_error == EIO || !(zio->io_flags & 4243 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 4244 zio == lio) { 4245 /* 4246 * For logical I/O requests, tell the SPA to log the 4247 * error and generate a logical data ereport. 4248 */ 4249 spa_log_error(spa, &zio->io_bookmark); 4250 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, 4251 &zio->io_bookmark, zio, 0, 0); 4252 } 4253 } 4254 4255 if (zio->io_error && zio == lio) { 4256 /* 4257 * Determine whether zio should be reexecuted. This will 4258 * propagate all the way to the root via zio_notify_parent(). 4259 */ 4260 ASSERT(vd == NULL && bp != NULL); 4261 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 4262 4263 if (IO_IS_ALLOCATING(zio) && 4264 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 4265 if (zio->io_error != ENOSPC) 4266 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 4267 else 4268 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4269 } 4270 4271 if ((zio->io_type == ZIO_TYPE_READ || 4272 zio->io_type == ZIO_TYPE_FREE) && 4273 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 4274 zio->io_error == ENXIO && 4275 spa_load_state(spa) == SPA_LOAD_NONE && 4276 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 4277 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4278 4279 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 4280 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4281 4282 /* 4283 * Here is a possibly good place to attempt to do 4284 * either combinatorial reconstruction or error correction 4285 * based on checksums. It also might be a good place 4286 * to send out preliminary ereports before we suspend 4287 * processing. 4288 */ 4289 } 4290 4291 /* 4292 * If there were logical child errors, they apply to us now. 4293 * We defer this until now to avoid conflating logical child 4294 * errors with errors that happened to the zio itself when 4295 * updating vdev stats and reporting FMA events above. 4296 */ 4297 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 4298 4299 if ((zio->io_error || zio->io_reexecute) && 4300 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 4301 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 4302 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 4303 4304 zio_gang_tree_free(&zio->io_gang_tree); 4305 4306 /* 4307 * Godfather I/Os should never suspend. 4308 */ 4309 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 4310 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 4311 zio->io_reexecute = 0; 4312 4313 if (zio->io_reexecute) { 4314 /* 4315 * This is a logical I/O that wants to reexecute. 4316 * 4317 * Reexecute is top-down. When an i/o fails, if it's not 4318 * the root, it simply notifies its parent and sticks around. 4319 * The parent, seeing that it still has children in zio_done(), 4320 * does the same. This percolates all the way up to the root. 4321 * The root i/o will reexecute or suspend the entire tree. 4322 * 4323 * This approach ensures that zio_reexecute() honors 4324 * all the original i/o dependency relationships, e.g. 4325 * parents not executing until children are ready. 4326 */ 4327 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 4328 4329 zio->io_gang_leader = NULL; 4330 4331 mutex_enter(&zio->io_lock); 4332 zio->io_state[ZIO_WAIT_DONE] = 1; 4333 mutex_exit(&zio->io_lock); 4334 4335 /* 4336 * "The Godfather" I/O monitors its children but is 4337 * not a true parent to them. It will track them through 4338 * the pipeline but severs its ties whenever they get into 4339 * trouble (e.g. suspended). This allows "The Godfather" 4340 * I/O to return status without blocking. 4341 */ 4342 zl = NULL; 4343 for (pio = zio_walk_parents(zio, &zl); pio != NULL; 4344 pio = pio_next) { 4345 zio_link_t *remove_zl = zl; 4346 pio_next = zio_walk_parents(zio, &zl); 4347 4348 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 4349 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 4350 zio_remove_child(pio, zio, remove_zl); 4351 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 4352 } 4353 } 4354 4355 if ((pio = zio_unique_parent(zio)) != NULL) { 4356 /* 4357 * We're not a root i/o, so there's nothing to do 4358 * but notify our parent. Don't propagate errors 4359 * upward since we haven't permanently failed yet. 4360 */ 4361 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 4362 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 4363 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 4364 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 4365 /* 4366 * We'd fail again if we reexecuted now, so suspend 4367 * until conditions improve (e.g. device comes online). 4368 */ 4369 zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR); 4370 } else { 4371 /* 4372 * Reexecution is potentially a huge amount of work. 4373 * Hand it off to the otherwise-unused claim taskq. 4374 */ 4375 ASSERT(zio->io_tqent.tqent_next == NULL); 4376 spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM, 4377 ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio, 4378 0, &zio->io_tqent); 4379 } 4380 return (ZIO_PIPELINE_STOP); 4381 } 4382 4383 ASSERT(zio->io_child_count == 0); 4384 ASSERT(zio->io_reexecute == 0); 4385 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 4386 4387 /* 4388 * Report any checksum errors, since the I/O is complete. 4389 */ 4390 while (zio->io_cksum_report != NULL) { 4391 zio_cksum_report_t *zcr = zio->io_cksum_report; 4392 zio->io_cksum_report = zcr->zcr_next; 4393 zcr->zcr_next = NULL; 4394 zcr->zcr_finish(zcr, NULL); 4395 zfs_ereport_free_checksum(zcr); 4396 } 4397 4398 /* 4399 * It is the responsibility of the done callback to ensure that this 4400 * particular zio is no longer discoverable for adoption, and as 4401 * such, cannot acquire any new parents. 4402 */ 4403 if (zio->io_done) 4404 zio->io_done(zio); 4405 4406 mutex_enter(&zio->io_lock); 4407 zio->io_state[ZIO_WAIT_DONE] = 1; 4408 mutex_exit(&zio->io_lock); 4409 4410 zl = NULL; 4411 for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) { 4412 zio_link_t *remove_zl = zl; 4413 pio_next = zio_walk_parents(zio, &zl); 4414 zio_remove_child(pio, zio, remove_zl); 4415 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 4416 } 4417 4418 if (zio->io_waiter != NULL) { 4419 mutex_enter(&zio->io_lock); 4420 zio->io_executor = NULL; 4421 cv_broadcast(&zio->io_cv); 4422 mutex_exit(&zio->io_lock); 4423 } else { 4424 zio_destroy(zio); 4425 } 4426 4427 return (ZIO_PIPELINE_STOP); 4428 } 4429 4430 /* 4431 * ========================================================================== 4432 * I/O pipeline definition 4433 * ========================================================================== 4434 */ 4435 static zio_pipe_stage_t *zio_pipeline[] = { 4436 NULL, 4437 zio_read_bp_init, 4438 zio_write_bp_init, 4439 zio_free_bp_init, 4440 zio_issue_async, 4441 zio_write_compress, 4442 zio_encrypt, 4443 zio_checksum_generate, 4444 zio_nop_write, 4445 zio_ddt_read_start, 4446 zio_ddt_read_done, 4447 zio_ddt_write, 4448 zio_ddt_free, 4449 zio_gang_assemble, 4450 zio_gang_issue, 4451 zio_dva_throttle, 4452 zio_dva_allocate, 4453 zio_dva_free, 4454 zio_dva_claim, 4455 zio_ready, 4456 zio_vdev_io_start, 4457 zio_vdev_io_done, 4458 zio_vdev_io_assess, 4459 zio_checksum_verify, 4460 zio_done 4461 }; 4462 4463 4464 4465 4466 /* 4467 * Compare two zbookmark_phys_t's to see which we would reach first in a 4468 * pre-order traversal of the object tree. 4469 * 4470 * This is simple in every case aside from the meta-dnode object. For all other 4471 * objects, we traverse them in order (object 1 before object 2, and so on). 4472 * However, all of these objects are traversed while traversing object 0, since 4473 * the data it points to is the list of objects. Thus, we need to convert to a 4474 * canonical representation so we can compare meta-dnode bookmarks to 4475 * non-meta-dnode bookmarks. 4476 * 4477 * We do this by calculating "equivalents" for each field of the zbookmark. 4478 * zbookmarks outside of the meta-dnode use their own object and level, and 4479 * calculate the level 0 equivalent (the first L0 blkid that is contained in the 4480 * blocks this bookmark refers to) by multiplying their blkid by their span 4481 * (the number of L0 blocks contained within one block at their level). 4482 * zbookmarks inside the meta-dnode calculate their object equivalent 4483 * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use 4484 * level + 1<<31 (any value larger than a level could ever be) for their level. 4485 * This causes them to always compare before a bookmark in their object 4486 * equivalent, compare appropriately to bookmarks in other objects, and to 4487 * compare appropriately to other bookmarks in the meta-dnode. 4488 */ 4489 int 4490 zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2, 4491 const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2) 4492 { 4493 /* 4494 * These variables represent the "equivalent" values for the zbookmark, 4495 * after converting zbookmarks inside the meta dnode to their 4496 * normal-object equivalents. 4497 */ 4498 uint64_t zb1obj, zb2obj; 4499 uint64_t zb1L0, zb2L0; 4500 uint64_t zb1level, zb2level; 4501 4502 if (zb1->zb_object == zb2->zb_object && 4503 zb1->zb_level == zb2->zb_level && 4504 zb1->zb_blkid == zb2->zb_blkid) 4505 return (0); 4506 4507 /* 4508 * BP_SPANB calculates the span in blocks. 4509 */ 4510 zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level); 4511 zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level); 4512 4513 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 4514 zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 4515 zb1L0 = 0; 4516 zb1level = zb1->zb_level + COMPARE_META_LEVEL; 4517 } else { 4518 zb1obj = zb1->zb_object; 4519 zb1level = zb1->zb_level; 4520 } 4521 4522 if (zb2->zb_object == DMU_META_DNODE_OBJECT) { 4523 zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 4524 zb2L0 = 0; 4525 zb2level = zb2->zb_level + COMPARE_META_LEVEL; 4526 } else { 4527 zb2obj = zb2->zb_object; 4528 zb2level = zb2->zb_level; 4529 } 4530 4531 /* Now that we have a canonical representation, do the comparison. */ 4532 if (zb1obj != zb2obj) 4533 return (zb1obj < zb2obj ? -1 : 1); 4534 else if (zb1L0 != zb2L0) 4535 return (zb1L0 < zb2L0 ? -1 : 1); 4536 else if (zb1level != zb2level) 4537 return (zb1level > zb2level ? -1 : 1); 4538 /* 4539 * This can (theoretically) happen if the bookmarks have the same object 4540 * and level, but different blkids, if the block sizes are not the same. 4541 * There is presently no way to change the indirect block sizes 4542 */ 4543 return (0); 4544 } 4545 4546 /* 4547 * This function checks the following: given that last_block is the place that 4548 * our traversal stopped last time, does that guarantee that we've visited 4549 * every node under subtree_root? Therefore, we can't just use the raw output 4550 * of zbookmark_compare. We have to pass in a modified version of 4551 * subtree_root; by incrementing the block id, and then checking whether 4552 * last_block is before or equal to that, we can tell whether or not having 4553 * visited last_block implies that all of subtree_root's children have been 4554 * visited. 4555 */ 4556 boolean_t 4557 zbookmark_subtree_completed(const dnode_phys_t *dnp, 4558 const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block) 4559 { 4560 zbookmark_phys_t mod_zb = *subtree_root; 4561 mod_zb.zb_blkid++; 4562 ASSERT(last_block->zb_level == 0); 4563 4564 /* The objset_phys_t isn't before anything. */ 4565 if (dnp == NULL) 4566 return (B_FALSE); 4567 4568 /* 4569 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the 4570 * data block size in sectors, because that variable is only used if 4571 * the bookmark refers to a block in the meta-dnode. Since we don't 4572 * know without examining it what object it refers to, and there's no 4573 * harm in passing in this value in other cases, we always pass it in. 4574 * 4575 * We pass in 0 for the indirect block size shift because zb2 must be 4576 * level 0. The indirect block size is only used to calculate the span 4577 * of the bookmark, but since the bookmark must be level 0, the span is 4578 * always 1, so the math works out. 4579 * 4580 * If you make changes to how the zbookmark_compare code works, be sure 4581 * to make sure that this code still works afterwards. 4582 */ 4583 return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift, 4584 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb, 4585 last_block) <= 0); 4586 } 4587