1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2014 Integros [integros.com] 26 */ 27 28 #include <sys/sysmacros.h> 29 #include <sys/zfs_context.h> 30 #include <sys/fm/fs/zfs.h> 31 #include <sys/spa.h> 32 #include <sys/txg.h> 33 #include <sys/spa_impl.h> 34 #include <sys/vdev_impl.h> 35 #include <sys/zio_impl.h> 36 #include <sys/zio_compress.h> 37 #include <sys/zio_checksum.h> 38 #include <sys/dmu_objset.h> 39 #include <sys/arc.h> 40 #include <sys/ddt.h> 41 #include <sys/blkptr.h> 42 #include <sys/zfeature.h> 43 #include <sys/metaslab_impl.h> 44 45 /* 46 * ========================================================================== 47 * I/O type descriptions 48 * ========================================================================== 49 */ 50 const char *zio_type_name[ZIO_TYPES] = { 51 "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim", 52 "zio_ioctl" 53 }; 54 55 boolean_t zio_dva_throttle_enabled = B_TRUE; 56 57 /* 58 * ========================================================================== 59 * I/O kmem caches 60 * ========================================================================== 61 */ 62 kmem_cache_t *zio_cache; 63 kmem_cache_t *zio_link_cache; 64 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 65 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 66 67 #ifdef _KERNEL 68 extern vmem_t *zio_alloc_arena; 69 #endif 70 71 #define ZIO_PIPELINE_CONTINUE 0x100 72 #define ZIO_PIPELINE_STOP 0x101 73 74 #define BP_SPANB(indblkshift, level) \ 75 (((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT))) 76 #define COMPARE_META_LEVEL 0x80000000ul 77 /* 78 * The following actions directly effect the spa's sync-to-convergence logic. 79 * The values below define the sync pass when we start performing the action. 80 * Care should be taken when changing these values as they directly impact 81 * spa_sync() performance. Tuning these values may introduce subtle performance 82 * pathologies and should only be done in the context of performance analysis. 83 * These tunables will eventually be removed and replaced with #defines once 84 * enough analysis has been done to determine optimal values. 85 * 86 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 87 * regular blocks are not deferred. 88 */ 89 int zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */ 90 int zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */ 91 int zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */ 92 93 /* 94 * An allocating zio is one that either currently has the DVA allocate 95 * stage set or will have it later in its lifetime. 96 */ 97 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 98 99 boolean_t zio_requeue_io_start_cut_in_line = B_TRUE; 100 101 #ifdef ZFS_DEBUG 102 int zio_buf_debug_limit = 16384; 103 #else 104 int zio_buf_debug_limit = 0; 105 #endif 106 107 static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t); 108 109 void 110 zio_init(void) 111 { 112 size_t c; 113 vmem_t *data_alloc_arena = NULL; 114 115 #ifdef _KERNEL 116 data_alloc_arena = zio_alloc_arena; 117 #endif 118 zio_cache = kmem_cache_create("zio_cache", 119 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 120 zio_link_cache = kmem_cache_create("zio_link_cache", 121 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 122 123 /* 124 * For small buffers, we want a cache for each multiple of 125 * SPA_MINBLOCKSIZE. For larger buffers, we want a cache 126 * for each quarter-power of 2. 127 */ 128 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 129 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 130 size_t p2 = size; 131 size_t align = 0; 132 size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; 133 134 while (!ISP2(p2)) 135 p2 &= p2 - 1; 136 137 #ifndef _KERNEL 138 /* 139 * If we are using watchpoints, put each buffer on its own page, 140 * to eliminate the performance overhead of trapping to the 141 * kernel when modifying a non-watched buffer that shares the 142 * page with a watched buffer. 143 */ 144 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 145 continue; 146 #endif 147 if (size <= 4 * SPA_MINBLOCKSIZE) { 148 align = SPA_MINBLOCKSIZE; 149 } else if (IS_P2ALIGNED(size, p2 >> 2)) { 150 align = MIN(p2 >> 2, PAGESIZE); 151 } 152 153 if (align != 0) { 154 char name[36]; 155 (void) sprintf(name, "zio_buf_%lu", (ulong_t)size); 156 zio_buf_cache[c] = kmem_cache_create(name, size, 157 align, NULL, NULL, NULL, NULL, NULL, cflags); 158 159 /* 160 * Since zio_data bufs do not appear in crash dumps, we 161 * pass KMC_NOTOUCH so that no allocator metadata is 162 * stored with the buffers. 163 */ 164 (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size); 165 zio_data_buf_cache[c] = kmem_cache_create(name, size, 166 align, NULL, NULL, NULL, NULL, data_alloc_arena, 167 cflags | KMC_NOTOUCH); 168 } 169 } 170 171 while (--c != 0) { 172 ASSERT(zio_buf_cache[c] != NULL); 173 if (zio_buf_cache[c - 1] == NULL) 174 zio_buf_cache[c - 1] = zio_buf_cache[c]; 175 176 ASSERT(zio_data_buf_cache[c] != NULL); 177 if (zio_data_buf_cache[c - 1] == NULL) 178 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 179 } 180 181 zio_inject_init(); 182 } 183 184 void 185 zio_fini(void) 186 { 187 size_t c; 188 kmem_cache_t *last_cache = NULL; 189 kmem_cache_t *last_data_cache = NULL; 190 191 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 192 if (zio_buf_cache[c] != last_cache) { 193 last_cache = zio_buf_cache[c]; 194 kmem_cache_destroy(zio_buf_cache[c]); 195 } 196 zio_buf_cache[c] = NULL; 197 198 if (zio_data_buf_cache[c] != last_data_cache) { 199 last_data_cache = zio_data_buf_cache[c]; 200 kmem_cache_destroy(zio_data_buf_cache[c]); 201 } 202 zio_data_buf_cache[c] = NULL; 203 } 204 205 kmem_cache_destroy(zio_link_cache); 206 kmem_cache_destroy(zio_cache); 207 208 zio_inject_fini(); 209 } 210 211 /* 212 * ========================================================================== 213 * Allocate and free I/O buffers 214 * ========================================================================== 215 */ 216 217 /* 218 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 219 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 220 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 221 * excess / transient data in-core during a crashdump. 222 */ 223 void * 224 zio_buf_alloc(size_t size) 225 { 226 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 227 228 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 229 230 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 231 } 232 233 /* 234 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 235 * crashdump if the kernel panics. This exists so that we will limit the amount 236 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 237 * of kernel heap dumped to disk when the kernel panics) 238 */ 239 void * 240 zio_data_buf_alloc(size_t size) 241 { 242 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 243 244 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 245 246 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 247 } 248 249 void 250 zio_buf_free(void *buf, size_t size) 251 { 252 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 253 254 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 255 256 kmem_cache_free(zio_buf_cache[c], buf); 257 } 258 259 void 260 zio_data_buf_free(void *buf, size_t size) 261 { 262 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 263 264 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 265 266 kmem_cache_free(zio_data_buf_cache[c], buf); 267 } 268 269 /* 270 * ========================================================================== 271 * Push and pop I/O transform buffers 272 * ========================================================================== 273 */ 274 void 275 zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize, 276 zio_transform_func_t *transform) 277 { 278 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 279 280 zt->zt_orig_data = zio->io_data; 281 zt->zt_orig_size = zio->io_size; 282 zt->zt_bufsize = bufsize; 283 zt->zt_transform = transform; 284 285 zt->zt_next = zio->io_transform_stack; 286 zio->io_transform_stack = zt; 287 288 zio->io_data = data; 289 zio->io_size = size; 290 } 291 292 void 293 zio_pop_transforms(zio_t *zio) 294 { 295 zio_transform_t *zt; 296 297 while ((zt = zio->io_transform_stack) != NULL) { 298 if (zt->zt_transform != NULL) 299 zt->zt_transform(zio, 300 zt->zt_orig_data, zt->zt_orig_size); 301 302 if (zt->zt_bufsize != 0) 303 zio_buf_free(zio->io_data, zt->zt_bufsize); 304 305 zio->io_data = zt->zt_orig_data; 306 zio->io_size = zt->zt_orig_size; 307 zio->io_transform_stack = zt->zt_next; 308 309 kmem_free(zt, sizeof (zio_transform_t)); 310 } 311 } 312 313 /* 314 * ========================================================================== 315 * I/O transform callbacks for subblocks and decompression 316 * ========================================================================== 317 */ 318 static void 319 zio_subblock(zio_t *zio, void *data, uint64_t size) 320 { 321 ASSERT(zio->io_size > size); 322 323 if (zio->io_type == ZIO_TYPE_READ) 324 bcopy(zio->io_data, data, size); 325 } 326 327 static void 328 zio_decompress(zio_t *zio, void *data, uint64_t size) 329 { 330 if (zio->io_error == 0 && 331 zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 332 zio->io_data, data, zio->io_size, size) != 0) 333 zio->io_error = SET_ERROR(EIO); 334 } 335 336 /* 337 * ========================================================================== 338 * I/O parent/child relationships and pipeline interlocks 339 * ========================================================================== 340 */ 341 zio_t * 342 zio_walk_parents(zio_t *cio, zio_link_t **zl) 343 { 344 list_t *pl = &cio->io_parent_list; 345 346 *zl = (*zl == NULL) ? list_head(pl) : list_next(pl, *zl); 347 if (*zl == NULL) 348 return (NULL); 349 350 ASSERT((*zl)->zl_child == cio); 351 return ((*zl)->zl_parent); 352 } 353 354 zio_t * 355 zio_walk_children(zio_t *pio, zio_link_t **zl) 356 { 357 list_t *cl = &pio->io_child_list; 358 359 *zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl); 360 if (*zl == NULL) 361 return (NULL); 362 363 ASSERT((*zl)->zl_parent == pio); 364 return ((*zl)->zl_child); 365 } 366 367 zio_t * 368 zio_unique_parent(zio_t *cio) 369 { 370 zio_link_t *zl = NULL; 371 zio_t *pio = zio_walk_parents(cio, &zl); 372 373 VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL); 374 return (pio); 375 } 376 377 void 378 zio_add_child(zio_t *pio, zio_t *cio) 379 { 380 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 381 382 /* 383 * Logical I/Os can have logical, gang, or vdev children. 384 * Gang I/Os can have gang or vdev children. 385 * Vdev I/Os can only have vdev children. 386 * The following ASSERT captures all of these constraints. 387 */ 388 ASSERT(cio->io_child_type <= pio->io_child_type); 389 390 zl->zl_parent = pio; 391 zl->zl_child = cio; 392 393 mutex_enter(&cio->io_lock); 394 mutex_enter(&pio->io_lock); 395 396 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 397 398 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 399 pio->io_children[cio->io_child_type][w] += !cio->io_state[w]; 400 401 list_insert_head(&pio->io_child_list, zl); 402 list_insert_head(&cio->io_parent_list, zl); 403 404 pio->io_child_count++; 405 cio->io_parent_count++; 406 407 mutex_exit(&pio->io_lock); 408 mutex_exit(&cio->io_lock); 409 } 410 411 static void 412 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 413 { 414 ASSERT(zl->zl_parent == pio); 415 ASSERT(zl->zl_child == cio); 416 417 mutex_enter(&cio->io_lock); 418 mutex_enter(&pio->io_lock); 419 420 list_remove(&pio->io_child_list, zl); 421 list_remove(&cio->io_parent_list, zl); 422 423 pio->io_child_count--; 424 cio->io_parent_count--; 425 426 mutex_exit(&pio->io_lock); 427 mutex_exit(&cio->io_lock); 428 429 kmem_cache_free(zio_link_cache, zl); 430 } 431 432 static boolean_t 433 zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) 434 { 435 uint64_t *countp = &zio->io_children[child][wait]; 436 boolean_t waiting = B_FALSE; 437 438 mutex_enter(&zio->io_lock); 439 ASSERT(zio->io_stall == NULL); 440 if (*countp != 0) { 441 zio->io_stage >>= 1; 442 ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN); 443 zio->io_stall = countp; 444 waiting = B_TRUE; 445 } 446 mutex_exit(&zio->io_lock); 447 448 return (waiting); 449 } 450 451 static void 452 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) 453 { 454 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 455 int *errorp = &pio->io_child_error[zio->io_child_type]; 456 457 mutex_enter(&pio->io_lock); 458 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 459 *errorp = zio_worst_error(*errorp, zio->io_error); 460 pio->io_reexecute |= zio->io_reexecute; 461 ASSERT3U(*countp, >, 0); 462 463 (*countp)--; 464 465 if (*countp == 0 && pio->io_stall == countp) { 466 zio_taskq_type_t type = 467 pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE : 468 ZIO_TASKQ_INTERRUPT; 469 pio->io_stall = NULL; 470 mutex_exit(&pio->io_lock); 471 /* 472 * Dispatch the parent zio in its own taskq so that 473 * the child can continue to make progress. This also 474 * prevents overflowing the stack when we have deeply nested 475 * parent-child relationships. 476 */ 477 zio_taskq_dispatch(pio, type, B_FALSE); 478 } else { 479 mutex_exit(&pio->io_lock); 480 } 481 } 482 483 static void 484 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 485 { 486 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 487 zio->io_error = zio->io_child_error[c]; 488 } 489 490 int 491 zio_timestamp_compare(const void *x1, const void *x2) 492 { 493 const zio_t *z1 = x1; 494 const zio_t *z2 = x2; 495 496 if (z1->io_queued_timestamp < z2->io_queued_timestamp) 497 return (-1); 498 if (z1->io_queued_timestamp > z2->io_queued_timestamp) 499 return (1); 500 501 if (z1->io_offset < z2->io_offset) 502 return (-1); 503 if (z1->io_offset > z2->io_offset) 504 return (1); 505 506 if (z1 < z2) 507 return (-1); 508 if (z1 > z2) 509 return (1); 510 511 return (0); 512 } 513 514 /* 515 * ========================================================================== 516 * Create the various types of I/O (read, write, free, etc) 517 * ========================================================================== 518 */ 519 static zio_t * 520 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 521 void *data, uint64_t size, zio_done_func_t *done, void *private, 522 zio_type_t type, zio_priority_t priority, enum zio_flag flags, 523 vdev_t *vd, uint64_t offset, const zbookmark_phys_t *zb, 524 enum zio_stage stage, enum zio_stage pipeline) 525 { 526 zio_t *zio; 527 528 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 529 ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0); 530 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 531 532 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 533 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 534 ASSERT(vd || stage == ZIO_STAGE_OPEN); 535 536 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 537 bzero(zio, sizeof (zio_t)); 538 539 mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL); 540 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 541 542 list_create(&zio->io_parent_list, sizeof (zio_link_t), 543 offsetof(zio_link_t, zl_parent_node)); 544 list_create(&zio->io_child_list, sizeof (zio_link_t), 545 offsetof(zio_link_t, zl_child_node)); 546 metaslab_trace_init(&zio->io_alloc_list); 547 548 if (vd != NULL) 549 zio->io_child_type = ZIO_CHILD_VDEV; 550 else if (flags & ZIO_FLAG_GANG_CHILD) 551 zio->io_child_type = ZIO_CHILD_GANG; 552 else if (flags & ZIO_FLAG_DDT_CHILD) 553 zio->io_child_type = ZIO_CHILD_DDT; 554 else 555 zio->io_child_type = ZIO_CHILD_LOGICAL; 556 557 if (bp != NULL) { 558 zio->io_bp = (blkptr_t *)bp; 559 zio->io_bp_copy = *bp; 560 zio->io_bp_orig = *bp; 561 if (type != ZIO_TYPE_WRITE || 562 zio->io_child_type == ZIO_CHILD_DDT) 563 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 564 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 565 zio->io_logical = zio; 566 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 567 pipeline |= ZIO_GANG_STAGES; 568 } 569 570 zio->io_spa = spa; 571 zio->io_txg = txg; 572 zio->io_done = done; 573 zio->io_private = private; 574 zio->io_type = type; 575 zio->io_priority = priority; 576 zio->io_vd = vd; 577 zio->io_offset = offset; 578 zio->io_orig_data = zio->io_data = data; 579 zio->io_orig_size = zio->io_size = size; 580 zio->io_orig_flags = zio->io_flags = flags; 581 zio->io_orig_stage = zio->io_stage = stage; 582 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 583 zio->io_pipeline_trace = ZIO_STAGE_OPEN; 584 585 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY); 586 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 587 588 if (zb != NULL) 589 zio->io_bookmark = *zb; 590 591 if (pio != NULL) { 592 if (zio->io_logical == NULL) 593 zio->io_logical = pio->io_logical; 594 if (zio->io_child_type == ZIO_CHILD_GANG) 595 zio->io_gang_leader = pio->io_gang_leader; 596 zio_add_child(pio, zio); 597 } 598 599 return (zio); 600 } 601 602 static void 603 zio_destroy(zio_t *zio) 604 { 605 metaslab_trace_fini(&zio->io_alloc_list); 606 list_destroy(&zio->io_parent_list); 607 list_destroy(&zio->io_child_list); 608 mutex_destroy(&zio->io_lock); 609 cv_destroy(&zio->io_cv); 610 kmem_cache_free(zio_cache, zio); 611 } 612 613 zio_t * 614 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 615 void *private, enum zio_flag flags) 616 { 617 zio_t *zio; 618 619 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 620 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 621 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 622 623 return (zio); 624 } 625 626 zio_t * 627 zio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags) 628 { 629 return (zio_null(NULL, spa, NULL, done, private, flags)); 630 } 631 632 void 633 zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp) 634 { 635 if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) { 636 zfs_panic_recover("blkptr at %p has invalid TYPE %llu", 637 bp, (longlong_t)BP_GET_TYPE(bp)); 638 } 639 if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS || 640 BP_GET_CHECKSUM(bp) <= ZIO_CHECKSUM_ON) { 641 zfs_panic_recover("blkptr at %p has invalid CHECKSUM %llu", 642 bp, (longlong_t)BP_GET_CHECKSUM(bp)); 643 } 644 if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS || 645 BP_GET_COMPRESS(bp) <= ZIO_COMPRESS_ON) { 646 zfs_panic_recover("blkptr at %p has invalid COMPRESS %llu", 647 bp, (longlong_t)BP_GET_COMPRESS(bp)); 648 } 649 if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) { 650 zfs_panic_recover("blkptr at %p has invalid LSIZE %llu", 651 bp, (longlong_t)BP_GET_LSIZE(bp)); 652 } 653 if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) { 654 zfs_panic_recover("blkptr at %p has invalid PSIZE %llu", 655 bp, (longlong_t)BP_GET_PSIZE(bp)); 656 } 657 658 if (BP_IS_EMBEDDED(bp)) { 659 if (BPE_GET_ETYPE(bp) > NUM_BP_EMBEDDED_TYPES) { 660 zfs_panic_recover("blkptr at %p has invalid ETYPE %llu", 661 bp, (longlong_t)BPE_GET_ETYPE(bp)); 662 } 663 } 664 665 /* 666 * Pool-specific checks. 667 * 668 * Note: it would be nice to verify that the blk_birth and 669 * BP_PHYSICAL_BIRTH() are not too large. However, spa_freeze() 670 * allows the birth time of log blocks (and dmu_sync()-ed blocks 671 * that are in the log) to be arbitrarily large. 672 */ 673 for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 674 uint64_t vdevid = DVA_GET_VDEV(&bp->blk_dva[i]); 675 if (vdevid >= spa->spa_root_vdev->vdev_children) { 676 zfs_panic_recover("blkptr at %p DVA %u has invalid " 677 "VDEV %llu", 678 bp, i, (longlong_t)vdevid); 679 continue; 680 } 681 vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid]; 682 if (vd == NULL) { 683 zfs_panic_recover("blkptr at %p DVA %u has invalid " 684 "VDEV %llu", 685 bp, i, (longlong_t)vdevid); 686 continue; 687 } 688 if (vd->vdev_ops == &vdev_hole_ops) { 689 zfs_panic_recover("blkptr at %p DVA %u has hole " 690 "VDEV %llu", 691 bp, i, (longlong_t)vdevid); 692 continue; 693 } 694 if (vd->vdev_ops == &vdev_missing_ops) { 695 /* 696 * "missing" vdevs are valid during import, but we 697 * don't have their detailed info (e.g. asize), so 698 * we can't perform any more checks on them. 699 */ 700 continue; 701 } 702 uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]); 703 uint64_t asize = DVA_GET_ASIZE(&bp->blk_dva[i]); 704 if (BP_IS_GANG(bp)) 705 asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE); 706 if (offset + asize > vd->vdev_asize) { 707 zfs_panic_recover("blkptr at %p DVA %u has invalid " 708 "OFFSET %llu", 709 bp, i, (longlong_t)offset); 710 } 711 } 712 } 713 714 zio_t * 715 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 716 void *data, uint64_t size, zio_done_func_t *done, void *private, 717 zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb) 718 { 719 zio_t *zio; 720 721 zfs_blkptr_verify(spa, bp); 722 723 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 724 data, size, done, private, 725 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 726 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 727 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 728 729 return (zio); 730 } 731 732 zio_t * 733 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 734 void *data, uint64_t size, const zio_prop_t *zp, 735 zio_done_func_t *ready, zio_done_func_t *children_ready, 736 zio_done_func_t *physdone, zio_done_func_t *done, 737 void *private, zio_priority_t priority, enum zio_flag flags, 738 const zbookmark_phys_t *zb) 739 { 740 zio_t *zio; 741 742 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 743 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 744 zp->zp_compress >= ZIO_COMPRESS_OFF && 745 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 746 DMU_OT_IS_VALID(zp->zp_type) && 747 zp->zp_level < 32 && 748 zp->zp_copies > 0 && 749 zp->zp_copies <= spa_max_replication(spa)); 750 751 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 752 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 753 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 754 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 755 756 zio->io_ready = ready; 757 zio->io_children_ready = children_ready; 758 zio->io_physdone = physdone; 759 zio->io_prop = *zp; 760 761 /* 762 * Data can be NULL if we are going to call zio_write_override() to 763 * provide the already-allocated BP. But we may need the data to 764 * verify a dedup hit (if requested). In this case, don't try to 765 * dedup (just take the already-allocated BP verbatim). 766 */ 767 if (data == NULL && zio->io_prop.zp_dedup_verify) { 768 zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE; 769 } 770 771 return (zio); 772 } 773 774 zio_t * 775 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, 776 uint64_t size, zio_done_func_t *done, void *private, 777 zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb) 778 { 779 zio_t *zio; 780 781 zio = zio_create(pio, spa, txg, bp, data, size, done, private, 782 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_IO_REWRITE, NULL, 0, zb, 783 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 784 785 return (zio); 786 } 787 788 void 789 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite) 790 { 791 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 792 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 793 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 794 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 795 796 /* 797 * We must reset the io_prop to match the values that existed 798 * when the bp was first written by dmu_sync() keeping in mind 799 * that nopwrite and dedup are mutually exclusive. 800 */ 801 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 802 zio->io_prop.zp_nopwrite = nopwrite; 803 zio->io_prop.zp_copies = copies; 804 zio->io_bp_override = bp; 805 } 806 807 void 808 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 809 { 810 811 /* 812 * The check for EMBEDDED is a performance optimization. We 813 * process the free here (by ignoring it) rather than 814 * putting it on the list and then processing it in zio_free_sync(). 815 */ 816 if (BP_IS_EMBEDDED(bp)) 817 return; 818 metaslab_check_free(spa, bp); 819 820 /* 821 * Frees that are for the currently-syncing txg, are not going to be 822 * deferred, and which will not need to do a read (i.e. not GANG or 823 * DEDUP), can be processed immediately. Otherwise, put them on the 824 * in-memory list for later processing. 825 */ 826 if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp) || 827 txg != spa->spa_syncing_txg || 828 spa_sync_pass(spa) >= zfs_sync_pass_deferred_free) { 829 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 830 } else { 831 VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 0))); 832 } 833 } 834 835 zio_t * 836 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 837 enum zio_flag flags) 838 { 839 zio_t *zio; 840 enum zio_stage stage = ZIO_FREE_PIPELINE; 841 842 ASSERT(!BP_IS_HOLE(bp)); 843 ASSERT(spa_syncing_txg(spa) == txg); 844 ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free); 845 846 if (BP_IS_EMBEDDED(bp)) 847 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 848 849 metaslab_check_free(spa, bp); 850 arc_freed(spa, bp); 851 852 /* 853 * GANG and DEDUP blocks can induce a read (for the gang block header, 854 * or the DDT), so issue them asynchronously so that this thread is 855 * not tied up. 856 */ 857 if (BP_IS_GANG(bp) || BP_GET_DEDUP(bp)) 858 stage |= ZIO_STAGE_ISSUE_ASYNC; 859 860 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 861 NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, flags, 862 NULL, 0, NULL, ZIO_STAGE_OPEN, stage); 863 864 return (zio); 865 } 866 867 zio_t * 868 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 869 zio_done_func_t *done, void *private, enum zio_flag flags) 870 { 871 zio_t *zio; 872 873 dprintf_bp(bp, "claiming in txg %llu", txg); 874 875 if (BP_IS_EMBEDDED(bp)) 876 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 877 878 /* 879 * A claim is an allocation of a specific block. Claims are needed 880 * to support immediate writes in the intent log. The issue is that 881 * immediate writes contain committed data, but in a txg that was 882 * *not* committed. Upon opening the pool after an unclean shutdown, 883 * the intent log claims all blocks that contain immediate write data 884 * so that the SPA knows they're in use. 885 * 886 * All claims *must* be resolved in the first txg -- before the SPA 887 * starts allocating blocks -- so that nothing is allocated twice. 888 * If txg == 0 we just verify that the block is claimable. 889 */ 890 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa)); 891 ASSERT(txg == spa_first_txg(spa) || txg == 0); 892 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */ 893 894 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 895 done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, flags, 896 NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 897 ASSERT0(zio->io_queued_timestamp); 898 899 return (zio); 900 } 901 902 zio_t * 903 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 904 zio_done_func_t *done, void *private, enum zio_flag flags) 905 { 906 zio_t *zio; 907 int c; 908 909 if (vd->vdev_children == 0) { 910 zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private, 911 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 912 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 913 914 zio->io_cmd = cmd; 915 } else { 916 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 917 918 for (c = 0; c < vd->vdev_children; c++) 919 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 920 done, private, flags)); 921 } 922 923 return (zio); 924 } 925 926 zio_t * 927 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 928 void *data, int checksum, zio_done_func_t *done, void *private, 929 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 930 { 931 zio_t *zio; 932 933 ASSERT(vd->vdev_children == 0); 934 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 935 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 936 ASSERT3U(offset + size, <=, vd->vdev_psize); 937 938 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 939 ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset, 940 NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 941 942 zio->io_prop.zp_checksum = checksum; 943 944 return (zio); 945 } 946 947 zio_t * 948 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 949 void *data, int checksum, zio_done_func_t *done, void *private, 950 zio_priority_t priority, enum zio_flag flags, boolean_t labels) 951 { 952 zio_t *zio; 953 954 ASSERT(vd->vdev_children == 0); 955 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 956 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 957 ASSERT3U(offset + size, <=, vd->vdev_psize); 958 959 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, done, private, 960 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd, offset, 961 NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 962 963 zio->io_prop.zp_checksum = checksum; 964 965 if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) { 966 /* 967 * zec checksums are necessarily destructive -- they modify 968 * the end of the write buffer to hold the verifier/checksum. 969 * Therefore, we must make a local copy in case the data is 970 * being written to multiple places in parallel. 971 */ 972 void *wbuf = zio_buf_alloc(size); 973 bcopy(data, wbuf, size); 974 zio_push_transform(zio, wbuf, size, size, NULL); 975 } 976 977 return (zio); 978 } 979 980 /* 981 * Create a child I/O to do some work for us. 982 */ 983 zio_t * 984 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 985 void *data, uint64_t size, int type, zio_priority_t priority, 986 enum zio_flag flags, zio_done_func_t *done, void *private) 987 { 988 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 989 zio_t *zio; 990 991 ASSERT(vd->vdev_parent == 992 (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev)); 993 994 if (type == ZIO_TYPE_READ && bp != NULL) { 995 /* 996 * If we have the bp, then the child should perform the 997 * checksum and the parent need not. This pushes error 998 * detection as close to the leaves as possible and 999 * eliminates redundant checksums in the interior nodes. 1000 */ 1001 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 1002 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 1003 } 1004 1005 if (vd->vdev_children == 0) 1006 offset += VDEV_LABEL_START_SIZE; 1007 1008 flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE; 1009 1010 /* 1011 * If we've decided to do a repair, the write is not speculative -- 1012 * even if the original read was. 1013 */ 1014 if (flags & ZIO_FLAG_IO_REPAIR) 1015 flags &= ~ZIO_FLAG_SPECULATIVE; 1016 1017 /* 1018 * If we're creating a child I/O that is not associated with a 1019 * top-level vdev, then the child zio is not an allocating I/O. 1020 * If this is a retried I/O then we ignore it since we will 1021 * have already processed the original allocating I/O. 1022 */ 1023 if (flags & ZIO_FLAG_IO_ALLOCATING && 1024 (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) { 1025 metaslab_class_t *mc = spa_normal_class(pio->io_spa); 1026 1027 ASSERT(mc->mc_alloc_throttle_enabled); 1028 ASSERT(type == ZIO_TYPE_WRITE); 1029 ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE); 1030 ASSERT(!(flags & ZIO_FLAG_IO_REPAIR)); 1031 ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) || 1032 pio->io_child_type == ZIO_CHILD_GANG); 1033 1034 flags &= ~ZIO_FLAG_IO_ALLOCATING; 1035 } 1036 1037 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, 1038 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 1039 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 1040 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 1041 1042 zio->io_physdone = pio->io_physdone; 1043 if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL) 1044 zio->io_logical->io_phys_children++; 1045 1046 return (zio); 1047 } 1048 1049 zio_t * 1050 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size, 1051 int type, zio_priority_t priority, enum zio_flag flags, 1052 zio_done_func_t *done, void *private) 1053 { 1054 zio_t *zio; 1055 1056 ASSERT(vd->vdev_ops->vdev_op_leaf); 1057 1058 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 1059 data, size, done, private, type, priority, 1060 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 1061 vd, offset, NULL, 1062 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 1063 1064 return (zio); 1065 } 1066 1067 void 1068 zio_flush(zio_t *zio, vdev_t *vd) 1069 { 1070 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 1071 NULL, NULL, 1072 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 1073 } 1074 1075 void 1076 zio_shrink(zio_t *zio, uint64_t size) 1077 { 1078 ASSERT(zio->io_executor == NULL); 1079 ASSERT(zio->io_orig_size == zio->io_size); 1080 ASSERT(size <= zio->io_size); 1081 1082 /* 1083 * We don't shrink for raidz because of problems with the 1084 * reconstruction when reading back less than the block size. 1085 * Note, BP_IS_RAIDZ() assumes no compression. 1086 */ 1087 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 1088 if (!BP_IS_RAIDZ(zio->io_bp)) 1089 zio->io_orig_size = zio->io_size = size; 1090 } 1091 1092 /* 1093 * ========================================================================== 1094 * Prepare to read and write logical blocks 1095 * ========================================================================== 1096 */ 1097 1098 static int 1099 zio_read_bp_init(zio_t *zio) 1100 { 1101 blkptr_t *bp = zio->io_bp; 1102 1103 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 1104 zio->io_child_type == ZIO_CHILD_LOGICAL && 1105 !(zio->io_flags & ZIO_FLAG_RAW)) { 1106 uint64_t psize = 1107 BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp); 1108 void *cbuf = zio_buf_alloc(psize); 1109 1110 zio_push_transform(zio, cbuf, psize, psize, zio_decompress); 1111 } 1112 1113 if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) { 1114 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1115 decode_embedded_bp_compressed(bp, zio->io_data); 1116 } else { 1117 ASSERT(!BP_IS_EMBEDDED(bp)); 1118 } 1119 1120 if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0) 1121 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1122 1123 if (BP_GET_TYPE(bp) == DMU_OT_DDT_ZAP) 1124 zio->io_flags |= ZIO_FLAG_DONT_CACHE; 1125 1126 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 1127 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 1128 1129 return (ZIO_PIPELINE_CONTINUE); 1130 } 1131 1132 static int 1133 zio_write_bp_init(zio_t *zio) 1134 { 1135 if (!IO_IS_ALLOCATING(zio)) 1136 return (ZIO_PIPELINE_CONTINUE); 1137 1138 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1139 1140 if (zio->io_bp_override) { 1141 blkptr_t *bp = zio->io_bp; 1142 zio_prop_t *zp = &zio->io_prop; 1143 1144 ASSERT(bp->blk_birth != zio->io_txg); 1145 ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0); 1146 1147 *bp = *zio->io_bp_override; 1148 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1149 1150 if (BP_IS_EMBEDDED(bp)) 1151 return (ZIO_PIPELINE_CONTINUE); 1152 1153 /* 1154 * If we've been overridden and nopwrite is set then 1155 * set the flag accordingly to indicate that a nopwrite 1156 * has already occurred. 1157 */ 1158 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1159 ASSERT(!zp->zp_dedup); 1160 ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum); 1161 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1162 return (ZIO_PIPELINE_CONTINUE); 1163 } 1164 1165 ASSERT(!zp->zp_nopwrite); 1166 1167 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1168 return (ZIO_PIPELINE_CONTINUE); 1169 1170 ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags & 1171 ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify); 1172 1173 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { 1174 BP_SET_DEDUP(bp, 1); 1175 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1176 return (ZIO_PIPELINE_CONTINUE); 1177 } 1178 1179 /* 1180 * We were unable to handle this as an override bp, treat 1181 * it as a regular write I/O. 1182 */ 1183 zio->io_bp_override = NULL; 1184 *bp = zio->io_bp_orig; 1185 zio->io_pipeline = zio->io_orig_pipeline; 1186 } 1187 1188 return (ZIO_PIPELINE_CONTINUE); 1189 } 1190 1191 static int 1192 zio_write_compress(zio_t *zio) 1193 { 1194 spa_t *spa = zio->io_spa; 1195 zio_prop_t *zp = &zio->io_prop; 1196 enum zio_compress compress = zp->zp_compress; 1197 blkptr_t *bp = zio->io_bp; 1198 uint64_t lsize = zio->io_size; 1199 uint64_t psize = lsize; 1200 int pass = 1; 1201 1202 /* 1203 * If our children haven't all reached the ready stage, 1204 * wait for them and then repeat this pipeline stage. 1205 */ 1206 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 1207 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) 1208 return (ZIO_PIPELINE_STOP); 1209 1210 if (!IO_IS_ALLOCATING(zio)) 1211 return (ZIO_PIPELINE_CONTINUE); 1212 1213 if (zio->io_children_ready != NULL) { 1214 /* 1215 * Now that all our children are ready, run the callback 1216 * associated with this zio in case it wants to modify the 1217 * data to be written. 1218 */ 1219 ASSERT3U(zp->zp_level, >, 0); 1220 zio->io_children_ready(zio); 1221 } 1222 1223 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1224 ASSERT(zio->io_bp_override == NULL); 1225 1226 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) { 1227 /* 1228 * We're rewriting an existing block, which means we're 1229 * working on behalf of spa_sync(). For spa_sync() to 1230 * converge, it must eventually be the case that we don't 1231 * have to allocate new blocks. But compression changes 1232 * the blocksize, which forces a reallocate, and makes 1233 * convergence take longer. Therefore, after the first 1234 * few passes, stop compressing to ensure convergence. 1235 */ 1236 pass = spa_sync_pass(spa); 1237 1238 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1239 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1240 ASSERT(!BP_GET_DEDUP(bp)); 1241 1242 if (pass >= zfs_sync_pass_dont_compress) 1243 compress = ZIO_COMPRESS_OFF; 1244 1245 /* Make sure someone doesn't change their mind on overwrites */ 1246 ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp), 1247 spa_max_replication(spa)) == BP_GET_NDVAS(bp)); 1248 } 1249 1250 if (compress != ZIO_COMPRESS_OFF) { 1251 void *cbuf = zio_buf_alloc(lsize); 1252 psize = zio_compress_data(compress, zio->io_data, cbuf, lsize); 1253 if (psize == 0 || psize == lsize) { 1254 compress = ZIO_COMPRESS_OFF; 1255 zio_buf_free(cbuf, lsize); 1256 } else if (!zp->zp_dedup && psize <= BPE_PAYLOAD_SIZE && 1257 zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && 1258 spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { 1259 encode_embedded_bp_compressed(bp, 1260 cbuf, compress, lsize, psize); 1261 BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); 1262 BP_SET_TYPE(bp, zio->io_prop.zp_type); 1263 BP_SET_LEVEL(bp, zio->io_prop.zp_level); 1264 zio_buf_free(cbuf, lsize); 1265 bp->blk_birth = zio->io_txg; 1266 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1267 ASSERT(spa_feature_is_active(spa, 1268 SPA_FEATURE_EMBEDDED_DATA)); 1269 return (ZIO_PIPELINE_CONTINUE); 1270 } else { 1271 /* 1272 * Round up compressed size up to the ashift 1273 * of the smallest-ashift device, and zero the tail. 1274 * This ensures that the compressed size of the BP 1275 * (and thus compressratio property) are correct, 1276 * in that we charge for the padding used to fill out 1277 * the last sector. 1278 */ 1279 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 1280 size_t rounded = (size_t)P2ROUNDUP(psize, 1281 1ULL << spa->spa_min_ashift); 1282 if (rounded >= lsize) { 1283 compress = ZIO_COMPRESS_OFF; 1284 zio_buf_free(cbuf, lsize); 1285 psize = lsize; 1286 } else { 1287 bzero((char *)cbuf + psize, rounded - psize); 1288 psize = rounded; 1289 zio_push_transform(zio, cbuf, 1290 psize, lsize, NULL); 1291 } 1292 } 1293 1294 /* 1295 * We were unable to handle this as an override bp, treat 1296 * it as a regular write I/O. 1297 */ 1298 zio->io_bp_override = NULL; 1299 *bp = zio->io_bp_orig; 1300 zio->io_pipeline = zio->io_orig_pipeline; 1301 } 1302 1303 /* 1304 * The final pass of spa_sync() must be all rewrites, but the first 1305 * few passes offer a trade-off: allocating blocks defers convergence, 1306 * but newly allocated blocks are sequential, so they can be written 1307 * to disk faster. Therefore, we allow the first few passes of 1308 * spa_sync() to allocate new blocks, but force rewrites after that. 1309 * There should only be a handful of blocks after pass 1 in any case. 1310 */ 1311 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg && 1312 BP_GET_PSIZE(bp) == psize && 1313 pass >= zfs_sync_pass_rewrite) { 1314 ASSERT(psize != 0); 1315 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1316 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1317 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1318 } else { 1319 BP_ZERO(bp); 1320 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1321 } 1322 1323 if (psize == 0) { 1324 if (zio->io_bp_orig.blk_birth != 0 && 1325 spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) { 1326 BP_SET_LSIZE(bp, lsize); 1327 BP_SET_TYPE(bp, zp->zp_type); 1328 BP_SET_LEVEL(bp, zp->zp_level); 1329 BP_SET_BIRTH(bp, zio->io_txg, 0); 1330 } 1331 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1332 } else { 1333 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1334 BP_SET_LSIZE(bp, lsize); 1335 BP_SET_TYPE(bp, zp->zp_type); 1336 BP_SET_LEVEL(bp, zp->zp_level); 1337 BP_SET_PSIZE(bp, psize); 1338 BP_SET_COMPRESS(bp, compress); 1339 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1340 BP_SET_DEDUP(bp, zp->zp_dedup); 1341 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1342 if (zp->zp_dedup) { 1343 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1344 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1345 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1346 } 1347 if (zp->zp_nopwrite) { 1348 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1349 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1350 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1351 } 1352 } 1353 return (ZIO_PIPELINE_CONTINUE); 1354 } 1355 1356 static int 1357 zio_free_bp_init(zio_t *zio) 1358 { 1359 blkptr_t *bp = zio->io_bp; 1360 1361 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1362 if (BP_GET_DEDUP(bp)) 1363 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1364 } 1365 1366 return (ZIO_PIPELINE_CONTINUE); 1367 } 1368 1369 /* 1370 * ========================================================================== 1371 * Execute the I/O pipeline 1372 * ========================================================================== 1373 */ 1374 1375 static void 1376 zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1377 { 1378 spa_t *spa = zio->io_spa; 1379 zio_type_t t = zio->io_type; 1380 int flags = (cutinline ? TQ_FRONT : 0); 1381 1382 /* 1383 * If we're a config writer or a probe, the normal issue and 1384 * interrupt threads may all be blocked waiting for the config lock. 1385 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1386 */ 1387 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1388 t = ZIO_TYPE_NULL; 1389 1390 /* 1391 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1392 */ 1393 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1394 t = ZIO_TYPE_NULL; 1395 1396 /* 1397 * If this is a high priority I/O, then use the high priority taskq if 1398 * available. 1399 */ 1400 if (zio->io_priority == ZIO_PRIORITY_NOW && 1401 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 1402 q++; 1403 1404 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 1405 1406 /* 1407 * NB: We are assuming that the zio can only be dispatched 1408 * to a single taskq at a time. It would be a grievous error 1409 * to dispatch the zio to another taskq at the same time. 1410 */ 1411 ASSERT(zio->io_tqent.tqent_next == NULL); 1412 spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio, 1413 flags, &zio->io_tqent); 1414 } 1415 1416 static boolean_t 1417 zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 1418 { 1419 kthread_t *executor = zio->io_executor; 1420 spa_t *spa = zio->io_spa; 1421 1422 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 1423 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1424 uint_t i; 1425 for (i = 0; i < tqs->stqs_count; i++) { 1426 if (taskq_member(tqs->stqs_taskq[i], executor)) 1427 return (B_TRUE); 1428 } 1429 } 1430 1431 return (B_FALSE); 1432 } 1433 1434 static int 1435 zio_issue_async(zio_t *zio) 1436 { 1437 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 1438 1439 return (ZIO_PIPELINE_STOP); 1440 } 1441 1442 void 1443 zio_interrupt(zio_t *zio) 1444 { 1445 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 1446 } 1447 1448 void 1449 zio_delay_interrupt(zio_t *zio) 1450 { 1451 /* 1452 * The timeout_generic() function isn't defined in userspace, so 1453 * rather than trying to implement the function, the zio delay 1454 * functionality has been disabled for userspace builds. 1455 */ 1456 1457 #ifdef _KERNEL 1458 /* 1459 * If io_target_timestamp is zero, then no delay has been registered 1460 * for this IO, thus jump to the end of this function and "skip" the 1461 * delay; issuing it directly to the zio layer. 1462 */ 1463 if (zio->io_target_timestamp != 0) { 1464 hrtime_t now = gethrtime(); 1465 1466 if (now >= zio->io_target_timestamp) { 1467 /* 1468 * This IO has already taken longer than the target 1469 * delay to complete, so we don't want to delay it 1470 * any longer; we "miss" the delay and issue it 1471 * directly to the zio layer. This is likely due to 1472 * the target latency being set to a value less than 1473 * the underlying hardware can satisfy (e.g. delay 1474 * set to 1ms, but the disks take 10ms to complete an 1475 * IO request). 1476 */ 1477 1478 DTRACE_PROBE2(zio__delay__miss, zio_t *, zio, 1479 hrtime_t, now); 1480 1481 zio_interrupt(zio); 1482 } else { 1483 hrtime_t diff = zio->io_target_timestamp - now; 1484 1485 DTRACE_PROBE3(zio__delay__hit, zio_t *, zio, 1486 hrtime_t, now, hrtime_t, diff); 1487 1488 (void) timeout_generic(CALLOUT_NORMAL, 1489 (void (*)(void *))zio_interrupt, zio, diff, 1, 0); 1490 } 1491 1492 return; 1493 } 1494 #endif 1495 1496 DTRACE_PROBE1(zio__delay__skip, zio_t *, zio); 1497 zio_interrupt(zio); 1498 } 1499 1500 /* 1501 * Execute the I/O pipeline until one of the following occurs: 1502 * 1503 * (1) the I/O completes 1504 * (2) the pipeline stalls waiting for dependent child I/Os 1505 * (3) the I/O issues, so we're waiting for an I/O completion interrupt 1506 * (4) the I/O is delegated by vdev-level caching or aggregation 1507 * (5) the I/O is deferred due to vdev-level queueing 1508 * (6) the I/O is handed off to another thread. 1509 * 1510 * In all cases, the pipeline stops whenever there's no CPU work; it never 1511 * burns a thread in cv_wait(). 1512 * 1513 * There's no locking on io_stage because there's no legitimate way 1514 * for multiple threads to be attempting to process the same I/O. 1515 */ 1516 static zio_pipe_stage_t *zio_pipeline[]; 1517 1518 void 1519 zio_execute(zio_t *zio) 1520 { 1521 zio->io_executor = curthread; 1522 1523 ASSERT3U(zio->io_queued_timestamp, >, 0); 1524 1525 while (zio->io_stage < ZIO_STAGE_DONE) { 1526 enum zio_stage pipeline = zio->io_pipeline; 1527 enum zio_stage stage = zio->io_stage; 1528 int rv; 1529 1530 ASSERT(!MUTEX_HELD(&zio->io_lock)); 1531 ASSERT(ISP2(stage)); 1532 ASSERT(zio->io_stall == NULL); 1533 1534 do { 1535 stage <<= 1; 1536 } while ((stage & pipeline) == 0); 1537 1538 ASSERT(stage <= ZIO_STAGE_DONE); 1539 1540 /* 1541 * If we are in interrupt context and this pipeline stage 1542 * will grab a config lock that is held across I/O, 1543 * or may wait for an I/O that needs an interrupt thread 1544 * to complete, issue async to avoid deadlock. 1545 * 1546 * For VDEV_IO_START, we cut in line so that the io will 1547 * be sent to disk promptly. 1548 */ 1549 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 1550 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 1551 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 1552 zio_requeue_io_start_cut_in_line : B_FALSE; 1553 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 1554 return; 1555 } 1556 1557 zio->io_stage = stage; 1558 zio->io_pipeline_trace |= zio->io_stage; 1559 rv = zio_pipeline[highbit64(stage) - 1](zio); 1560 1561 if (rv == ZIO_PIPELINE_STOP) 1562 return; 1563 1564 ASSERT(rv == ZIO_PIPELINE_CONTINUE); 1565 } 1566 } 1567 1568 /* 1569 * ========================================================================== 1570 * Initiate I/O, either sync or async 1571 * ========================================================================== 1572 */ 1573 int 1574 zio_wait(zio_t *zio) 1575 { 1576 int error; 1577 1578 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1579 ASSERT(zio->io_executor == NULL); 1580 1581 zio->io_waiter = curthread; 1582 ASSERT0(zio->io_queued_timestamp); 1583 zio->io_queued_timestamp = gethrtime(); 1584 1585 zio_execute(zio); 1586 1587 mutex_enter(&zio->io_lock); 1588 while (zio->io_executor != NULL) 1589 cv_wait(&zio->io_cv, &zio->io_lock); 1590 mutex_exit(&zio->io_lock); 1591 1592 error = zio->io_error; 1593 zio_destroy(zio); 1594 1595 return (error); 1596 } 1597 1598 void 1599 zio_nowait(zio_t *zio) 1600 { 1601 ASSERT(zio->io_executor == NULL); 1602 1603 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 1604 zio_unique_parent(zio) == NULL) { 1605 /* 1606 * This is a logical async I/O with no parent to wait for it. 1607 * We add it to the spa_async_root_zio "Godfather" I/O which 1608 * will ensure they complete prior to unloading the pool. 1609 */ 1610 spa_t *spa = zio->io_spa; 1611 1612 zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio); 1613 } 1614 1615 ASSERT0(zio->io_queued_timestamp); 1616 zio->io_queued_timestamp = gethrtime(); 1617 zio_execute(zio); 1618 } 1619 1620 /* 1621 * ========================================================================== 1622 * Reexecute or suspend/resume failed I/O 1623 * ========================================================================== 1624 */ 1625 1626 static void 1627 zio_reexecute(zio_t *pio) 1628 { 1629 zio_t *cio, *cio_next; 1630 1631 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 1632 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 1633 ASSERT(pio->io_gang_leader == NULL); 1634 ASSERT(pio->io_gang_tree == NULL); 1635 1636 pio->io_flags = pio->io_orig_flags; 1637 pio->io_stage = pio->io_orig_stage; 1638 pio->io_pipeline = pio->io_orig_pipeline; 1639 pio->io_reexecute = 0; 1640 pio->io_flags |= ZIO_FLAG_REEXECUTED; 1641 pio->io_pipeline_trace = 0; 1642 pio->io_error = 0; 1643 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1644 pio->io_state[w] = 0; 1645 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 1646 pio->io_child_error[c] = 0; 1647 1648 if (IO_IS_ALLOCATING(pio)) 1649 BP_ZERO(pio->io_bp); 1650 1651 /* 1652 * As we reexecute pio's children, new children could be created. 1653 * New children go to the head of pio's io_child_list, however, 1654 * so we will (correctly) not reexecute them. The key is that 1655 * the remainder of pio's io_child_list, from 'cio_next' onward, 1656 * cannot be affected by any side effects of reexecuting 'cio'. 1657 */ 1658 zio_link_t *zl = NULL; 1659 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 1660 cio_next = zio_walk_children(pio, &zl); 1661 mutex_enter(&pio->io_lock); 1662 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 1663 pio->io_children[cio->io_child_type][w]++; 1664 mutex_exit(&pio->io_lock); 1665 zio_reexecute(cio); 1666 } 1667 1668 /* 1669 * Now that all children have been reexecuted, execute the parent. 1670 * We don't reexecute "The Godfather" I/O here as it's the 1671 * responsibility of the caller to wait on him. 1672 */ 1673 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) { 1674 pio->io_queued_timestamp = gethrtime(); 1675 zio_execute(pio); 1676 } 1677 } 1678 1679 void 1680 zio_suspend(spa_t *spa, zio_t *zio) 1681 { 1682 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 1683 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 1684 "failure and the failure mode property for this pool " 1685 "is set to panic.", spa_name(spa)); 1686 1687 zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0); 1688 1689 mutex_enter(&spa->spa_suspend_lock); 1690 1691 if (spa->spa_suspend_zio_root == NULL) 1692 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 1693 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 1694 ZIO_FLAG_GODFATHER); 1695 1696 spa->spa_suspended = B_TRUE; 1697 1698 if (zio != NULL) { 1699 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 1700 ASSERT(zio != spa->spa_suspend_zio_root); 1701 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1702 ASSERT(zio_unique_parent(zio) == NULL); 1703 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 1704 zio_add_child(spa->spa_suspend_zio_root, zio); 1705 } 1706 1707 mutex_exit(&spa->spa_suspend_lock); 1708 } 1709 1710 int 1711 zio_resume(spa_t *spa) 1712 { 1713 zio_t *pio; 1714 1715 /* 1716 * Reexecute all previously suspended i/o. 1717 */ 1718 mutex_enter(&spa->spa_suspend_lock); 1719 spa->spa_suspended = B_FALSE; 1720 cv_broadcast(&spa->spa_suspend_cv); 1721 pio = spa->spa_suspend_zio_root; 1722 spa->spa_suspend_zio_root = NULL; 1723 mutex_exit(&spa->spa_suspend_lock); 1724 1725 if (pio == NULL) 1726 return (0); 1727 1728 zio_reexecute(pio); 1729 return (zio_wait(pio)); 1730 } 1731 1732 void 1733 zio_resume_wait(spa_t *spa) 1734 { 1735 mutex_enter(&spa->spa_suspend_lock); 1736 while (spa_suspended(spa)) 1737 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 1738 mutex_exit(&spa->spa_suspend_lock); 1739 } 1740 1741 /* 1742 * ========================================================================== 1743 * Gang blocks. 1744 * 1745 * A gang block is a collection of small blocks that looks to the DMU 1746 * like one large block. When zio_dva_allocate() cannot find a block 1747 * of the requested size, due to either severe fragmentation or the pool 1748 * being nearly full, it calls zio_write_gang_block() to construct the 1749 * block from smaller fragments. 1750 * 1751 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 1752 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 1753 * an indirect block: it's an array of block pointers. It consumes 1754 * only one sector and hence is allocatable regardless of fragmentation. 1755 * The gang header's bps point to its gang members, which hold the data. 1756 * 1757 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 1758 * as the verifier to ensure uniqueness of the SHA256 checksum. 1759 * Critically, the gang block bp's blk_cksum is the checksum of the data, 1760 * not the gang header. This ensures that data block signatures (needed for 1761 * deduplication) are independent of how the block is physically stored. 1762 * 1763 * Gang blocks can be nested: a gang member may itself be a gang block. 1764 * Thus every gang block is a tree in which root and all interior nodes are 1765 * gang headers, and the leaves are normal blocks that contain user data. 1766 * The root of the gang tree is called the gang leader. 1767 * 1768 * To perform any operation (read, rewrite, free, claim) on a gang block, 1769 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 1770 * in the io_gang_tree field of the original logical i/o by recursively 1771 * reading the gang leader and all gang headers below it. This yields 1772 * an in-core tree containing the contents of every gang header and the 1773 * bps for every constituent of the gang block. 1774 * 1775 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 1776 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 1777 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 1778 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 1779 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 1780 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 1781 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 1782 * of the gang header plus zio_checksum_compute() of the data to update the 1783 * gang header's blk_cksum as described above. 1784 * 1785 * The two-phase assemble/issue model solves the problem of partial failure -- 1786 * what if you'd freed part of a gang block but then couldn't read the 1787 * gang header for another part? Assembling the entire gang tree first 1788 * ensures that all the necessary gang header I/O has succeeded before 1789 * starting the actual work of free, claim, or write. Once the gang tree 1790 * is assembled, free and claim are in-memory operations that cannot fail. 1791 * 1792 * In the event that a gang write fails, zio_dva_unallocate() walks the 1793 * gang tree to immediately free (i.e. insert back into the space map) 1794 * everything we've allocated. This ensures that we don't get ENOSPC 1795 * errors during repeated suspend/resume cycles due to a flaky device. 1796 * 1797 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 1798 * the gang tree, we won't modify the block, so we can safely defer the free 1799 * (knowing that the block is still intact). If we *can* assemble the gang 1800 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 1801 * each constituent bp and we can allocate a new block on the next sync pass. 1802 * 1803 * In all cases, the gang tree allows complete recovery from partial failure. 1804 * ========================================================================== 1805 */ 1806 1807 static zio_t * 1808 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1809 { 1810 if (gn != NULL) 1811 return (pio); 1812 1813 return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp), 1814 NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 1815 &pio->io_bookmark)); 1816 } 1817 1818 zio_t * 1819 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1820 { 1821 zio_t *zio; 1822 1823 if (gn != NULL) { 1824 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1825 gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority, 1826 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1827 /* 1828 * As we rewrite each gang header, the pipeline will compute 1829 * a new gang block header checksum for it; but no one will 1830 * compute a new data checksum, so we do that here. The one 1831 * exception is the gang leader: the pipeline already computed 1832 * its data checksum because that stage precedes gang assembly. 1833 * (Presently, nothing actually uses interior data checksums; 1834 * this is just good hygiene.) 1835 */ 1836 if (gn != pio->io_gang_leader->io_gang_tree) { 1837 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 1838 data, BP_GET_PSIZE(bp)); 1839 } 1840 /* 1841 * If we are here to damage data for testing purposes, 1842 * leave the GBH alone so that we can detect the damage. 1843 */ 1844 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 1845 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 1846 } else { 1847 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 1848 data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority, 1849 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 1850 } 1851 1852 return (zio); 1853 } 1854 1855 /* ARGSUSED */ 1856 zio_t * 1857 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1858 { 1859 return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 1860 ZIO_GANG_CHILD_FLAGS(pio))); 1861 } 1862 1863 /* ARGSUSED */ 1864 zio_t * 1865 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data) 1866 { 1867 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 1868 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 1869 } 1870 1871 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 1872 NULL, 1873 zio_read_gang, 1874 zio_rewrite_gang, 1875 zio_free_gang, 1876 zio_claim_gang, 1877 NULL 1878 }; 1879 1880 static void zio_gang_tree_assemble_done(zio_t *zio); 1881 1882 static zio_gang_node_t * 1883 zio_gang_node_alloc(zio_gang_node_t **gnpp) 1884 { 1885 zio_gang_node_t *gn; 1886 1887 ASSERT(*gnpp == NULL); 1888 1889 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 1890 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 1891 *gnpp = gn; 1892 1893 return (gn); 1894 } 1895 1896 static void 1897 zio_gang_node_free(zio_gang_node_t **gnpp) 1898 { 1899 zio_gang_node_t *gn = *gnpp; 1900 1901 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1902 ASSERT(gn->gn_child[g] == NULL); 1903 1904 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 1905 kmem_free(gn, sizeof (*gn)); 1906 *gnpp = NULL; 1907 } 1908 1909 static void 1910 zio_gang_tree_free(zio_gang_node_t **gnpp) 1911 { 1912 zio_gang_node_t *gn = *gnpp; 1913 1914 if (gn == NULL) 1915 return; 1916 1917 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 1918 zio_gang_tree_free(&gn->gn_child[g]); 1919 1920 zio_gang_node_free(gnpp); 1921 } 1922 1923 static void 1924 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 1925 { 1926 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 1927 1928 ASSERT(gio->io_gang_leader == gio); 1929 ASSERT(BP_IS_GANG(bp)); 1930 1931 zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh, 1932 SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn, 1933 gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 1934 } 1935 1936 static void 1937 zio_gang_tree_assemble_done(zio_t *zio) 1938 { 1939 zio_t *gio = zio->io_gang_leader; 1940 zio_gang_node_t *gn = zio->io_private; 1941 blkptr_t *bp = zio->io_bp; 1942 1943 ASSERT(gio == zio_unique_parent(zio)); 1944 ASSERT(zio->io_child_count == 0); 1945 1946 if (zio->io_error) 1947 return; 1948 1949 if (BP_SHOULD_BYTESWAP(bp)) 1950 byteswap_uint64_array(zio->io_data, zio->io_size); 1951 1952 ASSERT(zio->io_data == gn->gn_gbh); 1953 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 1954 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1955 1956 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1957 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1958 if (!BP_IS_GANG(gbp)) 1959 continue; 1960 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 1961 } 1962 } 1963 1964 static void 1965 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data) 1966 { 1967 zio_t *gio = pio->io_gang_leader; 1968 zio_t *zio; 1969 1970 ASSERT(BP_IS_GANG(bp) == !!gn); 1971 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 1972 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 1973 1974 /* 1975 * If you're a gang header, your data is in gn->gn_gbh. 1976 * If you're a gang member, your data is in 'data' and gn == NULL. 1977 */ 1978 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data); 1979 1980 if (gn != NULL) { 1981 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 1982 1983 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 1984 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 1985 if (BP_IS_HOLE(gbp)) 1986 continue; 1987 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data); 1988 data = (char *)data + BP_GET_PSIZE(gbp); 1989 } 1990 } 1991 1992 if (gn == gio->io_gang_tree) 1993 ASSERT3P((char *)gio->io_data + gio->io_size, ==, data); 1994 1995 if (zio != pio) 1996 zio_nowait(zio); 1997 } 1998 1999 static int 2000 zio_gang_assemble(zio_t *zio) 2001 { 2002 blkptr_t *bp = zio->io_bp; 2003 2004 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 2005 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2006 2007 zio->io_gang_leader = zio; 2008 2009 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 2010 2011 return (ZIO_PIPELINE_CONTINUE); 2012 } 2013 2014 static int 2015 zio_gang_issue(zio_t *zio) 2016 { 2017 blkptr_t *bp = zio->io_bp; 2018 2019 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) 2020 return (ZIO_PIPELINE_STOP); 2021 2022 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 2023 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2024 2025 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 2026 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data); 2027 else 2028 zio_gang_tree_free(&zio->io_gang_tree); 2029 2030 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2031 2032 return (ZIO_PIPELINE_CONTINUE); 2033 } 2034 2035 static void 2036 zio_write_gang_member_ready(zio_t *zio) 2037 { 2038 zio_t *pio = zio_unique_parent(zio); 2039 zio_t *gio = zio->io_gang_leader; 2040 dva_t *cdva = zio->io_bp->blk_dva; 2041 dva_t *pdva = pio->io_bp->blk_dva; 2042 uint64_t asize; 2043 2044 if (BP_IS_HOLE(zio->io_bp)) 2045 return; 2046 2047 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 2048 2049 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 2050 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 2051 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 2052 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 2053 ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 2054 2055 mutex_enter(&pio->io_lock); 2056 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 2057 ASSERT(DVA_GET_GANG(&pdva[d])); 2058 asize = DVA_GET_ASIZE(&pdva[d]); 2059 asize += DVA_GET_ASIZE(&cdva[d]); 2060 DVA_SET_ASIZE(&pdva[d], asize); 2061 } 2062 mutex_exit(&pio->io_lock); 2063 } 2064 2065 static int 2066 zio_write_gang_block(zio_t *pio) 2067 { 2068 spa_t *spa = pio->io_spa; 2069 metaslab_class_t *mc = spa_normal_class(spa); 2070 blkptr_t *bp = pio->io_bp; 2071 zio_t *gio = pio->io_gang_leader; 2072 zio_t *zio; 2073 zio_gang_node_t *gn, **gnpp; 2074 zio_gbh_phys_t *gbh; 2075 uint64_t txg = pio->io_txg; 2076 uint64_t resid = pio->io_size; 2077 uint64_t lsize; 2078 int copies = gio->io_prop.zp_copies; 2079 int gbh_copies = MIN(copies + 1, spa_max_replication(spa)); 2080 zio_prop_t zp; 2081 int error; 2082 2083 int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER; 2084 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2085 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2086 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA)); 2087 2088 flags |= METASLAB_ASYNC_ALLOC; 2089 VERIFY(refcount_held(&mc->mc_alloc_slots, pio)); 2090 2091 /* 2092 * The logical zio has already placed a reservation for 2093 * 'copies' allocation slots but gang blocks may require 2094 * additional copies. These additional copies 2095 * (i.e. gbh_copies - copies) are guaranteed to succeed 2096 * since metaslab_class_throttle_reserve() always allows 2097 * additional reservations for gang blocks. 2098 */ 2099 VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies, 2100 pio, flags)); 2101 } 2102 2103 error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE, 2104 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags, 2105 &pio->io_alloc_list, pio); 2106 if (error) { 2107 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2108 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2109 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA)); 2110 2111 /* 2112 * If we failed to allocate the gang block header then 2113 * we remove any additional allocation reservations that 2114 * we placed here. The original reservation will 2115 * be removed when the logical I/O goes to the ready 2116 * stage. 2117 */ 2118 metaslab_class_throttle_unreserve(mc, 2119 gbh_copies - copies, pio); 2120 } 2121 pio->io_error = error; 2122 return (ZIO_PIPELINE_CONTINUE); 2123 } 2124 2125 if (pio == gio) { 2126 gnpp = &gio->io_gang_tree; 2127 } else { 2128 gnpp = pio->io_private; 2129 ASSERT(pio->io_ready == zio_write_gang_member_ready); 2130 } 2131 2132 gn = zio_gang_node_alloc(gnpp); 2133 gbh = gn->gn_gbh; 2134 bzero(gbh, SPA_GANGBLOCKSIZE); 2135 2136 /* 2137 * Create the gang header. 2138 */ 2139 zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL, 2140 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2141 2142 /* 2143 * Create and nowait the gang children. 2144 */ 2145 for (int g = 0; resid != 0; resid -= lsize, g++) { 2146 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 2147 SPA_MINBLOCKSIZE); 2148 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 2149 2150 zp.zp_checksum = gio->io_prop.zp_checksum; 2151 zp.zp_compress = ZIO_COMPRESS_OFF; 2152 zp.zp_type = DMU_OT_NONE; 2153 zp.zp_level = 0; 2154 zp.zp_copies = gio->io_prop.zp_copies; 2155 zp.zp_dedup = B_FALSE; 2156 zp.zp_dedup_verify = B_FALSE; 2157 zp.zp_nopwrite = B_FALSE; 2158 2159 zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 2160 (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, 2161 zio_write_gang_member_ready, NULL, NULL, NULL, 2162 &gn->gn_child[g], pio->io_priority, 2163 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2164 2165 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2166 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2167 ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA)); 2168 2169 /* 2170 * Gang children won't throttle but we should 2171 * account for their work, so reserve an allocation 2172 * slot for them here. 2173 */ 2174 VERIFY(metaslab_class_throttle_reserve(mc, 2175 zp.zp_copies, cio, flags)); 2176 } 2177 zio_nowait(cio); 2178 } 2179 2180 /* 2181 * Set pio's pipeline to just wait for zio to finish. 2182 */ 2183 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2184 2185 zio_nowait(zio); 2186 2187 return (ZIO_PIPELINE_CONTINUE); 2188 } 2189 2190 /* 2191 * The zio_nop_write stage in the pipeline determines if allocating a 2192 * new bp is necessary. The nopwrite feature can handle writes in 2193 * either syncing or open context (i.e. zil writes) and as a result is 2194 * mutually exclusive with dedup. 2195 * 2196 * By leveraging a cryptographically secure checksum, such as SHA256, we 2197 * can compare the checksums of the new data and the old to determine if 2198 * allocating a new block is required. Note that our requirements for 2199 * cryptographic strength are fairly weak: there can't be any accidental 2200 * hash collisions, but we don't need to be secure against intentional 2201 * (malicious) collisions. To trigger a nopwrite, you have to be able 2202 * to write the file to begin with, and triggering an incorrect (hash 2203 * collision) nopwrite is no worse than simply writing to the file. 2204 * That said, there are no known attacks against the checksum algorithms 2205 * used for nopwrite, assuming that the salt and the checksums 2206 * themselves remain secret. 2207 */ 2208 static int 2209 zio_nop_write(zio_t *zio) 2210 { 2211 blkptr_t *bp = zio->io_bp; 2212 blkptr_t *bp_orig = &zio->io_bp_orig; 2213 zio_prop_t *zp = &zio->io_prop; 2214 2215 ASSERT(BP_GET_LEVEL(bp) == 0); 2216 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 2217 ASSERT(zp->zp_nopwrite); 2218 ASSERT(!zp->zp_dedup); 2219 ASSERT(zio->io_bp_override == NULL); 2220 ASSERT(IO_IS_ALLOCATING(zio)); 2221 2222 /* 2223 * Check to see if the original bp and the new bp have matching 2224 * characteristics (i.e. same checksum, compression algorithms, etc). 2225 * If they don't then just continue with the pipeline which will 2226 * allocate a new bp. 2227 */ 2228 if (BP_IS_HOLE(bp_orig) || 2229 !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags & 2230 ZCHECKSUM_FLAG_NOPWRITE) || 2231 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 2232 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 2233 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 2234 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 2235 return (ZIO_PIPELINE_CONTINUE); 2236 2237 /* 2238 * If the checksums match then reset the pipeline so that we 2239 * avoid allocating a new bp and issuing any I/O. 2240 */ 2241 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 2242 ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags & 2243 ZCHECKSUM_FLAG_NOPWRITE); 2244 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 2245 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 2246 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 2247 ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop, 2248 sizeof (uint64_t)) == 0); 2249 2250 *bp = *bp_orig; 2251 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2252 zio->io_flags |= ZIO_FLAG_NOPWRITE; 2253 } 2254 2255 return (ZIO_PIPELINE_CONTINUE); 2256 } 2257 2258 /* 2259 * ========================================================================== 2260 * Dedup 2261 * ========================================================================== 2262 */ 2263 static void 2264 zio_ddt_child_read_done(zio_t *zio) 2265 { 2266 blkptr_t *bp = zio->io_bp; 2267 ddt_entry_t *dde = zio->io_private; 2268 ddt_phys_t *ddp; 2269 zio_t *pio = zio_unique_parent(zio); 2270 2271 mutex_enter(&pio->io_lock); 2272 ddp = ddt_phys_select(dde, bp); 2273 if (zio->io_error == 0) 2274 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 2275 if (zio->io_error == 0 && dde->dde_repair_data == NULL) 2276 dde->dde_repair_data = zio->io_data; 2277 else 2278 zio_buf_free(zio->io_data, zio->io_size); 2279 mutex_exit(&pio->io_lock); 2280 } 2281 2282 static int 2283 zio_ddt_read_start(zio_t *zio) 2284 { 2285 blkptr_t *bp = zio->io_bp; 2286 2287 ASSERT(BP_GET_DEDUP(bp)); 2288 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2289 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2290 2291 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2292 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2293 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 2294 ddt_phys_t *ddp = dde->dde_phys; 2295 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 2296 blkptr_t blk; 2297 2298 ASSERT(zio->io_vsd == NULL); 2299 zio->io_vsd = dde; 2300 2301 if (ddp_self == NULL) 2302 return (ZIO_PIPELINE_CONTINUE); 2303 2304 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2305 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 2306 continue; 2307 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 2308 &blk); 2309 zio_nowait(zio_read(zio, zio->io_spa, &blk, 2310 zio_buf_alloc(zio->io_size), zio->io_size, 2311 zio_ddt_child_read_done, dde, zio->io_priority, 2312 ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE, 2313 &zio->io_bookmark)); 2314 } 2315 return (ZIO_PIPELINE_CONTINUE); 2316 } 2317 2318 zio_nowait(zio_read(zio, zio->io_spa, bp, 2319 zio->io_data, zio->io_size, NULL, NULL, zio->io_priority, 2320 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 2321 2322 return (ZIO_PIPELINE_CONTINUE); 2323 } 2324 2325 static int 2326 zio_ddt_read_done(zio_t *zio) 2327 { 2328 blkptr_t *bp = zio->io_bp; 2329 2330 if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) 2331 return (ZIO_PIPELINE_STOP); 2332 2333 ASSERT(BP_GET_DEDUP(bp)); 2334 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 2335 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2336 2337 if (zio->io_child_error[ZIO_CHILD_DDT]) { 2338 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2339 ddt_entry_t *dde = zio->io_vsd; 2340 if (ddt == NULL) { 2341 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 2342 return (ZIO_PIPELINE_CONTINUE); 2343 } 2344 if (dde == NULL) { 2345 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 2346 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2347 return (ZIO_PIPELINE_STOP); 2348 } 2349 if (dde->dde_repair_data != NULL) { 2350 bcopy(dde->dde_repair_data, zio->io_data, zio->io_size); 2351 zio->io_child_error[ZIO_CHILD_DDT] = 0; 2352 } 2353 ddt_repair_done(ddt, dde); 2354 zio->io_vsd = NULL; 2355 } 2356 2357 ASSERT(zio->io_vsd == NULL); 2358 2359 return (ZIO_PIPELINE_CONTINUE); 2360 } 2361 2362 static boolean_t 2363 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 2364 { 2365 spa_t *spa = zio->io_spa; 2366 2367 /* 2368 * Note: we compare the original data, not the transformed data, 2369 * because when zio->io_bp is an override bp, we will not have 2370 * pushed the I/O transforms. That's an important optimization 2371 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 2372 */ 2373 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2374 zio_t *lio = dde->dde_lead_zio[p]; 2375 2376 if (lio != NULL) { 2377 return (lio->io_orig_size != zio->io_orig_size || 2378 bcmp(zio->io_orig_data, lio->io_orig_data, 2379 zio->io_orig_size) != 0); 2380 } 2381 } 2382 2383 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 2384 ddt_phys_t *ddp = &dde->dde_phys[p]; 2385 2386 if (ddp->ddp_phys_birth != 0) { 2387 arc_buf_t *abuf = NULL; 2388 arc_flags_t aflags = ARC_FLAG_WAIT; 2389 blkptr_t blk = *zio->io_bp; 2390 int error; 2391 2392 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 2393 2394 ddt_exit(ddt); 2395 2396 error = arc_read(NULL, spa, &blk, 2397 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 2398 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 2399 &aflags, &zio->io_bookmark); 2400 2401 if (error == 0) { 2402 if (arc_buf_size(abuf) != zio->io_orig_size || 2403 bcmp(abuf->b_data, zio->io_orig_data, 2404 zio->io_orig_size) != 0) 2405 error = SET_ERROR(EEXIST); 2406 arc_buf_destroy(abuf, &abuf); 2407 } 2408 2409 ddt_enter(ddt); 2410 return (error != 0); 2411 } 2412 } 2413 2414 return (B_FALSE); 2415 } 2416 2417 static void 2418 zio_ddt_child_write_ready(zio_t *zio) 2419 { 2420 int p = zio->io_prop.zp_copies; 2421 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2422 ddt_entry_t *dde = zio->io_private; 2423 ddt_phys_t *ddp = &dde->dde_phys[p]; 2424 zio_t *pio; 2425 2426 if (zio->io_error) 2427 return; 2428 2429 ddt_enter(ddt); 2430 2431 ASSERT(dde->dde_lead_zio[p] == zio); 2432 2433 ddt_phys_fill(ddp, zio->io_bp); 2434 2435 zio_link_t *zl = NULL; 2436 while ((pio = zio_walk_parents(zio, &zl)) != NULL) 2437 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 2438 2439 ddt_exit(ddt); 2440 } 2441 2442 static void 2443 zio_ddt_child_write_done(zio_t *zio) 2444 { 2445 int p = zio->io_prop.zp_copies; 2446 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 2447 ddt_entry_t *dde = zio->io_private; 2448 ddt_phys_t *ddp = &dde->dde_phys[p]; 2449 2450 ddt_enter(ddt); 2451 2452 ASSERT(ddp->ddp_refcnt == 0); 2453 ASSERT(dde->dde_lead_zio[p] == zio); 2454 dde->dde_lead_zio[p] = NULL; 2455 2456 if (zio->io_error == 0) { 2457 zio_link_t *zl = NULL; 2458 while (zio_walk_parents(zio, &zl) != NULL) 2459 ddt_phys_addref(ddp); 2460 } else { 2461 ddt_phys_clear(ddp); 2462 } 2463 2464 ddt_exit(ddt); 2465 } 2466 2467 static void 2468 zio_ddt_ditto_write_done(zio_t *zio) 2469 { 2470 int p = DDT_PHYS_DITTO; 2471 zio_prop_t *zp = &zio->io_prop; 2472 blkptr_t *bp = zio->io_bp; 2473 ddt_t *ddt = ddt_select(zio->io_spa, bp); 2474 ddt_entry_t *dde = zio->io_private; 2475 ddt_phys_t *ddp = &dde->dde_phys[p]; 2476 ddt_key_t *ddk = &dde->dde_key; 2477 2478 ddt_enter(ddt); 2479 2480 ASSERT(ddp->ddp_refcnt == 0); 2481 ASSERT(dde->dde_lead_zio[p] == zio); 2482 dde->dde_lead_zio[p] = NULL; 2483 2484 if (zio->io_error == 0) { 2485 ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum)); 2486 ASSERT(zp->zp_copies < SPA_DVAS_PER_BP); 2487 ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp)); 2488 if (ddp->ddp_phys_birth != 0) 2489 ddt_phys_free(ddt, ddk, ddp, zio->io_txg); 2490 ddt_phys_fill(ddp, bp); 2491 } 2492 2493 ddt_exit(ddt); 2494 } 2495 2496 static int 2497 zio_ddt_write(zio_t *zio) 2498 { 2499 spa_t *spa = zio->io_spa; 2500 blkptr_t *bp = zio->io_bp; 2501 uint64_t txg = zio->io_txg; 2502 zio_prop_t *zp = &zio->io_prop; 2503 int p = zp->zp_copies; 2504 int ditto_copies; 2505 zio_t *cio = NULL; 2506 zio_t *dio = NULL; 2507 ddt_t *ddt = ddt_select(spa, bp); 2508 ddt_entry_t *dde; 2509 ddt_phys_t *ddp; 2510 2511 ASSERT(BP_GET_DEDUP(bp)); 2512 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 2513 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 2514 2515 ddt_enter(ddt); 2516 dde = ddt_lookup(ddt, bp, B_TRUE); 2517 ddp = &dde->dde_phys[p]; 2518 2519 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 2520 /* 2521 * If we're using a weak checksum, upgrade to a strong checksum 2522 * and try again. If we're already using a strong checksum, 2523 * we can't resolve it, so just convert to an ordinary write. 2524 * (And automatically e-mail a paper to Nature?) 2525 */ 2526 if (!(zio_checksum_table[zp->zp_checksum].ci_flags & 2527 ZCHECKSUM_FLAG_DEDUP)) { 2528 zp->zp_checksum = spa_dedup_checksum(spa); 2529 zio_pop_transforms(zio); 2530 zio->io_stage = ZIO_STAGE_OPEN; 2531 BP_ZERO(bp); 2532 } else { 2533 zp->zp_dedup = B_FALSE; 2534 } 2535 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2536 ddt_exit(ddt); 2537 return (ZIO_PIPELINE_CONTINUE); 2538 } 2539 2540 ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); 2541 ASSERT(ditto_copies < SPA_DVAS_PER_BP); 2542 2543 if (ditto_copies > ddt_ditto_copies_present(dde) && 2544 dde->dde_lead_zio[DDT_PHYS_DITTO] == NULL) { 2545 zio_prop_t czp = *zp; 2546 2547 czp.zp_copies = ditto_copies; 2548 2549 /* 2550 * If we arrived here with an override bp, we won't have run 2551 * the transform stack, so we won't have the data we need to 2552 * generate a child i/o. So, toss the override bp and restart. 2553 * This is safe, because using the override bp is just an 2554 * optimization; and it's rare, so the cost doesn't matter. 2555 */ 2556 if (zio->io_bp_override) { 2557 zio_pop_transforms(zio); 2558 zio->io_stage = ZIO_STAGE_OPEN; 2559 zio->io_pipeline = ZIO_WRITE_PIPELINE; 2560 zio->io_bp_override = NULL; 2561 BP_ZERO(bp); 2562 ddt_exit(ddt); 2563 return (ZIO_PIPELINE_CONTINUE); 2564 } 2565 2566 dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2567 zio->io_orig_size, &czp, NULL, NULL, 2568 NULL, zio_ddt_ditto_write_done, dde, zio->io_priority, 2569 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2570 2571 zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); 2572 dde->dde_lead_zio[DDT_PHYS_DITTO] = dio; 2573 } 2574 2575 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 2576 if (ddp->ddp_phys_birth != 0) 2577 ddt_bp_fill(ddp, bp, txg); 2578 if (dde->dde_lead_zio[p] != NULL) 2579 zio_add_child(zio, dde->dde_lead_zio[p]); 2580 else 2581 ddt_phys_addref(ddp); 2582 } else if (zio->io_bp_override) { 2583 ASSERT(bp->blk_birth == txg); 2584 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 2585 ddt_phys_fill(ddp, bp); 2586 ddt_phys_addref(ddp); 2587 } else { 2588 cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, 2589 zio->io_orig_size, zp, 2590 zio_ddt_child_write_ready, NULL, NULL, 2591 zio_ddt_child_write_done, dde, zio->io_priority, 2592 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 2593 2594 zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL); 2595 dde->dde_lead_zio[p] = cio; 2596 } 2597 2598 ddt_exit(ddt); 2599 2600 if (cio) 2601 zio_nowait(cio); 2602 if (dio) 2603 zio_nowait(dio); 2604 2605 return (ZIO_PIPELINE_CONTINUE); 2606 } 2607 2608 ddt_entry_t *freedde; /* for debugging */ 2609 2610 static int 2611 zio_ddt_free(zio_t *zio) 2612 { 2613 spa_t *spa = zio->io_spa; 2614 blkptr_t *bp = zio->io_bp; 2615 ddt_t *ddt = ddt_select(spa, bp); 2616 ddt_entry_t *dde; 2617 ddt_phys_t *ddp; 2618 2619 ASSERT(BP_GET_DEDUP(bp)); 2620 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2621 2622 ddt_enter(ddt); 2623 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 2624 ddp = ddt_phys_select(dde, bp); 2625 ddt_phys_decref(ddp); 2626 ddt_exit(ddt); 2627 2628 return (ZIO_PIPELINE_CONTINUE); 2629 } 2630 2631 /* 2632 * ========================================================================== 2633 * Allocate and free blocks 2634 * ========================================================================== 2635 */ 2636 2637 static zio_t * 2638 zio_io_to_allocate(spa_t *spa) 2639 { 2640 zio_t *zio; 2641 2642 ASSERT(MUTEX_HELD(&spa->spa_alloc_lock)); 2643 2644 zio = avl_first(&spa->spa_alloc_tree); 2645 if (zio == NULL) 2646 return (NULL); 2647 2648 ASSERT(IO_IS_ALLOCATING(zio)); 2649 2650 /* 2651 * Try to place a reservation for this zio. If we're unable to 2652 * reserve then we throttle. 2653 */ 2654 if (!metaslab_class_throttle_reserve(spa_normal_class(spa), 2655 zio->io_prop.zp_copies, zio, 0)) { 2656 return (NULL); 2657 } 2658 2659 avl_remove(&spa->spa_alloc_tree, zio); 2660 ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE); 2661 2662 return (zio); 2663 } 2664 2665 static int 2666 zio_dva_throttle(zio_t *zio) 2667 { 2668 spa_t *spa = zio->io_spa; 2669 zio_t *nio; 2670 2671 if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE || 2672 !spa_normal_class(zio->io_spa)->mc_alloc_throttle_enabled || 2673 zio->io_child_type == ZIO_CHILD_GANG || 2674 zio->io_flags & ZIO_FLAG_NODATA) { 2675 return (ZIO_PIPELINE_CONTINUE); 2676 } 2677 2678 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2679 2680 ASSERT3U(zio->io_queued_timestamp, >, 0); 2681 ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE); 2682 2683 mutex_enter(&spa->spa_alloc_lock); 2684 2685 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2686 avl_add(&spa->spa_alloc_tree, zio); 2687 2688 nio = zio_io_to_allocate(zio->io_spa); 2689 mutex_exit(&spa->spa_alloc_lock); 2690 2691 if (nio == zio) 2692 return (ZIO_PIPELINE_CONTINUE); 2693 2694 if (nio != NULL) { 2695 ASSERT3U(nio->io_queued_timestamp, <=, 2696 zio->io_queued_timestamp); 2697 ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE); 2698 /* 2699 * We are passing control to a new zio so make sure that 2700 * it is processed by a different thread. We do this to 2701 * avoid stack overflows that can occur when parents are 2702 * throttled and children are making progress. We allow 2703 * it to go to the head of the taskq since it's already 2704 * been waiting. 2705 */ 2706 zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE); 2707 } 2708 return (ZIO_PIPELINE_STOP); 2709 } 2710 2711 void 2712 zio_allocate_dispatch(spa_t *spa) 2713 { 2714 zio_t *zio; 2715 2716 mutex_enter(&spa->spa_alloc_lock); 2717 zio = zio_io_to_allocate(spa); 2718 mutex_exit(&spa->spa_alloc_lock); 2719 if (zio == NULL) 2720 return; 2721 2722 ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE); 2723 ASSERT0(zio->io_error); 2724 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE); 2725 } 2726 2727 static int 2728 zio_dva_allocate(zio_t *zio) 2729 { 2730 spa_t *spa = zio->io_spa; 2731 metaslab_class_t *mc = spa_normal_class(spa); 2732 blkptr_t *bp = zio->io_bp; 2733 int error; 2734 int flags = 0; 2735 2736 if (zio->io_gang_leader == NULL) { 2737 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2738 zio->io_gang_leader = zio; 2739 } 2740 2741 ASSERT(BP_IS_HOLE(bp)); 2742 ASSERT0(BP_GET_NDVAS(bp)); 2743 ASSERT3U(zio->io_prop.zp_copies, >, 0); 2744 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 2745 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 2746 2747 if (zio->io_flags & ZIO_FLAG_NODATA) { 2748 flags |= METASLAB_DONT_THROTTLE; 2749 } 2750 if (zio->io_flags & ZIO_FLAG_GANG_CHILD) { 2751 flags |= METASLAB_GANG_CHILD; 2752 } 2753 if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) { 2754 flags |= METASLAB_ASYNC_ALLOC; 2755 } 2756 2757 error = metaslab_alloc(spa, mc, zio->io_size, bp, 2758 zio->io_prop.zp_copies, zio->io_txg, NULL, flags, 2759 &zio->io_alloc_list, zio); 2760 2761 if (error != 0) { 2762 spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, " 2763 "size %llu, error %d", spa_name(spa), zio, zio->io_size, 2764 error); 2765 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) 2766 return (zio_write_gang_block(zio)); 2767 zio->io_error = error; 2768 } 2769 2770 return (ZIO_PIPELINE_CONTINUE); 2771 } 2772 2773 static int 2774 zio_dva_free(zio_t *zio) 2775 { 2776 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 2777 2778 return (ZIO_PIPELINE_CONTINUE); 2779 } 2780 2781 static int 2782 zio_dva_claim(zio_t *zio) 2783 { 2784 int error; 2785 2786 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 2787 if (error) 2788 zio->io_error = error; 2789 2790 return (ZIO_PIPELINE_CONTINUE); 2791 } 2792 2793 /* 2794 * Undo an allocation. This is used by zio_done() when an I/O fails 2795 * and we want to give back the block we just allocated. 2796 * This handles both normal blocks and gang blocks. 2797 */ 2798 static void 2799 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 2800 { 2801 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 2802 ASSERT(zio->io_bp_override == NULL); 2803 2804 if (!BP_IS_HOLE(bp)) 2805 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 2806 2807 if (gn != NULL) { 2808 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2809 zio_dva_unallocate(zio, gn->gn_child[g], 2810 &gn->gn_gbh->zg_blkptr[g]); 2811 } 2812 } 2813 } 2814 2815 /* 2816 * Try to allocate an intent log block. Return 0 on success, errno on failure. 2817 */ 2818 int 2819 zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp, 2820 uint64_t size, boolean_t use_slog) 2821 { 2822 int error = 1; 2823 zio_alloc_list_t io_alloc_list; 2824 2825 ASSERT(txg > spa_syncing_txg(spa)); 2826 2827 metaslab_trace_init(&io_alloc_list); 2828 2829 if (use_slog) { 2830 error = metaslab_alloc(spa, spa_log_class(spa), size, 2831 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID, 2832 &io_alloc_list, NULL); 2833 } 2834 2835 if (error) { 2836 error = metaslab_alloc(spa, spa_normal_class(spa), size, 2837 new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID, 2838 &io_alloc_list, NULL); 2839 } 2840 metaslab_trace_fini(&io_alloc_list); 2841 2842 if (error == 0) { 2843 BP_SET_LSIZE(new_bp, size); 2844 BP_SET_PSIZE(new_bp, size); 2845 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 2846 BP_SET_CHECKSUM(new_bp, 2847 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 2848 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 2849 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 2850 BP_SET_LEVEL(new_bp, 0); 2851 BP_SET_DEDUP(new_bp, 0); 2852 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 2853 } 2854 2855 return (error); 2856 } 2857 2858 /* 2859 * Free an intent log block. 2860 */ 2861 void 2862 zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) 2863 { 2864 ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG); 2865 ASSERT(!BP_IS_GANG(bp)); 2866 2867 zio_free(spa, txg, bp); 2868 } 2869 2870 /* 2871 * ========================================================================== 2872 * Read and write to physical devices 2873 * ========================================================================== 2874 */ 2875 2876 2877 /* 2878 * Issue an I/O to the underlying vdev. Typically the issue pipeline 2879 * stops after this stage and will resume upon I/O completion. 2880 * However, there are instances where the vdev layer may need to 2881 * continue the pipeline when an I/O was not issued. Since the I/O 2882 * that was sent to the vdev layer might be different than the one 2883 * currently active in the pipeline (see vdev_queue_io()), we explicitly 2884 * force the underlying vdev layers to call either zio_execute() or 2885 * zio_interrupt() to ensure that the pipeline continues with the correct I/O. 2886 */ 2887 static int 2888 zio_vdev_io_start(zio_t *zio) 2889 { 2890 vdev_t *vd = zio->io_vd; 2891 uint64_t align; 2892 spa_t *spa = zio->io_spa; 2893 2894 ASSERT(zio->io_error == 0); 2895 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 2896 2897 if (vd == NULL) { 2898 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 2899 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 2900 2901 /* 2902 * The mirror_ops handle multiple DVAs in a single BP. 2903 */ 2904 vdev_mirror_ops.vdev_op_io_start(zio); 2905 return (ZIO_PIPELINE_STOP); 2906 } 2907 2908 ASSERT3P(zio->io_logical, !=, zio); 2909 2910 /* 2911 * We keep track of time-sensitive I/Os so that the scan thread 2912 * can quickly react to certain workloads. In particular, we care 2913 * about non-scrubbing, top-level reads and writes with the following 2914 * characteristics: 2915 * - synchronous writes of user data to non-slog devices 2916 * - any reads of user data 2917 * When these conditions are met, adjust the timestamp of spa_last_io 2918 * which allows the scan thread to adjust its workload accordingly. 2919 */ 2920 if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL && 2921 vd == vd->vdev_top && !vd->vdev_islog && 2922 zio->io_bookmark.zb_objset != DMU_META_OBJSET && 2923 zio->io_txg != spa_syncing_txg(spa)) { 2924 uint64_t old = spa->spa_last_io; 2925 uint64_t new = ddi_get_lbolt64(); 2926 if (old != new) 2927 (void) atomic_cas_64(&spa->spa_last_io, old, new); 2928 } 2929 2930 align = 1ULL << vd->vdev_top->vdev_ashift; 2931 2932 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 2933 P2PHASE(zio->io_size, align) != 0) { 2934 /* Transform logical writes to be a full physical block size. */ 2935 uint64_t asize = P2ROUNDUP(zio->io_size, align); 2936 char *abuf = zio_buf_alloc(asize); 2937 ASSERT(vd == vd->vdev_top); 2938 if (zio->io_type == ZIO_TYPE_WRITE) { 2939 bcopy(zio->io_data, abuf, zio->io_size); 2940 bzero(abuf + zio->io_size, asize - zio->io_size); 2941 } 2942 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 2943 } 2944 2945 /* 2946 * If this is not a physical io, make sure that it is properly aligned 2947 * before proceeding. 2948 */ 2949 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) { 2950 ASSERT0(P2PHASE(zio->io_offset, align)); 2951 ASSERT0(P2PHASE(zio->io_size, align)); 2952 } else { 2953 /* 2954 * For physical writes, we allow 512b aligned writes and assume 2955 * the device will perform a read-modify-write as necessary. 2956 */ 2957 ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE)); 2958 ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE)); 2959 } 2960 2961 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 2962 2963 /* 2964 * If this is a repair I/O, and there's no self-healing involved -- 2965 * that is, we're just resilvering what we expect to resilver -- 2966 * then don't do the I/O unless zio's txg is actually in vd's DTL. 2967 * This prevents spurious resilvering with nested replication. 2968 * For example, given a mirror of mirrors, (A+B)+(C+D), if only 2969 * A is out of date, we'll read from C+D, then use the data to 2970 * resilver A+B -- but we don't actually want to resilver B, just A. 2971 * The top-level mirror has no way to know this, so instead we just 2972 * discard unnecessary repairs as we work our way down the vdev tree. 2973 * The same logic applies to any form of nested replication: 2974 * ditto + mirror, RAID-Z + replacing, etc. This covers them all. 2975 */ 2976 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 2977 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 2978 zio->io_txg != 0 && /* not a delegated i/o */ 2979 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 2980 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 2981 zio_vdev_io_bypass(zio); 2982 return (ZIO_PIPELINE_CONTINUE); 2983 } 2984 2985 if (vd->vdev_ops->vdev_op_leaf && 2986 (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { 2987 2988 if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio)) 2989 return (ZIO_PIPELINE_CONTINUE); 2990 2991 if ((zio = vdev_queue_io(zio)) == NULL) 2992 return (ZIO_PIPELINE_STOP); 2993 2994 if (!vdev_accessible(vd, zio)) { 2995 zio->io_error = SET_ERROR(ENXIO); 2996 zio_interrupt(zio); 2997 return (ZIO_PIPELINE_STOP); 2998 } 2999 } 3000 3001 vd->vdev_ops->vdev_op_io_start(zio); 3002 return (ZIO_PIPELINE_STOP); 3003 } 3004 3005 static int 3006 zio_vdev_io_done(zio_t *zio) 3007 { 3008 vdev_t *vd = zio->io_vd; 3009 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 3010 boolean_t unexpected_error = B_FALSE; 3011 3012 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 3013 return (ZIO_PIPELINE_STOP); 3014 3015 ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); 3016 3017 if (vd != NULL && vd->vdev_ops->vdev_op_leaf) { 3018 3019 vdev_queue_io_done(zio); 3020 3021 if (zio->io_type == ZIO_TYPE_WRITE) 3022 vdev_cache_write(zio); 3023 3024 if (zio_injection_enabled && zio->io_error == 0) 3025 zio->io_error = zio_handle_device_injection(vd, 3026 zio, EIO); 3027 3028 if (zio_injection_enabled && zio->io_error == 0) 3029 zio->io_error = zio_handle_label_injection(zio, EIO); 3030 3031 if (zio->io_error) { 3032 if (!vdev_accessible(vd, zio)) { 3033 zio->io_error = SET_ERROR(ENXIO); 3034 } else { 3035 unexpected_error = B_TRUE; 3036 } 3037 } 3038 } 3039 3040 ops->vdev_op_io_done(zio); 3041 3042 if (unexpected_error) 3043 VERIFY(vdev_probe(vd, zio) == NULL); 3044 3045 return (ZIO_PIPELINE_CONTINUE); 3046 } 3047 3048 /* 3049 * For non-raidz ZIOs, we can just copy aside the bad data read from the 3050 * disk, and use that to finish the checksum ereport later. 3051 */ 3052 static void 3053 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 3054 const void *good_buf) 3055 { 3056 /* no processing needed */ 3057 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 3058 } 3059 3060 /*ARGSUSED*/ 3061 void 3062 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) 3063 { 3064 void *buf = zio_buf_alloc(zio->io_size); 3065 3066 bcopy(zio->io_data, buf, zio->io_size); 3067 3068 zcr->zcr_cbinfo = zio->io_size; 3069 zcr->zcr_cbdata = buf; 3070 zcr->zcr_finish = zio_vsd_default_cksum_finish; 3071 zcr->zcr_free = zio_buf_free; 3072 } 3073 3074 static int 3075 zio_vdev_io_assess(zio_t *zio) 3076 { 3077 vdev_t *vd = zio->io_vd; 3078 3079 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) 3080 return (ZIO_PIPELINE_STOP); 3081 3082 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 3083 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 3084 3085 if (zio->io_vsd != NULL) { 3086 zio->io_vsd_ops->vsd_free(zio); 3087 zio->io_vsd = NULL; 3088 } 3089 3090 if (zio_injection_enabled && zio->io_error == 0) 3091 zio->io_error = zio_handle_fault_injection(zio, EIO); 3092 3093 /* 3094 * If the I/O failed, determine whether we should attempt to retry it. 3095 * 3096 * On retry, we cut in line in the issue queue, since we don't want 3097 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 3098 */ 3099 if (zio->io_error && vd == NULL && 3100 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 3101 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 3102 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 3103 zio->io_error = 0; 3104 zio->io_flags |= ZIO_FLAG_IO_RETRY | 3105 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE; 3106 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 3107 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 3108 zio_requeue_io_start_cut_in_line); 3109 return (ZIO_PIPELINE_STOP); 3110 } 3111 3112 /* 3113 * If we got an error on a leaf device, convert it to ENXIO 3114 * if the device is not accessible at all. 3115 */ 3116 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 3117 !vdev_accessible(vd, zio)) 3118 zio->io_error = SET_ERROR(ENXIO); 3119 3120 /* 3121 * If we can't write to an interior vdev (mirror or RAID-Z), 3122 * set vdev_cant_write so that we stop trying to allocate from it. 3123 */ 3124 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 3125 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 3126 vd->vdev_cant_write = B_TRUE; 3127 } 3128 3129 /* 3130 * If a cache flush returns ENOTSUP or ENOTTY, we know that no future 3131 * attempts will ever succeed. In this case we set a persistent bit so 3132 * that we don't bother with it in the future. 3133 */ 3134 if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) && 3135 zio->io_type == ZIO_TYPE_IOCTL && 3136 zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL) 3137 vd->vdev_nowritecache = B_TRUE; 3138 3139 if (zio->io_error) 3140 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3141 3142 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 3143 zio->io_physdone != NULL) { 3144 ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED)); 3145 ASSERT(zio->io_child_type == ZIO_CHILD_VDEV); 3146 zio->io_physdone(zio->io_logical); 3147 } 3148 3149 return (ZIO_PIPELINE_CONTINUE); 3150 } 3151 3152 void 3153 zio_vdev_io_reissue(zio_t *zio) 3154 { 3155 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 3156 ASSERT(zio->io_error == 0); 3157 3158 zio->io_stage >>= 1; 3159 } 3160 3161 void 3162 zio_vdev_io_redone(zio_t *zio) 3163 { 3164 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 3165 3166 zio->io_stage >>= 1; 3167 } 3168 3169 void 3170 zio_vdev_io_bypass(zio_t *zio) 3171 { 3172 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 3173 ASSERT(zio->io_error == 0); 3174 3175 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 3176 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 3177 } 3178 3179 /* 3180 * ========================================================================== 3181 * Generate and verify checksums 3182 * ========================================================================== 3183 */ 3184 static int 3185 zio_checksum_generate(zio_t *zio) 3186 { 3187 blkptr_t *bp = zio->io_bp; 3188 enum zio_checksum checksum; 3189 3190 if (bp == NULL) { 3191 /* 3192 * This is zio_write_phys(). 3193 * We're either generating a label checksum, or none at all. 3194 */ 3195 checksum = zio->io_prop.zp_checksum; 3196 3197 if (checksum == ZIO_CHECKSUM_OFF) 3198 return (ZIO_PIPELINE_CONTINUE); 3199 3200 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 3201 } else { 3202 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 3203 ASSERT(!IO_IS_ALLOCATING(zio)); 3204 checksum = ZIO_CHECKSUM_GANG_HEADER; 3205 } else { 3206 checksum = BP_GET_CHECKSUM(bp); 3207 } 3208 } 3209 3210 zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size); 3211 3212 return (ZIO_PIPELINE_CONTINUE); 3213 } 3214 3215 static int 3216 zio_checksum_verify(zio_t *zio) 3217 { 3218 zio_bad_cksum_t info; 3219 blkptr_t *bp = zio->io_bp; 3220 int error; 3221 3222 ASSERT(zio->io_vd != NULL); 3223 3224 if (bp == NULL) { 3225 /* 3226 * This is zio_read_phys(). 3227 * We're either verifying a label checksum, or nothing at all. 3228 */ 3229 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 3230 return (ZIO_PIPELINE_CONTINUE); 3231 3232 ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); 3233 } 3234 3235 if ((error = zio_checksum_error(zio, &info)) != 0) { 3236 zio->io_error = error; 3237 if (error == ECKSUM && 3238 !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 3239 zfs_ereport_start_checksum(zio->io_spa, 3240 zio->io_vd, zio, zio->io_offset, 3241 zio->io_size, NULL, &info); 3242 } 3243 } 3244 3245 return (ZIO_PIPELINE_CONTINUE); 3246 } 3247 3248 /* 3249 * Called by RAID-Z to ensure we don't compute the checksum twice. 3250 */ 3251 void 3252 zio_checksum_verified(zio_t *zio) 3253 { 3254 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 3255 } 3256 3257 /* 3258 * ========================================================================== 3259 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 3260 * An error of 0 indicates success. ENXIO indicates whole-device failure, 3261 * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO 3262 * indicate errors that are specific to one I/O, and most likely permanent. 3263 * Any other error is presumed to be worse because we weren't expecting it. 3264 * ========================================================================== 3265 */ 3266 int 3267 zio_worst_error(int e1, int e2) 3268 { 3269 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 3270 int r1, r2; 3271 3272 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 3273 if (e1 == zio_error_rank[r1]) 3274 break; 3275 3276 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 3277 if (e2 == zio_error_rank[r2]) 3278 break; 3279 3280 return (r1 > r2 ? e1 : e2); 3281 } 3282 3283 /* 3284 * ========================================================================== 3285 * I/O completion 3286 * ========================================================================== 3287 */ 3288 static int 3289 zio_ready(zio_t *zio) 3290 { 3291 blkptr_t *bp = zio->io_bp; 3292 zio_t *pio, *pio_next; 3293 zio_link_t *zl = NULL; 3294 3295 if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || 3296 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) 3297 return (ZIO_PIPELINE_STOP); 3298 3299 if (zio->io_ready) { 3300 ASSERT(IO_IS_ALLOCATING(zio)); 3301 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 3302 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 3303 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 3304 3305 zio->io_ready(zio); 3306 } 3307 3308 if (bp != NULL && bp != &zio->io_bp_copy) 3309 zio->io_bp_copy = *bp; 3310 3311 if (zio->io_error != 0) { 3312 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3313 3314 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 3315 ASSERT(IO_IS_ALLOCATING(zio)); 3316 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 3317 /* 3318 * We were unable to allocate anything, unreserve and 3319 * issue the next I/O to allocate. 3320 */ 3321 metaslab_class_throttle_unreserve( 3322 spa_normal_class(zio->io_spa), 3323 zio->io_prop.zp_copies, zio); 3324 zio_allocate_dispatch(zio->io_spa); 3325 } 3326 } 3327 3328 mutex_enter(&zio->io_lock); 3329 zio->io_state[ZIO_WAIT_READY] = 1; 3330 pio = zio_walk_parents(zio, &zl); 3331 mutex_exit(&zio->io_lock); 3332 3333 /* 3334 * As we notify zio's parents, new parents could be added. 3335 * New parents go to the head of zio's io_parent_list, however, 3336 * so we will (correctly) not notify them. The remainder of zio's 3337 * io_parent_list, from 'pio_next' onward, cannot change because 3338 * all parents must wait for us to be done before they can be done. 3339 */ 3340 for (; pio != NULL; pio = pio_next) { 3341 pio_next = zio_walk_parents(zio, &zl); 3342 zio_notify_parent(pio, zio, ZIO_WAIT_READY); 3343 } 3344 3345 if (zio->io_flags & ZIO_FLAG_NODATA) { 3346 if (BP_IS_GANG(bp)) { 3347 zio->io_flags &= ~ZIO_FLAG_NODATA; 3348 } else { 3349 ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE); 3350 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 3351 } 3352 } 3353 3354 if (zio_injection_enabled && 3355 zio->io_spa->spa_syncing_txg == zio->io_txg) 3356 zio_handle_ignored_writes(zio); 3357 3358 return (ZIO_PIPELINE_CONTINUE); 3359 } 3360 3361 /* 3362 * Update the allocation throttle accounting. 3363 */ 3364 static void 3365 zio_dva_throttle_done(zio_t *zio) 3366 { 3367 zio_t *lio = zio->io_logical; 3368 zio_t *pio = zio_unique_parent(zio); 3369 vdev_t *vd = zio->io_vd; 3370 int flags = METASLAB_ASYNC_ALLOC; 3371 3372 ASSERT3P(zio->io_bp, !=, NULL); 3373 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 3374 ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE); 3375 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 3376 ASSERT(vd != NULL); 3377 ASSERT3P(vd, ==, vd->vdev_top); 3378 ASSERT(!(zio->io_flags & (ZIO_FLAG_IO_REPAIR | ZIO_FLAG_IO_RETRY))); 3379 ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING); 3380 ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE)); 3381 ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA)); 3382 3383 /* 3384 * Parents of gang children can have two flavors -- ones that 3385 * allocated the gang header (will have ZIO_FLAG_IO_REWRITE set) 3386 * and ones that allocated the constituent blocks. The allocation 3387 * throttle needs to know the allocating parent zio so we must find 3388 * it here. 3389 */ 3390 if (pio->io_child_type == ZIO_CHILD_GANG) { 3391 /* 3392 * If our parent is a rewrite gang child then our grandparent 3393 * would have been the one that performed the allocation. 3394 */ 3395 if (pio->io_flags & ZIO_FLAG_IO_REWRITE) 3396 pio = zio_unique_parent(pio); 3397 flags |= METASLAB_GANG_CHILD; 3398 } 3399 3400 ASSERT(IO_IS_ALLOCATING(pio)); 3401 ASSERT3P(zio, !=, zio->io_logical); 3402 ASSERT(zio->io_logical != NULL); 3403 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR)); 3404 ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE); 3405 3406 mutex_enter(&pio->io_lock); 3407 metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags); 3408 mutex_exit(&pio->io_lock); 3409 3410 metaslab_class_throttle_unreserve(spa_normal_class(zio->io_spa), 3411 1, pio); 3412 3413 /* 3414 * Call into the pipeline to see if there is more work that 3415 * needs to be done. If there is work to be done it will be 3416 * dispatched to another taskq thread. 3417 */ 3418 zio_allocate_dispatch(zio->io_spa); 3419 } 3420 3421 static int 3422 zio_done(zio_t *zio) 3423 { 3424 spa_t *spa = zio->io_spa; 3425 zio_t *lio = zio->io_logical; 3426 blkptr_t *bp = zio->io_bp; 3427 vdev_t *vd = zio->io_vd; 3428 uint64_t psize = zio->io_size; 3429 zio_t *pio, *pio_next; 3430 metaslab_class_t *mc = spa_normal_class(spa); 3431 zio_link_t *zl = NULL; 3432 3433 /* 3434 * If our children haven't all completed, 3435 * wait for them and then repeat this pipeline stage. 3436 */ 3437 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || 3438 zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || 3439 zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || 3440 zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) 3441 return (ZIO_PIPELINE_STOP); 3442 3443 /* 3444 * If the allocation throttle is enabled, then update the accounting. 3445 * We only track child I/Os that are part of an allocating async 3446 * write. We must do this since the allocation is performed 3447 * by the logical I/O but the actual write is done by child I/Os. 3448 */ 3449 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING && 3450 zio->io_child_type == ZIO_CHILD_VDEV) { 3451 ASSERT(mc->mc_alloc_throttle_enabled); 3452 zio_dva_throttle_done(zio); 3453 } 3454 3455 /* 3456 * If the allocation throttle is enabled, verify that 3457 * we have decremented the refcounts for every I/O that was throttled. 3458 */ 3459 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 3460 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 3461 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 3462 ASSERT(bp != NULL); 3463 metaslab_group_alloc_verify(spa, zio->io_bp, zio); 3464 VERIFY(refcount_not_held(&mc->mc_alloc_slots, zio)); 3465 } 3466 3467 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 3468 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 3469 ASSERT(zio->io_children[c][w] == 0); 3470 3471 if (bp != NULL && !BP_IS_EMBEDDED(bp)) { 3472 ASSERT(bp->blk_pad[0] == 0); 3473 ASSERT(bp->blk_pad[1] == 0); 3474 ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 || 3475 (bp == zio_unique_parent(zio)->io_bp)); 3476 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) && 3477 zio->io_bp_override == NULL && 3478 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 3479 ASSERT(!BP_SHOULD_BYTESWAP(bp)); 3480 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp)); 3481 ASSERT(BP_COUNT_GANG(bp) == 0 || 3482 (BP_COUNT_GANG(bp) == BP_GET_NDVAS(bp))); 3483 } 3484 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 3485 VERIFY(BP_EQUAL(bp, &zio->io_bp_orig)); 3486 } 3487 3488 /* 3489 * If there were child vdev/gang/ddt errors, they apply to us now. 3490 */ 3491 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 3492 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 3493 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 3494 3495 /* 3496 * If the I/O on the transformed data was successful, generate any 3497 * checksum reports now while we still have the transformed data. 3498 */ 3499 if (zio->io_error == 0) { 3500 while (zio->io_cksum_report != NULL) { 3501 zio_cksum_report_t *zcr = zio->io_cksum_report; 3502 uint64_t align = zcr->zcr_align; 3503 uint64_t asize = P2ROUNDUP(psize, align); 3504 char *abuf = zio->io_data; 3505 3506 if (asize != psize) { 3507 abuf = zio_buf_alloc(asize); 3508 bcopy(zio->io_data, abuf, psize); 3509 bzero(abuf + psize, asize - psize); 3510 } 3511 3512 zio->io_cksum_report = zcr->zcr_next; 3513 zcr->zcr_next = NULL; 3514 zcr->zcr_finish(zcr, abuf); 3515 zfs_ereport_free_checksum(zcr); 3516 3517 if (asize != psize) 3518 zio_buf_free(abuf, asize); 3519 } 3520 } 3521 3522 zio_pop_transforms(zio); /* note: may set zio->io_error */ 3523 3524 vdev_stat_update(zio, psize); 3525 3526 if (zio->io_error) { 3527 /* 3528 * If this I/O is attached to a particular vdev, 3529 * generate an error message describing the I/O failure 3530 * at the block level. We ignore these errors if the 3531 * device is currently unavailable. 3532 */ 3533 if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd)) 3534 zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0); 3535 3536 if ((zio->io_error == EIO || !(zio->io_flags & 3537 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 3538 zio == lio) { 3539 /* 3540 * For logical I/O requests, tell the SPA to log the 3541 * error and generate a logical data ereport. 3542 */ 3543 spa_log_error(spa, zio); 3544 zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio, 3545 0, 0); 3546 } 3547 } 3548 3549 if (zio->io_error && zio == lio) { 3550 /* 3551 * Determine whether zio should be reexecuted. This will 3552 * propagate all the way to the root via zio_notify_parent(). 3553 */ 3554 ASSERT(vd == NULL && bp != NULL); 3555 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3556 3557 if (IO_IS_ALLOCATING(zio) && 3558 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 3559 if (zio->io_error != ENOSPC) 3560 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 3561 else 3562 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3563 } 3564 3565 if ((zio->io_type == ZIO_TYPE_READ || 3566 zio->io_type == ZIO_TYPE_FREE) && 3567 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 3568 zio->io_error == ENXIO && 3569 spa_load_state(spa) == SPA_LOAD_NONE && 3570 spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE) 3571 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3572 3573 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 3574 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 3575 3576 /* 3577 * Here is a possibly good place to attempt to do 3578 * either combinatorial reconstruction or error correction 3579 * based on checksums. It also might be a good place 3580 * to send out preliminary ereports before we suspend 3581 * processing. 3582 */ 3583 } 3584 3585 /* 3586 * If there were logical child errors, they apply to us now. 3587 * We defer this until now to avoid conflating logical child 3588 * errors with errors that happened to the zio itself when 3589 * updating vdev stats and reporting FMA events above. 3590 */ 3591 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 3592 3593 if ((zio->io_error || zio->io_reexecute) && 3594 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 3595 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 3596 zio_dva_unallocate(zio, zio->io_gang_tree, bp); 3597 3598 zio_gang_tree_free(&zio->io_gang_tree); 3599 3600 /* 3601 * Godfather I/Os should never suspend. 3602 */ 3603 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 3604 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 3605 zio->io_reexecute = 0; 3606 3607 if (zio->io_reexecute) { 3608 /* 3609 * This is a logical I/O that wants to reexecute. 3610 * 3611 * Reexecute is top-down. When an i/o fails, if it's not 3612 * the root, it simply notifies its parent and sticks around. 3613 * The parent, seeing that it still has children in zio_done(), 3614 * does the same. This percolates all the way up to the root. 3615 * The root i/o will reexecute or suspend the entire tree. 3616 * 3617 * This approach ensures that zio_reexecute() honors 3618 * all the original i/o dependency relationships, e.g. 3619 * parents not executing until children are ready. 3620 */ 3621 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3622 3623 zio->io_gang_leader = NULL; 3624 3625 mutex_enter(&zio->io_lock); 3626 zio->io_state[ZIO_WAIT_DONE] = 1; 3627 mutex_exit(&zio->io_lock); 3628 3629 /* 3630 * "The Godfather" I/O monitors its children but is 3631 * not a true parent to them. It will track them through 3632 * the pipeline but severs its ties whenever they get into 3633 * trouble (e.g. suspended). This allows "The Godfather" 3634 * I/O to return status without blocking. 3635 */ 3636 zl = NULL; 3637 for (pio = zio_walk_parents(zio, &zl); pio != NULL; 3638 pio = pio_next) { 3639 zio_link_t *remove_zl = zl; 3640 pio_next = zio_walk_parents(zio, &zl); 3641 3642 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 3643 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 3644 zio_remove_child(pio, zio, remove_zl); 3645 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3646 } 3647 } 3648 3649 if ((pio = zio_unique_parent(zio)) != NULL) { 3650 /* 3651 * We're not a root i/o, so there's nothing to do 3652 * but notify our parent. Don't propagate errors 3653 * upward since we haven't permanently failed yet. 3654 */ 3655 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 3656 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 3657 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3658 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 3659 /* 3660 * We'd fail again if we reexecuted now, so suspend 3661 * until conditions improve (e.g. device comes online). 3662 */ 3663 zio_suspend(spa, zio); 3664 } else { 3665 /* 3666 * Reexecution is potentially a huge amount of work. 3667 * Hand it off to the otherwise-unused claim taskq. 3668 */ 3669 ASSERT(zio->io_tqent.tqent_next == NULL); 3670 spa_taskq_dispatch_ent(spa, ZIO_TYPE_CLAIM, 3671 ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio, 3672 0, &zio->io_tqent); 3673 } 3674 return (ZIO_PIPELINE_STOP); 3675 } 3676 3677 ASSERT(zio->io_child_count == 0); 3678 ASSERT(zio->io_reexecute == 0); 3679 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 3680 3681 /* 3682 * Report any checksum errors, since the I/O is complete. 3683 */ 3684 while (zio->io_cksum_report != NULL) { 3685 zio_cksum_report_t *zcr = zio->io_cksum_report; 3686 zio->io_cksum_report = zcr->zcr_next; 3687 zcr->zcr_next = NULL; 3688 zcr->zcr_finish(zcr, NULL); 3689 zfs_ereport_free_checksum(zcr); 3690 } 3691 3692 /* 3693 * It is the responsibility of the done callback to ensure that this 3694 * particular zio is no longer discoverable for adoption, and as 3695 * such, cannot acquire any new parents. 3696 */ 3697 if (zio->io_done) 3698 zio->io_done(zio); 3699 3700 mutex_enter(&zio->io_lock); 3701 zio->io_state[ZIO_WAIT_DONE] = 1; 3702 mutex_exit(&zio->io_lock); 3703 3704 zl = NULL; 3705 for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) { 3706 zio_link_t *remove_zl = zl; 3707 pio_next = zio_walk_parents(zio, &zl); 3708 zio_remove_child(pio, zio, remove_zl); 3709 zio_notify_parent(pio, zio, ZIO_WAIT_DONE); 3710 } 3711 3712 if (zio->io_waiter != NULL) { 3713 mutex_enter(&zio->io_lock); 3714 zio->io_executor = NULL; 3715 cv_broadcast(&zio->io_cv); 3716 mutex_exit(&zio->io_lock); 3717 } else { 3718 zio_destroy(zio); 3719 } 3720 3721 return (ZIO_PIPELINE_STOP); 3722 } 3723 3724 /* 3725 * ========================================================================== 3726 * I/O pipeline definition 3727 * ========================================================================== 3728 */ 3729 static zio_pipe_stage_t *zio_pipeline[] = { 3730 NULL, 3731 zio_read_bp_init, 3732 zio_write_bp_init, 3733 zio_free_bp_init, 3734 zio_issue_async, 3735 zio_write_compress, 3736 zio_checksum_generate, 3737 zio_nop_write, 3738 zio_ddt_read_start, 3739 zio_ddt_read_done, 3740 zio_ddt_write, 3741 zio_ddt_free, 3742 zio_gang_assemble, 3743 zio_gang_issue, 3744 zio_dva_throttle, 3745 zio_dva_allocate, 3746 zio_dva_free, 3747 zio_dva_claim, 3748 zio_ready, 3749 zio_vdev_io_start, 3750 zio_vdev_io_done, 3751 zio_vdev_io_assess, 3752 zio_checksum_verify, 3753 zio_done 3754 }; 3755 3756 3757 3758 3759 /* 3760 * Compare two zbookmark_phys_t's to see which we would reach first in a 3761 * pre-order traversal of the object tree. 3762 * 3763 * This is simple in every case aside from the meta-dnode object. For all other 3764 * objects, we traverse them in order (object 1 before object 2, and so on). 3765 * However, all of these objects are traversed while traversing object 0, since 3766 * the data it points to is the list of objects. Thus, we need to convert to a 3767 * canonical representation so we can compare meta-dnode bookmarks to 3768 * non-meta-dnode bookmarks. 3769 * 3770 * We do this by calculating "equivalents" for each field of the zbookmark. 3771 * zbookmarks outside of the meta-dnode use their own object and level, and 3772 * calculate the level 0 equivalent (the first L0 blkid that is contained in the 3773 * blocks this bookmark refers to) by multiplying their blkid by their span 3774 * (the number of L0 blocks contained within one block at their level). 3775 * zbookmarks inside the meta-dnode calculate their object equivalent 3776 * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use 3777 * level + 1<<31 (any value larger than a level could ever be) for their level. 3778 * This causes them to always compare before a bookmark in their object 3779 * equivalent, compare appropriately to bookmarks in other objects, and to 3780 * compare appropriately to other bookmarks in the meta-dnode. 3781 */ 3782 int 3783 zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2, 3784 const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2) 3785 { 3786 /* 3787 * These variables represent the "equivalent" values for the zbookmark, 3788 * after converting zbookmarks inside the meta dnode to their 3789 * normal-object equivalents. 3790 */ 3791 uint64_t zb1obj, zb2obj; 3792 uint64_t zb1L0, zb2L0; 3793 uint64_t zb1level, zb2level; 3794 3795 if (zb1->zb_object == zb2->zb_object && 3796 zb1->zb_level == zb2->zb_level && 3797 zb1->zb_blkid == zb2->zb_blkid) 3798 return (0); 3799 3800 /* 3801 * BP_SPANB calculates the span in blocks. 3802 */ 3803 zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level); 3804 zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level); 3805 3806 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 3807 zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 3808 zb1L0 = 0; 3809 zb1level = zb1->zb_level + COMPARE_META_LEVEL; 3810 } else { 3811 zb1obj = zb1->zb_object; 3812 zb1level = zb1->zb_level; 3813 } 3814 3815 if (zb2->zb_object == DMU_META_DNODE_OBJECT) { 3816 zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 3817 zb2L0 = 0; 3818 zb2level = zb2->zb_level + COMPARE_META_LEVEL; 3819 } else { 3820 zb2obj = zb2->zb_object; 3821 zb2level = zb2->zb_level; 3822 } 3823 3824 /* Now that we have a canonical representation, do the comparison. */ 3825 if (zb1obj != zb2obj) 3826 return (zb1obj < zb2obj ? -1 : 1); 3827 else if (zb1L0 != zb2L0) 3828 return (zb1L0 < zb2L0 ? -1 : 1); 3829 else if (zb1level != zb2level) 3830 return (zb1level > zb2level ? -1 : 1); 3831 /* 3832 * This can (theoretically) happen if the bookmarks have the same object 3833 * and level, but different blkids, if the block sizes are not the same. 3834 * There is presently no way to change the indirect block sizes 3835 */ 3836 return (0); 3837 } 3838 3839 /* 3840 * This function checks the following: given that last_block is the place that 3841 * our traversal stopped last time, does that guarantee that we've visited 3842 * every node under subtree_root? Therefore, we can't just use the raw output 3843 * of zbookmark_compare. We have to pass in a modified version of 3844 * subtree_root; by incrementing the block id, and then checking whether 3845 * last_block is before or equal to that, we can tell whether or not having 3846 * visited last_block implies that all of subtree_root's children have been 3847 * visited. 3848 */ 3849 boolean_t 3850 zbookmark_subtree_completed(const dnode_phys_t *dnp, 3851 const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block) 3852 { 3853 zbookmark_phys_t mod_zb = *subtree_root; 3854 mod_zb.zb_blkid++; 3855 ASSERT(last_block->zb_level == 0); 3856 3857 /* The objset_phys_t isn't before anything. */ 3858 if (dnp == NULL) 3859 return (B_FALSE); 3860 3861 /* 3862 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the 3863 * data block size in sectors, because that variable is only used if 3864 * the bookmark refers to a block in the meta-dnode. Since we don't 3865 * know without examining it what object it refers to, and there's no 3866 * harm in passing in this value in other cases, we always pass it in. 3867 * 3868 * We pass in 0 for the indirect block size shift because zb2 must be 3869 * level 0. The indirect block size is only used to calculate the span 3870 * of the bookmark, but since the bookmark must be level 0, the span is 3871 * always 1, so the math works out. 3872 * 3873 * If you make changes to how the zbookmark_compare code works, be sure 3874 * to make sure that this code still works afterwards. 3875 */ 3876 return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift, 3877 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb, 3878 last_block) <= 0); 3879 } 3880