1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 14 * Use is subject to license terms. 15 */ 16 17 /* 18 * Copyright (c) 2012, 2015 by Delphix. All rights reserved. 19 */ 20 21 #include <sys/zfs_context.h> 22 #include <sys/spa.h> 23 #include <sys/spa_impl.h> 24 #include <sys/dsl_pool.h> 25 #include <sys/dsl_scan.h> 26 #include <sys/vdev_impl.h> 27 #include <sys/vdev_draid.h> 28 #include <sys/zio.h> 29 #include <sys/zio_checksum.h> 30 #include <sys/abd.h> 31 #include <sys/fs/zfs.h> 32 33 /* 34 * Vdev mirror kstats 35 */ 36 static kstat_t *mirror_ksp = NULL; 37 38 typedef struct mirror_stats { 39 kstat_named_t vdev_mirror_stat_rotating_linear; 40 kstat_named_t vdev_mirror_stat_rotating_offset; 41 kstat_named_t vdev_mirror_stat_rotating_seek; 42 kstat_named_t vdev_mirror_stat_non_rotating_linear; 43 kstat_named_t vdev_mirror_stat_non_rotating_seek; 44 45 kstat_named_t vdev_mirror_stat_preferred_found; 46 kstat_named_t vdev_mirror_stat_preferred_not_found; 47 } mirror_stats_t; 48 49 static mirror_stats_t mirror_stats = { 50 /* New I/O follows directly the last I/O */ 51 { "rotating_linear", KSTAT_DATA_UINT64 }, 52 /* New I/O is within zfs_vdev_mirror_rotating_seek_offset of the last */ 53 { "rotating_offset", KSTAT_DATA_UINT64 }, 54 /* New I/O requires random seek */ 55 { "rotating_seek", KSTAT_DATA_UINT64 }, 56 /* New I/O follows directly the last I/O (nonrot) */ 57 { "non_rotating_linear", KSTAT_DATA_UINT64 }, 58 /* New I/O requires random seek (nonrot) */ 59 { "non_rotating_seek", KSTAT_DATA_UINT64 }, 60 /* Preferred child vdev found */ 61 { "preferred_found", KSTAT_DATA_UINT64 }, 62 /* Preferred child vdev not found or equal load */ 63 { "preferred_not_found", KSTAT_DATA_UINT64 }, 64 65 }; 66 67 #define MIRROR_STAT(stat) (mirror_stats.stat.value.ui64) 68 #define MIRROR_INCR(stat, val) atomic_add_64(&MIRROR_STAT(stat), val) 69 #define MIRROR_BUMP(stat) MIRROR_INCR(stat, 1) 70 71 void 72 vdev_mirror_stat_init(void) 73 { 74 mirror_ksp = kstat_create("zfs", 0, "vdev_mirror_stats", 75 "misc", KSTAT_TYPE_NAMED, 76 sizeof (mirror_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 77 if (mirror_ksp != NULL) { 78 mirror_ksp->ks_data = &mirror_stats; 79 kstat_install(mirror_ksp); 80 } 81 } 82 83 void 84 vdev_mirror_stat_fini(void) 85 { 86 if (mirror_ksp != NULL) { 87 kstat_delete(mirror_ksp); 88 mirror_ksp = NULL; 89 } 90 } 91 92 /* 93 * Virtual device vector for mirroring. 94 */ 95 typedef struct mirror_child { 96 vdev_t *mc_vd; 97 abd_t *mc_abd; 98 uint64_t mc_offset; 99 int mc_error; 100 int mc_load; 101 uint8_t mc_tried; 102 uint8_t mc_skipped; 103 uint8_t mc_speculative; 104 uint8_t mc_rebuilding; 105 } mirror_child_t; 106 107 typedef struct mirror_map { 108 int *mm_preferred; 109 int mm_preferred_cnt; 110 int mm_children; 111 boolean_t mm_resilvering; 112 boolean_t mm_rebuilding; 113 boolean_t mm_root; 114 mirror_child_t mm_child[]; 115 } mirror_map_t; 116 117 static const int vdev_mirror_shift = 21; 118 119 /* 120 * The load configuration settings below are tuned by default for 121 * the case where all devices are of the same rotational type. 122 * 123 * If there is a mixture of rotating and non-rotating media, setting 124 * zfs_vdev_mirror_non_rotating_seek_inc to 0 may well provide better results 125 * as it will direct more reads to the non-rotating vdevs which are more likely 126 * to have a higher performance. 127 */ 128 129 /* Rotating media load calculation configuration. */ 130 static int zfs_vdev_mirror_rotating_inc = 0; 131 static int zfs_vdev_mirror_rotating_seek_inc = 5; 132 static int zfs_vdev_mirror_rotating_seek_offset = 1 * 1024 * 1024; 133 134 /* Non-rotating media load calculation configuration. */ 135 static int zfs_vdev_mirror_non_rotating_inc = 0; 136 static int zfs_vdev_mirror_non_rotating_seek_inc = 1; 137 138 static inline size_t 139 vdev_mirror_map_size(int children) 140 { 141 return (offsetof(mirror_map_t, mm_child[children]) + 142 sizeof (int) * children); 143 } 144 145 static inline mirror_map_t * 146 vdev_mirror_map_alloc(int children, boolean_t resilvering, boolean_t root) 147 { 148 mirror_map_t *mm; 149 150 mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP); 151 mm->mm_children = children; 152 mm->mm_resilvering = resilvering; 153 mm->mm_root = root; 154 mm->mm_preferred = (int *)((uintptr_t)mm + 155 offsetof(mirror_map_t, mm_child[children])); 156 157 return (mm); 158 } 159 160 static void 161 vdev_mirror_map_free(zio_t *zio) 162 { 163 mirror_map_t *mm = zio->io_vsd; 164 165 kmem_free(mm, vdev_mirror_map_size(mm->mm_children)); 166 } 167 168 static const zio_vsd_ops_t vdev_mirror_vsd_ops = { 169 .vsd_free = vdev_mirror_map_free, 170 }; 171 172 static int 173 vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset) 174 { 175 uint64_t last_offset; 176 int64_t offset_diff; 177 int load; 178 179 /* All DVAs have equal weight at the root. */ 180 if (mm->mm_root) 181 return (INT_MAX); 182 183 /* 184 * We don't return INT_MAX if the device is resilvering i.e. 185 * vdev_resilver_txg != 0 as when tested performance was slightly 186 * worse overall when resilvering with compared to without. 187 */ 188 189 /* Fix zio_offset for leaf vdevs */ 190 if (vd->vdev_ops->vdev_op_leaf) 191 zio_offset += VDEV_LABEL_START_SIZE; 192 193 /* Standard load based on pending queue length. */ 194 load = vdev_queue_length(vd); 195 last_offset = vdev_queue_last_offset(vd); 196 197 if (vd->vdev_nonrot) { 198 /* Non-rotating media. */ 199 if (last_offset == zio_offset) { 200 MIRROR_BUMP(vdev_mirror_stat_non_rotating_linear); 201 return (load + zfs_vdev_mirror_non_rotating_inc); 202 } 203 204 /* 205 * Apply a seek penalty even for non-rotating devices as 206 * sequential I/O's can be aggregated into fewer operations on 207 * the device, thus avoiding unnecessary per-command overhead 208 * and boosting performance. 209 */ 210 MIRROR_BUMP(vdev_mirror_stat_non_rotating_seek); 211 return (load + zfs_vdev_mirror_non_rotating_seek_inc); 212 } 213 214 /* Rotating media I/O's which directly follow the last I/O. */ 215 if (last_offset == zio_offset) { 216 MIRROR_BUMP(vdev_mirror_stat_rotating_linear); 217 return (load + zfs_vdev_mirror_rotating_inc); 218 } 219 220 /* 221 * Apply half the seek increment to I/O's within seek offset 222 * of the last I/O issued to this vdev as they should incur less 223 * of a seek increment. 224 */ 225 offset_diff = (int64_t)(last_offset - zio_offset); 226 if (ABS(offset_diff) < zfs_vdev_mirror_rotating_seek_offset) { 227 MIRROR_BUMP(vdev_mirror_stat_rotating_offset); 228 return (load + (zfs_vdev_mirror_rotating_seek_inc / 2)); 229 } 230 231 /* Apply the full seek increment to all other I/O's. */ 232 MIRROR_BUMP(vdev_mirror_stat_rotating_seek); 233 return (load + zfs_vdev_mirror_rotating_seek_inc); 234 } 235 236 static boolean_t 237 vdev_mirror_rebuilding(vdev_t *vd) 238 { 239 if (vd->vdev_ops->vdev_op_leaf && vd->vdev_rebuild_txg) 240 return (B_TRUE); 241 242 for (int i = 0; i < vd->vdev_children; i++) { 243 if (vdev_mirror_rebuilding(vd->vdev_child[i])) { 244 return (B_TRUE); 245 } 246 } 247 248 return (B_FALSE); 249 } 250 251 /* 252 * Avoid inlining the function to keep vdev_mirror_io_start(), which 253 * is this functions only caller, as small as possible on the stack. 254 */ 255 noinline static mirror_map_t * 256 vdev_mirror_map_init(zio_t *zio) 257 { 258 mirror_map_t *mm = NULL; 259 mirror_child_t *mc; 260 vdev_t *vd = zio->io_vd; 261 int c; 262 263 if (vd == NULL) { 264 dva_t *dva = zio->io_bp->blk_dva; 265 spa_t *spa = zio->io_spa; 266 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 267 dva_t dva_copy[SPA_DVAS_PER_BP]; 268 269 /* 270 * The sequential scrub code sorts and issues all DVAs 271 * of a bp separately. Each of these IOs includes all 272 * original DVA copies so that repairs can be performed 273 * in the event of an error, but we only actually want 274 * to check the first DVA since the others will be 275 * checked by their respective sorted IOs. Only if we 276 * hit an error will we try all DVAs upon retrying. 277 * 278 * Note: This check is safe even if the user switches 279 * from a legacy scrub to a sequential one in the middle 280 * of processing, since scn_is_sorted isn't updated until 281 * all outstanding IOs from the previous scrub pass 282 * complete. 283 */ 284 if ((zio->io_flags & ZIO_FLAG_SCRUB) && 285 !(zio->io_flags & ZIO_FLAG_IO_RETRY) && 286 dsl_scan_scrubbing(spa->spa_dsl_pool) && 287 scn->scn_is_sorted) { 288 c = 1; 289 } else { 290 c = BP_GET_NDVAS(zio->io_bp); 291 } 292 293 /* 294 * If the pool cannot be written to, then infer that some 295 * DVAs might be invalid or point to vdevs that do not exist. 296 * We skip them. 297 */ 298 if (!spa_writeable(spa)) { 299 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 300 int j = 0; 301 for (int i = 0; i < c; i++) { 302 if (zfs_dva_valid(spa, &dva[i], zio->io_bp)) 303 dva_copy[j++] = dva[i]; 304 } 305 if (j == 0) { 306 zio->io_vsd = NULL; 307 zio->io_error = ENXIO; 308 return (NULL); 309 } 310 if (j < c) { 311 dva = dva_copy; 312 c = j; 313 } 314 } 315 316 mm = vdev_mirror_map_alloc(c, B_FALSE, B_TRUE); 317 for (c = 0; c < mm->mm_children; c++) { 318 mc = &mm->mm_child[c]; 319 320 mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c])); 321 mc->mc_offset = DVA_GET_OFFSET(&dva[c]); 322 if (mc->mc_vd == NULL) { 323 kmem_free(mm, vdev_mirror_map_size( 324 mm->mm_children)); 325 zio->io_vsd = NULL; 326 zio->io_error = ENXIO; 327 return (NULL); 328 } 329 } 330 } else { 331 /* 332 * If we are resilvering, then we should handle scrub reads 333 * differently; we shouldn't issue them to the resilvering 334 * device because it might not have those blocks. 335 * 336 * We are resilvering iff: 337 * 1) We are a replacing vdev (ie our name is "replacing-1" or 338 * "spare-1" or something like that), and 339 * 2) The pool is currently being resilvered. 340 * 341 * We cannot simply check vd->vdev_resilver_txg, because it's 342 * not set in this path. 343 * 344 * Nor can we just check our vdev_ops; there are cases (such as 345 * when a user types "zpool replace pool odev spare_dev" and 346 * spare_dev is in the spare list, or when a spare device is 347 * automatically used to replace a DEGRADED device) when 348 * resilvering is complete but both the original vdev and the 349 * spare vdev remain in the pool. That behavior is intentional. 350 * It helps implement the policy that a spare should be 351 * automatically removed from the pool after the user replaces 352 * the device that originally failed. 353 * 354 * If a spa load is in progress, then spa_dsl_pool may be 355 * uninitialized. But we shouldn't be resilvering during a spa 356 * load anyway. 357 */ 358 boolean_t replacing = (vd->vdev_ops == &vdev_replacing_ops || 359 vd->vdev_ops == &vdev_spare_ops) && 360 spa_load_state(vd->vdev_spa) == SPA_LOAD_NONE && 361 dsl_scan_resilvering(vd->vdev_spa->spa_dsl_pool); 362 mm = vdev_mirror_map_alloc(vd->vdev_children, replacing, 363 B_FALSE); 364 for (c = 0; c < mm->mm_children; c++) { 365 mc = &mm->mm_child[c]; 366 mc->mc_vd = vd->vdev_child[c]; 367 mc->mc_offset = zio->io_offset; 368 369 if (vdev_mirror_rebuilding(mc->mc_vd)) 370 mm->mm_rebuilding = mc->mc_rebuilding = B_TRUE; 371 } 372 } 373 374 return (mm); 375 } 376 377 static int 378 vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize, 379 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr) 380 { 381 int numerrors = 0; 382 int lasterror = 0; 383 384 if (vd->vdev_children == 0) { 385 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL; 386 return (SET_ERROR(EINVAL)); 387 } 388 389 vdev_open_children(vd, cr); 390 391 for (int c = 0; c < vd->vdev_children; c++) { 392 vdev_t *cvd = vd->vdev_child[c]; 393 394 if (cvd->vdev_open_error) { 395 lasterror = cvd->vdev_open_error; 396 numerrors++; 397 continue; 398 } 399 400 *asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1; 401 *max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1; 402 *logical_ashift = MAX(*logical_ashift, cvd->vdev_ashift); 403 } 404 for (int c = 0; c < vd->vdev_children; c++) { 405 vdev_t *cvd = vd->vdev_child[c]; 406 407 if (cvd->vdev_open_error) 408 continue; 409 *physical_ashift = vdev_best_ashift(*logical_ashift, 410 *physical_ashift, cvd->vdev_physical_ashift); 411 } 412 413 if (numerrors == vd->vdev_children) { 414 if (vdev_children_are_offline(vd)) 415 vd->vdev_stat.vs_aux = VDEV_AUX_CHILDREN_OFFLINE; 416 else 417 vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS; 418 return (lasterror); 419 } 420 421 return (0); 422 } 423 424 static void 425 vdev_mirror_close(vdev_t *vd) 426 { 427 for (int c = 0; c < vd->vdev_children; c++) 428 vdev_close(vd->vdev_child[c]); 429 } 430 431 static void 432 vdev_mirror_child_done(zio_t *zio) 433 { 434 mirror_child_t *mc = zio->io_private; 435 436 mc->mc_error = zio->io_error; 437 mc->mc_tried = 1; 438 mc->mc_skipped = 0; 439 } 440 441 /* 442 * Check the other, lower-index DVAs to see if they're on the same 443 * vdev as the child we picked. If they are, use them since they 444 * are likely to have been allocated from the primary metaslab in 445 * use at the time, and hence are more likely to have locality with 446 * single-copy data. 447 */ 448 static int 449 vdev_mirror_dva_select(zio_t *zio, int p) 450 { 451 dva_t *dva = zio->io_bp->blk_dva; 452 mirror_map_t *mm = zio->io_vsd; 453 int preferred; 454 int c; 455 456 preferred = mm->mm_preferred[p]; 457 for (p--; p >= 0; p--) { 458 c = mm->mm_preferred[p]; 459 if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred])) 460 preferred = c; 461 } 462 return (preferred); 463 } 464 465 static int 466 vdev_mirror_preferred_child_randomize(zio_t *zio) 467 { 468 mirror_map_t *mm = zio->io_vsd; 469 int p; 470 471 if (mm->mm_root) { 472 p = random_in_range(mm->mm_preferred_cnt); 473 return (vdev_mirror_dva_select(zio, p)); 474 } 475 476 /* 477 * To ensure we don't always favour the first matching vdev, 478 * which could lead to wear leveling issues on SSD's, we 479 * use the I/O offset as a pseudo random seed into the vdevs 480 * which have the lowest load. 481 */ 482 p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt; 483 return (mm->mm_preferred[p]); 484 } 485 486 static boolean_t 487 vdev_mirror_child_readable(mirror_child_t *mc) 488 { 489 vdev_t *vd = mc->mc_vd; 490 491 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops) 492 return (vdev_draid_readable(vd, mc->mc_offset)); 493 else 494 return (vdev_readable(vd)); 495 } 496 497 static boolean_t 498 vdev_mirror_child_missing(mirror_child_t *mc, uint64_t txg, uint64_t size) 499 { 500 vdev_t *vd = mc->mc_vd; 501 502 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops) 503 return (vdev_draid_missing(vd, mc->mc_offset, txg, size)); 504 else 505 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size)); 506 } 507 508 /* 509 * Try to find a vdev whose DTL doesn't contain the block we want to read 510 * preferring vdevs based on determined load. If we can't, try the read on 511 * any vdev we haven't already tried. 512 * 513 * Distributed spares are an exception to the above load rule. They are 514 * always preferred in order to detect gaps in the distributed spare which 515 * are created when another disk in the dRAID fails. In order to restore 516 * redundancy those gaps must be read to trigger the required repair IO. 517 */ 518 static int 519 vdev_mirror_child_select(zio_t *zio) 520 { 521 mirror_map_t *mm = zio->io_vsd; 522 uint64_t txg = zio->io_txg; 523 int c, lowest_load; 524 525 ASSERT(zio->io_bp == NULL || BP_GET_PHYSICAL_BIRTH(zio->io_bp) == txg); 526 527 lowest_load = INT_MAX; 528 mm->mm_preferred_cnt = 0; 529 for (c = 0; c < mm->mm_children; c++) { 530 mirror_child_t *mc; 531 532 mc = &mm->mm_child[c]; 533 if (mc->mc_tried || mc->mc_skipped) 534 continue; 535 536 if (mc->mc_vd == NULL || 537 !vdev_mirror_child_readable(mc)) { 538 mc->mc_error = SET_ERROR(ENXIO); 539 mc->mc_tried = 1; /* don't even try */ 540 mc->mc_skipped = 1; 541 continue; 542 } 543 544 if (vdev_mirror_child_missing(mc, txg, 1)) { 545 mc->mc_error = SET_ERROR(ESTALE); 546 mc->mc_skipped = 1; 547 mc->mc_speculative = 1; 548 continue; 549 } 550 551 if (mc->mc_vd->vdev_ops == &vdev_draid_spare_ops) { 552 mm->mm_preferred[0] = c; 553 mm->mm_preferred_cnt = 1; 554 break; 555 } 556 557 mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset); 558 if (mc->mc_load > lowest_load) 559 continue; 560 561 if (mc->mc_load < lowest_load) { 562 lowest_load = mc->mc_load; 563 mm->mm_preferred_cnt = 0; 564 } 565 mm->mm_preferred[mm->mm_preferred_cnt] = c; 566 mm->mm_preferred_cnt++; 567 } 568 569 if (mm->mm_preferred_cnt == 1) { 570 MIRROR_BUMP(vdev_mirror_stat_preferred_found); 571 return (mm->mm_preferred[0]); 572 } 573 574 if (mm->mm_preferred_cnt > 1) { 575 MIRROR_BUMP(vdev_mirror_stat_preferred_not_found); 576 return (vdev_mirror_preferred_child_randomize(zio)); 577 } 578 579 /* 580 * Every device is either missing or has this txg in its DTL. 581 * Look for any child we haven't already tried before giving up. 582 */ 583 for (c = 0; c < mm->mm_children; c++) { 584 if (!mm->mm_child[c].mc_tried) 585 return (c); 586 } 587 588 /* 589 * Every child failed. There's no place left to look. 590 */ 591 return (-1); 592 } 593 594 static void 595 vdev_mirror_io_start(zio_t *zio) 596 { 597 mirror_map_t *mm; 598 mirror_child_t *mc; 599 int c, children; 600 601 mm = vdev_mirror_map_init(zio); 602 zio->io_vsd = mm; 603 zio->io_vsd_ops = &vdev_mirror_vsd_ops; 604 605 if (mm == NULL) { 606 ASSERT(!spa_trust_config(zio->io_spa)); 607 ASSERT(zio->io_type == ZIO_TYPE_READ); 608 zio_execute(zio); 609 return; 610 } 611 612 if (zio->io_type == ZIO_TYPE_READ) { 613 if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_resilvering) { 614 /* 615 * For scrubbing reads we need to issue reads to all 616 * children. One child can reuse parent buffer, but 617 * for others we have to allocate separate ones to 618 * verify checksums if io_bp is non-NULL, or compare 619 * them in vdev_mirror_io_done() otherwise. 620 */ 621 boolean_t first = B_TRUE; 622 623 if (mm->mm_children > 1) 624 zio_batch_create(zio); 625 626 for (c = 0; c < mm->mm_children; c++) { 627 mc = &mm->mm_child[c]; 628 629 /* Don't issue ZIOs to offline children */ 630 if (!vdev_mirror_child_readable(mc)) { 631 mc->mc_error = SET_ERROR(ENXIO); 632 mc->mc_tried = 1; 633 mc->mc_skipped = 1; 634 continue; 635 } 636 637 mc->mc_abd = first ? zio->io_abd : 638 abd_alloc_sametype(zio->io_abd, 639 zio->io_size); 640 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 641 mc->mc_vd, mc->mc_offset, mc->mc_abd, 642 zio->io_size, zio->io_type, 643 zio->io_priority, 0, 644 vdev_mirror_child_done, mc)); 645 first = B_FALSE; 646 } 647 648 zio_execute(zio_batch_rele(zio)); 649 return; 650 } 651 /* 652 * For normal reads just pick one child. 653 */ 654 c = vdev_mirror_child_select(zio); 655 children = (c >= 0); 656 } else { 657 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 658 659 /* 660 * Writes go to all children. 661 */ 662 c = 0; 663 children = mm->mm_children; 664 } 665 666 if (children > 1) 667 zio_batch_create(zio); 668 669 while (children--) { 670 mc = &mm->mm_child[c++]; 671 672 /* 673 * When sequentially resilvering and the integrity of the data 674 * is speculative (ZIO_FLAG_SPECULATIVE), issue write repair IOs 675 * only to the vdev which is being rebuilt. Existing data on 676 * other children must never be overwritten with unconfirmed 677 * data to avoid unrecoverable damage to the pool. 678 */ 679 if ((zio->io_priority == ZIO_PRIORITY_REBUILD) && 680 (zio->io_flags & ZIO_FLAG_IO_REPAIR) && 681 !(zio->io_flags & ZIO_FLAG_SCRUB) && 682 (zio->io_flags & ZIO_FLAG_SPECULATIVE) && 683 mm->mm_rebuilding && !mc->mc_rebuilding) { 684 continue; 685 } 686 687 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 688 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size, 689 zio->io_type, zio->io_priority, 0, 690 vdev_mirror_child_done, mc)); 691 } 692 693 zio_execute(zio_batch_rele(zio)); 694 } 695 696 static int 697 vdev_mirror_worst_error(mirror_map_t *mm) 698 { 699 int error[2] = { 0, 0 }; 700 701 for (int c = 0; c < mm->mm_children; c++) { 702 mirror_child_t *mc = &mm->mm_child[c]; 703 int s = mc->mc_speculative; 704 error[s] = zio_worst_error(error[s], mc->mc_error); 705 } 706 707 return (error[0] ? error[0] : error[1]); 708 } 709 710 static void 711 vdev_mirror_io_done(zio_t *zio) 712 { 713 mirror_map_t *mm = zio->io_vsd; 714 mirror_child_t *mc; 715 int c; 716 int good_copies = 0; 717 int unexpected_errors = 0; 718 int last_good_copy = -1; 719 720 if (mm == NULL) 721 return; 722 723 for (c = 0; c < mm->mm_children; c++) { 724 mc = &mm->mm_child[c]; 725 726 if (mc->mc_error) { 727 if (!mc->mc_skipped) 728 unexpected_errors++; 729 } else if (mc->mc_tried) { 730 last_good_copy = c; 731 good_copies++; 732 } 733 } 734 735 if (zio->io_type == ZIO_TYPE_WRITE) { 736 /* 737 * XXX -- for now, treat partial writes as success. 738 * 739 * Now that we support write reallocation, it would be better 740 * to treat partial failure as real failure unless there are 741 * no non-degraded top-level vdevs left, and not update DTLs 742 * if we intend to reallocate. 743 */ 744 if (good_copies != mm->mm_children) { 745 /* 746 * Always require at least one good copy. 747 * 748 * For ditto blocks (io_vd == NULL), require 749 * all copies to be good. 750 * 751 * XXX -- for replacing vdevs, there's no great answer. 752 * If the old device is really dead, we may not even 753 * be able to access it -- so we only want to 754 * require good writes to the new device. But if 755 * the new device turns out to be flaky, we want 756 * to be able to detach it -- which requires all 757 * writes to the old device to have succeeded. 758 */ 759 if (good_copies == 0 || zio->io_vd == NULL) 760 zio->io_error = vdev_mirror_worst_error(mm); 761 } 762 return; 763 } 764 765 ASSERT(zio->io_type == ZIO_TYPE_READ); 766 767 /* 768 * Any Direct I/O read that has a checksum error must be treated as 769 * suspicious as the contents of the buffer could be getting 770 * manipulated while the I/O is taking place. The checksum verify error 771 * will be reported to the top-level Mirror VDEV. 772 * 773 * There will be no attampt at reading any additional data copies. If 774 * the buffer is still being manipulated while attempting to read from 775 * another child, there exists a possibly that the checksum could be 776 * verified as valid. However, the buffer contents could again get 777 * manipulated after verifying the checksum. This would lead to bad data 778 * being written out during self healing. 779 */ 780 if ((zio->io_flags & ZIO_FLAG_DIO_READ) && 781 (zio->io_post & ZIO_POST_DIO_CHKSUM_ERR)) { 782 zio_dio_chksum_verify_error_report(zio); 783 zio->io_error = vdev_mirror_worst_error(mm); 784 ASSERT3U(zio->io_error, ==, ECKSUM); 785 return; 786 } 787 788 /* 789 * If we don't have a good copy yet, keep trying other children. 790 */ 791 if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) { 792 ASSERT(c >= 0 && c < mm->mm_children); 793 mc = &mm->mm_child[c]; 794 zio_vdev_io_redone(zio); 795 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 796 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size, 797 ZIO_TYPE_READ, zio->io_priority, 0, 798 vdev_mirror_child_done, mc)); 799 return; 800 } 801 802 if (zio->io_flags & ZIO_FLAG_SCRUB && !mm->mm_resilvering) { 803 abd_t *best_abd = NULL; 804 if (last_good_copy >= 0) 805 best_abd = mm->mm_child[last_good_copy].mc_abd; 806 807 /* 808 * If we're scrubbing but don't have a BP available (because 809 * this vdev is under a raidz or draid vdev) then the best we 810 * can do is compare all of the copies read. If they're not 811 * identical then return a checksum error and the most likely 812 * correct data. The raidz code will issue a repair I/O if 813 * possible. 814 */ 815 if (zio->io_bp == NULL) { 816 ASSERT(zio->io_vd->vdev_ops == &vdev_replacing_ops || 817 zio->io_vd->vdev_ops == &vdev_spare_ops); 818 819 abd_t *pref_abd = NULL; 820 for (c = 0; c < last_good_copy; c++) { 821 mc = &mm->mm_child[c]; 822 if (mc->mc_error || !mc->mc_tried) 823 continue; 824 825 if (abd_cmp(mc->mc_abd, best_abd) != 0) 826 zio->io_error = SET_ERROR(ECKSUM); 827 828 /* 829 * The distributed spare is always prefered 830 * by vdev_mirror_child_select() so it's 831 * considered to be the best candidate. 832 */ 833 if (pref_abd == NULL && 834 mc->mc_vd->vdev_ops == 835 &vdev_draid_spare_ops) 836 pref_abd = mc->mc_abd; 837 838 /* 839 * In the absence of a preferred copy, use 840 * the parent pointer to avoid a memory copy. 841 */ 842 if (mc->mc_abd == zio->io_abd) 843 best_abd = mc->mc_abd; 844 } 845 if (pref_abd) 846 best_abd = pref_abd; 847 } else { 848 849 /* 850 * If we have a BP available, then checksums are 851 * already verified and we just need a buffer 852 * with valid data, preferring parent one to 853 * avoid a memory copy. 854 */ 855 for (c = 0; c < last_good_copy; c++) { 856 mc = &mm->mm_child[c]; 857 if (mc->mc_error || !mc->mc_tried) 858 continue; 859 if (mc->mc_abd == zio->io_abd) { 860 best_abd = mc->mc_abd; 861 break; 862 } 863 } 864 } 865 866 if (best_abd && best_abd != zio->io_abd) 867 abd_copy(zio->io_abd, best_abd, zio->io_size); 868 for (c = 0; c < mm->mm_children; c++) { 869 mc = &mm->mm_child[c]; 870 if (mc->mc_abd != zio->io_abd) 871 abd_free(mc->mc_abd); 872 mc->mc_abd = NULL; 873 } 874 } 875 876 if (good_copies == 0) { 877 zio->io_error = vdev_mirror_worst_error(mm); 878 ASSERT(zio->io_error != 0); 879 } 880 881 if (good_copies && spa_writeable(zio->io_spa) && 882 (unexpected_errors || 883 (zio->io_flags & ZIO_FLAG_RESILVER) || 884 ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_resilvering))) { 885 /* 886 * Use the good data we have in hand to repair damaged children. 887 */ 888 for (c = 0; c < mm->mm_children; c++) { 889 /* 890 * Don't rewrite known good children. 891 * Not only is it unnecessary, it could 892 * actually be harmful: if the system lost 893 * power while rewriting the only good copy, 894 * there would be no good copies left! 895 */ 896 mc = &mm->mm_child[c]; 897 898 if (mc->mc_error == 0) { 899 vdev_ops_t *ops = mc->mc_vd->vdev_ops; 900 901 if (mc->mc_tried) 902 continue; 903 /* 904 * We didn't try this child. We need to 905 * repair it if: 906 * 1. it's a scrub (in which case we have 907 * tried everything that was healthy) 908 * - or - 909 * 2. it's an indirect or distributed spare 910 * vdev (in which case it could point to any 911 * other vdev, which might have a bad DTL) 912 * - or - 913 * 3. the DTL indicates that this data is 914 * missing from this vdev 915 */ 916 if (!(zio->io_flags & ZIO_FLAG_SCRUB) && 917 ops != &vdev_indirect_ops && 918 ops != &vdev_draid_spare_ops && 919 !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL, 920 zio->io_txg, 1)) 921 continue; 922 mc->mc_error = SET_ERROR(ESTALE); 923 } 924 925 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 926 mc->mc_vd, mc->mc_offset, 927 zio->io_abd, zio->io_size, ZIO_TYPE_WRITE, 928 zio->io_priority == ZIO_PRIORITY_REBUILD ? 929 ZIO_PRIORITY_REBUILD : ZIO_PRIORITY_ASYNC_WRITE, 930 ZIO_FLAG_IO_REPAIR | (unexpected_errors ? 931 ZIO_FLAG_SELF_HEAL : 0), NULL, NULL)); 932 } 933 } 934 } 935 936 static void 937 vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded) 938 { 939 if (faulted == vd->vdev_children) { 940 if (vdev_children_are_offline(vd)) { 941 vdev_set_state(vd, B_FALSE, VDEV_STATE_OFFLINE, 942 VDEV_AUX_CHILDREN_OFFLINE); 943 } else { 944 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN, 945 VDEV_AUX_NO_REPLICAS); 946 } 947 } else if (degraded + faulted != 0) { 948 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE); 949 } else { 950 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE); 951 } 952 } 953 954 /* 955 * Return the maximum asize for a rebuild zio in the provided range. 956 */ 957 static uint64_t 958 vdev_mirror_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize, 959 uint64_t max_segment) 960 { 961 (void) start; 962 963 uint64_t psize = MIN(P2ROUNDUP(max_segment, 1 << vd->vdev_ashift), 964 SPA_MAXBLOCKSIZE); 965 966 return (MIN(asize, vdev_psize_to_asize(vd, psize))); 967 } 968 969 vdev_ops_t vdev_mirror_ops = { 970 .vdev_op_init = NULL, 971 .vdev_op_fini = NULL, 972 .vdev_op_open = vdev_mirror_open, 973 .vdev_op_close = vdev_mirror_close, 974 .vdev_op_psize_to_asize = vdev_default_asize, 975 .vdev_op_asize_to_psize = vdev_default_psize, 976 .vdev_op_min_asize = vdev_default_min_asize, 977 .vdev_op_min_alloc = NULL, 978 .vdev_op_io_start = vdev_mirror_io_start, 979 .vdev_op_io_done = vdev_mirror_io_done, 980 .vdev_op_state_change = vdev_mirror_state_change, 981 .vdev_op_need_resilver = vdev_default_need_resilver, 982 .vdev_op_hold = NULL, 983 .vdev_op_rele = NULL, 984 .vdev_op_remap = NULL, 985 .vdev_op_xlate = vdev_default_xlate, 986 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize, 987 .vdev_op_metaslab_init = NULL, 988 .vdev_op_config_generate = NULL, 989 .vdev_op_nparity = NULL, 990 .vdev_op_ndisks = NULL, 991 .vdev_op_type = VDEV_TYPE_MIRROR, /* name of this vdev type */ 992 .vdev_op_leaf = B_FALSE /* not a leaf vdev */ 993 }; 994 995 vdev_ops_t vdev_replacing_ops = { 996 .vdev_op_init = NULL, 997 .vdev_op_fini = NULL, 998 .vdev_op_open = vdev_mirror_open, 999 .vdev_op_close = vdev_mirror_close, 1000 .vdev_op_psize_to_asize = vdev_default_asize, 1001 .vdev_op_asize_to_psize = vdev_default_psize, 1002 .vdev_op_min_asize = vdev_default_min_asize, 1003 .vdev_op_min_alloc = NULL, 1004 .vdev_op_io_start = vdev_mirror_io_start, 1005 .vdev_op_io_done = vdev_mirror_io_done, 1006 .vdev_op_state_change = vdev_mirror_state_change, 1007 .vdev_op_need_resilver = vdev_default_need_resilver, 1008 .vdev_op_hold = NULL, 1009 .vdev_op_rele = NULL, 1010 .vdev_op_remap = NULL, 1011 .vdev_op_xlate = vdev_default_xlate, 1012 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize, 1013 .vdev_op_metaslab_init = NULL, 1014 .vdev_op_config_generate = NULL, 1015 .vdev_op_nparity = NULL, 1016 .vdev_op_ndisks = NULL, 1017 .vdev_op_type = VDEV_TYPE_REPLACING, /* name of this vdev type */ 1018 .vdev_op_leaf = B_FALSE /* not a leaf vdev */ 1019 }; 1020 1021 vdev_ops_t vdev_spare_ops = { 1022 .vdev_op_init = NULL, 1023 .vdev_op_fini = NULL, 1024 .vdev_op_open = vdev_mirror_open, 1025 .vdev_op_close = vdev_mirror_close, 1026 .vdev_op_psize_to_asize = vdev_default_asize, 1027 .vdev_op_asize_to_psize = vdev_default_psize, 1028 .vdev_op_min_asize = vdev_default_min_asize, 1029 .vdev_op_min_alloc = NULL, 1030 .vdev_op_io_start = vdev_mirror_io_start, 1031 .vdev_op_io_done = vdev_mirror_io_done, 1032 .vdev_op_state_change = vdev_mirror_state_change, 1033 .vdev_op_need_resilver = vdev_default_need_resilver, 1034 .vdev_op_hold = NULL, 1035 .vdev_op_rele = NULL, 1036 .vdev_op_remap = NULL, 1037 .vdev_op_xlate = vdev_default_xlate, 1038 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize, 1039 .vdev_op_metaslab_init = NULL, 1040 .vdev_op_config_generate = NULL, 1041 .vdev_op_nparity = NULL, 1042 .vdev_op_ndisks = NULL, 1043 .vdev_op_type = VDEV_TYPE_SPARE, /* name of this vdev type */ 1044 .vdev_op_leaf = B_FALSE /* not a leaf vdev */ 1045 }; 1046 1047 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_inc, INT, ZMOD_RW, 1048 "Rotating media load increment for non-seeking I/Os"); 1049 1050 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_inc, INT, 1051 ZMOD_RW, "Rotating media load increment for seeking I/Os"); 1052 1053 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_offset, INT, 1054 ZMOD_RW, 1055 "Offset in bytes from the last I/O which triggers " 1056 "a reduced rotating media seek increment"); 1057 1058 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_inc, INT, 1059 ZMOD_RW, "Non-rotating media load increment for non-seeking I/Os"); 1060 1061 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_seek_inc, INT, 1062 ZMOD_RW, "Non-rotating media load increment for seeking I/Os"); 1063