1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
29 */
30
31 #include <sys/zfs_context.h>
32 #include <sys/spa.h>
33 #include <sys/spa_impl.h>
34 #include <sys/dsl_pool.h>
35 #include <sys/dsl_scan.h>
36 #include <sys/vdev_impl.h>
37 #include <sys/vdev_draid.h>
38 #include <sys/zio.h>
39 #include <sys/zio_checksum.h>
40 #include <sys/abd.h>
41 #include <sys/fs/zfs.h>
42
43 /*
44 * Vdev mirror kstats
45 */
46 static kstat_t *mirror_ksp = NULL;
47
48 typedef struct mirror_stats {
49 kstat_named_t vdev_mirror_stat_rotating_linear;
50 kstat_named_t vdev_mirror_stat_rotating_offset;
51 kstat_named_t vdev_mirror_stat_rotating_seek;
52 kstat_named_t vdev_mirror_stat_non_rotating_linear;
53 kstat_named_t vdev_mirror_stat_non_rotating_seek;
54
55 kstat_named_t vdev_mirror_stat_preferred_found;
56 kstat_named_t vdev_mirror_stat_preferred_not_found;
57 } mirror_stats_t;
58
59 static mirror_stats_t mirror_stats = {
60 /* New I/O follows directly the last I/O */
61 { "rotating_linear", KSTAT_DATA_UINT64 },
62 /* New I/O is within zfs_vdev_mirror_rotating_seek_offset of the last */
63 { "rotating_offset", KSTAT_DATA_UINT64 },
64 /* New I/O requires random seek */
65 { "rotating_seek", KSTAT_DATA_UINT64 },
66 /* New I/O follows directly the last I/O (nonrot) */
67 { "non_rotating_linear", KSTAT_DATA_UINT64 },
68 /* New I/O requires random seek (nonrot) */
69 { "non_rotating_seek", KSTAT_DATA_UINT64 },
70 /* Preferred child vdev found */
71 { "preferred_found", KSTAT_DATA_UINT64 },
72 /* Preferred child vdev not found or equal load */
73 { "preferred_not_found", KSTAT_DATA_UINT64 },
74
75 };
76
77 #define MIRROR_STAT(stat) (mirror_stats.stat.value.ui64)
78 #define MIRROR_INCR(stat, val) atomic_add_64(&MIRROR_STAT(stat), val)
79 #define MIRROR_BUMP(stat) MIRROR_INCR(stat, 1)
80
81 void
vdev_mirror_stat_init(void)82 vdev_mirror_stat_init(void)
83 {
84 mirror_ksp = kstat_create("zfs", 0, "vdev_mirror_stats",
85 "misc", KSTAT_TYPE_NAMED,
86 sizeof (mirror_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
87 if (mirror_ksp != NULL) {
88 mirror_ksp->ks_data = &mirror_stats;
89 kstat_install(mirror_ksp);
90 }
91 }
92
93 void
vdev_mirror_stat_fini(void)94 vdev_mirror_stat_fini(void)
95 {
96 if (mirror_ksp != NULL) {
97 kstat_delete(mirror_ksp);
98 mirror_ksp = NULL;
99 }
100 }
101
102 /*
103 * Virtual device vector for mirroring.
104 */
105 typedef struct mirror_child {
106 vdev_t *mc_vd;
107 abd_t *mc_abd;
108 uint64_t mc_offset;
109 int mc_error;
110 int mc_load;
111 uint8_t mc_tried;
112 uint8_t mc_skipped;
113 uint8_t mc_speculative;
114 uint8_t mc_rebuilding;
115 } mirror_child_t;
116
117 typedef struct mirror_map {
118 int *mm_preferred;
119 int mm_preferred_cnt;
120 int mm_children;
121 boolean_t mm_resilvering;
122 boolean_t mm_rebuilding;
123 boolean_t mm_root;
124 mirror_child_t mm_child[];
125 } mirror_map_t;
126
127 static const int vdev_mirror_shift = 21;
128
129 /*
130 * The load configuration settings below are tuned by default for
131 * the case where all devices are of the same rotational type.
132 *
133 * If there is a mixture of rotating and non-rotating media, setting
134 * zfs_vdev_mirror_non_rotating_seek_inc to 0 may well provide better results
135 * as it will direct more reads to the non-rotating vdevs which are more likely
136 * to have a higher performance.
137 */
138
139 /* Rotating media load calculation configuration. */
140 static int zfs_vdev_mirror_rotating_inc = 0;
141 static int zfs_vdev_mirror_rotating_seek_inc = 5;
142 static int zfs_vdev_mirror_rotating_seek_offset = 1 * 1024 * 1024;
143
144 /* Non-rotating media load calculation configuration. */
145 static int zfs_vdev_mirror_non_rotating_inc = 0;
146 static int zfs_vdev_mirror_non_rotating_seek_inc = 1;
147
148 static inline size_t
vdev_mirror_map_size(int children)149 vdev_mirror_map_size(int children)
150 {
151 return (offsetof(mirror_map_t, mm_child[children]) +
152 sizeof (int) * children);
153 }
154
155 static inline mirror_map_t *
vdev_mirror_map_alloc(int children,boolean_t resilvering,boolean_t root)156 vdev_mirror_map_alloc(int children, boolean_t resilvering, boolean_t root)
157 {
158 mirror_map_t *mm;
159
160 mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP);
161 mm->mm_children = children;
162 mm->mm_resilvering = resilvering;
163 mm->mm_root = root;
164 mm->mm_preferred = (int *)((uintptr_t)mm +
165 offsetof(mirror_map_t, mm_child[children]));
166
167 return (mm);
168 }
169
170 static void
vdev_mirror_map_free(zio_t * zio)171 vdev_mirror_map_free(zio_t *zio)
172 {
173 mirror_map_t *mm = zio->io_vsd;
174
175 kmem_free(mm, vdev_mirror_map_size(mm->mm_children));
176 }
177
178 static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
179 .vsd_free = vdev_mirror_map_free,
180 };
181
182 static int
vdev_mirror_load(mirror_map_t * mm,vdev_t * vd,uint64_t zio_offset)183 vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset)
184 {
185 uint64_t last_offset;
186 int64_t offset_diff;
187 int load;
188
189 /* All DVAs have equal weight at the root. */
190 if (mm->mm_root)
191 return (INT_MAX);
192
193 /*
194 * We don't return INT_MAX if the device is resilvering i.e.
195 * vdev_resilver_txg != 0 as when tested performance was slightly
196 * worse overall when resilvering with compared to without.
197 */
198
199 /* Fix zio_offset for leaf vdevs */
200 if (vd->vdev_ops->vdev_op_leaf)
201 zio_offset += VDEV_LABEL_START_SIZE;
202
203 /* Standard load based on pending queue length. */
204 load = vdev_queue_length(vd);
205 last_offset = vdev_queue_last_offset(vd);
206
207 if (vd->vdev_nonrot) {
208 /* Non-rotating media. */
209 if (last_offset == zio_offset) {
210 MIRROR_BUMP(vdev_mirror_stat_non_rotating_linear);
211 return (load + zfs_vdev_mirror_non_rotating_inc);
212 }
213
214 /*
215 * Apply a seek penalty even for non-rotating devices as
216 * sequential I/O's can be aggregated into fewer operations on
217 * the device, thus avoiding unnecessary per-command overhead
218 * and boosting performance.
219 */
220 MIRROR_BUMP(vdev_mirror_stat_non_rotating_seek);
221 return (load + zfs_vdev_mirror_non_rotating_seek_inc);
222 }
223
224 /* Rotating media I/O's which directly follow the last I/O. */
225 if (last_offset == zio_offset) {
226 MIRROR_BUMP(vdev_mirror_stat_rotating_linear);
227 return (load + zfs_vdev_mirror_rotating_inc);
228 }
229
230 /*
231 * Apply half the seek increment to I/O's within seek offset
232 * of the last I/O issued to this vdev as they should incur less
233 * of a seek increment.
234 */
235 offset_diff = (int64_t)(last_offset - zio_offset);
236 if (ABS(offset_diff) < zfs_vdev_mirror_rotating_seek_offset) {
237 MIRROR_BUMP(vdev_mirror_stat_rotating_offset);
238 return (load + (zfs_vdev_mirror_rotating_seek_inc / 2));
239 }
240
241 /* Apply the full seek increment to all other I/O's. */
242 MIRROR_BUMP(vdev_mirror_stat_rotating_seek);
243 return (load + zfs_vdev_mirror_rotating_seek_inc);
244 }
245
246 static boolean_t
vdev_mirror_rebuilding(vdev_t * vd)247 vdev_mirror_rebuilding(vdev_t *vd)
248 {
249 if (vd->vdev_ops->vdev_op_leaf && vd->vdev_rebuild_txg)
250 return (B_TRUE);
251
252 for (int i = 0; i < vd->vdev_children; i++) {
253 if (vdev_mirror_rebuilding(vd->vdev_child[i])) {
254 return (B_TRUE);
255 }
256 }
257
258 return (B_FALSE);
259 }
260
261 /*
262 * Avoid inlining the function to keep vdev_mirror_io_start(), which
263 * is this functions only caller, as small as possible on the stack.
264 */
265 noinline static mirror_map_t *
vdev_mirror_map_init(zio_t * zio)266 vdev_mirror_map_init(zio_t *zio)
267 {
268 mirror_map_t *mm = NULL;
269 mirror_child_t *mc;
270 vdev_t *vd = zio->io_vd;
271 int c;
272
273 if (vd == NULL) {
274 dva_t *dva = zio->io_bp->blk_dva;
275 spa_t *spa = zio->io_spa;
276 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
277 dva_t dva_copy[SPA_DVAS_PER_BP];
278
279 /*
280 * The sequential scrub code sorts and issues all DVAs
281 * of a bp separately. Each of these IOs includes all
282 * original DVA copies so that repairs can be performed
283 * in the event of an error, but we only actually want
284 * to check the first DVA since the others will be
285 * checked by their respective sorted IOs. Only if we
286 * hit an error will we try all DVAs upon retrying.
287 *
288 * Note: This check is safe even if the user switches
289 * from a legacy scrub to a sequential one in the middle
290 * of processing, since scn_is_sorted isn't updated until
291 * all outstanding IOs from the previous scrub pass
292 * complete.
293 */
294 if ((zio->io_flags & ZIO_FLAG_SCRUB) &&
295 !(zio->io_flags & ZIO_FLAG_IO_RETRY) &&
296 dsl_scan_scrubbing(spa->spa_dsl_pool) &&
297 scn->scn_is_sorted) {
298 c = 1;
299 } else {
300 c = BP_GET_NDVAS(zio->io_bp);
301 }
302
303 /*
304 * If the pool cannot be written to, then infer that some
305 * DVAs might be invalid or point to vdevs that do not exist.
306 * We skip them.
307 */
308 if (!spa_writeable(spa)) {
309 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
310 int j = 0;
311 for (int i = 0; i < c; i++) {
312 if (zfs_dva_valid(spa, &dva[i], zio->io_bp))
313 dva_copy[j++] = dva[i];
314 }
315 if (j == 0) {
316 zio->io_vsd = NULL;
317 zio->io_error = ENXIO;
318 return (NULL);
319 }
320 if (j < c) {
321 dva = dva_copy;
322 c = j;
323 }
324 }
325
326 mm = vdev_mirror_map_alloc(c, B_FALSE, B_TRUE);
327 for (c = 0; c < mm->mm_children; c++) {
328 mc = &mm->mm_child[c];
329
330 mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
331 mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
332 if (mc->mc_vd == NULL) {
333 kmem_free(mm, vdev_mirror_map_size(
334 mm->mm_children));
335 zio->io_vsd = NULL;
336 zio->io_error = ENXIO;
337 return (NULL);
338 }
339 }
340 } else {
341 /*
342 * If we are resilvering, then we should handle scrub reads
343 * differently; we shouldn't issue them to the resilvering
344 * device because it might not have those blocks.
345 *
346 * We are resilvering iff:
347 * 1) We are a replacing vdev (ie our name is "replacing-1" or
348 * "spare-1" or something like that), and
349 * 2) The pool is currently being resilvered.
350 *
351 * We cannot simply check vd->vdev_resilver_txg, because it's
352 * not set in this path.
353 *
354 * Nor can we just check our vdev_ops; there are cases (such as
355 * when a user types "zpool replace pool odev spare_dev" and
356 * spare_dev is in the spare list, or when a spare device is
357 * automatically used to replace a DEGRADED device) when
358 * resilvering is complete but both the original vdev and the
359 * spare vdev remain in the pool. That behavior is intentional.
360 * It helps implement the policy that a spare should be
361 * automatically removed from the pool after the user replaces
362 * the device that originally failed.
363 *
364 * If a spa load is in progress, then spa_dsl_pool may be
365 * uninitialized. But we shouldn't be resilvering during a spa
366 * load anyway.
367 */
368 boolean_t replacing = (vd->vdev_ops == &vdev_replacing_ops ||
369 vd->vdev_ops == &vdev_spare_ops) &&
370 spa_load_state(vd->vdev_spa) == SPA_LOAD_NONE &&
371 dsl_scan_resilvering(vd->vdev_spa->spa_dsl_pool);
372 mm = vdev_mirror_map_alloc(vd->vdev_children, replacing,
373 B_FALSE);
374 for (c = 0; c < mm->mm_children; c++) {
375 mc = &mm->mm_child[c];
376 mc->mc_vd = vd->vdev_child[c];
377 mc->mc_offset = zio->io_offset;
378
379 if (vdev_mirror_rebuilding(mc->mc_vd))
380 mm->mm_rebuilding = mc->mc_rebuilding = B_TRUE;
381 }
382 }
383
384 return (mm);
385 }
386
387 static int
vdev_mirror_open(vdev_t * vd,uint64_t * asize,uint64_t * max_asize,uint64_t * logical_ashift,uint64_t * physical_ashift)388 vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
389 uint64_t *logical_ashift, uint64_t *physical_ashift)
390 {
391 int numerrors = 0;
392 int lasterror = 0;
393
394 if (vd->vdev_children == 0) {
395 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
396 return (SET_ERROR(EINVAL));
397 }
398
399 vdev_open_children(vd);
400
401 for (int c = 0; c < vd->vdev_children; c++) {
402 vdev_t *cvd = vd->vdev_child[c];
403
404 if (cvd->vdev_open_error) {
405 lasterror = cvd->vdev_open_error;
406 numerrors++;
407 continue;
408 }
409
410 *asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
411 *max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
412 *logical_ashift = MAX(*logical_ashift, cvd->vdev_ashift);
413 }
414 for (int c = 0; c < vd->vdev_children; c++) {
415 vdev_t *cvd = vd->vdev_child[c];
416
417 if (cvd->vdev_open_error)
418 continue;
419 *physical_ashift = vdev_best_ashift(*logical_ashift,
420 *physical_ashift, cvd->vdev_physical_ashift);
421 }
422
423 if (numerrors == vd->vdev_children) {
424 if (vdev_children_are_offline(vd))
425 vd->vdev_stat.vs_aux = VDEV_AUX_CHILDREN_OFFLINE;
426 else
427 vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
428 return (lasterror);
429 }
430
431 return (0);
432 }
433
434 static void
vdev_mirror_close(vdev_t * vd)435 vdev_mirror_close(vdev_t *vd)
436 {
437 for (int c = 0; c < vd->vdev_children; c++)
438 vdev_close(vd->vdev_child[c]);
439 }
440
441 static void
vdev_mirror_child_done(zio_t * zio)442 vdev_mirror_child_done(zio_t *zio)
443 {
444 mirror_child_t *mc = zio->io_private;
445
446 mc->mc_error = zio->io_error;
447 mc->mc_tried = 1;
448 mc->mc_skipped = 0;
449 }
450
451 /*
452 * Check the other, lower-index DVAs to see if they're on the same
453 * vdev as the child we picked. If they are, use them since they
454 * are likely to have been allocated from the primary metaslab in
455 * use at the time, and hence are more likely to have locality with
456 * single-copy data.
457 */
458 static int
vdev_mirror_dva_select(zio_t * zio,int p)459 vdev_mirror_dva_select(zio_t *zio, int p)
460 {
461 dva_t *dva = zio->io_bp->blk_dva;
462 mirror_map_t *mm = zio->io_vsd;
463 int preferred;
464 int c;
465
466 preferred = mm->mm_preferred[p];
467 for (p--; p >= 0; p--) {
468 c = mm->mm_preferred[p];
469 if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred]))
470 preferred = c;
471 }
472 return (preferred);
473 }
474
475 static int
vdev_mirror_preferred_child_randomize(zio_t * zio)476 vdev_mirror_preferred_child_randomize(zio_t *zio)
477 {
478 mirror_map_t *mm = zio->io_vsd;
479 int p;
480
481 if (mm->mm_root) {
482 p = random_in_range(mm->mm_preferred_cnt);
483 return (vdev_mirror_dva_select(zio, p));
484 }
485
486 /*
487 * To ensure we don't always favour the first matching vdev,
488 * which could lead to wear leveling issues on SSD's, we
489 * use the I/O offset as a pseudo random seed into the vdevs
490 * which have the lowest load.
491 */
492 p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt;
493 return (mm->mm_preferred[p]);
494 }
495
496 static boolean_t
vdev_mirror_child_readable(mirror_child_t * mc)497 vdev_mirror_child_readable(mirror_child_t *mc)
498 {
499 vdev_t *vd = mc->mc_vd;
500
501 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
502 return (vdev_draid_readable(vd, mc->mc_offset));
503 else
504 return (vdev_readable(vd));
505 }
506
507 static boolean_t
vdev_mirror_child_missing(mirror_child_t * mc,uint64_t txg,uint64_t size)508 vdev_mirror_child_missing(mirror_child_t *mc, uint64_t txg, uint64_t size)
509 {
510 vdev_t *vd = mc->mc_vd;
511
512 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
513 return (vdev_draid_missing(vd, mc->mc_offset, txg, size));
514 else
515 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size));
516 }
517
518 /*
519 * Try to find a vdev whose DTL doesn't contain the block we want to read
520 * preferring vdevs based on determined load. If we can't, try the read on
521 * any vdev we haven't already tried.
522 *
523 * Distributed spares are an exception to the above load rule. They are
524 * always preferred in order to detect gaps in the distributed spare which
525 * are created when another disk in the dRAID fails. In order to restore
526 * redundancy those gaps must be read to trigger the required repair IO.
527 */
528 static int
vdev_mirror_child_select(zio_t * zio)529 vdev_mirror_child_select(zio_t *zio)
530 {
531 mirror_map_t *mm = zio->io_vsd;
532 uint64_t txg = zio->io_txg;
533 int c, lowest_load;
534
535 ASSERT(zio->io_bp == NULL || BP_GET_BIRTH(zio->io_bp) == txg);
536
537 lowest_load = INT_MAX;
538 mm->mm_preferred_cnt = 0;
539 for (c = 0; c < mm->mm_children; c++) {
540 mirror_child_t *mc;
541
542 mc = &mm->mm_child[c];
543 if (mc->mc_tried || mc->mc_skipped)
544 continue;
545
546 if (mc->mc_vd == NULL ||
547 !vdev_mirror_child_readable(mc)) {
548 mc->mc_error = SET_ERROR(ENXIO);
549 mc->mc_tried = 1; /* don't even try */
550 mc->mc_skipped = 1;
551 continue;
552 }
553
554 if (vdev_mirror_child_missing(mc, txg, 1)) {
555 mc->mc_error = SET_ERROR(ESTALE);
556 mc->mc_skipped = 1;
557 mc->mc_speculative = 1;
558 continue;
559 }
560
561 if (mc->mc_vd->vdev_ops == &vdev_draid_spare_ops) {
562 mm->mm_preferred[0] = c;
563 mm->mm_preferred_cnt = 1;
564 break;
565 }
566
567 mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset);
568 if (mc->mc_load > lowest_load)
569 continue;
570
571 if (mc->mc_load < lowest_load) {
572 lowest_load = mc->mc_load;
573 mm->mm_preferred_cnt = 0;
574 }
575 mm->mm_preferred[mm->mm_preferred_cnt] = c;
576 mm->mm_preferred_cnt++;
577 }
578
579 if (mm->mm_preferred_cnt == 1) {
580 MIRROR_BUMP(vdev_mirror_stat_preferred_found);
581 return (mm->mm_preferred[0]);
582 }
583
584 if (mm->mm_preferred_cnt > 1) {
585 MIRROR_BUMP(vdev_mirror_stat_preferred_not_found);
586 return (vdev_mirror_preferred_child_randomize(zio));
587 }
588
589 /*
590 * Every device is either missing or has this txg in its DTL.
591 * Look for any child we haven't already tried before giving up.
592 */
593 for (c = 0; c < mm->mm_children; c++) {
594 if (!mm->mm_child[c].mc_tried)
595 return (c);
596 }
597
598 /*
599 * Every child failed. There's no place left to look.
600 */
601 return (-1);
602 }
603
604 static void
vdev_mirror_io_start(zio_t * zio)605 vdev_mirror_io_start(zio_t *zio)
606 {
607 mirror_map_t *mm;
608 mirror_child_t *mc;
609 int c, children;
610
611 mm = vdev_mirror_map_init(zio);
612 zio->io_vsd = mm;
613 zio->io_vsd_ops = &vdev_mirror_vsd_ops;
614
615 if (mm == NULL) {
616 ASSERT(!spa_trust_config(zio->io_spa));
617 ASSERT(zio->io_type == ZIO_TYPE_READ);
618 zio_execute(zio);
619 return;
620 }
621
622 if (zio->io_type == ZIO_TYPE_READ) {
623 if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_resilvering) {
624 /*
625 * For scrubbing reads we need to issue reads to all
626 * children. One child can reuse parent buffer, but
627 * for others we have to allocate separate ones to
628 * verify checksums if io_bp is non-NULL, or compare
629 * them in vdev_mirror_io_done() otherwise.
630 */
631 boolean_t first = B_TRUE;
632 for (c = 0; c < mm->mm_children; c++) {
633 mc = &mm->mm_child[c];
634
635 /* Don't issue ZIOs to offline children */
636 if (!vdev_mirror_child_readable(mc)) {
637 mc->mc_error = SET_ERROR(ENXIO);
638 mc->mc_tried = 1;
639 mc->mc_skipped = 1;
640 continue;
641 }
642
643 mc->mc_abd = first ? zio->io_abd :
644 abd_alloc_sametype(zio->io_abd,
645 zio->io_size);
646 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
647 mc->mc_vd, mc->mc_offset, mc->mc_abd,
648 zio->io_size, zio->io_type,
649 zio->io_priority, 0,
650 vdev_mirror_child_done, mc));
651 first = B_FALSE;
652 }
653 zio_execute(zio);
654 return;
655 }
656 /*
657 * For normal reads just pick one child.
658 */
659 c = vdev_mirror_child_select(zio);
660 children = (c >= 0);
661 } else {
662 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
663
664 /*
665 * Writes go to all children.
666 */
667 c = 0;
668 children = mm->mm_children;
669 }
670
671 while (children--) {
672 mc = &mm->mm_child[c];
673 c++;
674
675 /*
676 * When sequentially resilvering only issue write repair
677 * IOs to the vdev which is being rebuilt since performance
678 * is limited by the slowest child. This is an issue for
679 * faster replacement devices such as distributed spares.
680 */
681 if ((zio->io_priority == ZIO_PRIORITY_REBUILD) &&
682 (zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
683 !(zio->io_flags & ZIO_FLAG_SCRUB) &&
684 mm->mm_rebuilding && !mc->mc_rebuilding) {
685 continue;
686 }
687
688 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
689 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
690 zio->io_type, zio->io_priority, 0,
691 vdev_mirror_child_done, mc));
692 }
693
694 zio_execute(zio);
695 }
696
697 static int
vdev_mirror_worst_error(mirror_map_t * mm)698 vdev_mirror_worst_error(mirror_map_t *mm)
699 {
700 int error[2] = { 0, 0 };
701
702 for (int c = 0; c < mm->mm_children; c++) {
703 mirror_child_t *mc = &mm->mm_child[c];
704 int s = mc->mc_speculative;
705 error[s] = zio_worst_error(error[s], mc->mc_error);
706 }
707
708 return (error[0] ? error[0] : error[1]);
709 }
710
711 static void
vdev_mirror_io_done(zio_t * zio)712 vdev_mirror_io_done(zio_t *zio)
713 {
714 mirror_map_t *mm = zio->io_vsd;
715 mirror_child_t *mc;
716 int c;
717 int good_copies = 0;
718 int unexpected_errors = 0;
719 int last_good_copy = -1;
720
721 if (mm == NULL)
722 return;
723
724 for (c = 0; c < mm->mm_children; c++) {
725 mc = &mm->mm_child[c];
726
727 if (mc->mc_error) {
728 if (!mc->mc_skipped)
729 unexpected_errors++;
730 } else if (mc->mc_tried) {
731 last_good_copy = c;
732 good_copies++;
733 }
734 }
735
736 if (zio->io_type == ZIO_TYPE_WRITE) {
737 /*
738 * XXX -- for now, treat partial writes as success.
739 *
740 * Now that we support write reallocation, it would be better
741 * to treat partial failure as real failure unless there are
742 * no non-degraded top-level vdevs left, and not update DTLs
743 * if we intend to reallocate.
744 */
745 if (good_copies != mm->mm_children) {
746 /*
747 * Always require at least one good copy.
748 *
749 * For ditto blocks (io_vd == NULL), require
750 * all copies to be good.
751 *
752 * XXX -- for replacing vdevs, there's no great answer.
753 * If the old device is really dead, we may not even
754 * be able to access it -- so we only want to
755 * require good writes to the new device. But if
756 * the new device turns out to be flaky, we want
757 * to be able to detach it -- which requires all
758 * writes to the old device to have succeeded.
759 */
760 if (good_copies == 0 || zio->io_vd == NULL)
761 zio->io_error = vdev_mirror_worst_error(mm);
762 }
763 return;
764 }
765
766 ASSERT(zio->io_type == ZIO_TYPE_READ);
767
768 /*
769 * Any Direct I/O read that has a checksum error must be treated as
770 * suspicious as the contents of the buffer could be getting
771 * manipulated while the I/O is taking place. The checksum verify error
772 * will be reported to the top-level Mirror VDEV.
773 *
774 * There will be no attampt at reading any additional data copies. If
775 * the buffer is still being manipulated while attempting to read from
776 * another child, there exists a possibly that the checksum could be
777 * verified as valid. However, the buffer contents could again get
778 * manipulated after verifying the checksum. This would lead to bad data
779 * being written out during self healing.
780 */
781 if ((zio->io_flags & ZIO_FLAG_DIO_READ) &&
782 (zio->io_flags & ZIO_FLAG_DIO_CHKSUM_ERR)) {
783 zio_dio_chksum_verify_error_report(zio);
784 zio->io_error = vdev_mirror_worst_error(mm);
785 ASSERT3U(zio->io_error, ==, ECKSUM);
786 return;
787 }
788
789 /*
790 * If we don't have a good copy yet, keep trying other children.
791 */
792 if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
793 ASSERT(c >= 0 && c < mm->mm_children);
794 mc = &mm->mm_child[c];
795 zio_vdev_io_redone(zio);
796 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
797 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
798 ZIO_TYPE_READ, zio->io_priority, 0,
799 vdev_mirror_child_done, mc));
800 return;
801 }
802
803 if (zio->io_flags & ZIO_FLAG_SCRUB && !mm->mm_resilvering) {
804 abd_t *best_abd = NULL;
805 if (last_good_copy >= 0)
806 best_abd = mm->mm_child[last_good_copy].mc_abd;
807
808 /*
809 * If we're scrubbing but don't have a BP available (because
810 * this vdev is under a raidz or draid vdev) then the best we
811 * can do is compare all of the copies read. If they're not
812 * identical then return a checksum error and the most likely
813 * correct data. The raidz code will issue a repair I/O if
814 * possible.
815 */
816 if (zio->io_bp == NULL) {
817 ASSERT(zio->io_vd->vdev_ops == &vdev_replacing_ops ||
818 zio->io_vd->vdev_ops == &vdev_spare_ops);
819
820 abd_t *pref_abd = NULL;
821 for (c = 0; c < last_good_copy; c++) {
822 mc = &mm->mm_child[c];
823 if (mc->mc_error || !mc->mc_tried)
824 continue;
825
826 if (abd_cmp(mc->mc_abd, best_abd) != 0)
827 zio->io_error = SET_ERROR(ECKSUM);
828
829 /*
830 * The distributed spare is always prefered
831 * by vdev_mirror_child_select() so it's
832 * considered to be the best candidate.
833 */
834 if (pref_abd == NULL &&
835 mc->mc_vd->vdev_ops ==
836 &vdev_draid_spare_ops)
837 pref_abd = mc->mc_abd;
838
839 /*
840 * In the absence of a preferred copy, use
841 * the parent pointer to avoid a memory copy.
842 */
843 if (mc->mc_abd == zio->io_abd)
844 best_abd = mc->mc_abd;
845 }
846 if (pref_abd)
847 best_abd = pref_abd;
848 } else {
849
850 /*
851 * If we have a BP available, then checksums are
852 * already verified and we just need a buffer
853 * with valid data, preferring parent one to
854 * avoid a memory copy.
855 */
856 for (c = 0; c < last_good_copy; c++) {
857 mc = &mm->mm_child[c];
858 if (mc->mc_error || !mc->mc_tried)
859 continue;
860 if (mc->mc_abd == zio->io_abd) {
861 best_abd = mc->mc_abd;
862 break;
863 }
864 }
865 }
866
867 if (best_abd && best_abd != zio->io_abd)
868 abd_copy(zio->io_abd, best_abd, zio->io_size);
869 for (c = 0; c < mm->mm_children; c++) {
870 mc = &mm->mm_child[c];
871 if (mc->mc_abd != zio->io_abd)
872 abd_free(mc->mc_abd);
873 mc->mc_abd = NULL;
874 }
875 }
876
877 if (good_copies == 0) {
878 zio->io_error = vdev_mirror_worst_error(mm);
879 ASSERT(zio->io_error != 0);
880 }
881
882 if (good_copies && spa_writeable(zio->io_spa) &&
883 (unexpected_errors ||
884 (zio->io_flags & ZIO_FLAG_RESILVER) ||
885 ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_resilvering))) {
886 /*
887 * Use the good data we have in hand to repair damaged children.
888 */
889 for (c = 0; c < mm->mm_children; c++) {
890 /*
891 * Don't rewrite known good children.
892 * Not only is it unnecessary, it could
893 * actually be harmful: if the system lost
894 * power while rewriting the only good copy,
895 * there would be no good copies left!
896 */
897 mc = &mm->mm_child[c];
898
899 if (mc->mc_error == 0) {
900 vdev_ops_t *ops = mc->mc_vd->vdev_ops;
901
902 if (mc->mc_tried)
903 continue;
904 /*
905 * We didn't try this child. We need to
906 * repair it if:
907 * 1. it's a scrub (in which case we have
908 * tried everything that was healthy)
909 * - or -
910 * 2. it's an indirect or distributed spare
911 * vdev (in which case it could point to any
912 * other vdev, which might have a bad DTL)
913 * - or -
914 * 3. the DTL indicates that this data is
915 * missing from this vdev
916 */
917 if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
918 ops != &vdev_indirect_ops &&
919 ops != &vdev_draid_spare_ops &&
920 !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
921 zio->io_txg, 1))
922 continue;
923 mc->mc_error = SET_ERROR(ESTALE);
924 }
925
926 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
927 mc->mc_vd, mc->mc_offset,
928 zio->io_abd, zio->io_size, ZIO_TYPE_WRITE,
929 zio->io_priority == ZIO_PRIORITY_REBUILD ?
930 ZIO_PRIORITY_REBUILD : ZIO_PRIORITY_ASYNC_WRITE,
931 ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
932 ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
933 }
934 }
935 }
936
937 static void
vdev_mirror_state_change(vdev_t * vd,int faulted,int degraded)938 vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
939 {
940 if (faulted == vd->vdev_children) {
941 if (vdev_children_are_offline(vd)) {
942 vdev_set_state(vd, B_FALSE, VDEV_STATE_OFFLINE,
943 VDEV_AUX_CHILDREN_OFFLINE);
944 } else {
945 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
946 VDEV_AUX_NO_REPLICAS);
947 }
948 } else if (degraded + faulted != 0) {
949 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
950 } else {
951 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
952 }
953 }
954
955 /*
956 * Return the maximum asize for a rebuild zio in the provided range.
957 */
958 static uint64_t
vdev_mirror_rebuild_asize(vdev_t * vd,uint64_t start,uint64_t asize,uint64_t max_segment)959 vdev_mirror_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize,
960 uint64_t max_segment)
961 {
962 (void) start;
963
964 uint64_t psize = MIN(P2ROUNDUP(max_segment, 1 << vd->vdev_ashift),
965 SPA_MAXBLOCKSIZE);
966
967 return (MIN(asize, vdev_psize_to_asize(vd, psize)));
968 }
969
970 vdev_ops_t vdev_mirror_ops = {
971 .vdev_op_init = NULL,
972 .vdev_op_fini = NULL,
973 .vdev_op_open = vdev_mirror_open,
974 .vdev_op_close = vdev_mirror_close,
975 .vdev_op_asize = vdev_default_asize,
976 .vdev_op_min_asize = vdev_default_min_asize,
977 .vdev_op_min_alloc = NULL,
978 .vdev_op_io_start = vdev_mirror_io_start,
979 .vdev_op_io_done = vdev_mirror_io_done,
980 .vdev_op_state_change = vdev_mirror_state_change,
981 .vdev_op_need_resilver = vdev_default_need_resilver,
982 .vdev_op_hold = NULL,
983 .vdev_op_rele = NULL,
984 .vdev_op_remap = NULL,
985 .vdev_op_xlate = vdev_default_xlate,
986 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
987 .vdev_op_metaslab_init = NULL,
988 .vdev_op_config_generate = NULL,
989 .vdev_op_nparity = NULL,
990 .vdev_op_ndisks = NULL,
991 .vdev_op_type = VDEV_TYPE_MIRROR, /* name of this vdev type */
992 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
993 };
994
995 vdev_ops_t vdev_replacing_ops = {
996 .vdev_op_init = NULL,
997 .vdev_op_fini = NULL,
998 .vdev_op_open = vdev_mirror_open,
999 .vdev_op_close = vdev_mirror_close,
1000 .vdev_op_asize = vdev_default_asize,
1001 .vdev_op_min_asize = vdev_default_min_asize,
1002 .vdev_op_min_alloc = NULL,
1003 .vdev_op_io_start = vdev_mirror_io_start,
1004 .vdev_op_io_done = vdev_mirror_io_done,
1005 .vdev_op_state_change = vdev_mirror_state_change,
1006 .vdev_op_need_resilver = vdev_default_need_resilver,
1007 .vdev_op_hold = NULL,
1008 .vdev_op_rele = NULL,
1009 .vdev_op_remap = NULL,
1010 .vdev_op_xlate = vdev_default_xlate,
1011 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1012 .vdev_op_metaslab_init = NULL,
1013 .vdev_op_config_generate = NULL,
1014 .vdev_op_nparity = NULL,
1015 .vdev_op_ndisks = NULL,
1016 .vdev_op_type = VDEV_TYPE_REPLACING, /* name of this vdev type */
1017 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1018 };
1019
1020 vdev_ops_t vdev_spare_ops = {
1021 .vdev_op_init = NULL,
1022 .vdev_op_fini = NULL,
1023 .vdev_op_open = vdev_mirror_open,
1024 .vdev_op_close = vdev_mirror_close,
1025 .vdev_op_asize = vdev_default_asize,
1026 .vdev_op_min_asize = vdev_default_min_asize,
1027 .vdev_op_min_alloc = NULL,
1028 .vdev_op_io_start = vdev_mirror_io_start,
1029 .vdev_op_io_done = vdev_mirror_io_done,
1030 .vdev_op_state_change = vdev_mirror_state_change,
1031 .vdev_op_need_resilver = vdev_default_need_resilver,
1032 .vdev_op_hold = NULL,
1033 .vdev_op_rele = NULL,
1034 .vdev_op_remap = NULL,
1035 .vdev_op_xlate = vdev_default_xlate,
1036 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1037 .vdev_op_metaslab_init = NULL,
1038 .vdev_op_config_generate = NULL,
1039 .vdev_op_nparity = NULL,
1040 .vdev_op_ndisks = NULL,
1041 .vdev_op_type = VDEV_TYPE_SPARE, /* name of this vdev type */
1042 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1043 };
1044
1045 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_inc, INT, ZMOD_RW,
1046 "Rotating media load increment for non-seeking I/Os");
1047
1048 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_inc, INT,
1049 ZMOD_RW, "Rotating media load increment for seeking I/Os");
1050
1051 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_offset, INT,
1052 ZMOD_RW,
1053 "Offset in bytes from the last I/O which triggers "
1054 "a reduced rotating media seek increment");
1055
1056 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_inc, INT,
1057 ZMOD_RW, "Non-rotating media load increment for non-seeking I/Os");
1058
1059 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_seek_inc, INT,
1060 ZMOD_RW, "Non-rotating media load increment for seeking I/Os");
1061