1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
29 */
30
31 #include <sys/zfs_context.h>
32 #include <sys/spa.h>
33 #include <sys/spa_impl.h>
34 #include <sys/dsl_pool.h>
35 #include <sys/dsl_scan.h>
36 #include <sys/vdev_impl.h>
37 #include <sys/vdev_draid.h>
38 #include <sys/zio.h>
39 #include <sys/zio_checksum.h>
40 #include <sys/abd.h>
41 #include <sys/fs/zfs.h>
42
43 /*
44 * Vdev mirror kstats
45 */
46 static kstat_t *mirror_ksp = NULL;
47
48 typedef struct mirror_stats {
49 kstat_named_t vdev_mirror_stat_rotating_linear;
50 kstat_named_t vdev_mirror_stat_rotating_offset;
51 kstat_named_t vdev_mirror_stat_rotating_seek;
52 kstat_named_t vdev_mirror_stat_non_rotating_linear;
53 kstat_named_t vdev_mirror_stat_non_rotating_seek;
54
55 kstat_named_t vdev_mirror_stat_preferred_found;
56 kstat_named_t vdev_mirror_stat_preferred_not_found;
57 } mirror_stats_t;
58
59 static mirror_stats_t mirror_stats = {
60 /* New I/O follows directly the last I/O */
61 { "rotating_linear", KSTAT_DATA_UINT64 },
62 /* New I/O is within zfs_vdev_mirror_rotating_seek_offset of the last */
63 { "rotating_offset", KSTAT_DATA_UINT64 },
64 /* New I/O requires random seek */
65 { "rotating_seek", KSTAT_DATA_UINT64 },
66 /* New I/O follows directly the last I/O (nonrot) */
67 { "non_rotating_linear", KSTAT_DATA_UINT64 },
68 /* New I/O requires random seek (nonrot) */
69 { "non_rotating_seek", KSTAT_DATA_UINT64 },
70 /* Preferred child vdev found */
71 { "preferred_found", KSTAT_DATA_UINT64 },
72 /* Preferred child vdev not found or equal load */
73 { "preferred_not_found", KSTAT_DATA_UINT64 },
74
75 };
76
77 #define MIRROR_STAT(stat) (mirror_stats.stat.value.ui64)
78 #define MIRROR_INCR(stat, val) atomic_add_64(&MIRROR_STAT(stat), val)
79 #define MIRROR_BUMP(stat) MIRROR_INCR(stat, 1)
80
81 void
vdev_mirror_stat_init(void)82 vdev_mirror_stat_init(void)
83 {
84 mirror_ksp = kstat_create("zfs", 0, "vdev_mirror_stats",
85 "misc", KSTAT_TYPE_NAMED,
86 sizeof (mirror_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
87 if (mirror_ksp != NULL) {
88 mirror_ksp->ks_data = &mirror_stats;
89 kstat_install(mirror_ksp);
90 }
91 }
92
93 void
vdev_mirror_stat_fini(void)94 vdev_mirror_stat_fini(void)
95 {
96 if (mirror_ksp != NULL) {
97 kstat_delete(mirror_ksp);
98 mirror_ksp = NULL;
99 }
100 }
101
102 /*
103 * Virtual device vector for mirroring.
104 */
105 typedef struct mirror_child {
106 vdev_t *mc_vd;
107 abd_t *mc_abd;
108 uint64_t mc_offset;
109 int mc_error;
110 int mc_load;
111 uint8_t mc_tried;
112 uint8_t mc_skipped;
113 uint8_t mc_speculative;
114 uint8_t mc_rebuilding;
115 } mirror_child_t;
116
117 typedef struct mirror_map {
118 int *mm_preferred;
119 int mm_preferred_cnt;
120 int mm_children;
121 boolean_t mm_resilvering;
122 boolean_t mm_rebuilding;
123 boolean_t mm_root;
124 mirror_child_t mm_child[];
125 } mirror_map_t;
126
127 static const int vdev_mirror_shift = 21;
128
129 /*
130 * The load configuration settings below are tuned by default for
131 * the case where all devices are of the same rotational type.
132 *
133 * If there is a mixture of rotating and non-rotating media, setting
134 * zfs_vdev_mirror_non_rotating_seek_inc to 0 may well provide better results
135 * as it will direct more reads to the non-rotating vdevs which are more likely
136 * to have a higher performance.
137 */
138
139 /* Rotating media load calculation configuration. */
140 static int zfs_vdev_mirror_rotating_inc = 0;
141 static int zfs_vdev_mirror_rotating_seek_inc = 5;
142 static int zfs_vdev_mirror_rotating_seek_offset = 1 * 1024 * 1024;
143
144 /* Non-rotating media load calculation configuration. */
145 static int zfs_vdev_mirror_non_rotating_inc = 0;
146 static int zfs_vdev_mirror_non_rotating_seek_inc = 1;
147
148 static inline size_t
vdev_mirror_map_size(int children)149 vdev_mirror_map_size(int children)
150 {
151 return (offsetof(mirror_map_t, mm_child[children]) +
152 sizeof (int) * children);
153 }
154
155 static inline mirror_map_t *
vdev_mirror_map_alloc(int children,boolean_t resilvering,boolean_t root)156 vdev_mirror_map_alloc(int children, boolean_t resilvering, boolean_t root)
157 {
158 mirror_map_t *mm;
159
160 mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP);
161 mm->mm_children = children;
162 mm->mm_resilvering = resilvering;
163 mm->mm_root = root;
164 mm->mm_preferred = (int *)((uintptr_t)mm +
165 offsetof(mirror_map_t, mm_child[children]));
166
167 return (mm);
168 }
169
170 static void
vdev_mirror_map_free(zio_t * zio)171 vdev_mirror_map_free(zio_t *zio)
172 {
173 mirror_map_t *mm = zio->io_vsd;
174
175 kmem_free(mm, vdev_mirror_map_size(mm->mm_children));
176 }
177
178 static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
179 .vsd_free = vdev_mirror_map_free,
180 };
181
182 static int
vdev_mirror_load(mirror_map_t * mm,vdev_t * vd,uint64_t zio_offset)183 vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset)
184 {
185 uint64_t last_offset;
186 int64_t offset_diff;
187 int load;
188
189 /* All DVAs have equal weight at the root. */
190 if (mm->mm_root)
191 return (INT_MAX);
192
193 /*
194 * We don't return INT_MAX if the device is resilvering i.e.
195 * vdev_resilver_txg != 0 as when tested performance was slightly
196 * worse overall when resilvering with compared to without.
197 */
198
199 /* Fix zio_offset for leaf vdevs */
200 if (vd->vdev_ops->vdev_op_leaf)
201 zio_offset += VDEV_LABEL_START_SIZE;
202
203 /* Standard load based on pending queue length. */
204 load = vdev_queue_length(vd);
205 last_offset = vdev_queue_last_offset(vd);
206
207 if (vd->vdev_nonrot) {
208 /* Non-rotating media. */
209 if (last_offset == zio_offset) {
210 MIRROR_BUMP(vdev_mirror_stat_non_rotating_linear);
211 return (load + zfs_vdev_mirror_non_rotating_inc);
212 }
213
214 /*
215 * Apply a seek penalty even for non-rotating devices as
216 * sequential I/O's can be aggregated into fewer operations on
217 * the device, thus avoiding unnecessary per-command overhead
218 * and boosting performance.
219 */
220 MIRROR_BUMP(vdev_mirror_stat_non_rotating_seek);
221 return (load + zfs_vdev_mirror_non_rotating_seek_inc);
222 }
223
224 /* Rotating media I/O's which directly follow the last I/O. */
225 if (last_offset == zio_offset) {
226 MIRROR_BUMP(vdev_mirror_stat_rotating_linear);
227 return (load + zfs_vdev_mirror_rotating_inc);
228 }
229
230 /*
231 * Apply half the seek increment to I/O's within seek offset
232 * of the last I/O issued to this vdev as they should incur less
233 * of a seek increment.
234 */
235 offset_diff = (int64_t)(last_offset - zio_offset);
236 if (ABS(offset_diff) < zfs_vdev_mirror_rotating_seek_offset) {
237 MIRROR_BUMP(vdev_mirror_stat_rotating_offset);
238 return (load + (zfs_vdev_mirror_rotating_seek_inc / 2));
239 }
240
241 /* Apply the full seek increment to all other I/O's. */
242 MIRROR_BUMP(vdev_mirror_stat_rotating_seek);
243 return (load + zfs_vdev_mirror_rotating_seek_inc);
244 }
245
246 static boolean_t
vdev_mirror_rebuilding(vdev_t * vd)247 vdev_mirror_rebuilding(vdev_t *vd)
248 {
249 if (vd->vdev_ops->vdev_op_leaf && vd->vdev_rebuild_txg)
250 return (B_TRUE);
251
252 for (int i = 0; i < vd->vdev_children; i++) {
253 if (vdev_mirror_rebuilding(vd->vdev_child[i])) {
254 return (B_TRUE);
255 }
256 }
257
258 return (B_FALSE);
259 }
260
261 /*
262 * Avoid inlining the function to keep vdev_mirror_io_start(), which
263 * is this functions only caller, as small as possible on the stack.
264 */
265 noinline static mirror_map_t *
vdev_mirror_map_init(zio_t * zio)266 vdev_mirror_map_init(zio_t *zio)
267 {
268 mirror_map_t *mm = NULL;
269 mirror_child_t *mc;
270 vdev_t *vd = zio->io_vd;
271 int c;
272
273 if (vd == NULL) {
274 dva_t *dva = zio->io_bp->blk_dva;
275 spa_t *spa = zio->io_spa;
276 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
277 dva_t dva_copy[SPA_DVAS_PER_BP];
278
279 /*
280 * The sequential scrub code sorts and issues all DVAs
281 * of a bp separately. Each of these IOs includes all
282 * original DVA copies so that repairs can be performed
283 * in the event of an error, but we only actually want
284 * to check the first DVA since the others will be
285 * checked by their respective sorted IOs. Only if we
286 * hit an error will we try all DVAs upon retrying.
287 *
288 * Note: This check is safe even if the user switches
289 * from a legacy scrub to a sequential one in the middle
290 * of processing, since scn_is_sorted isn't updated until
291 * all outstanding IOs from the previous scrub pass
292 * complete.
293 */
294 if ((zio->io_flags & ZIO_FLAG_SCRUB) &&
295 !(zio->io_flags & ZIO_FLAG_IO_RETRY) &&
296 dsl_scan_scrubbing(spa->spa_dsl_pool) &&
297 scn->scn_is_sorted) {
298 c = 1;
299 } else {
300 c = BP_GET_NDVAS(zio->io_bp);
301 }
302
303 /*
304 * If the pool cannot be written to, then infer that some
305 * DVAs might be invalid or point to vdevs that do not exist.
306 * We skip them.
307 */
308 if (!spa_writeable(spa)) {
309 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
310 int j = 0;
311 for (int i = 0; i < c; i++) {
312 if (zfs_dva_valid(spa, &dva[i], zio->io_bp))
313 dva_copy[j++] = dva[i];
314 }
315 if (j == 0) {
316 zio->io_vsd = NULL;
317 zio->io_error = ENXIO;
318 return (NULL);
319 }
320 if (j < c) {
321 dva = dva_copy;
322 c = j;
323 }
324 }
325
326 mm = vdev_mirror_map_alloc(c, B_FALSE, B_TRUE);
327 for (c = 0; c < mm->mm_children; c++) {
328 mc = &mm->mm_child[c];
329
330 mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
331 mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
332 if (mc->mc_vd == NULL) {
333 kmem_free(mm, vdev_mirror_map_size(
334 mm->mm_children));
335 zio->io_vsd = NULL;
336 zio->io_error = ENXIO;
337 return (NULL);
338 }
339 }
340 } else {
341 /*
342 * If we are resilvering, then we should handle scrub reads
343 * differently; we shouldn't issue them to the resilvering
344 * device because it might not have those blocks.
345 *
346 * We are resilvering iff:
347 * 1) We are a replacing vdev (ie our name is "replacing-1" or
348 * "spare-1" or something like that), and
349 * 2) The pool is currently being resilvered.
350 *
351 * We cannot simply check vd->vdev_resilver_txg, because it's
352 * not set in this path.
353 *
354 * Nor can we just check our vdev_ops; there are cases (such as
355 * when a user types "zpool replace pool odev spare_dev" and
356 * spare_dev is in the spare list, or when a spare device is
357 * automatically used to replace a DEGRADED device) when
358 * resilvering is complete but both the original vdev and the
359 * spare vdev remain in the pool. That behavior is intentional.
360 * It helps implement the policy that a spare should be
361 * automatically removed from the pool after the user replaces
362 * the device that originally failed.
363 *
364 * If a spa load is in progress, then spa_dsl_pool may be
365 * uninitialized. But we shouldn't be resilvering during a spa
366 * load anyway.
367 */
368 boolean_t replacing = (vd->vdev_ops == &vdev_replacing_ops ||
369 vd->vdev_ops == &vdev_spare_ops) &&
370 spa_load_state(vd->vdev_spa) == SPA_LOAD_NONE &&
371 dsl_scan_resilvering(vd->vdev_spa->spa_dsl_pool);
372 mm = vdev_mirror_map_alloc(vd->vdev_children, replacing,
373 B_FALSE);
374 for (c = 0; c < mm->mm_children; c++) {
375 mc = &mm->mm_child[c];
376 mc->mc_vd = vd->vdev_child[c];
377 mc->mc_offset = zio->io_offset;
378
379 if (vdev_mirror_rebuilding(mc->mc_vd))
380 mm->mm_rebuilding = mc->mc_rebuilding = B_TRUE;
381 }
382 }
383
384 return (mm);
385 }
386
387 static int
vdev_mirror_open(vdev_t * vd,uint64_t * asize,uint64_t * max_asize,uint64_t * logical_ashift,uint64_t * physical_ashift)388 vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
389 uint64_t *logical_ashift, uint64_t *physical_ashift)
390 {
391 int numerrors = 0;
392 int lasterror = 0;
393
394 if (vd->vdev_children == 0) {
395 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
396 return (SET_ERROR(EINVAL));
397 }
398
399 vdev_open_children(vd);
400
401 for (int c = 0; c < vd->vdev_children; c++) {
402 vdev_t *cvd = vd->vdev_child[c];
403
404 if (cvd->vdev_open_error) {
405 lasterror = cvd->vdev_open_error;
406 numerrors++;
407 continue;
408 }
409
410 *asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
411 *max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
412 *logical_ashift = MAX(*logical_ashift, cvd->vdev_ashift);
413 }
414 for (int c = 0; c < vd->vdev_children; c++) {
415 vdev_t *cvd = vd->vdev_child[c];
416
417 if (cvd->vdev_open_error)
418 continue;
419 *physical_ashift = vdev_best_ashift(*logical_ashift,
420 *physical_ashift, cvd->vdev_physical_ashift);
421 }
422
423 if (numerrors == vd->vdev_children) {
424 if (vdev_children_are_offline(vd))
425 vd->vdev_stat.vs_aux = VDEV_AUX_CHILDREN_OFFLINE;
426 else
427 vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
428 return (lasterror);
429 }
430
431 return (0);
432 }
433
434 static void
vdev_mirror_close(vdev_t * vd)435 vdev_mirror_close(vdev_t *vd)
436 {
437 for (int c = 0; c < vd->vdev_children; c++)
438 vdev_close(vd->vdev_child[c]);
439 }
440
441 static void
vdev_mirror_child_done(zio_t * zio)442 vdev_mirror_child_done(zio_t *zio)
443 {
444 mirror_child_t *mc = zio->io_private;
445
446 mc->mc_error = zio->io_error;
447 mc->mc_tried = 1;
448 mc->mc_skipped = 0;
449 }
450
451 /*
452 * Check the other, lower-index DVAs to see if they're on the same
453 * vdev as the child we picked. If they are, use them since they
454 * are likely to have been allocated from the primary metaslab in
455 * use at the time, and hence are more likely to have locality with
456 * single-copy data.
457 */
458 static int
vdev_mirror_dva_select(zio_t * zio,int p)459 vdev_mirror_dva_select(zio_t *zio, int p)
460 {
461 dva_t *dva = zio->io_bp->blk_dva;
462 mirror_map_t *mm = zio->io_vsd;
463 int preferred;
464 int c;
465
466 preferred = mm->mm_preferred[p];
467 for (p--; p >= 0; p--) {
468 c = mm->mm_preferred[p];
469 if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred]))
470 preferred = c;
471 }
472 return (preferred);
473 }
474
475 static int
vdev_mirror_preferred_child_randomize(zio_t * zio)476 vdev_mirror_preferred_child_randomize(zio_t *zio)
477 {
478 mirror_map_t *mm = zio->io_vsd;
479 int p;
480
481 if (mm->mm_root) {
482 p = random_in_range(mm->mm_preferred_cnt);
483 return (vdev_mirror_dva_select(zio, p));
484 }
485
486 /*
487 * To ensure we don't always favour the first matching vdev,
488 * which could lead to wear leveling issues on SSD's, we
489 * use the I/O offset as a pseudo random seed into the vdevs
490 * which have the lowest load.
491 */
492 p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt;
493 return (mm->mm_preferred[p]);
494 }
495
496 static boolean_t
vdev_mirror_child_readable(mirror_child_t * mc)497 vdev_mirror_child_readable(mirror_child_t *mc)
498 {
499 vdev_t *vd = mc->mc_vd;
500
501 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
502 return (vdev_draid_readable(vd, mc->mc_offset));
503 else
504 return (vdev_readable(vd));
505 }
506
507 static boolean_t
vdev_mirror_child_missing(mirror_child_t * mc,uint64_t txg,uint64_t size)508 vdev_mirror_child_missing(mirror_child_t *mc, uint64_t txg, uint64_t size)
509 {
510 vdev_t *vd = mc->mc_vd;
511
512 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
513 return (vdev_draid_missing(vd, mc->mc_offset, txg, size));
514 else
515 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size));
516 }
517
518 /*
519 * Try to find a vdev whose DTL doesn't contain the block we want to read
520 * preferring vdevs based on determined load. If we can't, try the read on
521 * any vdev we haven't already tried.
522 *
523 * Distributed spares are an exception to the above load rule. They are
524 * always preferred in order to detect gaps in the distributed spare which
525 * are created when another disk in the dRAID fails. In order to restore
526 * redundancy those gaps must be read to trigger the required repair IO.
527 */
528 static int
vdev_mirror_child_select(zio_t * zio)529 vdev_mirror_child_select(zio_t *zio)
530 {
531 mirror_map_t *mm = zio->io_vsd;
532 uint64_t txg = zio->io_txg;
533 int c, lowest_load;
534
535 ASSERT(zio->io_bp == NULL || BP_GET_PHYSICAL_BIRTH(zio->io_bp) == txg);
536
537 lowest_load = INT_MAX;
538 mm->mm_preferred_cnt = 0;
539 for (c = 0; c < mm->mm_children; c++) {
540 mirror_child_t *mc;
541
542 mc = &mm->mm_child[c];
543 if (mc->mc_tried || mc->mc_skipped)
544 continue;
545
546 if (mc->mc_vd == NULL ||
547 !vdev_mirror_child_readable(mc)) {
548 mc->mc_error = SET_ERROR(ENXIO);
549 mc->mc_tried = 1; /* don't even try */
550 mc->mc_skipped = 1;
551 continue;
552 }
553
554 if (vdev_mirror_child_missing(mc, txg, 1)) {
555 mc->mc_error = SET_ERROR(ESTALE);
556 mc->mc_skipped = 1;
557 mc->mc_speculative = 1;
558 continue;
559 }
560
561 if (mc->mc_vd->vdev_ops == &vdev_draid_spare_ops) {
562 mm->mm_preferred[0] = c;
563 mm->mm_preferred_cnt = 1;
564 break;
565 }
566
567 mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset);
568 if (mc->mc_load > lowest_load)
569 continue;
570
571 if (mc->mc_load < lowest_load) {
572 lowest_load = mc->mc_load;
573 mm->mm_preferred_cnt = 0;
574 }
575 mm->mm_preferred[mm->mm_preferred_cnt] = c;
576 mm->mm_preferred_cnt++;
577 }
578
579 if (mm->mm_preferred_cnt == 1) {
580 MIRROR_BUMP(vdev_mirror_stat_preferred_found);
581 return (mm->mm_preferred[0]);
582 }
583
584 if (mm->mm_preferred_cnt > 1) {
585 MIRROR_BUMP(vdev_mirror_stat_preferred_not_found);
586 return (vdev_mirror_preferred_child_randomize(zio));
587 }
588
589 /*
590 * Every device is either missing or has this txg in its DTL.
591 * Look for any child we haven't already tried before giving up.
592 */
593 for (c = 0; c < mm->mm_children; c++) {
594 if (!mm->mm_child[c].mc_tried)
595 return (c);
596 }
597
598 /*
599 * Every child failed. There's no place left to look.
600 */
601 return (-1);
602 }
603
604 static void
vdev_mirror_io_start(zio_t * zio)605 vdev_mirror_io_start(zio_t *zio)
606 {
607 mirror_map_t *mm;
608 mirror_child_t *mc;
609 int c, children;
610
611 mm = vdev_mirror_map_init(zio);
612 zio->io_vsd = mm;
613 zio->io_vsd_ops = &vdev_mirror_vsd_ops;
614
615 if (mm == NULL) {
616 ASSERT(!spa_trust_config(zio->io_spa));
617 ASSERT(zio->io_type == ZIO_TYPE_READ);
618 zio_execute(zio);
619 return;
620 }
621
622 if (zio->io_type == ZIO_TYPE_READ) {
623 if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_resilvering) {
624 /*
625 * For scrubbing reads we need to issue reads to all
626 * children. One child can reuse parent buffer, but
627 * for others we have to allocate separate ones to
628 * verify checksums if io_bp is non-NULL, or compare
629 * them in vdev_mirror_io_done() otherwise.
630 */
631 boolean_t first = B_TRUE;
632 for (c = 0; c < mm->mm_children; c++) {
633 mc = &mm->mm_child[c];
634
635 /* Don't issue ZIOs to offline children */
636 if (!vdev_mirror_child_readable(mc)) {
637 mc->mc_error = SET_ERROR(ENXIO);
638 mc->mc_tried = 1;
639 mc->mc_skipped = 1;
640 continue;
641 }
642
643 mc->mc_abd = first ? zio->io_abd :
644 abd_alloc_sametype(zio->io_abd,
645 zio->io_size);
646 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
647 mc->mc_vd, mc->mc_offset, mc->mc_abd,
648 zio->io_size, zio->io_type,
649 zio->io_priority, 0,
650 vdev_mirror_child_done, mc));
651 first = B_FALSE;
652 }
653 zio_execute(zio);
654 return;
655 }
656 /*
657 * For normal reads just pick one child.
658 */
659 c = vdev_mirror_child_select(zio);
660 children = (c >= 0);
661 } else {
662 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
663
664 /*
665 * Writes go to all children.
666 */
667 c = 0;
668 children = mm->mm_children;
669 }
670
671 while (children--) {
672 mc = &mm->mm_child[c++];
673
674 /*
675 * When sequentially resilvering and the integrity of the data
676 * is speculative (ZIO_FLAG_SPECULATIVE), issue write repair IOs
677 * only to the vdev which is being rebuilt. Existing data on
678 * other children must never be overwritten with unconfirmed
679 * data to avoid unrecoverable damage to the pool.
680 */
681 if ((zio->io_priority == ZIO_PRIORITY_REBUILD) &&
682 (zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
683 !(zio->io_flags & ZIO_FLAG_SCRUB) &&
684 (zio->io_flags & ZIO_FLAG_SPECULATIVE) &&
685 mm->mm_rebuilding && !mc->mc_rebuilding) {
686 continue;
687 }
688
689 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
690 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
691 zio->io_type, zio->io_priority, 0,
692 vdev_mirror_child_done, mc));
693 }
694
695 zio_execute(zio);
696 }
697
698 static int
vdev_mirror_worst_error(mirror_map_t * mm)699 vdev_mirror_worst_error(mirror_map_t *mm)
700 {
701 int error[2] = { 0, 0 };
702
703 for (int c = 0; c < mm->mm_children; c++) {
704 mirror_child_t *mc = &mm->mm_child[c];
705 int s = mc->mc_speculative;
706 error[s] = zio_worst_error(error[s], mc->mc_error);
707 }
708
709 return (error[0] ? error[0] : error[1]);
710 }
711
712 static void
vdev_mirror_io_done(zio_t * zio)713 vdev_mirror_io_done(zio_t *zio)
714 {
715 mirror_map_t *mm = zio->io_vsd;
716 mirror_child_t *mc;
717 int c;
718 int good_copies = 0;
719 int unexpected_errors = 0;
720 int last_good_copy = -1;
721
722 if (mm == NULL)
723 return;
724
725 for (c = 0; c < mm->mm_children; c++) {
726 mc = &mm->mm_child[c];
727
728 if (mc->mc_error) {
729 if (!mc->mc_skipped)
730 unexpected_errors++;
731 } else if (mc->mc_tried) {
732 last_good_copy = c;
733 good_copies++;
734 }
735 }
736
737 if (zio->io_type == ZIO_TYPE_WRITE) {
738 /*
739 * XXX -- for now, treat partial writes as success.
740 *
741 * Now that we support write reallocation, it would be better
742 * to treat partial failure as real failure unless there are
743 * no non-degraded top-level vdevs left, and not update DTLs
744 * if we intend to reallocate.
745 */
746 if (good_copies != mm->mm_children) {
747 /*
748 * Always require at least one good copy.
749 *
750 * For ditto blocks (io_vd == NULL), require
751 * all copies to be good.
752 *
753 * XXX -- for replacing vdevs, there's no great answer.
754 * If the old device is really dead, we may not even
755 * be able to access it -- so we only want to
756 * require good writes to the new device. But if
757 * the new device turns out to be flaky, we want
758 * to be able to detach it -- which requires all
759 * writes to the old device to have succeeded.
760 */
761 if (good_copies == 0 || zio->io_vd == NULL)
762 zio->io_error = vdev_mirror_worst_error(mm);
763 }
764 return;
765 }
766
767 ASSERT(zio->io_type == ZIO_TYPE_READ);
768
769 /*
770 * Any Direct I/O read that has a checksum error must be treated as
771 * suspicious as the contents of the buffer could be getting
772 * manipulated while the I/O is taking place. The checksum verify error
773 * will be reported to the top-level Mirror VDEV.
774 *
775 * There will be no attampt at reading any additional data copies. If
776 * the buffer is still being manipulated while attempting to read from
777 * another child, there exists a possibly that the checksum could be
778 * verified as valid. However, the buffer contents could again get
779 * manipulated after verifying the checksum. This would lead to bad data
780 * being written out during self healing.
781 */
782 if ((zio->io_flags & ZIO_FLAG_DIO_READ) &&
783 (zio->io_post & ZIO_POST_DIO_CHKSUM_ERR)) {
784 zio_dio_chksum_verify_error_report(zio);
785 zio->io_error = vdev_mirror_worst_error(mm);
786 ASSERT3U(zio->io_error, ==, ECKSUM);
787 return;
788 }
789
790 /*
791 * If we don't have a good copy yet, keep trying other children.
792 */
793 if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
794 ASSERT(c >= 0 && c < mm->mm_children);
795 mc = &mm->mm_child[c];
796 zio_vdev_io_redone(zio);
797 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
798 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
799 ZIO_TYPE_READ, zio->io_priority, 0,
800 vdev_mirror_child_done, mc));
801 return;
802 }
803
804 if (zio->io_flags & ZIO_FLAG_SCRUB && !mm->mm_resilvering) {
805 abd_t *best_abd = NULL;
806 if (last_good_copy >= 0)
807 best_abd = mm->mm_child[last_good_copy].mc_abd;
808
809 /*
810 * If we're scrubbing but don't have a BP available (because
811 * this vdev is under a raidz or draid vdev) then the best we
812 * can do is compare all of the copies read. If they're not
813 * identical then return a checksum error and the most likely
814 * correct data. The raidz code will issue a repair I/O if
815 * possible.
816 */
817 if (zio->io_bp == NULL) {
818 ASSERT(zio->io_vd->vdev_ops == &vdev_replacing_ops ||
819 zio->io_vd->vdev_ops == &vdev_spare_ops);
820
821 abd_t *pref_abd = NULL;
822 for (c = 0; c < last_good_copy; c++) {
823 mc = &mm->mm_child[c];
824 if (mc->mc_error || !mc->mc_tried)
825 continue;
826
827 if (abd_cmp(mc->mc_abd, best_abd) != 0)
828 zio->io_error = SET_ERROR(ECKSUM);
829
830 /*
831 * The distributed spare is always prefered
832 * by vdev_mirror_child_select() so it's
833 * considered to be the best candidate.
834 */
835 if (pref_abd == NULL &&
836 mc->mc_vd->vdev_ops ==
837 &vdev_draid_spare_ops)
838 pref_abd = mc->mc_abd;
839
840 /*
841 * In the absence of a preferred copy, use
842 * the parent pointer to avoid a memory copy.
843 */
844 if (mc->mc_abd == zio->io_abd)
845 best_abd = mc->mc_abd;
846 }
847 if (pref_abd)
848 best_abd = pref_abd;
849 } else {
850
851 /*
852 * If we have a BP available, then checksums are
853 * already verified and we just need a buffer
854 * with valid data, preferring parent one to
855 * avoid a memory copy.
856 */
857 for (c = 0; c < last_good_copy; c++) {
858 mc = &mm->mm_child[c];
859 if (mc->mc_error || !mc->mc_tried)
860 continue;
861 if (mc->mc_abd == zio->io_abd) {
862 best_abd = mc->mc_abd;
863 break;
864 }
865 }
866 }
867
868 if (best_abd && best_abd != zio->io_abd)
869 abd_copy(zio->io_abd, best_abd, zio->io_size);
870 for (c = 0; c < mm->mm_children; c++) {
871 mc = &mm->mm_child[c];
872 if (mc->mc_abd != zio->io_abd)
873 abd_free(mc->mc_abd);
874 mc->mc_abd = NULL;
875 }
876 }
877
878 if (good_copies == 0) {
879 zio->io_error = vdev_mirror_worst_error(mm);
880 ASSERT(zio->io_error != 0);
881 }
882
883 if (good_copies && spa_writeable(zio->io_spa) &&
884 (unexpected_errors ||
885 (zio->io_flags & ZIO_FLAG_RESILVER) ||
886 ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_resilvering))) {
887 /*
888 * Use the good data we have in hand to repair damaged children.
889 */
890 for (c = 0; c < mm->mm_children; c++) {
891 /*
892 * Don't rewrite known good children.
893 * Not only is it unnecessary, it could
894 * actually be harmful: if the system lost
895 * power while rewriting the only good copy,
896 * there would be no good copies left!
897 */
898 mc = &mm->mm_child[c];
899
900 if (mc->mc_error == 0) {
901 vdev_ops_t *ops = mc->mc_vd->vdev_ops;
902
903 if (mc->mc_tried)
904 continue;
905 /*
906 * We didn't try this child. We need to
907 * repair it if:
908 * 1. it's a scrub (in which case we have
909 * tried everything that was healthy)
910 * - or -
911 * 2. it's an indirect or distributed spare
912 * vdev (in which case it could point to any
913 * other vdev, which might have a bad DTL)
914 * - or -
915 * 3. the DTL indicates that this data is
916 * missing from this vdev
917 */
918 if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
919 ops != &vdev_indirect_ops &&
920 ops != &vdev_draid_spare_ops &&
921 !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
922 zio->io_txg, 1))
923 continue;
924 mc->mc_error = SET_ERROR(ESTALE);
925 }
926
927 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
928 mc->mc_vd, mc->mc_offset,
929 zio->io_abd, zio->io_size, ZIO_TYPE_WRITE,
930 zio->io_priority == ZIO_PRIORITY_REBUILD ?
931 ZIO_PRIORITY_REBUILD : ZIO_PRIORITY_ASYNC_WRITE,
932 ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
933 ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
934 }
935 }
936 }
937
938 static void
vdev_mirror_state_change(vdev_t * vd,int faulted,int degraded)939 vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
940 {
941 if (faulted == vd->vdev_children) {
942 if (vdev_children_are_offline(vd)) {
943 vdev_set_state(vd, B_FALSE, VDEV_STATE_OFFLINE,
944 VDEV_AUX_CHILDREN_OFFLINE);
945 } else {
946 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
947 VDEV_AUX_NO_REPLICAS);
948 }
949 } else if (degraded + faulted != 0) {
950 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
951 } else {
952 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
953 }
954 }
955
956 /*
957 * Return the maximum asize for a rebuild zio in the provided range.
958 */
959 static uint64_t
vdev_mirror_rebuild_asize(vdev_t * vd,uint64_t start,uint64_t asize,uint64_t max_segment)960 vdev_mirror_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize,
961 uint64_t max_segment)
962 {
963 (void) start;
964
965 uint64_t psize = MIN(P2ROUNDUP(max_segment, 1 << vd->vdev_ashift),
966 SPA_MAXBLOCKSIZE);
967
968 return (MIN(asize, vdev_psize_to_asize(vd, psize)));
969 }
970
971 vdev_ops_t vdev_mirror_ops = {
972 .vdev_op_init = NULL,
973 .vdev_op_fini = NULL,
974 .vdev_op_open = vdev_mirror_open,
975 .vdev_op_close = vdev_mirror_close,
976 .vdev_op_psize_to_asize = vdev_default_asize,
977 .vdev_op_asize_to_psize = vdev_default_psize,
978 .vdev_op_min_asize = vdev_default_min_asize,
979 .vdev_op_min_alloc = NULL,
980 .vdev_op_io_start = vdev_mirror_io_start,
981 .vdev_op_io_done = vdev_mirror_io_done,
982 .vdev_op_state_change = vdev_mirror_state_change,
983 .vdev_op_need_resilver = vdev_default_need_resilver,
984 .vdev_op_hold = NULL,
985 .vdev_op_rele = NULL,
986 .vdev_op_remap = NULL,
987 .vdev_op_xlate = vdev_default_xlate,
988 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
989 .vdev_op_metaslab_init = NULL,
990 .vdev_op_config_generate = NULL,
991 .vdev_op_nparity = NULL,
992 .vdev_op_ndisks = NULL,
993 .vdev_op_type = VDEV_TYPE_MIRROR, /* name of this vdev type */
994 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
995 };
996
997 vdev_ops_t vdev_replacing_ops = {
998 .vdev_op_init = NULL,
999 .vdev_op_fini = NULL,
1000 .vdev_op_open = vdev_mirror_open,
1001 .vdev_op_close = vdev_mirror_close,
1002 .vdev_op_psize_to_asize = vdev_default_asize,
1003 .vdev_op_asize_to_psize = vdev_default_psize,
1004 .vdev_op_min_asize = vdev_default_min_asize,
1005 .vdev_op_min_alloc = NULL,
1006 .vdev_op_io_start = vdev_mirror_io_start,
1007 .vdev_op_io_done = vdev_mirror_io_done,
1008 .vdev_op_state_change = vdev_mirror_state_change,
1009 .vdev_op_need_resilver = vdev_default_need_resilver,
1010 .vdev_op_hold = NULL,
1011 .vdev_op_rele = NULL,
1012 .vdev_op_remap = NULL,
1013 .vdev_op_xlate = vdev_default_xlate,
1014 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1015 .vdev_op_metaslab_init = NULL,
1016 .vdev_op_config_generate = NULL,
1017 .vdev_op_nparity = NULL,
1018 .vdev_op_ndisks = NULL,
1019 .vdev_op_type = VDEV_TYPE_REPLACING, /* name of this vdev type */
1020 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1021 };
1022
1023 vdev_ops_t vdev_spare_ops = {
1024 .vdev_op_init = NULL,
1025 .vdev_op_fini = NULL,
1026 .vdev_op_open = vdev_mirror_open,
1027 .vdev_op_close = vdev_mirror_close,
1028 .vdev_op_psize_to_asize = vdev_default_asize,
1029 .vdev_op_asize_to_psize = vdev_default_psize,
1030 .vdev_op_min_asize = vdev_default_min_asize,
1031 .vdev_op_min_alloc = NULL,
1032 .vdev_op_io_start = vdev_mirror_io_start,
1033 .vdev_op_io_done = vdev_mirror_io_done,
1034 .vdev_op_state_change = vdev_mirror_state_change,
1035 .vdev_op_need_resilver = vdev_default_need_resilver,
1036 .vdev_op_hold = NULL,
1037 .vdev_op_rele = NULL,
1038 .vdev_op_remap = NULL,
1039 .vdev_op_xlate = vdev_default_xlate,
1040 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1041 .vdev_op_metaslab_init = NULL,
1042 .vdev_op_config_generate = NULL,
1043 .vdev_op_nparity = NULL,
1044 .vdev_op_ndisks = NULL,
1045 .vdev_op_type = VDEV_TYPE_SPARE, /* name of this vdev type */
1046 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1047 };
1048
1049 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_inc, INT, ZMOD_RW,
1050 "Rotating media load increment for non-seeking I/Os");
1051
1052 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_inc, INT,
1053 ZMOD_RW, "Rotating media load increment for seeking I/Os");
1054
1055 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_offset, INT,
1056 ZMOD_RW,
1057 "Offset in bytes from the last I/O which triggers "
1058 "a reduced rotating media seek increment");
1059
1060 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_inc, INT,
1061 ZMOD_RW, "Non-rotating media load increment for non-seeking I/Os");
1062
1063 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_seek_inc, INT,
1064 ZMOD_RW, "Non-rotating media load increment for seeking I/Os");
1065