1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
14 * Use is subject to license terms.
15 */
16
17 /*
18 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
19 */
20
21 #include <sys/zfs_context.h>
22 #include <sys/spa.h>
23 #include <sys/spa_impl.h>
24 #include <sys/dsl_pool.h>
25 #include <sys/dsl_scan.h>
26 #include <sys/vdev_impl.h>
27 #include <sys/vdev_draid.h>
28 #include <sys/zio.h>
29 #include <sys/zio_checksum.h>
30 #include <sys/abd.h>
31 #include <sys/fs/zfs.h>
32
33 /*
34 * Vdev mirror kstats
35 */
36 static kstat_t *mirror_ksp = NULL;
37
38 typedef struct mirror_stats {
39 kstat_named_t vdev_mirror_stat_rotating_linear;
40 kstat_named_t vdev_mirror_stat_rotating_offset;
41 kstat_named_t vdev_mirror_stat_rotating_seek;
42 kstat_named_t vdev_mirror_stat_non_rotating_linear;
43 kstat_named_t vdev_mirror_stat_non_rotating_seek;
44
45 kstat_named_t vdev_mirror_stat_preferred_found;
46 kstat_named_t vdev_mirror_stat_preferred_not_found;
47 } mirror_stats_t;
48
49 static mirror_stats_t mirror_stats = {
50 /* New I/O follows directly the last I/O */
51 { "rotating_linear", KSTAT_DATA_UINT64 },
52 /* New I/O is within zfs_vdev_mirror_rotating_seek_offset of the last */
53 { "rotating_offset", KSTAT_DATA_UINT64 },
54 /* New I/O requires random seek */
55 { "rotating_seek", KSTAT_DATA_UINT64 },
56 /* New I/O follows directly the last I/O (nonrot) */
57 { "non_rotating_linear", KSTAT_DATA_UINT64 },
58 /* New I/O requires random seek (nonrot) */
59 { "non_rotating_seek", KSTAT_DATA_UINT64 },
60 /* Preferred child vdev found */
61 { "preferred_found", KSTAT_DATA_UINT64 },
62 /* Preferred child vdev not found or equal load */
63 { "preferred_not_found", KSTAT_DATA_UINT64 },
64
65 };
66
67 #define MIRROR_STAT(stat) (mirror_stats.stat.value.ui64)
68 #define MIRROR_INCR(stat, val) atomic_add_64(&MIRROR_STAT(stat), val)
69 #define MIRROR_BUMP(stat) MIRROR_INCR(stat, 1)
70
71 void
vdev_mirror_stat_init(void)72 vdev_mirror_stat_init(void)
73 {
74 mirror_ksp = kstat_create("zfs", 0, "vdev_mirror_stats",
75 "misc", KSTAT_TYPE_NAMED,
76 sizeof (mirror_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
77 if (mirror_ksp != NULL) {
78 mirror_ksp->ks_data = &mirror_stats;
79 kstat_install(mirror_ksp);
80 }
81 }
82
83 void
vdev_mirror_stat_fini(void)84 vdev_mirror_stat_fini(void)
85 {
86 if (mirror_ksp != NULL) {
87 kstat_delete(mirror_ksp);
88 mirror_ksp = NULL;
89 }
90 }
91
92 /*
93 * Virtual device vector for mirroring.
94 */
95 typedef struct mirror_child {
96 vdev_t *mc_vd;
97 abd_t *mc_abd;
98 uint64_t mc_offset;
99 int mc_error;
100 int mc_load;
101 uint8_t mc_tried;
102 uint8_t mc_skipped;
103 uint8_t mc_speculative;
104 uint8_t mc_rebuilding;
105 } mirror_child_t;
106
107 typedef struct mirror_map {
108 int *mm_preferred;
109 int mm_preferred_cnt;
110 int mm_children;
111 boolean_t mm_resilvering;
112 boolean_t mm_rebuilding;
113 boolean_t mm_root;
114 mirror_child_t mm_child[];
115 } mirror_map_t;
116
117 static const int vdev_mirror_shift = 21;
118
119 /*
120 * The load configuration settings below are tuned by default for
121 * the case where all devices are of the same rotational type.
122 *
123 * If there is a mixture of rotating and non-rotating media, setting
124 * zfs_vdev_mirror_non_rotating_seek_inc to 0 may well provide better results
125 * as it will direct more reads to the non-rotating vdevs which are more likely
126 * to have a higher performance.
127 */
128
129 /* Rotating media load calculation configuration. */
130 static int zfs_vdev_mirror_rotating_inc = 0;
131 static int zfs_vdev_mirror_rotating_seek_inc = 5;
132 static int zfs_vdev_mirror_rotating_seek_offset = 1 * 1024 * 1024;
133
134 /* Non-rotating media load calculation configuration. */
135 static int zfs_vdev_mirror_non_rotating_inc = 0;
136 static int zfs_vdev_mirror_non_rotating_seek_inc = 1;
137
138 static inline size_t
vdev_mirror_map_size(int children)139 vdev_mirror_map_size(int children)
140 {
141 return (offsetof(mirror_map_t, mm_child[children]) +
142 sizeof (int) * children);
143 }
144
145 static inline mirror_map_t *
vdev_mirror_map_alloc(int children,boolean_t resilvering,boolean_t root)146 vdev_mirror_map_alloc(int children, boolean_t resilvering, boolean_t root)
147 {
148 mirror_map_t *mm;
149
150 mm = kmem_zalloc(vdev_mirror_map_size(children), KM_SLEEP);
151 mm->mm_children = children;
152 mm->mm_resilvering = resilvering;
153 mm->mm_root = root;
154 mm->mm_preferred = (int *)((uintptr_t)mm +
155 offsetof(mirror_map_t, mm_child[children]));
156
157 return (mm);
158 }
159
160 static void
vdev_mirror_map_free(zio_t * zio)161 vdev_mirror_map_free(zio_t *zio)
162 {
163 mirror_map_t *mm = zio->io_vsd;
164
165 kmem_free(mm, vdev_mirror_map_size(mm->mm_children));
166 }
167
168 static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
169 .vsd_free = vdev_mirror_map_free,
170 };
171
172 static int
vdev_mirror_load(mirror_map_t * mm,vdev_t * vd,uint64_t zio_offset)173 vdev_mirror_load(mirror_map_t *mm, vdev_t *vd, uint64_t zio_offset)
174 {
175 uint64_t last_offset;
176 int64_t offset_diff;
177 int load;
178
179 /* All DVAs have equal weight at the root. */
180 if (mm->mm_root)
181 return (INT_MAX);
182
183 /*
184 * We don't return INT_MAX if the device is resilvering i.e.
185 * vdev_resilver_txg != 0 as when tested performance was slightly
186 * worse overall when resilvering with compared to without.
187 */
188
189 /* Fix zio_offset for leaf vdevs */
190 if (vd->vdev_ops->vdev_op_leaf)
191 zio_offset += VDEV_LABEL_START_SIZE;
192
193 /* Standard load based on pending queue length. */
194 load = vdev_queue_length(vd);
195 last_offset = vdev_queue_last_offset(vd);
196
197 if (vd->vdev_nonrot) {
198 /* Non-rotating media. */
199 if (last_offset == zio_offset) {
200 MIRROR_BUMP(vdev_mirror_stat_non_rotating_linear);
201 return (load + zfs_vdev_mirror_non_rotating_inc);
202 }
203
204 /*
205 * Apply a seek penalty even for non-rotating devices as
206 * sequential I/O's can be aggregated into fewer operations on
207 * the device, thus avoiding unnecessary per-command overhead
208 * and boosting performance.
209 */
210 MIRROR_BUMP(vdev_mirror_stat_non_rotating_seek);
211 return (load + zfs_vdev_mirror_non_rotating_seek_inc);
212 }
213
214 /* Rotating media I/O's which directly follow the last I/O. */
215 if (last_offset == zio_offset) {
216 MIRROR_BUMP(vdev_mirror_stat_rotating_linear);
217 return (load + zfs_vdev_mirror_rotating_inc);
218 }
219
220 /*
221 * Apply half the seek increment to I/O's within seek offset
222 * of the last I/O issued to this vdev as they should incur less
223 * of a seek increment.
224 */
225 offset_diff = (int64_t)(last_offset - zio_offset);
226 if (ABS(offset_diff) < zfs_vdev_mirror_rotating_seek_offset) {
227 MIRROR_BUMP(vdev_mirror_stat_rotating_offset);
228 return (load + (zfs_vdev_mirror_rotating_seek_inc / 2));
229 }
230
231 /* Apply the full seek increment to all other I/O's. */
232 MIRROR_BUMP(vdev_mirror_stat_rotating_seek);
233 return (load + zfs_vdev_mirror_rotating_seek_inc);
234 }
235
236 static boolean_t
vdev_mirror_rebuilding(vdev_t * vd)237 vdev_mirror_rebuilding(vdev_t *vd)
238 {
239 if (vd->vdev_ops->vdev_op_leaf && vd->vdev_rebuild_txg)
240 return (B_TRUE);
241
242 for (int i = 0; i < vd->vdev_children; i++) {
243 if (vdev_mirror_rebuilding(vd->vdev_child[i])) {
244 return (B_TRUE);
245 }
246 }
247
248 return (B_FALSE);
249 }
250
251 /*
252 * Avoid inlining the function to keep vdev_mirror_io_start(), which
253 * is this functions only caller, as small as possible on the stack.
254 */
255 noinline static mirror_map_t *
vdev_mirror_map_init(zio_t * zio)256 vdev_mirror_map_init(zio_t *zio)
257 {
258 mirror_map_t *mm = NULL;
259 mirror_child_t *mc;
260 vdev_t *vd = zio->io_vd;
261 int c;
262
263 if (vd == NULL) {
264 dva_t *dva = zio->io_bp->blk_dva;
265 spa_t *spa = zio->io_spa;
266 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
267 dva_t dva_copy[SPA_DVAS_PER_BP];
268
269 /*
270 * The sequential scrub code sorts and issues all DVAs
271 * of a bp separately. Each of these IOs includes all
272 * original DVA copies so that repairs can be performed
273 * in the event of an error, but we only actually want
274 * to check the first DVA since the others will be
275 * checked by their respective sorted IOs. Only if we
276 * hit an error will we try all DVAs upon retrying.
277 *
278 * Note: This check is safe even if the user switches
279 * from a legacy scrub to a sequential one in the middle
280 * of processing, since scn_is_sorted isn't updated until
281 * all outstanding IOs from the previous scrub pass
282 * complete.
283 */
284 if ((zio->io_flags & ZIO_FLAG_SCRUB) &&
285 !(zio->io_flags & ZIO_FLAG_IO_RETRY) &&
286 dsl_scan_scrubbing(spa->spa_dsl_pool) &&
287 scn->scn_is_sorted) {
288 c = 1;
289 } else {
290 c = BP_GET_NDVAS(zio->io_bp);
291 }
292
293 /*
294 * If the pool cannot be written to, then infer that some
295 * DVAs might be invalid or point to vdevs that do not exist.
296 * We skip them.
297 */
298 if (!spa_writeable(spa)) {
299 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ);
300 int j = 0;
301 for (int i = 0; i < c; i++) {
302 if (zfs_dva_valid(spa, &dva[i], zio->io_bp))
303 dva_copy[j++] = dva[i];
304 }
305 if (j == 0) {
306 zio->io_vsd = NULL;
307 zio->io_error = ENXIO;
308 return (NULL);
309 }
310 if (j < c) {
311 dva = dva_copy;
312 c = j;
313 }
314 }
315
316 mm = vdev_mirror_map_alloc(c, B_FALSE, B_TRUE);
317 for (c = 0; c < mm->mm_children; c++) {
318 mc = &mm->mm_child[c];
319
320 mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
321 mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
322 if (mc->mc_vd == NULL) {
323 kmem_free(mm, vdev_mirror_map_size(
324 mm->mm_children));
325 zio->io_vsd = NULL;
326 zio->io_error = ENXIO;
327 return (NULL);
328 }
329 }
330 } else {
331 /*
332 * If we are resilvering, then we should handle scrub reads
333 * differently; we shouldn't issue them to the resilvering
334 * device because it might not have those blocks.
335 *
336 * We are resilvering iff:
337 * 1) We are a replacing vdev (ie our name is "replacing-1" or
338 * "spare-1" or something like that), and
339 * 2) The pool is currently being resilvered.
340 *
341 * We cannot simply check vd->vdev_resilver_txg, because it's
342 * not set in this path.
343 *
344 * Nor can we just check our vdev_ops; there are cases (such as
345 * when a user types "zpool replace pool odev spare_dev" and
346 * spare_dev is in the spare list, or when a spare device is
347 * automatically used to replace a DEGRADED device) when
348 * resilvering is complete but both the original vdev and the
349 * spare vdev remain in the pool. That behavior is intentional.
350 * It helps implement the policy that a spare should be
351 * automatically removed from the pool after the user replaces
352 * the device that originally failed.
353 *
354 * If a spa load is in progress, then spa_dsl_pool may be
355 * uninitialized. But we shouldn't be resilvering during a spa
356 * load anyway.
357 */
358 boolean_t replacing = (vd->vdev_ops == &vdev_replacing_ops ||
359 vd->vdev_ops == &vdev_spare_ops) &&
360 spa_load_state(vd->vdev_spa) == SPA_LOAD_NONE &&
361 dsl_scan_resilvering(vd->vdev_spa->spa_dsl_pool);
362 mm = vdev_mirror_map_alloc(vd->vdev_children, replacing,
363 B_FALSE);
364 for (c = 0; c < mm->mm_children; c++) {
365 mc = &mm->mm_child[c];
366 mc->mc_vd = vd->vdev_child[c];
367 mc->mc_offset = zio->io_offset;
368
369 if (vdev_mirror_rebuilding(mc->mc_vd))
370 mm->mm_rebuilding = mc->mc_rebuilding = B_TRUE;
371 }
372 }
373
374 return (mm);
375 }
376
377 static int
vdev_mirror_open(vdev_t * vd,uint64_t * asize,uint64_t * max_asize,uint64_t * logical_ashift,uint64_t * physical_ashift,cred_t * cr)378 vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
379 uint64_t *logical_ashift, uint64_t *physical_ashift, cred_t *cr)
380 {
381 int numerrors = 0;
382 int lasterror = 0;
383
384 if (vd->vdev_children == 0) {
385 vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
386 return (SET_ERROR(EINVAL));
387 }
388
389 vdev_open_children(vd, cr);
390
391 for (int c = 0; c < vd->vdev_children; c++) {
392 vdev_t *cvd = vd->vdev_child[c];
393
394 if (cvd->vdev_open_error) {
395 lasterror = cvd->vdev_open_error;
396 numerrors++;
397 continue;
398 }
399
400 *asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
401 *max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
402 *logical_ashift = MAX(*logical_ashift, cvd->vdev_ashift);
403 }
404 for (int c = 0; c < vd->vdev_children; c++) {
405 vdev_t *cvd = vd->vdev_child[c];
406
407 if (cvd->vdev_open_error)
408 continue;
409 *physical_ashift = vdev_best_ashift(*logical_ashift,
410 *physical_ashift, cvd->vdev_physical_ashift);
411 }
412
413 if (numerrors == vd->vdev_children) {
414 if (vdev_children_are_offline(vd))
415 vd->vdev_stat.vs_aux = VDEV_AUX_CHILDREN_OFFLINE;
416 else
417 vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
418 return (lasterror);
419 }
420
421 return (0);
422 }
423
424 static void
vdev_mirror_close(vdev_t * vd)425 vdev_mirror_close(vdev_t *vd)
426 {
427 for (int c = 0; c < vd->vdev_children; c++)
428 vdev_close(vd->vdev_child[c]);
429 }
430
431 static void
vdev_mirror_child_done(zio_t * zio)432 vdev_mirror_child_done(zio_t *zio)
433 {
434 mirror_child_t *mc = zio->io_private;
435
436 mc->mc_error = zio->io_error;
437 mc->mc_tried = 1;
438 mc->mc_skipped = 0;
439 }
440
441 /*
442 * Check the other, lower-index DVAs to see if they're on the same
443 * vdev as the child we picked. If they are, use them since they
444 * are likely to have been allocated from the primary metaslab in
445 * use at the time, and hence are more likely to have locality with
446 * single-copy data.
447 */
448 static int
vdev_mirror_dva_select(zio_t * zio,int p)449 vdev_mirror_dva_select(zio_t *zio, int p)
450 {
451 dva_t *dva = zio->io_bp->blk_dva;
452 mirror_map_t *mm = zio->io_vsd;
453 int preferred;
454 int c;
455
456 preferred = mm->mm_preferred[p];
457 for (p--; p >= 0; p--) {
458 c = mm->mm_preferred[p];
459 if (DVA_GET_VDEV(&dva[c]) == DVA_GET_VDEV(&dva[preferred]))
460 preferred = c;
461 }
462 return (preferred);
463 }
464
465 static int
vdev_mirror_preferred_child_randomize(zio_t * zio)466 vdev_mirror_preferred_child_randomize(zio_t *zio)
467 {
468 mirror_map_t *mm = zio->io_vsd;
469 int p;
470
471 if (mm->mm_root) {
472 p = random_in_range(mm->mm_preferred_cnt);
473 return (vdev_mirror_dva_select(zio, p));
474 }
475
476 /*
477 * To ensure we don't always favour the first matching vdev,
478 * which could lead to wear leveling issues on SSD's, we
479 * use the I/O offset as a pseudo random seed into the vdevs
480 * which have the lowest load.
481 */
482 p = (zio->io_offset >> vdev_mirror_shift) % mm->mm_preferred_cnt;
483 return (mm->mm_preferred[p]);
484 }
485
486 static boolean_t
vdev_mirror_child_readable(mirror_child_t * mc)487 vdev_mirror_child_readable(mirror_child_t *mc)
488 {
489 vdev_t *vd = mc->mc_vd;
490
491 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
492 return (vdev_draid_readable(vd, mc->mc_offset));
493 else
494 return (vdev_readable(vd));
495 }
496
497 static boolean_t
vdev_mirror_child_missing(mirror_child_t * mc,uint64_t txg,uint64_t size)498 vdev_mirror_child_missing(mirror_child_t *mc, uint64_t txg, uint64_t size)
499 {
500 vdev_t *vd = mc->mc_vd;
501
502 if (vd->vdev_top != NULL && vd->vdev_top->vdev_ops == &vdev_draid_ops)
503 return (vdev_draid_missing(vd, mc->mc_offset, txg, size));
504 else
505 return (vdev_dtl_contains(vd, DTL_MISSING, txg, size));
506 }
507
508 /*
509 * Try to find a vdev whose DTL doesn't contain the block we want to read
510 * preferring vdevs based on determined load. If we can't, try the read on
511 * any vdev we haven't already tried.
512 *
513 * Distributed spares are an exception to the above load rule. They are
514 * always preferred in order to detect gaps in the distributed spare which
515 * are created when another disk in the dRAID fails. In order to restore
516 * redundancy those gaps must be read to trigger the required repair IO.
517 */
518 static int
vdev_mirror_child_select(zio_t * zio)519 vdev_mirror_child_select(zio_t *zio)
520 {
521 mirror_map_t *mm = zio->io_vsd;
522 uint64_t txg = zio->io_txg;
523 int c, lowest_load;
524
525 ASSERT(zio->io_bp == NULL || BP_GET_PHYSICAL_BIRTH(zio->io_bp) == txg);
526
527 lowest_load = INT_MAX;
528 mm->mm_preferred_cnt = 0;
529 for (c = 0; c < mm->mm_children; c++) {
530 mirror_child_t *mc;
531
532 mc = &mm->mm_child[c];
533 if (mc->mc_tried || mc->mc_skipped)
534 continue;
535
536 if (mc->mc_vd == NULL ||
537 !vdev_mirror_child_readable(mc)) {
538 mc->mc_error = SET_ERROR(ENXIO);
539 mc->mc_tried = 1; /* don't even try */
540 mc->mc_skipped = 1;
541 continue;
542 }
543
544 if (vdev_mirror_child_missing(mc, txg, 1)) {
545 mc->mc_error = SET_ERROR(ESTALE);
546 mc->mc_skipped = 1;
547 mc->mc_speculative = 1;
548 continue;
549 }
550
551 if (mc->mc_vd->vdev_ops == &vdev_draid_spare_ops) {
552 mm->mm_preferred[0] = c;
553 mm->mm_preferred_cnt = 1;
554 break;
555 }
556
557 mc->mc_load = vdev_mirror_load(mm, mc->mc_vd, mc->mc_offset);
558 if (mc->mc_load > lowest_load)
559 continue;
560
561 if (mc->mc_load < lowest_load) {
562 lowest_load = mc->mc_load;
563 mm->mm_preferred_cnt = 0;
564 }
565 mm->mm_preferred[mm->mm_preferred_cnt] = c;
566 mm->mm_preferred_cnt++;
567 }
568
569 if (mm->mm_preferred_cnt == 1) {
570 MIRROR_BUMP(vdev_mirror_stat_preferred_found);
571 return (mm->mm_preferred[0]);
572 }
573
574 if (mm->mm_preferred_cnt > 1) {
575 MIRROR_BUMP(vdev_mirror_stat_preferred_not_found);
576 return (vdev_mirror_preferred_child_randomize(zio));
577 }
578
579 /*
580 * Every device is either missing or has this txg in its DTL.
581 * Look for any child we haven't already tried before giving up.
582 */
583 for (c = 0; c < mm->mm_children; c++) {
584 if (!mm->mm_child[c].mc_tried)
585 return (c);
586 }
587
588 /*
589 * Every child failed. There's no place left to look.
590 */
591 return (-1);
592 }
593
594 static void
vdev_mirror_io_start(zio_t * zio)595 vdev_mirror_io_start(zio_t *zio)
596 {
597 mirror_map_t *mm;
598 mirror_child_t *mc;
599 int c, children;
600
601 mm = vdev_mirror_map_init(zio);
602 zio->io_vsd = mm;
603 zio->io_vsd_ops = &vdev_mirror_vsd_ops;
604
605 if (mm == NULL) {
606 ASSERT(!spa_trust_config(zio->io_spa));
607 ASSERT(zio->io_type == ZIO_TYPE_READ);
608 zio_execute(zio);
609 return;
610 }
611
612 if (zio->io_type == ZIO_TYPE_READ) {
613 if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_resilvering) {
614 /*
615 * For scrubbing reads we need to issue reads to all
616 * children. One child can reuse parent buffer, but
617 * for others we have to allocate separate ones to
618 * verify checksums if io_bp is non-NULL, or compare
619 * them in vdev_mirror_io_done() otherwise.
620 */
621 boolean_t first = B_TRUE;
622
623 if (mm->mm_children > 1)
624 zio_batch_create(zio);
625
626 for (c = 0; c < mm->mm_children; c++) {
627 mc = &mm->mm_child[c];
628
629 /* Don't issue ZIOs to offline children */
630 if (!vdev_mirror_child_readable(mc)) {
631 mc->mc_error = SET_ERROR(ENXIO);
632 mc->mc_tried = 1;
633 mc->mc_skipped = 1;
634 continue;
635 }
636
637 mc->mc_abd = first ? zio->io_abd :
638 abd_alloc_sametype(zio->io_abd,
639 zio->io_size);
640 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
641 mc->mc_vd, mc->mc_offset, mc->mc_abd,
642 zio->io_size, zio->io_type,
643 zio->io_priority, 0,
644 vdev_mirror_child_done, mc));
645 first = B_FALSE;
646 }
647
648 zio_execute(zio_batch_rele(zio));
649 return;
650 }
651 /*
652 * For normal reads just pick one child.
653 */
654 c = vdev_mirror_child_select(zio);
655 children = (c >= 0);
656 } else {
657 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
658
659 /*
660 * Writes go to all children.
661 */
662 c = 0;
663 children = mm->mm_children;
664 }
665
666 if (children > 1)
667 zio_batch_create(zio);
668
669 while (children--) {
670 mc = &mm->mm_child[c++];
671
672 /*
673 * When sequentially resilvering and the integrity of the data
674 * is speculative (ZIO_FLAG_SPECULATIVE), issue write repair IOs
675 * only to the vdev which is being rebuilt. Existing data on
676 * other children must never be overwritten with unconfirmed
677 * data to avoid unrecoverable damage to the pool.
678 */
679 if ((zio->io_priority == ZIO_PRIORITY_REBUILD) &&
680 (zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
681 !(zio->io_flags & ZIO_FLAG_SCRUB) &&
682 (zio->io_flags & ZIO_FLAG_SPECULATIVE) &&
683 mm->mm_rebuilding && !mc->mc_rebuilding) {
684 continue;
685 }
686
687 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
688 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
689 zio->io_type, zio->io_priority, 0,
690 vdev_mirror_child_done, mc));
691 }
692
693 zio_execute(zio_batch_rele(zio));
694 }
695
696 static int
vdev_mirror_worst_error(mirror_map_t * mm)697 vdev_mirror_worst_error(mirror_map_t *mm)
698 {
699 int error[2] = { 0, 0 };
700
701 for (int c = 0; c < mm->mm_children; c++) {
702 mirror_child_t *mc = &mm->mm_child[c];
703 int s = mc->mc_speculative;
704 error[s] = zio_worst_error(error[s], mc->mc_error);
705 }
706
707 return (error[0] ? error[0] : error[1]);
708 }
709
710 static void
vdev_mirror_io_done(zio_t * zio)711 vdev_mirror_io_done(zio_t *zio)
712 {
713 mirror_map_t *mm = zio->io_vsd;
714 mirror_child_t *mc;
715 int c;
716 int good_copies = 0;
717 int unexpected_errors = 0;
718 int last_good_copy = -1;
719
720 if (mm == NULL)
721 return;
722
723 for (c = 0; c < mm->mm_children; c++) {
724 mc = &mm->mm_child[c];
725
726 if (mc->mc_error) {
727 if (!mc->mc_skipped)
728 unexpected_errors++;
729 } else if (mc->mc_tried) {
730 last_good_copy = c;
731 good_copies++;
732 }
733 }
734
735 if (zio->io_type == ZIO_TYPE_WRITE) {
736 /*
737 * XXX -- for now, treat partial writes as success.
738 *
739 * Now that we support write reallocation, it would be better
740 * to treat partial failure as real failure unless there are
741 * no non-degraded top-level vdevs left, and not update DTLs
742 * if we intend to reallocate.
743 */
744 if (good_copies != mm->mm_children) {
745 /*
746 * Always require at least one good copy.
747 *
748 * For ditto blocks (io_vd == NULL), require
749 * all copies to be good.
750 *
751 * XXX -- for replacing vdevs, there's no great answer.
752 * If the old device is really dead, we may not even
753 * be able to access it -- so we only want to
754 * require good writes to the new device. But if
755 * the new device turns out to be flaky, we want
756 * to be able to detach it -- which requires all
757 * writes to the old device to have succeeded.
758 */
759 if (good_copies == 0 || zio->io_vd == NULL)
760 zio->io_error = vdev_mirror_worst_error(mm);
761 }
762 return;
763 }
764
765 ASSERT(zio->io_type == ZIO_TYPE_READ);
766
767 /*
768 * Any Direct I/O read that has a checksum error must be treated as
769 * suspicious as the contents of the buffer could be getting
770 * manipulated while the I/O is taking place. The checksum verify error
771 * will be reported to the top-level Mirror VDEV.
772 *
773 * There will be no attampt at reading any additional data copies. If
774 * the buffer is still being manipulated while attempting to read from
775 * another child, there exists a possibly that the checksum could be
776 * verified as valid. However, the buffer contents could again get
777 * manipulated after verifying the checksum. This would lead to bad data
778 * being written out during self healing.
779 */
780 if ((zio->io_flags & ZIO_FLAG_DIO_READ) &&
781 (zio->io_post & ZIO_POST_DIO_CHKSUM_ERR)) {
782 zio_dio_chksum_verify_error_report(zio);
783 zio->io_error = vdev_mirror_worst_error(mm);
784 ASSERT3U(zio->io_error, ==, ECKSUM);
785 return;
786 }
787
788 /*
789 * If we don't have a good copy yet, keep trying other children.
790 */
791 if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
792 ASSERT(c >= 0 && c < mm->mm_children);
793 mc = &mm->mm_child[c];
794 zio_vdev_io_redone(zio);
795 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
796 mc->mc_vd, mc->mc_offset, zio->io_abd, zio->io_size,
797 ZIO_TYPE_READ, zio->io_priority, 0,
798 vdev_mirror_child_done, mc));
799 return;
800 }
801
802 if (zio->io_flags & ZIO_FLAG_SCRUB && !mm->mm_resilvering) {
803 abd_t *best_abd = NULL;
804 if (last_good_copy >= 0)
805 best_abd = mm->mm_child[last_good_copy].mc_abd;
806
807 /*
808 * If we're scrubbing but don't have a BP available (because
809 * this vdev is under a raidz or draid vdev) then the best we
810 * can do is compare all of the copies read. If they're not
811 * identical then return a checksum error and the most likely
812 * correct data. The raidz code will issue a repair I/O if
813 * possible.
814 */
815 if (zio->io_bp == NULL) {
816 ASSERT(zio->io_vd->vdev_ops == &vdev_replacing_ops ||
817 zio->io_vd->vdev_ops == &vdev_spare_ops);
818
819 abd_t *pref_abd = NULL;
820 for (c = 0; c < last_good_copy; c++) {
821 mc = &mm->mm_child[c];
822 if (mc->mc_error || !mc->mc_tried)
823 continue;
824
825 if (abd_cmp(mc->mc_abd, best_abd) != 0)
826 zio->io_error = SET_ERROR(ECKSUM);
827
828 /*
829 * The distributed spare is always prefered
830 * by vdev_mirror_child_select() so it's
831 * considered to be the best candidate.
832 */
833 if (pref_abd == NULL &&
834 mc->mc_vd->vdev_ops ==
835 &vdev_draid_spare_ops)
836 pref_abd = mc->mc_abd;
837
838 /*
839 * In the absence of a preferred copy, use
840 * the parent pointer to avoid a memory copy.
841 */
842 if (mc->mc_abd == zio->io_abd)
843 best_abd = mc->mc_abd;
844 }
845 if (pref_abd)
846 best_abd = pref_abd;
847 } else {
848
849 /*
850 * If we have a BP available, then checksums are
851 * already verified and we just need a buffer
852 * with valid data, preferring parent one to
853 * avoid a memory copy.
854 */
855 for (c = 0; c < last_good_copy; c++) {
856 mc = &mm->mm_child[c];
857 if (mc->mc_error || !mc->mc_tried)
858 continue;
859 if (mc->mc_abd == zio->io_abd) {
860 best_abd = mc->mc_abd;
861 break;
862 }
863 }
864 }
865
866 if (best_abd && best_abd != zio->io_abd)
867 abd_copy(zio->io_abd, best_abd, zio->io_size);
868 for (c = 0; c < mm->mm_children; c++) {
869 mc = &mm->mm_child[c];
870 if (mc->mc_abd != zio->io_abd)
871 abd_free(mc->mc_abd);
872 mc->mc_abd = NULL;
873 }
874 }
875
876 if (good_copies == 0) {
877 zio->io_error = vdev_mirror_worst_error(mm);
878 ASSERT(zio->io_error != 0);
879 }
880
881 if (good_copies && spa_writeable(zio->io_spa) &&
882 (unexpected_errors ||
883 (zio->io_flags & ZIO_FLAG_RESILVER) ||
884 ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_resilvering))) {
885 /*
886 * Use the good data we have in hand to repair damaged children.
887 */
888 for (c = 0; c < mm->mm_children; c++) {
889 /*
890 * Don't rewrite known good children.
891 * Not only is it unnecessary, it could
892 * actually be harmful: if the system lost
893 * power while rewriting the only good copy,
894 * there would be no good copies left!
895 */
896 mc = &mm->mm_child[c];
897
898 if (mc->mc_error == 0) {
899 vdev_ops_t *ops = mc->mc_vd->vdev_ops;
900
901 if (mc->mc_tried)
902 continue;
903 /*
904 * We didn't try this child. We need to
905 * repair it if:
906 * 1. it's a scrub (in which case we have
907 * tried everything that was healthy)
908 * - or -
909 * 2. it's an indirect or distributed spare
910 * vdev (in which case it could point to any
911 * other vdev, which might have a bad DTL)
912 * - or -
913 * 3. the DTL indicates that this data is
914 * missing from this vdev
915 */
916 if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
917 ops != &vdev_indirect_ops &&
918 ops != &vdev_draid_spare_ops &&
919 !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
920 zio->io_txg, 1))
921 continue;
922 mc->mc_error = SET_ERROR(ESTALE);
923 }
924
925 zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
926 mc->mc_vd, mc->mc_offset,
927 zio->io_abd, zio->io_size, ZIO_TYPE_WRITE,
928 zio->io_priority == ZIO_PRIORITY_REBUILD ?
929 ZIO_PRIORITY_REBUILD : ZIO_PRIORITY_ASYNC_WRITE,
930 ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
931 ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
932 }
933 }
934 }
935
936 static void
vdev_mirror_state_change(vdev_t * vd,int faulted,int degraded)937 vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
938 {
939 if (faulted == vd->vdev_children) {
940 if (vdev_children_are_offline(vd)) {
941 vdev_set_state(vd, B_FALSE, VDEV_STATE_OFFLINE,
942 VDEV_AUX_CHILDREN_OFFLINE);
943 } else {
944 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
945 VDEV_AUX_NO_REPLICAS);
946 }
947 } else if (degraded + faulted != 0) {
948 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
949 } else {
950 vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
951 }
952 }
953
954 /*
955 * Return the maximum asize for a rebuild zio in the provided range.
956 */
957 static uint64_t
vdev_mirror_rebuild_asize(vdev_t * vd,uint64_t start,uint64_t asize,uint64_t max_segment)958 vdev_mirror_rebuild_asize(vdev_t *vd, uint64_t start, uint64_t asize,
959 uint64_t max_segment)
960 {
961 (void) start;
962
963 uint64_t psize = MIN(P2ROUNDUP(max_segment, 1 << vd->vdev_ashift),
964 SPA_MAXBLOCKSIZE);
965
966 return (MIN(asize, vdev_psize_to_asize(vd, psize)));
967 }
968
969 vdev_ops_t vdev_mirror_ops = {
970 .vdev_op_init = NULL,
971 .vdev_op_fini = NULL,
972 .vdev_op_open = vdev_mirror_open,
973 .vdev_op_close = vdev_mirror_close,
974 .vdev_op_psize_to_asize = vdev_default_asize,
975 .vdev_op_asize_to_psize = vdev_default_psize,
976 .vdev_op_min_asize = vdev_default_min_asize,
977 .vdev_op_min_alloc = NULL,
978 .vdev_op_io_start = vdev_mirror_io_start,
979 .vdev_op_io_done = vdev_mirror_io_done,
980 .vdev_op_state_change = vdev_mirror_state_change,
981 .vdev_op_need_resilver = vdev_default_need_resilver,
982 .vdev_op_hold = NULL,
983 .vdev_op_rele = NULL,
984 .vdev_op_remap = NULL,
985 .vdev_op_xlate = vdev_default_xlate,
986 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
987 .vdev_op_metaslab_init = NULL,
988 .vdev_op_config_generate = NULL,
989 .vdev_op_nparity = NULL,
990 .vdev_op_ndisks = NULL,
991 .vdev_op_type = VDEV_TYPE_MIRROR, /* name of this vdev type */
992 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
993 };
994
995 vdev_ops_t vdev_replacing_ops = {
996 .vdev_op_init = NULL,
997 .vdev_op_fini = NULL,
998 .vdev_op_open = vdev_mirror_open,
999 .vdev_op_close = vdev_mirror_close,
1000 .vdev_op_psize_to_asize = vdev_default_asize,
1001 .vdev_op_asize_to_psize = vdev_default_psize,
1002 .vdev_op_min_asize = vdev_default_min_asize,
1003 .vdev_op_min_alloc = NULL,
1004 .vdev_op_io_start = vdev_mirror_io_start,
1005 .vdev_op_io_done = vdev_mirror_io_done,
1006 .vdev_op_state_change = vdev_mirror_state_change,
1007 .vdev_op_need_resilver = vdev_default_need_resilver,
1008 .vdev_op_hold = NULL,
1009 .vdev_op_rele = NULL,
1010 .vdev_op_remap = NULL,
1011 .vdev_op_xlate = vdev_default_xlate,
1012 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1013 .vdev_op_metaslab_init = NULL,
1014 .vdev_op_config_generate = NULL,
1015 .vdev_op_nparity = NULL,
1016 .vdev_op_ndisks = NULL,
1017 .vdev_op_type = VDEV_TYPE_REPLACING, /* name of this vdev type */
1018 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1019 };
1020
1021 vdev_ops_t vdev_spare_ops = {
1022 .vdev_op_init = NULL,
1023 .vdev_op_fini = NULL,
1024 .vdev_op_open = vdev_mirror_open,
1025 .vdev_op_close = vdev_mirror_close,
1026 .vdev_op_psize_to_asize = vdev_default_asize,
1027 .vdev_op_asize_to_psize = vdev_default_psize,
1028 .vdev_op_min_asize = vdev_default_min_asize,
1029 .vdev_op_min_alloc = NULL,
1030 .vdev_op_io_start = vdev_mirror_io_start,
1031 .vdev_op_io_done = vdev_mirror_io_done,
1032 .vdev_op_state_change = vdev_mirror_state_change,
1033 .vdev_op_need_resilver = vdev_default_need_resilver,
1034 .vdev_op_hold = NULL,
1035 .vdev_op_rele = NULL,
1036 .vdev_op_remap = NULL,
1037 .vdev_op_xlate = vdev_default_xlate,
1038 .vdev_op_rebuild_asize = vdev_mirror_rebuild_asize,
1039 .vdev_op_metaslab_init = NULL,
1040 .vdev_op_config_generate = NULL,
1041 .vdev_op_nparity = NULL,
1042 .vdev_op_ndisks = NULL,
1043 .vdev_op_type = VDEV_TYPE_SPARE, /* name of this vdev type */
1044 .vdev_op_leaf = B_FALSE /* not a leaf vdev */
1045 };
1046
1047 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_inc, INT, ZMOD_RW,
1048 "Rotating media load increment for non-seeking I/Os");
1049
1050 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_inc, INT,
1051 ZMOD_RW, "Rotating media load increment for seeking I/Os");
1052
1053 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, rotating_seek_offset, INT,
1054 ZMOD_RW,
1055 "Offset in bytes from the last I/O which triggers "
1056 "a reduced rotating media seek increment");
1057
1058 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_inc, INT,
1059 ZMOD_RW, "Non-rotating media load increment for non-seeking I/Os");
1060
1061 ZFS_MODULE_PARAM(zfs_vdev_mirror, zfs_vdev_mirror_, non_rotating_seek_inc, INT,
1062 ZMOD_RW, "Non-rotating media load increment for seeking I/Os");
1063