xref: /freebsd/sys/geom/raid/tr_raid1e.c (revision c243e4902be8df1e643c76b5f18b68bb77cc5268)
1 /*-
2  * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bio.h>
32 #include <sys/endian.h>
33 #include <sys/kernel.h>
34 #include <sys/kobj.h>
35 #include <sys/limits.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mutex.h>
39 #include <sys/sysctl.h>
40 #include <sys/systm.h>
41 #include <geom/geom.h>
42 #include "geom/raid/g_raid.h"
43 #include "g_raid_tr_if.h"
44 
45 #define N	2
46 
47 SYSCTL_DECL(_kern_geom_raid);
48 static SYSCTL_NODE(_kern_geom_raid, OID_AUTO, raid1e, CTLFLAG_RW, 0,
49     "RAID1E parameters");
50 
51 #define RAID1E_REBUILD_SLAB	(1 << 20) /* One transation in a rebuild */
52 static int g_raid1e_rebuild_slab = RAID1E_REBUILD_SLAB;
53 TUNABLE_INT("kern.geom.raid.raid1e.rebuild_slab_size",
54     &g_raid1e_rebuild_slab);
55 SYSCTL_UINT(_kern_geom_raid_raid1e, OID_AUTO, rebuild_slab_size, CTLFLAG_RW,
56     &g_raid1e_rebuild_slab, 0,
57     "Amount of the disk to rebuild each read/write cycle of the rebuild.");
58 
59 #define RAID1E_REBUILD_FAIR_IO 20 /* use 1/x of the available I/O */
60 static int g_raid1e_rebuild_fair_io = RAID1E_REBUILD_FAIR_IO;
61 TUNABLE_INT("kern.geom.raid.raid1e.rebuild_fair_io",
62     &g_raid1e_rebuild_fair_io);
63 SYSCTL_UINT(_kern_geom_raid_raid1e, OID_AUTO, rebuild_fair_io, CTLFLAG_RW,
64     &g_raid1e_rebuild_fair_io, 0,
65     "Fraction of the I/O bandwidth to use when disk busy for rebuild.");
66 
67 #define RAID1E_REBUILD_CLUSTER_IDLE 100
68 static int g_raid1e_rebuild_cluster_idle = RAID1E_REBUILD_CLUSTER_IDLE;
69 TUNABLE_INT("kern.geom.raid.raid1e.rebuild_cluster_idle",
70     &g_raid1e_rebuild_cluster_idle);
71 SYSCTL_UINT(_kern_geom_raid_raid1e, OID_AUTO, rebuild_cluster_idle, CTLFLAG_RW,
72     &g_raid1e_rebuild_cluster_idle, 0,
73     "Number of slabs to do each time we trigger a rebuild cycle");
74 
75 #define RAID1E_REBUILD_META_UPDATE 1024 /* update meta data every 1GB or so */
76 static int g_raid1e_rebuild_meta_update = RAID1E_REBUILD_META_UPDATE;
77 TUNABLE_INT("kern.geom.raid.raid1e.rebuild_meta_update",
78     &g_raid1e_rebuild_meta_update);
79 SYSCTL_UINT(_kern_geom_raid_raid1e, OID_AUTO, rebuild_meta_update, CTLFLAG_RW,
80     &g_raid1e_rebuild_meta_update, 0,
81     "When to update the meta data.");
82 
83 static MALLOC_DEFINE(M_TR_RAID1E, "tr_raid1e_data", "GEOM_RAID RAID1E data");
84 
85 #define TR_RAID1E_NONE 0
86 #define TR_RAID1E_REBUILD 1
87 #define TR_RAID1E_RESYNC 2
88 
89 #define TR_RAID1E_F_DOING_SOME	0x1
90 #define TR_RAID1E_F_LOCKED	0x2
91 #define TR_RAID1E_F_ABORT	0x4
92 
93 struct g_raid_tr_raid1e_object {
94 	struct g_raid_tr_object	 trso_base;
95 	int			 trso_starting;
96 	int			 trso_stopping;
97 	int			 trso_type;
98 	int			 trso_recover_slabs; /* slabs before rest */
99 	int			 trso_fair_io;
100 	int			 trso_meta_update;
101 	int			 trso_flags;
102 	struct g_raid_subdisk	*trso_failed_sd; /* like per volume */
103 	void			*trso_buffer;	 /* Buffer space */
104 	off_t			 trso_lock_pos; /* Locked range start. */
105 	off_t			 trso_lock_len; /* Locked range length. */
106 	struct bio		 trso_bio;
107 };
108 
109 static g_raid_tr_taste_t g_raid_tr_taste_raid1e;
110 static g_raid_tr_event_t g_raid_tr_event_raid1e;
111 static g_raid_tr_start_t g_raid_tr_start_raid1e;
112 static g_raid_tr_stop_t g_raid_tr_stop_raid1e;
113 static g_raid_tr_iostart_t g_raid_tr_iostart_raid1e;
114 static g_raid_tr_iodone_t g_raid_tr_iodone_raid1e;
115 static g_raid_tr_kerneldump_t g_raid_tr_kerneldump_raid1e;
116 static g_raid_tr_locked_t g_raid_tr_locked_raid1e;
117 static g_raid_tr_idle_t g_raid_tr_idle_raid1e;
118 static g_raid_tr_free_t g_raid_tr_free_raid1e;
119 
120 static kobj_method_t g_raid_tr_raid1e_methods[] = {
121 	KOBJMETHOD(g_raid_tr_taste,	g_raid_tr_taste_raid1e),
122 	KOBJMETHOD(g_raid_tr_event,	g_raid_tr_event_raid1e),
123 	KOBJMETHOD(g_raid_tr_start,	g_raid_tr_start_raid1e),
124 	KOBJMETHOD(g_raid_tr_stop,	g_raid_tr_stop_raid1e),
125 	KOBJMETHOD(g_raid_tr_iostart,	g_raid_tr_iostart_raid1e),
126 	KOBJMETHOD(g_raid_tr_iodone,	g_raid_tr_iodone_raid1e),
127 	KOBJMETHOD(g_raid_tr_kerneldump, g_raid_tr_kerneldump_raid1e),
128 	KOBJMETHOD(g_raid_tr_locked,	g_raid_tr_locked_raid1e),
129 	KOBJMETHOD(g_raid_tr_idle,	g_raid_tr_idle_raid1e),
130 	KOBJMETHOD(g_raid_tr_free,	g_raid_tr_free_raid1e),
131 	{ 0, 0 }
132 };
133 
134 static struct g_raid_tr_class g_raid_tr_raid1e_class = {
135 	"RAID1E",
136 	g_raid_tr_raid1e_methods,
137 	sizeof(struct g_raid_tr_raid1e_object),
138 	.trc_priority = 200
139 };
140 
141 static void g_raid_tr_raid1e_rebuild_abort(struct g_raid_tr_object *tr);
142 static void g_raid_tr_raid1e_maybe_rebuild(struct g_raid_tr_object *tr,
143     struct g_raid_subdisk *sd);
144 static int g_raid_tr_raid1e_select_read_disk(struct g_raid_volume *vol,
145     int no, off_t off, off_t len, u_int mask);
146 
147 static inline void
148 V2P(struct g_raid_volume *vol, off_t virt,
149     int *disk, off_t *offset, off_t *start)
150 {
151 	off_t nstrip;
152 	u_int strip_size;
153 
154 	strip_size = vol->v_strip_size;
155 	/* Strip number. */
156 	nstrip = virt / strip_size;
157 	/* Start position in strip. */
158 	*start = virt % strip_size;
159 	/* Disk number. */
160 	*disk = (nstrip * N) % vol->v_disks_count;
161 	/* Strip start position in disk. */
162 	*offset = ((nstrip * N) / vol->v_disks_count) * strip_size;
163 }
164 
165 static inline void
166 P2V(struct g_raid_volume *vol, int disk, off_t offset,
167     off_t *virt, int *copy)
168 {
169 	off_t nstrip, start;
170 	u_int strip_size;
171 
172 	strip_size = vol->v_strip_size;
173 	/* Start position in strip. */
174 	start = offset % strip_size;
175 	/* Physical strip number. */
176 	nstrip = (offset / strip_size) * vol->v_disks_count + disk;
177 	/* Number of physical strip (copy) inside virtual strip. */
178 	*copy = nstrip % N;
179 	/* Offset in virtual space. */
180 	*virt = (nstrip / N) * strip_size + start;
181 }
182 
183 static int
184 g_raid_tr_taste_raid1e(struct g_raid_tr_object *tr, struct g_raid_volume *vol)
185 {
186 	struct g_raid_tr_raid1e_object *trs;
187 
188 	trs = (struct g_raid_tr_raid1e_object *)tr;
189 	if (tr->tro_volume->v_raid_level != G_RAID_VOLUME_RL_RAID1E ||
190 	    tr->tro_volume->v_raid_level_qualifier != G_RAID_VOLUME_RLQ_R1EA)
191 		return (G_RAID_TR_TASTE_FAIL);
192 	trs->trso_starting = 1;
193 	return (G_RAID_TR_TASTE_SUCCEED);
194 }
195 
196 static int
197 g_raid_tr_update_state_raid1e_even(struct g_raid_volume *vol)
198 {
199 	struct g_raid_softc *sc;
200 	struct g_raid_subdisk *sd, *bestsd, *worstsd;
201 	int i, j, state, sstate;
202 
203 	sc = vol->v_softc;
204 	state = G_RAID_VOLUME_S_OPTIMAL;
205 	for (i = 0; i < vol->v_disks_count / N; i++) {
206 		bestsd = &vol->v_subdisks[i * N];
207 		for (j = 1; j < N; j++) {
208 			sd = &vol->v_subdisks[i * N + j];
209 			if (sd->sd_state > bestsd->sd_state)
210 				bestsd = sd;
211 			else if (sd->sd_state == bestsd->sd_state &&
212 			    (sd->sd_state == G_RAID_SUBDISK_S_REBUILD ||
213 			     sd->sd_state == G_RAID_SUBDISK_S_RESYNC) &&
214 			    sd->sd_rebuild_pos > bestsd->sd_rebuild_pos)
215 				bestsd = sd;
216 		}
217 		if (bestsd->sd_state >= G_RAID_SUBDISK_S_UNINITIALIZED &&
218 		    bestsd->sd_state != G_RAID_SUBDISK_S_ACTIVE) {
219 			/* We found reasonable candidate. */
220 			G_RAID_DEBUG1(1, sc,
221 			    "Promote subdisk %s:%d from %s to ACTIVE.",
222 			    vol->v_name, bestsd->sd_pos,
223 			    g_raid_subdisk_state2str(bestsd->sd_state));
224 			g_raid_change_subdisk_state(bestsd,
225 			    G_RAID_SUBDISK_S_ACTIVE);
226 			g_raid_write_metadata(sc,
227 			    vol, bestsd, bestsd->sd_disk);
228 		}
229 		worstsd = &vol->v_subdisks[i * N];
230 		for (j = 1; j < N; j++) {
231 			sd = &vol->v_subdisks[i * N + j];
232 			if (sd->sd_state < worstsd->sd_state)
233 				worstsd = sd;
234 		}
235 		if (worstsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)
236 			sstate = G_RAID_VOLUME_S_OPTIMAL;
237 		else if (worstsd->sd_state >= G_RAID_SUBDISK_S_STALE)
238 			sstate = G_RAID_VOLUME_S_SUBOPTIMAL;
239 		else if (bestsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)
240 			sstate = G_RAID_VOLUME_S_DEGRADED;
241 		else
242 			sstate = G_RAID_VOLUME_S_BROKEN;
243 		if (sstate < state)
244 			state = sstate;
245 	}
246 	return (state);
247 }
248 
249 static int
250 g_raid_tr_update_state_raid1e_odd(struct g_raid_volume *vol)
251 {
252 	struct g_raid_softc *sc;
253 	struct g_raid_subdisk *sd, *bestsd, *worstsd;
254 	int i, j, state, sstate;
255 
256 	sc = vol->v_softc;
257 	if (g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_ACTIVE) ==
258 	    vol->v_disks_count)
259 		return (G_RAID_VOLUME_S_OPTIMAL);
260 	for (i = 0; i < vol->v_disks_count; i++) {
261 		sd = &vol->v_subdisks[i];
262 		if (sd->sd_state == G_RAID_SUBDISK_S_UNINITIALIZED) {
263 			/* We found reasonable candidate. */
264 			G_RAID_DEBUG1(1, sc,
265 			    "Promote subdisk %s:%d from %s to STALE.",
266 			    vol->v_name, sd->sd_pos,
267 			    g_raid_subdisk_state2str(sd->sd_state));
268 			g_raid_change_subdisk_state(sd,
269 			    G_RAID_SUBDISK_S_STALE);
270 			g_raid_write_metadata(sc, vol, sd, sd->sd_disk);
271 		}
272 	}
273 	state = G_RAID_VOLUME_S_OPTIMAL;
274 	for (i = 0; i < vol->v_disks_count; i++) {
275 		bestsd = &vol->v_subdisks[i];
276 		worstsd = &vol->v_subdisks[i];
277 		for (j = 1; j < N; j++) {
278 			sd = &vol->v_subdisks[(i + j) % vol->v_disks_count];
279 			if (sd->sd_state > bestsd->sd_state)
280 				bestsd = sd;
281 			else if (sd->sd_state == bestsd->sd_state &&
282 			    (sd->sd_state == G_RAID_SUBDISK_S_REBUILD ||
283 			     sd->sd_state == G_RAID_SUBDISK_S_RESYNC) &&
284 			    sd->sd_rebuild_pos > bestsd->sd_rebuild_pos)
285 				bestsd = sd;
286 			if (sd->sd_state < worstsd->sd_state)
287 				worstsd = sd;
288 		}
289 		if (worstsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)
290 			sstate = G_RAID_VOLUME_S_OPTIMAL;
291 		else if (worstsd->sd_state >= G_RAID_SUBDISK_S_STALE)
292 			sstate = G_RAID_VOLUME_S_SUBOPTIMAL;
293 		else if (bestsd->sd_state >= G_RAID_SUBDISK_S_STALE)
294 			sstate = G_RAID_VOLUME_S_DEGRADED;
295 		else
296 			sstate = G_RAID_VOLUME_S_BROKEN;
297 		if (sstate < state)
298 			state = sstate;
299 	}
300 	return (state);
301 }
302 
303 static int
304 g_raid_tr_update_state_raid1e(struct g_raid_volume *vol,
305     struct g_raid_subdisk *sd)
306 {
307 	struct g_raid_tr_raid1e_object *trs;
308 	struct g_raid_softc *sc;
309 	u_int s;
310 
311 	sc = vol->v_softc;
312 	trs = (struct g_raid_tr_raid1e_object *)vol->v_tr;
313 	if (trs->trso_stopping &&
314 	    (trs->trso_flags & TR_RAID1E_F_DOING_SOME) == 0)
315 		s = G_RAID_VOLUME_S_STOPPED;
316 	else if (trs->trso_starting)
317 		s = G_RAID_VOLUME_S_STARTING;
318 	else {
319 		if ((vol->v_disks_count % N) == 0)
320 			s = g_raid_tr_update_state_raid1e_even(vol);
321 		else
322 			s = g_raid_tr_update_state_raid1e_odd(vol);
323 	}
324 	if (s != vol->v_state) {
325 		g_raid_event_send(vol, G_RAID_VOLUME_S_ALIVE(s) ?
326 		    G_RAID_VOLUME_E_UP : G_RAID_VOLUME_E_DOWN,
327 		    G_RAID_EVENT_VOLUME);
328 		g_raid_change_volume_state(vol, s);
329 		if (!trs->trso_starting && !trs->trso_stopping)
330 			g_raid_write_metadata(sc, vol, NULL, NULL);
331 	}
332 	if (!trs->trso_starting && !trs->trso_stopping)
333 		g_raid_tr_raid1e_maybe_rebuild(vol->v_tr, sd);
334 	return (0);
335 }
336 
337 static void
338 g_raid_tr_raid1e_fail_disk(struct g_raid_softc *sc, struct g_raid_subdisk *sd,
339     struct g_raid_disk *disk)
340 {
341 	struct g_raid_volume *vol;
342 
343 	vol = sd->sd_volume;
344 	/*
345 	 * We don't fail the last disk in the pack, since it still has decent
346 	 * data on it and that's better than failing the disk if it is the root
347 	 * file system.
348 	 *
349 	 * XXX should this be controlled via a tunable?  It makes sense for
350 	 * the volume that has / on it.  I can't think of a case where we'd
351 	 * want the volume to go away on this kind of event.
352 	 */
353 	if ((g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_ACTIVE) +
354 	     g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_RESYNC) +
355 	     g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_STALE) +
356 	     g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_UNINITIALIZED) <
357 	     vol->v_disks_count) &&
358 	    (sd->sd_state >= G_RAID_SUBDISK_S_UNINITIALIZED))
359 		return;
360 	g_raid_fail_disk(sc, sd, disk);
361 }
362 
363 static void
364 g_raid_tr_raid1e_rebuild_done(struct g_raid_tr_raid1e_object *trs)
365 {
366 	struct g_raid_volume *vol;
367 	struct g_raid_subdisk *sd;
368 
369 	vol = trs->trso_base.tro_volume;
370 	sd = trs->trso_failed_sd;
371 	g_raid_write_metadata(vol->v_softc, vol, sd, sd->sd_disk);
372 	free(trs->trso_buffer, M_TR_RAID1E);
373 	trs->trso_buffer = NULL;
374 	trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;
375 	trs->trso_type = TR_RAID1E_NONE;
376 	trs->trso_recover_slabs = 0;
377 	trs->trso_failed_sd = NULL;
378 	g_raid_tr_update_state_raid1e(vol, NULL);
379 }
380 
381 static void
382 g_raid_tr_raid1e_rebuild_finish(struct g_raid_tr_object *tr)
383 {
384 	struct g_raid_tr_raid1e_object *trs;
385 	struct g_raid_subdisk *sd;
386 
387 	trs = (struct g_raid_tr_raid1e_object *)tr;
388 	sd = trs->trso_failed_sd;
389 	G_RAID_DEBUG1(0, tr->tro_volume->v_softc,
390 	    "Subdisk %s:%d-%s rebuild completed.",
391 	    sd->sd_volume->v_name, sd->sd_pos,
392 	    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");
393 	g_raid_change_subdisk_state(sd, G_RAID_SUBDISK_S_ACTIVE);
394 	sd->sd_rebuild_pos = 0;
395 	g_raid_tr_raid1e_rebuild_done(trs);
396 }
397 
398 static void
399 g_raid_tr_raid1e_rebuild_abort(struct g_raid_tr_object *tr)
400 {
401 	struct g_raid_tr_raid1e_object *trs;
402 	struct g_raid_subdisk *sd;
403 	struct g_raid_volume *vol;
404 
405 	vol = tr->tro_volume;
406 	trs = (struct g_raid_tr_raid1e_object *)tr;
407 	sd = trs->trso_failed_sd;
408 	if (trs->trso_flags & TR_RAID1E_F_DOING_SOME) {
409 		G_RAID_DEBUG1(1, vol->v_softc,
410 		    "Subdisk %s:%d-%s rebuild is aborting.",
411 		    sd->sd_volume->v_name, sd->sd_pos,
412 		    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");
413 		trs->trso_flags |= TR_RAID1E_F_ABORT;
414 	} else {
415 		G_RAID_DEBUG1(0, vol->v_softc,
416 		    "Subdisk %s:%d-%s rebuild aborted.",
417 		    sd->sd_volume->v_name, sd->sd_pos,
418 		    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");
419 		trs->trso_flags &= ~TR_RAID1E_F_ABORT;
420 		if (trs->trso_flags & TR_RAID1E_F_LOCKED) {
421 			trs->trso_flags &= ~TR_RAID1E_F_LOCKED;
422 			g_raid_unlock_range(tr->tro_volume,
423 			    trs->trso_lock_pos, trs->trso_lock_len);
424 		}
425 		g_raid_tr_raid1e_rebuild_done(trs);
426 	}
427 }
428 
429 static void
430 g_raid_tr_raid1e_rebuild_some(struct g_raid_tr_object *tr)
431 {
432 	struct g_raid_tr_raid1e_object *trs;
433 	struct g_raid_softc *sc;
434 	struct g_raid_volume *vol;
435 	struct g_raid_subdisk *sd;
436 	struct bio *bp;
437 	off_t len, virtual, vend, offset, start;
438 	int disk, copy, best;
439 
440 	trs = (struct g_raid_tr_raid1e_object *)tr;
441 	if (trs->trso_flags & TR_RAID1E_F_DOING_SOME)
442 		return;
443 	vol = tr->tro_volume;
444 	sc = vol->v_softc;
445 	sd = trs->trso_failed_sd;
446 
447 	while (1) {
448 		if (sd->sd_rebuild_pos >= sd->sd_size) {
449 			g_raid_tr_raid1e_rebuild_finish(tr);
450 			return;
451 		}
452 		/* Get virtual offset from physical rebuild position. */
453 		P2V(vol, sd->sd_pos, sd->sd_rebuild_pos, &virtual, &copy);
454 		/* Get physical offset back to get first stripe position. */
455 		V2P(vol, virtual, &disk, &offset, &start);
456 		/* Calculate contignous data length. */
457 		len = MIN(g_raid1e_rebuild_slab,
458 		    sd->sd_size - sd->sd_rebuild_pos);
459 		if ((vol->v_disks_count % N) != 0)
460 			len = MIN(len, vol->v_strip_size - start);
461 		/* Find disk with most accurate data. */
462 		best = g_raid_tr_raid1e_select_read_disk(vol, disk,
463 		    offset + start, len, 0);
464 		if (best < 0) {
465 			/* There is no any valid disk. */
466 			g_raid_tr_raid1e_rebuild_abort(tr);
467 			return;
468 		} else if (best != copy) {
469 			/* Some other disk has better data. */
470 			break;
471 		}
472 		/* We have the most accurate data. Skip the range. */
473 		G_RAID_DEBUG1(3, sc, "Skipping rebuild for range %ju - %ju",
474 		    sd->sd_rebuild_pos, sd->sd_rebuild_pos + len);
475 		sd->sd_rebuild_pos += len;
476 	}
477 
478 	bp = &trs->trso_bio;
479 	memset(bp, 0, sizeof(*bp));
480 	bp->bio_offset = offset + start +
481 	    ((disk + best >= vol->v_disks_count) ? vol->v_strip_size : 0);
482 	bp->bio_length = len;
483 	bp->bio_data = trs->trso_buffer;
484 	bp->bio_cmd = BIO_READ;
485 	bp->bio_cflags = G_RAID_BIO_FLAG_SYNC;
486 	bp->bio_caller1 = &vol->v_subdisks[(disk + best) % vol->v_disks_count];
487 	G_RAID_LOGREQ(3, bp, "Queueing rebuild read");
488 	/*
489 	 * If we are crossing stripe boundary, correct affected virtual
490 	 * range we should lock.
491 	 */
492 	if (start + len > vol->v_strip_size) {
493 		P2V(vol, sd->sd_pos, sd->sd_rebuild_pos + len, &vend, &copy);
494 		len = vend - virtual;
495 	}
496 	trs->trso_flags |= TR_RAID1E_F_DOING_SOME;
497 	trs->trso_flags |= TR_RAID1E_F_LOCKED;
498 	trs->trso_lock_pos = virtual;
499 	trs->trso_lock_len = len;
500 	/* Lock callback starts I/O */
501 	g_raid_lock_range(sd->sd_volume, virtual, len, NULL, bp);
502 }
503 
504 static void
505 g_raid_tr_raid1e_rebuild_start(struct g_raid_tr_object *tr)
506 {
507 	struct g_raid_volume *vol;
508 	struct g_raid_tr_raid1e_object *trs;
509 	struct g_raid_subdisk *sd;
510 
511 	vol = tr->tro_volume;
512 	trs = (struct g_raid_tr_raid1e_object *)tr;
513 	if (trs->trso_failed_sd) {
514 		G_RAID_DEBUG1(1, vol->v_softc,
515 		    "Already rebuild in start rebuild. pos %jd\n",
516 		    (intmax_t)trs->trso_failed_sd->sd_rebuild_pos);
517 		return;
518 	}
519 	sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_RESYNC);
520 	if (sd == NULL)
521 		sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_REBUILD);
522 	if (sd == NULL) {
523 		sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_STALE);
524 		if (sd != NULL) {
525 			sd->sd_rebuild_pos = 0;
526 			g_raid_change_subdisk_state(sd,
527 			    G_RAID_SUBDISK_S_RESYNC);
528 			g_raid_write_metadata(vol->v_softc, vol, sd, NULL);
529 		} else {
530 			sd = g_raid_get_subdisk(vol,
531 			    G_RAID_SUBDISK_S_UNINITIALIZED);
532 			if (sd == NULL)
533 				sd = g_raid_get_subdisk(vol,
534 				    G_RAID_SUBDISK_S_NEW);
535 			if (sd != NULL) {
536 				sd->sd_rebuild_pos = 0;
537 				g_raid_change_subdisk_state(sd,
538 				    G_RAID_SUBDISK_S_REBUILD);
539 				g_raid_write_metadata(vol->v_softc,
540 				    vol, sd, NULL);
541 			}
542 		}
543 	}
544 	if (sd == NULL) {
545 		G_RAID_DEBUG1(1, vol->v_softc,
546 		    "No failed disk to rebuild.  night night.");
547 		return;
548 	}
549 	trs->trso_failed_sd = sd;
550 	G_RAID_DEBUG1(0, vol->v_softc,
551 	    "Subdisk %s:%d-%s rebuild start at %jd.",
552 	    sd->sd_volume->v_name, sd->sd_pos,
553 	    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]",
554 	    trs->trso_failed_sd->sd_rebuild_pos);
555 	trs->trso_type = TR_RAID1E_REBUILD;
556 	trs->trso_buffer = malloc(g_raid1e_rebuild_slab, M_TR_RAID1E, M_WAITOK);
557 	trs->trso_meta_update = g_raid1e_rebuild_meta_update;
558 	g_raid_tr_raid1e_rebuild_some(tr);
559 }
560 
561 static void
562 g_raid_tr_raid1e_maybe_rebuild(struct g_raid_tr_object *tr,
563     struct g_raid_subdisk *sd)
564 {
565 	struct g_raid_volume *vol;
566 	struct g_raid_tr_raid1e_object *trs;
567 	int nr;
568 
569 	vol = tr->tro_volume;
570 	trs = (struct g_raid_tr_raid1e_object *)tr;
571 	if (trs->trso_stopping)
572 		return;
573 	nr = g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_REBUILD) +
574 	    g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_RESYNC);
575 	switch(trs->trso_type) {
576 	case TR_RAID1E_NONE:
577 		if (vol->v_state < G_RAID_VOLUME_S_DEGRADED)
578 			return;
579 		if (nr == 0) {
580 			nr = g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_NEW) +
581 			    g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_STALE) +
582 			    g_raid_nsubdisks(vol, G_RAID_SUBDISK_S_UNINITIALIZED);
583 			if (nr == 0)
584 				return;
585 		}
586 		g_raid_tr_raid1e_rebuild_start(tr);
587 		break;
588 	case TR_RAID1E_REBUILD:
589 		if (vol->v_state < G_RAID_VOLUME_S_DEGRADED || nr == 0 ||
590 		    trs->trso_failed_sd == sd)
591 			g_raid_tr_raid1e_rebuild_abort(tr);
592 		break;
593 	case TR_RAID1E_RESYNC:
594 		break;
595 	}
596 }
597 
598 static int
599 g_raid_tr_event_raid1e(struct g_raid_tr_object *tr,
600     struct g_raid_subdisk *sd, u_int event)
601 {
602 
603 	g_raid_tr_update_state_raid1e(tr->tro_volume, sd);
604 	return (0);
605 }
606 
607 static int
608 g_raid_tr_start_raid1e(struct g_raid_tr_object *tr)
609 {
610 	struct g_raid_tr_raid1e_object *trs;
611 	struct g_raid_volume *vol;
612 
613 	trs = (struct g_raid_tr_raid1e_object *)tr;
614 	vol = tr->tro_volume;
615 	trs->trso_starting = 0;
616 	g_raid_tr_update_state_raid1e(vol, NULL);
617 	return (0);
618 }
619 
620 static int
621 g_raid_tr_stop_raid1e(struct g_raid_tr_object *tr)
622 {
623 	struct g_raid_tr_raid1e_object *trs;
624 	struct g_raid_volume *vol;
625 
626 	trs = (struct g_raid_tr_raid1e_object *)tr;
627 	vol = tr->tro_volume;
628 	trs->trso_starting = 0;
629 	trs->trso_stopping = 1;
630 	g_raid_tr_update_state_raid1e(vol, NULL);
631 	return (0);
632 }
633 
634 /*
635  * Select the disk to read from.  Take into account: subdisk state, running
636  * error recovery, average disk load, head position and possible cache hits.
637  */
638 #define ABS(x)		(((x) >= 0) ? (x) : (-(x)))
639 static int
640 g_raid_tr_raid1e_select_read_disk(struct g_raid_volume *vol,
641     int no, off_t off, off_t len, u_int mask)
642 {
643 	struct g_raid_subdisk *sd;
644 	off_t offset;
645 	int i, best, prio, bestprio;
646 
647 	best = -1;
648 	bestprio = INT_MAX;
649 	for (i = 0; i < N; i++) {
650 		sd = &vol->v_subdisks[(no + i) % vol->v_disks_count];
651 		offset = off;
652 		if (no + i >= vol->v_disks_count)
653 			offset += vol->v_strip_size;
654 
655 		prio = G_RAID_SUBDISK_LOAD(sd);
656 		if ((mask & (1 << sd->sd_pos)) != 0)
657 			continue;
658 		switch (sd->sd_state) {
659 		case G_RAID_SUBDISK_S_ACTIVE:
660 			break;
661 		case G_RAID_SUBDISK_S_RESYNC:
662 			if (offset + off < sd->sd_rebuild_pos)
663 				break;
664 			/* FALLTHROUGH */
665 		case G_RAID_SUBDISK_S_STALE:
666 			prio += i << 24;
667 			break;
668 		case G_RAID_SUBDISK_S_REBUILD:
669 			if (offset + off < sd->sd_rebuild_pos)
670 				break;
671 			/* FALLTHROUGH */
672 		default:
673 			continue;
674 		}
675 		prio += min(sd->sd_recovery, 255) << 16;
676 		/* If disk head is precisely in position - highly prefer it. */
677 		if (G_RAID_SUBDISK_POS(sd) == offset)
678 			prio -= 2 * G_RAID_SUBDISK_LOAD_SCALE;
679 		else
680 		/* If disk head is close to position - prefer it. */
681 		if (ABS(G_RAID_SUBDISK_POS(sd) - offset) <
682 		    G_RAID_SUBDISK_TRACK_SIZE)
683 			prio -= 1 * G_RAID_SUBDISK_LOAD_SCALE;
684 		if (prio < bestprio) {
685 			bestprio = prio;
686 			best = i;
687 		}
688 	}
689 	return (best);
690 }
691 
692 static void
693 g_raid_tr_iostart_raid1e_read(struct g_raid_tr_object *tr, struct bio *bp)
694 {
695 	struct g_raid_volume *vol;
696 	struct g_raid_subdisk *sd;
697 	struct bio_queue_head queue;
698 	struct bio *cbp;
699 	char *addr;
700 	off_t offset, start, length, remain;
701 	u_int no, strip_size;
702 	int best;
703 
704 	vol = tr->tro_volume;
705 	addr = bp->bio_data;
706 	strip_size = vol->v_strip_size;
707 	V2P(vol, bp->bio_offset, &no, &offset, &start);
708 	remain = bp->bio_length;
709 	bioq_init(&queue);
710 	while (remain > 0) {
711 		length = MIN(strip_size - start, remain);
712 		best = g_raid_tr_raid1e_select_read_disk(vol,
713 		    no, offset, length, 0);
714 		KASSERT(best >= 0, ("No readable disk in volume %s!",
715 		    vol->v_name));
716 		no += best;
717 		if (no >= vol->v_disks_count) {
718 			no -= vol->v_disks_count;
719 			offset += strip_size;
720 		}
721 		cbp = g_clone_bio(bp);
722 		if (cbp == NULL)
723 			goto failure;
724 		cbp->bio_offset = offset + start;
725 		cbp->bio_data = addr;
726 		cbp->bio_length = length;
727 		cbp->bio_caller1 = &vol->v_subdisks[no];
728 		bioq_insert_tail(&queue, cbp);
729 		no += N - best;
730 		if (no >= vol->v_disks_count) {
731 			no -= vol->v_disks_count;
732 			offset += strip_size;
733 		}
734 		remain -= length;
735 		addr += length;
736 		start = 0;
737 	}
738 	for (cbp = bioq_first(&queue); cbp != NULL;
739 	    cbp = bioq_first(&queue)) {
740 		bioq_remove(&queue, cbp);
741 		sd = cbp->bio_caller1;
742 		cbp->bio_caller1 = NULL;
743 		g_raid_subdisk_iostart(sd, cbp);
744 	}
745 	return;
746 failure:
747 	for (cbp = bioq_first(&queue); cbp != NULL;
748 	    cbp = bioq_first(&queue)) {
749 		bioq_remove(&queue, cbp);
750 		g_destroy_bio(cbp);
751 	}
752 	if (bp->bio_error == 0)
753 		bp->bio_error = ENOMEM;
754 	g_raid_iodone(bp, bp->bio_error);
755 }
756 
757 static void
758 g_raid_tr_iostart_raid1e_write(struct g_raid_tr_object *tr, struct bio *bp)
759 {
760 	struct g_raid_volume *vol;
761 	struct g_raid_subdisk *sd;
762 	struct bio_queue_head queue;
763 	struct bio *cbp;
764 	char *addr;
765 	off_t offset, start, length, remain;
766 	u_int no, strip_size;
767 	int i;
768 
769 	vol = tr->tro_volume;
770 	addr = bp->bio_data;
771 	strip_size = vol->v_strip_size;
772 	V2P(vol, bp->bio_offset, &no, &offset, &start);
773 	remain = bp->bio_length;
774 	bioq_init(&queue);
775 	while (remain > 0) {
776 		length = MIN(strip_size - start, remain);
777 		for (i = 0; i < N; i++) {
778 			sd = &vol->v_subdisks[no];
779 			switch (sd->sd_state) {
780 			case G_RAID_SUBDISK_S_ACTIVE:
781 			case G_RAID_SUBDISK_S_STALE:
782 			case G_RAID_SUBDISK_S_RESYNC:
783 				break;
784 			case G_RAID_SUBDISK_S_REBUILD:
785 				if (offset + start >= sd->sd_rebuild_pos)
786 					goto nextdisk;
787 				break;
788 			default:
789 				goto nextdisk;
790 			}
791 			cbp = g_clone_bio(bp);
792 			if (cbp == NULL)
793 				goto failure;
794 			cbp->bio_offset = offset + start;
795 			cbp->bio_data = addr;
796 			cbp->bio_length = length;
797 			cbp->bio_caller1 = sd;
798 			bioq_insert_tail(&queue, cbp);
799 nextdisk:
800 			if (++no >= vol->v_disks_count) {
801 				no = 0;
802 				offset += strip_size;
803 			}
804 		}
805 		remain -= length;
806 		addr += length;
807 		start = 0;
808 	}
809 	for (cbp = bioq_first(&queue); cbp != NULL;
810 	    cbp = bioq_first(&queue)) {
811 		bioq_remove(&queue, cbp);
812 		sd = cbp->bio_caller1;
813 		cbp->bio_caller1 = NULL;
814 		g_raid_subdisk_iostart(sd, cbp);
815 	}
816 	return;
817 failure:
818 	for (cbp = bioq_first(&queue); cbp != NULL;
819 	    cbp = bioq_first(&queue)) {
820 		bioq_remove(&queue, cbp);
821 		g_destroy_bio(cbp);
822 	}
823 	if (bp->bio_error == 0)
824 		bp->bio_error = ENOMEM;
825 	g_raid_iodone(bp, bp->bio_error);
826 }
827 
828 static void
829 g_raid_tr_iostart_raid1e(struct g_raid_tr_object *tr, struct bio *bp)
830 {
831 	struct g_raid_volume *vol;
832 	struct g_raid_tr_raid1e_object *trs;
833 
834 	vol = tr->tro_volume;
835 	trs = (struct g_raid_tr_raid1e_object *)tr;
836 	if (vol->v_state != G_RAID_VOLUME_S_OPTIMAL &&
837 	    vol->v_state != G_RAID_VOLUME_S_SUBOPTIMAL &&
838 	    vol->v_state != G_RAID_VOLUME_S_DEGRADED) {
839 		g_raid_iodone(bp, EIO);
840 		return;
841 	}
842 	/*
843 	 * If we're rebuilding, squeeze in rebuild activity every so often,
844 	 * even when the disk is busy.  Be sure to only count real I/O
845 	 * to the disk.  All 'SPECIAL' I/O is traffic generated to the disk
846 	 * by this module.
847 	 */
848 	if (trs->trso_failed_sd != NULL &&
849 	    !(bp->bio_cflags & G_RAID_BIO_FLAG_SPECIAL)) {
850 		/* Make this new or running now round short. */
851 		trs->trso_recover_slabs = 0;
852 		if (--trs->trso_fair_io <= 0) {
853 			trs->trso_fair_io = g_raid1e_rebuild_fair_io;
854 			g_raid_tr_raid1e_rebuild_some(tr);
855 		}
856 	}
857 	switch (bp->bio_cmd) {
858 	case BIO_READ:
859 		g_raid_tr_iostart_raid1e_read(tr, bp);
860 		break;
861 	case BIO_WRITE:
862 		g_raid_tr_iostart_raid1e_write(tr, bp);
863 		break;
864 	case BIO_DELETE:
865 		g_raid_iodone(bp, EIO);
866 		break;
867 	case BIO_FLUSH:
868 		g_raid_tr_flush_common(tr, bp);
869 		break;
870 	default:
871 		KASSERT(1 == 0, ("Invalid command here: %u (volume=%s)",
872 		    bp->bio_cmd, vol->v_name));
873 		break;
874 	}
875 }
876 
877 static void
878 g_raid_tr_iodone_raid1e(struct g_raid_tr_object *tr,
879     struct g_raid_subdisk *sd, struct bio *bp)
880 {
881 	struct bio *cbp;
882 	struct g_raid_subdisk *nsd;
883 	struct g_raid_volume *vol;
884 	struct bio *pbp;
885 	struct g_raid_tr_raid1e_object *trs;
886 	off_t virtual, offset, start;
887 	uintptr_t mask;
888 	int error, do_write, copy, disk, best;
889 
890 	trs = (struct g_raid_tr_raid1e_object *)tr;
891 	vol = tr->tro_volume;
892 	if (bp->bio_cflags & G_RAID_BIO_FLAG_SYNC) {
893 		if (trs->trso_type == TR_RAID1E_REBUILD) {
894 			nsd = trs->trso_failed_sd;
895 			if (bp->bio_cmd == BIO_READ) {
896 
897 				/* Immediately abort rebuild, if requested. */
898 				if (trs->trso_flags & TR_RAID1E_F_ABORT) {
899 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;
900 					g_raid_tr_raid1e_rebuild_abort(tr);
901 					return;
902 				}
903 
904 				/* On read error, skip and cross fingers. */
905 				if (bp->bio_error != 0) {
906 					G_RAID_LOGREQ(0, bp,
907 					    "Read error during rebuild (%d), "
908 					    "possible data loss!",
909 					    bp->bio_error);
910 					goto rebuild_round_done;
911 				}
912 
913 				/*
914 				 * The read operation finished, queue the
915 				 * write and get out.
916 				 */
917 				G_RAID_LOGREQ(3, bp, "Rebuild read done: %d",
918 				    bp->bio_error);
919 				bp->bio_cmd = BIO_WRITE;
920 				bp->bio_cflags = G_RAID_BIO_FLAG_SYNC;
921 				bp->bio_offset = nsd->sd_rebuild_pos;
922 				G_RAID_LOGREQ(3, bp, "Queueing rebuild write.");
923 				g_raid_subdisk_iostart(nsd, bp);
924 			} else {
925 				/*
926 				 * The write operation just finished.  Do
927 				 * another.  We keep cloning the master bio
928 				 * since it has the right buffers allocated to
929 				 * it.
930 				 */
931 				G_RAID_LOGREQ(3, bp, "Rebuild write done: %d",
932 				    bp->bio_error);
933 				if (bp->bio_error != 0 ||
934 				    trs->trso_flags & TR_RAID1E_F_ABORT) {
935 					if ((trs->trso_flags &
936 					    TR_RAID1E_F_ABORT) == 0) {
937 						g_raid_tr_raid1e_fail_disk(sd->sd_softc,
938 						    nsd, nsd->sd_disk);
939 					}
940 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;
941 					g_raid_tr_raid1e_rebuild_abort(tr);
942 					return;
943 				}
944 rebuild_round_done:
945 				trs->trso_flags &= ~TR_RAID1E_F_LOCKED;
946 				g_raid_unlock_range(tr->tro_volume,
947 				    trs->trso_lock_pos, trs->trso_lock_len);
948 				nsd->sd_rebuild_pos += bp->bio_length;
949 				if (nsd->sd_rebuild_pos >= nsd->sd_size) {
950 					g_raid_tr_raid1e_rebuild_finish(tr);
951 					return;
952 				}
953 
954 				/* Abort rebuild if we are stopping */
955 				if (trs->trso_stopping) {
956 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;
957 					g_raid_tr_raid1e_rebuild_abort(tr);
958 					return;
959 				}
960 
961 				if (--trs->trso_meta_update <= 0) {
962 					g_raid_write_metadata(vol->v_softc,
963 					    vol, nsd, nsd->sd_disk);
964 					trs->trso_meta_update =
965 					    g_raid1e_rebuild_meta_update;
966 					/* Compensate short rebuild I/Os. */
967 					if ((vol->v_disks_count % N) != 0 &&
968 					    vol->v_strip_size <
969 					     g_raid1e_rebuild_slab) {
970 						trs->trso_meta_update *=
971 						    g_raid1e_rebuild_slab;
972 						trs->trso_meta_update /=
973 						    vol->v_strip_size;
974 					}
975 				}
976 				trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;
977 				if (--trs->trso_recover_slabs <= 0)
978 					return;
979 				/* Run next rebuild iteration. */
980 				g_raid_tr_raid1e_rebuild_some(tr);
981 			}
982 		} else if (trs->trso_type == TR_RAID1E_RESYNC) {
983 			/*
984 			 * read good sd, read bad sd in parallel.  when both
985 			 * done, compare the buffers.  write good to the bad
986 			 * if different.  do the next bit of work.
987 			 */
988 			panic("Somehow, we think we're doing a resync");
989 		}
990 		return;
991 	}
992 	pbp = bp->bio_parent;
993 	pbp->bio_inbed++;
994 	mask = (intptr_t)bp->bio_caller2;
995 	if (bp->bio_cmd == BIO_READ && bp->bio_error != 0) {
996 		/*
997 		 * Read failed on first drive.  Retry the read error on
998 		 * another disk drive, if available, before erroring out the
999 		 * read.
1000 		 */
1001 		sd->sd_disk->d_read_errs++;
1002 		G_RAID_LOGREQ(0, bp,
1003 		    "Read error (%d), %d read errors total",
1004 		    bp->bio_error, sd->sd_disk->d_read_errs);
1005 
1006 		/*
1007 		 * If there are too many read errors, we move to degraded.
1008 		 * XXX Do we want to FAIL the drive (eg, make the user redo
1009 		 * everything to get it back in sync), or just degrade the
1010 		 * drive, which kicks off a resync?
1011 		 */
1012 		do_write = 0;
1013 		if (sd->sd_disk->d_read_errs > g_raid_read_err_thresh)
1014 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);
1015 		else if (mask == 0)
1016 			do_write = 1;
1017 
1018 		/* Restore what we were doing. */
1019 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);
1020 		V2P(vol, virtual, &disk, &offset, &start);
1021 
1022 		/* Find the other disk, and try to do the I/O to it. */
1023 		mask |= 1 << copy;
1024 		best = g_raid_tr_raid1e_select_read_disk(vol,
1025 		    disk, offset, start, mask);
1026 		if (best >= 0 && (cbp = g_clone_bio(pbp)) != NULL) {
1027 			disk += best;
1028 			if (disk >= vol->v_disks_count) {
1029 				disk -= vol->v_disks_count;
1030 				offset += vol->v_strip_size;
1031 			}
1032 			cbp->bio_offset = offset + start;
1033 			cbp->bio_length = bp->bio_length;
1034 			cbp->bio_data = bp->bio_data;
1035 			g_destroy_bio(bp);
1036 			nsd = &vol->v_subdisks[disk];
1037 			G_RAID_LOGREQ(2, cbp, "Retrying read from %d",
1038 			    nsd->sd_pos);
1039 			if (do_write)
1040 				mask |= 1 << 31;
1041 			if ((mask & (1 << 31)) != 0)
1042 				sd->sd_recovery++;
1043 			cbp->bio_caller2 = (void *)mask;
1044 			if (do_write) {
1045 				cbp->bio_caller1 = nsd;
1046 				/* Lock callback starts I/O */
1047 				g_raid_lock_range(sd->sd_volume,
1048 				    virtual, cbp->bio_length, pbp, cbp);
1049 			} else {
1050 				g_raid_subdisk_iostart(nsd, cbp);
1051 			}
1052 			return;
1053 		}
1054 		/*
1055 		 * We can't retry.  Return the original error by falling
1056 		 * through.  This will happen when there's only one good disk.
1057 		 * We don't need to fail the raid, since its actual state is
1058 		 * based on the state of the subdisks.
1059 		 */
1060 		G_RAID_LOGREQ(2, bp, "Couldn't retry read, failing it");
1061 	}
1062 	if (bp->bio_cmd == BIO_READ &&
1063 	    bp->bio_error == 0 &&
1064 	    (mask & (1 << 31)) != 0) {
1065 		G_RAID_LOGREQ(3, bp, "Recovered data from other drive");
1066 
1067 		/* Restore what we were doing. */
1068 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);
1069 		V2P(vol, virtual, &disk, &offset, &start);
1070 
1071 		/* Find best disk to write. */
1072 		best = g_raid_tr_raid1e_select_read_disk(vol,
1073 		    disk, offset, start, ~mask);
1074 		if (best >= 0 && (cbp = g_clone_bio(pbp)) != NULL) {
1075 			disk += best;
1076 			if (disk >= vol->v_disks_count) {
1077 				disk -= vol->v_disks_count;
1078 				offset += vol->v_strip_size;
1079 			}
1080 			cbp->bio_offset = offset + start;
1081 			cbp->bio_length = bp->bio_length;
1082 			cbp->bio_data = bp->bio_data;
1083 			cbp->bio_cmd = BIO_WRITE;
1084 			cbp->bio_cflags = G_RAID_BIO_FLAG_REMAP;
1085 			cbp->bio_caller2 = (void *)mask;
1086 			g_destroy_bio(bp);
1087 			G_RAID_LOGREQ(2, cbp,
1088 			    "Attempting bad sector remap on failing drive.");
1089 			g_raid_subdisk_iostart(&vol->v_subdisks[disk], cbp);
1090 			return;
1091 		}
1092 	}
1093 	if ((mask & (1 << 31)) != 0) {
1094 		/*
1095 		 * We're done with a recovery, mark the range as unlocked.
1096 		 * For any write errors, we agressively fail the disk since
1097 		 * there was both a READ and a WRITE error at this location.
1098 		 * Both types of errors generally indicates the drive is on
1099 		 * the verge of total failure anyway.  Better to stop trusting
1100 		 * it now.  However, we need to reset error to 0 in that case
1101 		 * because we're not failing the original I/O which succeeded.
1102 		 */
1103 
1104 		/* Restore what we were doing. */
1105 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);
1106 		V2P(vol, virtual, &disk, &offset, &start);
1107 
1108 		for (copy = 0; copy < N; copy++) {
1109 			if ((mask & (1 << copy) ) != 0)
1110 				vol->v_subdisks[(disk + copy) %
1111 				    vol->v_disks_count].sd_recovery--;
1112 		}
1113 
1114 		if (bp->bio_cmd == BIO_WRITE && bp->bio_error) {
1115 			G_RAID_LOGREQ(0, bp, "Remap write failed: "
1116 			    "failing subdisk.");
1117 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);
1118 			bp->bio_error = 0;
1119 		}
1120 		G_RAID_LOGREQ(2, bp, "REMAP done %d.", bp->bio_error);
1121 		g_raid_unlock_range(sd->sd_volume, virtual, bp->bio_length);
1122 	}
1123 	if (pbp->bio_cmd != BIO_READ) {
1124 		if (pbp->bio_inbed == 1 || pbp->bio_error != 0)
1125 			pbp->bio_error = bp->bio_error;
1126 		if (bp->bio_error != 0) {
1127 			G_RAID_LOGREQ(0, bp, "Write failed: failing subdisk.");
1128 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);
1129 		}
1130 		error = pbp->bio_error;
1131 	} else
1132 		error = bp->bio_error;
1133 	g_destroy_bio(bp);
1134 	if (pbp->bio_children == pbp->bio_inbed) {
1135 		pbp->bio_completed = pbp->bio_length;
1136 		g_raid_iodone(pbp, error);
1137 	}
1138 }
1139 
1140 static int
1141 g_raid_tr_kerneldump_raid1e(struct g_raid_tr_object *tr,
1142     void *virtual, vm_offset_t physical, off_t boffset, size_t blength)
1143 {
1144 	struct g_raid_volume *vol;
1145 	struct g_raid_subdisk *sd;
1146 	struct bio_queue_head queue;
1147 	char *addr;
1148 	off_t offset, start, length, remain;
1149 	u_int no, strip_size;
1150 	int i, error;
1151 
1152 	vol = tr->tro_volume;
1153 	addr = virtual;
1154 	strip_size = vol->v_strip_size;
1155 	V2P(vol, boffset, &no, &offset, &start);
1156 	remain = blength;
1157 	bioq_init(&queue);
1158 	while (remain > 0) {
1159 		length = MIN(strip_size - start, remain);
1160 		for (i = 0; i < N; i++) {
1161 			sd = &vol->v_subdisks[no];
1162 			switch (sd->sd_state) {
1163 			case G_RAID_SUBDISK_S_ACTIVE:
1164 			case G_RAID_SUBDISK_S_STALE:
1165 			case G_RAID_SUBDISK_S_RESYNC:
1166 				break;
1167 			case G_RAID_SUBDISK_S_REBUILD:
1168 				if (offset + start >= sd->sd_rebuild_pos)
1169 					goto nextdisk;
1170 				break;
1171 			default:
1172 				goto nextdisk;
1173 			}
1174 			error = g_raid_subdisk_kerneldump(sd,
1175 			    addr, 0, offset + start, length);
1176 			if (error != 0)
1177 				return (error);
1178 nextdisk:
1179 			if (++no >= vol->v_disks_count) {
1180 				no = 0;
1181 				offset += strip_size;
1182 			}
1183 		}
1184 		remain -= length;
1185 		addr += length;
1186 		start = 0;
1187 	}
1188 	return (0);
1189 }
1190 
1191 static int
1192 g_raid_tr_locked_raid1e(struct g_raid_tr_object *tr, void *argp)
1193 {
1194 	struct bio *bp;
1195 	struct g_raid_subdisk *sd;
1196 
1197 	bp = (struct bio *)argp;
1198 	sd = (struct g_raid_subdisk *)bp->bio_caller1;
1199 	g_raid_subdisk_iostart(sd, bp);
1200 
1201 	return (0);
1202 }
1203 
1204 static int
1205 g_raid_tr_idle_raid1e(struct g_raid_tr_object *tr)
1206 {
1207 	struct g_raid_tr_raid1e_object *trs;
1208 	struct g_raid_volume *vol;
1209 
1210 	vol = tr->tro_volume;
1211 	trs = (struct g_raid_tr_raid1e_object *)tr;
1212 	trs->trso_fair_io = g_raid1e_rebuild_fair_io;
1213 	trs->trso_recover_slabs = g_raid1e_rebuild_cluster_idle;
1214 	/* Compensate short rebuild I/Os. */
1215 	if ((vol->v_disks_count % N) != 0 &&
1216 	    vol->v_strip_size < g_raid1e_rebuild_slab) {
1217 		trs->trso_recover_slabs *= g_raid1e_rebuild_slab;
1218 		trs->trso_recover_slabs /= vol->v_strip_size;
1219 	}
1220 	if (trs->trso_type == TR_RAID1E_REBUILD)
1221 		g_raid_tr_raid1e_rebuild_some(tr);
1222 	return (0);
1223 }
1224 
1225 static int
1226 g_raid_tr_free_raid1e(struct g_raid_tr_object *tr)
1227 {
1228 	struct g_raid_tr_raid1e_object *trs;
1229 
1230 	trs = (struct g_raid_tr_raid1e_object *)tr;
1231 
1232 	if (trs->trso_buffer != NULL) {
1233 		free(trs->trso_buffer, M_TR_RAID1E);
1234 		trs->trso_buffer = NULL;
1235 	}
1236 	return (0);
1237 }
1238 
1239 G_RAID_TR_DECLARE(g_raid_tr_raid1e);
1240