xref: /freebsd/sys/geom/multipath/g_multipath.c (revision 3416500aef140042c64bc149cb1ec6620483bc44)
1 /*-
2  * Copyright (c) 2011-2013 Alexander Motin <mav@FreeBSD.org>
3  * Copyright (c) 2006-2007 Matthew Jacob <mjacob@FreeBSD.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 /*
28  * Based upon work by Pawel Jakub Dawidek <pjd@FreeBSD.org> for all of the
29  * fine geom examples, and by Poul Henning Kamp <phk@FreeBSD.org> for GEOM
30  * itself, all of which is most gratefully acknowledged.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/bio.h>
43 #include <sys/sbuf.h>
44 #include <sys/sysctl.h>
45 #include <sys/kthread.h>
46 #include <sys/malloc.h>
47 #include <geom/geom.h>
48 #include <geom/multipath/g_multipath.h>
49 
50 FEATURE(geom_multipath, "GEOM multipath support");
51 
52 SYSCTL_DECL(_kern_geom);
53 static SYSCTL_NODE(_kern_geom, OID_AUTO, multipath, CTLFLAG_RW, 0,
54     "GEOM_MULTIPATH tunables");
55 static u_int g_multipath_debug = 0;
56 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW,
57     &g_multipath_debug, 0, "Debug level");
58 static u_int g_multipath_exclusive = 1;
59 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
60     &g_multipath_exclusive, 0, "Exclusively open providers");
61 
62 static enum {
63 	GKT_NIL,
64 	GKT_RUN,
65 	GKT_DIE
66 } g_multipath_kt_state;
67 static struct bio_queue_head gmtbq;
68 static struct mtx gmtbq_mtx;
69 
70 static int g_multipath_read_metadata(struct g_consumer *cp,
71     struct g_multipath_metadata *md);
72 static int g_multipath_write_metadata(struct g_consumer *cp,
73     struct g_multipath_metadata *md);
74 
75 static void g_multipath_orphan(struct g_consumer *);
76 static void g_multipath_resize(struct g_consumer *);
77 static void g_multipath_start(struct bio *);
78 static void g_multipath_done(struct bio *);
79 static void g_multipath_done_error(struct bio *);
80 static void g_multipath_kt(void *);
81 
82 static int g_multipath_destroy(struct g_geom *);
83 static int
84 g_multipath_destroy_geom(struct gctl_req *, struct g_class *, struct g_geom *);
85 
86 static struct g_geom *g_multipath_find_geom(struct g_class *, const char *);
87 static int g_multipath_rotate(struct g_geom *);
88 
89 static g_taste_t g_multipath_taste;
90 static g_ctl_req_t g_multipath_config;
91 static g_init_t g_multipath_init;
92 static g_fini_t g_multipath_fini;
93 static g_dumpconf_t g_multipath_dumpconf;
94 
95 struct g_class g_multipath_class = {
96 	.name		= G_MULTIPATH_CLASS_NAME,
97 	.version	= G_VERSION,
98 	.ctlreq		= g_multipath_config,
99 	.taste		= g_multipath_taste,
100 	.destroy_geom	= g_multipath_destroy_geom,
101 	.init		= g_multipath_init,
102 	.fini		= g_multipath_fini
103 };
104 
105 #define	MP_FAIL		0x00000001
106 #define	MP_LOST		0x00000002
107 #define	MP_NEW		0x00000004
108 #define	MP_POSTED	0x00000008
109 #define	MP_BAD		(MP_FAIL | MP_LOST | MP_NEW)
110 #define	MP_WITHER	0x00000010
111 #define	MP_IDLE		0x00000020
112 #define	MP_IDLE_MASK	0xffffffe0
113 
114 static int
115 g_multipath_good(struct g_geom *gp)
116 {
117 	struct g_consumer *cp;
118 	int n = 0;
119 
120 	LIST_FOREACH(cp, &gp->consumer, consumer) {
121 		if ((cp->index & MP_BAD) == 0)
122 			n++;
123 	}
124 	return (n);
125 }
126 
127 static void
128 g_multipath_fault(struct g_consumer *cp, int cause)
129 {
130 	struct g_multipath_softc *sc;
131 	struct g_consumer *lcp;
132 	struct g_geom *gp;
133 
134 	gp = cp->geom;
135 	sc = gp->softc;
136 	cp->index |= cause;
137 	if (g_multipath_good(gp) == 0 && sc->sc_ndisks > 0) {
138 		LIST_FOREACH(lcp, &gp->consumer, consumer) {
139 			if (lcp->provider == NULL ||
140 			    (lcp->index & (MP_LOST | MP_NEW)))
141 				continue;
142 			if (sc->sc_ndisks > 1 && lcp == cp)
143 				continue;
144 			printf("GEOM_MULTIPATH: "
145 			    "all paths in %s were marked FAIL, restore %s\n",
146 			    sc->sc_name, lcp->provider->name);
147 			lcp->index &= ~MP_FAIL;
148 		}
149 	}
150 	if (cp != sc->sc_active)
151 		return;
152 	sc->sc_active = NULL;
153 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
154 		if ((lcp->index & MP_BAD) == 0) {
155 			sc->sc_active = lcp;
156 			break;
157 		}
158 	}
159 	if (sc->sc_active == NULL) {
160 		printf("GEOM_MULTIPATH: out of providers for %s\n",
161 		    sc->sc_name);
162 	} else if (sc->sc_active_active != 1) {
163 		printf("GEOM_MULTIPATH: %s is now active path in %s\n",
164 		    sc->sc_active->provider->name, sc->sc_name);
165 	}
166 }
167 
168 static struct g_consumer *
169 g_multipath_choose(struct g_geom *gp, struct bio *bp)
170 {
171 	struct g_multipath_softc *sc;
172 	struct g_consumer *best, *cp;
173 
174 	sc = gp->softc;
175 	if (sc->sc_active_active == 0 ||
176 	    (sc->sc_active_active == 2 && bp->bio_cmd != BIO_READ))
177 		return (sc->sc_active);
178 	best = NULL;
179 	LIST_FOREACH(cp, &gp->consumer, consumer) {
180 		if (cp->index & MP_BAD)
181 			continue;
182 		cp->index += MP_IDLE;
183 		if (best == NULL || cp->private < best->private ||
184 		    (cp->private == best->private && cp->index > best->index))
185 			best = cp;
186 	}
187 	if (best != NULL)
188 		best->index &= ~MP_IDLE_MASK;
189 	return (best);
190 }
191 
192 static void
193 g_mpd(void *arg, int flags __unused)
194 {
195 	struct g_geom *gp;
196 	struct g_multipath_softc *sc;
197 	struct g_consumer *cp;
198 	int w;
199 
200 	g_topology_assert();
201 	cp = arg;
202 	gp = cp->geom;
203 	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) {
204 		w = cp->acw;
205 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
206 		if (w > 0 && cp->provider != NULL &&
207 		    (cp->provider->geom->flags & G_GEOM_WITHER) == 0) {
208 			cp->index |= MP_WITHER;
209 			g_post_event(g_mpd, cp, M_WAITOK, NULL);
210 			return;
211 		}
212 	}
213 	sc = gp->softc;
214 	mtx_lock(&sc->sc_mtx);
215 	if (cp->provider) {
216 		printf("GEOM_MULTIPATH: %s removed from %s\n",
217 		    cp->provider->name, gp->name);
218 		g_detach(cp);
219 	}
220 	g_destroy_consumer(cp);
221 	mtx_unlock(&sc->sc_mtx);
222 	if (LIST_EMPTY(&gp->consumer))
223 		g_multipath_destroy(gp);
224 }
225 
226 static void
227 g_multipath_orphan(struct g_consumer *cp)
228 {
229 	struct g_multipath_softc *sc;
230 	uintptr_t *cnt;
231 
232 	g_topology_assert();
233 	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
234 	    cp->provider->name, cp->geom->name);
235 	sc = cp->geom->softc;
236 	cnt = (uintptr_t *)&cp->private;
237 	mtx_lock(&sc->sc_mtx);
238 	sc->sc_ndisks--;
239 	g_multipath_fault(cp, MP_LOST);
240 	if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
241 		cp->index |= MP_POSTED;
242 		mtx_unlock(&sc->sc_mtx);
243 		g_mpd(cp, 0);
244 	} else
245 		mtx_unlock(&sc->sc_mtx);
246 }
247 
248 static void
249 g_multipath_resize(struct g_consumer *cp)
250 {
251 	struct g_multipath_softc *sc;
252 	struct g_geom *gp;
253 	struct g_consumer *cp1;
254 	struct g_provider *pp;
255 	struct g_multipath_metadata md;
256 	off_t size, psize, ssize;
257 	int error;
258 
259 	g_topology_assert();
260 
261 	gp = cp->geom;
262 	pp = cp->provider;
263 	sc = gp->softc;
264 
265 	if (sc->sc_stopping)
266 		return;
267 
268 	if (pp->mediasize < sc->sc_size) {
269 		size = pp->mediasize;
270 		ssize = pp->sectorsize;
271 	} else {
272 		size = ssize = OFF_MAX;
273 		mtx_lock(&sc->sc_mtx);
274 		LIST_FOREACH(cp1, &gp->consumer, consumer) {
275 			pp = cp1->provider;
276 			if (pp == NULL)
277 				continue;
278 			if (pp->mediasize < size) {
279 				size = pp->mediasize;
280 				ssize = pp->sectorsize;
281 			}
282 		}
283 		mtx_unlock(&sc->sc_mtx);
284 		if (size == OFF_MAX || size == sc->sc_size)
285 			return;
286 	}
287 	psize = size - ((sc->sc_uuid[0] != 0) ? ssize : 0);
288 	printf("GEOM_MULTIPATH: %s size changed from %jd to %jd\n",
289 	    sc->sc_name, sc->sc_pp->mediasize, psize);
290 	if (sc->sc_uuid[0] != 0 && size < sc->sc_size) {
291 		error = g_multipath_read_metadata(cp, &md);
292 		if (error ||
293 		    (strcmp(md.md_magic, G_MULTIPATH_MAGIC) != 0) ||
294 		    (memcmp(md.md_uuid, sc->sc_uuid, sizeof(sc->sc_uuid)) != 0) ||
295 		    (strcmp(md.md_name, sc->sc_name) != 0) ||
296 		    (md.md_size != 0 && md.md_size != size) ||
297 		    (md.md_sectorsize != 0 && md.md_sectorsize != ssize)) {
298 			g_multipath_destroy(gp);
299 			return;
300 		}
301 	}
302 	sc->sc_size = size;
303 	g_resize_provider(sc->sc_pp, psize);
304 
305 	if (sc->sc_uuid[0] != 0) {
306 		pp = cp->provider;
307 		strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
308 		memcpy(md.md_uuid, sc->sc_uuid, sizeof (sc->sc_uuid));
309 		strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
310 		md.md_version = G_MULTIPATH_VERSION;
311 		md.md_size = size;
312 		md.md_sectorsize = ssize;
313 		md.md_active_active = sc->sc_active_active;
314 		error = g_multipath_write_metadata(cp, &md);
315 		if (error != 0)
316 			printf("GEOM_MULTIPATH: Can't update metadata on %s "
317 			    "(%d)\n", pp->name, error);
318 	}
319 }
320 
321 static void
322 g_multipath_start(struct bio *bp)
323 {
324 	struct g_multipath_softc *sc;
325 	struct g_geom *gp;
326 	struct g_consumer *cp;
327 	struct bio *cbp;
328 	uintptr_t *cnt;
329 
330 	gp = bp->bio_to->geom;
331 	sc = gp->softc;
332 	KASSERT(sc != NULL, ("NULL sc"));
333 	cbp = g_clone_bio(bp);
334 	if (cbp == NULL) {
335 		g_io_deliver(bp, ENOMEM);
336 		return;
337 	}
338 	mtx_lock(&sc->sc_mtx);
339 	cp = g_multipath_choose(gp, bp);
340 	if (cp == NULL) {
341 		mtx_unlock(&sc->sc_mtx);
342 		g_destroy_bio(cbp);
343 		g_io_deliver(bp, ENXIO);
344 		return;
345 	}
346 	if ((uintptr_t)bp->bio_driver1 < sc->sc_ndisks)
347 		bp->bio_driver1 = (void *)(uintptr_t)sc->sc_ndisks;
348 	cnt = (uintptr_t *)&cp->private;
349 	(*cnt)++;
350 	mtx_unlock(&sc->sc_mtx);
351 	cbp->bio_done = g_multipath_done;
352 	g_io_request(cbp, cp);
353 }
354 
355 static void
356 g_multipath_done(struct bio *bp)
357 {
358 	struct g_multipath_softc *sc;
359 	struct g_consumer *cp;
360 	uintptr_t *cnt;
361 
362 	if (bp->bio_error == ENXIO || bp->bio_error == EIO) {
363 		mtx_lock(&gmtbq_mtx);
364 		bioq_insert_tail(&gmtbq, bp);
365 		mtx_unlock(&gmtbq_mtx);
366 		wakeup(&g_multipath_kt_state);
367 	} else {
368 		cp = bp->bio_from;
369 		sc = cp->geom->softc;
370 		cnt = (uintptr_t *)&cp->private;
371 		mtx_lock(&sc->sc_mtx);
372 		(*cnt)--;
373 		if (*cnt == 0 && (cp->index & MP_LOST)) {
374 			if (g_post_event(g_mpd, cp, M_NOWAIT, NULL) == 0)
375 				cp->index |= MP_POSTED;
376 			mtx_unlock(&sc->sc_mtx);
377 		} else
378 			mtx_unlock(&sc->sc_mtx);
379 		g_std_done(bp);
380 	}
381 }
382 
383 static void
384 g_multipath_done_error(struct bio *bp)
385 {
386 	struct bio *pbp;
387 	struct g_geom *gp;
388 	struct g_multipath_softc *sc;
389 	struct g_consumer *cp;
390 	struct g_provider *pp;
391 	uintptr_t *cnt;
392 
393 	/*
394 	 * If we had a failure, we have to check first to see
395 	 * whether the consumer it failed on was the currently
396 	 * active consumer (i.e., this is the first in perhaps
397 	 * a number of failures). If so, we then switch consumers
398 	 * to the next available consumer.
399 	 */
400 
401 	pbp = bp->bio_parent;
402 	gp = pbp->bio_to->geom;
403 	sc = gp->softc;
404 	cp = bp->bio_from;
405 	pp = cp->provider;
406 	cnt = (uintptr_t *)&cp->private;
407 
408 	mtx_lock(&sc->sc_mtx);
409 	if ((cp->index & MP_FAIL) == 0) {
410 		printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
411 		    bp->bio_error, pp->name, sc->sc_name);
412 		g_multipath_fault(cp, MP_FAIL);
413 	}
414 	(*cnt)--;
415 	if (*cnt == 0 && (cp->index & (MP_LOST | MP_POSTED)) == MP_LOST) {
416 		cp->index |= MP_POSTED;
417 		mtx_unlock(&sc->sc_mtx);
418 		g_post_event(g_mpd, cp, M_WAITOK, NULL);
419 	} else
420 		mtx_unlock(&sc->sc_mtx);
421 
422 	/*
423 	 * If we can fruitfully restart the I/O, do so.
424 	 */
425 	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
426 		pbp->bio_inbed++;
427 		g_destroy_bio(bp);
428 		g_multipath_start(pbp);
429 	} else {
430 		g_std_done(bp);
431 	}
432 }
433 
434 static void
435 g_multipath_kt(void *arg)
436 {
437 
438 	g_multipath_kt_state = GKT_RUN;
439 	mtx_lock(&gmtbq_mtx);
440 	while (g_multipath_kt_state == GKT_RUN) {
441 		for (;;) {
442 			struct bio *bp;
443 
444 			bp = bioq_takefirst(&gmtbq);
445 			if (bp == NULL)
446 				break;
447 			mtx_unlock(&gmtbq_mtx);
448 			g_multipath_done_error(bp);
449 			mtx_lock(&gmtbq_mtx);
450 		}
451 		if (g_multipath_kt_state != GKT_RUN)
452 			break;
453 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
454 		    "gkt:wait", 0);
455 	}
456 	mtx_unlock(&gmtbq_mtx);
457 	wakeup(&g_multipath_kt_state);
458 	kproc_exit(0);
459 }
460 
461 
462 static int
463 g_multipath_access(struct g_provider *pp, int dr, int dw, int de)
464 {
465 	struct g_geom *gp;
466 	struct g_consumer *cp, *badcp = NULL;
467 	struct g_multipath_softc *sc;
468 	int error;
469 
470 	gp = pp->geom;
471 
472 	/* Error used if we have no valid consumers. */
473 	error = (dr > 0 || dw > 0 || de > 0) ? ENXIO : 0;
474 
475 	LIST_FOREACH(cp, &gp->consumer, consumer) {
476 		if (cp->index & MP_WITHER)
477 			continue;
478 
479 		error = g_access(cp, dr, dw, de);
480 		if (error) {
481 			badcp = cp;
482 			goto fail;
483 		}
484 	}
485 
486 	if (error != 0)
487 		return (error);
488 
489 	sc = gp->softc;
490 	sc->sc_opened += dr + dw + de;
491 	if (sc->sc_stopping && sc->sc_opened == 0)
492 		g_multipath_destroy(gp);
493 
494 	return (0);
495 
496 fail:
497 	LIST_FOREACH(cp, &gp->consumer, consumer) {
498 		if (cp == badcp)
499 			break;
500 		if (cp->index & MP_WITHER)
501 			continue;
502 
503 		(void) g_access(cp, -dr, -dw, -de);
504 	}
505 	return (error);
506 }
507 
508 static struct g_geom *
509 g_multipath_create(struct g_class *mp, struct g_multipath_metadata *md)
510 {
511 	struct g_multipath_softc *sc;
512 	struct g_geom *gp;
513 	struct g_provider *pp;
514 
515 	g_topology_assert();
516 
517 	LIST_FOREACH(gp, &mp->geom, geom) {
518 		sc = gp->softc;
519 		if (sc == NULL || sc->sc_stopping)
520 			continue;
521 		if (strcmp(gp->name, md->md_name) == 0) {
522 			printf("GEOM_MULTIPATH: name %s already exists\n",
523 			    md->md_name);
524 			return (NULL);
525 		}
526 	}
527 
528 	gp = g_new_geomf(mp, "%s", md->md_name);
529 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
530 	mtx_init(&sc->sc_mtx, "multipath", NULL, MTX_DEF);
531 	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
532 	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
533 	sc->sc_active_active = md->md_active_active;
534 	sc->sc_size = md->md_size;
535 	gp->softc = sc;
536 	gp->start = g_multipath_start;
537 	gp->orphan = g_multipath_orphan;
538 	gp->resize = g_multipath_resize;
539 	gp->access = g_multipath_access;
540 	gp->dumpconf = g_multipath_dumpconf;
541 
542 	pp = g_new_providerf(gp, "multipath/%s", md->md_name);
543 	pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
544 	if (md->md_size != 0) {
545 		pp->mediasize = md->md_size -
546 		    ((md->md_uuid[0] != 0) ? md->md_sectorsize : 0);
547 		pp->sectorsize = md->md_sectorsize;
548 	}
549 	sc->sc_pp = pp;
550 	g_error_provider(pp, 0);
551 	printf("GEOM_MULTIPATH: %s created\n", gp->name);
552 	return (gp);
553 }
554 
555 static int
556 g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
557 {
558 	struct g_multipath_softc *sc;
559 	struct g_consumer *cp, *nxtcp;
560 	int error, acr, acw, ace;
561 
562 	g_topology_assert();
563 
564 	sc = gp->softc;
565 	KASSERT(sc, ("no softc"));
566 
567 	/*
568 	 * Make sure that the passed provider isn't already attached
569 	 */
570 	LIST_FOREACH(cp, &gp->consumer, consumer) {
571 		if (cp->provider == pp)
572 			break;
573 	}
574 	if (cp) {
575 		printf("GEOM_MULTIPATH: provider %s already attached to %s\n",
576 		    pp->name, gp->name);
577 		return (EEXIST);
578 	}
579 	nxtcp = LIST_FIRST(&gp->consumer);
580 	cp = g_new_consumer(gp);
581 	cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
582 	cp->private = NULL;
583 	cp->index = MP_NEW;
584 	error = g_attach(cp, pp);
585 	if (error != 0) {
586 		printf("GEOM_MULTIPATH: cannot attach %s to %s",
587 		    pp->name, sc->sc_name);
588 		g_destroy_consumer(cp);
589 		return (error);
590 	}
591 
592 	/*
593 	 * Set access permissions on new consumer to match other consumers
594 	 */
595 	if (sc->sc_pp) {
596 		acr = sc->sc_pp->acr;
597 		acw = sc->sc_pp->acw;
598 		ace = sc->sc_pp->ace;
599 	} else
600 		acr = acw = ace = 0;
601 	if (g_multipath_exclusive) {
602 		acr++;
603 		acw++;
604 		ace++;
605 	}
606 	error = g_access(cp, acr, acw, ace);
607 	if (error) {
608 		printf("GEOM_MULTIPATH: cannot set access in "
609 		    "attaching %s to %s (%d)\n",
610 		    pp->name, sc->sc_name, error);
611 		g_detach(cp);
612 		g_destroy_consumer(cp);
613 		return (error);
614 	}
615 	if (sc->sc_size == 0) {
616 		sc->sc_size = pp->mediasize -
617 		    ((sc->sc_uuid[0] != 0) ? pp->sectorsize : 0);
618 		sc->sc_pp->mediasize = sc->sc_size;
619 		sc->sc_pp->sectorsize = pp->sectorsize;
620 	}
621 	if (sc->sc_pp->stripesize == 0 && sc->sc_pp->stripeoffset == 0) {
622 		sc->sc_pp->stripesize = pp->stripesize;
623 		sc->sc_pp->stripeoffset = pp->stripeoffset;
624 	}
625 	sc->sc_pp->flags |= pp->flags & G_PF_ACCEPT_UNMAPPED;
626 	mtx_lock(&sc->sc_mtx);
627 	cp->index = 0;
628 	sc->sc_ndisks++;
629 	mtx_unlock(&sc->sc_mtx);
630 	printf("GEOM_MULTIPATH: %s added to %s\n",
631 	    pp->name, sc->sc_name);
632 	if (sc->sc_active == NULL) {
633 		sc->sc_active = cp;
634 		if (sc->sc_active_active != 1)
635 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
636 			    pp->name, sc->sc_name);
637 	}
638 	return (0);
639 }
640 
641 static int
642 g_multipath_destroy(struct g_geom *gp)
643 {
644 	struct g_multipath_softc *sc;
645 	struct g_consumer *cp, *cp1;
646 
647 	g_topology_assert();
648 	if (gp->softc == NULL)
649 		return (ENXIO);
650 	sc = gp->softc;
651 	if (!sc->sc_stopping) {
652 		printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
653 		sc->sc_stopping = 1;
654 	}
655 	if (sc->sc_opened != 0) {
656 		g_wither_provider(sc->sc_pp, ENXIO);
657 		sc->sc_pp = NULL;
658 		return (EINPROGRESS);
659 	}
660 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
661 		mtx_lock(&sc->sc_mtx);
662 		if ((cp->index & MP_POSTED) == 0) {
663 			cp->index |= MP_POSTED;
664 			mtx_unlock(&sc->sc_mtx);
665 			g_mpd(cp, 0);
666 			if (cp1 == NULL)
667 				return(0);	/* Recursion happened. */
668 		} else
669 			mtx_unlock(&sc->sc_mtx);
670 	}
671 	if (!LIST_EMPTY(&gp->consumer))
672 		return (EINPROGRESS);
673 	mtx_destroy(&sc->sc_mtx);
674 	g_free(gp->softc);
675 	gp->softc = NULL;
676 	printf("GEOM_MULTIPATH: %s destroyed\n", gp->name);
677 	g_wither_geom(gp, ENXIO);
678 	return (0);
679 }
680 
681 static int
682 g_multipath_destroy_geom(struct gctl_req *req, struct g_class *mp,
683     struct g_geom *gp)
684 {
685 
686 	return (g_multipath_destroy(gp));
687 }
688 
689 static int
690 g_multipath_rotate(struct g_geom *gp)
691 {
692 	struct g_consumer *lcp, *first_good_cp = NULL;
693 	struct g_multipath_softc *sc = gp->softc;
694 	int active_cp_seen = 0;
695 
696 	g_topology_assert();
697 	if (sc == NULL)
698 		return (ENXIO);
699 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
700 		if ((lcp->index & MP_BAD) == 0) {
701 			if (first_good_cp == NULL)
702 				first_good_cp = lcp;
703 			if (active_cp_seen)
704 				break;
705 		}
706 		if (sc->sc_active == lcp)
707 			active_cp_seen = 1;
708 	}
709 	if (lcp == NULL)
710 		lcp = first_good_cp;
711 	if (lcp && lcp != sc->sc_active) {
712 		sc->sc_active = lcp;
713 		if (sc->sc_active_active != 1)
714 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
715 			    lcp->provider->name, sc->sc_name);
716 	}
717 	return (0);
718 }
719 
720 static void
721 g_multipath_init(struct g_class *mp)
722 {
723 	bioq_init(&gmtbq);
724 	mtx_init(&gmtbq_mtx, "gmtbq", NULL, MTX_DEF);
725 	kproc_create(g_multipath_kt, mp, NULL, 0, 0, "g_mp_kt");
726 }
727 
728 static void
729 g_multipath_fini(struct g_class *mp)
730 {
731 	if (g_multipath_kt_state == GKT_RUN) {
732 		mtx_lock(&gmtbq_mtx);
733 		g_multipath_kt_state = GKT_DIE;
734 		wakeup(&g_multipath_kt_state);
735 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
736 		    "gmp:fini", 0);
737 		mtx_unlock(&gmtbq_mtx);
738 	}
739 }
740 
741 static int
742 g_multipath_read_metadata(struct g_consumer *cp,
743     struct g_multipath_metadata *md)
744 {
745 	struct g_provider *pp;
746 	u_char *buf;
747 	int error;
748 
749 	g_topology_assert();
750 	error = g_access(cp, 1, 0, 0);
751 	if (error != 0)
752 		return (error);
753 	pp = cp->provider;
754 	g_topology_unlock();
755 	buf = g_read_data(cp, pp->mediasize - pp->sectorsize,
756 	    pp->sectorsize, &error);
757 	g_topology_lock();
758 	g_access(cp, -1, 0, 0);
759 	if (buf == NULL)
760 		return (error);
761 	multipath_metadata_decode(buf, md);
762 	g_free(buf);
763 	return (0);
764 }
765 
766 static int
767 g_multipath_write_metadata(struct g_consumer *cp,
768     struct g_multipath_metadata *md)
769 {
770 	struct g_provider *pp;
771 	u_char *buf;
772 	int error;
773 
774 	g_topology_assert();
775 	error = g_access(cp, 1, 1, 1);
776 	if (error != 0)
777 		return (error);
778 	pp = cp->provider;
779 	g_topology_unlock();
780 	buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
781 	multipath_metadata_encode(md, buf);
782 	error = g_write_data(cp, pp->mediasize - pp->sectorsize,
783 	    buf, pp->sectorsize);
784 	g_topology_lock();
785 	g_access(cp, -1, -1, -1);
786 	g_free(buf);
787 	return (error);
788 }
789 
790 static struct g_geom *
791 g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
792 {
793 	struct g_multipath_metadata md;
794 	struct g_multipath_softc *sc;
795 	struct g_consumer *cp;
796 	struct g_geom *gp, *gp1;
797 	int error, isnew;
798 
799 	g_topology_assert();
800 
801 	gp = g_new_geomf(mp, "multipath:taste");
802 	gp->start = g_multipath_start;
803 	gp->access = g_multipath_access;
804 	gp->orphan = g_multipath_orphan;
805 	cp = g_new_consumer(gp);
806 	g_attach(cp, pp);
807 	error = g_multipath_read_metadata(cp, &md);
808 	g_detach(cp);
809 	g_destroy_consumer(cp);
810 	g_destroy_geom(gp);
811 	if (error != 0)
812 		return (NULL);
813 	gp = NULL;
814 
815 	if (strcmp(md.md_magic, G_MULTIPATH_MAGIC) != 0) {
816 		if (g_multipath_debug)
817 			printf("%s is not MULTIPATH\n", pp->name);
818 		return (NULL);
819 	}
820 	if (md.md_version != G_MULTIPATH_VERSION) {
821 		printf("%s has version %d multipath id- this module is version "
822 		    " %d: rejecting\n", pp->name, md.md_version,
823 		    G_MULTIPATH_VERSION);
824 		return (NULL);
825 	}
826 	if (md.md_size != 0 && md.md_size != pp->mediasize)
827 		return (NULL);
828 	if (md.md_sectorsize != 0 && md.md_sectorsize != pp->sectorsize)
829 		return (NULL);
830 	if (g_multipath_debug)
831 		printf("MULTIPATH: %s/%s\n", md.md_name, md.md_uuid);
832 
833 	/*
834 	 * Let's check if such a device already is present. We check against
835 	 * uuid alone first because that's the true distinguishor. If that
836 	 * passes, then we check for name conflicts. If there are conflicts,
837 	 * modify the name.
838 	 *
839 	 * The whole purpose of this is to solve the problem that people don't
840 	 * pick good unique names, but good unique names (like uuids) are a
841 	 * pain to use. So, we allow people to build GEOMs with friendly names
842 	 * and uuids, and modify the names in case there's a collision.
843 	 */
844 	sc = NULL;
845 	LIST_FOREACH(gp, &mp->geom, geom) {
846 		sc = gp->softc;
847 		if (sc == NULL || sc->sc_stopping)
848 			continue;
849 		if (strncmp(md.md_uuid, sc->sc_uuid, sizeof(md.md_uuid)) == 0)
850 			break;
851 	}
852 
853 	LIST_FOREACH(gp1, &mp->geom, geom) {
854 		if (gp1 == gp)
855 			continue;
856 		sc = gp1->softc;
857 		if (sc == NULL || sc->sc_stopping)
858 			continue;
859 		if (strncmp(md.md_name, sc->sc_name, sizeof(md.md_name)) == 0)
860 			break;
861 	}
862 
863 	/*
864 	 * If gp is NULL, we had no extant MULTIPATH geom with this uuid.
865 	 *
866 	 * If gp1 is *not* NULL, that means we have a MULTIPATH geom extant
867 	 * with the same name (but a different UUID).
868 	 *
869 	 * If gp is NULL, then modify the name with a random number and
870   	 * complain, but allow the creation of the geom to continue.
871 	 *
872 	 * If gp is *not* NULL, just use the geom's name as we're attaching
873 	 * this disk to the (previously generated) name.
874 	 */
875 
876 	if (gp1) {
877 		sc = gp1->softc;
878 		if (gp == NULL) {
879 			char buf[16];
880 			u_long rand = random();
881 
882 			snprintf(buf, sizeof (buf), "%s-%lu", md.md_name, rand);
883 			printf("GEOM_MULTIPATH: geom %s/%s exists already\n",
884 			    sc->sc_name, sc->sc_uuid);
885 			printf("GEOM_MULTIPATH: %s will be (temporarily) %s\n",
886 			    md.md_uuid, buf);
887 			strlcpy(md.md_name, buf, sizeof(md.md_name));
888 		} else {
889 			strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
890 		}
891 	}
892 
893 	if (gp == NULL) {
894 		gp = g_multipath_create(mp, &md);
895 		if (gp == NULL) {
896 			printf("GEOM_MULTIPATH: cannot create geom %s/%s\n",
897 			    md.md_name, md.md_uuid);
898 			return (NULL);
899 		}
900 		isnew = 1;
901 	} else {
902 		isnew = 0;
903 	}
904 
905 	sc = gp->softc;
906 	KASSERT(sc != NULL, ("sc is NULL"));
907 	error = g_multipath_add_disk(gp, pp);
908 	if (error != 0) {
909 		if (isnew)
910 			g_multipath_destroy(gp);
911 		return (NULL);
912 	}
913 	return (gp);
914 }
915 
916 static void
917 g_multipath_ctl_add_name(struct gctl_req *req, struct g_class *mp,
918     const char *name)
919 {
920 	struct g_multipath_softc *sc;
921 	struct g_geom *gp;
922 	struct g_consumer *cp;
923 	struct g_provider *pp;
924 	const char *mpname;
925 	static const char devpf[6] = "/dev/";
926 	int error;
927 
928 	g_topology_assert();
929 
930 	mpname = gctl_get_asciiparam(req, "arg0");
931         if (mpname == NULL) {
932                 gctl_error(req, "No 'arg0' argument");
933                 return;
934         }
935 	gp = g_multipath_find_geom(mp, mpname);
936 	if (gp == NULL) {
937 		gctl_error(req, "Device %s is invalid", mpname);
938 		return;
939 	}
940 	sc = gp->softc;
941 
942 	if (strncmp(name, devpf, 5) == 0)
943 		name += 5;
944 	pp = g_provider_by_name(name);
945 	if (pp == NULL) {
946 		gctl_error(req, "Provider %s is invalid", name);
947 		return;
948 	}
949 
950 	/*
951 	 * Check to make sure parameters match.
952 	 */
953 	LIST_FOREACH(cp, &gp->consumer, consumer) {
954 		if (cp->provider == pp) {
955 			gctl_error(req, "provider %s is already there",
956 			    pp->name);
957 			return;
958 		}
959 	}
960 	if (sc->sc_pp->mediasize != 0 &&
961 	    sc->sc_pp->mediasize + (sc->sc_uuid[0] != 0 ? pp->sectorsize : 0)
962 	     != pp->mediasize) {
963 		gctl_error(req, "Providers size mismatch %jd != %jd",
964 		    (intmax_t) sc->sc_pp->mediasize +
965 			(sc->sc_uuid[0] != 0 ? pp->sectorsize : 0),
966 		    (intmax_t) pp->mediasize);
967 		return;
968 	}
969 	if (sc->sc_pp->sectorsize != 0 &&
970 	    sc->sc_pp->sectorsize != pp->sectorsize) {
971 		gctl_error(req, "Providers sectorsize mismatch %u != %u",
972 		    sc->sc_pp->sectorsize, pp->sectorsize);
973 		return;
974 	}
975 
976 	error = g_multipath_add_disk(gp, pp);
977 	if (error != 0)
978 		gctl_error(req, "Provider addition error: %d", error);
979 }
980 
981 static void
982 g_multipath_ctl_prefer(struct gctl_req *req, struct g_class *mp)
983 {
984 	struct g_geom *gp;
985 	struct g_multipath_softc *sc;
986 	struct g_consumer *cp;
987 	const char *name, *mpname;
988 	static const char devpf[6] = "/dev/";
989 	int *nargs;
990 
991 	g_topology_assert();
992 
993 	mpname = gctl_get_asciiparam(req, "arg0");
994         if (mpname == NULL) {
995                 gctl_error(req, "No 'arg0' argument");
996                 return;
997         }
998 	gp = g_multipath_find_geom(mp, mpname);
999 	if (gp == NULL) {
1000 		gctl_error(req, "Device %s is invalid", mpname);
1001 		return;
1002 	}
1003 	sc = gp->softc;
1004 
1005 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
1006 	if (nargs == NULL) {
1007 		gctl_error(req, "No 'nargs' argument");
1008 		return;
1009 	}
1010 	if (*nargs != 2) {
1011 		gctl_error(req, "missing device");
1012 		return;
1013 	}
1014 
1015 	name = gctl_get_asciiparam(req, "arg1");
1016 	if (name == NULL) {
1017 		gctl_error(req, "No 'arg1' argument");
1018 		return;
1019 	}
1020 	if (strncmp(name, devpf, 5) == 0) {
1021 		name += 5;
1022 	}
1023 
1024 	LIST_FOREACH(cp, &gp->consumer, consumer) {
1025 		if (cp->provider != NULL
1026                       && strcmp(cp->provider->name, name) == 0)
1027 		    break;
1028 	}
1029 
1030 	if (cp == NULL) {
1031 		gctl_error(req, "Provider %s not found", name);
1032 		return;
1033 	}
1034 
1035 	mtx_lock(&sc->sc_mtx);
1036 
1037 	if (cp->index & MP_BAD) {
1038 		gctl_error(req, "Consumer %s is invalid", name);
1039 		mtx_unlock(&sc->sc_mtx);
1040 		return;
1041 	}
1042 
1043 	/* Here when the consumer is present and in good shape */
1044 
1045 	sc->sc_active = cp;
1046 	if (!sc->sc_active_active)
1047 	    printf("GEOM_MULTIPATH: %s now active path in %s\n",
1048 		sc->sc_active->provider->name, sc->sc_name);
1049 
1050 	mtx_unlock(&sc->sc_mtx);
1051 }
1052 
1053 static void
1054 g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
1055 {
1056 	struct g_multipath_softc *sc;
1057 	struct g_geom *gp;
1058 	const char *mpname, *name;
1059 
1060 	mpname = gctl_get_asciiparam(req, "arg0");
1061         if (mpname == NULL) {
1062                 gctl_error(req, "No 'arg0' argument");
1063                 return;
1064         }
1065 	gp = g_multipath_find_geom(mp, mpname);
1066 	if (gp == NULL) {
1067 		gctl_error(req, "Device %s not found", mpname);
1068 		return;
1069 	}
1070 	sc = gp->softc;
1071 
1072 	name = gctl_get_asciiparam(req, "arg1");
1073 	if (name == NULL) {
1074 		gctl_error(req, "No 'arg1' argument");
1075 		return;
1076 	}
1077 	g_multipath_ctl_add_name(req, mp, name);
1078 }
1079 
1080 static void
1081 g_multipath_ctl_create(struct gctl_req *req, struct g_class *mp)
1082 {
1083 	struct g_multipath_metadata md;
1084 	struct g_multipath_softc *sc;
1085 	struct g_geom *gp;
1086 	const char *mpname, *name;
1087 	char param[16];
1088 	int *nargs, i, *val;
1089 
1090 	g_topology_assert();
1091 
1092 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
1093 	if (*nargs < 2) {
1094 		gctl_error(req, "wrong number of arguments.");
1095 		return;
1096 	}
1097 
1098 	mpname = gctl_get_asciiparam(req, "arg0");
1099         if (mpname == NULL) {
1100                 gctl_error(req, "No 'arg0' argument");
1101                 return;
1102         }
1103 	gp = g_multipath_find_geom(mp, mpname);
1104 	if (gp != NULL) {
1105 		gctl_error(req, "Device %s already exist", mpname);
1106 		return;
1107 	}
1108 
1109 	memset(&md, 0, sizeof(md));
1110 	strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
1111 	md.md_version = G_MULTIPATH_VERSION;
1112 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
1113 	md.md_size = 0;
1114 	md.md_sectorsize = 0;
1115 	md.md_uuid[0] = 0;
1116 	md.md_active_active = 0;
1117 	val = gctl_get_paraml(req, "active_active", sizeof(*val));
1118 	if (val != NULL && *val != 0)
1119 		md.md_active_active = 1;
1120 	val = gctl_get_paraml(req, "active_read", sizeof(*val));
1121 	if (val != NULL && *val != 0)
1122 		md.md_active_active = 2;
1123 	gp = g_multipath_create(mp, &md);
1124 	if (gp == NULL) {
1125 		gctl_error(req, "GEOM_MULTIPATH: cannot create geom %s/%s\n",
1126 		    md.md_name, md.md_uuid);
1127 		return;
1128 	}
1129 	sc = gp->softc;
1130 
1131 	for (i = 1; i < *nargs; i++) {
1132 		snprintf(param, sizeof(param), "arg%d", i);
1133 		name = gctl_get_asciiparam(req, param);
1134 		g_multipath_ctl_add_name(req, mp, name);
1135 	}
1136 
1137 	if (sc->sc_ndisks != (*nargs - 1))
1138 		g_multipath_destroy(gp);
1139 }
1140 
1141 static void
1142 g_multipath_ctl_configure(struct gctl_req *req, struct g_class *mp)
1143 {
1144 	struct g_multipath_softc *sc;
1145 	struct g_geom *gp;
1146 	struct g_consumer *cp;
1147 	struct g_provider *pp;
1148 	struct g_multipath_metadata md;
1149 	const char *name;
1150 	int error, *val;
1151 
1152 	g_topology_assert();
1153 
1154 	name = gctl_get_asciiparam(req, "arg0");
1155 	if (name == NULL) {
1156 		gctl_error(req, "No 'arg0' argument");
1157 		return;
1158 	}
1159 	gp = g_multipath_find_geom(mp, name);
1160 	if (gp == NULL) {
1161 		gctl_error(req, "Device %s is invalid", name);
1162 		return;
1163 	}
1164 	sc = gp->softc;
1165 	val = gctl_get_paraml(req, "active_active", sizeof(*val));
1166 	if (val != NULL && *val != 0)
1167 		sc->sc_active_active = 1;
1168 	val = gctl_get_paraml(req, "active_read", sizeof(*val));
1169 	if (val != NULL && *val != 0)
1170 		sc->sc_active_active = 2;
1171 	val = gctl_get_paraml(req, "active_passive", sizeof(*val));
1172 	if (val != NULL && *val != 0)
1173 		sc->sc_active_active = 0;
1174 	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
1175 		cp = sc->sc_active;
1176 		pp = cp->provider;
1177 		strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
1178 		memcpy(md.md_uuid, sc->sc_uuid, sizeof (sc->sc_uuid));
1179 		strlcpy(md.md_name, name, sizeof(md.md_name));
1180 		md.md_version = G_MULTIPATH_VERSION;
1181 		md.md_size = pp->mediasize;
1182 		md.md_sectorsize = pp->sectorsize;
1183 		md.md_active_active = sc->sc_active_active;
1184 		error = g_multipath_write_metadata(cp, &md);
1185 		if (error != 0)
1186 			gctl_error(req, "Can't update metadata on %s (%d)",
1187 			    pp->name, error);
1188 	}
1189 }
1190 
1191 static void
1192 g_multipath_ctl_fail(struct gctl_req *req, struct g_class *mp, int fail)
1193 {
1194 	struct g_multipath_softc *sc;
1195 	struct g_geom *gp;
1196 	struct g_consumer *cp;
1197 	const char *mpname, *name;
1198 	int found;
1199 
1200 	mpname = gctl_get_asciiparam(req, "arg0");
1201         if (mpname == NULL) {
1202                 gctl_error(req, "No 'arg0' argument");
1203                 return;
1204         }
1205 	gp = g_multipath_find_geom(mp, mpname);
1206 	if (gp == NULL) {
1207 		gctl_error(req, "Device %s not found", mpname);
1208 		return;
1209 	}
1210 	sc = gp->softc;
1211 
1212 	name = gctl_get_asciiparam(req, "arg1");
1213 	if (name == NULL) {
1214 		gctl_error(req, "No 'arg1' argument");
1215 		return;
1216 	}
1217 
1218 	found = 0;
1219 	mtx_lock(&sc->sc_mtx);
1220 	LIST_FOREACH(cp, &gp->consumer, consumer) {
1221 		if (cp->provider != NULL &&
1222 		    strcmp(cp->provider->name, name) == 0 &&
1223 		    (cp->index & MP_LOST) == 0) {
1224 			found = 1;
1225 			if (!fail == !(cp->index & MP_FAIL))
1226 				continue;
1227 			printf("GEOM_MULTIPATH: %s in %s is marked %s.\n",
1228 				name, sc->sc_name, fail ? "FAIL" : "OK");
1229 			if (fail) {
1230 				g_multipath_fault(cp, MP_FAIL);
1231 			} else {
1232 				cp->index &= ~MP_FAIL;
1233 			}
1234 		}
1235 	}
1236 	mtx_unlock(&sc->sc_mtx);
1237 	if (found == 0)
1238 		gctl_error(req, "Provider %s not found", name);
1239 }
1240 
1241 static void
1242 g_multipath_ctl_remove(struct gctl_req *req, struct g_class *mp)
1243 {
1244 	struct g_multipath_softc *sc;
1245 	struct g_geom *gp;
1246 	struct g_consumer *cp, *cp1;
1247 	const char *mpname, *name;
1248 	uintptr_t *cnt;
1249 	int found;
1250 
1251 	mpname = gctl_get_asciiparam(req, "arg0");
1252         if (mpname == NULL) {
1253                 gctl_error(req, "No 'arg0' argument");
1254                 return;
1255         }
1256 	gp = g_multipath_find_geom(mp, mpname);
1257 	if (gp == NULL) {
1258 		gctl_error(req, "Device %s not found", mpname);
1259 		return;
1260 	}
1261 	sc = gp->softc;
1262 
1263 	name = gctl_get_asciiparam(req, "arg1");
1264 	if (name == NULL) {
1265 		gctl_error(req, "No 'arg1' argument");
1266 		return;
1267 	}
1268 
1269 	found = 0;
1270 	mtx_lock(&sc->sc_mtx);
1271 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
1272 		if (cp->provider != NULL &&
1273 		    strcmp(cp->provider->name, name) == 0 &&
1274 		    (cp->index & MP_LOST) == 0) {
1275 			found = 1;
1276 			printf("GEOM_MULTIPATH: removing %s from %s\n",
1277 			    cp->provider->name, cp->geom->name);
1278 			sc->sc_ndisks--;
1279 			g_multipath_fault(cp, MP_LOST);
1280 			cnt = (uintptr_t *)&cp->private;
1281 			if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
1282 				cp->index |= MP_POSTED;
1283 				mtx_unlock(&sc->sc_mtx);
1284 				g_mpd(cp, 0);
1285 				if (cp1 == NULL)
1286 					return;	/* Recursion happened. */
1287 				mtx_lock(&sc->sc_mtx);
1288 			}
1289 		}
1290 	}
1291 	mtx_unlock(&sc->sc_mtx);
1292 	if (found == 0)
1293 		gctl_error(req, "Provider %s not found", name);
1294 }
1295 
1296 static struct g_geom *
1297 g_multipath_find_geom(struct g_class *mp, const char *name)
1298 {
1299 	struct g_geom *gp;
1300 	struct g_multipath_softc *sc;
1301 
1302 	LIST_FOREACH(gp, &mp->geom, geom) {
1303 		sc = gp->softc;
1304 		if (sc == NULL || sc->sc_stopping)
1305 			continue;
1306 		if (strcmp(gp->name, name) == 0)
1307 			return (gp);
1308 	}
1309 	return (NULL);
1310 }
1311 
1312 static void
1313 g_multipath_ctl_stop(struct gctl_req *req, struct g_class *mp)
1314 {
1315 	struct g_geom *gp;
1316 	const char *name;
1317 	int error;
1318 
1319 	g_topology_assert();
1320 
1321 	name = gctl_get_asciiparam(req, "arg0");
1322         if (name == NULL) {
1323                 gctl_error(req, "No 'arg0' argument");
1324                 return;
1325         }
1326 	gp = g_multipath_find_geom(mp, name);
1327 	if (gp == NULL) {
1328 		gctl_error(req, "Device %s is invalid", name);
1329 		return;
1330 	}
1331 	error = g_multipath_destroy(gp);
1332 	if (error != 0 && error != EINPROGRESS)
1333 		gctl_error(req, "failed to stop %s (err=%d)", name, error);
1334 }
1335 
1336 static void
1337 g_multipath_ctl_destroy(struct gctl_req *req, struct g_class *mp)
1338 {
1339 	struct g_geom *gp;
1340 	struct g_multipath_softc *sc;
1341 	struct g_consumer *cp;
1342 	struct g_provider *pp;
1343 	const char *name;
1344 	uint8_t *buf;
1345 	int error;
1346 
1347 	g_topology_assert();
1348 
1349 	name = gctl_get_asciiparam(req, "arg0");
1350         if (name == NULL) {
1351                 gctl_error(req, "No 'arg0' argument");
1352                 return;
1353         }
1354 	gp = g_multipath_find_geom(mp, name);
1355 	if (gp == NULL) {
1356 		gctl_error(req, "Device %s is invalid", name);
1357 		return;
1358 	}
1359 	sc = gp->softc;
1360 
1361 	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
1362 		cp = sc->sc_active;
1363 		pp = cp->provider;
1364 		error = g_access(cp, 1, 1, 1);
1365 		if (error != 0) {
1366 			gctl_error(req, "Can't open %s (%d)", pp->name, error);
1367 			goto destroy;
1368 		}
1369 		g_topology_unlock();
1370 		buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
1371 		error = g_write_data(cp, pp->mediasize - pp->sectorsize,
1372 		    buf, pp->sectorsize);
1373 		g_topology_lock();
1374 		g_access(cp, -1, -1, -1);
1375 		if (error != 0)
1376 			gctl_error(req, "Can't erase metadata on %s (%d)",
1377 			    pp->name, error);
1378 	}
1379 
1380 destroy:
1381 	error = g_multipath_destroy(gp);
1382 	if (error != 0 && error != EINPROGRESS)
1383 		gctl_error(req, "failed to destroy %s (err=%d)", name, error);
1384 }
1385 
1386 static void
1387 g_multipath_ctl_rotate(struct gctl_req *req, struct g_class *mp)
1388 {
1389 	struct g_geom *gp;
1390 	const char *name;
1391 	int error;
1392 
1393 	g_topology_assert();
1394 
1395 	name = gctl_get_asciiparam(req, "arg0");
1396         if (name == NULL) {
1397                 gctl_error(req, "No 'arg0' argument");
1398                 return;
1399         }
1400 	gp = g_multipath_find_geom(mp, name);
1401 	if (gp == NULL) {
1402 		gctl_error(req, "Device %s is invalid", name);
1403 		return;
1404 	}
1405 	error = g_multipath_rotate(gp);
1406 	if (error != 0) {
1407 		gctl_error(req, "failed to rotate %s (err=%d)", name, error);
1408 	}
1409 }
1410 
1411 static void
1412 g_multipath_ctl_getactive(struct gctl_req *req, struct g_class *mp)
1413 {
1414 	struct sbuf *sb;
1415 	struct g_geom *gp;
1416 	struct g_multipath_softc *sc;
1417 	struct g_consumer *cp;
1418 	const char *name;
1419 	int empty;
1420 
1421 	sb = sbuf_new_auto();
1422 
1423 	g_topology_assert();
1424 	name = gctl_get_asciiparam(req, "arg0");
1425         if (name == NULL) {
1426                 gctl_error(req, "No 'arg0' argument");
1427                 return;
1428         }
1429 	gp = g_multipath_find_geom(mp, name);
1430 	if (gp == NULL) {
1431 		gctl_error(req, "Device %s is invalid", name);
1432 		return;
1433 	}
1434 	sc = gp->softc;
1435 	if (sc->sc_active_active == 1) {
1436 		empty = 1;
1437 		LIST_FOREACH(cp, &gp->consumer, consumer) {
1438 			if (cp->index & MP_BAD)
1439 				continue;
1440 			if (!empty)
1441 				sbuf_cat(sb, " ");
1442 			sbuf_cat(sb, cp->provider->name);
1443 			empty = 0;
1444 		}
1445 		if (empty)
1446 			sbuf_cat(sb, "none");
1447 		sbuf_cat(sb, "\n");
1448 	} else if (sc->sc_active && sc->sc_active->provider) {
1449 		sbuf_printf(sb, "%s\n", sc->sc_active->provider->name);
1450 	} else {
1451 		sbuf_printf(sb, "none\n");
1452 	}
1453 	sbuf_finish(sb);
1454 	gctl_set_param_err(req, "output", sbuf_data(sb), sbuf_len(sb) + 1);
1455 	sbuf_delete(sb);
1456 }
1457 
1458 static void
1459 g_multipath_config(struct gctl_req *req, struct g_class *mp, const char *verb)
1460 {
1461 	uint32_t *version;
1462 	g_topology_assert();
1463 	version = gctl_get_paraml(req, "version", sizeof(*version));
1464 	if (version == NULL) {
1465 		gctl_error(req, "No 'version' argument");
1466 	} else if (*version != G_MULTIPATH_VERSION) {
1467 		gctl_error(req, "Userland and kernel parts are out of sync");
1468 	} else if (strcmp(verb, "add") == 0) {
1469 		g_multipath_ctl_add(req, mp);
1470 	} else if (strcmp(verb, "prefer") == 0) {
1471 		g_multipath_ctl_prefer(req, mp);
1472 	} else if (strcmp(verb, "create") == 0) {
1473 		g_multipath_ctl_create(req, mp);
1474 	} else if (strcmp(verb, "configure") == 0) {
1475 		g_multipath_ctl_configure(req, mp);
1476 	} else if (strcmp(verb, "stop") == 0) {
1477 		g_multipath_ctl_stop(req, mp);
1478 	} else if (strcmp(verb, "destroy") == 0) {
1479 		g_multipath_ctl_destroy(req, mp);
1480 	} else if (strcmp(verb, "fail") == 0) {
1481 		g_multipath_ctl_fail(req, mp, 1);
1482 	} else if (strcmp(verb, "restore") == 0) {
1483 		g_multipath_ctl_fail(req, mp, 0);
1484 	} else if (strcmp(verb, "remove") == 0) {
1485 		g_multipath_ctl_remove(req, mp);
1486 	} else if (strcmp(verb, "rotate") == 0) {
1487 		g_multipath_ctl_rotate(req, mp);
1488 	} else if (strcmp(verb, "getactive") == 0) {
1489 		g_multipath_ctl_getactive(req, mp);
1490 	} else {
1491 		gctl_error(req, "Unknown verb %s", verb);
1492 	}
1493 }
1494 
1495 static void
1496 g_multipath_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1497     struct g_consumer *cp, struct g_provider *pp)
1498 {
1499 	struct g_multipath_softc *sc;
1500 	int good;
1501 
1502 	g_topology_assert();
1503 
1504 	sc = gp->softc;
1505 	if (sc == NULL)
1506 		return;
1507 	if (cp != NULL) {
1508 		sbuf_printf(sb, "%s<State>%s</State>\n", indent,
1509 		    (cp->index & MP_NEW) ? "NEW" :
1510 		    (cp->index & MP_LOST) ? "LOST" :
1511 		    (cp->index & MP_FAIL) ? "FAIL" :
1512 		    (sc->sc_active_active == 1 || sc->sc_active == cp) ?
1513 		     "ACTIVE" :
1514 		     sc->sc_active_active == 2 ? "READ" : "PASSIVE");
1515 	} else {
1516 		good = g_multipath_good(gp);
1517 		sbuf_printf(sb, "%s<State>%s</State>\n", indent,
1518 		    good == 0 ? "BROKEN" :
1519 		    (good != sc->sc_ndisks || sc->sc_ndisks == 1) ?
1520 		    "DEGRADED" : "OPTIMAL");
1521 	}
1522 	if (cp == NULL && pp == NULL) {
1523 		sbuf_printf(sb, "%s<UUID>%s</UUID>\n", indent, sc->sc_uuid);
1524 		sbuf_printf(sb, "%s<Mode>Active/%s</Mode>\n", indent,
1525 		    sc->sc_active_active == 2 ? "Read" :
1526 		    sc->sc_active_active == 1 ? "Active" : "Passive");
1527 		sbuf_printf(sb, "%s<Type>%s</Type>\n", indent,
1528 		    sc->sc_uuid[0] == 0 ? "MANUAL" : "AUTOMATIC");
1529 	}
1530 }
1531 
1532 DECLARE_GEOM_CLASS(g_multipath_class, g_multipath);
1533