xref: /freebsd/sys/geom/multipath/g_multipath.c (revision fcb560670601b2a4d87bb31d7531c8dcc37ee71b)
1 /*-
2  * Copyright (c) 2011-2013 Alexander Motin <mav@FreeBSD.org>
3  * Copyright (c) 2006-2007 Matthew Jacob <mjacob@FreeBSD.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 /*
28  * Based upon work by Pawel Jakub Dawidek <pjd@FreeBSD.org> for all of the
29  * fine geom examples, and by Poul Henning Kamp <phk@FreeBSD.org> for GEOM
30  * itself, all of which is most gratefully acknowledged.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/bio.h>
43 #include <sys/sbuf.h>
44 #include <sys/sysctl.h>
45 #include <sys/kthread.h>
46 #include <sys/malloc.h>
47 #include <geom/geom.h>
48 #include <geom/multipath/g_multipath.h>
49 
50 FEATURE(geom_multipath, "GEOM multipath support");
51 
52 SYSCTL_DECL(_kern_geom);
53 static SYSCTL_NODE(_kern_geom, OID_AUTO, multipath, CTLFLAG_RW, 0,
54     "GEOM_MULTIPATH tunables");
55 static u_int g_multipath_debug = 0;
56 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW,
57     &g_multipath_debug, 0, "Debug level");
58 static u_int g_multipath_exclusive = 1;
59 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
60     &g_multipath_exclusive, 0, "Exclusively open providers");
61 
62 static enum {
63 	GKT_NIL,
64 	GKT_RUN,
65 	GKT_DIE
66 } g_multipath_kt_state;
67 static struct bio_queue_head gmtbq;
68 static struct mtx gmtbq_mtx;
69 
70 static int g_multipath_read_metadata(struct g_consumer *cp,
71     struct g_multipath_metadata *md);
72 static int g_multipath_write_metadata(struct g_consumer *cp,
73     struct g_multipath_metadata *md);
74 
75 static void g_multipath_orphan(struct g_consumer *);
76 static void g_multipath_resize(struct g_consumer *);
77 static void g_multipath_start(struct bio *);
78 static void g_multipath_done(struct bio *);
79 static void g_multipath_done_error(struct bio *);
80 static void g_multipath_kt(void *);
81 
82 static int g_multipath_destroy(struct g_geom *);
83 static int
84 g_multipath_destroy_geom(struct gctl_req *, struct g_class *, struct g_geom *);
85 
86 static struct g_geom *g_multipath_find_geom(struct g_class *, const char *);
87 static int g_multipath_rotate(struct g_geom *);
88 
89 static g_taste_t g_multipath_taste;
90 static g_ctl_req_t g_multipath_config;
91 static g_init_t g_multipath_init;
92 static g_fini_t g_multipath_fini;
93 static g_dumpconf_t g_multipath_dumpconf;
94 
95 struct g_class g_multipath_class = {
96 	.name		= G_MULTIPATH_CLASS_NAME,
97 	.version	= G_VERSION,
98 	.ctlreq		= g_multipath_config,
99 	.taste		= g_multipath_taste,
100 	.destroy_geom	= g_multipath_destroy_geom,
101 	.init		= g_multipath_init,
102 	.fini		= g_multipath_fini
103 };
104 
105 #define	MP_FAIL		0x00000001
106 #define	MP_LOST		0x00000002
107 #define	MP_NEW		0x00000004
108 #define	MP_POSTED	0x00000008
109 #define	MP_BAD		(MP_FAIL | MP_LOST | MP_NEW)
110 #define MP_IDLE		0x00000010
111 #define MP_IDLE_MASK	0xfffffff0
112 
113 static int
114 g_multipath_good(struct g_geom *gp)
115 {
116 	struct g_consumer *cp;
117 	int n = 0;
118 
119 	LIST_FOREACH(cp, &gp->consumer, consumer) {
120 		if ((cp->index & MP_BAD) == 0)
121 			n++;
122 	}
123 	return (n);
124 }
125 
126 static void
127 g_multipath_fault(struct g_consumer *cp, int cause)
128 {
129 	struct g_multipath_softc *sc;
130 	struct g_consumer *lcp;
131 	struct g_geom *gp;
132 
133 	gp = cp->geom;
134 	sc = gp->softc;
135 	cp->index |= cause;
136 	if (g_multipath_good(gp) == 0 && sc->sc_ndisks > 0) {
137 		LIST_FOREACH(lcp, &gp->consumer, consumer) {
138 			if (lcp->provider == NULL ||
139 			    (lcp->index & (MP_LOST | MP_NEW)))
140 				continue;
141 			if (sc->sc_ndisks > 1 && lcp == cp)
142 				continue;
143 			printf("GEOM_MULTIPATH: "
144 			    "all paths in %s were marked FAIL, restore %s\n",
145 			    sc->sc_name, lcp->provider->name);
146 			lcp->index &= ~MP_FAIL;
147 		}
148 	}
149 	if (cp != sc->sc_active)
150 		return;
151 	sc->sc_active = NULL;
152 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
153 		if ((lcp->index & MP_BAD) == 0) {
154 			sc->sc_active = lcp;
155 			break;
156 		}
157 	}
158 	if (sc->sc_active == NULL) {
159 		printf("GEOM_MULTIPATH: out of providers for %s\n",
160 		    sc->sc_name);
161 	} else if (sc->sc_active_active != 1) {
162 		printf("GEOM_MULTIPATH: %s is now active path in %s\n",
163 		    sc->sc_active->provider->name, sc->sc_name);
164 	}
165 }
166 
167 static struct g_consumer *
168 g_multipath_choose(struct g_geom *gp, struct bio *bp)
169 {
170 	struct g_multipath_softc *sc;
171 	struct g_consumer *best, *cp;
172 
173 	sc = gp->softc;
174 	if (sc->sc_active_active == 0 ||
175 	    (sc->sc_active_active == 2 && bp->bio_cmd != BIO_READ))
176 		return (sc->sc_active);
177 	best = NULL;
178 	LIST_FOREACH(cp, &gp->consumer, consumer) {
179 		if (cp->index & MP_BAD)
180 			continue;
181 		cp->index += MP_IDLE;
182 		if (best == NULL || cp->private < best->private ||
183 		    (cp->private == best->private && cp->index > best->index))
184 			best = cp;
185 	}
186 	if (best != NULL)
187 		best->index &= ~MP_IDLE_MASK;
188 	return (best);
189 }
190 
191 static void
192 g_mpd(void *arg, int flags __unused)
193 {
194 	struct g_geom *gp;
195 	struct g_multipath_softc *sc;
196 	struct g_consumer *cp;
197 	int w;
198 
199 	g_topology_assert();
200 	cp = arg;
201 	gp = cp->geom;
202 	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) {
203 		w = cp->acw;
204 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
205 		if (w > 0 && cp->provider != NULL &&
206 		    (cp->provider->geom->flags & G_GEOM_WITHER) == 0) {
207 			g_post_event(g_mpd, cp, M_WAITOK, NULL);
208 			return;
209 		}
210 	}
211 	sc = gp->softc;
212 	mtx_lock(&sc->sc_mtx);
213 	if (cp->provider) {
214 		printf("GEOM_MULTIPATH: %s removed from %s\n",
215 		    cp->provider->name, gp->name);
216 		g_detach(cp);
217 	}
218 	g_destroy_consumer(cp);
219 	mtx_unlock(&sc->sc_mtx);
220 	if (LIST_EMPTY(&gp->consumer))
221 		g_multipath_destroy(gp);
222 }
223 
224 static void
225 g_multipath_orphan(struct g_consumer *cp)
226 {
227 	struct g_multipath_softc *sc;
228 	uintptr_t *cnt;
229 
230 	g_topology_assert();
231 	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
232 	    cp->provider->name, cp->geom->name);
233 	sc = cp->geom->softc;
234 	cnt = (uintptr_t *)&cp->private;
235 	mtx_lock(&sc->sc_mtx);
236 	sc->sc_ndisks--;
237 	g_multipath_fault(cp, MP_LOST);
238 	if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
239 		cp->index |= MP_POSTED;
240 		mtx_unlock(&sc->sc_mtx);
241 		g_mpd(cp, 0);
242 	} else
243 		mtx_unlock(&sc->sc_mtx);
244 }
245 
246 static void
247 g_multipath_resize(struct g_consumer *cp)
248 {
249 	struct g_multipath_softc *sc;
250 	struct g_geom *gp;
251 	struct g_consumer *cp1;
252 	struct g_provider *pp;
253 	struct g_multipath_metadata md;
254 	off_t size, psize, ssize;
255 	int error;
256 
257 	g_topology_assert();
258 
259 	gp = cp->geom;
260 	pp = cp->provider;
261 	sc = gp->softc;
262 
263 	if (sc->sc_stopping)
264 		return;
265 
266 	if (pp->mediasize < sc->sc_size) {
267 		size = pp->mediasize;
268 		ssize = pp->sectorsize;
269 	} else {
270 		size = ssize = OFF_MAX;
271 		mtx_lock(&sc->sc_mtx);
272 		LIST_FOREACH(cp1, &gp->consumer, consumer) {
273 			pp = cp1->provider;
274 			if (pp == NULL)
275 				continue;
276 			if (pp->mediasize < size) {
277 				size = pp->mediasize;
278 				ssize = pp->sectorsize;
279 			}
280 		}
281 		mtx_unlock(&sc->sc_mtx);
282 		if (size == OFF_MAX || size == sc->sc_size)
283 			return;
284 	}
285 	psize = size - ((sc->sc_uuid[0] != 0) ? ssize : 0);
286 	printf("GEOM_MULTIPATH: %s size changed from %jd to %jd\n",
287 	    sc->sc_name, sc->sc_pp->mediasize, psize);
288 	if (sc->sc_uuid[0] != 0 && size < sc->sc_size) {
289 		error = g_multipath_read_metadata(cp, &md);
290 		if (error ||
291 		    (strcmp(md.md_magic, G_MULTIPATH_MAGIC) != 0) ||
292 		    (memcmp(md.md_uuid, sc->sc_uuid, sizeof(sc->sc_uuid)) != 0) ||
293 		    (strcmp(md.md_name, sc->sc_name) != 0) ||
294 		    (md.md_size != 0 && md.md_size != size) ||
295 		    (md.md_sectorsize != 0 && md.md_sectorsize != ssize)) {
296 			g_multipath_destroy(gp);
297 			return;
298 		}
299 	}
300 	sc->sc_size = size;
301 	g_resize_provider(sc->sc_pp, psize);
302 
303 	if (sc->sc_uuid[0] != 0) {
304 		pp = cp->provider;
305 		strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
306 		memcpy(md.md_uuid, sc->sc_uuid, sizeof (sc->sc_uuid));
307 		strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
308 		md.md_version = G_MULTIPATH_VERSION;
309 		md.md_size = size;
310 		md.md_sectorsize = ssize;
311 		md.md_active_active = sc->sc_active_active;
312 		error = g_multipath_write_metadata(cp, &md);
313 		if (error != 0)
314 			printf("GEOM_MULTIPATH: Can't update metadata on %s "
315 			    "(%d)\n", pp->name, error);
316 	}
317 }
318 
319 static void
320 g_multipath_start(struct bio *bp)
321 {
322 	struct g_multipath_softc *sc;
323 	struct g_geom *gp;
324 	struct g_consumer *cp;
325 	struct bio *cbp;
326 	uintptr_t *cnt;
327 
328 	gp = bp->bio_to->geom;
329 	sc = gp->softc;
330 	KASSERT(sc != NULL, ("NULL sc"));
331 	cbp = g_clone_bio(bp);
332 	if (cbp == NULL) {
333 		g_io_deliver(bp, ENOMEM);
334 		return;
335 	}
336 	mtx_lock(&sc->sc_mtx);
337 	cp = g_multipath_choose(gp, bp);
338 	if (cp == NULL) {
339 		mtx_unlock(&sc->sc_mtx);
340 		g_destroy_bio(cbp);
341 		g_io_deliver(bp, ENXIO);
342 		return;
343 	}
344 	if ((uintptr_t)bp->bio_driver1 < sc->sc_ndisks)
345 		bp->bio_driver1 = (void *)(uintptr_t)sc->sc_ndisks;
346 	cnt = (uintptr_t *)&cp->private;
347 	(*cnt)++;
348 	mtx_unlock(&sc->sc_mtx);
349 	cbp->bio_done = g_multipath_done;
350 	g_io_request(cbp, cp);
351 }
352 
353 static void
354 g_multipath_done(struct bio *bp)
355 {
356 	struct g_multipath_softc *sc;
357 	struct g_consumer *cp;
358 	uintptr_t *cnt;
359 
360 	if (bp->bio_error == ENXIO || bp->bio_error == EIO) {
361 		mtx_lock(&gmtbq_mtx);
362 		bioq_insert_tail(&gmtbq, bp);
363 		mtx_unlock(&gmtbq_mtx);
364 		wakeup(&g_multipath_kt_state);
365 	} else {
366 		cp = bp->bio_from;
367 		sc = cp->geom->softc;
368 		cnt = (uintptr_t *)&cp->private;
369 		mtx_lock(&sc->sc_mtx);
370 		(*cnt)--;
371 		if (*cnt == 0 && (cp->index & MP_LOST)) {
372 			cp->index |= MP_POSTED;
373 			mtx_unlock(&sc->sc_mtx);
374 			g_post_event(g_mpd, cp, M_WAITOK, NULL);
375 		} else
376 			mtx_unlock(&sc->sc_mtx);
377 		g_std_done(bp);
378 	}
379 }
380 
381 static void
382 g_multipath_done_error(struct bio *bp)
383 {
384 	struct bio *pbp;
385 	struct g_geom *gp;
386 	struct g_multipath_softc *sc;
387 	struct g_consumer *cp;
388 	struct g_provider *pp;
389 	uintptr_t *cnt;
390 
391 	/*
392 	 * If we had a failure, we have to check first to see
393 	 * whether the consumer it failed on was the currently
394 	 * active consumer (i.e., this is the first in perhaps
395 	 * a number of failures). If so, we then switch consumers
396 	 * to the next available consumer.
397 	 */
398 
399 	pbp = bp->bio_parent;
400 	gp = pbp->bio_to->geom;
401 	sc = gp->softc;
402 	cp = bp->bio_from;
403 	pp = cp->provider;
404 	cnt = (uintptr_t *)&cp->private;
405 
406 	mtx_lock(&sc->sc_mtx);
407 	if ((cp->index & MP_FAIL) == 0) {
408 		printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
409 		    bp->bio_error, pp->name, sc->sc_name);
410 		g_multipath_fault(cp, MP_FAIL);
411 	}
412 	(*cnt)--;
413 	if (*cnt == 0 && (cp->index & (MP_LOST | MP_POSTED)) == MP_LOST) {
414 		cp->index |= MP_POSTED;
415 		mtx_unlock(&sc->sc_mtx);
416 		g_post_event(g_mpd, cp, M_WAITOK, NULL);
417 	} else
418 		mtx_unlock(&sc->sc_mtx);
419 
420 	/*
421 	 * If we can fruitfully restart the I/O, do so.
422 	 */
423 	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
424 		pbp->bio_inbed++;
425 		g_destroy_bio(bp);
426 		g_multipath_start(pbp);
427 	} else {
428 		g_std_done(bp);
429 	}
430 }
431 
432 static void
433 g_multipath_kt(void *arg)
434 {
435 
436 	g_multipath_kt_state = GKT_RUN;
437 	mtx_lock(&gmtbq_mtx);
438 	while (g_multipath_kt_state == GKT_RUN) {
439 		for (;;) {
440 			struct bio *bp;
441 
442 			bp = bioq_takefirst(&gmtbq);
443 			if (bp == NULL)
444 				break;
445 			mtx_unlock(&gmtbq_mtx);
446 			g_multipath_done_error(bp);
447 			mtx_lock(&gmtbq_mtx);
448 		}
449 		if (g_multipath_kt_state != GKT_RUN)
450 			break;
451 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
452 		    "gkt:wait", 0);
453 	}
454 	mtx_unlock(&gmtbq_mtx);
455 	wakeup(&g_multipath_kt_state);
456 	kproc_exit(0);
457 }
458 
459 
460 static int
461 g_multipath_access(struct g_provider *pp, int dr, int dw, int de)
462 {
463 	struct g_geom *gp;
464 	struct g_consumer *cp, *badcp = NULL;
465 	struct g_multipath_softc *sc;
466 	int error;
467 
468 	gp = pp->geom;
469 
470 	LIST_FOREACH(cp, &gp->consumer, consumer) {
471 		error = g_access(cp, dr, dw, de);
472 		if (error) {
473 			badcp = cp;
474 			goto fail;
475 		}
476 	}
477 	sc = gp->softc;
478 	sc->sc_opened += dr + dw + de;
479 	if (sc->sc_stopping && sc->sc_opened == 0)
480 		g_multipath_destroy(gp);
481 	return (0);
482 
483 fail:
484 	LIST_FOREACH(cp, &gp->consumer, consumer) {
485 		if (cp == badcp)
486 			break;
487 		(void) g_access(cp, -dr, -dw, -de);
488 	}
489 	return (error);
490 }
491 
492 static struct g_geom *
493 g_multipath_create(struct g_class *mp, struct g_multipath_metadata *md)
494 {
495 	struct g_multipath_softc *sc;
496 	struct g_geom *gp;
497 	struct g_provider *pp;
498 
499 	g_topology_assert();
500 
501 	LIST_FOREACH(gp, &mp->geom, geom) {
502 		sc = gp->softc;
503 		if (sc == NULL || sc->sc_stopping)
504 			continue;
505 		if (strcmp(gp->name, md->md_name) == 0) {
506 			printf("GEOM_MULTIPATH: name %s already exists\n",
507 			    md->md_name);
508 			return (NULL);
509 		}
510 	}
511 
512 	gp = g_new_geomf(mp, "%s", md->md_name);
513 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
514 	mtx_init(&sc->sc_mtx, "multipath", NULL, MTX_DEF);
515 	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
516 	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
517 	sc->sc_active_active = md->md_active_active;
518 	sc->sc_size = md->md_size;
519 	gp->softc = sc;
520 	gp->start = g_multipath_start;
521 	gp->orphan = g_multipath_orphan;
522 	gp->resize = g_multipath_resize;
523 	gp->access = g_multipath_access;
524 	gp->dumpconf = g_multipath_dumpconf;
525 
526 	pp = g_new_providerf(gp, "multipath/%s", md->md_name);
527 	pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
528 	if (md->md_size != 0) {
529 		pp->mediasize = md->md_size -
530 		    ((md->md_uuid[0] != 0) ? md->md_sectorsize : 0);
531 		pp->sectorsize = md->md_sectorsize;
532 	}
533 	sc->sc_pp = pp;
534 	g_error_provider(pp, 0);
535 	printf("GEOM_MULTIPATH: %s created\n", gp->name);
536 	return (gp);
537 }
538 
539 static int
540 g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
541 {
542 	struct g_multipath_softc *sc;
543 	struct g_consumer *cp, *nxtcp;
544 	int error, acr, acw, ace;
545 
546 	g_topology_assert();
547 
548 	sc = gp->softc;
549 	KASSERT(sc, ("no softc"));
550 
551 	/*
552 	 * Make sure that the passed provider isn't already attached
553 	 */
554 	LIST_FOREACH(cp, &gp->consumer, consumer) {
555 		if (cp->provider == pp)
556 			break;
557 	}
558 	if (cp) {
559 		printf("GEOM_MULTIPATH: provider %s already attached to %s\n",
560 		    pp->name, gp->name);
561 		return (EEXIST);
562 	}
563 	nxtcp = LIST_FIRST(&gp->consumer);
564 	cp = g_new_consumer(gp);
565 	cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
566 	cp->private = NULL;
567 	cp->index = MP_NEW;
568 	error = g_attach(cp, pp);
569 	if (error != 0) {
570 		printf("GEOM_MULTIPATH: cannot attach %s to %s",
571 		    pp->name, sc->sc_name);
572 		g_destroy_consumer(cp);
573 		return (error);
574 	}
575 
576 	/*
577 	 * Set access permissions on new consumer to match other consumers
578 	 */
579 	if (sc->sc_pp) {
580 		acr = sc->sc_pp->acr;
581 		acw = sc->sc_pp->acw;
582 		ace = sc->sc_pp->ace;
583 	} else
584 		acr = acw = ace = 0;
585 	if (g_multipath_exclusive) {
586 		acr++;
587 		acw++;
588 		ace++;
589 	}
590 	error = g_access(cp, acr, acw, ace);
591 	if (error) {
592 		printf("GEOM_MULTIPATH: cannot set access in "
593 		    "attaching %s to %s (%d)\n",
594 		    pp->name, sc->sc_name, error);
595 		g_detach(cp);
596 		g_destroy_consumer(cp);
597 		return (error);
598 	}
599 	if (sc->sc_size == 0) {
600 		sc->sc_size = pp->mediasize -
601 		    ((sc->sc_uuid[0] != 0) ? pp->sectorsize : 0);
602 		sc->sc_pp->mediasize = sc->sc_size;
603 		sc->sc_pp->sectorsize = pp->sectorsize;
604 	}
605 	if (sc->sc_pp->stripesize == 0 && sc->sc_pp->stripeoffset == 0) {
606 		sc->sc_pp->stripesize = pp->stripesize;
607 		sc->sc_pp->stripeoffset = pp->stripeoffset;
608 	}
609 	sc->sc_pp->flags |= pp->flags & G_PF_ACCEPT_UNMAPPED;
610 	mtx_lock(&sc->sc_mtx);
611 	cp->index = 0;
612 	sc->sc_ndisks++;
613 	mtx_unlock(&sc->sc_mtx);
614 	printf("GEOM_MULTIPATH: %s added to %s\n",
615 	    pp->name, sc->sc_name);
616 	if (sc->sc_active == NULL) {
617 		sc->sc_active = cp;
618 		if (sc->sc_active_active != 1)
619 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
620 			    pp->name, sc->sc_name);
621 	}
622 	return (0);
623 }
624 
625 static int
626 g_multipath_destroy(struct g_geom *gp)
627 {
628 	struct g_multipath_softc *sc;
629 	struct g_consumer *cp, *cp1;
630 
631 	g_topology_assert();
632 	if (gp->softc == NULL)
633 		return (ENXIO);
634 	sc = gp->softc;
635 	if (!sc->sc_stopping) {
636 		printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
637 		sc->sc_stopping = 1;
638 	}
639 	if (sc->sc_opened != 0) {
640 		g_wither_provider(sc->sc_pp, ENXIO);
641 		sc->sc_pp = NULL;
642 		return (EINPROGRESS);
643 	}
644 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
645 		mtx_lock(&sc->sc_mtx);
646 		if ((cp->index & MP_POSTED) == 0) {
647 			cp->index |= MP_POSTED;
648 			mtx_unlock(&sc->sc_mtx);
649 			g_mpd(cp, 0);
650 			if (cp1 == NULL)
651 				return(0);	/* Recursion happened. */
652 		} else
653 			mtx_unlock(&sc->sc_mtx);
654 	}
655 	if (!LIST_EMPTY(&gp->consumer))
656 		return (EINPROGRESS);
657 	mtx_destroy(&sc->sc_mtx);
658 	g_free(gp->softc);
659 	gp->softc = NULL;
660 	printf("GEOM_MULTIPATH: %s destroyed\n", gp->name);
661 	g_wither_geom(gp, ENXIO);
662 	return (0);
663 }
664 
665 static int
666 g_multipath_destroy_geom(struct gctl_req *req, struct g_class *mp,
667     struct g_geom *gp)
668 {
669 
670 	return (g_multipath_destroy(gp));
671 }
672 
673 static int
674 g_multipath_rotate(struct g_geom *gp)
675 {
676 	struct g_consumer *lcp, *first_good_cp = NULL;
677 	struct g_multipath_softc *sc = gp->softc;
678 	int active_cp_seen = 0;
679 
680 	g_topology_assert();
681 	if (sc == NULL)
682 		return (ENXIO);
683 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
684 		if ((lcp->index & MP_BAD) == 0) {
685 			if (first_good_cp == NULL)
686 				first_good_cp = lcp;
687 			if (active_cp_seen)
688 				break;
689 		}
690 		if (sc->sc_active == lcp)
691 			active_cp_seen = 1;
692 	}
693 	if (lcp == NULL)
694 		lcp = first_good_cp;
695 	if (lcp && lcp != sc->sc_active) {
696 		sc->sc_active = lcp;
697 		if (sc->sc_active_active != 1)
698 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
699 			    lcp->provider->name, sc->sc_name);
700 	}
701 	return (0);
702 }
703 
704 static void
705 g_multipath_init(struct g_class *mp)
706 {
707 	bioq_init(&gmtbq);
708 	mtx_init(&gmtbq_mtx, "gmtbq", NULL, MTX_DEF);
709 	kproc_create(g_multipath_kt, mp, NULL, 0, 0, "g_mp_kt");
710 }
711 
712 static void
713 g_multipath_fini(struct g_class *mp)
714 {
715 	if (g_multipath_kt_state == GKT_RUN) {
716 		mtx_lock(&gmtbq_mtx);
717 		g_multipath_kt_state = GKT_DIE;
718 		wakeup(&g_multipath_kt_state);
719 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
720 		    "gmp:fini", 0);
721 		mtx_unlock(&gmtbq_mtx);
722 	}
723 }
724 
725 static int
726 g_multipath_read_metadata(struct g_consumer *cp,
727     struct g_multipath_metadata *md)
728 {
729 	struct g_provider *pp;
730 	u_char *buf;
731 	int error;
732 
733 	g_topology_assert();
734 	error = g_access(cp, 1, 0, 0);
735 	if (error != 0)
736 		return (error);
737 	pp = cp->provider;
738 	g_topology_unlock();
739 	buf = g_read_data(cp, pp->mediasize - pp->sectorsize,
740 	    pp->sectorsize, &error);
741 	g_topology_lock();
742 	g_access(cp, -1, 0, 0);
743 	if (buf == NULL)
744 		return (error);
745 	multipath_metadata_decode(buf, md);
746 	g_free(buf);
747 	return (0);
748 }
749 
750 static int
751 g_multipath_write_metadata(struct g_consumer *cp,
752     struct g_multipath_metadata *md)
753 {
754 	struct g_provider *pp;
755 	u_char *buf;
756 	int error;
757 
758 	g_topology_assert();
759 	error = g_access(cp, 1, 1, 1);
760 	if (error != 0)
761 		return (error);
762 	pp = cp->provider;
763 	g_topology_unlock();
764 	buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
765 	multipath_metadata_encode(md, buf);
766 	error = g_write_data(cp, pp->mediasize - pp->sectorsize,
767 	    buf, pp->sectorsize);
768 	g_topology_lock();
769 	g_access(cp, -1, -1, -1);
770 	g_free(buf);
771 	return (error);
772 }
773 
774 static struct g_geom *
775 g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
776 {
777 	struct g_multipath_metadata md;
778 	struct g_multipath_softc *sc;
779 	struct g_consumer *cp;
780 	struct g_geom *gp, *gp1;
781 	int error, isnew;
782 
783 	g_topology_assert();
784 
785 	gp = g_new_geomf(mp, "multipath:taste");
786 	gp->start = g_multipath_start;
787 	gp->access = g_multipath_access;
788 	gp->orphan = g_multipath_orphan;
789 	cp = g_new_consumer(gp);
790 	g_attach(cp, pp);
791 	error = g_multipath_read_metadata(cp, &md);
792 	g_detach(cp);
793 	g_destroy_consumer(cp);
794 	g_destroy_geom(gp);
795 	if (error != 0)
796 		return (NULL);
797 	gp = NULL;
798 
799 	if (strcmp(md.md_magic, G_MULTIPATH_MAGIC) != 0) {
800 		if (g_multipath_debug)
801 			printf("%s is not MULTIPATH\n", pp->name);
802 		return (NULL);
803 	}
804 	if (md.md_version != G_MULTIPATH_VERSION) {
805 		printf("%s has version %d multipath id- this module is version "
806 		    " %d: rejecting\n", pp->name, md.md_version,
807 		    G_MULTIPATH_VERSION);
808 		return (NULL);
809 	}
810 	if (md.md_size != 0 && md.md_size != pp->mediasize)
811 		return (NULL);
812 	if (md.md_sectorsize != 0 && md.md_sectorsize != pp->sectorsize)
813 		return (NULL);
814 	if (g_multipath_debug)
815 		printf("MULTIPATH: %s/%s\n", md.md_name, md.md_uuid);
816 
817 	/*
818 	 * Let's check if such a device already is present. We check against
819 	 * uuid alone first because that's the true distinguishor. If that
820 	 * passes, then we check for name conflicts. If there are conflicts,
821 	 * modify the name.
822 	 *
823 	 * The whole purpose of this is to solve the problem that people don't
824 	 * pick good unique names, but good unique names (like uuids) are a
825 	 * pain to use. So, we allow people to build GEOMs with friendly names
826 	 * and uuids, and modify the names in case there's a collision.
827 	 */
828 	sc = NULL;
829 	LIST_FOREACH(gp, &mp->geom, geom) {
830 		sc = gp->softc;
831 		if (sc == NULL || sc->sc_stopping)
832 			continue;
833 		if (strncmp(md.md_uuid, sc->sc_uuid, sizeof(md.md_uuid)) == 0)
834 			break;
835 	}
836 
837 	LIST_FOREACH(gp1, &mp->geom, geom) {
838 		if (gp1 == gp)
839 			continue;
840 		sc = gp1->softc;
841 		if (sc == NULL || sc->sc_stopping)
842 			continue;
843 		if (strncmp(md.md_name, sc->sc_name, sizeof(md.md_name)) == 0)
844 			break;
845 	}
846 
847 	/*
848 	 * If gp is NULL, we had no extant MULTIPATH geom with this uuid.
849 	 *
850 	 * If gp1 is *not* NULL, that means we have a MULTIPATH geom extant
851 	 * with the same name (but a different UUID).
852 	 *
853 	 * If gp is NULL, then modify the name with a random number and
854   	 * complain, but allow the creation of the geom to continue.
855 	 *
856 	 * If gp is *not* NULL, just use the geom's name as we're attaching
857 	 * this disk to the (previously generated) name.
858 	 */
859 
860 	if (gp1) {
861 		sc = gp1->softc;
862 		if (gp == NULL) {
863 			char buf[16];
864 			u_long rand = random();
865 
866 			snprintf(buf, sizeof (buf), "%s-%lu", md.md_name, rand);
867 			printf("GEOM_MULTIPATH: geom %s/%s exists already\n",
868 			    sc->sc_name, sc->sc_uuid);
869 			printf("GEOM_MULTIPATH: %s will be (temporarily) %s\n",
870 			    md.md_uuid, buf);
871 			strlcpy(md.md_name, buf, sizeof(md.md_name));
872 		} else {
873 			strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
874 		}
875 	}
876 
877 	if (gp == NULL) {
878 		gp = g_multipath_create(mp, &md);
879 		if (gp == NULL) {
880 			printf("GEOM_MULTIPATH: cannot create geom %s/%s\n",
881 			    md.md_name, md.md_uuid);
882 			return (NULL);
883 		}
884 		isnew = 1;
885 	} else {
886 		isnew = 0;
887 	}
888 
889 	sc = gp->softc;
890 	KASSERT(sc != NULL, ("sc is NULL"));
891 	error = g_multipath_add_disk(gp, pp);
892 	if (error != 0) {
893 		if (isnew)
894 			g_multipath_destroy(gp);
895 		return (NULL);
896 	}
897 	return (gp);
898 }
899 
900 static void
901 g_multipath_ctl_add_name(struct gctl_req *req, struct g_class *mp,
902     const char *name)
903 {
904 	struct g_multipath_softc *sc;
905 	struct g_geom *gp;
906 	struct g_consumer *cp;
907 	struct g_provider *pp;
908 	const char *mpname;
909 	static const char devpf[6] = "/dev/";
910 
911 	g_topology_assert();
912 
913 	mpname = gctl_get_asciiparam(req, "arg0");
914         if (mpname == NULL) {
915                 gctl_error(req, "No 'arg0' argument");
916                 return;
917         }
918 	gp = g_multipath_find_geom(mp, mpname);
919 	if (gp == NULL) {
920 		gctl_error(req, "Device %s is invalid", mpname);
921 		return;
922 	}
923 	sc = gp->softc;
924 
925 	if (strncmp(name, devpf, 5) == 0)
926 		name += 5;
927 	pp = g_provider_by_name(name);
928 	if (pp == NULL) {
929 		gctl_error(req, "Provider %s is invalid", name);
930 		return;
931 	}
932 
933 	/*
934 	 * Check to make sure parameters match.
935 	 */
936 	LIST_FOREACH(cp, &gp->consumer, consumer) {
937 		if (cp->provider == pp) {
938 			gctl_error(req, "provider %s is already there",
939 			    pp->name);
940 			return;
941 		}
942 	}
943 	if (sc->sc_pp->mediasize != 0 &&
944 	    sc->sc_pp->mediasize + (sc->sc_uuid[0] != 0 ? pp->sectorsize : 0)
945 	     != pp->mediasize) {
946 		gctl_error(req, "Providers size mismatch %jd != %jd",
947 		    (intmax_t) sc->sc_pp->mediasize +
948 			(sc->sc_uuid[0] != 0 ? pp->sectorsize : 0),
949 		    (intmax_t) pp->mediasize);
950 		return;
951 	}
952 	if (sc->sc_pp->sectorsize != 0 &&
953 	    sc->sc_pp->sectorsize != pp->sectorsize) {
954 		gctl_error(req, "Providers sectorsize mismatch %u != %u",
955 		    sc->sc_pp->sectorsize, pp->sectorsize);
956 		return;
957 	}
958 
959 	/*
960 	 * Now add....
961 	 */
962 	(void) g_multipath_add_disk(gp, pp);
963 }
964 
965 static void
966 g_multipath_ctl_prefer(struct gctl_req *req, struct g_class *mp)
967 {
968 	struct g_geom *gp;
969 	struct g_multipath_softc *sc;
970 	struct g_consumer *cp;
971 	const char *name, *mpname;
972 	static const char devpf[6] = "/dev/";
973 	int *nargs;
974 
975 	g_topology_assert();
976 
977 	mpname = gctl_get_asciiparam(req, "arg0");
978         if (mpname == NULL) {
979                 gctl_error(req, "No 'arg0' argument");
980                 return;
981         }
982 	gp = g_multipath_find_geom(mp, mpname);
983 	if (gp == NULL) {
984 		gctl_error(req, "Device %s is invalid", mpname);
985 		return;
986 	}
987 	sc = gp->softc;
988 
989 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
990 	if (nargs == NULL) {
991 		gctl_error(req, "No 'nargs' argument");
992 		return;
993 	}
994 	if (*nargs != 2) {
995 		gctl_error(req, "missing device");
996 		return;
997 	}
998 
999 	name = gctl_get_asciiparam(req, "arg1");
1000 	if (name == NULL) {
1001 		gctl_error(req, "No 'arg1' argument");
1002 		return;
1003 	}
1004 	if (strncmp(name, devpf, 5) == 0) {
1005 		name += 5;
1006 	}
1007 
1008 	LIST_FOREACH(cp, &gp->consumer, consumer) {
1009 		if (cp->provider != NULL
1010                       && strcmp(cp->provider->name, name) == 0)
1011 		    break;
1012 	}
1013 
1014 	if (cp == NULL) {
1015 		gctl_error(req, "Provider %s not found", name);
1016 		return;
1017 	}
1018 
1019 	mtx_lock(&sc->sc_mtx);
1020 
1021 	if (cp->index & MP_BAD) {
1022 		gctl_error(req, "Consumer %s is invalid", name);
1023 		mtx_unlock(&sc->sc_mtx);
1024 		return;
1025 	}
1026 
1027 	/* Here when the consumer is present and in good shape */
1028 
1029 	sc->sc_active = cp;
1030 	if (!sc->sc_active_active)
1031 	    printf("GEOM_MULTIPATH: %s now active path in %s\n",
1032 		sc->sc_active->provider->name, sc->sc_name);
1033 
1034 	mtx_unlock(&sc->sc_mtx);
1035 }
1036 
1037 static void
1038 g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
1039 {
1040 	struct g_multipath_softc *sc;
1041 	struct g_geom *gp;
1042 	const char *mpname, *name;
1043 
1044 	mpname = gctl_get_asciiparam(req, "arg0");
1045         if (mpname == NULL) {
1046                 gctl_error(req, "No 'arg0' argument");
1047                 return;
1048         }
1049 	gp = g_multipath_find_geom(mp, mpname);
1050 	if (gp == NULL) {
1051 		gctl_error(req, "Device %s not found", mpname);
1052 		return;
1053 	}
1054 	sc = gp->softc;
1055 
1056 	name = gctl_get_asciiparam(req, "arg1");
1057 	if (name == NULL) {
1058 		gctl_error(req, "No 'arg1' argument");
1059 		return;
1060 	}
1061 	g_multipath_ctl_add_name(req, mp, name);
1062 }
1063 
1064 static void
1065 g_multipath_ctl_create(struct gctl_req *req, struct g_class *mp)
1066 {
1067 	struct g_multipath_metadata md;
1068 	struct g_multipath_softc *sc;
1069 	struct g_geom *gp;
1070 	const char *mpname, *name;
1071 	char param[16];
1072 	int *nargs, i, *val;
1073 
1074 	g_topology_assert();
1075 
1076 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
1077 	if (*nargs < 2) {
1078 		gctl_error(req, "wrong number of arguments.");
1079 		return;
1080 	}
1081 
1082 	mpname = gctl_get_asciiparam(req, "arg0");
1083         if (mpname == NULL) {
1084                 gctl_error(req, "No 'arg0' argument");
1085                 return;
1086         }
1087 	gp = g_multipath_find_geom(mp, mpname);
1088 	if (gp != NULL) {
1089 		gctl_error(req, "Device %s already exist", mpname);
1090 		return;
1091 	}
1092 
1093 	memset(&md, 0, sizeof(md));
1094 	strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
1095 	md.md_version = G_MULTIPATH_VERSION;
1096 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
1097 	md.md_size = 0;
1098 	md.md_sectorsize = 0;
1099 	md.md_uuid[0] = 0;
1100 	md.md_active_active = 0;
1101 	val = gctl_get_paraml(req, "active_active", sizeof(*val));
1102 	if (val != NULL && *val != 0)
1103 		md.md_active_active = 1;
1104 	val = gctl_get_paraml(req, "active_read", sizeof(*val));
1105 	if (val != NULL && *val != 0)
1106 		md.md_active_active = 2;
1107 	gp = g_multipath_create(mp, &md);
1108 	if (gp == NULL) {
1109 		gctl_error(req, "GEOM_MULTIPATH: cannot create geom %s/%s\n",
1110 		    md.md_name, md.md_uuid);
1111 		return;
1112 	}
1113 	sc = gp->softc;
1114 
1115 	for (i = 1; i < *nargs; i++) {
1116 		snprintf(param, sizeof(param), "arg%d", i);
1117 		name = gctl_get_asciiparam(req, param);
1118 		g_multipath_ctl_add_name(req, mp, name);
1119 	}
1120 
1121 	if (sc->sc_ndisks != (*nargs - 1))
1122 		g_multipath_destroy(gp);
1123 }
1124 
1125 static void
1126 g_multipath_ctl_configure(struct gctl_req *req, struct g_class *mp)
1127 {
1128 	struct g_multipath_softc *sc;
1129 	struct g_geom *gp;
1130 	struct g_consumer *cp;
1131 	struct g_provider *pp;
1132 	struct g_multipath_metadata md;
1133 	const char *name;
1134 	int error, *val;
1135 
1136 	g_topology_assert();
1137 
1138 	name = gctl_get_asciiparam(req, "arg0");
1139 	if (name == NULL) {
1140 		gctl_error(req, "No 'arg0' argument");
1141 		return;
1142 	}
1143 	gp = g_multipath_find_geom(mp, name);
1144 	if (gp == NULL) {
1145 		gctl_error(req, "Device %s is invalid", name);
1146 		return;
1147 	}
1148 	sc = gp->softc;
1149 	val = gctl_get_paraml(req, "active_active", sizeof(*val));
1150 	if (val != NULL && *val != 0)
1151 		sc->sc_active_active = 1;
1152 	val = gctl_get_paraml(req, "active_read", sizeof(*val));
1153 	if (val != NULL && *val != 0)
1154 		sc->sc_active_active = 2;
1155 	val = gctl_get_paraml(req, "active_passive", sizeof(*val));
1156 	if (val != NULL && *val != 0)
1157 		sc->sc_active_active = 0;
1158 	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
1159 		cp = sc->sc_active;
1160 		pp = cp->provider;
1161 		strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
1162 		memcpy(md.md_uuid, sc->sc_uuid, sizeof (sc->sc_uuid));
1163 		strlcpy(md.md_name, name, sizeof(md.md_name));
1164 		md.md_version = G_MULTIPATH_VERSION;
1165 		md.md_size = pp->mediasize;
1166 		md.md_sectorsize = pp->sectorsize;
1167 		md.md_active_active = sc->sc_active_active;
1168 		error = g_multipath_write_metadata(cp, &md);
1169 		if (error != 0)
1170 			gctl_error(req, "Can't update metadata on %s (%d)",
1171 			    pp->name, error);
1172 	}
1173 }
1174 
1175 static void
1176 g_multipath_ctl_fail(struct gctl_req *req, struct g_class *mp, int fail)
1177 {
1178 	struct g_multipath_softc *sc;
1179 	struct g_geom *gp;
1180 	struct g_consumer *cp;
1181 	const char *mpname, *name;
1182 	int found;
1183 
1184 	mpname = gctl_get_asciiparam(req, "arg0");
1185         if (mpname == NULL) {
1186                 gctl_error(req, "No 'arg0' argument");
1187                 return;
1188         }
1189 	gp = g_multipath_find_geom(mp, mpname);
1190 	if (gp == NULL) {
1191 		gctl_error(req, "Device %s not found", mpname);
1192 		return;
1193 	}
1194 	sc = gp->softc;
1195 
1196 	name = gctl_get_asciiparam(req, "arg1");
1197 	if (name == NULL) {
1198 		gctl_error(req, "No 'arg1' argument");
1199 		return;
1200 	}
1201 
1202 	found = 0;
1203 	mtx_lock(&sc->sc_mtx);
1204 	LIST_FOREACH(cp, &gp->consumer, consumer) {
1205 		if (cp->provider != NULL &&
1206 		    strcmp(cp->provider->name, name) == 0 &&
1207 		    (cp->index & MP_LOST) == 0) {
1208 			found = 1;
1209 			if (!fail == !(cp->index & MP_FAIL))
1210 				continue;
1211 			printf("GEOM_MULTIPATH: %s in %s is marked %s.\n",
1212 				name, sc->sc_name, fail ? "FAIL" : "OK");
1213 			if (fail) {
1214 				g_multipath_fault(cp, MP_FAIL);
1215 			} else {
1216 				cp->index &= ~MP_FAIL;
1217 			}
1218 		}
1219 	}
1220 	mtx_unlock(&sc->sc_mtx);
1221 	if (found == 0)
1222 		gctl_error(req, "Provider %s not found", name);
1223 }
1224 
1225 static void
1226 g_multipath_ctl_remove(struct gctl_req *req, struct g_class *mp)
1227 {
1228 	struct g_multipath_softc *sc;
1229 	struct g_geom *gp;
1230 	struct g_consumer *cp, *cp1;
1231 	const char *mpname, *name;
1232 	uintptr_t *cnt;
1233 	int found;
1234 
1235 	mpname = gctl_get_asciiparam(req, "arg0");
1236         if (mpname == NULL) {
1237                 gctl_error(req, "No 'arg0' argument");
1238                 return;
1239         }
1240 	gp = g_multipath_find_geom(mp, mpname);
1241 	if (gp == NULL) {
1242 		gctl_error(req, "Device %s not found", mpname);
1243 		return;
1244 	}
1245 	sc = gp->softc;
1246 
1247 	name = gctl_get_asciiparam(req, "arg1");
1248 	if (name == NULL) {
1249 		gctl_error(req, "No 'arg1' argument");
1250 		return;
1251 	}
1252 
1253 	found = 0;
1254 	mtx_lock(&sc->sc_mtx);
1255 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
1256 		if (cp->provider != NULL &&
1257 		    strcmp(cp->provider->name, name) == 0 &&
1258 		    (cp->index & MP_LOST) == 0) {
1259 			found = 1;
1260 			printf("GEOM_MULTIPATH: removing %s from %s\n",
1261 			    cp->provider->name, cp->geom->name);
1262 			sc->sc_ndisks--;
1263 			g_multipath_fault(cp, MP_LOST);
1264 			cnt = (uintptr_t *)&cp->private;
1265 			if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
1266 				cp->index |= MP_POSTED;
1267 				mtx_unlock(&sc->sc_mtx);
1268 				g_mpd(cp, 0);
1269 				if (cp1 == NULL)
1270 					return;	/* Recursion happened. */
1271 				mtx_lock(&sc->sc_mtx);
1272 			}
1273 		}
1274 	}
1275 	mtx_unlock(&sc->sc_mtx);
1276 	if (found == 0)
1277 		gctl_error(req, "Provider %s not found", name);
1278 }
1279 
1280 static struct g_geom *
1281 g_multipath_find_geom(struct g_class *mp, const char *name)
1282 {
1283 	struct g_geom *gp;
1284 	struct g_multipath_softc *sc;
1285 
1286 	LIST_FOREACH(gp, &mp->geom, geom) {
1287 		sc = gp->softc;
1288 		if (sc == NULL || sc->sc_stopping)
1289 			continue;
1290 		if (strcmp(gp->name, name) == 0)
1291 			return (gp);
1292 	}
1293 	return (NULL);
1294 }
1295 
1296 static void
1297 g_multipath_ctl_stop(struct gctl_req *req, struct g_class *mp)
1298 {
1299 	struct g_geom *gp;
1300 	const char *name;
1301 	int error;
1302 
1303 	g_topology_assert();
1304 
1305 	name = gctl_get_asciiparam(req, "arg0");
1306         if (name == NULL) {
1307                 gctl_error(req, "No 'arg0' argument");
1308                 return;
1309         }
1310 	gp = g_multipath_find_geom(mp, name);
1311 	if (gp == NULL) {
1312 		gctl_error(req, "Device %s is invalid", name);
1313 		return;
1314 	}
1315 	error = g_multipath_destroy(gp);
1316 	if (error != 0 && error != EINPROGRESS)
1317 		gctl_error(req, "failed to stop %s (err=%d)", name, error);
1318 }
1319 
1320 static void
1321 g_multipath_ctl_destroy(struct gctl_req *req, struct g_class *mp)
1322 {
1323 	struct g_geom *gp;
1324 	struct g_multipath_softc *sc;
1325 	struct g_consumer *cp;
1326 	struct g_provider *pp;
1327 	const char *name;
1328 	uint8_t *buf;
1329 	int error;
1330 
1331 	g_topology_assert();
1332 
1333 	name = gctl_get_asciiparam(req, "arg0");
1334         if (name == NULL) {
1335                 gctl_error(req, "No 'arg0' argument");
1336                 return;
1337         }
1338 	gp = g_multipath_find_geom(mp, name);
1339 	if (gp == NULL) {
1340 		gctl_error(req, "Device %s is invalid", name);
1341 		return;
1342 	}
1343 	sc = gp->softc;
1344 
1345 	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
1346 		cp = sc->sc_active;
1347 		pp = cp->provider;
1348 		error = g_access(cp, 1, 1, 1);
1349 		if (error != 0) {
1350 			gctl_error(req, "Can't open %s (%d)", pp->name, error);
1351 			goto destroy;
1352 		}
1353 		g_topology_unlock();
1354 		buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
1355 		error = g_write_data(cp, pp->mediasize - pp->sectorsize,
1356 		    buf, pp->sectorsize);
1357 		g_topology_lock();
1358 		g_access(cp, -1, -1, -1);
1359 		if (error != 0)
1360 			gctl_error(req, "Can't erase metadata on %s (%d)",
1361 			    pp->name, error);
1362 	}
1363 
1364 destroy:
1365 	error = g_multipath_destroy(gp);
1366 	if (error != 0 && error != EINPROGRESS)
1367 		gctl_error(req, "failed to destroy %s (err=%d)", name, error);
1368 }
1369 
1370 static void
1371 g_multipath_ctl_rotate(struct gctl_req *req, struct g_class *mp)
1372 {
1373 	struct g_geom *gp;
1374 	const char *name;
1375 	int error;
1376 
1377 	g_topology_assert();
1378 
1379 	name = gctl_get_asciiparam(req, "arg0");
1380         if (name == NULL) {
1381                 gctl_error(req, "No 'arg0' argument");
1382                 return;
1383         }
1384 	gp = g_multipath_find_geom(mp, name);
1385 	if (gp == NULL) {
1386 		gctl_error(req, "Device %s is invalid", name);
1387 		return;
1388 	}
1389 	error = g_multipath_rotate(gp);
1390 	if (error != 0) {
1391 		gctl_error(req, "failed to rotate %s (err=%d)", name, error);
1392 	}
1393 }
1394 
1395 static void
1396 g_multipath_ctl_getactive(struct gctl_req *req, struct g_class *mp)
1397 {
1398 	struct sbuf *sb;
1399 	struct g_geom *gp;
1400 	struct g_multipath_softc *sc;
1401 	struct g_consumer *cp;
1402 	const char *name;
1403 	int empty;
1404 
1405 	sb = sbuf_new_auto();
1406 
1407 	g_topology_assert();
1408 	name = gctl_get_asciiparam(req, "arg0");
1409         if (name == NULL) {
1410                 gctl_error(req, "No 'arg0' argument");
1411                 return;
1412         }
1413 	gp = g_multipath_find_geom(mp, name);
1414 	if (gp == NULL) {
1415 		gctl_error(req, "Device %s is invalid", name);
1416 		return;
1417 	}
1418 	sc = gp->softc;
1419 	if (sc->sc_active_active == 1) {
1420 		empty = 1;
1421 		LIST_FOREACH(cp, &gp->consumer, consumer) {
1422 			if (cp->index & MP_BAD)
1423 				continue;
1424 			if (!empty)
1425 				sbuf_cat(sb, " ");
1426 			sbuf_cat(sb, cp->provider->name);
1427 			empty = 0;
1428 		}
1429 		if (empty)
1430 			sbuf_cat(sb, "none");
1431 		sbuf_cat(sb, "\n");
1432 	} else if (sc->sc_active && sc->sc_active->provider) {
1433 		sbuf_printf(sb, "%s\n", sc->sc_active->provider->name);
1434 	} else {
1435 		sbuf_printf(sb, "none\n");
1436 	}
1437 	sbuf_finish(sb);
1438 	gctl_set_param_err(req, "output", sbuf_data(sb), sbuf_len(sb) + 1);
1439 	sbuf_delete(sb);
1440 }
1441 
1442 static void
1443 g_multipath_config(struct gctl_req *req, struct g_class *mp, const char *verb)
1444 {
1445 	uint32_t *version;
1446 	g_topology_assert();
1447 	version = gctl_get_paraml(req, "version", sizeof(*version));
1448 	if (version == NULL) {
1449 		gctl_error(req, "No 'version' argument");
1450 	} else if (*version != G_MULTIPATH_VERSION) {
1451 		gctl_error(req, "Userland and kernel parts are out of sync");
1452 	} else if (strcmp(verb, "add") == 0) {
1453 		g_multipath_ctl_add(req, mp);
1454 	} else if (strcmp(verb, "prefer") == 0) {
1455 		g_multipath_ctl_prefer(req, mp);
1456 	} else if (strcmp(verb, "create") == 0) {
1457 		g_multipath_ctl_create(req, mp);
1458 	} else if (strcmp(verb, "configure") == 0) {
1459 		g_multipath_ctl_configure(req, mp);
1460 	} else if (strcmp(verb, "stop") == 0) {
1461 		g_multipath_ctl_stop(req, mp);
1462 	} else if (strcmp(verb, "destroy") == 0) {
1463 		g_multipath_ctl_destroy(req, mp);
1464 	} else if (strcmp(verb, "fail") == 0) {
1465 		g_multipath_ctl_fail(req, mp, 1);
1466 	} else if (strcmp(verb, "restore") == 0) {
1467 		g_multipath_ctl_fail(req, mp, 0);
1468 	} else if (strcmp(verb, "remove") == 0) {
1469 		g_multipath_ctl_remove(req, mp);
1470 	} else if (strcmp(verb, "rotate") == 0) {
1471 		g_multipath_ctl_rotate(req, mp);
1472 	} else if (strcmp(verb, "getactive") == 0) {
1473 		g_multipath_ctl_getactive(req, mp);
1474 	} else {
1475 		gctl_error(req, "Unknown verb %s", verb);
1476 	}
1477 }
1478 
1479 static void
1480 g_multipath_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1481     struct g_consumer *cp, struct g_provider *pp)
1482 {
1483 	struct g_multipath_softc *sc;
1484 	int good;
1485 
1486 	g_topology_assert();
1487 
1488 	sc = gp->softc;
1489 	if (sc == NULL)
1490 		return;
1491 	if (cp != NULL) {
1492 		sbuf_printf(sb, "%s<State>%s</State>\n", indent,
1493 		    (cp->index & MP_NEW) ? "NEW" :
1494 		    (cp->index & MP_LOST) ? "LOST" :
1495 		    (cp->index & MP_FAIL) ? "FAIL" :
1496 		    (sc->sc_active_active == 1 || sc->sc_active == cp) ?
1497 		     "ACTIVE" :
1498 		     sc->sc_active_active == 2 ? "READ" : "PASSIVE");
1499 	} else {
1500 		good = g_multipath_good(gp);
1501 		sbuf_printf(sb, "%s<State>%s</State>\n", indent,
1502 		    good == 0 ? "BROKEN" :
1503 		    (good != sc->sc_ndisks || sc->sc_ndisks == 1) ?
1504 		    "DEGRADED" : "OPTIMAL");
1505 	}
1506 	if (cp == NULL && pp == NULL) {
1507 		sbuf_printf(sb, "%s<UUID>%s</UUID>\n", indent, sc->sc_uuid);
1508 		sbuf_printf(sb, "%s<Mode>Active/%s</Mode>\n", indent,
1509 		    sc->sc_active_active == 2 ? "Read" :
1510 		    sc->sc_active_active == 1 ? "Active" : "Passive");
1511 		sbuf_printf(sb, "%s<Type>%s</Type>\n", indent,
1512 		    sc->sc_uuid[0] == 0 ? "MANUAL" : "AUTOMATIC");
1513 	}
1514 }
1515 
1516 DECLARE_GEOM_CLASS(g_multipath_class, g_multipath);
1517