xref: /freebsd/sys/geom/multipath/g_multipath.c (revision 9a14aa017b21c292740c00ee098195cd46642730)
1 /*-
2  * Copyright (c) 2011 Alexander Motin <mav@FreeBSD.org>
3  * Copyright (c) 2006-2007 Matthew Jacob <mjacob@FreeBSD.org>
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 /*
28  * Based upon work by Pawel Jakub Dawidek <pjd@FreeBSD.org> for all of the
29  * fine geom examples, and by Poul Henning Kamp <phk@FreeBSD.org> for GEOM
30  * itself, all of which is most gratefully acknowledged.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/bio.h>
42 #include <sys/sbuf.h>
43 #include <sys/sysctl.h>
44 #include <sys/kthread.h>
45 #include <sys/malloc.h>
46 #include <geom/geom.h>
47 #include <geom/multipath/g_multipath.h>
48 
49 FEATURE(geom_multipath, "GEOM multipath support");
50 
51 SYSCTL_DECL(_kern_geom);
52 static SYSCTL_NODE(_kern_geom, OID_AUTO, multipath, CTLFLAG_RW, 0,
53     "GEOM_MULTIPATH tunables");
54 static u_int g_multipath_debug = 0;
55 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, debug, CTLFLAG_RW,
56     &g_multipath_debug, 0, "Debug level");
57 static u_int g_multipath_exclusive = 1;
58 SYSCTL_UINT(_kern_geom_multipath, OID_AUTO, exclusive, CTLFLAG_RW,
59     &g_multipath_exclusive, 0, "Exclusively open providers");
60 
61 static enum {
62 	GKT_NIL,
63 	GKT_RUN,
64 	GKT_DIE
65 } g_multipath_kt_state;
66 static struct bio_queue_head gmtbq;
67 static struct mtx gmtbq_mtx;
68 
69 static void g_multipath_orphan(struct g_consumer *);
70 static void g_multipath_start(struct bio *);
71 static void g_multipath_done(struct bio *);
72 static void g_multipath_done_error(struct bio *);
73 static void g_multipath_kt(void *);
74 
75 static int g_multipath_destroy(struct g_geom *);
76 static int
77 g_multipath_destroy_geom(struct gctl_req *, struct g_class *, struct g_geom *);
78 
79 static struct g_geom *g_multipath_find_geom(struct g_class *, const char *);
80 static int g_multipath_rotate(struct g_geom *);
81 
82 static g_taste_t g_multipath_taste;
83 static g_ctl_req_t g_multipath_config;
84 static g_init_t g_multipath_init;
85 static g_fini_t g_multipath_fini;
86 static g_dumpconf_t g_multipath_dumpconf;
87 
88 struct g_class g_multipath_class = {
89 	.name		= G_MULTIPATH_CLASS_NAME,
90 	.version	= G_VERSION,
91 	.ctlreq		= g_multipath_config,
92 	.taste		= g_multipath_taste,
93 	.destroy_geom	= g_multipath_destroy_geom,
94 	.init		= g_multipath_init,
95 	.fini		= g_multipath_fini
96 };
97 
98 #define	MP_FAIL		0x00000001
99 #define	MP_LOST		0x00000002
100 #define	MP_NEW		0x00000004
101 #define	MP_POSTED	0x00000008
102 #define	MP_BAD		(MP_FAIL | MP_LOST | MP_NEW)
103 #define MP_IDLE		0x00000010
104 #define MP_IDLE_MASK	0xfffffff0
105 
106 static int
107 g_multipath_good(struct g_geom *gp)
108 {
109 	struct g_consumer *cp;
110 	int n = 0;
111 
112 	LIST_FOREACH(cp, &gp->consumer, consumer) {
113 		if ((cp->index & MP_BAD) == 0)
114 			n++;
115 	}
116 	return (n);
117 }
118 
119 static void
120 g_multipath_fault(struct g_consumer *cp, int cause)
121 {
122 	struct g_multipath_softc *sc;
123 	struct g_consumer *lcp;
124 	struct g_geom *gp;
125 
126 	gp = cp->geom;
127 	sc = gp->softc;
128 	cp->index |= cause;
129 	if (g_multipath_good(gp) == 0 && sc->sc_ndisks > 0) {
130 		LIST_FOREACH(lcp, &gp->consumer, consumer) {
131 			if (lcp->provider == NULL ||
132 			    (lcp->index & (MP_LOST | MP_NEW)))
133 				continue;
134 			if (sc->sc_ndisks > 1 && lcp == cp)
135 				continue;
136 			printf("GEOM_MULTIPATH: "
137 			    "all paths in %s were marked FAIL, restore %s\n",
138 			    sc->sc_name, lcp->provider->name);
139 			lcp->index &= ~MP_FAIL;
140 		}
141 	}
142 	if (cp != sc->sc_active)
143 		return;
144 	sc->sc_active = NULL;
145 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
146 		if ((lcp->index & MP_BAD) == 0) {
147 			sc->sc_active = lcp;
148 			break;
149 		}
150 	}
151 	if (sc->sc_active == NULL) {
152 		printf("GEOM_MULTIPATH: out of providers for %s\n",
153 		    sc->sc_name);
154 	} else if (!sc->sc_active_active) {
155 		printf("GEOM_MULTIPATH: %s is now active path in %s\n",
156 		    sc->sc_active->provider->name, sc->sc_name);
157 	}
158 }
159 
160 static struct g_consumer *
161 g_multipath_choose(struct g_geom *gp)
162 {
163 	struct g_multipath_softc *sc;
164 	struct g_consumer *best, *cp;
165 
166 	sc = gp->softc;
167 	if (!sc->sc_active_active)
168 		return (sc->sc_active);
169 	best = NULL;
170 	LIST_FOREACH(cp, &gp->consumer, consumer) {
171 		if (cp->index & MP_BAD)
172 			continue;
173 		cp->index += MP_IDLE;
174 		if (best == NULL || cp->private < best->private ||
175 		    (cp->private == best->private && cp->index > best->index))
176 			best = cp;
177 	}
178 	if (best != NULL)
179 		best->index &= ~MP_IDLE_MASK;
180 	return (best);
181 }
182 
183 static void
184 g_mpd(void *arg, int flags __unused)
185 {
186 	struct g_geom *gp;
187 	struct g_multipath_softc *sc;
188 	struct g_consumer *cp;
189 	int w;
190 
191 	g_topology_assert();
192 	cp = arg;
193 	gp = cp->geom;
194 	if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0) {
195 		w = cp->acw;
196 		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
197 		if (w > 0 && cp->provider != NULL &&
198 		    (cp->provider->geom->flags & G_GEOM_WITHER) == 0) {
199 			g_post_event(g_mpd, cp, M_WAITOK, NULL);
200 			return;
201 		}
202 	}
203 	sc = gp->softc;
204 	mtx_lock(&sc->sc_mtx);
205 	if (cp->provider) {
206 		printf("GEOM_MULTIPATH: %s removed from %s\n",
207 		    cp->provider->name, gp->name);
208 		g_detach(cp);
209 	}
210 	g_destroy_consumer(cp);
211 	mtx_unlock(&sc->sc_mtx);
212 	if (LIST_EMPTY(&gp->consumer))
213 		g_multipath_destroy(gp);
214 }
215 
216 static void
217 g_multipath_orphan(struct g_consumer *cp)
218 {
219 	struct g_multipath_softc *sc;
220 	uintptr_t *cnt;
221 
222 	g_topology_assert();
223 	printf("GEOM_MULTIPATH: %s in %s was disconnected\n",
224 	    cp->provider->name, cp->geom->name);
225 	sc = cp->geom->softc;
226 	cnt = (uintptr_t *)&cp->private;
227 	mtx_lock(&sc->sc_mtx);
228 	sc->sc_ndisks--;
229 	g_multipath_fault(cp, MP_LOST);
230 	if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
231 		cp->index |= MP_POSTED;
232 		mtx_unlock(&sc->sc_mtx);
233 		g_mpd(cp, 0);
234 	} else
235 		mtx_unlock(&sc->sc_mtx);
236 }
237 
238 static void
239 g_multipath_start(struct bio *bp)
240 {
241 	struct g_multipath_softc *sc;
242 	struct g_geom *gp;
243 	struct g_consumer *cp;
244 	struct bio *cbp;
245 	uintptr_t *cnt;
246 
247 	gp = bp->bio_to->geom;
248 	sc = gp->softc;
249 	KASSERT(sc != NULL, ("NULL sc"));
250 	cbp = g_clone_bio(bp);
251 	if (cbp == NULL) {
252 		g_io_deliver(bp, ENOMEM);
253 		return;
254 	}
255 	mtx_lock(&sc->sc_mtx);
256 	cp = g_multipath_choose(gp);
257 	if (cp == NULL) {
258 		mtx_unlock(&sc->sc_mtx);
259 		g_destroy_bio(cbp);
260 		g_io_deliver(bp, ENXIO);
261 		return;
262 	}
263 	if ((uintptr_t)bp->bio_driver1 < sc->sc_ndisks)
264 		bp->bio_driver1 = (void *)(uintptr_t)sc->sc_ndisks;
265 	cnt = (uintptr_t *)&cp->private;
266 	(*cnt)++;
267 	mtx_unlock(&sc->sc_mtx);
268 	cbp->bio_done = g_multipath_done;
269 	g_io_request(cbp, cp);
270 }
271 
272 static void
273 g_multipath_done(struct bio *bp)
274 {
275 	struct g_multipath_softc *sc;
276 	struct g_consumer *cp;
277 	uintptr_t *cnt;
278 
279 	if (bp->bio_error == ENXIO || bp->bio_error == EIO) {
280 		mtx_lock(&gmtbq_mtx);
281 		bioq_insert_tail(&gmtbq, bp);
282 		mtx_unlock(&gmtbq_mtx);
283 		wakeup(&g_multipath_kt_state);
284 	} else {
285 		cp = bp->bio_from;
286 		sc = cp->geom->softc;
287 		cnt = (uintptr_t *)&cp->private;
288 		mtx_lock(&sc->sc_mtx);
289 		(*cnt)--;
290 		if (*cnt == 0 && (cp->index & MP_LOST)) {
291 			cp->index |= MP_POSTED;
292 			mtx_unlock(&sc->sc_mtx);
293 			g_post_event(g_mpd, cp, M_WAITOK, NULL);
294 		} else
295 			mtx_unlock(&sc->sc_mtx);
296 		g_std_done(bp);
297 	}
298 }
299 
300 static void
301 g_multipath_done_error(struct bio *bp)
302 {
303 	struct bio *pbp;
304 	struct g_geom *gp;
305 	struct g_multipath_softc *sc;
306 	struct g_consumer *cp;
307 	struct g_provider *pp;
308 	uintptr_t *cnt;
309 
310 	/*
311 	 * If we had a failure, we have to check first to see
312 	 * whether the consumer it failed on was the currently
313 	 * active consumer (i.e., this is the first in perhaps
314 	 * a number of failures). If so, we then switch consumers
315 	 * to the next available consumer.
316 	 */
317 
318 	pbp = bp->bio_parent;
319 	gp = pbp->bio_to->geom;
320 	sc = gp->softc;
321 	cp = bp->bio_from;
322 	pp = cp->provider;
323 	cnt = (uintptr_t *)&cp->private;
324 
325 	mtx_lock(&sc->sc_mtx);
326 	printf("GEOM_MULTIPATH: Error %d, %s in %s marked FAIL\n",
327 	    bp->bio_error, pp->name, sc->sc_name);
328 	g_multipath_fault(cp, MP_FAIL);
329 	(*cnt)--;
330 	if (*cnt == 0 && (cp->index & (MP_LOST | MP_POSTED)) == MP_LOST) {
331 		cp->index |= MP_POSTED;
332 		mtx_unlock(&sc->sc_mtx);
333 		g_post_event(g_mpd, cp, M_WAITOK, NULL);
334 	} else
335 		mtx_unlock(&sc->sc_mtx);
336 
337 	/*
338 	 * If we can fruitfully restart the I/O, do so.
339 	 */
340 	if (pbp->bio_children < (uintptr_t)pbp->bio_driver1) {
341 		pbp->bio_inbed++;
342 		g_destroy_bio(bp);
343 		g_multipath_start(pbp);
344 	} else {
345 		g_std_done(bp);
346 	}
347 }
348 
349 static void
350 g_multipath_kt(void *arg)
351 {
352 
353 	g_multipath_kt_state = GKT_RUN;
354 	mtx_lock(&gmtbq_mtx);
355 	while (g_multipath_kt_state == GKT_RUN) {
356 		for (;;) {
357 			struct bio *bp;
358 
359 			bp = bioq_takefirst(&gmtbq);
360 			if (bp == NULL)
361 				break;
362 			mtx_unlock(&gmtbq_mtx);
363 			g_multipath_done_error(bp);
364 			mtx_lock(&gmtbq_mtx);
365 		}
366 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
367 		    "gkt:wait", hz / 10);
368 	}
369 	mtx_unlock(&gmtbq_mtx);
370 	wakeup(&g_multipath_kt_state);
371 	kproc_exit(0);
372 }
373 
374 
375 static int
376 g_multipath_access(struct g_provider *pp, int dr, int dw, int de)
377 {
378 	struct g_geom *gp;
379 	struct g_consumer *cp, *badcp = NULL;
380 	struct g_multipath_softc *sc;
381 	int error;
382 
383 	gp = pp->geom;
384 
385 	LIST_FOREACH(cp, &gp->consumer, consumer) {
386 		error = g_access(cp, dr, dw, de);
387 		if (error) {
388 			badcp = cp;
389 			goto fail;
390 		}
391 	}
392 	sc = gp->softc;
393 	sc->sc_opened += dr + dw + de;
394 	if (sc->sc_stopping && sc->sc_opened == 0)
395 		g_multipath_destroy(gp);
396 	return (0);
397 
398 fail:
399 	LIST_FOREACH(cp, &gp->consumer, consumer) {
400 		if (cp == badcp)
401 			break;
402 		(void) g_access(cp, -dr, -dw, -de);
403 	}
404 	return (error);
405 }
406 
407 static struct g_geom *
408 g_multipath_create(struct g_class *mp, struct g_multipath_metadata *md)
409 {
410 	struct g_multipath_softc *sc;
411 	struct g_geom *gp;
412 	struct g_provider *pp;
413 
414 	g_topology_assert();
415 
416 	LIST_FOREACH(gp, &mp->geom, geom) {
417 		sc = gp->softc;
418 		if (sc == NULL || sc->sc_stopping)
419 			continue;
420 		if (strcmp(gp->name, md->md_name) == 0) {
421 			printf("GEOM_MULTIPATH: name %s already exists\n",
422 			    md->md_name);
423 			return (NULL);
424 		}
425 	}
426 
427 	gp = g_new_geomf(mp, md->md_name);
428 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
429 	mtx_init(&sc->sc_mtx, "multipath", NULL, MTX_DEF);
430 	memcpy(sc->sc_uuid, md->md_uuid, sizeof (sc->sc_uuid));
431 	memcpy(sc->sc_name, md->md_name, sizeof (sc->sc_name));
432 	sc->sc_active_active = md->md_active_active;
433 	gp->softc = sc;
434 	gp->start = g_multipath_start;
435 	gp->orphan = g_multipath_orphan;
436 	gp->access = g_multipath_access;
437 	gp->dumpconf = g_multipath_dumpconf;
438 
439 	pp = g_new_providerf(gp, "multipath/%s", md->md_name);
440 	if (md->md_size != 0) {
441 		pp->mediasize = md->md_size -
442 		    ((md->md_uuid[0] != 0) ? md->md_sectorsize : 0);
443 		pp->sectorsize = md->md_sectorsize;
444 	}
445 	sc->sc_pp = pp;
446 	g_error_provider(pp, 0);
447 	printf("GEOM_MULTIPATH: %s created\n", gp->name);
448 	return (gp);
449 }
450 
451 static int
452 g_multipath_add_disk(struct g_geom *gp, struct g_provider *pp)
453 {
454 	struct g_multipath_softc *sc;
455 	struct g_consumer *cp, *nxtcp;
456 	int error, acr, acw, ace;
457 
458 	g_topology_assert();
459 
460 	sc = gp->softc;
461 	KASSERT(sc, ("no softc"));
462 
463 	/*
464 	 * Make sure that the passed provider isn't already attached
465 	 */
466 	LIST_FOREACH(cp, &gp->consumer, consumer) {
467 		if (cp->provider == pp)
468 			break;
469 	}
470 	if (cp) {
471 		printf("GEOM_MULTIPATH: provider %s already attached to %s\n",
472 		    pp->name, gp->name);
473 		return (EEXIST);
474 	}
475 	nxtcp = LIST_FIRST(&gp->consumer);
476 	cp = g_new_consumer(gp);
477 	cp->private = NULL;
478 	cp->index = MP_NEW;
479 	error = g_attach(cp, pp);
480 	if (error != 0) {
481 		printf("GEOM_MULTIPATH: cannot attach %s to %s",
482 		    pp->name, sc->sc_name);
483 		g_destroy_consumer(cp);
484 		return (error);
485 	}
486 
487 	/*
488 	 * Set access permissions on new consumer to match other consumers
489 	 */
490 	if (sc->sc_pp) {
491 		acr = sc->sc_pp->acr;
492 		acw = sc->sc_pp->acw;
493 		ace = sc->sc_pp->ace;
494 	} else
495 		acr = acw = ace = 0;
496 	if (g_multipath_exclusive) {
497 		acr++;
498 		acw++;
499 		ace++;
500 	}
501 	error = g_access(cp, acr, acw, ace);
502 	if (error) {
503 		printf("GEOM_MULTIPATH: cannot set access in "
504 		    "attaching %s to %s (%d)\n",
505 		    pp->name, sc->sc_name, error);
506 		g_detach(cp);
507 		g_destroy_consumer(cp);
508 		return (error);
509 	}
510 	if (sc->sc_pp != NULL && sc->sc_pp->mediasize == 0) {
511 		sc->sc_pp->mediasize = pp->mediasize -
512 		    ((sc->sc_uuid[0] != 0) ? pp->sectorsize : 0);
513 		sc->sc_pp->sectorsize = pp->sectorsize;
514 	}
515 	if (sc->sc_pp != NULL &&
516 	    sc->sc_pp->stripesize == 0 && sc->sc_pp->stripeoffset == 0) {
517 		sc->sc_pp->stripesize = pp->stripesize;
518 		sc->sc_pp->stripeoffset = pp->stripeoffset;
519 	}
520 	mtx_lock(&sc->sc_mtx);
521 	cp->index = 0;
522 	sc->sc_ndisks++;
523 	mtx_unlock(&sc->sc_mtx);
524 	printf("GEOM_MULTIPATH: %s added to %s\n",
525 	    pp->name, sc->sc_name);
526 	if (sc->sc_active == NULL) {
527 		sc->sc_active = cp;
528 		if (!sc->sc_active_active)
529 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
530 			    pp->name, sc->sc_name);
531 	}
532 	return (0);
533 }
534 
535 static int
536 g_multipath_destroy(struct g_geom *gp)
537 {
538 	struct g_multipath_softc *sc;
539 	struct g_consumer *cp, *cp1;
540 
541 	g_topology_assert();
542 	if (gp->softc == NULL)
543 		return (ENXIO);
544 	sc = gp->softc;
545 	if (!sc->sc_stopping) {
546 		printf("GEOM_MULTIPATH: destroying %s\n", gp->name);
547 		sc->sc_stopping = 1;
548 	}
549 	if (sc->sc_opened != 0) {
550 		if (sc->sc_pp != NULL) {
551 			g_wither_provider(sc->sc_pp, ENXIO);
552 			sc->sc_pp = NULL;
553 		}
554 		return (EINPROGRESS);
555 	}
556 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
557 		mtx_lock(&sc->sc_mtx);
558 		if ((cp->index & MP_POSTED) == 0) {
559 			cp->index |= MP_POSTED;
560 			mtx_unlock(&sc->sc_mtx);
561 			g_mpd(cp, 0);
562 			if (cp1 == NULL)
563 				return(0);	/* Recursion happened. */
564 		} else
565 			mtx_unlock(&sc->sc_mtx);
566 	}
567 	if (!LIST_EMPTY(&gp->consumer))
568 		return (EINPROGRESS);
569 	mtx_destroy(&sc->sc_mtx);
570 	g_free(gp->softc);
571 	gp->softc = NULL;
572 	printf("GEOM_MULTIPATH: %s destroyed\n", gp->name);
573 	g_wither_geom(gp, ENXIO);
574 	return (0);
575 }
576 
577 static int
578 g_multipath_destroy_geom(struct gctl_req *req, struct g_class *mp,
579     struct g_geom *gp)
580 {
581 
582 	return (g_multipath_destroy(gp));
583 }
584 
585 static int
586 g_multipath_rotate(struct g_geom *gp)
587 {
588 	struct g_consumer *lcp;
589 	struct g_multipath_softc *sc = gp->softc;
590 
591 	g_topology_assert();
592 	if (sc == NULL)
593 		return (ENXIO);
594 	LIST_FOREACH(lcp, &gp->consumer, consumer) {
595 		if ((lcp->index & MP_BAD) == 0) {
596 			if (sc->sc_active != lcp)
597 				break;
598 		}
599 	}
600 	if (lcp) {
601 		sc->sc_active = lcp;
602 		if (!sc->sc_active_active)
603 			printf("GEOM_MULTIPATH: %s is now active path in %s\n",
604 			    lcp->provider->name, sc->sc_name);
605 	}
606 	return (0);
607 }
608 
609 static void
610 g_multipath_init(struct g_class *mp)
611 {
612 	bioq_init(&gmtbq);
613 	mtx_init(&gmtbq_mtx, "gmtbq", NULL, MTX_DEF);
614 	if (kproc_create(g_multipath_kt, mp, NULL, 0, 0, "g_mp_kt") == 0)
615 		g_multipath_kt_state = GKT_RUN;
616 }
617 
618 static void
619 g_multipath_fini(struct g_class *mp)
620 {
621 	if (g_multipath_kt_state == GKT_RUN) {
622 		mtx_lock(&gmtbq_mtx);
623 		g_multipath_kt_state = GKT_DIE;
624 		wakeup(&g_multipath_kt_state);
625 		msleep(&g_multipath_kt_state, &gmtbq_mtx, PRIBIO,
626 		    "gmp:fini", 0);
627 		mtx_unlock(&gmtbq_mtx);
628 	}
629 }
630 
631 static int
632 g_multipath_read_metadata(struct g_consumer *cp,
633     struct g_multipath_metadata *md)
634 {
635 	struct g_provider *pp;
636 	u_char *buf;
637 	int error;
638 
639 	g_topology_assert();
640 	error = g_access(cp, 1, 0, 0);
641 	if (error != 0)
642 		return (error);
643 	pp = cp->provider;
644 	g_topology_unlock();
645 	buf = g_read_data(cp, pp->mediasize - pp->sectorsize,
646 	    pp->sectorsize, &error);
647 	g_topology_lock();
648 	g_access(cp, -1, 0, 0);
649 	if (buf == NULL)
650 		return (error);
651 	multipath_metadata_decode(buf, md);
652 	g_free(buf);
653 	return (0);
654 }
655 
656 static struct g_geom *
657 g_multipath_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
658 {
659 	struct g_multipath_metadata md;
660 	struct g_multipath_softc *sc;
661 	struct g_consumer *cp;
662 	struct g_geom *gp, *gp1;
663 	int error, isnew;
664 
665 	g_topology_assert();
666 
667 	gp = g_new_geomf(mp, "multipath:taste");
668 	gp->start = g_multipath_start;
669 	gp->access = g_multipath_access;
670 	gp->orphan = g_multipath_orphan;
671 	cp = g_new_consumer(gp);
672 	g_attach(cp, pp);
673 	error = g_multipath_read_metadata(cp, &md);
674 	g_detach(cp);
675 	g_destroy_consumer(cp);
676 	g_destroy_geom(gp);
677 	if (error != 0)
678 		return (NULL);
679 	gp = NULL;
680 
681 	if (strcmp(md.md_magic, G_MULTIPATH_MAGIC) != 0) {
682 		if (g_multipath_debug)
683 			printf("%s is not MULTIPATH\n", pp->name);
684 		return (NULL);
685 	}
686 	if (md.md_version != G_MULTIPATH_VERSION) {
687 		printf("%s has version %d multipath id- this module is version "
688 		    " %d: rejecting\n", pp->name, md.md_version,
689 		    G_MULTIPATH_VERSION);
690 		return (NULL);
691 	}
692 	if (md.md_size != 0 && md.md_size != pp->mediasize)
693 		return (NULL);
694 	if (md.md_sectorsize != 0 && md.md_sectorsize != pp->sectorsize)
695 		return (NULL);
696 	if (g_multipath_debug)
697 		printf("MULTIPATH: %s/%s\n", md.md_name, md.md_uuid);
698 
699 	/*
700 	 * Let's check if such a device already is present. We check against
701 	 * uuid alone first because that's the true distinguishor. If that
702 	 * passes, then we check for name conflicts. If there are conflicts,
703 	 * modify the name.
704 	 *
705 	 * The whole purpose of this is to solve the problem that people don't
706 	 * pick good unique names, but good unique names (like uuids) are a
707 	 * pain to use. So, we allow people to build GEOMs with friendly names
708 	 * and uuids, and modify the names in case there's a collision.
709 	 */
710 	sc = NULL;
711 	LIST_FOREACH(gp, &mp->geom, geom) {
712 		sc = gp->softc;
713 		if (sc == NULL || sc->sc_stopping)
714 			continue;
715 		if (strncmp(md.md_uuid, sc->sc_uuid, sizeof(md.md_uuid)) == 0)
716 			break;
717 	}
718 
719 	LIST_FOREACH(gp1, &mp->geom, geom) {
720 		if (gp1 == gp)
721 			continue;
722 		sc = gp1->softc;
723 		if (sc == NULL || sc->sc_stopping)
724 			continue;
725 		if (strncmp(md.md_name, sc->sc_name, sizeof(md.md_name)) == 0)
726 			break;
727 	}
728 
729 	/*
730 	 * If gp is NULL, we had no extant MULTIPATH geom with this uuid.
731 	 *
732 	 * If gp1 is *not* NULL, that means we have a MULTIPATH geom extant
733 	 * with the same name (but a different UUID).
734 	 *
735 	 * If gp is NULL, then modify the name with a random number and
736   	 * complain, but allow the creation of the geom to continue.
737 	 *
738 	 * If gp is *not* NULL, just use the geom's name as we're attaching
739 	 * this disk to the (previously generated) name.
740 	 */
741 
742 	if (gp1) {
743 		sc = gp1->softc;
744 		if (gp == NULL) {
745 			char buf[16];
746 			u_long rand = random();
747 
748 			snprintf(buf, sizeof (buf), "%s-%lu", md.md_name, rand);
749 			printf("GEOM_MULTIPATH: geom %s/%s exists already\n",
750 			    sc->sc_name, sc->sc_uuid);
751 			printf("GEOM_MULTIPATH: %s will be (temporarily) %s\n",
752 			    md.md_uuid, buf);
753 			strlcpy(md.md_name, buf, sizeof(md.md_name));
754 		} else {
755 			strlcpy(md.md_name, sc->sc_name, sizeof(md.md_name));
756 		}
757 	}
758 
759 	if (gp == NULL) {
760 		gp = g_multipath_create(mp, &md);
761 		if (gp == NULL) {
762 			printf("GEOM_MULTIPATH: cannot create geom %s/%s\n",
763 			    md.md_name, md.md_uuid);
764 			return (NULL);
765 		}
766 		isnew = 1;
767 	} else {
768 		isnew = 0;
769 	}
770 
771 	sc = gp->softc;
772 	KASSERT(sc != NULL, ("sc is NULL"));
773 	error = g_multipath_add_disk(gp, pp);
774 	if (error != 0) {
775 		if (isnew)
776 			g_multipath_destroy(gp);
777 		return (NULL);
778 	}
779 	return (gp);
780 }
781 
782 static void
783 g_multipath_ctl_add_name(struct gctl_req *req, struct g_class *mp,
784     const char *name)
785 {
786 	struct g_multipath_softc *sc;
787 	struct g_geom *gp;
788 	struct g_consumer *cp;
789 	struct g_provider *pp;
790 	const char *mpname;
791 	static const char devpf[6] = "/dev/";
792 
793 	g_topology_assert();
794 
795 	mpname = gctl_get_asciiparam(req, "arg0");
796         if (mpname == NULL) {
797                 gctl_error(req, "No 'arg0' argument");
798                 return;
799         }
800 	gp = g_multipath_find_geom(mp, mpname);
801 	if (gp == NULL) {
802 		gctl_error(req, "Device %s is invalid", mpname);
803 		return;
804 	}
805 	sc = gp->softc;
806 
807 	if (strncmp(name, devpf, 5) == 0)
808 		name += 5;
809 	pp = g_provider_by_name(name);
810 	if (pp == NULL) {
811 		gctl_error(req, "Provider %s is invalid", name);
812 		return;
813 	}
814 
815 	/*
816 	 * Check to make sure parameters match.
817 	 */
818 	LIST_FOREACH(cp, &gp->consumer, consumer) {
819 		if (cp->provider == pp) {
820 			gctl_error(req, "provider %s is already there",
821 			    pp->name);
822 			return;
823 		}
824 	}
825 	if (sc->sc_pp != NULL && sc->sc_pp->mediasize != 0 &&
826 	    sc->sc_pp->mediasize + (sc->sc_uuid[0] != 0 ? pp->sectorsize : 0)
827 	     != pp->mediasize) {
828 		gctl_error(req, "Providers size mismatch %jd != %jd",
829 		    (intmax_t) sc->sc_pp->mediasize +
830 			(sc->sc_uuid[0] != 0 ? pp->sectorsize : 0),
831 		    (intmax_t) pp->mediasize);
832 		return;
833 	}
834 	if (sc->sc_pp != NULL && sc->sc_pp->sectorsize != 0 &&
835 	    sc->sc_pp->sectorsize != pp->sectorsize) {
836 		gctl_error(req, "Providers sectorsize mismatch %u != %u",
837 		    sc->sc_pp->sectorsize, pp->sectorsize);
838 		return;
839 	}
840 
841 	/*
842 	 * Now add....
843 	 */
844 	(void) g_multipath_add_disk(gp, pp);
845 }
846 
847 static void
848 g_multipath_ctl_add(struct gctl_req *req, struct g_class *mp)
849 {
850 	struct g_multipath_softc *sc;
851 	struct g_geom *gp;
852 	const char *mpname, *name;
853 
854 	mpname = gctl_get_asciiparam(req, "arg0");
855         if (mpname == NULL) {
856                 gctl_error(req, "No 'arg0' argument");
857                 return;
858         }
859 	gp = g_multipath_find_geom(mp, mpname);
860 	if (gp == NULL) {
861 		gctl_error(req, "Device %s not found", mpname);
862 		return;
863 	}
864 	sc = gp->softc;
865 
866 	name = gctl_get_asciiparam(req, "arg1");
867 	if (name == NULL) {
868 		gctl_error(req, "No 'arg1' argument");
869 		return;
870 	}
871 	g_multipath_ctl_add_name(req, mp, name);
872 }
873 
874 static void
875 g_multipath_ctl_create(struct gctl_req *req, struct g_class *mp)
876 {
877 	struct g_multipath_metadata md;
878 	struct g_multipath_softc *sc;
879 	struct g_geom *gp;
880 	const char *mpname, *name;
881 	char param[16];
882 	int *nargs, i, *active_active;
883 
884 	g_topology_assert();
885 
886 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
887 	if (*nargs < 2) {
888 		gctl_error(req, "wrong number of arguments.");
889 		return;
890 	}
891 
892 	mpname = gctl_get_asciiparam(req, "arg0");
893         if (mpname == NULL) {
894                 gctl_error(req, "No 'arg0' argument");
895                 return;
896         }
897 	gp = g_multipath_find_geom(mp, mpname);
898 	if (gp != NULL) {
899 		gctl_error(req, "Device %s already exist", mpname);
900 		return;
901 	}
902 	sc = gp->softc;
903 
904 	memset(&md, 0, sizeof(md));
905 	strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic));
906 	md.md_version = G_MULTIPATH_VERSION;
907 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
908 	md.md_size = 0;
909 	md.md_sectorsize = 0;
910 	md.md_uuid[0] = 0;
911 	active_active = gctl_get_paraml(req, "active_active",
912 	    sizeof(*active_active));
913 	md.md_active_active =
914 	    (active_active == NULL || *active_active == 0) ? 0 : 1;
915 	gp = g_multipath_create(mp, &md);
916 	if (gp == NULL) {
917 		gctl_error(req, "GEOM_MULTIPATH: cannot create geom %s/%s\n",
918 		    md.md_name, md.md_uuid);
919 		return;
920 	}
921 	sc = gp->softc;
922 
923 	for (i = 1; i < *nargs; i++) {
924 		snprintf(param, sizeof(param), "arg%d", i);
925 		name = gctl_get_asciiparam(req, param);
926 		g_multipath_ctl_add_name(req, mp, name);
927 	}
928 
929 	if (sc->sc_ndisks != (*nargs - 1))
930 		g_multipath_destroy(gp);
931 }
932 
933 static void
934 g_multipath_ctl_fail(struct gctl_req *req, struct g_class *mp, int fail)
935 {
936 	struct g_multipath_softc *sc;
937 	struct g_geom *gp;
938 	struct g_consumer *cp;
939 	const char *mpname, *name;
940 	int found;
941 
942 	mpname = gctl_get_asciiparam(req, "arg0");
943         if (mpname == NULL) {
944                 gctl_error(req, "No 'arg0' argument");
945                 return;
946         }
947 	gp = g_multipath_find_geom(mp, mpname);
948 	if (gp == NULL) {
949 		gctl_error(req, "Device %s not found", mpname);
950 		return;
951 	}
952 	sc = gp->softc;
953 
954 	name = gctl_get_asciiparam(req, "arg1");
955 	if (name == NULL) {
956 		gctl_error(req, "No 'arg1' argument");
957 		return;
958 	}
959 
960 	found = 0;
961 	mtx_lock(&sc->sc_mtx);
962 	LIST_FOREACH(cp, &gp->consumer, consumer) {
963 		if (cp->provider != NULL &&
964 		    strcmp(cp->provider->name, name) == 0 &&
965 		    (cp->index & MP_LOST) == 0) {
966 			found = 1;
967 			printf("GEOM_MULTIPATH: %s in %s is marked %s.\n",
968 				name, sc->sc_name, fail ? "FAIL" : "OK");
969 			if (fail) {
970 				g_multipath_fault(cp, MP_FAIL);
971 			} else {
972 				cp->index &= ~MP_FAIL;
973 			}
974 		}
975 	}
976 	mtx_unlock(&sc->sc_mtx);
977 	if (found == 0)
978 		gctl_error(req, "Provider %s not found", name);
979 }
980 
981 static void
982 g_multipath_ctl_remove(struct gctl_req *req, struct g_class *mp)
983 {
984 	struct g_multipath_softc *sc;
985 	struct g_geom *gp;
986 	struct g_consumer *cp, *cp1;
987 	const char *mpname, *name;
988 	uintptr_t *cnt;
989 	int found;
990 
991 	mpname = gctl_get_asciiparam(req, "arg0");
992         if (mpname == NULL) {
993                 gctl_error(req, "No 'arg0' argument");
994                 return;
995         }
996 	gp = g_multipath_find_geom(mp, mpname);
997 	if (gp == NULL) {
998 		gctl_error(req, "Device %s not found", mpname);
999 		return;
1000 	}
1001 	sc = gp->softc;
1002 
1003 	name = gctl_get_asciiparam(req, "arg1");
1004 	if (name == NULL) {
1005 		gctl_error(req, "No 'arg1' argument");
1006 		return;
1007 	}
1008 
1009 	found = 0;
1010 	mtx_lock(&sc->sc_mtx);
1011 	LIST_FOREACH_SAFE(cp, &gp->consumer, consumer, cp1) {
1012 		if (cp->provider != NULL &&
1013 		    strcmp(cp->provider->name, name) == 0 &&
1014 		    (cp->index & MP_LOST) == 0) {
1015 			found = 1;
1016 			printf("GEOM_MULTIPATH: removing %s from %s\n",
1017 			    cp->provider->name, cp->geom->name);
1018 			sc->sc_ndisks--;
1019 			g_multipath_fault(cp, MP_LOST);
1020 			cnt = (uintptr_t *)&cp->private;
1021 			if (*cnt == 0 && (cp->index & MP_POSTED) == 0) {
1022 				cp->index |= MP_POSTED;
1023 				mtx_unlock(&sc->sc_mtx);
1024 				g_mpd(cp, 0);
1025 				if (cp1 == NULL)
1026 					return;	/* Recursion happened. */
1027 				mtx_lock(&sc->sc_mtx);
1028 			}
1029 		}
1030 	}
1031 	mtx_unlock(&sc->sc_mtx);
1032 	if (found == 0)
1033 		gctl_error(req, "Provider %s not found", name);
1034 }
1035 
1036 static struct g_geom *
1037 g_multipath_find_geom(struct g_class *mp, const char *name)
1038 {
1039 	struct g_geom *gp;
1040 	struct g_multipath_softc *sc;
1041 
1042 	LIST_FOREACH(gp, &mp->geom, geom) {
1043 		sc = gp->softc;
1044 		if (sc == NULL || sc->sc_stopping)
1045 			continue;
1046 		if (strcmp(gp->name, name) == 0)
1047 			return (gp);
1048 	}
1049 	return (NULL);
1050 }
1051 
1052 static void
1053 g_multipath_ctl_stop(struct gctl_req *req, struct g_class *mp)
1054 {
1055 	struct g_geom *gp;
1056 	const char *name;
1057 	int error;
1058 
1059 	g_topology_assert();
1060 
1061 	name = gctl_get_asciiparam(req, "arg0");
1062         if (name == NULL) {
1063                 gctl_error(req, "No 'arg0' argument");
1064                 return;
1065         }
1066 	gp = g_multipath_find_geom(mp, name);
1067 	if (gp == NULL) {
1068 		gctl_error(req, "Device %s is invalid", name);
1069 		return;
1070 	}
1071 	error = g_multipath_destroy(gp);
1072 	if (error != 0 && error != EINPROGRESS)
1073 		gctl_error(req, "failed to stop %s (err=%d)", name, error);
1074 }
1075 
1076 static void
1077 g_multipath_ctl_destroy(struct gctl_req *req, struct g_class *mp)
1078 {
1079 	struct g_geom *gp;
1080 	struct g_multipath_softc *sc;
1081 	struct g_consumer *cp;
1082 	struct g_provider *pp;
1083 	const char *name;
1084 	uint8_t *buf;
1085 	int error;
1086 
1087 	g_topology_assert();
1088 
1089 	name = gctl_get_asciiparam(req, "arg0");
1090         if (name == NULL) {
1091                 gctl_error(req, "No 'arg0' argument");
1092                 return;
1093         }
1094 	gp = g_multipath_find_geom(mp, name);
1095 	if (gp == NULL) {
1096 		gctl_error(req, "Device %s is invalid", name);
1097 		return;
1098 	}
1099 	sc = gp->softc;
1100 
1101 	if (sc->sc_uuid[0] != 0 && sc->sc_active != NULL) {
1102 		cp = sc->sc_active;
1103 		pp = cp->provider;
1104 		error = g_access(cp, 1, 1, 1);
1105 		if (error != 0) {
1106 			gctl_error(req, "Can't open %s (%d)", pp->name, error);
1107 			goto destroy;
1108 		}
1109 		g_topology_unlock();
1110 		buf = g_malloc(pp->sectorsize, M_WAITOK | M_ZERO);
1111 		error = g_write_data(cp, pp->mediasize - pp->sectorsize,
1112 		    buf, pp->sectorsize);
1113 		g_topology_lock();
1114 		g_access(cp, -1, -1, -1);
1115 		if (error != 0)
1116 			gctl_error(req, "Can't erase metadata on %s (%d)",
1117 			    pp->name, error);
1118 	}
1119 
1120 destroy:
1121 	error = g_multipath_destroy(gp);
1122 	if (error != 0 && error != EINPROGRESS)
1123 		gctl_error(req, "failed to destroy %s (err=%d)", name, error);
1124 }
1125 
1126 static void
1127 g_multipath_ctl_rotate(struct gctl_req *req, struct g_class *mp)
1128 {
1129 	struct g_geom *gp;
1130 	const char *name;
1131 	int error;
1132 
1133 	g_topology_assert();
1134 
1135 	name = gctl_get_asciiparam(req, "arg0");
1136         if (name == NULL) {
1137                 gctl_error(req, "No 'arg0' argument");
1138                 return;
1139         }
1140 	gp = g_multipath_find_geom(mp, name);
1141 	if (gp == NULL) {
1142 		gctl_error(req, "Device %s is invalid", name);
1143 		return;
1144 	}
1145 	error = g_multipath_rotate(gp);
1146 	if (error != 0) {
1147 		gctl_error(req, "failed to rotate %s (err=%d)", name, error);
1148 	}
1149 }
1150 
1151 static void
1152 g_multipath_ctl_getactive(struct gctl_req *req, struct g_class *mp)
1153 {
1154 	struct sbuf *sb;
1155 	struct g_geom *gp;
1156 	struct g_multipath_softc *sc;
1157 	struct g_consumer *cp;
1158 	const char *name;
1159 	int empty;
1160 
1161 	sb = sbuf_new_auto();
1162 
1163 	g_topology_assert();
1164 	name = gctl_get_asciiparam(req, "arg0");
1165         if (name == NULL) {
1166                 gctl_error(req, "No 'arg0' argument");
1167                 return;
1168         }
1169 	gp = g_multipath_find_geom(mp, name);
1170 	if (gp == NULL) {
1171 		gctl_error(req, "Device %s is invalid", name);
1172 		return;
1173 	}
1174 	sc = gp->softc;
1175 	if (sc->sc_active_active) {
1176 		empty = 1;
1177 		LIST_FOREACH(cp, &gp->consumer, consumer) {
1178 			if (cp->index & MP_BAD)
1179 				continue;
1180 			if (!empty)
1181 				sbuf_cat(sb, " ");
1182 			sbuf_cat(sb, cp->provider->name);
1183 			empty = 0;
1184 		}
1185 		if (empty)
1186 			sbuf_cat(sb, "none");
1187 		sbuf_cat(sb, "\n");
1188 	} else if (sc->sc_active && sc->sc_active->provider) {
1189 		sbuf_printf(sb, "%s\n", sc->sc_active->provider->name);
1190 	} else {
1191 		sbuf_printf(sb, "none\n");
1192 	}
1193 	sbuf_finish(sb);
1194 	gctl_set_param_err(req, "output", sbuf_data(sb), sbuf_len(sb) + 1);
1195 	sbuf_delete(sb);
1196 }
1197 
1198 static void
1199 g_multipath_config(struct gctl_req *req, struct g_class *mp, const char *verb)
1200 {
1201 	uint32_t *version;
1202 	g_topology_assert();
1203 	version = gctl_get_paraml(req, "version", sizeof(*version));
1204 	if (version == NULL) {
1205 		gctl_error(req, "No 'version' argument");
1206 	} else if (*version != G_MULTIPATH_VERSION) {
1207 		gctl_error(req, "Userland and kernel parts are out of sync");
1208 	} else if (strcmp(verb, "add") == 0) {
1209 		g_multipath_ctl_add(req, mp);
1210 	} else if (strcmp(verb, "create") == 0) {
1211 		g_multipath_ctl_create(req, mp);
1212 	} else if (strcmp(verb, "stop") == 0) {
1213 		g_multipath_ctl_stop(req, mp);
1214 	} else if (strcmp(verb, "destroy") == 0) {
1215 		g_multipath_ctl_destroy(req, mp);
1216 	} else if (strcmp(verb, "fail") == 0) {
1217 		g_multipath_ctl_fail(req, mp, 1);
1218 	} else if (strcmp(verb, "restore") == 0) {
1219 		g_multipath_ctl_fail(req, mp, 0);
1220 	} else if (strcmp(verb, "remove") == 0) {
1221 		g_multipath_ctl_remove(req, mp);
1222 	} else if (strcmp(verb, "rotate") == 0) {
1223 		g_multipath_ctl_rotate(req, mp);
1224 	} else if (strcmp(verb, "getactive") == 0) {
1225 		g_multipath_ctl_getactive(req, mp);
1226 	} else {
1227 		gctl_error(req, "Unknown verb %s", verb);
1228 	}
1229 }
1230 
1231 static void
1232 g_multipath_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
1233     struct g_consumer *cp, struct g_provider *pp)
1234 {
1235 	struct g_multipath_softc *sc;
1236 	int good;
1237 
1238 	g_topology_assert();
1239 
1240 	sc = gp->softc;
1241 	if (sc == NULL)
1242 		return;
1243 	if (cp != NULL) {
1244 		sbuf_printf(sb, "%s<State>%s</State>", indent,
1245 		    (cp->index & MP_NEW) ? "NEW" :
1246 		    (cp->index & MP_LOST) ? "LOST" :
1247 		    (cp->index & MP_FAIL) ? "FAIL" :
1248 		    (sc->sc_active_active || sc->sc_active == cp) ?
1249 		     "ACTIVE" : "PASSIVE");
1250 	} else {
1251 		good = g_multipath_good(gp);
1252 		sbuf_printf(sb, "%s<State>%s</State>", indent,
1253 		    good == 0 ? "BROKEN" :
1254 		    (good != sc->sc_ndisks || sc->sc_ndisks == 1) ?
1255 		    "DEGRADED" : "OPTIMAL");
1256 	}
1257 	if (cp == NULL && pp == NULL) {
1258 		sbuf_printf(sb, "%s<UUID>%s</UUID>", indent, sc->sc_uuid);
1259 		sbuf_printf(sb, "%s<Mode>Active/%s</Mode>", indent,
1260 		    sc->sc_active_active ? "Active" : "Passive");
1261 		sbuf_printf(sb, "%s<Type>%s</Type>", indent,
1262 		    sc->sc_uuid[0] == 0 ? "MANUAL" : "AUTOMATIC");
1263 	}
1264 }
1265 
1266 DECLARE_GEOM_CLASS(g_multipath_class, g_multipath);
1267