xref: /freebsd/sys/geom/geom_io.c (revision 3631c6382fa261edc7d8c7769e72f2ff7201468c)
1dd84a43cSPoul-Henning Kamp /*-
2dd84a43cSPoul-Henning Kamp  * Copyright (c) 2002 Poul-Henning Kamp
3dd84a43cSPoul-Henning Kamp  * Copyright (c) 2002 Networks Associates Technology, Inc.
4dd84a43cSPoul-Henning Kamp  * All rights reserved.
5dd84a43cSPoul-Henning Kamp  *
6dd84a43cSPoul-Henning Kamp  * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7dd84a43cSPoul-Henning Kamp  * and NAI Labs, the Security Research Division of Network Associates, Inc.
8dd84a43cSPoul-Henning Kamp  * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9dd84a43cSPoul-Henning Kamp  * DARPA CHATS research program.
10dd84a43cSPoul-Henning Kamp  *
11dd84a43cSPoul-Henning Kamp  * Redistribution and use in source and binary forms, with or without
12dd84a43cSPoul-Henning Kamp  * modification, are permitted provided that the following conditions
13dd84a43cSPoul-Henning Kamp  * are met:
14dd84a43cSPoul-Henning Kamp  * 1. Redistributions of source code must retain the above copyright
15dd84a43cSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer.
16dd84a43cSPoul-Henning Kamp  * 2. Redistributions in binary form must reproduce the above copyright
17dd84a43cSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer in the
18dd84a43cSPoul-Henning Kamp  *    documentation and/or other materials provided with the distribution.
19dd84a43cSPoul-Henning Kamp  * 3. The names of the authors may not be used to endorse or promote
20dd84a43cSPoul-Henning Kamp  *    products derived from this software without specific prior written
21dd84a43cSPoul-Henning Kamp  *    permission.
22dd84a43cSPoul-Henning Kamp  *
23dd84a43cSPoul-Henning Kamp  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24dd84a43cSPoul-Henning Kamp  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25dd84a43cSPoul-Henning Kamp  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26dd84a43cSPoul-Henning Kamp  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27dd84a43cSPoul-Henning Kamp  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28dd84a43cSPoul-Henning Kamp  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29dd84a43cSPoul-Henning Kamp  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30dd84a43cSPoul-Henning Kamp  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31dd84a43cSPoul-Henning Kamp  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32dd84a43cSPoul-Henning Kamp  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33dd84a43cSPoul-Henning Kamp  * SUCH DAMAGE.
34dd84a43cSPoul-Henning Kamp  */
35dd84a43cSPoul-Henning Kamp 
3650b1faefSDavid E. O'Brien #include <sys/cdefs.h>
3750b1faefSDavid E. O'Brien __FBSDID("$FreeBSD$");
38dd84a43cSPoul-Henning Kamp 
39dd84a43cSPoul-Henning Kamp #include <sys/param.h>
40dd84a43cSPoul-Henning Kamp #include <sys/systm.h>
41dd84a43cSPoul-Henning Kamp #include <sys/kernel.h>
42dd84a43cSPoul-Henning Kamp #include <sys/malloc.h>
43dd84a43cSPoul-Henning Kamp #include <sys/bio.h>
4449dbb61dSRobert Watson #include <sys/ktr.h>
4551460da8SJohn Baldwin #include <sys/proc.h>
463b378147SPawel Jakub Dawidek #include <sys/stack.h>
47dd84a43cSPoul-Henning Kamp 
48dd84a43cSPoul-Henning Kamp #include <sys/errno.h>
49dd84a43cSPoul-Henning Kamp #include <geom/geom.h>
50b1876192SPoul-Henning Kamp #include <geom/geom_int.h>
51e24cbd90SPoul-Henning Kamp #include <sys/devicestat.h>
52dd84a43cSPoul-Henning Kamp 
535ffb2c8bSPoul-Henning Kamp #include <vm/uma.h>
545ffb2c8bSPoul-Henning Kamp 
55dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_down;
56dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_up;
575fcf4e43SPoul-Henning Kamp static struct g_bioq g_bio_run_task;
58dd84a43cSPoul-Henning Kamp 
593432e4fdSPoul-Henning Kamp static u_int pace;
605ffb2c8bSPoul-Henning Kamp static uma_zone_t	biozone;
613432e4fdSPoul-Henning Kamp 
626231f75bSLuigi Rizzo /*
636231f75bSLuigi Rizzo  * The head of the list of classifiers used in g_io_request.
646231f75bSLuigi Rizzo  * Use g_register_classifier() and g_unregister_classifier()
656231f75bSLuigi Rizzo  * to add/remove entries to the list.
666231f75bSLuigi Rizzo  * Classifiers are invoked in registration order.
676231f75bSLuigi Rizzo  */
686231f75bSLuigi Rizzo static TAILQ_HEAD(g_classifier_tailq, g_classifier_hook)
696231f75bSLuigi Rizzo     g_classifier_tailq = TAILQ_HEAD_INITIALIZER(g_classifier_tailq);
706231f75bSLuigi Rizzo 
71dd84a43cSPoul-Henning Kamp #include <machine/atomic.h>
72dd84a43cSPoul-Henning Kamp 
73dd84a43cSPoul-Henning Kamp static void
74dd84a43cSPoul-Henning Kamp g_bioq_lock(struct g_bioq *bq)
75dd84a43cSPoul-Henning Kamp {
76dd84a43cSPoul-Henning Kamp 
77dd84a43cSPoul-Henning Kamp 	mtx_lock(&bq->bio_queue_lock);
78dd84a43cSPoul-Henning Kamp }
79dd84a43cSPoul-Henning Kamp 
80dd84a43cSPoul-Henning Kamp static void
81dd84a43cSPoul-Henning Kamp g_bioq_unlock(struct g_bioq *bq)
82dd84a43cSPoul-Henning Kamp {
83dd84a43cSPoul-Henning Kamp 
84dd84a43cSPoul-Henning Kamp 	mtx_unlock(&bq->bio_queue_lock);
85dd84a43cSPoul-Henning Kamp }
86dd84a43cSPoul-Henning Kamp 
87dd84a43cSPoul-Henning Kamp #if 0
88dd84a43cSPoul-Henning Kamp static void
89dd84a43cSPoul-Henning Kamp g_bioq_destroy(struct g_bioq *bq)
90dd84a43cSPoul-Henning Kamp {
91dd84a43cSPoul-Henning Kamp 
92dd84a43cSPoul-Henning Kamp 	mtx_destroy(&bq->bio_queue_lock);
93dd84a43cSPoul-Henning Kamp }
94dd84a43cSPoul-Henning Kamp #endif
95dd84a43cSPoul-Henning Kamp 
96dd84a43cSPoul-Henning Kamp static void
97dd84a43cSPoul-Henning Kamp g_bioq_init(struct g_bioq *bq)
98dd84a43cSPoul-Henning Kamp {
99dd84a43cSPoul-Henning Kamp 
100dd84a43cSPoul-Henning Kamp 	TAILQ_INIT(&bq->bio_queue);
1016008862bSJohn Baldwin 	mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF);
102dd84a43cSPoul-Henning Kamp }
103dd84a43cSPoul-Henning Kamp 
104dd84a43cSPoul-Henning Kamp static struct bio *
105dd84a43cSPoul-Henning Kamp g_bioq_first(struct g_bioq *bq)
106dd84a43cSPoul-Henning Kamp {
107dd84a43cSPoul-Henning Kamp 	struct bio *bp;
108dd84a43cSPoul-Henning Kamp 
109dd84a43cSPoul-Henning Kamp 	bp = TAILQ_FIRST(&bq->bio_queue);
110dd84a43cSPoul-Henning Kamp 	if (bp != NULL) {
111dcbd0fe5SPoul-Henning Kamp 		KASSERT((bp->bio_flags & BIO_ONQUEUE),
112dcbd0fe5SPoul-Henning Kamp 		    ("Bio not on queue bp=%p target %p", bp, bq));
113dcbd0fe5SPoul-Henning Kamp 		bp->bio_flags &= ~BIO_ONQUEUE;
114dd84a43cSPoul-Henning Kamp 		TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue);
115dd84a43cSPoul-Henning Kamp 		bq->bio_queue_length--;
116dd84a43cSPoul-Henning Kamp 	}
117dd84a43cSPoul-Henning Kamp 	return (bp);
118dd84a43cSPoul-Henning Kamp }
119dd84a43cSPoul-Henning Kamp 
120dd84a43cSPoul-Henning Kamp struct bio *
121dd84a43cSPoul-Henning Kamp g_new_bio(void)
122dd84a43cSPoul-Henning Kamp {
123dd84a43cSPoul-Henning Kamp 	struct bio *bp;
124dd84a43cSPoul-Henning Kamp 
1255ffb2c8bSPoul-Henning Kamp 	bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
1263b378147SPawel Jakub Dawidek #ifdef KTR
127b656c1b8SPawel Jakub Dawidek 	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
1283b378147SPawel Jakub Dawidek 		struct stack st;
1293b378147SPawel Jakub Dawidek 
1303b378147SPawel Jakub Dawidek 		CTR1(KTR_GEOM, "g_new_bio(): %p", bp);
1313b378147SPawel Jakub Dawidek 		stack_save(&st);
1323b378147SPawel Jakub Dawidek 		CTRSTACK(KTR_GEOM, &st, 3, 0);
1333b378147SPawel Jakub Dawidek 	}
1343b378147SPawel Jakub Dawidek #endif
135dd84a43cSPoul-Henning Kamp 	return (bp);
136dd84a43cSPoul-Henning Kamp }
137dd84a43cSPoul-Henning Kamp 
138a2033c96SPoul-Henning Kamp struct bio *
139a2033c96SPoul-Henning Kamp g_alloc_bio(void)
140a2033c96SPoul-Henning Kamp {
141a2033c96SPoul-Henning Kamp 	struct bio *bp;
142a2033c96SPoul-Henning Kamp 
143a2033c96SPoul-Henning Kamp 	bp = uma_zalloc(biozone, M_WAITOK | M_ZERO);
1443b378147SPawel Jakub Dawidek #ifdef KTR
145b656c1b8SPawel Jakub Dawidek 	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
1463b378147SPawel Jakub Dawidek 		struct stack st;
1473b378147SPawel Jakub Dawidek 
1483b378147SPawel Jakub Dawidek 		CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp);
1493b378147SPawel Jakub Dawidek 		stack_save(&st);
1503b378147SPawel Jakub Dawidek 		CTRSTACK(KTR_GEOM, &st, 3, 0);
1513b378147SPawel Jakub Dawidek 	}
1523b378147SPawel Jakub Dawidek #endif
153a2033c96SPoul-Henning Kamp 	return (bp);
154a2033c96SPoul-Henning Kamp }
155a2033c96SPoul-Henning Kamp 
156dd84a43cSPoul-Henning Kamp void
157dd84a43cSPoul-Henning Kamp g_destroy_bio(struct bio *bp)
158dd84a43cSPoul-Henning Kamp {
1593b378147SPawel Jakub Dawidek #ifdef KTR
160b656c1b8SPawel Jakub Dawidek 	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
1613b378147SPawel Jakub Dawidek 		struct stack st;
162dd84a43cSPoul-Henning Kamp 
1633b378147SPawel Jakub Dawidek 		CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp);
1643b378147SPawel Jakub Dawidek 		stack_save(&st);
1653b378147SPawel Jakub Dawidek 		CTRSTACK(KTR_GEOM, &st, 3, 0);
1663b378147SPawel Jakub Dawidek 	}
1673b378147SPawel Jakub Dawidek #endif
1685ffb2c8bSPoul-Henning Kamp 	uma_zfree(biozone, bp);
169dd84a43cSPoul-Henning Kamp }
170dd84a43cSPoul-Henning Kamp 
171dd84a43cSPoul-Henning Kamp struct bio *
172dd84a43cSPoul-Henning Kamp g_clone_bio(struct bio *bp)
173dd84a43cSPoul-Henning Kamp {
174dd84a43cSPoul-Henning Kamp 	struct bio *bp2;
175dd84a43cSPoul-Henning Kamp 
1765ffb2c8bSPoul-Henning Kamp 	bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
177a1bd3ee2SPoul-Henning Kamp 	if (bp2 != NULL) {
178936cc461SPoul-Henning Kamp 		bp2->bio_parent = bp;
179dd84a43cSPoul-Henning Kamp 		bp2->bio_cmd = bp->bio_cmd;
180dd84a43cSPoul-Henning Kamp 		bp2->bio_length = bp->bio_length;
181dd84a43cSPoul-Henning Kamp 		bp2->bio_offset = bp->bio_offset;
182dd84a43cSPoul-Henning Kamp 		bp2->bio_data = bp->bio_data;
183dd84a43cSPoul-Henning Kamp 		bp2->bio_attribute = bp->bio_attribute;
1846231f75bSLuigi Rizzo 		/* Inherit classification info from the parent */
1856231f75bSLuigi Rizzo 		bp2->bio_classifier1 = bp->bio_classifier1;
1866231f75bSLuigi Rizzo 		bp2->bio_classifier2 = bp->bio_classifier2;
187801bb689SPoul-Henning Kamp 		bp->bio_children++;
188a1bd3ee2SPoul-Henning Kamp 	}
1893b378147SPawel Jakub Dawidek #ifdef KTR
190b656c1b8SPawel Jakub Dawidek 	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
1913b378147SPawel Jakub Dawidek 		struct stack st;
1923b378147SPawel Jakub Dawidek 
193ad572235SRuslan Ermilov 		CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2);
1943b378147SPawel Jakub Dawidek 		stack_save(&st);
1953b378147SPawel Jakub Dawidek 		CTRSTACK(KTR_GEOM, &st, 3, 0);
1963b378147SPawel Jakub Dawidek 	}
1973b378147SPawel Jakub Dawidek #endif
198dd84a43cSPoul-Henning Kamp 	return(bp2);
199dd84a43cSPoul-Henning Kamp }
200dd84a43cSPoul-Henning Kamp 
2014bec0ff1SPawel Jakub Dawidek struct bio *
2024bec0ff1SPawel Jakub Dawidek g_duplicate_bio(struct bio *bp)
2034bec0ff1SPawel Jakub Dawidek {
2044bec0ff1SPawel Jakub Dawidek 	struct bio *bp2;
2054bec0ff1SPawel Jakub Dawidek 
2064bec0ff1SPawel Jakub Dawidek 	bp2 = uma_zalloc(biozone, M_WAITOK | M_ZERO);
2074bec0ff1SPawel Jakub Dawidek 	bp2->bio_parent = bp;
2084bec0ff1SPawel Jakub Dawidek 	bp2->bio_cmd = bp->bio_cmd;
2094bec0ff1SPawel Jakub Dawidek 	bp2->bio_length = bp->bio_length;
2104bec0ff1SPawel Jakub Dawidek 	bp2->bio_offset = bp->bio_offset;
2114bec0ff1SPawel Jakub Dawidek 	bp2->bio_data = bp->bio_data;
2124bec0ff1SPawel Jakub Dawidek 	bp2->bio_attribute = bp->bio_attribute;
2134bec0ff1SPawel Jakub Dawidek 	bp->bio_children++;
2144bec0ff1SPawel Jakub Dawidek #ifdef KTR
215b656c1b8SPawel Jakub Dawidek 	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
2164bec0ff1SPawel Jakub Dawidek 		struct stack st;
2174bec0ff1SPawel Jakub Dawidek 
2184bec0ff1SPawel Jakub Dawidek 		CTR2(KTR_GEOM, "g_duplicate_bio(%p): %p", bp, bp2);
2194bec0ff1SPawel Jakub Dawidek 		stack_save(&st);
2204bec0ff1SPawel Jakub Dawidek 		CTRSTACK(KTR_GEOM, &st, 3, 0);
2214bec0ff1SPawel Jakub Dawidek 	}
2224bec0ff1SPawel Jakub Dawidek #endif
2234bec0ff1SPawel Jakub Dawidek 	return(bp2);
2244bec0ff1SPawel Jakub Dawidek }
2254bec0ff1SPawel Jakub Dawidek 
226dd84a43cSPoul-Henning Kamp void
227dd84a43cSPoul-Henning Kamp g_io_init()
228dd84a43cSPoul-Henning Kamp {
229dd84a43cSPoul-Henning Kamp 
230dd84a43cSPoul-Henning Kamp 	g_bioq_init(&g_bio_run_down);
231dd84a43cSPoul-Henning Kamp 	g_bioq_init(&g_bio_run_up);
2325fcf4e43SPoul-Henning Kamp 	g_bioq_init(&g_bio_run_task);
2335ffb2c8bSPoul-Henning Kamp 	biozone = uma_zcreate("g_bio", sizeof (struct bio),
2345ffb2c8bSPoul-Henning Kamp 	    NULL, NULL,
2355ffb2c8bSPoul-Henning Kamp 	    NULL, NULL,
2365ffb2c8bSPoul-Henning Kamp 	    0, 0);
237dd84a43cSPoul-Henning Kamp }
238dd84a43cSPoul-Henning Kamp 
239dd84a43cSPoul-Henning Kamp int
2400d3f37a8SPoul-Henning Kamp g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr)
241dd84a43cSPoul-Henning Kamp {
242dd84a43cSPoul-Henning Kamp 	struct bio *bp;
243dd84a43cSPoul-Henning Kamp 	int error;
244dd84a43cSPoul-Henning Kamp 
245dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO, "bio_getattr(%s)", attr);
246a2033c96SPoul-Henning Kamp 	bp = g_alloc_bio();
247dd84a43cSPoul-Henning Kamp 	bp->bio_cmd = BIO_GETATTR;
248dd84a43cSPoul-Henning Kamp 	bp->bio_done = NULL;
249dd84a43cSPoul-Henning Kamp 	bp->bio_attribute = attr;
250dd84a43cSPoul-Henning Kamp 	bp->bio_length = *len;
251dd84a43cSPoul-Henning Kamp 	bp->bio_data = ptr;
252dd84a43cSPoul-Henning Kamp 	g_io_request(bp, cp);
25353706245SPoul-Henning Kamp 	error = biowait(bp, "ggetattr");
254dd84a43cSPoul-Henning Kamp 	*len = bp->bio_completed;
255dd84a43cSPoul-Henning Kamp 	g_destroy_bio(bp);
256dd84a43cSPoul-Henning Kamp 	return (error);
257dd84a43cSPoul-Henning Kamp }
258dd84a43cSPoul-Henning Kamp 
259c3618c65SPawel Jakub Dawidek int
260c3618c65SPawel Jakub Dawidek g_io_flush(struct g_consumer *cp)
261c3618c65SPawel Jakub Dawidek {
262c3618c65SPawel Jakub Dawidek 	struct bio *bp;
263c3618c65SPawel Jakub Dawidek 	int error;
264c3618c65SPawel Jakub Dawidek 
265c3618c65SPawel Jakub Dawidek 	g_trace(G_T_BIO, "bio_flush(%s)", cp->provider->name);
266c3618c65SPawel Jakub Dawidek 	bp = g_alloc_bio();
267c3618c65SPawel Jakub Dawidek 	bp->bio_cmd = BIO_FLUSH;
268f03f7a0cSJustin T. Gibbs 	bp->bio_flags |= BIO_ORDERED;
269c3618c65SPawel Jakub Dawidek 	bp->bio_done = NULL;
270c3618c65SPawel Jakub Dawidek 	bp->bio_attribute = NULL;
271c3618c65SPawel Jakub Dawidek 	bp->bio_offset = cp->provider->mediasize;
272c3618c65SPawel Jakub Dawidek 	bp->bio_length = 0;
273c3618c65SPawel Jakub Dawidek 	bp->bio_data = NULL;
274c3618c65SPawel Jakub Dawidek 	g_io_request(bp, cp);
275c3618c65SPawel Jakub Dawidek 	error = biowait(bp, "gflush");
276c3618c65SPawel Jakub Dawidek 	g_destroy_bio(bp);
277c3618c65SPawel Jakub Dawidek 	return (error);
278c3618c65SPawel Jakub Dawidek }
279c3618c65SPawel Jakub Dawidek 
280e39d70d4SPoul-Henning Kamp static int
281e39d70d4SPoul-Henning Kamp g_io_check(struct bio *bp)
282e39d70d4SPoul-Henning Kamp {
283e39d70d4SPoul-Henning Kamp 	struct g_consumer *cp;
284e39d70d4SPoul-Henning Kamp 	struct g_provider *pp;
285e39d70d4SPoul-Henning Kamp 
286e39d70d4SPoul-Henning Kamp 	cp = bp->bio_from;
287e39d70d4SPoul-Henning Kamp 	pp = bp->bio_to;
288e39d70d4SPoul-Henning Kamp 
289e39d70d4SPoul-Henning Kamp 	/* Fail if access counters dont allow the operation */
290e39d70d4SPoul-Henning Kamp 	switch(bp->bio_cmd) {
291e39d70d4SPoul-Henning Kamp 	case BIO_READ:
292e39d70d4SPoul-Henning Kamp 	case BIO_GETATTR:
293e39d70d4SPoul-Henning Kamp 		if (cp->acr == 0)
294e39d70d4SPoul-Henning Kamp 			return (EPERM);
295e39d70d4SPoul-Henning Kamp 		break;
296e39d70d4SPoul-Henning Kamp 	case BIO_WRITE:
297e39d70d4SPoul-Henning Kamp 	case BIO_DELETE:
298c3618c65SPawel Jakub Dawidek 	case BIO_FLUSH:
299e39d70d4SPoul-Henning Kamp 		if (cp->acw == 0)
300e39d70d4SPoul-Henning Kamp 			return (EPERM);
301e39d70d4SPoul-Henning Kamp 		break;
302e39d70d4SPoul-Henning Kamp 	default:
303e39d70d4SPoul-Henning Kamp 		return (EPERM);
304e39d70d4SPoul-Henning Kamp 	}
305e39d70d4SPoul-Henning Kamp 	/* if provider is marked for error, don't disturb. */
306e39d70d4SPoul-Henning Kamp 	if (pp->error)
307e39d70d4SPoul-Henning Kamp 		return (pp->error);
308*3631c638SAlexander Motin 	if (cp->flags & G_CF_ORPHAN)
309*3631c638SAlexander Motin 		return (ENXIO);
310e39d70d4SPoul-Henning Kamp 
311e39d70d4SPoul-Henning Kamp 	switch(bp->bio_cmd) {
312e39d70d4SPoul-Henning Kamp 	case BIO_READ:
313e39d70d4SPoul-Henning Kamp 	case BIO_WRITE:
314e39d70d4SPoul-Henning Kamp 	case BIO_DELETE:
3152a842317SAndriy Gapon 		/* Zero sectorsize or mediasize is probably a lack of media. */
3162a842317SAndriy Gapon 		if (pp->sectorsize == 0 || pp->mediasize == 0)
31743bff1a7SPoul-Henning Kamp 			return (ENXIO);
318e39d70d4SPoul-Henning Kamp 		/* Reject I/O not on sector boundary */
319e39d70d4SPoul-Henning Kamp 		if (bp->bio_offset % pp->sectorsize)
320e39d70d4SPoul-Henning Kamp 			return (EINVAL);
321e39d70d4SPoul-Henning Kamp 		/* Reject I/O not integral sector long */
322e39d70d4SPoul-Henning Kamp 		if (bp->bio_length % pp->sectorsize)
323e39d70d4SPoul-Henning Kamp 			return (EINVAL);
324d1b8bf47SPoul-Henning Kamp 		/* Reject requests before or past the end of media. */
325d1b8bf47SPoul-Henning Kamp 		if (bp->bio_offset < 0)
326d1b8bf47SPoul-Henning Kamp 			return (EIO);
327e39d70d4SPoul-Henning Kamp 		if (bp->bio_offset > pp->mediasize)
328e39d70d4SPoul-Henning Kamp 			return (EIO);
329e39d70d4SPoul-Henning Kamp 		break;
330e39d70d4SPoul-Henning Kamp 	default:
331e39d70d4SPoul-Henning Kamp 		break;
332e39d70d4SPoul-Henning Kamp 	}
333e39d70d4SPoul-Henning Kamp 	return (0);
334e39d70d4SPoul-Henning Kamp }
335e39d70d4SPoul-Henning Kamp 
3366231f75bSLuigi Rizzo /*
3376231f75bSLuigi Rizzo  * bio classification support.
3386231f75bSLuigi Rizzo  *
3396231f75bSLuigi Rizzo  * g_register_classifier() and g_unregister_classifier()
3406231f75bSLuigi Rizzo  * are used to add/remove a classifier from the list.
3416231f75bSLuigi Rizzo  * The list is protected using the g_bio_run_down lock,
3426231f75bSLuigi Rizzo  * because the classifiers are called in this path.
3436231f75bSLuigi Rizzo  *
3446231f75bSLuigi Rizzo  * g_io_request() passes bio's that are not already classified
3456231f75bSLuigi Rizzo  * (i.e. those with bio_classifier1 == NULL) to g_run_classifiers().
3466231f75bSLuigi Rizzo  * Classifiers can store their result in the two fields
3476231f75bSLuigi Rizzo  * bio_classifier1 and bio_classifier2.
3486231f75bSLuigi Rizzo  * A classifier that updates one of the fields should
3496231f75bSLuigi Rizzo  * return a non-zero value.
3506231f75bSLuigi Rizzo  * If no classifier updates the field, g_run_classifiers() sets
3516231f75bSLuigi Rizzo  * bio_classifier1 = BIO_NOTCLASSIFIED to avoid further calls.
3526231f75bSLuigi Rizzo  */
3536231f75bSLuigi Rizzo 
3546231f75bSLuigi Rizzo int
3556231f75bSLuigi Rizzo g_register_classifier(struct g_classifier_hook *hook)
3566231f75bSLuigi Rizzo {
3576231f75bSLuigi Rizzo 
3586231f75bSLuigi Rizzo 	g_bioq_lock(&g_bio_run_down);
3596231f75bSLuigi Rizzo 	TAILQ_INSERT_TAIL(&g_classifier_tailq, hook, link);
3606231f75bSLuigi Rizzo 	g_bioq_unlock(&g_bio_run_down);
3616231f75bSLuigi Rizzo 
3626231f75bSLuigi Rizzo 	return (0);
3636231f75bSLuigi Rizzo }
3646231f75bSLuigi Rizzo 
3656231f75bSLuigi Rizzo void
3666231f75bSLuigi Rizzo g_unregister_classifier(struct g_classifier_hook *hook)
3676231f75bSLuigi Rizzo {
3686231f75bSLuigi Rizzo 	struct g_classifier_hook *entry;
3696231f75bSLuigi Rizzo 
3706231f75bSLuigi Rizzo 	g_bioq_lock(&g_bio_run_down);
3716231f75bSLuigi Rizzo 	TAILQ_FOREACH(entry, &g_classifier_tailq, link) {
3726231f75bSLuigi Rizzo 		if (entry == hook) {
3736231f75bSLuigi Rizzo 			TAILQ_REMOVE(&g_classifier_tailq, hook, link);
3746231f75bSLuigi Rizzo 			break;
3756231f75bSLuigi Rizzo 		}
3766231f75bSLuigi Rizzo 	}
3776231f75bSLuigi Rizzo 	g_bioq_unlock(&g_bio_run_down);
3786231f75bSLuigi Rizzo }
3796231f75bSLuigi Rizzo 
3806231f75bSLuigi Rizzo static void
3816231f75bSLuigi Rizzo g_run_classifiers(struct bio *bp)
3826231f75bSLuigi Rizzo {
3836231f75bSLuigi Rizzo 	struct g_classifier_hook *hook;
3846231f75bSLuigi Rizzo 	int classified = 0;
3856231f75bSLuigi Rizzo 
3866231f75bSLuigi Rizzo 	TAILQ_FOREACH(hook, &g_classifier_tailq, link)
3876231f75bSLuigi Rizzo 		classified |= hook->func(hook->arg, bp);
3886231f75bSLuigi Rizzo 
3896231f75bSLuigi Rizzo 	if (!classified)
3906231f75bSLuigi Rizzo 		bp->bio_classifier1 = BIO_NOTCLASSIFIED;
3916231f75bSLuigi Rizzo }
3926231f75bSLuigi Rizzo 
393dd84a43cSPoul-Henning Kamp void
394dd84a43cSPoul-Henning Kamp g_io_request(struct bio *bp, struct g_consumer *cp)
395dd84a43cSPoul-Henning Kamp {
396801bb689SPoul-Henning Kamp 	struct g_provider *pp;
3970d883b11SAlexander Motin 	int first;
398dd84a43cSPoul-Henning Kamp 
399d0e17c1bSPoul-Henning Kamp 	KASSERT(cp != NULL, ("NULL cp in g_io_request"));
400d0e17c1bSPoul-Henning Kamp 	KASSERT(bp != NULL, ("NULL bp in g_io_request"));
401e060b6bdSPoul-Henning Kamp 	pp = cp->provider;
402801bb689SPoul-Henning Kamp 	KASSERT(pp != NULL, ("consumer not attached in g_io_request"));
40392ee312dSPawel Jakub Dawidek #ifdef DIAGNOSTIC
40492ee312dSPawel Jakub Dawidek 	KASSERT(bp->bio_driver1 == NULL,
40592ee312dSPawel Jakub Dawidek 	    ("bio_driver1 used by the consumer (geom %s)", cp->geom->name));
40692ee312dSPawel Jakub Dawidek 	KASSERT(bp->bio_driver2 == NULL,
40792ee312dSPawel Jakub Dawidek 	    ("bio_driver2 used by the consumer (geom %s)", cp->geom->name));
40892ee312dSPawel Jakub Dawidek 	KASSERT(bp->bio_pflags == 0,
40992ee312dSPawel Jakub Dawidek 	    ("bio_pflags used by the consumer (geom %s)", cp->geom->name));
41092ee312dSPawel Jakub Dawidek 	/*
41192ee312dSPawel Jakub Dawidek 	 * Remember consumer's private fields, so we can detect if they were
41292ee312dSPawel Jakub Dawidek 	 * modified by the provider.
41392ee312dSPawel Jakub Dawidek 	 */
41492ee312dSPawel Jakub Dawidek 	bp->_bio_caller1 = bp->bio_caller1;
41592ee312dSPawel Jakub Dawidek 	bp->_bio_caller2 = bp->bio_caller2;
41692ee312dSPawel Jakub Dawidek 	bp->_bio_cflags = bp->bio_cflags;
41792ee312dSPawel Jakub Dawidek #endif
418801bb689SPoul-Henning Kamp 
4191ded77b2SPawel Jakub Dawidek 	if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_GETATTR)) {
420c3618c65SPawel Jakub Dawidek 		KASSERT(bp->bio_data != NULL,
4211ded77b2SPawel Jakub Dawidek 		    ("NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd));
4221ded77b2SPawel Jakub Dawidek 	}
4231ded77b2SPawel Jakub Dawidek 	if (bp->bio_cmd & (BIO_DELETE|BIO_FLUSH)) {
4241ded77b2SPawel Jakub Dawidek 		KASSERT(bp->bio_data == NULL,
4251ded77b2SPawel Jakub Dawidek 		    ("non-NULL bp->data in g_io_request(cmd=%hhu)",
4261ded77b2SPawel Jakub Dawidek 		    bp->bio_cmd));
427c3618c65SPawel Jakub Dawidek 	}
428dcbd0fe5SPoul-Henning Kamp 	if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) {
429dcbd0fe5SPoul-Henning Kamp 		KASSERT(bp->bio_offset % cp->provider->sectorsize == 0,
430dcbd0fe5SPoul-Henning Kamp 		    ("wrong offset %jd for sectorsize %u",
431dcbd0fe5SPoul-Henning Kamp 		    bp->bio_offset, cp->provider->sectorsize));
432dcbd0fe5SPoul-Henning Kamp 		KASSERT(bp->bio_length % cp->provider->sectorsize == 0,
433dcbd0fe5SPoul-Henning Kamp 		    ("wrong length %jd for sectorsize %u",
434dcbd0fe5SPoul-Henning Kamp 		    bp->bio_length, cp->provider->sectorsize));
435dcbd0fe5SPoul-Henning Kamp 	}
436dcbd0fe5SPoul-Henning Kamp 
437f7717523SStephan Uphoff 	g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d",
438f7717523SStephan Uphoff 	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd);
439f7717523SStephan Uphoff 
440dd84a43cSPoul-Henning Kamp 	bp->bio_from = cp;
441801bb689SPoul-Henning Kamp 	bp->bio_to = pp;
4422fccec19SPoul-Henning Kamp 	bp->bio_error = 0;
4432fccec19SPoul-Henning Kamp 	bp->bio_completed = 0;
444dd84a43cSPoul-Henning Kamp 
44519fa21aaSPoul-Henning Kamp 	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
44619fa21aaSPoul-Henning Kamp 	    ("Bio already on queue bp=%p", bp));
44719fa21aaSPoul-Henning Kamp 	bp->bio_flags |= BIO_ONQUEUE;
44819fa21aaSPoul-Henning Kamp 
449a5be8eb5SAlexander Motin 	if (g_collectstats)
45019fa21aaSPoul-Henning Kamp 		binuptime(&bp->bio_t0);
451a5be8eb5SAlexander Motin 	else
452a5be8eb5SAlexander Motin 		getbinuptime(&bp->bio_t0);
4538827c821SPoul-Henning Kamp 
4548827c821SPoul-Henning Kamp 	/*
4558827c821SPoul-Henning Kamp 	 * The statistics collection is lockless, as such, but we
4568827c821SPoul-Henning Kamp 	 * can not update one instance of the statistics from more
4578827c821SPoul-Henning Kamp 	 * than one thread at a time, so grab the lock first.
4586231f75bSLuigi Rizzo 	 *
4596231f75bSLuigi Rizzo 	 * We also use the lock to protect the list of classifiers.
4608827c821SPoul-Henning Kamp 	 */
46119fa21aaSPoul-Henning Kamp 	g_bioq_lock(&g_bio_run_down);
4626231f75bSLuigi Rizzo 
4636231f75bSLuigi Rizzo 	if (!TAILQ_EMPTY(&g_classifier_tailq) && !bp->bio_classifier1)
4646231f75bSLuigi Rizzo 		g_run_classifiers(bp);
4656231f75bSLuigi Rizzo 
466cf457284SPoul-Henning Kamp 	if (g_collectstats & 1)
46719fa21aaSPoul-Henning Kamp 		devstat_start_transaction(pp->stat, &bp->bio_t0);
468cf457284SPoul-Henning Kamp 	if (g_collectstats & 2)
46919fa21aaSPoul-Henning Kamp 		devstat_start_transaction(cp->stat, &bp->bio_t0);
47019fa21aaSPoul-Henning Kamp 
47119fa21aaSPoul-Henning Kamp 	pp->nstart++;
472cf457284SPoul-Henning Kamp 	cp->nstart++;
4730d883b11SAlexander Motin 	first = TAILQ_EMPTY(&g_bio_run_down.bio_queue);
47419fa21aaSPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue);
47519fa21aaSPoul-Henning Kamp 	g_bio_run_down.bio_queue_length++;
47619fa21aaSPoul-Henning Kamp 	g_bioq_unlock(&g_bio_run_down);
477e39d70d4SPoul-Henning Kamp 
4782fccec19SPoul-Henning Kamp 	/* Pass it on down. */
4790d883b11SAlexander Motin 	if (first)
480dd84a43cSPoul-Henning Kamp 		wakeup(&g_wait_down);
481dd84a43cSPoul-Henning Kamp }
482dd84a43cSPoul-Henning Kamp 
483dd84a43cSPoul-Henning Kamp void
48472840432SPoul-Henning Kamp g_io_deliver(struct bio *bp, int error)
485dd84a43cSPoul-Henning Kamp {
486801bb689SPoul-Henning Kamp 	struct g_consumer *cp;
487801bb689SPoul-Henning Kamp 	struct g_provider *pp;
4880d883b11SAlexander Motin 	int first;
489dd84a43cSPoul-Henning Kamp 
490e060b6bdSPoul-Henning Kamp 	KASSERT(bp != NULL, ("NULL bp in g_io_deliver"));
491801bb689SPoul-Henning Kamp 	pp = bp->bio_to;
492f7eeab17SPoul-Henning Kamp 	KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver"));
493f7eeab17SPoul-Henning Kamp 	cp = bp->bio_from;
494f7eeab17SPoul-Henning Kamp 	if (cp == NULL) {
495f7eeab17SPoul-Henning Kamp 		bp->bio_error = error;
496f7eeab17SPoul-Henning Kamp 		bp->bio_done(bp);
497f7eeab17SPoul-Henning Kamp 		return;
498f7eeab17SPoul-Henning Kamp 	}
499801bb689SPoul-Henning Kamp 	KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver"));
500801bb689SPoul-Henning Kamp 	KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver"));
501fb231f36SEdward Tomasz Napierala #ifdef DIAGNOSTIC
502fb231f36SEdward Tomasz Napierala 	/*
503fb231f36SEdward Tomasz Napierala 	 * Some classes - GJournal in particular - can modify bio's
504fb231f36SEdward Tomasz Napierala 	 * private fields while the bio is in transit; G_GEOM_VOLATILE_BIO
505fb231f36SEdward Tomasz Napierala 	 * flag means it's an expected behaviour for that particular geom.
506fb231f36SEdward Tomasz Napierala 	 */
507fb231f36SEdward Tomasz Napierala 	if ((cp->geom->flags & G_GEOM_VOLATILE_BIO) == 0) {
508fb231f36SEdward Tomasz Napierala 		KASSERT(bp->bio_caller1 == bp->_bio_caller1,
509fb231f36SEdward Tomasz Napierala 		    ("bio_caller1 used by the provider %s", pp->name));
510fb231f36SEdward Tomasz Napierala 		KASSERT(bp->bio_caller2 == bp->_bio_caller2,
511fb231f36SEdward Tomasz Napierala 		    ("bio_caller2 used by the provider %s", pp->name));
512fb231f36SEdward Tomasz Napierala 		KASSERT(bp->bio_cflags == bp->_bio_cflags,
513fb231f36SEdward Tomasz Napierala 		    ("bio_cflags used by the provider %s", pp->name));
514fb231f36SEdward Tomasz Napierala 	}
515fb231f36SEdward Tomasz Napierala #endif
51646aeebecSPawel Jakub Dawidek 	KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0"));
51746aeebecSPawel Jakub Dawidek 	KASSERT(bp->bio_completed <= bp->bio_length,
51846aeebecSPawel Jakub Dawidek 	    ("bio_completed can't be greater than bio_length"));
5195ab413bfSPoul-Henning Kamp 
520dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO,
5210355b86eSPoul-Henning Kamp "g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd",
522801bb689SPoul-Henning Kamp 	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error,
5230355b86eSPoul-Henning Kamp 	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
524801bb689SPoul-Henning Kamp 
52519fa21aaSPoul-Henning Kamp 	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
52619fa21aaSPoul-Henning Kamp 	    ("Bio already on queue bp=%p", bp));
52719fa21aaSPoul-Henning Kamp 
528dcbd0fe5SPoul-Henning Kamp 	/*
529dcbd0fe5SPoul-Henning Kamp 	 * XXX: next two doesn't belong here
530dcbd0fe5SPoul-Henning Kamp 	 */
531e24cbd90SPoul-Henning Kamp 	bp->bio_bcount = bp->bio_length;
532e24cbd90SPoul-Henning Kamp 	bp->bio_resid = bp->bio_bcount - bp->bio_completed;
53319fa21aaSPoul-Henning Kamp 
5348827c821SPoul-Henning Kamp 	/*
5358827c821SPoul-Henning Kamp 	 * The statistics collection is lockless, as such, but we
5368827c821SPoul-Henning Kamp 	 * can not update one instance of the statistics from more
5378827c821SPoul-Henning Kamp 	 * than one thread at a time, so grab the lock first.
5388827c821SPoul-Henning Kamp 	 */
53919fa21aaSPoul-Henning Kamp 	g_bioq_lock(&g_bio_run_up);
540cf457284SPoul-Henning Kamp 	if (g_collectstats & 1)
541e24cbd90SPoul-Henning Kamp 		devstat_end_transaction_bio(pp->stat, bp);
542cf457284SPoul-Henning Kamp 	if (g_collectstats & 2)
543cf457284SPoul-Henning Kamp 		devstat_end_transaction_bio(cp->stat, bp);
5448827c821SPoul-Henning Kamp 
545c6ae9b5fSPoul-Henning Kamp 	cp->nend++;
546c6ae9b5fSPoul-Henning Kamp 	pp->nend++;
54719fa21aaSPoul-Henning Kamp 	if (error != ENOMEM) {
54819fa21aaSPoul-Henning Kamp 		bp->bio_error = error;
5490d883b11SAlexander Motin 		first = TAILQ_EMPTY(&g_bio_run_up.bio_queue);
55019fa21aaSPoul-Henning Kamp 		TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue);
551276f72c5SPoul-Henning Kamp 		bp->bio_flags |= BIO_ONQUEUE;
55219fa21aaSPoul-Henning Kamp 		g_bio_run_up.bio_queue_length++;
55319fa21aaSPoul-Henning Kamp 		g_bioq_unlock(&g_bio_run_up);
5540d883b11SAlexander Motin 		if (first)
55519fa21aaSPoul-Henning Kamp 			wakeup(&g_wait_up);
55619fa21aaSPoul-Henning Kamp 		return;
55719fa21aaSPoul-Henning Kamp 	}
55819fa21aaSPoul-Henning Kamp 	g_bioq_unlock(&g_bio_run_up);
559dd84a43cSPoul-Henning Kamp 
5602cc9686eSPoul-Henning Kamp 	if (bootverbose)
561801bb689SPoul-Henning Kamp 		printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name);
5621b949c05SPawel Jakub Dawidek 	bp->bio_children = 0;
5631b949c05SPawel Jakub Dawidek 	bp->bio_inbed = 0;
564801bb689SPoul-Henning Kamp 	g_io_request(bp, cp);
5653432e4fdSPoul-Henning Kamp 	pace++;
5663432e4fdSPoul-Henning Kamp 	return;
5673432e4fdSPoul-Henning Kamp }
568dd84a43cSPoul-Henning Kamp 
569dd84a43cSPoul-Henning Kamp void
570dd84a43cSPoul-Henning Kamp g_io_schedule_down(struct thread *tp __unused)
571dd84a43cSPoul-Henning Kamp {
572dd84a43cSPoul-Henning Kamp 	struct bio *bp;
573e39d70d4SPoul-Henning Kamp 	off_t excess;
574e39d70d4SPoul-Henning Kamp 	int error;
575dd84a43cSPoul-Henning Kamp 
576dd84a43cSPoul-Henning Kamp 	for(;;) {
577f0e185d7SPoul-Henning Kamp 		g_bioq_lock(&g_bio_run_down);
578dd84a43cSPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_down);
579f0e185d7SPoul-Henning Kamp 		if (bp == NULL) {
58049dbb61dSRobert Watson 			CTR0(KTR_GEOM, "g_down going to sleep");
581f0e185d7SPoul-Henning Kamp 			msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock,
5827fc019afSAlexander Motin 			    PRIBIO | PDROP, "-", 0);
583f0e185d7SPoul-Henning Kamp 			continue;
584f0e185d7SPoul-Henning Kamp 		}
58549dbb61dSRobert Watson 		CTR0(KTR_GEOM, "g_down has work to do");
586f0e185d7SPoul-Henning Kamp 		g_bioq_unlock(&g_bio_run_down);
587376ceb79SPoul-Henning Kamp 		if (pace > 0) {
58849dbb61dSRobert Watson 			CTR1(KTR_GEOM, "g_down pacing self (pace %d)", pace);
5894d70511aSJohn Baldwin 			pause("g_down", hz/10);
590376ceb79SPoul-Henning Kamp 			pace--;
591376ceb79SPoul-Henning Kamp 		}
592e39d70d4SPoul-Henning Kamp 		error = g_io_check(bp);
593e39d70d4SPoul-Henning Kamp 		if (error) {
59449dbb61dSRobert Watson 			CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider "
59549dbb61dSRobert Watson 			    "%s returned %d", bp, bp->bio_to->name, error);
596e39d70d4SPoul-Henning Kamp 			g_io_deliver(bp, error);
597e39d70d4SPoul-Henning Kamp 			continue;
598e39d70d4SPoul-Henning Kamp 		}
59949dbb61dSRobert Watson 		CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp,
60049dbb61dSRobert Watson 		    bp->bio_to->name);
601392d56b4SPoul-Henning Kamp 		switch (bp->bio_cmd) {
602392d56b4SPoul-Henning Kamp 		case BIO_READ:
603392d56b4SPoul-Henning Kamp 		case BIO_WRITE:
604392d56b4SPoul-Henning Kamp 		case BIO_DELETE:
605e39d70d4SPoul-Henning Kamp 			/* Truncate requests to the end of providers media. */
60649dbb61dSRobert Watson 			/*
60749dbb61dSRobert Watson 			 * XXX: What if we truncate because of offset being
60849dbb61dSRobert Watson 			 * bad, not length?
60949dbb61dSRobert Watson 			 */
610e39d70d4SPoul-Henning Kamp 			excess = bp->bio_offset + bp->bio_length;
611e39d70d4SPoul-Henning Kamp 			if (excess > bp->bio_to->mediasize) {
612e39d70d4SPoul-Henning Kamp 				excess -= bp->bio_to->mediasize;
613e39d70d4SPoul-Henning Kamp 				bp->bio_length -= excess;
61449dbb61dSRobert Watson 				if (excess > 0)
61549dbb61dSRobert Watson 					CTR3(KTR_GEOM, "g_down truncated bio "
61649dbb61dSRobert Watson 					    "%p provider %s by %d", bp,
61749dbb61dSRobert Watson 					    bp->bio_to->name, excess);
618e39d70d4SPoul-Henning Kamp 			}
619e39d70d4SPoul-Henning Kamp 			/* Deliver zero length transfers right here. */
620e39d70d4SPoul-Henning Kamp 			if (bp->bio_length == 0) {
621e39d70d4SPoul-Henning Kamp 				g_io_deliver(bp, 0);
62249dbb61dSRobert Watson 				CTR2(KTR_GEOM, "g_down terminated 0-length "
62349dbb61dSRobert Watson 				    "bp %p provider %s", bp, bp->bio_to->name);
624e39d70d4SPoul-Henning Kamp 				continue;
625e39d70d4SPoul-Henning Kamp 			}
626392d56b4SPoul-Henning Kamp 			break;
627392d56b4SPoul-Henning Kamp 		default:
628392d56b4SPoul-Henning Kamp 			break;
629392d56b4SPoul-Henning Kamp 		}
63051460da8SJohn Baldwin 		THREAD_NO_SLEEPING();
63149dbb61dSRobert Watson 		CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld "
63249dbb61dSRobert Watson 		    "len %ld", bp, bp->bio_to->name, bp->bio_offset,
63349dbb61dSRobert Watson 		    bp->bio_length);
634dd84a43cSPoul-Henning Kamp 		bp->bio_to->geom->start(bp);
63551460da8SJohn Baldwin 		THREAD_SLEEPING_OK();
636dd84a43cSPoul-Henning Kamp 	}
637dd84a43cSPoul-Henning Kamp }
638dd84a43cSPoul-Henning Kamp 
639dd84a43cSPoul-Henning Kamp void
6405fcf4e43SPoul-Henning Kamp bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg)
6415fcf4e43SPoul-Henning Kamp {
6425fcf4e43SPoul-Henning Kamp 	bp->bio_task = func;
6435fcf4e43SPoul-Henning Kamp 	bp->bio_task_arg = arg;
6445fcf4e43SPoul-Henning Kamp 	/*
6455fcf4e43SPoul-Henning Kamp 	 * The taskqueue is actually just a second queue off the "up"
6465fcf4e43SPoul-Henning Kamp 	 * queue, so we use the same lock.
6475fcf4e43SPoul-Henning Kamp 	 */
6485fcf4e43SPoul-Henning Kamp 	g_bioq_lock(&g_bio_run_up);
649dcbd0fe5SPoul-Henning Kamp 	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
650dcbd0fe5SPoul-Henning Kamp 	    ("Bio already on queue bp=%p target taskq", bp));
651dcbd0fe5SPoul-Henning Kamp 	bp->bio_flags |= BIO_ONQUEUE;
6525fcf4e43SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue);
6535fcf4e43SPoul-Henning Kamp 	g_bio_run_task.bio_queue_length++;
6545fcf4e43SPoul-Henning Kamp 	wakeup(&g_wait_up);
6555fcf4e43SPoul-Henning Kamp 	g_bioq_unlock(&g_bio_run_up);
6565fcf4e43SPoul-Henning Kamp }
6575fcf4e43SPoul-Henning Kamp 
6585fcf4e43SPoul-Henning Kamp 
6595fcf4e43SPoul-Henning Kamp void
660dd84a43cSPoul-Henning Kamp g_io_schedule_up(struct thread *tp __unused)
661dd84a43cSPoul-Henning Kamp {
662dd84a43cSPoul-Henning Kamp 	struct bio *bp;
663dd84a43cSPoul-Henning Kamp 	for(;;) {
664f0e185d7SPoul-Henning Kamp 		g_bioq_lock(&g_bio_run_up);
6655fcf4e43SPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_task);
6665fcf4e43SPoul-Henning Kamp 		if (bp != NULL) {
6675fcf4e43SPoul-Henning Kamp 			g_bioq_unlock(&g_bio_run_up);
66851460da8SJohn Baldwin 			THREAD_NO_SLEEPING();
66949dbb61dSRobert Watson 			CTR1(KTR_GEOM, "g_up processing task bp %p", bp);
6705fcf4e43SPoul-Henning Kamp 			bp->bio_task(bp->bio_task_arg);
67151460da8SJohn Baldwin 			THREAD_SLEEPING_OK();
6725fcf4e43SPoul-Henning Kamp 			continue;
6735fcf4e43SPoul-Henning Kamp 		}
674dd84a43cSPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_up);
675f0e185d7SPoul-Henning Kamp 		if (bp != NULL) {
676f0e185d7SPoul-Henning Kamp 			g_bioq_unlock(&g_bio_run_up);
67751460da8SJohn Baldwin 			THREAD_NO_SLEEPING();
67849dbb61dSRobert Watson 			CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off "
679c4901b67SSean Bruno 			    "%jd len %ld", bp, bp->bio_to->name,
68049dbb61dSRobert Watson 			    bp->bio_offset, bp->bio_length);
68153706245SPoul-Henning Kamp 			biodone(bp);
68251460da8SJohn Baldwin 			THREAD_SLEEPING_OK();
683f0e185d7SPoul-Henning Kamp 			continue;
684f0e185d7SPoul-Henning Kamp 		}
68549dbb61dSRobert Watson 		CTR0(KTR_GEOM, "g_up going to sleep");
686f0e185d7SPoul-Henning Kamp 		msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock,
6877fc019afSAlexander Motin 		    PRIBIO | PDROP, "-", 0);
688dd84a43cSPoul-Henning Kamp 	}
689dd84a43cSPoul-Henning Kamp }
690dd84a43cSPoul-Henning Kamp 
691dd84a43cSPoul-Henning Kamp void *
692dd84a43cSPoul-Henning Kamp g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error)
693dd84a43cSPoul-Henning Kamp {
694dd84a43cSPoul-Henning Kamp 	struct bio *bp;
695dd84a43cSPoul-Henning Kamp 	void *ptr;
696dd84a43cSPoul-Henning Kamp 	int errorc;
697dd84a43cSPoul-Henning Kamp 
6988dd5480dSPawel Jakub Dawidek 	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
6998dd5480dSPawel Jakub Dawidek 	    length <= MAXPHYS, ("g_read_data(): invalid length %jd",
7008dd5480dSPawel Jakub Dawidek 	    (intmax_t)length));
7013eb6ffdfSPoul-Henning Kamp 
702a2033c96SPoul-Henning Kamp 	bp = g_alloc_bio();
703dd84a43cSPoul-Henning Kamp 	bp->bio_cmd = BIO_READ;
704dd84a43cSPoul-Henning Kamp 	bp->bio_done = NULL;
705dd84a43cSPoul-Henning Kamp 	bp->bio_offset = offset;
706dd84a43cSPoul-Henning Kamp 	bp->bio_length = length;
707a163d034SWarner Losh 	ptr = g_malloc(length, M_WAITOK);
708dd84a43cSPoul-Henning Kamp 	bp->bio_data = ptr;
709dd84a43cSPoul-Henning Kamp 	g_io_request(bp, cp);
71053706245SPoul-Henning Kamp 	errorc = biowait(bp, "gread");
711dd84a43cSPoul-Henning Kamp 	if (error != NULL)
712dd84a43cSPoul-Henning Kamp 		*error = errorc;
713dd84a43cSPoul-Henning Kamp 	g_destroy_bio(bp);
714dd84a43cSPoul-Henning Kamp 	if (errorc) {
715dd84a43cSPoul-Henning Kamp 		g_free(ptr);
716dd84a43cSPoul-Henning Kamp 		ptr = NULL;
717dd84a43cSPoul-Henning Kamp 	}
718dd84a43cSPoul-Henning Kamp 	return (ptr);
719dd84a43cSPoul-Henning Kamp }
72090b1cd56SPoul-Henning Kamp 
72190b1cd56SPoul-Henning Kamp int
72290b1cd56SPoul-Henning Kamp g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length)
72390b1cd56SPoul-Henning Kamp {
72490b1cd56SPoul-Henning Kamp 	struct bio *bp;
72590b1cd56SPoul-Henning Kamp 	int error;
72690b1cd56SPoul-Henning Kamp 
7278dd5480dSPawel Jakub Dawidek 	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
7288dd5480dSPawel Jakub Dawidek 	    length <= MAXPHYS, ("g_write_data(): invalid length %jd",
7298dd5480dSPawel Jakub Dawidek 	    (intmax_t)length));
7303eb6ffdfSPoul-Henning Kamp 
731a2033c96SPoul-Henning Kamp 	bp = g_alloc_bio();
73290b1cd56SPoul-Henning Kamp 	bp->bio_cmd = BIO_WRITE;
73390b1cd56SPoul-Henning Kamp 	bp->bio_done = NULL;
73490b1cd56SPoul-Henning Kamp 	bp->bio_offset = offset;
73590b1cd56SPoul-Henning Kamp 	bp->bio_length = length;
73690b1cd56SPoul-Henning Kamp 	bp->bio_data = ptr;
73790b1cd56SPoul-Henning Kamp 	g_io_request(bp, cp);
73890b1cd56SPoul-Henning Kamp 	error = biowait(bp, "gwrite");
73990b1cd56SPoul-Henning Kamp 	g_destroy_bio(bp);
74090b1cd56SPoul-Henning Kamp 	return (error);
74190b1cd56SPoul-Henning Kamp }
74272e33095SPawel Jakub Dawidek 
7432b17fb95SPawel Jakub Dawidek int
7442b17fb95SPawel Jakub Dawidek g_delete_data(struct g_consumer *cp, off_t offset, off_t length)
7452b17fb95SPawel Jakub Dawidek {
7462b17fb95SPawel Jakub Dawidek 	struct bio *bp;
7472b17fb95SPawel Jakub Dawidek 	int error;
7482b17fb95SPawel Jakub Dawidek 
749eed6cda9SPoul-Henning Kamp 	KASSERT(length > 0 && length >= cp->provider->sectorsize,
750eed6cda9SPoul-Henning Kamp 	    ("g_delete_data(): invalid length %jd", (intmax_t)length));
7512b17fb95SPawel Jakub Dawidek 
7522b17fb95SPawel Jakub Dawidek 	bp = g_alloc_bio();
7532b17fb95SPawel Jakub Dawidek 	bp->bio_cmd = BIO_DELETE;
7542b17fb95SPawel Jakub Dawidek 	bp->bio_done = NULL;
7552b17fb95SPawel Jakub Dawidek 	bp->bio_offset = offset;
7562b17fb95SPawel Jakub Dawidek 	bp->bio_length = length;
7572b17fb95SPawel Jakub Dawidek 	bp->bio_data = NULL;
7582b17fb95SPawel Jakub Dawidek 	g_io_request(bp, cp);
7592b17fb95SPawel Jakub Dawidek 	error = biowait(bp, "gdelete");
7602b17fb95SPawel Jakub Dawidek 	g_destroy_bio(bp);
7612b17fb95SPawel Jakub Dawidek 	return (error);
7622b17fb95SPawel Jakub Dawidek }
7632b17fb95SPawel Jakub Dawidek 
76472e33095SPawel Jakub Dawidek void
76572e33095SPawel Jakub Dawidek g_print_bio(struct bio *bp)
76672e33095SPawel Jakub Dawidek {
76772e33095SPawel Jakub Dawidek 	const char *pname, *cmd = NULL;
76872e33095SPawel Jakub Dawidek 
76972e33095SPawel Jakub Dawidek 	if (bp->bio_to != NULL)
77072e33095SPawel Jakub Dawidek 		pname = bp->bio_to->name;
77172e33095SPawel Jakub Dawidek 	else
77272e33095SPawel Jakub Dawidek 		pname = "[unknown]";
77372e33095SPawel Jakub Dawidek 
77472e33095SPawel Jakub Dawidek 	switch (bp->bio_cmd) {
77572e33095SPawel Jakub Dawidek 	case BIO_GETATTR:
77672e33095SPawel Jakub Dawidek 		cmd = "GETATTR";
77772e33095SPawel Jakub Dawidek 		printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute);
77872e33095SPawel Jakub Dawidek 		return;
779c3618c65SPawel Jakub Dawidek 	case BIO_FLUSH:
780c3618c65SPawel Jakub Dawidek 		cmd = "FLUSH";
781c3618c65SPawel Jakub Dawidek 		printf("%s[%s]", pname, cmd);
782c3618c65SPawel Jakub Dawidek 		return;
78372e33095SPawel Jakub Dawidek 	case BIO_READ:
78472e33095SPawel Jakub Dawidek 		cmd = "READ";
7857ce513a5SEdward Tomasz Napierala 		break;
78672e33095SPawel Jakub Dawidek 	case BIO_WRITE:
78772e33095SPawel Jakub Dawidek 		cmd = "WRITE";
7887ce513a5SEdward Tomasz Napierala 		break;
78972e33095SPawel Jakub Dawidek 	case BIO_DELETE:
79072e33095SPawel Jakub Dawidek 		cmd = "DELETE";
7917ce513a5SEdward Tomasz Napierala 		break;
79272e33095SPawel Jakub Dawidek 	default:
79372e33095SPawel Jakub Dawidek 		cmd = "UNKNOWN";
79472e33095SPawel Jakub Dawidek 		printf("%s[%s()]", pname, cmd);
79572e33095SPawel Jakub Dawidek 		return;
79672e33095SPawel Jakub Dawidek 	}
7977ce513a5SEdward Tomasz Napierala 	printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd,
7987ce513a5SEdward Tomasz Napierala 	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
79972e33095SPawel Jakub Dawidek }
800