xref: /freebsd/sys/geom/geom_io.c (revision c6ae9b5fd2888dacabd9beb8f23c57b1700da1a5)
1dd84a43cSPoul-Henning Kamp /*-
2dd84a43cSPoul-Henning Kamp  * Copyright (c) 2002 Poul-Henning Kamp
3dd84a43cSPoul-Henning Kamp  * Copyright (c) 2002 Networks Associates Technology, Inc.
4dd84a43cSPoul-Henning Kamp  * All rights reserved.
5dd84a43cSPoul-Henning Kamp  *
6dd84a43cSPoul-Henning Kamp  * This software was developed for the FreeBSD Project by Poul-Henning Kamp
7dd84a43cSPoul-Henning Kamp  * and NAI Labs, the Security Research Division of Network Associates, Inc.
8dd84a43cSPoul-Henning Kamp  * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
9dd84a43cSPoul-Henning Kamp  * DARPA CHATS research program.
10dd84a43cSPoul-Henning Kamp  *
11dd84a43cSPoul-Henning Kamp  * Redistribution and use in source and binary forms, with or without
12dd84a43cSPoul-Henning Kamp  * modification, are permitted provided that the following conditions
13dd84a43cSPoul-Henning Kamp  * are met:
14dd84a43cSPoul-Henning Kamp  * 1. Redistributions of source code must retain the above copyright
15dd84a43cSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer.
16dd84a43cSPoul-Henning Kamp  * 2. Redistributions in binary form must reproduce the above copyright
17dd84a43cSPoul-Henning Kamp  *    notice, this list of conditions and the following disclaimer in the
18dd84a43cSPoul-Henning Kamp  *    documentation and/or other materials provided with the distribution.
19dd84a43cSPoul-Henning Kamp  * 3. The names of the authors may not be used to endorse or promote
20dd84a43cSPoul-Henning Kamp  *    products derived from this software without specific prior written
21dd84a43cSPoul-Henning Kamp  *    permission.
22dd84a43cSPoul-Henning Kamp  *
23dd84a43cSPoul-Henning Kamp  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24dd84a43cSPoul-Henning Kamp  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25dd84a43cSPoul-Henning Kamp  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26dd84a43cSPoul-Henning Kamp  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27dd84a43cSPoul-Henning Kamp  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28dd84a43cSPoul-Henning Kamp  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29dd84a43cSPoul-Henning Kamp  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30dd84a43cSPoul-Henning Kamp  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31dd84a43cSPoul-Henning Kamp  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32dd84a43cSPoul-Henning Kamp  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33dd84a43cSPoul-Henning Kamp  * SUCH DAMAGE.
34dd84a43cSPoul-Henning Kamp  *
35dd84a43cSPoul-Henning Kamp  * $FreeBSD$
36dd84a43cSPoul-Henning Kamp  */
37dd84a43cSPoul-Henning Kamp 
38dd84a43cSPoul-Henning Kamp 
39dd84a43cSPoul-Henning Kamp #include <sys/param.h>
400355b86eSPoul-Henning Kamp #include <sys/stdint.h>
41dd84a43cSPoul-Henning Kamp #ifndef _KERNEL
42dd84a43cSPoul-Henning Kamp #include <stdio.h>
43dd84a43cSPoul-Henning Kamp #include <string.h>
44dd84a43cSPoul-Henning Kamp #include <stdlib.h>
45dd84a43cSPoul-Henning Kamp #include <signal.h>
46dd84a43cSPoul-Henning Kamp #include <err.h>
47dd84a43cSPoul-Henning Kamp #include <sched.h>
48dd84a43cSPoul-Henning Kamp #else
49dd84a43cSPoul-Henning Kamp #include <sys/systm.h>
50dd84a43cSPoul-Henning Kamp #include <sys/kernel.h>
51dd84a43cSPoul-Henning Kamp #include <sys/malloc.h>
52dd84a43cSPoul-Henning Kamp #include <sys/bio.h>
53dd84a43cSPoul-Henning Kamp #endif
54dd84a43cSPoul-Henning Kamp 
55dd84a43cSPoul-Henning Kamp #include <sys/errno.h>
56dd84a43cSPoul-Henning Kamp #include <geom/geom.h>
57b1876192SPoul-Henning Kamp #include <geom/geom_int.h>
584ec35300SPoul-Henning Kamp #include <geom/geom_stats.h>
59dd84a43cSPoul-Henning Kamp 
60dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_down;
61dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_up;
62f0e185d7SPoul-Henning Kamp static struct g_bioq g_bio_run_task;
63dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_idle;
64dd84a43cSPoul-Henning Kamp 
653432e4fdSPoul-Henning Kamp static u_int pace;
663432e4fdSPoul-Henning Kamp 
67dd84a43cSPoul-Henning Kamp #include <machine/atomic.h>
68dd84a43cSPoul-Henning Kamp 
69dd84a43cSPoul-Henning Kamp static void
70dd84a43cSPoul-Henning Kamp g_bioq_lock(struct g_bioq *bq)
71dd84a43cSPoul-Henning Kamp {
72dd84a43cSPoul-Henning Kamp 
73dd84a43cSPoul-Henning Kamp 	mtx_lock(&bq->bio_queue_lock);
74dd84a43cSPoul-Henning Kamp }
75dd84a43cSPoul-Henning Kamp 
76dd84a43cSPoul-Henning Kamp static void
77dd84a43cSPoul-Henning Kamp g_bioq_unlock(struct g_bioq *bq)
78dd84a43cSPoul-Henning Kamp {
79dd84a43cSPoul-Henning Kamp 
80dd84a43cSPoul-Henning Kamp 	mtx_unlock(&bq->bio_queue_lock);
81dd84a43cSPoul-Henning Kamp }
82dd84a43cSPoul-Henning Kamp 
83dd84a43cSPoul-Henning Kamp #if 0
84dd84a43cSPoul-Henning Kamp static void
85dd84a43cSPoul-Henning Kamp g_bioq_destroy(struct g_bioq *bq)
86dd84a43cSPoul-Henning Kamp {
87dd84a43cSPoul-Henning Kamp 
88dd84a43cSPoul-Henning Kamp 	mtx_destroy(&bq->bio_queue_lock);
89dd84a43cSPoul-Henning Kamp }
90dd84a43cSPoul-Henning Kamp #endif
91dd84a43cSPoul-Henning Kamp 
92dd84a43cSPoul-Henning Kamp static void
93dd84a43cSPoul-Henning Kamp g_bioq_init(struct g_bioq *bq)
94dd84a43cSPoul-Henning Kamp {
95dd84a43cSPoul-Henning Kamp 
96dd84a43cSPoul-Henning Kamp 	TAILQ_INIT(&bq->bio_queue);
976008862bSJohn Baldwin 	mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF);
98dd84a43cSPoul-Henning Kamp }
99dd84a43cSPoul-Henning Kamp 
100dd84a43cSPoul-Henning Kamp static struct bio *
101dd84a43cSPoul-Henning Kamp g_bioq_first(struct g_bioq *bq)
102dd84a43cSPoul-Henning Kamp {
103dd84a43cSPoul-Henning Kamp 	struct bio *bp;
104dd84a43cSPoul-Henning Kamp 
105dd84a43cSPoul-Henning Kamp 	bp = TAILQ_FIRST(&bq->bio_queue);
106dd84a43cSPoul-Henning Kamp 	if (bp != NULL) {
107dd84a43cSPoul-Henning Kamp 		TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue);
108dd84a43cSPoul-Henning Kamp 		bq->bio_queue_length--;
109dd84a43cSPoul-Henning Kamp 	}
110dd84a43cSPoul-Henning Kamp 	return (bp);
111dd84a43cSPoul-Henning Kamp }
112dd84a43cSPoul-Henning Kamp 
113dd84a43cSPoul-Henning Kamp static void
114dd84a43cSPoul-Henning Kamp g_bioq_enqueue_tail(struct bio *bp, struct g_bioq *rq)
115dd84a43cSPoul-Henning Kamp {
116dd84a43cSPoul-Henning Kamp 
117dd84a43cSPoul-Henning Kamp 	g_bioq_lock(rq);
118dd84a43cSPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&rq->bio_queue, bp, bio_queue);
119dd84a43cSPoul-Henning Kamp 	rq->bio_queue_length++;
120dd84a43cSPoul-Henning Kamp 	g_bioq_unlock(rq);
121dd84a43cSPoul-Henning Kamp }
122dd84a43cSPoul-Henning Kamp 
123dd84a43cSPoul-Henning Kamp struct bio *
124dd84a43cSPoul-Henning Kamp g_new_bio(void)
125dd84a43cSPoul-Henning Kamp {
126dd84a43cSPoul-Henning Kamp 	struct bio *bp;
127dd84a43cSPoul-Henning Kamp 
128f0e185d7SPoul-Henning Kamp 	g_bioq_lock(&g_bio_idle);
129dd84a43cSPoul-Henning Kamp 	bp = g_bioq_first(&g_bio_idle);
130f0e185d7SPoul-Henning Kamp 	g_bioq_unlock(&g_bio_idle);
131dd84a43cSPoul-Henning Kamp 	if (bp == NULL)
132a1bd3ee2SPoul-Henning Kamp 		bp = g_malloc(sizeof *bp, M_NOWAIT | M_ZERO);
1330355b86eSPoul-Henning Kamp 	/* g_trace(G_T_BIO, "g_new_bio() = %p", bp); */
134dd84a43cSPoul-Henning Kamp 	return (bp);
135dd84a43cSPoul-Henning Kamp }
136dd84a43cSPoul-Henning Kamp 
137dd84a43cSPoul-Henning Kamp void
138dd84a43cSPoul-Henning Kamp g_destroy_bio(struct bio *bp)
139dd84a43cSPoul-Henning Kamp {
140dd84a43cSPoul-Henning Kamp 
1410355b86eSPoul-Henning Kamp 	/* g_trace(G_T_BIO, "g_destroy_bio(%p)", bp); */
142dd84a43cSPoul-Henning Kamp 	bzero(bp, sizeof *bp);
143dd84a43cSPoul-Henning Kamp 	g_bioq_enqueue_tail(bp, &g_bio_idle);
144dd84a43cSPoul-Henning Kamp }
145dd84a43cSPoul-Henning Kamp 
146dd84a43cSPoul-Henning Kamp struct bio *
147dd84a43cSPoul-Henning Kamp g_clone_bio(struct bio *bp)
148dd84a43cSPoul-Henning Kamp {
149dd84a43cSPoul-Henning Kamp 	struct bio *bp2;
150dd84a43cSPoul-Henning Kamp 
151dd84a43cSPoul-Henning Kamp 	bp2 = g_new_bio();
152a1bd3ee2SPoul-Henning Kamp 	if (bp2 != NULL) {
153936cc461SPoul-Henning Kamp 		bp2->bio_parent = bp;
154dd84a43cSPoul-Henning Kamp 		bp2->bio_cmd = bp->bio_cmd;
155dd84a43cSPoul-Henning Kamp 		bp2->bio_length = bp->bio_length;
156dd84a43cSPoul-Henning Kamp 		bp2->bio_offset = bp->bio_offset;
157dd84a43cSPoul-Henning Kamp 		bp2->bio_data = bp->bio_data;
158dd84a43cSPoul-Henning Kamp 		bp2->bio_attribute = bp->bio_attribute;
159801bb689SPoul-Henning Kamp 		bp->bio_children++;
160a1bd3ee2SPoul-Henning Kamp 	}
1610355b86eSPoul-Henning Kamp 	/* g_trace(G_T_BIO, "g_clone_bio(%p) = %p", bp, bp2); */
162dd84a43cSPoul-Henning Kamp 	return(bp2);
163dd84a43cSPoul-Henning Kamp }
164dd84a43cSPoul-Henning Kamp 
165dd84a43cSPoul-Henning Kamp void
166dd84a43cSPoul-Henning Kamp g_io_init()
167dd84a43cSPoul-Henning Kamp {
168dd84a43cSPoul-Henning Kamp 
169dd84a43cSPoul-Henning Kamp 	g_bioq_init(&g_bio_run_down);
170dd84a43cSPoul-Henning Kamp 	g_bioq_init(&g_bio_run_up);
171f0e185d7SPoul-Henning Kamp 	g_bioq_init(&g_bio_run_task);
172dd84a43cSPoul-Henning Kamp 	g_bioq_init(&g_bio_idle);
173dd84a43cSPoul-Henning Kamp }
174dd84a43cSPoul-Henning Kamp 
175dd84a43cSPoul-Henning Kamp int
1760d3f37a8SPoul-Henning Kamp g_io_setattr(const char *attr, struct g_consumer *cp, int len, void *ptr)
177dd84a43cSPoul-Henning Kamp {
178dd84a43cSPoul-Henning Kamp 	struct bio *bp;
179dd84a43cSPoul-Henning Kamp 	int error;
180dd84a43cSPoul-Henning Kamp 
181dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO, "bio_setattr(%s)", attr);
182dd84a43cSPoul-Henning Kamp 	bp = g_new_bio();
183dd84a43cSPoul-Henning Kamp 	bp->bio_cmd = BIO_SETATTR;
184dd84a43cSPoul-Henning Kamp 	bp->bio_done = NULL;
185dd84a43cSPoul-Henning Kamp 	bp->bio_attribute = attr;
186dd84a43cSPoul-Henning Kamp 	bp->bio_length = len;
187dd84a43cSPoul-Henning Kamp 	bp->bio_data = ptr;
188dd84a43cSPoul-Henning Kamp 	g_io_request(bp, cp);
18953706245SPoul-Henning Kamp 	error = biowait(bp, "gsetattr");
190dd84a43cSPoul-Henning Kamp 	g_destroy_bio(bp);
191dd84a43cSPoul-Henning Kamp 	return (error);
192dd84a43cSPoul-Henning Kamp }
193dd84a43cSPoul-Henning Kamp 
194dd84a43cSPoul-Henning Kamp 
195dd84a43cSPoul-Henning Kamp int
1960d3f37a8SPoul-Henning Kamp g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr)
197dd84a43cSPoul-Henning Kamp {
198dd84a43cSPoul-Henning Kamp 	struct bio *bp;
199dd84a43cSPoul-Henning Kamp 	int error;
200dd84a43cSPoul-Henning Kamp 
201dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO, "bio_getattr(%s)", attr);
202dd84a43cSPoul-Henning Kamp 	bp = g_new_bio();
203dd84a43cSPoul-Henning Kamp 	bp->bio_cmd = BIO_GETATTR;
204dd84a43cSPoul-Henning Kamp 	bp->bio_done = NULL;
205dd84a43cSPoul-Henning Kamp 	bp->bio_attribute = attr;
206dd84a43cSPoul-Henning Kamp 	bp->bio_length = *len;
207dd84a43cSPoul-Henning Kamp 	bp->bio_data = ptr;
208dd84a43cSPoul-Henning Kamp 	g_io_request(bp, cp);
20953706245SPoul-Henning Kamp 	error = biowait(bp, "ggetattr");
210dd84a43cSPoul-Henning Kamp 	*len = bp->bio_completed;
211dd84a43cSPoul-Henning Kamp 	g_destroy_bio(bp);
212dd84a43cSPoul-Henning Kamp 	return (error);
213dd84a43cSPoul-Henning Kamp }
214dd84a43cSPoul-Henning Kamp 
215e39d70d4SPoul-Henning Kamp static int
216e39d70d4SPoul-Henning Kamp g_io_check(struct bio *bp)
217e39d70d4SPoul-Henning Kamp {
218e39d70d4SPoul-Henning Kamp 	struct g_consumer *cp;
219e39d70d4SPoul-Henning Kamp 	struct g_provider *pp;
220e39d70d4SPoul-Henning Kamp 
221e39d70d4SPoul-Henning Kamp 	cp = bp->bio_from;
222e39d70d4SPoul-Henning Kamp 	pp = bp->bio_to;
223e39d70d4SPoul-Henning Kamp 
224e39d70d4SPoul-Henning Kamp 	/* Fail if access counters dont allow the operation */
225e39d70d4SPoul-Henning Kamp 	switch(bp->bio_cmd) {
226e39d70d4SPoul-Henning Kamp 	case BIO_READ:
227e39d70d4SPoul-Henning Kamp 	case BIO_GETATTR:
228e39d70d4SPoul-Henning Kamp 		if (cp->acr == 0)
229e39d70d4SPoul-Henning Kamp 			return (EPERM);
230e39d70d4SPoul-Henning Kamp 		break;
231e39d70d4SPoul-Henning Kamp 	case BIO_WRITE:
232e39d70d4SPoul-Henning Kamp 	case BIO_DELETE:
233e39d70d4SPoul-Henning Kamp 	case BIO_SETATTR:
234e39d70d4SPoul-Henning Kamp 		if (cp->acw == 0)
235e39d70d4SPoul-Henning Kamp 			return (EPERM);
236e39d70d4SPoul-Henning Kamp 		break;
237e39d70d4SPoul-Henning Kamp 	default:
238e39d70d4SPoul-Henning Kamp 		return (EPERM);
239e39d70d4SPoul-Henning Kamp 	}
240e39d70d4SPoul-Henning Kamp 	/* if provider is marked for error, don't disturb. */
241e39d70d4SPoul-Henning Kamp 	if (pp->error)
242e39d70d4SPoul-Henning Kamp 		return (pp->error);
243e39d70d4SPoul-Henning Kamp 
244e39d70d4SPoul-Henning Kamp 	switch(bp->bio_cmd) {
245e39d70d4SPoul-Henning Kamp 	case BIO_READ:
246e39d70d4SPoul-Henning Kamp 	case BIO_WRITE:
247e39d70d4SPoul-Henning Kamp 	case BIO_DELETE:
248e39d70d4SPoul-Henning Kamp 		/* Reject I/O not on sector boundary */
249e39d70d4SPoul-Henning Kamp 		if (bp->bio_offset % pp->sectorsize)
250e39d70d4SPoul-Henning Kamp 			return (EINVAL);
251e39d70d4SPoul-Henning Kamp 		/* Reject I/O not integral sector long */
252e39d70d4SPoul-Henning Kamp 		if (bp->bio_length % pp->sectorsize)
253e39d70d4SPoul-Henning Kamp 			return (EINVAL);
254e39d70d4SPoul-Henning Kamp 		/* Reject requests past the end of media. */
255e39d70d4SPoul-Henning Kamp 		if (bp->bio_offset > pp->mediasize)
256e39d70d4SPoul-Henning Kamp 			return (EIO);
257e39d70d4SPoul-Henning Kamp 		break;
258e39d70d4SPoul-Henning Kamp 	default:
259e39d70d4SPoul-Henning Kamp 		break;
260e39d70d4SPoul-Henning Kamp 	}
261e39d70d4SPoul-Henning Kamp 	return (0);
262e39d70d4SPoul-Henning Kamp }
263e39d70d4SPoul-Henning Kamp 
264dd84a43cSPoul-Henning Kamp void
265dd84a43cSPoul-Henning Kamp g_io_request(struct bio *bp, struct g_consumer *cp)
266dd84a43cSPoul-Henning Kamp {
267801bb689SPoul-Henning Kamp 	struct g_provider *pp;
268801bb689SPoul-Henning Kamp 	struct bintime bt;
269dd84a43cSPoul-Henning Kamp 
270801bb689SPoul-Henning Kamp 	pp = cp->provider;
271d0e17c1bSPoul-Henning Kamp 	KASSERT(cp != NULL, ("NULL cp in g_io_request"));
272d0e17c1bSPoul-Henning Kamp 	KASSERT(bp != NULL, ("NULL bp in g_io_request"));
273d0e17c1bSPoul-Henning Kamp 	KASSERT(bp->bio_data != NULL, ("NULL bp->data in g_io_request"));
274801bb689SPoul-Henning Kamp 	KASSERT(pp != NULL, ("consumer not attached in g_io_request"));
275801bb689SPoul-Henning Kamp 
276dd84a43cSPoul-Henning Kamp 	bp->bio_from = cp;
277801bb689SPoul-Henning Kamp 	bp->bio_to = pp;
2782fccec19SPoul-Henning Kamp 	bp->bio_error = 0;
2792fccec19SPoul-Henning Kamp 	bp->bio_completed = 0;
280dd84a43cSPoul-Henning Kamp 
281801bb689SPoul-Henning Kamp 	if (g_collectstats) {
282cce7303aSPoul-Henning Kamp 		binuptime(&bt);
283cce7303aSPoul-Henning Kamp 		bp->bio_t0 = bt;
284c6ae9b5fSPoul-Henning Kamp 		if (cp->nstart == cp->nend)
285cce7303aSPoul-Henning Kamp 			cp->stat->wentbusy = bt; /* Consumer is idle */
286c6ae9b5fSPoul-Henning Kamp 		if (pp->nstart == pp->nend)
287cce7303aSPoul-Henning Kamp 			pp->stat->wentbusy = bt; /* Provider is idle */
2884ec35300SPoul-Henning Kamp 		cp->stat->nop++;
2894ec35300SPoul-Henning Kamp 		pp->stat->nop++;
290c6ae9b5fSPoul-Henning Kamp 	}
291c6ae9b5fSPoul-Henning Kamp 	cp->nstart++;
292c6ae9b5fSPoul-Henning Kamp 	pp->nstart++;
293e39d70d4SPoul-Henning Kamp 
2942fccec19SPoul-Henning Kamp 	/* Pass it on down. */
295dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d",
296801bb689SPoul-Henning Kamp 	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd);
297dd84a43cSPoul-Henning Kamp 	g_bioq_enqueue_tail(bp, &g_bio_run_down);
298dd84a43cSPoul-Henning Kamp 	wakeup(&g_wait_down);
299dd84a43cSPoul-Henning Kamp }
300dd84a43cSPoul-Henning Kamp 
301dd84a43cSPoul-Henning Kamp void
30272840432SPoul-Henning Kamp g_io_deliver(struct bio *bp, int error)
303dd84a43cSPoul-Henning Kamp {
304801bb689SPoul-Henning Kamp 	struct g_consumer *cp;
305801bb689SPoul-Henning Kamp 	struct g_provider *pp;
306cce7303aSPoul-Henning Kamp 	struct bintime t1, dt;
307801bb689SPoul-Henning Kamp 	int idx;
308dd84a43cSPoul-Henning Kamp 
309801bb689SPoul-Henning Kamp 	cp = bp->bio_from;
310801bb689SPoul-Henning Kamp 	pp = bp->bio_to;
311d0e17c1bSPoul-Henning Kamp 	KASSERT(bp != NULL, ("NULL bp in g_io_deliver"));
312801bb689SPoul-Henning Kamp 	KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver"));
313801bb689SPoul-Henning Kamp 	KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver"));
314801bb689SPoul-Henning Kamp 	KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver"));
3155ab413bfSPoul-Henning Kamp 
316dd84a43cSPoul-Henning Kamp 	g_trace(G_T_BIO,
3170355b86eSPoul-Henning Kamp "g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd",
318801bb689SPoul-Henning Kamp 	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error,
3190355b86eSPoul-Henning Kamp 	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
320801bb689SPoul-Henning Kamp 
321cce7303aSPoul-Henning Kamp 	if (g_collectstats) {
322801bb689SPoul-Henning Kamp 		switch (bp->bio_cmd) {
323801bb689SPoul-Henning Kamp 		case BIO_READ:    idx =  G_STAT_IDX_READ;    break;
324801bb689SPoul-Henning Kamp 		case BIO_WRITE:   idx =  G_STAT_IDX_WRITE;   break;
325801bb689SPoul-Henning Kamp 		case BIO_DELETE:  idx =  G_STAT_IDX_DELETE;  break;
326801bb689SPoul-Henning Kamp 		case BIO_GETATTR: idx =  -1; break;
327801bb689SPoul-Henning Kamp 		case BIO_SETATTR: idx =  -1; break;
328801bb689SPoul-Henning Kamp 		default:
329801bb689SPoul-Henning Kamp 			panic("unknown bio_cmd in g_io_deliver");
330801bb689SPoul-Henning Kamp 			break;
331801bb689SPoul-Henning Kamp 		}
332801bb689SPoul-Henning Kamp 		binuptime(&t1);
333cce7303aSPoul-Henning Kamp 		/* Raise the "inconsistent" flag for userland */
33481e9eba3SPoul-Henning Kamp 		atomic_add_acq_int(&cp->stat->seq0, 1);
33581e9eba3SPoul-Henning Kamp 		atomic_add_acq_int(&pp->stat->seq0, 1);
336801bb689SPoul-Henning Kamp 		if (idx >= 0) {
337cce7303aSPoul-Henning Kamp 			/* Account the service time */
338cce7303aSPoul-Henning Kamp 			dt = t1;
339cce7303aSPoul-Henning Kamp 			bintime_sub(&dt, &bp->bio_t0);
340cce7303aSPoul-Henning Kamp 			bintime_add(&cp->stat->ops[idx].dt, &dt);
341cce7303aSPoul-Henning Kamp 			bintime_add(&pp->stat->ops[idx].dt, &dt);
342cce7303aSPoul-Henning Kamp 			/* ... and the metrics */
3434ec35300SPoul-Henning Kamp 			pp->stat->ops[idx].nbyte += bp->bio_completed;
3444ec35300SPoul-Henning Kamp 			cp->stat->ops[idx].nbyte += bp->bio_completed;
3454ec35300SPoul-Henning Kamp 			pp->stat->ops[idx].nop++;
3464ec35300SPoul-Henning Kamp 			cp->stat->ops[idx].nop++;
347cce7303aSPoul-Henning Kamp 			/* ... and any errors */
348801bb689SPoul-Henning Kamp 			if (error == ENOMEM) {
3494ec35300SPoul-Henning Kamp 				cp->stat->ops[idx].nmem++;
3504ec35300SPoul-Henning Kamp 				pp->stat->ops[idx].nmem++;
351801bb689SPoul-Henning Kamp 			} else if (error != 0) {
3524ec35300SPoul-Henning Kamp 				cp->stat->ops[idx].nerr++;
3534ec35300SPoul-Henning Kamp 				pp->stat->ops[idx].nerr++;
354801bb689SPoul-Henning Kamp 			}
355801bb689SPoul-Henning Kamp 		}
356cce7303aSPoul-Henning Kamp 		/* Account for busy time on the consumer */
357cce7303aSPoul-Henning Kamp 		dt = t1;
358cce7303aSPoul-Henning Kamp 		bintime_sub(&dt, &cp->stat->wentbusy);
359cce7303aSPoul-Henning Kamp 		bintime_add(&cp->stat->bt, &dt);
360cce7303aSPoul-Henning Kamp 		cp->stat->wentbusy = t1;
361cce7303aSPoul-Henning Kamp 		/* Account for busy time on the provider */
362cce7303aSPoul-Henning Kamp 		dt = t1;
363cce7303aSPoul-Henning Kamp 		bintime_sub(&dt, &pp->stat->wentbusy);
364cce7303aSPoul-Henning Kamp 		bintime_add(&pp->stat->bt, &dt);
365cce7303aSPoul-Henning Kamp 		pp->stat->wentbusy = t1;
366cce7303aSPoul-Henning Kamp 		/* Mark the structures as consistent again */
36781e9eba3SPoul-Henning Kamp 		atomic_add_acq_int(&cp->stat->seq1, 1);
36881e9eba3SPoul-Henning Kamp 		atomic_add_acq_int(&pp->stat->seq1, 1);
3694ec35300SPoul-Henning Kamp 		cp->stat->nend++;
370cce7303aSPoul-Henning Kamp 		pp->stat->nend++;
371c6ae9b5fSPoul-Henning Kamp 	}
372c6ae9b5fSPoul-Henning Kamp 	cp->nend++;
373c6ae9b5fSPoul-Henning Kamp 	pp->nend++;
374dd84a43cSPoul-Henning Kamp 
3753432e4fdSPoul-Henning Kamp 	if (error == ENOMEM) {
376801bb689SPoul-Henning Kamp 		printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name);
377801bb689SPoul-Henning Kamp 		g_io_request(bp, cp);
3783432e4fdSPoul-Henning Kamp 		pace++;
3793432e4fdSPoul-Henning Kamp 		return;
3803432e4fdSPoul-Henning Kamp 	}
38172840432SPoul-Henning Kamp 	bp->bio_error = error;
382dd84a43cSPoul-Henning Kamp 	g_bioq_enqueue_tail(bp, &g_bio_run_up);
383dd84a43cSPoul-Henning Kamp 	wakeup(&g_wait_up);
384dd84a43cSPoul-Henning Kamp }
385dd84a43cSPoul-Henning Kamp 
386dd84a43cSPoul-Henning Kamp void
387dd84a43cSPoul-Henning Kamp g_io_schedule_down(struct thread *tp __unused)
388dd84a43cSPoul-Henning Kamp {
389dd84a43cSPoul-Henning Kamp 	struct bio *bp;
390e39d70d4SPoul-Henning Kamp 	off_t excess;
391e39d70d4SPoul-Henning Kamp 	int error;
392f0e185d7SPoul-Henning Kamp 	struct mtx mymutex;
393f0e185d7SPoul-Henning Kamp 
394f0e185d7SPoul-Henning Kamp 	bzero(&mymutex, sizeof mymutex);
395f0e185d7SPoul-Henning Kamp 	mtx_init(&mymutex, "g_xdown", MTX_DEF, 0);
396dd84a43cSPoul-Henning Kamp 
397dd84a43cSPoul-Henning Kamp 	for(;;) {
398f0e185d7SPoul-Henning Kamp 		g_bioq_lock(&g_bio_run_down);
399dd84a43cSPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_down);
400f0e185d7SPoul-Henning Kamp 		if (bp == NULL) {
401f0e185d7SPoul-Henning Kamp 			msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock,
402f0e185d7SPoul-Henning Kamp 			    PRIBIO | PDROP, "g_down", hz/10);
403f0e185d7SPoul-Henning Kamp 			continue;
404f0e185d7SPoul-Henning Kamp 		}
405f0e185d7SPoul-Henning Kamp 		g_bioq_unlock(&g_bio_run_down);
406e39d70d4SPoul-Henning Kamp 		error = g_io_check(bp);
407e39d70d4SPoul-Henning Kamp 		if (error) {
408e39d70d4SPoul-Henning Kamp 			g_io_deliver(bp, error);
409e39d70d4SPoul-Henning Kamp 			continue;
410e39d70d4SPoul-Henning Kamp 		}
411392d56b4SPoul-Henning Kamp 		switch (bp->bio_cmd) {
412392d56b4SPoul-Henning Kamp 		case BIO_READ:
413392d56b4SPoul-Henning Kamp 		case BIO_WRITE:
414392d56b4SPoul-Henning Kamp 		case BIO_DELETE:
415e39d70d4SPoul-Henning Kamp 			/* Truncate requests to the end of providers media. */
416e39d70d4SPoul-Henning Kamp 			excess = bp->bio_offset + bp->bio_length;
417e39d70d4SPoul-Henning Kamp 			if (excess > bp->bio_to->mediasize) {
418e39d70d4SPoul-Henning Kamp 				excess -= bp->bio_to->mediasize;
419e39d70d4SPoul-Henning Kamp 				bp->bio_length -= excess;
420e39d70d4SPoul-Henning Kamp 			}
421e39d70d4SPoul-Henning Kamp 			/* Deliver zero length transfers right here. */
422e39d70d4SPoul-Henning Kamp 			if (bp->bio_length == 0) {
423e39d70d4SPoul-Henning Kamp 				g_io_deliver(bp, 0);
424e39d70d4SPoul-Henning Kamp 				continue;
425e39d70d4SPoul-Henning Kamp 			}
426392d56b4SPoul-Henning Kamp 			break;
427392d56b4SPoul-Henning Kamp 		default:
428392d56b4SPoul-Henning Kamp 			break;
429392d56b4SPoul-Henning Kamp 		}
430f0e185d7SPoul-Henning Kamp 		mtx_lock(&mymutex);
431dd84a43cSPoul-Henning Kamp 		bp->bio_to->geom->start(bp);
432f0e185d7SPoul-Henning Kamp 		mtx_unlock(&mymutex);
4333432e4fdSPoul-Henning Kamp 		if (pace) {
4343432e4fdSPoul-Henning Kamp 			pace--;
4353432e4fdSPoul-Henning Kamp 			break;
4363432e4fdSPoul-Henning Kamp 		}
437dd84a43cSPoul-Henning Kamp 	}
438dd84a43cSPoul-Henning Kamp }
439dd84a43cSPoul-Henning Kamp 
440dd84a43cSPoul-Henning Kamp void
441f0e185d7SPoul-Henning Kamp bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg)
442f0e185d7SPoul-Henning Kamp {
443f0e185d7SPoul-Henning Kamp 	bp->bio_task = func;
444f0e185d7SPoul-Henning Kamp 	bp->bio_task_arg = arg;
445f0e185d7SPoul-Henning Kamp 	/*
446f0e185d7SPoul-Henning Kamp 	 * The taskqueue is actually just a second queue off the "up"
447f0e185d7SPoul-Henning Kamp 	 * queue, so we use the same lock.
448f0e185d7SPoul-Henning Kamp 	 */
449f0e185d7SPoul-Henning Kamp 	g_bioq_lock(&g_bio_run_up);
450f0e185d7SPoul-Henning Kamp 	TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue);
451f0e185d7SPoul-Henning Kamp 	g_bio_run_task.bio_queue_length++;
452f0e185d7SPoul-Henning Kamp 	wakeup(&g_wait_up);
453f0e185d7SPoul-Henning Kamp 	g_bioq_unlock(&g_bio_run_up);
454f0e185d7SPoul-Henning Kamp }
455f0e185d7SPoul-Henning Kamp 
456f0e185d7SPoul-Henning Kamp 
457f0e185d7SPoul-Henning Kamp void
458dd84a43cSPoul-Henning Kamp g_io_schedule_up(struct thread *tp __unused)
459dd84a43cSPoul-Henning Kamp {
460dd84a43cSPoul-Henning Kamp 	struct bio *bp;
461f0e185d7SPoul-Henning Kamp 	struct mtx mymutex;
462dd84a43cSPoul-Henning Kamp 
463f0e185d7SPoul-Henning Kamp 	bzero(&mymutex, sizeof mymutex);
464f0e185d7SPoul-Henning Kamp 	mtx_init(&mymutex, "g_xup", MTX_DEF, 0);
465dd84a43cSPoul-Henning Kamp 	for(;;) {
466f0e185d7SPoul-Henning Kamp 		g_bioq_lock(&g_bio_run_up);
467f0e185d7SPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_task);
468f0e185d7SPoul-Henning Kamp 		if (bp != NULL) {
469f0e185d7SPoul-Henning Kamp 			g_bioq_unlock(&g_bio_run_up);
470f0e185d7SPoul-Henning Kamp 			mtx_lock(&mymutex);
471f0e185d7SPoul-Henning Kamp 			bp->bio_task(bp, bp->bio_task_arg);
472f0e185d7SPoul-Henning Kamp 			mtx_unlock(&mymutex);
473f0e185d7SPoul-Henning Kamp 			continue;
474f0e185d7SPoul-Henning Kamp 		}
475dd84a43cSPoul-Henning Kamp 		bp = g_bioq_first(&g_bio_run_up);
476f0e185d7SPoul-Henning Kamp 		if (bp != NULL) {
477f0e185d7SPoul-Henning Kamp 			g_bioq_unlock(&g_bio_run_up);
478f0e185d7SPoul-Henning Kamp 			mtx_lock(&mymutex);
47953706245SPoul-Henning Kamp 			biodone(bp);
480f0e185d7SPoul-Henning Kamp 			mtx_unlock(&mymutex);
481f0e185d7SPoul-Henning Kamp 			continue;
482f0e185d7SPoul-Henning Kamp 		}
483f0e185d7SPoul-Henning Kamp 		msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock,
484f0e185d7SPoul-Henning Kamp 		    PRIBIO | PDROP, "g_up", hz/10);
485dd84a43cSPoul-Henning Kamp 	}
486dd84a43cSPoul-Henning Kamp }
487dd84a43cSPoul-Henning Kamp 
488dd84a43cSPoul-Henning Kamp void *
489dd84a43cSPoul-Henning Kamp g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error)
490dd84a43cSPoul-Henning Kamp {
491dd84a43cSPoul-Henning Kamp 	struct bio *bp;
492dd84a43cSPoul-Henning Kamp 	void *ptr;
493dd84a43cSPoul-Henning Kamp 	int errorc;
494dd84a43cSPoul-Henning Kamp 
495dd84a43cSPoul-Henning Kamp 	bp = g_new_bio();
496dd84a43cSPoul-Henning Kamp 	bp->bio_cmd = BIO_READ;
497dd84a43cSPoul-Henning Kamp 	bp->bio_done = NULL;
498dd84a43cSPoul-Henning Kamp 	bp->bio_offset = offset;
499dd84a43cSPoul-Henning Kamp 	bp->bio_length = length;
500a163d034SWarner Losh 	ptr = g_malloc(length, M_WAITOK);
501dd84a43cSPoul-Henning Kamp 	bp->bio_data = ptr;
502dd84a43cSPoul-Henning Kamp 	g_io_request(bp, cp);
50353706245SPoul-Henning Kamp 	errorc = biowait(bp, "gread");
504dd84a43cSPoul-Henning Kamp 	if (error != NULL)
505dd84a43cSPoul-Henning Kamp 		*error = errorc;
506dd84a43cSPoul-Henning Kamp 	g_destroy_bio(bp);
507dd84a43cSPoul-Henning Kamp 	if (errorc) {
508dd84a43cSPoul-Henning Kamp 		g_free(ptr);
509dd84a43cSPoul-Henning Kamp 		ptr = NULL;
510dd84a43cSPoul-Henning Kamp 	}
511dd84a43cSPoul-Henning Kamp 	return (ptr);
512dd84a43cSPoul-Henning Kamp }
51390b1cd56SPoul-Henning Kamp 
51490b1cd56SPoul-Henning Kamp int
51590b1cd56SPoul-Henning Kamp g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length)
51690b1cd56SPoul-Henning Kamp {
51790b1cd56SPoul-Henning Kamp 	struct bio *bp;
51890b1cd56SPoul-Henning Kamp 	int error;
51990b1cd56SPoul-Henning Kamp 
52090b1cd56SPoul-Henning Kamp 	bp = g_new_bio();
52190b1cd56SPoul-Henning Kamp 	bp->bio_cmd = BIO_WRITE;
52290b1cd56SPoul-Henning Kamp 	bp->bio_done = NULL;
52390b1cd56SPoul-Henning Kamp 	bp->bio_offset = offset;
52490b1cd56SPoul-Henning Kamp 	bp->bio_length = length;
52590b1cd56SPoul-Henning Kamp 	bp->bio_data = ptr;
52690b1cd56SPoul-Henning Kamp 	g_io_request(bp, cp);
52790b1cd56SPoul-Henning Kamp 	error = biowait(bp, "gwrite");
52890b1cd56SPoul-Henning Kamp 	g_destroy_bio(bp);
52990b1cd56SPoul-Henning Kamp 	return (error);
53090b1cd56SPoul-Henning Kamp }
531