1dd84a43cSPoul-Henning Kamp /*- 2dd84a43cSPoul-Henning Kamp * Copyright (c) 2002 Poul-Henning Kamp 3dd84a43cSPoul-Henning Kamp * Copyright (c) 2002 Networks Associates Technology, Inc. 4dd84a43cSPoul-Henning Kamp * All rights reserved. 5dd84a43cSPoul-Henning Kamp * 6dd84a43cSPoul-Henning Kamp * This software was developed for the FreeBSD Project by Poul-Henning Kamp 7dd84a43cSPoul-Henning Kamp * and NAI Labs, the Security Research Division of Network Associates, Inc. 8dd84a43cSPoul-Henning Kamp * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 9dd84a43cSPoul-Henning Kamp * DARPA CHATS research program. 10dd84a43cSPoul-Henning Kamp * 11dd84a43cSPoul-Henning Kamp * Redistribution and use in source and binary forms, with or without 12dd84a43cSPoul-Henning Kamp * modification, are permitted provided that the following conditions 13dd84a43cSPoul-Henning Kamp * are met: 14dd84a43cSPoul-Henning Kamp * 1. Redistributions of source code must retain the above copyright 15dd84a43cSPoul-Henning Kamp * notice, this list of conditions and the following disclaimer. 16dd84a43cSPoul-Henning Kamp * 2. Redistributions in binary form must reproduce the above copyright 17dd84a43cSPoul-Henning Kamp * notice, this list of conditions and the following disclaimer in the 18dd84a43cSPoul-Henning Kamp * documentation and/or other materials provided with the distribution. 19dd84a43cSPoul-Henning Kamp * 3. The names of the authors may not be used to endorse or promote 20dd84a43cSPoul-Henning Kamp * products derived from this software without specific prior written 21dd84a43cSPoul-Henning Kamp * permission. 22dd84a43cSPoul-Henning Kamp * 23dd84a43cSPoul-Henning Kamp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24dd84a43cSPoul-Henning Kamp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25dd84a43cSPoul-Henning Kamp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26dd84a43cSPoul-Henning Kamp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27dd84a43cSPoul-Henning Kamp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28dd84a43cSPoul-Henning Kamp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29dd84a43cSPoul-Henning Kamp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30dd84a43cSPoul-Henning Kamp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31dd84a43cSPoul-Henning Kamp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32dd84a43cSPoul-Henning Kamp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33dd84a43cSPoul-Henning Kamp * SUCH DAMAGE. 34dd84a43cSPoul-Henning Kamp * 35dd84a43cSPoul-Henning Kamp * $FreeBSD$ 36dd84a43cSPoul-Henning Kamp */ 37dd84a43cSPoul-Henning Kamp 38dd84a43cSPoul-Henning Kamp 39dd84a43cSPoul-Henning Kamp #include <sys/param.h> 400355b86eSPoul-Henning Kamp #include <sys/stdint.h> 41dd84a43cSPoul-Henning Kamp #ifndef _KERNEL 42dd84a43cSPoul-Henning Kamp #include <stdio.h> 43dd84a43cSPoul-Henning Kamp #include <string.h> 44dd84a43cSPoul-Henning Kamp #include <stdlib.h> 45dd84a43cSPoul-Henning Kamp #include <signal.h> 46dd84a43cSPoul-Henning Kamp #include <err.h> 47dd84a43cSPoul-Henning Kamp #include <sched.h> 48dd84a43cSPoul-Henning Kamp #else 49dd84a43cSPoul-Henning Kamp #include <sys/systm.h> 50dd84a43cSPoul-Henning Kamp #include <sys/kernel.h> 51dd84a43cSPoul-Henning Kamp #include <sys/malloc.h> 52dd84a43cSPoul-Henning Kamp #include <sys/bio.h> 53dd84a43cSPoul-Henning Kamp #endif 54dd84a43cSPoul-Henning Kamp 55dd84a43cSPoul-Henning Kamp #include <sys/errno.h> 56dd84a43cSPoul-Henning Kamp #include <geom/geom.h> 57b1876192SPoul-Henning Kamp #include <geom/geom_int.h> 584ec35300SPoul-Henning Kamp #include <geom/geom_stats.h> 59dd84a43cSPoul-Henning Kamp 60dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_down; 61dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_up; 62f0e185d7SPoul-Henning Kamp static struct g_bioq g_bio_run_task; 63dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_idle; 64dd84a43cSPoul-Henning Kamp 653432e4fdSPoul-Henning Kamp static u_int pace; 663432e4fdSPoul-Henning Kamp 67dd84a43cSPoul-Henning Kamp #include <machine/atomic.h> 68dd84a43cSPoul-Henning Kamp 69dd84a43cSPoul-Henning Kamp static void 70dd84a43cSPoul-Henning Kamp g_bioq_lock(struct g_bioq *bq) 71dd84a43cSPoul-Henning Kamp { 72dd84a43cSPoul-Henning Kamp 73dd84a43cSPoul-Henning Kamp mtx_lock(&bq->bio_queue_lock); 74dd84a43cSPoul-Henning Kamp } 75dd84a43cSPoul-Henning Kamp 76dd84a43cSPoul-Henning Kamp static void 77dd84a43cSPoul-Henning Kamp g_bioq_unlock(struct g_bioq *bq) 78dd84a43cSPoul-Henning Kamp { 79dd84a43cSPoul-Henning Kamp 80dd84a43cSPoul-Henning Kamp mtx_unlock(&bq->bio_queue_lock); 81dd84a43cSPoul-Henning Kamp } 82dd84a43cSPoul-Henning Kamp 83dd84a43cSPoul-Henning Kamp #if 0 84dd84a43cSPoul-Henning Kamp static void 85dd84a43cSPoul-Henning Kamp g_bioq_destroy(struct g_bioq *bq) 86dd84a43cSPoul-Henning Kamp { 87dd84a43cSPoul-Henning Kamp 88dd84a43cSPoul-Henning Kamp mtx_destroy(&bq->bio_queue_lock); 89dd84a43cSPoul-Henning Kamp } 90dd84a43cSPoul-Henning Kamp #endif 91dd84a43cSPoul-Henning Kamp 92dd84a43cSPoul-Henning Kamp static void 93dd84a43cSPoul-Henning Kamp g_bioq_init(struct g_bioq *bq) 94dd84a43cSPoul-Henning Kamp { 95dd84a43cSPoul-Henning Kamp 96dd84a43cSPoul-Henning Kamp TAILQ_INIT(&bq->bio_queue); 976008862bSJohn Baldwin mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 98dd84a43cSPoul-Henning Kamp } 99dd84a43cSPoul-Henning Kamp 100dd84a43cSPoul-Henning Kamp static struct bio * 101dd84a43cSPoul-Henning Kamp g_bioq_first(struct g_bioq *bq) 102dd84a43cSPoul-Henning Kamp { 103dd84a43cSPoul-Henning Kamp struct bio *bp; 104dd84a43cSPoul-Henning Kamp 105dd84a43cSPoul-Henning Kamp bp = TAILQ_FIRST(&bq->bio_queue); 106dd84a43cSPoul-Henning Kamp if (bp != NULL) { 107dd84a43cSPoul-Henning Kamp TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 108dd84a43cSPoul-Henning Kamp bq->bio_queue_length--; 109dd84a43cSPoul-Henning Kamp } 110dd84a43cSPoul-Henning Kamp return (bp); 111dd84a43cSPoul-Henning Kamp } 112dd84a43cSPoul-Henning Kamp 113dd84a43cSPoul-Henning Kamp static void 114dd84a43cSPoul-Henning Kamp g_bioq_enqueue_tail(struct bio *bp, struct g_bioq *rq) 115dd84a43cSPoul-Henning Kamp { 116dd84a43cSPoul-Henning Kamp 117dd84a43cSPoul-Henning Kamp g_bioq_lock(rq); 118dd84a43cSPoul-Henning Kamp TAILQ_INSERT_TAIL(&rq->bio_queue, bp, bio_queue); 119dd84a43cSPoul-Henning Kamp rq->bio_queue_length++; 120dd84a43cSPoul-Henning Kamp g_bioq_unlock(rq); 121dd84a43cSPoul-Henning Kamp } 122dd84a43cSPoul-Henning Kamp 123dd84a43cSPoul-Henning Kamp struct bio * 124dd84a43cSPoul-Henning Kamp g_new_bio(void) 125dd84a43cSPoul-Henning Kamp { 126dd84a43cSPoul-Henning Kamp struct bio *bp; 127dd84a43cSPoul-Henning Kamp 128f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_idle); 129dd84a43cSPoul-Henning Kamp bp = g_bioq_first(&g_bio_idle); 130f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_idle); 131dd84a43cSPoul-Henning Kamp if (bp == NULL) 132a1bd3ee2SPoul-Henning Kamp bp = g_malloc(sizeof *bp, M_NOWAIT | M_ZERO); 1330355b86eSPoul-Henning Kamp /* g_trace(G_T_BIO, "g_new_bio() = %p", bp); */ 134dd84a43cSPoul-Henning Kamp return (bp); 135dd84a43cSPoul-Henning Kamp } 136dd84a43cSPoul-Henning Kamp 137dd84a43cSPoul-Henning Kamp void 138dd84a43cSPoul-Henning Kamp g_destroy_bio(struct bio *bp) 139dd84a43cSPoul-Henning Kamp { 140dd84a43cSPoul-Henning Kamp 1410355b86eSPoul-Henning Kamp /* g_trace(G_T_BIO, "g_destroy_bio(%p)", bp); */ 142dd84a43cSPoul-Henning Kamp bzero(bp, sizeof *bp); 143dd84a43cSPoul-Henning Kamp g_bioq_enqueue_tail(bp, &g_bio_idle); 144dd84a43cSPoul-Henning Kamp } 145dd84a43cSPoul-Henning Kamp 146dd84a43cSPoul-Henning Kamp struct bio * 147dd84a43cSPoul-Henning Kamp g_clone_bio(struct bio *bp) 148dd84a43cSPoul-Henning Kamp { 149dd84a43cSPoul-Henning Kamp struct bio *bp2; 150dd84a43cSPoul-Henning Kamp 151dd84a43cSPoul-Henning Kamp bp2 = g_new_bio(); 152a1bd3ee2SPoul-Henning Kamp if (bp2 != NULL) { 153936cc461SPoul-Henning Kamp bp2->bio_parent = bp; 154dd84a43cSPoul-Henning Kamp bp2->bio_cmd = bp->bio_cmd; 155dd84a43cSPoul-Henning Kamp bp2->bio_length = bp->bio_length; 156dd84a43cSPoul-Henning Kamp bp2->bio_offset = bp->bio_offset; 157dd84a43cSPoul-Henning Kamp bp2->bio_data = bp->bio_data; 158dd84a43cSPoul-Henning Kamp bp2->bio_attribute = bp->bio_attribute; 159801bb689SPoul-Henning Kamp bp->bio_children++; 160a1bd3ee2SPoul-Henning Kamp } 1610355b86eSPoul-Henning Kamp /* g_trace(G_T_BIO, "g_clone_bio(%p) = %p", bp, bp2); */ 162dd84a43cSPoul-Henning Kamp return(bp2); 163dd84a43cSPoul-Henning Kamp } 164dd84a43cSPoul-Henning Kamp 165dd84a43cSPoul-Henning Kamp void 166dd84a43cSPoul-Henning Kamp g_io_init() 167dd84a43cSPoul-Henning Kamp { 168dd84a43cSPoul-Henning Kamp 169dd84a43cSPoul-Henning Kamp g_bioq_init(&g_bio_run_down); 170dd84a43cSPoul-Henning Kamp g_bioq_init(&g_bio_run_up); 171f0e185d7SPoul-Henning Kamp g_bioq_init(&g_bio_run_task); 172dd84a43cSPoul-Henning Kamp g_bioq_init(&g_bio_idle); 173dd84a43cSPoul-Henning Kamp } 174dd84a43cSPoul-Henning Kamp 175dd84a43cSPoul-Henning Kamp int 1760d3f37a8SPoul-Henning Kamp g_io_setattr(const char *attr, struct g_consumer *cp, int len, void *ptr) 177dd84a43cSPoul-Henning Kamp { 178dd84a43cSPoul-Henning Kamp struct bio *bp; 179dd84a43cSPoul-Henning Kamp int error; 180dd84a43cSPoul-Henning Kamp 181dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, "bio_setattr(%s)", attr); 182dd84a43cSPoul-Henning Kamp bp = g_new_bio(); 183dd84a43cSPoul-Henning Kamp bp->bio_cmd = BIO_SETATTR; 184dd84a43cSPoul-Henning Kamp bp->bio_done = NULL; 185dd84a43cSPoul-Henning Kamp bp->bio_attribute = attr; 186dd84a43cSPoul-Henning Kamp bp->bio_length = len; 187dd84a43cSPoul-Henning Kamp bp->bio_data = ptr; 188dd84a43cSPoul-Henning Kamp g_io_request(bp, cp); 18953706245SPoul-Henning Kamp error = biowait(bp, "gsetattr"); 190dd84a43cSPoul-Henning Kamp g_destroy_bio(bp); 191dd84a43cSPoul-Henning Kamp return (error); 192dd84a43cSPoul-Henning Kamp } 193dd84a43cSPoul-Henning Kamp 194dd84a43cSPoul-Henning Kamp 195dd84a43cSPoul-Henning Kamp int 1960d3f37a8SPoul-Henning Kamp g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 197dd84a43cSPoul-Henning Kamp { 198dd84a43cSPoul-Henning Kamp struct bio *bp; 199dd84a43cSPoul-Henning Kamp int error; 200dd84a43cSPoul-Henning Kamp 201dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, "bio_getattr(%s)", attr); 202dd84a43cSPoul-Henning Kamp bp = g_new_bio(); 203dd84a43cSPoul-Henning Kamp bp->bio_cmd = BIO_GETATTR; 204dd84a43cSPoul-Henning Kamp bp->bio_done = NULL; 205dd84a43cSPoul-Henning Kamp bp->bio_attribute = attr; 206dd84a43cSPoul-Henning Kamp bp->bio_length = *len; 207dd84a43cSPoul-Henning Kamp bp->bio_data = ptr; 208dd84a43cSPoul-Henning Kamp g_io_request(bp, cp); 20953706245SPoul-Henning Kamp error = biowait(bp, "ggetattr"); 210dd84a43cSPoul-Henning Kamp *len = bp->bio_completed; 211dd84a43cSPoul-Henning Kamp g_destroy_bio(bp); 212dd84a43cSPoul-Henning Kamp return (error); 213dd84a43cSPoul-Henning Kamp } 214dd84a43cSPoul-Henning Kamp 215e39d70d4SPoul-Henning Kamp static int 216e39d70d4SPoul-Henning Kamp g_io_check(struct bio *bp) 217e39d70d4SPoul-Henning Kamp { 218e39d70d4SPoul-Henning Kamp struct g_consumer *cp; 219e39d70d4SPoul-Henning Kamp struct g_provider *pp; 220e39d70d4SPoul-Henning Kamp 221e39d70d4SPoul-Henning Kamp cp = bp->bio_from; 222e39d70d4SPoul-Henning Kamp pp = bp->bio_to; 223e39d70d4SPoul-Henning Kamp 224e39d70d4SPoul-Henning Kamp /* Fail if access counters dont allow the operation */ 225e39d70d4SPoul-Henning Kamp switch(bp->bio_cmd) { 226e39d70d4SPoul-Henning Kamp case BIO_READ: 227e39d70d4SPoul-Henning Kamp case BIO_GETATTR: 228e39d70d4SPoul-Henning Kamp if (cp->acr == 0) 229e39d70d4SPoul-Henning Kamp return (EPERM); 230e39d70d4SPoul-Henning Kamp break; 231e39d70d4SPoul-Henning Kamp case BIO_WRITE: 232e39d70d4SPoul-Henning Kamp case BIO_DELETE: 233e39d70d4SPoul-Henning Kamp case BIO_SETATTR: 234e39d70d4SPoul-Henning Kamp if (cp->acw == 0) 235e39d70d4SPoul-Henning Kamp return (EPERM); 236e39d70d4SPoul-Henning Kamp break; 237e39d70d4SPoul-Henning Kamp default: 238e39d70d4SPoul-Henning Kamp return (EPERM); 239e39d70d4SPoul-Henning Kamp } 240e39d70d4SPoul-Henning Kamp /* if provider is marked for error, don't disturb. */ 241e39d70d4SPoul-Henning Kamp if (pp->error) 242e39d70d4SPoul-Henning Kamp return (pp->error); 243e39d70d4SPoul-Henning Kamp 244e39d70d4SPoul-Henning Kamp switch(bp->bio_cmd) { 245e39d70d4SPoul-Henning Kamp case BIO_READ: 246e39d70d4SPoul-Henning Kamp case BIO_WRITE: 247e39d70d4SPoul-Henning Kamp case BIO_DELETE: 248e39d70d4SPoul-Henning Kamp /* Reject I/O not on sector boundary */ 249e39d70d4SPoul-Henning Kamp if (bp->bio_offset % pp->sectorsize) 250e39d70d4SPoul-Henning Kamp return (EINVAL); 251e39d70d4SPoul-Henning Kamp /* Reject I/O not integral sector long */ 252e39d70d4SPoul-Henning Kamp if (bp->bio_length % pp->sectorsize) 253e39d70d4SPoul-Henning Kamp return (EINVAL); 254e39d70d4SPoul-Henning Kamp /* Reject requests past the end of media. */ 255e39d70d4SPoul-Henning Kamp if (bp->bio_offset > pp->mediasize) 256e39d70d4SPoul-Henning Kamp return (EIO); 257e39d70d4SPoul-Henning Kamp break; 258e39d70d4SPoul-Henning Kamp default: 259e39d70d4SPoul-Henning Kamp break; 260e39d70d4SPoul-Henning Kamp } 261e39d70d4SPoul-Henning Kamp return (0); 262e39d70d4SPoul-Henning Kamp } 263e39d70d4SPoul-Henning Kamp 264dd84a43cSPoul-Henning Kamp void 265dd84a43cSPoul-Henning Kamp g_io_request(struct bio *bp, struct g_consumer *cp) 266dd84a43cSPoul-Henning Kamp { 267801bb689SPoul-Henning Kamp struct g_provider *pp; 268801bb689SPoul-Henning Kamp struct bintime bt; 269dd84a43cSPoul-Henning Kamp 270801bb689SPoul-Henning Kamp pp = cp->provider; 271d0e17c1bSPoul-Henning Kamp KASSERT(cp != NULL, ("NULL cp in g_io_request")); 272d0e17c1bSPoul-Henning Kamp KASSERT(bp != NULL, ("NULL bp in g_io_request")); 273d0e17c1bSPoul-Henning Kamp KASSERT(bp->bio_data != NULL, ("NULL bp->data in g_io_request")); 274801bb689SPoul-Henning Kamp KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 275801bb689SPoul-Henning Kamp 276dd84a43cSPoul-Henning Kamp bp->bio_from = cp; 277801bb689SPoul-Henning Kamp bp->bio_to = pp; 2782fccec19SPoul-Henning Kamp bp->bio_error = 0; 2792fccec19SPoul-Henning Kamp bp->bio_completed = 0; 280dd84a43cSPoul-Henning Kamp 281801bb689SPoul-Henning Kamp if (g_collectstats) { 282cce7303aSPoul-Henning Kamp binuptime(&bt); 283cce7303aSPoul-Henning Kamp bp->bio_t0 = bt; 284c6ae9b5fSPoul-Henning Kamp if (cp->nstart == cp->nend) 285cce7303aSPoul-Henning Kamp cp->stat->wentbusy = bt; /* Consumer is idle */ 286c6ae9b5fSPoul-Henning Kamp if (pp->nstart == pp->nend) 287cce7303aSPoul-Henning Kamp pp->stat->wentbusy = bt; /* Provider is idle */ 2884ec35300SPoul-Henning Kamp cp->stat->nop++; 2894ec35300SPoul-Henning Kamp pp->stat->nop++; 290c6ae9b5fSPoul-Henning Kamp } 291c6ae9b5fSPoul-Henning Kamp cp->nstart++; 292c6ae9b5fSPoul-Henning Kamp pp->nstart++; 293e39d70d4SPoul-Henning Kamp 2942fccec19SPoul-Henning Kamp /* Pass it on down. */ 295dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 296801bb689SPoul-Henning Kamp bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 297dd84a43cSPoul-Henning Kamp g_bioq_enqueue_tail(bp, &g_bio_run_down); 298dd84a43cSPoul-Henning Kamp wakeup(&g_wait_down); 299dd84a43cSPoul-Henning Kamp } 300dd84a43cSPoul-Henning Kamp 301dd84a43cSPoul-Henning Kamp void 30272840432SPoul-Henning Kamp g_io_deliver(struct bio *bp, int error) 303dd84a43cSPoul-Henning Kamp { 304801bb689SPoul-Henning Kamp struct g_consumer *cp; 305801bb689SPoul-Henning Kamp struct g_provider *pp; 306cce7303aSPoul-Henning Kamp struct bintime t1, dt; 307801bb689SPoul-Henning Kamp int idx; 308dd84a43cSPoul-Henning Kamp 309801bb689SPoul-Henning Kamp cp = bp->bio_from; 310801bb689SPoul-Henning Kamp pp = bp->bio_to; 311d0e17c1bSPoul-Henning Kamp KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 312801bb689SPoul-Henning Kamp KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 313801bb689SPoul-Henning Kamp KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 314801bb689SPoul-Henning Kamp KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 3155ab413bfSPoul-Henning Kamp 316dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, 3170355b86eSPoul-Henning Kamp "g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 318801bb689SPoul-Henning Kamp bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 3190355b86eSPoul-Henning Kamp (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 320801bb689SPoul-Henning Kamp 321cce7303aSPoul-Henning Kamp if (g_collectstats) { 322801bb689SPoul-Henning Kamp switch (bp->bio_cmd) { 323801bb689SPoul-Henning Kamp case BIO_READ: idx = G_STAT_IDX_READ; break; 324801bb689SPoul-Henning Kamp case BIO_WRITE: idx = G_STAT_IDX_WRITE; break; 325801bb689SPoul-Henning Kamp case BIO_DELETE: idx = G_STAT_IDX_DELETE; break; 326801bb689SPoul-Henning Kamp case BIO_GETATTR: idx = -1; break; 327801bb689SPoul-Henning Kamp case BIO_SETATTR: idx = -1; break; 328801bb689SPoul-Henning Kamp default: 329801bb689SPoul-Henning Kamp panic("unknown bio_cmd in g_io_deliver"); 330801bb689SPoul-Henning Kamp break; 331801bb689SPoul-Henning Kamp } 332801bb689SPoul-Henning Kamp binuptime(&t1); 333cce7303aSPoul-Henning Kamp /* Raise the "inconsistent" flag for userland */ 33481e9eba3SPoul-Henning Kamp atomic_add_acq_int(&cp->stat->seq0, 1); 33581e9eba3SPoul-Henning Kamp atomic_add_acq_int(&pp->stat->seq0, 1); 336801bb689SPoul-Henning Kamp if (idx >= 0) { 337cce7303aSPoul-Henning Kamp /* Account the service time */ 338cce7303aSPoul-Henning Kamp dt = t1; 339cce7303aSPoul-Henning Kamp bintime_sub(&dt, &bp->bio_t0); 340cce7303aSPoul-Henning Kamp bintime_add(&cp->stat->ops[idx].dt, &dt); 341cce7303aSPoul-Henning Kamp bintime_add(&pp->stat->ops[idx].dt, &dt); 342cce7303aSPoul-Henning Kamp /* ... and the metrics */ 3434ec35300SPoul-Henning Kamp pp->stat->ops[idx].nbyte += bp->bio_completed; 3444ec35300SPoul-Henning Kamp cp->stat->ops[idx].nbyte += bp->bio_completed; 3454ec35300SPoul-Henning Kamp pp->stat->ops[idx].nop++; 3464ec35300SPoul-Henning Kamp cp->stat->ops[idx].nop++; 347cce7303aSPoul-Henning Kamp /* ... and any errors */ 348801bb689SPoul-Henning Kamp if (error == ENOMEM) { 3494ec35300SPoul-Henning Kamp cp->stat->ops[idx].nmem++; 3504ec35300SPoul-Henning Kamp pp->stat->ops[idx].nmem++; 351801bb689SPoul-Henning Kamp } else if (error != 0) { 3524ec35300SPoul-Henning Kamp cp->stat->ops[idx].nerr++; 3534ec35300SPoul-Henning Kamp pp->stat->ops[idx].nerr++; 354801bb689SPoul-Henning Kamp } 355801bb689SPoul-Henning Kamp } 356cce7303aSPoul-Henning Kamp /* Account for busy time on the consumer */ 357cce7303aSPoul-Henning Kamp dt = t1; 358cce7303aSPoul-Henning Kamp bintime_sub(&dt, &cp->stat->wentbusy); 359cce7303aSPoul-Henning Kamp bintime_add(&cp->stat->bt, &dt); 360cce7303aSPoul-Henning Kamp cp->stat->wentbusy = t1; 361cce7303aSPoul-Henning Kamp /* Account for busy time on the provider */ 362cce7303aSPoul-Henning Kamp dt = t1; 363cce7303aSPoul-Henning Kamp bintime_sub(&dt, &pp->stat->wentbusy); 364cce7303aSPoul-Henning Kamp bintime_add(&pp->stat->bt, &dt); 365cce7303aSPoul-Henning Kamp pp->stat->wentbusy = t1; 366cce7303aSPoul-Henning Kamp /* Mark the structures as consistent again */ 36781e9eba3SPoul-Henning Kamp atomic_add_acq_int(&cp->stat->seq1, 1); 36881e9eba3SPoul-Henning Kamp atomic_add_acq_int(&pp->stat->seq1, 1); 3694ec35300SPoul-Henning Kamp cp->stat->nend++; 370cce7303aSPoul-Henning Kamp pp->stat->nend++; 371c6ae9b5fSPoul-Henning Kamp } 372c6ae9b5fSPoul-Henning Kamp cp->nend++; 373c6ae9b5fSPoul-Henning Kamp pp->nend++; 374dd84a43cSPoul-Henning Kamp 3753432e4fdSPoul-Henning Kamp if (error == ENOMEM) { 376801bb689SPoul-Henning Kamp printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 377801bb689SPoul-Henning Kamp g_io_request(bp, cp); 3783432e4fdSPoul-Henning Kamp pace++; 3793432e4fdSPoul-Henning Kamp return; 3803432e4fdSPoul-Henning Kamp } 38172840432SPoul-Henning Kamp bp->bio_error = error; 382dd84a43cSPoul-Henning Kamp g_bioq_enqueue_tail(bp, &g_bio_run_up); 383dd84a43cSPoul-Henning Kamp wakeup(&g_wait_up); 384dd84a43cSPoul-Henning Kamp } 385dd84a43cSPoul-Henning Kamp 386dd84a43cSPoul-Henning Kamp void 387dd84a43cSPoul-Henning Kamp g_io_schedule_down(struct thread *tp __unused) 388dd84a43cSPoul-Henning Kamp { 389dd84a43cSPoul-Henning Kamp struct bio *bp; 390e39d70d4SPoul-Henning Kamp off_t excess; 391e39d70d4SPoul-Henning Kamp int error; 392f0e185d7SPoul-Henning Kamp struct mtx mymutex; 393f0e185d7SPoul-Henning Kamp 394f0e185d7SPoul-Henning Kamp bzero(&mymutex, sizeof mymutex); 395f0e185d7SPoul-Henning Kamp mtx_init(&mymutex, "g_xdown", MTX_DEF, 0); 396dd84a43cSPoul-Henning Kamp 397dd84a43cSPoul-Henning Kamp for(;;) { 398f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_run_down); 399dd84a43cSPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_down); 400f0e185d7SPoul-Henning Kamp if (bp == NULL) { 401f0e185d7SPoul-Henning Kamp msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 402f0e185d7SPoul-Henning Kamp PRIBIO | PDROP, "g_down", hz/10); 403f0e185d7SPoul-Henning Kamp continue; 404f0e185d7SPoul-Henning Kamp } 405f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_down); 406e39d70d4SPoul-Henning Kamp error = g_io_check(bp); 407e39d70d4SPoul-Henning Kamp if (error) { 408e39d70d4SPoul-Henning Kamp g_io_deliver(bp, error); 409e39d70d4SPoul-Henning Kamp continue; 410e39d70d4SPoul-Henning Kamp } 411392d56b4SPoul-Henning Kamp switch (bp->bio_cmd) { 412392d56b4SPoul-Henning Kamp case BIO_READ: 413392d56b4SPoul-Henning Kamp case BIO_WRITE: 414392d56b4SPoul-Henning Kamp case BIO_DELETE: 415e39d70d4SPoul-Henning Kamp /* Truncate requests to the end of providers media. */ 416e39d70d4SPoul-Henning Kamp excess = bp->bio_offset + bp->bio_length; 417e39d70d4SPoul-Henning Kamp if (excess > bp->bio_to->mediasize) { 418e39d70d4SPoul-Henning Kamp excess -= bp->bio_to->mediasize; 419e39d70d4SPoul-Henning Kamp bp->bio_length -= excess; 420e39d70d4SPoul-Henning Kamp } 421e39d70d4SPoul-Henning Kamp /* Deliver zero length transfers right here. */ 422e39d70d4SPoul-Henning Kamp if (bp->bio_length == 0) { 423e39d70d4SPoul-Henning Kamp g_io_deliver(bp, 0); 424e39d70d4SPoul-Henning Kamp continue; 425e39d70d4SPoul-Henning Kamp } 426392d56b4SPoul-Henning Kamp break; 427392d56b4SPoul-Henning Kamp default: 428392d56b4SPoul-Henning Kamp break; 429392d56b4SPoul-Henning Kamp } 430f0e185d7SPoul-Henning Kamp mtx_lock(&mymutex); 431dd84a43cSPoul-Henning Kamp bp->bio_to->geom->start(bp); 432f0e185d7SPoul-Henning Kamp mtx_unlock(&mymutex); 4333432e4fdSPoul-Henning Kamp if (pace) { 4343432e4fdSPoul-Henning Kamp pace--; 4353432e4fdSPoul-Henning Kamp break; 4363432e4fdSPoul-Henning Kamp } 437dd84a43cSPoul-Henning Kamp } 438dd84a43cSPoul-Henning Kamp } 439dd84a43cSPoul-Henning Kamp 440dd84a43cSPoul-Henning Kamp void 441f0e185d7SPoul-Henning Kamp bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg) 442f0e185d7SPoul-Henning Kamp { 443f0e185d7SPoul-Henning Kamp bp->bio_task = func; 444f0e185d7SPoul-Henning Kamp bp->bio_task_arg = arg; 445f0e185d7SPoul-Henning Kamp /* 446f0e185d7SPoul-Henning Kamp * The taskqueue is actually just a second queue off the "up" 447f0e185d7SPoul-Henning Kamp * queue, so we use the same lock. 448f0e185d7SPoul-Henning Kamp */ 449f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_run_up); 450f0e185d7SPoul-Henning Kamp TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue); 451f0e185d7SPoul-Henning Kamp g_bio_run_task.bio_queue_length++; 452f0e185d7SPoul-Henning Kamp wakeup(&g_wait_up); 453f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 454f0e185d7SPoul-Henning Kamp } 455f0e185d7SPoul-Henning Kamp 456f0e185d7SPoul-Henning Kamp 457f0e185d7SPoul-Henning Kamp void 458dd84a43cSPoul-Henning Kamp g_io_schedule_up(struct thread *tp __unused) 459dd84a43cSPoul-Henning Kamp { 460dd84a43cSPoul-Henning Kamp struct bio *bp; 461f0e185d7SPoul-Henning Kamp struct mtx mymutex; 462dd84a43cSPoul-Henning Kamp 463f0e185d7SPoul-Henning Kamp bzero(&mymutex, sizeof mymutex); 464f0e185d7SPoul-Henning Kamp mtx_init(&mymutex, "g_xup", MTX_DEF, 0); 465dd84a43cSPoul-Henning Kamp for(;;) { 466f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_run_up); 467f0e185d7SPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_task); 468f0e185d7SPoul-Henning Kamp if (bp != NULL) { 469f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 470f0e185d7SPoul-Henning Kamp mtx_lock(&mymutex); 471f0e185d7SPoul-Henning Kamp bp->bio_task(bp, bp->bio_task_arg); 472f0e185d7SPoul-Henning Kamp mtx_unlock(&mymutex); 473f0e185d7SPoul-Henning Kamp continue; 474f0e185d7SPoul-Henning Kamp } 475dd84a43cSPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_up); 476f0e185d7SPoul-Henning Kamp if (bp != NULL) { 477f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 478f0e185d7SPoul-Henning Kamp mtx_lock(&mymutex); 47953706245SPoul-Henning Kamp biodone(bp); 480f0e185d7SPoul-Henning Kamp mtx_unlock(&mymutex); 481f0e185d7SPoul-Henning Kamp continue; 482f0e185d7SPoul-Henning Kamp } 483f0e185d7SPoul-Henning Kamp msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 484f0e185d7SPoul-Henning Kamp PRIBIO | PDROP, "g_up", hz/10); 485dd84a43cSPoul-Henning Kamp } 486dd84a43cSPoul-Henning Kamp } 487dd84a43cSPoul-Henning Kamp 488dd84a43cSPoul-Henning Kamp void * 489dd84a43cSPoul-Henning Kamp g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 490dd84a43cSPoul-Henning Kamp { 491dd84a43cSPoul-Henning Kamp struct bio *bp; 492dd84a43cSPoul-Henning Kamp void *ptr; 493dd84a43cSPoul-Henning Kamp int errorc; 494dd84a43cSPoul-Henning Kamp 495dd84a43cSPoul-Henning Kamp bp = g_new_bio(); 496dd84a43cSPoul-Henning Kamp bp->bio_cmd = BIO_READ; 497dd84a43cSPoul-Henning Kamp bp->bio_done = NULL; 498dd84a43cSPoul-Henning Kamp bp->bio_offset = offset; 499dd84a43cSPoul-Henning Kamp bp->bio_length = length; 500a163d034SWarner Losh ptr = g_malloc(length, M_WAITOK); 501dd84a43cSPoul-Henning Kamp bp->bio_data = ptr; 502dd84a43cSPoul-Henning Kamp g_io_request(bp, cp); 50353706245SPoul-Henning Kamp errorc = biowait(bp, "gread"); 504dd84a43cSPoul-Henning Kamp if (error != NULL) 505dd84a43cSPoul-Henning Kamp *error = errorc; 506dd84a43cSPoul-Henning Kamp g_destroy_bio(bp); 507dd84a43cSPoul-Henning Kamp if (errorc) { 508dd84a43cSPoul-Henning Kamp g_free(ptr); 509dd84a43cSPoul-Henning Kamp ptr = NULL; 510dd84a43cSPoul-Henning Kamp } 511dd84a43cSPoul-Henning Kamp return (ptr); 512dd84a43cSPoul-Henning Kamp } 51390b1cd56SPoul-Henning Kamp 51490b1cd56SPoul-Henning Kamp int 51590b1cd56SPoul-Henning Kamp g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 51690b1cd56SPoul-Henning Kamp { 51790b1cd56SPoul-Henning Kamp struct bio *bp; 51890b1cd56SPoul-Henning Kamp int error; 51990b1cd56SPoul-Henning Kamp 52090b1cd56SPoul-Henning Kamp bp = g_new_bio(); 52190b1cd56SPoul-Henning Kamp bp->bio_cmd = BIO_WRITE; 52290b1cd56SPoul-Henning Kamp bp->bio_done = NULL; 52390b1cd56SPoul-Henning Kamp bp->bio_offset = offset; 52490b1cd56SPoul-Henning Kamp bp->bio_length = length; 52590b1cd56SPoul-Henning Kamp bp->bio_data = ptr; 52690b1cd56SPoul-Henning Kamp g_io_request(bp, cp); 52790b1cd56SPoul-Henning Kamp error = biowait(bp, "gwrite"); 52890b1cd56SPoul-Henning Kamp g_destroy_bio(bp); 52990b1cd56SPoul-Henning Kamp return (error); 53090b1cd56SPoul-Henning Kamp } 531