1dd84a43cSPoul-Henning Kamp /*- 2dd84a43cSPoul-Henning Kamp * Copyright (c) 2002 Poul-Henning Kamp 3dd84a43cSPoul-Henning Kamp * Copyright (c) 2002 Networks Associates Technology, Inc. 4dd84a43cSPoul-Henning Kamp * All rights reserved. 5dd84a43cSPoul-Henning Kamp * 6dd84a43cSPoul-Henning Kamp * This software was developed for the FreeBSD Project by Poul-Henning Kamp 7dd84a43cSPoul-Henning Kamp * and NAI Labs, the Security Research Division of Network Associates, Inc. 8dd84a43cSPoul-Henning Kamp * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the 9dd84a43cSPoul-Henning Kamp * DARPA CHATS research program. 10dd84a43cSPoul-Henning Kamp * 11dd84a43cSPoul-Henning Kamp * Redistribution and use in source and binary forms, with or without 12dd84a43cSPoul-Henning Kamp * modification, are permitted provided that the following conditions 13dd84a43cSPoul-Henning Kamp * are met: 14dd84a43cSPoul-Henning Kamp * 1. Redistributions of source code must retain the above copyright 15dd84a43cSPoul-Henning Kamp * notice, this list of conditions and the following disclaimer. 16dd84a43cSPoul-Henning Kamp * 2. Redistributions in binary form must reproduce the above copyright 17dd84a43cSPoul-Henning Kamp * notice, this list of conditions and the following disclaimer in the 18dd84a43cSPoul-Henning Kamp * documentation and/or other materials provided with the distribution. 19dd84a43cSPoul-Henning Kamp * 3. The names of the authors may not be used to endorse or promote 20dd84a43cSPoul-Henning Kamp * products derived from this software without specific prior written 21dd84a43cSPoul-Henning Kamp * permission. 22dd84a43cSPoul-Henning Kamp * 23dd84a43cSPoul-Henning Kamp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24dd84a43cSPoul-Henning Kamp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25dd84a43cSPoul-Henning Kamp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26dd84a43cSPoul-Henning Kamp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27dd84a43cSPoul-Henning Kamp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28dd84a43cSPoul-Henning Kamp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29dd84a43cSPoul-Henning Kamp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30dd84a43cSPoul-Henning Kamp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31dd84a43cSPoul-Henning Kamp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32dd84a43cSPoul-Henning Kamp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33dd84a43cSPoul-Henning Kamp * SUCH DAMAGE. 34dd84a43cSPoul-Henning Kamp */ 35dd84a43cSPoul-Henning Kamp 3650b1faefSDavid E. O'Brien #include <sys/cdefs.h> 3750b1faefSDavid E. O'Brien __FBSDID("$FreeBSD$"); 38dd84a43cSPoul-Henning Kamp 39dd84a43cSPoul-Henning Kamp #include <sys/param.h> 40dd84a43cSPoul-Henning Kamp #include <sys/systm.h> 41dd84a43cSPoul-Henning Kamp #include <sys/kernel.h> 42dd84a43cSPoul-Henning Kamp #include <sys/malloc.h> 43dd84a43cSPoul-Henning Kamp #include <sys/bio.h> 4449dbb61dSRobert Watson #include <sys/ktr.h> 4551460da8SJohn Baldwin #include <sys/proc.h> 463b378147SPawel Jakub Dawidek #include <sys/stack.h> 47dd84a43cSPoul-Henning Kamp 48dd84a43cSPoul-Henning Kamp #include <sys/errno.h> 49dd84a43cSPoul-Henning Kamp #include <geom/geom.h> 50b1876192SPoul-Henning Kamp #include <geom/geom_int.h> 51e24cbd90SPoul-Henning Kamp #include <sys/devicestat.h> 52dd84a43cSPoul-Henning Kamp 535ffb2c8bSPoul-Henning Kamp #include <vm/uma.h> 545ffb2c8bSPoul-Henning Kamp 55dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_down; 56dd84a43cSPoul-Henning Kamp static struct g_bioq g_bio_run_up; 575fcf4e43SPoul-Henning Kamp static struct g_bioq g_bio_run_task; 58dd84a43cSPoul-Henning Kamp 593432e4fdSPoul-Henning Kamp static u_int pace; 605ffb2c8bSPoul-Henning Kamp static uma_zone_t biozone; 613432e4fdSPoul-Henning Kamp 626231f75bSLuigi Rizzo /* 636231f75bSLuigi Rizzo * The head of the list of classifiers used in g_io_request. 646231f75bSLuigi Rizzo * Use g_register_classifier() and g_unregister_classifier() 656231f75bSLuigi Rizzo * to add/remove entries to the list. 666231f75bSLuigi Rizzo * Classifiers are invoked in registration order. 676231f75bSLuigi Rizzo */ 686231f75bSLuigi Rizzo static TAILQ_HEAD(g_classifier_tailq, g_classifier_hook) 696231f75bSLuigi Rizzo g_classifier_tailq = TAILQ_HEAD_INITIALIZER(g_classifier_tailq); 706231f75bSLuigi Rizzo 71dd84a43cSPoul-Henning Kamp #include <machine/atomic.h> 72dd84a43cSPoul-Henning Kamp 73dd84a43cSPoul-Henning Kamp static void 74dd84a43cSPoul-Henning Kamp g_bioq_lock(struct g_bioq *bq) 75dd84a43cSPoul-Henning Kamp { 76dd84a43cSPoul-Henning Kamp 77dd84a43cSPoul-Henning Kamp mtx_lock(&bq->bio_queue_lock); 78dd84a43cSPoul-Henning Kamp } 79dd84a43cSPoul-Henning Kamp 80dd84a43cSPoul-Henning Kamp static void 81dd84a43cSPoul-Henning Kamp g_bioq_unlock(struct g_bioq *bq) 82dd84a43cSPoul-Henning Kamp { 83dd84a43cSPoul-Henning Kamp 84dd84a43cSPoul-Henning Kamp mtx_unlock(&bq->bio_queue_lock); 85dd84a43cSPoul-Henning Kamp } 86dd84a43cSPoul-Henning Kamp 87dd84a43cSPoul-Henning Kamp #if 0 88dd84a43cSPoul-Henning Kamp static void 89dd84a43cSPoul-Henning Kamp g_bioq_destroy(struct g_bioq *bq) 90dd84a43cSPoul-Henning Kamp { 91dd84a43cSPoul-Henning Kamp 92dd84a43cSPoul-Henning Kamp mtx_destroy(&bq->bio_queue_lock); 93dd84a43cSPoul-Henning Kamp } 94dd84a43cSPoul-Henning Kamp #endif 95dd84a43cSPoul-Henning Kamp 96dd84a43cSPoul-Henning Kamp static void 97dd84a43cSPoul-Henning Kamp g_bioq_init(struct g_bioq *bq) 98dd84a43cSPoul-Henning Kamp { 99dd84a43cSPoul-Henning Kamp 100dd84a43cSPoul-Henning Kamp TAILQ_INIT(&bq->bio_queue); 1016008862bSJohn Baldwin mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF); 102dd84a43cSPoul-Henning Kamp } 103dd84a43cSPoul-Henning Kamp 104dd84a43cSPoul-Henning Kamp static struct bio * 105dd84a43cSPoul-Henning Kamp g_bioq_first(struct g_bioq *bq) 106dd84a43cSPoul-Henning Kamp { 107dd84a43cSPoul-Henning Kamp struct bio *bp; 108dd84a43cSPoul-Henning Kamp 109dd84a43cSPoul-Henning Kamp bp = TAILQ_FIRST(&bq->bio_queue); 110dd84a43cSPoul-Henning Kamp if (bp != NULL) { 111dcbd0fe5SPoul-Henning Kamp KASSERT((bp->bio_flags & BIO_ONQUEUE), 112dcbd0fe5SPoul-Henning Kamp ("Bio not on queue bp=%p target %p", bp, bq)); 113dcbd0fe5SPoul-Henning Kamp bp->bio_flags &= ~BIO_ONQUEUE; 114dd84a43cSPoul-Henning Kamp TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue); 115dd84a43cSPoul-Henning Kamp bq->bio_queue_length--; 116dd84a43cSPoul-Henning Kamp } 117dd84a43cSPoul-Henning Kamp return (bp); 118dd84a43cSPoul-Henning Kamp } 119dd84a43cSPoul-Henning Kamp 120dd84a43cSPoul-Henning Kamp struct bio * 121dd84a43cSPoul-Henning Kamp g_new_bio(void) 122dd84a43cSPoul-Henning Kamp { 123dd84a43cSPoul-Henning Kamp struct bio *bp; 124dd84a43cSPoul-Henning Kamp 1255ffb2c8bSPoul-Henning Kamp bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 1263b378147SPawel Jakub Dawidek #ifdef KTR 127b656c1b8SPawel Jakub Dawidek if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 1283b378147SPawel Jakub Dawidek struct stack st; 1293b378147SPawel Jakub Dawidek 1303b378147SPawel Jakub Dawidek CTR1(KTR_GEOM, "g_new_bio(): %p", bp); 1313b378147SPawel Jakub Dawidek stack_save(&st); 1323b378147SPawel Jakub Dawidek CTRSTACK(KTR_GEOM, &st, 3, 0); 1333b378147SPawel Jakub Dawidek } 1343b378147SPawel Jakub Dawidek #endif 135dd84a43cSPoul-Henning Kamp return (bp); 136dd84a43cSPoul-Henning Kamp } 137dd84a43cSPoul-Henning Kamp 138a2033c96SPoul-Henning Kamp struct bio * 139a2033c96SPoul-Henning Kamp g_alloc_bio(void) 140a2033c96SPoul-Henning Kamp { 141a2033c96SPoul-Henning Kamp struct bio *bp; 142a2033c96SPoul-Henning Kamp 143a2033c96SPoul-Henning Kamp bp = uma_zalloc(biozone, M_WAITOK | M_ZERO); 1443b378147SPawel Jakub Dawidek #ifdef KTR 145b656c1b8SPawel Jakub Dawidek if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 1463b378147SPawel Jakub Dawidek struct stack st; 1473b378147SPawel Jakub Dawidek 1483b378147SPawel Jakub Dawidek CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp); 1493b378147SPawel Jakub Dawidek stack_save(&st); 1503b378147SPawel Jakub Dawidek CTRSTACK(KTR_GEOM, &st, 3, 0); 1513b378147SPawel Jakub Dawidek } 1523b378147SPawel Jakub Dawidek #endif 153a2033c96SPoul-Henning Kamp return (bp); 154a2033c96SPoul-Henning Kamp } 155a2033c96SPoul-Henning Kamp 156dd84a43cSPoul-Henning Kamp void 157dd84a43cSPoul-Henning Kamp g_destroy_bio(struct bio *bp) 158dd84a43cSPoul-Henning Kamp { 1593b378147SPawel Jakub Dawidek #ifdef KTR 160b656c1b8SPawel Jakub Dawidek if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 1613b378147SPawel Jakub Dawidek struct stack st; 162dd84a43cSPoul-Henning Kamp 1633b378147SPawel Jakub Dawidek CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp); 1643b378147SPawel Jakub Dawidek stack_save(&st); 1653b378147SPawel Jakub Dawidek CTRSTACK(KTR_GEOM, &st, 3, 0); 1663b378147SPawel Jakub Dawidek } 1673b378147SPawel Jakub Dawidek #endif 1685ffb2c8bSPoul-Henning Kamp uma_zfree(biozone, bp); 169dd84a43cSPoul-Henning Kamp } 170dd84a43cSPoul-Henning Kamp 171dd84a43cSPoul-Henning Kamp struct bio * 172dd84a43cSPoul-Henning Kamp g_clone_bio(struct bio *bp) 173dd84a43cSPoul-Henning Kamp { 174dd84a43cSPoul-Henning Kamp struct bio *bp2; 175dd84a43cSPoul-Henning Kamp 1765ffb2c8bSPoul-Henning Kamp bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO); 177a1bd3ee2SPoul-Henning Kamp if (bp2 != NULL) { 178936cc461SPoul-Henning Kamp bp2->bio_parent = bp; 179dd84a43cSPoul-Henning Kamp bp2->bio_cmd = bp->bio_cmd; 180dd84a43cSPoul-Henning Kamp bp2->bio_length = bp->bio_length; 181dd84a43cSPoul-Henning Kamp bp2->bio_offset = bp->bio_offset; 182dd84a43cSPoul-Henning Kamp bp2->bio_data = bp->bio_data; 183dd84a43cSPoul-Henning Kamp bp2->bio_attribute = bp->bio_attribute; 1846231f75bSLuigi Rizzo /* Inherit classification info from the parent */ 1856231f75bSLuigi Rizzo bp2->bio_classifier1 = bp->bio_classifier1; 1866231f75bSLuigi Rizzo bp2->bio_classifier2 = bp->bio_classifier2; 187801bb689SPoul-Henning Kamp bp->bio_children++; 188a1bd3ee2SPoul-Henning Kamp } 1893b378147SPawel Jakub Dawidek #ifdef KTR 190b656c1b8SPawel Jakub Dawidek if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 1913b378147SPawel Jakub Dawidek struct stack st; 1923b378147SPawel Jakub Dawidek 193ad572235SRuslan Ermilov CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2); 1943b378147SPawel Jakub Dawidek stack_save(&st); 1953b378147SPawel Jakub Dawidek CTRSTACK(KTR_GEOM, &st, 3, 0); 1963b378147SPawel Jakub Dawidek } 1973b378147SPawel Jakub Dawidek #endif 198dd84a43cSPoul-Henning Kamp return(bp2); 199dd84a43cSPoul-Henning Kamp } 200dd84a43cSPoul-Henning Kamp 2014bec0ff1SPawel Jakub Dawidek struct bio * 2024bec0ff1SPawel Jakub Dawidek g_duplicate_bio(struct bio *bp) 2034bec0ff1SPawel Jakub Dawidek { 2044bec0ff1SPawel Jakub Dawidek struct bio *bp2; 2054bec0ff1SPawel Jakub Dawidek 2064bec0ff1SPawel Jakub Dawidek bp2 = uma_zalloc(biozone, M_WAITOK | M_ZERO); 2074bec0ff1SPawel Jakub Dawidek bp2->bio_parent = bp; 2084bec0ff1SPawel Jakub Dawidek bp2->bio_cmd = bp->bio_cmd; 2094bec0ff1SPawel Jakub Dawidek bp2->bio_length = bp->bio_length; 2104bec0ff1SPawel Jakub Dawidek bp2->bio_offset = bp->bio_offset; 2114bec0ff1SPawel Jakub Dawidek bp2->bio_data = bp->bio_data; 2124bec0ff1SPawel Jakub Dawidek bp2->bio_attribute = bp->bio_attribute; 2134bec0ff1SPawel Jakub Dawidek bp->bio_children++; 2144bec0ff1SPawel Jakub Dawidek #ifdef KTR 215b656c1b8SPawel Jakub Dawidek if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) { 2164bec0ff1SPawel Jakub Dawidek struct stack st; 2174bec0ff1SPawel Jakub Dawidek 2184bec0ff1SPawel Jakub Dawidek CTR2(KTR_GEOM, "g_duplicate_bio(%p): %p", bp, bp2); 2194bec0ff1SPawel Jakub Dawidek stack_save(&st); 2204bec0ff1SPawel Jakub Dawidek CTRSTACK(KTR_GEOM, &st, 3, 0); 2214bec0ff1SPawel Jakub Dawidek } 2224bec0ff1SPawel Jakub Dawidek #endif 2234bec0ff1SPawel Jakub Dawidek return(bp2); 2244bec0ff1SPawel Jakub Dawidek } 2254bec0ff1SPawel Jakub Dawidek 226dd84a43cSPoul-Henning Kamp void 227dd84a43cSPoul-Henning Kamp g_io_init() 228dd84a43cSPoul-Henning Kamp { 229dd84a43cSPoul-Henning Kamp 230dd84a43cSPoul-Henning Kamp g_bioq_init(&g_bio_run_down); 231dd84a43cSPoul-Henning Kamp g_bioq_init(&g_bio_run_up); 2325fcf4e43SPoul-Henning Kamp g_bioq_init(&g_bio_run_task); 2335ffb2c8bSPoul-Henning Kamp biozone = uma_zcreate("g_bio", sizeof (struct bio), 2345ffb2c8bSPoul-Henning Kamp NULL, NULL, 2355ffb2c8bSPoul-Henning Kamp NULL, NULL, 2365ffb2c8bSPoul-Henning Kamp 0, 0); 237dd84a43cSPoul-Henning Kamp } 238dd84a43cSPoul-Henning Kamp 239dd84a43cSPoul-Henning Kamp int 2400d3f37a8SPoul-Henning Kamp g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr) 241dd84a43cSPoul-Henning Kamp { 242dd84a43cSPoul-Henning Kamp struct bio *bp; 243dd84a43cSPoul-Henning Kamp int error; 244dd84a43cSPoul-Henning Kamp 245dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, "bio_getattr(%s)", attr); 246a2033c96SPoul-Henning Kamp bp = g_alloc_bio(); 247dd84a43cSPoul-Henning Kamp bp->bio_cmd = BIO_GETATTR; 248dd84a43cSPoul-Henning Kamp bp->bio_done = NULL; 249dd84a43cSPoul-Henning Kamp bp->bio_attribute = attr; 250dd84a43cSPoul-Henning Kamp bp->bio_length = *len; 251dd84a43cSPoul-Henning Kamp bp->bio_data = ptr; 252dd84a43cSPoul-Henning Kamp g_io_request(bp, cp); 25353706245SPoul-Henning Kamp error = biowait(bp, "ggetattr"); 254dd84a43cSPoul-Henning Kamp *len = bp->bio_completed; 255dd84a43cSPoul-Henning Kamp g_destroy_bio(bp); 256dd84a43cSPoul-Henning Kamp return (error); 257dd84a43cSPoul-Henning Kamp } 258dd84a43cSPoul-Henning Kamp 259c3618c65SPawel Jakub Dawidek int 260c3618c65SPawel Jakub Dawidek g_io_flush(struct g_consumer *cp) 261c3618c65SPawel Jakub Dawidek { 262c3618c65SPawel Jakub Dawidek struct bio *bp; 263c3618c65SPawel Jakub Dawidek int error; 264c3618c65SPawel Jakub Dawidek 265c3618c65SPawel Jakub Dawidek g_trace(G_T_BIO, "bio_flush(%s)", cp->provider->name); 266c3618c65SPawel Jakub Dawidek bp = g_alloc_bio(); 267c3618c65SPawel Jakub Dawidek bp->bio_cmd = BIO_FLUSH; 268f03f7a0cSJustin T. Gibbs bp->bio_flags |= BIO_ORDERED; 269c3618c65SPawel Jakub Dawidek bp->bio_done = NULL; 270c3618c65SPawel Jakub Dawidek bp->bio_attribute = NULL; 271c3618c65SPawel Jakub Dawidek bp->bio_offset = cp->provider->mediasize; 272c3618c65SPawel Jakub Dawidek bp->bio_length = 0; 273c3618c65SPawel Jakub Dawidek bp->bio_data = NULL; 274c3618c65SPawel Jakub Dawidek g_io_request(bp, cp); 275c3618c65SPawel Jakub Dawidek error = biowait(bp, "gflush"); 276c3618c65SPawel Jakub Dawidek g_destroy_bio(bp); 277c3618c65SPawel Jakub Dawidek return (error); 278c3618c65SPawel Jakub Dawidek } 279c3618c65SPawel Jakub Dawidek 280e39d70d4SPoul-Henning Kamp static int 281e39d70d4SPoul-Henning Kamp g_io_check(struct bio *bp) 282e39d70d4SPoul-Henning Kamp { 283e39d70d4SPoul-Henning Kamp struct g_consumer *cp; 284e39d70d4SPoul-Henning Kamp struct g_provider *pp; 285e39d70d4SPoul-Henning Kamp 286e39d70d4SPoul-Henning Kamp cp = bp->bio_from; 287e39d70d4SPoul-Henning Kamp pp = bp->bio_to; 288e39d70d4SPoul-Henning Kamp 289e39d70d4SPoul-Henning Kamp /* Fail if access counters dont allow the operation */ 290e39d70d4SPoul-Henning Kamp switch(bp->bio_cmd) { 291e39d70d4SPoul-Henning Kamp case BIO_READ: 292e39d70d4SPoul-Henning Kamp case BIO_GETATTR: 293e39d70d4SPoul-Henning Kamp if (cp->acr == 0) 294e39d70d4SPoul-Henning Kamp return (EPERM); 295e39d70d4SPoul-Henning Kamp break; 296e39d70d4SPoul-Henning Kamp case BIO_WRITE: 297e39d70d4SPoul-Henning Kamp case BIO_DELETE: 298c3618c65SPawel Jakub Dawidek case BIO_FLUSH: 299e39d70d4SPoul-Henning Kamp if (cp->acw == 0) 300e39d70d4SPoul-Henning Kamp return (EPERM); 301e39d70d4SPoul-Henning Kamp break; 302e39d70d4SPoul-Henning Kamp default: 303e39d70d4SPoul-Henning Kamp return (EPERM); 304e39d70d4SPoul-Henning Kamp } 305e39d70d4SPoul-Henning Kamp /* if provider is marked for error, don't disturb. */ 306e39d70d4SPoul-Henning Kamp if (pp->error) 307e39d70d4SPoul-Henning Kamp return (pp->error); 308*3631c638SAlexander Motin if (cp->flags & G_CF_ORPHAN) 309*3631c638SAlexander Motin return (ENXIO); 310e39d70d4SPoul-Henning Kamp 311e39d70d4SPoul-Henning Kamp switch(bp->bio_cmd) { 312e39d70d4SPoul-Henning Kamp case BIO_READ: 313e39d70d4SPoul-Henning Kamp case BIO_WRITE: 314e39d70d4SPoul-Henning Kamp case BIO_DELETE: 3152a842317SAndriy Gapon /* Zero sectorsize or mediasize is probably a lack of media. */ 3162a842317SAndriy Gapon if (pp->sectorsize == 0 || pp->mediasize == 0) 31743bff1a7SPoul-Henning Kamp return (ENXIO); 318e39d70d4SPoul-Henning Kamp /* Reject I/O not on sector boundary */ 319e39d70d4SPoul-Henning Kamp if (bp->bio_offset % pp->sectorsize) 320e39d70d4SPoul-Henning Kamp return (EINVAL); 321e39d70d4SPoul-Henning Kamp /* Reject I/O not integral sector long */ 322e39d70d4SPoul-Henning Kamp if (bp->bio_length % pp->sectorsize) 323e39d70d4SPoul-Henning Kamp return (EINVAL); 324d1b8bf47SPoul-Henning Kamp /* Reject requests before or past the end of media. */ 325d1b8bf47SPoul-Henning Kamp if (bp->bio_offset < 0) 326d1b8bf47SPoul-Henning Kamp return (EIO); 327e39d70d4SPoul-Henning Kamp if (bp->bio_offset > pp->mediasize) 328e39d70d4SPoul-Henning Kamp return (EIO); 329e39d70d4SPoul-Henning Kamp break; 330e39d70d4SPoul-Henning Kamp default: 331e39d70d4SPoul-Henning Kamp break; 332e39d70d4SPoul-Henning Kamp } 333e39d70d4SPoul-Henning Kamp return (0); 334e39d70d4SPoul-Henning Kamp } 335e39d70d4SPoul-Henning Kamp 3366231f75bSLuigi Rizzo /* 3376231f75bSLuigi Rizzo * bio classification support. 3386231f75bSLuigi Rizzo * 3396231f75bSLuigi Rizzo * g_register_classifier() and g_unregister_classifier() 3406231f75bSLuigi Rizzo * are used to add/remove a classifier from the list. 3416231f75bSLuigi Rizzo * The list is protected using the g_bio_run_down lock, 3426231f75bSLuigi Rizzo * because the classifiers are called in this path. 3436231f75bSLuigi Rizzo * 3446231f75bSLuigi Rizzo * g_io_request() passes bio's that are not already classified 3456231f75bSLuigi Rizzo * (i.e. those with bio_classifier1 == NULL) to g_run_classifiers(). 3466231f75bSLuigi Rizzo * Classifiers can store their result in the two fields 3476231f75bSLuigi Rizzo * bio_classifier1 and bio_classifier2. 3486231f75bSLuigi Rizzo * A classifier that updates one of the fields should 3496231f75bSLuigi Rizzo * return a non-zero value. 3506231f75bSLuigi Rizzo * If no classifier updates the field, g_run_classifiers() sets 3516231f75bSLuigi Rizzo * bio_classifier1 = BIO_NOTCLASSIFIED to avoid further calls. 3526231f75bSLuigi Rizzo */ 3536231f75bSLuigi Rizzo 3546231f75bSLuigi Rizzo int 3556231f75bSLuigi Rizzo g_register_classifier(struct g_classifier_hook *hook) 3566231f75bSLuigi Rizzo { 3576231f75bSLuigi Rizzo 3586231f75bSLuigi Rizzo g_bioq_lock(&g_bio_run_down); 3596231f75bSLuigi Rizzo TAILQ_INSERT_TAIL(&g_classifier_tailq, hook, link); 3606231f75bSLuigi Rizzo g_bioq_unlock(&g_bio_run_down); 3616231f75bSLuigi Rizzo 3626231f75bSLuigi Rizzo return (0); 3636231f75bSLuigi Rizzo } 3646231f75bSLuigi Rizzo 3656231f75bSLuigi Rizzo void 3666231f75bSLuigi Rizzo g_unregister_classifier(struct g_classifier_hook *hook) 3676231f75bSLuigi Rizzo { 3686231f75bSLuigi Rizzo struct g_classifier_hook *entry; 3696231f75bSLuigi Rizzo 3706231f75bSLuigi Rizzo g_bioq_lock(&g_bio_run_down); 3716231f75bSLuigi Rizzo TAILQ_FOREACH(entry, &g_classifier_tailq, link) { 3726231f75bSLuigi Rizzo if (entry == hook) { 3736231f75bSLuigi Rizzo TAILQ_REMOVE(&g_classifier_tailq, hook, link); 3746231f75bSLuigi Rizzo break; 3756231f75bSLuigi Rizzo } 3766231f75bSLuigi Rizzo } 3776231f75bSLuigi Rizzo g_bioq_unlock(&g_bio_run_down); 3786231f75bSLuigi Rizzo } 3796231f75bSLuigi Rizzo 3806231f75bSLuigi Rizzo static void 3816231f75bSLuigi Rizzo g_run_classifiers(struct bio *bp) 3826231f75bSLuigi Rizzo { 3836231f75bSLuigi Rizzo struct g_classifier_hook *hook; 3846231f75bSLuigi Rizzo int classified = 0; 3856231f75bSLuigi Rizzo 3866231f75bSLuigi Rizzo TAILQ_FOREACH(hook, &g_classifier_tailq, link) 3876231f75bSLuigi Rizzo classified |= hook->func(hook->arg, bp); 3886231f75bSLuigi Rizzo 3896231f75bSLuigi Rizzo if (!classified) 3906231f75bSLuigi Rizzo bp->bio_classifier1 = BIO_NOTCLASSIFIED; 3916231f75bSLuigi Rizzo } 3926231f75bSLuigi Rizzo 393dd84a43cSPoul-Henning Kamp void 394dd84a43cSPoul-Henning Kamp g_io_request(struct bio *bp, struct g_consumer *cp) 395dd84a43cSPoul-Henning Kamp { 396801bb689SPoul-Henning Kamp struct g_provider *pp; 3970d883b11SAlexander Motin int first; 398dd84a43cSPoul-Henning Kamp 399d0e17c1bSPoul-Henning Kamp KASSERT(cp != NULL, ("NULL cp in g_io_request")); 400d0e17c1bSPoul-Henning Kamp KASSERT(bp != NULL, ("NULL bp in g_io_request")); 401e060b6bdSPoul-Henning Kamp pp = cp->provider; 402801bb689SPoul-Henning Kamp KASSERT(pp != NULL, ("consumer not attached in g_io_request")); 40392ee312dSPawel Jakub Dawidek #ifdef DIAGNOSTIC 40492ee312dSPawel Jakub Dawidek KASSERT(bp->bio_driver1 == NULL, 40592ee312dSPawel Jakub Dawidek ("bio_driver1 used by the consumer (geom %s)", cp->geom->name)); 40692ee312dSPawel Jakub Dawidek KASSERT(bp->bio_driver2 == NULL, 40792ee312dSPawel Jakub Dawidek ("bio_driver2 used by the consumer (geom %s)", cp->geom->name)); 40892ee312dSPawel Jakub Dawidek KASSERT(bp->bio_pflags == 0, 40992ee312dSPawel Jakub Dawidek ("bio_pflags used by the consumer (geom %s)", cp->geom->name)); 41092ee312dSPawel Jakub Dawidek /* 41192ee312dSPawel Jakub Dawidek * Remember consumer's private fields, so we can detect if they were 41292ee312dSPawel Jakub Dawidek * modified by the provider. 41392ee312dSPawel Jakub Dawidek */ 41492ee312dSPawel Jakub Dawidek bp->_bio_caller1 = bp->bio_caller1; 41592ee312dSPawel Jakub Dawidek bp->_bio_caller2 = bp->bio_caller2; 41692ee312dSPawel Jakub Dawidek bp->_bio_cflags = bp->bio_cflags; 41792ee312dSPawel Jakub Dawidek #endif 418801bb689SPoul-Henning Kamp 4191ded77b2SPawel Jakub Dawidek if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_GETATTR)) { 420c3618c65SPawel Jakub Dawidek KASSERT(bp->bio_data != NULL, 4211ded77b2SPawel Jakub Dawidek ("NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd)); 4221ded77b2SPawel Jakub Dawidek } 4231ded77b2SPawel Jakub Dawidek if (bp->bio_cmd & (BIO_DELETE|BIO_FLUSH)) { 4241ded77b2SPawel Jakub Dawidek KASSERT(bp->bio_data == NULL, 4251ded77b2SPawel Jakub Dawidek ("non-NULL bp->data in g_io_request(cmd=%hhu)", 4261ded77b2SPawel Jakub Dawidek bp->bio_cmd)); 427c3618c65SPawel Jakub Dawidek } 428dcbd0fe5SPoul-Henning Kamp if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_DELETE)) { 429dcbd0fe5SPoul-Henning Kamp KASSERT(bp->bio_offset % cp->provider->sectorsize == 0, 430dcbd0fe5SPoul-Henning Kamp ("wrong offset %jd for sectorsize %u", 431dcbd0fe5SPoul-Henning Kamp bp->bio_offset, cp->provider->sectorsize)); 432dcbd0fe5SPoul-Henning Kamp KASSERT(bp->bio_length % cp->provider->sectorsize == 0, 433dcbd0fe5SPoul-Henning Kamp ("wrong length %jd for sectorsize %u", 434dcbd0fe5SPoul-Henning Kamp bp->bio_length, cp->provider->sectorsize)); 435dcbd0fe5SPoul-Henning Kamp } 436dcbd0fe5SPoul-Henning Kamp 437f7717523SStephan Uphoff g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", 438f7717523SStephan Uphoff bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); 439f7717523SStephan Uphoff 440dd84a43cSPoul-Henning Kamp bp->bio_from = cp; 441801bb689SPoul-Henning Kamp bp->bio_to = pp; 4422fccec19SPoul-Henning Kamp bp->bio_error = 0; 4432fccec19SPoul-Henning Kamp bp->bio_completed = 0; 444dd84a43cSPoul-Henning Kamp 44519fa21aaSPoul-Henning Kamp KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 44619fa21aaSPoul-Henning Kamp ("Bio already on queue bp=%p", bp)); 44719fa21aaSPoul-Henning Kamp bp->bio_flags |= BIO_ONQUEUE; 44819fa21aaSPoul-Henning Kamp 449a5be8eb5SAlexander Motin if (g_collectstats) 45019fa21aaSPoul-Henning Kamp binuptime(&bp->bio_t0); 451a5be8eb5SAlexander Motin else 452a5be8eb5SAlexander Motin getbinuptime(&bp->bio_t0); 4538827c821SPoul-Henning Kamp 4548827c821SPoul-Henning Kamp /* 4558827c821SPoul-Henning Kamp * The statistics collection is lockless, as such, but we 4568827c821SPoul-Henning Kamp * can not update one instance of the statistics from more 4578827c821SPoul-Henning Kamp * than one thread at a time, so grab the lock first. 4586231f75bSLuigi Rizzo * 4596231f75bSLuigi Rizzo * We also use the lock to protect the list of classifiers. 4608827c821SPoul-Henning Kamp */ 46119fa21aaSPoul-Henning Kamp g_bioq_lock(&g_bio_run_down); 4626231f75bSLuigi Rizzo 4636231f75bSLuigi Rizzo if (!TAILQ_EMPTY(&g_classifier_tailq) && !bp->bio_classifier1) 4646231f75bSLuigi Rizzo g_run_classifiers(bp); 4656231f75bSLuigi Rizzo 466cf457284SPoul-Henning Kamp if (g_collectstats & 1) 46719fa21aaSPoul-Henning Kamp devstat_start_transaction(pp->stat, &bp->bio_t0); 468cf457284SPoul-Henning Kamp if (g_collectstats & 2) 46919fa21aaSPoul-Henning Kamp devstat_start_transaction(cp->stat, &bp->bio_t0); 47019fa21aaSPoul-Henning Kamp 47119fa21aaSPoul-Henning Kamp pp->nstart++; 472cf457284SPoul-Henning Kamp cp->nstart++; 4730d883b11SAlexander Motin first = TAILQ_EMPTY(&g_bio_run_down.bio_queue); 47419fa21aaSPoul-Henning Kamp TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue); 47519fa21aaSPoul-Henning Kamp g_bio_run_down.bio_queue_length++; 47619fa21aaSPoul-Henning Kamp g_bioq_unlock(&g_bio_run_down); 477e39d70d4SPoul-Henning Kamp 4782fccec19SPoul-Henning Kamp /* Pass it on down. */ 4790d883b11SAlexander Motin if (first) 480dd84a43cSPoul-Henning Kamp wakeup(&g_wait_down); 481dd84a43cSPoul-Henning Kamp } 482dd84a43cSPoul-Henning Kamp 483dd84a43cSPoul-Henning Kamp void 48472840432SPoul-Henning Kamp g_io_deliver(struct bio *bp, int error) 485dd84a43cSPoul-Henning Kamp { 486801bb689SPoul-Henning Kamp struct g_consumer *cp; 487801bb689SPoul-Henning Kamp struct g_provider *pp; 4880d883b11SAlexander Motin int first; 489dd84a43cSPoul-Henning Kamp 490e060b6bdSPoul-Henning Kamp KASSERT(bp != NULL, ("NULL bp in g_io_deliver")); 491801bb689SPoul-Henning Kamp pp = bp->bio_to; 492f7eeab17SPoul-Henning Kamp KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver")); 493f7eeab17SPoul-Henning Kamp cp = bp->bio_from; 494f7eeab17SPoul-Henning Kamp if (cp == NULL) { 495f7eeab17SPoul-Henning Kamp bp->bio_error = error; 496f7eeab17SPoul-Henning Kamp bp->bio_done(bp); 497f7eeab17SPoul-Henning Kamp return; 498f7eeab17SPoul-Henning Kamp } 499801bb689SPoul-Henning Kamp KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver")); 500801bb689SPoul-Henning Kamp KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver")); 501fb231f36SEdward Tomasz Napierala #ifdef DIAGNOSTIC 502fb231f36SEdward Tomasz Napierala /* 503fb231f36SEdward Tomasz Napierala * Some classes - GJournal in particular - can modify bio's 504fb231f36SEdward Tomasz Napierala * private fields while the bio is in transit; G_GEOM_VOLATILE_BIO 505fb231f36SEdward Tomasz Napierala * flag means it's an expected behaviour for that particular geom. 506fb231f36SEdward Tomasz Napierala */ 507fb231f36SEdward Tomasz Napierala if ((cp->geom->flags & G_GEOM_VOLATILE_BIO) == 0) { 508fb231f36SEdward Tomasz Napierala KASSERT(bp->bio_caller1 == bp->_bio_caller1, 509fb231f36SEdward Tomasz Napierala ("bio_caller1 used by the provider %s", pp->name)); 510fb231f36SEdward Tomasz Napierala KASSERT(bp->bio_caller2 == bp->_bio_caller2, 511fb231f36SEdward Tomasz Napierala ("bio_caller2 used by the provider %s", pp->name)); 512fb231f36SEdward Tomasz Napierala KASSERT(bp->bio_cflags == bp->_bio_cflags, 513fb231f36SEdward Tomasz Napierala ("bio_cflags used by the provider %s", pp->name)); 514fb231f36SEdward Tomasz Napierala } 515fb231f36SEdward Tomasz Napierala #endif 51646aeebecSPawel Jakub Dawidek KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0")); 51746aeebecSPawel Jakub Dawidek KASSERT(bp->bio_completed <= bp->bio_length, 51846aeebecSPawel Jakub Dawidek ("bio_completed can't be greater than bio_length")); 5195ab413bfSPoul-Henning Kamp 520dd84a43cSPoul-Henning Kamp g_trace(G_T_BIO, 5210355b86eSPoul-Henning Kamp "g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd", 522801bb689SPoul-Henning Kamp bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error, 5230355b86eSPoul-Henning Kamp (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 524801bb689SPoul-Henning Kamp 52519fa21aaSPoul-Henning Kamp KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 52619fa21aaSPoul-Henning Kamp ("Bio already on queue bp=%p", bp)); 52719fa21aaSPoul-Henning Kamp 528dcbd0fe5SPoul-Henning Kamp /* 529dcbd0fe5SPoul-Henning Kamp * XXX: next two doesn't belong here 530dcbd0fe5SPoul-Henning Kamp */ 531e24cbd90SPoul-Henning Kamp bp->bio_bcount = bp->bio_length; 532e24cbd90SPoul-Henning Kamp bp->bio_resid = bp->bio_bcount - bp->bio_completed; 53319fa21aaSPoul-Henning Kamp 5348827c821SPoul-Henning Kamp /* 5358827c821SPoul-Henning Kamp * The statistics collection is lockless, as such, but we 5368827c821SPoul-Henning Kamp * can not update one instance of the statistics from more 5378827c821SPoul-Henning Kamp * than one thread at a time, so grab the lock first. 5388827c821SPoul-Henning Kamp */ 53919fa21aaSPoul-Henning Kamp g_bioq_lock(&g_bio_run_up); 540cf457284SPoul-Henning Kamp if (g_collectstats & 1) 541e24cbd90SPoul-Henning Kamp devstat_end_transaction_bio(pp->stat, bp); 542cf457284SPoul-Henning Kamp if (g_collectstats & 2) 543cf457284SPoul-Henning Kamp devstat_end_transaction_bio(cp->stat, bp); 5448827c821SPoul-Henning Kamp 545c6ae9b5fSPoul-Henning Kamp cp->nend++; 546c6ae9b5fSPoul-Henning Kamp pp->nend++; 54719fa21aaSPoul-Henning Kamp if (error != ENOMEM) { 54819fa21aaSPoul-Henning Kamp bp->bio_error = error; 5490d883b11SAlexander Motin first = TAILQ_EMPTY(&g_bio_run_up.bio_queue); 55019fa21aaSPoul-Henning Kamp TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue); 551276f72c5SPoul-Henning Kamp bp->bio_flags |= BIO_ONQUEUE; 55219fa21aaSPoul-Henning Kamp g_bio_run_up.bio_queue_length++; 55319fa21aaSPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 5540d883b11SAlexander Motin if (first) 55519fa21aaSPoul-Henning Kamp wakeup(&g_wait_up); 55619fa21aaSPoul-Henning Kamp return; 55719fa21aaSPoul-Henning Kamp } 55819fa21aaSPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 559dd84a43cSPoul-Henning Kamp 5602cc9686eSPoul-Henning Kamp if (bootverbose) 561801bb689SPoul-Henning Kamp printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name); 5621b949c05SPawel Jakub Dawidek bp->bio_children = 0; 5631b949c05SPawel Jakub Dawidek bp->bio_inbed = 0; 564801bb689SPoul-Henning Kamp g_io_request(bp, cp); 5653432e4fdSPoul-Henning Kamp pace++; 5663432e4fdSPoul-Henning Kamp return; 5673432e4fdSPoul-Henning Kamp } 568dd84a43cSPoul-Henning Kamp 569dd84a43cSPoul-Henning Kamp void 570dd84a43cSPoul-Henning Kamp g_io_schedule_down(struct thread *tp __unused) 571dd84a43cSPoul-Henning Kamp { 572dd84a43cSPoul-Henning Kamp struct bio *bp; 573e39d70d4SPoul-Henning Kamp off_t excess; 574e39d70d4SPoul-Henning Kamp int error; 575dd84a43cSPoul-Henning Kamp 576dd84a43cSPoul-Henning Kamp for(;;) { 577f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_run_down); 578dd84a43cSPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_down); 579f0e185d7SPoul-Henning Kamp if (bp == NULL) { 58049dbb61dSRobert Watson CTR0(KTR_GEOM, "g_down going to sleep"); 581f0e185d7SPoul-Henning Kamp msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock, 5827fc019afSAlexander Motin PRIBIO | PDROP, "-", 0); 583f0e185d7SPoul-Henning Kamp continue; 584f0e185d7SPoul-Henning Kamp } 58549dbb61dSRobert Watson CTR0(KTR_GEOM, "g_down has work to do"); 586f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_down); 587376ceb79SPoul-Henning Kamp if (pace > 0) { 58849dbb61dSRobert Watson CTR1(KTR_GEOM, "g_down pacing self (pace %d)", pace); 5894d70511aSJohn Baldwin pause("g_down", hz/10); 590376ceb79SPoul-Henning Kamp pace--; 591376ceb79SPoul-Henning Kamp } 592e39d70d4SPoul-Henning Kamp error = g_io_check(bp); 593e39d70d4SPoul-Henning Kamp if (error) { 59449dbb61dSRobert Watson CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider " 59549dbb61dSRobert Watson "%s returned %d", bp, bp->bio_to->name, error); 596e39d70d4SPoul-Henning Kamp g_io_deliver(bp, error); 597e39d70d4SPoul-Henning Kamp continue; 598e39d70d4SPoul-Henning Kamp } 59949dbb61dSRobert Watson CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp, 60049dbb61dSRobert Watson bp->bio_to->name); 601392d56b4SPoul-Henning Kamp switch (bp->bio_cmd) { 602392d56b4SPoul-Henning Kamp case BIO_READ: 603392d56b4SPoul-Henning Kamp case BIO_WRITE: 604392d56b4SPoul-Henning Kamp case BIO_DELETE: 605e39d70d4SPoul-Henning Kamp /* Truncate requests to the end of providers media. */ 60649dbb61dSRobert Watson /* 60749dbb61dSRobert Watson * XXX: What if we truncate because of offset being 60849dbb61dSRobert Watson * bad, not length? 60949dbb61dSRobert Watson */ 610e39d70d4SPoul-Henning Kamp excess = bp->bio_offset + bp->bio_length; 611e39d70d4SPoul-Henning Kamp if (excess > bp->bio_to->mediasize) { 612e39d70d4SPoul-Henning Kamp excess -= bp->bio_to->mediasize; 613e39d70d4SPoul-Henning Kamp bp->bio_length -= excess; 61449dbb61dSRobert Watson if (excess > 0) 61549dbb61dSRobert Watson CTR3(KTR_GEOM, "g_down truncated bio " 61649dbb61dSRobert Watson "%p provider %s by %d", bp, 61749dbb61dSRobert Watson bp->bio_to->name, excess); 618e39d70d4SPoul-Henning Kamp } 619e39d70d4SPoul-Henning Kamp /* Deliver zero length transfers right here. */ 620e39d70d4SPoul-Henning Kamp if (bp->bio_length == 0) { 621e39d70d4SPoul-Henning Kamp g_io_deliver(bp, 0); 62249dbb61dSRobert Watson CTR2(KTR_GEOM, "g_down terminated 0-length " 62349dbb61dSRobert Watson "bp %p provider %s", bp, bp->bio_to->name); 624e39d70d4SPoul-Henning Kamp continue; 625e39d70d4SPoul-Henning Kamp } 626392d56b4SPoul-Henning Kamp break; 627392d56b4SPoul-Henning Kamp default: 628392d56b4SPoul-Henning Kamp break; 629392d56b4SPoul-Henning Kamp } 63051460da8SJohn Baldwin THREAD_NO_SLEEPING(); 63149dbb61dSRobert Watson CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld " 63249dbb61dSRobert Watson "len %ld", bp, bp->bio_to->name, bp->bio_offset, 63349dbb61dSRobert Watson bp->bio_length); 634dd84a43cSPoul-Henning Kamp bp->bio_to->geom->start(bp); 63551460da8SJohn Baldwin THREAD_SLEEPING_OK(); 636dd84a43cSPoul-Henning Kamp } 637dd84a43cSPoul-Henning Kamp } 638dd84a43cSPoul-Henning Kamp 639dd84a43cSPoul-Henning Kamp void 6405fcf4e43SPoul-Henning Kamp bio_taskqueue(struct bio *bp, bio_task_t *func, void *arg) 6415fcf4e43SPoul-Henning Kamp { 6425fcf4e43SPoul-Henning Kamp bp->bio_task = func; 6435fcf4e43SPoul-Henning Kamp bp->bio_task_arg = arg; 6445fcf4e43SPoul-Henning Kamp /* 6455fcf4e43SPoul-Henning Kamp * The taskqueue is actually just a second queue off the "up" 6465fcf4e43SPoul-Henning Kamp * queue, so we use the same lock. 6475fcf4e43SPoul-Henning Kamp */ 6485fcf4e43SPoul-Henning Kamp g_bioq_lock(&g_bio_run_up); 649dcbd0fe5SPoul-Henning Kamp KASSERT(!(bp->bio_flags & BIO_ONQUEUE), 650dcbd0fe5SPoul-Henning Kamp ("Bio already on queue bp=%p target taskq", bp)); 651dcbd0fe5SPoul-Henning Kamp bp->bio_flags |= BIO_ONQUEUE; 6525fcf4e43SPoul-Henning Kamp TAILQ_INSERT_TAIL(&g_bio_run_task.bio_queue, bp, bio_queue); 6535fcf4e43SPoul-Henning Kamp g_bio_run_task.bio_queue_length++; 6545fcf4e43SPoul-Henning Kamp wakeup(&g_wait_up); 6555fcf4e43SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 6565fcf4e43SPoul-Henning Kamp } 6575fcf4e43SPoul-Henning Kamp 6585fcf4e43SPoul-Henning Kamp 6595fcf4e43SPoul-Henning Kamp void 660dd84a43cSPoul-Henning Kamp g_io_schedule_up(struct thread *tp __unused) 661dd84a43cSPoul-Henning Kamp { 662dd84a43cSPoul-Henning Kamp struct bio *bp; 663dd84a43cSPoul-Henning Kamp for(;;) { 664f0e185d7SPoul-Henning Kamp g_bioq_lock(&g_bio_run_up); 6655fcf4e43SPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_task); 6665fcf4e43SPoul-Henning Kamp if (bp != NULL) { 6675fcf4e43SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 66851460da8SJohn Baldwin THREAD_NO_SLEEPING(); 66949dbb61dSRobert Watson CTR1(KTR_GEOM, "g_up processing task bp %p", bp); 6705fcf4e43SPoul-Henning Kamp bp->bio_task(bp->bio_task_arg); 67151460da8SJohn Baldwin THREAD_SLEEPING_OK(); 6725fcf4e43SPoul-Henning Kamp continue; 6735fcf4e43SPoul-Henning Kamp } 674dd84a43cSPoul-Henning Kamp bp = g_bioq_first(&g_bio_run_up); 675f0e185d7SPoul-Henning Kamp if (bp != NULL) { 676f0e185d7SPoul-Henning Kamp g_bioq_unlock(&g_bio_run_up); 67751460da8SJohn Baldwin THREAD_NO_SLEEPING(); 67849dbb61dSRobert Watson CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off " 679c4901b67SSean Bruno "%jd len %ld", bp, bp->bio_to->name, 68049dbb61dSRobert Watson bp->bio_offset, bp->bio_length); 68153706245SPoul-Henning Kamp biodone(bp); 68251460da8SJohn Baldwin THREAD_SLEEPING_OK(); 683f0e185d7SPoul-Henning Kamp continue; 684f0e185d7SPoul-Henning Kamp } 68549dbb61dSRobert Watson CTR0(KTR_GEOM, "g_up going to sleep"); 686f0e185d7SPoul-Henning Kamp msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock, 6877fc019afSAlexander Motin PRIBIO | PDROP, "-", 0); 688dd84a43cSPoul-Henning Kamp } 689dd84a43cSPoul-Henning Kamp } 690dd84a43cSPoul-Henning Kamp 691dd84a43cSPoul-Henning Kamp void * 692dd84a43cSPoul-Henning Kamp g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error) 693dd84a43cSPoul-Henning Kamp { 694dd84a43cSPoul-Henning Kamp struct bio *bp; 695dd84a43cSPoul-Henning Kamp void *ptr; 696dd84a43cSPoul-Henning Kamp int errorc; 697dd84a43cSPoul-Henning Kamp 6988dd5480dSPawel Jakub Dawidek KASSERT(length > 0 && length >= cp->provider->sectorsize && 6998dd5480dSPawel Jakub Dawidek length <= MAXPHYS, ("g_read_data(): invalid length %jd", 7008dd5480dSPawel Jakub Dawidek (intmax_t)length)); 7013eb6ffdfSPoul-Henning Kamp 702a2033c96SPoul-Henning Kamp bp = g_alloc_bio(); 703dd84a43cSPoul-Henning Kamp bp->bio_cmd = BIO_READ; 704dd84a43cSPoul-Henning Kamp bp->bio_done = NULL; 705dd84a43cSPoul-Henning Kamp bp->bio_offset = offset; 706dd84a43cSPoul-Henning Kamp bp->bio_length = length; 707a163d034SWarner Losh ptr = g_malloc(length, M_WAITOK); 708dd84a43cSPoul-Henning Kamp bp->bio_data = ptr; 709dd84a43cSPoul-Henning Kamp g_io_request(bp, cp); 71053706245SPoul-Henning Kamp errorc = biowait(bp, "gread"); 711dd84a43cSPoul-Henning Kamp if (error != NULL) 712dd84a43cSPoul-Henning Kamp *error = errorc; 713dd84a43cSPoul-Henning Kamp g_destroy_bio(bp); 714dd84a43cSPoul-Henning Kamp if (errorc) { 715dd84a43cSPoul-Henning Kamp g_free(ptr); 716dd84a43cSPoul-Henning Kamp ptr = NULL; 717dd84a43cSPoul-Henning Kamp } 718dd84a43cSPoul-Henning Kamp return (ptr); 719dd84a43cSPoul-Henning Kamp } 72090b1cd56SPoul-Henning Kamp 72190b1cd56SPoul-Henning Kamp int 72290b1cd56SPoul-Henning Kamp g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length) 72390b1cd56SPoul-Henning Kamp { 72490b1cd56SPoul-Henning Kamp struct bio *bp; 72590b1cd56SPoul-Henning Kamp int error; 72690b1cd56SPoul-Henning Kamp 7278dd5480dSPawel Jakub Dawidek KASSERT(length > 0 && length >= cp->provider->sectorsize && 7288dd5480dSPawel Jakub Dawidek length <= MAXPHYS, ("g_write_data(): invalid length %jd", 7298dd5480dSPawel Jakub Dawidek (intmax_t)length)); 7303eb6ffdfSPoul-Henning Kamp 731a2033c96SPoul-Henning Kamp bp = g_alloc_bio(); 73290b1cd56SPoul-Henning Kamp bp->bio_cmd = BIO_WRITE; 73390b1cd56SPoul-Henning Kamp bp->bio_done = NULL; 73490b1cd56SPoul-Henning Kamp bp->bio_offset = offset; 73590b1cd56SPoul-Henning Kamp bp->bio_length = length; 73690b1cd56SPoul-Henning Kamp bp->bio_data = ptr; 73790b1cd56SPoul-Henning Kamp g_io_request(bp, cp); 73890b1cd56SPoul-Henning Kamp error = biowait(bp, "gwrite"); 73990b1cd56SPoul-Henning Kamp g_destroy_bio(bp); 74090b1cd56SPoul-Henning Kamp return (error); 74190b1cd56SPoul-Henning Kamp } 74272e33095SPawel Jakub Dawidek 7432b17fb95SPawel Jakub Dawidek int 7442b17fb95SPawel Jakub Dawidek g_delete_data(struct g_consumer *cp, off_t offset, off_t length) 7452b17fb95SPawel Jakub Dawidek { 7462b17fb95SPawel Jakub Dawidek struct bio *bp; 7472b17fb95SPawel Jakub Dawidek int error; 7482b17fb95SPawel Jakub Dawidek 749eed6cda9SPoul-Henning Kamp KASSERT(length > 0 && length >= cp->provider->sectorsize, 750eed6cda9SPoul-Henning Kamp ("g_delete_data(): invalid length %jd", (intmax_t)length)); 7512b17fb95SPawel Jakub Dawidek 7522b17fb95SPawel Jakub Dawidek bp = g_alloc_bio(); 7532b17fb95SPawel Jakub Dawidek bp->bio_cmd = BIO_DELETE; 7542b17fb95SPawel Jakub Dawidek bp->bio_done = NULL; 7552b17fb95SPawel Jakub Dawidek bp->bio_offset = offset; 7562b17fb95SPawel Jakub Dawidek bp->bio_length = length; 7572b17fb95SPawel Jakub Dawidek bp->bio_data = NULL; 7582b17fb95SPawel Jakub Dawidek g_io_request(bp, cp); 7592b17fb95SPawel Jakub Dawidek error = biowait(bp, "gdelete"); 7602b17fb95SPawel Jakub Dawidek g_destroy_bio(bp); 7612b17fb95SPawel Jakub Dawidek return (error); 7622b17fb95SPawel Jakub Dawidek } 7632b17fb95SPawel Jakub Dawidek 76472e33095SPawel Jakub Dawidek void 76572e33095SPawel Jakub Dawidek g_print_bio(struct bio *bp) 76672e33095SPawel Jakub Dawidek { 76772e33095SPawel Jakub Dawidek const char *pname, *cmd = NULL; 76872e33095SPawel Jakub Dawidek 76972e33095SPawel Jakub Dawidek if (bp->bio_to != NULL) 77072e33095SPawel Jakub Dawidek pname = bp->bio_to->name; 77172e33095SPawel Jakub Dawidek else 77272e33095SPawel Jakub Dawidek pname = "[unknown]"; 77372e33095SPawel Jakub Dawidek 77472e33095SPawel Jakub Dawidek switch (bp->bio_cmd) { 77572e33095SPawel Jakub Dawidek case BIO_GETATTR: 77672e33095SPawel Jakub Dawidek cmd = "GETATTR"; 77772e33095SPawel Jakub Dawidek printf("%s[%s(attr=%s)]", pname, cmd, bp->bio_attribute); 77872e33095SPawel Jakub Dawidek return; 779c3618c65SPawel Jakub Dawidek case BIO_FLUSH: 780c3618c65SPawel Jakub Dawidek cmd = "FLUSH"; 781c3618c65SPawel Jakub Dawidek printf("%s[%s]", pname, cmd); 782c3618c65SPawel Jakub Dawidek return; 78372e33095SPawel Jakub Dawidek case BIO_READ: 78472e33095SPawel Jakub Dawidek cmd = "READ"; 7857ce513a5SEdward Tomasz Napierala break; 78672e33095SPawel Jakub Dawidek case BIO_WRITE: 78772e33095SPawel Jakub Dawidek cmd = "WRITE"; 7887ce513a5SEdward Tomasz Napierala break; 78972e33095SPawel Jakub Dawidek case BIO_DELETE: 79072e33095SPawel Jakub Dawidek cmd = "DELETE"; 7917ce513a5SEdward Tomasz Napierala break; 79272e33095SPawel Jakub Dawidek default: 79372e33095SPawel Jakub Dawidek cmd = "UNKNOWN"; 79472e33095SPawel Jakub Dawidek printf("%s[%s()]", pname, cmd); 79572e33095SPawel Jakub Dawidek return; 79672e33095SPawel Jakub Dawidek } 7977ce513a5SEdward Tomasz Napierala printf("%s[%s(offset=%jd, length=%jd)]", pname, cmd, 7987ce513a5SEdward Tomasz Napierala (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length); 79972e33095SPawel Jakub Dawidek } 800