scsi/targets/sd_xbuf.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/scsi/scsi.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/thread.h>
#include <sys/var.h>

#include "sd_xbuf.h"

/*
 * xbuf.c: buf(9s) extension facility.
 *
 * The buf(9S) extension facility is intended to allow block drivers to
 * allocate additional memory that is associated with a particular buf(9S)
 * struct.  It is further intended to help in addressing the usual set of
 * problems associated with such allocations, in particular those involving
 * recovery from allocation failures, especially in code paths that the
 * system relies on to free memory.
 *
 * CAVEAT: Currently this code is completely private to the sd driver and in
 * NO WAY constitutes a public or supported interface of any kind. It is
 * envisioned that this may one day migrate into the Solaris DDI, but until
 * that time this ought to be considered completely unstable and is subject
 * to change without notice. This code may NOT in any way be utilized by
 * ANY code outside the sd driver.
 */


static int xbuf_iostart(ddi_xbuf_attr_t xap);
static void xbuf_dispatch(ddi_xbuf_attr_t xap);
static void xbuf_restart_callback(void *arg);
static int xbuf_brk_done(struct buf *bp);


/*
 * Note: Should this be exposed to the caller.... do we want to give the
 * caller the fexibility of specifying the parameters for the thread pool?
 * Note: these values are just estimates at this time, based upon what
 * seems reasonable for the sd driver. It may be preferable to make these
 * parameters self-scaling in a real (future) implementation.
 */
#define	XBUF_TQ_MINALLOC	64
#define	XBUF_TQ_MAXALLOC	512
#define	XBUF_DISPATCH_DELAY	(drv_usectohz(50000))	/* 50 msec */

static taskq_t *xbuf_tq = NULL;
static int xbuf_attr_tq_minalloc = XBUF_TQ_MINALLOC;
static int xbuf_attr_tq_maxalloc = XBUF_TQ_MAXALLOC;

static kmutex_t	xbuf_mutex = { 0 };
static uint32_t	xbuf_refcount = 0;

/*
 * Private wrapper for buf cloned via ddi_xbuf_qstrategy()
 */
struct xbuf_brk {
	kmutex_t mutex;
	struct buf *bp0;
	uint8_t nbufs;	/* number of buf allocated */
	uint8_t active; /* number of active xfer */

	size_t brksize;	/* break size used for this buf */
	int brkblk;

	/* xfer position */
	off_t off;
	off_t noff;
	daddr_t blkno;
};

_NOTE(DATA_READABLE_WITHOUT_LOCK(xbuf_brk::off))

/*
 * Hack needed in the prototype so buf breakup will work.
 * Here we can rely on the sd code not changing the value in
 * b_forw.
 */
#define	b_clone_private b_forw


/* ARGSUSED */
DDII ddi_xbuf_attr_t
ddi_xbuf_attr_create(size_t xsize,
    void (*xa_strategy)(struct buf *bp, ddi_xbuf_t xp, void *attr_arg),
    void *attr_arg, uint32_t active_limit, uint32_t reserve_limit,
    major_t major, int flags)
{
	ddi_xbuf_attr_t	xap;

	xap = kmem_zalloc(sizeof (struct __ddi_xbuf_attr), KM_SLEEP);

	mutex_init(&xap->xa_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&xap->xa_reserve_mutex, NULL, MUTEX_DRIVER, NULL);

	/* Future: Allow the caller to specify alignment requirements? */
	xap->xa_allocsize	= max(xsize, sizeof (void *));
	xap->xa_active_limit	= active_limit;
	xap->xa_active_lowater	= xap->xa_active_limit / 2;
	xap->xa_reserve_limit	= reserve_limit;
	xap->xa_strategy	= xa_strategy;
	xap->xa_attr_arg	= attr_arg;

	mutex_enter(&xbuf_mutex);
	if (xbuf_refcount == 0) {
		ASSERT(xbuf_tq == NULL);
		/*
		 * Note: Would be nice if: (1) #threads in the taskq pool (set
		 * to the value of 'ncpus' at the time the taskq is created)
		 * could adjust automatically with DR; (2) the taskq
		 * minalloc/maxalloc counts could be grown/shrunk on the fly.
		 */
		xbuf_tq = taskq_create("xbuf_taskq", ncpus,
		    (v.v_maxsyspri - 2), xbuf_attr_tq_minalloc,
		    xbuf_attr_tq_maxalloc, TASKQ_PREPOPULATE);
	}
	xbuf_refcount++;
	mutex_exit(&xbuf_mutex);

	/* In this prototype we just always use the global system pool. */
	xap->xa_tq = xbuf_tq;

	return (xap);
}


DDII void
ddi_xbuf_attr_destroy(ddi_xbuf_attr_t xap)
{
	ddi_xbuf_t	xp;

	mutex_destroy(&xap->xa_mutex);
	mutex_destroy(&xap->xa_reserve_mutex);

	/* Free any xbufs on the reserve list */
	while (xap->xa_reserve_count != 0) {
		xp = xap->xa_reserve_headp;
		xap->xa_reserve_headp = *((void **)xp);
		xap->xa_reserve_count--;
		kmem_free(xp, xap->xa_allocsize);
	}
	ASSERT(xap->xa_reserve_headp == NULL);

	mutex_enter(&xbuf_mutex);
	ASSERT((xbuf_refcount != 0) && (xbuf_tq != NULL));
	xbuf_refcount--;
	if (xbuf_refcount == 0) {
		taskq_destroy(xbuf_tq);
		xbuf_tq = NULL;
	}
	mutex_exit(&xbuf_mutex);

	kmem_free(xap, sizeof (struct __ddi_xbuf_attr));
}


/* ARGSUSED */
DDII void
ddi_xbuf_attr_register_devinfo(ddi_xbuf_attr_t xbuf_attr, dev_info_t *dip)
{
	/* Currently a no-op in this prototype */
}


/* ARGSUSED */
DDII void
ddi_xbuf_attr_unregister_devinfo(ddi_xbuf_attr_t xbuf_attr, dev_info_t *dip)
{
	/* Currently a no-op in this prototype */
}

DDII int
ddi_xbuf_attr_setup_brk(ddi_xbuf_attr_t xap, size_t size)
{
	if (size < DEV_BSIZE)
		return (0);

	mutex_enter(&xap->xa_mutex);
	xap->xa_brksize = size & ~(DEV_BSIZE - 1);
	mutex_exit(&xap->xa_mutex);
	return (1);
}


/*
 * Enqueue the given buf and attempt to initiate IO.
 * Called from the driver strategy(9E) routine.
 */

DDII int
ddi_xbuf_qstrategy(struct buf *bp, ddi_xbuf_attr_t xap)
{
	ASSERT(xap != NULL);
	ASSERT(!mutex_owned(&xap->xa_mutex));
	ASSERT(!mutex_owned(&xap->xa_reserve_mutex));

	mutex_enter(&xap->xa_mutex);

	ASSERT((bp->b_bcount & (DEV_BSIZE - 1)) == 0);

	/*
	 * Breakup buf if necessary. bp->b_private is temporarily
	 * used to save xbuf_brk
	 */
	if (xap->xa_brksize && bp->b_bcount > xap->xa_brksize) {
		struct xbuf_brk *brkp;

		brkp = kmem_zalloc(sizeof (struct xbuf_brk), KM_SLEEP);
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*brkp))
		mutex_init(&brkp->mutex, NULL, MUTEX_DRIVER, NULL);
		brkp->bp0 = bp;
		brkp->brksize = xap->xa_brksize;
		brkp->brkblk = btodt(xap->xa_brksize);
		brkp->noff = xap->xa_brksize;
		brkp->blkno = bp->b_blkno;
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*brkp))
		bp->b_private = brkp;
	} else {
		bp->b_private = NULL;
	}

	/* Enqueue buf */
	if (xap->xa_headp == NULL) {
		xap->xa_headp = xap->xa_tailp = bp;
	} else {
		xap->xa_tailp->av_forw = bp;
		xap->xa_tailp = bp;
	}
	bp->av_forw = NULL;

	xap->xa_pending++;
	mutex_exit(&xap->xa_mutex);
	return (xbuf_iostart(xap));
}


/*
 * Drivers call this immediately before calling biodone(9F), to notify the
 * framework that the indicated xbuf is no longer being used by the driver.
 * May be called under interrupt context.
 */

DDII int
ddi_xbuf_done(struct buf *bp, ddi_xbuf_attr_t xap)
{
	ddi_xbuf_t xp;
	int done;

	ASSERT(bp != NULL);
	ASSERT(xap != NULL);
	ASSERT(!mutex_owned(&xap->xa_mutex));
	ASSERT(!mutex_owned(&xap->xa_reserve_mutex));

	xp = ddi_xbuf_get(bp, xap);

	mutex_enter(&xap->xa_mutex);

#ifdef	SDDEBUG
	if (xap->xa_active_limit != 0) {
		ASSERT(xap->xa_active_count > 0);
	}
#endif
	xap->xa_active_count--;

	if (xap->xa_reserve_limit != 0) {
		mutex_enter(&xap->xa_reserve_mutex);
		if (xap->xa_reserve_count < xap->xa_reserve_limit) {
			/* Put this xbuf onto the reserve list & exit */
			*((void **)xp) = xap->xa_reserve_headp;
			xap->xa_reserve_headp = xp;
			xap->xa_reserve_count++;
			mutex_exit(&xap->xa_reserve_mutex);
			goto done;
		}
		mutex_exit(&xap->xa_reserve_mutex);
	}

	kmem_free(xp, xap->xa_allocsize);	/* return it to the system */

done:
	if (bp->b_iodone == xbuf_brk_done) {
		struct xbuf_brk *brkp = (struct xbuf_brk *)bp->b_clone_private;

		brkp->active--;
		if (brkp->active || xap->xa_headp == brkp->bp0) {
			done = 0;
		} else {
			brkp->off = -1;	/* mark bp0 as completed */
			done = 1;
		}
	} else {
		done = 1;
	}

	if ((xap->xa_active_limit == 0) ||
	    (xap->xa_active_count <= xap->xa_active_lowater)) {
		xbuf_dispatch(xap);
	}

	mutex_exit(&xap->xa_mutex);
	return (done);
}

static int
xbuf_brk_done(struct buf *bp)
{
	struct xbuf_brk *brkp = (struct xbuf_brk *)bp->b_clone_private;
	struct buf *bp0 = brkp->bp0;
	int done;

	mutex_enter(&brkp->mutex);
	if (bp->b_flags & B_ERROR && !(bp0->b_flags & B_ERROR)) {
		bp0->b_flags |= B_ERROR;
		bp0->b_error = bp->b_error;
	}
	if (bp->b_resid)
		bp0->b_resid = bp0->b_bcount;

	freerbuf(bp);
	brkp->nbufs--;

	done = (brkp->off == -1 && brkp->nbufs == 0);
	mutex_exit(&brkp->mutex);

	/* All buf segments done */
	if (done) {
		mutex_destroy(&brkp->mutex);
		kmem_free(brkp, sizeof (struct xbuf_brk));
		biodone(bp0);
	}
	return (0);
}

DDII void
ddi_xbuf_dispatch(ddi_xbuf_attr_t xap)
{
	mutex_enter(&xap->xa_mutex);
	if ((xap->xa_active_limit == 0) ||
	    (xap->xa_active_count <= xap->xa_active_lowater)) {
		xbuf_dispatch(xap);
	}
	mutex_exit(&xap->xa_mutex);
}


/*
 * ISSUE: in this prototype we cannot really implement ddi_xbuf_get()
 * unless we explicitly hide the xbuf pointer somewhere in the buf
 * during allocation, and then rely on the driver never changing it.
 * We can probably get away with using b_private for this for now,
 * tho it really is kinda gnarly.....
 */

/* ARGSUSED */
DDII ddi_xbuf_t
ddi_xbuf_get(struct buf *bp, ddi_xbuf_attr_t xap)
{
	return (bp->b_private);
}


/*
 * Initiate IOs for bufs on the queue.  Called from kernel thread or taskq
 * thread context. May execute concurrently for the same ddi_xbuf_attr_t.
 */

static int
xbuf_iostart(ddi_xbuf_attr_t xap)
{
	struct buf *bp;
	ddi_xbuf_t xp;

	ASSERT(xap != NULL);
	ASSERT(!mutex_owned(&xap->xa_mutex));
	ASSERT(!mutex_owned(&xap->xa_reserve_mutex));

	/*
	 * For each request on the queue, attempt to allocate the specified
	 * xbuf extension area, and call the driver's iostart() routine.
	 * We process as many requests on the queue as we can, until either
	 * (1) we run out of requests; or
	 * (2) we run out of resources; or
	 * (3) we reach the maximum limit for the given ddi_xbuf_attr_t.
	 */
	for (;;) {
		mutex_enter(&xap->xa_mutex);

		if ((bp = xap->xa_headp) == NULL) {
			break;	/* queue empty */
		}

		if ((xap->xa_active_limit != 0) &&
		    (xap->xa_active_count >= xap->xa_active_limit)) {
			break;	/* allocation limit reached */
		}

		/*
		 * If the reserve_limit is non-zero then work with the
		 * reserve else always allocate a new struct.
		 */
		if (xap->xa_reserve_limit != 0) {
			/*
			 * Don't penalize EVERY I/O by always allocating a new
			 * struct. for the sake of maintaining and not touching
			 * a reserve for a pathalogical condition that may never
			 * happen. Use the reserve entries first, this uses it
			 * like a local pool rather than a reserve that goes
			 * untouched. Make sure it's re-populated whenever it
			 * gets fully depleted just in case it really is needed.
			 * This is safe because under the pathalogical
			 * condition, when the system runs out of memory such
			 * that the below allocs fail, the reserve will still
			 * be available whether the entries are saved away on
			 * the queue unused or in-transport somewhere. Thus
			 * progress can still continue, however slowly.
			 */
			mutex_enter(&xap->xa_reserve_mutex);
			if (xap->xa_reserve_count != 0) {
				ASSERT(xap->xa_reserve_headp != NULL);
				/* Grab an xbuf from the reserve */
				xp = xap->xa_reserve_headp;
				xap->xa_reserve_headp = *((void **)xp);
				ASSERT(xap->xa_reserve_count > 0);
				xap->xa_reserve_count--;
			} else {
				/*
				 * Either this is the first time through,
				 * or the reserve has been totally depleted.
				 * Re-populate the reserve (pool). Excess
				 * structs. get released in the done path.
				 */
				while (xap->xa_reserve_count <
				    xap->xa_reserve_limit) {
					xp = kmem_alloc(xap->xa_allocsize,
					    KM_NOSLEEP);
					if (xp == NULL) {
						break;
					}
					*((void **)xp) = xap->xa_reserve_headp;
					xap->xa_reserve_headp = xp;
					xap->xa_reserve_count++;
				}
				/* And one more to use right now. */
				xp = kmem_alloc(xap->xa_allocsize, KM_NOSLEEP);
			}
			mutex_exit(&xap->xa_reserve_mutex);
		} else {
			/*
			 * Try to alloc a new xbuf struct. If this fails just
			 * exit for now. We'll get back here again either upon
			 * cmd completion or via the timer handler.
			 * Question: what if the allocation attempt for the very
			 * first cmd. fails? There are no outstanding cmds so
			 * how do we get back here?
			 * Should look at un_ncmds_in_transport, if it's zero
			 * then schedule xbuf_restart_callback via the timer.
			 * Athough that breaks the architecture by bringing
			 * softstate data into this code.
			 */
			xp = kmem_alloc(xap->xa_allocsize, KM_NOSLEEP);
		}
		if (xp == NULL) {
			break; /* Can't process a cmd. right now. */
		}

		/*
		 * Always run the counter. It's used/needed when xa_active_limit
		 * is non-zero which is the typical (and right now only) case.
		 */
		xap->xa_active_count++;

		if (bp->b_private) {
			struct xbuf_brk *brkp = bp->b_private;
			struct buf *bp0 = bp;

			brkp->active++;

			mutex_enter(&brkp->mutex);
			brkp->nbufs++;
			mutex_exit(&brkp->mutex);

			if (brkp->noff < bp0->b_bcount) {
				bp = bioclone(bp0, brkp->off, brkp->brksize,
				    bp0->b_edev, brkp->blkno, xbuf_brk_done,
				    NULL, KM_SLEEP);

				/* update xfer position */
				brkp->off = brkp->noff;
				brkp->noff += brkp->brksize;
				brkp->blkno += brkp->brkblk;
			} else {
				bp = bioclone(bp0, brkp->off,
				    bp0->b_bcount - brkp->off, bp0->b_edev,
				    brkp->blkno, xbuf_brk_done, NULL, KM_SLEEP);

				/* unlink the buf from the list */
				xap->xa_headp = bp0->av_forw;
				bp0->av_forw = NULL;
			}
			bp->b_clone_private = (struct buf *)brkp;
		} else {
			/* unlink the buf from the list */
			xap->xa_headp = bp->av_forw;
			bp->av_forw = NULL;
		}

		/*
		 * Hack needed in the prototype so ddi_xbuf_get() will work.
		 * Here we can rely on the sd code not changing the value in
		 * b_private (in fact it wants it there). See ddi_get_xbuf()
		 */
		bp->b_private = xp;

		/* call the driver's iostart routine */
		mutex_exit(&xap->xa_mutex);
		(*(xap->xa_strategy))(bp, xp, xap->xa_attr_arg);
	}

	ASSERT(xap->xa_pending > 0);
	xap->xa_pending--;
	mutex_exit(&xap->xa_mutex);
	return (0);
}

static void
xbuf_taskq_cb(void *arg)
{
	(void) xbuf_iostart(arg);
}

/*
 * Re-start IO processing if there is anything on the queue, AND if the
 * restart function is not already running/pending for this ddi_xbuf_attr_t
 */
static void
xbuf_dispatch(ddi_xbuf_attr_t xap)
{
	ASSERT(xap != NULL);
	ASSERT(xap->xa_tq != NULL);
	ASSERT(mutex_owned(&xap->xa_mutex));

	if ((xap->xa_headp != NULL) && (xap->xa_timeid == NULL) &&
	    (xap->xa_pending == 0)) {
		/*
		 * First try to see if we can dispatch the restart function
		 * immediately, in a taskq thread.  If this fails, then
		 * schedule a timeout(9F) callback to try again later.
		 */
		if (taskq_dispatch(xap->xa_tq,
		    xbuf_taskq_cb, xap, KM_NOSLEEP) == TASKQID_INVALID) {
			/*
			 * Unable to enqueue the request for the taskq thread,
			 * try again later.  Note that this will keep re-trying
			 * until taskq_dispatch() succeeds.
			 */
			xap->xa_timeid = timeout(xbuf_restart_callback, xap,
			    XBUF_DISPATCH_DELAY);
		} else {
			/*
			 * This indicates that xbuf_iostart() will soon be
			 * run for this ddi_xbuf_attr_t, and we do not need to
			 * schedule another invocation via timeout/taskq
			 */
			xap->xa_pending++;
		}
	}
}

/* timeout(9F) callback routine for xbuf restart mechanism. */
static void
xbuf_restart_callback(void *arg)
{
	ddi_xbuf_attr_t	xap = arg;

	ASSERT(xap != NULL);
	ASSERT(xap->xa_tq != NULL);
	ASSERT(!mutex_owned(&xap->xa_mutex));

	mutex_enter(&xap->xa_mutex);
	xap->xa_timeid = NULL;
	xbuf_dispatch(xap);
	mutex_exit(&xap->xa_mutex);
}


DDII void
ddi_xbuf_flushq(ddi_xbuf_attr_t xap, int (*funcp)(struct buf *))
{
	struct buf *bp;
	struct buf *next_bp;
	struct buf *prev_bp = NULL;

	ASSERT(xap != NULL);
	ASSERT(xap->xa_tq != NULL);
	ASSERT(!mutex_owned(&xap->xa_mutex));

	mutex_enter(&xap->xa_mutex);

	for (bp = xap->xa_headp; bp != NULL; bp = next_bp) {

		next_bp = bp->av_forw;	/* Save for next iteration */

		/*
		 * If the user-supplied function is non-NULL and returns
		 * FALSE, then just leave the current bp on the queue.
		 */
		if ((funcp != NULL) && (!(*funcp)(bp))) {
			prev_bp = bp;
			continue;
		}

		/* de-queue the bp */
		if (bp == xap->xa_headp) {
			xap->xa_headp = next_bp;
			if (xap->xa_headp == NULL) {
				xap->xa_tailp = NULL;
			}
		} else {
			ASSERT(xap->xa_headp != NULL);
			ASSERT(prev_bp != NULL);
			if (bp == xap->xa_tailp) {
				ASSERT(next_bp == NULL);
				xap->xa_tailp = prev_bp;
			}
			prev_bp->av_forw = next_bp;
		}
		bp->av_forw = NULL;

		/* Add the bp to the flush queue */
		if (xap->xa_flush_headp == NULL) {
			ASSERT(xap->xa_flush_tailp == NULL);
			xap->xa_flush_headp = xap->xa_flush_tailp = bp;
		} else {
			ASSERT(xap->xa_flush_tailp != NULL);
			xap->xa_flush_tailp->av_forw = bp;
			xap->xa_flush_tailp = bp;
		}
	}

	while ((bp = xap->xa_flush_headp) != NULL) {
		xap->xa_flush_headp = bp->av_forw;
		if (xap->xa_flush_headp == NULL) {
			xap->xa_flush_tailp = NULL;
		}
		mutex_exit(&xap->xa_mutex);
		bioerror(bp, EIO);
		bp->b_resid = bp->b_bcount;
		biodone(bp);
		mutex_enter(&xap->xa_mutex);
	}

	mutex_exit(&xap->xa_mutex);
}