xref: /illumos-gate/usr/src/uts/common/io/bofi.c (revision f334afcfaebea1b7dc3430015651d8d748fa8a3e)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
 */


#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/buf.h>
#include <sys/errno.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/kmem.h>
#include <sys/proc.h>
#include <sys/cpuvar.h>
#include <sys/ddi_impldefs.h>
#include <sys/ddi.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/io/ddi.h>
#include <sys/sysevent/eventdefs.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/debug.h>
#include <sys/bofi.h>
#include <sys/bofi_impl.h>

#ifdef __sparc
#include <sys/dvma.h>
#endif

/*
 * Testing the resilience of a hardened device driver requires a suitably wide
 * range of different types of "typical" hardware faults to be injected,
 * preferably in a controlled and repeatable fashion. This is not in general
 * possible via hardware, so the "fault injection test harness" is provided.
 * This works by intercepting calls from the driver to various DDI routines,
 * and then corrupting the result of those DDI routine calls as if the
 * hardware had caused the corruption.
 *
 * Conceptually, the bofi driver consists of two parts:
 *
 * A driver interface that supports a number of ioctls which allow error
 * definitions ("errdefs") to be defined and subsequently managed. The
 * driver is a clone driver, so each open will create a separate
 * invocation. Any errdefs created by using ioctls to that invocation
 * will automatically be deleted when that invocation is closed.
 *
 * Intercept routines: When the bofi driver is attached, it edits the
 * bus_ops structure of the bus nexus specified by the "bofi-nexus"
 * field in the "bofi.conf" file, thus allowing the
 * bofi driver to intercept various ddi functions. These intercept
 * routines primarily carry out fault injections based on the errdefs
 * created for that device.
 *
 * Faults can be injected into:
 *
 * DMA (corrupting data for DMA to/from memory areas defined by
 * ddi_dma_setup(), ddi_dma_bind_handle(), etc)
 *
 * Physical IO (corrupting data sent/received via ddi_get8(), ddi_put8(),
 * etc),
 *
 * Interrupts (generating spurious interrupts, losing interrupts,
 * delaying interrupts).
 *
 * By default, ddi routines called from all drivers will be intercepted
 * and faults potentially injected. However, the "bofi-to-test" field in
 * the "bofi.conf" file can be set to a space-separated list of drivers to
 * test (or by preceding each driver name in the list with an "!", a list
 * of drivers not to test).
 *
 * In addition to fault injection, the bofi driver does a number of static
 * checks which are controlled by properties in the "bofi.conf" file.
 *
 * "bofi-ddi-check" - if set will validate that there are no PIO access
 * other than those using the DDI routines (ddi_get8(), ddi_put8(), etc).
 *
 * "bofi-range-check" - if set to values 1 (warning) or 2 (panic), will
 * validate that calls to ddi_get8(), ddi_put8(), etc are not made
 * specifying addresses outside the range of the access_handle.
 *
 * "bofi-sync-check" - if set will validate that calls to ddi_dma_sync()
 * are being made correctly.
 */

extern void *bp_mapin_common(struct buf *, int);

static int bofi_ddi_check;
static int bofi_sync_check;
static int bofi_range_check;

static struct bofi_link bofi_link_array[BOFI_NLINKS], *bofi_link_freelist;

#define	LLSZMASK (sizeof (uint64_t)-1)

#define	HDL_HASH_TBL_SIZE 64
static struct bofi_shadow hhash_table[HDL_HASH_TBL_SIZE];
static struct bofi_shadow dhash_table[HDL_HASH_TBL_SIZE];
#define	HDL_DHASH(x) \
	(&dhash_table[((uintptr_t)(x) >> 3) & (HDL_HASH_TBL_SIZE-1)])
#define	HDL_HHASH(x) \
	(&hhash_table[((uintptr_t)(x) >> 5) & (HDL_HASH_TBL_SIZE-1)])

static struct bofi_shadow shadow_list;
static struct bofi_errent *errent_listp;

static char driver_list[NAMESIZE];
static int driver_list_size;
static int driver_list_neg;
static char nexus_name[NAMESIZE];

static int initialized = 0;

#define	NCLONES 2560
static int clone_tab[NCLONES];

static dev_info_t *our_dip;

static kmutex_t bofi_mutex;
static kmutex_t clone_tab_mutex;
static kmutex_t bofi_low_mutex;
static ddi_iblock_cookie_t bofi_low_cookie;
static uint_t	bofi_signal(caddr_t arg);
static int	bofi_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
static int	bofi_attach(dev_info_t *, ddi_attach_cmd_t);
static int	bofi_detach(dev_info_t *, ddi_detach_cmd_t);
static int	bofi_open(dev_t *, int, int, cred_t *);
static int	bofi_close(dev_t, int, int, cred_t *);
static int	bofi_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
static int	bofi_errdef_alloc(struct bofi_errdef *, char *,
		    struct bofi_errent *);
static int	bofi_errdef_free(struct bofi_errent *);
static void	bofi_start(struct bofi_errctl *, char *);
static void	bofi_stop(struct bofi_errctl *, char *);
static void	bofi_broadcast(struct bofi_errctl *, char *);
static void	bofi_clear_acc_chk(struct bofi_errctl *, char *);
static void	bofi_clear_errors(struct bofi_errctl *, char *);
static void	bofi_clear_errdefs(struct bofi_errctl *, char *);
static int	bofi_errdef_check(struct bofi_errstate *,
		    struct acc_log_elem **);
static int	bofi_errdef_check_w(struct bofi_errstate *,
		    struct acc_log_elem **);
static int	bofi_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
		    off_t, off_t, caddr_t *);
static int	bofi_dma_allochdl(dev_info_t *, dev_info_t *,
		    ddi_dma_attr_t *, int (*)(caddr_t), caddr_t,
		    ddi_dma_handle_t *);
static int	bofi_dma_freehdl(dev_info_t *, dev_info_t *,
		    ddi_dma_handle_t);
static int	bofi_dma_bindhdl(dev_info_t *, dev_info_t *,
		    ddi_dma_handle_t, struct ddi_dma_req *, ddi_dma_cookie_t *,
		    uint_t *);
static int	bofi_dma_unbindhdl(dev_info_t *, dev_info_t *,
		    ddi_dma_handle_t);
static int	bofi_dma_flush(dev_info_t *, dev_info_t *, ddi_dma_handle_t,
		    off_t, size_t, uint_t);
static int	bofi_dma_ctl(dev_info_t *, dev_info_t *, ddi_dma_handle_t,
		    enum ddi_dma_ctlops, off_t *, size_t *, caddr_t *, uint_t);
static int	bofi_dma_win(dev_info_t *, dev_info_t *, ddi_dma_handle_t,
		    uint_t, off_t *, size_t *, ddi_dma_cookie_t *, uint_t *);
static int	bofi_intr_ops(dev_info_t *dip, dev_info_t *rdip,
		    ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp,
		    void *result);
static int	bofi_fm_ereport_callback(sysevent_t *ev, void *cookie);

evchan_t *bofi_error_chan;

#define	FM_SIMULATED_DMA "simulated.dma"
#define	FM_SIMULATED_PIO "simulated.pio"

#if defined(__sparc)
static void	bofi_dvma_kaddr_load(ddi_dma_handle_t, caddr_t, uint_t,
		    uint_t, ddi_dma_cookie_t *);
static void	bofi_dvma_unload(ddi_dma_handle_t, uint_t, uint_t);
static void	bofi_dvma_sync(ddi_dma_handle_t, uint_t, uint_t);
static void	bofi_dvma_reserve(dev_info_t *, ddi_dma_handle_t);
#endif
static int	driver_under_test(dev_info_t *);
static int	bofi_check_acc_hdl(ddi_acc_impl_t *);
static int	bofi_check_dma_hdl(ddi_dma_impl_t *);
static int	bofi_post_event(dev_info_t *dip, dev_info_t *rdip,
		    ddi_eventcookie_t eventhdl, void *impl_data);

static struct bus_ops bofi_bus_ops = {
	BUSO_REV,
	bofi_map,
	NULL,
	NULL,
	NULL,
	i_ddi_map_fault,
	NULL,
	bofi_dma_allochdl,
	bofi_dma_freehdl,
	bofi_dma_bindhdl,
	bofi_dma_unbindhdl,
	bofi_dma_flush,
	bofi_dma_win,
	bofi_dma_ctl,
	NULL,
	ddi_bus_prop_op,
	ndi_busop_get_eventcookie,
	ndi_busop_add_eventcall,
	ndi_busop_remove_eventcall,
	bofi_post_event,
	NULL,
	0,
	0,
	0,
	0,
	0,
	0,
	0,
	bofi_intr_ops
};

static struct cb_ops bofi_cb_ops = {
	bofi_open,		/* open */
	bofi_close,		/* close */
	nodev,			/* strategy */
	nodev,			/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	bofi_ioctl,		/* ioctl */
	nodev,			/* devmap */
	nodev,			/* mmap */
	nodev,			/* segmap */
	nochpoll,		/* chpoll */
	ddi_prop_op,		/* prop_op */
	NULL,			/* for STREAMS drivers */
	D_MP,			/* driver compatibility flag */
	CB_REV,			/* cb_ops revision */
	nodev,			/* aread */
	nodev			/* awrite */
};

static struct dev_ops bofi_ops = {
	DEVO_REV,		/* driver build version */
	0,			/* device reference count */
	bofi_getinfo,
	nulldev,
	nulldev,		/* probe */
	bofi_attach,
	bofi_detach,
	nulldev,		/* reset */
	&bofi_cb_ops,
	(struct bus_ops *)NULL,
	nulldev,		/* power */
	ddi_quiesce_not_needed,		/* quiesce */
};

/* module configuration stuff */
static void    *statep;

static struct modldrv modldrv = {
	&mod_driverops,
	"bofi driver",
	&bofi_ops
};

static struct modlinkage modlinkage = {
	MODREV_1,
	&modldrv,
	0
};

static struct bus_ops save_bus_ops;

#if defined(__sparc)
static struct dvma_ops bofi_dvma_ops = {
	DVMAO_REV,
	bofi_dvma_kaddr_load,
	bofi_dvma_unload,
	bofi_dvma_sync
};
#endif

/*
 * support routine - map user page into kernel virtual
 */
static caddr_t
dmareq_mapin(offset_t len, caddr_t addr, struct as *as, int flag)
{
	struct buf buf;
	struct proc proc;

	/*
	 * mock up a buf structure so we can call bp_mapin_common()
	 */
	buf.b_flags = B_PHYS;
	buf.b_un.b_addr = (caddr_t)addr;
	buf.b_bcount = (size_t)len;
	proc.p_as = as;
	buf.b_proc = &proc;
	return (bp_mapin_common(&buf, flag));
}


/*
 * support routine - map page chain into kernel virtual
 */
static caddr_t
dmareq_pp_mapin(offset_t len, uint_t offset, page_t *pp, int flag)
{
	struct buf buf;

	/*
	 * mock up a buf structure so we can call bp_mapin_common()
	 */
	buf.b_flags = B_PAGEIO;
	buf.b_un.b_addr = (caddr_t)(uintptr_t)offset;
	buf.b_bcount = (size_t)len;
	buf.b_pages = pp;
	return (bp_mapin_common(&buf, flag));
}


/*
 * support routine - map page array into kernel virtual
 */
static caddr_t
dmareq_pplist_mapin(uint_t len, caddr_t addr, page_t **pplist, struct as *as,
    int flag)
{
	struct buf buf;
	struct proc proc;

	/*
	 * mock up a buf structure so we can call bp_mapin_common()
	 */
	buf.b_flags = B_PHYS|B_SHADOW;
	buf.b_un.b_addr = addr;
	buf.b_bcount = len;
	buf.b_shadow = pplist;
	proc.p_as = as;
	buf.b_proc = &proc;
	return (bp_mapin_common(&buf, flag));
}


/*
 * support routine - map dmareq into kernel virtual if not already
 * fills in *lenp with length
 * *mapaddr will be new kernel virtual address - or null if no mapping needed
 */
static caddr_t
ddi_dmareq_mapin(struct ddi_dma_req *dmareqp, caddr_t *mapaddrp,
    offset_t *lenp)
{
	int sleep = (dmareqp->dmar_fp == DDI_DMA_SLEEP) ? VM_SLEEP: VM_NOSLEEP;

	*lenp = dmareqp->dmar_object.dmao_size;
	if (dmareqp->dmar_object.dmao_type == DMA_OTYP_PAGES) {
		*mapaddrp = dmareq_pp_mapin(dmareqp->dmar_object.dmao_size,
		    dmareqp->dmar_object.dmao_obj.pp_obj.pp_offset,
		    dmareqp->dmar_object.dmao_obj.pp_obj.pp_pp, sleep);
		return (*mapaddrp);
	} else if (dmareqp->dmar_object.dmao_obj.virt_obj.v_priv != NULL) {
		*mapaddrp = dmareq_pplist_mapin(dmareqp->dmar_object.dmao_size,
		    dmareqp->dmar_object.dmao_obj.virt_obj.v_addr,
		    dmareqp->dmar_object.dmao_obj.virt_obj.v_priv,
		    dmareqp->dmar_object.dmao_obj.virt_obj.v_as, sleep);
		return (*mapaddrp);
	} else if (dmareqp->dmar_object.dmao_obj.virt_obj.v_as == &kas) {
		*mapaddrp = NULL;
		return (dmareqp->dmar_object.dmao_obj.virt_obj.v_addr);
	} else if (dmareqp->dmar_object.dmao_obj.virt_obj.v_as == NULL) {
		*mapaddrp = NULL;
		return (dmareqp->dmar_object.dmao_obj.virt_obj.v_addr);
	} else {
		*mapaddrp = dmareq_mapin(dmareqp->dmar_object.dmao_size,
		    dmareqp->dmar_object.dmao_obj.virt_obj.v_addr,
		    dmareqp->dmar_object.dmao_obj.virt_obj.v_as, sleep);
		return (*mapaddrp);
	}
}


/*
 * support routine - free off kernel virtual mapping as allocated by
 * ddi_dmareq_mapin()
 */
static void
ddi_dmareq_mapout(caddr_t addr, offset_t len, int map_flags, page_t *pp,
    page_t **pplist)
{
	struct buf buf;

	if (addr == NULL)
		return;
	/*
	 * mock up a buf structure
	 */
	buf.b_flags = B_REMAPPED | map_flags;
	buf.b_un.b_addr = addr;
	buf.b_bcount = (size_t)len;
	buf.b_pages = pp;
	buf.b_shadow = pplist;
	bp_mapout(&buf);
}

static time_t
bofi_gettime()
{
	timestruc_t ts;

	gethrestime(&ts);
	return (ts.tv_sec);
}

/*
 * reset the bus_ops structure of the specified nexus to point to
 * the original values in the save_bus_ops structure.
 *
 * Note that both this routine and modify_bus_ops() rely on the current
 * behavior of the framework in that nexus drivers are not unloadable
 *
 */

static int
reset_bus_ops(char *name, struct bus_ops *bop)
{
	struct modctl *modp;
	struct modldrv *mp;
	struct bus_ops *bp;
	struct dev_ops *ops;

	mutex_enter(&mod_lock);
	/*
	 * find specified module
	 */
	modp = &modules;
	do {
		if (strcmp(name, modp->mod_modname) == 0) {
			if (!modp->mod_linkage) {
				mutex_exit(&mod_lock);
				return (0);
			}
			mp = modp->mod_linkage->ml_linkage[0];
			if (!mp || !mp->drv_dev_ops) {
				mutex_exit(&mod_lock);
				return (0);
			}
			ops = mp->drv_dev_ops;
			bp = ops->devo_bus_ops;
			if (!bp) {
				mutex_exit(&mod_lock);
				return (0);
			}
			if (ops->devo_refcnt > 0) {
				/*
				 * As long as devices are active with modified
				 * bus ops bofi must not go away. There may be
				 * drivers with modified access or dma handles.
				 */
				mutex_exit(&mod_lock);
				return (0);
			}
			cmn_err(CE_NOTE, "bofi reset bus_ops for %s",
			    mp->drv_linkinfo);
			bp->bus_intr_op = bop->bus_intr_op;
			bp->bus_post_event = bop->bus_post_event;
			bp->bus_map = bop->bus_map;
			bp->bus_dma_map = bop->bus_dma_map;
			bp->bus_dma_allochdl = bop->bus_dma_allochdl;
			bp->bus_dma_freehdl = bop->bus_dma_freehdl;
			bp->bus_dma_bindhdl = bop->bus_dma_bindhdl;
			bp->bus_dma_unbindhdl = bop->bus_dma_unbindhdl;
			bp->bus_dma_flush = bop->bus_dma_flush;
			bp->bus_dma_win = bop->bus_dma_win;
			bp->bus_dma_ctl = bop->bus_dma_ctl;
			mutex_exit(&mod_lock);
			return (1);
		}
	} while ((modp = modp->mod_next) != &modules);
	mutex_exit(&mod_lock);
	return (0);
}

/*
 * modify the bus_ops structure of the specified nexus to point to bofi
 * routines, saving the original values in the save_bus_ops structure
 */

static int
modify_bus_ops(char *name, struct bus_ops *bop)
{
	struct modctl *modp;
	struct modldrv *mp;
	struct bus_ops *bp;
	struct dev_ops *ops;

	if (ddi_name_to_major(name) == -1)
		return (0);

	mutex_enter(&mod_lock);
	/*
	 * find specified module
	 */
	modp = &modules;
	do {
		if (strcmp(name, modp->mod_modname) == 0) {
			if (!modp->mod_linkage) {
				mutex_exit(&mod_lock);
				return (0);
			}
			mp = modp->mod_linkage->ml_linkage[0];
			if (!mp || !mp->drv_dev_ops) {
				mutex_exit(&mod_lock);
				return (0);
			}
			ops = mp->drv_dev_ops;
			bp = ops->devo_bus_ops;
			if (!bp) {
				mutex_exit(&mod_lock);
				return (0);
			}
			if (ops->devo_refcnt == 0) {
				/*
				 * If there is no device active for this
				 * module then there is nothing to do for bofi.
				 */
				mutex_exit(&mod_lock);
				return (0);
			}
			cmn_err(CE_NOTE, "bofi modify bus_ops for %s",
			    mp->drv_linkinfo);
			save_bus_ops = *bp;
			bp->bus_intr_op = bop->bus_intr_op;
			bp->bus_post_event = bop->bus_post_event;
			bp->bus_map = bop->bus_map;
			bp->bus_dma_map = bop->bus_dma_map;
			bp->bus_dma_allochdl = bop->bus_dma_allochdl;
			bp->bus_dma_freehdl = bop->bus_dma_freehdl;
			bp->bus_dma_bindhdl = bop->bus_dma_bindhdl;
			bp->bus_dma_unbindhdl = bop->bus_dma_unbindhdl;
			bp->bus_dma_flush = bop->bus_dma_flush;
			bp->bus_dma_win = bop->bus_dma_win;
			bp->bus_dma_ctl = bop->bus_dma_ctl;
			mutex_exit(&mod_lock);
			return (1);
		}
	} while ((modp = modp->mod_next) != &modules);
	mutex_exit(&mod_lock);
	return (0);
}


int
_init(void)
{
	int    e;

	e = ddi_soft_state_init(&statep, sizeof (struct bofi_errent), 1);
	if (e != 0)
		return (e);
	if ((e = mod_install(&modlinkage)) != 0)
		ddi_soft_state_fini(&statep);
	return (e);
}


int
_fini(void)
{
	int e;

	if ((e = mod_remove(&modlinkage)) != 0)
		return (e);
	ddi_soft_state_fini(&statep);
	return (e);
}


int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}


static int
bofi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	char *name;
	char buf[80];
	int i;
	int s, ss;
	int size = NAMESIZE;
	int new_string;
	char *ptr;

	if (cmd != DDI_ATTACH)
		return (DDI_FAILURE);
	/*
	 * only one instance - but we clone using the open routine
	 */
	if (ddi_get_instance(dip) > 0)
		return (DDI_FAILURE);

	if (!initialized) {
		if ((name = ddi_get_name(dip)) == NULL)
			return (DDI_FAILURE);
		(void) snprintf(buf, sizeof (buf), "%s,ctl", name);
		if (ddi_create_minor_node(dip, buf, S_IFCHR, 0,
		    DDI_PSEUDO, 0) == DDI_FAILURE)
			return (DDI_FAILURE);

		if (ddi_get_soft_iblock_cookie(dip, DDI_SOFTINT_MED,
		    &bofi_low_cookie) != DDI_SUCCESS) {
			ddi_remove_minor_node(dip, buf);
			return (DDI_FAILURE); /* fail attach */
		}
		/*
		 * get nexus name (from conf file)
		 */
		if (ddi_prop_op(DDI_DEV_T_ANY, dip, PROP_LEN_AND_VAL_BUF, 0,
		    "bofi-nexus", nexus_name, &size) != DDI_PROP_SUCCESS) {
			ddi_remove_minor_node(dip, buf);
			return (DDI_FAILURE);
		}
		/*
		 * get whether to do dma map kmem private checking
		 */
		if ((bofi_range_check = ddi_prop_lookup_string(DDI_DEV_T_ANY,
		    dip, 0, "bofi-range-check", &ptr)) != DDI_PROP_SUCCESS)
			bofi_range_check = 0;
		else if (strcmp(ptr, "panic") == 0)
			bofi_range_check = 2;
		else if (strcmp(ptr, "warn") == 0)
			bofi_range_check = 1;
		else
			bofi_range_check = 0;
		ddi_prop_free(ptr);

		/*
		 * get whether to prevent direct access to register
		 */
		if ((bofi_ddi_check = ddi_prop_lookup_string(DDI_DEV_T_ANY,
		    dip, 0, "bofi-ddi-check", &ptr)) != DDI_PROP_SUCCESS)
			bofi_ddi_check = 0;
		else if (strcmp(ptr, "on") == 0)
			bofi_ddi_check = 1;
		else
			bofi_ddi_check = 0;
		ddi_prop_free(ptr);

		/*
		 * get whether to do copy on ddi_dma_sync
		 */
		if ((bofi_sync_check = ddi_prop_lookup_string(DDI_DEV_T_ANY,
		    dip, 0, "bofi-sync-check", &ptr)) != DDI_PROP_SUCCESS)
			bofi_sync_check = 0;
		else if (strcmp(ptr, "on") == 0)
			bofi_sync_check = 1;
		else
			bofi_sync_check = 0;
		ddi_prop_free(ptr);

		/*
		 * get driver-under-test names (from conf file)
		 */
		size = NAMESIZE;
		if (ddi_prop_op(DDI_DEV_T_ANY, dip, PROP_LEN_AND_VAL_BUF, 0,
		    "bofi-to-test", driver_list, &size) != DDI_PROP_SUCCESS)
			driver_list[0] = 0;
		/*
		 * and convert into a sequence of strings
		 */
		driver_list_neg = 1;
		new_string = 1;
		driver_list_size = strlen(driver_list);
		for (i = 0; i < driver_list_size; i++) {
			if (driver_list[i] == ' ') {
				driver_list[i] = '\0';
				new_string = 1;
			} else if (new_string) {
				if (driver_list[i] != '!')
					driver_list_neg = 0;
				new_string = 0;
			}
		}
		/*
		 * initialize mutex, lists
		 */
		mutex_init(&clone_tab_mutex, NULL, MUTEX_DRIVER,
		    NULL);
		/*
		 * fake up iblock cookie - need to protect outselves
		 * against drivers that use hilevel interrupts
		 */
		ss = spl8();
		s = spl8();
		splx(ss);
		mutex_init(&bofi_mutex, NULL, MUTEX_SPIN, (void *)(uintptr_t)s);
		mutex_init(&bofi_low_mutex, NULL, MUTEX_DRIVER,
		    (void *)bofi_low_cookie);
		shadow_list.next = &shadow_list;
		shadow_list.prev = &shadow_list;
		for (i = 0; i < HDL_HASH_TBL_SIZE; i++) {
			hhash_table[i].hnext = &hhash_table[i];
			hhash_table[i].hprev = &hhash_table[i];
			dhash_table[i].dnext = &dhash_table[i];
			dhash_table[i].dprev = &dhash_table[i];
		}
		for (i = 1; i < BOFI_NLINKS; i++)
			bofi_link_array[i].link = &bofi_link_array[i-1];
		bofi_link_freelist = &bofi_link_array[BOFI_NLINKS - 1];
		/*
		 * overlay bus_ops structure
		 */
		if (modify_bus_ops(nexus_name, &bofi_bus_ops) == 0) {
			ddi_remove_minor_node(dip, buf);
			mutex_destroy(&clone_tab_mutex);
			mutex_destroy(&bofi_mutex);
			mutex_destroy(&bofi_low_mutex);
			return (DDI_FAILURE);
		}
		if (sysevent_evc_bind(FM_ERROR_CHAN, &bofi_error_chan, 0) == 0)
			(void) sysevent_evc_subscribe(bofi_error_chan, "bofi",
			    EC_FM, bofi_fm_ereport_callback, NULL, 0);

		/*
		 * save dip for getinfo
		 */
		our_dip = dip;
		ddi_report_dev(dip);
		initialized = 1;
	}
	return (DDI_SUCCESS);
}


static int
bofi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	char *name;
	char buf[80];

	if (cmd != DDI_DETACH)
		return (DDI_FAILURE);
	if (ddi_get_instance(dip) > 0)
		return (DDI_FAILURE);
	if ((name = ddi_get_name(dip)) == NULL)
		return (DDI_FAILURE);
	(void) snprintf(buf, sizeof (buf), "%s,ctl", name);
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	/*
	 * make sure test bofi is no longer in use
	 */
	if (shadow_list.next != &shadow_list || errent_listp != NULL) {
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (DDI_FAILURE);
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);

	/*
	 * restore bus_ops structure
	 */
	if (reset_bus_ops(nexus_name, &save_bus_ops) == 0)
		return (DDI_FAILURE);

	(void) sysevent_evc_unbind(bofi_error_chan);

	mutex_destroy(&clone_tab_mutex);
	mutex_destroy(&bofi_mutex);
	mutex_destroy(&bofi_low_mutex);
	ddi_remove_minor_node(dip, buf);
	our_dip = NULL;
	initialized = 0;
	return (DDI_SUCCESS);
}


/* ARGSUSED */
static int
bofi_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	dev_t	dev = (dev_t)arg;
	int	minor = (int)getminor(dev);
	int	retval;

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if (minor != 0 || our_dip == NULL) {
			*result = (void *)NULL;
			retval = DDI_FAILURE;
		} else {
			*result = (void *)our_dip;
			retval = DDI_SUCCESS;
		}
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		retval = DDI_SUCCESS;
		break;
	default:
		retval = DDI_FAILURE;
	}
	return (retval);
}


/* ARGSUSED */
static int
bofi_open(dev_t *devp, int flag, int otyp, cred_t *credp)
{
	int	minor = (int)getminor(*devp);
	struct bofi_errent *softc;

	/*
	 * only allow open on minor=0 - the clone device
	 */
	if (minor != 0)
		return (ENXIO);
	/*
	 * fail if not attached
	 */
	if (!initialized)
		return (ENXIO);
	/*
	 * find a free slot and grab it
	 */
	mutex_enter(&clone_tab_mutex);
	for (minor = 1; minor < NCLONES; minor++) {
		if (clone_tab[minor] == 0) {
			clone_tab[minor] = 1;
			break;
		}
	}
	mutex_exit(&clone_tab_mutex);
	if (minor == NCLONES)
		return (EAGAIN);
	/*
	 * soft state structure for this clone is used to maintain a list
	 * of allocated errdefs so they can be freed on close
	 */
	if (ddi_soft_state_zalloc(statep, minor) != DDI_SUCCESS) {
		mutex_enter(&clone_tab_mutex);
		clone_tab[minor] = 0;
		mutex_exit(&clone_tab_mutex);
		return (EAGAIN);
	}
	softc = ddi_get_soft_state(statep, minor);
	softc->cnext = softc;
	softc->cprev = softc;

	*devp = makedevice(getmajor(*devp), minor);
	return (0);
}


/* ARGSUSED */
static int
bofi_close(dev_t dev, int flag, int otyp, cred_t *credp)
{
	int	minor = (int)getminor(dev);
	struct bofi_errent *softc;
	struct bofi_errent *ep, *next_ep;

	softc = ddi_get_soft_state(statep, minor);
	if (softc == NULL)
		return (ENXIO);
	/*
	 * find list of errdefs and free them off
	 */
	for (ep = softc->cnext; ep != softc; ) {
		next_ep = ep->cnext;
		(void) bofi_errdef_free(ep);
		ep = next_ep;
	}
	/*
	 * free clone tab slot
	 */
	mutex_enter(&clone_tab_mutex);
	clone_tab[minor] = 0;
	mutex_exit(&clone_tab_mutex);

	ddi_soft_state_free(statep, minor);
	return (0);
}


/* ARGSUSED */
static int
bofi_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
    int *rvalp)
{
	struct bofi_errent *softc;
	int	minor = (int)getminor(dev);
	struct bofi_errdef errdef;
	struct bofi_errctl errctl;
	struct bofi_errstate errstate;
	void *ed_handle;
	struct bofi_get_handles get_handles;
	struct bofi_get_hdl_info hdl_info;
	struct handle_info *hdlip;
	struct handle_info *hib;

	char *buffer;
	char *bufptr;
	char *endbuf;
	int req_count, count, err;
	char *namep;
	struct bofi_shadow *hp;
	int retval;
	struct bofi_shadow *hhashp;
	int i;

	switch (cmd) {
	case BOFI_ADD_DEF:
		/*
		 * add a new error definition
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errdef32	errdef_32;

			if (ddi_copyin((void *)arg, &errdef_32,
			    sizeof (struct bofi_errdef32), mode)) {
				return (EFAULT);
			}
			errdef.namesize = errdef_32.namesize;
			(void) strncpy(errdef.name, errdef_32.name, NAMESIZE);
			errdef.instance = errdef_32.instance;
			errdef.rnumber = errdef_32.rnumber;
			errdef.offset = errdef_32.offset;
			errdef.len = errdef_32.len;
			errdef.access_type = errdef_32.access_type;
			errdef.access_count = errdef_32.access_count;
			errdef.fail_count = errdef_32.fail_count;
			errdef.acc_chk = errdef_32.acc_chk;
			errdef.optype = errdef_32.optype;
			errdef.operand = errdef_32.operand;
			errdef.log.logsize = errdef_32.log.logsize;
			errdef.log.entries = errdef_32.log.entries;
			errdef.log.flags = errdef_32.log.flags;
			errdef.log.wrapcnt = errdef_32.log.wrapcnt;
			errdef.log.start_time = errdef_32.log.start_time;
			errdef.log.stop_time = errdef_32.log.stop_time;
			errdef.log.logbase =
			    (caddr_t)(uintptr_t)errdef_32.log.logbase;
			errdef.errdef_handle = errdef_32.errdef_handle;
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyin((void *)arg, &errdef,
			    sizeof (struct bofi_errdef), mode))
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyin((void *)arg, &errdef,
		    sizeof (struct bofi_errdef), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		/*
		 * do some validation
		 */
		if (errdef.fail_count == 0)
			errdef.optype = 0;
		if (errdef.optype != 0) {
			if (errdef.access_type & BOFI_INTR &&
			    errdef.optype != BOFI_DELAY_INTR &&
			    errdef.optype != BOFI_LOSE_INTR &&
			    errdef.optype != BOFI_EXTRA_INTR)
				return (EINVAL);
			if ((errdef.access_type & (BOFI_DMA_RW|BOFI_PIO_R)) &&
			    errdef.optype == BOFI_NO_TRANSFER)
				return (EINVAL);
			if ((errdef.access_type & (BOFI_PIO_RW)) &&
			    errdef.optype != BOFI_EQUAL &&
			    errdef.optype != BOFI_OR &&
			    errdef.optype != BOFI_XOR &&
			    errdef.optype != BOFI_AND &&
			    errdef.optype != BOFI_NO_TRANSFER)
				return (EINVAL);
		}
		/*
		 * find softstate for this clone, so we can tag
		 * new errdef on to it
		 */
		softc = ddi_get_soft_state(statep, minor);
		if (softc == NULL)
			return (ENXIO);
		/*
		 * read in name
		 */
		if (errdef.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errdef.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errdef.name, errdef.namesize);

		if (bofi_errdef_alloc(&errdef, namep, softc) != DDI_SUCCESS) {
			(void) bofi_errdef_free((struct bofi_errent *)
			    (uintptr_t)errdef.errdef_handle);
			kmem_free(namep, errdef.namesize+1);
			return (EINVAL);
		}
		/*
		 * copy out errdef again, including filled in errdef_handle
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errdef32	errdef_32;

			errdef_32.namesize = errdef.namesize;
			(void) strncpy(errdef_32.name, errdef.name, NAMESIZE);
			errdef_32.instance = errdef.instance;
			errdef_32.rnumber = errdef.rnumber;
			errdef_32.offset = errdef.offset;
			errdef_32.len = errdef.len;
			errdef_32.access_type = errdef.access_type;
			errdef_32.access_count = errdef.access_count;
			errdef_32.fail_count = errdef.fail_count;
			errdef_32.acc_chk = errdef.acc_chk;
			errdef_32.optype = errdef.optype;
			errdef_32.operand = errdef.operand;
			errdef_32.log.logsize = errdef.log.logsize;
			errdef_32.log.entries = errdef.log.entries;
			errdef_32.log.flags = errdef.log.flags;
			errdef_32.log.wrapcnt = errdef.log.wrapcnt;
			errdef_32.log.start_time = errdef.log.start_time;
			errdef_32.log.stop_time = errdef.log.stop_time;
			errdef_32.log.logbase =
			    (caddr32_t)(uintptr_t)errdef.log.logbase;
			errdef_32.errdef_handle = errdef.errdef_handle;
			if (ddi_copyout(&errdef_32, (void *)arg,
			    sizeof (struct bofi_errdef32), mode) != 0) {
				(void) bofi_errdef_free((struct bofi_errent *)
				    errdef.errdef_handle);
				kmem_free(namep, errdef.namesize+1);
				return (EFAULT);
			}
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyout(&errdef, (void *)arg,
			    sizeof (struct bofi_errdef), mode) != 0) {
				(void) bofi_errdef_free((struct bofi_errent *)
				    errdef.errdef_handle);
				kmem_free(namep, errdef.namesize+1);
				return (EFAULT);
			}
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyout(&errdef, (void *)arg,
		    sizeof (struct bofi_errdef), mode) != 0) {
			(void) bofi_errdef_free((struct bofi_errent *)
			    (uintptr_t)errdef.errdef_handle);
			kmem_free(namep, errdef.namesize+1);
			return (EFAULT);
		}
#endif /* _MULTI_DATAMODEL */
		return (0);
	case BOFI_DEL_DEF:
		/*
		 * delete existing errdef
		 */
		if (ddi_copyin((void *)arg, &ed_handle,
		    sizeof (void *), mode) != 0)
			return (EFAULT);
		return (bofi_errdef_free((struct bofi_errent *)ed_handle));
	case BOFI_START:
		/*
		 * start all errdefs corresponding to
		 * this name and instance
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_start(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_STOP:
		/*
		 * stop all errdefs corresponding to
		 * this name and instance
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_stop(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_BROADCAST:
		/*
		 * wakeup all errdefs corresponding to
		 * this name and instance
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_broadcast(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_CLEAR_ACC_CHK:
		/*
		 * clear "acc_chk" for all errdefs corresponding to
		 * this name and instance
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_clear_acc_chk(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_CLEAR_ERRORS:
		/*
		 * set "fail_count" to 0 for all errdefs corresponding to
		 * this name and instance whose "access_count"
		 * has expired.
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_clear_errors(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_CLEAR_ERRDEFS:
		/*
		 * set "access_count" and "fail_count" to 0 for all errdefs
		 * corresponding to this name and instance
		 */
		if (ddi_copyin((void *)arg, &errctl,
		    sizeof (struct bofi_errctl), mode) != 0)
			return (EFAULT);
		/*
		 * copy in name
		 */
		if (errctl.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(errctl.namesize+1, KM_SLEEP);
		(void) strncpy(namep, errctl.name, errctl.namesize);
		bofi_clear_errdefs(&errctl, namep);
		kmem_free(namep, errctl.namesize+1);
		return (0);
	case BOFI_CHK_STATE:
	{
		struct acc_log_elem *klg;
		size_t uls;
		/*
		 * get state for this errdef - read in dummy errstate
		 * with just the errdef_handle filled in
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errstate32	errstate_32;

			if (ddi_copyin((void *)arg, &errstate_32,
			    sizeof (struct bofi_errstate32), mode) != 0) {
				return (EFAULT);
			}
			errstate.fail_time = errstate_32.fail_time;
			errstate.msg_time = errstate_32.msg_time;
			errstate.access_count = errstate_32.access_count;
			errstate.fail_count = errstate_32.fail_count;
			errstate.acc_chk = errstate_32.acc_chk;
			errstate.errmsg_count = errstate_32.errmsg_count;
			(void) strncpy(errstate.buffer, errstate_32.buffer,
			    ERRMSGSIZE);
			errstate.severity = errstate_32.severity;
			errstate.log.logsize = errstate_32.log.logsize;
			errstate.log.entries = errstate_32.log.entries;
			errstate.log.flags = errstate_32.log.flags;
			errstate.log.wrapcnt = errstate_32.log.wrapcnt;
			errstate.log.start_time = errstate_32.log.start_time;
			errstate.log.stop_time = errstate_32.log.stop_time;
			errstate.log.logbase =
			    (caddr_t)(uintptr_t)errstate_32.log.logbase;
			errstate.errdef_handle = errstate_32.errdef_handle;
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyin((void *)arg, &errstate,
			    sizeof (struct bofi_errstate), mode) != 0)
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyin((void *)arg, &errstate,
		    sizeof (struct bofi_errstate), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		if ((retval = bofi_errdef_check(&errstate, &klg)) == EINVAL)
			return (EINVAL);
		/*
		 * copy out real errstate structure
		 */
		uls = errstate.log.logsize;
		if (errstate.log.entries > uls && uls)
			/* insufficient user memory */
			errstate.log.entries = uls;
		/* always pass back a time */
		if (errstate.log.stop_time == 0ul)
			(void) drv_getparm(TIME, &(errstate.log.stop_time));

#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errstate32	errstate_32;

			errstate_32.fail_time = errstate.fail_time;
			errstate_32.msg_time = errstate.msg_time;
			errstate_32.access_count = errstate.access_count;
			errstate_32.fail_count = errstate.fail_count;
			errstate_32.acc_chk = errstate.acc_chk;
			errstate_32.errmsg_count = errstate.errmsg_count;
			(void) strncpy(errstate_32.buffer, errstate.buffer,
			    ERRMSGSIZE);
			errstate_32.severity = errstate.severity;
			errstate_32.log.logsize = errstate.log.logsize;
			errstate_32.log.entries = errstate.log.entries;
			errstate_32.log.flags = errstate.log.flags;
			errstate_32.log.wrapcnt = errstate.log.wrapcnt;
			errstate_32.log.start_time = errstate.log.start_time;
			errstate_32.log.stop_time = errstate.log.stop_time;
			errstate_32.log.logbase =
			    (caddr32_t)(uintptr_t)errstate.log.logbase;
			errstate_32.errdef_handle = errstate.errdef_handle;
			if (ddi_copyout(&errstate_32, (void *)arg,
			    sizeof (struct bofi_errstate32), mode) != 0)
				return (EFAULT);
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyout(&errstate, (void *)arg,
			    sizeof (struct bofi_errstate), mode) != 0)
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyout(&errstate, (void *)arg,
		    sizeof (struct bofi_errstate), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		if (uls && errstate.log.entries &&
		    ddi_copyout(klg, errstate.log.logbase,
		    errstate.log.entries * sizeof (struct acc_log_elem),
		    mode) != 0) {
			return (EFAULT);
		}
		return (retval);
	}
	case BOFI_CHK_STATE_W:
	{
		struct acc_log_elem *klg;
		size_t uls;
		/*
		 * get state for this errdef - read in dummy errstate
		 * with just the errdef_handle filled in. Then wait for
		 * a ddi_report_fault message to come back
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errstate32	errstate_32;

			if (ddi_copyin((void *)arg, &errstate_32,
			    sizeof (struct bofi_errstate32), mode) != 0) {
				return (EFAULT);
			}
			errstate.fail_time = errstate_32.fail_time;
			errstate.msg_time = errstate_32.msg_time;
			errstate.access_count = errstate_32.access_count;
			errstate.fail_count = errstate_32.fail_count;
			errstate.acc_chk = errstate_32.acc_chk;
			errstate.errmsg_count = errstate_32.errmsg_count;
			(void) strncpy(errstate.buffer, errstate_32.buffer,
			    ERRMSGSIZE);
			errstate.severity = errstate_32.severity;
			errstate.log.logsize = errstate_32.log.logsize;
			errstate.log.entries = errstate_32.log.entries;
			errstate.log.flags = errstate_32.log.flags;
			errstate.log.wrapcnt = errstate_32.log.wrapcnt;
			errstate.log.start_time = errstate_32.log.start_time;
			errstate.log.stop_time = errstate_32.log.stop_time;
			errstate.log.logbase =
			    (caddr_t)(uintptr_t)errstate_32.log.logbase;
			errstate.errdef_handle = errstate_32.errdef_handle;
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyin((void *)arg, &errstate,
			    sizeof (struct bofi_errstate), mode) != 0)
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyin((void *)arg, &errstate,
		    sizeof (struct bofi_errstate), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		if ((retval = bofi_errdef_check_w(&errstate, &klg)) == EINVAL)
			return (EINVAL);
		/*
		 * copy out real errstate structure
		 */
		uls = errstate.log.logsize;
		uls = errstate.log.logsize;
		if (errstate.log.entries > uls && uls)
			/* insufficient user memory */
			errstate.log.entries = uls;
		/* always pass back a time */
		if (errstate.log.stop_time == 0ul)
			(void) drv_getparm(TIME, &(errstate.log.stop_time));

#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_errstate32	errstate_32;

			errstate_32.fail_time = errstate.fail_time;
			errstate_32.msg_time = errstate.msg_time;
			errstate_32.access_count = errstate.access_count;
			errstate_32.fail_count = errstate.fail_count;
			errstate_32.acc_chk = errstate.acc_chk;
			errstate_32.errmsg_count = errstate.errmsg_count;
			(void) strncpy(errstate_32.buffer, errstate.buffer,
			    ERRMSGSIZE);
			errstate_32.severity = errstate.severity;
			errstate_32.log.logsize = errstate.log.logsize;
			errstate_32.log.entries = errstate.log.entries;
			errstate_32.log.flags = errstate.log.flags;
			errstate_32.log.wrapcnt = errstate.log.wrapcnt;
			errstate_32.log.start_time = errstate.log.start_time;
			errstate_32.log.stop_time = errstate.log.stop_time;
			errstate_32.log.logbase =
			    (caddr32_t)(uintptr_t)errstate.log.logbase;
			errstate_32.errdef_handle = errstate.errdef_handle;
			if (ddi_copyout(&errstate_32, (void *)arg,
			    sizeof (struct bofi_errstate32), mode) != 0)
				return (EFAULT);
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyout(&errstate, (void *)arg,
			    sizeof (struct bofi_errstate), mode) != 0)
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyout(&errstate, (void *)arg,
		    sizeof (struct bofi_errstate), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */

		if (uls && errstate.log.entries &&
		    ddi_copyout(klg, errstate.log.logbase,
		    errstate.log.entries * sizeof (struct acc_log_elem),
		    mode) != 0) {
			return (EFAULT);
		}
		return (retval);
	}
	case BOFI_GET_HANDLES:
		/*
		 * display existing handles
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_get_handles32	get_handles_32;

			if (ddi_copyin((void *)arg, &get_handles_32,
			    sizeof (get_handles_32), mode) != 0) {
				return (EFAULT);
			}
			get_handles.namesize = get_handles_32.namesize;
			(void) strncpy(get_handles.name, get_handles_32.name,
			    NAMESIZE);
			get_handles.instance = get_handles_32.instance;
			get_handles.count = get_handles_32.count;
			get_handles.buffer =
			    (caddr_t)(uintptr_t)get_handles_32.buffer;
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyin((void *)arg, &get_handles,
			    sizeof (get_handles), mode) != 0)
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyin((void *)arg, &get_handles,
		    sizeof (get_handles), mode) != 0)
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		/*
		 * read in name
		 */
		if (get_handles.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(get_handles.namesize+1, KM_SLEEP);
		(void) strncpy(namep, get_handles.name, get_handles.namesize);
		req_count = get_handles.count;
		bufptr = buffer = kmem_zalloc(req_count, KM_SLEEP);
		endbuf = bufptr + req_count;
		/*
		 * display existing handles
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		for (i = 0; i < HDL_HASH_TBL_SIZE; i++) {
			hhashp = &hhash_table[i];
			for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext) {
				if (!driver_under_test(hp->dip))
					continue;
				if (ddi_name_to_major(ddi_get_name(hp->dip)) !=
				    ddi_name_to_major(namep))
					continue;
				if (hp->instance != get_handles.instance)
					continue;
				/*
				 * print information per handle - note that
				 * DMA* means an unbound DMA handle
				 */
				(void) snprintf(bufptr, (size_t)(endbuf-bufptr),
				    "  %s %d %s ", hp->name, hp->instance,
				    (hp->type == BOFI_INT_HDL) ? "INTR" :
				    (hp->type == BOFI_ACC_HDL) ? "PIO" :
				    (hp->type == BOFI_DMA_HDL) ? "DMA" :
				    (hp->hparrayp != NULL) ? "DVMA" : "DMA*");
				bufptr += strlen(bufptr);
				if (hp->type == BOFI_ACC_HDL) {
					if (hp->len == INT_MAX - hp->offset)
						(void) snprintf(bufptr,
						    (size_t)(endbuf-bufptr),
						    "reg set %d off 0x%llx\n",
						    hp->rnumber, hp->offset);
					else
						(void) snprintf(bufptr,
						    (size_t)(endbuf-bufptr),
						    "reg set %d off 0x%llx"
						    " len 0x%llx\n",
						    hp->rnumber, hp->offset,
						    hp->len);
				} else if (hp->type == BOFI_DMA_HDL)
					(void) snprintf(bufptr,
					    (size_t)(endbuf-bufptr),
					    "handle no %d len 0x%llx"
					    " addr 0x%p\n", hp->rnumber,
					    hp->len, (void *)hp->addr);
				else if (hp->type == BOFI_NULL &&
				    hp->hparrayp == NULL)
					(void) snprintf(bufptr,
					    (size_t)(endbuf-bufptr),
					    "handle no %d\n", hp->rnumber);
				else
					(void) snprintf(bufptr,
					    (size_t)(endbuf-bufptr), "\n");
				bufptr += strlen(bufptr);
			}
		}
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		err = ddi_copyout(buffer, get_handles.buffer, req_count, mode);
		kmem_free(namep, get_handles.namesize+1);
		kmem_free(buffer, req_count);
		if (err != 0)
			return (EFAULT);
		else
			return (0);
	case BOFI_GET_HANDLE_INFO:
		/*
		 * display existing handles
		 */
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_get_hdl_info32	hdl_info_32;

			if (ddi_copyin((void *)arg, &hdl_info_32,
			    sizeof (hdl_info_32), mode)) {
				return (EFAULT);
			}
			hdl_info.namesize = hdl_info_32.namesize;
			(void) strncpy(hdl_info.name, hdl_info_32.name,
			    NAMESIZE);
			hdl_info.count = hdl_info_32.count;
			hdl_info.hdli = (caddr_t)(uintptr_t)hdl_info_32.hdli;
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyin((void *)arg, &hdl_info,
			    sizeof (hdl_info), mode))
				return (EFAULT);
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyin((void *)arg, &hdl_info,
		    sizeof (hdl_info), mode))
			return (EFAULT);
#endif /* _MULTI_DATAMODEL */
		if (hdl_info.namesize > NAMESIZE)
			return (EINVAL);
		namep = kmem_zalloc(hdl_info.namesize + 1, KM_SLEEP);
		(void) strncpy(namep, hdl_info.name, hdl_info.namesize);
		req_count = hdl_info.count;
		count = hdl_info.count = 0; /* the actual no of handles */
		if (req_count > 0) {
			hib = hdlip =
			    kmem_zalloc(req_count * sizeof (struct handle_info),
			    KM_SLEEP);
		} else {
			hib = hdlip = 0;
			req_count = hdl_info.count = 0;
		}

		/*
		 * display existing handles
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		for (i = 0; i < HDL_HASH_TBL_SIZE; i++) {
			hhashp = &hhash_table[i];
			for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext) {
				if (!driver_under_test(hp->dip) ||
				    ddi_name_to_major(ddi_get_name(hp->dip)) !=
				    ddi_name_to_major(namep) ||
				    ++(hdl_info.count) > req_count ||
				    count == req_count)
					continue;

				hdlip->instance = hp->instance;
				hdlip->rnumber = hp->rnumber;
				switch (hp->type) {
				case BOFI_ACC_HDL:
					hdlip->access_type = BOFI_PIO_RW;
					hdlip->offset = hp->offset;
					hdlip->len = hp->len;
					break;
				case BOFI_DMA_HDL:
					hdlip->access_type = 0;
					if (hp->flags & DDI_DMA_WRITE)
						hdlip->access_type |=
						    BOFI_DMA_W;
					if (hp->flags & DDI_DMA_READ)
						hdlip->access_type |=
						    BOFI_DMA_R;
					hdlip->len = hp->len;
					hdlip->addr_cookie =
					    (uint64_t)(uintptr_t)hp->addr;
					break;
				case BOFI_INT_HDL:
					hdlip->access_type = BOFI_INTR;
					break;
				default:
					hdlip->access_type = 0;
					break;
				}
				hdlip++;
				count++;
			}
		}
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		err = 0;
#ifdef _MULTI_DATAMODEL
		switch (ddi_model_convert_from(mode & FMODELS)) {
		case DDI_MODEL_ILP32:
		{
			/*
			 * For use when a 32 bit app makes a call into a
			 * 64 bit ioctl
			 */
			struct bofi_get_hdl_info32	hdl_info_32;

			hdl_info_32.namesize = hdl_info.namesize;
			(void) strncpy(hdl_info_32.name, hdl_info.name,
			    NAMESIZE);
			hdl_info_32.count = hdl_info.count;
			hdl_info_32.hdli = (caddr32_t)(uintptr_t)hdl_info.hdli;
			if (ddi_copyout(&hdl_info_32, (void *)arg,
			    sizeof (hdl_info_32), mode) != 0) {
				kmem_free(namep, hdl_info.namesize+1);
				if (req_count > 0)
					kmem_free(hib,
					    req_count * sizeof (*hib));
				return (EFAULT);
			}
			break;
		}
		case DDI_MODEL_NONE:
			if (ddi_copyout(&hdl_info, (void *)arg,
			    sizeof (hdl_info), mode) != 0) {
				kmem_free(namep, hdl_info.namesize+1);
				if (req_count > 0)
					kmem_free(hib,
					    req_count * sizeof (*hib));
				return (EFAULT);
			}
			break;
		}
#else /* ! _MULTI_DATAMODEL */
		if (ddi_copyout(&hdl_info, (void *)arg,
		    sizeof (hdl_info), mode) != 0) {
			kmem_free(namep, hdl_info.namesize+1);
			if (req_count > 0)
				kmem_free(hib, req_count * sizeof (*hib));
			return (EFAULT);
		}
#endif /* ! _MULTI_DATAMODEL */
		if (count > 0) {
			if (ddi_copyout(hib, hdl_info.hdli,
			    count * sizeof (*hib), mode) != 0) {
				kmem_free(namep, hdl_info.namesize+1);
				if (req_count > 0)
					kmem_free(hib,
					    req_count * sizeof (*hib));
				return (EFAULT);
			}
		}
		kmem_free(namep, hdl_info.namesize+1);
		if (req_count > 0)
			kmem_free(hib, req_count * sizeof (*hib));
		return (err);
	default:
		return (ENOTTY);
	}
}


/*
 * add a new error definition
 */
static int
bofi_errdef_alloc(struct bofi_errdef *errdefp, char *namep,
    struct bofi_errent *softc)
{
	struct bofi_errent *ep;
	struct bofi_shadow *hp;
	struct bofi_link   *lp;

	/*
	 * allocate errdef structure and put on in-use list
	 */
	ep = kmem_zalloc(sizeof (struct bofi_errent), KM_SLEEP);
	ep->errdef = *errdefp;
	ep->name = namep;
	ep->errdef.errdef_handle = (uint64_t)(uintptr_t)ep;
	ep->errstate.severity = DDI_SERVICE_RESTORED;
	ep->errstate.errdef_handle = (uint64_t)(uintptr_t)ep;
	cv_init(&ep->cv, NULL, CV_DRIVER, NULL);
	/*
	 * allocate space for logging
	 */
	ep->errdef.log.entries = 0;
	ep->errdef.log.wrapcnt = 0;
	if (ep->errdef.access_type & BOFI_LOG)
		ep->logbase = kmem_alloc(sizeof (struct acc_log_elem) *
		    ep->errdef.log.logsize, KM_SLEEP);
	else
		ep->logbase = NULL;
	/*
	 * put on in-use list
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	ep->next = errent_listp;
	errent_listp = ep;
	/*
	 * and add it to the per-clone list
	 */
	ep->cnext = softc->cnext;
	softc->cnext->cprev = ep;
	ep->cprev = softc;
	softc->cnext = ep;

	/*
	 * look for corresponding shadow handle structures and if we find any
	 * tag this errdef structure on to their link lists.
	 */
	for (hp = shadow_list.next; hp != &shadow_list; hp = hp->next) {
		if (ddi_name_to_major(hp->name) == ddi_name_to_major(namep) &&
		    hp->instance == errdefp->instance &&
		    (((errdefp->access_type & BOFI_DMA_RW) &&
		    (ep->errdef.rnumber == -1 ||
		    hp->rnumber == ep->errdef.rnumber) &&
		    hp->type == BOFI_DMA_HDL &&
		    (((uintptr_t)(hp->addr + ep->errdef.offset +
		    ep->errdef.len) & ~LLSZMASK) >
		    ((uintptr_t)((hp->addr + ep->errdef.offset) +
		    LLSZMASK) & ~LLSZMASK))) ||
		    ((errdefp->access_type & BOFI_INTR) &&
		    hp->type == BOFI_INT_HDL) ||
		    ((errdefp->access_type & BOFI_PIO_RW) &&
		    hp->type == BOFI_ACC_HDL &&
		    (errdefp->rnumber == -1 ||
		    hp->rnumber == errdefp->rnumber) &&
		    (errdefp->len == 0 ||
		    hp->offset < errdefp->offset + errdefp->len) &&
		    hp->offset + hp->len > errdefp->offset))) {
			lp = bofi_link_freelist;
			if (lp != NULL) {
				bofi_link_freelist = lp->link;
				lp->errentp = ep;
				lp->link = hp->link;
				hp->link = lp;
			}
		}
	}
	errdefp->errdef_handle = (uint64_t)(uintptr_t)ep;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	ep->softintr_id = NULL;
	return (ddi_add_softintr(our_dip, DDI_SOFTINT_MED, &ep->softintr_id,
	    NULL, NULL, bofi_signal, (caddr_t)&ep->errdef));
}


/*
 * delete existing errdef
 */
static int
bofi_errdef_free(struct bofi_errent *ep)
{
	struct bofi_errent *hep, *prev_hep;
	struct bofi_link *lp, *prev_lp, *next_lp;
	struct bofi_shadow *hp;

	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	/*
	 * don't just assume its a valid ep - check that its on the
	 * in-use list
	 */
	prev_hep = NULL;
	for (hep = errent_listp; hep != NULL; ) {
		if (hep == ep)
			break;
		prev_hep = hep;
		hep = hep->next;
	}
	if (hep == NULL) {
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (EINVAL);
	}
	/*
	 * found it - delete from in-use list
	 */

	if (prev_hep)
		prev_hep->next = hep->next;
	else
		errent_listp = hep->next;
	/*
	 * and take it off the per-clone list
	 */
	hep->cnext->cprev = hep->cprev;
	hep->cprev->cnext = hep->cnext;
	/*
	 * see if we are on any shadow handle link lists - and if we
	 * are then take us off
	 */
	for (hp = shadow_list.next; hp != &shadow_list; hp = hp->next) {
		prev_lp = NULL;
		for (lp = hp->link; lp != NULL; ) {
			if (lp->errentp == ep) {
				if (prev_lp)
					prev_lp->link = lp->link;
				else
					hp->link = lp->link;
				next_lp = lp->link;
				lp->link = bofi_link_freelist;
				bofi_link_freelist = lp;
				lp = next_lp;
			} else {
				prev_lp = lp;
				lp = lp->link;
			}
		}
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);

	cv_destroy(&ep->cv);
	kmem_free(ep->name, ep->errdef.namesize+1);
	if ((ep->errdef.access_type & BOFI_LOG) &&
	    ep->errdef.log.logsize && ep->logbase) /* double check */
		kmem_free(ep->logbase,
		    sizeof (struct acc_log_elem) * ep->errdef.log.logsize);

	if (ep->softintr_id)
		ddi_remove_softintr(ep->softintr_id);
	kmem_free(ep, sizeof (struct bofi_errent));
	return (0);
}


/*
 * start all errdefs corresponding to this name and instance
 */
static void
bofi_start(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			ep->state |= BOFI_DEV_ACTIVE;
			(void) drv_getparm(TIME, &(ep->errdef.log.start_time));
			ep->errdef.log.stop_time = 0ul;
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * stop all errdefs corresponding to this name and instance
 */
static void
bofi_stop(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			ep->state &= ~BOFI_DEV_ACTIVE;
			if (ep->errdef.log.stop_time == 0ul)
				(void) drv_getparm(TIME,
				    &(ep->errdef.log.stop_time));
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * wake up any thread waiting on this errdefs
 */
static uint_t
bofi_signal(caddr_t arg)
{
	struct bofi_errdef *edp = (struct bofi_errdef *)arg;
	struct bofi_errent *hep;
	struct bofi_errent *ep =
	    (struct bofi_errent *)(uintptr_t)edp->errdef_handle;

	mutex_enter(&bofi_low_mutex);
	for (hep = errent_listp; hep != NULL; ) {
		if (hep == ep)
			break;
		hep = hep->next;
	}
	if (hep == NULL) {
		mutex_exit(&bofi_low_mutex);
		return (DDI_INTR_UNCLAIMED);
	}
	if ((ep->errdef.access_type & BOFI_LOG) &&
	    (edp->log.flags & BOFI_LOG_FULL)) {
		edp->log.stop_time = bofi_gettime();
		ep->state |= BOFI_NEW_MESSAGE;
		if (ep->state & BOFI_MESSAGE_WAIT)
			cv_broadcast(&ep->cv);
		ep->state &= ~BOFI_MESSAGE_WAIT;
	}
	if (ep->errstate.msg_time != 0) {
		ep->state |= BOFI_NEW_MESSAGE;
		if (ep->state & BOFI_MESSAGE_WAIT)
			cv_broadcast(&ep->cv);
		ep->state &= ~BOFI_MESSAGE_WAIT;
	}
	mutex_exit(&bofi_low_mutex);
	return (DDI_INTR_CLAIMED);
}


/*
 * wake up all errdefs corresponding to this name and instance
 */
static void
bofi_broadcast(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			/*
			 * wake up sleepers
			 */
			ep->state |= BOFI_NEW_MESSAGE;
			if (ep->state & BOFI_MESSAGE_WAIT)
				cv_broadcast(&ep->cv);
			ep->state &= ~BOFI_MESSAGE_WAIT;
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * clear "acc_chk" for all errdefs corresponding to this name and instance
 * and wake them up.
 */
static void
bofi_clear_acc_chk(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			mutex_enter(&bofi_mutex);
			if (ep->errdef.access_count == 0 &&
			    ep->errdef.fail_count == 0)
				ep->errdef.acc_chk = 0;
			mutex_exit(&bofi_mutex);
			/*
			 * wake up sleepers
			 */
			ep->state |= BOFI_NEW_MESSAGE;
			if (ep->state & BOFI_MESSAGE_WAIT)
				cv_broadcast(&ep->cv);
			ep->state &= ~BOFI_MESSAGE_WAIT;
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * set "fail_count" to 0 for all errdefs corresponding to this name and instance
 * whose "access_count" has expired, set "acc_chk" to 0 and wake them up.
 */
static void
bofi_clear_errors(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			mutex_enter(&bofi_mutex);
			if (ep->errdef.access_count == 0) {
				ep->errdef.acc_chk = 0;
				ep->errdef.fail_count = 0;
				mutex_exit(&bofi_mutex);
				if (ep->errdef.log.stop_time == 0ul)
					(void) drv_getparm(TIME,
					    &(ep->errdef.log.stop_time));
			} else
				mutex_exit(&bofi_mutex);
			/*
			 * wake up sleepers
			 */
			ep->state |= BOFI_NEW_MESSAGE;
			if (ep->state & BOFI_MESSAGE_WAIT)
				cv_broadcast(&ep->cv);
			ep->state &= ~BOFI_MESSAGE_WAIT;
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * set "access_count" and "fail_count" to 0 for all errdefs corresponding to
 * this name and instance, set "acc_chk" to 0, and wake them up.
 */
static void
bofi_clear_errdefs(struct bofi_errctl *errctlp, char *namep)
{
	struct bofi_errent *ep;

	/*
	 * look for any errdefs with matching name and instance
	 */
	mutex_enter(&bofi_low_mutex);
	for (ep = errent_listp; ep != NULL; ep = ep->next)
		if (strncmp(namep, ep->name, NAMESIZE) == 0 &&
		    errctlp->instance == ep->errdef.instance) {
			mutex_enter(&bofi_mutex);
			ep->errdef.acc_chk = 0;
			ep->errdef.access_count = 0;
			ep->errdef.fail_count = 0;
			mutex_exit(&bofi_mutex);
			if (ep->errdef.log.stop_time == 0ul)
				(void) drv_getparm(TIME,
				    &(ep->errdef.log.stop_time));
			/*
			 * wake up sleepers
			 */
			ep->state |= BOFI_NEW_MESSAGE;
			if (ep->state & BOFI_MESSAGE_WAIT)
				cv_broadcast(&ep->cv);
			ep->state &= ~BOFI_MESSAGE_WAIT;
		}
	mutex_exit(&bofi_low_mutex);
}


/*
 * get state for this errdef
 */
static int
bofi_errdef_check(struct bofi_errstate *errstatep, struct acc_log_elem **logpp)
{
	struct bofi_errent *hep;
	struct bofi_errent *ep;

	ep = (struct bofi_errent *)(uintptr_t)errstatep->errdef_handle;
	mutex_enter(&bofi_low_mutex);
	/*
	 * don't just assume its a valid ep - check that its on the
	 * in-use list
	 */
	for (hep = errent_listp; hep != NULL; hep = hep->next)
		if (hep == ep)
			break;
	if (hep == NULL) {
		mutex_exit(&bofi_low_mutex);
		return (EINVAL);
	}
	mutex_enter(&bofi_mutex);
	ep->errstate.access_count = ep->errdef.access_count;
	ep->errstate.fail_count = ep->errdef.fail_count;
	ep->errstate.acc_chk = ep->errdef.acc_chk;
	ep->errstate.log = ep->errdef.log;
	*logpp = ep->logbase;
	*errstatep = ep->errstate;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (0);
}


/*
 * Wait for a ddi_report_fault message to come back for this errdef
 * Then return state for this errdef.
 * fault report is intercepted by bofi_post_event, which triggers
 * bofi_signal via a softint, which will wake up this routine if
 * we are waiting
 */
static int
bofi_errdef_check_w(struct bofi_errstate *errstatep,
    struct acc_log_elem **logpp)
{
	struct bofi_errent *hep;
	struct bofi_errent *ep;
	int rval = 0;

	ep = (struct bofi_errent *)(uintptr_t)errstatep->errdef_handle;
	mutex_enter(&bofi_low_mutex);
retry:
	/*
	 * don't just assume its a valid ep - check that its on the
	 * in-use list
	 */
	for (hep = errent_listp; hep != NULL; hep = hep->next)
		if (hep == ep)
			break;
	if (hep == NULL) {
		mutex_exit(&bofi_low_mutex);
		return (EINVAL);
	}
	/*
	 * wait for ddi_report_fault for the devinfo corresponding
	 * to this errdef
	 */
	if (rval == 0 && !(ep->state & BOFI_NEW_MESSAGE)) {
		ep->state |= BOFI_MESSAGE_WAIT;
		if (cv_wait_sig(&ep->cv, &bofi_low_mutex) == 0) {
			if (!(ep->state & BOFI_NEW_MESSAGE))
				rval = EINTR;
		}
		goto retry;
	}
	ep->state &= ~BOFI_NEW_MESSAGE;
	/*
	 * we either didn't need to sleep, we've been woken up or we've been
	 * signaled - either way return state now
	 */
	mutex_enter(&bofi_mutex);
	ep->errstate.access_count = ep->errdef.access_count;
	ep->errstate.fail_count = ep->errdef.fail_count;
	ep->errstate.acc_chk = ep->errdef.acc_chk;
	ep->errstate.log = ep->errdef.log;
	*logpp = ep->logbase;
	*errstatep = ep->errstate;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (rval);
}


/*
 * support routine - check if requested driver is defined as under test in the
 * conf file.
 */
static int
driver_under_test(dev_info_t *rdip)
{
	int i;
	char	*rname;
	major_t rmaj;

	rname = ddi_get_name(rdip);
	rmaj = ddi_name_to_major(rname);

	/*
	 * Enforce the user to specifically request the following drivers.
	 */
	for (i = 0; i < driver_list_size; i += (1 + strlen(&driver_list[i]))) {
		if (driver_list_neg == 0) {
			if (rmaj == ddi_name_to_major(&driver_list[i]))
				return (1);
		} else {
			if (rmaj == ddi_name_to_major(&driver_list[i+1]))
				return (0);
		}
	}
	if (driver_list_neg == 0)
		return (0);
	else
		return (1);

}


static void
log_acc_event(struct bofi_errent *ep, uint_t at, offset_t offset, off_t len,
    size_t repcount, uint64_t *valuep)
{
	struct bofi_errdef *edp = &(ep->errdef);
	struct acc_log *log = &edp->log;

	ASSERT(log != NULL);
	ASSERT(MUTEX_HELD(&bofi_mutex));

	if (log->flags & BOFI_LOG_REPIO)
		repcount = 1;
	else if (repcount == 0 && edp->access_count > 0 &&
	    (log->flags & BOFI_LOG_FULL) == 0)
		edp->access_count += 1;

	if (repcount && log->entries < log->logsize) {
		struct acc_log_elem *elem = ep->logbase + log->entries;

		if (log->flags & BOFI_LOG_TIMESTAMP)
			elem->access_time = bofi_gettime();
		elem->access_type = at;
		elem->offset = offset;
		elem->value = valuep ? *valuep : 0ll;
		elem->size = len;
		elem->repcount = repcount;
		++log->entries;
		if (log->entries == log->logsize) {
			log->flags |= BOFI_LOG_FULL;
			ddi_trigger_softintr(((struct bofi_errent *)
			    (uintptr_t)edp->errdef_handle)->softintr_id);
		}
	}
	if ((log->flags & BOFI_LOG_WRAP) && edp->access_count <= 1) {
		log->wrapcnt++;
		edp->access_count = log->logsize;
		log->entries = 0;	/* wrap back to the start */
	}
}


/*
 * got a condition match on dma read/write - check counts and corrupt
 * data if necessary
 *
 * bofi_mutex always held when this is called.
 */
static void
do_dma_corrupt(struct bofi_shadow *hp, struct bofi_errent *ep,
    uint_t synctype, off_t off, off_t length)
{
	uint64_t operand;
	int i;
	off_t len;
	caddr_t logaddr;
	uint64_t *addr;
	uint64_t *endaddr;
	ddi_dma_impl_t *hdlp;
	ndi_err_t *errp;

	ASSERT(MUTEX_HELD(&bofi_mutex));
	if ((ep->errdef.access_count ||
	    ep->errdef.fail_count) &&
	    (ep->errdef.access_type & BOFI_LOG)) {
		uint_t atype;

		if (synctype == DDI_DMA_SYNC_FORDEV)
			atype = BOFI_DMA_W;
		else if (synctype == DDI_DMA_SYNC_FORCPU ||
		    synctype == DDI_DMA_SYNC_FORKERNEL)
			atype = BOFI_DMA_R;
		else
			atype = 0;
		if ((off <= ep->errdef.offset &&
		    off + length > ep->errdef.offset) ||
		    (off > ep->errdef.offset &&
		    off < ep->errdef.offset + ep->errdef.len)) {
			logaddr = (caddr_t)((uintptr_t)(hp->addr +
			    off + LLSZMASK) & ~LLSZMASK);

			log_acc_event(ep, atype, logaddr - hp->addr,
			    length, 1, 0);
		}
	}
	if (ep->errdef.access_count > 1) {
		ep->errdef.access_count--;
	} else if (ep->errdef.fail_count > 0) {
		ep->errdef.fail_count--;
		ep->errdef.access_count = 0;
		/*
		 * OK do the corruption
		 */
		if (ep->errstate.fail_time == 0)
			ep->errstate.fail_time = bofi_gettime();
		/*
		 * work out how much to corrupt
		 *
		 * Make sure endaddr isn't greater than hp->addr + hp->len.
		 * If endaddr becomes less than addr len becomes negative
		 * and the following loop isn't entered.
		 */
		addr = (uint64_t *)((uintptr_t)((hp->addr +
		    ep->errdef.offset) + LLSZMASK) & ~LLSZMASK);
		endaddr = (uint64_t *)((uintptr_t)(hp->addr + min(hp->len,
		    ep->errdef.offset + ep->errdef.len)) & ~LLSZMASK);
		len = endaddr - addr;
		operand = ep->errdef.operand;
		hdlp = (ddi_dma_impl_t *)(hp->hdl.dma_handle);
		errp = &hdlp->dmai_error;
		if (ep->errdef.acc_chk & 2) {
			uint64_t ena;
			char buf[FM_MAX_CLASS];

			errp->err_status = DDI_FM_NONFATAL;
			(void) snprintf(buf, FM_MAX_CLASS, FM_SIMULATED_DMA);
			ena = fm_ena_generate(0, FM_ENA_FMT1);
			ddi_fm_ereport_post(hp->dip, buf, ena,
			    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
			    FM_EREPORT_VERS0, NULL);
		}
		switch (ep->errdef.optype) {
		case BOFI_EQUAL :
			for (i = 0; i < len; i++)
				*(addr + i) = operand;
			break;
		case BOFI_AND :
			for (i = 0; i < len; i++)
				*(addr + i) &= operand;
			break;
		case BOFI_OR :
			for (i = 0; i < len; i++)
				*(addr + i) |= operand;
			break;
		case BOFI_XOR :
			for (i = 0; i < len; i++)
				*(addr + i) ^= operand;
			break;
		default:
			/* do nothing */
			break;
		}
	}
}


static uint64_t do_bofi_rd8(struct bofi_shadow *, caddr_t);
static uint64_t do_bofi_rd16(struct bofi_shadow *, caddr_t);
static uint64_t do_bofi_rd32(struct bofi_shadow *, caddr_t);
static uint64_t do_bofi_rd64(struct bofi_shadow *, caddr_t);


/*
 * check all errdefs linked to this shadow handle. If we've got a condition
 * match check counts and corrupt data if necessary
 *
 * bofi_mutex always held when this is called.
 *
 * because of possibility of BOFI_NO_TRANSFER, we couldn't get data
 * from io-space before calling this, so we pass in the func to do the
 * transfer as a parameter.
 */
static uint64_t
do_pior_corrupt(struct bofi_shadow *hp, caddr_t addr,
    uint64_t (*func)(), size_t repcount, size_t accsize)
{
	struct bofi_errent *ep;
	struct bofi_link   *lp;
	uint64_t operand;
	uintptr_t minlen;
	intptr_t base;
	int done_get = 0;
	uint64_t get_val, gv;
	ddi_acc_impl_t *hdlp;
	ndi_err_t *errp;

	ASSERT(MUTEX_HELD(&bofi_mutex));
	/*
	 * check through all errdefs associated with this shadow handle
	 */
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		ep = lp->errentp;
		if (ep->errdef.len == 0)
			minlen = hp->len;
		else
			minlen = min(hp->len, ep->errdef.len);
		base = addr - hp->addr - ep->errdef.offset + hp->offset;
		if ((ep->errdef.access_type & BOFI_PIO_R) &&
		    (ep->state & BOFI_DEV_ACTIVE) &&
		    base >= 0 && base < minlen) {
			/*
			 * condition match for pio read
			 */
			if (ep->errdef.access_count > 1) {
				ep->errdef.access_count--;
				if (done_get == 0) {
					done_get = 1;
					gv = get_val = func(hp, addr);
				}
				if (ep->errdef.access_type & BOFI_LOG) {
					log_acc_event(ep, BOFI_PIO_R,
					    addr - hp->addr,
					    accsize, repcount, &gv);
				}
			} else if (ep->errdef.fail_count > 0) {
				ep->errdef.fail_count--;
				ep->errdef.access_count = 0;
				/*
				 * OK do corruption
				 */
				if (ep->errstate.fail_time == 0)
					ep->errstate.fail_time = bofi_gettime();
				operand = ep->errdef.operand;
				if (done_get == 0) {
					if (ep->errdef.optype ==
					    BOFI_NO_TRANSFER)
						/*
						 * no transfer - bomb out
						 */
						return (operand);
					done_get = 1;
					gv = get_val = func(hp, addr);

				}
				if (ep->errdef.access_type & BOFI_LOG) {
					log_acc_event(ep, BOFI_PIO_R,
					    addr - hp->addr,
					    accsize, repcount, &gv);
				}
				hdlp = (ddi_acc_impl_t *)(hp->hdl.acc_handle);
				errp = hdlp->ahi_err;
				if (ep->errdef.acc_chk & 1) {
					uint64_t ena;
					char buf[FM_MAX_CLASS];

					errp->err_status = DDI_FM_NONFATAL;
					(void) snprintf(buf, FM_MAX_CLASS,
					    FM_SIMULATED_PIO);
					ena = fm_ena_generate(0, FM_ENA_FMT1);
					ddi_fm_ereport_post(hp->dip, buf, ena,
					    DDI_NOSLEEP, FM_VERSION,
					    DATA_TYPE_UINT8, FM_EREPORT_VERS0,
					    NULL);
				}
				switch (ep->errdef.optype) {
				case BOFI_EQUAL :
					get_val = operand;
					break;
				case BOFI_AND :
					get_val &= operand;
					break;
				case BOFI_OR :
					get_val |= operand;
					break;
				case BOFI_XOR :
					get_val ^= operand;
					break;
				default:
					/* do nothing */
					break;
				}
			}
		}
	}
	if (done_get == 0)
		return (func(hp, addr));
	else
		return (get_val);
}


/*
 * check all errdefs linked to this shadow handle. If we've got a condition
 * match check counts and corrupt data if necessary
 *
 * bofi_mutex always held when this is called.
 *
 * because of possibility of BOFI_NO_TRANSFER, we return 0 if no data
 * is to be written out to io-space, 1 otherwise
 */
static int
do_piow_corrupt(struct bofi_shadow *hp, caddr_t addr, uint64_t *valuep,
    size_t size, size_t repcount)
{
	struct bofi_errent *ep;
	struct bofi_link   *lp;
	uintptr_t minlen;
	intptr_t base;
	uint64_t v = *valuep;
	ddi_acc_impl_t *hdlp;
	ndi_err_t *errp;

	ASSERT(MUTEX_HELD(&bofi_mutex));
	/*
	 * check through all errdefs associated with this shadow handle
	 */
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		ep = lp->errentp;
		if (ep->errdef.len == 0)
			minlen = hp->len;
		else
			minlen = min(hp->len, ep->errdef.len);
		base = (caddr_t)addr - hp->addr - ep->errdef.offset +hp->offset;
		if ((ep->errdef.access_type & BOFI_PIO_W) &&
		    (ep->state & BOFI_DEV_ACTIVE) &&
		    base >= 0 && base < minlen) {
			/*
			 * condition match for pio write
			 */

			if (ep->errdef.access_count > 1) {
				ep->errdef.access_count--;
				if (ep->errdef.access_type & BOFI_LOG)
					log_acc_event(ep, BOFI_PIO_W,
					    addr - hp->addr, size,
					    repcount, &v);
			} else if (ep->errdef.fail_count > 0) {
				ep->errdef.fail_count--;
				ep->errdef.access_count = 0;
				if (ep->errdef.access_type & BOFI_LOG)
					log_acc_event(ep, BOFI_PIO_W,
					    addr - hp->addr, size,
					    repcount, &v);
				/*
				 * OK do corruption
				 */
				if (ep->errstate.fail_time == 0)
					ep->errstate.fail_time = bofi_gettime();
				hdlp = (ddi_acc_impl_t *)(hp->hdl.acc_handle);
				errp = hdlp->ahi_err;
				if (ep->errdef.acc_chk & 1) {
					uint64_t ena;
					char buf[FM_MAX_CLASS];

					errp->err_status = DDI_FM_NONFATAL;
					(void) snprintf(buf, FM_MAX_CLASS,
					    FM_SIMULATED_PIO);
					ena = fm_ena_generate(0, FM_ENA_FMT1);
					ddi_fm_ereport_post(hp->dip, buf, ena,
					    DDI_NOSLEEP, FM_VERSION,
					    DATA_TYPE_UINT8, FM_EREPORT_VERS0,
					    NULL);
				}
				switch (ep->errdef.optype) {
				case BOFI_EQUAL :
					*valuep = ep->errdef.operand;
					break;
				case BOFI_AND :
					*valuep &= ep->errdef.operand;
					break;
				case BOFI_OR :
					*valuep |= ep->errdef.operand;
					break;
				case BOFI_XOR :
					*valuep ^= ep->errdef.operand;
					break;
				case BOFI_NO_TRANSFER :
					/*
					 * no transfer - bomb out
					 */
					return (0);
				default:
					/* do nothing */
					break;
				}
			}
		}
	}
	return (1);
}


static uint64_t
do_bofi_rd8(struct bofi_shadow *hp, caddr_t addr)
{
	return (hp->save.acc.ahi_get8(&hp->save.acc, (uint8_t *)addr));
}

#define	BOFI_READ_CHECKS(type) \
	if (bofi_ddi_check) \
		addr = (type *)((uintptr_t)addr - 64 + hp->addr); \
	if (bofi_range_check && ((caddr_t)addr < hp->addr || \
	    (caddr_t)addr - hp->addr >= hp->len)) { \
		cmn_err((bofi_range_check == 2) ? CE_PANIC : CE_WARN, \
		    "ddi_get() out of range addr %p not in %p/%llx", \
		    (void *)addr, (void *)hp->addr, hp->len); \
		return (0); \
	}

/*
 * our getb() routine - use tryenter
 */
static uint8_t
bofi_rd8(ddi_acc_impl_t *handle, uint8_t *addr)
{
	struct bofi_shadow *hp;
	uint8_t retval;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_READ_CHECKS(uint8_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex))
		return (hp->save.acc.ahi_get8(&hp->save.acc, addr));
	retval = (uint8_t)do_pior_corrupt(hp, (caddr_t)addr, do_bofi_rd8, 1,
	    1);
	mutex_exit(&bofi_mutex);
	return (retval);
}


static uint64_t
do_bofi_rd16(struct bofi_shadow *hp, caddr_t addr)
{
	return (hp->save.acc.ahi_get16(&hp->save.acc, (uint16_t *)addr));
}


/*
 * our getw() routine - use tryenter
 */
static uint16_t
bofi_rd16(ddi_acc_impl_t *handle, uint16_t *addr)
{
	struct bofi_shadow *hp;
	uint16_t retval;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_READ_CHECKS(uint16_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex))
		return (hp->save.acc.ahi_get16(&hp->save.acc, addr));
	retval = (uint16_t)do_pior_corrupt(hp, (caddr_t)addr, do_bofi_rd16, 1,
	    2);
	mutex_exit(&bofi_mutex);
	return (retval);
}


static uint64_t
do_bofi_rd32(struct bofi_shadow *hp, caddr_t addr)
{
	return (hp->save.acc.ahi_get32(&hp->save.acc, (uint32_t *)addr));
}


/*
 * our getl() routine - use tryenter
 */
static uint32_t
bofi_rd32(ddi_acc_impl_t *handle, uint32_t *addr)
{
	struct bofi_shadow *hp;
	uint32_t retval;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_READ_CHECKS(uint32_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex))
		return (hp->save.acc.ahi_get32(&hp->save.acc, addr));
	retval = (uint32_t)do_pior_corrupt(hp, (caddr_t)addr, do_bofi_rd32, 1,
	    4);
	mutex_exit(&bofi_mutex);
	return (retval);
}


static uint64_t
do_bofi_rd64(struct bofi_shadow *hp, caddr_t addr)
{
	return (hp->save.acc.ahi_get64(&hp->save.acc, (uint64_t *)addr));
}


/*
 * our getll() routine - use tryenter
 */
static uint64_t
bofi_rd64(ddi_acc_impl_t *handle, uint64_t *addr)
{
	struct bofi_shadow *hp;
	uint64_t retval;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_READ_CHECKS(uint64_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex))
		return (hp->save.acc.ahi_get64(&hp->save.acc, addr));
	retval = (uint64_t)do_pior_corrupt(hp, (caddr_t)addr, do_bofi_rd64, 1,
	    8);
	mutex_exit(&bofi_mutex);
	return (retval);
}

#define	BOFI_WRITE_TESTS(type) \
	if (bofi_ddi_check) \
		addr = (type *)((uintptr_t)addr - 64 + hp->addr); \
	if (bofi_range_check && ((caddr_t)addr < hp->addr || \
	    (caddr_t)addr - hp->addr >= hp->len)) { \
		cmn_err((bofi_range_check == 2) ? CE_PANIC : CE_WARN, \
		    "ddi_put() out of range addr %p not in %p/%llx\n", \
		    (void *)addr, (void *)hp->addr, hp->len); \
		return; \
	}

/*
 * our putb() routine - use tryenter
 */
static void
bofi_wr8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t value)
{
	struct bofi_shadow *hp;
	uint64_t llvalue = value;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_WRITE_TESTS(uint8_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_put8(&hp->save.acc, addr, (uint8_t)llvalue);
		return;
	}
	if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 1, 1))
		hp->save.acc.ahi_put8(&hp->save.acc, addr, (uint8_t)llvalue);
	mutex_exit(&bofi_mutex);
}


/*
 * our putw() routine - use tryenter
 */
static void
bofi_wr16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t value)
{
	struct bofi_shadow *hp;
	uint64_t llvalue = value;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_WRITE_TESTS(uint16_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_put16(&hp->save.acc, addr, (uint16_t)llvalue);
		return;
	}
	if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 2, 1))
		hp->save.acc.ahi_put16(&hp->save.acc, addr, (uint16_t)llvalue);
	mutex_exit(&bofi_mutex);
}


/*
 * our putl() routine - use tryenter
 */
static void
bofi_wr32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t value)
{
	struct bofi_shadow *hp;
	uint64_t llvalue = value;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_WRITE_TESTS(uint32_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_put32(&hp->save.acc, addr, (uint32_t)llvalue);
		return;
	}
	if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 4, 1))
		hp->save.acc.ahi_put32(&hp->save.acc, addr, (uint32_t)llvalue);
	mutex_exit(&bofi_mutex);
}


/*
 * our putll() routine - use tryenter
 */
static void
bofi_wr64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t value)
{
	struct bofi_shadow *hp;
	uint64_t llvalue = value;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_WRITE_TESTS(uint64_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_put64(&hp->save.acc, addr, (uint64_t)llvalue);
		return;
	}
	if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 8, 1))
		hp->save.acc.ahi_put64(&hp->save.acc, addr, (uint64_t)llvalue);
	mutex_exit(&bofi_mutex);
}

#define	BOFI_REP_READ_TESTS(type) \
	if (bofi_ddi_check) \
		dev_addr = (type *)((uintptr_t)dev_addr - 64 + hp->addr); \
	if (bofi_range_check && ((caddr_t)dev_addr < hp->addr || \
	    (caddr_t)(dev_addr + repcount) - hp->addr > hp->len)) { \
		cmn_err((bofi_range_check == 2) ? CE_PANIC : CE_WARN, \
		    "ddi_rep_get() out of range addr %p not in %p/%llx\n", \
		    (void *)dev_addr, (void *)hp->addr, hp->len); \
		if ((caddr_t)dev_addr < hp->addr || \
		    (caddr_t)dev_addr - hp->addr >= hp->len) \
			return; \
		repcount = (type *)(hp->addr + hp->len) - dev_addr; \
	}

/*
 * our rep_getb() routine - use tryenter
 */
static void
bofi_rep_rd8(ddi_acc_impl_t *handle, uint8_t *host_addr, uint8_t *dev_addr,
    size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint8_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_READ_TESTS(uint8_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_get8(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		*(host_addr + i) = (uint8_t)do_pior_corrupt(hp, (caddr_t)addr,
		    do_bofi_rd8, i ? 0 : repcount, 1);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_getw() routine - use tryenter
 */
static void
bofi_rep_rd16(ddi_acc_impl_t *handle, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint16_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_READ_TESTS(uint16_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_get16(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		*(host_addr + i) = (uint16_t)do_pior_corrupt(hp, (caddr_t)addr,
		    do_bofi_rd16, i ? 0 : repcount, 2);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_getl() routine - use tryenter
 */
static void
bofi_rep_rd32(ddi_acc_impl_t *handle, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint32_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_READ_TESTS(uint32_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_get32(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		*(host_addr + i) = (uint32_t)do_pior_corrupt(hp, (caddr_t)addr,
		    do_bofi_rd32, i ? 0 : repcount, 4);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_getll() routine - use tryenter
 */
static void
bofi_rep_rd64(ddi_acc_impl_t *handle, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint64_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_READ_TESTS(uint64_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_get64(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		*(host_addr + i) = (uint64_t)do_pior_corrupt(hp, (caddr_t)addr,
		    do_bofi_rd64, i ? 0 : repcount, 8);
	}
	mutex_exit(&bofi_mutex);
}

#define	BOFI_REP_WRITE_TESTS(type) \
	if (bofi_ddi_check) \
		dev_addr = (type *)((uintptr_t)dev_addr - 64 + hp->addr); \
	if (bofi_range_check && ((caddr_t)dev_addr < hp->addr || \
	    (caddr_t)(dev_addr + repcount) - hp->addr > hp->len)) { \
		cmn_err((bofi_range_check == 2) ? CE_PANIC : CE_WARN, \
		    "ddi_rep_put() out of range addr %p not in %p/%llx\n", \
		    (void *)dev_addr, (void *)hp->addr, hp->len); \
		if ((caddr_t)dev_addr < hp->addr || \
		    (caddr_t)dev_addr - hp->addr >= hp->len) \
			return; \
		repcount = (type *)(hp->addr + hp->len) - dev_addr; \
	}

/*
 * our rep_putb() routine - use tryenter
 */
static void
bofi_rep_wr8(ddi_acc_impl_t *handle, uint8_t *host_addr, uint8_t *dev_addr,
    size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint64_t llvalue;
	uint8_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_WRITE_TESTS(uint8_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_put8(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		llvalue = *(host_addr + i);
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 1, i ? 0 :
		    repcount))
			hp->save.acc.ahi_put8(&hp->save.acc, addr,
			    (uint8_t)llvalue);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_putw() routine - use tryenter
 */
static void
bofi_rep_wr16(ddi_acc_impl_t *handle, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint64_t llvalue;
	uint16_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_WRITE_TESTS(uint16_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_put16(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		llvalue = *(host_addr + i);
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 2, i ? 0 :
		    repcount))
			hp->save.acc.ahi_put16(&hp->save.acc, addr,
			    (uint16_t)llvalue);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_putl() routine - use tryenter
 */
static void
bofi_rep_wr32(ddi_acc_impl_t *handle, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint64_t llvalue;
	uint32_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_WRITE_TESTS(uint32_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_put32(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		llvalue = *(host_addr + i);
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 4, i ? 0 :
		    repcount))
			hp->save.acc.ahi_put32(&hp->save.acc, addr,
			    (uint32_t)llvalue);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our rep_putll() routine - use tryenter
 */
static void
bofi_rep_wr64(ddi_acc_impl_t *handle, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	struct bofi_shadow *hp;
	int i;
	uint64_t llvalue;
	uint64_t *addr;

	hp = handle->ahi_common.ah_bus_private;
	BOFI_REP_WRITE_TESTS(uint64_t)
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		hp->save.acc.ahi_rep_put64(&hp->save.acc, host_addr, dev_addr,
		    repcount, flags);
		return;
	}
	for (i = 0; i < repcount; i++) {
		llvalue = *(host_addr + i);
		addr = dev_addr + ((flags == DDI_DEV_AUTOINCR) ? i : 0);
		if (do_piow_corrupt(hp, (caddr_t)addr, &llvalue, 8, i ? 0 :
		    repcount))
			hp->save.acc.ahi_put64(&hp->save.acc, addr,
			    (uint64_t)llvalue);
	}
	mutex_exit(&bofi_mutex);
}


/*
 * our ddi_map routine
 */
static int
bofi_map(dev_info_t *dip, dev_info_t *rdip,
    ddi_map_req_t *reqp, off_t offset, off_t len, caddr_t *vaddrp)
{
	ddi_acc_impl_t *ap;
	struct bofi_shadow *hp;
	struct bofi_errent *ep;
	struct bofi_link   *lp, *next_lp;
	int retval;
	struct bofi_shadow *dhashp;
	struct bofi_shadow *hhashp;

	switch (reqp->map_op) {
	case DDI_MO_MAP_LOCKED:
		/*
		 * for this case get nexus to do real work first
		 */
		retval = save_bus_ops.bus_map(dip, rdip, reqp, offset, len,
		    vaddrp);
		if (retval != DDI_SUCCESS)
			return (retval);

		ap = (ddi_acc_impl_t *)reqp->map_handlep;
		if (ap == NULL)
			return (DDI_SUCCESS);
		/*
		 * if driver_list is set, only intercept those drivers
		 */
		if (!driver_under_test(ap->ahi_common.ah_dip))
			return (DDI_SUCCESS);

		/*
		 * support for ddi_regs_map_setup()
		 * - allocate shadow handle structure and fill it in
		 */
		hp = kmem_zalloc(sizeof (struct bofi_shadow), KM_SLEEP);
		(void) strncpy(hp->name, ddi_get_name(ap->ahi_common.ah_dip),
		    NAMESIZE);
		hp->instance = ddi_get_instance(ap->ahi_common.ah_dip);
		hp->dip = ap->ahi_common.ah_dip;
		hp->addr = *vaddrp;
		/*
		 * return spurious value to catch direct access to registers
		 */
		if (bofi_ddi_check)
			*vaddrp = (caddr_t)64;
		hp->rnumber = ((ddi_acc_hdl_t *)ap)->ah_rnumber;
		hp->offset = offset;
		if (len == 0)
			hp->len = INT_MAX - offset;
		else
			hp->len = min(len, INT_MAX - offset);
		hp->hdl.acc_handle = (ddi_acc_handle_t)ap;
		hp->link = NULL;
		hp->type = BOFI_ACC_HDL;
		/*
		 * save existing function pointers and plug in our own
		 */
		hp->save.acc = *ap;
		ap->ahi_get8 = bofi_rd8;
		ap->ahi_get16 = bofi_rd16;
		ap->ahi_get32 = bofi_rd32;
		ap->ahi_get64 = bofi_rd64;
		ap->ahi_put8 = bofi_wr8;
		ap->ahi_put16 = bofi_wr16;
		ap->ahi_put32 = bofi_wr32;
		ap->ahi_put64 = bofi_wr64;
		ap->ahi_rep_get8 = bofi_rep_rd8;
		ap->ahi_rep_get16 = bofi_rep_rd16;
		ap->ahi_rep_get32 = bofi_rep_rd32;
		ap->ahi_rep_get64 = bofi_rep_rd64;
		ap->ahi_rep_put8 = bofi_rep_wr8;
		ap->ahi_rep_put16 = bofi_rep_wr16;
		ap->ahi_rep_put32 = bofi_rep_wr32;
		ap->ahi_rep_put64 = bofi_rep_wr64;
		ap->ahi_fault_check = bofi_check_acc_hdl;
#if defined(__sparc)
#else
		ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
#endif
		/*
		 * stick in a pointer to our shadow handle
		 */
		ap->ahi_common.ah_bus_private = hp;
		/*
		 * add to dhash, hhash and inuse lists
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		hp->next = shadow_list.next;
		shadow_list.next->prev = hp;
		hp->prev = &shadow_list;
		shadow_list.next = hp;
		hhashp = HDL_HHASH(ap);
		hp->hnext = hhashp->hnext;
		hhashp->hnext->hprev = hp;
		hp->hprev = hhashp;
		hhashp->hnext = hp;
		dhashp = HDL_DHASH(hp->dip);
		hp->dnext = dhashp->dnext;
		dhashp->dnext->dprev = hp;
		hp->dprev = dhashp;
		dhashp->dnext = hp;
		/*
		 * chain on any pre-existing errdefs that apply to this
		 * acc_handle
		 */
		for (ep = errent_listp; ep != NULL; ep = ep->next) {
			if (ddi_name_to_major(hp->name) ==
			    ddi_name_to_major(ep->name) &&
			    hp->instance == ep->errdef.instance &&
			    (ep->errdef.access_type & BOFI_PIO_RW) &&
			    (ep->errdef.rnumber == -1 ||
			    hp->rnumber == ep->errdef.rnumber) &&
			    (ep->errdef.len == 0 ||
			    offset < ep->errdef.offset + ep->errdef.len) &&
			    offset + hp->len > ep->errdef.offset) {
				lp = bofi_link_freelist;
				if (lp != NULL) {
					bofi_link_freelist = lp->link;
					lp->errentp = ep;
					lp->link = hp->link;
					hp->link = lp;
				}
			}
		}
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (DDI_SUCCESS);
	case DDI_MO_UNMAP:

		ap = (ddi_acc_impl_t *)reqp->map_handlep;
		if (ap == NULL)
			break;
		/*
		 * support for ddi_regs_map_free()
		 * - check we really have a shadow handle for this one
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		hhashp = HDL_HHASH(ap);
		for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
			if (hp->hdl.acc_handle == (ddi_acc_handle_t)ap)
				break;
		if (hp == hhashp) {
			mutex_exit(&bofi_mutex);
			mutex_exit(&bofi_low_mutex);
			break;
		}
		/*
		 * got a shadow handle - restore original pointers
		 */
		*ap = hp->save.acc;
		*vaddrp = hp->addr;
		/*
		 * remove from dhash, hhash and inuse lists
		 */
		hp->hnext->hprev = hp->hprev;
		hp->hprev->hnext = hp->hnext;
		hp->dnext->dprev = hp->dprev;
		hp->dprev->dnext = hp->dnext;
		hp->next->prev = hp->prev;
		hp->prev->next = hp->next;
		/*
		 * free any errdef link structures tagged onto the shadow handle
		 */
		for (lp = hp->link; lp != NULL; ) {
			next_lp = lp->link;
			lp->link = bofi_link_freelist;
			bofi_link_freelist = lp;
			lp = next_lp;
		}
		hp->link = NULL;
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		/*
		 * finally delete shadow handle
		 */
		kmem_free(hp, sizeof (struct bofi_shadow));
		break;
	default:
		break;
	}
	return (save_bus_ops.bus_map(dip, rdip, reqp, offset, len, vaddrp));
}


/*
 * chain any pre-existing errdefs on to newly created dma handle
 * if required call do_dma_corrupt() to corrupt data
 */
static void
chain_on_errdefs(struct bofi_shadow *hp)
{
	struct bofi_errent *ep;
	struct bofi_link   *lp;

	ASSERT(MUTEX_HELD(&bofi_mutex));
	/*
	 * chain on any pre-existing errdefs that apply to this dma_handle
	 */
	for (ep = errent_listp; ep != NULL; ep = ep->next) {
		if (ddi_name_to_major(hp->name) ==
		    ddi_name_to_major(ep->name) &&
		    hp->instance == ep->errdef.instance &&
		    (ep->errdef.rnumber == -1 ||
		    hp->rnumber == ep->errdef.rnumber) &&
		    ((ep->errdef.access_type & BOFI_DMA_RW) &&
		    (((uintptr_t)(hp->addr + ep->errdef.offset +
		    ep->errdef.len) & ~LLSZMASK) >
		    ((uintptr_t)((hp->addr + ep->errdef.offset) +
		    LLSZMASK) & ~LLSZMASK)))) {
			/*
			 * got a match - link it on
			 */
			lp = bofi_link_freelist;
			if (lp != NULL) {
				bofi_link_freelist = lp->link;
				lp->errentp = ep;
				lp->link = hp->link;
				hp->link = lp;
				if ((ep->errdef.access_type & BOFI_DMA_W) &&
				    (hp->flags & DDI_DMA_WRITE) &&
				    (ep->state & BOFI_DEV_ACTIVE)) {
					do_dma_corrupt(hp, ep,
					    DDI_DMA_SYNC_FORDEV,
					    0, hp->len);
				}
			}
		}
	}
}


/*
 * need to do copy byte-by-byte in case one of pages is little-endian
 */
static void
xbcopy(void *from, void *to, u_longlong_t len)
{
	uchar_t *f = from;
	uchar_t *t = to;

	while (len--)
		*t++ = *f++;
}


/*
 * our ddi_dma_allochdl routine
 */
static int
bofi_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attrp,
    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
{
	int retval = DDI_DMA_NORESOURCES;
	struct bofi_shadow *hp, *xhp;
	int maxrnumber = 0;
	struct bofi_shadow *dhashp;
	struct bofi_shadow *hhashp;
	ddi_dma_impl_t *mp;

	/*
	 * if driver_list is set, only intercept those drivers
	 */
	if (!driver_under_test(rdip))
		return (save_bus_ops.bus_dma_allochdl(dip, rdip, attrp,
		    waitfp, arg, handlep));

	/*
	 * allocate shadow handle structure and fill it in
	 */
	hp = kmem_zalloc(sizeof (struct bofi_shadow),
	    ((waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP));
	if (hp == NULL) {
		/*
		 * what to do here? Wait a bit and try again
		 */
		if (waitfp != DDI_DMA_DONTWAIT)
			(void) timeout((void (*)())(uintptr_t)waitfp, arg, 10);
		return (retval);
	}
	(void) strncpy(hp->name, ddi_get_name(rdip), NAMESIZE);
	hp->instance = ddi_get_instance(rdip);
	hp->dip = rdip;
	hp->link = NULL;
	hp->type = BOFI_NULL;
	/*
	 * call nexus to do the real work
	 */
	retval = save_bus_ops.bus_dma_allochdl(dip, rdip, attrp, waitfp, arg,
	    handlep);
	if (retval != DDI_SUCCESS) {
		kmem_free(hp, sizeof (struct bofi_shadow));
		return (retval);
	}
	/*
	 * now point set dma_handle to point to real handle
	 */
	hp->hdl.dma_handle = *handlep;
	mp = (ddi_dma_impl_t *)*handlep;
	mp->dmai_fault_check = bofi_check_dma_hdl;
	/*
	 * bind and unbind are cached in devinfo - must overwrite them
	 * - note that our bind and unbind are quite happy dealing with
	 * any handles for this devinfo that were previously allocated
	 */
	if (save_bus_ops.bus_dma_bindhdl == DEVI(rdip)->devi_bus_dma_bindfunc)
		DEVI(rdip)->devi_bus_dma_bindfunc = bofi_dma_bindhdl;
	if (save_bus_ops.bus_dma_unbindhdl ==
	    DEVI(rdip)->devi_bus_dma_unbindfunc)
		DEVI(rdip)->devi_bus_dma_unbindfunc = bofi_dma_unbindhdl;
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	/*
	 * get an "rnumber" for this handle - really just seeking to
	 * get a unique number - generally only care for early allocated
	 * handles - so we get as far as INT_MAX, just stay there
	 */
	dhashp = HDL_DHASH(hp->dip);
	for (xhp = dhashp->dnext; xhp != dhashp; xhp = xhp->dnext)
		if (ddi_name_to_major(xhp->name) ==
		    ddi_name_to_major(hp->name) &&
		    xhp->instance == hp->instance &&
		    (xhp->type == BOFI_DMA_HDL ||
		    xhp->type == BOFI_NULL))
			if (xhp->rnumber >= maxrnumber) {
				if (xhp->rnumber == INT_MAX)
					maxrnumber = INT_MAX;
				else
					maxrnumber = xhp->rnumber + 1;
			}
	hp->rnumber = maxrnumber;
	/*
	 * add to dhash, hhash and inuse lists
	 */
	hp->next = shadow_list.next;
	shadow_list.next->prev = hp;
	hp->prev = &shadow_list;
	shadow_list.next = hp;
	hhashp = HDL_HHASH(*handlep);
	hp->hnext = hhashp->hnext;
	hhashp->hnext->hprev = hp;
	hp->hprev = hhashp;
	hhashp->hnext = hp;
	dhashp = HDL_DHASH(hp->dip);
	hp->dnext = dhashp->dnext;
	dhashp->dnext->dprev = hp;
	hp->dprev = dhashp;
	dhashp->dnext = hp;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (retval);
}


/*
 * our ddi_dma_freehdl routine
 */
static int
bofi_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
{
	int retval;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;

	/*
	 * find shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	/*
	 * call nexus to do the real work
	 */
	retval = save_bus_ops.bus_dma_freehdl(dip, rdip, handle);
	if (retval != DDI_SUCCESS) {
		return (retval);
	}
	/*
	 * did we really have a shadow for this handle
	 */
	if (hp == hhashp)
		return (retval);
	/*
	 * yes we have - see if it's still bound
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	if (hp->type != BOFI_NULL)
		panic("driver freeing bound dma_handle");
	/*
	 * remove from dhash, hhash and inuse lists
	 */
	hp->hnext->hprev = hp->hprev;
	hp->hprev->hnext = hp->hnext;
	hp->dnext->dprev = hp->dprev;
	hp->dprev->dnext = hp->dnext;
	hp->next->prev = hp->prev;
	hp->prev->next = hp->next;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);

	kmem_free(hp, sizeof (struct bofi_shadow));
	return (retval);
}


/*
 * our ddi_dma_bindhdl routine
 */
static int
bofi_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
    ddi_dma_handle_t handle, struct ddi_dma_req *dmareqp,
    ddi_dma_cookie_t *cookiep, uint_t *ccountp)
{
	int retval = DDI_DMA_NORESOURCES;
	auto struct ddi_dma_req dmareq;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;
	ddi_dma_impl_t *mp;
	unsigned long pagemask = ddi_ptob(rdip, 1) - 1;

	/*
	 * check we really have a shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	if (hp == hhashp) {
		/*
		 * no we don't - just call nexus to do the real work
		 */
		return save_bus_ops.bus_dma_bindhdl(dip, rdip, handle, dmareqp,
		    cookiep, ccountp);
	}
	/*
	 * yes we have - see if it's already bound
	 */
	if (hp->type != BOFI_NULL)
		return (DDI_DMA_INUSE);

	hp->flags = dmareqp->dmar_flags;
	if (dmareqp->dmar_object.dmao_type == DMA_OTYP_PAGES) {
		hp->map_flags = B_PAGEIO;
		hp->map_pp = dmareqp->dmar_object.dmao_obj.pp_obj.pp_pp;
	} else if (dmareqp->dmar_object.dmao_obj.virt_obj.v_priv != NULL) {
		hp->map_flags = B_SHADOW;
		hp->map_pplist = dmareqp->dmar_object.dmao_obj.virt_obj.v_priv;
	} else {
		hp->map_flags = 0;
	}
	/*
	 * get a kernel virtual mapping
	 */
	hp->addr = ddi_dmareq_mapin(dmareqp, &hp->mapaddr, &hp->len);
	if (hp->addr == NULL)
		goto error;
	if (bofi_sync_check) {
		/*
		 * Take a copy and pass pointers to this up to nexus instead.
		 * Data will be copied from the original on explicit
		 * and implicit ddi_dma_sync()
		 *
		 * - maintain page alignment because some devices assume it.
		 */
		hp->origaddr = hp->addr;
		hp->allocaddr = ddi_umem_alloc(
		    ((uintptr_t)hp->addr & pagemask) + hp->len,
		    (dmareqp->dmar_fp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP,
		    &hp->umem_cookie);
		if (hp->allocaddr == NULL)
			goto error;
		hp->addr = hp->allocaddr + ((uintptr_t)hp->addr & pagemask);
		if (dmareqp->dmar_flags & DDI_DMA_WRITE)
			xbcopy(hp->origaddr, hp->addr, hp->len);
		dmareq = *dmareqp;
		dmareq.dmar_object.dmao_size = hp->len;
		dmareq.dmar_object.dmao_type = DMA_OTYP_VADDR;
		dmareq.dmar_object.dmao_obj.virt_obj.v_as = &kas;
		dmareq.dmar_object.dmao_obj.virt_obj.v_addr = hp->addr;
		dmareq.dmar_object.dmao_obj.virt_obj.v_priv = NULL;
		dmareqp = &dmareq;
	}
	/*
	 * call nexus to do the real work
	 */
	retval = save_bus_ops.bus_dma_bindhdl(dip, rdip, handle, dmareqp,
	    cookiep, ccountp);
	if (retval != DDI_SUCCESS)
		goto error2;
	/*
	 * unset DMP_NOSYNC
	 */
	mp = (ddi_dma_impl_t *)handle;
	mp->dmai_rflags &= ~DMP_NOSYNC;
	/*
	 * chain on any pre-existing errdefs that apply to this
	 * acc_handle and corrupt if required (as there is an implicit
	 * ddi_dma_sync() in this call)
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hp->type = BOFI_DMA_HDL;
	chain_on_errdefs(hp);
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (retval);

error:
	if (dmareqp->dmar_fp != DDI_DMA_DONTWAIT) {
		/*
		 * what to do here? Wait a bit and try again
		 */
		(void) timeout((void (*)())(uintptr_t)dmareqp->dmar_fp,
		    dmareqp->dmar_arg, 10);
	}
error2:
	if (hp) {
		ddi_dmareq_mapout(hp->mapaddr, hp->len, hp->map_flags,
		    hp->map_pp, hp->map_pplist);
		if (bofi_sync_check && hp->allocaddr)
			ddi_umem_free(hp->umem_cookie);
		hp->mapaddr = NULL;
		hp->allocaddr = NULL;
		hp->origaddr = NULL;
	}
	return (retval);
}


/*
 * our ddi_dma_unbindhdl routine
 */
static int
bofi_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
{
	struct bofi_link *lp, *next_lp;
	struct bofi_errent *ep;
	int retval;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;

	/*
	 * call nexus to do the real work
	 */
	retval = save_bus_ops.bus_dma_unbindhdl(dip, rdip, handle);
	if (retval != DDI_SUCCESS)
		return (retval);
	/*
	 * check we really have a shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle)
			break;
	if (hp == hhashp) {
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (retval);
	}
	/*
	 * yes we have - see if it's already unbound
	 */
	if (hp->type == BOFI_NULL)
		panic("driver unbinding unbound dma_handle");
	/*
	 * free any errdef link structures tagged on to this
	 * shadow handle
	 */
	for (lp = hp->link; lp != NULL; ) {
		next_lp = lp->link;
		/*
		 * there is an implicit sync_for_cpu on free -
		 * may need to corrupt
		 */
		ep = lp->errentp;
		if ((ep->errdef.access_type & BOFI_DMA_R) &&
		    (hp->flags & DDI_DMA_READ) &&
		    (ep->state & BOFI_DEV_ACTIVE)) {
			do_dma_corrupt(hp, ep, DDI_DMA_SYNC_FORCPU, 0, hp->len);
		}
		lp->link = bofi_link_freelist;
		bofi_link_freelist = lp;
		lp = next_lp;
	}
	hp->link = NULL;
	hp->type = BOFI_NULL;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);

	if (bofi_sync_check && (hp->flags & DDI_DMA_READ))
		/*
		 * implicit sync_for_cpu - copy data back
		 */
		if (hp->allocaddr)
			xbcopy(hp->addr, hp->origaddr, hp->len);
	ddi_dmareq_mapout(hp->mapaddr, hp->len, hp->map_flags,
	    hp->map_pp, hp->map_pplist);
	if (bofi_sync_check && hp->allocaddr)
		ddi_umem_free(hp->umem_cookie);
	hp->mapaddr = NULL;
	hp->allocaddr = NULL;
	hp->origaddr = NULL;
	return (retval);
}


/*
 * our ddi_dma_sync routine
 */
static int
bofi_dma_flush(dev_info_t *dip, dev_info_t *rdip,
    ddi_dma_handle_t handle, off_t off, size_t len, uint_t flags)
{
	struct bofi_link *lp;
	struct bofi_errent *ep;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;
	int retval;

	if (flags == DDI_DMA_SYNC_FORCPU || flags == DDI_DMA_SYNC_FORKERNEL) {
		/*
		 * in this case get nexus driver to do sync first
		 */
		retval = save_bus_ops.bus_dma_flush(dip, rdip, handle, off,
		    len, flags);
		if (retval != DDI_SUCCESS)
			return (retval);
	}
	/*
	 * check we really have a shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle &&
		    hp->type == BOFI_DMA_HDL)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	if (hp != hhashp) {
		/*
		 * yes - do we need to copy data from original
		 */
		if (bofi_sync_check && flags == DDI_DMA_SYNC_FORDEV)
			if (hp->allocaddr)
				xbcopy(hp->origaddr+off, hp->addr+off,
				    len ? len : (hp->len - off));
		/*
		 * yes - check if we need to corrupt the data
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		for (lp = hp->link; lp != NULL; lp = lp->link) {
			ep = lp->errentp;
			if ((((ep->errdef.access_type & BOFI_DMA_R) &&
			    (flags == DDI_DMA_SYNC_FORCPU ||
			    flags == DDI_DMA_SYNC_FORKERNEL)) ||
			    ((ep->errdef.access_type & BOFI_DMA_W) &&
			    (flags == DDI_DMA_SYNC_FORDEV))) &&
			    (ep->state & BOFI_DEV_ACTIVE)) {
				do_dma_corrupt(hp, ep, flags, off,
				    len ? len : (hp->len - off));
			}
		}
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		/*
		 *  do we need to copy data to original
		 */
		if (bofi_sync_check && (flags == DDI_DMA_SYNC_FORCPU ||
		    flags == DDI_DMA_SYNC_FORKERNEL))
			if (hp->allocaddr)
				xbcopy(hp->addr+off, hp->origaddr+off,
				    len ? len : (hp->len - off));
	}
	if (flags == DDI_DMA_SYNC_FORDEV)
		/*
		 * in this case get nexus driver to do sync last
		 */
		retval = save_bus_ops.bus_dma_flush(dip, rdip, handle, off,
		    len, flags);
	return (retval);
}


/*
 * our dma_win routine
 */
static int
bofi_dma_win(dev_info_t *dip, dev_info_t *rdip,
    ddi_dma_handle_t handle, uint_t win, off_t *offp,
    size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
{
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;
	int retval;
	ddi_dma_impl_t *mp;

	/*
	 * call nexus to do the real work
	 */
	retval = save_bus_ops.bus_dma_win(dip, rdip, handle, win, offp, lenp,
	    cookiep, ccountp);
	if (retval != DDI_SUCCESS)
		return (retval);
	/*
	 * check we really have a shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle)
			break;
	if (hp != hhashp) {
		/*
		 * yes - make sure DMP_NOSYNC is unset
		 */
		mp = (ddi_dma_impl_t *)handle;
		mp->dmai_rflags &= ~DMP_NOSYNC;
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (retval);
}


/*
 * our dma_ctl routine
 */
static int
bofi_dma_ctl(dev_info_t *dip, dev_info_t *rdip,
    ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
    off_t *offp, size_t *lenp, caddr_t *objp, uint_t flags)
{
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;
	int retval;
	int i;
	struct bofi_shadow *dummyhp;

	/*
	 * get nexus to do real work
	 */
	retval = save_bus_ops.bus_dma_ctl(dip, rdip, handle, request, offp,
	    lenp, objp, flags);
	if (retval != DDI_SUCCESS)
		return (retval);
	/*
	 * if driver_list is set, only intercept those drivers
	 */
	if (!driver_under_test(rdip))
		return (DDI_SUCCESS);

#if defined(__sparc)
	/*
	 * check if this is a dvma_reserve - that one's like a
	 * dma_allochdl and needs to be handled separately
	 */
	if (request == DDI_DMA_RESERVE) {
		bofi_dvma_reserve(rdip, *(ddi_dma_handle_t *)objp);
		return (DDI_SUCCESS);
	}
#endif
	/*
	 * check we really have a shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == handle)
			break;
	if (hp == hhashp) {
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (retval);
	}
	/*
	 * yes we have - see what kind of command this is
	 */
	switch (request) {
	case DDI_DMA_RELEASE:
		/*
		 * dvma release - release dummy handle and all the index handles
		 */
		dummyhp = hp;
		dummyhp->hnext->hprev = dummyhp->hprev;
		dummyhp->hprev->hnext = dummyhp->hnext;
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		for (i = 0; i < dummyhp->len; i++) {
			hp = dummyhp->hparrayp[i];
			/*
			 * chek none of the index handles were still loaded
			 */
			if (hp->type != BOFI_NULL)
				panic("driver releasing loaded dvma");
			/*
			 * remove from dhash and inuse lists
			 */
			mutex_enter(&bofi_low_mutex);
			mutex_enter(&bofi_mutex);
			hp->dnext->dprev = hp->dprev;
			hp->dprev->dnext = hp->dnext;
			hp->next->prev = hp->prev;
			hp->prev->next = hp->next;
			mutex_exit(&bofi_mutex);
			mutex_exit(&bofi_low_mutex);

			if (bofi_sync_check && hp->allocaddr)
				ddi_umem_free(hp->umem_cookie);
			kmem_free(hp, sizeof (struct bofi_shadow));
		}
		kmem_free(dummyhp->hparrayp, dummyhp->len *
		    sizeof (struct bofi_shadow *));
		kmem_free(dummyhp, sizeof (struct bofi_shadow));
		return (retval);
	default:
		break;
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	return (retval);
}

#if defined(__sparc)
/*
 * dvma reserve case from bofi_dma_ctl()
 */
static void
bofi_dvma_reserve(dev_info_t *rdip, ddi_dma_handle_t handle)
{
	struct bofi_shadow *hp;
	struct bofi_shadow *dummyhp;
	struct bofi_shadow *dhashp;
	struct bofi_shadow *hhashp;
	ddi_dma_impl_t *mp;
	struct fast_dvma *nexus_private;
	int i, count;

	mp = (ddi_dma_impl_t *)handle;
	count = mp->dmai_ndvmapages;
	/*
	 * allocate dummy shadow handle structure
	 */
	dummyhp = kmem_zalloc(sizeof (*dummyhp), KM_SLEEP);
	if (mp->dmai_rflags & DMP_BYPASSNEXUS) {
		/*
		 * overlay our routines over the nexus's dvma routines
		 */
		nexus_private = (struct fast_dvma *)mp->dmai_nexus_private;
		dummyhp->save.dvma_ops = *(nexus_private->ops);
		nexus_private->ops = &bofi_dvma_ops;
	}
	/*
	 * now fill in the dummy handle. This just gets put on hhash queue
	 * so our dvma routines can find and index off to the handle they
	 * really want.
	 */
	(void) strncpy(dummyhp->name, ddi_get_name(rdip), NAMESIZE);
	dummyhp->instance = ddi_get_instance(rdip);
	dummyhp->rnumber = -1;
	dummyhp->dip = rdip;
	dummyhp->len = count;
	dummyhp->hdl.dma_handle = handle;
	dummyhp->link = NULL;
	dummyhp->type = BOFI_NULL;
	/*
	 * allocate space for real handles
	 */
	dummyhp->hparrayp = kmem_alloc(count *
	    sizeof (struct bofi_shadow *), KM_SLEEP);
	for (i = 0; i < count; i++) {
		/*
		 * allocate shadow handle structures and fill them in
		 */
		hp = kmem_zalloc(sizeof (*hp), KM_SLEEP);
		(void) strncpy(hp->name, ddi_get_name(rdip), NAMESIZE);
		hp->instance = ddi_get_instance(rdip);
		hp->rnumber = -1;
		hp->dip = rdip;
		hp->hdl.dma_handle = 0;
		hp->link = NULL;
		hp->type = BOFI_NULL;
		if (bofi_sync_check) {
			unsigned long pagemask = ddi_ptob(rdip, 1) - 1;
			/*
			 * Take a copy and set this to be hp->addr
			 * Data will be copied to and from the original on
			 * explicit and implicit ddi_dma_sync()
			 *
			 * - maintain page alignment because some devices
			 * assume it.
			 */
			hp->allocaddr = ddi_umem_alloc(
			    ((int)(uintptr_t)hp->addr & pagemask)
			    + pagemask + 1,
			    KM_SLEEP, &hp->umem_cookie);
			hp->addr = hp->allocaddr +
			    ((int)(uintptr_t)hp->addr & pagemask);
		}
		/*
		 * add to dhash and inuse lists.
		 * these don't go on hhash queue.
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		hp->next = shadow_list.next;
		shadow_list.next->prev = hp;
		hp->prev = &shadow_list;
		shadow_list.next = hp;
		dhashp = HDL_DHASH(hp->dip);
		hp->dnext = dhashp->dnext;
		dhashp->dnext->dprev = hp;
		hp->dprev = dhashp;
		dhashp->dnext = hp;
		dummyhp->hparrayp[i] = hp;
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
	}
	/*
	 * add dummy handle to hhash list only
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(handle);
	dummyhp->hnext = hhashp->hnext;
	hhashp->hnext->hprev = dummyhp;
	dummyhp->hprev = hhashp;
	hhashp->hnext = dummyhp;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
}

/*
 * our dvma_kaddr_load()
 */
static void
bofi_dvma_kaddr_load(ddi_dma_handle_t h, caddr_t a, uint_t len, uint_t index,
    ddi_dma_cookie_t *cp)
{
	struct bofi_shadow *dummyhp;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;
	struct bofi_errent *ep;
	struct bofi_link   *lp;

	/*
	 * check we really have a dummy shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(h);
	for (dummyhp = hhashp->hnext; dummyhp != hhashp;
	    dummyhp = dummyhp->hnext)
		if (dummyhp->hdl.dma_handle == h)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	if (dummyhp == hhashp) {
		/*
		 * no dummy shadow - panic
		 */
		panic("driver dvma_kaddr_load with no reserve");
	}

	/*
	 * find real hp
	 */
	hp = dummyhp->hparrayp[index];
	/*
	 * check its not already loaded
	 */
	if (hp->type != BOFI_NULL)
		panic("driver loading loaded dvma");
	/*
	 * if were doing copying, just need to change origaddr and get
	 * nexus to map hp->addr again
	 * if not, set hp->addr to new address.
	 * - note these are always kernel virtual addresses - no need to map
	 */
	if (bofi_sync_check && hp->allocaddr) {
		hp->origaddr = a;
		a = hp->addr;
	} else
		hp->addr = a;
	hp->len = len;
	/*
	 * get nexus to do the real work
	 */
	dummyhp->save.dvma_ops.dvma_kaddr_load(h, a, len, index, cp);
	/*
	 * chain on any pre-existing errdefs that apply to this dma_handle
	 * no need to corrupt - there's no implicit dma_sync on this one
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hp->type = BOFI_DMA_HDL;
	for (ep = errent_listp; ep != NULL; ep = ep->next) {
		if (ddi_name_to_major(hp->name) ==
		    ddi_name_to_major(ep->name) &&
		    hp->instance == ep->errdef.instance &&
		    (ep->errdef.rnumber == -1 ||
		    hp->rnumber == ep->errdef.rnumber) &&
		    ((ep->errdef.access_type & BOFI_DMA_RW) &&
		    (((uintptr_t)(hp->addr + ep->errdef.offset +
		    ep->errdef.len) & ~LLSZMASK) >
		    ((uintptr_t)((hp->addr + ep->errdef.offset) +
		    LLSZMASK) & ~LLSZMASK)))) {
			lp = bofi_link_freelist;
			if (lp != NULL) {
				bofi_link_freelist = lp->link;
				lp->errentp = ep;
				lp->link = hp->link;
				hp->link = lp;
			}
		}
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
}

/*
 * our dvma_unload()
 */
static void
bofi_dvma_unload(ddi_dma_handle_t h, uint_t index, uint_t view)
{
	struct bofi_link *lp, *next_lp;
	struct bofi_errent *ep;
	struct bofi_shadow *dummyhp;
	struct bofi_shadow *hp;
	struct bofi_shadow *hhashp;

	/*
	 * check we really have a dummy shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(h);
	for (dummyhp = hhashp->hnext; dummyhp != hhashp;
	    dummyhp = dummyhp->hnext)
		if (dummyhp->hdl.dma_handle == h)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	if (dummyhp == hhashp) {
		/*
		 * no dummy shadow - panic
		 */
		panic("driver dvma_unload with no reserve");
	}
	dummyhp->save.dvma_ops.dvma_unload(h, index, view);
	/*
	 * find real hp
	 */
	hp = dummyhp->hparrayp[index];
	/*
	 * check its not already unloaded
	 */
	if (hp->type == BOFI_NULL)
		panic("driver unloading unloaded dvma");
	/*
	 * free any errdef link structures tagged on to this
	 * shadow handle - do corruption if necessary
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	for (lp = hp->link; lp != NULL; ) {
		next_lp = lp->link;
		ep = lp->errentp;
		if ((ep->errdef.access_type & BOFI_DMA_R) &&
		    (view == DDI_DMA_SYNC_FORCPU ||
		    view == DDI_DMA_SYNC_FORKERNEL) &&
		    (ep->state & BOFI_DEV_ACTIVE)) {
			do_dma_corrupt(hp, ep, view, 0, hp->len);
		}
		lp->link = bofi_link_freelist;
		bofi_link_freelist = lp;
		lp = next_lp;
	}
	hp->link = NULL;
	hp->type = BOFI_NULL;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	/*
	 * if there is an explicit sync_for_cpu, then do copy to original
	 */
	if (bofi_sync_check &&
	    (view == DDI_DMA_SYNC_FORCPU || view == DDI_DMA_SYNC_FORKERNEL))
		if (hp->allocaddr)
			xbcopy(hp->addr, hp->origaddr, hp->len);
}

/*
 * our dvma_unload()
 */
static void
bofi_dvma_sync(ddi_dma_handle_t h, uint_t index, uint_t view)
{
	struct bofi_link *lp;
	struct bofi_errent *ep;
	struct bofi_shadow *hp;
	struct bofi_shadow *dummyhp;
	struct bofi_shadow *hhashp;

	/*
	 * check we really have a dummy shadow for this handle
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	hhashp = HDL_HHASH(h);
	for (dummyhp = hhashp->hnext; dummyhp != hhashp;
	    dummyhp = dummyhp->hnext)
		if (dummyhp->hdl.dma_handle == h)
			break;
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	if (dummyhp == hhashp) {
		/*
		 * no dummy shadow - panic
		 */
		panic("driver dvma_sync with no reserve");
	}
	/*
	 * find real hp
	 */
	hp = dummyhp->hparrayp[index];
	/*
	 * check its already loaded
	 */
	if (hp->type == BOFI_NULL)
		panic("driver syncing unloaded dvma");
	if (view == DDI_DMA_SYNC_FORCPU || view == DDI_DMA_SYNC_FORKERNEL)
		/*
		 * in this case do sync first
		 */
		dummyhp->save.dvma_ops.dvma_sync(h, index, view);
	/*
	 * if there is an explicit sync_for_dev, then do copy from original
	 */
	if (bofi_sync_check && view == DDI_DMA_SYNC_FORDEV) {
		if (hp->allocaddr)
			xbcopy(hp->origaddr, hp->addr, hp->len);
	}
	/*
	 * do corruption if necessary
	 */
	mutex_enter(&bofi_low_mutex);
	mutex_enter(&bofi_mutex);
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		ep = lp->errentp;
		if ((((ep->errdef.access_type & BOFI_DMA_R) &&
		    (view == DDI_DMA_SYNC_FORCPU ||
		    view == DDI_DMA_SYNC_FORKERNEL)) ||
		    ((ep->errdef.access_type & BOFI_DMA_W) &&
		    (view == DDI_DMA_SYNC_FORDEV))) &&
		    (ep->state & BOFI_DEV_ACTIVE)) {
			do_dma_corrupt(hp, ep, view, 0, hp->len);
		}
	}
	mutex_exit(&bofi_mutex);
	mutex_exit(&bofi_low_mutex);
	/*
	 * if there is an explicit sync_for_cpu, then do copy to original
	 */
	if (bofi_sync_check &&
	    (view == DDI_DMA_SYNC_FORCPU || view == DDI_DMA_SYNC_FORKERNEL)) {
		if (hp->allocaddr)
			xbcopy(hp->addr, hp->origaddr, hp->len);
	}
	if (view == DDI_DMA_SYNC_FORDEV)
		/*
		 * in this case do sync last
		 */
		dummyhp->save.dvma_ops.dvma_sync(h, index, view);
}
#endif

/*
 * bofi intercept routine - gets called instead of users interrupt routine
 */
static uint_t
bofi_intercept_intr(caddr_t xp, caddr_t arg2)
{
	struct bofi_errent *ep;
	struct bofi_link   *lp;
	struct bofi_shadow *hp;
	int intr_count = 1;
	int i;
	uint_t retval = DDI_INTR_UNCLAIMED;
	uint_t result;
	int unclaimed_counter = 0;
	int jabber_detected = 0;

	hp = (struct bofi_shadow *)xp;
	/*
	 * check if nothing to do
	 */
	if (hp->link == NULL)
		return (hp->save.intr.int_handler
		    (hp->save.intr.int_handler_arg1, arg2));
	mutex_enter(&bofi_mutex);
	/*
	 * look for any errdefs
	 */
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		ep = lp->errentp;
		if (ep->state & BOFI_DEV_ACTIVE) {
			/*
			 * got one
			 */
			if ((ep->errdef.access_count ||
			    ep->errdef.fail_count) &&
			    (ep->errdef.access_type & BOFI_LOG))
				log_acc_event(ep, BOFI_INTR, 0, 0, 1, 0);
			if (ep->errdef.access_count > 1) {
				ep->errdef.access_count--;
			} else if (ep->errdef.fail_count > 0) {
				ep->errdef.fail_count--;
				ep->errdef.access_count = 0;
				/*
				 * OK do "corruption"
				 */
				if (ep->errstate.fail_time == 0)
					ep->errstate.fail_time = bofi_gettime();
				switch (ep->errdef.optype) {
				case BOFI_DELAY_INTR:
					if (!hp->hilevel) {
						drv_usecwait
						    (ep->errdef.operand);
					}
					break;
				case BOFI_LOSE_INTR:
					intr_count = 0;
					break;
				case BOFI_EXTRA_INTR:
					intr_count += ep->errdef.operand;
					break;
				default:
					break;
				}
			}
		}
	}
	mutex_exit(&bofi_mutex);
	/*
	 * send extra or fewer interrupts as requested
	 */
	for (i = 0; i < intr_count; i++) {
		result = hp->save.intr.int_handler
		    (hp->save.intr.int_handler_arg1, arg2);
		if (result == DDI_INTR_CLAIMED)
			unclaimed_counter >>= 1;
		else if (++unclaimed_counter >= 20)
			jabber_detected = 1;
		if (i == 0)
			retval = result;
	}
	/*
	 * if more than 1000 spurious interrupts requested and
	 * jabber not detected - give warning
	 */
	if (intr_count > 1000 && !jabber_detected)
		panic("undetected interrupt jabber: %s%d",
		    hp->name, hp->instance);
	/*
	 * return first response - or "unclaimed" if none
	 */
	return (retval);
}


/*
 * our ddi_check_acc_hdl
 */
/* ARGSUSED */
static int
bofi_check_acc_hdl(ddi_acc_impl_t *handle)
{
	struct bofi_shadow *hp;
	struct bofi_link   *lp;
	uint_t result = 0;

	hp = handle->ahi_common.ah_bus_private;
	if (!hp->link || !mutex_tryenter(&bofi_mutex)) {
		return (0);
	}
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		/*
		 * OR in error state from all associated
		 * errdef structures
		 */
		if (lp->errentp->errdef.access_count == 0 &&
		    (lp->errentp->state & BOFI_DEV_ACTIVE)) {
			result = (lp->errentp->errdef.acc_chk & 1);
		}
	}
	mutex_exit(&bofi_mutex);
	return (result);
}

/*
 * our ddi_check_dma_hdl
 */
/* ARGSUSED */
static int
bofi_check_dma_hdl(ddi_dma_impl_t *handle)
{
	struct bofi_shadow *hp;
	struct bofi_link   *lp;
	struct bofi_shadow *hhashp;
	uint_t result = 0;

	if (!mutex_tryenter(&bofi_mutex)) {
		return (0);
	}
	hhashp = HDL_HHASH(handle);
	for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext)
		if (hp->hdl.dma_handle == (ddi_dma_handle_t)handle)
			break;
	if (hp == hhashp) {
		mutex_exit(&bofi_mutex);
		return (0);
	}
	if (!hp->link) {
		mutex_exit(&bofi_mutex);
		return (0);
	}
	for (lp = hp->link; lp != NULL; lp = lp->link) {
		/*
		 * OR in error state from all associated
		 * errdef structures
		 */
		if (lp->errentp->errdef.access_count == 0 &&
		    (lp->errentp->state & BOFI_DEV_ACTIVE)) {
			result = ((lp->errentp->errdef.acc_chk & 2) ? 1 : 0);
		}
	}
	mutex_exit(&bofi_mutex);
	return (result);
}


/* ARGSUSED */
static int
bofi_post_event(dev_info_t *dip, dev_info_t *rdip,
    ddi_eventcookie_t eventhdl, void *impl_data)
{
	ddi_eventcookie_t ec;
	struct ddi_fault_event_data *arg;
	struct bofi_errent *ep;
	struct bofi_shadow *hp;
	struct bofi_shadow *dhashp;
	struct bofi_link   *lp;

	ASSERT(eventhdl);
	if (ddi_get_eventcookie(dip, DDI_DEVI_FAULT_EVENT, &ec) != DDI_SUCCESS)
		return (DDI_FAILURE);

	if (ec != eventhdl)
		return (save_bus_ops.bus_post_event(dip, rdip, eventhdl,
		    impl_data));

	arg = (struct ddi_fault_event_data *)impl_data;
	mutex_enter(&bofi_mutex);
	/*
	 * find shadow handles with appropriate dev_infos
	 * and set error reported on all associated errdef structures
	 */
	dhashp = HDL_DHASH(arg->f_dip);
	for (hp = dhashp->dnext; hp != dhashp; hp = hp->dnext) {
		if (hp->dip == arg->f_dip) {
			for (lp = hp->link; lp != NULL; lp = lp->link) {
				ep = lp->errentp;
				ep->errstate.errmsg_count++;
				if ((ep->errstate.msg_time == 0 ||
				    ep->errstate.severity > arg->f_impact) &&
				    (ep->state & BOFI_DEV_ACTIVE)) {
					ep->errstate.msg_time = bofi_gettime();
					ep->errstate.severity = arg->f_impact;
					(void) strncpy(ep->errstate.buffer,
					    arg->f_message, ERRMSGSIZE);
					ddi_trigger_softintr(ep->softintr_id);
				}
			}
		}
	}
	mutex_exit(&bofi_mutex);
	return (save_bus_ops.bus_post_event(dip, rdip, eventhdl, impl_data));
}

/*ARGSUSED*/
static int
bofi_fm_ereport_callback(sysevent_t *ev, void *cookie)
{
	char *class = "";
	char *path = "";
	char *ptr;
	nvlist_t *nvlist;
	nvlist_t *detector;
	ddi_fault_impact_t impact;
	struct bofi_errent *ep;
	struct bofi_shadow *hp;
	struct bofi_link   *lp;
	char service_class[FM_MAX_CLASS];
	char hppath[MAXPATHLEN];
	int service_ereport = 0;

	(void) sysevent_get_attr_list(ev, &nvlist);
	(void) nvlist_lookup_string(nvlist, FM_CLASS, &class);
	if (nvlist_lookup_nvlist(nvlist, FM_EREPORT_DETECTOR, &detector) == 0)
		(void) nvlist_lookup_string(detector, FM_FMRI_DEV_PATH, &path);

	(void) snprintf(service_class, FM_MAX_CLASS, "%s.%s.%s.",
	    FM_EREPORT_CLASS, DDI_IO_CLASS, DDI_FM_SERVICE_IMPACT);
	if (strncmp(class, service_class, strlen(service_class) - 1) == 0)
		service_ereport = 1;

	mutex_enter(&bofi_mutex);
	/*
	 * find shadow handles with appropriate dev_infos
	 * and set error reported on all associated errdef structures
	 */
	for (hp = shadow_list.next; hp != &shadow_list; hp = hp->next) {
		(void) ddi_pathname(hp->dip, hppath);
		if (strcmp(path, hppath) != 0)
			continue;
		for (lp = hp->link; lp != NULL; lp = lp->link) {
			ep = lp->errentp;
			ep->errstate.errmsg_count++;
			if (!(ep->state & BOFI_DEV_ACTIVE))
				continue;
			if (ep->errstate.msg_time != 0)
				continue;
			if (service_ereport) {
				ptr = class + strlen(service_class);
				if (strcmp(ptr, DDI_FM_SERVICE_LOST) == 0)
					impact = DDI_SERVICE_LOST;
				else if (strcmp(ptr,
				    DDI_FM_SERVICE_DEGRADED) == 0)
					impact = DDI_SERVICE_DEGRADED;
				else if (strcmp(ptr,
				    DDI_FM_SERVICE_RESTORED) == 0)
					impact = DDI_SERVICE_RESTORED;
				else
					impact = DDI_SERVICE_UNAFFECTED;
				if (ep->errstate.severity > impact)
					ep->errstate.severity = impact;
			} else if (ep->errstate.buffer[0] == '\0') {
				(void) strncpy(ep->errstate.buffer, class,
				    ERRMSGSIZE);
			}
			if (ep->errstate.buffer[0] != '\0' &&
			    ep->errstate.severity < DDI_SERVICE_RESTORED) {
				ep->errstate.msg_time = bofi_gettime();
				ddi_trigger_softintr(ep->softintr_id);
			}
		}
	}
	nvlist_free(nvlist);
	mutex_exit(&bofi_mutex);
	return (0);
}

/*
 * our intr_ops routine
 */
static int
bofi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t intr_op,
    ddi_intr_handle_impl_t *hdlp, void *result)
{
	int retval;
	struct bofi_shadow *hp;
	struct bofi_shadow *dhashp;
	struct bofi_shadow *hhashp;
	struct bofi_errent *ep;
	struct bofi_link   *lp, *next_lp;

	switch (intr_op) {
	case DDI_INTROP_ADDISR:
		/*
		 * if driver_list is set, only intercept those drivers
		 */
		if (!driver_under_test(rdip))
			return (save_bus_ops.bus_intr_op(dip, rdip,
			    intr_op, hdlp, result));
		/*
		 * allocate shadow handle structure and fill in
		 */
		hp = kmem_zalloc(sizeof (struct bofi_shadow), KM_SLEEP);
		(void) strncpy(hp->name, ddi_get_name(rdip), NAMESIZE);
		hp->instance = ddi_get_instance(rdip);
		hp->save.intr.int_handler = hdlp->ih_cb_func;
		hp->save.intr.int_handler_arg1 = hdlp->ih_cb_arg1;
		hdlp->ih_cb_func = (ddi_intr_handler_t *)bofi_intercept_intr;
		hdlp->ih_cb_arg1 = (caddr_t)hp;
		hp->bofi_inum = hdlp->ih_inum;
		hp->dip = rdip;
		hp->link = NULL;
		hp->type = BOFI_INT_HDL;
		/*
		 * save whether hilevel or not
		 */

		if (hdlp->ih_pri >= ddi_intr_get_hilevel_pri())
			hp->hilevel = 1;
		else
			hp->hilevel = 0;

		/*
		 * call nexus to do real work, but specifying our handler, and
		 * our shadow handle as argument
		 */
		retval = save_bus_ops.bus_intr_op(dip, rdip,
		    intr_op, hdlp, result);
		if (retval != DDI_SUCCESS) {
			kmem_free(hp, sizeof (struct bofi_shadow));
			return (retval);
		}
		/*
		 * add to dhash, hhash and inuse lists
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		hp->next = shadow_list.next;
		shadow_list.next->prev = hp;
		hp->prev = &shadow_list;
		shadow_list.next = hp;
		hhashp = HDL_HHASH(hdlp->ih_inum);
		hp->hnext = hhashp->hnext;
		hhashp->hnext->hprev = hp;
		hp->hprev = hhashp;
		hhashp->hnext = hp;
		dhashp = HDL_DHASH(hp->dip);
		hp->dnext = dhashp->dnext;
		dhashp->dnext->dprev = hp;
		hp->dprev = dhashp;
		dhashp->dnext = hp;
		/*
		 * chain on any pre-existing errdefs that apply to this
		 * acc_handle
		 */
		for (ep = errent_listp; ep != NULL; ep = ep->next) {
			if (ddi_name_to_major(hp->name) ==
			    ddi_name_to_major(ep->name) &&
			    hp->instance == ep->errdef.instance &&
			    (ep->errdef.access_type & BOFI_INTR)) {
				lp = bofi_link_freelist;
				if (lp != NULL) {
					bofi_link_freelist = lp->link;
					lp->errentp = ep;
					lp->link = hp->link;
					hp->link = lp;
				}
			}
		}
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		return (retval);
	case DDI_INTROP_REMISR:
		/*
		 * call nexus routine first
		 */
		retval = save_bus_ops.bus_intr_op(dip, rdip,
		    intr_op, hdlp, result);
		/*
		 * find shadow handle
		 */
		mutex_enter(&bofi_low_mutex);
		mutex_enter(&bofi_mutex);
		hhashp = HDL_HHASH(hdlp->ih_inum);
		for (hp = hhashp->hnext; hp != hhashp; hp = hp->hnext) {
			if (hp->dip == rdip &&
			    hp->type == BOFI_INT_HDL &&
			    hp->bofi_inum == hdlp->ih_inum) {
				break;
			}
		}
		if (hp == hhashp) {
			mutex_exit(&bofi_mutex);
			mutex_exit(&bofi_low_mutex);
			return (retval);
		}
		/*
		 * found one - remove from dhash, hhash and inuse lists
		 */
		hp->hnext->hprev = hp->hprev;
		hp->hprev->hnext = hp->hnext;
		hp->dnext->dprev = hp->dprev;
		hp->dprev->dnext = hp->dnext;
		hp->next->prev = hp->prev;
		hp->prev->next = hp->next;
		/*
		 * free any errdef link structures
		 * tagged on to this shadow handle
		 */
		for (lp = hp->link; lp != NULL; ) {
			next_lp = lp->link;
			lp->link = bofi_link_freelist;
			bofi_link_freelist = lp;
			lp = next_lp;
		}
		hp->link = NULL;
		mutex_exit(&bofi_mutex);
		mutex_exit(&bofi_low_mutex);
		kmem_free(hp, sizeof (struct bofi_shadow));
		return (retval);
	default:
		return (save_bus_ops.bus_intr_op(dip, rdip,
		    intr_op, hdlp, result));
	}
}