xref: /titanic_52/usr/src/uts/common/io/scsi/adapters/smrt/smrt_ciss.c (revision bfd83fd302218f780e15314e3e7cf95617cadc29)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright (c) 2017, Joyent, Inc.
 */

#include <sys/scsi/adapters/smrt/smrt.h>

/*
 * Discovery, Resets, Periodics, and Events
 * ----------------------------------------
 *
 * Discovery is the act of figuring out what logical and physical volumes exist
 * under the controller.  Discovery happens in response to the following events:
 *
 *   o iports for virtual and physical devices being attached
 *   o Controller event notifications indicating potential topology changes
 *   o After a reset of the controller, before we can perform I/O again
 *
 * Because we have to perform discovery after a reset, which can happen during
 * panic(), that also means that discovery may be run in panic context.  We
 * also need to emphasize the need for discovery to happen after a controller
 * reset.  Once a reset is initiated, we cannot be certain about the addresses
 * of any of the existing targets until the reset has completed.  The driver
 * performs I/Os to addresses that the controller provides.  The controller
 * specification says that these addresses may change after a controller reset.
 *
 * Unfortunately, all of this combined means that making sure we can correctly
 * run discovery is somewhat complicated.  In non-panic contexts, discovery is
 * always run from a taskq.  We'll kick off the discovery in the taskq if
 * nothing is pending at that time.  The state is managed by bits in the
 * smrt_status member of the smrt_t.  There are four bits at this time:
 *
 *	SMRT_CTLR_DISCOVERY_REQUESTED	This flag indicates that something has
 *					requested that a discovery be performed.
 *					If no flags are set when this is set,
 *					then we will kick off discovery.  All
 *					discovery requests are initiated via the
 *					smrt_discover_request() function.
 *
 *	SMRT_CTLR_DISCOVERY_RUNNING	This flag is set at the start of us
 *					running a discovery.  It is removed when
 *					discovery finishes.
 *
 *	SMRT_CTLR_DISCOVERY_PERIODIC	This flag is set in a number of
 *					circumstances, which will be described
 *					in a subsequent section.  This indicates
 *					that the periodic must kick off the
 *					discovery process.
 *
 *	SMRT_CTLR_DISCOVERY_REQUIRED	This flag indicates that at some point a
 *					controller reset occurred and we need to
 *					have a successful discovery to finish
 *					the act of resetting and allowing I/O to
 *					continue.
 *
 * In general, a request to discover kicks off the taskq to discover entries, if
 * it hasn't already been requested or started.  This also allows us to coalesce
 * multiple requests, if needed.  Note that if a request comes in when a
 * discovery is ongoing, we do not kick off discovery again.  Instead, we set
 * the SMRT_CTLR_DISCOVERY_REQUESTED flag which will rerun discovery after the
 * initial pass has completed.
 *
 * When a discovery starts, the first thing it does is clear the
 * SMRT_CTLR_DISCOVERY_REQUESTED flag.  This is important, because any
 * additional requests for discovery that come in after this has started likely
 * indicate that we've missed something.  As such, when the discovery process
 * finishes, if it sees the REQUESTED flag, then it will need to set the
 * PERIODIC flag.  The PERIODIC flag is used to indicate that we should run
 * discovery again, but not kick if off immediately.  Instead, it should be
 * driven by the normal periodic behavior.
 *
 * If for some reason the act of discovery fails, or we fail to dispatch
 * discovery due to a transient error, then we will flag PERIODIC so that the
 * periodic tick will try and run things again.
 *
 * Now, we need to talk about SMRT_CTLR_DISCOVERY_REQUIRED.  This flag is set
 * after a reset occurs.  The reset thread will be blocked on this.
 * Importantly, none of the code in the discovery path can ask for a controller
 * reset at this time.  If at the end of a discovery, this flag is set, then we
 * will signal the reset thread that it should check on its status by
 * broadcasting on the smrt_cv_finishq.  At that point, the reset thread will
 * continue.
 *
 * Panic Context
 * -------------
 *
 * All of this talk of threads and taskqs is well and good, but as an HBA
 * driver, we have a serious responsibility to try and deal with panic sanely.
 * In panic context, we will directly call the discovery functions and not poll
 * for them to occur.
 *
 * However, because our discovery relies on the target maps, which aren't safe
 * for panic context at this time, we have to take a different approach.  We
 * leverage the fact that we have a generation number stored with every
 * discovery.  If we try to do an I/O to a device where the generation doesn't
 * match, then we know that it disappeared and should not be used.  We also
 * sanity check the model, serial numbers, and WWNs to make sure that these are
 * the same devices.  If they are, then we'll end up updating the address
 * structures.
 *
 * Now, it is possible that when we were panicking, we had a thread that was in
 * the process of running a discovery or even resetting the system.  Once we're
 * in panic, those threads aren't running, so if they didn't end up producing a
 * new view of the world that the SCSI framework is using, then it shouldn't
 * really matter, as we won't have updated the list of devices.  Importantly,
 * once we're in that context, we're not going to be attaching or detaching
 * targets.  If we get a request for one of these targets which has disappeared,
 * we're going to have to end up giving up.
 *
 * Request Attributes
 * ------------------
 *
 * The CISS specification allows for three different kinds of attributes that
 * describe how requests are queued to the controller.  These are:
 *
 * 	HEAD OF QUEUE		The request should go to the head of the
 * 				controller queue.  This is used for resets and
 * 				aborts to ensure that they're not blocked behind
 * 				additional I/O.
 *
 * 	SIMPLE			This queues the request for normal processing.
 * 				Commands queued this way are not special with
 * 				respect to one another.  We use this for all I/O
 * 				and discovery commands.
 *
 *	ORDERED			This attribute is used to indicate that commands
 *				should be submitted and processed in some order.
 *				This is used primarily for the event
 *				notification bits so we can ensure that at the
 *				return of a cancellation of the event
 *				notification, that any outstanding request has
 *				been honored.
 */

static int smrt_ctlr_versions(smrt_t *, uint16_t, smrt_versions_t *);
static void smrt_discover(void *);

/*
 * The maximum number of seconds to wait for the controller to come online.
 */
unsigned smrt_ciss_init_time = 90;

/*
 * A tunable that determines the number of events per tick that we'll process
 * via asynchronous event notification.  If this rate is very high, then we will
 * not submit the event and it will be picked up at the next tick of the
 * periodic.
 */
uint_t smrt_event_intervention_threshold = 1000;

/*
 * Converts a LUN Address to a BMIC Identifier.  The BMIC Identifier is used
 * when performing various physical commands and generally should stay the same
 * for a given device across inserts and removals; however, not across
 * controller resets.  These are calculated based on what the CISS specification
 * calls the 'Level 2' target and bus, which don't have a real meaning in the
 * SAS world otherwise.
 */
uint16_t
smrt_lun_addr_to_bmic(PhysDevAddr_t *paddr)
{
	uint16_t id;

	id = (paddr->Target[1].PeripDev.Bus - 1) << 8;
	id += paddr->Target[1].PeripDev.Dev;

	return (id);
}

void
smrt_write_lun_addr_phys(LUNAddr_t *lun, boolean_t masked, unsigned bus,
    unsigned target)
{
	lun->PhysDev.Mode = masked ? MASK_PERIPHERIAL_DEV_ADDR :
	    PERIPHERIAL_DEV_ADDR;

	lun->PhysDev.TargetId = target;
	lun->PhysDev.Bus = bus;

	bzero(&lun->PhysDev.Target, sizeof (lun->PhysDev.Target));
}

/*
 * According to the CISS Specification, the controller is always addressed in
 * Mask Perhiperhal mode with a bus and target ID of zero.  This is used by
 * commands that need to write to the controller itself, which is generally
 * discovery and other commands.
 */
void
smrt_write_controller_lun_addr(LUNAddr_t *lun)
{
	smrt_write_lun_addr_phys(lun, B_TRUE, 0, 0);
}

void
smrt_write_message_common(smrt_command_t *smcm, uint8_t type, int timeout_secs)
{
	switch (type) {
	case CISS_MSG_ABORT:
	case CISS_MSG_RESET:
	case CISS_MSG_NOP:
		break;

	default:
		panic("unknown message type");
	}

	smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_MSG;
	smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_HEADOFQUEUE;
	smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_NONE;
	smcm->smcm_va_cmd->Request.Timeout = LE_16(timeout_secs);
	smcm->smcm_va_cmd->Request.CDBLen = CISS_CDBLEN;
	smcm->smcm_va_cmd->Request.CDB[0] = type;
}

void
smrt_write_message_abort_one(smrt_command_t *smcm, uint32_t tag)
{
	smrt_tag_t cisstag;

	/*
	 * When aborting a particular command, the request is addressed
	 * to the controller.
	 */
	smrt_write_lun_addr_phys(&smcm->smcm_va_cmd->Header.LUN,
	    B_TRUE, 0, 0);

	smrt_write_message_common(smcm, CISS_MSG_ABORT, 0);

	/*
	 * Abort a single command.
	 */
	smcm->smcm_va_cmd->Request.CDB[1] = CISS_ABORT_TASK;

	/*
	 * The CISS Specification says that the tag value for a task-level
	 * abort should be in the CDB in bytes 4-11.
	 */
	bzero(&cisstag, sizeof (cisstag));
	cisstag.tag_value = tag;
	bcopy(&cisstag, &smcm->smcm_va_cmd->Request.CDB[4],
	    sizeof (cisstag));
}

void
smrt_write_message_abort_all(smrt_command_t *smcm, LUNAddr_t *addr)
{
	/*
	 * When aborting all tasks for a particular Logical Volume,
	 * the command is addressed not to the controller but to
	 * the Volume itself.
	 */
	smcm->smcm_va_cmd->Header.LUN = *addr;

	smrt_write_message_common(smcm, CISS_MSG_ABORT, 0);

	/*
	 * Abort all commands for a particular Logical Volume.
	 */
	smcm->smcm_va_cmd->Request.CDB[1] = CISS_ABORT_TASKSET;
}

void
smrt_write_message_event_notify(smrt_command_t *smcm)
{
	smrt_event_notify_req_t senr;

	smrt_write_controller_lun_addr(&smcm->smcm_va_cmd->Header.LUN);

	smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_CMD;
	smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_ORDERED;
	smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_READ;
	smcm->smcm_va_cmd->Request.Timeout = 0;
	smcm->smcm_va_cmd->Request.CDBLen = sizeof (senr);

	bzero(&senr, sizeof (senr));
	senr.senr_opcode = CISS_SCMD_READ;
	senr.senr_subcode = CISS_BMIC_NOTIFY_ON_EVENT;
	senr.senr_flags = BE_32(0);
	senr.senr_size = BE_32(SMRT_EVENT_NOTIFY_BUFLEN);

	bcopy(&senr, &smcm->smcm_va_cmd->Request.CDB[0],
	    MIN(CISS_CDBLEN, sizeof (senr)));
}

void
smrt_write_message_cancel_event_notify(smrt_command_t *smcm)
{
	smrt_event_notify_req_t senr;

	smrt_write_controller_lun_addr(&smcm->smcm_va_cmd->Header.LUN);

	smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_CMD;
	smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_ORDERED;
	smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_WRITE;
	smcm->smcm_va_cmd->Request.Timeout = LE_16(SMRT_ASYNC_CANCEL_TIMEOUT);
	smcm->smcm_va_cmd->Request.CDBLen = sizeof (senr);

	bzero(&senr, sizeof (senr));
	senr.senr_opcode = CISS_SCMD_WRITE;
	senr.senr_subcode = CISS_BMIC_NOTIFY_ON_EVENT_CANCEL;
	senr.senr_size = BE_32(SMRT_EVENT_NOTIFY_BUFLEN);

	bcopy(&senr, &smcm->smcm_va_cmd->Request.CDB[0],
	    MIN(CISS_CDBLEN, sizeof (senr)));
}

void
smrt_write_message_reset_ctlr(smrt_command_t *smcm)
{
	smrt_write_lun_addr_phys(&smcm->smcm_va_cmd->Header.LUN,
	    B_TRUE, 0, 0);

	smrt_write_message_common(smcm, CISS_MSG_RESET, 0);

	smcm->smcm_va_cmd->Request.CDB[1] = CISS_RESET_CTLR;
}

void
smrt_write_message_nop(smrt_command_t *smcm, int timeout_secs)
{
	/*
	 * No-op messages are always sent to the controller.
	 */
	smrt_write_lun_addr_phys(&smcm->smcm_va_cmd->Header.LUN,
	    B_TRUE, 0, 0);

	smrt_write_message_common(smcm, CISS_MSG_NOP, timeout_secs);
}

/*
 * This routine is executed regularly by ddi_periodic_add(9F).  It checks the
 * health of the controller and looks for submitted commands that have timed
 * out.
 */
void
smrt_periodic(void *arg)
{
	smrt_t *smrt = arg;

	mutex_enter(&smrt->smrt_mutex);

	/*
	 * Before we even check if the controller is running to process
	 * everything else, we must first check if we had a request to kick off
	 * discovery.  We do this before the check if the controller is running,
	 * as this may be required to finish a discovery.
	 */
	if ((smrt->smrt_status & SMRT_CTLR_DISCOVERY_PERIODIC) != 0 &&
	    (smrt->smrt_status & SMRT_CTLR_DISCOVERY_RUNNING) == 0 &&
	    (smrt->smrt_status & SMRT_CTLR_STATUS_RESETTING) == 0) {
		if (ddi_taskq_dispatch(smrt->smrt_discover_taskq,
		    smrt_discover, smrt, DDI_NOSLEEP) != DDI_SUCCESS) {
			smrt->smrt_stats.smrts_discovery_tq_errors++;
		} else {
			smrt->smrt_status &= ~SMRT_CTLR_DISCOVERY_PERIODIC;
		}
	}

	if (!(smrt->smrt_status & SMRT_CTLR_STATUS_RUNNING)) {
		/*
		 * The device is currently not active, e.g. due to an
		 * in-progress controller reset.
		 */
		mutex_exit(&smrt->smrt_mutex);
		return;
	}

	/*
	 * Check on the health of the controller firmware.  Note that if the
	 * controller has locked up, this routine will panic the system.
	 */
	smrt_lockup_check(smrt);

	/*
	 * Reset the event notification threshold counter.
	 */
	smrt->smrt_event_count = 0;

	/*
	 * Check inflight commands to see if they have timed out.
	 */
	for (smrt_command_t *smcm = avl_first(&smrt->smrt_inflight);
	    smcm != NULL; smcm = AVL_NEXT(&smrt->smrt_inflight, smcm)) {
		if (smcm->smcm_status & SMRT_CMD_STATUS_POLLED) {
			/*
			 * Polled commands are timed out by the polling
			 * routine.
			 */
			continue;
		}

		if (smcm->smcm_status & SMRT_CMD_STATUS_ABORT_SENT) {
			/*
			 * This command has been aborted; either it will
			 * complete or the controller will be reset.
			 */
			continue;
		}

		if (list_link_active(&smcm->smcm_link_abort)) {
			/*
			 * Already on the abort queue.
			 */
			continue;
		}

		if (smcm->smcm_expiry == 0) {
			/*
			 * This command has no expiry time.
			 */
			continue;
		}

		if (gethrtime() > smcm->smcm_expiry) {
			list_insert_tail(&smrt->smrt_abortq, smcm);
			smcm->smcm_status |= SMRT_CMD_STATUS_TIMEOUT;
		}
	}

	/*
	 * Process the abort queue.
	 */
	(void) smrt_process_abortq(smrt);

	/*
	 * Check if we have an outstanding event intervention request.  Note,
	 * the command in question should always be in a state such that it is
	 * usable by the system here.  The command is always prepared again by
	 * the normal event notification path, even if a reset has occurred.
	 * The reset will be processed before we'd ever consider running an
	 * event again.  Note, if we fail to submit this, then we leave this for
	 * the next occurrence of the periodic.
	 */
	if (smrt->smrt_status & SMRT_CTLR_ASYNC_INTERVENTION) {
		smrt->smrt_stats.smrts_events_intervened++;

		if (smrt_submit(smrt, smrt->smrt_event_cmd) == 0) {
			smrt->smrt_status &= ~SMRT_CTLR_ASYNC_INTERVENTION;
		}
	}

	mutex_exit(&smrt->smrt_mutex);
}

int
smrt_retrieve(smrt_t *smrt)
{
	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	switch (smrt->smrt_ctlr_mode) {
	case SMRT_CTLR_MODE_SIMPLE:
		smrt_retrieve_simple(smrt);
		return (DDI_SUCCESS);

	case SMRT_CTLR_MODE_UNKNOWN:
		break;
	}

	panic("unknown controller mode");
	/* LINTED: E_FUNC_NO_RET_VAL */
}

/*
 * Grab a new tag number for this command.  We aim to avoid reusing tag numbers
 * as much as possible, so as to avoid spurious double completion from the
 * controller.
 */
static void
smrt_set_new_tag(smrt_t *smrt, smrt_command_t *smcm)
{
	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	/*
	 * Loop until we find a tag that is not in use.  The tag space is
	 * very large (~30 bits) and the maximum number of inflight commands
	 * is comparatively small (~1024 in current controllers).
	 */
	for (;;) {
		uint32_t new_tag = smrt->smrt_next_tag;

		if (++smrt->smrt_next_tag > SMRT_MAX_TAG_NUMBER) {
			smrt->smrt_next_tag = SMRT_MIN_TAG_NUMBER;
		}

		if (smrt_lookup_inflight(smrt, new_tag) != NULL) {
			/*
			 * This tag is already used on an inflight command.
			 * Choose another.
			 */
			continue;
		}

		/*
		 * Set the tag for the command and also write it into the
		 * appropriate part of the request block.
		 */
		smcm->smcm_tag = new_tag;
		smcm->smcm_va_cmd->Header.Tag.tag_value = new_tag;
		return;
	}
}

/*
 * Submit a command to the controller.
 */
int
smrt_submit(smrt_t *smrt, smrt_command_t *smcm)
{
	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));
	VERIFY(smcm->smcm_type != SMRT_CMDTYPE_PREINIT);

	/*
	 * Anything that asks us to ignore the running state of the controller
	 * must be wired up to poll for completion.
	 */
	if (smcm->smcm_status & SMRT_CMD_IGNORE_RUNNING) {
		VERIFY(smcm->smcm_status & SMRT_CMD_STATUS_POLLED);
	}

	/*
	 * If the controller is currently being reset, do not allow command
	 * submission.  However, if this is one of the commands needed to finish
	 * reset, as indicated on the command structure, allow it.
	 */
	if (!(smrt->smrt_status & SMRT_CTLR_STATUS_RUNNING) &&
	    !(smcm->smcm_status & SMRT_CMD_IGNORE_RUNNING)) {
		return (EIO);
	}

	/*
	 * Do not allow submission of more concurrent commands than the
	 * controller supports.
	 */
	if (avl_numnodes(&smrt->smrt_inflight) >= smrt->smrt_maxcmds) {
		return (EAGAIN);
	}

	/*
	 * Synchronise the Command Block DMA resources to ensure that the
	 * device has a consistent view before we pass it the command.
	 */
	if (ddi_dma_sync(smcm->smcm_contig.smdma_dma_handle, 0, 0,
	    DDI_DMA_SYNC_FORDEV) != DDI_SUCCESS) {
		dev_err(smrt->smrt_dip, CE_PANIC, "DMA sync failure");
		return (EIO);
	}

	/*
	 * Ensure that this command is not re-used without issuing a new
	 * tag number and performing any appropriate cleanup.
	 */
	VERIFY(!(smcm->smcm_status & SMRT_CMD_STATUS_USED));
	smcm->smcm_status |= SMRT_CMD_STATUS_USED;

	/*
	 * Assign a tag that is not currently in use
	 */
	smrt_set_new_tag(smrt, smcm);

	/*
	 * Insert this command into the inflight AVL.
	 */
	avl_index_t where;
	if (avl_find(&smrt->smrt_inflight, smcm, &where) != NULL) {
		dev_err(smrt->smrt_dip, CE_PANIC, "duplicate submit tag %x",
		    smcm->smcm_tag);
	}
	avl_insert(&smrt->smrt_inflight, smcm, where);
	if (smrt->smrt_stats.smrts_max_inflight <
	    avl_numnodes(&smrt->smrt_inflight)) {
		smrt->smrt_stats.smrts_max_inflight =
		    avl_numnodes(&smrt->smrt_inflight);
	}

	VERIFY(!(smcm->smcm_status & SMRT_CMD_STATUS_INFLIGHT));
	smcm->smcm_status |= SMRT_CMD_STATUS_INFLIGHT;

	smcm->smcm_time_submit = gethrtime();

	switch (smrt->smrt_ctlr_mode) {
	case SMRT_CTLR_MODE_SIMPLE:
		smrt_submit_simple(smrt, smcm);
		return (0);

	case SMRT_CTLR_MODE_UNKNOWN:
		break;
	}
	panic("unknown controller mode");
	/* LINTED: E_FUNC_NO_RET_VAL */
}

static void
smrt_process_finishq_sync(smrt_command_t *smcm)
{
	smrt_t *smrt = smcm->smcm_ctlr;

	if (ddi_dma_sync(smcm->smcm_contig.smdma_dma_handle, 0, 0,
	    DDI_DMA_SYNC_FORCPU) != DDI_SUCCESS) {
		dev_err(smrt->smrt_dip, CE_PANIC, "finishq DMA sync failure");
	}
}

static void
smrt_process_finishq_one(smrt_command_t *smcm)
{
	smrt_t *smrt = smcm->smcm_ctlr;

	VERIFY(!(smcm->smcm_status & SMRT_CMD_STATUS_COMPLETE));
	smcm->smcm_status |= SMRT_CMD_STATUS_COMPLETE;

	switch (smcm->smcm_type) {
	case SMRT_CMDTYPE_INTERNAL:
		cv_broadcast(&smcm->smcm_ctlr->smrt_cv_finishq);
		return;

	case SMRT_CMDTYPE_SCSA:
		smrt_hba_complete(smcm);
		return;

	case SMRT_CMDTYPE_EVENT:
		smrt_event_complete(smcm);
		return;

	case SMRT_CMDTYPE_ABORTQ:
		/*
		 * Abort messages sent as part of abort queue processing
		 * do not require any completion activity.
		 */
		mutex_exit(&smrt->smrt_mutex);
		smrt_command_free(smcm);
		mutex_enter(&smrt->smrt_mutex);
		return;

	case SMRT_CMDTYPE_PREINIT:
		dev_err(smrt->smrt_dip, CE_PANIC, "preinit command "
		    "completed after initialisation");
		return;
	}

	panic("unknown command type");
}

/*
 * Process commands in the completion queue.
 */
void
smrt_process_finishq(smrt_t *smrt)
{
	smrt_command_t *smcm;

	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	while ((smcm = list_remove_head(&smrt->smrt_finishq)) != NULL) {
		/*
		 * Synchronise the Command Block before we read from it or
		 * free it, to ensure that any writes from the controller are
		 * visible.
		 */
		smrt_process_finishq_sync(smcm);

		/*
		 * Check if this command was in line to be aborted.
		 */
		if (list_link_active(&smcm->smcm_link_abort)) {
			/*
			 * This command was in line, but the controller
			 * subsequently completed the command before we
			 * were able to do so.
			 */
			list_remove(&smrt->smrt_abortq, smcm);
			smcm->smcm_status &= ~SMRT_CMD_STATUS_TIMEOUT;
		}

		/*
		 * Check if this command has been abandoned by the original
		 * submitter.  If it has, free it now to avoid a leak.
		 */
		if (smcm->smcm_status & SMRT_CMD_STATUS_ABANDONED) {
			mutex_exit(&smrt->smrt_mutex);
			smrt_command_free(smcm);
			mutex_enter(&smrt->smrt_mutex);
			continue;
		}

		if (smcm->smcm_status & SMRT_CMD_STATUS_POLLED) {
			/*
			 * This command will be picked up and processed
			 * by "smrt_poll_for()" once the CV is triggered
			 * at the end of processing.
			 */
			smcm->smcm_status |= SMRT_CMD_STATUS_POLL_COMPLETE;
			continue;
		}

		smrt_process_finishq_one(smcm);
	}

	cv_broadcast(&smrt->smrt_cv_finishq);
}

/*
 * Process commands in the abort queue.
 */
void
smrt_process_abortq(smrt_t *smrt)
{
	smrt_command_t *smcm;
	smrt_command_t *abort_smcm = NULL;

	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	if (list_is_empty(&smrt->smrt_abortq)) {
		goto out;
	}

another:
	mutex_exit(&smrt->smrt_mutex);
	if ((abort_smcm = smrt_command_alloc(smrt, SMRT_CMDTYPE_ABORTQ,
	    KM_NOSLEEP)) == NULL) {
		/*
		 * No resources available to send abort messages.  We will
		 * try again the next time around.
		 */
		mutex_enter(&smrt->smrt_mutex);
		goto out;
	}
	mutex_enter(&smrt->smrt_mutex);

	while ((smcm = list_remove_head(&smrt->smrt_abortq)) != NULL) {
		if (!(smcm->smcm_status & SMRT_CMD_STATUS_INFLIGHT)) {
			/*
			 * This message is not currently inflight, so
			 * no abort is needed.
			 */
			continue;
		}

		if (smcm->smcm_status & SMRT_CMD_STATUS_ABORT_SENT) {
			/*
			 * An abort message has already been sent for
			 * this command.
			 */
			continue;
		}

		/*
		 * Send an abort message for the command.
		 */
		smrt_write_message_abort_one(abort_smcm, smcm->smcm_tag);
		if (smrt_submit(smrt, abort_smcm) != 0) {
			/*
			 * The command could not be submitted to the
			 * controller.  Put it back in the abort queue
			 * and give up for now.
			 */
			list_insert_head(&smrt->smrt_abortq, smcm);
			goto out;
		}
		smcm->smcm_status |= SMRT_CMD_STATUS_ABORT_SENT;

		/*
		 * Record some debugging information about the abort we
		 * sent:
		 */
		smcm->smcm_abort_time = gethrtime();
		smcm->smcm_abort_tag = abort_smcm->smcm_tag;

		/*
		 * The abort message was sent.  Release it and
		 * allocate another command.
		 */
		abort_smcm = NULL;
		goto another;
	}

out:
	cv_broadcast(&smrt->smrt_cv_finishq);
	if (abort_smcm != NULL) {
		mutex_exit(&smrt->smrt_mutex);
		smrt_command_free(abort_smcm);
		mutex_enter(&smrt->smrt_mutex);
	}
}

int
smrt_poll_for(smrt_t *smrt, smrt_command_t *smcm)
{
	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));
	VERIFY(smcm->smcm_status & SMRT_CMD_STATUS_POLLED);

	while (!(smcm->smcm_status & SMRT_CMD_STATUS_POLL_COMPLETE)) {
		if (smcm->smcm_expiry != 0) {
			/*
			 * This command has an expiry time.  Check to see
			 * if it has already passed:
			 */
			if (smcm->smcm_expiry < gethrtime()) {
				return (ETIMEDOUT);
			}
		}

		if (ddi_in_panic()) {
			/*
			 * When the system is panicking, there are no
			 * interrupts or other threads.  Drive the polling loop
			 * on our own, but with a small delay to avoid
			 * aggrevating the controller while we're trying to
			 * dump.
			 */
			(void) smrt_retrieve(smrt);
			smrt_process_finishq(smrt);
			drv_usecwait(100);
			continue;
		}

		/*
		 * Wait for command completion to return through the regular
		 * interrupt handling path.
		 */
		if (smcm->smcm_expiry == 0) {
			cv_wait(&smrt->smrt_cv_finishq, &smrt->smrt_mutex);
		} else {
			/*
			 * Wait only until the expiry time for this command.
			 */
			(void) cv_timedwait_sig_hrtime(&smrt->smrt_cv_finishq,
			    &smrt->smrt_mutex, smcm->smcm_expiry);
		}
	}

	/*
	 * Fire the completion callback for this command.  The callback
	 * is responsible for freeing the command, so it may not be
	 * referenced again once this call returns.
	 */
	smrt_process_finishq_one(smcm);

	return (0);
}

void
smrt_intr_set(smrt_t *smrt, boolean_t enabled)
{
	/*
	 * Read the Interrupt Mask Register.
	 */
	uint32_t imr = smrt_get32(smrt, CISS_I2O_INTERRUPT_MASK);

	switch (smrt->smrt_ctlr_mode) {
	case SMRT_CTLR_MODE_SIMPLE:
		if (enabled) {
			imr &= ~CISS_IMR_BIT_SIMPLE_INTR_DISABLE;
		} else {
			imr |= CISS_IMR_BIT_SIMPLE_INTR_DISABLE;
		}
		smrt_put32(smrt, CISS_I2O_INTERRUPT_MASK, imr);
		return;

	case SMRT_CTLR_MODE_UNKNOWN:
		break;
	}
	panic("unknown controller mode");
}

/*
 * Signal to the controller that we have updated the Configuration Table by
 * writing to the Inbound Doorbell Register.  The controller will, after some
 * number of seconds, acknowledge this by clearing the bit.
 *
 * If successful, return DDI_SUCCESS.  If the controller takes too long to
 * acknowledge, return DDI_FAILURE.
 */
int
smrt_cfgtbl_flush(smrt_t *smrt)
{
	/*
	 * Read the current value of the Inbound Doorbell Register.
	 */
	uint32_t idr = smrt_get32(smrt, CISS_I2O_INBOUND_DOORBELL);

	/*
	 * Signal the Configuration Table change to the controller.
	 */
	idr |= CISS_IDR_BIT_CFGTBL_CHANGE;
	smrt_put32(smrt, CISS_I2O_INBOUND_DOORBELL, idr);

	/*
	 * Wait for the controller to acknowledge the change.
	 */
	for (unsigned i = 0; i < smrt_ciss_init_time; i++) {
		idr = smrt_get32(smrt, CISS_I2O_INBOUND_DOORBELL);

		if ((idr & CISS_IDR_BIT_CFGTBL_CHANGE) == 0) {
			return (DDI_SUCCESS);
		}

		/*
		 * Wait for one second before trying again.
		 */
		delay(drv_usectohz(1000000));
	}

	dev_err(smrt->smrt_dip, CE_WARN, "time out expired before controller "
	    "configuration completed");
	return (DDI_FAILURE);
}

int
smrt_cfgtbl_transport_has_support(smrt_t *smrt, int xport)
{
	VERIFY(xport == CISS_CFGTBL_XPORT_SIMPLE);

	/*
	 * Read the current value of the "Supported Transport Methods" field in
	 * the Configuration Table.
	 */
	uint32_t xport_active = ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->TransportSupport);

	/*
	 * Check that the desired transport method is supported by the
	 * controller:
	 */
	if ((xport_active & xport) == 0) {
		dev_err(smrt->smrt_dip, CE_WARN, "controller does not support "
		    "method \"%s\"", xport == CISS_CFGTBL_XPORT_SIMPLE ?
		    "simple" : "performant");
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

void
smrt_cfgtbl_transport_set(smrt_t *smrt, int xport)
{
	VERIFY(xport == CISS_CFGTBL_XPORT_SIMPLE);

	ddi_put32(smrt->smrt_ct_handle, &smrt->smrt_ct->TransportRequest,
	    xport);
}

int
smrt_cfgtbl_transport_confirm(smrt_t *smrt, int xport)
{
	VERIFY(xport == CISS_CFGTBL_XPORT_SIMPLE);

	/*
	 * Read the current value of the TransportActive field in the
	 * Configuration Table.
	 */
	uint32_t xport_active = ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->TransportActive);

	/*
	 * Check that the desired transport method is now active:
	 */
	if ((xport_active & xport) == 0) {
		dev_err(smrt->smrt_dip, CE_WARN, "failed to enable transport "
		    "method \"%s\"", xport == CISS_CFGTBL_XPORT_SIMPLE ?
		    "simple" : "performant");
		return (DDI_FAILURE);
	}

	/*
	 * Ensure that the controller is now ready to accept commands.
	 */
	if ((xport_active & CISS_CFGTBL_READY_FOR_COMMANDS) == 0) {
		dev_err(smrt->smrt_dip, CE_WARN, "controller not ready to "
		    "accept commands");
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

uint32_t
smrt_ctlr_get_maxsgelements(smrt_t *smrt)
{
	return (ddi_get32(smrt->smrt_ct_handle, &smrt->smrt_ct->MaxSGElements));
}

uint32_t
smrt_ctlr_get_cmdsoutmax(smrt_t *smrt)
{
	return (ddi_get32(smrt->smrt_ct_handle, &smrt->smrt_ct->CmdsOutMax));
}

static uint32_t
smrt_ctlr_get_hostdrvsup(smrt_t *smrt)
{
	return (ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->HostDrvrSupport));
}

int
smrt_ctlr_init(smrt_t *smrt)
{
	uint8_t signature[4] = { 'C', 'I', 'S', 'S' };
	int e;

	if ((e = smrt_ctlr_wait_for_state(smrt,
	    SMRT_WAIT_STATE_READY)) != DDI_SUCCESS) {
		return (e);
	}

	/*
	 * The configuration table contains an ASCII signature ("CISS") which
	 * should be checked as we initialise the controller.
	 * See: "9.1 Configuration Table" in CISS Specification.
	 */
	for (unsigned i = 0; i < 4; i++) {
		if (ddi_get8(smrt->smrt_ct_handle,
		    &smrt->smrt_ct->Signature[i]) != signature[i]) {
			dev_err(smrt->smrt_dip, CE_WARN, "invalid signature "
			    "detected");
			return (DDI_FAILURE);
		}
	}

	/*
	 * Initialise an appropriate Transport Method.  For now, this driver
	 * only supports the "Simple" method.
	 */
	if ((e = smrt_ctlr_init_simple(smrt)) != DDI_SUCCESS) {
		return (e);
	}

	/*
	 * Save some common feature support bitfields.
	 */
	smrt->smrt_host_support = smrt_ctlr_get_hostdrvsup(smrt);
	smrt->smrt_bus_support = ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->BusTypes);

	/*
	 * Read initial controller heartbeat value and mark the current
	 * reading time.
	 */
	smrt->smrt_last_heartbeat = ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->HeartBeat);
	smrt->smrt_last_heartbeat_time = gethrtime();

	/*
	 * Determine the firmware version of the controller so that we can
	 * select which type of interrupts to use.
	 */
	if ((e = smrt_ctlr_versions(smrt, SMRT_DISCOVER_TIMEOUT,
	    &smrt->smrt_versions)) != 0) {
		dev_err(smrt->smrt_dip, CE_WARN, "could not identify "
		    "controller (%d)", e);
		return (DDI_FAILURE);
	}

	dev_err(smrt->smrt_dip, CE_NOTE, "!firmware rev %s",
	    smrt->smrt_versions.smrtv_firmware_rev);

	return (DDI_SUCCESS);
}

void
smrt_ctlr_teardown(smrt_t *smrt)
{
	smrt->smrt_status &= ~SMRT_CTLR_STATUS_RUNNING;

	switch (smrt->smrt_ctlr_mode) {
	case SMRT_CTLR_MODE_SIMPLE:
		smrt_ctlr_teardown_simple(smrt);
		return;

	case SMRT_CTLR_MODE_UNKNOWN:
		return;
	}

	panic("unknown controller mode");
}

int
smrt_ctlr_wait_for_state(smrt_t *smrt, smrt_wait_state_t state)
{
	unsigned wait_usec = 100 * 1000;
	unsigned wait_count = SMRT_WAIT_DELAY_SECONDS * 1000000 / wait_usec;

	VERIFY(state == SMRT_WAIT_STATE_READY ||
	    state == SMRT_WAIT_STATE_UNREADY);

	/*
	 * Read from the Scratchpad Register until the expected ready signature
	 * is detected.  This behaviour is not described in the CISS
	 * specification.
	 *
	 * If the device is not in the desired state immediately, sleep for a
	 * second and try again.  If the device has not become ready in 300
	 * seconds, give up.
	 */
	for (unsigned i = 0; i < wait_count; i++) {
		uint32_t spr = smrt_get32(smrt, CISS_I2O_SCRATCHPAD);

		switch (state) {
		case SMRT_WAIT_STATE_READY:
			if (spr == CISS_SCRATCHPAD_INITIALISED) {
				return (DDI_SUCCESS);
			}
			break;

		case SMRT_WAIT_STATE_UNREADY:
			if (spr != CISS_SCRATCHPAD_INITIALISED) {
				return (DDI_SUCCESS);
			}
			break;
		}

		if (ddi_in_panic()) {
			/*
			 * There is no sleep for the panicking, so we
			 * must spin wait:
			 */
			drv_usecwait(wait_usec);
		} else {
			/*
			 * Wait for a quarter second and try again.
			 */
			delay(drv_usectohz(wait_usec));
		}
	}

	dev_err(smrt->smrt_dip, CE_WARN, "time out waiting for controller "
	    "to enter state \"%s\"", state == SMRT_WAIT_STATE_READY ?
	    "ready": "unready");
	return (DDI_FAILURE);
}

void
smrt_lockup_check(smrt_t *smrt)
{
	/*
	 * Read the current controller heartbeat value.
	 */
	uint32_t heartbeat = ddi_get32(smrt->smrt_ct_handle,
	    &smrt->smrt_ct->HeartBeat);

	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	/*
	 * Check to see if the value is the same as last time we looked:
	 */
	if (heartbeat != smrt->smrt_last_heartbeat) {
		/*
		 * The heartbeat value has changed, which suggests that the
		 * firmware in the controller has not yet come to a complete
		 * stop.  Record the new value, as well as the current time.
		 */
		smrt->smrt_last_heartbeat = heartbeat;
		smrt->smrt_last_heartbeat_time = gethrtime();
		return;
	}

	/*
	 * The controller _might_ have been able to signal to us that is
	 * has locked up.  This is a truly unfathomable state of affairs:
	 * If the firmware can tell it has flown off the rails, why not
	 * simply reset the controller?
	 */
	uint32_t odr = smrt_get32(smrt, CISS_I2O_OUTBOUND_DOORBELL_STATUS);
	uint32_t spr = smrt_get32(smrt, CISS_I2O_SCRATCHPAD);
	if ((odr & CISS_ODR_BIT_LOCKUP) != 0) {
		dev_err(smrt->smrt_dip, CE_PANIC, "HP SmartArray firmware has "
		    "reported a critical fault (odr %08x spr %08x)",
		    odr, spr);
	}

	if (gethrtime() > smrt->smrt_last_heartbeat_time + 60 * NANOSEC) {
		dev_err(smrt->smrt_dip, CE_PANIC, "HP SmartArray firmware has "
		    "stopped responding (odr %08x spr %08x)",
		    odr, spr);
	}
}

/*
 * Probe the controller with the IDENTIFY CONTROLLER request.  This is a BMIC
 * command, so it must be submitted to the controller and we must poll for its
 * completion.  This functionality is only presently used during controller
 * initialisation, so it uses the special pre-initialisation path for command
 * allocation and submission.
 */
static int
smrt_ctlr_identify(smrt_t *smrt, uint16_t timeout,
    smrt_identify_controller_t *resp)
{
	smrt_command_t *smcm;
	smrt_identify_controller_req_t smicr;
	int r;
	size_t sz;

	/*
	 * Allocate a command with a data buffer; the controller will fill it
	 * with identification information.  There is some suggestion in the
	 * firmware-level specification that the buffer length should be a
	 * multiple of 512 bytes for some controllers, so we round up.
	 */
	sz = P2ROUNDUP_TYPED(sizeof (*resp), 512, size_t);
	if ((smcm = smrt_command_alloc_preinit(smrt, sz, KM_SLEEP)) == NULL) {
		return (ENOMEM);
	}

	smrt_write_controller_lun_addr(&smcm->smcm_va_cmd->Header.LUN);

	smcm->smcm_va_cmd->Request.CDBLen = sizeof (smicr);
	smcm->smcm_va_cmd->Request.Timeout = timeout;
	smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_CMD;
	smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_SIMPLE;
	smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_READ;

	/*
	 * Construct the IDENTIFY CONTROLLER request CDB.  Note that any
	 * reserved fields in the request must be filled with zeroes.
	 */
	bzero(&smicr, sizeof (smicr));
	smicr.smicr_opcode = CISS_SCMD_BMIC_READ;
	smicr.smicr_lun = 0;
	smicr.smicr_command = CISS_BMIC_IDENTIFY_CONTROLLER;
	bcopy(&smicr, &smcm->smcm_va_cmd->Request.CDB[0],
	    MIN(CISS_CDBLEN, sizeof (smicr)));

	/*
	 * Send the command to the device and poll for its completion.
	 */
	smcm->smcm_status |= SMRT_CMD_STATUS_POLLED;
	smcm->smcm_expiry = gethrtime() + timeout * NANOSEC;
	if ((r = smrt_preinit_command_simple(smrt, smcm)) != 0) {
		VERIFY3S(r, ==, ETIMEDOUT);
		VERIFY0(smcm->smcm_status & SMRT_CMD_STATUS_POLL_COMPLETE);

		/*
		 * This command timed out, but the driver is not presently
		 * initialised to the point where we can try to abort it.
		 * The command was created with the PREINIT type, so it
		 * does not appear in the global command tracking list.
		 * In order to avoid problems with DMA from the controller,
		 * we have to leak the command allocation.
		 */
		smcm = NULL;
		goto out;
	}

	if (smcm->smcm_status & SMRT_CMD_STATUS_RESET_SENT) {
		/*
		 * The controller was reset while we were trying to identify
		 * it.  Report failure.
		 */
		r = EIO;
		goto out;
	}

	if (smcm->smcm_status & SMRT_CMD_STATUS_ERROR) {
		ErrorInfo_t *ei = smcm->smcm_va_err;

		if (ei->CommandStatus != CISS_CMD_DATA_UNDERRUN) {
			dev_err(smrt->smrt_dip, CE_WARN, "identify "
			    "controller error: status 0x%x",
			    ei->CommandStatus);
			r = EIO;
			goto out;
		}
	}

	if (resp != NULL) {
		/*
		 * Copy the identify response out for the caller.
		 */
		bcopy(smcm->smcm_internal->smcmi_va, resp, sizeof (*resp));
	}

	r = 0;

out:
	if (smcm != NULL) {
		smrt_command_free(smcm);
	}
	return (r);
}

/*
 * The firmware versions in an IDENTIFY CONTROLLER response generally take
 * the form of a four byte ASCII string containing a dotted decimal version
 * number; e.g., "8.00".
 *
 * This function sanitises the firmware version, replacing unexpected
 * values with a question mark.
 */
static void
smrt_copy_firmware_version(uint8_t *src, char *dst)
{
	for (unsigned i = 0; i < 4; i++) {
		/*
		 * Make sure that this is a 7-bit clean ASCII value.
		 */
		char c = src[i] <= 0x7f ? (char)(src[i] & 0x7f) : '?';

		if (isalnum(c) || c == '.' || c == ' ') {
			dst[i] = c;
		} else {
			dst[i] = '?';
		}
	}
	dst[4] = '\0';
}

/*
 * Using an IDENTIFY CONTROLLER request, determine firmware and controller
 * version details.  See the comments for "smrt_ctlr_identify()" for more
 * details about calling context.
 */
static int
smrt_ctlr_versions(smrt_t *smrt, uint16_t timeout, smrt_versions_t *smrtv)
{
	smrt_identify_controller_t smic;
	int r;

	if ((r = smrt_ctlr_identify(smrt, timeout, &smic)) != 0) {
		return (r);
	}

	smrtv->smrtv_hardware_version = smic.smic_hardware_version;
	smrt_copy_firmware_version(smic.smic_firmware_rev,
	    smrtv->smrtv_firmware_rev);
	smrt_copy_firmware_version(smic.smic_recovery_rev,
	    smrtv->smrtv_recovery_rev);
	smrt_copy_firmware_version(smic.smic_bootblock_rev,
	    smrtv->smrtv_bootblock_rev);

	return (0);
}

int
smrt_ctlr_reset(smrt_t *smrt)
{
	smrt_command_t *smcm, *smcm_nop;
	int r;

	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));

	if (ddi_in_panic()) {
		goto skip_check;
	}

	if (smrt->smrt_status & SMRT_CTLR_STATUS_RESETTING) {
		/*
		 * Don't pile on.  One reset is enough.  Wait until
		 * it's complete, and then return success.
		 */
		while (smrt->smrt_status & SMRT_CTLR_STATUS_RESETTING) {
			cv_wait(&smrt->smrt_cv_finishq, &smrt->smrt_mutex);
		}
		return (0);
	}
	smrt->smrt_status |= SMRT_CTLR_STATUS_RESETTING;
	smrt->smrt_last_reset_start = gethrtime();
	smrt->smrt_stats.smrts_ctlr_resets++;

skip_check:
	/*
	 * Allocate two commands: one for the soft reset message, which we
	 * cannot free until the controller has reset; and one for the ping we
	 * will use to determine when it is once again functional.
	 */
	mutex_exit(&smrt->smrt_mutex);
	if ((smcm = smrt_command_alloc(smrt, SMRT_CMDTYPE_INTERNAL,
	    KM_NOSLEEP)) == NULL) {
		mutex_enter(&smrt->smrt_mutex);
		return (ENOMEM);
	}
	if ((smcm_nop = smrt_command_alloc(smrt, SMRT_CMDTYPE_INTERNAL,
	    KM_NOSLEEP)) == NULL) {
		smrt_command_free(smcm);
		mutex_enter(&smrt->smrt_mutex);
		return (ENOMEM);
	}
	mutex_enter(&smrt->smrt_mutex);

	/*
	 * Send a soft reset command to the controller.  If this command
	 * succeeds, there will likely be no completion notification.  Instead,
	 * the device should become unavailable for some period of time and
	 * then become available again.  Once available again, we know the soft
	 * reset has completed and should abort all in-flight commands.
	 */
	smrt_write_message_reset_ctlr(smcm);

	/*
	 * Disable interrupts now.
	 */
	smrt_intr_set(smrt, B_FALSE);

	dev_err(smrt->smrt_dip, CE_WARN, "attempting controller soft reset");
	smcm->smcm_status |= SMRT_CMD_STATUS_POLLED;
	if ((r = smrt_submit(smrt, smcm)) != 0) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "submit failed (%d)", r);
	}

	/*
	 * Mark every currently inflight command as being reset, including the
	 * soft reset command we just sent.  Once we confirm the reset works,
	 * we can safely report that these commands have failed.
	 */
	for (smrt_command_t *t = avl_first(&smrt->smrt_inflight);
	    t != NULL; t = AVL_NEXT(&smrt->smrt_inflight, t)) {
		t->smcm_status |= SMRT_CMD_STATUS_RESET_SENT;
	}

	/*
	 * Now that we have submitted our soft reset command, prevent
	 * the rest of the driver from interacting with the controller.
	 */
	smrt->smrt_status &= ~SMRT_CTLR_STATUS_RUNNING;

	/*
	 * We do not expect a completion from the controller for our soft
	 * reset command, but we also cannot remove it from the inflight
	 * list until we know the controller has actually reset.  To do
	 * otherwise would potentially allow the controller to scribble
	 * on the memory we were using.
	 */
	smcm->smcm_status |= SMRT_CMD_STATUS_ABANDONED;

	if (smrt_ctlr_wait_for_state(smrt, SMRT_WAIT_STATE_UNREADY) !=
	    DDI_SUCCESS) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "controller did not become unready");
	}
	dev_err(smrt->smrt_dip, CE_NOTE, "soft reset: controller unready");

	if (smrt_ctlr_wait_for_state(smrt, SMRT_WAIT_STATE_READY) !=
	    DDI_SUCCESS) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "controller did not come become ready");
	}
	dev_err(smrt->smrt_dip, CE_NOTE, "soft reset: controller ready");

	/*
	 * In at least the Smart Array P420i, the controller can take 30-45
	 * seconds after the scratchpad register shows it as being available
	 * before it is ready to receive commands.  In order to avoid hitting
	 * it too early with our post-reset ping, we will sleep for 10 seconds
	 * here.
	 */
	if (ddi_in_panic()) {
		drv_usecwait(10 * MICROSEC);
	} else {
		delay(drv_usectohz(10 * MICROSEC));
	}

	smrt_ctlr_teardown(smrt);
	if (smrt_ctlr_init(smrt) != DDI_SUCCESS) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "controller transport could not be configured");
	}
	dev_err(smrt->smrt_dip, CE_NOTE, "soft reset: controller configured");

	smrt_write_message_nop(smcm_nop, 0);
	smcm_nop->smcm_status |= SMRT_CMD_STATUS_POLLED |
	    SMRT_CMD_IGNORE_RUNNING;
	if ((r = smrt_submit(smrt, smcm_nop)) != 0) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "ping could not be submitted (%d)", r);
	}

	/*
	 * Interrupts are still masked at this stage.  Poll manually in
	 * a way that will not trigger regular finish queue processing:
	 */
	VERIFY(smcm_nop->smcm_status & SMRT_CMD_STATUS_INFLIGHT);
	for (unsigned i = 0; i < 600; i++) {
		smrt_retrieve_simple(smrt);

		if (!(smcm_nop->smcm_status & SMRT_CMD_STATUS_INFLIGHT)) {
			/*
			 * Remove the ping command from the finish queue and
			 * process it manually.  This processing must mirror
			 * what would have been done in smrt_process_finishq().
			 */
			VERIFY(list_link_active(&smcm_nop->smcm_link_finish));
			list_remove(&smrt->smrt_finishq, smcm_nop);
			smrt_process_finishq_sync(smcm_nop);
			smcm_nop->smcm_status |= SMRT_CMD_STATUS_POLL_COMPLETE;
			smrt_process_finishq_one(smcm_nop);
			break;
		}

		if (ddi_in_panic()) {
			drv_usecwait(100 * 1000);
		} else {
			delay(drv_usectohz(100 * 1000));
		}
	}

	if (!(smcm_nop->smcm_status & SMRT_CMD_STATUS_COMPLETE)) {
		dev_err(smrt->smrt_dip, CE_PANIC, "soft reset failed: "
		    "ping did not complete");
	} else if (smcm_nop->smcm_status & SMRT_CMD_STATUS_ERROR) {
		dev_err(smrt->smrt_dip, CE_WARN, "soft reset: ping completed "
		    "in error (status %u)",
		    (unsigned)smcm_nop->smcm_va_err->CommandStatus);
	} else {
		dev_err(smrt->smrt_dip, CE_NOTE, "soft reset: ping completed");
	}

	/*
	 * Now that the controller is working again, we can abort any
	 * commands that were inflight during the reset.
	 */
	smrt_command_t *nt;
	for (smrt_command_t *t = avl_first(&smrt->smrt_inflight);
	    t != NULL; t = nt) {
		nt = AVL_NEXT(&smrt->smrt_inflight, t);

		if (t->smcm_status & SMRT_CMD_STATUS_RESET_SENT) {
			avl_remove(&smrt->smrt_inflight, t);
			t->smcm_status &= ~SMRT_CMD_STATUS_INFLIGHT;

			list_insert_tail(&smrt->smrt_finishq, t);
		}
	}

	/*
	 * Quiesce our discovery thread.  Note, because
	 * SMRT_CTLR_STATUS_RESTARTING is set, nothing can cause it to be
	 * enabled again.
	 */
	if (!ddi_in_panic()) {
		mutex_exit(&smrt->smrt_mutex);
		ddi_taskq_wait(smrt->smrt_discover_taskq);
		mutex_enter(&smrt->smrt_mutex);
	}

	/*
	 * Re-enable interrupts.  Now, we must kick off a discovery to make sure
	 * that the system is in a sane state and that we can perform I/O.
	 */
	smrt_intr_set(smrt, B_TRUE);
	smrt->smrt_status &= ~SMRT_CTLR_STATUS_RESETTING;
	smrt->smrt_status |= SMRT_CTLR_DISCOVERY_REQUIRED;

	/*
	 * Attempt a discovery to make sure that the drivers sees a realistic
	 * view of the world.  If we're not in panic context, spin for the
	 * asynchronous process to complete, otherwise we're in panic context
	 * and this is going to happen regardless if we want it to or not.
	 * Before we kick off the request to run discovery, we reset the
	 * discovery request flags as we know that nothing else can consider
	 * running discovery and we don't want to delay until the next smrt
	 * periodic tick if we can avoid it.  In panic context, if this failed,
	 * then we won't make it back.
	 */
	VERIFY0(smrt->smrt_status & SMRT_CTLR_DISCOVERY_RUNNING);
	smrt->smrt_status &= ~(SMRT_CTLR_DISCOVERY_MASK);
	smrt_discover(smrt);
	if (!ddi_in_panic()) {
		while (smrt->smrt_status & SMRT_CTLR_DISCOVERY_REQUIRED) {
			cv_wait(&smrt->smrt_cv_finishq, &smrt->smrt_mutex);
		}
	}

	smrt->smrt_status |= SMRT_CTLR_STATUS_RUNNING;
	smrt->smrt_last_reset_finish = gethrtime();

	/*
	 * Wake anybody that was waiting for the reset to complete.
	 */
	cv_broadcast(&smrt->smrt_cv_finishq);

	/*
	 * Process the completion queue one last time before we let go
	 * of the mutex.
	 */
	smrt_process_finishq(smrt);

	mutex_exit(&smrt->smrt_mutex);
	smrt_command_free(smcm_nop);
	mutex_enter(&smrt->smrt_mutex);
	return (0);
}

int
smrt_event_init(smrt_t *smrt)
{
	int ret;
	smrt_command_t *event, *cancel;

	event = smrt_command_alloc(smrt, SMRT_CMDTYPE_EVENT, KM_NOSLEEP);
	if (event == NULL)
		return (ENOMEM);
	if (smrt_command_attach_internal(smrt, event, SMRT_EVENT_NOTIFY_BUFLEN,
	    KM_NOSLEEP) != 0) {
		smrt_command_free(event);
		return (ENOMEM);
	}
	smrt_write_message_event_notify(event);

	cancel = smrt_command_alloc(smrt, SMRT_CMDTYPE_INTERNAL, KM_NOSLEEP);
	if (cancel == NULL) {
		smrt_command_free(event);
		return (ENOMEM);
	}
	if (smrt_command_attach_internal(smrt, cancel, SMRT_EVENT_NOTIFY_BUFLEN,
	    KM_NOSLEEP) != 0) {
		smrt_command_free(event);
		smrt_command_free(cancel);
		return (ENOMEM);
	}
	smrt_write_message_cancel_event_notify(cancel);

	cv_init(&smrt->smrt_event_queue, NULL, CV_DRIVER, NULL);

	mutex_enter(&smrt->smrt_mutex);
	if ((ret = smrt_submit(smrt, event)) != 0) {
		mutex_exit(&smrt->smrt_mutex);
		smrt_command_free(event);
		smrt_command_free(cancel);
		return (ret);
	}

	smrt->smrt_event_cmd = event;
	smrt->smrt_event_cancel_cmd = cancel;
	mutex_exit(&smrt->smrt_mutex);

	return (0);
}

void
smrt_event_complete(smrt_command_t *smcm)
{
	smrt_event_notify_t *sen;
	boolean_t log, rescan;

	boolean_t intervene = B_FALSE;
	smrt_t *smrt = smcm->smcm_ctlr;

	VERIFY(MUTEX_HELD(&smrt->smrt_mutex));
	VERIFY3P(smcm, ==, smrt->smrt_event_cmd);
	VERIFY0(smrt->smrt_status & SMRT_CTLR_ASYNC_INTERVENTION);

	smrt->smrt_stats.smrts_events_received++;

	if (smrt->smrt_status & SMRT_CTLR_STATUS_DETACHING) {
		cv_signal(&smrt->smrt_event_queue);
		return;
	}

	if (smrt->smrt_status & SMRT_CTLR_STATUS_RESETTING) {
		intervene = B_TRUE;
		goto clean;
	}

	/*
	 * The event notification command failed for some reason.  Attempt to
	 * drive on and try again at the next intervention period.  Because this
	 * may represent a programmer error (though it's hard to know), we wait
	 * until the next intervention period and don't panic.
	 */
	if (smcm->smcm_status & SMRT_CMD_STATUS_ERROR) {
		ErrorInfo_t *ei = smcm->smcm_va_err;
		intervene = B_TRUE;

		smrt->smrt_stats.smrts_events_errors++;
		dev_err(smrt->smrt_dip, CE_WARN, "!event notification request "
		    "error: status 0x%x", ei->CommandStatus);
		goto clean;
	}

	sen = smcm->smcm_internal->smcmi_va;
	log = rescan = B_FALSE;
	switch (sen->sen_class) {
	case SMRT_EVENT_CLASS_PROTOCOL:
		/*
		 * Most of the event protocol class events aren't really
		 * actionable.  However, subclass 1 indicates errors.  Today,
		 * the only error is an event overflow.  If there's an event
		 * overflow, then we must assume that we need to rescan.
		 */
		if (sen->sen_subclass == SMRT_EVENT_PROTOCOL_SUBCLASS_ERROR) {
			rescan = B_TRUE;
		}
		break;
	case SMRT_EVENT_CLASS_HOTPLUG:
		/*
		 * We want to log all hotplug events.  However we only need to
		 * scan these if the subclass indicates the event is for a disk.
		 */
		log = B_TRUE;
		if (sen->sen_subclass == SMRT_EVENT_HOTPLUG_SUBCLASS_DRIVE) {
			rescan = B_TRUE;
		}
		break;
	case SMRT_EVENT_CLASS_HWERROR:
	case SMRT_EVENT_CLASS_ENVIRONMENT:
		log = B_TRUE;
		break;
	case SMRT_EVENT_CLASS_PHYS:
		log = B_TRUE;
		/*
		 * This subclass indicates some change for physical drives.  As
		 * such, this should trigger a rescan.
		 */
		if (sen->sen_subclass == SMRT_EVENT_PHYS_SUBCLASS_STATE) {
			rescan = B_TRUE;
		}
		break;
	case SMRT_EVENT_CLASS_LOGVOL:
		rescan = B_TRUE;
		log = B_TRUE;
		break;
	default:
		/*
		 * While there are other classes of events, it's hard to say how
		 * actionable they are for the moment.  If we revamp this such
		 * that it becomes an ireport based system, then we should just
		 * always log these.  We opt not to at the moment to try and be
		 * kind to the system log.
		 */
		break;
	}

	/*
	 * Ideally, this would be an ireport that we could pass onto
	 * administrators; however, since we don't have any way to generate
	 * that, we provide a subset of the event information.
	 */
	if (log) {
		const char *rmsg;
		if (rescan == B_TRUE) {
			rmsg = "rescanning";
		} else {
			rmsg = "not rescanning";
		}
		if (sen->sen_message[0] != '\0') {
			sen->sen_message[sizeof (sen->sen_message) - 1] = '\0';
			dev_err(smrt->smrt_dip, CE_NOTE, "!controller event "
			    "class/sub-class/detail %x, %x, %x: %s; %s devices",
			    sen->sen_class, sen->sen_subclass, sen->sen_detail,
			    sen->sen_message, rmsg);
		} else {
			dev_err(smrt->smrt_dip, CE_NOTE, "!controller event "
			    "class/sub-class/detail %x, %x, %x; %s devices",
			    sen->sen_class, sen->sen_subclass, sen->sen_detail,
			    rmsg);
		}
	}

	if (rescan)
		smrt_discover_request(smrt);

clean:
	mutex_exit(&smrt->smrt_mutex);
	smrt_command_reuse(smcm);
	bzero(smcm->smcm_internal->smcmi_va, SMRT_EVENT_NOTIFY_BUFLEN);
	mutex_enter(&smrt->smrt_mutex);

	/*
	 * Make sure we're not _now_ detaching or resetting.
	 */
	if (smrt->smrt_status & SMRT_CTLR_STATUS_DETACHING) {
		cv_signal(&smrt->smrt_event_queue);
		return;
	}

	if ((smrt->smrt_status & SMRT_CTLR_STATUS_RESETTING) != 0 ||
	    intervene == B_TRUE) {
		smrt->smrt_status |= SMRT_CTLR_ASYNC_INTERVENTION;
		return;
	}

	/*
	 * Check out command count per tick.  If it's too high, leave it for
	 * intervention to solve.  Likely there is some serious driver or
	 * firmware error going on.
	 */
	smrt->smrt_event_count++;
	if (smrt->smrt_event_count > smrt_event_intervention_threshold) {
		smrt->smrt_status |= SMRT_CTLR_ASYNC_INTERVENTION;
		return;
	}

	if (smrt_submit(smrt, smcm) != 0) {
		smrt->smrt_status |= SMRT_CTLR_ASYNC_INTERVENTION;
	}
}

void
smrt_event_fini(smrt_t *smrt)
{
	int ret;
	smrt_command_t *event, *cancel;
	mutex_enter(&smrt->smrt_mutex);

	/*
	 * If intervention has been requested, there is nothing for us to do. We
	 * clear the flag so nothing else accidentally sees this and takes
	 * action.  We also don't need to bother sending a cancellation request,
	 * as there is no outstanding event.
	 */
	if (smrt->smrt_status & SMRT_CTLR_ASYNC_INTERVENTION) {
		smrt->smrt_status &= ~SMRT_CTLR_ASYNC_INTERVENTION;
		goto free;
	}

	/*
	 * Submit a cancel request for the event notification queue.  Because we
	 * submit both the cancel event and the regular notification event as an
	 * ordered command, we know that by the time this completes, that the
	 * existing one will have completed.
	 */
	smrt->smrt_event_cancel_cmd->smcm_status |= SMRT_CMD_STATUS_POLLED;
	if ((ret = smrt_submit(smrt, smrt->smrt_event_cancel_cmd)) != 0) {
		/*
		 * This is unfortunate.  We've failed to submit the command.  At
		 * this point all we can do is reset the device.  If the reset
		 * succeeds, we're done and we can clear all the memory.  If it
		 * fails, then all we can do is just leak the command and scream
		 * to the system, sorry.
		 */
		if (smrt_ctlr_reset(smrt) != 0) {
			dev_err(smrt->smrt_dip, CE_WARN, "failed to reset "
			    "device after failure to submit cancellation "
			    "(%d), abandoning smrt_command_t at address %p",
			    ret, smrt->smrt_event_cmd);
			smrt->smrt_event_cmd = NULL;
			goto free;
		}
	}

	smrt->smrt_event_cancel_cmd->smcm_expiry = gethrtime() +
	    SMRT_ASYNC_CANCEL_TIMEOUT * NANOSEC;
	if ((ret = smrt_poll_for(smrt, smrt->smrt_event_cancel_cmd)) != 0) {
		VERIFY3S(ret, ==, ETIMEDOUT);
		VERIFY0(smrt->smrt_event_cancel_cmd->smcm_status &
		    SMRT_CMD_STATUS_POLL_COMPLETE);

		/*
		 * The command timed out.  All we can do is hope a reset will
		 * work.
		 */
		if (smrt_ctlr_reset(smrt) != 0) {
			dev_err(smrt->smrt_dip, CE_WARN, "failed to reset "
			    "device after failure to poll for async "
			    "cancellation command abandoning smrt_command_t "
			    "event command at address %p and cancellation "
			    "command at %p", smrt->smrt_event_cmd,
			    smrt->smrt_event_cancel_cmd);
			smrt->smrt_event_cmd = NULL;
			smrt->smrt_event_cancel_cmd = NULL;
			goto free;
		}

	}

	/*
	 * Well, in the end, it's results that count.
	 */
	if (smrt->smrt_event_cancel_cmd->smcm_status &
	    SMRT_CMD_STATUS_RESET_SENT) {
		goto free;
	}

	if (smrt->smrt_event_cancel_cmd->smcm_status & SMRT_CMD_STATUS_ERROR) {
		ErrorInfo_t *ei = smrt->smrt_event_cancel_cmd->smcm_va_err;

		/*
		 * This can return a CISS_CMD_TARGET_STATUS entry when the
		 * controller doesn't think a command is outstanding.  It is
		 * possible we raced, so don't think too much about that case.
		 * Anything else leaves us between a rock and a hard place, the
		 * only way out is a reset.
		 */
		if (ei->CommandStatus != CISS_CMD_TARGET_STATUS &&
		    smrt_ctlr_reset(smrt) != 0) {
			dev_err(smrt->smrt_dip, CE_WARN, "failed to reset  "
			    "device after receiving an error on the async "
			    "cancellation command (%d); abandoning "
			    "smrt_command_t event command at address %p and "
			    "cancellation command at %p", ei->CommandStatus,
			    smrt->smrt_event_cmd, smrt->smrt_event_cancel_cmd);
			smrt->smrt_event_cmd = NULL;
			smrt->smrt_event_cancel_cmd = NULL;
			goto free;
		}
	}

free:
	event = smrt->smrt_event_cmd;
	smrt->smrt_event_cmd = NULL;
	cancel = smrt->smrt_event_cancel_cmd;
	smrt->smrt_event_cancel_cmd = NULL;
	mutex_exit(&smrt->smrt_mutex);
	if (event != NULL)
		smrt_command_free(event);
	if (cancel != NULL)
		smrt_command_free(cancel);
	cv_destroy(&smrt->smrt_event_queue);
}

/*
 * We've been asked to do a discovery in panic context.  This would have
 * occurred because there was a device reset.  Because we can't rely on the
 * target maps, all we can do at the moment is go over all the active targets
 * and note which ones no longer exist.  If this target was required to dump,
 * then the dump code will encounter a fatal error.  If not, then we should
 * count ourselves surprisingly lucky.
 */
static void
smrt_discover_panic_check(smrt_t *smrt)
{
	smrt_target_t *smtg;

	ASSERT(MUTEX_HELD(&smrt->smrt_mutex));
	for (smtg = list_head(&smrt->smrt_targets); smtg != NULL;
	    smtg = list_next(&smrt->smrt_targets, smtg)) {
		uint64_t gen;

		if (smtg->smtg_physical) {
			smrt_physical_t *smpt = smtg->smtg_lun.smtg_phys;
			/*
			 * Don't worry about drives that aren't visible.
			 */
			if (!smpt->smpt_visible)
				continue;
			gen = smpt->smpt_gen;
		} else {
			smrt_volume_t *smlv = smtg->smtg_lun.smtg_vol;
			gen = smlv->smlv_gen;
		}

		if (gen != smrt->smrt_discover_gen) {
			dev_err(smrt->smrt_dip, CE_WARN, "target %s "
			    "disappeared during post-panic discovery",
			    scsi_device_unit_address(smtg->smtg_scsi_dev));
			smtg->smtg_gone = B_TRUE;
		}
	}
}

static void
smrt_discover(void *arg)
{
	int log = 0, phys = 0;
	smrt_t *smrt = arg;
	uint64_t gen;
	boolean_t runphys, runvirt;

	mutex_enter(&smrt->smrt_mutex);
	smrt->smrt_status |= SMRT_CTLR_DISCOVERY_RUNNING;
	smrt->smrt_status &= ~SMRT_CTLR_DISCOVERY_REQUESTED;

	smrt->smrt_discover_gen++;
	gen = smrt->smrt_discover_gen;
	runphys = smrt->smrt_phys_tgtmap != NULL;
	runvirt = smrt->smrt_virt_tgtmap != NULL;
	mutex_exit(&smrt->smrt_mutex);
	if (runphys)
		phys = smrt_phys_discover(smrt, SMRT_DISCOVER_TIMEOUT, gen);
	if (runvirt)
		log = smrt_logvol_discover(smrt, SMRT_DISCOVER_TIMEOUT, gen);
	mutex_enter(&smrt->smrt_mutex);

	if (phys != 0 || log != 0) {
		if (!ddi_in_panic()) {
			smrt->smrt_status |= SMRT_CTLR_DISCOVERY_PERIODIC;
		} else {
			panic("smrt_t %p failed to perform discovery after "
			    "a reset in panic context, unable to continue. "
			    "logvol: %d, phys: %d", smrt, log, phys);
		}
	} else {
		if (!ddi_in_panic() &&
		    smrt->smrt_status & SMRT_CTLR_DISCOVERY_REQUIRED) {
			smrt->smrt_status &= ~SMRT_CTLR_DISCOVERY_REQUIRED;
			cv_broadcast(&smrt->smrt_cv_finishq);
		}

		if (ddi_in_panic()) {
			smrt_discover_panic_check(smrt);
		}
	}
	smrt->smrt_status &= ~SMRT_CTLR_DISCOVERY_RUNNING;
	if (smrt->smrt_status & SMRT_CTLR_DISCOVERY_REQUESTED)
		smrt->smrt_status |= SMRT_CTLR_DISCOVERY_PERIODIC;
	mutex_exit(&smrt->smrt_mutex);
}

/*
 * Request discovery, which is always run via a taskq.
 */
void
smrt_discover_request(smrt_t *smrt)
{
	boolean_t run;
	ASSERT(MUTEX_HELD(&smrt->smrt_mutex));

	if (ddi_in_panic()) {
		smrt_discover(smrt);
		return;
	}

	run = (smrt->smrt_status & SMRT_CTLR_DISCOVERY_MASK) == 0;
	smrt->smrt_status |= SMRT_CTLR_DISCOVERY_REQUESTED;
	if (run && ddi_taskq_dispatch(smrt->smrt_discover_taskq,
	    smrt_discover, smrt, DDI_NOSLEEP) != DDI_SUCCESS) {
		smrt->smrt_status |= SMRT_CTLR_DISCOVERY_PERIODIC;
		smrt->smrt_stats.smrts_discovery_tq_errors++;
	}
}