xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/lmrc/lmrc.c (revision 8119dad84d6416f13557b0ba8e2aaf9064cbcfd3)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2024 Racktop Systems, Inc.
 */

/*
 * This file implements the interfaces for communicating with the MegaRAID HBA.
 * There are three basic interfaces:
 * - the device registers, which provide basic information about the controller
 *   hardware and the features it supports, as well as control registers used
 *   during sending and reception of I/O frames
 * - Fusion-MPT v2.5, perhaps later, which defines the format of the I/O frames
 *   used for communicating with the HBA and virtual and physical devices that
 *   are attached to it
 * - MFI, the MegaRAID Firmware Interface, which are sent and received as MPT
 *   payloads to control and communicate with the RAID controller.
 */

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/scsi/scsi.h>

#include <sys/scsi/adapters/mfi/mfi.h>
#include <sys/scsi/adapters/mfi/mfi_evt.h>
#include <sys/scsi/adapters/mfi/mfi_pd.h>

#include <sys/cpuvar.h>

#include "lmrc.h"
#include "lmrc_reg.h"
#include "lmrc_raid.h"
#include "lmrc_phys.h"

static uint32_t lmrc_read_reg(lmrc_t *, uint32_t);
static void lmrc_write_reg(lmrc_t *, uint32_t, uint32_t);
static int lmrc_transition_to_ready(lmrc_t *);
static void lmrc_process_mptmfi_passthru(lmrc_t *, lmrc_mpt_cmd_t *);
static int lmrc_poll_mfi(lmrc_t *, lmrc_mfi_cmd_t *, uint8_t);
static boolean_t lmrc_check_fw_fault(lmrc_t *);
static int lmrc_get_event_log_info(lmrc_t *, mfi_evt_log_info_t *);
static void lmrc_aen_handler(void *);
static void lmrc_complete_aen(lmrc_t *, lmrc_mfi_cmd_t *);
static int lmrc_register_aen(lmrc_t *, uint32_t);

/*
 * Device register access functions.
 *
 * Due to the way ddi_get* and ddi_put* work, we'll need to calculate the
 * absolute virtual address of the registers ourselves.
 *
 * For read accesses, employ a erratum workaround for Aero controllers. In some
 * cases, reads of certain registers will intermittently return all zeros. As a
 * workaround, retry the read up to three times until a non-zero value is read.
 * Supposedly this is enough, every other driver I looked at does this.
 */
static uint32_t
lmrc_read_reg_1(lmrc_t *lmrc, uint32_t reg)
{
	uint32_t *addr = (uint32_t *)((uintptr_t)lmrc->l_regmap + reg);
	return (ddi_get32(lmrc->l_reghandle, addr));
}

static uint32_t
lmrc_read_reg(lmrc_t *lmrc, uint32_t reg)
{
	if (lmrc->l_class != LMRC_ACLASS_AERO)
		return (lmrc_read_reg_1(lmrc, reg));

	/* Workaround for the hardware erratum in Aero controllers */
	for (uint_t i = 0; i < 3; i++) {
		uint32_t val = lmrc_read_reg_1(lmrc, reg);

		if (val != 0)
			return (val);
	}

	return (0);
}

static void
lmrc_write_reg(lmrc_t *lmrc, uint32_t reg, uint32_t val)
{
	uint32_t *addr = (uint32_t *)((uintptr_t)lmrc->l_regmap + reg);
	ddi_put32(lmrc->l_reghandle, addr, val);
}

static void
lmrc_write_reg64(lmrc_t *lmrc, uint32_t reg, uint64_t val)
{
	uint64_t *addr = (uint64_t *)((uintptr_t)lmrc->l_regmap + reg);
	ddi_put64(lmrc->l_reghandle, addr, val);
}

/*
 * Interrupt control
 *
 * There are two interrupt registers for host driver use, HostInterruptStatus
 * and HostInterruptMask. Most of the bits in each register are reserved and
 * must masked and/or preserved when used.
 */
void
lmrc_disable_intr(lmrc_t *lmrc)
{
	uint32_t mask = lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET);

	/* Disable all known interrupt: reset, reply, and doorbell. */
	mask |= MPI2_HIM_RESET_IRQ_MASK;
	mask |= MPI2_HIM_REPLY_INT_MASK;
	mask |= MPI2_HIM_IOC2SYS_DB_MASK;

	lmrc_write_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET, mask);

	/* Dummy read to force pci flush. Probably bogus but harmless. */
	(void) lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET);
}

void
lmrc_enable_intr(lmrc_t *lmrc)
{
	uint32_t mask = lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET);

	/* Enable the reply interrupts and the doorbell interrupts. */
	mask &= ~MPI2_HIM_REPLY_INT_MASK;
	mask &= ~MPI2_HIM_IOC2SYS_DB_MASK;

	/* Clear outstanding interrupts before enabling any. */
	lmrc_write_reg(lmrc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0);
	/* Dummy read to force pci flush. Probably bogus but harmless. */
	(void) lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);

	lmrc_write_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET, mask);
	/* Dummy read to force pci flush. Probably bogus but harmless. */
	(void) lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_MASK_OFFSET);
}

uint_t
lmrc_intr_ack(lmrc_t *lmrc)
{
	uint32_t mask =
	    MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT | MPI2_HIS_IOC2SYS_DB_STATUS;
	uint32_t status;

	status = lmrc_read_reg(lmrc, MPI2_HOST_INTERRUPT_STATUS_OFFSET);

	if ((status & mask) == 0)
		return (DDI_INTR_UNCLAIMED);

	if (lmrc_check_acc_handle(lmrc->l_reghandle) != DDI_SUCCESS) {
		ddi_fm_service_impact(lmrc->l_dip, DDI_SERVICE_LOST);
		return (DDI_INTR_UNCLAIMED);
	}

	return (DDI_INTR_CLAIMED);
}

/*
 * Fusion-MPT requests
 *
 * The controller expects to have access to a large chunk of DMA memory, into
 * which the driver writes fixed-size I/O requests for the controller to
 * process. To notify the hardware about a new request, a request descriptor is
 * written to the queue port registers which includes the SMID of the request.
 * This memory isn't really a queue, though, as it seems there are no
 * constraints about ordering of the requests. All that matters is that there
 * is a valid request at the address that corresponds with the SMID in the
 * descriptor.
 *
 * If the hardware supports MPI 2.6 atomic request descriptors, which are a
 * 32bit subset of the 64bit MPI 2.0/2.5 request descriptors, the descriptor is
 * sent to the controller in a single 32bit write into a device register.
 *
 * For all other descriptor types, we'll employ a 64bit write to the queue
 * registers, assuming that provides the required atomicity.
 */
void
lmrc_send_atomic_request(lmrc_t *lmrc, lmrc_atomic_req_desc_t req_desc)
{
	if (lmrc->l_atomic_desc_support) {
		lmrc_write_reg(lmrc,
		    MPI26_ATOMIC_REQUEST_DESCRIPTOR_POST_OFFSET,
		    req_desc.rd_reg);
	} else {
		lmrc_req_desc_t rd;

		bzero(&rd, sizeof (rd));
		rd.rd_atomic = req_desc;

		lmrc_send_request(lmrc, rd);
	}
}

void
lmrc_send_request(lmrc_t *lmrc, lmrc_req_desc_t req_desc)
{
	lmrc_write_reg64(lmrc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET,
	    req_desc.rd_reg);
}

lmrc_atomic_req_desc_t
lmrc_build_atomic_request(lmrc_t *lmrc, lmrc_mpt_cmd_t *mpt, uint8_t flags)
{
	lmrc_atomic_req_desc_t req_desc;

	VERIFY3U(mpt->mpt_smid, !=, 0);

	/*
	 * Select the reply queue based on the CPU id to distribute reply load
	 * among queues.
	 */
	mpt->mpt_queue = CPU->cpu_id % lmrc->l_max_reply_queues;

	bzero(&req_desc, sizeof (req_desc));

	req_desc.rd_atomic.RequestFlags = flags;
	req_desc.rd_atomic.MSIxIndex = mpt->mpt_queue;
	req_desc.rd_atomic.SMID = mpt->mpt_smid;

	return (req_desc);
}

/*
 * Reply Processing
 *
 * The controller will post replies to completed requests in the DMA memory
 * provided for that purpose. This memory is divided in equally-sized chunks,
 * each being a separate reply queue that is also associated with an interrupt
 * vector. The replies are fixed size structures and will be written by the
 * hardware in order of completion into the queue. For each queue, there is a
 * register to tell the hardware which replies have been consumed by the driver.
 *
 * In response to an interrupt, the driver will walk the reply queue associated
 * with the interrupt vector at the last known position and processess all
 * completed replies. After a number of replies has been processed, or if no
 * more replies are ready to be processed, the controller will be notified about
 * the last reply index to be processed by writing the appropriate register.
 */

/*
 * lmrc_get_next_reply_desc
 *
 * Get the next unprocessed reply descriptor for a queue, or NULL if there is
 * none.
 */
static Mpi2ReplyDescriptorsUnion_t *
lmrc_get_next_reply_desc(lmrc_t *lmrc, int queue)
{
	Mpi2ReplyDescriptorsUnion_t *desc;

	desc = lmrc->l_reply_dma.ld_buf;

	desc += (queue * lmrc->l_reply_alloc_sz) / sizeof (*desc);
	desc += lmrc->l_last_reply_idx[queue];

	VERIFY3S(ddi_dma_sync(lmrc->l_reply_dma.ld_hdl,
	    (void *)desc - lmrc->l_reply_dma.ld_buf, sizeof (*desc),
	    DDI_DMA_SYNC_FORKERNEL), ==, DDI_SUCCESS);

	/*
	 * Check if this is an unused reply descriptor, indicating that
	 * we've reached the end of replies in this queue.
	 *
	 * Even if the descriptor is only "half unused" we can't use it.
	 */
	if (desc->Words.Low == MPI2_RPY_DESCRIPT_UNUSED_WORD0_MARK ||
	    desc->Words.High == MPI2_RPY_DESCRIPT_UNUSED_WORD1_MARK)
		return (NULL);

	/* advance last reply index, wrap around if necessary */
	lmrc->l_last_reply_idx[queue]++;
	if (lmrc->l_last_reply_idx[queue] >= lmrc->l_reply_q_depth)
		lmrc->l_last_reply_idx[queue] = 0;

	return (desc);
}

/*
 * lmrc_write_rphi
 *
 * Write the Reply Post Host Index register for queue.
 */
static void
lmrc_write_rphi(lmrc_t *lmrc, uint32_t queue)
{
	int reg = 0;
	uint32_t val = (queue << 24) | lmrc->l_last_reply_idx[queue];

	if (lmrc->l_intr_type != DDI_INTR_TYPE_MSIX)
		VERIFY3U(queue, ==, 0);

	if (lmrc->l_msix_combined) {
		reg = queue / 8;
		val &= 0x07ffffff;
	}

	lmrc_write_reg(lmrc, lmrc->l_rphi[reg], val);
}

/*
 * lmrc_process_mpt_pkt
 *
 * Process a reply to a MPT IO request. Update the scsi_pkt according to status,
 * ex_status, and data_len, setting up the ARQ pkt if necessary.
 */
static void
lmrc_process_mpt_pkt(lmrc_t *lmrc, struct scsi_pkt *pkt, uint8_t status,
    uint8_t ex_status, uint32_t data_len)
{
	pkt->pkt_statistics = 0;
	pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
	    STATE_XFERRED_DATA | STATE_GOT_STATUS;

	pkt->pkt_resid = pkt->pkt_dma_len - data_len;

	switch (status) {
	case MFI_STAT_OK:
	case MFI_STAT_LD_CC_IN_PROGRESS:
	case MFI_STAT_LD_RECON_IN_PROGRESS:
		pkt->pkt_reason = CMD_CMPLT;
		pkt->pkt_scbp[0] = STATUS_GOOD;
		break;

	case MFI_STAT_SCSI_DONE_WITH_ERROR:
	case MFI_STAT_LD_LBA_OUT_OF_RANGE: {
		struct scsi_arq_status *arq =
		    (struct scsi_arq_status *)pkt->pkt_scbp;

		pkt->pkt_reason = CMD_CMPLT;
		arq->sts_status.sts_chk = 1;

		pkt->pkt_state |= STATE_ARQ_DONE;
		arq->sts_rqpkt_reason = CMD_CMPLT;
		arq->sts_rqpkt_resid = 0;
		arq->sts_rqpkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
		    STATE_SENT_CMD | STATE_XFERRED_DATA;
		*(uint8_t *)&arq->sts_rqpkt_status = STATUS_GOOD;
		break;
	}
	case MFI_STAT_LD_OFFLINE:
	case MFI_STAT_DEVICE_NOT_FOUND:
		pkt->pkt_reason = CMD_DEV_GONE;
		pkt->pkt_statistics = STAT_DISCON;
		break;

	default:
		dev_err(lmrc->l_dip, CE_WARN, "!command failed, status = %x, "
		    "ex_status = %x, cdb[0] = %x", status, ex_status,
		    pkt->pkt_cdbp[0]);
		pkt->pkt_reason = CMD_TRAN_ERR;
		break;
	}
}

/*
 * lmrc_poll_for_reply
 *
 * During a panic we'll have to resort to polled I/O to write core dumps.
 * Repeatedly check the reply queue for a new reply associated with the
 * given request descriptor and complete it, or return an error if we get
 * no reply within a reasonable time.
 */
int
lmrc_poll_for_reply(lmrc_t *lmrc, lmrc_mpt_cmd_t *mpt)
{
	clock_t max_wait = LMRC_IO_TIMEOUT * MILLISEC * 10;
	Mpi25SCSIIORequest_t *io_req = mpt->mpt_io_frame;
	Mpi2ReplyDescriptorsUnion_t *desc;
	uint16_t desc_smid;

	VERIFY(ddi_in_panic());

	/*
	 * Walk the reply queue. Discard entries which we aren't
	 * looking for.
	 */
	do {
		desc = lmrc_get_next_reply_desc(lmrc, mpt->mpt_queue);
		if (desc == NULL) {
			if (max_wait == 0)
				return (TRAN_FATAL_ERROR);

			drv_usecwait(100);
			max_wait--;
			continue;
		}

		desc_smid = desc->SCSIIOSuccess.SMID;

		/* reset descriptor */
		desc->Words.Low = MPI2_RPY_DESCRIPT_UNUSED_WORD0_MARK;
		desc->Words.High = MPI2_RPY_DESCRIPT_UNUSED_WORD1_MARK;

		lmrc_write_rphi(lmrc, mpt->mpt_queue);
	} while (desc == NULL || desc_smid != mpt->mpt_smid);

	VERIFY3S(ddi_dma_sync(lmrc->l_ioreq_dma.ld_hdl,
	    (void *)io_req - lmrc->l_ioreq_dma.ld_buf,
	    LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE, DDI_DMA_SYNC_FORKERNEL),
	    ==, DDI_SUCCESS);

	/* If this is I/O, process it. */
	if (io_req->Function == LMRC_MPI2_FUNCTION_LD_IO_REQUEST ||
	    io_req->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
		lmrc_process_mpt_pkt(lmrc, mpt->mpt_pkt,
		    io_req->VendorRegion.rc_status,
		    io_req->VendorRegion.rc_exstatus, io_req->DataLength);
	}

	return (TRAN_ACCEPT);
}

/*
 * lmrc_process_replies
 *
 * Process all new reply entries in a queue in response to an interrupt.
 */
int
lmrc_process_replies(lmrc_t *lmrc, uint8_t queue)
{
	int nprocessed = 0;
	Mpi2ReplyDescriptorsUnion_t *desc;

	for (desc = lmrc_get_next_reply_desc(lmrc, queue);
	    desc != NULL;
	    desc = lmrc_get_next_reply_desc(lmrc, queue)) {
		Mpi2SCSIIOSuccessReplyDescriptor_t *reply =
		    &desc->SCSIIOSuccess;
		uint16_t smid = reply->SMID;
		lmrc_mpt_cmd_t *mpt = lmrc->l_mpt_cmds[smid - 1];
		lmrc_tgt_t *tgt = NULL;
		Mpi25SCSIIORequest_t *io_req;
		struct scsi_pkt *pkt;
		struct scsi_device *sd;

		VERIFY3U(reply->SMID, <=, lmrc->l_max_fw_cmds);

		mutex_enter(&mpt->mpt_lock);
		mpt->mpt_complete = B_TRUE;
		pkt = mpt->mpt_pkt;
		io_req = mpt->mpt_io_frame;

		VERIFY3S(ddi_dma_sync(lmrc->l_ioreq_dma.ld_hdl,
		    (void *)io_req - lmrc->l_ioreq_dma.ld_buf,
		    LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE,
		    DDI_DMA_SYNC_FORKERNEL), ==, DDI_SUCCESS);


		switch (io_req->Function) {
		case MPI2_FUNCTION_SCSI_TASK_MGMT:
			VERIFY0(pkt);
			VERIFY0(list_link_active(&mpt->mpt_node));
			cv_signal(&mpt->mpt_cv);
			break;

		case MPI2_FUNCTION_SCSI_IO_REQUEST:
		case LMRC_MPI2_FUNCTION_LD_IO_REQUEST:
			VERIFY(pkt != NULL);

			sd = scsi_address_device(&pkt->pkt_address);
			VERIFY(sd != NULL);

			tgt = scsi_device_hba_private_get(sd);
			VERIFY(tgt != NULL);

			lmrc_process_mpt_pkt(lmrc, pkt,
			    io_req->VendorRegion.rc_status,
			    io_req->VendorRegion.rc_exstatus,
			    io_req->DataLength);

			break;

		case LMRC_MPI2_FUNCTION_PASSTHRU_IO_REQUEST:
			VERIFY0(pkt);
			VERIFY0(list_link_active(&mpt->mpt_node));
			lmrc_process_mptmfi_passthru(lmrc, mpt);
			break;

		default:
			mutex_exit(&mpt->mpt_lock);
			dev_err(lmrc->l_dip, CE_PANIC,
			    "reply received for unknown Function %x",
			    io_req->Function);
		}

		mutex_exit(&mpt->mpt_lock);

		if (pkt != NULL) {
			lmrc_tgt_rem_active_mpt(tgt, mpt);
			atomic_dec_uint(&lmrc->l_fw_outstanding_cmds);
			scsi_hba_pkt_comp(pkt);
		}

		/* reset descriptor */
		desc->Words.Low = MPI2_RPY_DESCRIPT_UNUSED_WORD0_MARK;
		desc->Words.High = MPI2_RPY_DESCRIPT_UNUSED_WORD1_MARK;

		nprocessed++;

		if (nprocessed % LMRC_THRESHOLD_REPLY_COUNT == 0)
			lmrc_write_rphi(lmrc, queue);
	}

	if (nprocessed != 0 && nprocessed % LMRC_THRESHOLD_REPLY_COUNT != 0)
		lmrc_write_rphi(lmrc, queue);

	return (DDI_INTR_CLAIMED);
}


/*
 * MFI - MegaRAID Firmware Interface
 */

/*
 * lmrc_build_mptmfi_passthru
 *
 * MFI commands are send as MPT MFI passthrough I/O requests. To be able to send
 * a MFI frame to the RAID controller, we need to have a MPT command set up as
 * MPT I/O request and a one-entry SGL pointing to the MFI command.
 *
 * As there's only a small number of MFI commands compared to the amound of MPT
 * commands, the MPT command for each MFI is pre-allocated at attach time and
 * initialized here.
 */
int
lmrc_build_mptmfi_passthru(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
{
	Mpi25SCSIIORequest_t *io_req;
	const ddi_dma_cookie_t *cookie;
	lmrc_mpt_cmd_t *mpt;

	mpt = lmrc_get_mpt(lmrc);
	if (mpt == NULL)
		return (DDI_FAILURE);

	/* lmrc_get_mpt() should return the mpt locked */
	ASSERT(mutex_owned(&mpt->mpt_lock));

	mfi->mfi_mpt = mpt;
	mpt->mpt_mfi = mfi;

	io_req = mpt->mpt_io_frame;
	io_req->Function = LMRC_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
	io_req->ChainOffset = lmrc->l_chain_offset_mfi_pthru;

	cookie = ddi_dma_cookie_one(mfi->mfi_frame_dma.ld_hdl);
	lmrc_dma_build_sgl(lmrc, mpt, cookie, 1);

	VERIFY3S(ddi_dma_sync(lmrc->l_ioreq_dma.ld_hdl,
	    (void *)io_req - lmrc->l_ioreq_dma.ld_buf,
	    LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE, DDI_DMA_SYNC_FORDEV),
	    ==, DDI_SUCCESS);

	/*
	 * As we're not sending this command to the hardware any time soon,
	 * drop the mutex before we return.
	 */
	mutex_exit(&mpt->mpt_lock);

	return (DDI_SUCCESS);
}

/*
 * lmrc_process_mptmfi_passthru
 *
 * When a MPT MFI passthrough command completes, invoke the callback if there
 * is one. Panic if an invalid command completed as that should never happen.
 */
static void
lmrc_process_mptmfi_passthru(lmrc_t *lmrc, lmrc_mpt_cmd_t *mpt)
{
	lmrc_mfi_cmd_t *mfi;
	mfi_header_t *hdr;

	VERIFY3P(mpt->mpt_mfi, !=, NULL);
	mfi = mpt->mpt_mfi;
	VERIFY0(list_link_active(&mfi->mfi_node));

	hdr = &mfi->mfi_frame->mf_hdr;

	if ((hdr->mh_flags & MFI_FRAME_DIR_READ) != 0)
		(void) ddi_dma_sync(mfi->mfi_data_dma.ld_hdl, 0,
		    mfi->mfi_data_dma.ld_len, DDI_DMA_SYNC_FORKERNEL);

	switch (hdr->mh_cmd) {
	case MFI_CMD_DCMD:
	case MFI_CMD_LD_SCSI_IO:
	case MFI_CMD_PD_SCSI_IO:
	case MFI_CMD_ABORT:
		mutex_enter(&mfi->mfi_lock);
		if (mfi->mfi_callback != NULL)
			mfi->mfi_callback(lmrc, mfi);
		mutex_exit(&mfi->mfi_lock);
		break;

	case MFI_CMD_INVALID:
	default:
		dev_err(lmrc->l_dip, CE_PANIC,
		    "invalid MFI cmd completion received, cmd = %x",
		    hdr->mh_cmd);
		break;
	}
}

/*
 * lmrc_issue_mfi
 *
 * Post a MFI command to the firmware. Reset the cmd_status to invalid. Build
 * a MPT MFI passthru command if necessary and a MPT atomic request descriptor
 * before posting the request. The MFI command's mutex must be held.
 */
void
lmrc_issue_mfi(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi, lmrc_mfi_cmd_cb_t *cb)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	lmrc_atomic_req_desc_t req_desc;

	ASSERT(mutex_owned(&mfi->mfi_lock));

	if ((hdr->mh_flags & MFI_FRAME_DONT_POST_IN_REPLY_QUEUE) == 0) {
		VERIFY3U(cb, !=, NULL);
		mfi->mfi_callback = cb;
	} else {
		VERIFY3U(cb, ==, NULL);
	}

	hdr->mh_cmd_status = MFI_STAT_INVALID_STATUS;

	req_desc = lmrc_build_atomic_request(lmrc, mfi->mfi_mpt,
	    MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO);

	(void) ddi_dma_sync(mfi->mfi_frame_dma.ld_hdl, 0,
	    mfi->mfi_frame_dma.ld_len, DDI_DMA_SYNC_FORDEV);

	if ((hdr->mh_flags & MFI_FRAME_DIR_WRITE) != 0)
		(void) ddi_dma_sync(mfi->mfi_data_dma.ld_hdl, 0,
		    mfi->mfi_data_dma.ld_len, DDI_DMA_SYNC_FORDEV);

	lmrc_send_atomic_request(lmrc, req_desc);
}

/*
 * lmrc_poll_mfi
 *
 * Poll a MFI command for completion, waiting up to max_wait secs. Repeatedly
 * check the command status until it changes to something that is not invalid.
 *
 * Trigger an online controller reset on timeout.
 */
static int
lmrc_poll_mfi(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi, uint8_t max_wait)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	lmrc_dma_t *dma = &mfi->mfi_frame_dma;
	clock_t timeout = ddi_get_lbolt() + drv_usectohz(max_wait * MICROSEC);
	clock_t now;

	ASSERT(mutex_owned(&mfi->mfi_lock));

	do {
		(void) ddi_dma_sync(dma->ld_hdl, 0, dma->ld_len,
		    DDI_DMA_SYNC_FORKERNEL);
		if (hdr->mh_cmd_status != MFI_STAT_INVALID_STATUS)
			break;

		(void) cv_reltimedwait(&mfi->mfi_cv, &mfi->mfi_lock,
		    drv_usectohz(MILLISEC), TR_MILLISEC);
		now = ddi_get_lbolt();
	} while (!lmrc->l_fw_fault && now <= timeout);

	if (hdr->mh_cmd_status != MFI_STAT_INVALID_STATUS)
		return (DDI_SUCCESS);

	if (now > timeout) {
		dev_err(lmrc->l_dip, CE_WARN,
		    "!%s: command timeout after %ds", __func__, max_wait);

		/*
		 * Signal the housekeeping thread to check for FW/HW faults,
		 * performing a reset if necessary.
		 */
		cv_signal(&lmrc->l_thread_cv);
	}

	return (DDI_FAILURE);
}

/*
 * lmrc_wait_mfi
 *
 * Wait for up to max_wait secs for a MFI command to complete. The cmd mutex
 * must be held.
 *
 * Trigger an online controller reset on timeout.
 */
int
lmrc_wait_mfi(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi, uint8_t max_wait)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	lmrc_dma_t *dma = &mfi->mfi_frame_dma;
	clock_t timeout = ddi_get_lbolt() + drv_usectohz(max_wait * MICROSEC);
	int ret;

	ASSERT(mutex_owned(&mfi->mfi_lock));

	do {
		ret = cv_timedwait(&mfi->mfi_cv, &mfi->mfi_lock, timeout);

		(void) ddi_dma_sync(dma->ld_hdl, 0, dma->ld_len,
		    DDI_DMA_SYNC_FORKERNEL);

	} while (!lmrc->l_fw_fault &&
	    hdr->mh_cmd_status == MFI_STAT_INVALID_STATUS && ret != -1);

	if (!lmrc->l_fw_fault && ret != -1)
		return (DDI_SUCCESS);

	if (ret == -1) {
		dev_err(lmrc->l_dip, CE_WARN, "!%s: blocked command timeout "
		    "after %ds, cmd = %d, status = %d", __func__, max_wait,
		    hdr->mh_cmd, hdr->mh_cmd_status);

		/*
		 * Signal the housekeeping thread to check for FW/HW faults,
		 * performing a reset if necessary.
		 */
		cv_signal(&lmrc->l_thread_cv);
	}

	return (DDI_FAILURE);
}

/*
 * lmrc_wakeup_mfi
 *
 * Signal the CV associated with a MFI command to wake up the thread waiting
 * for its completion.
 */
void
lmrc_wakeup_mfi(lmrc_t *lmrc, lmrc_mfi_cmd_t *cmd)
{
	ASSERT(mutex_owned(&cmd->mfi_lock));
	cv_signal(&cmd->mfi_cv);
}

/*
 * lmrc_issue_blocked_mfi
 *
 * Post a MFI command to the firmware and wait for the command to complete.
 */
int
lmrc_issue_blocked_mfi(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	int ret;

	mutex_enter(&mfi->mfi_lock);
	lmrc_issue_mfi(lmrc, mfi, lmrc_wakeup_mfi);
	ret = lmrc_wait_mfi(lmrc, mfi, LMRC_INTERNAL_CMD_WAIT_TIME);
	mutex_exit(&mfi->mfi_lock);

	if (ret == DDI_SUCCESS && hdr->mh_cmd_status == MFI_STAT_OK)
		return (DDI_SUCCESS);

	dev_err(lmrc->l_dip, CE_WARN,
	    "!%s: blocked command failure, cmd = %d, status = %d",
	    __func__, hdr->mh_cmd, hdr->mh_cmd_status);

	return (ret);
}

/*
 * lmrc_abort_cb
 *
 * Callback for any command that is to be aborted.
 *
 * If the command completed normally before it could be aborted, set the status
 * to indicate the intended abortion.
 */
static void
lmrc_abort_cb(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;

	if (hdr->mh_cmd_status == MFI_STAT_OK)
		hdr->mh_cmd_status = MFI_STAT_NOT_FOUND;
}

/*
 * lmrc_abort_mfi
 *
 * Abort a MFI command. This is a bit tricky as the hardware may still complete
 * it at any time.
 *
 * The mutex of the command to be aborted must be held to prevent it from
 * completing behind our back. We'll replace its callback with our own, issue an
 * ABORT command, and drop the mutex before we wait for the ABORT command to
 * complete.
 */
static int
lmrc_abort_cmd(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi_to_abort)
{
	lmrc_mfi_cmd_t *mfi = lmrc_get_mfi(lmrc);
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	mfi_abort_payload_t *abort = &mfi->mfi_frame->mf_abort;
	lmrc_mfi_cmd_cb_t *orig_cb = mfi_to_abort->mfi_callback;
	int ret;

	ASSERT(mutex_owned(&mfi_to_abort->mfi_lock));

	/* Replace the commands callback with our own. */
	mfi_to_abort->mfi_callback = lmrc_abort_cb;

	hdr->mh_cmd = MFI_CMD_ABORT;
	abort->ma_abort_context = mfi_to_abort->mfi_idx;
	lmrc_dma_set_addr64(&mfi_to_abort->mfi_frame_dma,
	    &abort->ma_abort_mfi_phys_addr);

	/* Send the ABORT. */
	mutex_enter(&mfi->mfi_lock);
	lmrc_issue_mfi(lmrc, mfi, lmrc_wakeup_mfi);

	/*
	 * Drop the mutex of the command to be aborted, allowing it to proceed
	 * while we wait for the ABORT command to complete.
	 */
	mutex_exit(&mfi_to_abort->mfi_lock);
	ret = lmrc_wait_mfi(lmrc, mfi, LMRC_INTERNAL_CMD_WAIT_TIME);
	mutex_exit(&mfi->mfi_lock);

	/*
	 * The ABORT command may fail if cmd_to_abort has completed already.
	 * Treat any other failure as fatal, restore the callback and fail.
	 */
	if (ret != DDI_SUCCESS && hdr->mh_cmd_status != MFI_STAT_NOT_FOUND) {
		mutex_enter(&mfi_to_abort->mfi_lock);
		mfi_to_abort->mfi_callback = orig_cb;
		goto out;
	}

	/*
	 * Wait for the aborted command to complete. If we time out on this
	 * there's little we can do here, so we restore the callback and fail.
	 */
	mutex_enter(&mfi_to_abort->mfi_lock);
	ret = lmrc_poll_mfi(lmrc, mfi_to_abort, LMRC_INTERNAL_CMD_WAIT_TIME);
	mfi_to_abort->mfi_callback = orig_cb;

	if (ret != DDI_SUCCESS)
		goto out;

	/* Wake up anyone waiting on the aborted command. */
	if (mfi_to_abort->mfi_callback != NULL)
		mfi_to_abort->mfi_callback(lmrc, mfi_to_abort);

out:
	lmrc_put_mfi(mfi);
	ASSERT(mutex_owned(&mfi_to_abort->mfi_lock));
	return (ret);
}


/*
 * Controller Initialization and Housekeeping
 */

/*
 * lmrc_check_fw_fault
 *
 * Check the firmware state. If faulted, return B_TRUE.
 * Return B_FALSE otherwise.
 */
static boolean_t
lmrc_check_fw_fault(lmrc_t *lmrc)
{
	uint32_t status = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);
	uint32_t fw_state = LMRC_FW_STATE(status);

	if (fw_state == LMRC_FW_STATE_FAULT)
		return (B_TRUE);

	return (B_FALSE);
}

/*
 * lmrc_wait_for_reg
 *
 * Repeatedly read the register and check that 'bits' match 'exp'.
 */
static boolean_t
lmrc_wait_for_reg(lmrc_t *lmrc, uint32_t reg, uint32_t bits, uint32_t exp,
    uint64_t max_wait)
{
	uint32_t val;
	uint64_t i;

	max_wait *= MILLISEC / 100;

	for (i = 0; i < max_wait; i++) {
		delay(drv_usectohz(100 * MILLISEC));
		val = lmrc_read_reg(lmrc, reg);

		if ((val & bits) == exp)
			return (B_TRUE);
	}

	return (B_FALSE);
}

static int
lmrc_hard_reset(lmrc_t *lmrc)
{
	int ret = DDI_SUCCESS;

	/* Write the reset key sequence. */
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_FLUSH_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_1ST_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_2ND_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_3RD_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_4TH_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_5TH_KEY_VALUE);
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_6TH_KEY_VALUE);

	/* Check diag write enable. */
	if (!lmrc_wait_for_reg(lmrc, MPI2_HOST_DIAGNOSTIC_OFFSET,
	    MPI2_DIAG_DIAG_WRITE_ENABLE, MPI2_DIAG_DIAG_WRITE_ENABLE,
	    LMRC_RESET_TIMEOUT)) {
		dev_err(lmrc->l_dip, CE_WARN, "diag unlock failed");
		return (DDI_FAILURE);
	}

	/* Reset IOC. */
	lmrc_write_reg(lmrc, MPI2_HOST_DIAGNOSTIC_OFFSET,
	    lmrc_read_reg(lmrc, MPI2_HOST_DIAGNOSTIC_OFFSET) |
	    MPI2_DIAG_RESET_ADAPTER);
	delay(drv_usectohz(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC));

	/* Check the reset adapter bit. */
	if ((lmrc_read_reg(lmrc, MPI2_HOST_DIAGNOSTIC_OFFSET) &
	    MPI2_DIAG_RESET_ADAPTER) == 0)
		goto out;

	delay(drv_usectohz(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC));

	/* Check the reset adapter bit again. */
	if ((lmrc_read_reg(lmrc, MPI2_HOST_DIAGNOSTIC_OFFSET) &
	    MPI2_DIAG_RESET_ADAPTER) == 0)
		goto out;

	ret = DDI_FAILURE;
out:
	lmrc_write_reg(lmrc, MPI2_WRITE_SEQUENCE_OFFSET,
	    MPI2_WRSEQ_FLUSH_KEY_VALUE);
	return (ret);
}

/*
 * lmrc_reset_ctrl
 *
 * Attempt to reset the controller, if the hardware supports it.
 * If reset is unsupported or the reset fails repeatedly, we shut the
 * controller down.
 */
static int
lmrc_reset_ctrl(lmrc_t *lmrc)
{
	uint32_t status, fw_state, reset_adapter;
	int max_wait, i;

	if (lmrc->l_disable_online_ctrl_reset)
		return (DDI_FAILURE);

	status = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);
	fw_state = LMRC_FW_STATE(status);
	reset_adapter = LMRC_FW_RESET_ADAPTER(status);

	if (fw_state == LMRC_FW_STATE_FAULT && reset_adapter == 0) {
		dev_err(lmrc->l_dip, CE_WARN,
		    "FW in fault state, but reset not supported");
		goto out;
	}

	for (i = 0; i < LMRC_MAX_RESET_TRIES; i++) {
		dev_err(lmrc->l_dip, CE_WARN, "resetting...");

		if (lmrc_hard_reset(lmrc) != DDI_SUCCESS)
			continue;

		/* Wait for the FW state to move beyond INIT. */
		max_wait = LMRC_IO_TIMEOUT * MILLISEC / 100;
		do {
			status = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);
			fw_state = LMRC_FW_STATE(status);

			if (fw_state <= LMRC_FW_STATE_FW_INIT)
				delay(drv_usectohz(100 * MILLISEC));
		} while (fw_state <= LMRC_FW_STATE_FW_INIT && max_wait > 0);

		if (fw_state <= LMRC_FW_STATE_FW_INIT) {
			dev_err(lmrc->l_dip, CE_WARN,
			    "fw state <= LMRC_FW_STATE_FW_INIT, state = %x",
			    fw_state);
			continue;
		}

		return (DDI_SUCCESS);
	}

	dev_err(lmrc->l_dip, CE_WARN, "reset failed");
out:
	/* Stop the controller. */
	lmrc_write_reg(lmrc, MPI2_DOORBELL_OFFSET, MFI_STOP_ADP);
	(void) lmrc_read_reg(lmrc, MPI2_DOORBELL_OFFSET);

	return (DDI_FAILURE);
}

/*
 * lmrc_tgt_complete_cmd
 *
 * In case of a controller reset, complete the cmd and clean up. This is done
 * in a taskq to avoid locking and list manipulation headaches.
 */
static void
lmrc_tgt_complete_cmd(void *arg)
{
	lmrc_scsa_cmd_t *cmd = arg;
	struct scsi_pkt *pkt;
	lmrc_t *lmrc;

	mutex_enter(&cmd->sc_mpt->mpt_lock);

	/* Just in case the command completed before the taskq was run... */
	if (cmd->sc_mpt->mpt_complete) {
		mutex_exit(&cmd->sc_mpt->mpt_lock);
		return;
	}

	lmrc = cmd->sc_mpt->mpt_lmrc;
	pkt = cmd->sc_mpt->mpt_pkt;

	pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD;
	pkt->pkt_reason = CMD_RESET;
	pkt->pkt_statistics = STAT_BUS_RESET;
	mutex_exit(&cmd->sc_mpt->mpt_lock);

	lmrc_tgt_rem_active_mpt(cmd->sc_tgt, cmd->sc_mpt);
	atomic_dec_uint(&lmrc->l_fw_outstanding_cmds);

	scsi_hba_pkt_comp(pkt);
}

/*
 * lmrc_tgt_complete_cmds
 *
 * Walk the list of active commands of a target. Schedule a taskq to handle the
 * timeout processing and clean up.
 */
static void
lmrc_tgt_complete_cmds(lmrc_t *lmrc, lmrc_tgt_t *tgt)
{
	lmrc_mpt_cmd_t *mpt;

	mutex_enter(&tgt->tgt_mpt_active_lock);
	if (list_is_empty(&tgt->tgt_mpt_active)) {
		mutex_exit(&tgt->tgt_mpt_active_lock);
		return;
	}

	for (mpt = lmrc_tgt_first_active_mpt(tgt);
	    mpt != NULL;
	    mpt = lmrc_tgt_next_active_mpt(tgt, mpt)) {
		lmrc_scsa_cmd_t *cmd = mpt->mpt_pkt->pkt_ha_private;

		ASSERT(mutex_owned(&mpt->mpt_lock));
		VERIFY(mpt->mpt_pkt != NULL);
		VERIFY(cmd != NULL);

		if (mpt->mpt_complete)
			continue;

		taskq_dispatch_ent(lmrc->l_taskq, lmrc_tgt_complete_cmd, cmd,
		    TQ_NOSLEEP, &mpt->mpt_tqent);
	}
	mutex_exit(&tgt->tgt_mpt_active_lock);
}

/*
 * lmrc_tgt_timeout_cmds
 *
 * Walk the list of active commands of a target. Try to abort commands which are
 * overdue.
 */
static int
lmrc_tgt_timeout_cmds(lmrc_t *lmrc, lmrc_tgt_t *tgt)
{
	lmrc_mpt_cmd_t *mpt;
	int ret = DDI_SUCCESS;

	mutex_enter(&tgt->tgt_mpt_active_lock);
	if (list_is_empty(&tgt->tgt_mpt_active))
		goto out;

	for (mpt = lmrc_tgt_first_active_mpt(tgt);
	    mpt != NULL;
	    mpt = lmrc_tgt_next_active_mpt(tgt, mpt)) {
		hrtime_t now;

		ASSERT(mutex_owned(&mpt->mpt_lock));
		VERIFY(mpt->mpt_pkt != NULL);

		/* Just in case the command completed by now... */
		if (mpt->mpt_complete)
			continue;

		now = gethrtime();

		if (now > mpt->mpt_timeout) {
			/*
			 * Give the packet a bit more time for the abort to
			 * complete.
			 */
			mpt->mpt_timeout = now + LMRC_IO_TIMEOUT * NANOSEC;

			/*
			 * If the abort failed for whatever reason,
			 * we can stop here as only a controller reset
			 * can get us back into a sane state.
			 */
			if (lmrc_abort_mpt(lmrc, tgt, mpt) != 1) {
				mutex_exit(&mpt->mpt_lock);
				ret = DDI_FAILURE;
				goto out;
			}
		}
	}

out:
	mutex_exit(&tgt->tgt_mpt_active_lock);
	return (ret);
}

/*
 * lmrc_thread
 *
 * Check whether the controller is FW fault state. Check all targets for
 * commands which have timed out.
 */
void
lmrc_thread(void *arg)
{
	lmrc_t *lmrc = arg;

	do {
		int i;

		/* Wake up at least once a minute. */
		mutex_enter(&lmrc->l_thread_lock);
		(void) cv_reltimedwait(&lmrc->l_thread_cv, &lmrc->l_thread_lock,
		    drv_usectohz(60 * MICROSEC), TR_SEC);
		mutex_exit(&lmrc->l_thread_lock);

		if (lmrc->l_thread_stop)
			continue;

		lmrc->l_fw_fault = lmrc_check_fw_fault(lmrc);

		/*
		 * Check all targets for timed-out commands. If we find any
		 * and fail to abort them, we pretend the FW has faulted to
		 * trigger a reset.
		 */
		if (!lmrc->l_fw_fault) {
			for (i = 0; i < ARRAY_SIZE(lmrc->l_targets); i++) {
				if (lmrc_tgt_timeout_cmds(lmrc,
				    &lmrc->l_targets[i]) != DDI_SUCCESS) {
					lmrc->l_fw_fault = B_TRUE;
					break;
				}
			}
		}

		/*
		 * If the FW is faulted, try to recover by performing a reset.
		 */
		if (lmrc->l_fw_fault) {
			int ret;

			lmrc_disable_intr(lmrc);

			/*
			 * Even if the reset failed, it will have stopped the
			 * controller and we can complete all outstanding
			 * commands.
			 */
			ret = lmrc_reset_ctrl(lmrc);

			(void) lmrc_abort_outstanding_mfi(lmrc,
			    LMRC_MAX_MFI_CMDS);

			for (i = 0; i < ARRAY_SIZE(lmrc->l_targets); i++)
				lmrc_tgt_complete_cmds(lmrc,
				    &lmrc->l_targets[i]);

			if (ret != DDI_SUCCESS) {
				dev_err(lmrc->l_dip, CE_WARN, "reset failed");
				continue;
			}

			if (lmrc_transition_to_ready(lmrc) != DDI_SUCCESS)
				continue;

			if (lmrc_ioc_init(lmrc) != DDI_SUCCESS)
				continue;

			lmrc_enable_intr(lmrc);

			if (lmrc_start_aen(lmrc) != DDI_SUCCESS) {
				dev_err(lmrc->l_dip, CE_WARN,
				    "failed to re-initiate AEN");
				continue;
			}

			lmrc->l_fw_fault = lmrc_check_fw_fault(lmrc);
		}
	} while (!lmrc->l_thread_stop);

	thread_exit();
}

/*
 * lmrc_transition_to_ready
 *
 * Move firmware to ready state. At attach time, the FW can potentially be in
 * any one of several possible states. If the FW is in operational, waiting-for-
 * handshake states, take steps to bring it to ready state. Otherwise, wait for
 * the FW to reach ready state.
 */
static int
lmrc_transition_to_ready(lmrc_t *lmrc)
{
	uint32_t status, new_status;
	uint32_t fw_state;
	uint8_t max_wait;
	uint_t i;

	status = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);
	fw_state = LMRC_FW_STATE(status);
	max_wait = LMRC_RESET_TIMEOUT;

	while (fw_state != LMRC_FW_STATE_READY) {
		switch (fw_state) {
		case LMRC_FW_STATE_FAULT:
			dev_err(lmrc->l_dip, CE_NOTE, "FW is in fault state!");
			if (lmrc_reset_ctrl(lmrc) != DDI_SUCCESS)
				return (DDI_FAILURE);
			break;

		case LMRC_FW_STATE_WAIT_HANDSHAKE:
			/* Set the CLR bit in inbound doorbell */
			lmrc_write_reg(lmrc, MPI2_DOORBELL_OFFSET,
			    MFI_INIT_CLEAR_HANDSHAKE | MFI_INIT_HOTPLUG);
			break;

		case LMRC_FW_STATE_BOOT_MSG_PENDING:
			lmrc_write_reg(lmrc, MPI2_DOORBELL_OFFSET,
			    MFI_INIT_HOTPLUG);
			break;

		case LMRC_FW_STATE_OPERATIONAL:
			/* Bring it to READY state, wait up to 10s */
			lmrc_disable_intr(lmrc);
			lmrc_write_reg(lmrc, MPI2_DOORBELL_OFFSET,
			    MFI_RESET_FLAGS);
			(void) lmrc_wait_for_reg(lmrc, MPI2_DOORBELL_OFFSET, 1,
			    0, 10);
			break;

		case LMRC_FW_STATE_UNDEFINED:
			/* This state should not last for more than 2 sec */
		case LMRC_FW_STATE_BB_INIT:
		case LMRC_FW_STATE_FW_INIT:
		case LMRC_FW_STATE_FW_INIT_2:
		case LMRC_FW_STATE_DEVICE_SCAN:
		case LMRC_FW_STATE_FLUSH_CACHE:
			break;
		default:
			dev_err(lmrc->l_dip, CE_WARN, "Unknown FW state %x",
			    fw_state);
			return (DDI_FAILURE);
		}

		/*
		 * The current state should not last for more than max_wait
		 * seconds.
		 */
		for (i = 0; i < max_wait * 1000; i++) {
			new_status = lmrc_read_reg(lmrc,
			    MPI26_SCRATCHPAD0_OFFSET);

			if (status != new_status)
				break;

			delay(drv_usectohz(MILLISEC));
		}

		if (new_status == status) {
			dev_err(lmrc->l_dip, CE_WARN,
			    "FW state (%x) hasn't changed in %d seconds",
			    fw_state, max_wait);
			return (DDI_FAILURE);
		}

		status = new_status;
		fw_state = LMRC_FW_STATE(status);
	}

	if (lmrc_check_acc_handle(lmrc->l_reghandle) != DDI_FM_OK)
		return (DDI_FAILURE);

	return (DDI_SUCCESS);
}

/*
 * lmrc_adapter_init
 *
 * Get the hardware and firmware into a usable state, and fetch some basic
 * information from the registers to calculate sizes of basic data structures.
 */
int
lmrc_adapter_init(lmrc_t *lmrc)
{
	uint32_t reg;
	int ret;
	int i;

	ret = lmrc_transition_to_ready(lmrc);
	if (ret != DDI_SUCCESS)
		return (ret);

	/*
	 * Get maximum RAID map size.
	 */
	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD2_OFFSET);
	lmrc->l_max_raid_map_sz = LMRC_MAX_RAID_MAP_SZ(reg);

	lmrc->l_max_reply_queues = 1;
	lmrc->l_rphi[0] = MPI2_REPLY_POST_HOST_INDEX_OFFSET;

	/*
	 * Apparently, bit 27 of the scratch pad register indicates whether
	 * MSI-X is supported by the firmware.
	 */
	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);

	if (LMRC_FW_MSIX_ENABLED(reg)) {
		lmrc->l_fw_msix_enabled = B_TRUE;

		reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD1_OFFSET);
		lmrc->l_max_reply_queues = LMRC_MAX_REPLY_QUEUES_EXT(reg);

		if (lmrc->l_max_reply_queues > LMRC_MAX_REPLY_POST_HOST_INDEX) {
			lmrc->l_msix_combined = B_TRUE;
			lmrc->l_rphi[0] =
			    MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET;
		}

		/*
		 * Compute reply post index register addresses 1-15.
		 */
		for (i = 1; i < LMRC_MAX_REPLY_POST_HOST_INDEX; i++) {
			lmrc->l_rphi[i] = i * 0x10 +
			    MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET;
		}
	}

	/*
	 * Get the number of commands the firmware supports. Use one less,
	 * because reply_q_depth is based on one more than this. XXX: Why?
	 */
	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD0_OFFSET);
	lmrc->l_max_fw_cmds = LMRC_FW_MAX_CMD(reg) - 1;

	if (lmrc->l_max_fw_cmds < LMRC_MAX_MFI_CMDS) {
		dev_err(lmrc->l_dip, CE_WARN, "!max_fw_cmds too low: %d",
		    lmrc->l_max_fw_cmds);
		return (DDI_FAILURE);
	}

	/*
	 * Reserve some commands for MFI, the remainder is for SCSI commands.
	 */
	lmrc->l_max_scsi_cmds = lmrc->l_max_fw_cmds - LMRC_MAX_MFI_CMDS;

	/*
	 * XXX: This magic calculation isn't explained anywhere. Let's see...
	 * lmrc_max_fw_cmds + 1 gives us what was reported in the register,
	 * That + 15 is for rounding it up the next multiple of 16, which
	 * / 16 * 16 does.
	 * And apparently we want twice that much for queue depth. Why?
	 *
	 * So in reality, the queue depth is based on at least one more than
	 * lmrc_max_fw_cmds, but it could be even more. That makes the above
	 * statement about lmrc_max_fw_cmds questionable.
	 */
	lmrc->l_reply_q_depth = (lmrc->l_max_fw_cmds + 1 + 15) / 16 * 16 * 2;

	/* Allocation size of one reply queue, based on depth. */
	lmrc->l_reply_alloc_sz =
	    sizeof (Mpi2ReplyDescriptorsUnion_t) * lmrc->l_reply_q_depth;

	/* Allocation size of the DMA memory used for all MPI I/O frames. */
	lmrc->l_io_frames_alloc_sz = LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE *
	    (lmrc->l_max_fw_cmds + 2);

	/*
	 * If LMRC_EXT_CHAIN_SIZE_SUPPORT is set in scratch pad 1, firmware
	 * supports an extended IO chain frame which is 4 times the size of a
	 * legacy firmware frame.
	 * Legacy Firmware frame size is (8 * 128) = 1K
	 * 1M IO Firmware frame size is (8 * 128 * 4) = 4K
	 */
	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD1_OFFSET);
	lmrc->l_max_chain_frame_sz = LMRC_MAX_CHAIN_SIZE(reg) *
	    (LMRC_EXT_CHAIN_SIZE_SUPPORT(reg) ? LMRC_1MB_IO : LMRC_256K_IO);

	/*
	 * Check whether the controller supports DMA to the full 64bit address
	 * space.
	 */
	lmrc->l_64bit_dma_support = LMRC_64BIT_DMA_SUPPORT(reg);

	/*
	 * We use a I/O frame size of 256 bytes, that is what
	 * LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE is set to.
	 *
	 * The offset of the SGL in the I/O frame is 128, so
	 * there are 128 bytes left for 8 SGEs of 16 bytes each.
	 */
	lmrc->l_max_sge_in_main_msg =
	    (LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
	    offsetof(Mpi25SCSIIORequest_t, SGL)) / sizeof (Mpi25SGEIOUnion_t);

	/*
	 * Similarly, number of SGE in a SGE chain frame.
	 */
	lmrc->l_max_sge_in_chain =
	    lmrc->l_max_chain_frame_sz / sizeof (Mpi25SGEIOUnion_t);

	/*
	 * The total number of SGE we support in a transfer is sum of
	 * the above two, minus one for the link (last SGE in main msg).
	 *
	 * XXX: So why -2?
	 */
	lmrc->l_max_num_sge =
	    lmrc->l_max_sge_in_main_msg + lmrc->l_max_sge_in_chain - 2;

	/*
	 * The offset of the last SGE in the I/O request, used for linking
	 * the SGE chain frame if necessary.
	 */
	lmrc->l_chain_offset_io_request =
	    (LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
	    sizeof (Mpi25SGEIOUnion_t)) / sizeof (Mpi25SGEIOUnion_t);

	/*
	 * For MFI passthru, the link to the SGE chain frame is always
	 * the first SGE in the I/O frame, the other SGEs in the I/O frame
	 * will not be used.
	 */
	lmrc->l_chain_offset_mfi_pthru =
	    offsetof(Mpi25SCSIIORequest_t, SGL) / sizeof (Mpi25SGEIOUnion_t);


	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD3_OFFSET);
	if (LMRC_NVME_PAGE_SHIFT(reg) > LMRC_DEFAULT_NVME_PAGE_SHIFT) {
		lmrc->l_nvme_page_sz = 1 << LMRC_NVME_PAGE_SHIFT(reg);
		dev_err(lmrc->l_dip, CE_NOTE, "!NVME page size: %ld",
		    lmrc->l_nvme_page_sz);
	}

	reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD1_OFFSET);
	lmrc->l_fw_sync_cache_support = LMRC_SYNC_CACHE_SUPPORT(reg);

	if (lmrc->l_class == LMRC_ACLASS_AERO) {
		reg = lmrc_read_reg(lmrc, MPI26_SCRATCHPAD1_OFFSET);
		lmrc->l_atomic_desc_support =
		    LMRC_ATOMIC_DESCRIPTOR_SUPPORT(reg);
	}

	return (DDI_SUCCESS);
}

/*
 * lmrc_ioc_init
 *
 * Manually build a MFI IOC INIT command to setup basic operating parameters
 * such as the DMA parameters for the I/O request frames and the reply post
 * queues. Send the IOC INIT command using a special request descriptor which
 * directly includes the physical address of the MFI command frame.
 *
 * After this command completes, the controller is ready to accept MPT commands
 * using the normal method of placing it in the I/O request DMA memory and
 * writing a MPT request descripter to the appropriate registers.
 */
int
lmrc_ioc_init(lmrc_t *lmrc)
{
	lmrc_mfi_cmd_t *mfi = lmrc_get_mfi(lmrc);
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	mfi_init_payload_t *init = &mfi->mfi_frame->mf_init;
	lmrc_req_desc_t req_desc;
	Mpi2IOCInitRequest_t *IOCInitMsg;
	lmrc_dma_t dma;
	int ret = DDI_SUCCESS;

	ret = lmrc_dma_alloc(lmrc, lmrc->l_dma_attr, &dma,
	    sizeof (Mpi2IOCInitRequest_t), 256, DDI_DMA_CONSISTENT);
	if (ret != DDI_SUCCESS) {
		lmrc_put_mfi(mfi);
		dev_err(lmrc->l_dip, CE_WARN,
		    "!%s: failed to allocate IOC command", __func__);
		return (DDI_FAILURE);
	}

	IOCInitMsg = dma.ld_buf;
	IOCInitMsg->Function = MPI2_FUNCTION_IOC_INIT;
	IOCInitMsg->WhoInit = MPI2_WHOINIT_HOST_DRIVER;
	IOCInitMsg->MsgVersion = MPI2_VERSION;
	IOCInitMsg->HeaderVersion = MPI2_HEADER_VERSION;
	IOCInitMsg->SystemRequestFrameSize =
	    LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4;
	IOCInitMsg->ReplyDescriptorPostQueueDepth = lmrc->l_reply_q_depth;
	lmrc_dma_set_addr64(&lmrc->l_reply_dma,
	    (uint64_t *)&IOCInitMsg->ReplyDescriptorPostQueueAddress);
	lmrc_dma_set_addr64(&lmrc->l_ioreq_dma,
	    (uint64_t *)&IOCInitMsg->SystemRequestFrameBaseAddress);
	IOCInitMsg->HostMSIxVectors = lmrc->l_max_reply_queues;
	/* XXX: Why NVMe? */
	IOCInitMsg->HostPageSize = LMRC_DEFAULT_NVME_PAGE_SHIFT;

	hdr->mh_cmd = MFI_CMD_INIT;
	hdr->mh_cmd_status = MFI_STAT_INVALID_STATUS;
	hdr->mh_flags = MFI_FRAME_DONT_POST_IN_REPLY_QUEUE;

	hdr->mh_drv_opts.mc_support_additional_msix = 1;
	hdr->mh_drv_opts.mc_support_max_255lds = 1;
	hdr->mh_drv_opts.mc_support_ndrive_r1_lb = 1;
	hdr->mh_drv_opts.mc_support_security_protocol_cmds_fw = 1;
	hdr->mh_drv_opts.mc_support_ext_io_size = 1;

	hdr->mh_data_xfer_len = lmrc_dma_get_size(&dma);

	lmrc_dma_set_addr64(&dma, &init->mi_queue_info_new_phys_addr);

	lmrc_dma_set_addr64(&mfi->mfi_frame_dma, &req_desc.rd_reg);
	VERIFY0(req_desc.rd_mfa_io.RequestFlags);
	req_desc.rd_mfa_io.RequestFlags = LMRC_REQ_DESCRIPT_FLAGS_MFA;

	lmrc_disable_intr(lmrc);
	if (!lmrc_wait_for_reg(lmrc, MPI2_DOORBELL_OFFSET, 1, 0, 10))
		return (DDI_FAILURE);

	(void) ddi_dma_sync(dma.ld_hdl, 0, dma.ld_len, DDI_DMA_SYNC_FORDEV);
	(void) ddi_dma_sync(mfi->mfi_frame_dma.ld_hdl, 0,
	    mfi->mfi_frame_dma.ld_len, DDI_DMA_SYNC_FORDEV);

	lmrc_send_request(lmrc, req_desc);

	mutex_enter(&mfi->mfi_lock);
	ret = lmrc_poll_mfi(lmrc, mfi, LMRC_INTERNAL_CMD_WAIT_TIME);
	mutex_exit(&mfi->mfi_lock);

	if (ret != DDI_SUCCESS) {
		if (hdr->mh_cmd_status != MFI_STAT_INVALID_STATUS)
			dev_err(lmrc->l_dip, CE_WARN,
			    "!IOC Init failed, status = 0x%x",
			    hdr->mh_cmd_status);
	}

	lmrc_dma_free(&dma);
	lmrc_put_mfi(mfi);

	return (ret);
}

/*
 * lmrc_get_ctrl_info
 *
 * Build a MFI DCMD to get controller information from FW. Update the copy in
 * the soft state.
 */
static int
lmrc_get_ctrl_info(lmrc_t *lmrc)
{
	mfi_ctrl_info_t *ci = lmrc->l_ctrl_info;
	lmrc_mfi_cmd_t *mfi;
	int ret;

	mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ, MFI_DCMD_CTRL_GET_INFO,
	    sizeof (mfi_ctrl_info_t), 1);

	if (mfi == NULL)
		return (DDI_FAILURE);

	ret = lmrc_issue_blocked_mfi(lmrc, mfi);

	if (ret != DDI_SUCCESS)
		goto out;

	(void) ddi_dma_sync(mfi->mfi_data_dma.ld_hdl, 0,
	    mfi->mfi_data_dma.ld_len, DDI_DMA_SYNC_FORKERNEL);
	bcopy(mfi->mfi_data_dma.ld_buf, ci, sizeof (mfi_ctrl_info_t));

out:
	lmrc_put_dcmd(lmrc, mfi);
	return (ret);
}

/*
 * lmrc_fw_init
 *
 * Complete firmware initialization. At this point, we can already send MFI
 * commands. so we can start by getting the controller information from the
 * firmware and set up things in our soft state. Next we issue the commands
 * to get the PD map and RAID map, which will complete asynchronously when
 * new information is available and then re-send themselves.
 */
int
lmrc_fw_init(lmrc_t *lmrc)
{
	int drv_max_lds = MFI_MAX_LOGICAL_DRIVES;
	mfi_ctrl_info_t *ci = lmrc->l_ctrl_info;
	int ret;

	ret = lmrc_get_ctrl_info(lmrc);
	if (ret != DDI_SUCCESS) {
		dev_err(lmrc->l_dip, CE_WARN, "!Unable to get FW ctrl info.");
		return (DDI_FAILURE);
	}

	lmrc->l_disable_online_ctrl_reset =
	    ci->ci_prop.cp_disable_online_ctrl_reset == 1;

	lmrc->l_max_256_vd_support =
	    ci->ci_adapter_opts3.ao3_support_max_ext_lds == 1;

	if (ci->ci_max_lds > 64) {
		lmrc->l_max_256_vd_support = B_TRUE;
		drv_max_lds = LMRC_MAX_LOGICAL_DRIVES_EXT;
	}

	lmrc->l_fw_supported_vd_count = min(ci->ci_max_lds, drv_max_lds);

	lmrc->l_fw_supported_pd_count = min(ci->ci_max_pds,
	    MFI_MAX_PHYSICAL_DRIVES);

	lmrc->l_max_map_sz = lmrc->l_current_map_sz =
	    lmrc->l_max_raid_map_sz * LMRC_MIN_MAP_SIZE;

	lmrc->l_use_seqnum_jbod_fp =
	    ci->ci_adapter_opts3.ao3_use_seq_num_jbod_FP != 0;

	lmrc->l_pdmap_tgtid_support =
	    ci->ci_adapter_opts4.ao4_support_pd_map_target_id != 0;

	return (DDI_SUCCESS);
}


/*
 * lmrc_ctrl_shutdown
 *
 * Called by lmrc_quiesce() to send a shutdown command to the controller.
 * Cannot use locks, therefore cannot use lmrc_get_dcmd() or lmrc_get_mfi().
 */
int
lmrc_ctrl_shutdown(lmrc_t *lmrc)
{
	lmrc_mfi_cmd_t *mfi = list_remove_head(&lmrc->l_mfi_cmd_list);
	mfi_header_t *hdr;
	mfi_dcmd_payload_t *dcmd;

	if (mfi == NULL)
		return (DDI_FAILURE);

	hdr = &mfi->mfi_frame->mf_hdr;
	dcmd = &mfi->mfi_frame->mf_dcmd;

	hdr->mh_cmd = MFI_CMD_DCMD;
	hdr->mh_flags = MFI_FRAME_DONT_POST_IN_REPLY_QUEUE;
	dcmd->md_opcode = MFI_DCMD_CTRL_SHUTDOWN;

	lmrc_disable_intr(lmrc);
	lmrc_issue_mfi(lmrc, mfi, NULL);

	return (DDI_SUCCESS);
}

/*
 * driver target state management
 *
 * The soft state of the controller instance keeps a pre-allocated array of
 * target structures for all possible targets, even though only a small number
 * of them are likely to be used. Each target structure contains back link to
 * the soft state and a mutex, which are never cleared or changed when a target
 * is added or removed.
 */

/*
 * lmrc_tgt_init
 *
 * Initialize the tgt structure for a newly discovered tgt. The same tgt
 * structure is used for PDs and LDs, the distinction can be made by the
 * presence or absence of tgt_pd_info. LDs are always of type disk, the type
 * of PDs is taken from their pd_info. If a device has no SAS WWN, we'll fake
 * the interconnect type to be PARALLEL to make sure device address isn't
 * misunderstood as a WWN by devfsadm.
 */
void
lmrc_tgt_init(lmrc_tgt_t *tgt, uint16_t dev_id, char *addr,
    mfi_pd_info_t *pd_info)
{
	rw_enter(&tgt->tgt_lock, RW_WRITER);

	bzero(&tgt->tgt_dev_id,
	    sizeof (lmrc_tgt_t) - offsetof(lmrc_tgt_t, tgt_dev_id));

	tgt->tgt_dev_id = dev_id;
	tgt->tgt_pd_info = pd_info;
	tgt->tgt_interconnect_type = INTERCONNECT_SAS;

	if (pd_info == NULL) {
		tgt->tgt_type = DTYPE_DIRECT;
	} else {
		tgt->tgt_type = pd_info->pd_scsi_dev_type;
	}

	(void) strlcpy(tgt->tgt_wwnstr, addr, sizeof (tgt->tgt_wwnstr));
	if (scsi_wwnstr_to_wwn(tgt->tgt_wwnstr, &tgt->tgt_wwn) != DDI_SUCCESS) {
		tgt->tgt_interconnect_type = INTERCONNECT_PARALLEL;
		tgt->tgt_wwn = dev_id;
	}

	rw_exit(&tgt->tgt_lock);
}

/*
 * lmrc_tgt_clear
 *
 * Reset the tgt structure of a target which is no longer present.
 */
void
lmrc_tgt_clear(lmrc_tgt_t *tgt)
{
	rw_enter(&tgt->tgt_lock, RW_WRITER);

	if (tgt->tgt_pd_info != NULL)
		kmem_free(tgt->tgt_pd_info, sizeof (mfi_pd_info_t));

	bzero(&tgt->tgt_dev_id,
	    sizeof (lmrc_tgt_t) - offsetof(lmrc_tgt_t, tgt_dev_id));
	tgt->tgt_dev_id = LMRC_DEVHDL_INVALID;
	rw_exit(&tgt->tgt_lock);
}

/*
 * lmrc_tgt_find
 *
 * Walk the target list and find a tgt matching the given scsi_device.
 * Return the tgt read-locked. The targets_lock mutex must be held the
 * whole time.
 */
lmrc_tgt_t *
lmrc_tgt_find(lmrc_t *lmrc, struct scsi_device *sd)
{
	const char *ua = scsi_device_unit_address(sd);
	char *comma, wwnstr[SCSI_WWN_BUFLEN];
	uint64_t wwn;
	unsigned long tgtid;
	lmrc_tgt_t *tgt;
	size_t i;

	VERIFY(ua != NULL);

	(void) strlcpy(wwnstr, ua, sizeof (wwnstr));

	/*
	 * If the unit address is a valid target ID and within range for
	 * VD IDs, use that.
	 */
	if (ddi_strtoul(wwnstr, &comma, 10, &tgtid) == 0 &&
	    *comma == ',' &&
	    tgtid <= lmrc->l_fw_supported_vd_count) {
		tgt = &lmrc->l_targets[tgtid];

		rw_enter(&tgt->tgt_lock, RW_READER);
		if (tgt->tgt_dev_id == tgtid &&
		    tgt->tgt_wwn == tgtid) {
			return (tgt);
		}
		rw_exit(&tgt->tgt_lock);
	}

	/* Chop off ",lun" as scsi_wwnstr_to_wwn() can't handle it. */
	comma = strchr(wwnstr, ',');
	if (comma != NULL)
		*comma = '\0';

	/* Else, if unit address is a valid WWN, look for that. */
	if (scsi_wwnstr_to_wwn(wwnstr, &wwn) == DDI_SUCCESS) {
		for (i = 0; i < ARRAY_SIZE(lmrc->l_targets); i++) {
			tgt = &lmrc->l_targets[i];

			rw_enter(&tgt->tgt_lock, RW_READER);
			if (tgt->tgt_wwn == wwn) {
				return (tgt);
			}
			rw_exit(&tgt->tgt_lock);
		}
	} else {
		/* Do it the hard way and compare wwnstr. */
		for (i = 0; i < ARRAY_SIZE(lmrc->l_targets); i++) {
			tgt = &lmrc->l_targets[i];

			rw_enter(&tgt->tgt_lock, RW_READER);
			if (strcmp(tgt->tgt_wwnstr, wwnstr) == 0) {
				return (tgt);
			}
			rw_exit(&tgt->tgt_lock);
		}
	}

	return (NULL);
}

/*
 * MPT/MFI command management
 *
 * For each kind of command, MFI and MPT, the driver keeps an array of pre-
 * allocated and pre-initialized commands. Additionally, it keeps two lists of
 * currently unused commands. A set of functions is provided for each list to
 * get and put commands from/to the list. Commands are initialized during get(),
 * because having completed commands on the list can help in certain cases
 * during debugging.
 *
 * MPT commands in use for I/O are kept on a active command list of the target
 * they are addressing. All other types of commands are not kept on any list
 * while they are being processed by the hardware. When walking the command
 * arrays, busy commands not associated with a target can be distinguished by
 * not being linked on any list.
 */

/*
 * lmrc_get_mpt
 *
 * Get a MPT command from the list and initialize it. Return the command locked.
 * Return NULL if the MPT command list is empty.
 */
lmrc_mpt_cmd_t *
lmrc_get_mpt(lmrc_t *lmrc)
{
	lmrc_mpt_cmd_t *mpt;
	Mpi25SCSIIORequest_t *io_req;

	mutex_enter(&lmrc->l_mpt_cmd_lock);
	mpt = list_remove_head(&lmrc->l_mpt_cmd_list);
	mutex_exit(&lmrc->l_mpt_cmd_lock);
	if (mpt == NULL)
		return (NULL);

	mutex_enter(&mpt->mpt_lock);
	bzero(mpt->mpt_io_frame, LMRC_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
	bzero(mpt->mpt_chain_dma.ld_buf, mpt->mpt_chain_dma.ld_len);
	bzero(mpt->mpt_sense_dma.ld_buf, mpt->mpt_sense_dma.ld_len);

	mpt->mpt_mfi = NULL;
	mpt->mpt_pkt = NULL;

	/* Set the offset of the SGL entries inside the MPT command. */
	io_req = mpt->mpt_io_frame;
	io_req->SGLOffset0 = offsetof(Mpi25SCSIIORequest_t, SGL) / 4;

	mpt->mpt_complete = B_FALSE;
	cv_init(&mpt->mpt_cv, NULL, CV_DRIVER, NULL);

	return (mpt);
}

/*
 * lmrc_put_mpt
 *
 * Put a MPT command back on the list. The command lock must be held when this
 * function is called, being unlocked only after the command has been put on
 * the free list. The command CV is destroyed, thereby asserting that no one is
 * still waiting on it.
 */
void
lmrc_put_mpt(lmrc_mpt_cmd_t *mpt)
{
	lmrc_t *lmrc = mpt->mpt_lmrc;

	VERIFY(lmrc != NULL);

	ASSERT0(list_link_active(&mpt->mpt_node));
	ASSERT(mutex_owned(&mpt->mpt_lock));
	cv_destroy(&mpt->mpt_cv);

	mutex_enter(&lmrc->l_mpt_cmd_lock);
	list_insert_tail(&lmrc->l_mpt_cmd_list, mpt);
	mutex_exit(&lmrc->l_mpt_cmd_lock);
	mutex_exit(&mpt->mpt_lock);
}

/*
 * lmrc_get_mfi
 *
 * Get a MFI command from the list and initialize it.
 */
lmrc_mfi_cmd_t *
lmrc_get_mfi(lmrc_t *lmrc)
{
	lmrc_mfi_cmd_t *mfi;

	mutex_enter(&lmrc->l_mfi_cmd_lock);
	mfi = list_remove_head(&lmrc->l_mfi_cmd_list);
	mutex_exit(&lmrc->l_mfi_cmd_lock);
	VERIFY(mfi != NULL);

	mutex_enter(&mfi->mfi_lock);
	bzero(mfi->mfi_frame, sizeof (mfi_frame_t));
	mfi->mfi_frame->mf_hdr.mh_context = mfi->mfi_idx;
	mfi->mfi_callback = NULL;

	cv_init(&mfi->mfi_cv, NULL, CV_DRIVER, NULL);
	mutex_exit(&mfi->mfi_lock);

	return (mfi);
}

/*
 * lmrc_put_mfi
 *
 * Put a MFI command back on the list. Destroy the CV, thereby
 * asserting that no one is waiting on it.
 */
void
lmrc_put_mfi(lmrc_mfi_cmd_t *mfi)
{
	lmrc_t *lmrc = mfi->mfi_lmrc;

	VERIFY(lmrc != NULL);

	ASSERT0(list_link_active(&mfi->mfi_node));

	mutex_enter(&mfi->mfi_lock);

	cv_destroy(&mfi->mfi_cv);

	mutex_enter(&lmrc->l_mfi_cmd_lock);
	list_insert_tail(&lmrc->l_mfi_cmd_list, mfi);
	mutex_exit(&lmrc->l_mfi_cmd_lock);
	mutex_exit(&mfi->mfi_lock);
}

/*
 * lmrc_abort_outstanding_mfi
 *
 * Walk the MFI cmd array and abort each command which is still outstanding,
 * which is indicated by not being linked on l_mfi_cmd_list.
 *
 * As a special case, if the FW is in fault state, just call each commands
 * completion callback.
 */
int
lmrc_abort_outstanding_mfi(lmrc_t *lmrc, const size_t ncmd)
{
	int ret;
	int i;

	for (i = 0; i < ncmd; i++) {
		lmrc_mfi_cmd_t *mfi = lmrc->l_mfi_cmds[i];

		mutex_enter(&mfi->mfi_lock);
		if (list_link_active(&mfi->mfi_node)) {
			mutex_exit(&mfi->mfi_lock);
			continue;
		}

		/*
		 * If the FW is faulted, wake up anyone waiting on the command
		 * to clean it up.
		 */
		if (lmrc->l_fw_fault) {
			if (mfi->mfi_callback != NULL)
				mfi->mfi_callback(lmrc, mfi);
			mutex_exit(&mfi->mfi_lock);
			continue;
		}

		ret = lmrc_abort_cmd(lmrc, mfi);
		mutex_exit(&mfi->mfi_lock);
		if (ret != DDI_SUCCESS)
			return (ret);

		lmrc_dma_free(&mfi->mfi_data_dma);
		lmrc_put_mfi(mfi);
	}

	return (DDI_SUCCESS);
}

/*
 * lmrc_get_dcmd
 *
 * Build a MFI DCMD with DMA memory for data transfers.
 */
lmrc_mfi_cmd_t *
lmrc_get_dcmd(lmrc_t *lmrc, uint16_t flags, uint32_t opcode, uint32_t xferlen,
    uint_t align)
{
	lmrc_mfi_cmd_t *mfi = lmrc_get_mfi(lmrc);
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;
	mfi_dcmd_payload_t *dcmd = &mfi->mfi_frame->mf_dcmd;
	lmrc_dma_t *dma = &mfi->mfi_data_dma;
	int ret;

	hdr->mh_cmd = MFI_CMD_DCMD;
	hdr->mh_flags = flags;

	dcmd->md_opcode = opcode;

	if ((flags & MFI_FRAME_DIR_READ) != 0 ||
	    (flags & MFI_FRAME_DIR_WRITE) != 0) {
		ret = lmrc_dma_alloc(lmrc, lmrc->l_dma_attr, dma, xferlen,
		    align, DDI_DMA_CONSISTENT);
		if (ret != DDI_SUCCESS) {
			lmrc_put_mfi(mfi);
			return (NULL);
		}

		hdr->mh_flags |= MFI_FRAME_SGL64;
		hdr->mh_sge_count = 1;
		hdr->mh_data_xfer_len = lmrc_dma_get_size(dma);

		dcmd->md_sgl.ms64_length = lmrc_dma_get_size(dma);
		lmrc_dma_set_addr64(dma, &dcmd->md_sgl.ms64_phys_addr);
	}

	return (mfi);
}

/*
 * lmrc_put_dcmd
 *
 * Free the DMA memory of a MFI DCMD and return the command back on the list.
 */
void
lmrc_put_dcmd(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
{
	lmrc_dma_free(&mfi->mfi_data_dma);
	lmrc_put_mfi(mfi);
}


/*
 * Asynchronous Event Notifications
 */
/*
 * lmrc_get_event_log_info
 *
 * Get the Event Log Info from the firmware.
 */
static int
lmrc_get_event_log_info(lmrc_t *lmrc, mfi_evt_log_info_t *eli)
{
	lmrc_mfi_cmd_t *mfi;
	int ret;

	mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ,
	    MFI_DCMD_CTRL_EVENT_GET_INFO, sizeof (mfi_evt_log_info_t), 1);

	if (mfi == NULL)
		return (DDI_FAILURE);

	ret = lmrc_issue_blocked_mfi(lmrc, mfi);

	if (ret != DDI_SUCCESS)
		goto out;

	bcopy(mfi->mfi_data_dma.ld_buf, eli, sizeof (mfi_evt_log_info_t));

out:
	lmrc_put_dcmd(lmrc, mfi);
	return (ret);
}

/*
 * lmrc_aen_handler
 *
 * Check the event code and handle it as needed. In the case of PD or LD related
 * events, invoke their special handlers.
 */
static void
lmrc_aen_handler(void *arg)
{
	lmrc_mfi_cmd_t *mfi = arg;
	lmrc_t *lmrc = mfi->mfi_lmrc;
	mfi_evt_detail_t *evt = mfi->mfi_data_dma.ld_buf;
	mfi_dcmd_payload_t *dcmd = &mfi->mfi_frame->mf_dcmd;
	int ret = DDI_FAILURE;

	/* Controller & Configuration specific events */
	switch (evt->evt_code) {
	case MFI_EVT_CFG_CLEARED:
	case MFI_EVT_CTRL_HOST_BUS_SCAN_REQD:
	case MFI_EVT_FOREIGN_CFG_IMPORTED:
		ret = lmrc_get_pd_list(lmrc);
		if (ret != DDI_SUCCESS)
			break;

		ret = lmrc_get_ld_list(lmrc);
		break;

	case MFI_EVT_CTRL_PROP_CHANGED:
		ret = lmrc_get_ctrl_info(lmrc);
		break;

	case MFI_EVT_CTRL_PATROL_READ_START:
	case MFI_EVT_CTRL_PATROL_READ_RESUMED:
	case MFI_EVT_CTRL_PATROL_READ_COMPLETE:
	case MFI_EVT_CTRL_PATROL_READ_CANT_START:
	case MFI_EVT_CTRL_PERF_COLLECTION:
	case MFI_EVT_CTRL_BOOTDEV_SET:
	case MFI_EVT_CTRL_BOOTDEV_RESET:
	case MFI_EVT_CTRL_PERSONALITY_CHANGE:
	case MFI_EVT_CTRL_PERSONALITY_CHANGE_PEND:
	case MFI_EVT_CTRL_NR_OF_VALID_SNAPDUMP:
		break;

	default:
		/* LD-specific events */
		if ((evt->evt_cl.evt_locale & MFI_EVT_LOCALE_LD) != 0)
			ret = lmrc_raid_aen_handler(lmrc, evt);

		/* PD-specific events */
		else if ((evt->evt_cl.evt_locale & MFI_EVT_LOCALE_PD) != 0)
			ret = lmrc_phys_aen_handler(lmrc, evt);

		if (ret != DDI_SUCCESS) {
			dev_err(lmrc->l_dip, CE_NOTE, "!unknown AEN received, "
			    "seqnum = %d, timestamp = %d, code = %x, "
			    "locale = %x, class = %d, argtype = %d",
			    evt->evt_seqnum, evt->evt_timestamp, evt->evt_code,
			    evt->evt_cl.evt_locale, evt->evt_cl.evt_class,
			    evt->evt_argtype);
		}
	}

	dev_err(lmrc->l_dip, CE_NOTE, "!%s", evt->evt_descr);

	/*
	 * Just reuse the command in its entirety. Increase the sequence
	 * number.
	 */
	dcmd->md_mbox_32[0] = evt->evt_seqnum + 1;
	mutex_enter(&mfi->mfi_lock);
	lmrc_issue_mfi(lmrc, mfi, lmrc_complete_aen);
	mutex_exit(&mfi->mfi_lock);
}

/*
 * lmrc_complete_aen
 *
 * An AEN was received, so schedule a taskq to process it.
 */
static void
lmrc_complete_aen(lmrc_t *lmrc, lmrc_mfi_cmd_t *mfi)
{
	mfi_header_t *hdr = &mfi->mfi_frame->mf_hdr;

	ASSERT(mutex_owned(&mfi->mfi_lock));

	if (hdr->mh_cmd_status != MFI_STAT_OK) {
		/* Was the command aborted? */
		if (hdr->mh_cmd_status == MFI_STAT_NOT_FOUND)
			return;

		dev_err(lmrc->l_dip, CE_WARN,
		    "!AEN failed, status = %d",
		    hdr->mh_cmd_status);
		taskq_dispatch_ent(lmrc->l_taskq, (task_func_t *)lmrc_put_mfi,
		    mfi, TQ_NOSLEEP, &mfi->mfi_tqent);
		return;
	}

	taskq_dispatch_ent(lmrc->l_taskq, lmrc_aen_handler, mfi, TQ_NOSLEEP,
	    &mfi->mfi_tqent);
}

/*
 * lmrc_register_aen
 *
 * In FreeBSD, this function checks for an existing AEN. If its class and locale
 * already include what is requested here they just return. In the other case,
 * the existing AEN is aborted and a new one is created, which includes
 * the previous locale and class and new ones.
 *
 * Given that the driver (same as in FreeBSD) calls this function during attach
 * to create an AEN with LOCALE_ALL and CLASS_DEBUG, all of this would be dead
 * code anyway.
 */
static int
lmrc_register_aen(lmrc_t *lmrc, uint32_t seqnum)
{
	lmrc_mfi_cmd_t *mfi;
	mfi_dcmd_payload_t *dcmd;

	mfi = lmrc_get_dcmd(lmrc, MFI_FRAME_DIR_READ, MFI_DCMD_CTRL_EVENT_WAIT,
	    sizeof (mfi_evt_detail_t), 1);

	if (mfi == NULL)
		return (DDI_FAILURE);

	dcmd = &mfi->mfi_frame->mf_dcmd;
	dcmd->md_mbox_32[0] = seqnum;
	dcmd->md_mbox_16[2] = MFI_EVT_LOCALE_ALL;
	dcmd->md_mbox_8[7] = MFI_EVT_CLASS_DEBUG;

	mutex_enter(&mfi->mfi_lock);
	lmrc_issue_mfi(lmrc, mfi, lmrc_complete_aen);
	mutex_exit(&mfi->mfi_lock);

	return (DDI_SUCCESS);
}

/*
 * lmrc_start_aen
 *
 * Set up and enable AEN processing.
 */
int
lmrc_start_aen(lmrc_t *lmrc)
{
	mfi_evt_log_info_t eli;
	int ret;

	bzero(&eli, sizeof (eli));

	/* Get the latest sequence number from the Event Log Info. */
	ret = lmrc_get_event_log_info(lmrc, &eli);
	if (ret != DDI_SUCCESS)
		return (ret);

	/* Register AEN with FW for latest sequence number + 1. */
	ret = lmrc_register_aen(lmrc, eli.eli_newest_seqnum + 1);
	return (ret);
}