xref: /illumos-gate/usr/src/uts/common/io/fibre-channel/fca/qlge/qlge.c (revision 0dc2366f7b9f9f36e10909b1e95edbf2a261c2ac)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 QLogic Corporation. All rights reserved.
 */

#include <qlge.h>
#include <sys/atomic.h>
#include <sys/strsubr.h>
#include <sys/pattr.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <inet/ip.h>



/*
 * Local variables
 */
static struct ether_addr ql_ether_broadcast_addr =
	{0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
static char version[] = "QLogic GLDv3 Driver " VERSIONSTR;

/*
 * Local function prototypes
 */
static void ql_free_resources(dev_info_t *, qlge_t *);
static void ql_fini_kstats(qlge_t *);
static uint32_t ql_get_link_state(qlge_t *);
static void ql_read_conf(qlge_t *);
static int ql_alloc_phys(dev_info_t *, ddi_dma_handle_t *,
    ddi_device_acc_attr_t *, uint_t, ddi_acc_handle_t *,
    size_t, size_t, caddr_t *, ddi_dma_cookie_t *);
static void ql_free_phys(ddi_dma_handle_t *, ddi_acc_handle_t *);
static int ql_set_routing_reg(qlge_t *, uint32_t, uint32_t, int);
static int ql_route_initialize(qlge_t *);
static int ql_attach(dev_info_t *, ddi_attach_cmd_t);
static int ql_detach(dev_info_t *, ddi_detach_cmd_t);
static int ql_bringdown_adapter(qlge_t *);
static int ql_bringup_adapter(qlge_t *);
static int ql_asic_reset(qlge_t *);
static void ql_wake_mpi_reset_soft_intr(qlge_t *);
static void ql_stop_timer(qlge_t *qlge);

/*
 * TX dma maping handlers allow multiple sscatter-gather lists
 */
ddi_dma_attr_t  tx_mapping_dma_attr = {
	DMA_ATTR_V0,			/* dma_attr_version */
	QL_DMA_LOW_ADDRESS,		/* low DMA address range */
	QL_DMA_HIGH_64BIT_ADDRESS,	/* high DMA address range */
	QL_DMA_XFER_COUNTER,		/* DMA counter register */
	QL_DMA_ADDRESS_ALIGNMENT,	/* DMA address alignment, default - 8 */
	QL_DMA_BURSTSIZES,		/* DMA burstsizes */
	QL_DMA_MIN_XFER_SIZE,		/* min effective DMA size */
	QL_DMA_MAX_XFER_SIZE,		/* max DMA xfer size */
	QL_DMA_SEGMENT_BOUNDARY,	/* segment boundary */
	QL_MAX_TX_DMA_HANDLES,		/* s/g list length */
	QL_DMA_GRANULARITY,		/* granularity of device */
	QL_DMA_XFER_FLAGS		/* DMA transfer flags */
};

/*
 * Receive buffers and Request/Response queues do not allow scatter-gather lists
 */
ddi_dma_attr_t  dma_attr = {
	DMA_ATTR_V0,			/* dma_attr_version */
	QL_DMA_LOW_ADDRESS,		/* low DMA address range */
	QL_DMA_HIGH_64BIT_ADDRESS,	/* high DMA address range */
	QL_DMA_XFER_COUNTER,		/* DMA counter register */
	QL_DMA_ADDRESS_ALIGNMENT,	/* DMA address alignment, default - 8 */
	QL_DMA_BURSTSIZES,		/* DMA burstsizes */
	QL_DMA_MIN_XFER_SIZE,		/* min effective DMA size */
	QL_DMA_MAX_XFER_SIZE,		/* max DMA xfer size */
	QL_DMA_SEGMENT_BOUNDARY,	/* segment boundary */
	1,				/* s/g list length, i.e no sg list */
	QL_DMA_GRANULARITY,		/* granularity of device */
	QL_DMA_XFER_FLAGS		/* DMA transfer flags */
};

/*
 * DMA access attribute structure.
 */
/* device register access from host */
ddi_device_acc_attr_t ql_dev_acc_attr = {
	DDI_DEVICE_ATTR_V0,
	DDI_STRUCTURE_LE_ACC,
	DDI_STRICTORDER_ACC
};

/* host ring descriptors */
ddi_device_acc_attr_t ql_desc_acc_attr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

/* host ring buffer */
ddi_device_acc_attr_t ql_buf_acc_attr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

/*
 * Hash key table for Receive Side Scaling (RSS) support
 */
const uint8_t key_data[] = {
	0x23, 0x64, 0xa1, 0xaa, 0x37, 0xc0, 0xed, 0x05, 0x2b, 0x36,
	0x50, 0x5c, 0x45, 0x1e, 0x7e, 0xc8, 0x5d, 0x2a, 0x54, 0x2f,
	0xe4, 0x3d, 0x0f, 0xbb, 0x91, 0xd9, 0x25, 0x60, 0xd4, 0xf8,
	0x12, 0xa0, 0x59, 0x4b, 0x9e, 0x8a, 0x51, 0xda, 0xcd, 0x49};

/*
 * Shadow Registers:
 * Outbound queues have a consumer index that is maintained by the chip.
 * Inbound queues have a producer index that is maintained by the chip.
 * For lower overhead, these registers are "shadowed" to host memory
 * which allows the device driver to track the queue progress without
 * PCI reads. When an entry is placed on an inbound queue, the chip will
 * update the relevant index register and then copy the value to the
 * shadow register in host memory.
 */

static inline unsigned int
ql_read_sh_reg(const volatile void *addr)
{
	return (*(volatile uint32_t *)addr);
}

/*
 * Read 32 bit atomically
 */
uint32_t
ql_atomic_read_32(volatile uint32_t *target)
{
	/*
	 * atomic_add_32_nv returns the new value after the add,
	 * we are adding 0 so we should get the original value
	 */
	return (atomic_add_32_nv(target, 0));
}

/*
 * Set 32 bit atomically
 */
void
ql_atomic_set_32(volatile uint32_t *target, uint32_t newval)
{
	(void) atomic_swap_32(target, newval);
}


/*
 * Setup device PCI configuration registers.
 * Kernel context.
 */
static void
ql_pci_config(qlge_t *qlge)
{
	uint16_t w;

	qlge->vendor_id = (uint16_t)pci_config_get16(qlge->pci_handle,
	    PCI_CONF_VENID);
	qlge->device_id = (uint16_t)pci_config_get16(qlge->pci_handle,
	    PCI_CONF_DEVID);

	/*
	 * we want to respect framework's setting of PCI
	 * configuration space command register and also
	 * want to make sure that all bits of interest to us
	 * are properly set in PCI Command register(0x04).
	 * PCI_COMM_IO		0x1	 I/O access enable
	 * PCI_COMM_MAE		0x2	 Memory access enable
	 * PCI_COMM_ME		0x4	 bus master enable
	 * PCI_COMM_MEMWR_INVAL	0x10	 memory write and invalidate enable.
	 */
	w = (uint16_t)pci_config_get16(qlge->pci_handle, PCI_CONF_COMM);
	w = (uint16_t)(w & (~PCI_COMM_IO));
	w = (uint16_t)(w | PCI_COMM_MAE | PCI_COMM_ME |
	    /* PCI_COMM_MEMWR_INVAL | */
	    PCI_COMM_PARITY_DETECT | PCI_COMM_SERR_ENABLE);

	pci_config_put16(qlge->pci_handle, PCI_CONF_COMM, w);

	ql_dump_pci_config(qlge);
}

/*
 * This routine parforms the neccessary steps to set GLD mac information
 * such as Function number, xgmac mask and shift bits
 */
static int
ql_set_mac_info(qlge_t *qlge)
{
	uint32_t value;
	int rval = DDI_SUCCESS;
	uint32_t fn0_net, fn1_net;

	/* set default value */
	qlge->fn0_net = FN0_NET;
	qlge->fn1_net = FN1_NET;

	if (ql_read_processor_data(qlge, MPI_REG, &value) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "%s(%d) read MPI register failed",
		    __func__, qlge->instance);
	} else {
		fn0_net = (value >> 1) & 0x07;
		fn1_net = (value >> 5) & 0x07;
		if ((fn0_net > 4) || (fn1_net > 4) || (fn0_net == fn1_net)) {
			cmn_err(CE_WARN, "%s(%d) bad mpi register value %x, \n"
			    "nic0 function number %d,"
			    "nic1 function number %d "
			    "use default\n",
			    __func__, qlge->instance, value, fn0_net, fn1_net);
		} else {
			qlge->fn0_net = fn0_net;
			qlge->fn1_net = fn1_net;
		}
	}

	/* Get the function number that the driver is associated with */
	value = ql_read_reg(qlge, REG_STATUS);
	qlge->func_number = (uint8_t)((value >> 6) & 0x03);
	QL_PRINT(DBG_INIT, ("status register is:%x, func_number: %d\n",
	    value, qlge->func_number));

	/* The driver is loaded on a non-NIC function? */
	if ((qlge->func_number != qlge->fn0_net) &&
	    (qlge->func_number != qlge->fn1_net)) {
		cmn_err(CE_WARN,
		    "Invalid function number = 0x%x\n", qlge->func_number);
		return (DDI_FAILURE);
	}
	/* network port 0? */
	if (qlge->func_number == qlge->fn0_net) {
		qlge->xgmac_sem_mask = QL_PORT0_XGMAC_SEM_MASK;
		qlge->xgmac_sem_bits = QL_PORT0_XGMAC_SEM_BITS;
	} else {
		qlge->xgmac_sem_mask = QL_PORT1_XGMAC_SEM_MASK;
		qlge->xgmac_sem_bits = QL_PORT1_XGMAC_SEM_BITS;
	}

	return (rval);

}

/*
 * write to doorbell register
 */
void
ql_write_doorbell_reg(qlge_t *qlge, uint32_t *addr, uint32_t data)
{
	ddi_put32(qlge->dev_doorbell_reg_handle, addr, data);
}

/*
 * read from doorbell register
 */
uint32_t
ql_read_doorbell_reg(qlge_t *qlge, uint32_t *addr)
{
	uint32_t ret;

	ret = ddi_get32(qlge->dev_doorbell_reg_handle, addr);

	return	(ret);
}

/*
 * This function waits for a specific bit to come ready
 * in a given register.  It is used mostly by the initialize
 * process, but is also used in kernel thread API such as
 * netdev->set_multi, netdev->set_mac_address, netdev->vlan_rx_add_vid.
 */
static int
ql_wait_reg_rdy(qlge_t *qlge, uint32_t reg, uint32_t bit, uint32_t err_bit)
{
	uint32_t temp;
	int count = UDELAY_COUNT;

	while (count) {
		temp = ql_read_reg(qlge, reg);

		/* check for errors */
		if ((temp & err_bit) != 0) {
			break;
		} else if ((temp & bit) != 0)
			return (DDI_SUCCESS);
		qlge_delay(UDELAY_DELAY);
		count--;
	}
	cmn_err(CE_WARN,
	    "Waiting for reg %x to come ready failed.", reg);
	return (DDI_FAILURE);
}

/*
 * The CFG register is used to download TX and RX control blocks
 * to the chip. This function waits for an operation to complete.
 */
static int
ql_wait_cfg(qlge_t *qlge, uint32_t bit)
{
	int count = UDELAY_COUNT;
	uint32_t temp;

	while (count) {
		temp = ql_read_reg(qlge, REG_CONFIGURATION);
		if ((temp & CFG_LE) != 0) {
			break;
		}
		if ((temp & bit) == 0)
			return (DDI_SUCCESS);
		qlge_delay(UDELAY_DELAY);
		count--;
	}
	cmn_err(CE_WARN,
	    "Waiting for cfg register bit %x failed.", bit);
	return (DDI_FAILURE);
}


/*
 * Used to issue init control blocks to hw. Maps control block,
 * sets address, triggers download, waits for completion.
 */
static int
ql_write_cfg(qlge_t *qlge, uint32_t bit, uint64_t phy_addr, uint16_t q_id)
{
	int status = DDI_SUCCESS;
	uint32_t mask;
	uint32_t value;

	status = ql_sem_spinlock(qlge, SEM_ICB_MASK);
	if (status != DDI_SUCCESS) {
		goto exit;
	}
	status = ql_wait_cfg(qlge, bit);
	if (status != DDI_SUCCESS) {
		goto exit;
	}

	ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_LOWER, LS_64BITS(phy_addr));
	ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_UPPER, MS_64BITS(phy_addr));

	mask = CFG_Q_MASK | (bit << 16);
	value = bit | (q_id << CFG_Q_SHIFT);
	ql_write_reg(qlge, REG_CONFIGURATION, (mask | value));

	/*
	 * Wait for the bit to clear after signaling hw.
	 */
	status = ql_wait_cfg(qlge, bit);
	ql_sem_unlock(qlge, SEM_ICB_MASK); /* does flush too */

exit:
	return (status);
}

/*
 * Initialize adapter instance
 */
static int
ql_init_instance(qlge_t *qlge)
{
	int i;

	/* Default value */
	qlge->mac_flags = QL_MAC_INIT;
	qlge->mtu = ETHERMTU;		/* set normal size as default */
	qlge->page_size = VM_PAGE_SIZE;	/* default page size */
	/* Set up the default ring sizes. */
	qlge->tx_ring_size = NUM_TX_RING_ENTRIES;
	qlge->rx_ring_size = NUM_RX_RING_ENTRIES;

	/* Set up the coalescing parameters. */
	qlge->rx_coalesce_usecs = DFLT_RX_COALESCE_WAIT;
	qlge->tx_coalesce_usecs = DFLT_TX_COALESCE_WAIT;
	qlge->rx_max_coalesced_frames = DFLT_RX_INTER_FRAME_WAIT;
	qlge->tx_max_coalesced_frames = DFLT_TX_INTER_FRAME_WAIT;
	qlge->payload_copy_thresh = DFLT_PAYLOAD_COPY_THRESH;
	qlge->ql_dbgprnt = 0;
#if QL_DEBUG
	qlge->ql_dbgprnt = QL_DEBUG;
#endif /* QL_DEBUG */

	/*
	 * TODO: Should be obtained from configuration or based off
	 * number of active cpus SJP 4th Mar. 09
	 */
	qlge->tx_ring_count = 1;
	qlge->rss_ring_count = 4;
	qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count;

	for (i = 0; i < MAX_RX_RINGS; i++) {
		qlge->rx_polls[i] = 0;
		qlge->rx_interrupts[i] = 0;
	}

	/*
	 * Set up the operating parameters.
	 */
	qlge->multicast_list_count = 0;

	/*
	 * Set up the max number of unicast list
	 */
	qlge->unicst_total = MAX_UNICAST_LIST_SIZE;
	qlge->unicst_avail = MAX_UNICAST_LIST_SIZE;

	/*
	 * read user defined properties in .conf file
	 */
	ql_read_conf(qlge); /* mtu, pause, LSO etc */

	QL_PRINT(DBG_INIT, ("mtu is %d \n", qlge->mtu));

	/* choose Memory Space mapping and get Vendor Id, Device ID etc */
	ql_pci_config(qlge);
	qlge->ip_hdr_offset = 0;

	if (qlge->device_id == 0x8000) {
		/* Schultz card */
		qlge->cfg_flags |= CFG_CHIP_8100;
		/* enable just ipv4 chksum offload for Schultz */
		qlge->cfg_flags |= CFG_CKSUM_FULL_IPv4;
		/*
		 * Schultz firmware does not do pseduo IP header checksum
		 * calculation, needed to be done by driver
		 */
		qlge->cfg_flags |= CFG_HW_UNABLE_PSEUDO_HDR_CKSUM;
		if (qlge->lso_enable)
			qlge->cfg_flags |= CFG_LSO;
		qlge->cfg_flags |= CFG_SUPPORT_SCATTER_GATHER;
		/* Schultz must split packet header */
		qlge->cfg_flags |= CFG_ENABLE_SPLIT_HEADER;
		qlge->max_read_mbx = 5;
		qlge->ip_hdr_offset = 2;
	}

	/* Set Function Number and some of the iocb mac information */
	if (ql_set_mac_info(qlge) != DDI_SUCCESS)
		return (DDI_FAILURE);

	/* Read network settings from NVRAM */
	/* After nvram is read successfully, update dev_addr */
	if (ql_get_flash_params(qlge) == DDI_SUCCESS) {
		QL_PRINT(DBG_INIT, ("mac%d address is \n", qlge->func_number));
		for (i = 0; i < ETHERADDRL; i++) {
			qlge->dev_addr.ether_addr_octet[i] =
			    qlge->nic_config.factory_MAC[i];
		}
	} else {
		cmn_err(CE_WARN, "%s(%d): Failed to read flash memory",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}

	bcopy(qlge->dev_addr.ether_addr_octet,
	    qlge->unicst_addr[0].addr.ether_addr_octet,
	    ETHERADDRL);
	QL_DUMP(DBG_INIT, "\t flash mac address dump:\n",
	    &qlge->dev_addr.ether_addr_octet[0], 8, ETHERADDRL);

	qlge->port_link_state = LS_DOWN;

	return (DDI_SUCCESS);
}


/*
 * This hardware semaphore provides the mechanism for exclusive access to
 * resources shared between the NIC driver, MPI firmware,
 * FCOE firmware and the FC driver.
 */
static int
ql_sem_trylock(qlge_t *qlge, uint32_t sem_mask)
{
	uint32_t sem_bits = 0;

	switch (sem_mask) {
	case SEM_XGMAC0_MASK:
		sem_bits = SEM_SET << SEM_XGMAC0_SHIFT;
		break;
	case SEM_XGMAC1_MASK:
		sem_bits = SEM_SET << SEM_XGMAC1_SHIFT;
		break;
	case SEM_ICB_MASK:
		sem_bits = SEM_SET << SEM_ICB_SHIFT;
		break;
	case SEM_MAC_ADDR_MASK:
		sem_bits = SEM_SET << SEM_MAC_ADDR_SHIFT;
		break;
	case SEM_FLASH_MASK:
		sem_bits = SEM_SET << SEM_FLASH_SHIFT;
		break;
	case SEM_PROBE_MASK:
		sem_bits = SEM_SET << SEM_PROBE_SHIFT;
		break;
	case SEM_RT_IDX_MASK:
		sem_bits = SEM_SET << SEM_RT_IDX_SHIFT;
		break;
	case SEM_PROC_REG_MASK:
		sem_bits = SEM_SET << SEM_PROC_REG_SHIFT;
		break;
	default:
		cmn_err(CE_WARN, "Bad Semaphore mask!.");
		return (DDI_FAILURE);
	}

	ql_write_reg(qlge, REG_SEMAPHORE, sem_bits | sem_mask);
	return (!(ql_read_reg(qlge, REG_SEMAPHORE) & sem_bits));
}

/*
 * Lock a specific bit of Semaphore register to gain
 * access to a particular shared register
 */
int
ql_sem_spinlock(qlge_t *qlge, uint32_t sem_mask)
{
	unsigned int wait_count = 30;

	while (wait_count) {
		if (!ql_sem_trylock(qlge, sem_mask))
			return (DDI_SUCCESS);
		qlge_delay(100);
		wait_count--;
	}
	cmn_err(CE_WARN, "%s(%d) sem_mask 0x%x lock timeout ",
	    __func__, qlge->instance, sem_mask);
	return (DDI_FAILURE);
}

/*
 * Unock a specific bit of Semaphore register to release
 * access to a particular shared register
 */
void
ql_sem_unlock(qlge_t *qlge, uint32_t sem_mask)
{
	ql_write_reg(qlge, REG_SEMAPHORE, sem_mask);
	(void) ql_read_reg(qlge, REG_SEMAPHORE);	/* flush */
}

/*
 * Get property value from configuration file.
 *
 * string = property string pointer.
 *
 * Returns:
 * 0xFFFFFFFF = no property else property value.
 */
static uint32_t
ql_get_prop(qlge_t *qlge, char *string)
{
	char buf[256];
	uint32_t data;

	/* Get adapter instance parameter. */
	(void) sprintf(buf, "hba%d-%s", qlge->instance, string);
	data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0, buf,
	    (int)0xffffffff);

	/* Adapter instance parameter found? */
	if (data == 0xffffffff) {
		/* No, get default parameter. */
		data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0,
		    string, (int)0xffffffff);
	}

	return (data);
}

/*
 * Read user setting from configuration file.
 */
static void
ql_read_conf(qlge_t *qlge)
{
	uint32_t data;

	/* clear configuration flags */
	qlge->cfg_flags = 0;

	/* Get default rx_copy enable/disable. */
	if ((data = ql_get_prop(qlge, "force-rx-copy")) == 0xffffffff ||
	    data == 0) {
		qlge->cfg_flags &= ~CFG_RX_COPY_MODE;
		qlge->rx_copy = B_FALSE;
		QL_PRINT(DBG_INIT, ("rx copy mode disabled\n"));
	} else if (data == 1) {
		qlge->cfg_flags |= CFG_RX_COPY_MODE;
		qlge->rx_copy = B_TRUE;
		QL_PRINT(DBG_INIT, ("rx copy mode enabled\n"));
	}

	/* Get mtu packet size. */
	data = ql_get_prop(qlge, "mtu");
	if ((data == ETHERMTU) || (data == JUMBO_MTU)) {
		if (qlge->mtu != data) {
			qlge->mtu = data;
			cmn_err(CE_NOTE, "new mtu is %d\n", qlge->mtu);
		}
	}

	/* Get pause mode, default is Per Priority mode. */
	qlge->pause = PAUSE_MODE_PER_PRIORITY;
	data = ql_get_prop(qlge, "pause");
	if (data <= PAUSE_MODE_PER_PRIORITY) {
		if (qlge->pause != data) {
			qlge->pause = data;
			cmn_err(CE_NOTE, "new pause mode %d\n", qlge->pause);
		}
	}

	/* Get tx_max_coalesced_frames. */
	qlge->tx_max_coalesced_frames = 5;
	data = ql_get_prop(qlge, "tx_max_coalesced_frames");
	/* if data is valid */
	if ((data != 0xffffffff) && data) {
		if (qlge->tx_max_coalesced_frames != data) {
			qlge->tx_max_coalesced_frames = (uint16_t)data;
		}
	}

	/* Get split header payload_copy_thresh. */
	qlge->payload_copy_thresh = 6;
	data = ql_get_prop(qlge, "payload_copy_thresh");
	/* if data is valid */
	if ((data != 0xffffffff) && (data != 0)) {
		if (qlge->payload_copy_thresh != data) {
			qlge->payload_copy_thresh = data;
		}
	}

	/* large send offload (LSO) capability. */
	qlge->lso_enable = 1;
	data = ql_get_prop(qlge, "lso_enable");
	/* if data is valid */
	if (data != 0xffffffff) {
		if (qlge->lso_enable != data) {
			qlge->lso_enable = (uint16_t)data;
		}
	}
}

/*
 * Enable global interrupt
 */
static void
ql_enable_global_interrupt(qlge_t *qlge)
{
	ql_write_reg(qlge, REG_INTERRUPT_ENABLE,
	    (INTR_EN_EI << 16) | INTR_EN_EI);
	qlge->flags |= INTERRUPTS_ENABLED;
}

/*
 * Disable global interrupt
 */
static void
ql_disable_global_interrupt(qlge_t *qlge)
{
	ql_write_reg(qlge, REG_INTERRUPT_ENABLE, (INTR_EN_EI << 16));
	qlge->flags &= ~INTERRUPTS_ENABLED;
}

/*
 * Enable one ring interrupt
 */
void
ql_enable_completion_interrupt(qlge_t *qlge, uint32_t intr)
{
	struct intr_ctx *ctx = qlge->intr_ctx + intr;

	QL_PRINT(DBG_INTR, ("%s(%d): To enable intr %d, irq_cnt %d \n",
	    __func__, qlge->instance, intr, ctx->irq_cnt));

	if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) {
		/*
		 * Always enable if we're MSIX multi interrupts and
		 * it's not the default (zeroeth) interrupt.
		 */
		ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask);
		return;
	}

	if (!atomic_dec_32_nv(&ctx->irq_cnt)) {
		mutex_enter(&qlge->hw_mutex);
		ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask);
		mutex_exit(&qlge->hw_mutex);
		QL_PRINT(DBG_INTR,
		    ("%s(%d): write %x to intr enable register \n",
		    __func__, qlge->instance, ctx->intr_en_mask));
	}
}

/*
 * ql_forced_disable_completion_interrupt
 * Used by call from OS, may be called without
 * a pending interrupt so force the disable
 */
uint32_t
ql_forced_disable_completion_interrupt(qlge_t *qlge, uint32_t intr)
{
	uint32_t var = 0;
	struct intr_ctx *ctx = qlge->intr_ctx + intr;

	QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n",
	    __func__, qlge->instance, intr, ctx->irq_cnt));

	if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) {
		ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
		var = ql_read_reg(qlge, REG_STATUS);
		return (var);
	}

	mutex_enter(&qlge->hw_mutex);
	ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
	var = ql_read_reg(qlge, REG_STATUS);
	mutex_exit(&qlge->hw_mutex);

	return (var);
}

/*
 * Disable a completion interrupt
 */
void
ql_disable_completion_interrupt(qlge_t *qlge, uint32_t intr)
{
	struct intr_ctx *ctx;

	ctx = qlge->intr_ctx + intr;
	QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n",
	    __func__, qlge->instance, intr, ctx->irq_cnt));
	/*
	 * HW disables for us if we're MSIX multi interrupts and
	 * it's not the default (zeroeth) interrupt.
	 */
	if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && (intr != 0))
		return;

	if (ql_atomic_read_32(&ctx->irq_cnt) == 0) {
		mutex_enter(&qlge->hw_mutex);
		ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask);
		mutex_exit(&qlge->hw_mutex);
	}
	atomic_inc_32(&ctx->irq_cnt);
}

/*
 * Enable all completion interrupts
 */
static void
ql_enable_all_completion_interrupts(qlge_t *qlge)
{
	int i;
	uint32_t value = 1;

	for (i = 0; i < qlge->intr_cnt; i++) {
		/*
		 * Set the count to 1 for Legacy / MSI interrupts or for the
		 * default interrupt (0)
		 */
		if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0) {
			ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value);
		}
		ql_enable_completion_interrupt(qlge, i);
	}
}

/*
 * Disable all completion interrupts
 */
static void
ql_disable_all_completion_interrupts(qlge_t *qlge)
{
	int i;
	uint32_t value = 0;

	for (i = 0; i < qlge->intr_cnt; i++) {

		/*
		 * Set the count to 0 for Legacy / MSI interrupts or for the
		 * default interrupt (0)
		 */
		if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0)
			ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value);

		ql_disable_completion_interrupt(qlge, i);
	}
}

/*
 * Update small buffer queue producer index
 */
static void
ql_update_sbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring)
{
	/* Update the buffer producer index */
	QL_PRINT(DBG_RX, ("sbq: updating prod idx = %d.\n",
	    rx_ring->sbq_prod_idx));
	ql_write_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg,
	    rx_ring->sbq_prod_idx);
}

/*
 * Update large buffer queue producer index
 */
static void
ql_update_lbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring)
{
	/* Update the buffer producer index */
	QL_PRINT(DBG_RX, ("lbq: updating prod idx = %d.\n",
	    rx_ring->lbq_prod_idx));
	ql_write_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg,
	    rx_ring->lbq_prod_idx);
}

/*
 * Adds a small buffer descriptor to end of its in use list,
 * assumes sbq_lock is already taken
 */
static void
ql_add_sbuf_to_in_use_list(struct rx_ring *rx_ring,
    struct bq_desc *sbq_desc)
{
	uint32_t inuse_idx = rx_ring->sbq_use_tail;

	rx_ring->sbuf_in_use[inuse_idx] = sbq_desc;
	inuse_idx++;
	if (inuse_idx >= rx_ring->sbq_len)
		inuse_idx = 0;
	rx_ring->sbq_use_tail = inuse_idx;
	atomic_inc_32(&rx_ring->sbuf_in_use_count);
	ASSERT(rx_ring->sbuf_in_use_count <= rx_ring->sbq_len);
}

/*
 * Get a small buffer descriptor from its in use list
 */
static struct bq_desc *
ql_get_sbuf_from_in_use_list(struct rx_ring *rx_ring)
{
	struct bq_desc *sbq_desc = NULL;
	uint32_t inuse_idx;

	/* Pick from head of in use list */
	inuse_idx = rx_ring->sbq_use_head;
	sbq_desc = rx_ring->sbuf_in_use[inuse_idx];
	rx_ring->sbuf_in_use[inuse_idx] = NULL;

	if (sbq_desc != NULL) {
		inuse_idx++;
		if (inuse_idx >= rx_ring->sbq_len)
			inuse_idx = 0;
		rx_ring->sbq_use_head = inuse_idx;
		atomic_dec_32(&rx_ring->sbuf_in_use_count);
		atomic_inc_32(&rx_ring->rx_indicate);
		sbq_desc->upl_inuse = 1;
		/* if mp is NULL */
		if (sbq_desc->mp == NULL) {
			/* try to remap mp again */
			sbq_desc->mp =
			    desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
			    rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
		}
	}

	return (sbq_desc);
}

/*
 * Add a small buffer descriptor to its free list
 */
static void
ql_add_sbuf_to_free_list(struct rx_ring *rx_ring,
    struct bq_desc *sbq_desc)
{
	uint32_t free_idx;

	/* Add to the end of free list */
	free_idx = rx_ring->sbq_free_tail;
	rx_ring->sbuf_free[free_idx] = sbq_desc;
	ASSERT(rx_ring->sbuf_free_count <= rx_ring->sbq_len);
	free_idx++;
	if (free_idx >= rx_ring->sbq_len)
		free_idx = 0;
	rx_ring->sbq_free_tail = free_idx;
	atomic_inc_32(&rx_ring->sbuf_free_count);
}

/*
 * Get a small buffer descriptor from its free list
 */
static struct bq_desc *
ql_get_sbuf_from_free_list(struct rx_ring *rx_ring)
{
	struct bq_desc *sbq_desc;
	uint32_t free_idx;

	free_idx = rx_ring->sbq_free_head;
	/* Pick from top of free list */
	sbq_desc = rx_ring->sbuf_free[free_idx];
	rx_ring->sbuf_free[free_idx] = NULL;
	if (sbq_desc != NULL) {
		free_idx++;
		if (free_idx >= rx_ring->sbq_len)
			free_idx = 0;
		rx_ring->sbq_free_head = free_idx;
		atomic_dec_32(&rx_ring->sbuf_free_count);
		ASSERT(rx_ring->sbuf_free_count != 0);
	}
	return (sbq_desc);
}

/*
 * Add a large buffer descriptor to its in use list
 */
static void
ql_add_lbuf_to_in_use_list(struct rx_ring *rx_ring,
    struct bq_desc *lbq_desc)
{
	uint32_t inuse_idx;

	inuse_idx = rx_ring->lbq_use_tail;

	rx_ring->lbuf_in_use[inuse_idx] = lbq_desc;
	inuse_idx++;
	if (inuse_idx >= rx_ring->lbq_len)
		inuse_idx = 0;
	rx_ring->lbq_use_tail = inuse_idx;
	atomic_inc_32(&rx_ring->lbuf_in_use_count);
}

/*
 * Get a large buffer descriptor from in use list
 */
static struct bq_desc *
ql_get_lbuf_from_in_use_list(struct rx_ring *rx_ring)
{
	struct bq_desc *lbq_desc;
	uint32_t inuse_idx;

	/* Pick from head of in use list */
	inuse_idx = rx_ring->lbq_use_head;
	lbq_desc = rx_ring->lbuf_in_use[inuse_idx];
	rx_ring->lbuf_in_use[inuse_idx] = NULL;

	if (lbq_desc != NULL) {
		inuse_idx++;
		if (inuse_idx >= rx_ring->lbq_len)
			inuse_idx = 0;
		rx_ring->lbq_use_head = inuse_idx;
		atomic_dec_32(&rx_ring->lbuf_in_use_count);
		atomic_inc_32(&rx_ring->rx_indicate);
		lbq_desc->upl_inuse = 1;

		/* if mp is NULL */
		if (lbq_desc->mp == NULL) {
			/* try to remap mp again */
			lbq_desc->mp =
			    desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
			    rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
		}
	}
	return (lbq_desc);
}

/*
 * Add a large buffer descriptor to free list
 */
static void
ql_add_lbuf_to_free_list(struct rx_ring *rx_ring,
    struct bq_desc *lbq_desc)
{
	uint32_t free_idx;

	/* Add to the end of free list */
	free_idx = rx_ring->lbq_free_tail;
	rx_ring->lbuf_free[free_idx] = lbq_desc;
	free_idx++;
	if (free_idx >= rx_ring->lbq_len)
		free_idx = 0;
	rx_ring->lbq_free_tail = free_idx;
	atomic_inc_32(&rx_ring->lbuf_free_count);
	ASSERT(rx_ring->lbuf_free_count <= rx_ring->lbq_len);
}

/*
 * Get a large buffer descriptor from its free list
 */
static struct bq_desc *
ql_get_lbuf_from_free_list(struct rx_ring *rx_ring)
{
	struct bq_desc *lbq_desc;
	uint32_t free_idx;

	free_idx = rx_ring->lbq_free_head;
	/* Pick from head of free list */
	lbq_desc = rx_ring->lbuf_free[free_idx];
	rx_ring->lbuf_free[free_idx] = NULL;

	if (lbq_desc != NULL) {
		free_idx++;
		if (free_idx >= rx_ring->lbq_len)
			free_idx = 0;
		rx_ring->lbq_free_head = free_idx;
		atomic_dec_32(&rx_ring->lbuf_free_count);
		ASSERT(rx_ring->lbuf_free_count != 0);
	}
	return (lbq_desc);
}

/*
 * Add a small buffer descriptor to free list
 */
static void
ql_refill_sbuf_free_list(struct bq_desc *sbq_desc, boolean_t alloc_memory)
{
	struct rx_ring *rx_ring = sbq_desc->rx_ring;
	uint64_t *sbq_entry;
	qlge_t *qlge = (qlge_t *)rx_ring->qlge;
	/*
	 * Sync access
	 */
	mutex_enter(&rx_ring->sbq_lock);

	sbq_desc->upl_inuse = 0;

	/*
	 * If we are freeing the buffers as a result of adapter unload, get out
	 */
	if ((sbq_desc->free_buf != NULL) ||
	    (qlge->mac_flags == QL_MAC_DETACH)) {
		if (sbq_desc->free_buf == NULL)
			atomic_dec_32(&rx_ring->rx_indicate);
		mutex_exit(&rx_ring->sbq_lock);
		return;
	}
#ifdef QLGE_LOAD_UNLOAD
	if (rx_ring->rx_indicate == 0)
		cmn_err(CE_WARN, "sbq: indicate wrong");
#endif
#ifdef QLGE_TRACK_BUFFER_USAGE
	uint32_t sb_consumer_idx;
	uint32_t sb_producer_idx;
	uint32_t num_free_buffers;
	uint32_t temp;

	temp = ql_read_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg);
	sb_producer_idx = temp & 0x0000ffff;
	sb_consumer_idx = (temp >> 16);

	if (sb_consumer_idx > sb_producer_idx)
		num_free_buffers = NUM_SMALL_BUFFERS -
		    (sb_consumer_idx - sb_producer_idx);
	else
		num_free_buffers = sb_producer_idx - sb_consumer_idx;

	if (num_free_buffers < qlge->rx_sb_low_count[rx_ring->cq_id])
		qlge->rx_sb_low_count[rx_ring->cq_id] = num_free_buffers;

#endif

	ASSERT(sbq_desc->mp == NULL);

#ifdef QLGE_LOAD_UNLOAD
	if (rx_ring->rx_indicate > 0xFF000000)
		cmn_err(CE_WARN, "sbq: indicate(%d) wrong: %d mac_flags %d,"
		    " sbq_desc index %d.",
		    rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags,
		    sbq_desc->index);
#endif
	if (alloc_memory) {
		sbq_desc->mp =
		    desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
		    rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
		if (sbq_desc->mp == NULL) {
			rx_ring->rx_failed_sbq_allocs++;
		}
	}

	/* Got the packet from the stack decrement rx_indicate count */
	atomic_dec_32(&rx_ring->rx_indicate);

	ql_add_sbuf_to_free_list(rx_ring, sbq_desc);

	/* Rearm if possible */
	if ((rx_ring->sbuf_free_count >= MIN_BUFFERS_FREE_COUNT) &&
	    (qlge->mac_flags == QL_MAC_STARTED)) {
		sbq_entry = rx_ring->sbq_dma.vaddr;
		sbq_entry += rx_ring->sbq_prod_idx;

		while (rx_ring->sbuf_free_count > MIN_BUFFERS_ARM_COUNT) {
			/* Get first one from free list */
			sbq_desc = ql_get_sbuf_from_free_list(rx_ring);

			*sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr);
			sbq_entry++;
			rx_ring->sbq_prod_idx++;
			if (rx_ring->sbq_prod_idx >= rx_ring->sbq_len) {
				rx_ring->sbq_prod_idx = 0;
				sbq_entry = rx_ring->sbq_dma.vaddr;
			}
			/* Add to end of in use list */
			ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc);
		}

		/* Update small buffer queue producer index */
		ql_update_sbq_prod_idx(qlge, rx_ring);
	}

	mutex_exit(&rx_ring->sbq_lock);
	QL_PRINT(DBG_RX_RING, ("%s(%d) exited, sbuf_free_count %d\n",
	    __func__, qlge->instance, rx_ring->sbuf_free_count));
}

/*
 * rx recycle call back function
 */
static void
ql_release_to_sbuf_free_list(caddr_t p)
{
	struct bq_desc *sbq_desc = (struct bq_desc *)(void *)p;

	if (sbq_desc == NULL)
		return;
	ql_refill_sbuf_free_list(sbq_desc, B_TRUE);
}

/*
 * Add a large buffer descriptor to free list
 */
static void
ql_refill_lbuf_free_list(struct bq_desc *lbq_desc, boolean_t alloc_memory)
{
	struct rx_ring *rx_ring = lbq_desc->rx_ring;
	uint64_t *lbq_entry;
	qlge_t *qlge = rx_ring->qlge;

	/* Sync access */
	mutex_enter(&rx_ring->lbq_lock);

	lbq_desc->upl_inuse = 0;
	/*
	 * If we are freeing the buffers as a result of adapter unload, get out
	 */
	if ((lbq_desc->free_buf != NULL) ||
	    (qlge->mac_flags == QL_MAC_DETACH)) {
		if (lbq_desc->free_buf == NULL)
			atomic_dec_32(&rx_ring->rx_indicate);
		mutex_exit(&rx_ring->lbq_lock);
		return;
	}
#ifdef QLGE_LOAD_UNLOAD
	if (rx_ring->rx_indicate == 0)
		cmn_err(CE_WARN, "lbq: indicate wrong");
#endif
#ifdef QLGE_TRACK_BUFFER_USAGE
	uint32_t lb_consumer_idx;
	uint32_t lb_producer_idx;
	uint32_t num_free_buffers;
	uint32_t temp;

	temp = ql_read_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg);

	lb_producer_idx = temp & 0x0000ffff;
	lb_consumer_idx = (temp >> 16);

	if (lb_consumer_idx > lb_producer_idx)
		num_free_buffers = NUM_LARGE_BUFFERS -
		    (lb_consumer_idx - lb_producer_idx);
	else
		num_free_buffers = lb_producer_idx - lb_consumer_idx;

	if (num_free_buffers < qlge->rx_lb_low_count[rx_ring->cq_id]) {
		qlge->rx_lb_low_count[rx_ring->cq_id] = num_free_buffers;
	}
#endif

	ASSERT(lbq_desc->mp == NULL);
#ifdef QLGE_LOAD_UNLOAD
	if (rx_ring->rx_indicate > 0xFF000000)
		cmn_err(CE_WARN, "lbq: indicate(%d) wrong: %d mac_flags %d,"
		    "lbq_desc index %d",
		    rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags,
		    lbq_desc->index);
#endif
	if (alloc_memory) {
		lbq_desc->mp =
		    desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
		    rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
		if (lbq_desc->mp == NULL) {
			rx_ring->rx_failed_lbq_allocs++;
		}
	}

	/* Got the packet from the stack decrement rx_indicate count */
	atomic_dec_32(&rx_ring->rx_indicate);

	ql_add_lbuf_to_free_list(rx_ring, lbq_desc);

	/* Rearm if possible */
	if ((rx_ring->lbuf_free_count >= MIN_BUFFERS_FREE_COUNT) &&
	    (qlge->mac_flags == QL_MAC_STARTED)) {
		lbq_entry = rx_ring->lbq_dma.vaddr;
		lbq_entry += rx_ring->lbq_prod_idx;
		while (rx_ring->lbuf_free_count > MIN_BUFFERS_ARM_COUNT) {
			/* Get first one from free list */
			lbq_desc = ql_get_lbuf_from_free_list(rx_ring);

			*lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr);
			lbq_entry++;
			rx_ring->lbq_prod_idx++;
			if (rx_ring->lbq_prod_idx >= rx_ring->lbq_len) {
				rx_ring->lbq_prod_idx = 0;
				lbq_entry = rx_ring->lbq_dma.vaddr;
			}

			/* Add to end of in use list */
			ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc);
		}

		/* Update large buffer queue producer index */
		ql_update_lbq_prod_idx(rx_ring->qlge, rx_ring);
	}

	mutex_exit(&rx_ring->lbq_lock);
	QL_PRINT(DBG_RX_RING, ("%s exitd, lbuf_free_count %d\n",
	    __func__, rx_ring->lbuf_free_count));
}
/*
 * rx recycle call back function
 */
static void
ql_release_to_lbuf_free_list(caddr_t p)
{
	struct bq_desc *lbq_desc = (struct bq_desc *)(void *)p;

	if (lbq_desc == NULL)
		return;
	ql_refill_lbuf_free_list(lbq_desc, B_TRUE);
}

/*
 * free small buffer queue buffers
 */
static void
ql_free_sbq_buffers(struct rx_ring *rx_ring)
{
	struct bq_desc *sbq_desc;
	uint32_t i;
	uint32_t j = rx_ring->sbq_free_head;
	int  force_cnt = 0;

	for (i = 0; i < rx_ring->sbuf_free_count; i++) {
		sbq_desc = rx_ring->sbuf_free[j];
		sbq_desc->free_buf = 1;
		j++;
		if (j >= rx_ring->sbq_len) {
			j = 0;
		}
		if (sbq_desc->mp != NULL) {
			freemsg(sbq_desc->mp);
			sbq_desc->mp = NULL;
		}
	}
	rx_ring->sbuf_free_count = 0;

	j = rx_ring->sbq_use_head;
	for (i = 0; i < rx_ring->sbuf_in_use_count; i++) {
		sbq_desc = rx_ring->sbuf_in_use[j];
		sbq_desc->free_buf = 1;
		j++;
		if (j >= rx_ring->sbq_len) {
			j = 0;
		}
		if (sbq_desc->mp != NULL) {
			freemsg(sbq_desc->mp);
			sbq_desc->mp = NULL;
		}
	}
	rx_ring->sbuf_in_use_count = 0;

	sbq_desc = &rx_ring->sbq_desc[0];
	for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) {
		/*
		 * Set flag so that the callback does not allocate a new buffer
		 */
		sbq_desc->free_buf = 1;
		if (sbq_desc->upl_inuse != 0) {
			force_cnt++;
		}
		if (sbq_desc->bd_dma.dma_handle != NULL) {
			ql_free_phys(&sbq_desc->bd_dma.dma_handle,
			    &sbq_desc->bd_dma.acc_handle);
			sbq_desc->bd_dma.dma_handle = NULL;
			sbq_desc->bd_dma.acc_handle = NULL;
		}
	}
#ifdef QLGE_LOAD_UNLOAD
	cmn_err(CE_NOTE, "sbq: free %d inuse %d force %d\n",
	    rx_ring->sbuf_free_count, rx_ring->sbuf_in_use_count, force_cnt);
#endif
	if (rx_ring->sbuf_in_use != NULL) {
		kmem_free(rx_ring->sbuf_in_use, (rx_ring->sbq_len *
		    sizeof (struct bq_desc *)));
		rx_ring->sbuf_in_use = NULL;
	}

	if (rx_ring->sbuf_free != NULL) {
		kmem_free(rx_ring->sbuf_free, (rx_ring->sbq_len *
		    sizeof (struct bq_desc *)));
		rx_ring->sbuf_free = NULL;
	}
}

/* Allocate small buffers */
static int
ql_alloc_sbufs(qlge_t *qlge, struct rx_ring *rx_ring)
{
	struct bq_desc *sbq_desc;
	int i;
	ddi_dma_cookie_t dma_cookie;

	rx_ring->sbuf_free = kmem_zalloc(rx_ring->sbq_len *
	    sizeof (struct bq_desc *), KM_NOSLEEP);
	if (rx_ring->sbuf_free == NULL) {
		cmn_err(CE_WARN,
		    "!%s: sbuf_free_list alloc: failed",
		    __func__);
		rx_ring->sbuf_free_count = 0;
		goto alloc_sbuf_err;
	}

	rx_ring->sbuf_in_use = kmem_zalloc(rx_ring->sbq_len *
	    sizeof (struct bq_desc *), KM_NOSLEEP);
	if (rx_ring->sbuf_in_use == NULL) {
		cmn_err(CE_WARN,
		    "!%s: sbuf_inuse_list alloc: failed",
		    __func__);
		rx_ring->sbuf_in_use_count = 0;
		goto alloc_sbuf_err;
	}
	rx_ring->sbq_use_head = 0;
	rx_ring->sbq_use_tail = 0;
	rx_ring->sbq_free_head = 0;
	rx_ring->sbq_free_tail = 0;
	sbq_desc = &rx_ring->sbq_desc[0];

	for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) {
		/* Allocate buffer */
		if (ql_alloc_phys(qlge->dip, &sbq_desc->bd_dma.dma_handle,
		    &ql_buf_acc_attr,
		    DDI_DMA_READ | DDI_DMA_STREAMING,
		    &sbq_desc->bd_dma.acc_handle,
		    (size_t)rx_ring->sbq_buf_size,	/* mem size */
		    (size_t)0,				/* default alignment */
		    (caddr_t *)&sbq_desc->bd_dma.vaddr,
		    &dma_cookie) != 0) {
			cmn_err(CE_WARN,
			    "!%s: ddi_dma_alloc_handle: failed",
			    __func__);
			goto alloc_sbuf_err;
		}

		/* Set context for Return buffer callback */
		sbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress;
		sbq_desc->rx_recycle.free_func = ql_release_to_sbuf_free_list;
		sbq_desc->rx_recycle.free_arg  = (caddr_t)sbq_desc;
		sbq_desc->rx_ring = rx_ring;
		sbq_desc->upl_inuse = 0;
		sbq_desc->free_buf = 0;

		sbq_desc->mp =
		    desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr),
		    rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle);
		if (sbq_desc->mp == NULL) {
			cmn_err(CE_WARN, "%s: desballoc() failed", __func__);
			goto alloc_sbuf_err;
		}
		ql_add_sbuf_to_free_list(rx_ring, sbq_desc);
	}

	return (DDI_SUCCESS);

alloc_sbuf_err:
	ql_free_sbq_buffers(rx_ring);
	return (DDI_FAILURE);
}

static void
ql_free_lbq_buffers(struct rx_ring *rx_ring)
{
	struct bq_desc *lbq_desc;
	uint32_t i, j;
	int force_cnt = 0;

	j = rx_ring->lbq_free_head;
	for (i = 0; i < rx_ring->lbuf_free_count; i++) {
		lbq_desc = rx_ring->lbuf_free[j];
		lbq_desc->free_buf = 1;
		j++;
		if (j >= rx_ring->lbq_len)
			j = 0;
		if (lbq_desc->mp != NULL) {
			freemsg(lbq_desc->mp);
			lbq_desc->mp = NULL;
		}
	}
	rx_ring->lbuf_free_count = 0;

	j = rx_ring->lbq_use_head;
	for (i = 0; i < rx_ring->lbuf_in_use_count; i++) {
		lbq_desc = rx_ring->lbuf_in_use[j];
		lbq_desc->free_buf = 1;
		j++;
		if (j >= rx_ring->lbq_len) {
			j = 0;
		}
		if (lbq_desc->mp != NULL) {
			freemsg(lbq_desc->mp);
			lbq_desc->mp = NULL;
		}
	}
	rx_ring->lbuf_in_use_count = 0;

	lbq_desc = &rx_ring->lbq_desc[0];
	for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) {
		/* Set flag so that callback will not allocate a new buffer */
		lbq_desc->free_buf = 1;
		if (lbq_desc->upl_inuse != 0) {
			force_cnt++;
		}
		if (lbq_desc->bd_dma.dma_handle != NULL) {
			ql_free_phys(&lbq_desc->bd_dma.dma_handle,
			    &lbq_desc->bd_dma.acc_handle);
			lbq_desc->bd_dma.dma_handle = NULL;
			lbq_desc->bd_dma.acc_handle = NULL;
		}
	}
#ifdef QLGE_LOAD_UNLOAD
	if (force_cnt) {
		cmn_err(CE_WARN, "lbq: free %d inuse %d force %d",
		    rx_ring->lbuf_free_count, rx_ring->lbuf_in_use_count,
		    force_cnt);
	}
#endif
	if (rx_ring->lbuf_in_use != NULL) {
		kmem_free(rx_ring->lbuf_in_use, (rx_ring->lbq_len *
		    sizeof (struct bq_desc *)));
		rx_ring->lbuf_in_use = NULL;
	}

	if (rx_ring->lbuf_free != NULL) {
		kmem_free(rx_ring->lbuf_free, (rx_ring->lbq_len *
		    sizeof (struct bq_desc *)));
		rx_ring->lbuf_free = NULL;
	}
}

/* Allocate large buffers */
static int
ql_alloc_lbufs(qlge_t *qlge, struct rx_ring *rx_ring)
{
	struct bq_desc *lbq_desc;
	ddi_dma_cookie_t dma_cookie;
	int i;
	uint32_t lbq_buf_size;

	rx_ring->lbuf_free = kmem_zalloc(rx_ring->lbq_len *
	    sizeof (struct bq_desc *), KM_NOSLEEP);
	if (rx_ring->lbuf_free == NULL) {
		cmn_err(CE_WARN,
		    "!%s: lbuf_free_list alloc: failed",
		    __func__);
		rx_ring->lbuf_free_count = 0;
		goto alloc_lbuf_err;
	}

	rx_ring->lbuf_in_use = kmem_zalloc(rx_ring->lbq_len *
	    sizeof (struct bq_desc *), KM_NOSLEEP);

	if (rx_ring->lbuf_in_use == NULL) {
		cmn_err(CE_WARN,
		    "!%s: lbuf_inuse_list alloc: failed",
		    __func__);
		rx_ring->lbuf_in_use_count = 0;
		goto alloc_lbuf_err;
	}
	rx_ring->lbq_use_head = 0;
	rx_ring->lbq_use_tail = 0;
	rx_ring->lbq_free_head = 0;
	rx_ring->lbq_free_tail = 0;

	lbq_buf_size = (qlge->mtu == ETHERMTU) ?
	    NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE;

	lbq_desc = &rx_ring->lbq_desc[0];
	for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) {
		rx_ring->lbq_buf_size = lbq_buf_size;
		/* Allocate buffer */
		if (ql_alloc_phys(qlge->dip, &lbq_desc->bd_dma.dma_handle,
		    &ql_buf_acc_attr,
		    DDI_DMA_READ | DDI_DMA_STREAMING,
		    &lbq_desc->bd_dma.acc_handle,
		    (size_t)rx_ring->lbq_buf_size,  /* mem size */
		    (size_t)0, /* default alignment */
		    (caddr_t *)&lbq_desc->bd_dma.vaddr,
		    &dma_cookie) != 0) {
			cmn_err(CE_WARN,
			    "!%s: ddi_dma_alloc_handle: failed",
			    __func__);
			goto alloc_lbuf_err;
		}

		/* Set context for Return buffer callback */
		lbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress;
		lbq_desc->rx_recycle.free_func = ql_release_to_lbuf_free_list;
		lbq_desc->rx_recycle.free_arg  = (caddr_t)lbq_desc;
		lbq_desc->rx_ring = rx_ring;
		lbq_desc->upl_inuse = 0;
		lbq_desc->free_buf = 0;

		lbq_desc->mp =
		    desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr),
		    rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle);
		if (lbq_desc->mp == NULL) {
			cmn_err(CE_WARN, "%s: desballoc() failed", __func__);
			goto alloc_lbuf_err;
		}
		ql_add_lbuf_to_free_list(rx_ring, lbq_desc);
	} /* For all large buffers */

	return (DDI_SUCCESS);

alloc_lbuf_err:
	ql_free_lbq_buffers(rx_ring);
	return (DDI_FAILURE);
}

/*
 * Free rx buffers
 */
static void
ql_free_rx_buffers(qlge_t *qlge)
{
	int i;
	struct rx_ring *rx_ring;

	for (i = 0; i < qlge->rx_ring_count; i++) {
		rx_ring = &qlge->rx_ring[i];
		if (rx_ring->type != TX_Q) {
			ql_free_lbq_buffers(rx_ring);
			ql_free_sbq_buffers(rx_ring);
		}
	}
}

/*
 * Allocate rx buffers
 */
static int
ql_alloc_rx_buffers(qlge_t *qlge)
{
	struct rx_ring *rx_ring;
	int i;

	for (i = 0; i < qlge->rx_ring_count; i++) {
		rx_ring = &qlge->rx_ring[i];
		if (rx_ring->type != TX_Q) {
			if (ql_alloc_sbufs(qlge, rx_ring) != DDI_SUCCESS)
				goto alloc_err;
			if (ql_alloc_lbufs(qlge, rx_ring) != DDI_SUCCESS)
				goto alloc_err;
		}
	}
#ifdef QLGE_TRACK_BUFFER_USAGE
	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (qlge->rx_ring[i].type == RX_Q) {
			qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS;
			qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS;
		}
		qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES;
	}
#endif
	return (DDI_SUCCESS);

alloc_err:

	return (DDI_FAILURE);
}

/*
 * Initialize large buffer queue ring
 */
static void
ql_init_lbq_ring(struct rx_ring *rx_ring)
{
	uint16_t i;
	struct bq_desc *lbq_desc;

	bzero(rx_ring->lbq_desc, rx_ring->lbq_len * sizeof (struct bq_desc));
	for (i = 0; i < rx_ring->lbq_len; i++) {
		lbq_desc = &rx_ring->lbq_desc[i];
		lbq_desc->index = i;
	}
}

/*
 * Initialize small buffer queue ring
 */
static void
ql_init_sbq_ring(struct rx_ring *rx_ring)
{
	uint16_t i;
	struct bq_desc *sbq_desc;

	bzero(rx_ring->sbq_desc, rx_ring->sbq_len * sizeof (struct bq_desc));
	for (i = 0; i < rx_ring->sbq_len; i++) {
		sbq_desc = &rx_ring->sbq_desc[i];
		sbq_desc->index = i;
	}
}

/*
 * Calculate the pseudo-header checksum if hardware can not do
 */
static void
ql_pseudo_cksum(uint8_t *buf)
{
	uint32_t cksum;
	uint16_t iphl;
	uint16_t proto;

	iphl = (uint16_t)(4 * (buf[0] & 0xF));
	cksum = (((uint16_t)buf[2])<<8) + buf[3] - iphl;
	cksum += proto = buf[9];
	cksum += (((uint16_t)buf[12])<<8) + buf[13];
	cksum += (((uint16_t)buf[14])<<8) + buf[15];
	cksum += (((uint16_t)buf[16])<<8) + buf[17];
	cksum += (((uint16_t)buf[18])<<8) + buf[19];
	cksum = (cksum>>16) + (cksum & 0xFFFF);
	cksum = (cksum>>16) + (cksum & 0xFFFF);

	/*
	 * Point it to the TCP/UDP header, and
	 * update the checksum field.
	 */
	buf += iphl + ((proto == IPPROTO_TCP) ?
	    TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET);

	*(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum);

}

/*
 * Transmit an incoming packet.
 */
mblk_t *
ql_ring_tx(void *arg, mblk_t *mp)
{
	struct tx_ring *tx_ring = (struct tx_ring *)arg;
	qlge_t *qlge = tx_ring->qlge;
	mblk_t *next;
	int rval;
	uint32_t tx_count = 0;

	if (qlge->port_link_state == LS_DOWN) {
		/* can not send message while link is down */
		mblk_t *tp;
		cmn_err(CE_WARN, "tx failed due to link down");

		while (mp != NULL) {
			tp = mp->b_next;
			mp->b_next = NULL;
			freemsg(mp);
			mp = tp;
		}
		goto exit;
	}

	mutex_enter(&tx_ring->tx_lock);
	/* if mac is not started, driver is not ready, can not send */
	if (tx_ring->mac_flags != QL_MAC_STARTED) {
		cmn_err(CE_WARN, "%s(%d)ring not started, mode %d "
		    " return packets",
		    __func__, qlge->instance, tx_ring->mac_flags);
		mutex_exit(&tx_ring->tx_lock);
		goto exit;
	}

	/* we must try to send all */
	while (mp != NULL) {
		/*
		 * if number of available slots is less than a threshold,
		 * then quit
		 */
		if (tx_ring->tx_free_count <= TX_STOP_THRESHOLD) {
			tx_ring->queue_stopped = 1;
			rval = DDI_FAILURE;
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_WARN, "%s(%d) no resources",
			    __func__, qlge->instance);
#endif
			tx_ring->defer++;
			/*
			 * If we return the buffer back we are expected to call
			 * mac_tx_ring_update() when resources are available
			 */
			break;
		}

		next = mp->b_next;
		mp->b_next = NULL;

		rval = ql_send_common(tx_ring, mp);

		if (rval != DDI_SUCCESS) {
			mp->b_next = next;
			break;
		}
		tx_count++;
		mp = next;
	}

	/*
	 * After all msg blocks are mapped or copied to tx buffer,
	 * trigger the hardware to send!
	 */
	if (tx_count > 0) {
		ql_write_doorbell_reg(tx_ring->qlge, tx_ring->prod_idx_db_reg,
		    tx_ring->prod_idx);
	}

	mutex_exit(&tx_ring->tx_lock);
exit:
	return (mp);
}


/*
 * This function builds an mblk list for the given inbound
 * completion.
 */

static mblk_t *
ql_build_rx_mp(qlge_t *qlge, struct rx_ring *rx_ring,
    struct ib_mac_iocb_rsp *ib_mac_rsp)
{
	mblk_t *mp = NULL;
	mblk_t *mp1 = NULL;	/* packet header */
	mblk_t *mp2 = NULL;	/* packet content */
	struct bq_desc *lbq_desc;
	struct bq_desc *sbq_desc;
	uint32_t err_flag = (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK);
	uint32_t payload_len = le32_to_cpu(ib_mac_rsp->data_len);
	uint32_t header_len = le32_to_cpu(ib_mac_rsp->hdr_len);
	uint32_t pkt_len = payload_len + header_len;
	uint32_t done;
	uint64_t *curr_ial_ptr;
	uint32_t ial_data_addr_low;
	uint32_t actual_data_addr_low;
	mblk_t *mp_ial = NULL;	/* ial chained packets */
	uint32_t size;

	/*
	 * Check if error flags are set
	 */
	if (err_flag != 0) {
		if ((err_flag & IB_MAC_IOCB_RSP_ERR_OVERSIZE) != 0)
			rx_ring->frame_too_long++;
		if ((err_flag & IB_MAC_IOCB_RSP_ERR_UNDERSIZE) != 0)
			rx_ring->frame_too_short++;
		if ((err_flag & IB_MAC_IOCB_RSP_ERR_CRC) != 0)
			rx_ring->fcs_err++;
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_WARN, "bad packet, type 0x%x", err_flag);
#endif
		QL_DUMP(DBG_RX, "qlge_ring_rx: bad response iocb dump\n",
		    (uint8_t *)ib_mac_rsp, 8,
		    (size_t)sizeof (struct ib_mac_iocb_rsp));
	}

	/* header should not be in large buffer */
	if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HL) {
		cmn_err(CE_WARN, "header in large buffer or invalid!");
		err_flag |= 1;
	}
	/*
	 * Handle the header buffer if present.
	 * packet header must be valid and saved in one small buffer
	 * broadcast/multicast packets' headers not splitted
	 */
	if ((ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV) &&
	    (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) {
		QL_PRINT(DBG_RX, ("Header of %d bytes in small buffer.\n",
		    header_len));
		/* Sync access */
		sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);

		ASSERT(sbq_desc != NULL);

		/*
		 * Validate addresses from the ASIC with the
		 * expected sbuf address
		 */
		if (cpu_to_le64(sbq_desc->bd_dma.dma_addr)
		    != ib_mac_rsp->hdr_addr) {
			/* Small buffer address mismatch */
			cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
			    " in wrong small buffer",
			    __func__, qlge->instance, rx_ring->cq_id);
			goto fetal_error;
		}
		/* get this packet */
		mp1 = sbq_desc->mp;
		if ((err_flag != 0)|| (mp1 == NULL)) {
			/* failed on this packet, put it back for re-arming */
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_WARN, "get header from small buffer fail");
#endif
			ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
			mp1 = NULL;
		} else {
			/* Flush DMA'd data */
			(void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle,
			    0, header_len, DDI_DMA_SYNC_FORKERNEL);

			if ((qlge->ip_hdr_offset != 0)&&
			    (header_len < SMALL_BUFFER_SIZE)) {
				/*
				 * copy entire header to a 2 bytes boundary
				 * address for 8100 adapters so that the IP
				 * header can be on a 4 byte boundary address
				 */
				bcopy(mp1->b_rptr,
				    (mp1->b_rptr + SMALL_BUFFER_SIZE +
				    qlge->ip_hdr_offset),
				    header_len);
				mp1->b_rptr += SMALL_BUFFER_SIZE +
				    qlge->ip_hdr_offset;
			}

			/*
			 * Adjust the mp payload_len to match
			 * the packet header payload_len
			 */
			mp1->b_wptr = mp1->b_rptr + header_len;
			mp1->b_next = mp1->b_cont = NULL;
			QL_DUMP(DBG_RX, "\t RX packet header dump:\n",
			    (uint8_t *)mp1->b_rptr, 8, header_len);
		}
	}

	/*
	 * packet data or whole packet can be in small or one or
	 * several large buffer(s)
	 */
	if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) {
		/*
		 * The data is in a single small buffer.
		 */
		sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);

		ASSERT(sbq_desc != NULL);

		QL_PRINT(DBG_RX,
		    ("%d bytes in a single small buffer, sbq_desc = %p, "
		    "sbq_desc->bd_dma.dma_addr = %x,"
		    " ib_mac_rsp->data_addr = %x, mp = %p\n",
		    payload_len, sbq_desc, sbq_desc->bd_dma.dma_addr,
		    ib_mac_rsp->data_addr, sbq_desc->mp));

		/*
		 * Validate  addresses from the ASIC with the
		 * expected sbuf address
		 */
		if (cpu_to_le64(sbq_desc->bd_dma.dma_addr)
		    != ib_mac_rsp->data_addr) {
			/* Small buffer address mismatch */
			cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
			    " in wrong small buffer",
			    __func__, qlge->instance, rx_ring->cq_id);
			goto fetal_error;
		}
		/* get this packet */
		mp2 = sbq_desc->mp;
		if ((err_flag != 0) || (mp2 == NULL)) {
#ifdef QLGE_LOAD_UNLOAD
			/* failed on this packet, put it back for re-arming */
			cmn_err(CE_WARN, "ignore bad data from small buffer");
#endif
			ql_refill_sbuf_free_list(sbq_desc, B_FALSE);
			mp2 = NULL;
		} else {
			/* Adjust the buffer length to match the payload_len */
			mp2->b_wptr = mp2->b_rptr + payload_len;
			mp2->b_next = mp2->b_cont = NULL;
			/* Flush DMA'd data */
			(void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle,
			    0, payload_len, DDI_DMA_SYNC_FORKERNEL);
			QL_DUMP(DBG_RX, "\t RX packet payload dump:\n",
			    (uint8_t *)mp2->b_rptr, 8, payload_len);
			/*
			 * if payload is too small , copy to
			 * the end of packet header
			 */
			if ((mp1 != NULL) &&
			    (payload_len <= qlge->payload_copy_thresh) &&
			    (pkt_len <
			    (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) {
				bcopy(mp2->b_rptr, mp1->b_wptr, payload_len);
				mp1->b_wptr += payload_len;
				freemsg(mp2);
				mp2 = NULL;
			}
		}
	} else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
		/*
		 * The data is in a single large buffer.
		 */
		lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring);

		QL_PRINT(DBG_RX,
		    ("%d bytes in a single large buffer, lbq_desc = %p, "
		    "lbq_desc->bd_dma.dma_addr = %x,"
		    " ib_mac_rsp->data_addr = %x, mp = %p\n",
		    payload_len, lbq_desc, lbq_desc->bd_dma.dma_addr,
		    ib_mac_rsp->data_addr, lbq_desc->mp));

		ASSERT(lbq_desc != NULL);

		/*
		 * Validate  addresses from the ASIC with
		 * the expected lbuf address
		 */
		if (cpu_to_le64(lbq_desc->bd_dma.dma_addr)
		    != ib_mac_rsp->data_addr) {
			/* Large buffer address mismatch */
			cmn_err(CE_WARN, "%s(%d) ring%d packet saved"
			    " in wrong large buffer",
			    __func__, qlge->instance, rx_ring->cq_id);
			goto fetal_error;
		}
		mp2 = lbq_desc->mp;
		if ((err_flag != 0) || (mp2 == NULL)) {
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_WARN, "ignore bad data from large buffer");
#endif
			/* failed on this packet, put it back for re-arming */
			ql_refill_lbuf_free_list(lbq_desc, B_FALSE);
			mp2 = NULL;
		} else {
			/*
			 * Adjust the buffer length to match
			 * the packet payload_len
			 */
			mp2->b_wptr = mp2->b_rptr + payload_len;
			mp2->b_next = mp2->b_cont = NULL;
			/* Flush DMA'd data */
			(void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle,
			    0, payload_len, DDI_DMA_SYNC_FORKERNEL);
			QL_DUMP(DBG_RX, "\t RX packet payload dump:\n",
			    (uint8_t *)mp2->b_rptr, 8, payload_len);
			/*
			 * if payload is too small , copy to
			 * the end of packet header
			 */
			if ((mp1 != NULL) &&
			    (payload_len <= qlge->payload_copy_thresh) &&
			    (pkt_len<
			    (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) {
				bcopy(mp2->b_rptr, mp1->b_wptr, payload_len);
				mp1->b_wptr += payload_len;
				freemsg(mp2);
				mp2 = NULL;
			}
		}
	} else if (payload_len) {
		/*
		 * payload available but not in sml nor lrg buffer,
		 * so, it is saved in IAL
		 */
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_NOTE, "packet chained in IAL \n");
#endif
		/* lrg buf addresses are saved in one small buffer */
		sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring);
		curr_ial_ptr = (uint64_t *)sbq_desc->bd_dma.vaddr;
		done = 0;
		while (!done) {
			ial_data_addr_low =
			    (uint32_t)(le64_to_cpu(*curr_ial_ptr) &
			    0xFFFFFFFE);
			/* check if this is the last packet fragment */
			done = (uint32_t)(le64_to_cpu(*curr_ial_ptr) & 1);
			curr_ial_ptr++;
			/*
			 * The data is in one or several large buffer(s).
			 */
			lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring);
			actual_data_addr_low =
			    (uint32_t)(lbq_desc->bd_dma.dma_addr &
			    0xFFFFFFFE);
			if (ial_data_addr_low != actual_data_addr_low) {
				cmn_err(CE_WARN,
				    "packet saved in wrong ial lrg buffer"
				    " expected %x, actual %lx",
				    ial_data_addr_low,
				    (uintptr_t)lbq_desc->bd_dma.dma_addr);
				goto fetal_error;
			}

			if (mp_ial == NULL) {
				mp_ial = mp2 = lbq_desc->mp;
			} else {
				mp2->b_cont = lbq_desc->mp;
				mp2 = lbq_desc->mp;
			}
			mp2->b_next = NULL;
			mp2->b_cont = NULL;
			size = (payload_len < rx_ring->lbq_buf_size)?
			    payload_len : rx_ring->lbq_buf_size;
			mp2->b_wptr = mp2->b_rptr + size;
			/* Flush DMA'd data */
			(void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle,
			    0, size, DDI_DMA_SYNC_FORKERNEL);
			payload_len -= size;
			QL_DUMP(DBG_RX, "\t Mac data dump:\n",
			    (uint8_t *)mp2->b_rptr, 8, size);
		}
		mp2 = mp_ial;
		freemsg(sbq_desc->mp);
	}
	/*
	 * some packets' hdr not split, then send mp2 upstream, otherwise,
	 * concatenate message block mp2 to the tail of message header, mp1
	 */
	if (!err_flag) {
		if (mp1) {
			if (mp2) {
				QL_PRINT(DBG_RX, ("packet in mp1 and mp2\n"));
				linkb(mp1, mp2); /* mp1->b_cont = mp2; */
				mp = mp1;
			} else {
				QL_PRINT(DBG_RX, ("packet in mp1 only\n"));
				mp = mp1;
			}
		} else if (mp2) {
			QL_PRINT(DBG_RX, ("packet in mp2 only\n"));
			mp = mp2;
		}
	}
	return (mp);

fetal_error:
	/* Fetal Error! */
	*mp->b_wptr = 0;
	return (mp);

}

/*
 * Bump completion queue consumer index.
 */
static void
ql_update_cq(struct rx_ring *rx_ring)
{
	rx_ring->cnsmr_idx++;
	rx_ring->curr_entry++;
	if (rx_ring->cnsmr_idx >= rx_ring->cq_len) {
		rx_ring->cnsmr_idx = 0;
		rx_ring->curr_entry = rx_ring->cq_dma.vaddr;
	}
}

/*
 * Update completion queue consumer index.
 */
static void
ql_write_cq_idx(struct rx_ring *rx_ring)
{
	qlge_t *qlge = rx_ring->qlge;

	ql_write_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg,
	    rx_ring->cnsmr_idx);
}

/*
 * Processes a SYS-Chip Event Notification Completion Event.
 * The incoming notification event that describes a link up/down
 * or some sorts of error happens.
 */
static void
ql_process_chip_ae_intr(qlge_t *qlge,
    struct ib_sys_event_iocb_rsp *ib_sys_event_rsp_ptr)
{
	uint8_t eventType = ib_sys_event_rsp_ptr->event_type;
	uint32_t soft_req = 0;

	switch (eventType) {
		case SYS_EVENT_PORT_LINK_UP:	/* 0x0h */
			QL_PRINT(DBG_MBX, ("Port Link Up\n"));
			break;

		case SYS_EVENT_PORT_LINK_DOWN:	/* 0x1h */
			QL_PRINT(DBG_MBX, ("Port Link Down\n"));
			break;

		case SYS_EVENT_MULTIPLE_CAM_HITS : /* 0x6h */
			cmn_err(CE_WARN, "A multiple CAM hits look up error "
			    "occurred");
			soft_req |= NEED_HW_RESET;
			break;

		case SYS_EVENT_SOFT_ECC_ERR:	/* 0x7h */
			cmn_err(CE_WARN, "Soft ECC error detected");
			soft_req |= NEED_HW_RESET;
			break;

		case SYS_EVENT_MGMT_FATAL_ERR:	/* 0x8h */
			cmn_err(CE_WARN, "Management (MPI) Processor fatal"
			    " error occured");
			soft_req |= NEED_MPI_RESET;
			break;

		case SYS_EVENT_MAC_INTERRUPT:	/* 0x9h */
			QL_PRINT(DBG_MBX, ("MAC Interrupt"));
			break;

		case SYS_EVENT_PCI_ERR_READING_SML_LRG_BUF:	/* 0x40h */
			cmn_err(CE_WARN, "PCI Error reading small/large "
			    "buffers occured");
			soft_req |= NEED_HW_RESET;
			break;

		default:
			QL_PRINT(DBG_RX, ("%s(%d) unknown Sys Event: "
			    "type 0x%x occured",
			    __func__, qlge->instance, eventType));
			break;
	}

	if ((soft_req & NEED_MPI_RESET) != 0) {
		ql_wake_mpi_reset_soft_intr(qlge);
	} else if ((soft_req & NEED_HW_RESET) != 0) {
		ql_wake_asic_reset_soft_intr(qlge);
	}
}

/*
 * set received packet checksum flag
 */
void
ql_set_rx_cksum(mblk_t *mp, struct ib_mac_iocb_rsp *net_rsp)
{
	uint32_t flags;

	/* Not TCP or UDP packet? nothing more to do */
	if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) == 0) &&
	    ((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) == 0))
	return;

	/* No CKO support for IPv6 */
	if ((net_rsp->flags3 & IB_MAC_IOCB_RSP_V6) != 0)
		return;

	/*
	 * If checksum error, don't set flags; stack will calculate
	 * checksum, detect the error and update statistics
	 */
	if (((net_rsp->flags1 & IB_MAC_IOCB_RSP_TE) != 0) ||
	    ((net_rsp->flags1 & IB_MAC_IOCB_RSP_IE) != 0))
		return;

	/* TCP or UDP packet and checksum valid */
	if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) != 0) &&
	    ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) {
		flags = HCK_FULLCKSUM_OK;
		mac_hcksum_set(mp, 0, 0, 0, 0, flags);
	}
	if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) != 0) &&
	    ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) {
		flags = HCK_FULLCKSUM_OK;
		mac_hcksum_set(mp, 0, 0, 0, 0, flags);
	}
}

/*
 * This function goes through h/w descriptor in one specified rx ring,
 * receives the data if the descriptor status shows the data is ready.
 * It returns a chain of mblks containing the received data, to be
 * passed up to mac_rx_ring().
 */
mblk_t *
ql_ring_rx(struct rx_ring *rx_ring, int poll_bytes)
{
	qlge_t *qlge = rx_ring->qlge;
	uint32_t prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
	struct ib_mac_iocb_rsp *net_rsp;
	mblk_t *mp;
	mblk_t *mblk_head;
	mblk_t **mblk_tail;
	uint32_t received_bytes = 0;
	boolean_t done = B_FALSE;
	uint32_t length;

#ifdef QLGE_TRACK_BUFFER_USAGE
	uint32_t consumer_idx;
	uint32_t producer_idx;
	uint32_t num_free_entries;
	uint32_t temp;

	temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg);
	consumer_idx = temp & 0x0000ffff;
	producer_idx = (temp >> 16);

	if (consumer_idx > producer_idx)
		num_free_entries = (consumer_idx - producer_idx);
	else
		num_free_entries = NUM_RX_RING_ENTRIES - (
		    producer_idx - consumer_idx);

	if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id])
		qlge->cq_low_count[rx_ring->cq_id] = num_free_entries;

#endif
	mblk_head = NULL;
	mblk_tail = &mblk_head;

	while (!done && (prod != rx_ring->cnsmr_idx)) {
		QL_PRINT(DBG_RX,
		    ("%s cq_id = %d, prod = %d, cnsmr = %d.\n",
		    __func__, rx_ring->cq_id, prod, rx_ring->cnsmr_idx));

		net_rsp = (struct ib_mac_iocb_rsp *)rx_ring->curr_entry;
		(void) ddi_dma_sync(rx_ring->cq_dma.dma_handle,
		    (off_t)((uintptr_t)net_rsp -
		    (uintptr_t)rx_ring->cq_dma.vaddr),
		    (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL);
		QL_DUMP(DBG_RX, "qlge_ring_rx: rx completion iocb\n",
		    rx_ring->curr_entry, 8, (size_t)sizeof (*net_rsp));

		switch (net_rsp->opcode) {

		case OPCODE_IB_MAC_IOCB:
			/* Adding length of pkt header and payload */
			length = le32_to_cpu(net_rsp->data_len) +
			    le32_to_cpu(net_rsp->hdr_len);
			if ((poll_bytes != QLGE_POLL_ALL) &&
			    ((received_bytes + length) > poll_bytes)) {
				done = B_TRUE;
				continue;
			}
			received_bytes += length;

			mp = ql_build_rx_mp(qlge, rx_ring, net_rsp);
			if (mp != NULL) {
				if (rx_ring->mac_flags != QL_MAC_STARTED) {
					/*
					 * Increment number of packets we have
					 * indicated to the stack, should be
					 * decremented when we get it back
					 * or when freemsg is called
					 */
					ASSERT(rx_ring->rx_indicate
					    <= rx_ring->cq_len);
#ifdef QLGE_LOAD_UNLOAD
					cmn_err(CE_WARN, "%s do not send to OS,"
					    " mac_flags %d, indicate %d",
					    __func__, rx_ring->mac_flags,
					    rx_ring->rx_indicate);
#endif
					QL_PRINT(DBG_RX,
					    ("cq_id = %d, packet "
					    "dropped, mac not "
					    "enabled.\n",
					    rx_ring->cq_id));
					rx_ring->rx_pkt_dropped_mac_unenabled++;

					/* rx_lock is expected to be held */
					mutex_exit(&rx_ring->rx_lock);
					freemsg(mp);
					mutex_enter(&rx_ring->rx_lock);
					mp = NULL;
				}

				if (mp != NULL) {
					/*
					 * IP full packet has been
					 * successfully verified by
					 * H/W and is correct
					 */
					ql_set_rx_cksum(mp, net_rsp);

					rx_ring->rx_packets++;
					rx_ring->rx_bytes += length;
					*mblk_tail = mp;
					mblk_tail = &mp->b_next;
				}
			} else {
				QL_PRINT(DBG_RX,
				    ("cq_id = %d, packet dropped\n",
				    rx_ring->cq_id));
				rx_ring->rx_packets_dropped_no_buffer++;
			}
			break;

		case OPCODE_IB_SYS_EVENT_IOCB:
			ql_process_chip_ae_intr(qlge,
			    (struct ib_sys_event_iocb_rsp *)
			    net_rsp);
			break;

		default:
			cmn_err(CE_WARN,
			    "%s Ring(%d)Hit default case, not handled!"
			    " dropping the packet, "
			    "opcode = %x.", __func__, rx_ring->cq_id,
			    net_rsp->opcode);
			break;
		}
		/* increment cnsmr_idx and curr_entry */
		ql_update_cq(rx_ring);
		prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);

	}
	/* update cnsmr_idx */
	ql_write_cq_idx(rx_ring);
	/* do not enable interrupt for polling mode */
	if (poll_bytes == QLGE_POLL_ALL)
		ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq);
	return (mblk_head);
}

/* Process an outbound completion from an rx ring. */
static void
ql_process_mac_tx_intr(qlge_t *qlge, struct ob_mac_iocb_rsp *mac_rsp)
{
	struct tx_ring *tx_ring;
	struct tx_ring_desc *tx_ring_desc;
	int j;

	tx_ring = &qlge->tx_ring[mac_rsp->txq_idx];
	tx_ring_desc = tx_ring->wq_desc;
	tx_ring_desc += mac_rsp->tid;

	if (tx_ring_desc->tx_type == USE_DMA) {
		QL_PRINT(DBG_TX, ("%s(%d): tx type USE_DMA\n",
		    __func__, qlge->instance));

		/*
		 * Release the DMA resource that is used for
		 * DMA binding.
		 */
		for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) {
			(void) ddi_dma_unbind_handle(
			    tx_ring_desc->tx_dma_handle[j]);
		}

		tx_ring_desc->tx_dma_handle_used = 0;
		/*
		 * Free the mblk after sending completed
		 */
		if (tx_ring_desc->mp != NULL) {
			freemsg(tx_ring_desc->mp);
			tx_ring_desc->mp = NULL;
		}
	}

	tx_ring->obytes += tx_ring_desc->tx_bytes;
	tx_ring->opackets++;

	if (mac_rsp->flags1 & (OB_MAC_IOCB_RSP_E | OB_MAC_IOCB_RSP_S |
	    OB_MAC_IOCB_RSP_L | OB_MAC_IOCB_RSP_B)) {
		tx_ring->errxmt++;
		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_E) {
			/* EMPTY */
			QL_PRINT(DBG_TX,
			    ("Total descriptor length did not match "
			    "transfer length.\n"));
		}
		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_S) {
			/* EMPTY */
			QL_PRINT(DBG_TX,
			    ("Frame too short to be legal, not sent.\n"));
		}
		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_L) {
			/* EMPTY */
			QL_PRINT(DBG_TX,
			    ("Frame too long, but sent anyway.\n"));
		}
		if (mac_rsp->flags3 & OB_MAC_IOCB_RSP_B) {
			/* EMPTY */
			QL_PRINT(DBG_TX,
			    ("PCI backplane error. Frame not sent.\n"));
		}
	}
	atomic_inc_32(&tx_ring->tx_free_count);
}

/*
 * clean up tx completion iocbs
 */
static int
ql_clean_outbound_rx_ring(struct rx_ring *rx_ring)
{
	qlge_t *qlge = rx_ring->qlge;
	uint32_t prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
	struct ob_mac_iocb_rsp *net_rsp = NULL;
	int count = 0;
	struct tx_ring *tx_ring;
	boolean_t resume_tx = B_FALSE;

	mutex_enter(&rx_ring->rx_lock);
#ifdef QLGE_TRACK_BUFFER_USAGE
	{
	uint32_t consumer_idx;
	uint32_t producer_idx;
	uint32_t num_free_entries;
	uint32_t temp;

	temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg);
	consumer_idx = temp & 0x0000ffff;
	producer_idx = (temp >> 16);

	if (consumer_idx > producer_idx)
		num_free_entries = (consumer_idx - producer_idx);
	else
		num_free_entries = NUM_RX_RING_ENTRIES -
		    (producer_idx - consumer_idx);

	if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id])
		qlge->cq_low_count[rx_ring->cq_id] = num_free_entries;

	}
#endif
	/* While there are entries in the completion queue. */
	while (prod != rx_ring->cnsmr_idx) {

		QL_PRINT(DBG_RX,
		    ("%s cq_id = %d, prod = %d, cnsmr = %d.\n", __func__,
		    rx_ring->cq_id, prod, rx_ring->cnsmr_idx));

		net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry;
		(void) ddi_dma_sync(rx_ring->cq_dma.dma_handle,
		    (off_t)((uintptr_t)net_rsp -
		    (uintptr_t)rx_ring->cq_dma.vaddr),
		    (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL);

		QL_DUMP(DBG_RX, "ql_clean_outbound_rx_ring: "
		    "response packet data\n",
		    rx_ring->curr_entry, 8,
		    (size_t)sizeof (*net_rsp));

		switch (net_rsp->opcode) {

		case OPCODE_OB_MAC_OFFLOAD_IOCB:
		case OPCODE_OB_MAC_IOCB:
			ql_process_mac_tx_intr(qlge, net_rsp);
			break;

		default:
			cmn_err(CE_WARN,
			    "%s Hit default case, not handled! "
			    "dropping the packet,"
			    " opcode = %x.",
			    __func__, net_rsp->opcode);
			break;
		}
		count++;
		ql_update_cq(rx_ring);
		prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
	}
	ql_write_cq_idx(rx_ring);

	mutex_exit(&rx_ring->rx_lock);

	tx_ring = &qlge->tx_ring[net_rsp->txq_idx];

	mutex_enter(&tx_ring->tx_lock);

	if (tx_ring->queue_stopped &&
	    (tx_ring->tx_free_count > TX_RESUME_THRESHOLD)) {
		/*
		 * The queue got stopped because the tx_ring was full.
		 * Wake it up, because it's now at least 25% empty.
		 */
		tx_ring->queue_stopped = 0;
		resume_tx = B_TRUE;
	}

	mutex_exit(&tx_ring->tx_lock);
	/* Don't hold the lock during OS callback */
	if (resume_tx)
		RESUME_TX(tx_ring);
	return (count);
}

/*
 * reset asic when error happens
 */
/* ARGSUSED */
static uint_t
ql_asic_reset_work(caddr_t arg1, caddr_t arg2)
{
	qlge_t *qlge = (qlge_t *)((void *)arg1);
	int status;

	mutex_enter(&qlge->gen_mutex);
	status = ql_bringdown_adapter(qlge);
	if (status != DDI_SUCCESS)
		goto error;

	status = ql_bringup_adapter(qlge);
	if (status != DDI_SUCCESS)
		goto error;
	mutex_exit(&qlge->gen_mutex);
	return (DDI_INTR_CLAIMED);

error:
	mutex_exit(&qlge->gen_mutex);
	cmn_err(CE_WARN,
	    "qlge up/down cycle failed, closing device");
	return (DDI_INTR_CLAIMED);
}

/*
 * Reset MPI
 */
/* ARGSUSED */
static uint_t
ql_mpi_reset_work(caddr_t arg1, caddr_t arg2)
{
	qlge_t *qlge = (qlge_t *)((void *)arg1);

	(void) ql_reset_mpi_risc(qlge);
	return (DDI_INTR_CLAIMED);
}

/*
 * Process MPI mailbox messages
 */
/* ARGSUSED */
static uint_t
ql_mpi_event_work(caddr_t arg1, caddr_t arg2)
{
	qlge_t *qlge = (qlge_t *)((void *)arg1);

	ql_do_mpi_intr(qlge);
	return (DDI_INTR_CLAIMED);
}

/* Fire up a handler to reset the MPI processor. */
void
ql_wake_asic_reset_soft_intr(qlge_t *qlge)
{
	(void) ddi_intr_trigger_softint(qlge->asic_reset_intr_hdl, NULL);
}

static void
ql_wake_mpi_reset_soft_intr(qlge_t *qlge)
{
	(void) ddi_intr_trigger_softint(qlge->mpi_reset_intr_hdl, NULL);
}

static void
ql_wake_mpi_event_soft_intr(qlge_t *qlge)
{
	(void) ddi_intr_trigger_softint(qlge->mpi_event_intr_hdl, NULL);
}

/*
 * This handles a fatal error, MPI activity, and the default
 * rx_ring in an MSI-X multiple interrupt vector environment.
 * In MSI/Legacy environment it also process the rest of
 * the rx_rings.
 */
/* ARGSUSED */
static uint_t
ql_isr(caddr_t arg1, caddr_t arg2)
{
	struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
	qlge_t *qlge = rx_ring->qlge;
	struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];
	uint32_t var, prod;
	int i;
	int work_done = 0;

	mblk_t *mp;

	_NOTE(ARGUNUSED(arg2));

	++qlge->rx_interrupts[rx_ring->cq_id];

	if (ql_atomic_read_32(&qlge->intr_ctx[0].irq_cnt)) {
		ql_write_reg(qlge, REG_RSVD7, 0xfeed0002);
		var = ql_read_reg(qlge, REG_ERROR_STATUS);
		var = ql_read_reg(qlge, REG_STATUS);
		var = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1);
		return (DDI_INTR_CLAIMED);
	}

	ql_disable_completion_interrupt(qlge, intr_ctx->intr);

	/*
	 * Check the default queue and wake handler if active.
	 */
	rx_ring = &qlge->rx_ring[0];
	prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
	QL_PRINT(DBG_INTR, ("rx-ring[0] prod index 0x%x, consumer 0x%x ",
	    prod, rx_ring->cnsmr_idx));
	/* check if interrupt is due to incoming packet */
	if (prod != rx_ring->cnsmr_idx) {
		QL_PRINT(DBG_INTR, ("Waking handler for rx_ring[0].\n"));
		ql_disable_completion_interrupt(qlge, intr_ctx->intr);
		mutex_enter(&rx_ring->rx_lock);
		mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
		mutex_exit(&rx_ring->rx_lock);

		if (mp != NULL)
			RX_UPSTREAM(rx_ring, mp);
		work_done++;
	} else {
		/*
		 * If interrupt is not due to incoming packet, read status
		 * register to see if error happens or mailbox interrupt.
		 */
		var = ql_read_reg(qlge, REG_STATUS);
		if ((var & STATUS_FE) != 0) {
			ql_write_reg(qlge, REG_RSVD7, 0xfeed0003);

			cmn_err(CE_WARN, "Got fatal error, STS = %x.", var);
			var = ql_read_reg(qlge, REG_ERROR_STATUS);
			cmn_err(CE_WARN,
			    "Resetting chip. Error Status Register = 0x%x",
			    var);
			ql_wake_asic_reset_soft_intr(qlge);
			return (DDI_INTR_CLAIMED);
		}

		/*
		 * Check MPI processor activity.
		 */
		if ((var & STATUS_PI) != 0) {
			/*
			 * We've got an async event or mailbox completion.
			 * Handle it and clear the source of the interrupt.
			 */
			ql_write_reg(qlge, REG_RSVD7, 0xfeed0004);

			QL_PRINT(DBG_INTR, ("Got MPI processor interrupt.\n"));
			ql_disable_completion_interrupt(qlge, intr_ctx->intr);
			ql_wake_mpi_event_soft_intr(qlge);
			work_done++;
		}
	}

	if (qlge->intr_type != DDI_INTR_TYPE_MSIX) {
		/*
		 * Start the DPC for each active queue.
		 */
		for (i = 1; i < qlge->rx_ring_count; i++) {
			rx_ring = &qlge->rx_ring[i];

			if (ql_read_sh_reg(rx_ring->prod_idx_sh_reg) !=
			    rx_ring->cnsmr_idx) {
				QL_PRINT(DBG_INTR,
				    ("Waking handler for rx_ring[%d].\n", i));

				ql_disable_completion_interrupt(qlge,
				    rx_ring->irq);
				if (rx_ring->type == TX_Q) {
					(void) ql_clean_outbound_rx_ring(
					    rx_ring);
					ql_enable_completion_interrupt(
					    rx_ring->qlge, rx_ring->irq);
				} else {
					mutex_enter(&rx_ring->rx_lock);
					mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
					mutex_exit(&rx_ring->rx_lock);
					if (mp != NULL)
						RX_UPSTREAM(rx_ring, mp);
#ifdef QLGE_LOAD_UNLOAD
					if (rx_ring->mac_flags ==
					    QL_MAC_STOPPED)
						cmn_err(CE_NOTE,
						    "%s rx_indicate(%d) %d\n",
						    __func__, i,
						    rx_ring->rx_indicate);
#endif
				}
				work_done++;
			}
		}
	}

	ql_enable_completion_interrupt(qlge, intr_ctx->intr);

	return (work_done ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
}

/*
 * MSI-X Multiple Vector Interrupt Handler for outbound (TX) completions.
 */
/* ARGSUSED */
static uint_t
ql_msix_tx_isr(caddr_t arg1, caddr_t arg2)
{
	struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
	qlge_t *qlge = rx_ring->qlge;
	_NOTE(ARGUNUSED(arg2));

	++qlge->rx_interrupts[rx_ring->cq_id];
	(void) ql_clean_outbound_rx_ring(rx_ring);
	ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq);

	return (DDI_INTR_CLAIMED);
}

/*
 * Poll n_bytes of chained incoming packets
 */
mblk_t *
ql_ring_rx_poll(void *arg, int n_bytes)
{
	struct rx_ring *rx_ring = (struct rx_ring *)arg;
	qlge_t *qlge = rx_ring->qlge;
	mblk_t *mp = NULL;
	uint32_t var;

	ASSERT(n_bytes >= 0);
	QL_PRINT(DBG_GLD, ("%s for ring(%d) to read max %d bytes\n",
	    __func__, rx_ring->cq_id, n_bytes));

	++qlge->rx_polls[rx_ring->cq_id];

	if (n_bytes == 0)
		return (mp);
	mutex_enter(&rx_ring->rx_lock);
	mp = ql_ring_rx(rx_ring, n_bytes);
	mutex_exit(&rx_ring->rx_lock);

	if ((rx_ring->cq_id == 0) && (mp == NULL)) {
		var = ql_read_reg(qlge, REG_STATUS);
		/*
		 * Check for fatal error.
		 */
		if ((var & STATUS_FE) != 0) {
			ql_write_reg(qlge, REG_RSVD7, 0xfeed0003);
			var = ql_read_reg(qlge, REG_ERROR_STATUS);
			cmn_err(CE_WARN, "Got fatal error %x.", var);
			ql_wake_asic_reset_soft_intr(qlge);
		}
		/*
		 * Check MPI processor activity.
		 */
		if ((var & STATUS_PI) != 0) {
			/*
			 * We've got an async event or mailbox completion.
			 * Handle it and clear the source of the interrupt.
			 */
			ql_write_reg(qlge, REG_RSVD7, 0xfeed0004);
			ql_do_mpi_intr(qlge);
		}
	}

	return (mp);
}

/*
 * MSI-X Multiple Vector Interrupt Handler for inbound (RX) completions.
 */
/* ARGSUSED */
static uint_t
ql_msix_rx_isr(caddr_t arg1, caddr_t arg2)
{
	struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1);
	qlge_t *qlge = rx_ring->qlge;
	mblk_t *mp;
	_NOTE(ARGUNUSED(arg2));

	QL_PRINT(DBG_INTR, ("%s for ring %d\n", __func__, rx_ring->cq_id));

	++qlge->rx_interrupts[rx_ring->cq_id];

	mutex_enter(&rx_ring->rx_lock);
	mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL);
	mutex_exit(&rx_ring->rx_lock);

	if (mp != NULL)
		RX_UPSTREAM(rx_ring, mp);

	return (DDI_INTR_CLAIMED);
}


/*
 *
 * Allocate DMA Buffer for ioctl service
 *
 */
static int
ql_alloc_ioctl_dma_buf(qlge_t *qlge)
{
	uint64_t phy_addr;
	uint64_t alloc_size;
	ddi_dma_cookie_t dma_cookie;

	alloc_size = qlge->ioctl_buf_dma_attr.mem_len =
	    max(WCS_MPI_CODE_RAM_LENGTH, MEMC_MPI_RAM_LENGTH);
	if (ql_alloc_phys(qlge->dip, &qlge->ioctl_buf_dma_attr.dma_handle,
	    &ql_buf_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &qlge->ioctl_buf_dma_attr.acc_handle,
	    (size_t)alloc_size,  /* mem size */
	    (size_t)0,  /* alignment */
	    (caddr_t *)&qlge->ioctl_buf_dma_attr.vaddr,
	    &dma_cookie) != 0) {
		cmn_err(CE_WARN, "%s(%d): ioctl DMA allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}

	phy_addr = dma_cookie.dmac_laddress;

	if (qlge->ioctl_buf_dma_attr.vaddr == NULL) {
		cmn_err(CE_WARN, "%s(%d): failed.", __func__, qlge->instance);
		return (DDI_FAILURE);
	}

	qlge->ioctl_buf_dma_attr.dma_addr = phy_addr;

	QL_PRINT(DBG_MBX, ("%s: ioctl_dma_buf_virt_addr = 0x%lx, "
	    "phy_addr = 0x%lx\n",
	    __func__, qlge->ioctl_buf_dma_attr.vaddr, phy_addr));

	return (DDI_SUCCESS);
}


/*
 * Function to free physical memory.
 */
static void
ql_free_phys(ddi_dma_handle_t *dma_handle, ddi_acc_handle_t *acc_handle)
{
	if (dma_handle != NULL) {
		(void) ddi_dma_unbind_handle(*dma_handle);
		if (acc_handle != NULL)
			ddi_dma_mem_free(acc_handle);
		ddi_dma_free_handle(dma_handle);
	}
}

/*
 * Function to free ioctl dma buffer.
 */
static void
ql_free_ioctl_dma_buf(qlge_t *qlge)
{
	if (qlge->ioctl_buf_dma_attr.dma_handle != NULL) {
		ql_free_phys(&qlge->ioctl_buf_dma_attr.dma_handle,
		    &qlge->ioctl_buf_dma_attr.acc_handle);

		qlge->ioctl_buf_dma_attr.vaddr = NULL;
		qlge->ioctl_buf_dma_attr.dma_handle = NULL;
	}
}

/*
 * Free shadow register space used for request and completion queues
 */
static void
ql_free_shadow_space(qlge_t *qlge)
{
	if (qlge->host_copy_shadow_dma_attr.dma_handle != NULL) {
		ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle,
		    &qlge->host_copy_shadow_dma_attr.acc_handle);
		bzero(&qlge->host_copy_shadow_dma_attr,
		    sizeof (qlge->host_copy_shadow_dma_attr));
	}

	if (qlge->buf_q_ptr_base_addr_dma_attr.dma_handle != NULL) {
		ql_free_phys(&qlge->buf_q_ptr_base_addr_dma_attr.dma_handle,
		    &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle);
		bzero(&qlge->buf_q_ptr_base_addr_dma_attr,
		    sizeof (qlge->buf_q_ptr_base_addr_dma_attr));
	}
}

/*
 * Allocate shadow register space for request and completion queues
 */
static int
ql_alloc_shadow_space(qlge_t *qlge)
{
	ddi_dma_cookie_t dma_cookie;

	if (ql_alloc_phys(qlge->dip,
	    &qlge->host_copy_shadow_dma_attr.dma_handle,
	    &ql_dev_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &qlge->host_copy_shadow_dma_attr.acc_handle,
	    (size_t)VM_PAGE_SIZE,  /* mem size */
	    (size_t)4, /* 4 bytes alignment */
	    (caddr_t *)&qlge->host_copy_shadow_dma_attr.vaddr,
	    &dma_cookie) != 0) {
		bzero(&qlge->host_copy_shadow_dma_attr,
		    sizeof (qlge->host_copy_shadow_dma_attr));

		cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory for "
		    "response shadow registers", __func__, qlge->instance);
		return (DDI_FAILURE);
	}

	qlge->host_copy_shadow_dma_attr.dma_addr = dma_cookie.dmac_laddress;

	if (ql_alloc_phys(qlge->dip,
	    &qlge->buf_q_ptr_base_addr_dma_attr.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle,
	    (size_t)VM_PAGE_SIZE,  /* mem size */
	    (size_t)4, /* 4 bytes alignment */
	    (caddr_t *)&qlge->buf_q_ptr_base_addr_dma_attr.vaddr,
	    &dma_cookie) != 0) {
		bzero(&qlge->buf_q_ptr_base_addr_dma_attr,
		    sizeof (qlge->buf_q_ptr_base_addr_dma_attr));

		cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory "
		    "for request shadow registers",
		    __func__, qlge->instance);
		goto err_wqp_sh_area;
	}
	qlge->buf_q_ptr_base_addr_dma_attr.dma_addr = dma_cookie.dmac_laddress;

	return (DDI_SUCCESS);

err_wqp_sh_area:
	ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle,
	    &qlge->host_copy_shadow_dma_attr.acc_handle);
	bzero(&qlge->host_copy_shadow_dma_attr,
	    sizeof (qlge->host_copy_shadow_dma_attr));

	return (DDI_FAILURE);
}

/*
 * Initialize a tx ring
 */
static void
ql_init_tx_ring(struct tx_ring *tx_ring)
{
	int i;
	struct ob_mac_iocb_req *mac_iocb_ptr = tx_ring->wq_dma.vaddr;
	struct tx_ring_desc *tx_ring_desc = tx_ring->wq_desc;

	for (i = 0; i < tx_ring->wq_len; i++) {
		tx_ring_desc->index = i;
		tx_ring_desc->queue_entry = mac_iocb_ptr;
		mac_iocb_ptr++;
		tx_ring_desc++;
	}
	tx_ring->tx_free_count = tx_ring->wq_len;
	tx_ring->queue_stopped = 0;
}

/*
 * Free one tx ring resources
 */
static void
ql_free_tx_resources(struct tx_ring *tx_ring)
{
	struct tx_ring_desc *tx_ring_desc;
	int i, j;

	ql_free_phys(&tx_ring->wq_dma.dma_handle, &tx_ring->wq_dma.acc_handle);
	bzero(&tx_ring->wq_dma, sizeof (tx_ring->wq_dma));

	if (tx_ring->wq_desc != NULL) {
		tx_ring_desc = tx_ring->wq_desc;
		for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
			for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) {
				if (tx_ring_desc->tx_dma_handle[j]) {
					/*
					 * The unbinding will happen in tx
					 * completion, here we just free the
					 * handles
					 */
					ddi_dma_free_handle(
					    &(tx_ring_desc->tx_dma_handle[j]));
					tx_ring_desc->tx_dma_handle[j] = NULL;
				}
			}
			if (tx_ring_desc->oal != NULL) {
				tx_ring_desc->oal_dma_addr = 0;
				tx_ring_desc->oal = NULL;
				tx_ring_desc->copy_buffer = NULL;
				tx_ring_desc->copy_buffer_dma_addr = 0;

				ql_free_phys(&tx_ring_desc->oal_dma.dma_handle,
				    &tx_ring_desc->oal_dma.acc_handle);
			}
		}
		kmem_free(tx_ring->wq_desc,
		    tx_ring->wq_len * sizeof (struct tx_ring_desc));
		tx_ring->wq_desc = NULL;
	}
	/* free the wqicb struct */
	if (tx_ring->wqicb_dma.dma_handle) {
		ql_free_phys(&tx_ring->wqicb_dma.dma_handle,
		    &tx_ring->wqicb_dma.acc_handle);
		bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma));
	}
}

/*
 * Allocate work (request) queue memory and transmit
 * descriptors for this transmit ring
 */
static int
ql_alloc_tx_resources(qlge_t *qlge, struct tx_ring *tx_ring)
{
	ddi_dma_cookie_t dma_cookie;
	struct tx_ring_desc *tx_ring_desc;
	int i, j;
	uint32_t length;

	/* allocate dma buffers for obiocbs */
	if (ql_alloc_phys(qlge->dip, &tx_ring->wq_dma.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &tx_ring->wq_dma.acc_handle,
	    (size_t)tx_ring->wq_size,	/* mem size */
	    (size_t)128, /* alignment:128 bytes boundary */
	    (caddr_t *)&tx_ring->wq_dma.vaddr,
	    &dma_cookie) != 0) {
		bzero(&tx_ring->wq_dma, sizeof (&tx_ring->wq_dma));
		cmn_err(CE_WARN, "%s(%d): reqQ allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}
	tx_ring->wq_dma.dma_addr = dma_cookie.dmac_laddress;

	tx_ring->wq_desc =
	    kmem_zalloc(tx_ring->wq_len * sizeof (struct tx_ring_desc),
	    KM_NOSLEEP);
	if (tx_ring->wq_desc == NULL) {
		goto err;
	} else {
		tx_ring_desc = tx_ring->wq_desc;
		/*
		 * Allocate a large enough structure to hold the following
		 * 1. oal buffer MAX_SGELEMENTS * sizeof (oal_entry) bytes
		 * 2. copy buffer of QL_MAX_COPY_LENGTH bytes
		 */
		for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
			length = (sizeof (struct oal_entry) * MAX_SG_ELEMENTS)
			    + QL_MAX_COPY_LENGTH;

			if (ql_alloc_phys(qlge->dip,
			    &tx_ring_desc->oal_dma.dma_handle,
			    &ql_desc_acc_attr,
			    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
			    &tx_ring_desc->oal_dma.acc_handle,
			    (size_t)length,	/* mem size */
			    (size_t)0, /* default alignment:8 bytes boundary */
			    (caddr_t *)&tx_ring_desc->oal_dma.vaddr,
			    &dma_cookie) != 0) {
				bzero(&tx_ring_desc->oal_dma,
				    sizeof (tx_ring_desc->oal_dma));
				cmn_err(CE_WARN, "%s(%d): reqQ tx buf &"
				    "oal alloc failed.",
				    __func__, qlge->instance);
				return (DDI_FAILURE);
			}

			tx_ring_desc->oal = tx_ring_desc->oal_dma.vaddr;
			tx_ring_desc->oal_dma_addr = dma_cookie.dmac_laddress;
			tx_ring_desc->copy_buffer =
			    (caddr_t)((uint8_t *)tx_ring_desc->oal
			    + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS));
			tx_ring_desc->copy_buffer_dma_addr =
			    (tx_ring_desc->oal_dma_addr
			    + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS));

			/* Allocate dma handles for transmit buffers */
			for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) {
				if (ddi_dma_alloc_handle(qlge->dip,
				    &tx_mapping_dma_attr,
				    DDI_DMA_DONTWAIT,
				    0, &tx_ring_desc->tx_dma_handle[j])
				    != DDI_SUCCESS) {
					cmn_err(CE_WARN,
					    "!%s: ddi_dma_alloc_handle: "
					    "tx_dma_handle "
					    "alloc failed", __func__);
					goto err;
				}
			}
		}
	}
	/* alloc a wqicb control block to load this tx ring to hw */
	if (ql_alloc_phys(qlge->dip, &tx_ring->wqicb_dma.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &tx_ring->wqicb_dma.acc_handle,
	    (size_t)sizeof (struct wqicb_t),	/* mem size */
	    (size_t)0, /* alignment:128 bytes boundary */
	    (caddr_t *)&tx_ring->wqicb_dma.vaddr,
	    &dma_cookie) != 0) {
		bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma));
		cmn_err(CE_WARN, "%s(%d): wqicb allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}
	tx_ring->wqicb_dma.dma_addr = dma_cookie.dmac_laddress;

	return (DDI_SUCCESS);

err:
	ql_free_tx_resources(tx_ring);
	return (DDI_FAILURE);
}

/*
 * Free one rx ring resources
 */
static void
ql_free_rx_resources(struct rx_ring *rx_ring)
{
	/* Free the small buffer queue. */
	if (rx_ring->sbq_dma.dma_handle) {
		ql_free_phys(&rx_ring->sbq_dma.dma_handle,
		    &rx_ring->sbq_dma.acc_handle);
		bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma));
	}

	/* Free the small buffer queue control blocks. */
	kmem_free(rx_ring->sbq_desc, rx_ring->sbq_len *
	    sizeof (struct bq_desc));
	rx_ring->sbq_desc = NULL;

	/* Free the large buffer queue. */
	if (rx_ring->lbq_dma.dma_handle) {
		ql_free_phys(&rx_ring->lbq_dma.dma_handle,
		    &rx_ring->lbq_dma.acc_handle);
		bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma));
	}

	/* Free the large buffer queue control blocks. */
	kmem_free(rx_ring->lbq_desc, rx_ring->lbq_len *
	    sizeof (struct bq_desc));
	rx_ring->lbq_desc = NULL;

	/* Free cqicb struct */
	if (rx_ring->cqicb_dma.dma_handle) {
		ql_free_phys(&rx_ring->cqicb_dma.dma_handle,
		    &rx_ring->cqicb_dma.acc_handle);
		bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma));
	}
	/* Free the rx queue. */
	if (rx_ring->cq_dma.dma_handle) {
		ql_free_phys(&rx_ring->cq_dma.dma_handle,
		    &rx_ring->cq_dma.acc_handle);
		bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma));
	}
}

/*
 * Allocate queues and buffers for this completions queue based
 * on the values in the parameter structure.
 */
static int
ql_alloc_rx_resources(qlge_t *qlge, struct rx_ring *rx_ring)
{
	ddi_dma_cookie_t dma_cookie;

	if (ql_alloc_phys(qlge->dip, &rx_ring->cq_dma.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &rx_ring->cq_dma.acc_handle,
	    (size_t)rx_ring->cq_size,  /* mem size */
	    (size_t)128, /* alignment:128 bytes boundary */
	    (caddr_t *)&rx_ring->cq_dma.vaddr,
	    &dma_cookie) != 0)	{
		bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma));
		cmn_err(CE_WARN, "%s(%d): rspQ allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}
	rx_ring->cq_dma.dma_addr = dma_cookie.dmac_laddress;

	if (rx_ring->sbq_len != 0) {
		/*
		 * Allocate small buffer queue.
		 */
		if (ql_alloc_phys(qlge->dip, &rx_ring->sbq_dma.dma_handle,
		    &ql_desc_acc_attr,
		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
		    &rx_ring->sbq_dma.acc_handle,
		    (size_t)rx_ring->sbq_size,  /* mem size */
		    (size_t)128, /* alignment:128 bytes boundary */
		    (caddr_t *)&rx_ring->sbq_dma.vaddr,
		    &dma_cookie) != 0) {
			bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma));
			cmn_err(CE_WARN,
			    "%s(%d): small buffer queue allocation failed.",
			    __func__, qlge->instance);
			goto err_mem;
		}
		rx_ring->sbq_dma.dma_addr = dma_cookie.dmac_laddress;

		/*
		 * Allocate small buffer queue control blocks.
		 */
		rx_ring->sbq_desc =
		    kmem_zalloc(rx_ring->sbq_len * sizeof (struct bq_desc),
		    KM_NOSLEEP);
		if (rx_ring->sbq_desc == NULL) {
			cmn_err(CE_WARN,
			    "sbq control block allocation failed.");
			goto err_mem;
		}

		ql_init_sbq_ring(rx_ring);
	}

	if (rx_ring->lbq_len != 0) {
		/*
		 * Allocate large buffer queue.
		 */
		if (ql_alloc_phys(qlge->dip, &rx_ring->lbq_dma.dma_handle,
		    &ql_desc_acc_attr,
		    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
		    &rx_ring->lbq_dma.acc_handle,
		    (size_t)rx_ring->lbq_size,  /* mem size */
		    (size_t)128, /* alignment:128 bytes boundary */
		    (caddr_t *)&rx_ring->lbq_dma.vaddr,
		    &dma_cookie) != 0) {
			bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma));
			cmn_err(CE_WARN, "%s(%d): lbq allocation failed.",
			    __func__, qlge->instance);
			goto err_mem;
		}
		rx_ring->lbq_dma.dma_addr = dma_cookie.dmac_laddress;

		/*
		 * Allocate large buffer queue control blocks.
		 */
		rx_ring->lbq_desc =
		    kmem_zalloc(rx_ring->lbq_len * sizeof (struct bq_desc),
		    KM_NOSLEEP);
		if (rx_ring->lbq_desc == NULL) {
			cmn_err(CE_WARN,
			    "Large buffer queue control block allocation "
			    "failed.");
			goto err_mem;
		}
		ql_init_lbq_ring(rx_ring);
	}

	if (ql_alloc_phys(qlge->dip, &rx_ring->cqicb_dma.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &rx_ring->cqicb_dma.acc_handle,
	    (size_t)sizeof (struct cqicb_t),  /* mem size */
	    (size_t)0, /* alignment:128 bytes boundary */
	    (caddr_t *)&rx_ring->cqicb_dma.vaddr,
	    &dma_cookie) != 0) {
		bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma));
		cmn_err(CE_WARN, "%s(%d): cqicb allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}
	rx_ring->cqicb_dma.dma_addr = dma_cookie.dmac_laddress;

	return (DDI_SUCCESS);

err_mem:
	ql_free_rx_resources(rx_ring);
	return (DDI_FAILURE);
}

/*
 * Frees tx/rx queues memory resources
 */
static void
ql_free_mem_resources(qlge_t *qlge)
{
	int i;

	if (qlge->ricb_dma.dma_handle) {
		/* free the ricb struct */
		ql_free_phys(&qlge->ricb_dma.dma_handle,
		    &qlge->ricb_dma.acc_handle);
		bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma));
	}

	ql_free_rx_buffers(qlge);

	ql_free_ioctl_dma_buf(qlge);

	for (i = 0; i < qlge->tx_ring_count; i++)
		ql_free_tx_resources(&qlge->tx_ring[i]);

	for (i = 0; i < qlge->rx_ring_count; i++)
		ql_free_rx_resources(&qlge->rx_ring[i]);

	ql_free_shadow_space(qlge);
}

/*
 * Allocate buffer queues, large buffers and small buffers etc
 *
 * This API is called in the gld_attach member function. It is called
 * only once.  Later reset,reboot should not re-allocate all rings and
 * buffers.
 */
static int
ql_alloc_mem_resources(qlge_t *qlge)
{
	int i;
	ddi_dma_cookie_t dma_cookie;

	/* Allocate space for our shadow registers */
	if (ql_alloc_shadow_space(qlge))
		return (DDI_FAILURE);

	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (ql_alloc_rx_resources(qlge, &qlge->rx_ring[i]) != 0) {
			cmn_err(CE_WARN, "RX resource allocation failed.");
			goto err_mem;
		}
	}
	/* Allocate tx queue resources */
	for (i = 0; i < qlge->tx_ring_count; i++) {
		if (ql_alloc_tx_resources(qlge, &qlge->tx_ring[i]) != 0) {
			cmn_err(CE_WARN, "Tx resource allocation failed.");
			goto err_mem;
		}
	}

	if (ql_alloc_ioctl_dma_buf(qlge) != DDI_SUCCESS) {
		goto err_mem;
	}

	if (ql_alloc_rx_buffers(qlge) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "?%s(%d): ql_alloc_rx_buffers failed",
		    __func__, qlge->instance);
		goto err_mem;
	}

	qlge->sequence |= INIT_ALLOC_RX_BUF;

	if (ql_alloc_phys(qlge->dip, &qlge->ricb_dma.dma_handle,
	    &ql_desc_acc_attr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &qlge->ricb_dma.acc_handle,
	    (size_t)sizeof (struct ricb),  /* mem size */
	    (size_t)0, /* alignment:128 bytes boundary */
	    (caddr_t *)&qlge->ricb_dma.vaddr,
	    &dma_cookie) != 0) {
		bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma));
		cmn_err(CE_WARN, "%s(%d): ricb allocation failed.",
		    __func__, qlge->instance);
		return (DDI_FAILURE);
	}
	qlge->ricb_dma.dma_addr = dma_cookie.dmac_laddress;

	return (DDI_SUCCESS);

err_mem:
	ql_free_mem_resources(qlge);
	return (DDI_FAILURE);
}


/*
 * Function used to allocate physical memory and zero it.
 */

static int
ql_alloc_phys(dev_info_t *dip, ddi_dma_handle_t *dma_handle,
    ddi_device_acc_attr_t *device_acc_attr,
    uint_t dma_flags,
    ddi_acc_handle_t *acc_handle,
    size_t size,
    size_t alignment,
    caddr_t *vaddr,
    ddi_dma_cookie_t *dma_cookie)
{
	size_t rlen;
	uint_t cnt;

	/*
	 * Workaround for SUN XMITS buffer must end and start on 8 byte
	 * boundary. Else, hardware will overrun the buffer. Simple fix is
	 * to make sure buffer has enough room for overrun.
	 */
	if (size & 7) {
		size += 8 - (size & 7);
	}

	/* Adjust the alignment if requested */
	if (alignment) {
		dma_attr.dma_attr_align = alignment;
	}

	/*
	 * Allocate DMA handle
	 */
	if (ddi_dma_alloc_handle(dip, &dma_attr, DDI_DMA_SLEEP, NULL,
	    dma_handle) != DDI_SUCCESS) {
		cmn_err(CE_WARN, QL_BANG "%s:  ddi_dma_alloc_handle FAILED",
		    __func__);
		return (QL_ERROR);
	}
	/*
	 * Allocate DMA memory
	 */
	if (ddi_dma_mem_alloc(*dma_handle, size, device_acc_attr,
	    dma_flags & (DDI_DMA_CONSISTENT|DDI_DMA_STREAMING), DDI_DMA_SLEEP,
	    NULL, vaddr, &rlen, acc_handle) != DDI_SUCCESS) {
		ddi_dma_free_handle(dma_handle);
	}
	if (vaddr == NULL) {
		cmn_err(CE_WARN, "alloc_phys: Memory alloc Failed");
		ddi_dma_free_handle(dma_handle);
		return (QL_ERROR);
	}

	if (ddi_dma_addr_bind_handle(*dma_handle, NULL, *vaddr, rlen,
	    dma_flags, DDI_DMA_SLEEP, NULL,
	    dma_cookie, &cnt) != DDI_DMA_MAPPED) {
		ddi_dma_mem_free(acc_handle);

		ddi_dma_free_handle(dma_handle);
		cmn_err(CE_WARN, "%s ddi_dma_addr_bind_handle FAILED",
		    __func__);
		return (QL_ERROR);
	}

	if (cnt != 1) {

		ql_free_phys(dma_handle, acc_handle);

		cmn_err(CE_WARN, "%s: cnt != 1; Failed segment count",
		    __func__);
		return (QL_ERROR);
	}

	bzero((caddr_t)*vaddr, rlen);

	return (0);
}

/*
 * Add interrupt handlers based on the interrupt type.
 * Before adding the interrupt handlers, the interrupt vectors should
 * have been allocated, and the rx/tx rings have also been allocated.
 */
static int
ql_add_intr_handlers(qlge_t *qlge)
{
	int vector = 0;
	int rc, i;
	uint32_t value;
	struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];

	switch (qlge->intr_type) {
	case DDI_INTR_TYPE_MSIX:
		/*
		 * Add interrupt handler for rx and tx rings: vector[0 -
		 * (qlge->intr_cnt -1)].
		 */
		value = 0;
		for (vector = 0; vector < qlge->intr_cnt; vector++) {
			ql_atomic_set_32(&intr_ctx->irq_cnt, value);

			/*
			 * associate interrupt vector with interrupt handler
			 */
			rc = ddi_intr_add_handler(qlge->htable[vector],
			    (ddi_intr_handler_t *)intr_ctx->handler,
			    (void *)&qlge->rx_ring[vector], NULL);

			if (rc != DDI_SUCCESS) {
				QL_PRINT(DBG_INIT,
				    ("Add rx interrupt handler failed. "
				    "return: %d, vector: %d", rc, vector));
				for (vector--; vector >= 0; vector--) {
					(void) ddi_intr_remove_handler(
					    qlge->htable[vector]);
				}
				return (DDI_FAILURE);
			}
			intr_ctx++;
		}
		break;

	case DDI_INTR_TYPE_MSI:
		/*
		 * Add interrupt handlers for the only vector
		 */
		ql_atomic_set_32(&intr_ctx->irq_cnt, value);

		rc = ddi_intr_add_handler(qlge->htable[vector],
		    ql_isr,
		    (caddr_t)&qlge->rx_ring[0], NULL);

		if (rc != DDI_SUCCESS) {
			QL_PRINT(DBG_INIT,
			    ("Add MSI interrupt handler failed: %d\n", rc));
			return (DDI_FAILURE);
		}
		break;

	case DDI_INTR_TYPE_FIXED:
		/*
		 * Add interrupt handlers for the only vector
		 */
		ql_atomic_set_32(&intr_ctx->irq_cnt, value);

		rc = ddi_intr_add_handler(qlge->htable[vector],
		    ql_isr,
		    (caddr_t)&qlge->rx_ring[0], NULL);

		if (rc != DDI_SUCCESS) {
			QL_PRINT(DBG_INIT,
			    ("Add legacy interrupt handler failed: %d\n", rc));
			return (DDI_FAILURE);
		}
		break;

	default:
		return (DDI_FAILURE);
	}

	/* Enable interrupts */
	/* Block enable */
	if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) {
		QL_PRINT(DBG_INIT, ("Block enabling %d interrupt(s)\n",
		    qlge->intr_cnt));
		(void) ddi_intr_block_enable(qlge->htable, qlge->intr_cnt);
	} else { /* Non block enable */
		for (i = 0; i < qlge->intr_cnt; i++) {
			QL_PRINT(DBG_INIT, ("Non Block Enabling interrupt %d\n,"
			    "handle 0x%x\n", i, qlge->htable[i]));
			(void) ddi_intr_enable(qlge->htable[i]);
		}
	}
	qlge->sequence |= INIT_INTR_ENABLED;

	return (DDI_SUCCESS);
}

/*
 * Here we build the intr_ctx structures based on
 * our rx_ring count and intr vector count.
 * The intr_ctx structure is used to hook each vector
 * to possibly different handlers.
 */
static void
ql_resolve_queues_to_irqs(qlge_t *qlge)
{
	int i = 0;
	struct intr_ctx *intr_ctx = &qlge->intr_ctx[0];

	if (qlge->intr_type == DDI_INTR_TYPE_MSIX) {
		/*
		 * Each rx_ring has its own intr_ctx since we
		 * have separate vectors for each queue.
		 * This only true when MSI-X is enabled.
		 */
		for (i = 0; i < qlge->intr_cnt; i++, intr_ctx++) {
			qlge->rx_ring[i].irq = i;
			intr_ctx->intr = i;
			intr_ctx->qlge = qlge;

			/*
			 * We set up each vectors enable/disable/read bits so
			 * there's no bit/mask calculations in critical path.
			 */
			intr_ctx->intr_en_mask =
			    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
			    INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK |
			    INTR_EN_IHD | i;
			intr_ctx->intr_dis_mask =
			    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
			    INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK |
			    INTR_EN_IHD | i;
			intr_ctx->intr_read_mask =
			    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
			    INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD
			    | i;

			if (i == 0) {
				/*
				 * Default queue handles bcast/mcast plus
				 * async events.
				 */
				intr_ctx->handler = ql_isr;
			} else if (qlge->rx_ring[i].type == TX_Q) {
				/*
				 * Outbound queue is for outbound completions
				 * only.
				 */
				intr_ctx->handler = ql_msix_tx_isr;
			} else {
				/*
				 * Inbound queues handle unicast frames only.
				 */
				intr_ctx->handler = ql_msix_rx_isr;
			}
		}
	} else {
		/*
		 * All rx_rings use the same intr_ctx since
		 * there is only one vector.
		 */
		intr_ctx->intr = 0;
		intr_ctx->qlge = qlge;
		/*
		 * We set up each vectors enable/disable/read bits so
		 * there's no bit/mask calculations in the critical path.
		 */
		intr_ctx->intr_en_mask =
		    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
		    INTR_EN_TYPE_ENABLE;
		intr_ctx->intr_dis_mask =
		    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
		    INTR_EN_TYPE_DISABLE;
		intr_ctx->intr_read_mask =
		    INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
		    INTR_EN_TYPE_READ;
		/*
		 * Single interrupt means one handler for all rings.
		 */
		intr_ctx->handler = ql_isr;
		for (i = 0; i < qlge->rx_ring_count; i++)
			qlge->rx_ring[i].irq = 0;
	}
}


/*
 * Free allocated interrupts.
 */
static void
ql_free_irq_vectors(qlge_t *qlge)
{
	int i;
	int rc;

	if (qlge->sequence & INIT_INTR_ENABLED) {
		/* Disable all interrupts */
		if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) {
			/* Call ddi_intr_block_disable() */
			(void) ddi_intr_block_disable(qlge->htable,
			    qlge->intr_cnt);
		} else {
			for (i = 0; i < qlge->intr_cnt; i++) {
				(void) ddi_intr_disable(qlge->htable[i]);
			}
		}

		qlge->sequence &= ~INIT_INTR_ENABLED;
	}

	for (i = 0; i < qlge->intr_cnt; i++) {

		if (qlge->sequence & INIT_ADD_INTERRUPT)
			(void) ddi_intr_remove_handler(qlge->htable[i]);

		if (qlge->sequence & INIT_INTR_ALLOC) {
			rc = ddi_intr_free(qlge->htable[i]);
			if (rc != DDI_SUCCESS) {
				/* EMPTY */
				QL_PRINT(DBG_INIT, ("Free intr failed: %d",
				    rc));
			}
		}
	}
	if (qlge->sequence & INIT_INTR_ALLOC)
		qlge->sequence &= ~INIT_INTR_ALLOC;

	if (qlge->sequence & INIT_ADD_INTERRUPT)
		qlge->sequence &= ~INIT_ADD_INTERRUPT;

	if (qlge->htable) {
		kmem_free(qlge->htable, qlge->intr_size);
		qlge->htable = NULL;
	}
}

/*
 * Allocate interrupt vectors
 * For legacy and MSI, only 1 handle is needed.
 * For MSI-X, if fewer than 2 vectors are available, return failure.
 * Upon success, this maps the vectors to rx and tx rings for
 * interrupts.
 */
static int
ql_request_irq_vectors(qlge_t *qlge, int intr_type)
{
	dev_info_t *devinfo;
	uint32_t request, orig;
	int count, avail, actual;
	int minimum;
	int rc;

	devinfo = qlge->dip;

	switch (intr_type) {
	case DDI_INTR_TYPE_FIXED:
		request = 1;	/* Request 1 legacy interrupt handle */
		minimum = 1;
		QL_PRINT(DBG_INIT, ("interrupt type: legacy\n"));
		break;

	case DDI_INTR_TYPE_MSI:
		request = 1;	/* Request 1 MSI interrupt handle */
		minimum = 1;
		QL_PRINT(DBG_INIT, ("interrupt type: MSI\n"));
		break;

	case DDI_INTR_TYPE_MSIX:
		/*
		 * Ideal number of vectors for the adapter is
		 * # rss rings + tx completion rings for default completion
		 * queue.
		 */
		request = qlge->rx_ring_count;

		orig = request;
		if (request > (MAX_RX_RINGS))
			request = MAX_RX_RINGS;
		minimum = 2;
		QL_PRINT(DBG_INIT, ("interrupt type: MSI-X\n"));
		break;

	default:
		QL_PRINT(DBG_INIT, ("Invalid parameter\n"));
		return (DDI_FAILURE);
	}

	QL_PRINT(DBG_INIT, ("interrupt handles requested: %d  minimum: %d\n",
	    request, minimum));

	/*
	 * Get number of supported interrupts
	 */
	rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
	if ((rc != DDI_SUCCESS) || (count < minimum)) {
		QL_PRINT(DBG_INIT, ("Get interrupt number failed. Return: %d, "
		    "count: %d\n", rc, count));
		return (DDI_FAILURE);
	}
	QL_PRINT(DBG_INIT, ("interrupts supported: %d\n", count));

	/*
	 * Get number of available interrupts
	 */
	rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
	if ((rc != DDI_SUCCESS) || (avail < minimum)) {
		QL_PRINT(DBG_INIT,
		    ("Get interrupt available number failed. Return:"
		    " %d, available: %d\n", rc, avail));
		return (DDI_FAILURE);
	}
	QL_PRINT(DBG_INIT, ("interrupts available: %d\n", avail));

	if (avail < request) {
		QL_PRINT(DBG_INIT, ("Request %d handles, %d available\n",
		    request, avail));
		request = avail;
	}

	actual = 0;
	qlge->intr_cnt = 0;

	/*
	 * Allocate an array of interrupt handles
	 */
	qlge->intr_size = (size_t)(request * sizeof (ddi_intr_handle_t));
	qlge->htable = kmem_alloc(qlge->intr_size, KM_SLEEP);

	rc = ddi_intr_alloc(devinfo, qlge->htable, intr_type, 0,
	    (int)request, &actual, DDI_INTR_ALLOC_NORMAL);
	if (rc != DDI_SUCCESS) {
		cmn_err(CE_WARN, "%s(%d) Allocate interrupts failed. return:"
		    " %d, request: %d, actual: %d",
		    __func__, qlge->instance, rc, request, actual);
		goto ql_intr_alloc_fail;
	}
	qlge->intr_cnt = actual;

	qlge->sequence |= INIT_INTR_ALLOC;

	/*
	 * If the actual number of vectors is less than the minumum
	 * then fail.
	 */
	if (actual < minimum) {
		cmn_err(CE_WARN,
		    "Insufficient interrupt handles available: %d", actual);
		goto ql_intr_alloc_fail;
	}

	/*
	 * For MSI-X, actual might force us to reduce number of tx & rx rings
	 */
	if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
		if (actual < MAX_RX_RINGS) {
			qlge->tx_ring_count = 1;
			qlge->rss_ring_count = actual - 1;
			qlge->rx_ring_count = qlge->tx_ring_count +
			    qlge->rss_ring_count;
		}
	}
	/*
	 * Get priority for first vector, assume remaining are all the same
	 */
	rc = ddi_intr_get_pri(qlge->htable[0], &qlge->intr_pri);
	if (rc != DDI_SUCCESS) {
		QL_PRINT(DBG_INIT, ("Get interrupt priority failed: %d\n", rc));
		goto ql_intr_alloc_fail;
	}

	rc = ddi_intr_get_cap(qlge->htable[0], &qlge->intr_cap);
	if (rc != DDI_SUCCESS) {
		QL_PRINT(DBG_INIT, ("Get interrupt cap failed: %d\n", rc));
		goto ql_intr_alloc_fail;
	}

	qlge->intr_type = intr_type;

	return (DDI_SUCCESS);

ql_intr_alloc_fail:
	ql_free_irq_vectors(qlge);

	return (DDI_FAILURE);
}

/*
 * Allocate interrupt vector(s) for one of the following interrupt types, MSI-X,
 * MSI or Legacy. In MSI and Legacy modes we only support a single receive and
 * transmit queue.
 */
int
ql_alloc_irqs(qlge_t *qlge)
{
	int intr_types;
	int rval;

	/*
	 * Get supported interrupt types
	 */
	if (ddi_intr_get_supported_types(qlge->dip, &intr_types)
	    != DDI_SUCCESS) {
		cmn_err(CE_WARN, "%s(%d):ddi_intr_get_supported_types failed",
		    __func__, qlge->instance);

		return (DDI_FAILURE);
	}

	QL_PRINT(DBG_INIT, ("%s(%d) Interrupt types supported %d\n",
	    __func__, qlge->instance, intr_types));

	/* Install MSI-X interrupts */
	if ((intr_types & DDI_INTR_TYPE_MSIX) != 0) {
		QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt supported %d\n",
		    __func__, qlge->instance, intr_types));
		rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSIX);
		if (rval == DDI_SUCCESS) {
			return (rval);
		}
		QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt allocation failed,"
		    " trying MSI interrupts ...\n", __func__, qlge->instance));
	}

	/*
	 * We will have 2 completion queues in MSI / Legacy mode,
	 * Queue 0 for default completions
	 * Queue 1 for transmit completions
	 */
	qlge->rss_ring_count = 1; /* Default completion queue (0) for all */
	qlge->tx_ring_count = 1; /* Single tx completion queue */
	qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count;

	QL_PRINT(DBG_INIT, ("%s(%d) Falling back to single completion queue \n",
	    __func__, qlge->instance));
	/*
	 * Add the h/w interrupt handler and initialise mutexes
	 */
	rval = DDI_FAILURE;

	/*
	 * If OS supports MSIX interrupt but fails to allocate, then try
	 * MSI interrupt. If MSI interrupt allocation fails also, then roll
	 * back to fixed interrupt.
	 */
	if (intr_types & DDI_INTR_TYPE_MSI) {
		rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSI);
		if (rval == DDI_SUCCESS) {
			qlge->intr_type = DDI_INTR_TYPE_MSI;
			QL_PRINT(DBG_INIT, ("%s(%d) use MSI Interrupt \n",
			    __func__, qlge->instance));
		}
	}

	/* Try Fixed interrupt Legacy mode */
	if (rval != DDI_SUCCESS) {
		rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_FIXED);
		if (rval != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d):Legacy mode interrupt "
			    "allocation failed",
			    __func__, qlge->instance);
		} else {
			qlge->intr_type = DDI_INTR_TYPE_FIXED;
			QL_PRINT(DBG_INIT, ("%s(%d) use Fixed Interrupt \n",
			    __func__, qlge->instance));
		}
	}

	return (rval);
}

static void
ql_free_rx_tx_locks(qlge_t *qlge)
{
	int i;
	struct rx_ring *rx_ring;
	struct tx_ring *tx_ring;

	for (i = 0; i < qlge->tx_ring_count; i++) {
		tx_ring = &qlge->tx_ring[i];
		mutex_destroy(&tx_ring->tx_lock);
	}

	for (i = 0; i < qlge->rx_ring_count; i++) {
		rx_ring = &qlge->rx_ring[i];
		mutex_destroy(&rx_ring->rx_lock);
		mutex_destroy(&rx_ring->sbq_lock);
		mutex_destroy(&rx_ring->lbq_lock);
	}
}

/*
 * Frees all resources allocated during attach.
 *
 * Input:
 * dip = pointer to device information structure.
 * sequence = bits indicating resources to free.
 *
 * Context:
 * Kernel context.
 */
static void
ql_free_resources(dev_info_t *dip, qlge_t *qlge)
{

	/* Disable driver timer */
	ql_stop_timer(qlge);

	if (qlge->sequence & INIT_MAC_REGISTERED) {
		(void) mac_unregister(qlge->mh);
		qlge->sequence &= ~INIT_MAC_REGISTERED;
	}

	if (qlge->sequence & INIT_MAC_ALLOC) {
		/* Nothing to do, macp is already freed */
		qlge->sequence &= ~INIT_MAC_ALLOC;
	}

	if (qlge->sequence & INIT_PCI_CONFIG_SETUP) {
		pci_config_teardown(&qlge->pci_handle);
		qlge->sequence &= ~INIT_PCI_CONFIG_SETUP;
	}

	if (qlge->sequence & INIT_ADD_INTERRUPT) {
		ql_free_irq_vectors(qlge);
		qlge->sequence &= ~INIT_ADD_INTERRUPT;
	}

	if (qlge->sequence & INIT_ADD_SOFT_INTERRUPT) {
		(void) ddi_intr_remove_softint(qlge->mpi_event_intr_hdl);
		(void) ddi_intr_remove_softint(qlge->mpi_reset_intr_hdl);
		(void) ddi_intr_remove_softint(qlge->asic_reset_intr_hdl);
		qlge->sequence &= ~INIT_ADD_SOFT_INTERRUPT;
	}

	if (qlge->sequence & INIT_KSTATS) {
		ql_fini_kstats(qlge);
		qlge->sequence &= ~INIT_KSTATS;
	}

	if (qlge->sequence & INIT_MUTEX) {
		mutex_destroy(&qlge->gen_mutex);
		mutex_destroy(&qlge->hw_mutex);
		mutex_destroy(&qlge->mbx_mutex);
		cv_destroy(&qlge->cv_mbx_intr);
		qlge->sequence &= ~INIT_MUTEX;
	}

	if (qlge->sequence & INIT_LOCKS_CREATED) {
		ql_free_rx_tx_locks(qlge);
		qlge->sequence &= ~INIT_LOCKS_CREATED;
	}

	if (qlge->sequence & INIT_MEMORY_ALLOC) {
		ql_free_mem_resources(qlge);
		qlge->sequence &= ~INIT_MEMORY_ALLOC;
	}

	if (qlge->sequence & INIT_REGS_SETUP) {
		ddi_regs_map_free(&qlge->dev_handle);
		qlge->sequence &= ~INIT_REGS_SETUP;
	}

	if (qlge->sequence & INIT_DOORBELL_REGS_SETUP) {
		ddi_regs_map_free(&qlge->dev_doorbell_reg_handle);
		qlge->sequence &= ~INIT_DOORBELL_REGS_SETUP;
	}

	/*
	 * free flash flt table that allocated in attach stage
	 */
	if ((qlge->flt.ql_flt_entry_ptr != NULL)&&
	    (qlge->flt.header.length != 0)) {
		kmem_free(qlge->flt.ql_flt_entry_ptr, qlge->flt.header.length);
		qlge->flt.ql_flt_entry_ptr = NULL;
	}

	/* finally, free qlge structure */
	if (qlge->sequence & INIT_SOFTSTATE_ALLOC) {
		kmem_free(qlge, sizeof (qlge_t));
	}

	ddi_prop_remove_all(dip);
	ddi_set_driver_private(dip, NULL);

}

/*
 * Set promiscuous mode of the driver
 * Caller must catch HW_LOCK
 */
void
ql_set_promiscuous(qlge_t *qlge, int mode)
{
	if (mode) {
		(void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT,
		    RT_IDX_VALID, 1);
	} else {
		(void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT,
		    RT_IDX_VALID, 0);
	}
}
/*
 * Write 'data1' to Mac Protocol Address Index Register and
 * 'data2' to Mac Protocol Address Data Register
 *  Assuming that the Mac Protocol semaphore lock has been acquired.
 */
static int
ql_write_mac_proto_regs(qlge_t *qlge, uint32_t data1, uint32_t data2)
{
	int return_value = DDI_SUCCESS;

	if (ql_wait_reg_bit(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
	    MAC_PROTOCOL_ADDRESS_INDEX_MW, BIT_SET, 5) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "Wait for MAC_PROTOCOL Address Register "
		    "timeout.");
		return_value = DDI_FAILURE;
		goto out;
	}
	ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX /* A8 */, data1);
	ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA /* 0xAC */, data2);
out:
	return (return_value);
}
/*
 * Enable the 'index'ed multicast address in the host memory's multicast_list
 */
int
ql_add_multicast_address(qlge_t *qlge, int index)
{
	int rtn_val = DDI_FAILURE;
	uint32_t offset;
	uint32_t value1, value2;

	/* Acquire the required semaphore */
	if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) {
		return (rtn_val);
	}

	/* Program Offset0 - lower 32 bits of the MAC address */
	offset = 0;
	value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST |
	    (index << 4) | offset;
	value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[5]));
	if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS)
		goto out;

	/* Program offset1: upper 16 bits of the MAC address */
	offset = 1;
	value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST |
	    (index<<4) | offset;
	value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[0] << 8)
	    |qlge->multicast_list[index].addr.ether_addr_octet[1]);
	if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
		goto out;
	}
	rtn_val = DDI_SUCCESS;
out:
	ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK);
	return (rtn_val);
}

/*
 * Disable the 'index'ed multicast address in the host memory's multicast_list
 */
int
ql_remove_multicast_address(qlge_t *qlge, int index)
{
	int rtn_val = DDI_FAILURE;
	uint32_t offset;
	uint32_t value1, value2;

	/* Acquire the required semaphore */
	if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) {
		return (rtn_val);
	}
	/* Program Offset0 - lower 32 bits of the MAC address */
	offset = 0;
	value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4);
	value2 =
	    ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8)
	    |(qlge->multicast_list[index].addr.ether_addr_octet[5]));
	if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
		goto out;
	}
	/* Program offset1: upper 16 bits of the MAC address */
	offset = 1;
	value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4);
	value2 = 0;
	if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) {
		goto out;
	}
	rtn_val = DDI_SUCCESS;
out:
	ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK);
	return (rtn_val);
}

/*
 * Add a new multicast address to the list of supported list
 * This API is called after OS called gld_set_multicast (GLDv2)
 * or m_multicst (GLDv3)
 *
 * Restriction:
 * The number of maximum multicast address is limited by hardware.
 */
int
ql_add_to_multicast_list(qlge_t *qlge, uint8_t *ep)
{
	uint32_t index = qlge->multicast_list_count;
	int rval = DDI_SUCCESS;
	int status;

	if ((ep[0] & 01) == 0) {
		rval = EINVAL;
		goto exit;
	}

	/* if there is an availabe space in multicast_list, then add it */
	if (index < MAX_MULTICAST_LIST_SIZE) {
		bcopy(ep, qlge->multicast_list[index].addr.ether_addr_octet,
		    ETHERADDRL);
		/* increment the total number of addresses in multicast list */
		(void) ql_add_multicast_address(qlge, index);
		qlge->multicast_list_count++;
		QL_PRINT(DBG_GLD,
		    ("%s(%d): added to index of multicast list= 0x%x, "
		    "total %d\n", __func__, qlge->instance, index,
		    qlge->multicast_list_count));

		if (index > MAX_MULTICAST_HW_SIZE) {
			if (!qlge->multicast_promisc) {
				status = ql_set_routing_reg(qlge,
				    RT_IDX_ALLMULTI_SLOT,
				    RT_IDX_MCAST, 1);
				if (status) {
					cmn_err(CE_WARN,
					    "Failed to init routing reg "
					    "for mcast promisc mode.");
					rval = ENOENT;
					goto exit;
				}
				qlge->multicast_promisc = B_TRUE;
			}
		}
	} else {
		rval = ENOENT;
	}
exit:
	return (rval);
}

/*
 * Remove an old multicast address from the list of supported multicast
 * addresses. This API is called after OS called gld_set_multicast (GLDv2)
 * or m_multicst (GLDv3)
 * The number of maximum multicast address is limited by hardware.
 */
int
ql_remove_from_multicast_list(qlge_t *qlge, uint8_t *ep)
{
	uint32_t total = qlge->multicast_list_count;
	int i = 0;
	int rmv_index = 0;
	size_t length = sizeof (ql_multicast_addr);
	int status;

	for (i = 0; i < total; i++) {
		if (bcmp(ep, &qlge->multicast_list[i].addr, ETHERADDRL) != 0) {
			continue;
		}

		rmv_index = i;
		/* block move the reset of other multicast address forward */
		length = ((total -1) -i) * sizeof (ql_multicast_addr);
		if (length > 0) {
			bcopy(&qlge->multicast_list[i+1],
			    &qlge->multicast_list[i], length);
		}
		qlge->multicast_list_count--;
		if (qlge->multicast_list_count <= MAX_MULTICAST_HW_SIZE) {
			/*
			 * there is a deletion in multicast list table,
			 * re-enable them
			 */
			for (i = rmv_index; i < qlge->multicast_list_count;
			    i++) {
				(void) ql_add_multicast_address(qlge, i);
			}
			/* and disable the last one */
			(void) ql_remove_multicast_address(qlge, i);

			/* disable multicast promiscuous mode */
			if (qlge->multicast_promisc) {
				status = ql_set_routing_reg(qlge,
				    RT_IDX_ALLMULTI_SLOT,
				    RT_IDX_MCAST, 0);
				if (status) {
					cmn_err(CE_WARN,
					    "Failed to init routing reg for "
					    "mcast promisc mode.");
					goto exit;
				}
				/* write to config register */
				qlge->multicast_promisc = B_FALSE;
			}
		}
		break;
	}
exit:
	return (DDI_SUCCESS);
}

/*
 * Read a XGMAC register
 */
int
ql_read_xgmac_reg(qlge_t *qlge, uint32_t addr, uint32_t *val)
{
	int rtn_val = DDI_FAILURE;

	/* wait for XGMAC Address register RDY bit set */
	if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY,
	    BIT_SET, 10) != DDI_SUCCESS) {
		goto out;
	}
	/* start rx transaction */
	ql_write_reg(qlge, REG_XGMAC_ADDRESS, addr|XGMAC_ADDRESS_READ_TRANSACT);

	/*
	 * wait for XGMAC Address register RDY bit set,
	 * which indicates data is ready
	 */
	if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY,
	    BIT_SET, 10) != DDI_SUCCESS) {
		goto out;
	}
	/* read data from XGAMC_DATA register */
	*val = ql_read_reg(qlge, REG_XGMAC_DATA);
	rtn_val = DDI_SUCCESS;
out:
	return (rtn_val);
}

/*
 * Implement checksum offload for IPv4 IP packets
 */
static void
ql_hw_csum_setup(qlge_t *qlge, uint32_t pflags, caddr_t bp,
    struct ob_mac_iocb_req *mac_iocb_ptr)
{
	struct ip *iphdr = NULL;
	struct ether_header *ethhdr;
	struct ether_vlan_header *ethvhdr;
	struct tcphdr *tcp_hdr;
	uint32_t etherType;
	int mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len;
	int ip_hdr_off, tcp_udp_hdr_off, hdr_off;

	ethhdr  = (struct ether_header *)((void *)bp);
	ethvhdr = (struct ether_vlan_header *)((void *)bp);
	/* Is this vlan packet? */
	if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) {
		mac_hdr_len = sizeof (struct ether_vlan_header);
		etherType = ntohs(ethvhdr->ether_type);
	} else {
		mac_hdr_len = sizeof (struct ether_header);
		etherType = ntohs(ethhdr->ether_type);
	}
	/* Is this IPv4 or IPv6 packet? */
	if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len)) ==
	    IPV4_VERSION) {
		if (etherType == ETHERTYPE_IP /* 0800 */) {
			iphdr = (struct ip *)(void *)(bp+mac_hdr_len);
		} else {
			/* EMPTY */
			QL_PRINT(DBG_TX,
			    ("%s(%d) : IPv4 None IP packet type 0x%x\n",
			    __func__, qlge->instance, etherType));
		}
	}
	/* ipV4 packets */
	if (iphdr != NULL) {

		ip_hdr_len = IPH_HDR_LENGTH(iphdr);
		QL_PRINT(DBG_TX,
		    ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH:"
		    " %d bytes \n", __func__, qlge->instance, ip_hdr_len));

		ip_hdr_off = mac_hdr_len;
		QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n",
		    __func__, qlge->instance, ip_hdr_len));

		mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 |
		    OB_MAC_IOCB_REQ_IPv4);

		if (pflags & HCK_IPV4_HDRCKSUM) {
			QL_PRINT(DBG_TX, ("%s(%d) : Do IPv4 header checksum\n",
			    __func__, qlge->instance));
			mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB;
			mac_iocb_ptr->flag2 = (uint8_t)(mac_iocb_ptr->flag2 |
			    OB_MAC_IOCB_REQ_IC);
			iphdr->ip_sum = 0;
			mac_iocb_ptr->hdr_off = (uint16_t)
			    cpu_to_le16(ip_hdr_off);
		}
		if (pflags & HCK_FULLCKSUM) {
			if (iphdr->ip_p == IPPROTO_TCP) {
				tcp_hdr =
				    (struct tcphdr *)(void *)
				    ((uint8_t *)(void *)iphdr + ip_hdr_len);
				QL_PRINT(DBG_TX, ("%s(%d) : Do TCP checksum\n",
				    __func__, qlge->instance));
				mac_iocb_ptr->opcode =
				    OPCODE_OB_MAC_OFFLOAD_IOCB;
				mac_iocb_ptr->flag1 =
				    (uint8_t)(mac_iocb_ptr->flag1 |
				    OB_MAC_IOCB_REQ_TC);
				mac_iocb_ptr->flag2 =
				    (uint8_t)(mac_iocb_ptr->flag2 |
				    OB_MAC_IOCB_REQ_IC);
				iphdr->ip_sum = 0;
				tcp_udp_hdr_off = mac_hdr_len+ip_hdr_len;
				tcp_udp_hdr_len = tcp_hdr->th_off*4;
				QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n",
				    __func__, qlge->instance, tcp_udp_hdr_len));
				hdr_off = ip_hdr_off;
				tcp_udp_hdr_off <<= 6;
				hdr_off |= tcp_udp_hdr_off;
				mac_iocb_ptr->hdr_off =
				    (uint16_t)cpu_to_le16(hdr_off);
				mac_iocb_ptr->protocol_hdr_len = (uint16_t)
				    cpu_to_le16(mac_hdr_len + ip_hdr_len +
				    tcp_udp_hdr_len);

				/*
				 * if the chip is unable to do pseudo header
				 * cksum calculation, do it in then put the
				 * result to the data passed to the chip
				 */
				if (qlge->cfg_flags &
				    CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) {
					ql_pseudo_cksum((uint8_t *)iphdr);
				}
			} else if (iphdr->ip_p == IPPROTO_UDP) {
				QL_PRINT(DBG_TX, ("%s(%d) : Do UDP checksum\n",
				    __func__, qlge->instance));
				mac_iocb_ptr->opcode =
				    OPCODE_OB_MAC_OFFLOAD_IOCB;
				mac_iocb_ptr->flag1 =
				    (uint8_t)(mac_iocb_ptr->flag1 |
				    OB_MAC_IOCB_REQ_UC);
				mac_iocb_ptr->flag2 =
				    (uint8_t)(mac_iocb_ptr->flag2 |
				    OB_MAC_IOCB_REQ_IC);
				iphdr->ip_sum = 0;
				tcp_udp_hdr_off = mac_hdr_len + ip_hdr_len;
				tcp_udp_hdr_len = sizeof (struct udphdr);
				QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n",
				    __func__, qlge->instance, tcp_udp_hdr_len));
				hdr_off = ip_hdr_off;
				tcp_udp_hdr_off <<= 6;
				hdr_off |= tcp_udp_hdr_off;
				mac_iocb_ptr->hdr_off =
				    (uint16_t)cpu_to_le16(hdr_off);
				mac_iocb_ptr->protocol_hdr_len = (uint16_t)
				    cpu_to_le16(mac_hdr_len + ip_hdr_len
				    + tcp_udp_hdr_len);

				/*
				 * if the chip is unable to calculate pseudo
				 * hdr cksum,do it in then put the result to
				 * the data passed to the chip
				 */
				if (qlge->cfg_flags &
				    CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) {
					ql_pseudo_cksum((uint8_t *)iphdr);
				}
			}
		}
	}
}

/*
 * For TSO/LSO:
 * MAC frame transmission with TCP large segment offload is performed in the
 * same way as the MAC frame transmission with checksum offload with the
 * exception that the maximum TCP segment size (MSS) must be specified to
 * allow the chip to segment the data into legal sized frames.
 * The host also needs to calculate a pseudo-header checksum over the
 * following fields:
 * Source IP Address, Destination IP Address, and the Protocol.
 * The TCP length is not included in the pseudo-header calculation.
 * The pseudo-header checksum is place in the TCP checksum field of the
 * prototype header.
 */
static void
ql_lso_pseudo_cksum(uint8_t *buf)
{
	uint32_t cksum;
	uint16_t iphl;
	uint16_t proto;

	/*
	 * Calculate the LSO pseudo-header checksum.
	 */
	iphl = (uint16_t)(4 * (buf[0] & 0xF));
	cksum = proto = buf[9];
	cksum += (((uint16_t)buf[12])<<8) + buf[13];
	cksum += (((uint16_t)buf[14])<<8) + buf[15];
	cksum += (((uint16_t)buf[16])<<8) + buf[17];
	cksum += (((uint16_t)buf[18])<<8) + buf[19];
	cksum = (cksum>>16) + (cksum & 0xFFFF);
	cksum = (cksum>>16) + (cksum & 0xFFFF);

	/*
	 * Point it to the TCP/UDP header, and
	 * update the checksum field.
	 */
	buf += iphl + ((proto == IPPROTO_TCP) ?
	    TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET);

	*(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum);
}

/*
 * Tell the hardware to do Large Send Offload (LSO)
 *
 * Some fields in ob_mac_iocb need to be set so hardware can know what is
 * the incoming packet, TCP or UDP, whether a VLAN tag needs to be inserted
 * in the right place of the packet etc, thus, hardware can process the
 * packet correctly.
 */
static void
ql_hw_lso_setup(qlge_t *qlge, uint32_t mss, caddr_t bp,
    struct ob_mac_iocb_req *mac_iocb_ptr)
{
	struct ip *iphdr = NULL;
	struct ether_header *ethhdr;
	struct ether_vlan_header *ethvhdr;
	struct tcphdr *tcp_hdr;
	struct udphdr *udp_hdr;
	uint32_t etherType;
	uint16_t mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len;
	uint16_t ip_hdr_off, tcp_udp_hdr_off, hdr_off;

	ethhdr = (struct ether_header *)(void *)bp;
	ethvhdr = (struct ether_vlan_header *)(void *)bp;

	/* Is this vlan packet? */
	if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) {
		mac_hdr_len = sizeof (struct ether_vlan_header);
		etherType = ntohs(ethvhdr->ether_type);
	} else {
		mac_hdr_len = sizeof (struct ether_header);
		etherType = ntohs(ethhdr->ether_type);
	}
	/* Is this IPv4 or IPv6 packet? */
	if (IPH_HDR_VERSION((ipha_t *)(void *)(bp + mac_hdr_len)) ==
	    IPV4_VERSION) {
		if (etherType == ETHERTYPE_IP /* 0800 */) {
			iphdr 	= (struct ip *)(void *)(bp+mac_hdr_len);
		} else {
			/* EMPTY */
			QL_PRINT(DBG_TX, ("%s(%d) : IPv4 None IP packet"
			    " type 0x%x\n",
			    __func__, qlge->instance, etherType));
		}
	}

	if (iphdr != NULL) { /* ipV4 packets */
		ip_hdr_len = (uint16_t)IPH_HDR_LENGTH(iphdr);
		QL_PRINT(DBG_TX,
		    ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH: %d"
		    " bytes \n", __func__, qlge->instance, ip_hdr_len));

		ip_hdr_off = mac_hdr_len;
		QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n",
		    __func__, qlge->instance, ip_hdr_len));

		mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 |
		    OB_MAC_IOCB_REQ_IPv4);
		if (qlge->cfg_flags & CFG_CKSUM_FULL_IPv4) {
			if (iphdr->ip_p == IPPROTO_TCP) {
				tcp_hdr = (struct tcphdr *)(void *)
				    ((uint8_t *)(void *)iphdr +
				    ip_hdr_len);
				QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on TCP "
				    "packet\n",
				    __func__, qlge->instance));
				mac_iocb_ptr->opcode =
				    OPCODE_OB_MAC_OFFLOAD_IOCB;
				mac_iocb_ptr->flag1 =
				    (uint8_t)(mac_iocb_ptr->flag1 |
				    OB_MAC_IOCB_REQ_LSO);
				iphdr->ip_sum = 0;
				tcp_udp_hdr_off =
				    (uint16_t)(mac_hdr_len+ip_hdr_len);
				tcp_udp_hdr_len =
				    (uint16_t)(tcp_hdr->th_off*4);
				QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n",
				    __func__, qlge->instance, tcp_udp_hdr_len));
				hdr_off = ip_hdr_off;
				tcp_udp_hdr_off <<= 6;
				hdr_off |= tcp_udp_hdr_off;
				mac_iocb_ptr->hdr_off =
				    (uint16_t)cpu_to_le16(hdr_off);
				mac_iocb_ptr->protocol_hdr_len = (uint16_t)
				    cpu_to_le16(mac_hdr_len + ip_hdr_len +
				    tcp_udp_hdr_len);
				mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss);

				/*
				 * if the chip is unable to calculate pseudo
				 * header checksum, do it in then put the result
				 * to the data passed to the chip
				 */
				if (qlge->cfg_flags &
				    CFG_HW_UNABLE_PSEUDO_HDR_CKSUM)
					ql_lso_pseudo_cksum((uint8_t *)iphdr);
			} else if (iphdr->ip_p == IPPROTO_UDP) {
				udp_hdr = (struct udphdr *)(void *)
				    ((uint8_t *)(void *)iphdr
				    + ip_hdr_len);
				QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on UDP "
				    "packet\n",
				    __func__, qlge->instance));
				mac_iocb_ptr->opcode =
				    OPCODE_OB_MAC_OFFLOAD_IOCB;
				mac_iocb_ptr->flag1 =
				    (uint8_t)(mac_iocb_ptr->flag1 |
				    OB_MAC_IOCB_REQ_LSO);
				iphdr->ip_sum = 0;
				tcp_udp_hdr_off =
				    (uint16_t)(mac_hdr_len+ip_hdr_len);
				tcp_udp_hdr_len =
				    (uint16_t)(udp_hdr->uh_ulen*4);
				QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n",
				    __func__, qlge->instance, tcp_udp_hdr_len));
				hdr_off = ip_hdr_off;
				tcp_udp_hdr_off <<= 6;
				hdr_off |= tcp_udp_hdr_off;
				mac_iocb_ptr->hdr_off =
				    (uint16_t)cpu_to_le16(hdr_off);
				mac_iocb_ptr->protocol_hdr_len = (uint16_t)
				    cpu_to_le16(mac_hdr_len + ip_hdr_len +
				    tcp_udp_hdr_len);
				mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss);

				/*
				 * if the chip is unable to do pseudo header
				 * checksum calculation, do it here then put the
				 * result to the data passed to the chip
				 */
				if (qlge->cfg_flags &
				    CFG_HW_UNABLE_PSEUDO_HDR_CKSUM)
					ql_lso_pseudo_cksum((uint8_t *)iphdr);
			}
		}
	}
}

/*
 * Generic packet sending function which is used to send one packet.
 */
int
ql_send_common(struct tx_ring *tx_ring, mblk_t *mp)
{
	struct tx_ring_desc *tx_cb;
	struct ob_mac_iocb_req *mac_iocb_ptr;
	mblk_t *tp;
	size_t msg_len = 0;
	size_t off;
	caddr_t bp;
	size_t nbyte, total_len;
	uint_t i = 0;
	int j = 0, frags = 0;
	uint32_t phy_addr_low, phy_addr_high;
	uint64_t phys_addr;
	clock_t now;
	uint32_t pflags = 0;
	uint32_t mss = 0;
	enum tx_mode_t tx_mode;
	struct oal_entry *oal_entry;
	int status;
	uint_t ncookies, oal_entries, max_oal_entries;
	size_t max_seg_len = 0;
	boolean_t use_lso = B_FALSE;
	struct oal_entry *tx_entry = NULL;
	struct oal_entry *last_oal_entry;
	qlge_t *qlge = tx_ring->qlge;
	ddi_dma_cookie_t dma_cookie;
	size_t tx_buf_len = QL_MAX_COPY_LENGTH;
	int force_pullup = 0;

	tp = mp;
	total_len = msg_len = 0;
	max_oal_entries = TX_DESC_PER_IOCB + MAX_SG_ELEMENTS-1;

	/* Calculate number of data and segments in the incoming message */
	for (tp = mp; tp != NULL; tp = tp->b_cont) {
		nbyte = MBLKL(tp);
		total_len += nbyte;
		max_seg_len = max(nbyte, max_seg_len);
		QL_PRINT(DBG_TX, ("Requested sending data in %d segments, "
		    "total length: %d\n", frags, nbyte));
		frags++;
	}

	if (total_len >= QL_LSO_MAX) {
		freemsg(mp);
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_NOTE, "%s: quit, packet oversize %d\n",
		    __func__, (int)total_len);
#endif
		return (NULL);
	}

	bp = (caddr_t)mp->b_rptr;
	if (bp[0] & 1) {
		if (bcmp(bp, ql_ether_broadcast_addr.ether_addr_octet,
		    ETHERADDRL) == 0) {
			QL_PRINT(DBG_TX, ("Broadcast packet\n"));
			tx_ring->brdcstxmt++;
		} else {
			QL_PRINT(DBG_TX, ("multicast packet\n"));
			tx_ring->multixmt++;
		}
	}

	tx_ring->obytes += total_len;
	tx_ring->opackets ++;

	QL_PRINT(DBG_TX, ("total requested sending data length: %d, in %d segs,"
	    " max seg len: %d\n", total_len, frags, max_seg_len));

	/* claim a free slot in tx ring */
	tx_cb = &tx_ring->wq_desc[tx_ring->prod_idx];

	/* get the tx descriptor */
	mac_iocb_ptr = tx_cb->queue_entry;

	bzero((void *)mac_iocb_ptr, sizeof (*mac_iocb_ptr));

	ASSERT(tx_cb->mp == NULL);

	/*
	 * Decide to use DMA map or copy mode.
	 * DMA map mode must be used when the total msg length is more than the
	 * tx buffer length.
	 */

	if (total_len > tx_buf_len)
		tx_mode = USE_DMA;
	else if	(max_seg_len > QL_MAX_COPY_LENGTH)
		tx_mode = USE_DMA;
	else
		tx_mode = USE_COPY;

	if (qlge->chksum_cap) {
		mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags);
		QL_PRINT(DBG_TX, ("checksum flag is :0x%x, card capability "
		    "is 0x%x \n", pflags, qlge->chksum_cap));
		if (qlge->lso_enable) {
			uint32_t lso_flags = 0;
			mac_lso_get(mp, &mss, &lso_flags);
			use_lso = (lso_flags == HW_LSO);
		}
		QL_PRINT(DBG_TX, ("mss :%d, use_lso %x \n",
		    mss, use_lso));
	}

do_pullup:

	/* concatenate all frags into one large packet if too fragmented */
	if (((tx_mode == USE_DMA)&&(frags > QL_MAX_TX_DMA_HANDLES)) ||
	    force_pullup) {
		mblk_t *mp1;
		if ((mp1 = msgpullup(mp, -1)) != NULL) {
			freemsg(mp);
			mp = mp1;
			frags = 1;
		} else {
			tx_ring->tx_fail_dma_bind++;
			goto bad;
		}
	}

	tx_cb->tx_bytes = (uint32_t)total_len;
	tx_cb->mp = mp;
	tx_cb->tx_dma_handle_used = 0;

	if (tx_mode == USE_DMA) {
		msg_len = total_len;

		mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
		mac_iocb_ptr->tid = tx_ring->prod_idx;
		mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len);
		mac_iocb_ptr->txq_idx = tx_ring->wq_id;

		tx_entry = &mac_iocb_ptr->oal_entry[0];
		oal_entry = NULL;

		for (tp = mp, oal_entries = j = 0; tp != NULL;
		    tp = tp->b_cont) {
			/* if too many tx dma handles needed */
			if (j >= QL_MAX_TX_DMA_HANDLES) {
				tx_ring->tx_no_dma_handle++;
				if (!force_pullup) {
					force_pullup = 1;
					goto do_pullup;
				} else {
					goto bad;
				}
			}
			nbyte = (uint16_t)MBLKL(tp);
			if (nbyte == 0)
				continue;

			status = ddi_dma_addr_bind_handle(
			    tx_cb->tx_dma_handle[j], NULL,
			    (caddr_t)tp->b_rptr, nbyte,
			    DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
			    0, &dma_cookie, &ncookies);

			QL_PRINT(DBG_TX, ("map sending data segment: %d, "
			    "length: %d, spans in %d cookies\n",
			    j, nbyte, ncookies));

			if (status != DDI_DMA_MAPPED) {
				goto bad;
			}
			/*
			 * Each fragment can span several cookies. One cookie
			 * will use one tx descriptor to transmit.
			 */
			for (i = ncookies; i > 0; i--, tx_entry++,
			    oal_entries++) {
				/*
				 * The number of TX descriptors that can be
				 *  saved in tx iocb and oal list is limited
				 */
				if (oal_entries > max_oal_entries) {
					tx_ring->tx_no_dma_cookie++;
					if (!force_pullup) {
						force_pullup = 1;
						goto do_pullup;
					} else {
						goto bad;
					}
				}

				if ((oal_entries == TX_DESC_PER_IOCB) &&
				    !oal_entry) {
					/*
					 * Time to switch to an oal list
					 * The last entry should be copied
					 * to first entry in the oal list
					 */
					oal_entry = tx_cb->oal;
					tx_entry =
					    &mac_iocb_ptr->oal_entry[
					    TX_DESC_PER_IOCB-1];
					bcopy(tx_entry, oal_entry,
					    sizeof (*oal_entry));

					/*
					 * last entry should be updated to
					 * point to the extended oal list itself
					 */
					tx_entry->buf_addr_low =
					    cpu_to_le32(
					    LS_64BITS(tx_cb->oal_dma_addr));
					tx_entry->buf_addr_high =
					    cpu_to_le32(
					    MS_64BITS(tx_cb->oal_dma_addr));
					/*
					 * Point tx_entry to the oal list
					 * second entry
					 */
					tx_entry = &oal_entry[1];
				}

				tx_entry->buf_len =
				    (uint32_t)cpu_to_le32(dma_cookie.dmac_size);
				phys_addr = dma_cookie.dmac_laddress;
				tx_entry->buf_addr_low =
				    cpu_to_le32(LS_64BITS(phys_addr));
				tx_entry->buf_addr_high =
				    cpu_to_le32(MS_64BITS(phys_addr));

				last_oal_entry = tx_entry;

				if (i > 1)
					ddi_dma_nextcookie(
					    tx_cb->tx_dma_handle[j],
					    &dma_cookie);
			}
			j++;
		}
		/*
		 * if OAL is used, the last oal entry in tx iocb indicates
		 * number of additional address/len pairs in OAL
		 */
		if (oal_entries > TX_DESC_PER_IOCB) {
			tx_entry = &mac_iocb_ptr->oal_entry[TX_DESC_PER_IOCB-1];
			tx_entry->buf_len = (uint32_t)
			    (cpu_to_le32((sizeof (struct oal_entry) *
			    (oal_entries -TX_DESC_PER_IOCB+1))|OAL_CONT_ENTRY));
		}
		last_oal_entry->buf_len = cpu_to_le32(
		    le32_to_cpu(last_oal_entry->buf_len)|OAL_LAST_ENTRY);

		tx_cb->tx_dma_handle_used = j;
		QL_PRINT(DBG_TX, ("total tx_dma_handle_used %d cookies %d \n",
		    j, oal_entries));

		bp = (caddr_t)mp->b_rptr;
	}
	if (tx_mode == USE_COPY) {
		bp = tx_cb->copy_buffer;
		off = 0;
		nbyte = 0;
		frags = 0;
		/*
		 * Copy up to tx_buf_len of the transmit data
		 * from mp to tx buffer
		 */
		for (tp = mp; tp != NULL; tp = tp->b_cont) {
			nbyte = MBLKL(tp);
			if ((off + nbyte) <= tx_buf_len) {
				bcopy(tp->b_rptr, &bp[off], nbyte);
				off += nbyte;
				frags ++;
			}
		}

		msg_len = off;

		mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
		mac_iocb_ptr->tid = tx_ring->prod_idx;
		mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len);
		mac_iocb_ptr->txq_idx = tx_ring->wq_id;

		QL_PRINT(DBG_TX, ("Copy Mode:actual sent data length is: %d, "
		    "from %d segaments\n", msg_len, frags));

		phys_addr = tx_cb->copy_buffer_dma_addr;
		phy_addr_low = cpu_to_le32(LS_64BITS(phys_addr));
		phy_addr_high = cpu_to_le32(MS_64BITS(phys_addr));

		QL_DUMP(DBG_TX, "\t requested sending data:\n",
		    (uint8_t *)tx_cb->copy_buffer, 8, total_len);

		mac_iocb_ptr->oal_entry[0].buf_len = (uint32_t)
		    cpu_to_le32(msg_len | OAL_LAST_ENTRY);
		mac_iocb_ptr->oal_entry[0].buf_addr_low  = phy_addr_low;
		mac_iocb_ptr->oal_entry[0].buf_addr_high = phy_addr_high;

		freemsg(mp); /* no need, we have copied */
		tx_cb->mp = NULL;
	} /* End of Copy Mode */

	/* Do TSO/LSO on TCP packet? */
	if (use_lso && mss) {
		ql_hw_lso_setup(qlge, mss, bp, mac_iocb_ptr);
	} else if (pflags & qlge->chksum_cap) {
		/* Do checksum offloading */
		ql_hw_csum_setup(qlge, pflags, bp, mac_iocb_ptr);
	}

	/* let device know the latest outbound IOCB */
	(void) ddi_dma_sync(tx_ring->wq_dma.dma_handle,
	    (off_t)((uintptr_t)mac_iocb_ptr - (uintptr_t)tx_ring->wq_dma.vaddr),
	    (size_t)sizeof (*mac_iocb_ptr), DDI_DMA_SYNC_FORDEV);

	if (tx_mode == USE_DMA) {
		/* let device know the latest outbound OAL if necessary */
		if (oal_entries > TX_DESC_PER_IOCB) {
			(void) ddi_dma_sync(tx_cb->oal_dma.dma_handle,
			    (off_t)0,
			    (sizeof (struct oal_entry) *
			    (oal_entries -TX_DESC_PER_IOCB+1)),
			    DDI_DMA_SYNC_FORDEV);
		}
	} else { /* for USE_COPY mode, tx buffer has changed */
		/* let device know the latest change */
		(void) ddi_dma_sync(tx_cb->oal_dma.dma_handle,
		/* copy buf offset */
		    (off_t)(sizeof (oal_entry) * MAX_SG_ELEMENTS),
		    msg_len, DDI_DMA_SYNC_FORDEV);
	}

	/* save how the packet was sent */
	tx_cb->tx_type = tx_mode;

	QL_DUMP_REQ_PKT(qlge, mac_iocb_ptr, tx_cb->oal, oal_entries);
	/* reduce the number of available tx slot */
	atomic_dec_32(&tx_ring->tx_free_count);

	tx_ring->prod_idx++;
	if (tx_ring->prod_idx >= tx_ring->wq_len)
		tx_ring->prod_idx = 0;

	now = ddi_get_lbolt();
	qlge->last_tx_time = now;

	return (DDI_SUCCESS);

bad:
	/*
	 * if for any reason driver can not send, delete
	 * the message pointer, mp
	 */
	now = ddi_get_lbolt();
	freemsg(mp);
	mp = NULL;
	for (i = 0; i < j; i++)
		(void) ddi_dma_unbind_handle(tx_cb->tx_dma_handle[i]);

	QL_PRINT(DBG_TX, ("%s(%d) failed at 0x%x",
	    __func__, qlge->instance, (int)now));

	return (DDI_SUCCESS);
}


/*
 * Initializes hardware and driver software flags before the driver
 * is finally ready to work.
 */
int
ql_do_start(qlge_t *qlge)
{
	int i;
	struct rx_ring *rx_ring;
	uint16_t lbq_buf_size;
	int rings_done;

	ASSERT(qlge != NULL);

	mutex_enter(&qlge->hw_mutex);

	/* Reset adapter */
	(void) ql_asic_reset(qlge);

	lbq_buf_size = (uint16_t)
	    ((qlge->mtu == ETHERMTU)? NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE);
	if (qlge->rx_ring[0].lbq_buf_size != lbq_buf_size) {
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_NOTE, "realloc buffers old: %d new: %d\n",
		    qlge->rx_ring[0].lbq_buf_size, lbq_buf_size);
#endif
		/*
		 * Check if any ring has buffers still with upper layers
		 * If buffers are pending with upper layers, we use the
		 * existing buffers and don't reallocate new ones
		 * Unfortunately there is no way to evict buffers from
		 * upper layers. Using buffers with the current size may
		 * cause slightly sub-optimal performance, but that seems
		 * to be the easiest way to handle this situation.
		 */
		rings_done = 0;
		for (i = 0; i < qlge->rx_ring_count; i++) {
			rx_ring = &qlge->rx_ring[i];
			if (rx_ring->rx_indicate == 0)
				rings_done++;
			else
				break;
		}
		/*
		 * No buffers pending with upper layers;
		 * reallocte them for new MTU size
		 */
		if (rings_done >= qlge->rx_ring_count) {
			/* free large buffer pool */
			for (i = 0; i < qlge->rx_ring_count; i++) {
				rx_ring = &qlge->rx_ring[i];
				if (rx_ring->type != TX_Q) {
					ql_free_sbq_buffers(rx_ring);
					ql_free_lbq_buffers(rx_ring);
				}
			}
			/* reallocate large buffer pool */
			for (i = 0; i < qlge->rx_ring_count; i++) {
				rx_ring = &qlge->rx_ring[i];
				if (rx_ring->type != TX_Q) {
					(void) ql_alloc_sbufs(qlge, rx_ring);
					(void) ql_alloc_lbufs(qlge, rx_ring);
				}
			}
		}
	}

	if (ql_bringup_adapter(qlge) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "qlge bringup adapter failed");
		ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED);
		mutex_exit(&qlge->hw_mutex);
		return (DDI_FAILURE);
	}

	mutex_exit(&qlge->hw_mutex);

	/* Get current link state */
	qlge->port_link_state = ql_get_link_state(qlge);

	if (qlge->port_link_state == LS_UP) {
		QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n",
		    __func__, qlge->instance));
		/* If driver detects a carrier on */
		CARRIER_ON(qlge);
	} else {
		QL_PRINT(DBG_GLD, ("%s(%d) Link down\n",
		    __func__, qlge->instance));
		/* If driver detects a lack of carrier */
		CARRIER_OFF(qlge);
	}
	qlge->mac_flags = QL_MAC_STARTED;
	return (DDI_SUCCESS);
}

/*
 * Stop currently running driver
 * Driver needs to stop routing new packets to driver and wait until
 * all pending tx/rx buffers to be free-ed.
 */
int
ql_do_stop(qlge_t *qlge)
{
	int rc = DDI_FAILURE;
	uint32_t i, j, k;
	struct bq_desc *sbq_desc, *lbq_desc;
	struct rx_ring *rx_ring;

	ASSERT(qlge != NULL);

	CARRIER_OFF(qlge);

	rc = ql_bringdown_adapter(qlge);
	if (rc != DDI_SUCCESS) {
		cmn_err(CE_WARN, "qlge bringdown adapter failed.");
	} else
		rc = DDI_SUCCESS;

	for (k = 0; k < qlge->rx_ring_count; k++) {
		rx_ring = &qlge->rx_ring[k];
		if (rx_ring->type != TX_Q) {
			j = rx_ring->lbq_use_head;
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_NOTE, "ring %d: move %d lbufs in use list"
			    " to free list %d\n total %d\n",
			    k, rx_ring->lbuf_in_use_count,
			    rx_ring->lbuf_free_count,
			    rx_ring->lbuf_in_use_count +
			    rx_ring->lbuf_free_count);
#endif
			for (i = 0; i < rx_ring->lbuf_in_use_count; i++) {
				lbq_desc = rx_ring->lbuf_in_use[j];
				j++;
				if (j >= rx_ring->lbq_len) {
					j = 0;
				}
				if (lbq_desc->mp) {
					atomic_inc_32(&rx_ring->rx_indicate);
					freemsg(lbq_desc->mp);
				}
			}
			rx_ring->lbq_use_head = j;
			rx_ring->lbq_use_tail = j;
			rx_ring->lbuf_in_use_count = 0;
			j = rx_ring->sbq_use_head;
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_NOTE, "ring %d: move %d sbufs in use list,"
			    " to free list %d\n total %d \n",
			    k, rx_ring->sbuf_in_use_count,
			    rx_ring->sbuf_free_count,
			    rx_ring->sbuf_in_use_count +
			    rx_ring->sbuf_free_count);
#endif
			for (i = 0; i < rx_ring->sbuf_in_use_count; i++) {
				sbq_desc = rx_ring->sbuf_in_use[j];
				j++;
				if (j >= rx_ring->sbq_len) {
					j = 0;
				}
				if (sbq_desc->mp) {
					atomic_inc_32(&rx_ring->rx_indicate);
					freemsg(sbq_desc->mp);
				}
			}
			rx_ring->sbq_use_head = j;
			rx_ring->sbq_use_tail = j;
			rx_ring->sbuf_in_use_count = 0;
		}
	}

	qlge->mac_flags = QL_MAC_STOPPED;

	return (rc);
}

/*
 * Support
 */

void
ql_disable_isr(qlge_t *qlge)
{
	/*
	 * disable the hardware interrupt
	 */
	ISP_DISABLE_GLOBAL_INTRS(qlge);

	qlge->flags &= ~INTERRUPTS_ENABLED;
}



/*
 * busy wait for 'usecs' microseconds.
 */
void
qlge_delay(clock_t usecs)
{
	drv_usecwait(usecs);
}

/*
 * retrieve firmware details.
 */

pci_cfg_t *
ql_get_pci_config(qlge_t *qlge)
{
	return (&(qlge->pci_cfg));
}

/*
 * Get current Link status
 */
static uint32_t
ql_get_link_state(qlge_t *qlge)
{
	uint32_t bitToCheck = 0;
	uint32_t temp, linkState;

	if (qlge->func_number == qlge->fn0_net) {
		bitToCheck = STS_PL0;
	} else {
		bitToCheck = STS_PL1;
	}
	temp = ql_read_reg(qlge, REG_STATUS);
	QL_PRINT(DBG_GLD, ("%s(%d) chip status reg: 0x%x\n",
	    __func__, qlge->instance, temp));

	if (temp & bitToCheck) {
		linkState = LS_UP;
	} else {
		linkState = LS_DOWN;
	}
	if (CFG_IST(qlge, CFG_CHIP_8100)) {
		/* for Schultz, link Speed is fixed to 10G, full duplex */
		qlge->speed  = SPEED_10G;
		qlge->duplex = 1;
	}
	return (linkState);
}
/*
 * Get current link status and report to OS
 */
static void
ql_get_and_report_link_state(qlge_t *qlge)
{
	uint32_t cur_link_state;

	/* Get current link state */
	cur_link_state = ql_get_link_state(qlge);
	/* if link state has changed */
	if (cur_link_state != qlge->port_link_state) {

		qlge->port_link_state = cur_link_state;

		if (qlge->port_link_state == LS_UP) {
			QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n",
			    __func__, qlge->instance));
			/* If driver detects a carrier on */
			CARRIER_ON(qlge);
		} else {
			QL_PRINT(DBG_GLD, ("%s(%d) Link down\n",
			    __func__, qlge->instance));
			/* If driver detects a lack of carrier */
			CARRIER_OFF(qlge);
		}
	}
}

/*
 * timer callback function executed after timer expires
 */
static void
ql_timer(void* arg)
{
	ql_get_and_report_link_state((qlge_t *)arg);
}

/*
 * stop the running timer if activated
 */
static void
ql_stop_timer(qlge_t *qlge)
{
	timeout_id_t timer_id;
	/* Disable driver timer */
	if (qlge->ql_timer_timeout_id != NULL) {
		timer_id = qlge->ql_timer_timeout_id;
		qlge->ql_timer_timeout_id = NULL;
		(void) untimeout(timer_id);
	}
}

/*
 * stop then restart timer
 */
void
ql_restart_timer(qlge_t *qlge)
{
	ql_stop_timer(qlge);
	qlge->ql_timer_ticks = TICKS_PER_SEC / 4;
	qlge->ql_timer_timeout_id = timeout(ql_timer,
	    (void *)qlge, qlge->ql_timer_ticks);
}

/* ************************************************************************* */
/*
 *		Hardware K-Stats Data Structures and Subroutines
 */
/* ************************************************************************* */
static const ql_ksindex_t ql_kstats_hw[] = {
	/* PCI related hardware information */
	{ 0, "Vendor Id"			},
	{ 1, "Device Id"			},
	{ 2, "Command"				},
	{ 3, "Status"				},
	{ 4, "Revision Id"			},
	{ 5, "Cache Line Size"			},
	{ 6, "Latency Timer"			},
	{ 7, "Header Type"			},
	{ 9, "I/O base addr"			},
	{ 10, "Control Reg Base addr low"	},
	{ 11, "Control Reg Base addr high"	},
	{ 12, "Doorbell Reg Base addr low"	},
	{ 13, "Doorbell Reg Base addr high"	},
	{ 14, "Subsystem Vendor Id"		},
	{ 15, "Subsystem Device ID"		},
	{ 16, "PCIe Device Control"		},
	{ 17, "PCIe Link Status"		},

	{ -1,	NULL				},
};

/*
 * kstat update function for PCI registers
 */
static int
ql_kstats_get_pci_regs(kstat_t *ksp, int flag)
{
	qlge_t *qlge;
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	qlge = ksp->ks_private;
	knp = ksp->ks_data;
	(knp++)->value.ui32 = qlge->pci_cfg.vendor_id;
	(knp++)->value.ui32 = qlge->pci_cfg.device_id;
	(knp++)->value.ui32 = qlge->pci_cfg.command;
	(knp++)->value.ui32 = qlge->pci_cfg.status;
	(knp++)->value.ui32 = qlge->pci_cfg.revision;
	(knp++)->value.ui32 = qlge->pci_cfg.cache_line_size;
	(knp++)->value.ui32 = qlge->pci_cfg.latency_timer;
	(knp++)->value.ui32 = qlge->pci_cfg.header_type;
	(knp++)->value.ui32 = qlge->pci_cfg.io_base_address;
	(knp++)->value.ui32 =
	    qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_lower;
	(knp++)->value.ui32 =
	    qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_upper;
	(knp++)->value.ui32 =
	    qlge->pci_cfg.pci_doorbell_mem_base_address_lower;
	(knp++)->value.ui32 =
	    qlge->pci_cfg.pci_doorbell_mem_base_address_upper;
	(knp++)->value.ui32 = qlge->pci_cfg.sub_vendor_id;
	(knp++)->value.ui32 = qlge->pci_cfg.sub_device_id;
	(knp++)->value.ui32 = qlge->pci_cfg.pcie_device_control;
	(knp++)->value.ui32 = qlge->pci_cfg.link_status;

	return (0);
}

static const ql_ksindex_t ql_kstats_mii[] = {
	/* MAC/MII related hardware information */
	{ 0, "mtu"},

	{ -1, NULL},
};


/*
 * kstat update function for MII related information.
 */
static int
ql_kstats_mii_update(kstat_t *ksp, int flag)
{
	qlge_t *qlge;
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	qlge = ksp->ks_private;
	knp = ksp->ks_data;

	(knp++)->value.ui32 = qlge->mtu;

	return (0);
}

static const ql_ksindex_t ql_kstats_reg[] = {
	/* Register information */
	{ 0, "System (0x08)"			},
	{ 1, "Reset/Fail Over(0x0Ch"		},
	{ 2, "Function Specific Control(0x10)"	},
	{ 3, "Status (0x30)"			},
	{ 4, "Intr Enable (0x34)"		},
	{ 5, "Intr Status1 (0x3C)"		},
	{ 6, "Error Status (0x54)"		},
	{ 7, "XGMAC Flow Control(0x11C)"	},
	{ 8, "XGMAC Tx Pause Frames(0x230)"	},
	{ 9, "XGMAC Rx Pause Frames(0x388)"	},
	{ 10, "XGMAC Rx FIFO Drop Count(0x5B8)"	},
	{ 11, "interrupts actually allocated"	},
	{ 12, "interrupts on rx ring 0"		},
	{ 13, "interrupts on rx ring 1"		},
	{ 14, "interrupts on rx ring 2"		},
	{ 15, "interrupts on rx ring 3"		},
	{ 16, "interrupts on rx ring 4"		},
	{ 17, "interrupts on rx ring 5"		},
	{ 18, "interrupts on rx ring 6"		},
	{ 19, "interrupts on rx ring 7"		},
	{ 20, "polls on rx ring 0"		},
	{ 21, "polls on rx ring 1"		},
	{ 22, "polls on rx ring 2"		},
	{ 23, "polls on rx ring 3"		},
	{ 24, "polls on rx ring 4"		},
	{ 25, "polls on rx ring 5"		},
	{ 26, "polls on rx ring 6"		},
	{ 27, "polls on rx ring 7"		},
	{ 28, "tx no resource on ring 0"	},
	{ 29, "tx dma bind fail on ring 0"	},
	{ 30, "tx dma no handle on ring 0"	},
	{ 31, "tx dma no cookie on ring 0"	},
	{ 32, "MPI firmware major version"},
	{ 33, "MPI firmware minor version"},
	{ 34, "MPI firmware sub version"},

	{ -1, NULL},
};


/*
 * kstat update function for device register set
 */
static int
ql_kstats_get_reg_and_dev_stats(kstat_t *ksp, int flag)
{
	qlge_t *qlge;
	kstat_named_t *knp;
	uint32_t val32;
	int i = 0;
	struct tx_ring *tx_ring;

	if (flag != KSTAT_READ)
		return (EACCES);

	qlge = ksp->ks_private;
	knp = ksp->ks_data;

	(knp++)->value.ui32 = ql_read_reg(qlge, REG_SYSTEM);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_RESET_FAILOVER);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_STATUS);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_ENABLE);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1);
	(knp++)->value.ui32 = ql_read_reg(qlge, REG_ERROR_STATUS);

	if (ql_sem_spinlock(qlge, qlge->xgmac_sem_mask)) {
		return (0);
	}
	(void) ql_read_xgmac_reg(qlge, REG_XGMAC_FLOW_CONTROL, &val32);
	(knp++)->value.ui32 = val32;

	(void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_TX_PAUSE_PKTS, &val32);
	(knp++)->value.ui32 = val32;

	(void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_PAUSE_PKTS, &val32);
	(knp++)->value.ui32 = val32;

	(void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_FIFO_DROPS, &val32);
	(knp++)->value.ui32 = val32;

	ql_sem_unlock(qlge, qlge->xgmac_sem_mask);

	(knp++)->value.ui32 = qlge->intr_cnt;

	for (i = 0; i < 8; i++) {
		(knp++)->value.ui32 = qlge->rx_interrupts[i];
	}

	for (i = 0; i < 8; i++) {
		(knp++)->value.ui32 = qlge->rx_polls[i];
	}

	tx_ring = &qlge->tx_ring[0];
	(knp++)->value.ui32 = tx_ring->defer;
	(knp++)->value.ui32 = tx_ring->tx_fail_dma_bind;
	(knp++)->value.ui32 = tx_ring->tx_no_dma_handle;
	(knp++)->value.ui32 = tx_ring->tx_no_dma_cookie;

	(knp++)->value.ui32 = qlge->fw_version_info.major_version;
	(knp++)->value.ui32 = qlge->fw_version_info.minor_version;
	(knp++)->value.ui32 = qlge->fw_version_info.sub_minor_version;

	return (0);
}


static kstat_t *
ql_setup_named_kstat(qlge_t *qlge, int instance, char *name,
    const ql_ksindex_t *ksip, size_t size, int (*update)(kstat_t *, int))
{
	kstat_t *ksp;
	kstat_named_t *knp;
	char *np;
	int type;

	size /= sizeof (ql_ksindex_t);
	ksp = kstat_create(ADAPTER_NAME, instance, name, "net",
	    KSTAT_TYPE_NAMED, ((uint32_t)size) - 1, KSTAT_FLAG_PERSISTENT);
	if (ksp == NULL)
		return (NULL);

	ksp->ks_private = qlge;
	ksp->ks_update = update;
	for (knp = ksp->ks_data; (np = ksip->name) != NULL; ++knp, ++ksip) {
		switch (*np) {
		default:
			type = KSTAT_DATA_UINT32;
			break;
		case '&':
			np += 1;
			type = KSTAT_DATA_CHAR;
			break;
		}
		kstat_named_init(knp, np, (uint8_t)type);
	}
	kstat_install(ksp);

	return (ksp);
}

/*
 * Setup various kstat
 */
int
ql_init_kstats(qlge_t *qlge)
{
	/* Hardware KStats */
	qlge->ql_kstats[QL_KSTAT_CHIP] = ql_setup_named_kstat(qlge,
	    qlge->instance, "chip", ql_kstats_hw,
	    sizeof (ql_kstats_hw), ql_kstats_get_pci_regs);
	if (qlge->ql_kstats[QL_KSTAT_CHIP] == NULL) {
		return (DDI_FAILURE);
	}

	/* MII KStats */
	qlge->ql_kstats[QL_KSTAT_LINK] = ql_setup_named_kstat(qlge,
	    qlge->instance, "mii", ql_kstats_mii,
	    sizeof (ql_kstats_mii), ql_kstats_mii_update);
	if (qlge->ql_kstats[QL_KSTAT_LINK] == NULL) {
		return (DDI_FAILURE);
	}

	/* REG KStats */
	qlge->ql_kstats[QL_KSTAT_REG] = ql_setup_named_kstat(qlge,
	    qlge->instance, "reg", ql_kstats_reg,
	    sizeof (ql_kstats_reg), ql_kstats_get_reg_and_dev_stats);
	if (qlge->ql_kstats[QL_KSTAT_REG] == NULL) {
		return (DDI_FAILURE);
	}
	return (DDI_SUCCESS);
}

/*
 * delete all kstat
 */
void
ql_fini_kstats(qlge_t *qlge)
{
	int i;

	for (i = 0; i < QL_KSTAT_COUNT; i++) {
		if (qlge->ql_kstats[i] != NULL)
			kstat_delete(qlge->ql_kstats[i]);
	}
}

/* ************************************************************************* */
/*
 *                                 kstat end
 */
/* ************************************************************************* */

/*
 * Setup the parameters for receive and transmit rings including buffer sizes
 * and completion queue sizes
 */
static int
ql_setup_rings(qlge_t *qlge)
{
	uint8_t i;
	struct rx_ring *rx_ring;
	struct tx_ring *tx_ring;
	uint16_t lbq_buf_size;

	lbq_buf_size = (uint16_t)
	    ((qlge->mtu == ETHERMTU)? NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE);

	/*
	 * rx_ring[0] is always the default queue.
	 */
	/*
	 * qlge->rx_ring_count:
	 * Total number of rx_rings. This includes a number
	 * of outbound completion handler rx_rings, and a
	 * number of inbound completion handler rx_rings.
	 * rss is only enabled if we have more than 1 rx completion
	 * queue. If we have a single rx completion queue
	 * then all rx completions go to this queue and
	 * the last completion queue
	 */

	qlge->tx_ring_first_cq_id = qlge->rss_ring_count;

	for (i = 0; i < qlge->tx_ring_count; i++) {
		tx_ring = &qlge->tx_ring[i];
		bzero((void *)tx_ring, sizeof (*tx_ring));
		tx_ring->qlge = qlge;
		tx_ring->wq_id = i;
		tx_ring->wq_len = qlge->tx_ring_size;
		tx_ring->wq_size = (uint32_t)(
		    tx_ring->wq_len * sizeof (struct ob_mac_iocb_req));

		/*
		 * The completion queue ID for the tx rings start
		 * immediately after the last rss completion queue.
		 */
		tx_ring->cq_id = (uint16_t)(i + qlge->tx_ring_first_cq_id);
	}

	for (i = 0; i < qlge->rx_ring_count; i++) {
		rx_ring = &qlge->rx_ring[i];
		bzero((void *)rx_ring, sizeof (*rx_ring));
		rx_ring->qlge = qlge;
		rx_ring->cq_id = i;
		if (i != 0)
			rx_ring->cpu = (i) % qlge->rx_ring_count;
		else
			rx_ring->cpu = 0;

		if (i < qlge->rss_ring_count) {
			/*
			 * Inbound completions (RSS) queues
			 * Default queue is queue 0 which handles
			 * unicast plus bcast/mcast and async events.
			 * Other inbound queues handle unicast frames only.
			 */
			rx_ring->cq_len = qlge->rx_ring_size;
			rx_ring->cq_size = (uint32_t)
			    (rx_ring->cq_len * sizeof (struct net_rsp_iocb));
			rx_ring->lbq_len = NUM_LARGE_BUFFERS;
			rx_ring->lbq_size = (uint32_t)
			    (rx_ring->lbq_len * sizeof (uint64_t));
			rx_ring->lbq_buf_size = lbq_buf_size;
			rx_ring->sbq_len = NUM_SMALL_BUFFERS;
			rx_ring->sbq_size = (uint32_t)
			    (rx_ring->sbq_len * sizeof (uint64_t));
			rx_ring->sbq_buf_size = SMALL_BUFFER_SIZE * 2;
			rx_ring->type = RX_Q;

			QL_PRINT(DBG_GLD,
			    ("%s(%d)Allocating rss completion queue %d "
			    "on cpu %d\n", __func__, qlge->instance,
			    rx_ring->cq_id, rx_ring->cpu));
		} else {
			/*
			 * Outbound queue handles outbound completions only
			 */
			/* outbound cq is same size as tx_ring it services. */
			rx_ring->cq_len = qlge->tx_ring_size;
			rx_ring->cq_size = (uint32_t)
			    (rx_ring->cq_len * sizeof (struct net_rsp_iocb));
			rx_ring->lbq_len = 0;
			rx_ring->lbq_size = 0;
			rx_ring->lbq_buf_size = 0;
			rx_ring->sbq_len = 0;
			rx_ring->sbq_size = 0;
			rx_ring->sbq_buf_size = 0;
			rx_ring->type = TX_Q;

			QL_PRINT(DBG_GLD,
			    ("%s(%d)Allocating TX completion queue %d on"
			    " cpu %d\n", __func__, qlge->instance,
			    rx_ring->cq_id, rx_ring->cpu));
		}
	}

	return (DDI_SUCCESS);
}

static int
ql_start_rx_ring(qlge_t *qlge, struct rx_ring *rx_ring)
{
	struct cqicb_t *cqicb = (struct cqicb_t *)rx_ring->cqicb_dma.vaddr;
	void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr +
	    (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE)
	/* first shadow area is used by wqicb's host copy of consumer index */
	    + sizeof (uint64_t);
	uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr +
	    (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE)
	    + sizeof (uint64_t);
	/* lrg/sml bufq pointers */
	uint8_t *buf_q_base_reg =
	    (uint8_t *)qlge->buf_q_ptr_base_addr_dma_attr.vaddr +
	    (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE);
	uint64_t buf_q_base_reg_dma =
	    qlge->buf_q_ptr_base_addr_dma_attr.dma_addr +
	    (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE);
	caddr_t doorbell_area =
	    qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * (128 + rx_ring->cq_id));
	int err = 0;
	uint16_t bq_len;
	uint64_t tmp;
	uint64_t *base_indirect_ptr;
	int page_entries;

	/* Set up the shadow registers for this ring. */
	rx_ring->prod_idx_sh_reg = shadow_reg;
	rx_ring->prod_idx_sh_reg_dma = shadow_reg_dma;

	rx_ring->lbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg;
	rx_ring->lbq_base_indirect_dma = buf_q_base_reg_dma;

	QL_PRINT(DBG_INIT, ("%s rx ring(%d): prod_idx virtual addr = 0x%lx,"
	    " phys_addr 0x%lx\n", __func__, rx_ring->cq_id,
	    rx_ring->prod_idx_sh_reg, rx_ring->prod_idx_sh_reg_dma));

	buf_q_base_reg += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t));
	buf_q_base_reg_dma += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t));
	rx_ring->sbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg;
	rx_ring->sbq_base_indirect_dma = buf_q_base_reg_dma;

	/* PCI doorbell mem area + 0x00 for consumer index register */
	rx_ring->cnsmr_idx_db_reg = (uint32_t *)(void *)doorbell_area;
	rx_ring->cnsmr_idx = 0;
	*rx_ring->prod_idx_sh_reg = 0;
	rx_ring->curr_entry = rx_ring->cq_dma.vaddr;

	/* PCI doorbell mem area + 0x04 for valid register */
	rx_ring->valid_db_reg = (uint32_t *)(void *)
	    ((uint8_t *)(void *)doorbell_area + 0x04);

	/* PCI doorbell mem area + 0x18 for large buffer consumer */
	rx_ring->lbq_prod_idx_db_reg = (uint32_t *)(void *)
	    ((uint8_t *)(void *)doorbell_area + 0x18);

	/* PCI doorbell mem area + 0x1c */
	rx_ring->sbq_prod_idx_db_reg = (uint32_t *)(void *)
	    ((uint8_t *)(void *)doorbell_area + 0x1c);

	bzero((void *)cqicb, sizeof (*cqicb));

	cqicb->msix_vect = (uint8_t)rx_ring->irq;

	bq_len = (uint16_t)((rx_ring->cq_len == 65536) ?
	    (uint16_t)0 : (uint16_t)rx_ring->cq_len);
	cqicb->len = (uint16_t)cpu_to_le16(bq_len | LEN_V | LEN_CPP_CONT);

	cqicb->cq_base_addr_lo =
	    cpu_to_le32(LS_64BITS(rx_ring->cq_dma.dma_addr));
	cqicb->cq_base_addr_hi =
	    cpu_to_le32(MS_64BITS(rx_ring->cq_dma.dma_addr));

	cqicb->prod_idx_addr_lo =
	    cpu_to_le32(LS_64BITS(rx_ring->prod_idx_sh_reg_dma));
	cqicb->prod_idx_addr_hi =
	    cpu_to_le32(MS_64BITS(rx_ring->prod_idx_sh_reg_dma));

	/*
	 * Set up the control block load flags.
	 */
	cqicb->flags = FLAGS_LC | /* Load queue base address */
	    FLAGS_LV | /* Load MSI-X vector */
	    FLAGS_LI;  /* Load irq delay values */
	if (rx_ring->lbq_len) {
		/* Load lbq values */
		cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LL);
		tmp = (uint64_t)rx_ring->lbq_dma.dma_addr;
		base_indirect_ptr = (uint64_t *)rx_ring->lbq_base_indirect;
		page_entries = 0;
		do {
			*base_indirect_ptr = cpu_to_le64(tmp);
			tmp += VM_PAGE_SIZE;
			base_indirect_ptr++;
			page_entries++;
		} while (page_entries < (int)(
		    ((rx_ring->lbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE)));

		cqicb->lbq_addr_lo =
		    cpu_to_le32(LS_64BITS(rx_ring->lbq_base_indirect_dma));
		cqicb->lbq_addr_hi =
		    cpu_to_le32(MS_64BITS(rx_ring->lbq_base_indirect_dma));
		bq_len = (uint16_t)((rx_ring->lbq_buf_size == 65536) ?
		    (uint16_t)0 : (uint16_t)rx_ring->lbq_buf_size);
		cqicb->lbq_buf_size = (uint16_t)cpu_to_le16(bq_len);
		bq_len = (uint16_t)((rx_ring->lbq_len == 65536) ? (uint16_t)0 :
		    (uint16_t)rx_ring->lbq_len);
		cqicb->lbq_len = (uint16_t)cpu_to_le16(bq_len);
		rx_ring->lbq_prod_idx = 0;
		rx_ring->lbq_curr_idx = 0;
	}
	if (rx_ring->sbq_len) {
		/* Load sbq values */
		cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LS);
		tmp = (uint64_t)rx_ring->sbq_dma.dma_addr;
		base_indirect_ptr = (uint64_t *)rx_ring->sbq_base_indirect;
		page_entries = 0;

		do {
			*base_indirect_ptr = cpu_to_le64(tmp);
			tmp += VM_PAGE_SIZE;
			base_indirect_ptr++;
			page_entries++;
		} while (page_entries < (uint32_t)
		    (((rx_ring->sbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE)));

		cqicb->sbq_addr_lo =
		    cpu_to_le32(LS_64BITS(rx_ring->sbq_base_indirect_dma));
		cqicb->sbq_addr_hi =
		    cpu_to_le32(MS_64BITS(rx_ring->sbq_base_indirect_dma));
		cqicb->sbq_buf_size = (uint16_t)
		    cpu_to_le16((uint16_t)(rx_ring->sbq_buf_size/2));
		bq_len = (uint16_t)((rx_ring->sbq_len == 65536) ?
		    (uint16_t)0 : (uint16_t)rx_ring->sbq_len);
		cqicb->sbq_len = (uint16_t)cpu_to_le16(bq_len);
		rx_ring->sbq_prod_idx = 0;
		rx_ring->sbq_curr_idx = 0;
	}
	switch (rx_ring->type) {
	case TX_Q:
		cqicb->irq_delay = (uint16_t)
		    cpu_to_le16(qlge->tx_coalesce_usecs);
		cqicb->pkt_delay = (uint16_t)
		    cpu_to_le16(qlge->tx_max_coalesced_frames);
		break;

	case DEFAULT_Q:
		cqicb->irq_delay = 0;
		cqicb->pkt_delay = 0;
		break;

	case RX_Q:
		/*
		 * Inbound completion handling rx_rings run in
		 * separate NAPI contexts.
		 */
		cqicb->irq_delay = (uint16_t)
		    cpu_to_le16(qlge->rx_coalesce_usecs);
		cqicb->pkt_delay = (uint16_t)
		    cpu_to_le16(qlge->rx_max_coalesced_frames);
		break;
	default:
		cmn_err(CE_WARN, "Invalid rx_ring->type = %d.",
		    rx_ring->type);
	}
	QL_PRINT(DBG_INIT, ("Initializing rx completion queue %d.\n",
	    rx_ring->cq_id));
	/* QL_DUMP_CQICB(qlge, cqicb); */
	err = ql_write_cfg(qlge, CFG_LCQ, rx_ring->cqicb_dma.dma_addr,
	    rx_ring->cq_id);
	if (err) {
		cmn_err(CE_WARN, "Failed to load CQICB.");
		return (err);
	}

	rx_ring->rx_packets_dropped_no_buffer = 0;
	rx_ring->rx_pkt_dropped_mac_unenabled = 0;
	rx_ring->rx_failed_sbq_allocs = 0;
	rx_ring->rx_failed_lbq_allocs = 0;
	rx_ring->rx_packets = 0;
	rx_ring->rx_bytes = 0;
	rx_ring->frame_too_long = 0;
	rx_ring->frame_too_short = 0;
	rx_ring->fcs_err = 0;

	return (err);
}

/*
 * start RSS
 */
static int
ql_start_rss(qlge_t *qlge)
{
	struct ricb *ricb = (struct ricb *)qlge->ricb_dma.vaddr;
	int status = 0;
	int i;
	uint8_t *hash_id = (uint8_t *)ricb->hash_cq_id;

	bzero((void *)ricb, sizeof (*ricb));

	ricb->base_cq = RSS_L4K;
	ricb->flags =
	    (RSS_L6K | RSS_LI | RSS_LB | RSS_LM | RSS_RI4 | RSS_RI6 | RSS_RT4 |
	    RSS_RT6);
	ricb->mask = (uint16_t)cpu_to_le16(RSS_HASH_CQ_ID_MAX - 1);

	/*
	 * Fill out the Indirection Table.
	 */
	for (i = 0; i < RSS_HASH_CQ_ID_MAX; i++)
		hash_id[i] = (uint8_t)(i & (qlge->rss_ring_count - 1));

	(void) memcpy(&ricb->ipv6_hash_key[0], key_data, 40);
	(void) memcpy(&ricb->ipv4_hash_key[0], key_data, 16);

	QL_PRINT(DBG_INIT, ("Initializing RSS.\n"));

	status = ql_write_cfg(qlge, CFG_LR, qlge->ricb_dma.dma_addr, 0);
	if (status) {
		cmn_err(CE_WARN, "Failed to load RICB.");
		return (status);
	}

	return (status);
}

/*
 * load a tx ring control block to hw and start this ring
 */
static int
ql_start_tx_ring(qlge_t *qlge, struct tx_ring *tx_ring)
{
	struct wqicb_t *wqicb = (struct wqicb_t *)tx_ring->wqicb_dma.vaddr;
	caddr_t doorbell_area =
	    qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * tx_ring->wq_id);
	void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr +
	    (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE;
	uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr +
	    (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE;
	int err = 0;

	/*
	 * Assign doorbell registers for this tx_ring.
	 */

	/* TX PCI doorbell mem area for tx producer index */
	tx_ring->prod_idx_db_reg = (uint32_t *)(void *)doorbell_area;
	tx_ring->prod_idx = 0;
	/* TX PCI doorbell mem area + 0x04 */
	tx_ring->valid_db_reg = (uint32_t *)(void *)
	    ((uint8_t *)(void *)doorbell_area + 0x04);

	/*
	 * Assign shadow registers for this tx_ring.
	 */
	tx_ring->cnsmr_idx_sh_reg = shadow_reg;
	tx_ring->cnsmr_idx_sh_reg_dma = shadow_reg_dma;
	*tx_ring->cnsmr_idx_sh_reg = 0;

	QL_PRINT(DBG_INIT, ("%s tx ring(%d): cnsmr_idx virtual addr = 0x%lx,"
	    " phys_addr 0x%lx\n",
	    __func__, tx_ring->wq_id, tx_ring->cnsmr_idx_sh_reg,
	    tx_ring->cnsmr_idx_sh_reg_dma));

	wqicb->len =
	    (uint16_t)cpu_to_le16(tx_ring->wq_len | Q_LEN_V | Q_LEN_CPP_CONT);
	wqicb->flags = cpu_to_le16(Q_FLAGS_LC |
	    Q_FLAGS_LB | Q_FLAGS_LI | Q_FLAGS_LO);
	wqicb->cq_id_rss = (uint16_t)cpu_to_le16(tx_ring->cq_id);
	wqicb->rid = 0;
	wqicb->wq_addr_lo = cpu_to_le32(LS_64BITS(tx_ring->wq_dma.dma_addr));
	wqicb->wq_addr_hi = cpu_to_le32(MS_64BITS(tx_ring->wq_dma.dma_addr));
	wqicb->cnsmr_idx_addr_lo =
	    cpu_to_le32(LS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma));
	wqicb->cnsmr_idx_addr_hi =
	    cpu_to_le32(MS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma));

	ql_init_tx_ring(tx_ring);
	/* QL_DUMP_WQICB(qlge, wqicb); */
	err = ql_write_cfg(qlge, CFG_LRQ, tx_ring->wqicb_dma.dma_addr,
	    tx_ring->wq_id);

	if (err) {
		cmn_err(CE_WARN, "Failed to load WQICB.");
		return (err);
	}
	return (err);
}

/*
 * Set up a MAC, multicast or VLAN address for the
 * inbound frame matching.
 */
int
ql_set_mac_addr_reg(qlge_t *qlge, uint8_t *addr, uint32_t type,
    uint16_t index)
{
	uint32_t offset = 0;
	int status = DDI_SUCCESS;

	switch (type) {
	case MAC_ADDR_TYPE_MULTI_MAC:
	case MAC_ADDR_TYPE_CAM_MAC: {
		uint32_t cam_output;
		uint32_t upper = (addr[0] << 8) | addr[1];
		uint32_t lower =
		    (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
		    (addr[5]);

		QL_PRINT(DBG_INIT, ("Adding %s ", (type ==
		    MAC_ADDR_TYPE_MULTI_MAC) ?
		    "MULTICAST" : "UNICAST"));
		QL_PRINT(DBG_INIT,
		    ("addr %02x %02x %02x %02x %02x %02x at index %d in "
		    "the CAM.\n",
		    addr[0], addr[1], addr[2], addr[3], addr[4],
		    addr[5], index));

		status = ql_wait_reg_rdy(qlge,
		    REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
		if (status)
			goto exit;
		/* offset 0 - lower 32 bits of the MAC address */
		ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
		    (offset++) |
		    (index << MAC_ADDR_IDX_SHIFT) | /* index */
		    type);	/* type */
		ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, lower);
		status = ql_wait_reg_rdy(qlge,
		    REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
		if (status)
			goto exit;
		/* offset 1 - upper 16 bits of the MAC address */
		ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
		    (offset++) |
		    (index << MAC_ADDR_IDX_SHIFT) | /* index */
		    type);	/* type */
		ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, upper);
		status = ql_wait_reg_rdy(qlge,
		    REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0);
		if (status)
			goto exit;
		/* offset 2 - CQ ID associated with this MAC address */
		ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX,
		    (offset) | (index << MAC_ADDR_IDX_SHIFT) |	/* index */
		    type);	/* type */
		/*
		 * This field should also include the queue id
		 * and possibly the function id.  Right now we hardcode
		 * the route field to NIC core.
		 */
		if (type == MAC_ADDR_TYPE_CAM_MAC) {
			cam_output = (CAM_OUT_ROUTE_NIC |
			    (qlge->func_number << CAM_OUT_FUNC_SHIFT) |
			    (0 <<
			    CAM_OUT_CQ_ID_SHIFT));

			/* route to NIC core */
			ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA,
			    cam_output);
			}
		break;
		}
	default:
		cmn_err(CE_WARN,
		    "Address type %d not yet supported.", type);
		status = DDI_FAILURE;
	}
exit:
	return (status);
}

/*
 * The NIC function for this chip has 16 routing indexes.  Each one can be used
 * to route different frame types to various inbound queues.  We send broadcast
 * multicast/error frames to the default queue for slow handling,
 * and CAM hit/RSS frames to the fast handling queues.
 */
static int
ql_set_routing_reg(qlge_t *qlge, uint32_t index, uint32_t mask, int enable)
{
	int status;
	uint32_t value = 0;

	QL_PRINT(DBG_INIT,
	    ("%s %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s mask %s the routing reg.\n",
	    (enable ? "Adding" : "Removing"),
	    ((index == RT_IDX_ALL_ERR_SLOT) ? "MAC ERROR/ALL ERROR" : ""),
	    ((index == RT_IDX_IP_CSUM_ERR_SLOT) ? "IP CSUM ERROR" : ""),
	    ((index ==
	    RT_IDX_TCP_UDP_CSUM_ERR_SLOT) ? "TCP/UDP CSUM ERROR" : ""),
	    ((index == RT_IDX_BCAST_SLOT) ? "BROADCAST" : ""),
	    ((index == RT_IDX_MCAST_MATCH_SLOT) ? "MULTICAST MATCH" : ""),
	    ((index == RT_IDX_ALLMULTI_SLOT) ? "ALL MULTICAST MATCH" : ""),
	    ((index == RT_IDX_UNUSED6_SLOT) ? "UNUSED6" : ""),
	    ((index == RT_IDX_UNUSED7_SLOT) ? "UNUSED7" : ""),
	    ((index == RT_IDX_RSS_MATCH_SLOT) ? "RSS ALL/IPV4 MATCH" : ""),
	    ((index == RT_IDX_RSS_IPV6_SLOT) ? "RSS IPV6" : ""),
	    ((index == RT_IDX_RSS_TCP4_SLOT) ? "RSS TCP4" : ""),
	    ((index == RT_IDX_RSS_TCP6_SLOT) ? "RSS TCP6" : ""),
	    ((index == RT_IDX_CAM_HIT_SLOT) ? "CAM HIT" : ""),
	    ((index == RT_IDX_UNUSED013) ? "UNUSED13" : ""),
	    ((index == RT_IDX_UNUSED014) ? "UNUSED14" : ""),
	    ((index == RT_IDX_PROMISCUOUS_SLOT) ? "PROMISCUOUS" : ""),
	    (enable ? "to" : "from")));

	switch (mask) {
	case RT_IDX_CAM_HIT:
		value = RT_IDX_DST_CAM_Q | /* dest */
		    RT_IDX_TYPE_NICQ | /* type */
		    (RT_IDX_CAM_HIT_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_VALID: /* Promiscuous Mode frames. */
		value = RT_IDX_DST_DFLT_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_PROMISCUOUS_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_ERR:	/* Pass up MAC,IP,TCP/UDP error frames. */
		value = RT_IDX_DST_DFLT_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_ALL_ERR_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_BCAST:	/* Pass up Broadcast frames to default Q. */
		value = RT_IDX_DST_DFLT_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_BCAST_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_MCAST:	/* Pass up All Multicast frames. */
		value = RT_IDX_DST_CAM_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_ALLMULTI_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_MCAST_MATCH:	/* Pass up matched Multicast frames. */
		value = RT_IDX_DST_CAM_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_MCAST_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case RT_IDX_RSS_MATCH:	/* Pass up matched RSS frames. */
		value = RT_IDX_DST_RSS |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (RT_IDX_RSS_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */
		break;

	case 0:	/* Clear the E-bit on an entry. */
		value = RT_IDX_DST_DFLT_Q |	/* dest */
		    RT_IDX_TYPE_NICQ |	/* type */
		    (index << RT_IDX_IDX_SHIFT); /* index */
		break;

	default:
		cmn_err(CE_WARN, "Mask type %d not yet supported.",
		    mask);
		status = -EPERM;
		goto exit;
	}

	if (value != 0) {
		status = ql_wait_reg_rdy(qlge, REG_ROUTING_INDEX, RT_IDX_MW, 0);
		if (status)
			goto exit;
		value |= (enable ? RT_IDX_E : 0);
		ql_write_reg(qlge, REG_ROUTING_INDEX, value);
		ql_write_reg(qlge, REG_ROUTING_DATA, enable ? mask : 0);
	}

exit:
	return (status);
}

/*
 * Clear all the entries in the routing table.
 * Caller must get semaphore in advance.
 */

static int
ql_stop_routing(qlge_t *qlge)
{
	int status = 0;
	int i;
	/* Clear all the entries in the routing table. */
	for (i = 0; i < 16; i++) {
		status = ql_set_routing_reg(qlge, i, 0, 0);
		if (status) {
			cmn_err(CE_WARN, "Stop routing failed. ");
		}
	}
	return (status);
}

/* Initialize the frame-to-queue routing. */
static int
ql_route_initialize(qlge_t *qlge)
{
	int status = 0;

	status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
	if (status != DDI_SUCCESS)
		return (status);

	/* Clear all the entries in the routing table. */
	status = ql_stop_routing(qlge);
	if (status) {
		goto exit;
	}
	status = ql_set_routing_reg(qlge, RT_IDX_BCAST_SLOT, RT_IDX_BCAST, 1);
	if (status) {
		cmn_err(CE_WARN,
		    "Failed to init routing register for broadcast packets.");
		goto exit;
	}
	/*
	 * If we have more than one inbound queue, then turn on RSS in the
	 * routing block.
	 */
	if (qlge->rss_ring_count > 1) {
		status = ql_set_routing_reg(qlge, RT_IDX_RSS_MATCH_SLOT,
		    RT_IDX_RSS_MATCH, 1);
		if (status) {
			cmn_err(CE_WARN,
			    "Failed to init routing register for MATCH RSS "
			    "packets.");
			goto exit;
		}
	}

	status = ql_set_routing_reg(qlge, RT_IDX_CAM_HIT_SLOT,
	    RT_IDX_CAM_HIT, 1);
	if (status) {
		cmn_err(CE_WARN,
		    "Failed to init routing register for CAM packets.");
		goto exit;
	}

	status = ql_set_routing_reg(qlge, RT_IDX_MCAST_MATCH_SLOT,
	    RT_IDX_MCAST_MATCH, 1);
	if (status) {
		cmn_err(CE_WARN,
		    "Failed to init routing register for Multicast "
		    "packets.");
	}

exit:
	ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
	return (status);
}

/*
 * Initialize hardware
 */
static int
ql_device_initialize(qlge_t *qlge)
{
	uint32_t value, mask, required_max_frame_size;
	int i;
	int status = 0;
	uint16_t pause = PAUSE_MODE_DISABLED;
	boolean_t update_port_config = B_FALSE;
	/*
	 * Set up the System register to halt on errors.
	 */
	value = SYS_EFE | SYS_FAE;
	mask = value << 16;
	ql_write_reg(qlge, REG_SYSTEM, mask | value);

	/* Set the default queue. */
	value = NIC_RCV_CFG_DFQ;
	mask = NIC_RCV_CFG_DFQ_MASK;

	ql_write_reg(qlge, REG_NIC_RECEIVE_CONFIGURATION, mask | value);

	/* Enable the MPI interrupt. */
	ql_write_reg(qlge, REG_INTERRUPT_MASK, (INTR_MASK_PI << 16)
	    | INTR_MASK_PI);
	/* Enable the function, set pagesize, enable error checking. */
	value = FSC_FE | FSC_EPC_INBOUND | FSC_EPC_OUTBOUND |
	    FSC_EC | FSC_VM_PAGE_4K | FSC_DBRST_1024;
	/* Set/clear header splitting. */
	if (CFG_IST(qlge, CFG_ENABLE_SPLIT_HEADER)) {
		value |= FSC_SH;
		ql_write_reg(qlge, REG_SPLIT_HEADER, SMALL_BUFFER_SIZE);
	}
	mask = FSC_VM_PAGESIZE_MASK |
	    FSC_DBL_MASK | FSC_DBRST_MASK | (value << 16);
	ql_write_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL, mask | value);
	/*
	 * check current port max frame size, if different from OS setting,
	 * then we need to change
	 */
	required_max_frame_size =
	    (qlge->mtu == ETHERMTU)? NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE;

	if (ql_get_port_cfg(qlge) == DDI_SUCCESS) {
		/* if correct frame size but different from required size */
		if (qlge->port_cfg_info.max_frame_size !=
		    required_max_frame_size) {
			QL_PRINT(DBG_MBX,
			    ("update frame size, current %d, new %d\n",
			    qlge->port_cfg_info.max_frame_size,
			    required_max_frame_size));
			qlge->port_cfg_info.max_frame_size =
			    required_max_frame_size;
			update_port_config = B_TRUE;
		}
		if (qlge->port_cfg_info.link_cfg & STD_PAUSE)
			pause = PAUSE_MODE_STANDARD;
		else if (qlge->port_cfg_info.link_cfg & PP_PAUSE)
			pause = PAUSE_MODE_PER_PRIORITY;
		if (pause != qlge->pause) {
			update_port_config = B_TRUE;
		}
		/*
		 * Always update port config for now to work around
		 * a hardware bug
		 */
		update_port_config = B_TRUE;

		/* if need to update port configuration */
		if (update_port_config)
			(void) ql_set_port_cfg(qlge);
	} else
		cmn_err(CE_WARN, "ql_get_port_cfg failed");

	/* Start up the rx queues. */
	for (i = 0; i < qlge->rx_ring_count; i++) {
		status = ql_start_rx_ring(qlge, &qlge->rx_ring[i]);
		if (status) {
			cmn_err(CE_WARN,
			    "Failed to start rx ring[%d]", i);
			return (status);
		}
	}

	/*
	 * If there is more than one inbound completion queue
	 * then download a RICB to configure RSS.
	 */
	if (qlge->rss_ring_count > 1) {
		status = ql_start_rss(qlge);
		if (status) {
			cmn_err(CE_WARN, "Failed to start RSS.");
			return (status);
		}
	}

	/* Start up the tx queues. */
	for (i = 0; i < qlge->tx_ring_count; i++) {
		status = ql_start_tx_ring(qlge, &qlge->tx_ring[i]);
		if (status) {
			cmn_err(CE_WARN,
			    "Failed to start tx ring[%d]", i);
			return (status);
		}
	}
	qlge->selected_tx_ring = 0;
	/* Set the frame routing filter. */
	status = ql_route_initialize(qlge);
	if (status) {
		cmn_err(CE_WARN,
		    "Failed to init CAM/Routing tables.");
		return (status);
	}

	return (status);
}

/*
 * Issue soft reset to chip.
 */
static int
ql_asic_reset(qlge_t *qlge)
{
	uint32_t value;
	int max_wait_time = 3;
	int status = DDI_SUCCESS;

	ql_write_reg(qlge, REG_RESET_FAILOVER, FUNCTION_RESET_MASK
	    |FUNCTION_RESET);

	max_wait_time = 3;
	do {
		value =  ql_read_reg(qlge, REG_RESET_FAILOVER);
		if ((value & FUNCTION_RESET) == 0)
			break;
		qlge_delay(QL_ONE_SEC_DELAY);
	} while ((--max_wait_time));

	if (max_wait_time == 0) {
		cmn_err(CE_WARN,
		    "TIMEOUT!!! errored out of resetting the chip!");
		status = DDI_FAILURE;
	}

	return (status);
}

/*
 * If there are more than MIN_BUFFERS_ARM_COUNT small buffer descriptors in
 * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list
 * to be used by hardware.
 */
static void
ql_arm_sbuf(qlge_t *qlge, struct rx_ring *rx_ring)
{
	struct bq_desc *sbq_desc;
	int i;
	uint64_t *sbq_entry = rx_ring->sbq_dma.vaddr;
	uint32_t arm_count;

	if (rx_ring->sbuf_free_count > rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT)
		arm_count = (rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT);
	else {
		/* Adjust to a multiple of 16 */
		arm_count = (rx_ring->sbuf_free_count / 16) * 16;
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_NOTE, "adjust sbuf arm_count %d\n", arm_count);
#endif
	}
	for (i = 0; i < arm_count; i++) {
		sbq_desc = ql_get_sbuf_from_free_list(rx_ring);
		if (sbq_desc == NULL)
			break;
		/* Arm asic */
		*sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr);
		sbq_entry++;

		/* link the descriptors to in_use_list */
		ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc);
		rx_ring->sbq_prod_idx++;
	}
	ql_update_sbq_prod_idx(qlge, rx_ring);
}

/*
 * If there are more than MIN_BUFFERS_ARM_COUNT large buffer descriptors in
 * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list
 * to be used by hardware.
 */
static void
ql_arm_lbuf(qlge_t *qlge, struct rx_ring *rx_ring)
{
	struct bq_desc *lbq_desc;
	int i;
	uint64_t *lbq_entry = rx_ring->lbq_dma.vaddr;
	uint32_t arm_count;

	if (rx_ring->lbuf_free_count > rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT)
		arm_count = (rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT);
	else {
		/* Adjust to a multiple of 16 */
		arm_count = (rx_ring->lbuf_free_count / 16) * 16;
#ifdef QLGE_LOAD_UNLOAD
		cmn_err(CE_NOTE, "adjust lbuf arm_count %d\n", arm_count);
#endif
	}
	for (i = 0; i < arm_count; i++) {
		lbq_desc = ql_get_lbuf_from_free_list(rx_ring);
		if (lbq_desc == NULL)
			break;
		/* Arm asic */
		*lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr);
		lbq_entry++;

		/* link the descriptors to in_use_list */
		ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc);
		rx_ring->lbq_prod_idx++;
	}
	ql_update_lbq_prod_idx(qlge, rx_ring);
}


/*
 * Initializes the adapter by configuring request and response queues,
 * allocates and ARMs small and large receive buffers to the
 * hardware
 */
static int
ql_bringup_adapter(qlge_t *qlge)
{
	int i;

	if (ql_device_initialize(qlge) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "?%s(%d): ql_device_initialize failed",
		    __func__, qlge->instance);
		goto err_bringup;
	}
	qlge->sequence |= INIT_ADAPTER_UP;

#ifdef QLGE_TRACK_BUFFER_USAGE
	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (qlge->rx_ring[i].type != TX_Q) {
			qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS;
			qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS;
		}
		qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES;
	}
#endif
	/* Arm buffers */
	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (qlge->rx_ring[i].type != TX_Q) {
			ql_arm_sbuf(qlge, &qlge->rx_ring[i]);
			ql_arm_lbuf(qlge, &qlge->rx_ring[i]);
		}
	}

	/* Enable work/request queues */
	for (i = 0; i < qlge->tx_ring_count; i++) {
		if (qlge->tx_ring[i].valid_db_reg)
			ql_write_doorbell_reg(qlge,
			    qlge->tx_ring[i].valid_db_reg,
			    REQ_Q_VALID);
	}

	/* Enable completion queues */
	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (qlge->rx_ring[i].valid_db_reg)
			ql_write_doorbell_reg(qlge,
			    qlge->rx_ring[i].valid_db_reg,
			    RSP_Q_VALID);
	}

	for (i = 0; i < qlge->tx_ring_count; i++) {
		mutex_enter(&qlge->tx_ring[i].tx_lock);
		qlge->tx_ring[i].mac_flags = QL_MAC_STARTED;
		mutex_exit(&qlge->tx_ring[i].tx_lock);
	}

	for (i = 0; i < qlge->rx_ring_count; i++) {
		mutex_enter(&qlge->rx_ring[i].rx_lock);
		qlge->rx_ring[i].mac_flags = QL_MAC_STARTED;
		mutex_exit(&qlge->rx_ring[i].rx_lock);
	}

	/* This mutex will get re-acquired in enable_completion interrupt */
	mutex_exit(&qlge->hw_mutex);
	/* Traffic can start flowing now */
	ql_enable_all_completion_interrupts(qlge);
	mutex_enter(&qlge->hw_mutex);

	ql_enable_global_interrupt(qlge);

	qlge->sequence |= ADAPTER_INIT;
	return (DDI_SUCCESS);

err_bringup:
	(void) ql_asic_reset(qlge);
	return (DDI_FAILURE);
}

/*
 * Initialize mutexes of each rx/tx rings
 */
static int
ql_init_rx_tx_locks(qlge_t *qlge)
{
	struct tx_ring *tx_ring;
	struct rx_ring *rx_ring;
	int i;

	for (i = 0; i < qlge->tx_ring_count; i++) {
		tx_ring = &qlge->tx_ring[i];
		mutex_init(&tx_ring->tx_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
	}

	for (i = 0; i < qlge->rx_ring_count; i++) {
		rx_ring = &qlge->rx_ring[i];
		mutex_init(&rx_ring->rx_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
		mutex_init(&rx_ring->sbq_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
		mutex_init(&rx_ring->lbq_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
	}

	return (DDI_SUCCESS);
}

/*
 * ql_attach - Driver attach.
 */
static int
ql_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int instance;
	qlge_t *qlge;
	int rval;
	uint16_t w;
	mac_register_t *macp = NULL;
	rval = DDI_FAILURE;

	/* first get the instance */
	instance = ddi_get_instance(dip);

	switch (cmd) {
	case DDI_ATTACH:
		/*
		 * Check that hardware is installed in a DMA-capable slot
		 */
		if (ddi_slaveonly(dip) == DDI_SUCCESS) {
			cmn_err(CE_WARN, "?%s(%d): Not installed in a "
			    "DMA-capable slot", ADAPTER_NAME, instance);
			break;
		}

		/*
		 * No support for high-level interrupts
		 */
		if (ddi_intr_hilevel(dip, 0) != 0) {
			cmn_err(CE_WARN, "?%s(%d): No support for high-level"
			    " intrs", ADAPTER_NAME, instance);
			break;
		}

		/*
		 * Allocate our per-device-instance structure
		 */

		qlge = (qlge_t *)kmem_zalloc(sizeof (*qlge), KM_SLEEP);
		ASSERT(qlge != NULL);

		qlge->sequence |= INIT_SOFTSTATE_ALLOC;

		qlge->dip = dip;
		qlge->instance = instance;

		/*
		 * Setup the ISP8x00 registers address mapping to be
		 * accessed by this particular driver.
		 * 0x0   Configuration Space
		 * 0x1   I/O Space
		 * 0x2   1st Memory Space address - Control Register Set
		 * 0x3   2nd Memory Space address - Doorbell Memory Space
		 */

		w = 2;
		if (ddi_regs_map_setup(dip, w, (caddr_t *)&qlge->iobase, 0,
		    sizeof (dev_reg_t), &ql_dev_acc_attr,
		    &qlge->dev_handle) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): Unable to map device "
			    "registers", ADAPTER_NAME, instance);
			ql_free_resources(dip, qlge);
			break;
		}

		QL_PRINT(DBG_GLD, ("ql_attach: I/O base = 0x%x\n",
		    qlge->iobase));

		qlge->sequence |= INIT_REGS_SETUP;

		/* map Doorbell memory space */
		w = 3;
		if (ddi_regs_map_setup(dip, w,
		    (caddr_t *)&qlge->doorbell_reg_iobase, 0,
		    0x100000 /* sizeof (dev_doorbell_reg_t) */,
		    &ql_dev_acc_attr,
		    &qlge->dev_doorbell_reg_handle) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): Unable to map Doorbell "
			    "registers",
			    ADAPTER_NAME, instance);
			ql_free_resources(dip, qlge);
			break;
		}

		QL_PRINT(DBG_GLD, ("ql_attach: Doorbell I/O base = 0x%x\n",
		    qlge->doorbell_reg_iobase));

		qlge->sequence |= INIT_DOORBELL_REGS_SETUP;

		/*
		 * Allocate a macinfo structure for this instance
		 */
		if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
			cmn_err(CE_WARN, "%s(%d): mac_alloc failed",
			    __func__, instance);
			ql_free_resources(dip, qlge);
			return (NULL);
		}
		/* save adapter status to dip private data */
		ddi_set_driver_private(dip, qlge);
		QL_PRINT(DBG_INIT, ("%s(%d): Allocate macinfo structure done\n",
		    ADAPTER_NAME, instance));

		qlge->sequence |= INIT_MAC_ALLOC;

		/*
		 * Attach this instance of the device
		 */
		/* Setup PCI Local Bus Configuration resource. */
		if (pci_config_setup(dip, &qlge->pci_handle) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d):Unable to get PCI resources",
			    ADAPTER_NAME, instance);
			ql_free_resources(dip, qlge);
			break;
		}

		qlge->sequence |= INIT_PCI_CONFIG_SETUP;

		if (ql_init_instance(qlge) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): Unable to initialize device "
			    "instance", ADAPTER_NAME, instance);
			ql_free_resources(dip, qlge);
			break;
		}

		/* Setup interrupt vectors */
		if (ql_alloc_irqs(qlge) != DDI_SUCCESS) {
			ql_free_resources(dip, qlge);
			break;
		}
		qlge->sequence |= INIT_INTR_ALLOC;

		/* Configure queues */
		if (ql_setup_rings(qlge) != DDI_SUCCESS) {
			ql_free_resources(dip, qlge);
			break;
		}

		qlge->sequence |= INIT_SETUP_RINGS;
		/*
		 * Map queues to interrupt vectors
		 */
		ql_resolve_queues_to_irqs(qlge);
		/*
		 * Add interrupt handlers
		 */
		if (ql_add_intr_handlers(qlge) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "Failed to add interrupt "
			    "handlers");
			ql_free_resources(dip, qlge);
			break;
		}

		qlge->sequence |= INIT_ADD_INTERRUPT;
		QL_PRINT(DBG_GLD, ("%s(%d): Add interrupt handler done\n",
		    ADAPTER_NAME, instance));

		/* Initialize mutex, need the interrupt priority */
		(void) ql_init_rx_tx_locks(qlge);

		qlge->sequence |= INIT_LOCKS_CREATED;

		/*
		 * Use a soft interrupt to do something that we do not want
		 * to do in regular network functions or with mutexs being held
		 */
		if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_event_intr_hdl,
		    DDI_INTR_SOFTPRI_MIN, ql_mpi_event_work, (caddr_t)qlge)
		    != DDI_SUCCESS) {
			ql_free_resources(dip, qlge);
			break;
		}

		if (ddi_intr_add_softint(qlge->dip, &qlge->asic_reset_intr_hdl,
		    DDI_INTR_SOFTPRI_MIN, ql_asic_reset_work, (caddr_t)qlge)
		    != DDI_SUCCESS) {
			ql_free_resources(dip, qlge);
			break;
		}

		if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_reset_intr_hdl,
		    DDI_INTR_SOFTPRI_MIN, ql_mpi_reset_work, (caddr_t)qlge)
		    != DDI_SUCCESS) {
			ql_free_resources(dip, qlge);
			break;
		}

		qlge->sequence |= INIT_ADD_SOFT_INTERRUPT;

		/*
		 * mutex to protect the adapter state structure.
		 * initialize mutexes according to the interrupt priority
		 */
		mutex_init(&qlge->gen_mutex, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
		mutex_init(&qlge->hw_mutex, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));
		mutex_init(&qlge->mbx_mutex, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(qlge->intr_pri));

		/* Mailbox wait and interrupt conditional variable. */
		cv_init(&qlge->cv_mbx_intr, NULL, CV_DRIVER, NULL);

		qlge->sequence |= INIT_MUTEX;

		/*
		 * KStats
		 */
		if (ql_init_kstats(qlge) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): KState initialization failed",
			    ADAPTER_NAME, instance);
			ql_free_resources(dip, qlge);
			break;
		}
		qlge->sequence |= INIT_KSTATS;

		/*
		 * Initialize gld macinfo structure
		 */
		ql_gld3_init(qlge, macp);

		if (mac_register(macp, &qlge->mh) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): mac_register failed",
			    __func__, instance);
			ql_free_resources(dip, qlge);
			break;
		}
		qlge->sequence |= INIT_MAC_REGISTERED;
		QL_PRINT(DBG_GLD, ("%s(%d): mac_register done\n",
		    ADAPTER_NAME, instance));

		mac_free(macp);
		macp = NULL;

		qlge->mac_flags = QL_MAC_ATTACHED;

		/*
		 * Allocate memory resources
		 */
		if (ql_alloc_mem_resources(qlge) != DDI_SUCCESS) {
			cmn_err(CE_WARN, "%s(%d): memory allocation failed",
			    __func__, qlge->instance);
			ql_free_mem_resources(qlge);
			ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED);
			return (DDI_FAILURE);
		}
		qlge->sequence |= INIT_MEMORY_ALLOC;

		ddi_report_dev(dip);

		rval = DDI_SUCCESS;
	break;
/*
 * DDI_RESUME
 * When called  with  cmd  set  to  DDI_RESUME,  attach()  must
 * restore  the hardware state of a device (power may have been
 * removed from the device), allow  pending  requests  to  con-
 * tinue,  and  service  new requests. In this case, the driver
 * must not  make  any  assumptions  about  the  state  of  the
 * hardware,  but  must  restore the state of the device except
 * for the power level of components.
 *
 */
	case DDI_RESUME:

		if ((qlge = (qlge_t *)QL_GET_DEV(dip)) == NULL)
			return (DDI_FAILURE);

		QL_PRINT(DBG_GLD, ("%s(%d)-DDI_RESUME\n",
		    __func__, qlge->instance));

		mutex_enter(&qlge->gen_mutex);
		rval = ql_do_start(qlge);
		mutex_exit(&qlge->gen_mutex);
		break;

	default:
		break;
	}
	return (rval);
}

/*
 * Unbind all pending tx dma handles during driver bring down
 */
static void
ql_unbind_pending_tx_dma_handle(struct tx_ring *tx_ring)
{
	struct tx_ring_desc *tx_ring_desc;
	int i, j;

	if (tx_ring->wq_desc) {
		tx_ring_desc = tx_ring->wq_desc;
		for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) {
			for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) {
				if (tx_ring_desc->tx_dma_handle[j]) {
					(void) ddi_dma_unbind_handle(
					    tx_ring_desc->tx_dma_handle[j]);
				}
			}
			tx_ring_desc->tx_dma_handle_used = 0;
		} /* end of for loop */
	}
}
/*
 * Wait for all the packets sent to the chip to finish transmission
 * to prevent buffers to be unmapped before or during a transmit operation
 */
static int
ql_wait_tx_quiesce(qlge_t *qlge)
{
	int count = MAX_TX_WAIT_COUNT, i;
	int rings_done;
	volatile struct tx_ring *tx_ring;
	uint32_t consumer_idx;
	uint32_t producer_idx;
	uint32_t temp;
	int done = 0;
	int rval = DDI_FAILURE;

	while (!done) {
		rings_done = 0;

		for (i = 0; i < qlge->tx_ring_count; i++) {
			tx_ring = &qlge->tx_ring[i];
			temp = ql_read_doorbell_reg(qlge,
			    tx_ring->prod_idx_db_reg);
			producer_idx = temp & 0x0000ffff;
			consumer_idx = (temp >> 16);

			/*
			 * Get the pending iocb count, ones which have not been
			 * pulled down by the chip
			 */
			if (producer_idx >= consumer_idx)
				temp = (producer_idx - consumer_idx);
			else
				temp = (tx_ring->wq_len - consumer_idx) +
				    producer_idx;

			if ((tx_ring->tx_free_count + temp) >= tx_ring->wq_len)
				rings_done++;
			else {
				done = 1;
				break;
			}
		}

		/* If all the rings are done */
		if (rings_done >= qlge->tx_ring_count) {
#ifdef QLGE_LOAD_UNLOAD
			cmn_err(CE_NOTE, "%s(%d) done successfully \n",
			    __func__, qlge->instance);
#endif
			rval = DDI_SUCCESS;
			break;
		}

		qlge_delay(100);

		count--;
		if (!count) {

			count = MAX_TX_WAIT_COUNT;
#ifdef QLGE_LOAD_UNLOAD
			volatile struct rx_ring *rx_ring;
			cmn_err(CE_NOTE, "%s(%d): Waiting for %d pending"
			    " Transmits on queue %d to complete .\n",
			    __func__, qlge->instance,
			    (qlge->tx_ring[i].wq_len -
			    qlge->tx_ring[i].tx_free_count),
			    i);

			rx_ring = &qlge->rx_ring[i+1];
			temp = ql_read_doorbell_reg(qlge,
			    rx_ring->cnsmr_idx_db_reg);
			consumer_idx = temp & 0x0000ffff;
			producer_idx = (temp >> 16);
			cmn_err(CE_NOTE, "%s(%d): Transmit completion queue %d,"
			    " Producer %d, Consumer %d\n",
			    __func__, qlge->instance,
			    i+1,
			    producer_idx, consumer_idx);

			temp = ql_read_doorbell_reg(qlge,
			    tx_ring->prod_idx_db_reg);
			producer_idx = temp & 0x0000ffff;
			consumer_idx = (temp >> 16);
			cmn_err(CE_NOTE, "%s(%d): Transmit request queue %d,"
			    " Producer %d, Consumer %d\n",
			    __func__, qlge->instance, i,
			    producer_idx, consumer_idx);
#endif

			/* For now move on */
			break;
		}
	}
	/* Stop the request queue */
	mutex_enter(&qlge->hw_mutex);
	for (i = 0; i < qlge->tx_ring_count; i++) {
		if (qlge->tx_ring[i].valid_db_reg) {
			ql_write_doorbell_reg(qlge,
			    qlge->tx_ring[i].valid_db_reg, 0);
		}
	}
	mutex_exit(&qlge->hw_mutex);
	return (rval);
}

/*
 * Wait for all the receives indicated to the stack to come back
 */
static int
ql_wait_rx_complete(qlge_t *qlge)
{
	int i;
	/* Disable all the completion queues */
	mutex_enter(&qlge->hw_mutex);
	for (i = 0; i < qlge->rx_ring_count; i++) {
		if (qlge->rx_ring[i].valid_db_reg) {
			ql_write_doorbell_reg(qlge,
			    qlge->rx_ring[i].valid_db_reg, 0);
		}
	}
	mutex_exit(&qlge->hw_mutex);

	/* Wait for OS to return all rx buffers */
	qlge_delay(QL_ONE_SEC_DELAY);
	return (DDI_SUCCESS);
}

/*
 * stop the driver
 */
static int
ql_bringdown_adapter(qlge_t *qlge)
{
	int i;
	int status = DDI_SUCCESS;

	qlge->mac_flags = QL_MAC_BRINGDOWN;
	if (qlge->sequence & ADAPTER_INIT) {
		/* stop forwarding external packets to driver */
		status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
		if (status)
			return (status);
		(void) ql_stop_routing(qlge);
		ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
		/*
		 * Set the flag for receive and transmit
		 * operations to cease
		 */
		for (i = 0; i < qlge->tx_ring_count; i++) {
			mutex_enter(&qlge->tx_ring[i].tx_lock);
			qlge->tx_ring[i].mac_flags = QL_MAC_STOPPED;
			mutex_exit(&qlge->tx_ring[i].tx_lock);
		}

		for (i = 0; i < qlge->rx_ring_count; i++) {
			mutex_enter(&qlge->rx_ring[i].rx_lock);
			qlge->rx_ring[i].mac_flags = QL_MAC_STOPPED;
			mutex_exit(&qlge->rx_ring[i].rx_lock);
		}

		/*
		 * Need interrupts to be running while the transmit
		 * completions are cleared. Wait for the packets
		 * queued to the chip to be sent out
		 */
		(void) ql_wait_tx_quiesce(qlge);
		/* Interrupts not needed from now */
		ql_disable_all_completion_interrupts(qlge);

		mutex_enter(&qlge->hw_mutex);
		/* Disable Global interrupt */
		ql_disable_global_interrupt(qlge);
		mutex_exit(&qlge->hw_mutex);

		/* Wait for all the indicated packets to come back */
		status = ql_wait_rx_complete(qlge);

		mutex_enter(&qlge->hw_mutex);
		/* Reset adapter */
		(void) ql_asic_reset(qlge);
		/*
		 * Unbind all tx dma handles to prevent pending tx descriptors'
		 * dma handles from being re-used.
		 */
		for (i = 0; i < qlge->tx_ring_count; i++) {
			ql_unbind_pending_tx_dma_handle(&qlge->tx_ring[i]);
		}

		qlge->sequence &= ~ADAPTER_INIT;

		mutex_exit(&qlge->hw_mutex);
	}
	return (status);
}

/*
 * ql_detach
 * Used to remove all the states associated with a given
 * instances of a device node prior to the removal of that
 * instance from the system.
 */
static int
ql_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	qlge_t *qlge;
	int rval;

	rval = DDI_SUCCESS;

	switch (cmd) {
	case DDI_DETACH:

		if ((qlge = QL_GET_DEV(dip)) == NULL)
			return (DDI_FAILURE);
		rval = ql_bringdown_adapter(qlge);
		if (rval != DDI_SUCCESS)
			break;

		qlge->mac_flags = QL_MAC_DETACH;

		/* free memory resources */
		if (qlge->sequence & INIT_MEMORY_ALLOC) {
			ql_free_mem_resources(qlge);
			qlge->sequence &= ~INIT_MEMORY_ALLOC;
		}
		ql_free_resources(dip, qlge);

		break;

	case DDI_SUSPEND:
		if ((qlge = QL_GET_DEV(dip)) == NULL)
			return (DDI_FAILURE);

		mutex_enter(&qlge->gen_mutex);
		if ((qlge->mac_flags == QL_MAC_ATTACHED) ||
		    (qlge->mac_flags == QL_MAC_STARTED)) {
			(void) ql_do_stop(qlge);
		}
		qlge->mac_flags = QL_MAC_SUSPENDED;
		mutex_exit(&qlge->gen_mutex);

		break;
	default:
		rval = DDI_FAILURE;
		break;
	}

	return (rval);
}

/*
 * quiesce(9E) entry point.
 *
 * This function is called when the system is single-threaded at high
 * PIL with preemption disabled. Therefore, this function must not be
 * blocked.
 *
 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 */
int
ql_quiesce(dev_info_t *dip)
{
	qlge_t *qlge;
	int i;

	if ((qlge = QL_GET_DEV(dip)) == NULL)
		return (DDI_FAILURE);

	if (CFG_IST(qlge, CFG_CHIP_8100)) {
		/* stop forwarding external packets to driver */
		(void) ql_sem_spinlock(qlge, SEM_RT_IDX_MASK);
		(void) ql_stop_routing(qlge);
		ql_sem_unlock(qlge, SEM_RT_IDX_MASK);
		/* Stop all the request queues */
		for (i = 0; i < qlge->tx_ring_count; i++) {
			if (qlge->tx_ring[i].valid_db_reg) {
				ql_write_doorbell_reg(qlge,
				    qlge->tx_ring[i].valid_db_reg, 0);
			}
		}
		qlge_delay(QL_ONE_SEC_DELAY/4);
		/* Interrupts not needed from now */
		/* Disable MPI interrupt */
		ql_write_reg(qlge, REG_INTERRUPT_MASK,
		    (INTR_MASK_PI << 16));
		ql_disable_global_interrupt(qlge);

		/* Disable all the rx completion queues */
		for (i = 0; i < qlge->rx_ring_count; i++) {
			if (qlge->rx_ring[i].valid_db_reg) {
				ql_write_doorbell_reg(qlge,
				    qlge->rx_ring[i].valid_db_reg, 0);
			}
		}
		qlge_delay(QL_ONE_SEC_DELAY/4);
		qlge->mac_flags = QL_MAC_STOPPED;
		/* Reset adapter */
		(void) ql_asic_reset(qlge);
		qlge_delay(100);
	}

	return (DDI_SUCCESS);
}

QL_STREAM_OPS(ql_ops, ql_attach, ql_detach);

/*
 * Loadable Driver Interface Structures.
 * Declare and initialize the module configuration section...
 */
static struct modldrv modldrv = {
	&mod_driverops,		/* type of module: driver */
	version,		/* name of module */
	&ql_ops			/* driver dev_ops */
};

static struct modlinkage modlinkage = {
	MODREV_1, 	&modldrv,	NULL
};

/*
 * Loadable Module Routines
 */

/*
 * _init
 * Initializes a loadable module. It is called before any other
 * routine in a loadable module.
 */
int
_init(void)
{
	int rval;

	mac_init_ops(&ql_ops, ADAPTER_NAME);
	rval = mod_install(&modlinkage);
	if (rval != DDI_SUCCESS) {
		mac_fini_ops(&ql_ops);
		cmn_err(CE_WARN, "?Unable to install/attach driver '%s'",
		    ADAPTER_NAME);
	}

	return (rval);
}

/*
 * _fini
 * Prepares a module for unloading. It is called when the system
 * wants to unload a module. If the module determines that it can
 * be unloaded, then _fini() returns the value returned by
 * mod_remove(). Upon successful return from _fini() no other
 * routine in the module will be called before _init() is called.
 */
int
_fini(void)
{
	int rval;

	rval = mod_remove(&modlinkage);
	if (rval == DDI_SUCCESS) {
		mac_fini_ops(&ql_ops);
	}

	return (rval);
}

/*
 * _info
 * Returns information about loadable module.
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}