/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 QLogic Corporation. All rights reserved. */ /* * Copyright (c) 2018, Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include /* * Local variables */ static struct ether_addr ql_ether_broadcast_addr = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; static char version[] = "GLDv3 QLogic 81XX " VERSIONSTR; /* * Local function prototypes */ static void ql_free_resources(qlge_t *); static void ql_fini_kstats(qlge_t *); static uint32_t ql_get_link_state(qlge_t *); static void ql_read_conf(qlge_t *); static int ql_alloc_phys(dev_info_t *, ddi_dma_handle_t *, ddi_device_acc_attr_t *, uint_t, ddi_acc_handle_t *, size_t, size_t, caddr_t *, ddi_dma_cookie_t *); static int ql_alloc_phys_rbuf(dev_info_t *, ddi_dma_handle_t *, ddi_device_acc_attr_t *, uint_t, ddi_acc_handle_t *, size_t, size_t, caddr_t *, ddi_dma_cookie_t *); static void ql_free_phys(ddi_dma_handle_t *, ddi_acc_handle_t *); static int ql_set_routing_reg(qlge_t *, uint32_t, uint32_t, int); static int ql_attach(dev_info_t *, ddi_attach_cmd_t); static int ql_detach(dev_info_t *, ddi_detach_cmd_t); static int ql_bringdown_adapter(qlge_t *); static int ql_bringup_adapter(qlge_t *); static int ql_asic_reset(qlge_t *); static void ql_wake_mpi_reset_soft_intr(qlge_t *); static void ql_stop_timer(qlge_t *qlge); static void ql_fm_fini(qlge_t *qlge); int ql_clean_outbound_rx_ring(struct rx_ring *rx_ring); /* * TX dma maping handlers allow multiple sscatter-gather lists */ ddi_dma_attr_t tx_mapping_dma_attr = { DMA_ATTR_V0, /* dma_attr_version */ QL_DMA_LOW_ADDRESS, /* low DMA address range */ QL_DMA_HIGH_64BIT_ADDRESS, /* high DMA address range */ QL_DMA_XFER_COUNTER, /* DMA counter register */ QL_DMA_ADDRESS_ALIGNMENT, /* DMA address alignment, default - 8 */ QL_DMA_BURSTSIZES, /* DMA burstsizes */ QL_DMA_MIN_XFER_SIZE, /* min effective DMA size */ QL_DMA_MAX_XFER_SIZE, /* max DMA xfer size */ QL_DMA_SEGMENT_BOUNDARY, /* segment boundary */ QL_MAX_TX_DMA_HANDLES, /* s/g list length */ QL_DMA_GRANULARITY, /* granularity of device */ DDI_DMA_RELAXED_ORDERING /* DMA transfer flags */ }; /* * Receive buffers and Request/Response queues do not allow scatter-gather lists */ ddi_dma_attr_t dma_attr = { DMA_ATTR_V0, /* dma_attr_version */ QL_DMA_LOW_ADDRESS, /* low DMA address range */ QL_DMA_HIGH_64BIT_ADDRESS, /* high DMA address range */ QL_DMA_XFER_COUNTER, /* DMA counter register */ QL_DMA_ADDRESS_ALIGNMENT, /* DMA address alignment, default - 8 */ QL_DMA_BURSTSIZES, /* DMA burstsizes */ QL_DMA_MIN_XFER_SIZE, /* min effective DMA size */ QL_DMA_MAX_XFER_SIZE, /* max DMA xfer size */ QL_DMA_SEGMENT_BOUNDARY, /* segment boundary */ 1, /* s/g list length, i.e no sg list */ QL_DMA_GRANULARITY, /* granularity of device */ QL_DMA_XFER_FLAGS /* DMA transfer flags */ }; /* * Receive buffers do not allow scatter-gather lists */ ddi_dma_attr_t dma_attr_rbuf = { DMA_ATTR_V0, /* dma_attr_version */ QL_DMA_LOW_ADDRESS, /* low DMA address range */ QL_DMA_HIGH_64BIT_ADDRESS, /* high DMA address range */ QL_DMA_XFER_COUNTER, /* DMA counter register */ 0x1, /* DMA address alignment, default - 8 */ QL_DMA_BURSTSIZES, /* DMA burstsizes */ QL_DMA_MIN_XFER_SIZE, /* min effective DMA size */ QL_DMA_MAX_XFER_SIZE, /* max DMA xfer size */ QL_DMA_SEGMENT_BOUNDARY, /* segment boundary */ 1, /* s/g list length, i.e no sg list */ QL_DMA_GRANULARITY, /* granularity of device */ DDI_DMA_RELAXED_ORDERING /* DMA transfer flags */ }; /* * DMA access attribute structure. */ /* device register access from host */ ddi_device_acc_attr_t ql_dev_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_STRUCTURE_LE_ACC, DDI_STRICTORDER_ACC }; /* host ring descriptors */ ddi_device_acc_attr_t ql_desc_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; /* host ring buffer */ ddi_device_acc_attr_t ql_buf_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; /* * Hash key table for Receive Side Scaling (RSS) support */ const uint8_t key_data[] = { 0x23, 0x64, 0xa1, 0xaa, 0x37, 0xc0, 0xed, 0x05, 0x2b, 0x36, 0x50, 0x5c, 0x45, 0x1e, 0x7e, 0xc8, 0x5d, 0x2a, 0x54, 0x2f, 0xe4, 0x3d, 0x0f, 0xbb, 0x91, 0xd9, 0x25, 0x60, 0xd4, 0xf8, 0x12, 0xa0, 0x59, 0x4b, 0x9e, 0x8a, 0x51, 0xda, 0xcd, 0x49}; /* * Shadow Registers: * Outbound queues have a consumer index that is maintained by the chip. * Inbound queues have a producer index that is maintained by the chip. * For lower overhead, these registers are "shadowed" to host memory * which allows the device driver to track the queue progress without * PCI reads. When an entry is placed on an inbound queue, the chip will * update the relevant index register and then copy the value to the * shadow register in host memory. * Currently, ql_read_sh_reg only read Inbound queues'producer index. */ static inline unsigned int ql_read_sh_reg(qlge_t *qlge, struct rx_ring *rx_ring) { uint32_t rtn; /* re-synchronize shadow prod index dma buffer before reading */ (void) ddi_dma_sync(qlge->host_copy_shadow_dma_attr.dma_handle, rx_ring->prod_idx_sh_reg_offset, sizeof (uint32_t), DDI_DMA_SYNC_FORKERNEL); rtn = ddi_get32(qlge->host_copy_shadow_dma_attr.acc_handle, (uint32_t *)rx_ring->prod_idx_sh_reg); return (rtn); } /* * Read 32 bit atomically */ uint32_t ql_atomic_read_32(volatile uint32_t *target) { /* * atomic_add_32_nv returns the new value after the add, * we are adding 0 so we should get the original value */ return (atomic_add_32_nv(target, 0)); } /* * Set 32 bit atomically */ void ql_atomic_set_32(volatile uint32_t *target, uint32_t newval) { (void) atomic_swap_32(target, newval); } /* * Setup device PCI configuration registers. * Kernel context. */ static void ql_pci_config(qlge_t *qlge) { uint16_t w; qlge->vendor_id = (uint16_t)pci_config_get16(qlge->pci_handle, PCI_CONF_VENID); qlge->device_id = (uint16_t)pci_config_get16(qlge->pci_handle, PCI_CONF_DEVID); /* * we want to respect framework's setting of PCI * configuration space command register and also * want to make sure that all bits of interest to us * are properly set in PCI Command register(0x04). * PCI_COMM_IO 0x1 I/O access enable * PCI_COMM_MAE 0x2 Memory access enable * PCI_COMM_ME 0x4 bus master enable * PCI_COMM_MEMWR_INVAL 0x10 memory write and invalidate enable. */ w = (uint16_t)pci_config_get16(qlge->pci_handle, PCI_CONF_COMM); w = (uint16_t)(w & (~PCI_COMM_IO)); w = (uint16_t)(w | PCI_COMM_MAE | PCI_COMM_ME | /* PCI_COMM_MEMWR_INVAL | */ PCI_COMM_PARITY_DETECT | PCI_COMM_SERR_ENABLE); pci_config_put16(qlge->pci_handle, PCI_CONF_COMM, w); w = pci_config_get16(qlge->pci_handle, 0x54); w = (uint16_t)(w & (~0x7000)); w = (uint16_t)(w | 0x5000); pci_config_put16(qlge->pci_handle, 0x54, w); ql_dump_pci_config(qlge); } /* * This routine parforms the neccessary steps to set GLD mac information * such as Function number, xgmac mask and shift bits */ static int ql_set_mac_info(qlge_t *qlge) { uint32_t value; int rval = DDI_FAILURE; uint32_t fn0_net, fn1_net; /* set default value */ qlge->fn0_net = FN0_NET; qlge->fn1_net = FN1_NET; if (ql_read_processor_data(qlge, MPI_REG, &value) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d) read MPI register failed", __func__, qlge->instance); goto exit; } else { fn0_net = (value >> 1) & 0x07; fn1_net = (value >> 5) & 0x07; if ((fn0_net > 4) || (fn1_net > 4) || (fn0_net == fn1_net)) { cmn_err(CE_WARN, "%s(%d) bad mpi register value %x, \n" "nic0 function number %d," "nic1 function number %d " "use default\n", __func__, qlge->instance, value, fn0_net, fn1_net); goto exit; } else { qlge->fn0_net = fn0_net; qlge->fn1_net = fn1_net; } } /* Get the function number that the driver is associated with */ value = ql_read_reg(qlge, REG_STATUS); qlge->func_number = (uint8_t)((value >> 6) & 0x03); QL_PRINT(DBG_INIT, ("status register is:%x, func_number: %d\n", value, qlge->func_number)); /* The driver is loaded on a non-NIC function? */ if ((qlge->func_number != qlge->fn0_net) && (qlge->func_number != qlge->fn1_net)) { cmn_err(CE_WARN, "Invalid function number = 0x%x\n", qlge->func_number); goto exit; } /* network port 0? */ if (qlge->func_number == qlge->fn0_net) { qlge->xgmac_sem_mask = QL_PORT0_XGMAC_SEM_MASK; qlge->xgmac_sem_bits = QL_PORT0_XGMAC_SEM_BITS; } else { qlge->xgmac_sem_mask = QL_PORT1_XGMAC_SEM_MASK; qlge->xgmac_sem_bits = QL_PORT1_XGMAC_SEM_BITS; } rval = DDI_SUCCESS; exit: return (rval); } /* * write to doorbell register */ void ql_write_doorbell_reg(qlge_t *qlge, uint32_t *addr, uint32_t data) { ddi_put32(qlge->dev_doorbell_reg_handle, addr, data); } /* * read from doorbell register */ uint32_t ql_read_doorbell_reg(qlge_t *qlge, uint32_t *addr) { uint32_t ret; ret = ddi_get32(qlge->dev_doorbell_reg_handle, addr); return (ret); } /* * This function waits for a specific bit to come ready * in a given register. It is used mostly by the initialize * process, but is also used in kernel thread API such as * netdev->set_multi, netdev->set_mac_address, netdev->vlan_rx_add_vid. */ static int ql_wait_reg_rdy(qlge_t *qlge, uint32_t reg, uint32_t bit, uint32_t err_bit) { uint32_t temp; int count = UDELAY_COUNT; while (count) { temp = ql_read_reg(qlge, reg); /* check for errors */ if ((temp & err_bit) != 0) { break; } else if ((temp & bit) != 0) return (DDI_SUCCESS); qlge_delay(UDELAY_DELAY); count--; } cmn_err(CE_WARN, "Waiting for reg %x to come ready failed.", reg); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_NO_RESPONSE); atomic_or_32(&qlge->flags, ADAPTER_ERROR); } return (DDI_FAILURE); } /* * The CFG register is used to download TX and RX control blocks * to the chip. This function waits for an operation to complete. */ static int ql_wait_cfg(qlge_t *qlge, uint32_t bit) { return (ql_wait_reg_bit(qlge, REG_CONFIGURATION, bit, BIT_RESET, 0)); } /* * Used to issue init control blocks to hw. Maps control block, * sets address, triggers download, waits for completion. */ static int ql_write_cfg(qlge_t *qlge, uint32_t bit, uint64_t phy_addr, uint16_t q_id) { int status = DDI_SUCCESS; uint32_t mask; uint32_t value; status = ql_sem_spinlock(qlge, SEM_ICB_MASK); if (status != DDI_SUCCESS) { goto exit; } status = ql_wait_cfg(qlge, bit); if (status != DDI_SUCCESS) { goto exit; } ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_LOWER, LS_64BITS(phy_addr)); ql_write_reg(qlge, REG_ICB_ACCESS_ADDRESS_UPPER, MS_64BITS(phy_addr)); mask = CFG_Q_MASK | (bit << 16); value = bit | (q_id << CFG_Q_SHIFT); ql_write_reg(qlge, REG_CONFIGURATION, (mask | value)); /* * Wait for the bit to clear after signaling hw. */ status = ql_wait_cfg(qlge, bit); ql_sem_unlock(qlge, SEM_ICB_MASK); /* does flush too */ exit: return (status); } /* * Initialize adapter instance */ static int ql_init_instance(qlge_t *qlge) { int i; /* Default value */ qlge->mac_flags = QL_MAC_INIT; qlge->mtu = ETHERMTU; /* set normal size as default */ qlge->page_size = VM_PAGE_SIZE; /* default page size */ for (i = 0; i < MAX_RX_RINGS; i++) { qlge->rx_polls[i] = 0; qlge->rx_interrupts[i] = 0; } /* * Set up the operating parameters. */ qlge->multicast_list_count = 0; /* * Set up the max number of unicast list */ qlge->unicst_total = MAX_UNICAST_LIST_SIZE; qlge->unicst_avail = MAX_UNICAST_LIST_SIZE; /* * read user defined properties in .conf file */ ql_read_conf(qlge); /* mtu, pause, LSO etc */ qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count; QL_PRINT(DBG_INIT, ("mtu is %d \n", qlge->mtu)); /* choose Memory Space mapping and get Vendor Id, Device ID etc */ ql_pci_config(qlge); qlge->ip_hdr_offset = 0; if (qlge->device_id == 0x8000) { /* Schultz card */ qlge->cfg_flags |= CFG_CHIP_8100; /* enable just ipv4 chksum offload for Schultz */ qlge->cfg_flags |= CFG_CKSUM_FULL_IPv4; /* * Schultz firmware does not do pseduo IP header checksum * calculation, needed to be done by driver */ qlge->cfg_flags |= CFG_HW_UNABLE_PSEUDO_HDR_CKSUM; if (qlge->lso_enable) qlge->cfg_flags |= CFG_LSO; qlge->cfg_flags |= CFG_SUPPORT_SCATTER_GATHER; /* Schultz must split packet header */ qlge->cfg_flags |= CFG_ENABLE_SPLIT_HEADER; qlge->max_read_mbx = 5; qlge->ip_hdr_offset = 2; } /* Set Function Number and some of the iocb mac information */ if (ql_set_mac_info(qlge) != DDI_SUCCESS) return (DDI_FAILURE); /* Read network settings from NVRAM */ /* After nvram is read successfully, update dev_addr */ if (ql_get_flash_params(qlge) == DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("mac%d address is \n", qlge->func_number)); for (i = 0; i < ETHERADDRL; i++) { qlge->dev_addr.ether_addr_octet[i] = qlge->nic_config.factory_MAC[i]; } } else { cmn_err(CE_WARN, "%s(%d): Failed to read flash memory", __func__, qlge->instance); return (DDI_FAILURE); } bcopy(qlge->dev_addr.ether_addr_octet, qlge->unicst_addr[0].addr.ether_addr_octet, ETHERADDRL); QL_DUMP(DBG_INIT, "\t flash mac address dump:\n", &qlge->dev_addr.ether_addr_octet[0], 8, ETHERADDRL); qlge->port_link_state = LS_DOWN; return (DDI_SUCCESS); } /* * This hardware semaphore provides the mechanism for exclusive access to * resources shared between the NIC driver, MPI firmware, * FCOE firmware and the FC driver. */ static int ql_sem_trylock(qlge_t *qlge, uint32_t sem_mask) { uint32_t sem_bits = 0; switch (sem_mask) { case SEM_XGMAC0_MASK: sem_bits = SEM_SET << SEM_XGMAC0_SHIFT; break; case SEM_XGMAC1_MASK: sem_bits = SEM_SET << SEM_XGMAC1_SHIFT; break; case SEM_ICB_MASK: sem_bits = SEM_SET << SEM_ICB_SHIFT; break; case SEM_MAC_ADDR_MASK: sem_bits = SEM_SET << SEM_MAC_ADDR_SHIFT; break; case SEM_FLASH_MASK: sem_bits = SEM_SET << SEM_FLASH_SHIFT; break; case SEM_PROBE_MASK: sem_bits = SEM_SET << SEM_PROBE_SHIFT; break; case SEM_RT_IDX_MASK: sem_bits = SEM_SET << SEM_RT_IDX_SHIFT; break; case SEM_PROC_REG_MASK: sem_bits = SEM_SET << SEM_PROC_REG_SHIFT; break; default: cmn_err(CE_WARN, "Bad Semaphore mask!."); return (DDI_FAILURE); } ql_write_reg(qlge, REG_SEMAPHORE, sem_bits | sem_mask); return (!(ql_read_reg(qlge, REG_SEMAPHORE) & sem_bits)); } /* * Lock a specific bit of Semaphore register to gain * access to a particular shared register */ int ql_sem_spinlock(qlge_t *qlge, uint32_t sem_mask) { unsigned int wait_count = 30; while (wait_count) { if (!ql_sem_trylock(qlge, sem_mask)) return (DDI_SUCCESS); qlge_delay(100); wait_count--; } cmn_err(CE_WARN, "%s(%d) sem_mask 0x%x lock timeout ", __func__, qlge->instance, sem_mask); return (DDI_FAILURE); } /* * Unock a specific bit of Semaphore register to release * access to a particular shared register */ void ql_sem_unlock(qlge_t *qlge, uint32_t sem_mask) { ql_write_reg(qlge, REG_SEMAPHORE, sem_mask); (void) ql_read_reg(qlge, REG_SEMAPHORE); /* flush */ } /* * Get property value from configuration file. * * string = property string pointer. * * Returns: * 0xFFFFFFFF = no property else property value. */ static uint32_t ql_get_prop(qlge_t *qlge, char *string) { char buf[256]; uint32_t data; /* Get adapter instance parameter. */ (void) sprintf(buf, "hba%d-%s", qlge->instance, string); data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0, buf, (int)0xffffffff); /* Adapter instance parameter found? */ if (data == 0xffffffff) { /* No, get default parameter. */ data = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, qlge->dip, 0, string, (int)0xffffffff); } return (data); } /* * Read user setting from configuration file. */ static void ql_read_conf(qlge_t *qlge) { uint32_t data; /* clear configuration flags */ qlge->cfg_flags = 0; /* Set up the default ring sizes. */ qlge->tx_ring_size = NUM_TX_RING_ENTRIES; data = ql_get_prop(qlge, "tx_ring_size"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->tx_ring_size != data) { qlge->tx_ring_size = (uint16_t)data; } } qlge->rx_ring_size = NUM_RX_RING_ENTRIES; data = ql_get_prop(qlge, "rx_ring_size"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->rx_ring_size != data) { qlge->rx_ring_size = (uint16_t)data; } } qlge->tx_ring_count = 8; data = ql_get_prop(qlge, "tx_ring_count"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->tx_ring_count != data) { qlge->tx_ring_count = (uint16_t)data; } } qlge->rss_ring_count = 8; data = ql_get_prop(qlge, "rss_ring_count"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->rss_ring_count != data) { qlge->rss_ring_count = (uint16_t)data; } } /* Get default rx_copy enable/disable. */ if ((data = ql_get_prop(qlge, "force-rx-copy")) == 0xffffffff || data == 0) { qlge->rx_copy = B_FALSE; QL_PRINT(DBG_INIT, ("rx copy mode disabled\n")); } else if (data == 1) { qlge->rx_copy = B_TRUE; QL_PRINT(DBG_INIT, ("rx copy mode enabled\n")); } qlge->rx_copy_threshold = qlge->rx_ring_size / 4; data = ql_get_prop(qlge, "rx_copy_threshold"); if ((data != 0xffffffff) && (data != 0)) { qlge->rx_copy_threshold = data; cmn_err(CE_NOTE, "!new rx_copy_threshold %d \n", qlge->rx_copy_threshold); } /* Get mtu packet size. */ data = ql_get_prop(qlge, "mtu"); if ((data == ETHERMTU) || (data == JUMBO_MTU)) { if (qlge->mtu != data) { qlge->mtu = data; cmn_err(CE_NOTE, "new mtu is %d\n", qlge->mtu); } } if (qlge->mtu == JUMBO_MTU) { qlge->rx_coalesce_usecs = DFLT_RX_COALESCE_WAIT_JUMBO; qlge->tx_coalesce_usecs = DFLT_TX_COALESCE_WAIT_JUMBO; qlge->rx_max_coalesced_frames = DFLT_RX_INTER_FRAME_WAIT_JUMBO; qlge->tx_max_coalesced_frames = DFLT_TX_INTER_FRAME_WAIT_JUMBO; } /* Get pause mode, default is Per Priority mode. */ qlge->pause = PAUSE_MODE_PER_PRIORITY; data = ql_get_prop(qlge, "pause"); if (data <= PAUSE_MODE_PER_PRIORITY) { if (qlge->pause != data) { qlge->pause = data; cmn_err(CE_NOTE, "new pause mode %d\n", qlge->pause); } } /* Receive interrupt delay */ qlge->rx_coalesce_usecs = DFLT_RX_COALESCE_WAIT; data = ql_get_prop(qlge, "rx_intr_delay"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->rx_coalesce_usecs != data) { qlge->rx_coalesce_usecs = (uint16_t)data; } } /* Rx inter-packet delay. */ qlge->rx_max_coalesced_frames = DFLT_RX_INTER_FRAME_WAIT; data = ql_get_prop(qlge, "rx_ipkt_delay"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->rx_max_coalesced_frames != data) { qlge->rx_max_coalesced_frames = (uint16_t)data; } } /* Transmit interrupt delay */ qlge->tx_coalesce_usecs = DFLT_TX_COALESCE_WAIT; data = ql_get_prop(qlge, "tx_intr_delay"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->tx_coalesce_usecs != data) { qlge->tx_coalesce_usecs = (uint16_t)data; } } /* Tx inter-packet delay. */ qlge->tx_max_coalesced_frames = DFLT_TX_INTER_FRAME_WAIT; data = ql_get_prop(qlge, "tx_ipkt_delay"); /* if data is valid */ if ((data != 0xffffffff) && data) { if (qlge->tx_max_coalesced_frames != data) { qlge->tx_max_coalesced_frames = (uint16_t)data; } } /* Get split header payload_copy_thresh. */ qlge->payload_copy_thresh = DFLT_PAYLOAD_COPY_THRESH; data = ql_get_prop(qlge, "payload_copy_thresh"); /* if data is valid */ if ((data != 0xffffffff) && (data != 0)) { if (qlge->payload_copy_thresh != data) { qlge->payload_copy_thresh = data; } } /* large send offload (LSO) capability. */ qlge->lso_enable = 1; data = ql_get_prop(qlge, "lso_enable"); /* if data is valid */ if ((data == 0) || (data == 1)) { if (qlge->lso_enable != data) { qlge->lso_enable = (uint16_t)data; } } /* dcbx capability. */ qlge->dcbx_enable = 1; data = ql_get_prop(qlge, "dcbx_enable"); /* if data is valid */ if ((data == 0) || (data == 1)) { if (qlge->dcbx_enable != data) { qlge->dcbx_enable = (uint16_t)data; } } /* fault management enable */ qlge->fm_enable = B_TRUE; data = ql_get_prop(qlge, "fm-enable"); if ((data == 0x1) || (data == 0)) { qlge->fm_enable = (boolean_t)data; } } /* * Enable global interrupt */ static void ql_enable_global_interrupt(qlge_t *qlge) { ql_write_reg(qlge, REG_INTERRUPT_ENABLE, (INTR_EN_EI << 16) | INTR_EN_EI); qlge->flags |= INTERRUPTS_ENABLED; } /* * Disable global interrupt */ static void ql_disable_global_interrupt(qlge_t *qlge) { ql_write_reg(qlge, REG_INTERRUPT_ENABLE, (INTR_EN_EI << 16)); qlge->flags &= ~INTERRUPTS_ENABLED; } /* * Enable one ring interrupt */ void ql_enable_completion_interrupt(qlge_t *qlge, uint32_t intr) { struct intr_ctx *ctx = qlge->intr_ctx + intr; QL_PRINT(DBG_INTR, ("%s(%d): To enable intr %d, irq_cnt %d \n", __func__, qlge->instance, intr, ctx->irq_cnt)); if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) { /* * Always enable if we're MSIX multi interrupts and * it's not the default (zeroeth) interrupt. */ ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask); return; } if (!atomic_dec_32_nv(&ctx->irq_cnt)) { mutex_enter(&qlge->hw_mutex); ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_en_mask); mutex_exit(&qlge->hw_mutex); QL_PRINT(DBG_INTR, ("%s(%d): write %x to intr enable register \n", __func__, qlge->instance, ctx->intr_en_mask)); } } /* * ql_forced_disable_completion_interrupt * Used by call from OS, may be called without * a pending interrupt so force the disable */ uint32_t ql_forced_disable_completion_interrupt(qlge_t *qlge, uint32_t intr) { uint32_t var = 0; struct intr_ctx *ctx = qlge->intr_ctx + intr; QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n", __func__, qlge->instance, intr, ctx->irq_cnt)); if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && intr) { ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask); var = ql_read_reg(qlge, REG_STATUS); return (var); } mutex_enter(&qlge->hw_mutex); ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask); var = ql_read_reg(qlge, REG_STATUS); mutex_exit(&qlge->hw_mutex); return (var); } /* * Disable a completion interrupt */ void ql_disable_completion_interrupt(qlge_t *qlge, uint32_t intr) { struct intr_ctx *ctx; ctx = qlge->intr_ctx + intr; QL_PRINT(DBG_INTR, ("%s(%d): To disable intr %d, irq_cnt %d \n", __func__, qlge->instance, intr, ctx->irq_cnt)); /* * HW disables for us if we're MSIX multi interrupts and * it's not the default (zeroeth) interrupt. */ if ((qlge->intr_type == DDI_INTR_TYPE_MSIX) && (intr != 0)) return; if (ql_atomic_read_32(&ctx->irq_cnt) == 0) { mutex_enter(&qlge->hw_mutex); ql_write_reg(qlge, REG_INTERRUPT_ENABLE, ctx->intr_dis_mask); mutex_exit(&qlge->hw_mutex); } atomic_inc_32(&ctx->irq_cnt); } /* * Enable all completion interrupts */ static void ql_enable_all_completion_interrupts(qlge_t *qlge) { int i; uint32_t value = 1; for (i = 0; i < qlge->intr_cnt; i++) { /* * Set the count to 1 for Legacy / MSI interrupts or for the * default interrupt (0) */ if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0) { ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value); } ql_enable_completion_interrupt(qlge, i); } } /* * Disable all completion interrupts */ static void ql_disable_all_completion_interrupts(qlge_t *qlge) { int i; uint32_t value = 0; for (i = 0; i < qlge->intr_cnt; i++) { /* * Set the count to 0 for Legacy / MSI interrupts or for the * default interrupt (0) */ if ((qlge->intr_type != DDI_INTR_TYPE_MSIX) || i == 0) ql_atomic_set_32(&qlge->intr_ctx[i].irq_cnt, value); ql_disable_completion_interrupt(qlge, i); } } /* * Update small buffer queue producer index */ static void ql_update_sbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring) { /* Update the buffer producer index */ QL_PRINT(DBG_RX, ("sbq: updating prod idx = %d.\n", rx_ring->sbq_prod_idx)); ql_write_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg, rx_ring->sbq_prod_idx); } /* * Update large buffer queue producer index */ static void ql_update_lbq_prod_idx(qlge_t *qlge, struct rx_ring *rx_ring) { /* Update the buffer producer index */ QL_PRINT(DBG_RX, ("lbq: updating prod idx = %d.\n", rx_ring->lbq_prod_idx)); ql_write_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg, rx_ring->lbq_prod_idx); } /* * Adds a small buffer descriptor to end of its in use list, * assumes sbq_lock is already taken */ static void ql_add_sbuf_to_in_use_list(struct rx_ring *rx_ring, struct bq_desc *sbq_desc) { uint32_t inuse_idx = rx_ring->sbq_use_tail; rx_ring->sbuf_in_use[inuse_idx] = sbq_desc; inuse_idx++; if (inuse_idx >= rx_ring->sbq_len) inuse_idx = 0; rx_ring->sbq_use_tail = inuse_idx; atomic_inc_32(&rx_ring->sbuf_in_use_count); ASSERT(rx_ring->sbuf_in_use_count <= rx_ring->sbq_len); } /* * Get a small buffer descriptor from its in use list */ static struct bq_desc * ql_get_sbuf_from_in_use_list(struct rx_ring *rx_ring) { struct bq_desc *sbq_desc = NULL; uint32_t inuse_idx; /* Pick from head of in use list */ inuse_idx = rx_ring->sbq_use_head; sbq_desc = rx_ring->sbuf_in_use[inuse_idx]; rx_ring->sbuf_in_use[inuse_idx] = NULL; if (sbq_desc != NULL) { inuse_idx++; if (inuse_idx >= rx_ring->sbq_len) inuse_idx = 0; rx_ring->sbq_use_head = inuse_idx; atomic_dec_32(&rx_ring->sbuf_in_use_count); atomic_inc_32(&rx_ring->rx_indicate); sbq_desc->upl_inuse = 1; /* if mp is NULL */ if (sbq_desc->mp == NULL) { /* try to remap mp again */ sbq_desc->mp = desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr), rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle); } } return (sbq_desc); } /* * Add a small buffer descriptor to its free list */ static void ql_add_sbuf_to_free_list(struct rx_ring *rx_ring, struct bq_desc *sbq_desc) { uint32_t free_idx; /* Add to the end of free list */ free_idx = rx_ring->sbq_free_tail; rx_ring->sbuf_free[free_idx] = sbq_desc; ASSERT(rx_ring->sbuf_free_count <= rx_ring->sbq_len); free_idx++; if (free_idx >= rx_ring->sbq_len) free_idx = 0; rx_ring->sbq_free_tail = free_idx; atomic_inc_32(&rx_ring->sbuf_free_count); } /* * Get a small buffer descriptor from its free list */ static struct bq_desc * ql_get_sbuf_from_free_list(struct rx_ring *rx_ring) { struct bq_desc *sbq_desc; uint32_t free_idx; free_idx = rx_ring->sbq_free_head; /* Pick from top of free list */ sbq_desc = rx_ring->sbuf_free[free_idx]; rx_ring->sbuf_free[free_idx] = NULL; if (sbq_desc != NULL) { free_idx++; if (free_idx >= rx_ring->sbq_len) free_idx = 0; rx_ring->sbq_free_head = free_idx; atomic_dec_32(&rx_ring->sbuf_free_count); } return (sbq_desc); } /* * Add a large buffer descriptor to its in use list */ static void ql_add_lbuf_to_in_use_list(struct rx_ring *rx_ring, struct bq_desc *lbq_desc) { uint32_t inuse_idx; inuse_idx = rx_ring->lbq_use_tail; rx_ring->lbuf_in_use[inuse_idx] = lbq_desc; inuse_idx++; if (inuse_idx >= rx_ring->lbq_len) inuse_idx = 0; rx_ring->lbq_use_tail = inuse_idx; atomic_inc_32(&rx_ring->lbuf_in_use_count); } /* * Get a large buffer descriptor from in use list */ static struct bq_desc * ql_get_lbuf_from_in_use_list(struct rx_ring *rx_ring) { struct bq_desc *lbq_desc; uint32_t inuse_idx; /* Pick from head of in use list */ inuse_idx = rx_ring->lbq_use_head; lbq_desc = rx_ring->lbuf_in_use[inuse_idx]; rx_ring->lbuf_in_use[inuse_idx] = NULL; if (lbq_desc != NULL) { inuse_idx++; if (inuse_idx >= rx_ring->lbq_len) inuse_idx = 0; rx_ring->lbq_use_head = inuse_idx; atomic_dec_32(&rx_ring->lbuf_in_use_count); atomic_inc_32(&rx_ring->rx_indicate); lbq_desc->upl_inuse = 1; /* if mp is NULL */ if (lbq_desc->mp == NULL) { /* try to remap mp again */ lbq_desc->mp = desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr), rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle); } } return (lbq_desc); } /* * Add a large buffer descriptor to free list */ static void ql_add_lbuf_to_free_list(struct rx_ring *rx_ring, struct bq_desc *lbq_desc) { uint32_t free_idx; /* Add to the end of free list */ free_idx = rx_ring->lbq_free_tail; rx_ring->lbuf_free[free_idx] = lbq_desc; free_idx++; if (free_idx >= rx_ring->lbq_len) free_idx = 0; rx_ring->lbq_free_tail = free_idx; atomic_inc_32(&rx_ring->lbuf_free_count); ASSERT(rx_ring->lbuf_free_count <= rx_ring->lbq_len); } /* * Get a large buffer descriptor from its free list */ static struct bq_desc * ql_get_lbuf_from_free_list(struct rx_ring *rx_ring) { struct bq_desc *lbq_desc; uint32_t free_idx; free_idx = rx_ring->lbq_free_head; /* Pick from head of free list */ lbq_desc = rx_ring->lbuf_free[free_idx]; rx_ring->lbuf_free[free_idx] = NULL; if (lbq_desc != NULL) { free_idx++; if (free_idx >= rx_ring->lbq_len) free_idx = 0; rx_ring->lbq_free_head = free_idx; atomic_dec_32(&rx_ring->lbuf_free_count); } return (lbq_desc); } /* * Add a small buffer descriptor to free list */ static void ql_refill_sbuf_free_list(struct bq_desc *sbq_desc, boolean_t alloc_memory) { struct rx_ring *rx_ring = sbq_desc->rx_ring; uint64_t *sbq_entry; qlge_t *qlge = (qlge_t *)rx_ring->qlge; /* * Sync access */ mutex_enter(&rx_ring->sbq_lock); sbq_desc->upl_inuse = 0; /* * If we are freeing the buffers as a result of adapter unload, get out */ if ((sbq_desc->free_buf != 0) || (qlge->mac_flags == QL_MAC_DETACH)) { if (sbq_desc->free_buf == 0) atomic_dec_32(&rx_ring->rx_indicate); mutex_exit(&rx_ring->sbq_lock); return; } #ifdef QLGE_LOAD_UNLOAD if (rx_ring->rx_indicate == 0) cmn_err(CE_WARN, "sbq: indicate wrong"); #endif #ifdef QLGE_TRACK_BUFFER_USAGE uint32_t sb_consumer_idx; uint32_t sb_producer_idx; uint32_t num_free_buffers; uint32_t temp; temp = ql_read_doorbell_reg(qlge, rx_ring->sbq_prod_idx_db_reg); sb_producer_idx = temp & 0x0000ffff; sb_consumer_idx = (temp >> 16); if (sb_consumer_idx > sb_producer_idx) num_free_buffers = NUM_SMALL_BUFFERS - (sb_consumer_idx - sb_producer_idx); else num_free_buffers = sb_producer_idx - sb_consumer_idx; if (num_free_buffers < qlge->rx_sb_low_count[rx_ring->cq_id]) qlge->rx_sb_low_count[rx_ring->cq_id] = num_free_buffers; #endif #ifdef QLGE_LOAD_UNLOAD if (rx_ring->rx_indicate > 0xFF000000) cmn_err(CE_WARN, "sbq: indicate(%d) wrong: %d mac_flags %d," " sbq_desc index %d.", rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags, sbq_desc->index); #endif if (alloc_memory) { sbq_desc->mp = desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr), rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle); if (sbq_desc->mp == NULL) { rx_ring->rx_failed_sbq_allocs++; } } /* Got the packet from the stack decrement rx_indicate count */ atomic_dec_32(&rx_ring->rx_indicate); ql_add_sbuf_to_free_list(rx_ring, sbq_desc); /* Rearm if possible */ if ((rx_ring->sbuf_free_count >= MIN_BUFFERS_FREE_COUNT) && (qlge->mac_flags == QL_MAC_STARTED)) { sbq_entry = rx_ring->sbq_dma.vaddr; sbq_entry += rx_ring->sbq_prod_idx; while (rx_ring->sbuf_free_count > MIN_BUFFERS_ARM_COUNT) { /* Get first one from free list */ sbq_desc = ql_get_sbuf_from_free_list(rx_ring); *sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr); sbq_entry++; rx_ring->sbq_prod_idx++; if (rx_ring->sbq_prod_idx >= rx_ring->sbq_len) { rx_ring->sbq_prod_idx = 0; sbq_entry = rx_ring->sbq_dma.vaddr; } /* Add to end of in use list */ ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc); } /* Update small buffer queue producer index */ ql_update_sbq_prod_idx(qlge, rx_ring); } mutex_exit(&rx_ring->sbq_lock); QL_PRINT(DBG_RX_RING, ("%s(%d) exited, sbuf_free_count %d\n", __func__, qlge->instance, rx_ring->sbuf_free_count)); } /* * rx recycle call back function */ static void ql_release_to_sbuf_free_list(caddr_t p) { struct bq_desc *sbq_desc = (struct bq_desc *)(void *)p; if (sbq_desc == NULL) return; ql_refill_sbuf_free_list(sbq_desc, B_TRUE); } /* * Add a large buffer descriptor to free list */ static void ql_refill_lbuf_free_list(struct bq_desc *lbq_desc, boolean_t alloc_memory) { struct rx_ring *rx_ring = lbq_desc->rx_ring; uint64_t *lbq_entry; qlge_t *qlge = rx_ring->qlge; /* Sync access */ mutex_enter(&rx_ring->lbq_lock); lbq_desc->upl_inuse = 0; /* * If we are freeing the buffers as a result of adapter unload, get out */ if ((lbq_desc->free_buf != 0) || (qlge->mac_flags == QL_MAC_DETACH)) { if (lbq_desc->free_buf == 0) atomic_dec_32(&rx_ring->rx_indicate); mutex_exit(&rx_ring->lbq_lock); return; } #ifdef QLGE_LOAD_UNLOAD if (rx_ring->rx_indicate == 0) cmn_err(CE_WARN, "lbq: indicate wrong"); #endif #ifdef QLGE_TRACK_BUFFER_USAGE uint32_t lb_consumer_idx; uint32_t lb_producer_idx; uint32_t num_free_buffers; uint32_t temp; temp = ql_read_doorbell_reg(qlge, rx_ring->lbq_prod_idx_db_reg); lb_producer_idx = temp & 0x0000ffff; lb_consumer_idx = (temp >> 16); if (lb_consumer_idx > lb_producer_idx) num_free_buffers = NUM_LARGE_BUFFERS - (lb_consumer_idx - lb_producer_idx); else num_free_buffers = lb_producer_idx - lb_consumer_idx; if (num_free_buffers < qlge->rx_lb_low_count[rx_ring->cq_id]) { qlge->rx_lb_low_count[rx_ring->cq_id] = num_free_buffers; } #endif #ifdef QLGE_LOAD_UNLOAD if (rx_ring->rx_indicate > 0xFF000000) cmn_err(CE_WARN, "lbq: indicate(%d) wrong: %d mac_flags %d," "lbq_desc index %d", rx_ring->cq_id, rx_ring->rx_indicate, rx_ring->mac_flags, lbq_desc->index); #endif if (alloc_memory) { lbq_desc->mp = desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr), rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle); if (lbq_desc->mp == NULL) { rx_ring->rx_failed_lbq_allocs++; } } /* Got the packet from the stack decrement rx_indicate count */ atomic_dec_32(&rx_ring->rx_indicate); ql_add_lbuf_to_free_list(rx_ring, lbq_desc); /* Rearm if possible */ if ((rx_ring->lbuf_free_count >= MIN_BUFFERS_FREE_COUNT) && (qlge->mac_flags == QL_MAC_STARTED)) { lbq_entry = rx_ring->lbq_dma.vaddr; lbq_entry += rx_ring->lbq_prod_idx; while (rx_ring->lbuf_free_count > MIN_BUFFERS_ARM_COUNT) { /* Get first one from free list */ lbq_desc = ql_get_lbuf_from_free_list(rx_ring); *lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr); lbq_entry++; rx_ring->lbq_prod_idx++; if (rx_ring->lbq_prod_idx >= rx_ring->lbq_len) { rx_ring->lbq_prod_idx = 0; lbq_entry = rx_ring->lbq_dma.vaddr; } /* Add to end of in use list */ ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc); } /* Update large buffer queue producer index */ ql_update_lbq_prod_idx(rx_ring->qlge, rx_ring); } mutex_exit(&rx_ring->lbq_lock); QL_PRINT(DBG_RX_RING, ("%s exitd, lbuf_free_count %d\n", __func__, rx_ring->lbuf_free_count)); } /* * rx recycle call back function */ static void ql_release_to_lbuf_free_list(caddr_t p) { struct bq_desc *lbq_desc = (struct bq_desc *)(void *)p; if (lbq_desc == NULL) return; ql_refill_lbuf_free_list(lbq_desc, B_TRUE); } /* * free small buffer queue buffers */ static void ql_free_sbq_buffers(struct rx_ring *rx_ring) { struct bq_desc *sbq_desc; uint32_t i; uint32_t j = rx_ring->sbq_free_head; int force_cnt = 0; for (i = 0; i < rx_ring->sbuf_free_count; i++) { sbq_desc = rx_ring->sbuf_free[j]; sbq_desc->free_buf = 1; j++; if (j >= rx_ring->sbq_len) { j = 0; } if (sbq_desc->mp != NULL) { freemsg(sbq_desc->mp); sbq_desc->mp = NULL; } } rx_ring->sbuf_free_count = 0; j = rx_ring->sbq_use_head; for (i = 0; i < rx_ring->sbuf_in_use_count; i++) { sbq_desc = rx_ring->sbuf_in_use[j]; sbq_desc->free_buf = 1; j++; if (j >= rx_ring->sbq_len) { j = 0; } if (sbq_desc->mp != NULL) { freemsg(sbq_desc->mp); sbq_desc->mp = NULL; } } rx_ring->sbuf_in_use_count = 0; sbq_desc = &rx_ring->sbq_desc[0]; for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) { /* * Set flag so that the callback does not allocate a new buffer */ sbq_desc->free_buf = 1; if (sbq_desc->upl_inuse != 0) { force_cnt++; } if (sbq_desc->bd_dma.dma_handle != NULL) { ql_free_phys(&sbq_desc->bd_dma.dma_handle, &sbq_desc->bd_dma.acc_handle); sbq_desc->bd_dma.dma_handle = NULL; sbq_desc->bd_dma.acc_handle = NULL; } } #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "sbq: free %d inuse %d force %d\n", rx_ring->sbuf_free_count, rx_ring->sbuf_in_use_count, force_cnt); #endif if (rx_ring->sbuf_in_use != NULL) { kmem_free(rx_ring->sbuf_in_use, (rx_ring->sbq_len * sizeof (struct bq_desc *))); rx_ring->sbuf_in_use = NULL; } if (rx_ring->sbuf_free != NULL) { kmem_free(rx_ring->sbuf_free, (rx_ring->sbq_len * sizeof (struct bq_desc *))); rx_ring->sbuf_free = NULL; } } /* Allocate small buffers */ static int ql_alloc_sbufs(qlge_t *qlge, struct rx_ring *rx_ring) { struct bq_desc *sbq_desc; int i; ddi_dma_cookie_t dma_cookie; rx_ring->sbq_use_head = 0; rx_ring->sbq_use_tail = 0; rx_ring->sbuf_in_use_count = 0; rx_ring->sbq_free_head = 0; rx_ring->sbq_free_tail = 0; rx_ring->sbuf_free_count = 0; rx_ring->sbuf_free = kmem_zalloc(rx_ring->sbq_len * sizeof (struct bq_desc *), KM_NOSLEEP); if (rx_ring->sbuf_free == NULL) { cmn_err(CE_WARN, "!%s: sbuf_free_list alloc: failed", __func__); goto alloc_sbuf_err; } rx_ring->sbuf_in_use = kmem_zalloc(rx_ring->sbq_len * sizeof (struct bq_desc *), KM_NOSLEEP); if (rx_ring->sbuf_in_use == NULL) { cmn_err(CE_WARN, "!%s: sbuf_inuse_list alloc: failed", __func__); goto alloc_sbuf_err; } sbq_desc = &rx_ring->sbq_desc[0]; for (i = 0; i < rx_ring->sbq_len; i++, sbq_desc++) { /* Allocate buffer */ if (ql_alloc_phys_rbuf(qlge->dip, &sbq_desc->bd_dma.dma_handle, &ql_buf_acc_attr, DDI_DMA_READ | DDI_DMA_STREAMING, &sbq_desc->bd_dma.acc_handle, (size_t)rx_ring->sbq_buf_size, /* mem size */ (size_t)0, /* default alignment */ (caddr_t *)&sbq_desc->bd_dma.vaddr, &dma_cookie) != 0) { cmn_err(CE_WARN, "!%s: ddi_dma_alloc_handle: failed", __func__); goto alloc_sbuf_err; } /* Set context for Return buffer callback */ sbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress; sbq_desc->rx_recycle.free_func = ql_release_to_sbuf_free_list; sbq_desc->rx_recycle.free_arg = (caddr_t)sbq_desc; sbq_desc->rx_ring = rx_ring; sbq_desc->upl_inuse = 0; sbq_desc->free_buf = 0; sbq_desc->mp = desballoc((unsigned char *)(sbq_desc->bd_dma.vaddr), rx_ring->sbq_buf_size, 0, &sbq_desc->rx_recycle); if (sbq_desc->mp == NULL) { cmn_err(CE_WARN, "%s: desballoc() failed", __func__); goto alloc_sbuf_err; } ql_add_sbuf_to_free_list(rx_ring, sbq_desc); } return (DDI_SUCCESS); alloc_sbuf_err: ql_free_sbq_buffers(rx_ring); return (DDI_FAILURE); } static void ql_free_lbq_buffers(struct rx_ring *rx_ring) { struct bq_desc *lbq_desc; uint32_t i, j; int force_cnt = 0; j = rx_ring->lbq_free_head; for (i = 0; i < rx_ring->lbuf_free_count; i++) { lbq_desc = rx_ring->lbuf_free[j]; lbq_desc->free_buf = 1; j++; if (j >= rx_ring->lbq_len) j = 0; if (lbq_desc->mp != NULL) { freemsg(lbq_desc->mp); lbq_desc->mp = NULL; } } rx_ring->lbuf_free_count = 0; j = rx_ring->lbq_use_head; for (i = 0; i < rx_ring->lbuf_in_use_count; i++) { lbq_desc = rx_ring->lbuf_in_use[j]; lbq_desc->free_buf = 1; j++; if (j >= rx_ring->lbq_len) { j = 0; } if (lbq_desc->mp != NULL) { freemsg(lbq_desc->mp); lbq_desc->mp = NULL; } } rx_ring->lbuf_in_use_count = 0; lbq_desc = &rx_ring->lbq_desc[0]; for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) { /* Set flag so that callback will not allocate a new buffer */ lbq_desc->free_buf = 1; if (lbq_desc->upl_inuse != 0) { force_cnt++; } if (lbq_desc->bd_dma.dma_handle != NULL) { ql_free_phys(&lbq_desc->bd_dma.dma_handle, &lbq_desc->bd_dma.acc_handle); lbq_desc->bd_dma.dma_handle = NULL; lbq_desc->bd_dma.acc_handle = NULL; } } #ifdef QLGE_LOAD_UNLOAD if (force_cnt) { cmn_err(CE_WARN, "lbq: free %d inuse %d force %d", rx_ring->lbuf_free_count, rx_ring->lbuf_in_use_count, force_cnt); } #endif if (rx_ring->lbuf_in_use != NULL) { kmem_free(rx_ring->lbuf_in_use, (rx_ring->lbq_len * sizeof (struct bq_desc *))); rx_ring->lbuf_in_use = NULL; } if (rx_ring->lbuf_free != NULL) { kmem_free(rx_ring->lbuf_free, (rx_ring->lbq_len * sizeof (struct bq_desc *))); rx_ring->lbuf_free = NULL; } } /* Allocate large buffers */ static int ql_alloc_lbufs(qlge_t *qlge, struct rx_ring *rx_ring) { struct bq_desc *lbq_desc; ddi_dma_cookie_t dma_cookie; int i; uint32_t lbq_buf_size; rx_ring->lbq_use_head = 0; rx_ring->lbq_use_tail = 0; rx_ring->lbuf_in_use_count = 0; rx_ring->lbq_free_head = 0; rx_ring->lbq_free_tail = 0; rx_ring->lbuf_free_count = 0; rx_ring->lbuf_free = kmem_zalloc(rx_ring->lbq_len * sizeof (struct bq_desc *), KM_NOSLEEP); if (rx_ring->lbuf_free == NULL) { cmn_err(CE_WARN, "!%s: lbuf_free_list alloc: failed", __func__); goto alloc_lbuf_err; } rx_ring->lbuf_in_use = kmem_zalloc(rx_ring->lbq_len * sizeof (struct bq_desc *), KM_NOSLEEP); if (rx_ring->lbuf_in_use == NULL) { cmn_err(CE_WARN, "!%s: lbuf_inuse_list alloc: failed", __func__); goto alloc_lbuf_err; } lbq_buf_size = (qlge->mtu == ETHERMTU) ? LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE; lbq_desc = &rx_ring->lbq_desc[0]; for (i = 0; i < rx_ring->lbq_len; i++, lbq_desc++) { rx_ring->lbq_buf_size = lbq_buf_size; /* Allocate buffer */ if (ql_alloc_phys_rbuf(qlge->dip, &lbq_desc->bd_dma.dma_handle, &ql_buf_acc_attr, DDI_DMA_READ | DDI_DMA_STREAMING, &lbq_desc->bd_dma.acc_handle, (size_t)rx_ring->lbq_buf_size, /* mem size */ (size_t)0, /* default alignment */ (caddr_t *)&lbq_desc->bd_dma.vaddr, &dma_cookie) != 0) { cmn_err(CE_WARN, "!%s: ddi_dma_alloc_handle: failed", __func__); goto alloc_lbuf_err; } /* Set context for Return buffer callback */ lbq_desc->bd_dma.dma_addr = dma_cookie.dmac_laddress; lbq_desc->rx_recycle.free_func = ql_release_to_lbuf_free_list; lbq_desc->rx_recycle.free_arg = (caddr_t)lbq_desc; lbq_desc->rx_ring = rx_ring; lbq_desc->upl_inuse = 0; lbq_desc->free_buf = 0; lbq_desc->mp = desballoc((unsigned char *)(lbq_desc->bd_dma.vaddr), rx_ring->lbq_buf_size, 0, &lbq_desc->rx_recycle); if (lbq_desc->mp == NULL) { cmn_err(CE_WARN, "%s: desballoc() failed", __func__); goto alloc_lbuf_err; } ql_add_lbuf_to_free_list(rx_ring, lbq_desc); } /* For all large buffers */ return (DDI_SUCCESS); alloc_lbuf_err: ql_free_lbq_buffers(rx_ring); return (DDI_FAILURE); } /* * Free rx buffers */ static void ql_free_rx_buffers(qlge_t *qlge) { int i; struct rx_ring *rx_ring; for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (rx_ring->type != TX_Q) { ql_free_lbq_buffers(rx_ring); ql_free_sbq_buffers(rx_ring); } } } /* * Allocate rx buffers */ static int ql_alloc_rx_buffers(qlge_t *qlge) { struct rx_ring *rx_ring; int i; for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (rx_ring->type != TX_Q) { if (ql_alloc_sbufs(qlge, rx_ring) != DDI_SUCCESS) goto alloc_err; if (ql_alloc_lbufs(qlge, rx_ring) != DDI_SUCCESS) goto alloc_err; } } #ifdef QLGE_TRACK_BUFFER_USAGE for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].type == RX_Q) { qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS; qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS; } qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES; } #endif return (DDI_SUCCESS); alloc_err: ql_free_rx_buffers(qlge); return (DDI_FAILURE); } /* * Initialize large buffer queue ring */ static void ql_init_lbq_ring(struct rx_ring *rx_ring) { uint16_t i; struct bq_desc *lbq_desc; bzero(rx_ring->lbq_desc, rx_ring->lbq_len * sizeof (struct bq_desc)); for (i = 0; i < rx_ring->lbq_len; i++) { lbq_desc = &rx_ring->lbq_desc[i]; lbq_desc->index = i; } } /* * Initialize small buffer queue ring */ static void ql_init_sbq_ring(struct rx_ring *rx_ring) { uint16_t i; struct bq_desc *sbq_desc; bzero(rx_ring->sbq_desc, rx_ring->sbq_len * sizeof (struct bq_desc)); for (i = 0; i < rx_ring->sbq_len; i++) { sbq_desc = &rx_ring->sbq_desc[i]; sbq_desc->index = i; } } /* * Calculate the pseudo-header checksum if hardware can not do */ static void ql_pseudo_cksum(uint8_t *buf) { uint32_t cksum; uint16_t iphl; uint16_t proto; iphl = (uint16_t)(4 * (buf[0] & 0xF)); cksum = (((uint16_t)buf[2])<<8) + buf[3] - iphl; cksum += proto = buf[9]; cksum += (((uint16_t)buf[12])<<8) + buf[13]; cksum += (((uint16_t)buf[14])<<8) + buf[15]; cksum += (((uint16_t)buf[16])<<8) + buf[17]; cksum += (((uint16_t)buf[18])<<8) + buf[19]; cksum = (cksum>>16) + (cksum & 0xFFFF); cksum = (cksum>>16) + (cksum & 0xFFFF); /* * Point it to the TCP/UDP header, and * update the checksum field. */ buf += iphl + ((proto == IPPROTO_TCP) ? TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET); *(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum); } /* * Transmit an incoming packet. */ mblk_t * ql_ring_tx(void *arg, mblk_t *mp) { struct tx_ring *tx_ring = (struct tx_ring *)arg; qlge_t *qlge = tx_ring->qlge; mblk_t *next; int rval; uint32_t tx_count = 0; if (qlge->port_link_state == LS_DOWN) { /* can not send message while link is down */ mblk_t *tp; while (mp != NULL) { tp = mp->b_next; mp->b_next = NULL; freemsg(mp); mp = tp; } goto exit; } mutex_enter(&tx_ring->tx_lock); /* if mac is not started, driver is not ready, can not send */ if (tx_ring->mac_flags != QL_MAC_STARTED) { cmn_err(CE_WARN, "%s(%d)ring not started, mode %d " " return packets", __func__, qlge->instance, tx_ring->mac_flags); mutex_exit(&tx_ring->tx_lock); goto exit; } /* we must try to send all */ while (mp != NULL) { /* * if number of available slots is less than a threshold, * then quit */ if (tx_ring->tx_free_count <= TX_STOP_THRESHOLD) { tx_ring->queue_stopped = 1; rval = DDI_FAILURE; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "%s(%d) no resources", __func__, qlge->instance); #endif tx_ring->defer++; /* * If we return the buffer back we are expected to call * mac_tx_ring_update() when resources are available */ break; } next = mp->b_next; mp->b_next = NULL; rval = ql_send_common(tx_ring, mp); if (rval != DDI_SUCCESS) { mp->b_next = next; break; } tx_count++; mp = next; } /* * After all msg blocks are mapped or copied to tx buffer, * trigger the hardware to send! */ if (tx_count > 0) { ql_write_doorbell_reg(tx_ring->qlge, tx_ring->prod_idx_db_reg, tx_ring->prod_idx); } mutex_exit(&tx_ring->tx_lock); exit: return (mp); } /* * This function builds an mblk list for the given inbound * completion. */ static mblk_t * ql_build_rx_mp(qlge_t *qlge, struct rx_ring *rx_ring, struct ib_mac_iocb_rsp *ib_mac_rsp) { mblk_t *mp = NULL; mblk_t *mp1 = NULL; /* packet header */ mblk_t *mp2 = NULL; /* packet content */ struct bq_desc *lbq_desc; struct bq_desc *sbq_desc; uint32_t err_flag = (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK); uint32_t payload_len = le32_to_cpu(ib_mac_rsp->data_len); uint32_t header_len = le32_to_cpu(ib_mac_rsp->hdr_len); uint32_t pkt_len = payload_len + header_len; uint32_t done; uint64_t *curr_ial_ptr; uint32_t ial_data_addr_low; uint32_t actual_data_addr_low; mblk_t *mp_ial = NULL; /* ial chained packets */ uint32_t size; uint32_t cp_offset; boolean_t rx_copy = B_FALSE; mblk_t *tp = NULL; /* * Check if error flags are set */ if (err_flag != 0) { if ((err_flag & IB_MAC_IOCB_RSP_ERR_OVERSIZE) != 0) rx_ring->frame_too_long++; if ((err_flag & IB_MAC_IOCB_RSP_ERR_UNDERSIZE) != 0) rx_ring->frame_too_short++; if ((err_flag & IB_MAC_IOCB_RSP_ERR_CRC) != 0) rx_ring->fcs_err++; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "bad packet, type 0x%x", err_flag); #endif QL_DUMP(DBG_RX, "qlge_ring_rx: bad response iocb dump\n", (uint8_t *)ib_mac_rsp, 8, (size_t)sizeof (struct ib_mac_iocb_rsp)); } /* header should not be in large buffer */ if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HL) { cmn_err(CE_WARN, "header in large buffer or invalid!"); err_flag |= 1; } /* if whole packet is too big than rx buffer size */ if (pkt_len > qlge->max_frame_size) { cmn_err(CE_WARN, "ql_build_rx_mpframe too long(%d)!", pkt_len); err_flag |= 1; } if (qlge->rx_copy || (rx_ring->sbuf_in_use_count <= qlge->rx_copy_threshold) || (rx_ring->lbuf_in_use_count <= qlge->rx_copy_threshold)) { rx_copy = B_TRUE; } /* if using rx copy mode, we need to allocate a big enough buffer */ if (rx_copy) { qlge->stats.norcvbuf++; tp = allocb(payload_len + header_len + qlge->ip_hdr_offset, BPRI_MED); if (tp == NULL) { cmn_err(CE_WARN, "rx copy failed to allocate memory"); } else { tp->b_rptr += qlge->ip_hdr_offset; } } /* * Handle the header buffer if present. * packet header must be valid and saved in one small buffer * broadcast/multicast packets' headers not splitted */ if ((ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV) && (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) { QL_PRINT(DBG_RX, ("Header of %d bytes in small buffer.\n", header_len)); /* Sync access */ sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring); ASSERT(sbq_desc != NULL); /* * Validate addresses from the ASIC with the * expected sbuf address */ if (cpu_to_le64(sbq_desc->bd_dma.dma_addr) != ib_mac_rsp->hdr_addr) { /* Small buffer address mismatch */ cmn_err(CE_WARN, "%s(%d) ring%d packet saved" " in wrong small buffer", __func__, qlge->instance, rx_ring->cq_id); goto fatal_error; } /* get this packet */ mp1 = sbq_desc->mp; /* Flush DMA'd data */ (void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle, 0, header_len, DDI_DMA_SYNC_FORKERNEL); if ((err_flag != 0)|| (mp1 == NULL)) { /* failed on this packet, put it back for re-arming */ #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "get header from small buffer fail"); #endif ql_refill_sbuf_free_list(sbq_desc, B_FALSE); mp1 = NULL; } else if (rx_copy) { if (tp != NULL) { bcopy(sbq_desc->bd_dma.vaddr, tp->b_rptr, header_len); } ql_refill_sbuf_free_list(sbq_desc, B_FALSE); mp1 = NULL; } else { if ((qlge->ip_hdr_offset != 0)&& (header_len < SMALL_BUFFER_SIZE)) { /* * copy entire header to a 2 bytes boundary * address for 8100 adapters so that the IP * header can be on a 4 byte boundary address */ bcopy(mp1->b_rptr, (mp1->b_rptr + SMALL_BUFFER_SIZE + qlge->ip_hdr_offset), header_len); mp1->b_rptr += SMALL_BUFFER_SIZE + qlge->ip_hdr_offset; } /* * Adjust the mp payload_len to match * the packet header payload_len */ mp1->b_wptr = mp1->b_rptr + header_len; mp1->b_next = mp1->b_cont = NULL; QL_DUMP(DBG_RX, "\t RX packet header dump:\n", (uint8_t *)mp1->b_rptr, 8, header_len); } } /* * packet data or whole packet can be in small or one or * several large buffer(s) */ if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) { /* * The data is in a single small buffer. */ sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring); ASSERT(sbq_desc != NULL); QL_PRINT(DBG_RX, ("%d bytes in a single small buffer, sbq_desc = %p, " "sbq_desc->bd_dma.dma_addr = %x," " ib_mac_rsp->data_addr = %x, mp = %p\n", payload_len, sbq_desc, sbq_desc->bd_dma.dma_addr, ib_mac_rsp->data_addr, sbq_desc->mp)); /* * Validate addresses from the ASIC with the * expected sbuf address */ if (cpu_to_le64(sbq_desc->bd_dma.dma_addr) != ib_mac_rsp->data_addr) { /* Small buffer address mismatch */ cmn_err(CE_WARN, "%s(%d) ring%d packet saved" " in wrong small buffer", __func__, qlge->instance, rx_ring->cq_id); goto fatal_error; } /* get this packet */ mp2 = sbq_desc->mp; (void) ddi_dma_sync(sbq_desc->bd_dma.dma_handle, 0, payload_len, DDI_DMA_SYNC_FORKERNEL); if ((err_flag != 0) || (mp2 == NULL)) { #ifdef QLGE_LOAD_UNLOAD /* failed on this packet, put it back for re-arming */ cmn_err(CE_WARN, "ignore bad data from small buffer"); #endif ql_refill_sbuf_free_list(sbq_desc, B_FALSE); mp2 = NULL; } else if (rx_copy) { if (tp != NULL) { bcopy(sbq_desc->bd_dma.vaddr, tp->b_rptr + header_len, payload_len); tp->b_wptr = tp->b_rptr + header_len + payload_len; } ql_refill_sbuf_free_list(sbq_desc, B_FALSE); mp2 = NULL; } else { /* Adjust the buffer length to match the payload_len */ mp2->b_wptr = mp2->b_rptr + payload_len; mp2->b_next = mp2->b_cont = NULL; /* Flush DMA'd data */ QL_DUMP(DBG_RX, "\t RX packet payload dump:\n", (uint8_t *)mp2->b_rptr, 8, payload_len); /* * if payload is too small , copy to * the end of packet header */ if ((mp1 != NULL) && (payload_len <= qlge->payload_copy_thresh) && (pkt_len < (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) { bcopy(mp2->b_rptr, mp1->b_wptr, payload_len); mp1->b_wptr += payload_len; freemsg(mp2); mp2 = NULL; } } } else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) { /* * The data is in a single large buffer. */ lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring); QL_PRINT(DBG_RX, ("%d bytes in a single large buffer, lbq_desc = %p, " "lbq_desc->bd_dma.dma_addr = %x," " ib_mac_rsp->data_addr = %x, mp = %p\n", payload_len, lbq_desc, lbq_desc->bd_dma.dma_addr, ib_mac_rsp->data_addr, lbq_desc->mp)); ASSERT(lbq_desc != NULL); /* * Validate addresses from the ASIC with * the expected lbuf address */ if (cpu_to_le64(lbq_desc->bd_dma.dma_addr) != ib_mac_rsp->data_addr) { /* Large buffer address mismatch */ cmn_err(CE_WARN, "%s(%d) ring%d packet saved" " in wrong large buffer", __func__, qlge->instance, rx_ring->cq_id); goto fatal_error; } mp2 = lbq_desc->mp; /* Flush DMA'd data */ (void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle, 0, payload_len, DDI_DMA_SYNC_FORKERNEL); if ((err_flag != 0) || (mp2 == NULL)) { #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "ignore bad data from large buffer"); #endif /* failed on this packet, put it back for re-arming */ ql_refill_lbuf_free_list(lbq_desc, B_FALSE); mp2 = NULL; } else if (rx_copy) { if (tp != NULL) { bcopy(lbq_desc->bd_dma.vaddr, tp->b_rptr + header_len, payload_len); tp->b_wptr = tp->b_rptr + header_len + payload_len; } ql_refill_lbuf_free_list(lbq_desc, B_FALSE); mp2 = NULL; } else { /* * Adjust the buffer length to match * the packet payload_len */ mp2->b_wptr = mp2->b_rptr + payload_len; mp2->b_next = mp2->b_cont = NULL; QL_DUMP(DBG_RX, "\t RX packet payload dump:\n", (uint8_t *)mp2->b_rptr, 8, payload_len); /* * if payload is too small , copy to * the end of packet header */ if ((mp1 != NULL) && (payload_len <= qlge->payload_copy_thresh) && (pkt_len< (SMALL_BUFFER_SIZE - qlge->ip_hdr_offset))) { bcopy(mp2->b_rptr, mp1->b_wptr, payload_len); mp1->b_wptr += payload_len; freemsg(mp2); mp2 = NULL; } } } else if (payload_len) { /* ial case */ /* * payload available but not in sml nor lrg buffer, * so, it is saved in IAL */ #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "packet chained in IAL \n"); #endif /* lrg buf addresses are saved in one small buffer */ sbq_desc = ql_get_sbuf_from_in_use_list(rx_ring); curr_ial_ptr = (uint64_t *)sbq_desc->bd_dma.vaddr; done = 0; cp_offset = 0; while (!done) { ial_data_addr_low = (uint32_t)(le64_to_cpu(*curr_ial_ptr) & 0xFFFFFFFE); /* check if this is the last packet fragment */ done = (uint32_t)(le64_to_cpu(*curr_ial_ptr) & 1); curr_ial_ptr++; /* * The data is in one or several large buffer(s). */ lbq_desc = ql_get_lbuf_from_in_use_list(rx_ring); actual_data_addr_low = (uint32_t)(lbq_desc->bd_dma.dma_addr & 0xFFFFFFFE); if (ial_data_addr_low != actual_data_addr_low) { cmn_err(CE_WARN, "packet saved in wrong ial lrg buffer" " expected %x, actual %lx", ial_data_addr_low, (uintptr_t)lbq_desc->bd_dma.dma_addr); goto fatal_error; } size = (payload_len < rx_ring->lbq_buf_size)? payload_len : rx_ring->lbq_buf_size; payload_len -= size; mp2 = lbq_desc->mp; if ((err_flag != 0) || (mp2 == NULL)) { #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "ignore bad data from large buffer"); #endif ql_refill_lbuf_free_list(lbq_desc, B_FALSE); mp2 = NULL; } else if (rx_copy) { if (tp != NULL) { (void) ddi_dma_sync( lbq_desc->bd_dma.dma_handle, 0, size, DDI_DMA_SYNC_FORKERNEL); bcopy(lbq_desc->bd_dma.vaddr, tp->b_rptr + header_len + cp_offset, size); tp->b_wptr = tp->b_rptr + size + cp_offset + header_len; cp_offset += size; } ql_refill_lbuf_free_list(lbq_desc, B_FALSE); mp2 = NULL; } else { if (mp_ial == NULL) { mp_ial = mp2; } else { linkb(mp_ial, mp2); } mp2->b_next = NULL; mp2->b_cont = NULL; mp2->b_wptr = mp2->b_rptr + size; /* Flush DMA'd data */ (void) ddi_dma_sync(lbq_desc->bd_dma.dma_handle, 0, size, DDI_DMA_SYNC_FORKERNEL); QL_PRINT(DBG_RX, ("ial %d payload received \n", size)); QL_DUMP(DBG_RX, "\t Mac data dump:\n", (uint8_t *)mp2->b_rptr, 8, size); } } if (err_flag != 0) { #ifdef QLGE_LOAD_UNLOAD /* failed on this packet, put it back for re-arming */ cmn_err(CE_WARN, "ignore bad data from small buffer"); #endif ql_refill_sbuf_free_list(sbq_desc, B_FALSE); } else { mp2 = mp_ial; freemsg(sbq_desc->mp); } } /* * some packets' hdr not split, then send mp2 upstream, otherwise, * concatenate message block mp2 to the tail of message header, mp1 */ if (!err_flag) { if (rx_copy) { if (tp != NULL) { tp->b_next = NULL; tp->b_cont = NULL; tp->b_wptr = tp->b_rptr + header_len + payload_len; } mp = tp; } else { if (mp1) { if (mp2) { QL_PRINT(DBG_RX, ("packet in mp1 and mp2\n")); /* mp1->b_cont = mp2; */ linkb(mp1, mp2); mp = mp1; } else { QL_PRINT(DBG_RX, ("packet in mp1 only\n")); mp = mp1; } } else if (mp2) { QL_PRINT(DBG_RX, ("packet in mp2 only\n")); mp = mp2; } } } return (mp); fatal_error: /* fatal Error! */ if (qlge->fm_enable) { ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED); ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); atomic_or_32(&qlge->flags, ADAPTER_ERROR); } if (tp) { freemsg(tp); } /* *mp->b_wptr = 0; */ ql_wake_asic_reset_soft_intr(qlge); return (NULL); } /* * Bump completion queue consumer index. */ static void ql_update_cq(struct rx_ring *rx_ring) { rx_ring->cnsmr_idx++; rx_ring->curr_entry++; if (rx_ring->cnsmr_idx >= rx_ring->cq_len) { rx_ring->cnsmr_idx = 0; rx_ring->curr_entry = rx_ring->cq_dma.vaddr; } } /* * Update completion queue consumer index. */ static void ql_write_cq_idx(struct rx_ring *rx_ring) { qlge_t *qlge = rx_ring->qlge; ql_write_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg, rx_ring->cnsmr_idx); } /* * Processes a SYS-Chip Event Notification Completion Event. * The incoming notification event that describes a link up/down * or some sorts of error happens. */ static void ql_process_chip_ae_intr(qlge_t *qlge, struct ib_sys_event_iocb_rsp *ib_sys_event_rsp_ptr) { uint8_t eventType = ib_sys_event_rsp_ptr->event_type; uint32_t soft_req = 0; switch (eventType) { case SYS_EVENT_PORT_LINK_UP: /* 0x0h */ QL_PRINT(DBG_MBX, ("Port Link Up\n")); break; case SYS_EVENT_PORT_LINK_DOWN: /* 0x1h */ QL_PRINT(DBG_MBX, ("Port Link Down\n")); break; case SYS_EVENT_MULTIPLE_CAM_HITS : /* 0x6h */ cmn_err(CE_WARN, "A multiple CAM hits look up error " "occurred"); soft_req |= NEED_HW_RESET; break; case SYS_EVENT_SOFT_ECC_ERR: /* 0x7h */ cmn_err(CE_WARN, "Soft ECC error detected"); soft_req |= NEED_HW_RESET; break; case SYS_EVENT_MGMT_FATAL_ERR: /* 0x8h */ cmn_err(CE_WARN, "Management (MPI) Processor fatal" " error occured"); soft_req |= NEED_MPI_RESET; break; case SYS_EVENT_MAC_INTERRUPT: /* 0x9h */ QL_PRINT(DBG_MBX, ("MAC Interrupt")); break; case SYS_EVENT_PCI_ERR_READING_SML_LRG_BUF: /* 0x40h */ cmn_err(CE_WARN, "PCI Error reading small/large " "buffers occured"); soft_req |= NEED_HW_RESET; break; default: QL_PRINT(DBG_RX, ("%s(%d) unknown Sys Event: " "type 0x%x occured", __func__, qlge->instance, eventType)); break; } if ((soft_req & NEED_MPI_RESET) != 0) { ql_wake_mpi_reset_soft_intr(qlge); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED); } } else if ((soft_req & NEED_HW_RESET) != 0) { ql_wake_asic_reset_soft_intr(qlge); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_DEGRADED); } } } /* * set received packet checksum flag */ void ql_set_rx_cksum(mblk_t *mp, struct ib_mac_iocb_rsp *net_rsp) { uint32_t flags; /* Not TCP or UDP packet? nothing more to do */ if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) == 0) && ((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) == 0)) return; /* No CKO support for IPv6 */ if ((net_rsp->flags3 & IB_MAC_IOCB_RSP_V6) != 0) return; /* * If checksum error, don't set flags; stack will calculate * checksum, detect the error and update statistics */ if (((net_rsp->flags1 & IB_MAC_IOCB_RSP_TE) != 0) || ((net_rsp->flags1 & IB_MAC_IOCB_RSP_IE) != 0)) return; /* TCP or UDP packet and checksum valid */ if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_T) != 0) && ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) { flags = HCK_FULLCKSUM_OK; mac_hcksum_set(mp, 0, 0, 0, 0, flags); } if (((net_rsp->flags2 & IB_MAC_IOCB_RSP_U) != 0) && ((net_rsp->flags1 & IB_MAC_IOCB_RSP_NU) == 0)) { flags = HCK_FULLCKSUM_OK; mac_hcksum_set(mp, 0, 0, 0, 0, flags); } } /* * This function goes through h/w descriptor in one specified rx ring, * receives the data if the descriptor status shows the data is ready. * It returns a chain of mblks containing the received data, to be * passed up to mac_rx_ring(). */ mblk_t * ql_ring_rx(struct rx_ring *rx_ring, int poll_bytes) { qlge_t *qlge = rx_ring->qlge; uint32_t prod = ql_read_sh_reg(qlge, rx_ring); struct ib_mac_iocb_rsp *net_rsp; mblk_t *mp; mblk_t *mblk_head; mblk_t **mblk_tail; uint32_t received_bytes = 0; uint32_t length; #ifdef QLGE_PERFORMANCE uint32_t pkt_ct = 0; #endif #ifdef QLGE_TRACK_BUFFER_USAGE uint32_t consumer_idx; uint32_t producer_idx; uint32_t num_free_entries; uint32_t temp; temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg); consumer_idx = temp & 0x0000ffff; producer_idx = (temp >> 16); if (consumer_idx > producer_idx) num_free_entries = (consumer_idx - producer_idx); else num_free_entries = NUM_RX_RING_ENTRIES - ( producer_idx - consumer_idx); if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id]) qlge->cq_low_count[rx_ring->cq_id] = num_free_entries; #endif mblk_head = NULL; mblk_tail = &mblk_head; while ((prod != rx_ring->cnsmr_idx)) { QL_PRINT(DBG_RX, ("%s cq_id = %d, prod = %d, cnsmr = %d.\n", __func__, rx_ring->cq_id, prod, rx_ring->cnsmr_idx)); net_rsp = (struct ib_mac_iocb_rsp *)rx_ring->curr_entry; (void) ddi_dma_sync(rx_ring->cq_dma.dma_handle, (off_t)((uintptr_t)net_rsp - (uintptr_t)rx_ring->cq_dma.vaddr), (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL); QL_DUMP(DBG_RX, "qlge_ring_rx: rx completion iocb\n", rx_ring->curr_entry, 8, (size_t)sizeof (*net_rsp)); switch (net_rsp->opcode) { case OPCODE_IB_MAC_IOCB: /* Adding length of pkt header and payload */ length = le32_to_cpu(net_rsp->data_len) + le32_to_cpu(net_rsp->hdr_len); if ((poll_bytes != QLGE_POLL_ALL) && ((received_bytes + length) > poll_bytes)) { continue; } received_bytes += length; #ifdef QLGE_PERFORMANCE pkt_ct++; #endif mp = ql_build_rx_mp(qlge, rx_ring, net_rsp); if (mp != NULL) { if (rx_ring->mac_flags != QL_MAC_STARTED) { /* * Increment number of packets we have * indicated to the stack, should be * decremented when we get it back * or when freemsg is called */ ASSERT(rx_ring->rx_indicate <= rx_ring->cq_len); #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_WARN, "%s do not send to OS," " mac_flags %d, indicate %d", __func__, rx_ring->mac_flags, rx_ring->rx_indicate); #endif QL_PRINT(DBG_RX, ("cq_id = %d, packet " "dropped, mac not " "enabled.\n", rx_ring->cq_id)); rx_ring->rx_pkt_dropped_mac_unenabled++; /* rx_lock is expected to be held */ mutex_exit(&rx_ring->rx_lock); freemsg(mp); mutex_enter(&rx_ring->rx_lock); mp = NULL; } if (mp != NULL) { /* * IP full packet has been * successfully verified by * H/W and is correct */ ql_set_rx_cksum(mp, net_rsp); rx_ring->rx_packets++; rx_ring->rx_bytes += length; *mblk_tail = mp; mblk_tail = &mp->b_next; } } else { QL_PRINT(DBG_RX, ("cq_id = %d, packet dropped\n", rx_ring->cq_id)); rx_ring->rx_packets_dropped_no_buffer++; } break; case OPCODE_IB_SYS_EVENT_IOCB: ql_process_chip_ae_intr(qlge, (struct ib_sys_event_iocb_rsp *) net_rsp); break; default: cmn_err(CE_WARN, "%s Ring(%d)Hit default case, not handled!" " dropping the packet, " "opcode = %x.", __func__, rx_ring->cq_id, net_rsp->opcode); break; } /* increment cnsmr_idx and curr_entry */ ql_update_cq(rx_ring); prod = ql_read_sh_reg(qlge, rx_ring); } #ifdef QLGE_PERFORMANCE if (pkt_ct >= 7) rx_ring->hist[7]++; else if (pkt_ct == 6) rx_ring->hist[6]++; else if (pkt_ct == 5) rx_ring->hist[5]++; else if (pkt_ct == 4) rx_ring->hist[4]++; else if (pkt_ct == 3) rx_ring->hist[3]++; else if (pkt_ct == 2) rx_ring->hist[2]++; else if (pkt_ct == 1) rx_ring->hist[1]++; else if (pkt_ct == 0) rx_ring->hist[0]++; #endif /* update cnsmr_idx */ ql_write_cq_idx(rx_ring); /* do not enable interrupt for polling mode */ if (poll_bytes == QLGE_POLL_ALL) ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq); return (mblk_head); } /* Process an outbound completion from an rx ring. */ static void ql_process_mac_tx_intr(qlge_t *qlge, struct ob_mac_iocb_rsp *mac_rsp) { struct tx_ring *tx_ring; struct tx_ring_desc *tx_ring_desc; int j; tx_ring = &qlge->tx_ring[mac_rsp->txq_idx]; tx_ring_desc = tx_ring->wq_desc; tx_ring_desc += mac_rsp->tid; if (tx_ring_desc->tx_type == USE_DMA) { QL_PRINT(DBG_TX, ("%s(%d): tx type USE_DMA\n", __func__, qlge->instance)); /* * Release the DMA resource that is used for * DMA binding. */ for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) { (void) ddi_dma_unbind_handle( tx_ring_desc->tx_dma_handle[j]); } tx_ring_desc->tx_dma_handle_used = 0; /* * Free the mblk after sending completed */ if (tx_ring_desc->mp != NULL) { freemsg(tx_ring_desc->mp); tx_ring_desc->mp = NULL; } } tx_ring->obytes += tx_ring_desc->tx_bytes; tx_ring->opackets++; if (mac_rsp->flags1 & (OB_MAC_IOCB_RSP_E | OB_MAC_IOCB_RSP_S | OB_MAC_IOCB_RSP_L | OB_MAC_IOCB_RSP_B)) { tx_ring->errxmt++; if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_E) { /* EMPTY */ QL_PRINT(DBG_TX, ("Total descriptor length did not match " "transfer length.\n")); } if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_S) { /* EMPTY */ QL_PRINT(DBG_TX, ("Frame too short to be legal, not sent.\n")); } if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_L) { /* EMPTY */ QL_PRINT(DBG_TX, ("Frame too long, but sent anyway.\n")); } if (mac_rsp->flags3 & OB_MAC_IOCB_RSP_B) { /* EMPTY */ QL_PRINT(DBG_TX, ("PCI backplane error. Frame not sent.\n")); } } atomic_inc_32(&tx_ring->tx_free_count); } /* * clean up tx completion iocbs */ int ql_clean_outbound_rx_ring(struct rx_ring *rx_ring) { qlge_t *qlge = rx_ring->qlge; uint32_t prod = ql_read_sh_reg(qlge, rx_ring); struct ob_mac_iocb_rsp *net_rsp = NULL; int count = 0; struct tx_ring *tx_ring; boolean_t resume_tx = B_FALSE; mutex_enter(&rx_ring->rx_lock); #ifdef QLGE_TRACK_BUFFER_USAGE { uint32_t consumer_idx; uint32_t producer_idx; uint32_t num_free_entries; uint32_t temp; temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg); consumer_idx = temp & 0x0000ffff; producer_idx = (temp >> 16); if (consumer_idx > producer_idx) num_free_entries = (consumer_idx - producer_idx); else num_free_entries = NUM_RX_RING_ENTRIES - (producer_idx - consumer_idx); if (num_free_entries < qlge->cq_low_count[rx_ring->cq_id]) qlge->cq_low_count[rx_ring->cq_id] = num_free_entries; } #endif /* While there are entries in the completion queue. */ while (prod != rx_ring->cnsmr_idx) { QL_PRINT(DBG_RX, ("%s cq_id = %d, prod = %d, cnsmr = %d.\n", __func__, rx_ring->cq_id, prod, rx_ring->cnsmr_idx)); net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry; (void) ddi_dma_sync(rx_ring->cq_dma.dma_handle, (off_t)((uintptr_t)net_rsp - (uintptr_t)rx_ring->cq_dma.vaddr), (size_t)sizeof (*net_rsp), DDI_DMA_SYNC_FORKERNEL); QL_DUMP(DBG_RX, "ql_clean_outbound_rx_ring: " "response packet data\n", rx_ring->curr_entry, 8, (size_t)sizeof (*net_rsp)); switch (net_rsp->opcode) { case OPCODE_OB_MAC_OFFLOAD_IOCB: case OPCODE_OB_MAC_IOCB: ql_process_mac_tx_intr(qlge, net_rsp); break; default: cmn_err(CE_WARN, "%s Hit default case, not handled! " "dropping the packet," " opcode = %x.", __func__, net_rsp->opcode); break; } count++; ql_update_cq(rx_ring); prod = ql_read_sh_reg(qlge, rx_ring); } ql_write_cq_idx(rx_ring); mutex_exit(&rx_ring->rx_lock); net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry; tx_ring = &qlge->tx_ring[net_rsp->txq_idx]; mutex_enter(&tx_ring->tx_lock); if (tx_ring->queue_stopped && (tx_ring->tx_free_count > TX_RESUME_THRESHOLD)) { /* * The queue got stopped because the tx_ring was full. * Wake it up, because it's now at least 25% empty. */ tx_ring->queue_stopped = 0; resume_tx = B_TRUE; } mutex_exit(&tx_ring->tx_lock); /* Don't hold the lock during OS callback */ if (resume_tx) RESUME_TX(tx_ring); return (count); } /* * reset asic when error happens */ /* ARGSUSED */ static uint_t ql_asic_reset_work(caddr_t arg1, caddr_t arg2) { qlge_t *qlge = (qlge_t *)((void *)arg1); int status; mutex_enter(&qlge->gen_mutex); (void) ql_do_stop(qlge); /* * Write default ethernet address to chip register Mac * Address slot 0 and Enable Primary Mac Function. */ mutex_enter(&qlge->hw_mutex); (void) ql_unicst_set(qlge, (uint8_t *)qlge->unicst_addr[0].addr.ether_addr_octet, 0); mutex_exit(&qlge->hw_mutex); qlge->mac_flags = QL_MAC_INIT; status = ql_do_start(qlge); if (status != DDI_SUCCESS) goto error; qlge->mac_flags = QL_MAC_STARTED; mutex_exit(&qlge->gen_mutex); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_RESTORED); return (DDI_INTR_CLAIMED); error: mutex_exit(&qlge->gen_mutex); cmn_err(CE_WARN, "qlge up/down cycle failed, closing device"); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); atomic_or_32(&qlge->flags, ADAPTER_ERROR); } return (DDI_INTR_CLAIMED); } /* * Reset MPI */ /* ARGSUSED */ static uint_t ql_mpi_reset_work(caddr_t arg1, caddr_t arg2) { qlge_t *qlge = (qlge_t *)((void *)arg1); (void) ql_reset_mpi_risc(qlge); return (DDI_INTR_CLAIMED); } /* * Process MPI mailbox messages */ /* ARGSUSED */ static uint_t ql_mpi_event_work(caddr_t arg1, caddr_t arg2) { qlge_t *qlge = (qlge_t *)((void *)arg1); ql_do_mpi_intr(qlge); return (DDI_INTR_CLAIMED); } /* Fire up a handler to reset the MPI processor. */ void ql_wake_asic_reset_soft_intr(qlge_t *qlge) { (void) ddi_intr_trigger_softint(qlge->asic_reset_intr_hdl, NULL); } static void ql_wake_mpi_reset_soft_intr(qlge_t *qlge) { (void) ddi_intr_trigger_softint(qlge->mpi_reset_intr_hdl, NULL); } static void ql_wake_mpi_event_soft_intr(qlge_t *qlge) { (void) ddi_intr_trigger_softint(qlge->mpi_event_intr_hdl, NULL); } /* * This handles a fatal error, MPI activity, and the default * rx_ring in an MSI-X multiple interrupt vector environment. * In MSI/Legacy environment it also process the rest of * the rx_rings. */ /* ARGSUSED */ static uint_t ql_isr(caddr_t arg1, caddr_t arg2) { struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1); struct rx_ring *ob_ring; qlge_t *qlge = rx_ring->qlge; struct intr_ctx *intr_ctx = &qlge->intr_ctx[0]; uint32_t var, prod; int i; int work_done = 0; mblk_t *mp; _NOTE(ARGUNUSED(arg2)); ++qlge->rx_interrupts[rx_ring->cq_id]; if (ql_atomic_read_32(&qlge->intr_ctx[0].irq_cnt)) { ql_write_reg(qlge, REG_RSVD7, 0xfeed0002); var = ql_read_reg(qlge, REG_ERROR_STATUS); var = ql_read_reg(qlge, REG_STATUS); var = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1); return (DDI_INTR_CLAIMED); } ql_disable_completion_interrupt(qlge, intr_ctx->intr); /* * process send completes on first stride tx ring if available */ if (qlge->isr_stride) { ob_ring = &qlge->rx_ring[qlge->isr_stride]; if (ql_read_sh_reg(qlge, ob_ring) != ob_ring->cnsmr_idx) { (void) ql_clean_outbound_rx_ring(ob_ring); } } /* * Check the default queue and wake handler if active. */ rx_ring = &qlge->rx_ring[0]; prod = ql_read_sh_reg(qlge, rx_ring); QL_PRINT(DBG_INTR, ("rx-ring[0] prod index 0x%x, consumer 0x%x ", prod, rx_ring->cnsmr_idx)); /* check if interrupt is due to incoming packet */ if (prod != rx_ring->cnsmr_idx) { QL_PRINT(DBG_INTR, ("Waking handler for rx_ring[0].\n")); ql_disable_completion_interrupt(qlge, intr_ctx->intr); mutex_enter(&rx_ring->rx_lock); mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL); mutex_exit(&rx_ring->rx_lock); if (mp != NULL) RX_UPSTREAM(rx_ring, mp); work_done++; } else { /* * If interrupt is not due to incoming packet, read status * register to see if error happens or mailbox interrupt. */ var = ql_read_reg(qlge, REG_STATUS); if ((var & STATUS_FE) != 0) { ql_write_reg(qlge, REG_RSVD7, 0xfeed0003); if (qlge->fm_enable) { atomic_or_32(&qlge->flags, ADAPTER_ERROR); ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); } cmn_err(CE_WARN, "Got fatal error, STS = %x.", var); var = ql_read_reg(qlge, REG_ERROR_STATUS); cmn_err(CE_WARN, "Resetting chip. Error Status Register = 0x%x", var); ql_wake_asic_reset_soft_intr(qlge); return (DDI_INTR_CLAIMED); } /* * Check MPI processor activity. */ if ((var & STATUS_PI) != 0) { /* * We've got an async event or mailbox completion. * Handle it and clear the source of the interrupt. */ ql_write_reg(qlge, REG_RSVD7, 0xfeed0004); QL_PRINT(DBG_INTR, ("Got MPI processor interrupt.\n")); ql_disable_completion_interrupt(qlge, intr_ctx->intr); ql_wake_mpi_event_soft_intr(qlge); work_done++; } } if (qlge->intr_type != DDI_INTR_TYPE_MSIX) { /* * Start the DPC for each active queue. */ for (i = 1; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (ql_read_sh_reg(qlge, rx_ring) != rx_ring->cnsmr_idx) { QL_PRINT(DBG_INTR, ("Waking handler for rx_ring[%d].\n", i)); ql_disable_completion_interrupt(qlge, rx_ring->irq); if (rx_ring->type == TX_Q) { (void) ql_clean_outbound_rx_ring( rx_ring); ql_enable_completion_interrupt( rx_ring->qlge, rx_ring->irq); } else { mutex_enter(&rx_ring->rx_lock); mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL); mutex_exit(&rx_ring->rx_lock); if (mp != NULL) RX_UPSTREAM(rx_ring, mp); #ifdef QLGE_LOAD_UNLOAD if (rx_ring->mac_flags == QL_MAC_STOPPED) cmn_err(CE_NOTE, "%s rx_indicate(%d) %d\n", __func__, i, rx_ring->rx_indicate); #endif } work_done++; } } } ql_enable_completion_interrupt(qlge, intr_ctx->intr); return (work_done ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); } /* * MSI-X Multiple Vector Interrupt Handler for outbound (TX) completions. */ /* ARGSUSED */ static uint_t ql_msix_tx_isr(caddr_t arg1, caddr_t arg2) { struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1); qlge_t *qlge = rx_ring->qlge; _NOTE(ARGUNUSED(arg2)); ++qlge->rx_interrupts[rx_ring->cq_id]; (void) ql_clean_outbound_rx_ring(rx_ring); ql_enable_completion_interrupt(rx_ring->qlge, rx_ring->irq); return (DDI_INTR_CLAIMED); } /* * MSI-X Multiple Vector Interrupt Handler */ /* ARGSUSED */ static uint_t ql_msix_isr(caddr_t arg1, caddr_t arg2) { struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1); struct rx_ring *ob_ring; qlge_t *qlge = rx_ring->qlge; mblk_t *mp; _NOTE(ARGUNUSED(arg2)); QL_PRINT(DBG_INTR, ("%s for ring %d\n", __func__, rx_ring->cq_id)); ql_disable_completion_interrupt(qlge, rx_ring->irq); /* * process send completes on stride tx ring if available */ if (qlge->isr_stride) { ob_ring = rx_ring + qlge->isr_stride; if (ql_read_sh_reg(qlge, ob_ring) != ob_ring->cnsmr_idx) { ++qlge->rx_interrupts[ob_ring->cq_id]; (void) ql_clean_outbound_rx_ring(ob_ring); } } ++qlge->rx_interrupts[rx_ring->cq_id]; mutex_enter(&rx_ring->rx_lock); mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL); mutex_exit(&rx_ring->rx_lock); if (mp != NULL) RX_UPSTREAM(rx_ring, mp); return (DDI_INTR_CLAIMED); } /* * Poll n_bytes of chained incoming packets */ mblk_t * ql_ring_rx_poll(void *arg, int n_bytes) { struct rx_ring *rx_ring = (struct rx_ring *)arg; qlge_t *qlge = rx_ring->qlge; mblk_t *mp = NULL; uint32_t var; ASSERT(n_bytes >= 0); QL_PRINT(DBG_GLD, ("%s for ring(%d) to read max %d bytes\n", __func__, rx_ring->cq_id, n_bytes)); ++qlge->rx_polls[rx_ring->cq_id]; if (n_bytes == 0) return (mp); mutex_enter(&rx_ring->rx_lock); mp = ql_ring_rx(rx_ring, n_bytes); mutex_exit(&rx_ring->rx_lock); if ((rx_ring->cq_id == 0) && (mp == NULL)) { var = ql_read_reg(qlge, REG_STATUS); /* * Check for fatal error. */ if ((var & STATUS_FE) != 0) { ql_write_reg(qlge, REG_RSVD7, 0xfeed0003); var = ql_read_reg(qlge, REG_ERROR_STATUS); cmn_err(CE_WARN, "Got fatal error %x.", var); ql_wake_asic_reset_soft_intr(qlge); if (qlge->fm_enable) { atomic_or_32(&qlge->flags, ADAPTER_ERROR); ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); } } /* * Check MPI processor activity. */ if ((var & STATUS_PI) != 0) { /* * We've got an async event or mailbox completion. * Handle it and clear the source of the interrupt. */ ql_write_reg(qlge, REG_RSVD7, 0xfeed0004); ql_do_mpi_intr(qlge); } } return (mp); } /* * MSI-X Multiple Vector Interrupt Handler for inbound (RX) completions. */ /* ARGSUSED */ static uint_t ql_msix_rx_isr(caddr_t arg1, caddr_t arg2) { struct rx_ring *rx_ring = (struct rx_ring *)((void *)arg1); qlge_t *qlge = rx_ring->qlge; mblk_t *mp; _NOTE(ARGUNUSED(arg2)); QL_PRINT(DBG_INTR, ("%s for ring %d\n", __func__, rx_ring->cq_id)); ++qlge->rx_interrupts[rx_ring->cq_id]; mutex_enter(&rx_ring->rx_lock); mp = ql_ring_rx(rx_ring, QLGE_POLL_ALL); mutex_exit(&rx_ring->rx_lock); if (mp != NULL) RX_UPSTREAM(rx_ring, mp); return (DDI_INTR_CLAIMED); } /* * * Allocate DMA Buffer for ioctl service * */ static int ql_alloc_ioctl_dma_buf(qlge_t *qlge) { uint64_t phy_addr; uint64_t alloc_size; ddi_dma_cookie_t dma_cookie; alloc_size = qlge->ioctl_buf_dma_attr.mem_len = max(WCS_MPI_CODE_RAM_LENGTH, MEMC_MPI_RAM_LENGTH); if (ql_alloc_phys(qlge->dip, &qlge->ioctl_buf_dma_attr.dma_handle, &ql_buf_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &qlge->ioctl_buf_dma_attr.acc_handle, (size_t)alloc_size, /* mem size */ (size_t)0, /* alignment */ (caddr_t *)&qlge->ioctl_buf_dma_attr.vaddr, &dma_cookie) != 0) { cmn_err(CE_WARN, "%s(%d): ioctl DMA allocation failed.", __func__, qlge->instance); return (DDI_FAILURE); } phy_addr = dma_cookie.dmac_laddress; if (qlge->ioctl_buf_dma_attr.vaddr == NULL) { cmn_err(CE_WARN, "%s(%d): failed.", __func__, qlge->instance); return (DDI_FAILURE); } qlge->ioctl_buf_dma_attr.dma_addr = phy_addr; QL_PRINT(DBG_MBX, ("%s: ioctl_dma_buf_virt_addr = 0x%lx, " "phy_addr = 0x%lx\n", __func__, qlge->ioctl_buf_dma_attr.vaddr, phy_addr)); return (DDI_SUCCESS); } /* * Function to free physical memory. */ static void ql_free_phys(ddi_dma_handle_t *dma_handle, ddi_acc_handle_t *acc_handle) { if (*dma_handle != NULL) { (void) ddi_dma_unbind_handle(*dma_handle); if (*acc_handle != NULL) ddi_dma_mem_free(acc_handle); ddi_dma_free_handle(dma_handle); *acc_handle = NULL; *dma_handle = NULL; } } /* * Function to free ioctl dma buffer. */ static void ql_free_ioctl_dma_buf(qlge_t *qlge) { if (qlge->ioctl_buf_dma_attr.dma_handle != NULL) { ql_free_phys(&qlge->ioctl_buf_dma_attr.dma_handle, &qlge->ioctl_buf_dma_attr.acc_handle); qlge->ioctl_buf_dma_attr.vaddr = NULL; qlge->ioctl_buf_dma_attr.dma_handle = NULL; } } /* * Free shadow register space used for request and completion queues */ static void ql_free_shadow_space(qlge_t *qlge) { if (qlge->host_copy_shadow_dma_attr.dma_handle != NULL) { ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle, &qlge->host_copy_shadow_dma_attr.acc_handle); bzero(&qlge->host_copy_shadow_dma_attr, sizeof (qlge->host_copy_shadow_dma_attr)); } if (qlge->buf_q_ptr_base_addr_dma_attr.dma_handle != NULL) { ql_free_phys(&qlge->buf_q_ptr_base_addr_dma_attr.dma_handle, &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle); bzero(&qlge->buf_q_ptr_base_addr_dma_attr, sizeof (qlge->buf_q_ptr_base_addr_dma_attr)); } } /* * Allocate shadow register space for request and completion queues */ static int ql_alloc_shadow_space(qlge_t *qlge) { ddi_dma_cookie_t dma_cookie; if (ql_alloc_phys(qlge->dip, &qlge->host_copy_shadow_dma_attr.dma_handle, &ql_dev_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &qlge->host_copy_shadow_dma_attr.acc_handle, (size_t)VM_PAGE_SIZE, /* mem size */ (size_t)4, /* 4 bytes alignment */ (caddr_t *)&qlge->host_copy_shadow_dma_attr.vaddr, &dma_cookie) != 0) { bzero(&qlge->host_copy_shadow_dma_attr, sizeof (qlge->host_copy_shadow_dma_attr)); cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory for " "response shadow registers", __func__, qlge->instance); return (DDI_FAILURE); } qlge->host_copy_shadow_dma_attr.dma_addr = dma_cookie.dmac_laddress; if (ql_alloc_phys(qlge->dip, &qlge->buf_q_ptr_base_addr_dma_attr.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &qlge->buf_q_ptr_base_addr_dma_attr.acc_handle, (size_t)VM_PAGE_SIZE, /* mem size */ (size_t)4, /* 4 bytes alignment */ (caddr_t *)&qlge->buf_q_ptr_base_addr_dma_attr.vaddr, &dma_cookie) != 0) { bzero(&qlge->buf_q_ptr_base_addr_dma_attr, sizeof (qlge->buf_q_ptr_base_addr_dma_attr)); cmn_err(CE_WARN, "%s(%d): Unable to allocate DMA memory " "for request shadow registers", __func__, qlge->instance); goto err_wqp_sh_area; } qlge->buf_q_ptr_base_addr_dma_attr.dma_addr = dma_cookie.dmac_laddress; return (DDI_SUCCESS); err_wqp_sh_area: ql_free_phys(&qlge->host_copy_shadow_dma_attr.dma_handle, &qlge->host_copy_shadow_dma_attr.acc_handle); bzero(&qlge->host_copy_shadow_dma_attr, sizeof (qlge->host_copy_shadow_dma_attr)); return (DDI_FAILURE); } /* * Initialize a tx ring */ static void ql_init_tx_ring(struct tx_ring *tx_ring) { int i; struct ob_mac_iocb_req *mac_iocb_ptr = tx_ring->wq_dma.vaddr; struct tx_ring_desc *tx_ring_desc = tx_ring->wq_desc; for (i = 0; i < tx_ring->wq_len; i++) { tx_ring_desc->index = i; tx_ring_desc->queue_entry = mac_iocb_ptr; mac_iocb_ptr++; tx_ring_desc++; } tx_ring->tx_free_count = tx_ring->wq_len; tx_ring->queue_stopped = 0; } /* * Free one tx ring resources */ static void ql_free_tx_resources(struct tx_ring *tx_ring) { struct tx_ring_desc *tx_ring_desc; int i, j; if (tx_ring->wq_dma.dma_handle != NULL) { ql_free_phys(&tx_ring->wq_dma.dma_handle, &tx_ring->wq_dma.acc_handle); bzero(&tx_ring->wq_dma, sizeof (tx_ring->wq_dma)); } if (tx_ring->wq_desc != NULL) { tx_ring_desc = tx_ring->wq_desc; for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) { for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) { if (tx_ring_desc->tx_dma_handle[j]) { /* * The unbinding will happen in tx * completion, here we just free the * handles */ ddi_dma_free_handle( &(tx_ring_desc->tx_dma_handle[j])); tx_ring_desc->tx_dma_handle[j] = NULL; } } if (tx_ring_desc->oal != NULL) { tx_ring_desc->oal_dma_addr = 0; tx_ring_desc->oal = NULL; tx_ring_desc->copy_buffer = NULL; tx_ring_desc->copy_buffer_dma_addr = 0; ql_free_phys(&tx_ring_desc->oal_dma.dma_handle, &tx_ring_desc->oal_dma.acc_handle); } } kmem_free(tx_ring->wq_desc, tx_ring->wq_len * sizeof (struct tx_ring_desc)); tx_ring->wq_desc = NULL; } /* free the wqicb struct */ if (tx_ring->wqicb_dma.dma_handle) { ql_free_phys(&tx_ring->wqicb_dma.dma_handle, &tx_ring->wqicb_dma.acc_handle); bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma)); } } /* * Allocate work (request) queue memory and transmit * descriptors for this transmit ring */ static int ql_alloc_tx_resources(qlge_t *qlge, struct tx_ring *tx_ring) { ddi_dma_cookie_t dma_cookie; struct tx_ring_desc *tx_ring_desc; int i, j; uint32_t length; /* allocate dma buffers for obiocbs */ if (ql_alloc_phys(qlge->dip, &tx_ring->wq_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &tx_ring->wq_dma.acc_handle, (size_t)tx_ring->wq_size, /* mem size */ (size_t)128, /* alignment:128 bytes boundary */ (caddr_t *)&tx_ring->wq_dma.vaddr, &dma_cookie) != 0) { bzero(&tx_ring->wq_dma, sizeof (tx_ring->wq_dma)); cmn_err(CE_WARN, "%s(%d): reqQ allocation failed.", __func__, qlge->instance); return (DDI_FAILURE); } tx_ring->wq_dma.dma_addr = dma_cookie.dmac_laddress; tx_ring->wq_desc = kmem_zalloc(tx_ring->wq_len * sizeof (struct tx_ring_desc), KM_NOSLEEP); if (tx_ring->wq_desc == NULL) { goto err; } else { tx_ring_desc = tx_ring->wq_desc; /* * Allocate a large enough structure to hold the following * 1. oal buffer MAX_SGELEMENTS * sizeof (oal_entry) bytes * 2. copy buffer of QL_MAX_COPY_LENGTH bytes */ length = (sizeof (struct oal_entry) * MAX_SG_ELEMENTS) + QL_MAX_COPY_LENGTH; for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) { if (ql_alloc_phys(qlge->dip, &tx_ring_desc->oal_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &tx_ring_desc->oal_dma.acc_handle, (size_t)length, /* mem size */ (size_t)0, /* default alignment:8 bytes boundary */ (caddr_t *)&tx_ring_desc->oal_dma.vaddr, &dma_cookie) != 0) { bzero(&tx_ring_desc->oal_dma, sizeof (tx_ring_desc->oal_dma)); cmn_err(CE_WARN, "%s(%d): reqQ tx buf &" "oal alloc failed.", __func__, qlge->instance); goto err; } tx_ring_desc->oal = tx_ring_desc->oal_dma.vaddr; tx_ring_desc->oal_dma_addr = dma_cookie.dmac_laddress; tx_ring_desc->copy_buffer = (caddr_t)((uint8_t *)tx_ring_desc->oal + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS)); tx_ring_desc->copy_buffer_dma_addr = (tx_ring_desc->oal_dma_addr + (sizeof (struct oal_entry) * MAX_SG_ELEMENTS)); /* Allocate dma handles for transmit buffers */ for (j = 0; j < QL_MAX_TX_DMA_HANDLES; j++) { if (ddi_dma_alloc_handle(qlge->dip, &tx_mapping_dma_attr, DDI_DMA_DONTWAIT, 0, &tx_ring_desc->tx_dma_handle[j]) != DDI_SUCCESS) { tx_ring_desc->tx_dma_handle[j] = NULL; cmn_err(CE_WARN, "!%s: ddi_dma_alloc_handle: " "tx_dma_handle " "alloc failed", __func__); ql_free_phys( &tx_ring_desc->oal_dma.dma_handle, &tx_ring_desc->oal_dma.acc_handle); goto err; } } } } /* alloc a wqicb control block to load this tx ring to hw */ if (ql_alloc_phys(qlge->dip, &tx_ring->wqicb_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &tx_ring->wqicb_dma.acc_handle, (size_t)sizeof (struct wqicb_t), /* mem size */ (size_t)0, /* alignment:128 bytes boundary */ (caddr_t *)&tx_ring->wqicb_dma.vaddr, &dma_cookie) != 0) { bzero(&tx_ring->wqicb_dma, sizeof (tx_ring->wqicb_dma)); cmn_err(CE_WARN, "%s(%d): wqicb allocation failed.", __func__, qlge->instance); goto err; } tx_ring->wqicb_dma.dma_addr = dma_cookie.dmac_laddress; return (DDI_SUCCESS); err: ql_free_tx_resources(tx_ring); return (DDI_FAILURE); } /* * Free one rx ring resources */ static void ql_free_rx_resources(struct rx_ring *rx_ring) { /* Free the small buffer queue. */ if (rx_ring->sbq_dma.dma_handle) { ql_free_phys(&rx_ring->sbq_dma.dma_handle, &rx_ring->sbq_dma.acc_handle); bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma)); } /* Free the small buffer queue control blocks. */ if (rx_ring->sbq_desc != NULL) { kmem_free(rx_ring->sbq_desc, rx_ring->sbq_len * sizeof (struct bq_desc)); rx_ring->sbq_desc = NULL; } /* Free the large buffer queue. */ if (rx_ring->lbq_dma.dma_handle) { ql_free_phys(&rx_ring->lbq_dma.dma_handle, &rx_ring->lbq_dma.acc_handle); bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma)); } /* Free the large buffer queue control blocks. */ if (rx_ring->lbq_desc != NULL) { kmem_free(rx_ring->lbq_desc, rx_ring->lbq_len * sizeof (struct bq_desc)); rx_ring->lbq_desc = NULL; } /* Free cqicb struct */ if (rx_ring->cqicb_dma.dma_handle) { ql_free_phys(&rx_ring->cqicb_dma.dma_handle, &rx_ring->cqicb_dma.acc_handle); bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma)); } /* Free the rx queue. */ if (rx_ring->cq_dma.dma_handle) { ql_free_phys(&rx_ring->cq_dma.dma_handle, &rx_ring->cq_dma.acc_handle); bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma)); } } /* * Allocate queues and buffers for this completions queue based * on the values in the parameter structure. */ static int ql_alloc_rx_resources(qlge_t *qlge, struct rx_ring *rx_ring) { ddi_dma_cookie_t dma_cookie; if (ql_alloc_phys(qlge->dip, &rx_ring->cq_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &rx_ring->cq_dma.acc_handle, (size_t)rx_ring->cq_size, /* mem size */ (size_t)128, /* alignment:128 bytes boundary */ (caddr_t *)&rx_ring->cq_dma.vaddr, &dma_cookie) != 0) { bzero(&rx_ring->cq_dma, sizeof (rx_ring->cq_dma)); cmn_err(CE_WARN, "%s(%d): rspQ allocation failed.", __func__, qlge->instance); return (DDI_FAILURE); } rx_ring->cq_dma.dma_addr = dma_cookie.dmac_laddress; if (rx_ring->sbq_len != 0) { /* * Allocate small buffer queue. */ if (ql_alloc_phys(qlge->dip, &rx_ring->sbq_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &rx_ring->sbq_dma.acc_handle, (size_t)rx_ring->sbq_size, /* mem size */ (size_t)128, /* alignment:128 bytes boundary */ (caddr_t *)&rx_ring->sbq_dma.vaddr, &dma_cookie) != 0) { bzero(&rx_ring->sbq_dma, sizeof (rx_ring->sbq_dma)); cmn_err(CE_WARN, "%s(%d): small buffer queue allocation failed.", __func__, qlge->instance); goto err_mem; } rx_ring->sbq_dma.dma_addr = dma_cookie.dmac_laddress; /* * Allocate small buffer queue control blocks. */ rx_ring->sbq_desc = kmem_zalloc(rx_ring->sbq_len * sizeof (struct bq_desc), KM_NOSLEEP); if (rx_ring->sbq_desc == NULL) { cmn_err(CE_WARN, "sbq control block allocation failed."); goto err_mem; } ql_init_sbq_ring(rx_ring); } if (rx_ring->lbq_len != 0) { /* * Allocate large buffer queue. */ if (ql_alloc_phys(qlge->dip, &rx_ring->lbq_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &rx_ring->lbq_dma.acc_handle, (size_t)rx_ring->lbq_size, /* mem size */ (size_t)128, /* alignment:128 bytes boundary */ (caddr_t *)&rx_ring->lbq_dma.vaddr, &dma_cookie) != 0) { bzero(&rx_ring->lbq_dma, sizeof (rx_ring->lbq_dma)); cmn_err(CE_WARN, "%s(%d): lbq allocation failed.", __func__, qlge->instance); goto err_mem; } rx_ring->lbq_dma.dma_addr = dma_cookie.dmac_laddress; /* * Allocate large buffer queue control blocks. */ rx_ring->lbq_desc = kmem_zalloc(rx_ring->lbq_len * sizeof (struct bq_desc), KM_NOSLEEP); if (rx_ring->lbq_desc == NULL) { cmn_err(CE_WARN, "Large buffer queue control block allocation " "failed."); goto err_mem; } ql_init_lbq_ring(rx_ring); } if (ql_alloc_phys(qlge->dip, &rx_ring->cqicb_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &rx_ring->cqicb_dma.acc_handle, (size_t)sizeof (struct cqicb_t), /* mem size */ (size_t)0, /* alignment:128 bytes boundary */ (caddr_t *)&rx_ring->cqicb_dma.vaddr, &dma_cookie) != 0) { bzero(&rx_ring->cqicb_dma, sizeof (rx_ring->cqicb_dma)); cmn_err(CE_WARN, "%s(%d): cqicb allocation failed.", __func__, qlge->instance); goto err_mem; } rx_ring->cqicb_dma.dma_addr = dma_cookie.dmac_laddress; return (DDI_SUCCESS); err_mem: ql_free_rx_resources(rx_ring); return (DDI_FAILURE); } /* * Frees tx/rx queues memory resources */ static void ql_free_mem_resources(qlge_t *qlge) { int i; if (qlge->ricb_dma.dma_handle) { /* free the ricb struct */ ql_free_phys(&qlge->ricb_dma.dma_handle, &qlge->ricb_dma.acc_handle); bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma)); } ql_free_rx_buffers(qlge); ql_free_ioctl_dma_buf(qlge); for (i = 0; i < qlge->tx_ring_count; i++) ql_free_tx_resources(&qlge->tx_ring[i]); for (i = 0; i < qlge->rx_ring_count; i++) ql_free_rx_resources(&qlge->rx_ring[i]); ql_free_shadow_space(qlge); } /* * Allocate buffer queues, large buffers and small buffers etc * * This API is called in the gld_attach member function. It is called * only once. Later reset,reboot should not re-allocate all rings and * buffers. */ static int ql_alloc_mem_resources(qlge_t *qlge) { int i; ddi_dma_cookie_t dma_cookie; /* Allocate space for our shadow registers */ if (ql_alloc_shadow_space(qlge)) return (DDI_FAILURE); for (i = 0; i < qlge->rx_ring_count; i++) { if (ql_alloc_rx_resources(qlge, &qlge->rx_ring[i]) != 0) { cmn_err(CE_WARN, "RX resource allocation failed."); goto err_mem; } } /* Allocate tx queue resources */ for (i = 0; i < qlge->tx_ring_count; i++) { if (ql_alloc_tx_resources(qlge, &qlge->tx_ring[i]) != 0) { cmn_err(CE_WARN, "Tx resource allocation failed."); goto err_mem; } } if (ql_alloc_ioctl_dma_buf(qlge) != DDI_SUCCESS) { goto err_mem; } if (ql_alloc_rx_buffers(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "?%s(%d): ql_alloc_rx_buffers failed", __func__, qlge->instance); goto err_mem; } qlge->sequence |= INIT_ALLOC_RX_BUF; if (ql_alloc_phys(qlge->dip, &qlge->ricb_dma.dma_handle, &ql_desc_acc_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, &qlge->ricb_dma.acc_handle, (size_t)sizeof (struct ricb), /* mem size */ (size_t)0, /* alignment:128 bytes boundary */ (caddr_t *)&qlge->ricb_dma.vaddr, &dma_cookie) != 0) { bzero(&qlge->ricb_dma, sizeof (qlge->ricb_dma)); cmn_err(CE_WARN, "%s(%d): ricb allocation failed.", __func__, qlge->instance); goto err_mem; } qlge->ricb_dma.dma_addr = dma_cookie.dmac_laddress; return (DDI_SUCCESS); err_mem: ql_free_mem_resources(qlge); return (DDI_FAILURE); } /* * Function used to allocate physical memory and zero it. */ static int ql_alloc_phys_rbuf(dev_info_t *dip, ddi_dma_handle_t *dma_handle, ddi_device_acc_attr_t *device_acc_attr, uint_t dma_flags, ddi_acc_handle_t *acc_handle, size_t size, size_t alignment, caddr_t *vaddr, ddi_dma_cookie_t *dma_cookie) { size_t rlen; uint_t cnt; /* * Workaround for SUN XMITS buffer must end and start on 8 byte * boundary. Else, hardware will overrun the buffer. Simple fix is * to make sure buffer has enough room for overrun. */ if (size & 7) { size += 8 - (size & 7); } /* Adjust the alignment if requested */ if (alignment) { dma_attr.dma_attr_align = alignment; } /* * Allocate DMA handle */ if (ddi_dma_alloc_handle(dip, &dma_attr_rbuf, DDI_DMA_DONTWAIT, NULL, dma_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, QL_BANG "%s: ddi_dma_alloc_handle FAILED", __func__); *dma_handle = NULL; return (QL_ERROR); } /* * Allocate DMA memory */ if (ddi_dma_mem_alloc(*dma_handle, size, device_acc_attr, dma_flags & (DDI_DMA_CONSISTENT|DDI_DMA_STREAMING), DDI_DMA_DONTWAIT, NULL, vaddr, &rlen, acc_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, "alloc_phys: DMA Memory alloc Failed"); ddi_dma_free_handle(dma_handle); *acc_handle = NULL; *dma_handle = NULL; return (QL_ERROR); } if (ddi_dma_addr_bind_handle(*dma_handle, NULL, *vaddr, rlen, dma_flags, DDI_DMA_DONTWAIT, NULL, dma_cookie, &cnt) != DDI_DMA_MAPPED) { ddi_dma_mem_free(acc_handle); ddi_dma_free_handle(dma_handle); cmn_err(CE_WARN, "%s ddi_dma_addr_bind_handle FAILED", __func__); *acc_handle = NULL; *dma_handle = NULL; return (QL_ERROR); } if (cnt != 1) { ql_free_phys(dma_handle, acc_handle); cmn_err(CE_WARN, "%s: cnt != 1; Failed segment count", __func__); return (QL_ERROR); } bzero((caddr_t)*vaddr, rlen); return (0); } /* * Function used to allocate physical memory and zero it. */ static int ql_alloc_phys(dev_info_t *dip, ddi_dma_handle_t *dma_handle, ddi_device_acc_attr_t *device_acc_attr, uint_t dma_flags, ddi_acc_handle_t *acc_handle, size_t size, size_t alignment, caddr_t *vaddr, ddi_dma_cookie_t *dma_cookie) { size_t rlen; uint_t cnt; /* * Workaround for SUN XMITS buffer must end and start on 8 byte * boundary. Else, hardware will overrun the buffer. Simple fix is * to make sure buffer has enough room for overrun. */ if (size & 7) { size += 8 - (size & 7); } /* Adjust the alignment if requested */ if (alignment) { dma_attr.dma_attr_align = alignment; } /* * Allocate DMA handle */ if (ddi_dma_alloc_handle(dip, &dma_attr, DDI_DMA_DONTWAIT, NULL, dma_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, QL_BANG "%s: ddi_dma_alloc_handle FAILED", __func__); *dma_handle = NULL; return (QL_ERROR); } /* * Allocate DMA memory */ if (ddi_dma_mem_alloc(*dma_handle, size, device_acc_attr, dma_flags & (DDI_DMA_CONSISTENT|DDI_DMA_STREAMING), DDI_DMA_DONTWAIT, NULL, vaddr, &rlen, acc_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, "alloc_phys: DMA Memory alloc Failed"); ddi_dma_free_handle(dma_handle); *acc_handle = NULL; *dma_handle = NULL; return (QL_ERROR); } if (ddi_dma_addr_bind_handle(*dma_handle, NULL, *vaddr, rlen, dma_flags, DDI_DMA_DONTWAIT, NULL, dma_cookie, &cnt) != DDI_DMA_MAPPED) { ddi_dma_mem_free(acc_handle); ddi_dma_free_handle(dma_handle); cmn_err(CE_WARN, "%s ddi_dma_addr_bind_handle FAILED", __func__); *acc_handle = NULL; *dma_handle = NULL; return (QL_ERROR); } if (cnt != 1) { ql_free_phys(dma_handle, acc_handle); cmn_err(CE_WARN, "%s: cnt != 1; Failed segment count", __func__); return (QL_ERROR); } bzero((caddr_t)*vaddr, rlen); return (0); } /* * Add interrupt handlers based on the interrupt type. * Before adding the interrupt handlers, the interrupt vectors should * have been allocated, and the rx/tx rings have also been allocated. */ static int ql_add_intr_handlers(qlge_t *qlge) { int vector = 0; int rc, i; uint32_t value; struct intr_ctx *intr_ctx = &qlge->intr_ctx[0]; switch (qlge->intr_type) { case DDI_INTR_TYPE_MSIX: /* * Add interrupt handler for rx and tx rings: vector[0 - * (qlge->intr_cnt -1)]. */ value = 0; for (vector = 0; vector < qlge->intr_cnt; vector++) { ql_atomic_set_32(&intr_ctx->irq_cnt, value); /* * associate interrupt vector with interrupt handler */ rc = ddi_intr_add_handler(qlge->htable[vector], (ddi_intr_handler_t *)intr_ctx->handler, (void *)&qlge->rx_ring[vector], NULL); QL_PRINT(DBG_INIT, ("rx_ring[%d] 0x%p\n", vector, &qlge->rx_ring[vector])); if (rc != DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("Add rx interrupt handler failed. " "return: %d, vector: %d", rc, vector)); for (vector--; vector >= 0; vector--) { (void) ddi_intr_remove_handler( qlge->htable[vector]); } return (DDI_FAILURE); } intr_ctx++; } break; case DDI_INTR_TYPE_MSI: /* * Add interrupt handlers for the only vector */ ql_atomic_set_32(&intr_ctx->irq_cnt, value); rc = ddi_intr_add_handler(qlge->htable[vector], ql_isr, (caddr_t)&qlge->rx_ring[0], NULL); if (rc != DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("Add MSI interrupt handler failed: %d\n", rc)); return (DDI_FAILURE); } break; case DDI_INTR_TYPE_FIXED: /* * Add interrupt handlers for the only vector */ ql_atomic_set_32(&intr_ctx->irq_cnt, value); rc = ddi_intr_add_handler(qlge->htable[vector], ql_isr, (caddr_t)&qlge->rx_ring[0], NULL); if (rc != DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("Add legacy interrupt handler failed: %d\n", rc)); return (DDI_FAILURE); } break; default: return (DDI_FAILURE); } /* Enable interrupts */ /* Block enable */ if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) { QL_PRINT(DBG_INIT, ("Block enabling %d interrupt(s)\n", qlge->intr_cnt)); (void) ddi_intr_block_enable(qlge->htable, qlge->intr_cnt); } else { /* Non block enable */ for (i = 0; i < qlge->intr_cnt; i++) { QL_PRINT(DBG_INIT, ("Non Block Enabling interrupt %d " "handle 0x%x\n", i, qlge->htable[i])); (void) ddi_intr_enable(qlge->htable[i]); } } qlge->sequence |= INIT_INTR_ENABLED; return (DDI_SUCCESS); } /* * Here we build the intr_ctx structures based on * our rx_ring count and intr vector count. * The intr_ctx structure is used to hook each vector * to possibly different handlers. */ static void ql_resolve_queues_to_irqs(qlge_t *qlge) { int i = 0; struct intr_ctx *intr_ctx = &qlge->intr_ctx[0]; if (qlge->intr_type == DDI_INTR_TYPE_MSIX) { /* * Each rx_ring has its own intr_ctx since we * have separate vectors for each queue. * This only true when MSI-X is enabled. */ for (i = 0; i < qlge->intr_cnt; i++, intr_ctx++) { qlge->rx_ring[i].irq = i; intr_ctx->intr = i; intr_ctx->qlge = qlge; /* * We set up each vectors enable/disable/read bits so * there's no bit/mask calculations in critical path. */ intr_ctx->intr_en_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK | INTR_EN_IHD | i; intr_ctx->intr_dis_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK | INTR_EN_IHD | i; intr_ctx->intr_read_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD | i; if (i == 0) { /* * Default queue handles bcast/mcast plus * async events. */ intr_ctx->handler = ql_isr; } else if (qlge->rx_ring[i].type == TX_Q) { /* * Outbound queue is for outbound completions * only. */ if (qlge->isr_stride) intr_ctx->handler = ql_msix_isr; else intr_ctx->handler = ql_msix_tx_isr; } else { /* * Inbound queues handle unicast frames only. */ if (qlge->isr_stride) intr_ctx->handler = ql_msix_isr; else intr_ctx->handler = ql_msix_rx_isr; } } i = qlge->intr_cnt; for (; i < qlge->rx_ring_count; i++, intr_ctx++) { int iv = i - qlge->isr_stride; qlge->rx_ring[i].irq = iv; intr_ctx->intr = iv; intr_ctx->qlge = qlge; /* * We set up each vectors enable/disable/read bits so * there's no bit/mask calculations in critical path. */ intr_ctx->intr_en_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK | INTR_EN_IHD | iv; intr_ctx->intr_dis_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK | INTR_EN_IHD | iv; intr_ctx->intr_read_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD | iv; if (qlge->rx_ring[i].type == TX_Q) { /* * Outbound queue is for outbound completions * only. */ intr_ctx->handler = ql_msix_isr; } else { /* * Inbound queues handle unicast frames only. */ intr_ctx->handler = ql_msix_rx_isr; } } } else { /* * All rx_rings use the same intr_ctx since * there is only one vector. */ intr_ctx->intr = 0; intr_ctx->qlge = qlge; /* * We set up each vectors enable/disable/read bits so * there's no bit/mask calculations in the critical path. */ intr_ctx->intr_en_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_ENABLE; intr_ctx->intr_dis_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_DISABLE; intr_ctx->intr_read_mask = INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_READ; /* * Single interrupt means one handler for all rings. */ intr_ctx->handler = ql_isr; for (i = 0; i < qlge->rx_ring_count; i++) qlge->rx_ring[i].irq = 0; } } /* * Free allocated interrupts. */ static void ql_free_irq_vectors(qlge_t *qlge) { int i; int rc; if (qlge->sequence & INIT_INTR_ENABLED) { /* Disable all interrupts */ if (qlge->intr_cap & DDI_INTR_FLAG_BLOCK) { /* Call ddi_intr_block_disable() */ (void) ddi_intr_block_disable(qlge->htable, qlge->intr_cnt); } else { for (i = 0; i < qlge->intr_cnt; i++) { (void) ddi_intr_disable(qlge->htable[i]); } } qlge->sequence &= ~INIT_INTR_ENABLED; } for (i = 0; i < qlge->intr_cnt; i++) { if (qlge->sequence & INIT_ADD_INTERRUPT) (void) ddi_intr_remove_handler(qlge->htable[i]); if (qlge->sequence & INIT_INTR_ALLOC) { rc = ddi_intr_free(qlge->htable[i]); if (rc != DDI_SUCCESS) { /* EMPTY */ QL_PRINT(DBG_INIT, ("Free intr failed: %d", rc)); } } } if (qlge->sequence & INIT_INTR_ALLOC) qlge->sequence &= ~INIT_INTR_ALLOC; if (qlge->sequence & INIT_ADD_INTERRUPT) qlge->sequence &= ~INIT_ADD_INTERRUPT; if (qlge->htable) { kmem_free(qlge->htable, qlge->intr_size); qlge->htable = NULL; } } /* * Allocate interrupt vectors * For legacy and MSI, only 1 handle is needed. * For MSI-X, if fewer than 2 vectors are available, return failure. * Upon success, this maps the vectors to rx and tx rings for * interrupts. */ static int ql_request_irq_vectors(qlge_t *qlge, int intr_type) { dev_info_t *devinfo; uint32_t request, orig; int count, avail, actual; int minimum; int rc; devinfo = qlge->dip; switch (intr_type) { case DDI_INTR_TYPE_FIXED: request = 1; /* Request 1 legacy interrupt handle */ minimum = 1; QL_PRINT(DBG_INIT, ("interrupt type: legacy\n")); break; case DDI_INTR_TYPE_MSI: request = 1; /* Request 1 MSI interrupt handle */ minimum = 1; QL_PRINT(DBG_INIT, ("interrupt type: MSI\n")); break; case DDI_INTR_TYPE_MSIX: /* * Ideal number of vectors for the adapter is * # rss rings + tx completion rings for default completion * queue. */ request = qlge->rx_ring_count; orig = request; if (request > (MAX_RX_RINGS)) request = MAX_RX_RINGS; minimum = 2; QL_PRINT(DBG_INIT, ("interrupt type: MSI-X\n")); break; default: QL_PRINT(DBG_INIT, ("Invalid parameter\n")); return (DDI_FAILURE); } QL_PRINT(DBG_INIT, ("interrupt handles requested: %d minimum: %d\n", request, minimum)); /* * Get number of supported interrupts */ rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); if ((rc != DDI_SUCCESS) || (count < minimum)) { QL_PRINT(DBG_INIT, ("Get interrupt number failed. Return: %d, " "count: %d\n", rc, count)); return (DDI_FAILURE); } QL_PRINT(DBG_INIT, ("interrupts supported: %d\n", count)); /* * Get number of available interrupts */ rc = ddi_intr_get_navail(devinfo, intr_type, &avail); if ((rc != DDI_SUCCESS) || (avail < minimum)) { QL_PRINT(DBG_INIT, ("Get interrupt available number failed. Return:" " %d, available: %d\n", rc, avail)); return (DDI_FAILURE); } QL_PRINT(DBG_INIT, ("interrupts available: %d\n", avail)); if (avail < request) { QL_PRINT(DBG_INIT, ("Request %d handles, %d available\n", request, avail)); request = avail; } actual = 0; qlge->intr_cnt = 0; /* * Allocate an array of interrupt handles */ qlge->intr_size = (size_t)(request * sizeof (ddi_intr_handle_t)); qlge->htable = kmem_alloc(qlge->intr_size, KM_SLEEP); rc = ddi_intr_alloc(devinfo, qlge->htable, intr_type, 0, (int)request, &actual, DDI_INTR_ALLOC_NORMAL); if (rc != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d) Allocate interrupts failed. return:" " %d, request: %d, actual: %d", __func__, qlge->instance, rc, request, actual); goto ql_intr_alloc_fail; } qlge->intr_cnt = actual; qlge->sequence |= INIT_INTR_ALLOC; /* * If the actual number of vectors is less than the minumum * then fail. */ if (actual < minimum) { cmn_err(CE_WARN, "Insufficient interrupt handles available: %d", actual); goto ql_intr_alloc_fail; } /* * For MSI-X, actual might force us to reduce number of tx & rx rings */ if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { if (actual >= (orig / 2)) { count = orig / 2; qlge->rss_ring_count = count; qlge->tx_ring_count = count; qlge->isr_stride = count; } else if (actual >= (orig / 4)) { count = orig / 4; qlge->rss_ring_count = count; qlge->tx_ring_count = count; qlge->isr_stride = count; } else if (actual >= (orig / 8)) { count = orig / 8; qlge->rss_ring_count = count; qlge->tx_ring_count = count; qlge->isr_stride = count; } else if (actual < MAX_RX_RINGS) { qlge->tx_ring_count = 1; qlge->rss_ring_count = actual - 1; } qlge->intr_cnt = count; qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count; } cmn_err(CE_NOTE, "!qlge(%d) tx %d, rss %d, stride %d\n", qlge->instance, qlge->tx_ring_count, qlge->rss_ring_count, qlge->isr_stride); /* * Get priority for first vector, assume remaining are all the same */ rc = ddi_intr_get_pri(qlge->htable[0], &qlge->intr_pri); if (rc != DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("Get interrupt priority failed: %d\n", rc)); goto ql_intr_alloc_fail; } rc = ddi_intr_get_cap(qlge->htable[0], &qlge->intr_cap); if (rc != DDI_SUCCESS) { QL_PRINT(DBG_INIT, ("Get interrupt cap failed: %d\n", rc)); goto ql_intr_alloc_fail; } qlge->intr_type = intr_type; return (DDI_SUCCESS); ql_intr_alloc_fail: ql_free_irq_vectors(qlge); return (DDI_FAILURE); } /* * Allocate interrupt vector(s) for one of the following interrupt types, MSI-X, * MSI or Legacy. In MSI and Legacy modes we only support a single receive and * transmit queue. */ int ql_alloc_irqs(qlge_t *qlge) { int intr_types; int rval; /* * Get supported interrupt types */ if (ddi_intr_get_supported_types(qlge->dip, &intr_types) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d):ddi_intr_get_supported_types failed", __func__, qlge->instance); return (DDI_FAILURE); } QL_PRINT(DBG_INIT, ("%s(%d) Interrupt types supported %d\n", __func__, qlge->instance, intr_types)); /* Install MSI-X interrupts */ if ((intr_types & DDI_INTR_TYPE_MSIX) != 0) { QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt supported %d\n", __func__, qlge->instance, intr_types)); rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSIX); if (rval == DDI_SUCCESS) { return (rval); } QL_PRINT(DBG_INIT, ("%s(%d) MSI-X interrupt allocation failed," " trying MSI interrupts ...\n", __func__, qlge->instance)); } /* * We will have 2 completion queues in MSI / Legacy mode, * Queue 0 for default completions * Queue 1 for transmit completions */ qlge->rss_ring_count = 1; /* Default completion queue (0) for all */ qlge->tx_ring_count = 1; /* Single tx completion queue */ qlge->rx_ring_count = qlge->tx_ring_count + qlge->rss_ring_count; QL_PRINT(DBG_INIT, ("%s(%d) Falling back to single completion queue \n", __func__, qlge->instance)); /* * Add the h/w interrupt handler and initialise mutexes */ rval = DDI_FAILURE; /* * If OS supports MSIX interrupt but fails to allocate, then try * MSI interrupt. If MSI interrupt allocation fails also, then roll * back to fixed interrupt. */ if (intr_types & DDI_INTR_TYPE_MSI) { rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_MSI); if (rval == DDI_SUCCESS) { qlge->intr_type = DDI_INTR_TYPE_MSI; QL_PRINT(DBG_INIT, ("%s(%d) use MSI Interrupt \n", __func__, qlge->instance)); } } /* Try Fixed interrupt Legacy mode */ if (rval != DDI_SUCCESS) { rval = ql_request_irq_vectors(qlge, DDI_INTR_TYPE_FIXED); if (rval != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d):Legacy mode interrupt " "allocation failed", __func__, qlge->instance); } else { qlge->intr_type = DDI_INTR_TYPE_FIXED; QL_PRINT(DBG_INIT, ("%s(%d) use Fixed Interrupt \n", __func__, qlge->instance)); } } return (rval); } static void ql_free_rx_tx_locks(qlge_t *qlge) { int i; struct rx_ring *rx_ring; struct tx_ring *tx_ring; for (i = 0; i < qlge->tx_ring_count; i++) { tx_ring = &qlge->tx_ring[i]; mutex_destroy(&tx_ring->tx_lock); } for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; mutex_destroy(&rx_ring->rx_lock); mutex_destroy(&rx_ring->sbq_lock); mutex_destroy(&rx_ring->lbq_lock); } } /* * Frees all resources allocated during attach. * * Input: * dip = pointer to device information structure. * sequence = bits indicating resources to free. * * Context: * Kernel context. */ static void ql_free_resources(qlge_t *qlge) { /* Disable driver timer */ ql_stop_timer(qlge); if (qlge->sequence & INIT_MAC_REGISTERED) { (void) mac_unregister(qlge->mh); qlge->sequence &= ~INIT_MAC_REGISTERED; } if (qlge->sequence & INIT_MAC_ALLOC) { /* Nothing to do, macp is already freed */ qlge->sequence &= ~INIT_MAC_ALLOC; } if (qlge->sequence & INIT_PCI_CONFIG_SETUP) { pci_config_teardown(&qlge->pci_handle); qlge->sequence &= ~INIT_PCI_CONFIG_SETUP; } if (qlge->sequence & INIT_INTR_ALLOC) { ql_free_irq_vectors(qlge); qlge->sequence &= ~INIT_ADD_INTERRUPT; } if (qlge->sequence & INIT_ADD_SOFT_INTERRUPT) { (void) ddi_intr_remove_softint(qlge->mpi_event_intr_hdl); (void) ddi_intr_remove_softint(qlge->mpi_reset_intr_hdl); (void) ddi_intr_remove_softint(qlge->asic_reset_intr_hdl); qlge->sequence &= ~INIT_ADD_SOFT_INTERRUPT; } if (qlge->sequence & INIT_KSTATS) { ql_fini_kstats(qlge); qlge->sequence &= ~INIT_KSTATS; } if (qlge->sequence & INIT_MUTEX) { mutex_destroy(&qlge->gen_mutex); mutex_destroy(&qlge->hw_mutex); mutex_destroy(&qlge->mbx_mutex); cv_destroy(&qlge->cv_mbx_intr); qlge->sequence &= ~INIT_MUTEX; } if (qlge->sequence & INIT_LOCKS_CREATED) { ql_free_rx_tx_locks(qlge); qlge->sequence &= ~INIT_LOCKS_CREATED; } if (qlge->sequence & INIT_MEMORY_ALLOC) { ql_free_mem_resources(qlge); qlge->sequence &= ~INIT_MEMORY_ALLOC; } if (qlge->sequence & INIT_REGS_SETUP) { ddi_regs_map_free(&qlge->dev_handle); qlge->sequence &= ~INIT_REGS_SETUP; } if (qlge->sequence & INIT_DOORBELL_REGS_SETUP) { ddi_regs_map_free(&qlge->dev_doorbell_reg_handle); qlge->sequence &= ~INIT_DOORBELL_REGS_SETUP; } /* * free flash flt table that allocated in attach stage */ if ((qlge->flt.ql_flt_entry_ptr != NULL)&& (qlge->flt.header.length != 0)) { kmem_free(qlge->flt.ql_flt_entry_ptr, qlge->flt.header.length); qlge->flt.ql_flt_entry_ptr = NULL; } if (qlge->sequence & INIT_FM) { ql_fm_fini(qlge); qlge->sequence &= ~INIT_FM; } ddi_prop_remove_all(qlge->dip); ddi_set_driver_private(qlge->dip, NULL); /* finally, free qlge structure */ if (qlge->sequence & INIT_SOFTSTATE_ALLOC) { kmem_free(qlge, sizeof (qlge_t)); } } /* * Set promiscuous mode of the driver * Caller must catch HW_LOCK */ void ql_set_promiscuous(qlge_t *qlge, int mode) { if (mode) { (void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 1); } else { (void) ql_set_routing_reg(qlge, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 0); } } /* * Write 'data1' to Mac Protocol Address Index Register and * 'data2' to Mac Protocol Address Data Register * Assuming that the Mac Protocol semaphore lock has been acquired. */ static int ql_write_mac_proto_regs(qlge_t *qlge, uint32_t data1, uint32_t data2) { int return_value = DDI_SUCCESS; if (ql_wait_reg_bit(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_PROTOCOL_ADDRESS_INDEX_MW, BIT_SET, 5) != DDI_SUCCESS) { cmn_err(CE_WARN, "Wait for MAC_PROTOCOL Address Register " "timeout."); return_value = DDI_FAILURE; goto out; } ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX /* A8 */, data1); ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA /* 0xAC */, data2); out: return (return_value); } /* * Enable the 'index'ed multicast address in the host memory's multicast_list */ int ql_add_multicast_address(qlge_t *qlge, int index) { int rtn_val = DDI_FAILURE; uint32_t offset; uint32_t value1, value2; /* Acquire the required semaphore */ if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) { return (rtn_val); } /* Program Offset0 - lower 32 bits of the MAC address */ offset = 0; value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST | (index << 4) | offset; value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24) |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16) |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8) |(qlge->multicast_list[index].addr.ether_addr_octet[5])); if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) goto out; /* Program offset1: upper 16 bits of the MAC address */ offset = 1; value1 = MAC_PROTOCOL_ADDRESS_ENABLE | MAC_PROTOCOL_TYPE_MULTICAST | (index<<4) | offset; value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[0] << 8) |qlge->multicast_list[index].addr.ether_addr_octet[1]); if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) { goto out; } rtn_val = DDI_SUCCESS; out: ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK); return (rtn_val); } /* * Disable the 'index'ed multicast address in the host memory's multicast_list */ int ql_remove_multicast_address(qlge_t *qlge, int index) { int rtn_val = DDI_FAILURE; uint32_t offset; uint32_t value1, value2; /* Acquire the required semaphore */ if (ql_sem_spinlock(qlge, QL_MAC_PROTOCOL_SEM_MASK) != DDI_SUCCESS) { return (rtn_val); } /* Program Offset0 - lower 32 bits of the MAC address */ offset = 0; value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4); value2 = ((qlge->multicast_list[index].addr.ether_addr_octet[2] << 24) |(qlge->multicast_list[index].addr.ether_addr_octet[3] << 16) |(qlge->multicast_list[index].addr.ether_addr_octet[4] << 8) |(qlge->multicast_list[index].addr.ether_addr_octet[5])); if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) { goto out; } /* Program offset1: upper 16 bits of the MAC address */ offset = 1; value1 = (MAC_PROTOCOL_TYPE_MULTICAST | offset)|(index<<4); value2 = 0; if (ql_write_mac_proto_regs(qlge, value1, value2) != DDI_SUCCESS) { goto out; } rtn_val = DDI_SUCCESS; out: ql_sem_unlock(qlge, QL_MAC_PROTOCOL_SEM_MASK); return (rtn_val); } /* * Add a new multicast address to the list of supported list * This API is called after OS called gld_set_multicast (GLDv2) * or m_multicst (GLDv3) * * Restriction: * The number of maximum multicast address is limited by hardware. */ int ql_add_to_multicast_list(qlge_t *qlge, uint8_t *ep) { uint32_t index = qlge->multicast_list_count; int rval = DDI_SUCCESS; int status; if ((ep[0] & 01) == 0) { rval = EINVAL; goto exit; } /* if there is an availabe space in multicast_list, then add it */ if (index < MAX_MULTICAST_LIST_SIZE) { bcopy(ep, qlge->multicast_list[index].addr.ether_addr_octet, ETHERADDRL); /* increment the total number of addresses in multicast list */ (void) ql_add_multicast_address(qlge, index); qlge->multicast_list_count++; QL_PRINT(DBG_GLD, ("%s(%d): added to index of multicast list= 0x%x, " "total %d\n", __func__, qlge->instance, index, qlge->multicast_list_count)); if (index > MAX_MULTICAST_HW_SIZE) { if (!qlge->multicast_promisc) { status = ql_set_routing_reg(qlge, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 1); if (status) { cmn_err(CE_WARN, "Failed to init routing reg " "for mcast promisc mode."); rval = ENOENT; goto exit; } qlge->multicast_promisc = B_TRUE; } } } else { rval = ENOENT; } exit: return (rval); } /* * Remove an old multicast address from the list of supported multicast * addresses. This API is called after OS called gld_set_multicast (GLDv2) * or m_multicst (GLDv3) * The number of maximum multicast address is limited by hardware. */ int ql_remove_from_multicast_list(qlge_t *qlge, uint8_t *ep) { uint32_t total = qlge->multicast_list_count; int i = 0; int rmv_index = 0; size_t length = sizeof (ql_multicast_addr); int status; for (i = 0; i < total; i++) { if (bcmp(ep, &qlge->multicast_list[i].addr, ETHERADDRL) != 0) { continue; } rmv_index = i; /* block move the reset of other multicast address forward */ length = ((total -1) -i) * sizeof (ql_multicast_addr); if (length > 0) { bcopy(&qlge->multicast_list[i+1], &qlge->multicast_list[i], length); } qlge->multicast_list_count--; if (qlge->multicast_list_count <= MAX_MULTICAST_HW_SIZE) { /* * there is a deletion in multicast list table, * re-enable them */ for (i = rmv_index; i < qlge->multicast_list_count; i++) { (void) ql_add_multicast_address(qlge, i); } /* and disable the last one */ (void) ql_remove_multicast_address(qlge, i); /* disable multicast promiscuous mode */ if (qlge->multicast_promisc) { status = ql_set_routing_reg(qlge, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 0); if (status) { cmn_err(CE_WARN, "Failed to init routing reg for " "mcast promisc mode."); goto exit; } /* write to config register */ qlge->multicast_promisc = B_FALSE; } } break; } exit: return (DDI_SUCCESS); } /* * Read a XGMAC register */ int ql_read_xgmac_reg(qlge_t *qlge, uint32_t addr, uint32_t *val) { int rtn_val = DDI_FAILURE; /* wait for XGMAC Address register RDY bit set */ if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY, BIT_SET, 10) != DDI_SUCCESS) { goto out; } /* start rx transaction */ ql_write_reg(qlge, REG_XGMAC_ADDRESS, addr|XGMAC_ADDRESS_READ_TRANSACT); /* * wait for XGMAC Address register RDY bit set, * which indicates data is ready */ if (ql_wait_reg_bit(qlge, REG_XGMAC_ADDRESS, XGMAC_ADDRESS_RDY, BIT_SET, 10) != DDI_SUCCESS) { goto out; } /* read data from XGAMC_DATA register */ *val = ql_read_reg(qlge, REG_XGMAC_DATA); rtn_val = DDI_SUCCESS; out: return (rtn_val); } /* * Implement checksum offload for IPv4 IP packets */ static void ql_hw_csum_setup(qlge_t *qlge, uint32_t pflags, caddr_t bp, struct ob_mac_iocb_req *mac_iocb_ptr) { struct ip *iphdr = NULL; struct ether_header *ethhdr; struct ether_vlan_header *ethvhdr; struct tcphdr *tcp_hdr; uint32_t etherType; int mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len; int ip_hdr_off, tcp_udp_hdr_off, hdr_off; ethhdr = (struct ether_header *)((void *)bp); ethvhdr = (struct ether_vlan_header *)((void *)bp); /* Is this vlan packet? */ if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) { mac_hdr_len = sizeof (struct ether_vlan_header); etherType = ntohs(ethvhdr->ether_type); } else { mac_hdr_len = sizeof (struct ether_header); etherType = ntohs(ethhdr->ether_type); } /* Is this IPv4 or IPv6 packet? */ if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len)) == IPV4_VERSION) { if (etherType == ETHERTYPE_IP /* 0800 */) { iphdr = (struct ip *)(void *)(bp+mac_hdr_len); } else { /* EMPTY */ QL_PRINT(DBG_TX, ("%s(%d) : IPv4 None IP packet type 0x%x\n", __func__, qlge->instance, etherType)); } } /* ipV4 packets */ if (iphdr != NULL) { ip_hdr_len = IPH_HDR_LENGTH(iphdr); QL_PRINT(DBG_TX, ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH:" " %d bytes \n", __func__, qlge->instance, ip_hdr_len)); ip_hdr_off = mac_hdr_len; QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n", __func__, qlge->instance, ip_hdr_len)); mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 | OB_MAC_IOCB_REQ_IPv4); if (pflags & HCK_IPV4_HDRCKSUM) { QL_PRINT(DBG_TX, ("%s(%d) : Do IPv4 header checksum\n", __func__, qlge->instance)); mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB; mac_iocb_ptr->flag2 = (uint8_t)(mac_iocb_ptr->flag2 | OB_MAC_IOCB_REQ_IC); iphdr->ip_sum = 0; mac_iocb_ptr->hdr_off = (uint16_t) cpu_to_le16(ip_hdr_off); } if (pflags & HCK_FULLCKSUM) { if (iphdr->ip_p == IPPROTO_TCP) { tcp_hdr = (struct tcphdr *)(void *) ((uint8_t *)(void *)iphdr + ip_hdr_len); QL_PRINT(DBG_TX, ("%s(%d) : Do TCP checksum\n", __func__, qlge->instance)); mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB; mac_iocb_ptr->flag1 = (uint8_t)(mac_iocb_ptr->flag1 | OB_MAC_IOCB_REQ_TC); mac_iocb_ptr->flag2 = (uint8_t)(mac_iocb_ptr->flag2 | OB_MAC_IOCB_REQ_IC); iphdr->ip_sum = 0; tcp_udp_hdr_off = mac_hdr_len+ip_hdr_len; tcp_udp_hdr_len = tcp_hdr->th_off*4; QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n", __func__, qlge->instance, tcp_udp_hdr_len)); hdr_off = ip_hdr_off; tcp_udp_hdr_off <<= 6; hdr_off |= tcp_udp_hdr_off; mac_iocb_ptr->hdr_off = (uint16_t)cpu_to_le16(hdr_off); mac_iocb_ptr->protocol_hdr_len = (uint16_t) cpu_to_le16(mac_hdr_len + ip_hdr_len + tcp_udp_hdr_len); /* * if the chip is unable to do pseudo header * cksum calculation, do it in then put the * result to the data passed to the chip */ if (qlge->cfg_flags & CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) { ql_pseudo_cksum((uint8_t *)iphdr); } } else if (iphdr->ip_p == IPPROTO_UDP) { QL_PRINT(DBG_TX, ("%s(%d) : Do UDP checksum\n", __func__, qlge->instance)); mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB; mac_iocb_ptr->flag1 = (uint8_t)(mac_iocb_ptr->flag1 | OB_MAC_IOCB_REQ_UC); mac_iocb_ptr->flag2 = (uint8_t)(mac_iocb_ptr->flag2 | OB_MAC_IOCB_REQ_IC); iphdr->ip_sum = 0; tcp_udp_hdr_off = mac_hdr_len + ip_hdr_len; tcp_udp_hdr_len = sizeof (struct udphdr); QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n", __func__, qlge->instance, tcp_udp_hdr_len)); hdr_off = ip_hdr_off; tcp_udp_hdr_off <<= 6; hdr_off |= tcp_udp_hdr_off; mac_iocb_ptr->hdr_off = (uint16_t)cpu_to_le16(hdr_off); mac_iocb_ptr->protocol_hdr_len = (uint16_t) cpu_to_le16(mac_hdr_len + ip_hdr_len + tcp_udp_hdr_len); /* * if the chip is unable to calculate pseudo * hdr cksum,do it in then put the result to * the data passed to the chip */ if (qlge->cfg_flags & CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) { ql_pseudo_cksum((uint8_t *)iphdr); } } } } } /* * For TSO/LSO: * MAC frame transmission with TCP large segment offload is performed in the * same way as the MAC frame transmission with checksum offload with the * exception that the maximum TCP segment size (MSS) must be specified to * allow the chip to segment the data into legal sized frames. * The host also needs to calculate a pseudo-header checksum over the * following fields: * Source IP Address, Destination IP Address, and the Protocol. * The TCP length is not included in the pseudo-header calculation. * The pseudo-header checksum is place in the TCP checksum field of the * prototype header. */ static void ql_lso_pseudo_cksum(uint8_t *buf) { uint32_t cksum; uint16_t iphl; uint16_t proto; /* * Calculate the LSO pseudo-header checksum. */ iphl = (uint16_t)(4 * (buf[0] & 0xF)); cksum = proto = buf[9]; cksum += (((uint16_t)buf[12])<<8) + buf[13]; cksum += (((uint16_t)buf[14])<<8) + buf[15]; cksum += (((uint16_t)buf[16])<<8) + buf[17]; cksum += (((uint16_t)buf[18])<<8) + buf[19]; cksum = (cksum>>16) + (cksum & 0xFFFF); cksum = (cksum>>16) + (cksum & 0xFFFF); /* * Point it to the TCP/UDP header, and * update the checksum field. */ buf += iphl + ((proto == IPPROTO_TCP) ? TCP_CKSUM_OFFSET : UDP_CKSUM_OFFSET); *(uint16_t *)(void *)buf = (uint16_t)htons((uint16_t)cksum); } /* * For IPv4 IP packets, distribute the tx packets evenly among tx rings */ typedef uint32_t ub4; /* unsigned 4-byte quantities */ typedef uint8_t ub1; #define hashsize(n) ((ub4)1<<(n)) #define hashmask(n) (hashsize(n)-1) #define mix(a, b, c) \ { \ a -= b; a -= c; a ^= (c>>13); \ b -= c; b -= a; b ^= (a<<8); \ c -= a; c -= b; c ^= (b>>13); \ a -= b; a -= c; a ^= (c>>12); \ b -= c; b -= a; b ^= (a<<16); \ c -= a; c -= b; c ^= (b>>5); \ a -= b; a -= c; a ^= (c>>3); \ b -= c; b -= a; b ^= (a<<10); \ c -= a; c -= b; c ^= (b>>15); \ } ub4 hash(ub1 *k, ub4 length, ub4 initval) { ub4 a, b, c, len; /* Set up the internal state */ len = length; a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ c = initval; /* the previous hash value */ /* handle most of the key */ while (len >= 12) { a += (k[0] +((ub4)k[1]<<8) +((ub4)k[2]<<16) +((ub4)k[3]<<24)); b += (k[4] +((ub4)k[5]<<8) +((ub4)k[6]<<16) +((ub4)k[7]<<24)); c += (k[8] +((ub4)k[9]<<8) +((ub4)k[10]<<16)+((ub4)k[11]<<24)); mix(a, b, c); k += 12; len -= 12; } /* handle the last 11 bytes */ c += length; /* all the case statements fall through */ switch (len) { /* FALLTHRU */ case 11: c += ((ub4)k[10]<<24); /* FALLTHRU */ case 10: c += ((ub4)k[9]<<16); /* FALLTHRU */ case 9 : c += ((ub4)k[8]<<8); /* the first byte of c is reserved for the length */ /* FALLTHRU */ case 8 : b += ((ub4)k[7]<<24); /* FALLTHRU */ case 7 : b += ((ub4)k[6]<<16); /* FALLTHRU */ case 6 : b += ((ub4)k[5]<<8); /* FALLTHRU */ case 5 : b += k[4]; /* FALLTHRU */ case 4 : a += ((ub4)k[3]<<24); /* FALLTHRU */ case 3 : a += ((ub4)k[2]<<16); /* FALLTHRU */ case 2 : a += ((ub4)k[1]<<8); /* FALLTHRU */ case 1 : a += k[0]; /* case 0: nothing left to add */ } mix(a, b, c); /* report the result */ return (c); } uint8_t ql_tx_hashing(qlge_t *qlge, caddr_t bp) { struct ip *iphdr = NULL; struct ether_header *ethhdr; struct ether_vlan_header *ethvhdr; struct tcphdr *tcp_hdr; struct udphdr *udp_hdr; uint32_t etherType; int mac_hdr_len, ip_hdr_len; uint32_t h = 0; /* 0 by default */ uint8_t tx_ring_id = 0; uint32_t ip_src_addr = 0; uint32_t ip_desc_addr = 0; uint16_t src_port = 0; uint16_t dest_port = 0; uint8_t key[12]; QL_PRINT(DBG_TX, ("%s(%d) entered \n", __func__, qlge->instance)); ethhdr = (struct ether_header *)((void *)bp); ethvhdr = (struct ether_vlan_header *)((void *)bp); if (qlge->tx_ring_count == 1) return (tx_ring_id); /* Is this vlan packet? */ if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) { mac_hdr_len = sizeof (struct ether_vlan_header); etherType = ntohs(ethvhdr->ether_type); } else { mac_hdr_len = sizeof (struct ether_header); etherType = ntohs(ethhdr->ether_type); } /* Is this IPv4 or IPv6 packet? */ if (etherType == ETHERTYPE_IP /* 0800 */) { if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len)) == IPV4_VERSION) { iphdr = (struct ip *)(void *)(bp+mac_hdr_len); } if (((unsigned long)iphdr) & 0x3) { /* IP hdr not 4-byte aligned */ return (tx_ring_id); } } /* ipV4 packets */ if (iphdr) { ip_hdr_len = IPH_HDR_LENGTH(iphdr); ip_src_addr = iphdr->ip_src.s_addr; ip_desc_addr = iphdr->ip_dst.s_addr; if (iphdr->ip_p == IPPROTO_TCP) { tcp_hdr = (struct tcphdr *)(void *) ((uint8_t *)iphdr + ip_hdr_len); src_port = tcp_hdr->th_sport; dest_port = tcp_hdr->th_dport; } else if (iphdr->ip_p == IPPROTO_UDP) { udp_hdr = (struct udphdr *)(void *) ((uint8_t *)iphdr + ip_hdr_len); src_port = udp_hdr->uh_sport; dest_port = udp_hdr->uh_dport; } key[0] = (uint8_t)((ip_src_addr) &0xFF); key[1] = (uint8_t)((ip_src_addr >> 8) &0xFF); key[2] = (uint8_t)((ip_src_addr >> 16) &0xFF); key[3] = (uint8_t)((ip_src_addr >> 24) &0xFF); key[4] = (uint8_t)((ip_desc_addr) &0xFF); key[5] = (uint8_t)((ip_desc_addr >> 8) &0xFF); key[6] = (uint8_t)((ip_desc_addr >> 16) &0xFF); key[7] = (uint8_t)((ip_desc_addr >> 24) &0xFF); key[8] = (uint8_t)((src_port) &0xFF); key[9] = (uint8_t)((src_port >> 8) &0xFF); key[10] = (uint8_t)((dest_port) &0xFF); key[11] = (uint8_t)((dest_port >> 8) &0xFF); h = hash(key, 12, 0); /* return 32 bit */ tx_ring_id = (h & (qlge->tx_ring_count - 1)); if (tx_ring_id >= qlge->tx_ring_count) { cmn_err(CE_WARN, "%s bad tx_ring_id %d\n", __func__, tx_ring_id); tx_ring_id = 0; } } return (tx_ring_id); } /* * Tell the hardware to do Large Send Offload (LSO) * * Some fields in ob_mac_iocb need to be set so hardware can know what is * the incoming packet, TCP or UDP, whether a VLAN tag needs to be inserted * in the right place of the packet etc, thus, hardware can process the * packet correctly. */ static void ql_hw_lso_setup(qlge_t *qlge, uint32_t mss, caddr_t bp, struct ob_mac_iocb_req *mac_iocb_ptr) { struct ip *iphdr = NULL; struct ether_header *ethhdr; struct ether_vlan_header *ethvhdr; struct tcphdr *tcp_hdr; struct udphdr *udp_hdr; uint32_t etherType; uint16_t mac_hdr_len, ip_hdr_len, tcp_udp_hdr_len; uint16_t ip_hdr_off, tcp_udp_hdr_off, hdr_off; ethhdr = (struct ether_header *)(void *)bp; ethvhdr = (struct ether_vlan_header *)(void *)bp; /* Is this vlan packet? */ if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) { mac_hdr_len = sizeof (struct ether_vlan_header); etherType = ntohs(ethvhdr->ether_type); } else { mac_hdr_len = sizeof (struct ether_header); etherType = ntohs(ethhdr->ether_type); } /* Is this IPv4 or IPv6 packet? */ if (IPH_HDR_VERSION((ipha_t *)(void *)(bp + mac_hdr_len)) == IPV4_VERSION) { if (etherType == ETHERTYPE_IP /* 0800 */) { iphdr = (struct ip *)(void *)(bp+mac_hdr_len); } else { /* EMPTY */ QL_PRINT(DBG_TX, ("%s(%d) : IPv4 None IP packet" " type 0x%x\n", __func__, qlge->instance, etherType)); } } if (iphdr != NULL) { /* ipV4 packets */ ip_hdr_len = (uint16_t)IPH_HDR_LENGTH(iphdr); QL_PRINT(DBG_TX, ("%s(%d) : IPv4 header length using IPH_HDR_LENGTH: %d" " bytes \n", __func__, qlge->instance, ip_hdr_len)); ip_hdr_off = mac_hdr_len; QL_PRINT(DBG_TX, ("%s(%d) : ip_hdr_len=%d\n", __func__, qlge->instance, ip_hdr_len)); mac_iocb_ptr->flag0 = (uint8_t)(mac_iocb_ptr->flag0 | OB_MAC_IOCB_REQ_IPv4); if (qlge->cfg_flags & CFG_CKSUM_FULL_IPv4) { if (iphdr->ip_p == IPPROTO_TCP) { tcp_hdr = (struct tcphdr *)(void *) ((uint8_t *)(void *)iphdr + ip_hdr_len); QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on TCP " "packet\n", __func__, qlge->instance)); mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB; mac_iocb_ptr->flag1 = (uint8_t)(mac_iocb_ptr->flag1 | OB_MAC_IOCB_REQ_LSO); iphdr->ip_sum = 0; tcp_udp_hdr_off = (uint16_t)(mac_hdr_len+ip_hdr_len); tcp_udp_hdr_len = (uint16_t)(tcp_hdr->th_off*4); QL_PRINT(DBG_TX, ("%s(%d): tcp header len:%d\n", __func__, qlge->instance, tcp_udp_hdr_len)); hdr_off = ip_hdr_off; tcp_udp_hdr_off <<= 6; hdr_off |= tcp_udp_hdr_off; mac_iocb_ptr->hdr_off = (uint16_t)cpu_to_le16(hdr_off); mac_iocb_ptr->protocol_hdr_len = (uint16_t) cpu_to_le16(mac_hdr_len + ip_hdr_len + tcp_udp_hdr_len); mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss); /* * if the chip is unable to calculate pseudo * header checksum, do it in then put the result * to the data passed to the chip */ if (qlge->cfg_flags & CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) ql_lso_pseudo_cksum((uint8_t *)iphdr); } else if (iphdr->ip_p == IPPROTO_UDP) { udp_hdr = (struct udphdr *)(void *) ((uint8_t *)(void *)iphdr + ip_hdr_len); QL_PRINT(DBG_TX, ("%s(%d) : Do TSO on UDP " "packet\n", __func__, qlge->instance)); mac_iocb_ptr->opcode = OPCODE_OB_MAC_OFFLOAD_IOCB; mac_iocb_ptr->flag1 = (uint8_t)(mac_iocb_ptr->flag1 | OB_MAC_IOCB_REQ_LSO); iphdr->ip_sum = 0; tcp_udp_hdr_off = (uint16_t)(mac_hdr_len+ip_hdr_len); tcp_udp_hdr_len = (uint16_t)(udp_hdr->uh_ulen*4); QL_PRINT(DBG_TX, ("%s(%d):udp header len:%d\n", __func__, qlge->instance, tcp_udp_hdr_len)); hdr_off = ip_hdr_off; tcp_udp_hdr_off <<= 6; hdr_off |= tcp_udp_hdr_off; mac_iocb_ptr->hdr_off = (uint16_t)cpu_to_le16(hdr_off); mac_iocb_ptr->protocol_hdr_len = (uint16_t) cpu_to_le16(mac_hdr_len + ip_hdr_len + tcp_udp_hdr_len); mac_iocb_ptr->mss = (uint16_t)cpu_to_le16(mss); /* * if the chip is unable to do pseudo header * checksum calculation, do it here then put the * result to the data passed to the chip */ if (qlge->cfg_flags & CFG_HW_UNABLE_PSEUDO_HDR_CKSUM) ql_lso_pseudo_cksum((uint8_t *)iphdr); } } } } /* * Generic packet sending function which is used to send one packet. */ int ql_send_common(struct tx_ring *tx_ring, mblk_t *mp) { struct tx_ring_desc *tx_cb; struct ob_mac_iocb_req *mac_iocb_ptr; mblk_t *tp; size_t msg_len = 0; size_t off; caddr_t bp; size_t nbyte, total_len; uint_t i = 0; int j = 0, frags = 0; uint32_t phy_addr_low, phy_addr_high; uint64_t phys_addr; clock_t now; uint32_t pflags = 0; uint32_t mss = 0; enum tx_mode_t tx_mode; struct oal_entry *oal_entry; int status; uint_t ncookies, oal_entries, max_oal_entries; size_t max_seg_len = 0; boolean_t use_lso = B_FALSE; struct oal_entry *tx_entry = NULL; struct oal_entry *last_oal_entry; qlge_t *qlge = tx_ring->qlge; ddi_dma_cookie_t dma_cookie; size_t tx_buf_len = QL_MAX_COPY_LENGTH; int force_pullup = 0; tp = mp; total_len = msg_len = 0; max_oal_entries = TX_DESC_PER_IOCB + MAX_SG_ELEMENTS-1; /* Calculate number of data and segments in the incoming message */ for (tp = mp; tp != NULL; tp = tp->b_cont) { nbyte = MBLKL(tp); total_len += nbyte; max_seg_len = max(nbyte, max_seg_len); QL_PRINT(DBG_TX, ("Requested sending data in %d segments, " "total length: %d\n", frags, nbyte)); frags++; } if (total_len >= QL_LSO_MAX) { freemsg(mp); #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "%s: quit, packet oversize %d\n", __func__, (int)total_len); #endif return (0); } bp = (caddr_t)mp->b_rptr; if (bp[0] & 1) { if (bcmp(bp, ql_ether_broadcast_addr.ether_addr_octet, ETHERADDRL) == 0) { QL_PRINT(DBG_TX, ("Broadcast packet\n")); tx_ring->brdcstxmt++; } else { QL_PRINT(DBG_TX, ("multicast packet\n")); tx_ring->multixmt++; } } tx_ring->obytes += total_len; tx_ring->opackets ++; QL_PRINT(DBG_TX, ("total requested sending data length: %d, in %d segs," " max seg len: %d\n", total_len, frags, max_seg_len)); /* claim a free slot in tx ring */ tx_cb = &tx_ring->wq_desc[tx_ring->prod_idx]; /* get the tx descriptor */ mac_iocb_ptr = tx_cb->queue_entry; bzero((void *)mac_iocb_ptr, 20); ASSERT(tx_cb->mp == NULL); /* * Decide to use DMA map or copy mode. * DMA map mode must be used when the total msg length is more than the * tx buffer length. */ if (total_len > tx_buf_len) tx_mode = USE_DMA; else if (max_seg_len > QL_MAX_COPY_LENGTH) tx_mode = USE_DMA; else tx_mode = USE_COPY; if (qlge->chksum_cap) { mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags); QL_PRINT(DBG_TX, ("checksum flag is :0x%x, card capability " "is 0x%x \n", pflags, qlge->chksum_cap)); if (qlge->lso_enable) { uint32_t lso_flags = 0; mac_lso_get(mp, &mss, &lso_flags); use_lso = (lso_flags == HW_LSO); } QL_PRINT(DBG_TX, ("mss :%d, use_lso %x \n", mss, use_lso)); } do_pullup: /* concatenate all frags into one large packet if too fragmented */ if (((tx_mode == USE_DMA)&&(frags > QL_MAX_TX_DMA_HANDLES)) || force_pullup) { mblk_t *mp1; if ((mp1 = msgpullup(mp, -1)) != NULL) { freemsg(mp); mp = mp1; frags = 1; } else { tx_ring->tx_fail_dma_bind++; goto bad; } } tx_cb->tx_bytes = (uint32_t)total_len; tx_cb->mp = mp; tx_cb->tx_dma_handle_used = 0; if (tx_mode == USE_DMA) { msg_len = total_len; mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB; mac_iocb_ptr->tid = tx_ring->prod_idx; mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len); mac_iocb_ptr->txq_idx = tx_ring->wq_id; tx_entry = &mac_iocb_ptr->oal_entry[0]; oal_entry = NULL; for (tp = mp, oal_entries = j = 0; tp != NULL; tp = tp->b_cont) { /* if too many tx dma handles needed */ if (j >= QL_MAX_TX_DMA_HANDLES) { tx_ring->tx_no_dma_handle++; if (!force_pullup) { force_pullup = 1; goto do_pullup; } else { goto bad; } } nbyte = (uint16_t)MBLKL(tp); if (nbyte == 0) continue; status = ddi_dma_addr_bind_handle( tx_cb->tx_dma_handle[j], NULL, (caddr_t)tp->b_rptr, nbyte, DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0, &dma_cookie, &ncookies); QL_PRINT(DBG_TX, ("map sending data segment: %d, " "length: %d, spans in %d cookies\n", j, nbyte, ncookies)); if (status != DDI_DMA_MAPPED) { goto bad; } /* * Each fragment can span several cookies. One cookie * will use one tx descriptor to transmit. */ for (i = ncookies; i > 0; i--, tx_entry++, oal_entries++) { /* * The number of TX descriptors that can be * saved in tx iocb and oal list is limited */ if (oal_entries > max_oal_entries) { tx_ring->tx_no_dma_cookie++; if (!force_pullup) { force_pullup = 1; goto do_pullup; } else { goto bad; } } if ((oal_entries == TX_DESC_PER_IOCB) && !oal_entry) { /* * Time to switch to an oal list * The last entry should be copied * to first entry in the oal list */ oal_entry = tx_cb->oal; tx_entry = &mac_iocb_ptr->oal_entry[ TX_DESC_PER_IOCB-1]; bcopy(tx_entry, oal_entry, sizeof (*oal_entry)); /* * last entry should be updated to * point to the extended oal list itself */ tx_entry->buf_addr_low = cpu_to_le32( LS_64BITS(tx_cb->oal_dma_addr)); tx_entry->buf_addr_high = cpu_to_le32( MS_64BITS(tx_cb->oal_dma_addr)); /* * Point tx_entry to the oal list * second entry */ tx_entry = &oal_entry[1]; } tx_entry->buf_len = (uint32_t)cpu_to_le32(dma_cookie.dmac_size); phys_addr = dma_cookie.dmac_laddress; tx_entry->buf_addr_low = cpu_to_le32(LS_64BITS(phys_addr)); tx_entry->buf_addr_high = cpu_to_le32(MS_64BITS(phys_addr)); last_oal_entry = tx_entry; if (i > 1) ddi_dma_nextcookie( tx_cb->tx_dma_handle[j], &dma_cookie); } j++; } /* * if OAL is used, the last oal entry in tx iocb indicates * number of additional address/len pairs in OAL */ if (oal_entries > TX_DESC_PER_IOCB) { tx_entry = &mac_iocb_ptr->oal_entry[TX_DESC_PER_IOCB-1]; tx_entry->buf_len = (uint32_t) (cpu_to_le32((sizeof (struct oal_entry) * (oal_entries -TX_DESC_PER_IOCB+1))|OAL_CONT_ENTRY)); } last_oal_entry->buf_len = cpu_to_le32( le32_to_cpu(last_oal_entry->buf_len)|OAL_LAST_ENTRY); tx_cb->tx_dma_handle_used = j; QL_PRINT(DBG_TX, ("total tx_dma_handle_used %d cookies %d \n", j, oal_entries)); bp = (caddr_t)mp->b_rptr; } if (tx_mode == USE_COPY) { bp = tx_cb->copy_buffer; off = 0; nbyte = 0; frags = 0; /* * Copy up to tx_buf_len of the transmit data * from mp to tx buffer */ for (tp = mp; tp != NULL; tp = tp->b_cont) { nbyte = MBLKL(tp); if ((off + nbyte) <= tx_buf_len) { bcopy(tp->b_rptr, &bp[off], nbyte); off += nbyte; frags ++; } } msg_len = off; mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB; mac_iocb_ptr->tid = tx_ring->prod_idx; mac_iocb_ptr->frame_len = (uint32_t)cpu_to_le32(msg_len); mac_iocb_ptr->txq_idx = tx_ring->wq_id; QL_PRINT(DBG_TX, ("Copy Mode:actual sent data length is: %d, " "from %d segaments\n", msg_len, frags)); phys_addr = tx_cb->copy_buffer_dma_addr; phy_addr_low = cpu_to_le32(LS_64BITS(phys_addr)); phy_addr_high = cpu_to_le32(MS_64BITS(phys_addr)); QL_DUMP(DBG_TX, "\t requested sending data:\n", (uint8_t *)tx_cb->copy_buffer, 8, total_len); mac_iocb_ptr->oal_entry[0].buf_len = (uint32_t) cpu_to_le32(msg_len | OAL_LAST_ENTRY); mac_iocb_ptr->oal_entry[0].buf_addr_low = phy_addr_low; mac_iocb_ptr->oal_entry[0].buf_addr_high = phy_addr_high; freemsg(mp); /* no need, we have copied */ tx_cb->mp = NULL; } /* End of Copy Mode */ /* Do TSO/LSO on TCP packet? */ if (use_lso && mss) { ql_hw_lso_setup(qlge, mss, bp, mac_iocb_ptr); } else if (pflags & qlge->chksum_cap) { /* Do checksum offloading */ ql_hw_csum_setup(qlge, pflags, bp, mac_iocb_ptr); } /* let device know the latest outbound IOCB */ (void) ddi_dma_sync(tx_ring->wq_dma.dma_handle, (off_t)((uintptr_t)mac_iocb_ptr - (uintptr_t)tx_ring->wq_dma.vaddr), (size_t)sizeof (*mac_iocb_ptr), DDI_DMA_SYNC_FORDEV); if (tx_mode == USE_DMA) { /* let device know the latest outbound OAL if necessary */ if (oal_entries > TX_DESC_PER_IOCB) { (void) ddi_dma_sync(tx_cb->oal_dma.dma_handle, (off_t)0, (sizeof (struct oal_entry) * (oal_entries -TX_DESC_PER_IOCB+1)), DDI_DMA_SYNC_FORDEV); } } else { /* for USE_COPY mode, tx buffer has changed */ /* let device know the latest change */ (void) ddi_dma_sync(tx_cb->oal_dma.dma_handle, /* copy buf offset */ (off_t)(sizeof (oal_entry) * MAX_SG_ELEMENTS), msg_len, DDI_DMA_SYNC_FORDEV); } /* save how the packet was sent */ tx_cb->tx_type = tx_mode; QL_DUMP_REQ_PKT(qlge, mac_iocb_ptr, tx_cb->oal, oal_entries); /* reduce the number of available tx slot */ atomic_dec_32(&tx_ring->tx_free_count); tx_ring->prod_idx++; if (tx_ring->prod_idx >= tx_ring->wq_len) tx_ring->prod_idx = 0; now = ddi_get_lbolt(); qlge->last_tx_time = now; return (DDI_SUCCESS); bad: /* * if for any reason driver can not send, delete * the message pointer, mp */ now = ddi_get_lbolt(); freemsg(mp); mp = NULL; tx_cb->mp = NULL; for (i = 0; i < j; i++) (void) ddi_dma_unbind_handle(tx_cb->tx_dma_handle[i]); QL_PRINT(DBG_TX, ("%s(%d) failed at 0x%x", __func__, qlge->instance, (int)now)); return (DDI_SUCCESS); } /* * Initializes hardware and driver software flags before the driver * is finally ready to work. */ int ql_do_start(qlge_t *qlge) { int i; struct rx_ring *rx_ring; uint16_t lbq_buf_size; int rings_done; ASSERT(qlge != NULL); mutex_enter(&qlge->hw_mutex); /* Reset adapter */ (void) ql_asic_reset(qlge); lbq_buf_size = (uint16_t) ((qlge->mtu == ETHERMTU)? LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE); if (qlge->rx_ring[0].lbq_buf_size != lbq_buf_size) { #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "realloc buffers old: %d new: %d\n", qlge->rx_ring[0].lbq_buf_size, lbq_buf_size); #endif /* * Check if any ring has buffers still with upper layers * If buffers are pending with upper layers, we use the * existing buffers and don't reallocate new ones * Unfortunately there is no way to evict buffers from * upper layers. Using buffers with the current size may * cause slightly sub-optimal performance, but that seems * to be the easiest way to handle this situation. */ rings_done = 0; for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (rx_ring->rx_indicate == 0) rings_done++; else break; } /* * No buffers pending with upper layers; * reallocte them for new MTU size */ if (rings_done >= qlge->rx_ring_count) { /* free large buffer pool */ for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (rx_ring->type != TX_Q) { ql_free_sbq_buffers(rx_ring); ql_free_lbq_buffers(rx_ring); } } /* reallocate large buffer pool */ for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; if (rx_ring->type != TX_Q) { (void) ql_alloc_sbufs(qlge, rx_ring); (void) ql_alloc_lbufs(qlge, rx_ring); } } } } if (ql_bringup_adapter(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "qlge bringup adapter failed"); mutex_exit(&qlge->hw_mutex); if (qlge->fm_enable) { atomic_or_32(&qlge->flags, ADAPTER_ERROR); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); } return (DDI_FAILURE); } mutex_exit(&qlge->hw_mutex); /* if adapter is up successfully but was bad before */ if (qlge->flags & ADAPTER_ERROR) { atomic_and_32(&qlge->flags, ~ADAPTER_ERROR); if (qlge->fm_enable) { ddi_fm_service_impact(qlge->dip, DDI_SERVICE_RESTORED); } } /* Get current link state */ qlge->port_link_state = ql_get_link_state(qlge); if (qlge->port_link_state == LS_UP) { QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n", __func__, qlge->instance)); /* If driver detects a carrier on */ CARRIER_ON(qlge); } else { QL_PRINT(DBG_GLD, ("%s(%d) Link down\n", __func__, qlge->instance)); /* If driver detects a lack of carrier */ CARRIER_OFF(qlge); } qlge->mac_flags = QL_MAC_STARTED; return (DDI_SUCCESS); } /* * Stop currently running driver * Driver needs to stop routing new packets to driver and wait until * all pending tx/rx buffers to be free-ed. */ int ql_do_stop(qlge_t *qlge) { int rc = DDI_FAILURE; uint32_t i, j, k; struct bq_desc *sbq_desc, *lbq_desc; struct rx_ring *rx_ring; ASSERT(qlge != NULL); CARRIER_OFF(qlge); rc = ql_bringdown_adapter(qlge); if (rc != DDI_SUCCESS) { cmn_err(CE_WARN, "qlge bringdown adapter failed."); } else rc = DDI_SUCCESS; for (k = 0; k < qlge->rx_ring_count; k++) { rx_ring = &qlge->rx_ring[k]; if (rx_ring->type != TX_Q) { j = rx_ring->lbq_use_head; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "ring %d: move %d lbufs in use list" " to free list %d\n total %d\n", k, rx_ring->lbuf_in_use_count, rx_ring->lbuf_free_count, rx_ring->lbuf_in_use_count + rx_ring->lbuf_free_count); #endif for (i = 0; i < rx_ring->lbuf_in_use_count; i++) { lbq_desc = rx_ring->lbuf_in_use[j]; j++; if (j >= rx_ring->lbq_len) { j = 0; } if (lbq_desc->mp) { atomic_inc_32(&rx_ring->rx_indicate); freemsg(lbq_desc->mp); } } rx_ring->lbq_use_head = j; rx_ring->lbq_use_tail = j; rx_ring->lbuf_in_use_count = 0; j = rx_ring->sbq_use_head; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "ring %d: move %d sbufs in use list," " to free list %d\n total %d \n", k, rx_ring->sbuf_in_use_count, rx_ring->sbuf_free_count, rx_ring->sbuf_in_use_count + rx_ring->sbuf_free_count); #endif for (i = 0; i < rx_ring->sbuf_in_use_count; i++) { sbq_desc = rx_ring->sbuf_in_use[j]; j++; if (j >= rx_ring->sbq_len) { j = 0; } if (sbq_desc->mp) { atomic_inc_32(&rx_ring->rx_indicate); freemsg(sbq_desc->mp); } } rx_ring->sbq_use_head = j; rx_ring->sbq_use_tail = j; rx_ring->sbuf_in_use_count = 0; } } qlge->mac_flags = QL_MAC_STOPPED; return (rc); } /* * Support */ void ql_disable_isr(qlge_t *qlge) { /* * disable the hardware interrupt */ ISP_DISABLE_GLOBAL_INTRS(qlge); qlge->flags &= ~INTERRUPTS_ENABLED; } /* * busy wait for 'usecs' microseconds. */ void qlge_delay(clock_t usecs) { drv_usecwait(usecs); } /* * retrieve firmware details. */ pci_cfg_t * ql_get_pci_config(qlge_t *qlge) { return (&(qlge->pci_cfg)); } /* * Get current Link status */ static uint32_t ql_get_link_state(qlge_t *qlge) { uint32_t bitToCheck = 0; uint32_t temp, linkState; if (qlge->func_number == qlge->fn0_net) { bitToCheck = STS_PL0; } else { bitToCheck = STS_PL1; } temp = ql_read_reg(qlge, REG_STATUS); QL_PRINT(DBG_GLD, ("%s(%d) chip status reg: 0x%x\n", __func__, qlge->instance, temp)); if (temp & bitToCheck) { linkState = LS_UP; } else { linkState = LS_DOWN; } if (CFG_IST(qlge, CFG_CHIP_8100)) { /* for Schultz, link Speed is fixed to 10G, full duplex */ qlge->speed = SPEED_10G; qlge->duplex = 1; } return (linkState); } /* * Get current link status and report to OS */ static void ql_get_and_report_link_state(qlge_t *qlge) { uint32_t cur_link_state; /* Get current link state */ cur_link_state = ql_get_link_state(qlge); /* if link state has changed */ if (cur_link_state != qlge->port_link_state) { qlge->port_link_state = cur_link_state; if (qlge->port_link_state == LS_UP) { QL_PRINT(DBG_GLD, ("%s(%d) Link UP !!\n", __func__, qlge->instance)); /* If driver detects a carrier on */ CARRIER_ON(qlge); } else { QL_PRINT(DBG_GLD, ("%s(%d) Link down\n", __func__, qlge->instance)); /* If driver detects a lack of carrier */ CARRIER_OFF(qlge); } } } /* * timer callback function executed after timer expires */ static void ql_timer(void* arg) { ql_get_and_report_link_state((qlge_t *)arg); } /* * stop the running timer if activated */ static void ql_stop_timer(qlge_t *qlge) { timeout_id_t timer_id; /* Disable driver timer */ if (qlge->ql_timer_timeout_id != NULL) { timer_id = qlge->ql_timer_timeout_id; qlge->ql_timer_timeout_id = NULL; (void) untimeout(timer_id); } } /* * stop then restart timer */ void ql_restart_timer(qlge_t *qlge) { ql_stop_timer(qlge); qlge->ql_timer_ticks = TICKS_PER_SEC / 4; qlge->ql_timer_timeout_id = timeout(ql_timer, (void *)qlge, qlge->ql_timer_ticks); } /* ************************************************************************* */ /* * Hardware K-Stats Data Structures and Subroutines */ /* ************************************************************************* */ static const ql_ksindex_t ql_kstats_hw[] = { /* PCI related hardware information */ { 0, "Vendor Id" }, { 1, "Device Id" }, { 2, "Command" }, { 3, "Status" }, { 4, "Revision Id" }, { 5, "Cache Line Size" }, { 6, "Latency Timer" }, { 7, "Header Type" }, { 9, "I/O base addr" }, { 10, "Control Reg Base addr low" }, { 11, "Control Reg Base addr high" }, { 12, "Doorbell Reg Base addr low" }, { 13, "Doorbell Reg Base addr high" }, { 14, "Subsystem Vendor Id" }, { 15, "Subsystem Device ID" }, { 16, "PCIe Device Control" }, { 17, "PCIe Link Status" }, { -1, NULL }, }; /* * kstat update function for PCI registers */ static int ql_kstats_get_pci_regs(kstat_t *ksp, int flag) { qlge_t *qlge; kstat_named_t *knp; if (flag != KSTAT_READ) return (EACCES); qlge = ksp->ks_private; knp = ksp->ks_data; (knp++)->value.ui32 = qlge->pci_cfg.vendor_id; (knp++)->value.ui32 = qlge->pci_cfg.device_id; (knp++)->value.ui32 = qlge->pci_cfg.command; (knp++)->value.ui32 = qlge->pci_cfg.status; (knp++)->value.ui32 = qlge->pci_cfg.revision; (knp++)->value.ui32 = qlge->pci_cfg.cache_line_size; (knp++)->value.ui32 = qlge->pci_cfg.latency_timer; (knp++)->value.ui32 = qlge->pci_cfg.header_type; (knp++)->value.ui32 = qlge->pci_cfg.io_base_address; (knp++)->value.ui32 = qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_lower; (knp++)->value.ui32 = qlge->pci_cfg.pci_cntl_reg_set_mem_base_address_upper; (knp++)->value.ui32 = qlge->pci_cfg.pci_doorbell_mem_base_address_lower; (knp++)->value.ui32 = qlge->pci_cfg.pci_doorbell_mem_base_address_upper; (knp++)->value.ui32 = qlge->pci_cfg.sub_vendor_id; (knp++)->value.ui32 = qlge->pci_cfg.sub_device_id; (knp++)->value.ui32 = qlge->pci_cfg.pcie_device_control; (knp++)->value.ui32 = qlge->pci_cfg.link_status; return (0); } static const ql_ksindex_t ql_kstats_mii[] = { /* MAC/MII related hardware information */ { 0, "mtu"}, { -1, NULL}, }; /* * kstat update function for MII related information. */ static int ql_kstats_mii_update(kstat_t *ksp, int flag) { qlge_t *qlge; kstat_named_t *knp; if (flag != KSTAT_READ) return (EACCES); qlge = ksp->ks_private; knp = ksp->ks_data; (knp++)->value.ui32 = qlge->mtu; return (0); } static const ql_ksindex_t ql_kstats_reg[] = { /* Register information */ { 0, "System (0x08)" }, { 1, "Reset/Fail Over(0x0Ch" }, { 2, "Function Specific Control(0x10)" }, { 3, "Status (0x30)" }, { 4, "Intr Enable (0x34)" }, { 5, "Intr Status1 (0x3C)" }, { 6, "Error Status (0x54)" }, { 7, "XGMAC Flow Control(0x11C)" }, { 8, "XGMAC Tx Pause Frames(0x230)" }, { 9, "XGMAC Rx Pause Frames(0x388)" }, { 10, "XGMAC Rx FIFO Drop Count(0x5B8)" }, { 11, "interrupts actually allocated" }, { 12, "interrupts on rx ring 0" }, { 13, "interrupts on rx ring 1" }, { 14, "interrupts on rx ring 2" }, { 15, "interrupts on rx ring 3" }, { 16, "interrupts on rx ring 4" }, { 17, "interrupts on rx ring 5" }, { 18, "interrupts on rx ring 6" }, { 19, "interrupts on rx ring 7" }, { 20, "polls on rx ring 0" }, { 21, "polls on rx ring 1" }, { 22, "polls on rx ring 2" }, { 23, "polls on rx ring 3" }, { 24, "polls on rx ring 4" }, { 25, "polls on rx ring 5" }, { 26, "polls on rx ring 6" }, { 27, "polls on rx ring 7" }, { 28, "tx no resource on ring 0" }, { 29, "tx dma bind fail on ring 0" }, { 30, "tx dma no handle on ring 0" }, { 31, "tx dma no cookie on ring 0" }, { 32, "MPI firmware major version" }, { 33, "MPI firmware minor version" }, { 34, "MPI firmware sub version" }, { 35, "rx no resource" }, { -1, NULL}, }; /* * kstat update function for device register set */ static int ql_kstats_get_reg_and_dev_stats(kstat_t *ksp, int flag) { qlge_t *qlge; kstat_named_t *knp; uint32_t val32; int i = 0; struct tx_ring *tx_ring; struct rx_ring *rx_ring; if (flag != KSTAT_READ) return (EACCES); qlge = ksp->ks_private; knp = ksp->ks_data; (knp++)->value.ui32 = ql_read_reg(qlge, REG_SYSTEM); (knp++)->value.ui32 = ql_read_reg(qlge, REG_RESET_FAILOVER); (knp++)->value.ui32 = ql_read_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL); (knp++)->value.ui32 = ql_read_reg(qlge, REG_STATUS); (knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_ENABLE); (knp++)->value.ui32 = ql_read_reg(qlge, REG_INTERRUPT_STATUS_1); (knp++)->value.ui32 = ql_read_reg(qlge, REG_ERROR_STATUS); if (ql_sem_spinlock(qlge, qlge->xgmac_sem_mask)) { return (0); } (void) ql_read_xgmac_reg(qlge, REG_XGMAC_FLOW_CONTROL, &val32); (knp++)->value.ui32 = val32; (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_TX_PAUSE_PKTS, &val32); (knp++)->value.ui32 = val32; (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_PAUSE_PKTS, &val32); (knp++)->value.ui32 = val32; (void) ql_read_xgmac_reg(qlge, REG_XGMAC_MAC_RX_FIFO_DROPS, &val32); (knp++)->value.ui32 = val32; ql_sem_unlock(qlge, qlge->xgmac_sem_mask); (knp++)->value.ui32 = qlge->intr_cnt; for (i = 0; i < 8; i++) { (knp++)->value.ui32 = qlge->rx_interrupts[i]; } for (i = 0; i < 8; i++) { (knp++)->value.ui32 = qlge->rx_polls[i]; } tx_ring = &qlge->tx_ring[0]; (knp++)->value.ui32 = tx_ring->defer; (knp++)->value.ui32 = tx_ring->tx_fail_dma_bind; (knp++)->value.ui32 = tx_ring->tx_no_dma_handle; (knp++)->value.ui32 = tx_ring->tx_no_dma_cookie; (knp++)->value.ui32 = qlge->fw_version_info.major_version; (knp++)->value.ui32 = qlge->fw_version_info.minor_version; (knp++)->value.ui32 = qlge->fw_version_info.sub_minor_version; for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; val32 += rx_ring->rx_packets_dropped_no_buffer; } (knp++)->value.ui32 = val32; return (0); } static kstat_t * ql_setup_named_kstat(qlge_t *qlge, int instance, char *name, const ql_ksindex_t *ksip, size_t size, int (*update)(kstat_t *, int)) { kstat_t *ksp; kstat_named_t *knp; char *np; int type; size /= sizeof (ql_ksindex_t); ksp = kstat_create(ADAPTER_NAME, instance, name, "net", KSTAT_TYPE_NAMED, ((uint32_t)size) - 1, KSTAT_FLAG_PERSISTENT); if (ksp == NULL) return (NULL); ksp->ks_private = qlge; ksp->ks_update = update; for (knp = ksp->ks_data; (np = ksip->name) != NULL; ++knp, ++ksip) { switch (*np) { default: type = KSTAT_DATA_UINT32; break; case '&': np += 1; type = KSTAT_DATA_CHAR; break; } kstat_named_init(knp, np, (uint8_t)type); } kstat_install(ksp); return (ksp); } /* * Setup various kstat */ int ql_init_kstats(qlge_t *qlge) { /* Hardware KStats */ qlge->ql_kstats[QL_KSTAT_CHIP] = ql_setup_named_kstat(qlge, qlge->instance, "chip", ql_kstats_hw, sizeof (ql_kstats_hw), ql_kstats_get_pci_regs); if (qlge->ql_kstats[QL_KSTAT_CHIP] == NULL) { return (DDI_FAILURE); } /* MII KStats */ qlge->ql_kstats[QL_KSTAT_LINK] = ql_setup_named_kstat(qlge, qlge->instance, "mii", ql_kstats_mii, sizeof (ql_kstats_mii), ql_kstats_mii_update); if (qlge->ql_kstats[QL_KSTAT_LINK] == NULL) { return (DDI_FAILURE); } /* REG KStats */ qlge->ql_kstats[QL_KSTAT_REG] = ql_setup_named_kstat(qlge, qlge->instance, "reg", ql_kstats_reg, sizeof (ql_kstats_reg), ql_kstats_get_reg_and_dev_stats); if (qlge->ql_kstats[QL_KSTAT_REG] == NULL) { return (DDI_FAILURE); } return (DDI_SUCCESS); } /* * delete all kstat */ void ql_fini_kstats(qlge_t *qlge) { int i; for (i = 0; i < QL_KSTAT_COUNT; i++) { if (qlge->ql_kstats[i] != NULL) kstat_delete(qlge->ql_kstats[i]); } } /* ************************************************************************* */ /* * kstat end */ /* ************************************************************************* */ /* * Setup the parameters for receive and transmit rings including buffer sizes * and completion queue sizes */ static int ql_setup_rings(qlge_t *qlge) { uint8_t i; struct rx_ring *rx_ring; struct tx_ring *tx_ring; uint16_t lbq_buf_size; lbq_buf_size = (uint16_t) ((qlge->mtu == ETHERMTU)? LRG_BUF_NORMAL_SIZE : LRG_BUF_JUMBO_SIZE); /* * rx_ring[0] is always the default queue. */ /* * qlge->rx_ring_count: * Total number of rx_rings. This includes a number * of outbound completion handler rx_rings, and a * number of inbound completion handler rx_rings. * rss is only enabled if we have more than 1 rx completion * queue. If we have a single rx completion queue * then all rx completions go to this queue and * the last completion queue */ qlge->tx_ring_first_cq_id = qlge->rss_ring_count; for (i = 0; i < qlge->tx_ring_count; i++) { tx_ring = &qlge->tx_ring[i]; bzero((void *)tx_ring, sizeof (*tx_ring)); tx_ring->qlge = qlge; tx_ring->wq_id = i; tx_ring->wq_len = qlge->tx_ring_size; tx_ring->wq_size = (uint32_t)( tx_ring->wq_len * sizeof (struct ob_mac_iocb_req)); /* * The completion queue ID for the tx rings start * immediately after the last rss completion queue. */ tx_ring->cq_id = (uint16_t)(i + qlge->tx_ring_first_cq_id); } for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; bzero((void *)rx_ring, sizeof (*rx_ring)); rx_ring->qlge = qlge; rx_ring->cq_id = i; if (i != 0) rx_ring->cpu = (i) % qlge->rx_ring_count; else rx_ring->cpu = 0; if (i < qlge->rss_ring_count) { /* * Inbound completions (RSS) queues * Default queue is queue 0 which handles * unicast plus bcast/mcast and async events. * Other inbound queues handle unicast frames only. */ rx_ring->cq_len = qlge->rx_ring_size; rx_ring->cq_size = (uint32_t) (rx_ring->cq_len * sizeof (struct net_rsp_iocb)); rx_ring->lbq_len = NUM_LARGE_BUFFERS; rx_ring->lbq_size = (uint32_t) (rx_ring->lbq_len * sizeof (uint64_t)); rx_ring->lbq_buf_size = lbq_buf_size; rx_ring->sbq_len = NUM_SMALL_BUFFERS; rx_ring->sbq_size = (uint32_t) (rx_ring->sbq_len * sizeof (uint64_t)); rx_ring->sbq_buf_size = SMALL_BUFFER_SIZE * 2; rx_ring->type = RX_Q; QL_PRINT(DBG_GLD, ("%s(%d)Allocating rss completion queue %d " "on cpu %d\n", __func__, qlge->instance, rx_ring->cq_id, rx_ring->cpu)); } else { /* * Outbound queue handles outbound completions only */ /* outbound cq is same size as tx_ring it services. */ QL_PRINT(DBG_INIT, ("rx_ring 0x%p i %d\n", rx_ring, i)); rx_ring->cq_len = qlge->tx_ring_size; rx_ring->cq_size = (uint32_t) (rx_ring->cq_len * sizeof (struct net_rsp_iocb)); rx_ring->lbq_len = 0; rx_ring->lbq_size = 0; rx_ring->lbq_buf_size = 0; rx_ring->sbq_len = 0; rx_ring->sbq_size = 0; rx_ring->sbq_buf_size = 0; rx_ring->type = TX_Q; QL_PRINT(DBG_GLD, ("%s(%d)Allocating TX completion queue %d on" " cpu %d\n", __func__, qlge->instance, rx_ring->cq_id, rx_ring->cpu)); } } return (DDI_SUCCESS); } static int ql_start_rx_ring(qlge_t *qlge, struct rx_ring *rx_ring) { struct cqicb_t *cqicb = (struct cqicb_t *)rx_ring->cqicb_dma.vaddr; void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr + (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE) /* first shadow area is used by wqicb's host copy of consumer index */ + sizeof (uint64_t); uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr + (rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE) + sizeof (uint64_t); /* lrg/sml bufq pointers */ uint8_t *buf_q_base_reg = (uint8_t *)qlge->buf_q_ptr_base_addr_dma_attr.vaddr + (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE); uint64_t buf_q_base_reg_dma = qlge->buf_q_ptr_base_addr_dma_attr.dma_addr + (rx_ring->cq_id * sizeof (uint64_t) * BUF_Q_PTR_SPACE); caddr_t doorbell_area = qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * (128 + rx_ring->cq_id)); int err = 0; uint16_t bq_len; uint64_t tmp; uint64_t *base_indirect_ptr; int page_entries; /* Set up the shadow registers for this ring. */ rx_ring->prod_idx_sh_reg = shadow_reg; rx_ring->prod_idx_sh_reg_dma = shadow_reg_dma; rx_ring->prod_idx_sh_reg_offset = (off_t)(((rx_ring->cq_id * sizeof (uint64_t) * RX_TX_RING_SHADOW_SPACE) + sizeof (uint64_t))); rx_ring->lbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg; rx_ring->lbq_base_indirect_dma = buf_q_base_reg_dma; QL_PRINT(DBG_INIT, ("%s rx ring(%d): prod_idx virtual addr = 0x%lx," " phys_addr 0x%lx\n", __func__, rx_ring->cq_id, rx_ring->prod_idx_sh_reg, rx_ring->prod_idx_sh_reg_dma)); buf_q_base_reg += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t)); buf_q_base_reg_dma += ((BUF_Q_PTR_SPACE / 2) * sizeof (uint64_t)); rx_ring->sbq_base_indirect = (uint64_t *)(void *)buf_q_base_reg; rx_ring->sbq_base_indirect_dma = buf_q_base_reg_dma; /* PCI doorbell mem area + 0x00 for consumer index register */ rx_ring->cnsmr_idx_db_reg = (uint32_t *)(void *)doorbell_area; rx_ring->cnsmr_idx = 0; *rx_ring->prod_idx_sh_reg = 0; rx_ring->curr_entry = rx_ring->cq_dma.vaddr; /* PCI doorbell mem area + 0x04 for valid register */ rx_ring->valid_db_reg = (uint32_t *)(void *) ((uint8_t *)(void *)doorbell_area + 0x04); /* PCI doorbell mem area + 0x18 for large buffer consumer */ rx_ring->lbq_prod_idx_db_reg = (uint32_t *)(void *) ((uint8_t *)(void *)doorbell_area + 0x18); /* PCI doorbell mem area + 0x1c */ rx_ring->sbq_prod_idx_db_reg = (uint32_t *)(void *) ((uint8_t *)(void *)doorbell_area + 0x1c); bzero((void *)cqicb, sizeof (*cqicb)); cqicb->msix_vect = (uint8_t)rx_ring->irq; bq_len = (uint16_t)((rx_ring->cq_len == 65536) ? (uint16_t)0 : (uint16_t)rx_ring->cq_len); cqicb->len = (uint16_t)cpu_to_le16(bq_len | LEN_V | LEN_CPP_CONT); cqicb->cq_base_addr_lo = cpu_to_le32(LS_64BITS(rx_ring->cq_dma.dma_addr)); cqicb->cq_base_addr_hi = cpu_to_le32(MS_64BITS(rx_ring->cq_dma.dma_addr)); cqicb->prod_idx_addr_lo = cpu_to_le32(LS_64BITS(rx_ring->prod_idx_sh_reg_dma)); cqicb->prod_idx_addr_hi = cpu_to_le32(MS_64BITS(rx_ring->prod_idx_sh_reg_dma)); /* * Set up the control block load flags. */ cqicb->flags = FLAGS_LC | /* Load queue base address */ FLAGS_LV | /* Load MSI-X vector */ FLAGS_LI; /* Load irq delay values */ if (rx_ring->lbq_len) { /* Load lbq values */ cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LL); tmp = (uint64_t)rx_ring->lbq_dma.dma_addr; base_indirect_ptr = (uint64_t *)rx_ring->lbq_base_indirect; page_entries = 0; do { *base_indirect_ptr = cpu_to_le64(tmp); tmp += VM_PAGE_SIZE; base_indirect_ptr++; page_entries++; } while (page_entries < (int)( ((rx_ring->lbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE))); cqicb->lbq_addr_lo = cpu_to_le32(LS_64BITS(rx_ring->lbq_base_indirect_dma)); cqicb->lbq_addr_hi = cpu_to_le32(MS_64BITS(rx_ring->lbq_base_indirect_dma)); bq_len = (uint16_t)((rx_ring->lbq_buf_size == 65536) ? (uint16_t)0 : (uint16_t)rx_ring->lbq_buf_size); cqicb->lbq_buf_size = (uint16_t)cpu_to_le16(bq_len); bq_len = (uint16_t)((rx_ring->lbq_len == 65536) ? (uint16_t)0 : (uint16_t)rx_ring->lbq_len); cqicb->lbq_len = (uint16_t)cpu_to_le16(bq_len); rx_ring->lbq_prod_idx = 0; rx_ring->lbq_curr_idx = 0; } if (rx_ring->sbq_len) { /* Load sbq values */ cqicb->flags = (uint8_t)(cqicb->flags | FLAGS_LS); tmp = (uint64_t)rx_ring->sbq_dma.dma_addr; base_indirect_ptr = (uint64_t *)rx_ring->sbq_base_indirect; page_entries = 0; do { *base_indirect_ptr = cpu_to_le64(tmp); tmp += VM_PAGE_SIZE; base_indirect_ptr++; page_entries++; } while (page_entries < (uint32_t) (((rx_ring->sbq_len * sizeof (uint64_t)) / VM_PAGE_SIZE))); cqicb->sbq_addr_lo = cpu_to_le32(LS_64BITS(rx_ring->sbq_base_indirect_dma)); cqicb->sbq_addr_hi = cpu_to_le32(MS_64BITS(rx_ring->sbq_base_indirect_dma)); cqicb->sbq_buf_size = (uint16_t) cpu_to_le16((uint16_t)(rx_ring->sbq_buf_size/2)); bq_len = (uint16_t)((rx_ring->sbq_len == 65536) ? (uint16_t)0 : (uint16_t)rx_ring->sbq_len); cqicb->sbq_len = (uint16_t)cpu_to_le16(bq_len); rx_ring->sbq_prod_idx = 0; rx_ring->sbq_curr_idx = 0; } switch (rx_ring->type) { case TX_Q: cqicb->irq_delay = (uint16_t) cpu_to_le16(qlge->tx_coalesce_usecs); cqicb->pkt_delay = (uint16_t) cpu_to_le16(qlge->tx_max_coalesced_frames); break; case DEFAULT_Q: cqicb->irq_delay = (uint16_t) cpu_to_le16(qlge->rx_coalesce_usecs); cqicb->pkt_delay = (uint16_t) cpu_to_le16(qlge->rx_max_coalesced_frames); break; case RX_Q: /* * Inbound completion handling rx_rings run in * separate NAPI contexts. */ cqicb->irq_delay = (uint16_t) cpu_to_le16(qlge->rx_coalesce_usecs); cqicb->pkt_delay = (uint16_t) cpu_to_le16(qlge->rx_max_coalesced_frames); break; default: cmn_err(CE_WARN, "Invalid rx_ring->type = %d.", rx_ring->type); } QL_PRINT(DBG_INIT, ("Initializing rx completion queue %d.\n", rx_ring->cq_id)); /* QL_DUMP_CQICB(qlge, cqicb); */ err = ql_write_cfg(qlge, CFG_LCQ, rx_ring->cqicb_dma.dma_addr, rx_ring->cq_id); if (err) { cmn_err(CE_WARN, "Failed to load CQICB."); return (err); } rx_ring->rx_packets_dropped_no_buffer = 0; rx_ring->rx_pkt_dropped_mac_unenabled = 0; rx_ring->rx_failed_sbq_allocs = 0; rx_ring->rx_failed_lbq_allocs = 0; rx_ring->rx_packets = 0; rx_ring->rx_bytes = 0; rx_ring->frame_too_long = 0; rx_ring->frame_too_short = 0; rx_ring->fcs_err = 0; return (err); } /* * start RSS */ static int ql_start_rss(qlge_t *qlge) { struct ricb *ricb = (struct ricb *)qlge->ricb_dma.vaddr; int status = 0; int i; uint8_t *hash_id = (uint8_t *)ricb->hash_cq_id; bzero((void *)ricb, sizeof (*ricb)); ricb->base_cq = RSS_L4K; ricb->flags = (RSS_L6K | RSS_LI | RSS_LB | RSS_LM | RSS_RI4 | RSS_RI6 | RSS_RT4 | RSS_RT6); ricb->mask = (uint16_t)cpu_to_le16(RSS_HASH_CQ_ID_MAX - 1); /* * Fill out the Indirection Table. */ for (i = 0; i < RSS_HASH_CQ_ID_MAX; i++) hash_id[i] = (uint8_t)(i & (qlge->rss_ring_count - 1)); (void) memcpy(&ricb->ipv6_hash_key[0], key_data, 40); (void) memcpy(&ricb->ipv4_hash_key[0], key_data, 16); QL_PRINT(DBG_INIT, ("Initializing RSS.\n")); status = ql_write_cfg(qlge, CFG_LR, qlge->ricb_dma.dma_addr, 0); if (status) { cmn_err(CE_WARN, "Failed to load RICB."); return (status); } return (status); } /* * load a tx ring control block to hw and start this ring */ static int ql_start_tx_ring(qlge_t *qlge, struct tx_ring *tx_ring) { struct wqicb_t *wqicb = (struct wqicb_t *)tx_ring->wqicb_dma.vaddr; caddr_t doorbell_area = qlge->doorbell_reg_iobase + (VM_PAGE_SIZE * tx_ring->wq_id); void *shadow_reg = (uint8_t *)qlge->host_copy_shadow_dma_attr.vaddr + (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE; uint64_t shadow_reg_dma = qlge->host_copy_shadow_dma_attr.dma_addr + (tx_ring->wq_id * sizeof (uint64_t)) * RX_TX_RING_SHADOW_SPACE; int err = 0; /* * Assign doorbell registers for this tx_ring. */ /* TX PCI doorbell mem area for tx producer index */ tx_ring->prod_idx_db_reg = (uint32_t *)(void *)doorbell_area; tx_ring->prod_idx = 0; /* TX PCI doorbell mem area + 0x04 */ tx_ring->valid_db_reg = (uint32_t *)(void *) ((uint8_t *)(void *)doorbell_area + 0x04); /* * Assign shadow registers for this tx_ring. */ tx_ring->cnsmr_idx_sh_reg = shadow_reg; tx_ring->cnsmr_idx_sh_reg_dma = shadow_reg_dma; *tx_ring->cnsmr_idx_sh_reg = 0; QL_PRINT(DBG_INIT, ("%s tx ring(%d): cnsmr_idx virtual addr = 0x%lx," " phys_addr 0x%lx\n", __func__, tx_ring->wq_id, tx_ring->cnsmr_idx_sh_reg, tx_ring->cnsmr_idx_sh_reg_dma)); wqicb->len = (uint16_t)cpu_to_le16(tx_ring->wq_len | Q_LEN_V | Q_LEN_CPP_CONT); wqicb->flags = cpu_to_le16(Q_FLAGS_LC | Q_FLAGS_LB | Q_FLAGS_LI | Q_FLAGS_LO); wqicb->cq_id_rss = (uint16_t)cpu_to_le16(tx_ring->cq_id); wqicb->rid = 0; wqicb->wq_addr_lo = cpu_to_le32(LS_64BITS(tx_ring->wq_dma.dma_addr)); wqicb->wq_addr_hi = cpu_to_le32(MS_64BITS(tx_ring->wq_dma.dma_addr)); wqicb->cnsmr_idx_addr_lo = cpu_to_le32(LS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma)); wqicb->cnsmr_idx_addr_hi = cpu_to_le32(MS_64BITS(tx_ring->cnsmr_idx_sh_reg_dma)); ql_init_tx_ring(tx_ring); /* QL_DUMP_WQICB(qlge, wqicb); */ err = ql_write_cfg(qlge, CFG_LRQ, tx_ring->wqicb_dma.dma_addr, tx_ring->wq_id); if (err) { cmn_err(CE_WARN, "Failed to load WQICB."); return (err); } return (err); } /* * Set up a MAC, multicast or VLAN address for the * inbound frame matching. */ int ql_set_mac_addr_reg(qlge_t *qlge, uint8_t *addr, uint32_t type, uint16_t index) { uint32_t offset = 0; int status = DDI_SUCCESS; switch (type) { case MAC_ADDR_TYPE_MULTI_MAC: case MAC_ADDR_TYPE_CAM_MAC: { uint32_t cam_output; uint32_t upper = (addr[0] << 8) | addr[1]; uint32_t lower = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) | (addr[5]); QL_PRINT(DBG_INIT, ("Adding %s ", (type == MAC_ADDR_TYPE_MULTI_MAC) ? "MULTICAST" : "UNICAST")); QL_PRINT(DBG_INIT, ("addr %02x %02x %02x %02x %02x %02x at index %d in " "the CAM.\n", addr[0], addr[1], addr[2], addr[3], addr[4], addr[5], index)); status = ql_wait_reg_rdy(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0); if (status) goto exit; /* offset 0 - lower 32 bits of the MAC address */ ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, (offset++) | (index << MAC_ADDR_IDX_SHIFT) | /* index */ type); /* type */ ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, lower); status = ql_wait_reg_rdy(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0); if (status) goto exit; /* offset 1 - upper 16 bits of the MAC address */ ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, (offset++) | (index << MAC_ADDR_IDX_SHIFT) | /* index */ type); /* type */ ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, upper); status = ql_wait_reg_rdy(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, MAC_ADDR_MW, 0); if (status) goto exit; /* offset 2 - CQ ID associated with this MAC address */ ql_write_reg(qlge, REG_MAC_PROTOCOL_ADDRESS_INDEX, (offset) | (index << MAC_ADDR_IDX_SHIFT) | /* index */ type); /* type */ /* * This field should also include the queue id * and possibly the function id. Right now we hardcode * the route field to NIC core. */ if (type == MAC_ADDR_TYPE_CAM_MAC) { cam_output = (CAM_OUT_ROUTE_NIC | (qlge->func_number << CAM_OUT_FUNC_SHIFT) | (0 << CAM_OUT_CQ_ID_SHIFT)); /* route to NIC core */ ql_write_reg(qlge, REG_MAC_PROTOCOL_DATA, cam_output); } break; } default: cmn_err(CE_WARN, "Address type %d not yet supported.", type); status = DDI_FAILURE; } exit: return (status); } /* * The NIC function for this chip has 16 routing indexes. Each one can be used * to route different frame types to various inbound queues. We send broadcast * multicast/error frames to the default queue for slow handling, * and CAM hit/RSS frames to the fast handling queues. */ static int ql_set_routing_reg(qlge_t *qlge, uint32_t index, uint32_t mask, int enable) { int status; uint32_t value = 0; QL_PRINT(DBG_INIT, ("%s %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s mask %s the routing reg.\n", (enable ? "Adding" : "Removing"), ((index == RT_IDX_ALL_ERR_SLOT) ? "MAC ERROR/ALL ERROR" : ""), ((index == RT_IDX_IP_CSUM_ERR_SLOT) ? "IP CSUM ERROR" : ""), ((index == RT_IDX_TCP_UDP_CSUM_ERR_SLOT) ? "TCP/UDP CSUM ERROR" : ""), ((index == RT_IDX_BCAST_SLOT) ? "BROADCAST" : ""), ((index == RT_IDX_MCAST_MATCH_SLOT) ? "MULTICAST MATCH" : ""), ((index == RT_IDX_ALLMULTI_SLOT) ? "ALL MULTICAST MATCH" : ""), ((index == RT_IDX_UNUSED6_SLOT) ? "UNUSED6" : ""), ((index == RT_IDX_UNUSED7_SLOT) ? "UNUSED7" : ""), ((index == RT_IDX_RSS_MATCH_SLOT) ? "RSS ALL/IPV4 MATCH" : ""), ((index == RT_IDX_RSS_IPV6_SLOT) ? "RSS IPV6" : ""), ((index == RT_IDX_RSS_TCP4_SLOT) ? "RSS TCP4" : ""), ((index == RT_IDX_RSS_TCP6_SLOT) ? "RSS TCP6" : ""), ((index == RT_IDX_CAM_HIT_SLOT) ? "CAM HIT" : ""), ((index == RT_IDX_UNUSED013) ? "UNUSED13" : ""), ((index == RT_IDX_UNUSED014) ? "UNUSED14" : ""), ((index == RT_IDX_PROMISCUOUS_SLOT) ? "PROMISCUOUS" : ""), (enable ? "to" : "from"))); switch (mask) { case RT_IDX_CAM_HIT: value = RT_IDX_DST_CAM_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_CAM_HIT_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_VALID: /* Promiscuous Mode frames. */ value = RT_IDX_DST_DFLT_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_PROMISCUOUS_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_ERR: /* Pass up MAC,IP,TCP/UDP error frames. */ value = RT_IDX_DST_DFLT_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_ALL_ERR_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_BCAST: /* Pass up Broadcast frames to default Q. */ value = RT_IDX_DST_DFLT_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_BCAST_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_MCAST: /* Pass up All Multicast frames. */ value = RT_IDX_DST_CAM_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_ALLMULTI_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_MCAST_MATCH: /* Pass up matched Multicast frames. */ value = RT_IDX_DST_CAM_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_MCAST_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case RT_IDX_RSS_MATCH: /* Pass up matched RSS frames. */ value = RT_IDX_DST_RSS | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (RT_IDX_RSS_MATCH_SLOT << RT_IDX_IDX_SHIFT); /* index */ break; case 0: /* Clear the E-bit on an entry. */ value = RT_IDX_DST_DFLT_Q | /* dest */ RT_IDX_TYPE_NICQ | /* type */ (index << RT_IDX_IDX_SHIFT); /* index */ break; default: cmn_err(CE_WARN, "Mask type %d not yet supported.", mask); status = -EPERM; goto exit; } if (value != 0) { status = ql_wait_reg_rdy(qlge, REG_ROUTING_INDEX, RT_IDX_MW, 0); if (status) goto exit; value |= (enable ? RT_IDX_E : 0); ql_write_reg(qlge, REG_ROUTING_INDEX, value); ql_write_reg(qlge, REG_ROUTING_DATA, enable ? mask : 0); } exit: return (status); } /* * Clear all the entries in the routing table. * Caller must get semaphore in advance. */ static int ql_stop_routing(qlge_t *qlge) { int status = 0; int i; /* Clear all the entries in the routing table. */ for (i = 0; i < 16; i++) { status = ql_set_routing_reg(qlge, i, 0, 0); if (status) { cmn_err(CE_WARN, "Stop routing failed. "); } } return (status); } /* Initialize the frame-to-queue routing. */ int ql_route_initialize(qlge_t *qlge) { int status = 0; status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK); if (status != DDI_SUCCESS) return (status); /* Clear all the entries in the routing table. */ status = ql_stop_routing(qlge); if (status) { goto exit; } status = ql_set_routing_reg(qlge, RT_IDX_BCAST_SLOT, RT_IDX_BCAST, 1); if (status) { cmn_err(CE_WARN, "Failed to init routing register for broadcast packets."); goto exit; } /* * If we have more than one inbound queue, then turn on RSS in the * routing block. */ if (qlge->rss_ring_count > 1) { status = ql_set_routing_reg(qlge, RT_IDX_RSS_MATCH_SLOT, RT_IDX_RSS_MATCH, 1); if (status) { cmn_err(CE_WARN, "Failed to init routing register for MATCH RSS " "packets."); goto exit; } } status = ql_set_routing_reg(qlge, RT_IDX_CAM_HIT_SLOT, RT_IDX_CAM_HIT, 1); if (status) { cmn_err(CE_WARN, "Failed to init routing register for CAM packets."); goto exit; } status = ql_set_routing_reg(qlge, RT_IDX_MCAST_MATCH_SLOT, RT_IDX_MCAST_MATCH, 1); if (status) { cmn_err(CE_WARN, "Failed to init routing register for Multicast " "packets."); } exit: ql_sem_unlock(qlge, SEM_RT_IDX_MASK); return (status); } /* * Initialize hardware */ static int ql_device_initialize(qlge_t *qlge) { uint32_t value, mask; int i; int status = 0; uint16_t pause = PAUSE_MODE_DISABLED; boolean_t update_port_config = B_FALSE; uint32_t pause_bit_mask; boolean_t dcbx_enable = B_FALSE; uint32_t dcbx_bit_mask = 0x10; /* * Set up the System register to halt on errors. */ value = SYS_EFE | SYS_FAE; mask = value << 16; ql_write_reg(qlge, REG_SYSTEM, mask | value); /* Set the default queue. */ value = NIC_RCV_CFG_DFQ; mask = NIC_RCV_CFG_DFQ_MASK; ql_write_reg(qlge, REG_NIC_RECEIVE_CONFIGURATION, mask | value); /* Enable the MPI interrupt. */ ql_write_reg(qlge, REG_INTERRUPT_MASK, (INTR_MASK_PI << 16) | INTR_MASK_PI); /* Enable the function, set pagesize, enable error checking. */ value = FSC_FE | FSC_EPC_INBOUND | FSC_EPC_OUTBOUND | FSC_EC | FSC_VM_PAGE_4K | FSC_DBRST_1024; /* Set/clear header splitting. */ if (CFG_IST(qlge, CFG_ENABLE_SPLIT_HEADER)) { value |= FSC_SH; ql_write_reg(qlge, REG_SPLIT_HEADER, SMALL_BUFFER_SIZE); } mask = FSC_VM_PAGESIZE_MASK | FSC_DBL_MASK | FSC_DBRST_MASK | (value << 16); ql_write_reg(qlge, REG_FUNCTION_SPECIFIC_CONTROL, mask | value); /* * check current port max frame size, if different from OS setting, * then we need to change */ qlge->max_frame_size = (qlge->mtu == ETHERMTU)? NORMAL_FRAME_SIZE : JUMBO_FRAME_SIZE; mutex_enter(&qlge->mbx_mutex); status = ql_get_port_cfg(qlge); mutex_exit(&qlge->mbx_mutex); if (status == DDI_SUCCESS) { /* if current frame size is smaller than required size */ if (qlge->port_cfg_info.max_frame_size < qlge->max_frame_size) { QL_PRINT(DBG_MBX, ("update frame size, current %d, new %d\n", qlge->port_cfg_info.max_frame_size, qlge->max_frame_size)); qlge->port_cfg_info.max_frame_size = qlge->max_frame_size; qlge->port_cfg_info.link_cfg |= ENABLE_JUMBO; update_port_config = B_TRUE; } if (qlge->port_cfg_info.link_cfg & STD_PAUSE) pause = PAUSE_MODE_STANDARD; else if (qlge->port_cfg_info.link_cfg & PP_PAUSE) pause = PAUSE_MODE_PER_PRIORITY; if (pause != qlge->pause) { pause_bit_mask = 0x60; /* bit 5-6 */ /* clear pause bits */ qlge->port_cfg_info.link_cfg &= ~pause_bit_mask; if (qlge->pause == PAUSE_MODE_STANDARD) qlge->port_cfg_info.link_cfg |= STD_PAUSE; else if (qlge->pause == PAUSE_MODE_PER_PRIORITY) qlge->port_cfg_info.link_cfg |= PP_PAUSE; update_port_config = B_TRUE; } if (qlge->port_cfg_info.link_cfg & DCBX_ENABLE) dcbx_enable = B_TRUE; if (dcbx_enable != qlge->dcbx_enable) { qlge->port_cfg_info.link_cfg &= ~dcbx_bit_mask; if (qlge->dcbx_enable) qlge->port_cfg_info.link_cfg |= DCBX_ENABLE; } update_port_config = B_TRUE; /* if need to update port configuration */ if (update_port_config) { mutex_enter(&qlge->mbx_mutex); (void) ql_set_mpi_port_config(qlge, qlge->port_cfg_info); mutex_exit(&qlge->mbx_mutex); } } else cmn_err(CE_WARN, "ql_get_port_cfg failed"); /* Start up the rx queues. */ for (i = 0; i < qlge->rx_ring_count; i++) { status = ql_start_rx_ring(qlge, &qlge->rx_ring[i]); if (status) { cmn_err(CE_WARN, "Failed to start rx ring[%d]", i); return (status); } } /* * If there is more than one inbound completion queue * then download a RICB to configure RSS. */ if (qlge->rss_ring_count > 1) { status = ql_start_rss(qlge); if (status) { cmn_err(CE_WARN, "Failed to start RSS."); return (status); } } /* Start up the tx queues. */ for (i = 0; i < qlge->tx_ring_count; i++) { status = ql_start_tx_ring(qlge, &qlge->tx_ring[i]); if (status) { cmn_err(CE_WARN, "Failed to start tx ring[%d]", i); return (status); } } qlge->selected_tx_ring = 0; /* Set the frame routing filter. */ status = ql_route_initialize(qlge); if (status) { cmn_err(CE_WARN, "Failed to init CAM/Routing tables."); return (status); } return (status); } /* * Issue soft reset to chip. */ static int ql_asic_reset(qlge_t *qlge) { int status = DDI_SUCCESS; ql_write_reg(qlge, REG_RESET_FAILOVER, FUNCTION_RESET_MASK |FUNCTION_RESET); if (ql_wait_reg_bit(qlge, REG_RESET_FAILOVER, FUNCTION_RESET, BIT_RESET, 0) != DDI_SUCCESS) { cmn_err(CE_WARN, "TIMEOUT!!! errored out of resetting the chip!"); status = DDI_FAILURE; } return (status); } /* * If there are more than MIN_BUFFERS_ARM_COUNT small buffer descriptors in * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list * to be used by hardware. */ static void ql_arm_sbuf(qlge_t *qlge, struct rx_ring *rx_ring) { struct bq_desc *sbq_desc; int i; uint64_t *sbq_entry = rx_ring->sbq_dma.vaddr; uint32_t arm_count; if (rx_ring->sbuf_free_count > rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT) arm_count = (rx_ring->sbq_len-MIN_BUFFERS_ARM_COUNT); else { /* Adjust to a multiple of 16 */ arm_count = (rx_ring->sbuf_free_count / 16) * 16; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "adjust sbuf arm_count %d\n", arm_count); #endif } for (i = 0; i < arm_count; i++) { sbq_desc = ql_get_sbuf_from_free_list(rx_ring); if (sbq_desc == NULL) break; /* Arm asic */ *sbq_entry = cpu_to_le64(sbq_desc->bd_dma.dma_addr); sbq_entry++; /* link the descriptors to in_use_list */ ql_add_sbuf_to_in_use_list(rx_ring, sbq_desc); rx_ring->sbq_prod_idx++; } ql_update_sbq_prod_idx(qlge, rx_ring); } /* * If there are more than MIN_BUFFERS_ARM_COUNT large buffer descriptors in * its free list, move xMIN_BUFFERS_ARM_COUNT descriptors to its in use list * to be used by hardware. */ static void ql_arm_lbuf(qlge_t *qlge, struct rx_ring *rx_ring) { struct bq_desc *lbq_desc; int i; uint64_t *lbq_entry = rx_ring->lbq_dma.vaddr; uint32_t arm_count; if (rx_ring->lbuf_free_count > rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT) arm_count = (rx_ring->lbq_len-MIN_BUFFERS_ARM_COUNT); else { /* Adjust to a multiple of 16 */ arm_count = (rx_ring->lbuf_free_count / 16) * 16; #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "adjust lbuf arm_count %d\n", arm_count); #endif } for (i = 0; i < arm_count; i++) { lbq_desc = ql_get_lbuf_from_free_list(rx_ring); if (lbq_desc == NULL) break; /* Arm asic */ *lbq_entry = cpu_to_le64(lbq_desc->bd_dma.dma_addr); lbq_entry++; /* link the descriptors to in_use_list */ ql_add_lbuf_to_in_use_list(rx_ring, lbq_desc); rx_ring->lbq_prod_idx++; } ql_update_lbq_prod_idx(qlge, rx_ring); } /* * Initializes the adapter by configuring request and response queues, * allocates and ARMs small and large receive buffers to the * hardware */ static int ql_bringup_adapter(qlge_t *qlge) { int i; if (ql_device_initialize(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "?%s(%d): ql_device_initialize failed", __func__, qlge->instance); goto err_bringup; } qlge->sequence |= INIT_ADAPTER_UP; #ifdef QLGE_TRACK_BUFFER_USAGE for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].type != TX_Q) { qlge->rx_sb_low_count[i] = NUM_SMALL_BUFFERS; qlge->rx_lb_low_count[i] = NUM_LARGE_BUFFERS; } qlge->cq_low_count[i] = NUM_RX_RING_ENTRIES; } #endif /* Arm buffers */ for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].type != TX_Q) { ql_arm_sbuf(qlge, &qlge->rx_ring[i]); ql_arm_lbuf(qlge, &qlge->rx_ring[i]); } } /* Enable work/request queues */ for (i = 0; i < qlge->tx_ring_count; i++) { if (qlge->tx_ring[i].valid_db_reg) ql_write_doorbell_reg(qlge, qlge->tx_ring[i].valid_db_reg, REQ_Q_VALID); } /* Enable completion queues */ for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].valid_db_reg) ql_write_doorbell_reg(qlge, qlge->rx_ring[i].valid_db_reg, RSP_Q_VALID); } for (i = 0; i < qlge->tx_ring_count; i++) { mutex_enter(&qlge->tx_ring[i].tx_lock); qlge->tx_ring[i].mac_flags = QL_MAC_STARTED; mutex_exit(&qlge->tx_ring[i].tx_lock); } for (i = 0; i < qlge->rx_ring_count; i++) { mutex_enter(&qlge->rx_ring[i].rx_lock); qlge->rx_ring[i].mac_flags = QL_MAC_STARTED; mutex_exit(&qlge->rx_ring[i].rx_lock); } /* This mutex will get re-acquired in enable_completion interrupt */ mutex_exit(&qlge->hw_mutex); /* Traffic can start flowing now */ ql_enable_all_completion_interrupts(qlge); mutex_enter(&qlge->hw_mutex); ql_enable_global_interrupt(qlge); qlge->sequence |= ADAPTER_INIT; return (DDI_SUCCESS); err_bringup: (void) ql_asic_reset(qlge); return (DDI_FAILURE); } /* * Initialize mutexes of each rx/tx rings */ static int ql_init_rx_tx_locks(qlge_t *qlge) { struct tx_ring *tx_ring; struct rx_ring *rx_ring; int i; for (i = 0; i < qlge->tx_ring_count; i++) { tx_ring = &qlge->tx_ring[i]; mutex_init(&tx_ring->tx_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); } for (i = 0; i < qlge->rx_ring_count; i++) { rx_ring = &qlge->rx_ring[i]; mutex_init(&rx_ring->rx_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); mutex_init(&rx_ring->sbq_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); mutex_init(&rx_ring->lbq_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); } return (DDI_SUCCESS); } /*ARGSUSED*/ /* * Simply call pci_ereport_post which generates ereports for errors * that occur in the PCI local bus configuration status registers. */ static int ql_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) { pci_ereport_post(dip, err, NULL); return (err->fme_status); } static void ql_fm_init(qlge_t *qlge) { ddi_iblock_cookie_t iblk; QL_PRINT(DBG_INIT, ("ql_fm_init(%d) entered, FMA capability %x\n", qlge->instance, qlge->fm_capabilities)); /* * Register capabilities with IO Fault Services. The capabilities * set above may not be supported by the parent nexus, in that case * some capability bits may be cleared. */ if (qlge->fm_capabilities) ddi_fm_init(qlge->dip, &qlge->fm_capabilities, &iblk); /* * Initialize pci ereport capabilities if ereport capable */ if (DDI_FM_EREPORT_CAP(qlge->fm_capabilities) || DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) { pci_ereport_setup(qlge->dip); } /* Register error callback if error callback capable */ if (DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) { ddi_fm_handler_register(qlge->dip, ql_fm_error_cb, (void*) qlge); } /* * DDI_FLGERR_ACC indicates: * Driver will check its access handle(s) for faults on * a regular basis by calling ddi_fm_acc_err_get * Driver is able to cope with incorrect results of I/O * operations resulted from an I/O fault */ if (DDI_FM_ACC_ERR_CAP(qlge->fm_capabilities)) { ql_dev_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; } /* * DDI_DMA_FLAGERR indicates: * Driver will check its DMA handle(s) for faults on a * regular basis using ddi_fm_dma_err_get * Driver is able to cope with incorrect results of DMA * operations resulted from an I/O fault */ if (DDI_FM_DMA_ERR_CAP(qlge->fm_capabilities)) { tx_mapping_dma_attr.dma_attr_flags = DDI_DMA_FLAGERR; dma_attr.dma_attr_flags = DDI_DMA_FLAGERR; } QL_PRINT(DBG_INIT, ("ql_fm_init(%d) done\n", qlge->instance)); } static void ql_fm_fini(qlge_t *qlge) { QL_PRINT(DBG_INIT, ("ql_fm_fini(%d) entered\n", qlge->instance)); /* Only unregister FMA capabilities if we registered some */ if (qlge->fm_capabilities) { /* * Release any resources allocated by pci_ereport_setup() */ if (DDI_FM_EREPORT_CAP(qlge->fm_capabilities) || DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) pci_ereport_teardown(qlge->dip); /* * Un-register error callback if error callback capable */ if (DDI_FM_ERRCB_CAP(qlge->fm_capabilities)) ddi_fm_handler_unregister(qlge->dip); /* Unregister from IO Fault Services */ ddi_fm_fini(qlge->dip); } QL_PRINT(DBG_INIT, ("ql_fm_fini(%d) done\n", qlge->instance)); } /* * ql_attach - Driver attach. */ static int ql_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { int instance; qlge_t *qlge = NULL; int rval; uint16_t w; mac_register_t *macp = NULL; uint32_t data; rval = DDI_FAILURE; /* first get the instance */ instance = ddi_get_instance(dip); switch (cmd) { case DDI_ATTACH: /* * Allocate our per-device-instance structure */ qlge = (qlge_t *)kmem_zalloc(sizeof (*qlge), KM_SLEEP); ASSERT(qlge != NULL); qlge->sequence |= INIT_SOFTSTATE_ALLOC; qlge->dip = dip; qlge->instance = instance; /* Set up the coalescing parameters. */ qlge->ql_dbgprnt = 0; #if QL_DEBUG qlge->ql_dbgprnt = QL_DEBUG; #endif /* QL_DEBUG */ /* * Initialize for fma support */ /* fault management (fm) capabilities. */ qlge->fm_capabilities = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE; data = ql_get_prop(qlge, "fm-capable"); if (data <= 0xf) { qlge->fm_capabilities = data; } ql_fm_init(qlge); qlge->sequence |= INIT_FM; QL_PRINT(DBG_INIT, ("ql_attach(%d): fma init done\n", qlge->instance)); /* * Setup the ISP8x00 registers address mapping to be * accessed by this particular driver. * 0x0 Configuration Space * 0x1 I/O Space * 0x2 1st Memory Space address - Control Register Set * 0x3 2nd Memory Space address - Doorbell Memory Space */ w = 2; if (ddi_regs_map_setup(dip, w, (caddr_t *)&qlge->iobase, 0, sizeof (dev_reg_t), &ql_dev_acc_attr, &qlge->dev_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): Unable to map device " "registers", ADAPTER_NAME, instance); break; } QL_PRINT(DBG_GLD, ("ql_attach: I/O base = 0x%x\n", qlge->iobase)); qlge->sequence |= INIT_REGS_SETUP; /* map Doorbell memory space */ w = 3; if (ddi_regs_map_setup(dip, w, (caddr_t *)&qlge->doorbell_reg_iobase, 0, 0x100000 /* sizeof (dev_doorbell_reg_t) */, &ql_dev_acc_attr, &qlge->dev_doorbell_reg_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): Unable to map Doorbell " "registers", ADAPTER_NAME, instance); break; } QL_PRINT(DBG_GLD, ("ql_attach: Doorbell I/O base = 0x%x\n", qlge->doorbell_reg_iobase)); qlge->sequence |= INIT_DOORBELL_REGS_SETUP; /* * Allocate a macinfo structure for this instance */ if ((macp = mac_alloc(MAC_VERSION)) == NULL) { cmn_err(CE_WARN, "%s(%d): mac_alloc failed", __func__, instance); break; } /* save adapter status to dip private data */ ddi_set_driver_private(dip, qlge); QL_PRINT(DBG_INIT, ("%s(%d): Allocate macinfo structure done\n", ADAPTER_NAME, instance)); qlge->sequence |= INIT_MAC_ALLOC; /* * Attach this instance of the device */ /* Setup PCI Local Bus Configuration resource. */ if (pci_config_setup(dip, &qlge->pci_handle) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d):Unable to get PCI resources", ADAPTER_NAME, instance); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); } break; } qlge->sequence |= INIT_PCI_CONFIG_SETUP; QL_PRINT(DBG_GLD, ("ql_attach(%d): pci_config_setup done\n", instance)); if (ql_init_instance(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): Unable to initialize device " "instance", ADAPTER_NAME, instance); if (qlge->fm_enable) { ql_fm_ereport(qlge, DDI_FM_DEVICE_INVAL_STATE); ddi_fm_service_impact(qlge->dip, DDI_SERVICE_LOST); } break; } QL_PRINT(DBG_GLD, ("ql_attach(%d): ql_init_instance done\n", instance)); /* Setup interrupt vectors */ if (ql_alloc_irqs(qlge) != DDI_SUCCESS) { break; } qlge->sequence |= INIT_INTR_ALLOC; QL_PRINT(DBG_GLD, ("ql_attach(%d): ql_alloc_irqs done\n", instance)); /* Configure queues */ if (ql_setup_rings(qlge) != DDI_SUCCESS) { break; } qlge->sequence |= INIT_SETUP_RINGS; QL_PRINT(DBG_GLD, ("ql_attach(%d): setup rings done\n", instance)); /* * Allocate memory resources */ if (ql_alloc_mem_resources(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): memory allocation failed", __func__, qlge->instance); break; } qlge->sequence |= INIT_MEMORY_ALLOC; QL_PRINT(DBG_GLD, ("ql_alloc_mem_resources(%d) done\n", instance)); /* * Map queues to interrupt vectors */ ql_resolve_queues_to_irqs(qlge); /* Initialize mutex, need the interrupt priority */ (void) ql_init_rx_tx_locks(qlge); qlge->sequence |= INIT_LOCKS_CREATED; QL_PRINT(DBG_INIT, ("%s(%d): ql_init_rx_tx_locks done\n", ADAPTER_NAME, instance)); /* * Use a soft interrupt to do something that we do not want * to do in regular network functions or with mutexs being held */ if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_event_intr_hdl, DDI_INTR_SOFTPRI_MIN, ql_mpi_event_work, (caddr_t)qlge) != DDI_SUCCESS) { break; } if (ddi_intr_add_softint(qlge->dip, &qlge->asic_reset_intr_hdl, DDI_INTR_SOFTPRI_MIN, ql_asic_reset_work, (caddr_t)qlge) != DDI_SUCCESS) { break; } if (ddi_intr_add_softint(qlge->dip, &qlge->mpi_reset_intr_hdl, DDI_INTR_SOFTPRI_MIN, ql_mpi_reset_work, (caddr_t)qlge) != DDI_SUCCESS) { break; } qlge->sequence |= INIT_ADD_SOFT_INTERRUPT; QL_PRINT(DBG_INIT, ("%s(%d): ddi_intr_add_softint done\n", ADAPTER_NAME, instance)); /* * mutex to protect the adapter state structure. * initialize mutexes according to the interrupt priority */ mutex_init(&qlge->gen_mutex, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); mutex_init(&qlge->hw_mutex, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); mutex_init(&qlge->mbx_mutex, NULL, MUTEX_DRIVER, DDI_INTR_PRI(qlge->intr_pri)); /* Mailbox wait and interrupt conditional variable. */ cv_init(&qlge->cv_mbx_intr, NULL, CV_DRIVER, NULL); qlge->sequence |= INIT_MUTEX; QL_PRINT(DBG_INIT, ("%s(%d): mutex_init done\n", ADAPTER_NAME, instance)); /* * KStats */ if (ql_init_kstats(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): KState initialization failed", ADAPTER_NAME, instance); break; } qlge->sequence |= INIT_KSTATS; QL_PRINT(DBG_INIT, ("%s(%d): ql_init_kstats done\n", ADAPTER_NAME, instance)); /* * Initialize gld macinfo structure */ ql_gld3_init(qlge, macp); /* * Add interrupt handlers */ if (ql_add_intr_handlers(qlge) != DDI_SUCCESS) { cmn_err(CE_WARN, "Failed to add interrupt " "handlers"); break; } qlge->sequence |= INIT_ADD_INTERRUPT; QL_PRINT(DBG_INIT, ("%s(%d): Add interrupt handler done\n", ADAPTER_NAME, instance)); /* * MAC Register */ if (mac_register(macp, &qlge->mh) != DDI_SUCCESS) { cmn_err(CE_WARN, "%s(%d): mac_register failed", __func__, instance); break; } qlge->sequence |= INIT_MAC_REGISTERED; QL_PRINT(DBG_GLD, ("%s(%d): mac_register done\n", ADAPTER_NAME, instance)); mac_free(macp); macp = NULL; qlge->mac_flags = QL_MAC_ATTACHED; ddi_report_dev(dip); rval = DDI_SUCCESS; break; /* * DDI_RESUME * When called with cmd set to DDI_RESUME, attach() must * restore the hardware state of a device (power may have been * removed from the device), allow pending requests to con- * tinue, and service new requests. In this case, the driver * must not make any assumptions about the state of the * hardware, but must restore the state of the device except * for the power level of components. * */ case DDI_RESUME: if ((qlge = (qlge_t *)QL_GET_DEV(dip)) == NULL) return (DDI_FAILURE); QL_PRINT(DBG_GLD, ("%s(%d)-DDI_RESUME\n", __func__, qlge->instance)); mutex_enter(&qlge->gen_mutex); rval = ql_do_start(qlge); mutex_exit(&qlge->gen_mutex); break; default: break; } /* if failed to attach */ if ((cmd == DDI_ATTACH) && (rval != DDI_SUCCESS) && (qlge != NULL)) { cmn_err(CE_WARN, "qlge driver attach failed, sequence %x", qlge->sequence); ql_free_resources(qlge); } return (rval); } /* * Unbind all pending tx dma handles during driver bring down */ static void ql_unbind_pending_tx_dma_handle(struct tx_ring *tx_ring) { struct tx_ring_desc *tx_ring_desc; int i, j; if (tx_ring->wq_desc) { tx_ring_desc = tx_ring->wq_desc; for (i = 0; i < tx_ring->wq_len; i++, tx_ring_desc++) { for (j = 0; j < tx_ring_desc->tx_dma_handle_used; j++) { if (tx_ring_desc->tx_dma_handle[j]) { (void) ddi_dma_unbind_handle( tx_ring_desc->tx_dma_handle[j]); } } tx_ring_desc->tx_dma_handle_used = 0; } /* end of for loop */ } } /* * Wait for all the packets sent to the chip to finish transmission * to prevent buffers to be unmapped before or during a transmit operation */ static int ql_wait_tx_quiesce(qlge_t *qlge) { int count = MAX_TX_WAIT_COUNT, i; int rings_done; volatile struct tx_ring *tx_ring; uint32_t consumer_idx; uint32_t producer_idx; uint32_t temp; int done = 0; int rval = DDI_FAILURE; while (!done) { rings_done = 0; for (i = 0; i < qlge->tx_ring_count; i++) { tx_ring = &qlge->tx_ring[i]; temp = ql_read_doorbell_reg(qlge, tx_ring->prod_idx_db_reg); producer_idx = temp & 0x0000ffff; consumer_idx = (temp >> 16); if (qlge->isr_stride) { struct rx_ring *ob_ring; ob_ring = &qlge->rx_ring[tx_ring->cq_id]; if (producer_idx != ob_ring->cnsmr_idx) { cmn_err(CE_NOTE, " force clean \n"); (void) ql_clean_outbound_rx_ring( ob_ring); } } /* * Get the pending iocb count, ones which have not been * pulled down by the chip */ if (producer_idx >= consumer_idx) temp = (producer_idx - consumer_idx); else temp = (tx_ring->wq_len - consumer_idx) + producer_idx; if ((tx_ring->tx_free_count + temp) >= tx_ring->wq_len) rings_done++; else { done = 1; break; } } /* If all the rings are done */ if (rings_done >= qlge->tx_ring_count) { #ifdef QLGE_LOAD_UNLOAD cmn_err(CE_NOTE, "%s(%d) done successfully \n", __func__, qlge->instance); #endif rval = DDI_SUCCESS; break; } qlge_delay(100); count--; if (!count) { count = MAX_TX_WAIT_COUNT; #ifdef QLGE_LOAD_UNLOAD volatile struct rx_ring *rx_ring; cmn_err(CE_NOTE, "%s(%d): Waiting for %d pending" " Transmits on queue %d to complete .\n", __func__, qlge->instance, (qlge->tx_ring[i].wq_len - qlge->tx_ring[i].tx_free_count), i); rx_ring = &qlge->rx_ring[i+1]; temp = ql_read_doorbell_reg(qlge, rx_ring->cnsmr_idx_db_reg); consumer_idx = temp & 0x0000ffff; producer_idx = (temp >> 16); cmn_err(CE_NOTE, "%s(%d): Transmit completion queue %d," " Producer %d, Consumer %d\n", __func__, qlge->instance, i+1, producer_idx, consumer_idx); temp = ql_read_doorbell_reg(qlge, tx_ring->prod_idx_db_reg); producer_idx = temp & 0x0000ffff; consumer_idx = (temp >> 16); cmn_err(CE_NOTE, "%s(%d): Transmit request queue %d," " Producer %d, Consumer %d\n", __func__, qlge->instance, i, producer_idx, consumer_idx); #endif /* For now move on */ break; } } /* Stop the request queue */ mutex_enter(&qlge->hw_mutex); for (i = 0; i < qlge->tx_ring_count; i++) { if (qlge->tx_ring[i].valid_db_reg) { ql_write_doorbell_reg(qlge, qlge->tx_ring[i].valid_db_reg, 0); } } mutex_exit(&qlge->hw_mutex); return (rval); } /* * Wait for all the receives indicated to the stack to come back */ static int ql_wait_rx_complete(qlge_t *qlge) { int i; /* Disable all the completion queues */ mutex_enter(&qlge->hw_mutex); for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].valid_db_reg) { ql_write_doorbell_reg(qlge, qlge->rx_ring[i].valid_db_reg, 0); } } mutex_exit(&qlge->hw_mutex); /* Wait for OS to return all rx buffers */ qlge_delay(QL_ONE_SEC_DELAY); return (DDI_SUCCESS); } /* * stop the driver */ static int ql_bringdown_adapter(qlge_t *qlge) { int i; int status = DDI_SUCCESS; qlge->mac_flags = QL_MAC_BRINGDOWN; if (qlge->sequence & ADAPTER_INIT) { /* stop forwarding external packets to driver */ status = ql_sem_spinlock(qlge, SEM_RT_IDX_MASK); if (status) return (status); (void) ql_stop_routing(qlge); ql_sem_unlock(qlge, SEM_RT_IDX_MASK); /* * Set the flag for receive and transmit * operations to cease */ for (i = 0; i < qlge->tx_ring_count; i++) { mutex_enter(&qlge->tx_ring[i].tx_lock); qlge->tx_ring[i].mac_flags = QL_MAC_STOPPED; mutex_exit(&qlge->tx_ring[i].tx_lock); } for (i = 0; i < qlge->rx_ring_count; i++) { mutex_enter(&qlge->rx_ring[i].rx_lock); qlge->rx_ring[i].mac_flags = QL_MAC_STOPPED; mutex_exit(&qlge->rx_ring[i].rx_lock); } /* * Need interrupts to be running while the transmit * completions are cleared. Wait for the packets * queued to the chip to be sent out */ (void) ql_wait_tx_quiesce(qlge); /* Interrupts not needed from now */ ql_disable_all_completion_interrupts(qlge); mutex_enter(&qlge->hw_mutex); /* Disable Global interrupt */ ql_disable_global_interrupt(qlge); mutex_exit(&qlge->hw_mutex); /* Wait for all the indicated packets to come back */ status = ql_wait_rx_complete(qlge); mutex_enter(&qlge->hw_mutex); /* Reset adapter */ (void) ql_asic_reset(qlge); /* * Unbind all tx dma handles to prevent pending tx descriptors' * dma handles from being re-used. */ for (i = 0; i < qlge->tx_ring_count; i++) { ql_unbind_pending_tx_dma_handle(&qlge->tx_ring[i]); } qlge->sequence &= ~ADAPTER_INIT; mutex_exit(&qlge->hw_mutex); } return (status); } /* * ql_detach * Used to remove all the states associated with a given * instances of a device node prior to the removal of that * instance from the system. */ static int ql_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { qlge_t *qlge; int rval; rval = DDI_SUCCESS; switch (cmd) { case DDI_DETACH: if ((qlge = QL_GET_DEV(dip)) == NULL) return (DDI_FAILURE); rval = ql_bringdown_adapter(qlge); if (rval != DDI_SUCCESS) break; qlge->mac_flags = QL_MAC_DETACH; /* free memory resources */ if (qlge->sequence & INIT_MEMORY_ALLOC) { ql_free_mem_resources(qlge); qlge->sequence &= ~INIT_MEMORY_ALLOC; } ql_free_resources(qlge); break; case DDI_SUSPEND: if ((qlge = QL_GET_DEV(dip)) == NULL) return (DDI_FAILURE); mutex_enter(&qlge->gen_mutex); if ((qlge->mac_flags == QL_MAC_ATTACHED) || (qlge->mac_flags == QL_MAC_STARTED)) { (void) ql_do_stop(qlge); } qlge->mac_flags = QL_MAC_SUSPENDED; mutex_exit(&qlge->gen_mutex); break; default: rval = DDI_FAILURE; break; } return (rval); } /* * quiesce(9E) entry point. * * This function is called when the system is single-threaded at high * PIL with preemption disabled. Therefore, this function must not be * blocked. * * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. */ int ql_quiesce(dev_info_t *dip) { qlge_t *qlge; int i; if ((qlge = QL_GET_DEV(dip)) == NULL) return (DDI_FAILURE); if (CFG_IST(qlge, CFG_CHIP_8100)) { /* stop forwarding external packets to driver */ (void) ql_sem_spinlock(qlge, SEM_RT_IDX_MASK); (void) ql_stop_routing(qlge); ql_sem_unlock(qlge, SEM_RT_IDX_MASK); /* Stop all the request queues */ for (i = 0; i < qlge->tx_ring_count; i++) { if (qlge->tx_ring[i].valid_db_reg) { ql_write_doorbell_reg(qlge, qlge->tx_ring[i].valid_db_reg, 0); } } qlge_delay(QL_ONE_SEC_DELAY/4); /* Interrupts not needed from now */ /* Disable MPI interrupt */ ql_write_reg(qlge, REG_INTERRUPT_MASK, (INTR_MASK_PI << 16)); ql_disable_global_interrupt(qlge); /* Disable all the rx completion queues */ for (i = 0; i < qlge->rx_ring_count; i++) { if (qlge->rx_ring[i].valid_db_reg) { ql_write_doorbell_reg(qlge, qlge->rx_ring[i].valid_db_reg, 0); } } qlge_delay(QL_ONE_SEC_DELAY/4); qlge->mac_flags = QL_MAC_STOPPED; /* Reset adapter */ (void) ql_asic_reset(qlge); qlge_delay(100); } return (DDI_SUCCESS); } QL_STREAM_OPS(ql_ops, ql_attach, ql_detach); /* * Loadable Driver Interface Structures. * Declare and initialize the module configuration section... */ static struct modldrv modldrv = { &mod_driverops, /* type of module: driver */ version, /* name of module */ &ql_ops /* driver dev_ops */ }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; /* * Loadable Module Routines */ /* * _init * Initializes a loadable module. It is called before any other * routine in a loadable module. */ int _init(void) { int rval; mac_init_ops(&ql_ops, ADAPTER_NAME); rval = mod_install(&modlinkage); if (rval != DDI_SUCCESS) { mac_fini_ops(&ql_ops); cmn_err(CE_WARN, "?Unable to install/attach driver '%s'", ADAPTER_NAME); } return (rval); } /* * _fini * Prepares a module for unloading. It is called when the system * wants to unload a module. If the module determines that it can * be unloaded, then _fini() returns the value returned by * mod_remove(). Upon successful return from _fini() no other * routine in the module will be called before _init() is called. */ int _fini(void) { int rval; rval = mod_remove(&modlinkage); if (rval == DDI_SUCCESS) { mac_fini_ops(&ql_ops); } return (rval); } /* * _info * Returns information about loadable module. */ int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); }