/* * CDDL HEADER START * * Copyright(c) 2007-2008 Intel Corporation. All rights reserved. * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at: * http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When using or redistributing this file, you may do so under the * License only. No other modification of this header is permitted. * * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms of the CDDL. */ #pragma ident "%Z%%M% %I% %E% SMI" #include "ixgbe_sw.h" static char ident[] = "Intel 10Gb Ethernet 1.0.0"; /* * Local function protoypes */ static int ixgbe_register_mac(ixgbe_t *); static int ixgbe_identify_hardware(ixgbe_t *); static int ixgbe_regs_map(ixgbe_t *); static void ixgbe_init_properties(ixgbe_t *); static int ixgbe_init_driver_settings(ixgbe_t *); static void ixgbe_init_locks(ixgbe_t *); static void ixgbe_destroy_locks(ixgbe_t *); static int ixgbe_init(ixgbe_t *); static int ixgbe_chip_start(ixgbe_t *); static void ixgbe_chip_stop(ixgbe_t *); static int ixgbe_reset(ixgbe_t *); static void ixgbe_tx_clean(ixgbe_t *); static boolean_t ixgbe_tx_drain(ixgbe_t *); static boolean_t ixgbe_rx_drain(ixgbe_t *); static int ixgbe_alloc_rings(ixgbe_t *); static int ixgbe_init_rings(ixgbe_t *); static void ixgbe_free_rings(ixgbe_t *); static void ixgbe_fini_rings(ixgbe_t *); static void ixgbe_setup_rings(ixgbe_t *); static void ixgbe_setup_rx(ixgbe_t *); static void ixgbe_setup_tx(ixgbe_t *); static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); static void ixgbe_setup_rss(ixgbe_t *); static void ixgbe_init_unicst(ixgbe_t *); static void ixgbe_setup_multicst(ixgbe_t *); static void ixgbe_get_hw_state(ixgbe_t *); static void ixgbe_get_conf(ixgbe_t *); static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); static boolean_t ixgbe_driver_link_check(ixgbe_t *); static void ixgbe_local_timer(void *); static void ixgbe_arm_watchdog_timer(ixgbe_t *); static void ixgbe_start_watchdog_timer(ixgbe_t *); static void ixgbe_restart_watchdog_timer(ixgbe_t *); static void ixgbe_stop_watchdog_timer(ixgbe_t *); static void ixgbe_disable_adapter_interrupts(ixgbe_t *); static void ixgbe_enable_adapter_interrupts(ixgbe_t *); static boolean_t is_valid_mac_addr(uint8_t *); static boolean_t ixgbe_stall_check(ixgbe_t *); static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); static void ixgbe_set_internal_mac_loopback(ixgbe_t *); static boolean_t ixgbe_find_mac_address(ixgbe_t *); static int ixgbe_alloc_intrs(ixgbe_t *); static int ixgbe_alloc_intr_handles(ixgbe_t *, int); static int ixgbe_add_intr_handlers(ixgbe_t *); static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); static void ixgbe_set_ivar(ixgbe_t *, uint16_t, uint8_t); static int ixgbe_map_rings_to_vectors(ixgbe_t *); static void ixgbe_setup_adapter_vector(ixgbe_t *); static void ixgbe_rem_intr_handlers(ixgbe_t *); static void ixgbe_rem_intrs(ixgbe_t *); static int ixgbe_enable_intrs(ixgbe_t *); static int ixgbe_disable_intrs(ixgbe_t *); static uint_t ixgbe_intr_legacy(void *, void *); static uint_t ixgbe_intr_msi(void *, void *); static uint_t ixgbe_intr_rx(void *, void *); static uint_t ixgbe_intr_tx_other(void *, void *); static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); static void ixgbe_intr_other_work(ixgbe_t *); static void ixgbe_get_driver_control(struct ixgbe_hw *); static void ixgbe_release_driver_control(struct ixgbe_hw *); static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); static int ixgbe_resume(dev_info_t *); static int ixgbe_suspend(dev_info_t *); static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data); static void ixgbe_fm_init(ixgbe_t *); static void ixgbe_fm_fini(ixgbe_t *); static struct cb_ops ixgbe_cb_ops = { nulldev, /* cb_open */ nulldev, /* cb_close */ nodev, /* cb_strategy */ nodev, /* cb_print */ nodev, /* cb_dump */ nodev, /* cb_read */ nodev, /* cb_write */ nodev, /* cb_ioctl */ nodev, /* cb_devmap */ nodev, /* cb_mmap */ nodev, /* cb_segmap */ nochpoll, /* cb_chpoll */ ddi_prop_op, /* cb_prop_op */ NULL, /* cb_stream */ D_MP | D_HOTPLUG, /* cb_flag */ CB_REV, /* cb_rev */ nodev, /* cb_aread */ nodev /* cb_awrite */ }; static struct dev_ops ixgbe_dev_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ NULL, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ ixgbe_attach, /* devo_attach */ ixgbe_detach, /* devo_detach */ nodev, /* devo_reset */ &ixgbe_cb_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ ddi_power /* devo_power */ }; static struct modldrv ixgbe_modldrv = { &mod_driverops, /* Type of module. This one is a driver */ ident, /* Discription string */ &ixgbe_dev_ops /* driver ops */ }; static struct modlinkage ixgbe_modlinkage = { MODREV_1, &ixgbe_modldrv, NULL }; /* * Access attributes for register mapping */ ddi_device_acc_attr_t ixgbe_regs_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_STRUCTURE_LE_ACC, DDI_STRICTORDER_ACC, DDI_FLAGERR_ACC }; /* * Loopback property */ static lb_property_t lb_normal = { normal, "normal", IXGBE_LB_NONE }; static lb_property_t lb_mac = { internal, "MAC", IXGBE_LB_INTERNAL_MAC }; #define IXGBE_M_CALLBACK_FLAGS (MC_IOCTL | MC_GETCAPAB) static mac_callbacks_t ixgbe_m_callbacks = { IXGBE_M_CALLBACK_FLAGS, ixgbe_m_stat, ixgbe_m_start, ixgbe_m_stop, ixgbe_m_promisc, ixgbe_m_multicst, ixgbe_m_unicst, ixgbe_m_tx, NULL, ixgbe_m_ioctl, ixgbe_m_getcapab }; /* * Module Initialization Functions. */ int _init(void) { int status; mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); status = mod_install(&ixgbe_modlinkage); if (status != DDI_SUCCESS) { mac_fini_ops(&ixgbe_dev_ops); } return (status); } int _fini(void) { int status; status = mod_remove(&ixgbe_modlinkage); if (status == DDI_SUCCESS) { mac_fini_ops(&ixgbe_dev_ops); } return (status); } int _info(struct modinfo *modinfop) { int status; status = mod_info(&ixgbe_modlinkage, modinfop); return (status); } /* * ixgbe_attach - Driver attach. * * This function is the device specific initialization entry * point. This entry point is required and must be written. * The DDI_ATTACH command must be provided in the attach entry * point. When attach() is called with cmd set to DDI_ATTACH, * all normal kernel services (such as kmem_alloc(9F)) are * available for use by the driver. * * The attach() function will be called once for each instance * of the device on the system with cmd set to DDI_ATTACH. * Until attach() succeeds, the only driver entry points which * may be called are open(9E) and getinfo(9E). */ static int ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) { ixgbe_t *ixgbe; struct ixgbe_osdep *osdep; struct ixgbe_hw *hw; int instance; /* * Check the command and perform corresponding operations */ switch (cmd) { default: return (DDI_FAILURE); case DDI_RESUME: return (ixgbe_resume(devinfo)); case DDI_ATTACH: break; } /* Get the device instance */ instance = ddi_get_instance(devinfo); /* Allocate memory for the instance data structure */ ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); ixgbe->dip = devinfo; ixgbe->instance = instance; hw = &ixgbe->hw; osdep = &ixgbe->osdep; hw->back = osdep; osdep->ixgbe = ixgbe; /* Attach the instance pointer to the dev_info data structure */ ddi_set_driver_private(devinfo, ixgbe); /* * Initialize for fma support */ ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, "PROP_FM_CAPABLE", 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); ixgbe_fm_init(ixgbe); ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; /* * Map PCI config space registers */ if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { ixgbe_error(ixgbe, "Failed to map PCI configurations"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; /* * Identify the chipset family */ if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to identify hardware"); goto attach_fail; } /* * Map device registers */ if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to map device registers"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; /* * Initialize driver parameters */ ixgbe_init_properties(ixgbe); ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; /* * Allocate interrupts */ if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to allocate interrupts"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; /* * Allocate rx/tx rings based on the ring numbers. * The actual numbers of rx/tx rings are decided by the number of * allocated interrupt vectors, so we should allocate the rings after * interrupts are allocated. */ if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; /* * Map rings to interrupt vectors */ if (ixgbe_map_rings_to_vectors(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to map rings to vectors"); goto attach_fail; } /* * Add interrupt handlers */ if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to add interrupt handlers"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; /* * Initialize driver parameters */ if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize driver settings"); goto attach_fail; } /* * Initialize mutexes for this device. * Do this before enabling the interrupt handler and * register the softint to avoid the condition where * interrupt handler can try using uninitialized mutex. */ ixgbe_init_locks(ixgbe); ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; /* * Initialize chipset hardware */ if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize adapter"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); goto attach_fail; } /* * Initialize DMA and hardware settings for rx/tx rings */ if (ixgbe_init_rings(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize rings"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_INIT_RINGS; /* * Initialize statistics */ if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize statistics"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; /* * Initialize NDD parameters */ if (ixgbe_nd_init(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize ndd"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_NDD; /* * Register the driver to the MAC */ if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to register MAC"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; /* * Now that mutex locks are initialized, and the chip is also * initialized, enable interrupts. */ if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); goto attach_fail; } ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; ixgbe->ixgbe_state |= IXGBE_INITIALIZED; return (DDI_SUCCESS); attach_fail: ixgbe_unconfigure(devinfo, ixgbe); return (DDI_FAILURE); } /* * ixgbe_detach - Driver detach. * * The detach() function is the complement of the attach routine. * If cmd is set to DDI_DETACH, detach() is used to remove the * state associated with a given instance of a device node * prior to the removal of that instance from the system. * * The detach() function will be called once for each instance * of the device for which there has been a successful attach() * once there are no longer any opens on the device. * * Interrupts routine are disabled, All memory allocated by this * driver are freed. */ static int ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) { ixgbe_t *ixgbe; /* * Check detach command */ switch (cmd) { default: return (DDI_FAILURE); case DDI_SUSPEND: return (ixgbe_suspend(devinfo)); case DDI_DETACH: break; } /* * Get the pointer to the driver private data structure */ ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); if (ixgbe == NULL) return (DDI_FAILURE); /* * Unregister MAC. If failed, we have to fail the detach */ if (mac_unregister(ixgbe->mac_hdl) != 0) { ixgbe_error(ixgbe, "Failed to unregister MAC"); return (DDI_FAILURE); } ixgbe->attach_progress &= ~ATTACH_PROGRESS_MAC; /* * If the device is still running, it needs to be stopped first. * This check is necessary because under some specific circumstances, * the detach routine can be called without stopping the interface * first. */ mutex_enter(&ixgbe->gen_lock); if (ixgbe->ixgbe_state & IXGBE_STARTED) { ixgbe->ixgbe_state &= ~IXGBE_STARTED; ixgbe_stop(ixgbe); mutex_exit(&ixgbe->gen_lock); /* Disable and stop the watchdog timer */ ixgbe_disable_watchdog_timer(ixgbe); } else mutex_exit(&ixgbe->gen_lock); /* * Check if there are still rx buffers held by the upper layer. * If so, fail the detach. */ if (!ixgbe_rx_drain(ixgbe)) return (DDI_FAILURE); /* * Do the remaining unconfigure routines */ ixgbe_unconfigure(devinfo, ixgbe); return (DDI_SUCCESS); } static void ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) { /* * Disable interrupt */ if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { (void) ixgbe_disable_intrs(ixgbe); } /* * Unregister MAC */ if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { (void) mac_unregister(ixgbe->mac_hdl); } /* * Free ndd parameters */ if (ixgbe->attach_progress & ATTACH_PROGRESS_NDD) { ixgbe_nd_cleanup(ixgbe); } /* * Free statistics */ if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { kstat_delete((kstat_t *)ixgbe->ixgbe_ks); } /* * Remove interrupt handlers */ if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { ixgbe_rem_intr_handlers(ixgbe); } /* * Remove interrupts */ if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { ixgbe_rem_intrs(ixgbe); } /* * Remove driver properties */ if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { (void) ddi_prop_remove_all(devinfo); } /* * Release the DMA resources of rx/tx rings */ if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT_RINGS) { ixgbe_fini_rings(ixgbe); } /* * Stop the chipset */ if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { mutex_enter(&ixgbe->gen_lock); ixgbe_chip_stop(ixgbe); mutex_exit(&ixgbe->gen_lock); } /* * Free register handle */ if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { if (ixgbe->osdep.reg_handle != NULL) ddi_regs_map_free(&ixgbe->osdep.reg_handle); } /* * Free PCI config handle */ if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { if (ixgbe->osdep.cfg_handle != NULL) pci_config_teardown(&ixgbe->osdep.cfg_handle); } /* * Free locks */ if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { ixgbe_destroy_locks(ixgbe); } /* * Free the rx/tx rings */ if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { ixgbe_free_rings(ixgbe); } /* * Unregister FMA capabilities */ if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { ixgbe_fm_fini(ixgbe); } /* * Free the driver data structure */ kmem_free(ixgbe, sizeof (ixgbe_t)); ddi_set_driver_private(devinfo, NULL); } /* * ixgbe_register_mac - Register the driver and its function pointers with * the GLD interface. */ static int ixgbe_register_mac(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; mac_register_t *mac; int status; if ((mac = mac_alloc(MAC_VERSION)) == NULL) return (IXGBE_FAILURE); mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; mac->m_driver = ixgbe; mac->m_dip = ixgbe->dip; mac->m_src_addr = hw->mac.addr; mac->m_callbacks = &ixgbe_m_callbacks; mac->m_min_sdu = 0; mac->m_max_sdu = ixgbe->default_mtu; mac->m_margin = VLAN_TAGSZ; status = mac_register(mac, &ixgbe->mac_hdl); mac_free(mac); return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); } /* * ixgbe_identify_hardware - Identify the type of the chipset. */ static int ixgbe_identify_hardware(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; struct ixgbe_osdep *osdep = &ixgbe->osdep; /* * Get the device id */ hw->vendor_id = pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); hw->device_id = pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); hw->revision_id = pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); hw->subsystem_device_id = pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); hw->subsystem_vendor_id = pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); return (IXGBE_SUCCESS); } /* * ixgbe_regs_map - Map the device registers. * */ static int ixgbe_regs_map(ixgbe_t *ixgbe) { dev_info_t *devinfo = ixgbe->dip; struct ixgbe_hw *hw = &ixgbe->hw; struct ixgbe_osdep *osdep = &ixgbe->osdep; off_t mem_size; /* * First get the size of device registers to be mapped. */ if (ddi_dev_regsize(devinfo, 1, &mem_size) != DDI_SUCCESS) { return (IXGBE_FAILURE); } /* * Call ddi_regs_map_setup() to map registers */ if ((ddi_regs_map_setup(devinfo, 1, (caddr_t *)&hw->hw_addr, 0, mem_size, &ixgbe_regs_acc_attr, &osdep->reg_handle)) != DDI_SUCCESS) { return (IXGBE_FAILURE); } return (IXGBE_SUCCESS); } /* * ixgbe_init_properties - Initialize driver properties. */ static void ixgbe_init_properties(ixgbe_t *ixgbe) { /* * Get conf file properties, including link settings * jumbo frames, ring number, descriptor number, etc. */ ixgbe_get_conf(ixgbe); } /* * ixgbe_init_driver_settings - Initialize driver settings. * * The settings include hardware function pointers, bus information, * rx/tx rings settings, link state, and any other parameters that * need to be setup during driver initialization. */ static int ixgbe_init_driver_settings(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; ixgbe_rx_ring_t *rx_ring; ixgbe_tx_ring_t *tx_ring; uint32_t rx_size; uint32_t tx_size; int i; /* * Initialize chipset specific hardware function pointers */ if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { return (IXGBE_FAILURE); } /* * Set rx buffer size * * The IP header alignment room is counted in the calculation. * The rx buffer size is in unit of 1K that is required by the * chipset hardware. */ rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; ixgbe->rx_buf_size = ((rx_size >> 10) + ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; /* * Set tx buffer size */ tx_size = ixgbe->max_frame_size; ixgbe->tx_buf_size = ((tx_size >> 10) + ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; /* * Initialize rx/tx rings parameters */ for (i = 0; i < ixgbe->num_rx_rings; i++) { rx_ring = &ixgbe->rx_rings[i]; rx_ring->index = i; rx_ring->ixgbe = ixgbe; rx_ring->ring_size = ixgbe->rx_ring_size; rx_ring->free_list_size = ixgbe->rx_ring_size; rx_ring->copy_thresh = ixgbe->rx_copy_thresh; rx_ring->limit_per_intr = ixgbe->rx_limit_per_intr; } for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; tx_ring->index = i; tx_ring->ixgbe = ixgbe; if (ixgbe->tx_head_wb_enable) tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; else tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; tx_ring->ring_size = ixgbe->tx_ring_size; tx_ring->free_list_size = ixgbe->tx_ring_size + (ixgbe->tx_ring_size >> 1); tx_ring->copy_thresh = ixgbe->tx_copy_thresh; tx_ring->recycle_thresh = ixgbe->tx_recycle_thresh; tx_ring->overload_thresh = ixgbe->tx_overload_thresh; tx_ring->resched_thresh = ixgbe->tx_resched_thresh; } /* * Initialize values of interrupt throttling rate */ for (i = 1; i < IXGBE_MAX_RING_VECTOR; i++) ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; /* * The initial link state should be "unknown" */ ixgbe->link_state = LINK_STATE_UNKNOWN; return (IXGBE_SUCCESS); } /* * ixgbe_init_locks - Initialize locks. */ static void ixgbe_init_locks(ixgbe_t *ixgbe) { ixgbe_rx_ring_t *rx_ring; ixgbe_tx_ring_t *tx_ring; int i; for (i = 0; i < ixgbe->num_rx_rings; i++) { rx_ring = &ixgbe->rx_rings[i]; mutex_init(&rx_ring->rx_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); mutex_init(&rx_ring->recycle_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); } for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; mutex_init(&tx_ring->tx_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); mutex_init(&tx_ring->recycle_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); mutex_init(&tx_ring->tcb_head_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); mutex_init(&tx_ring->tcb_tail_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); } mutex_init(&ixgbe->gen_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); mutex_init(&ixgbe->watchdog_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); } /* * ixgbe_destroy_locks - Destroy locks. */ static void ixgbe_destroy_locks(ixgbe_t *ixgbe) { ixgbe_rx_ring_t *rx_ring; ixgbe_tx_ring_t *tx_ring; int i; for (i = 0; i < ixgbe->num_rx_rings; i++) { rx_ring = &ixgbe->rx_rings[i]; mutex_destroy(&rx_ring->rx_lock); mutex_destroy(&rx_ring->recycle_lock); } for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; mutex_destroy(&tx_ring->tx_lock); mutex_destroy(&tx_ring->recycle_lock); mutex_destroy(&tx_ring->tcb_head_lock); mutex_destroy(&tx_ring->tcb_tail_lock); } mutex_destroy(&ixgbe->gen_lock); mutex_destroy(&ixgbe->watchdog_lock); } static int ixgbe_resume(dev_info_t *devinfo) { ixgbe_t *ixgbe; ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); if (ixgbe == NULL) return (DDI_FAILURE); mutex_enter(&ixgbe->gen_lock); if (ixgbe->ixgbe_state & IXGBE_STARTED) { if (ixgbe_start(ixgbe) != IXGBE_SUCCESS) { mutex_exit(&ixgbe->gen_lock); return (DDI_FAILURE); } /* * Enable and start the watchdog timer */ ixgbe_enable_watchdog_timer(ixgbe); } ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; mutex_exit(&ixgbe->gen_lock); return (DDI_SUCCESS); } static int ixgbe_suspend(dev_info_t *devinfo) { ixgbe_t *ixgbe; ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); if (ixgbe == NULL) return (DDI_FAILURE); mutex_enter(&ixgbe->gen_lock); ixgbe->ixgbe_state |= IXGBE_SUSPENDED; ixgbe_stop(ixgbe); mutex_exit(&ixgbe->gen_lock); /* * Disable and stop the watchdog timer */ ixgbe_disable_watchdog_timer(ixgbe); return (DDI_SUCCESS); } /* * ixgbe_init - Initialize the device. */ static int ixgbe_init(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; mutex_enter(&ixgbe->gen_lock); /* * Reset chipset to put the hardware in a known state * before we try to do anything with the eeprom. */ if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto init_fail; } /* * Need to init eeprom before validating the checksum. */ if (ixgbe_init_eeprom_params(hw) < 0) { ixgbe_error(ixgbe, "Unable to intitialize the eeprom interface."); ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto init_fail; } /* * NVM validation */ if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { /* * Some PCI-E parts fail the first check due to * the link being in sleep state. Call it again, * if it fails a second time it's a real issue. */ if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { ixgbe_error(ixgbe, "Invalid NVM checksum. Please contact " "the vendor to update the NVM."); ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto init_fail; } } /* * Setup default flow control thresholds - enable/disable * & flow control type is controlled by ixgbe.conf */ hw->fc.high_water = DEFAULT_FCRTH; hw->fc.low_water = DEFAULT_FCRTL; hw->fc.pause_time = DEFAULT_FCPAUSE; hw->fc.send_xon = B_TRUE; /* * Don't wait for auto-negotiation to complete */ hw->phy.autoneg_wait_to_complete = B_FALSE; /* * Initialize link settings */ (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); /* * Initialize the chipset hardware */ if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto init_fail; } if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { goto init_fail; } if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { goto init_fail; } mutex_exit(&ixgbe->gen_lock); return (IXGBE_SUCCESS); init_fail: /* * Reset PHY */ (void) ixgbe_reset_phy(hw); mutex_exit(&ixgbe->gen_lock); ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); return (IXGBE_FAILURE); } /* * ixgbe_init_rings - Allocate DMA resources for all rx/tx rings and * initialize relevant hardware settings. */ static int ixgbe_init_rings(ixgbe_t *ixgbe) { int i; /* * Allocate buffers for all the rx/tx rings */ if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) return (IXGBE_FAILURE); /* * Setup the rx/tx rings */ mutex_enter(&ixgbe->gen_lock); for (i = 0; i < ixgbe->num_rx_rings; i++) mutex_enter(&ixgbe->rx_rings[i].rx_lock); for (i = 0; i < ixgbe->num_tx_rings; i++) mutex_enter(&ixgbe->tx_rings[i].tx_lock); ixgbe_setup_rings(ixgbe); for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); mutex_exit(&ixgbe->gen_lock); return (IXGBE_SUCCESS); } /* * ixgbe_fini_rings - Release DMA resources of all rx/tx rings. */ static void ixgbe_fini_rings(ixgbe_t *ixgbe) { /* * Release the DMA/memory resources of rx/tx rings */ ixgbe_free_dma(ixgbe); } /* * ixgbe_chip_start - Initialize and start the chipset hardware. */ static int ixgbe_chip_start(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; int i; ASSERT(mutex_owned(&ixgbe->gen_lock)); /* * Get the mac address * This function should handle SPARC case correctly. */ if (!ixgbe_find_mac_address(ixgbe)) { ixgbe_error(ixgbe, "Failed to get the mac address"); return (IXGBE_FAILURE); } /* * Validate the mac address */ (void) ixgbe_init_rx_addrs(hw); if (!is_valid_mac_addr(hw->mac.addr)) { ixgbe_error(ixgbe, "Invalid mac address"); return (IXGBE_FAILURE); } /* * Configure/Initialize hardware */ if (ixgbe_init_hw(hw) != IXGBE_SUCCESS) { ixgbe_error(ixgbe, "Failed to initialize hardware"); return (IXGBE_FAILURE); } /* * Setup adapter interrupt vectors */ ixgbe_setup_adapter_vector(ixgbe); /* * Initialize unicast addresses. */ ixgbe_init_unicst(ixgbe); /* * Setup and initialize the mctable structures. */ ixgbe_setup_multicst(ixgbe); /* * Set interrupt throttling rate */ for (i = 0; i < ixgbe->intr_cnt; i++) IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); /* * Save the state of the phy */ ixgbe_get_hw_state(ixgbe); /* * Make sure driver has control */ ixgbe_get_driver_control(hw); return (IXGBE_SUCCESS); } /* * ixgbe_chip_stop - Stop the chipset hardware */ static void ixgbe_chip_stop(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; ASSERT(mutex_owned(&ixgbe->gen_lock)); /* * Tell firmware driver is no longer in control */ ixgbe_release_driver_control(hw); /* * Reset the chipset */ (void) ixgbe_reset_hw(hw); /* * Reset PHY */ (void) ixgbe_reset_phy(hw); } /* * ixgbe_reset - Reset the chipset and re-start the driver. * * It involves stopping and re-starting the chipset, * and re-configuring the rx/tx rings. */ static int ixgbe_reset(ixgbe_t *ixgbe) { int i; mutex_enter(&ixgbe->gen_lock); ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); ixgbe->ixgbe_state &= ~IXGBE_STARTED; /* * Disable the adapter interrupts to stop any rx/tx activities * before draining pending data and resetting hardware. */ ixgbe_disable_adapter_interrupts(ixgbe); /* * Drain the pending transmit packets */ (void) ixgbe_tx_drain(ixgbe); for (i = 0; i < ixgbe->num_rx_rings; i++) mutex_enter(&ixgbe->rx_rings[i].rx_lock); for (i = 0; i < ixgbe->num_tx_rings; i++) mutex_enter(&ixgbe->tx_rings[i].tx_lock); /* * Stop the chipset hardware */ ixgbe_chip_stop(ixgbe); /* * Clean the pending tx data/resources */ ixgbe_tx_clean(ixgbe); /* * Start the chipset hardware */ if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto reset_failure; } if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { goto reset_failure; } /* * Setup the rx/tx rings */ ixgbe_setup_rings(ixgbe); /* * Enable adapter interrupts * The interrupts must be enabled after the driver state is START */ ixgbe_enable_adapter_interrupts(ixgbe); for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); ixgbe->ixgbe_state |= IXGBE_STARTED; mutex_exit(&ixgbe->gen_lock); return (IXGBE_SUCCESS); reset_failure: for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); mutex_exit(&ixgbe->gen_lock); ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); return (IXGBE_FAILURE); } /* * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. */ static void ixgbe_tx_clean(ixgbe_t *ixgbe) { ixgbe_tx_ring_t *tx_ring; tx_control_block_t *tcb; link_list_t pending_list; uint32_t desc_num; struct ixgbe_hw *hw = &ixgbe->hw; int i, j; LINK_LIST_INIT(&pending_list); for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; mutex_enter(&tx_ring->recycle_lock); /* * Clean the pending tx data - the pending packets in the * work_list that have no chances to be transmitted again. * * We must ensure the chipset is stopped or the link is down * before cleaning the transmit packets. */ desc_num = 0; for (j = 0; j < tx_ring->ring_size; j++) { tcb = tx_ring->work_list[j]; if (tcb != NULL) { desc_num += tcb->desc_num; tx_ring->work_list[j] = NULL; ixgbe_free_tcb(tcb); LIST_PUSH_TAIL(&pending_list, &tcb->link); } } if (desc_num > 0) { atomic_add_32(&tx_ring->tbd_free, desc_num); ASSERT(tx_ring->tbd_free == tx_ring->ring_size); /* * Reset the head and tail pointers of the tbd ring; * Reset the writeback head if it's enable. */ tx_ring->tbd_head = 0; tx_ring->tbd_tail = 0; if (ixgbe->tx_head_wb_enable) *tx_ring->tbd_head_wb = 0; IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_TDH(tx_ring->index), 0); IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_TDT(tx_ring->index), 0); } mutex_exit(&tx_ring->recycle_lock); /* * Add the tx control blocks in the pending list to * the free list. */ ixgbe_put_free_list(tx_ring, &pending_list); } } /* * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be * transmitted. */ static boolean_t ixgbe_tx_drain(ixgbe_t *ixgbe) { ixgbe_tx_ring_t *tx_ring; boolean_t done; int i, j; /* * Wait for a specific time to allow pending tx packets * to be transmitted. * * Check the counter tbd_free to see if transmission is done. * No lock protection is needed here. * * Return B_TRUE if all pending packets have been transmitted; * Otherwise return B_FALSE; */ for (i = 0; i < TX_DRAIN_TIME; i++) { done = B_TRUE; for (j = 0; j < ixgbe->num_tx_rings; j++) { tx_ring = &ixgbe->tx_rings[j]; done = done && (tx_ring->tbd_free == tx_ring->ring_size); } if (done) break; msec_delay(1); } return (done); } /* * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. */ static boolean_t ixgbe_rx_drain(ixgbe_t *ixgbe) { ixgbe_rx_ring_t *rx_ring; boolean_t done; int i, j; /* * Polling the rx free list to check if those rx buffers held by * the upper layer are released. * * Check the counter rcb_free to see if all pending buffers are * released. No lock protection is needed here. * * Return B_TRUE if all pending buffers have been released; * Otherwise return B_FALSE; */ for (i = 0; i < RX_DRAIN_TIME; i++) { done = B_TRUE; for (j = 0; j < ixgbe->num_rx_rings; j++) { rx_ring = &ixgbe->rx_rings[j]; done = done && (rx_ring->rcb_free == rx_ring->free_list_size); } if (done) break; msec_delay(1); } return (done); } /* * ixgbe_start - Start the driver/chipset. */ int ixgbe_start(ixgbe_t *ixgbe) { int i; ASSERT(mutex_owned(&ixgbe->gen_lock)); for (i = 0; i < ixgbe->num_rx_rings; i++) mutex_enter(&ixgbe->rx_rings[i].rx_lock); for (i = 0; i < ixgbe->num_tx_rings; i++) mutex_enter(&ixgbe->tx_rings[i].tx_lock); /* * Start the chipset hardware */ if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); goto start_failure; } if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { goto start_failure; } /* * Setup the rx/tx rings */ ixgbe_setup_rings(ixgbe); /* * Enable adapter interrupts * The interrupts must be enabled after the driver state is START */ ixgbe_enable_adapter_interrupts(ixgbe); for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); return (IXGBE_SUCCESS); start_failure: for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); return (IXGBE_FAILURE); } /* * ixgbe_stop - Stop the driver/chipset. */ void ixgbe_stop(ixgbe_t *ixgbe) { int i; ASSERT(mutex_owned(&ixgbe->gen_lock)); /* * Disable the adapter interrupts */ ixgbe_disable_adapter_interrupts(ixgbe); /* * Drain the pending tx packets */ (void) ixgbe_tx_drain(ixgbe); for (i = 0; i < ixgbe->num_rx_rings; i++) mutex_enter(&ixgbe->rx_rings[i].rx_lock); for (i = 0; i < ixgbe->num_tx_rings; i++) mutex_enter(&ixgbe->tx_rings[i].tx_lock); /* * Stop the chipset hardware */ ixgbe_chip_stop(ixgbe); if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); } /* * Clean the pending tx data/resources */ ixgbe_tx_clean(ixgbe); for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->tx_rings[i].tx_lock); for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) mutex_exit(&ixgbe->rx_rings[i].rx_lock); } /* * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. */ static int ixgbe_alloc_rings(ixgbe_t *ixgbe) { /* * Allocate memory space for rx rings */ ixgbe->rx_rings = kmem_zalloc( sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, KM_NOSLEEP); if (ixgbe->rx_rings == NULL) { return (IXGBE_FAILURE); } /* * Allocate memory space for tx rings */ ixgbe->tx_rings = kmem_zalloc( sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, KM_NOSLEEP); if (ixgbe->tx_rings == NULL) { kmem_free(ixgbe->rx_rings, sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); ixgbe->rx_rings = NULL; return (IXGBE_FAILURE); } return (IXGBE_SUCCESS); } /* * ixgbe_free_rings - Free the memory space of rx/tx rings. */ static void ixgbe_free_rings(ixgbe_t *ixgbe) { if (ixgbe->rx_rings != NULL) { kmem_free(ixgbe->rx_rings, sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); ixgbe->rx_rings = NULL; } if (ixgbe->tx_rings != NULL) { kmem_free(ixgbe->tx_rings, sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); ixgbe->tx_rings = NULL; } } /* * ixgbe_setup_rings - Setup rx/tx rings. */ static void ixgbe_setup_rings(ixgbe_t *ixgbe) { /* * Setup the rx/tx rings, including the following: * * 1. Setup the descriptor ring and the control block buffers; * 2. Initialize necessary registers for receive/transmit; * 3. Initialize software pointers/parameters for receive/transmit; */ ixgbe_setup_rx(ixgbe); ixgbe_setup_tx(ixgbe); } static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) { ixgbe_t *ixgbe = rx_ring->ixgbe; struct ixgbe_hw *hw = &ixgbe->hw; rx_control_block_t *rcb; union ixgbe_adv_rx_desc *rbd; uint32_t size; uint32_t buf_low; uint32_t buf_high; uint32_t reg_val; int i; ASSERT(mutex_owned(&rx_ring->rx_lock)); ASSERT(mutex_owned(&ixgbe->gen_lock)); for (i = 0; i < ixgbe->rx_ring_size; i++) { rcb = rx_ring->work_list[i]; rbd = &rx_ring->rbd_ring[i]; rbd->read.pkt_addr = rcb->rx_buf.dma_address; rbd->read.hdr_addr = NULL; } /* * Initialize the length register */ size = rx_ring->ring_size * sizeof (union ixgbe_adv_rx_desc); IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->index), size); /* * Initialize the base address registers */ buf_low = (uint32_t)rx_ring->rbd_area.dma_address; buf_high = (uint32_t)(rx_ring->rbd_area.dma_address >> 32); IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->index), buf_high); IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->index), buf_low); /* * Setup head & tail pointers */ IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->index), rx_ring->ring_size - 1); IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->index), 0); rx_ring->rbd_next = 0; /* * Note: Considering the case that the chipset is being reset * and there are still some buffers held by the upper layer, * we should not reset the values of rcb_head, rcb_tail and * rcb_free if the state is not IXGBE_UNKNOWN. */ if (ixgbe->ixgbe_state == IXGBE_UNKNOWN) { rx_ring->rcb_head = 0; rx_ring->rcb_tail = 0; rx_ring->rcb_free = rx_ring->free_list_size; } /* * Setup the Receive Descriptor Control Register (RXDCTL) * PTHRESH=32 descriptors (half the internal cache) * HTHRESH=0 descriptors (to minimize latency on fetch) * WTHRESH defaults to 1 (writeback each descriptor) */ reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->index)); reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ reg_val |= 0x0020; /* pthresh */ IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->index), reg_val); /* * Setup the Split and Replication Receive Control Register. * Set the rx buffer size and the advanced descriptor type. */ reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->index), reg_val); } static void ixgbe_setup_rx(ixgbe_t *ixgbe) { ixgbe_rx_ring_t *rx_ring; struct ixgbe_hw *hw = &ixgbe->hw; uint32_t reg_val; int i; /* * Set filter control in FCTRL to accept broadcast packets and do * not pass pause frames to host. Flow control settings are already * in this register, so preserve them. */ reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); /* * Enable the receive unit. This must be done after filter * control is set in FCTRL. */ reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); /* * ixgbe_setup_rx_ring must be called after configuring RXCTRL */ for (i = 0; i < ixgbe->num_rx_rings; i++) { rx_ring = &ixgbe->rx_rings[i]; ixgbe_setup_rx_ring(rx_ring); } /* * The Max Frame Size in MHADD will be internally increased by four * bytes if the packet has a VLAN field, so includes MTU, ethernet * header and frame check sequence. */ reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); /* * Setup Jumbo Frame enable bit */ if (ixgbe->default_mtu > ETHERMTU) { reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); reg_val |= IXGBE_HLREG0_JUMBOEN; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); } /* * Hardware checksum settings */ if (ixgbe->rx_hcksum_enable) { reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); } /* * Setup RSS for multiple receive queues */ if (ixgbe->num_rx_rings > 1) ixgbe_setup_rss(ixgbe); } static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) { ixgbe_t *ixgbe = tx_ring->ixgbe; struct ixgbe_hw *hw = &ixgbe->hw; uint32_t size; uint32_t buf_low; uint32_t buf_high; uint32_t reg_val; ASSERT(mutex_owned(&tx_ring->tx_lock)); ASSERT(mutex_owned(&ixgbe->gen_lock)); /* * Initialize the length register */ size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); /* * Initialize the base address registers */ buf_low = (uint32_t)tx_ring->tbd_area.dma_address; buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); /* * setup TXDCTL(tx_ring->index) */ reg_val = IXGBE_TXDCTL_ENABLE; IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); /* * Setup head & tail pointers */ IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); /* * Setup head write-back */ if (ixgbe->tx_head_wb_enable) { /* * The memory of the head write-back is allocated using * the extra tbd beyond the tail of the tbd ring. */ tx_ring->tbd_head_wb = (uint32_t *) ((uintptr_t)tx_ring->tbd_area.address + size); *tx_ring->tbd_head_wb = 0; buf_low = (uint32_t) (tx_ring->tbd_area.dma_address + size); buf_high = (uint32_t) ((tx_ring->tbd_area.dma_address + size) >> 32); /* Set the head write-back enable bit */ buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); /* * Turn off relaxed ordering for head write back or it will * cause problems with the tx recycling */ reg_val = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)); reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); } else { tx_ring->tbd_head_wb = NULL; } tx_ring->tbd_head = 0; tx_ring->tbd_tail = 0; tx_ring->tbd_free = tx_ring->ring_size; /* * Note: Considering the case that the chipset is being reset, * and there are still some tcb in the pending list, * we should not reset the values of tcb_head, tcb_tail and * tcb_free if the state is not IXGBE_UNKNOWN. */ if (ixgbe->ixgbe_state == IXGBE_UNKNOWN) { tx_ring->tcb_head = 0; tx_ring->tcb_tail = 0; tx_ring->tcb_free = tx_ring->free_list_size; } /* * Initialize hardware checksum offload settings */ tx_ring->hcksum_context.hcksum_flags = 0; tx_ring->hcksum_context.ip_hdr_len = 0; tx_ring->hcksum_context.mac_hdr_len = 0; tx_ring->hcksum_context.l4_proto = 0; } static void ixgbe_setup_tx(ixgbe_t *ixgbe) { ixgbe_tx_ring_t *tx_ring; int i; for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; ixgbe_setup_tx_ring(tx_ring); } } /* * ixgbe_setup_rss - Setup receive-side scaling feature. */ static void ixgbe_setup_rss(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; uint32_t i, j, mrqc, rxcsum; uint32_t random; uint32_t reta; /* * Fill out redirection table */ j = 0; reta = 0; for (i = 0; i < 128; i++) { reta = (reta << 8) | (j * 0x11); if (j == 3) IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); j = ((j + 1) % 4); } /* * Fill out hash function seeds with a random constant */ for (i = 0; i < 10; i++) { (void) random_get_pseudo_bytes((uint8_t *)&random, sizeof (uint32_t)); IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); } /* * enable RSS & perform hash on these packet types */ mrqc = IXGBE_MRQC_RSSEN | IXGBE_MRQC_RSS_FIELD_IPV4 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP | IXGBE_MRQC_RSS_FIELD_IPV4_UDP | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | IXGBE_MRQC_RSS_FIELD_IPV6_EX | IXGBE_MRQC_RSS_FIELD_IPV6 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP | IXGBE_MRQC_RSS_FIELD_IPV6_UDP | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); /* * Disable Packet Checksum to enable RSS for multiple receive queues. * * It is an adapter hardware limitation that Packet Checksum is * mutually exclusive with RSS. */ rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); rxcsum |= IXGBE_RXCSUM_PCSD; rxcsum &= ~IXGBE_RXCSUM_IPPCSE; IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); } /* * ixgbe_init_unicst - Initialize the unicast addresses. */ static void ixgbe_init_unicst(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; int slot; /* * Here we should consider two situations: * * 1. Chipset is initialized the first time * Initialize the multiple unicast addresses, and * save the default mac address. * * 2. Chipset is reset * Recover the multiple unicast addresses from the * software data structure to the RAR registers. */ if (!ixgbe->unicst_init) { /* * Initialize the multiple unicast addresses */ ixgbe->unicst_total = MAX_NUM_UNICAST_ADDRESSES; ixgbe->unicst_avail = ixgbe->unicst_total - 1; bcopy(hw->mac.addr, ixgbe->unicst_addr[0].mac.addr, ETHERADDRL); ixgbe->unicst_addr[0].mac.set = 1; for (slot = 1; slot < ixgbe->unicst_total; slot++) ixgbe->unicst_addr[slot].mac.set = 0; ixgbe->unicst_init = B_TRUE; } else { /* * Recover the default mac address */ bcopy(ixgbe->unicst_addr[0].mac.addr, hw->mac.addr, ETHERADDRL); /* Re-configure the RAR registers */ for (slot = 1; slot < ixgbe->unicst_total; slot++) (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, NULL, NULL); } } /* * ixgbe_unicst_set - Set the unicast address to the specified slot. */ int ixgbe_unicst_set(ixgbe_t *ixgbe, const uint8_t *mac_addr, mac_addr_slot_t slot) { struct ixgbe_hw *hw = &ixgbe->hw; ASSERT(mutex_owned(&ixgbe->gen_lock)); /* * Save the unicast address in the software data structure */ bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); /* * Set the unicast address to the RAR register */ (void) ixgbe_set_rar(hw, slot, (uint8_t *)mac_addr, NULL, NULL); if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); return (EIO); } return (0); } /* * ixgbe_multicst_add - Add a multicst address. */ int ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) { ASSERT(mutex_owned(&ixgbe->gen_lock)); if ((multiaddr[0] & 01) == 0) { return (EINVAL); } if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { return (ENOENT); } bcopy(multiaddr, &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); ixgbe->mcast_count++; /* * Update the multicast table in the hardware */ ixgbe_setup_multicst(ixgbe); if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); return (EIO); } return (0); } /* * ixgbe_multicst_remove - Remove a multicst address. */ int ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) { int i; ASSERT(mutex_owned(&ixgbe->gen_lock)); for (i = 0; i < ixgbe->mcast_count; i++) { if (bcmp(multiaddr, &ixgbe->mcast_table[i], ETHERADDRL) == 0) { for (i++; i < ixgbe->mcast_count; i++) { ixgbe->mcast_table[i - 1] = ixgbe->mcast_table[i]; } ixgbe->mcast_count--; break; } } /* * Update the multicast table in the hardware */ ixgbe_setup_multicst(ixgbe); if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); return (EIO); } return (0); } /* * ixgbe_setup_multicast - Setup multicast data structures. * * This routine initializes all of the multicast related structures * and save them in the hardware registers. */ static void ixgbe_setup_multicst(ixgbe_t *ixgbe) { uint8_t *mc_addr_list; uint32_t mc_addr_count; struct ixgbe_hw *hw = &ixgbe->hw; ASSERT(mutex_owned(&ixgbe->gen_lock)); ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); mc_addr_list = (uint8_t *)ixgbe->mcast_table; mc_addr_count = ixgbe->mcast_count; /* * Update the multicast addresses to the MTA registers */ (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, ixgbe_mc_table_itr); } /* * ixgbe_get_conf - Get driver configurations set in driver.conf. * * This routine gets user-configured values out of the configuration * file ixgbe.conf. * * For each configurable value, there is a minimum, a maximum, and a * default. * If user does not configure a value, use the default. * If user configures below the minimum, use the minumum. * If user configures above the maximum, use the maxumum. */ static void ixgbe_get_conf(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; uint32_t flow_control; /* * ixgbe driver supports the following user configurations: * * Jumbo frame configuration: * default_mtu * * Ethernet flow control configuration: * flow_control * * Multiple rings configurations: * tx_queue_number * tx_ring_size * rx_queue_number * rx_ring_size * * Call ixgbe_get_prop() to get the value for a specific * configuration parameter. */ /* * Jumbo frame configuration - max_frame_size controls host buffer * allocation, so includes MTU, ethernet header, vlan tag and * frame check sequence. */ ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, MIN_MTU, MAX_MTU, DEFAULT_MTU); ixgbe->max_frame_size = ixgbe->default_mtu + sizeof (struct ether_vlan_header) + ETHERFCSL; /* * Ethernet flow control configuration */ flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, ixgbe_fc_none, 3, ixgbe_fc_full); if (flow_control == 3) flow_control = ixgbe_fc_default; hw->fc.type = flow_control; /* * Multiple rings configurations */ ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, MIN_TX_QUEUE_NUM, MAX_TX_QUEUE_NUM, DEFAULT_TX_QUEUE_NUM); ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, MIN_RX_QUEUE_NUM, MAX_RX_QUEUE_NUM, DEFAULT_RX_QUEUE_NUM); ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); /* * Tunable used to force an interrupt type. The only use is * for testing of the lesser interrupt types. * 0 = don't force interrupt type * 1 = force interrupt type MSIX * 2 = force interrupt type MSI * 3 = force interrupt type Legacy */ ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); ixgbe_log(ixgbe, "interrupt force: %d\n", ixgbe->intr_force); ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 0, 1, 1); ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 0, 1, 1); ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 0, 1, 0); ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 0, 1, 1); ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, DEFAULT_TX_COPY_THRESHOLD); ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, DEFAULT_RX_COPY_THRESHOLD); ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, DEFAULT_RX_LIMIT_PER_INTR); ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, MIN_INTR_THROTTLING, MAX_INTR_THROTTLING, DEFAULT_INTR_THROTTLING); } /* * ixgbe_get_prop - Get a property value out of the configuration file * ixgbe.conf. * * Caller provides the name of the property, a default value, a minimum * value, and a maximum value. * * Return configured value of the property, with default, minimum and * maximum properly applied. */ static int ixgbe_get_prop(ixgbe_t *ixgbe, char *propname, /* name of the property */ int minval, /* minimum acceptable value */ int maxval, /* maximim acceptable value */ int defval) /* default value */ { int value; /* * Call ddi_prop_get_int() to read the conf settings */ value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, DDI_PROP_DONTPASS, propname, defval); if (value > maxval) value = maxval; if (value < minval) value = minval; return (value); } /* * ixgbe_driver_setup_link - Using the link properties to setup the link. */ int ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) { struct ixgbe_mac_info *mac; struct ixgbe_phy_info *phy; boolean_t invalid; mac = &ixgbe->hw.mac; phy = &ixgbe->hw.phy; invalid = B_FALSE; if (ixgbe->param_adv_autoneg_cap == 1) { mac->autoneg = B_TRUE; phy->autoneg_advertised = 0; /* * No half duplex support with 10Gb parts */ if (ixgbe->param_adv_10000fdx_cap == 1) phy->autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; if (ixgbe->param_adv_1000fdx_cap == 1) phy->autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; if (ixgbe->param_adv_100fdx_cap == 1) phy->autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; if (phy->autoneg_advertised == 0) invalid = B_TRUE; } else { ixgbe->hw.mac.autoneg = B_FALSE; } if (invalid) { ixgbe_notice(ixgbe, "Invalid link settings. Setup link to " "autonegotiation with full link capabilities."); ixgbe->hw.mac.autoneg = B_TRUE; } if (setup_hw) { if (ixgbe_setup_link(&ixgbe->hw) != IXGBE_SUCCESS) return (IXGBE_FAILURE); } return (IXGBE_SUCCESS); } /* * ixgbe_driver_link_check - Link status processing. */ static boolean_t ixgbe_driver_link_check(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; boolean_t link_up = B_FALSE; boolean_t link_changed = B_FALSE; ASSERT(mutex_owned(&ixgbe->gen_lock)); (void) ixgbe_check_link(hw, &speed, &link_up); if (link_up) { /* * The Link is up, check whether it was marked as down earlier */ if (ixgbe->link_state != LINK_STATE_UP) { switch (speed) { case IXGBE_LINK_SPEED_10GB_FULL: ixgbe->link_speed = SPEED_10GB; break; case IXGBE_LINK_SPEED_1GB_FULL: ixgbe->link_speed = SPEED_1GB; break; case IXGBE_LINK_SPEED_100_FULL: ixgbe->link_speed = SPEED_100; } ixgbe->link_duplex = LINK_DUPLEX_FULL; ixgbe->link_state = LINK_STATE_UP; ixgbe->link_down_timeout = 0; link_changed = B_TRUE; } } else { if (ixgbe->link_state != LINK_STATE_DOWN) { ixgbe->link_speed = 0; ixgbe->link_duplex = 0; ixgbe->link_state = LINK_STATE_DOWN; link_changed = B_TRUE; } if (ixgbe->ixgbe_state & IXGBE_STARTED) { if (ixgbe->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) { ixgbe->link_down_timeout++; } else if (ixgbe->link_down_timeout == MAX_LINK_DOWN_TIMEOUT) { ixgbe_tx_clean(ixgbe); ixgbe->link_down_timeout++; } } } return (link_changed); } /* * ixgbe_local_timer - Driver watchdog function. * * This function will handle the transmit stall check, link status check and * other routines. */ static void ixgbe_local_timer(void *arg) { ixgbe_t *ixgbe = (ixgbe_t *)arg; if (ixgbe_stall_check(ixgbe)) { ixgbe->reset_count++; if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); } ixgbe_restart_watchdog_timer(ixgbe); } /* * ixgbe_stall_check - Check for transmit stall. * * This function checks if the adapter is stalled (in transmit). * * It is called each time the watchdog timeout is invoked. * If the transmit descriptor reclaim continuously fails, * the watchdog value will increment by 1. If the watchdog * value exceeds the threshold, the ixgbe is assumed to * have stalled and need to be reset. */ static boolean_t ixgbe_stall_check(ixgbe_t *ixgbe) { ixgbe_tx_ring_t *tx_ring; boolean_t result; int i; if (ixgbe->link_state != LINK_STATE_UP) return (B_FALSE); /* * If any tx ring is stalled, we'll reset the chipset */ result = B_FALSE; for (i = 0; i < ixgbe->num_tx_rings; i++) { tx_ring = &ixgbe->tx_rings[i]; if (tx_ring->recycle_fail > 0) tx_ring->stall_watchdog++; else tx_ring->stall_watchdog = 0; if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { result = B_TRUE; break; } } if (result) { tx_ring->stall_watchdog = 0; tx_ring->recycle_fail = 0; } return (result); } /* * is_valid_mac_addr - Check if the mac address is valid. */ static boolean_t is_valid_mac_addr(uint8_t *mac_addr) { const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; const uint8_t addr_test2[6] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || !(bcmp(addr_test2, mac_addr, ETHERADDRL))) return (B_FALSE); return (B_TRUE); } static boolean_t ixgbe_find_mac_address(ixgbe_t *ixgbe) { #ifdef __sparc struct ixgbe_hw *hw = &ixgbe->hw; uchar_t *bytes; struct ether_addr sysaddr; uint_t nelts; int err; boolean_t found = B_FALSE; /* * The "vendor's factory-set address" may already have * been extracted from the chip, but if the property * "local-mac-address" is set we use that instead. * * We check whether it looks like an array of 6 * bytes (which it should, if OBP set it). If we can't * make sense of it this way, we'll ignore it. */ err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); if (err == DDI_PROP_SUCCESS) { if (nelts == ETHERADDRL) { while (nelts--) hw->mac.addr[nelts] = bytes[nelts]; found = B_TRUE; } ddi_prop_free(bytes); } /* * Look up the OBP property "local-mac-address?". If the user has set * 'local-mac-address? = false', use "the system address" instead. */ if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { if (localetheraddr(NULL, &sysaddr) != 0) { bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); found = B_TRUE; } } ddi_prop_free(bytes); } /* * Finally(!), if there's a valid "mac-address" property (created * if we netbooted from this interface), we must use this instead * of any of the above to ensure that the NFS/install server doesn't * get confused by the address changing as Solaris takes over! */ err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); if (err == DDI_PROP_SUCCESS) { if (nelts == ETHERADDRL) { while (nelts--) hw->mac.addr[nelts] = bytes[nelts]; found = B_TRUE; } ddi_prop_free(bytes); } if (found) { bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); return (B_TRUE); } #else _NOTE(ARGUNUSED(ixgbe)); #endif return (B_TRUE); } #pragma inline(ixgbe_arm_watchdog_timer) static void ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) { /* * Fire a watchdog timer */ ixgbe->watchdog_tid = timeout(ixgbe_local_timer, (void *)ixgbe, 1 * drv_usectohz(1000000)); } /* * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. */ void ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) { mutex_enter(&ixgbe->watchdog_lock); if (!ixgbe->watchdog_enable) { ixgbe->watchdog_enable = B_TRUE; ixgbe->watchdog_start = B_TRUE; ixgbe_arm_watchdog_timer(ixgbe); } mutex_exit(&ixgbe->watchdog_lock); } /* * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. */ void ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) { timeout_id_t tid; mutex_enter(&ixgbe->watchdog_lock); ixgbe->watchdog_enable = B_FALSE; ixgbe->watchdog_start = B_FALSE; tid = ixgbe->watchdog_tid; ixgbe->watchdog_tid = 0; mutex_exit(&ixgbe->watchdog_lock); if (tid != 0) (void) untimeout(tid); } /* * ixgbe_start_watchdog_timer - Start the driver watchdog timer. */ static void ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) { mutex_enter(&ixgbe->watchdog_lock); if (ixgbe->watchdog_enable) { if (!ixgbe->watchdog_start) { ixgbe->watchdog_start = B_TRUE; ixgbe_arm_watchdog_timer(ixgbe); } } mutex_exit(&ixgbe->watchdog_lock); } /* * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. */ static void ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) { mutex_enter(&ixgbe->watchdog_lock); if (ixgbe->watchdog_start) ixgbe_arm_watchdog_timer(ixgbe); mutex_exit(&ixgbe->watchdog_lock); } /* * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. */ static void ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) { timeout_id_t tid; mutex_enter(&ixgbe->watchdog_lock); ixgbe->watchdog_start = B_FALSE; tid = ixgbe->watchdog_tid; ixgbe->watchdog_tid = 0; mutex_exit(&ixgbe->watchdog_lock); if (tid != 0) (void) untimeout(tid); } /* * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. */ static void ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; /* * mask all interrupts off */ IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); /* * for MSI-X, also disable autoclear */ if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); } IXGBE_WRITE_FLUSH(hw); } /* * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. */ static void ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; uint32_t eims, eiac, gpie; gpie = 0; eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ /* * msi-x mode */ if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { /* enable autoclear but not on bits 29:20 */ eiac = (eims & ~0x3ff00000); /* general purpose interrupt enable */ gpie |= (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |IXGBE_GPIE_OCD); /* * non-msi-x mode */ } else { /* disable autoclear, leave gpie at default */ eiac = 0; } IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims); IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); IXGBE_WRITE_FLUSH(hw); } /* * ixgbe_loopback_ioctl - Loopback support. */ enum ioc_reply ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) { lb_info_sz_t *lbsp; lb_property_t *lbpp; uint32_t *lbmp; uint32_t size; uint32_t value; if (mp->b_cont == NULL) return (IOC_INVAL); switch (iocp->ioc_cmd) { default: return (IOC_INVAL); case LB_GET_INFO_SIZE: size = sizeof (lb_info_sz_t); if (iocp->ioc_count != size) return (IOC_INVAL); value = sizeof (lb_normal); value += sizeof (lb_mac); lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; *lbsp = value; break; case LB_GET_INFO: value = sizeof (lb_normal); value += sizeof (lb_mac); size = value; if (iocp->ioc_count != size) return (IOC_INVAL); value = 0; lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; lbpp[value++] = lb_normal; lbpp[value++] = lb_mac; break; case LB_GET_MODE: size = sizeof (uint32_t); if (iocp->ioc_count != size) return (IOC_INVAL); lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; *lbmp = ixgbe->loopback_mode; break; case LB_SET_MODE: size = 0; if (iocp->ioc_count != sizeof (uint32_t)) return (IOC_INVAL); lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) return (IOC_INVAL); break; } iocp->ioc_count = size; iocp->ioc_error = 0; if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); return (IOC_INVAL); } return (IOC_REPLY); } /* * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. */ static boolean_t ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) { struct ixgbe_hw *hw; if (mode == ixgbe->loopback_mode) return (B_TRUE); hw = &ixgbe->hw; ixgbe->loopback_mode = mode; if (mode == IXGBE_LB_NONE) { /* * Reset the chip */ hw->phy.autoneg_wait_to_complete = B_TRUE; (void) ixgbe_reset(ixgbe); hw->phy.autoneg_wait_to_complete = B_FALSE; return (B_TRUE); } mutex_enter(&ixgbe->gen_lock); switch (mode) { default: mutex_exit(&ixgbe->gen_lock); return (B_FALSE); case IXGBE_LB_INTERNAL_MAC: ixgbe_set_internal_mac_loopback(ixgbe); break; } mutex_exit(&ixgbe->gen_lock); return (B_TRUE); } /* * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. */ static void ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) { struct ixgbe_hw *hw; uint32_t reg; uint8_t atlas; hw = &ixgbe->hw; /* * Setup MAC loopback */ reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); reg |= IXGBE_HLREG0_LPBK; IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); reg &= ~IXGBE_AUTOC_LMS_MASK; IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); /* * Disable Atlas Tx lanes to keep packets in loopback and not on wire */ if (hw->mac.type == ixgbe_mac_82598EB) { (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, &atlas); atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, atlas); (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, &atlas); atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, atlas); (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, &atlas); atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, atlas); (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, &atlas); atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, atlas); } } #pragma inline(ixgbe_intr_rx_work) /* * ixgbe_intr_rx_work - RX processing of ISR. */ static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) { mblk_t *mp; mutex_enter(&rx_ring->rx_lock); mp = ixgbe_rx(rx_ring); mutex_exit(&rx_ring->rx_lock); if (mp != NULL) mac_rx(rx_ring->ixgbe->mac_hdl, NULL, mp); } #pragma inline(ixgbe_intr_tx_work) /* * ixgbe_intr_tx_work - TX processing of ISR. */ static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) { /* * Recycle the tx descriptors */ tx_ring->tx_recycle(tx_ring); /* * Schedule the re-transmit */ if (tx_ring->reschedule && (tx_ring->tbd_free >= tx_ring->resched_thresh)) { tx_ring->reschedule = B_FALSE; mac_tx_update(tx_ring->ixgbe->mac_hdl); IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); } } #pragma inline(ixgbe_intr_other_work) /* * ixgbe_intr_other_work - Other processing of ISR. */ static void ixgbe_intr_other_work(ixgbe_t *ixgbe) { boolean_t link_changed; ixgbe_stop_watchdog_timer(ixgbe); mutex_enter(&ixgbe->gen_lock); /* * Take care of link status change */ link_changed = ixgbe_driver_link_check(ixgbe); /* * Get new phy state */ ixgbe_get_hw_state(ixgbe); mutex_exit(&ixgbe->gen_lock); if (link_changed) mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); ixgbe_start_watchdog_timer(ixgbe); } /* * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. */ static uint_t ixgbe_intr_legacy(void *arg1, void *arg2) { _NOTE(ARGUNUSED(arg2)); ixgbe_t *ixgbe = (ixgbe_t *)arg1; struct ixgbe_hw *hw = &ixgbe->hw; ixgbe_tx_ring_t *tx_ring; uint32_t eicr; mblk_t *mp; boolean_t tx_reschedule; boolean_t link_changed; uint_t result; mutex_enter(&ixgbe->gen_lock); if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { mutex_exit(&ixgbe->gen_lock); return (DDI_INTR_UNCLAIMED); } mp = NULL; tx_reschedule = B_FALSE; link_changed = B_FALSE; /* * Any bit set in eicr: claim this interrupt */ eicr = IXGBE_READ_REG(hw, IXGBE_EICR); if (eicr) { /* * For legacy interrupt, we have only one interrupt, * so we have only one rx ring and one tx ring enabled. */ ASSERT(ixgbe->num_rx_rings == 1); ASSERT(ixgbe->num_tx_rings == 1); /* * For legacy interrupt, we can't differentiate * between tx and rx, so always clean both */ if (eicr & IXGBE_EICR_RTX_QUEUE) { /* * Clean the rx descriptors */ mp = ixgbe_rx(&ixgbe->rx_rings[0]); /* * Recycle the tx descriptors */ tx_ring = &ixgbe->tx_rings[0]; tx_ring->tx_recycle(tx_ring); /* * Schedule the re-transmit */ tx_reschedule = (tx_ring->reschedule && (tx_ring->tbd_free >= tx_ring->resched_thresh)); } if (eicr & IXGBE_EICR_LSC) { /* take care of link status change */ link_changed = ixgbe_driver_link_check(ixgbe); /* Get new phy state */ ixgbe_get_hw_state(ixgbe); } result = DDI_INTR_CLAIMED; } else { /* * No interrupt cause bits set: don't claim this interrupt. */ result = DDI_INTR_UNCLAIMED; } mutex_exit(&ixgbe->gen_lock); /* * Do the following work outside of the gen_lock */ if (mp != NULL) mac_rx(ixgbe->mac_hdl, NULL, mp); if (tx_reschedule) { tx_ring->reschedule = B_FALSE; mac_tx_update(ixgbe->mac_hdl); IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); } if (link_changed) mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); return (result); } /* * ixgbe_intr_msi - Interrupt handler for MSI. */ static uint_t ixgbe_intr_msi(void *arg1, void *arg2) { _NOTE(ARGUNUSED(arg2)); ixgbe_t *ixgbe = (ixgbe_t *)arg1; struct ixgbe_hw *hw = &ixgbe->hw; uint32_t eicr; eicr = IXGBE_READ_REG(hw, IXGBE_EICR); /* * For MSI interrupt, we have only one vector, * so we have only one rx ring and one tx ring enabled. */ ASSERT(ixgbe->num_rx_rings == 1); ASSERT(ixgbe->num_tx_rings == 1); /* * For MSI interrupt, we can't differentiate * between tx and rx, so always clean both. */ if (eicr & IXGBE_EICR_RTX_QUEUE) { ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); } if (eicr & IXGBE_EICR_LSC) { ixgbe_intr_other_work(ixgbe); } return (DDI_INTR_CLAIMED); } /* * ixgbe_intr_rx - Interrupt handler for rx. */ static uint_t ixgbe_intr_rx(void *arg1, void *arg2) { _NOTE(ARGUNUSED(arg2)); ixgbe_ring_vector_t *vect = (ixgbe_ring_vector_t *)arg1; ixgbe_t *ixgbe = vect->ixgbe; int r_idx; /* * clean each rx ring that has its bit set in the map */ r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); while (r_idx >= 0) { ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), (ixgbe->num_rx_rings - 1)); } return (DDI_INTR_CLAIMED); } /* * ixgbe_intr_tx_other - Interrupt handler for both tx and other. * * Always look for Tx cleanup work. Only look for other work if the right * bits are set in the Interrupt Cause Register. */ static uint_t ixgbe_intr_tx_other(void *arg1, void *arg2) { _NOTE(ARGUNUSED(arg2)); ixgbe_t *ixgbe = (ixgbe_t *)arg1; struct ixgbe_hw *hw = &ixgbe->hw; uint32_t eicr; eicr = IXGBE_READ_REG(hw, IXGBE_EICR); /* * Always look for Tx cleanup work. We don't have separate * transmit vectors, so we have only one tx ring enabled. */ ASSERT(ixgbe->num_tx_rings == 1); ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); /* * Check for "other" causes. */ if (eicr & IXGBE_EICR_LSC) { ixgbe_intr_other_work(ixgbe); } return (DDI_INTR_CLAIMED); } /* * ixgbe_alloc_intrs - Allocate interrupts for the driver. * * Normal sequence is to try MSI-X; if not sucessful, try MSI; * if not successful, try Legacy. * ixgbe->intr_force can be used to force sequence to start with * any of the 3 types. * If MSI-X is not used, number of tx/rx rings is forced to 1. */ static int ixgbe_alloc_intrs(ixgbe_t *ixgbe) { dev_info_t *devinfo; int intr_types; int rc; devinfo = ixgbe->dip; /* * Get supported interrupt types */ rc = ddi_intr_get_supported_types(devinfo, &intr_types); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Get supported interrupt types failed: %d", rc); return (IXGBE_FAILURE); } IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); ixgbe->intr_type = 0; /* * Install MSI-X interrupts */ if ((intr_types & DDI_INTR_TYPE_MSIX) && (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); if (rc == IXGBE_SUCCESS) return (IXGBE_SUCCESS); ixgbe_log(ixgbe, "Allocate MSI-X failed, trying MSI interrupts..."); } /* * MSI-X not used, force rings to 1 */ ixgbe->num_rx_rings = 1; ixgbe->num_tx_rings = 1; ixgbe_log(ixgbe, "MSI-X not used, force rx and tx queue number to 1"); /* * Install MSI interrupts */ if ((intr_types & DDI_INTR_TYPE_MSI) && (ixgbe->intr_force <= IXGBE_INTR_MSI)) { rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); if (rc == IXGBE_SUCCESS) return (IXGBE_SUCCESS); ixgbe_log(ixgbe, "Allocate MSI failed, trying Legacy interrupts..."); } /* * Install legacy interrupts */ if (intr_types & DDI_INTR_TYPE_FIXED) { rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); if (rc == IXGBE_SUCCESS) return (IXGBE_SUCCESS); ixgbe_log(ixgbe, "Allocate Legacy interrupts failed"); } /* * If none of the 3 types succeeded, return failure */ return (IXGBE_FAILURE); } /* * ixgbe_alloc_intr_handles - Allocate interrupt handles. * * For legacy and MSI, only 1 handle is needed. For MSI-X, * if fewer than 2 handles are available, return failure. * Upon success, this sets the number of Rx rings to a number that * matches the handles available for Rx interrupts. */ static int ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) { dev_info_t *devinfo; int request, count, avail, actual; int rx_rings, minimum; int rc; devinfo = ixgbe->dip; /* * Currently only 1 tx ring is supported. More tx rings * will be supported with future enhancement. */ if (ixgbe->num_tx_rings > 1) { ixgbe->num_tx_rings = 1; ixgbe_log(ixgbe, "Use only 1 MSI-X vector for tx, " "force tx queue number to 1"); } switch (intr_type) { case DDI_INTR_TYPE_FIXED: request = 1; /* Request 1 legacy interrupt handle */ minimum = 1; IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); break; case DDI_INTR_TYPE_MSI: request = 1; /* Request 1 MSI interrupt handle */ minimum = 1; IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); break; case DDI_INTR_TYPE_MSIX: /* * Best number of vectors for the adapter is * # rx rings + # tx rings + 1 for other * But currently we only support number of vectors of * # rx rings + 1 for tx & other */ request = ixgbe->num_rx_rings + 1; minimum = 2; IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); break; default: ixgbe_log(ixgbe, "invalid call to ixgbe_alloc_intr_handles(): %d\n", intr_type); return (IXGBE_FAILURE); } IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", request, minimum); /* * Get number of supported interrupts */ rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); if ((rc != DDI_SUCCESS) || (count < minimum)) { ixgbe_log(ixgbe, "Get interrupt number failed. Return: %d, count: %d", rc, count); return (IXGBE_FAILURE); } IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); /* * Get number of available interrupts */ rc = ddi_intr_get_navail(devinfo, intr_type, &avail); if ((rc != DDI_SUCCESS) || (avail < minimum)) { ixgbe_log(ixgbe, "Get interrupt available number failed. " "Return: %d, available: %d", rc, avail); return (IXGBE_FAILURE); } IXGBE_DEBUGLOG_1(ixgbe, "interrupts available: %d", avail); if (avail < request) { ixgbe_log(ixgbe, "Request %d handles, %d available", request, avail); request = avail; } actual = 0; ixgbe->intr_cnt = 0; /* * Allocate an array of interrupt handles */ ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, request, &actual, DDI_INTR_ALLOC_NORMAL); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Allocate interrupts failed. " "return: %d, request: %d, actual: %d", rc, request, actual); goto alloc_handle_fail; } IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); ixgbe->intr_cnt = actual; /* * Now we know the actual number of vectors. Here we assume that * tx and other will share 1 vector and all remaining (must be at * least 1 remaining) will be used for rx. */ if (actual < minimum) { ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", actual); goto alloc_handle_fail; } /* * For MSI-X, actual might force us to reduce number of rx rings */ if (intr_type == DDI_INTR_TYPE_MSIX) { rx_rings = actual - 1; if (rx_rings < ixgbe->num_rx_rings) { ixgbe_log(ixgbe, "MSI-X vectors force Rx queue number to %d", rx_rings); ixgbe->num_rx_rings = rx_rings; } } /* * Get priority for first vector, assume remaining are all the same */ rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Get interrupt priority failed: %d", rc); goto alloc_handle_fail; } rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); goto alloc_handle_fail; } ixgbe->intr_type = intr_type; return (IXGBE_SUCCESS); alloc_handle_fail: ixgbe_rem_intrs(ixgbe); return (IXGBE_FAILURE); } /* * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. * * Before adding the interrupt handlers, the interrupt vectors have * been allocated, and the rx/tx rings have also been allocated. */ static int ixgbe_add_intr_handlers(ixgbe_t *ixgbe) { ixgbe_rx_ring_t *rx_ring; int vector; int rc; int i; vector = 0; switch (ixgbe->intr_type) { case DDI_INTR_TYPE_MSIX: /* * Add interrupt handler for tx + other */ rc = ddi_intr_add_handler(ixgbe->htable[vector], (ddi_intr_handler_t *)ixgbe_intr_tx_other, (void *)ixgbe, NULL); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Add tx/other interrupt handler failed: %d", rc); return (IXGBE_FAILURE); } vector++; /* * Add interrupt handler for each rx ring */ for (i = 0; i < ixgbe->num_rx_rings; i++) { rx_ring = &ixgbe->rx_rings[i]; /* * install pointer to vect_map[vector] */ rc = ddi_intr_add_handler(ixgbe->htable[vector], (ddi_intr_handler_t *)ixgbe_intr_rx, (void *)&ixgbe->vect_map[vector], NULL); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Add rx interrupt handler failed. " "return: %d, rx ring: %d", rc, i); for (vector--; vector >= 0; vector--) { (void) ddi_intr_remove_handler( ixgbe->htable[vector]); } return (IXGBE_FAILURE); } rx_ring->intr_vector = vector; vector++; } break; case DDI_INTR_TYPE_MSI: /* * Add interrupt handlers for the only vector */ rc = ddi_intr_add_handler(ixgbe->htable[vector], (ddi_intr_handler_t *)ixgbe_intr_msi, (void *)ixgbe, NULL); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Add MSI interrupt handler failed: %d", rc); return (IXGBE_FAILURE); } rx_ring = &ixgbe->rx_rings[0]; rx_ring->intr_vector = vector; vector++; break; case DDI_INTR_TYPE_FIXED: /* * Add interrupt handlers for the only vector */ rc = ddi_intr_add_handler(ixgbe->htable[vector], (ddi_intr_handler_t *)ixgbe_intr_legacy, (void *)ixgbe, NULL); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Add legacy interrupt handler failed: %d", rc); return (IXGBE_FAILURE); } rx_ring = &ixgbe->rx_rings[0]; rx_ring->intr_vector = vector; vector++; break; default: return (IXGBE_FAILURE); } ASSERT(vector == ixgbe->intr_cnt); return (IXGBE_SUCCESS); } #pragma inline(ixgbe_map_rxring_to_vector) /* * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. */ static void ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) { ixgbe->vect_map[v_idx].ixgbe = ixgbe; /* * Set bit in map */ BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); /* * Count bits set */ ixgbe->vect_map[v_idx].rxr_cnt++; /* * Remember bit position */ ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; } #pragma inline(ixgbe_map_txring_to_vector) /* * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. */ static void ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) { ixgbe->vect_map[v_idx].ixgbe = ixgbe; /* * Set bit in map */ BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); /* * Count bits set */ ixgbe->vect_map[v_idx].txr_cnt++; /* * Remember bit position */ ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; } /* * ixgbe_set_ivar - Set the given entry in the given interrupt vector * allocation register (IVAR). */ static void ixgbe_set_ivar(ixgbe_t *ixgbe, uint16_t int_alloc_entry, uint8_t msix_vector) { struct ixgbe_hw *hw = &ixgbe->hw; u32 ivar, index; msix_vector |= IXGBE_IVAR_ALLOC_VAL; index = (int_alloc_entry >> 2) & 0x1F; ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); ivar &= ~(0xFF << (8 * (int_alloc_entry & 0x3))); ivar |= (msix_vector << (8 * (int_alloc_entry & 0x3))); IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); } /* * ixgbe_map_rings_to_vectors - Map descriptor rings to interrupt vectors. * * For msi-x, this currently implements only the scheme which is * 1 vector for tx + other, 1 vector for each rx ring. */ static int ixgbe_map_rings_to_vectors(ixgbe_t *ixgbe) { int i, vector = 0; int vect_remain = ixgbe->intr_cnt; /* initialize vector map */ bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); /* * non-MSI-X case is very simple: all interrupts on vector 0 */ if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { ixgbe_map_rxring_to_vector(ixgbe, 0, 0); ixgbe_map_txring_to_vector(ixgbe, 0, 0); return (IXGBE_SUCCESS); } /* * Ring/vector mapping for MSI-X */ /* * Map vector 0 to tx */ ixgbe_map_txring_to_vector(ixgbe, 0, vector++); vect_remain--; /* * Map remaining vectors to rx rings */ for (i = 0; i < vect_remain; i++) { ixgbe_map_rxring_to_vector(ixgbe, i, vector++); } return (IXGBE_SUCCESS); } /* * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). * * This relies on queue/vector mapping already set up in the * vect_map[] structures */ static void ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; ixgbe_ring_vector_t *vect; /* vector bitmap */ int r_idx; /* ring index */ int v_idx; /* vector index */ /* * Clear any previous entries */ for (v_idx = 0; v_idx < IXGBE_IVAR_REG_NUM; v_idx++) IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); /* * "Other" is always on vector 0 */ ixgbe_set_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0); /* * For each interrupt vector, populate the IVAR table */ for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { vect = &ixgbe->vect_map[v_idx]; /* * For each rx ring bit set */ r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); while (r_idx >= 0) { ixgbe_set_ivar(ixgbe, IXGBE_IVAR_RX_QUEUE(r_idx), v_idx); r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), (ixgbe->num_rx_rings - 1)); } /* * For each tx ring bit set */ r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); while (r_idx >= 0) { ixgbe_set_ivar(ixgbe, IXGBE_IVAR_TX_QUEUE(r_idx), v_idx); r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), (ixgbe->num_tx_rings - 1)); } } } /* * ixgbe_rem_intr_handlers - Remove the interrupt handlers. */ static void ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) { int i; int rc; for (i = 0; i < ixgbe->intr_cnt; i++) { rc = ddi_intr_remove_handler(ixgbe->htable[i]); if (rc != DDI_SUCCESS) { IXGBE_DEBUGLOG_1(ixgbe, "Remove intr handler failed: %d", rc); } } } /* * ixgbe_rem_intrs - Remove the allocated interrupts. */ static void ixgbe_rem_intrs(ixgbe_t *ixgbe) { int i; int rc; for (i = 0; i < ixgbe->intr_cnt; i++) { rc = ddi_intr_free(ixgbe->htable[i]); if (rc != DDI_SUCCESS) { IXGBE_DEBUGLOG_1(ixgbe, "Free intr failed: %d", rc); } } kmem_free(ixgbe->htable, ixgbe->intr_size); ixgbe->htable = NULL; } /* * ixgbe_enable_intrs - Enable all the ddi interrupts. */ static int ixgbe_enable_intrs(ixgbe_t *ixgbe) { int i; int rc; /* * Enable interrupts */ if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { /* * Call ddi_intr_block_enable() for MSI */ rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Enable block intr failed: %d", rc); return (IXGBE_FAILURE); } } else { /* * Call ddi_intr_enable() for Legacy/MSI non block enable */ for (i = 0; i < ixgbe->intr_cnt; i++) { rc = ddi_intr_enable(ixgbe->htable[i]); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Enable intr failed: %d", rc); return (IXGBE_FAILURE); } } } return (IXGBE_SUCCESS); } /* * ixgbe_disable_intrs - Disable all the interrupts. */ static int ixgbe_disable_intrs(ixgbe_t *ixgbe) { int i; int rc; /* * Disable all interrupts */ if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Disable block intr failed: %d", rc); return (IXGBE_FAILURE); } } else { for (i = 0; i < ixgbe->intr_cnt; i++) { rc = ddi_intr_disable(ixgbe->htable[i]); if (rc != DDI_SUCCESS) { ixgbe_log(ixgbe, "Disable intr failed: %d", rc); return (IXGBE_FAILURE); } } } return (IXGBE_SUCCESS); } /* * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. */ static void ixgbe_get_hw_state(ixgbe_t *ixgbe) { struct ixgbe_hw *hw = &ixgbe->hw; uint32_t links; uint32_t pcs1g_anlp = 0; uint32_t pcs1g_ana = 0; ASSERT(mutex_owned(&ixgbe->gen_lock)); ixgbe->param_lp_1000fdx_cap = 0; ixgbe->param_lp_100fdx_cap = 0; links = IXGBE_READ_REG(hw, IXGBE_LINKS); if (links & IXGBE_LINKS_PCS_1G_EN) { pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); ixgbe->param_lp_1000fdx_cap = (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; ixgbe->param_lp_100fdx_cap = (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; } ixgbe->param_1000fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; ixgbe->param_100fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; } /* * ixgbe_get_driver_control - Notify that driver is in control of device. */ static void ixgbe_get_driver_control(struct ixgbe_hw *hw) { uint32_t ctrl_ext; /* * Notify firmware that driver is in control of device */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); } /* * ixgbe_release_driver_control - Notify that driver is no longer in control * of device. */ static void ixgbe_release_driver_control(struct ixgbe_hw *hw) { uint32_t ctrl_ext; /* * Notify firmware that driver is no longer in control of device */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); } /* * ixgbe_atomic_reserve - Atomic decrease operation. */ int ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) { uint32_t oldval; uint32_t newval; /* * ATOMICALLY */ do { oldval = *count_p; if (oldval < n) return (-1); newval = oldval - n; } while (atomic_cas_32(count_p, oldval, newval) != oldval); return (newval); } /* * ixgbe_mc_table_itr - Traverse the entries in the multicast table. */ static uint8_t * ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) { _NOTE(ARGUNUSED(hw)); _NOTE(ARGUNUSED(vmdq)); uint8_t *addr = *upd_ptr; uint8_t *new_ptr; new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; *upd_ptr = new_ptr; return (addr); } /* * FMA support */ int ixgbe_check_acc_handle(ddi_acc_handle_t handle) { ddi_fm_error_t de; ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); return (de.fme_status); } int ixgbe_check_dma_handle(ddi_dma_handle_t handle) { ddi_fm_error_t de; ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); return (de.fme_status); } /* * ixgbe_fm_error_cb - The IO fault service error handling callback function. */ static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) { _NOTE(ARGUNUSED(impl_data)); /* * as the driver can always deal with an error in any dma or * access handle, we can just return the fme_status value. */ pci_ereport_post(dip, err, NULL); return (err->fme_status); } static void ixgbe_fm_init(ixgbe_t *ixgbe) { ddi_iblock_cookie_t iblk; int fma_acc_flag, fma_dma_flag; /* * Only register with IO Fault Services if we have some capability */ if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; fma_acc_flag = 1; } else { ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; fma_acc_flag = 0; } if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { fma_dma_flag = 1; } else { fma_dma_flag = 0; } ixgbe_set_fma_flags(fma_acc_flag, fma_dma_flag); if (ixgbe->fm_capabilities) { /* * Register capabilities with IO Fault Services */ ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); /* * Initialize pci ereport capabilities if ereport capable */ if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) pci_ereport_setup(ixgbe->dip); /* * Register error callback if error callback capable */ if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) ddi_fm_handler_register(ixgbe->dip, ixgbe_fm_error_cb, (void*) ixgbe); } } static void ixgbe_fm_fini(ixgbe_t *ixgbe) { /* * Only unregister FMA capabilities if they are registered */ if (ixgbe->fm_capabilities) { /* * Release any resources allocated by pci_ereport_setup() */ if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) pci_ereport_teardown(ixgbe->dip); /* * Un-register error callback if error callback capable */ if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) ddi_fm_handler_unregister(ixgbe->dip); /* * Unregister from IO Fault Service */ ddi_fm_fini(ixgbe->dip); } } void ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) { uint64_t ena; char buf[FM_MAX_CLASS]; (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); ena = fm_ena_generate(0, FM_ENA_FMT1); if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); } }