1*71d10453SEric Joyner /* SPDX-License-Identifier: BSD-3-Clause */ 2*71d10453SEric Joyner /* Copyright (c) 2020, Intel Corporation 3*71d10453SEric Joyner * All rights reserved. 4*71d10453SEric Joyner * 5*71d10453SEric Joyner * Redistribution and use in source and binary forms, with or without 6*71d10453SEric Joyner * modification, are permitted provided that the following conditions are met: 7*71d10453SEric Joyner * 8*71d10453SEric Joyner * 1. Redistributions of source code must retain the above copyright notice, 9*71d10453SEric Joyner * this list of conditions and the following disclaimer. 10*71d10453SEric Joyner * 11*71d10453SEric Joyner * 2. Redistributions in binary form must reproduce the above copyright 12*71d10453SEric Joyner * notice, this list of conditions and the following disclaimer in the 13*71d10453SEric Joyner * documentation and/or other materials provided with the distribution. 14*71d10453SEric Joyner * 15*71d10453SEric Joyner * 3. Neither the name of the Intel Corporation nor the names of its 16*71d10453SEric Joyner * contributors may be used to endorse or promote products derived from 17*71d10453SEric Joyner * this software without specific prior written permission. 18*71d10453SEric Joyner * 19*71d10453SEric Joyner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20*71d10453SEric Joyner * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21*71d10453SEric Joyner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22*71d10453SEric Joyner * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23*71d10453SEric Joyner * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24*71d10453SEric Joyner * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25*71d10453SEric Joyner * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26*71d10453SEric Joyner * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27*71d10453SEric Joyner * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28*71d10453SEric Joyner * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29*71d10453SEric Joyner * POSSIBILITY OF SUCH DAMAGE. 30*71d10453SEric Joyner */ 31*71d10453SEric Joyner /*$FreeBSD$*/ 32*71d10453SEric Joyner 33*71d10453SEric Joyner /** 34*71d10453SEric Joyner * @file if_ice_iflib.c 35*71d10453SEric Joyner * @brief iflib driver implementation 36*71d10453SEric Joyner * 37*71d10453SEric Joyner * Contains the main entry point for the iflib driver implementation. It 38*71d10453SEric Joyner * implements the various ifdi driver methods, and sets up the module and 39*71d10453SEric Joyner * driver values to load an iflib driver. 40*71d10453SEric Joyner */ 41*71d10453SEric Joyner 42*71d10453SEric Joyner #include "ice_iflib.h" 43*71d10453SEric Joyner #include "ice_drv_info.h" 44*71d10453SEric Joyner #include "ice_switch.h" 45*71d10453SEric Joyner #include "ice_sched.h" 46*71d10453SEric Joyner 47*71d10453SEric Joyner #include <sys/module.h> 48*71d10453SEric Joyner #include <sys/sockio.h> 49*71d10453SEric Joyner #include <sys/smp.h> 50*71d10453SEric Joyner #include <dev/pci/pcivar.h> 51*71d10453SEric Joyner #include <dev/pci/pcireg.h> 52*71d10453SEric Joyner 53*71d10453SEric Joyner /* 54*71d10453SEric Joyner * Device method prototypes 55*71d10453SEric Joyner */ 56*71d10453SEric Joyner 57*71d10453SEric Joyner static void *ice_register(device_t); 58*71d10453SEric Joyner static int ice_if_attach_pre(if_ctx_t); 59*71d10453SEric Joyner static int ice_attach_pre_recovery_mode(struct ice_softc *sc); 60*71d10453SEric Joyner static int ice_if_attach_post(if_ctx_t); 61*71d10453SEric Joyner static void ice_attach_post_recovery_mode(struct ice_softc *sc); 62*71d10453SEric Joyner static int ice_if_detach(if_ctx_t); 63*71d10453SEric Joyner static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); 64*71d10453SEric Joyner static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); 65*71d10453SEric Joyner static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix); 66*71d10453SEric Joyner static void ice_if_queues_free(if_ctx_t ctx); 67*71d10453SEric Joyner static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 68*71d10453SEric Joyner static void ice_if_intr_enable(if_ctx_t ctx); 69*71d10453SEric Joyner static void ice_if_intr_disable(if_ctx_t ctx); 70*71d10453SEric Joyner static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 71*71d10453SEric Joyner static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 72*71d10453SEric Joyner static int ice_if_promisc_set(if_ctx_t ctx, int flags); 73*71d10453SEric Joyner static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 74*71d10453SEric Joyner static int ice_if_media_change(if_ctx_t ctx); 75*71d10453SEric Joyner static void ice_if_init(if_ctx_t ctx); 76*71d10453SEric Joyner static void ice_if_timer(if_ctx_t ctx, uint16_t qid); 77*71d10453SEric Joyner static void ice_if_update_admin_status(if_ctx_t ctx); 78*71d10453SEric Joyner static void ice_if_multi_set(if_ctx_t ctx); 79*71d10453SEric Joyner static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag); 80*71d10453SEric Joyner static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag); 81*71d10453SEric Joyner static void ice_if_stop(if_ctx_t ctx); 82*71d10453SEric Joyner static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter); 83*71d10453SEric Joyner static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); 84*71d10453SEric Joyner static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req); 85*71d10453SEric Joyner 86*71d10453SEric Joyner static int ice_msix_que(void *arg); 87*71d10453SEric Joyner static int ice_msix_admin(void *arg); 88*71d10453SEric Joyner 89*71d10453SEric Joyner /* 90*71d10453SEric Joyner * Helper function prototypes 91*71d10453SEric Joyner */ 92*71d10453SEric Joyner static int ice_pci_mapping(struct ice_softc *sc); 93*71d10453SEric Joyner static void ice_free_pci_mapping(struct ice_softc *sc); 94*71d10453SEric Joyner static void ice_update_link_status(struct ice_softc *sc, bool update_media); 95*71d10453SEric Joyner static void ice_init_device_features(struct ice_softc *sc); 96*71d10453SEric Joyner static void ice_init_tx_tracking(struct ice_vsi *vsi); 97*71d10453SEric Joyner static void ice_handle_reset_event(struct ice_softc *sc); 98*71d10453SEric Joyner static void ice_handle_pf_reset_request(struct ice_softc *sc); 99*71d10453SEric Joyner static void ice_prepare_for_reset(struct ice_softc *sc); 100*71d10453SEric Joyner static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc); 101*71d10453SEric Joyner static void ice_rebuild(struct ice_softc *sc); 102*71d10453SEric Joyner static void ice_rebuild_recovery_mode(struct ice_softc *sc); 103*71d10453SEric Joyner static void ice_free_irqvs(struct ice_softc *sc); 104*71d10453SEric Joyner static void ice_update_rx_mbuf_sz(struct ice_softc *sc); 105*71d10453SEric Joyner static void ice_poll_for_media_avail(struct ice_softc *sc); 106*71d10453SEric Joyner static void ice_setup_scctx(struct ice_softc *sc); 107*71d10453SEric Joyner static int ice_allocate_msix(struct ice_softc *sc); 108*71d10453SEric Joyner static void ice_admin_timer(void *arg); 109*71d10453SEric Joyner static void ice_transition_recovery_mode(struct ice_softc *sc); 110*71d10453SEric Joyner static void ice_transition_safe_mode(struct ice_softc *sc); 111*71d10453SEric Joyner 112*71d10453SEric Joyner /* 113*71d10453SEric Joyner * Device Interface Declaration 114*71d10453SEric Joyner */ 115*71d10453SEric Joyner 116*71d10453SEric Joyner /** 117*71d10453SEric Joyner * @var ice_methods 118*71d10453SEric Joyner * @brief ice driver method entry points 119*71d10453SEric Joyner * 120*71d10453SEric Joyner * List of device methods implementing the generic device interface used by 121*71d10453SEric Joyner * the device stack to interact with the ice driver. Since this is an iflib 122*71d10453SEric Joyner * driver, most of the methods point to the generic iflib implementation. 123*71d10453SEric Joyner */ 124*71d10453SEric Joyner static device_method_t ice_methods[] = { 125*71d10453SEric Joyner /* Device interface */ 126*71d10453SEric Joyner DEVMETHOD(device_register, ice_register), 127*71d10453SEric Joyner DEVMETHOD(device_probe, iflib_device_probe_vendor), 128*71d10453SEric Joyner DEVMETHOD(device_attach, iflib_device_attach), 129*71d10453SEric Joyner DEVMETHOD(device_detach, iflib_device_detach), 130*71d10453SEric Joyner DEVMETHOD(device_shutdown, iflib_device_shutdown), 131*71d10453SEric Joyner DEVMETHOD(device_suspend, iflib_device_suspend), 132*71d10453SEric Joyner DEVMETHOD(device_resume, iflib_device_resume), 133*71d10453SEric Joyner DEVMETHOD_END 134*71d10453SEric Joyner }; 135*71d10453SEric Joyner 136*71d10453SEric Joyner /** 137*71d10453SEric Joyner * @var ice_iflib_methods 138*71d10453SEric Joyner * @brief iflib method entry points 139*71d10453SEric Joyner * 140*71d10453SEric Joyner * List of device methods used by the iflib stack to interact with this 141*71d10453SEric Joyner * driver. These are the real main entry points used to interact with this 142*71d10453SEric Joyner * driver. 143*71d10453SEric Joyner */ 144*71d10453SEric Joyner static device_method_t ice_iflib_methods[] = { 145*71d10453SEric Joyner DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre), 146*71d10453SEric Joyner DEVMETHOD(ifdi_attach_post, ice_if_attach_post), 147*71d10453SEric Joyner DEVMETHOD(ifdi_detach, ice_if_detach), 148*71d10453SEric Joyner DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc), 149*71d10453SEric Joyner DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc), 150*71d10453SEric Joyner DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign), 151*71d10453SEric Joyner DEVMETHOD(ifdi_queues_free, ice_if_queues_free), 152*71d10453SEric Joyner DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set), 153*71d10453SEric Joyner DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable), 154*71d10453SEric Joyner DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable), 155*71d10453SEric Joyner DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable), 156*71d10453SEric Joyner DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable), 157*71d10453SEric Joyner DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set), 158*71d10453SEric Joyner DEVMETHOD(ifdi_media_status, ice_if_media_status), 159*71d10453SEric Joyner DEVMETHOD(ifdi_media_change, ice_if_media_change), 160*71d10453SEric Joyner DEVMETHOD(ifdi_init, ice_if_init), 161*71d10453SEric Joyner DEVMETHOD(ifdi_stop, ice_if_stop), 162*71d10453SEric Joyner DEVMETHOD(ifdi_timer, ice_if_timer), 163*71d10453SEric Joyner DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status), 164*71d10453SEric Joyner DEVMETHOD(ifdi_multi_set, ice_if_multi_set), 165*71d10453SEric Joyner DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register), 166*71d10453SEric Joyner DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister), 167*71d10453SEric Joyner DEVMETHOD(ifdi_get_counter, ice_if_get_counter), 168*71d10453SEric Joyner DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl), 169*71d10453SEric Joyner DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req), 170*71d10453SEric Joyner DEVMETHOD_END 171*71d10453SEric Joyner }; 172*71d10453SEric Joyner 173*71d10453SEric Joyner /** 174*71d10453SEric Joyner * @var ice_driver 175*71d10453SEric Joyner * @brief driver structure for the generic device stack 176*71d10453SEric Joyner * 177*71d10453SEric Joyner * driver_t definition used to setup the generic device methods. 178*71d10453SEric Joyner */ 179*71d10453SEric Joyner static driver_t ice_driver = { 180*71d10453SEric Joyner .name = "ice", 181*71d10453SEric Joyner .methods = ice_methods, 182*71d10453SEric Joyner .size = sizeof(struct ice_softc), 183*71d10453SEric Joyner }; 184*71d10453SEric Joyner 185*71d10453SEric Joyner /** 186*71d10453SEric Joyner * @var ice_iflib_driver 187*71d10453SEric Joyner * @brief driver structure for the iflib stack 188*71d10453SEric Joyner * 189*71d10453SEric Joyner * driver_t definition used to setup the iflib device methods. 190*71d10453SEric Joyner */ 191*71d10453SEric Joyner static driver_t ice_iflib_driver = { 192*71d10453SEric Joyner .name = "ice", 193*71d10453SEric Joyner .methods = ice_iflib_methods, 194*71d10453SEric Joyner .size = sizeof(struct ice_softc), 195*71d10453SEric Joyner }; 196*71d10453SEric Joyner 197*71d10453SEric Joyner extern struct if_txrx ice_txrx; 198*71d10453SEric Joyner extern struct if_txrx ice_recovery_txrx; 199*71d10453SEric Joyner 200*71d10453SEric Joyner /** 201*71d10453SEric Joyner * @var ice_sctx 202*71d10453SEric Joyner * @brief ice driver shared context 203*71d10453SEric Joyner * 204*71d10453SEric Joyner * Structure defining shared values (context) that is used by all instances of 205*71d10453SEric Joyner * the device. Primarily used to setup details about how the iflib stack 206*71d10453SEric Joyner * should treat this driver. Also defines the default, minimum, and maximum 207*71d10453SEric Joyner * number of descriptors in each ring. 208*71d10453SEric Joyner */ 209*71d10453SEric Joyner static struct if_shared_ctx ice_sctx = { 210*71d10453SEric Joyner .isc_magic = IFLIB_MAGIC, 211*71d10453SEric Joyner .isc_q_align = PAGE_SIZE, 212*71d10453SEric Joyner 213*71d10453SEric Joyner .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 214*71d10453SEric Joyner /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but 215*71d10453SEric Joyner * that doesn't make sense since that would be larger than the maximum 216*71d10453SEric Joyner * size of a single packet. 217*71d10453SEric Joyner */ 218*71d10453SEric Joyner .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 219*71d10453SEric Joyner 220*71d10453SEric Joyner /* XXX: This is only used by iflib to ensure that 221*71d10453SEric Joyner * scctx->isc_tx_tso_size_max + the VLAN header is a valid size. 222*71d10453SEric Joyner */ 223*71d10453SEric Joyner .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 224*71d10453SEric Joyner /* XXX: This is used by iflib to set the number of segments in the TSO 225*71d10453SEric Joyner * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the 226*71d10453SEric Joyner * related ifnet parameter. 227*71d10453SEric Joyner */ 228*71d10453SEric Joyner .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 229*71d10453SEric Joyner 230*71d10453SEric Joyner .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 231*71d10453SEric Joyner .isc_rx_nsegments = ICE_MAX_RX_SEGS, 232*71d10453SEric Joyner .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 233*71d10453SEric Joyner 234*71d10453SEric Joyner .isc_nfl = 1, 235*71d10453SEric Joyner .isc_ntxqs = 1, 236*71d10453SEric Joyner .isc_nrxqs = 1, 237*71d10453SEric Joyner 238*71d10453SEric Joyner .isc_admin_intrcnt = 1, 239*71d10453SEric Joyner .isc_vendor_info = ice_vendor_info_array, 240*71d10453SEric Joyner .isc_driver_version = __DECONST(char *, ice_driver_version), 241*71d10453SEric Joyner .isc_driver = &ice_iflib_driver, 242*71d10453SEric Joyner 243*71d10453SEric Joyner /* 244*71d10453SEric Joyner * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available 245*71d10453SEric Joyner * for hardware checksum offload 246*71d10453SEric Joyner * 247*71d10453SEric Joyner * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the 248*71d10453SEric Joyner * IP sum field, required by our hardware to calculate valid TSO 249*71d10453SEric Joyner * checksums. 250*71d10453SEric Joyner * 251*71d10453SEric Joyner * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs 252*71d10453SEric Joyner * even when the interface is down. 253*71d10453SEric Joyner * 254*71d10453SEric Joyner * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X 255*71d10453SEric Joyner * vectors manually instead of relying on iflib code to do this. 256*71d10453SEric Joyner */ 257*71d10453SEric Joyner .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 258*71d10453SEric Joyner IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX, 259*71d10453SEric Joyner 260*71d10453SEric Joyner .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 261*71d10453SEric Joyner .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 262*71d10453SEric Joyner .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 263*71d10453SEric Joyner .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 264*71d10453SEric Joyner .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 265*71d10453SEric Joyner .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 266*71d10453SEric Joyner }; 267*71d10453SEric Joyner 268*71d10453SEric Joyner /** 269*71d10453SEric Joyner * @var ice_devclass 270*71d10453SEric Joyner * @brief ice driver device class 271*71d10453SEric Joyner * 272*71d10453SEric Joyner * device class used to setup the ice driver module kobject class. 273*71d10453SEric Joyner */ 274*71d10453SEric Joyner devclass_t ice_devclass; 275*71d10453SEric Joyner DRIVER_MODULE(ice, pci, ice_driver, ice_devclass, ice_module_event_handler, 0); 276*71d10453SEric Joyner 277*71d10453SEric Joyner MODULE_VERSION(ice, 1); 278*71d10453SEric Joyner MODULE_DEPEND(ice, pci, 1, 1, 1); 279*71d10453SEric Joyner MODULE_DEPEND(ice, ether, 1, 1, 1); 280*71d10453SEric Joyner MODULE_DEPEND(ice, iflib, 1, 1, 1); 281*71d10453SEric Joyner MODULE_DEPEND(ice, firmware, 1, 1, 1); 282*71d10453SEric Joyner 283*71d10453SEric Joyner IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array); 284*71d10453SEric Joyner 285*71d10453SEric Joyner /* Static driver-wide sysctls */ 286*71d10453SEric Joyner #include "ice_iflib_sysctls.h" 287*71d10453SEric Joyner 288*71d10453SEric Joyner /** 289*71d10453SEric Joyner * ice_pci_mapping - Map PCI BAR memory 290*71d10453SEric Joyner * @sc: device private softc 291*71d10453SEric Joyner * 292*71d10453SEric Joyner * Map PCI BAR 0 for device operation. 293*71d10453SEric Joyner */ 294*71d10453SEric Joyner static int 295*71d10453SEric Joyner ice_pci_mapping(struct ice_softc *sc) 296*71d10453SEric Joyner { 297*71d10453SEric Joyner int rc; 298*71d10453SEric Joyner 299*71d10453SEric Joyner /* Map BAR0 */ 300*71d10453SEric Joyner rc = ice_map_bar(sc->dev, &sc->bar0, 0); 301*71d10453SEric Joyner if (rc) 302*71d10453SEric Joyner return rc; 303*71d10453SEric Joyner 304*71d10453SEric Joyner return 0; 305*71d10453SEric Joyner } 306*71d10453SEric Joyner 307*71d10453SEric Joyner /** 308*71d10453SEric Joyner * ice_free_pci_mapping - Release PCI BAR memory 309*71d10453SEric Joyner * @sc: device private softc 310*71d10453SEric Joyner * 311*71d10453SEric Joyner * Release PCI BARs which were previously mapped by ice_pci_mapping(). 312*71d10453SEric Joyner */ 313*71d10453SEric Joyner static void 314*71d10453SEric Joyner ice_free_pci_mapping(struct ice_softc *sc) 315*71d10453SEric Joyner { 316*71d10453SEric Joyner /* Free BAR0 */ 317*71d10453SEric Joyner ice_free_bar(sc->dev, &sc->bar0); 318*71d10453SEric Joyner } 319*71d10453SEric Joyner 320*71d10453SEric Joyner /* 321*71d10453SEric Joyner * Device methods 322*71d10453SEric Joyner */ 323*71d10453SEric Joyner 324*71d10453SEric Joyner /** 325*71d10453SEric Joyner * ice_register - register device method callback 326*71d10453SEric Joyner * @dev: the device being registered 327*71d10453SEric Joyner * 328*71d10453SEric Joyner * Returns a pointer to the shared context structure, which is used by iflib. 329*71d10453SEric Joyner */ 330*71d10453SEric Joyner static void * 331*71d10453SEric Joyner ice_register(device_t dev __unused) 332*71d10453SEric Joyner { 333*71d10453SEric Joyner return &ice_sctx; 334*71d10453SEric Joyner } /* ice_register */ 335*71d10453SEric Joyner 336*71d10453SEric Joyner /** 337*71d10453SEric Joyner * ice_setup_scctx - Setup the iflib softc context structure 338*71d10453SEric Joyner * @sc: the device private structure 339*71d10453SEric Joyner * 340*71d10453SEric Joyner * Setup the parameters in if_softc_ctx_t structure used by the iflib stack 341*71d10453SEric Joyner * when loading. 342*71d10453SEric Joyner */ 343*71d10453SEric Joyner static void 344*71d10453SEric Joyner ice_setup_scctx(struct ice_softc *sc) 345*71d10453SEric Joyner { 346*71d10453SEric Joyner if_softc_ctx_t scctx = sc->scctx; 347*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 348*71d10453SEric Joyner bool safe_mode, recovery_mode; 349*71d10453SEric Joyner 350*71d10453SEric Joyner safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE); 351*71d10453SEric Joyner recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE); 352*71d10453SEric Joyner 353*71d10453SEric Joyner /* 354*71d10453SEric Joyner * If the driver loads in Safe mode or Recovery mode, limit iflib to 355*71d10453SEric Joyner * a single queue pair. 356*71d10453SEric Joyner */ 357*71d10453SEric Joyner if (safe_mode || recovery_mode) { 358*71d10453SEric Joyner scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1; 359*71d10453SEric Joyner scctx->isc_ntxqsets_max = 1; 360*71d10453SEric Joyner scctx->isc_nrxqsets_max = 1; 361*71d10453SEric Joyner } else { 362*71d10453SEric Joyner /* 363*71d10453SEric Joyner * iflib initially sets the isc_ntxqsets and isc_nrxqsets to 364*71d10453SEric Joyner * the values of the override sysctls. Cache these initial 365*71d10453SEric Joyner * values so that the driver can be aware of what the iflib 366*71d10453SEric Joyner * sysctl value is when setting up MSI-X vectors. 367*71d10453SEric Joyner */ 368*71d10453SEric Joyner sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets; 369*71d10453SEric Joyner sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets; 370*71d10453SEric Joyner 371*71d10453SEric Joyner if (scctx->isc_ntxqsets == 0) 372*71d10453SEric Joyner scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size; 373*71d10453SEric Joyner if (scctx->isc_nrxqsets == 0) 374*71d10453SEric Joyner scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size; 375*71d10453SEric Joyner 376*71d10453SEric Joyner scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq; 377*71d10453SEric Joyner scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq; 378*71d10453SEric Joyner 379*71d10453SEric Joyner /* 380*71d10453SEric Joyner * Sanity check that the iflib sysctl values are within the 381*71d10453SEric Joyner * maximum supported range. 382*71d10453SEric Joyner */ 383*71d10453SEric Joyner if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max) 384*71d10453SEric Joyner sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max; 385*71d10453SEric Joyner if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max) 386*71d10453SEric Joyner sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max; 387*71d10453SEric Joyner } 388*71d10453SEric Joyner 389*71d10453SEric Joyner scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 390*71d10453SEric Joyner * sizeof(struct ice_tx_desc), DBA_ALIGN); 391*71d10453SEric Joyner scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 392*71d10453SEric Joyner * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 393*71d10453SEric Joyner 394*71d10453SEric Joyner scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 395*71d10453SEric Joyner scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 396*71d10453SEric Joyner scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 397*71d10453SEric Joyner scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 398*71d10453SEric Joyner 399*71d10453SEric Joyner scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR); 400*71d10453SEric Joyner scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size; 401*71d10453SEric Joyner 402*71d10453SEric Joyner /* 403*71d10453SEric Joyner * If the driver loads in recovery mode, disable Tx/Rx functionality 404*71d10453SEric Joyner */ 405*71d10453SEric Joyner if (recovery_mode) 406*71d10453SEric Joyner scctx->isc_txrx = &ice_recovery_txrx; 407*71d10453SEric Joyner else 408*71d10453SEric Joyner scctx->isc_txrx = &ice_txrx; 409*71d10453SEric Joyner 410*71d10453SEric Joyner /* 411*71d10453SEric Joyner * If the driver loads in Safe mode or Recovery mode, disable 412*71d10453SEric Joyner * advanced features including hardware offloads. 413*71d10453SEric Joyner */ 414*71d10453SEric Joyner if (safe_mode || recovery_mode) { 415*71d10453SEric Joyner scctx->isc_capenable = ICE_SAFE_CAPS; 416*71d10453SEric Joyner scctx->isc_tx_csum_flags = 0; 417*71d10453SEric Joyner } else { 418*71d10453SEric Joyner scctx->isc_capenable = ICE_FULL_CAPS; 419*71d10453SEric Joyner scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 420*71d10453SEric Joyner } 421*71d10453SEric Joyner 422*71d10453SEric Joyner scctx->isc_capabilities = scctx->isc_capenable; 423*71d10453SEric Joyner } /* ice_setup_scctx */ 424*71d10453SEric Joyner 425*71d10453SEric Joyner /** 426*71d10453SEric Joyner * ice_if_attach_pre - Early device attach logic 427*71d10453SEric Joyner * @ctx: the iflib context structure 428*71d10453SEric Joyner * 429*71d10453SEric Joyner * Called by iflib during the attach process. Earliest main driver entry 430*71d10453SEric Joyner * point which performs necessary hardware and driver initialization. Called 431*71d10453SEric Joyner * before the Tx and Rx queues are allocated. 432*71d10453SEric Joyner */ 433*71d10453SEric Joyner static int 434*71d10453SEric Joyner ice_if_attach_pre(if_ctx_t ctx) 435*71d10453SEric Joyner { 436*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 437*71d10453SEric Joyner enum ice_fw_modes fw_mode; 438*71d10453SEric Joyner enum ice_status status; 439*71d10453SEric Joyner if_softc_ctx_t scctx; 440*71d10453SEric Joyner struct ice_hw *hw; 441*71d10453SEric Joyner device_t dev; 442*71d10453SEric Joyner int err; 443*71d10453SEric Joyner 444*71d10453SEric Joyner device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n"); 445*71d10453SEric Joyner 446*71d10453SEric Joyner sc->ctx = ctx; 447*71d10453SEric Joyner sc->media = iflib_get_media(ctx); 448*71d10453SEric Joyner sc->sctx = iflib_get_sctx(ctx); 449*71d10453SEric Joyner sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx); 450*71d10453SEric Joyner 451*71d10453SEric Joyner dev = sc->dev = iflib_get_dev(ctx); 452*71d10453SEric Joyner scctx = sc->scctx = iflib_get_softc_ctx(ctx); 453*71d10453SEric Joyner 454*71d10453SEric Joyner hw = &sc->hw; 455*71d10453SEric Joyner hw->back = sc; 456*71d10453SEric Joyner 457*71d10453SEric Joyner snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name), 458*71d10453SEric Joyner "%s:admin", device_get_nameunit(dev)); 459*71d10453SEric Joyner mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF); 460*71d10453SEric Joyner callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0); 461*71d10453SEric Joyner 462*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 463*71d10453SEric Joyner 464*71d10453SEric Joyner if (ice_pci_mapping(sc)) { 465*71d10453SEric Joyner err = (ENXIO); 466*71d10453SEric Joyner goto destroy_admin_timer; 467*71d10453SEric Joyner } 468*71d10453SEric Joyner 469*71d10453SEric Joyner /* Save off the PCI information */ 470*71d10453SEric Joyner ice_save_pci_info(hw, dev); 471*71d10453SEric Joyner 472*71d10453SEric Joyner /* create tunables as early as possible */ 473*71d10453SEric Joyner ice_add_device_tunables(sc); 474*71d10453SEric Joyner 475*71d10453SEric Joyner /* Setup ControlQ lengths */ 476*71d10453SEric Joyner ice_set_ctrlq_len(hw); 477*71d10453SEric Joyner 478*71d10453SEric Joyner fw_mode = ice_get_fw_mode(hw); 479*71d10453SEric Joyner if (fw_mode == ICE_FW_MODE_REC) { 480*71d10453SEric Joyner device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 481*71d10453SEric Joyner 482*71d10453SEric Joyner err = ice_attach_pre_recovery_mode(sc); 483*71d10453SEric Joyner if (err) 484*71d10453SEric Joyner goto free_pci_mapping; 485*71d10453SEric Joyner 486*71d10453SEric Joyner return (0); 487*71d10453SEric Joyner } 488*71d10453SEric Joyner 489*71d10453SEric Joyner /* Initialize the hw data structure */ 490*71d10453SEric Joyner status = ice_init_hw(hw); 491*71d10453SEric Joyner if (status) { 492*71d10453SEric Joyner if (status == ICE_ERR_FW_API_VER) { 493*71d10453SEric Joyner /* Enter recovery mode, so that the driver remains 494*71d10453SEric Joyner * loaded. This way, if the system administrator 495*71d10453SEric Joyner * cannot update the driver, they may still attempt to 496*71d10453SEric Joyner * downgrade the NVM. 497*71d10453SEric Joyner */ 498*71d10453SEric Joyner err = ice_attach_pre_recovery_mode(sc); 499*71d10453SEric Joyner if (err) 500*71d10453SEric Joyner goto free_pci_mapping; 501*71d10453SEric Joyner 502*71d10453SEric Joyner return (0); 503*71d10453SEric Joyner } else { 504*71d10453SEric Joyner err = EIO; 505*71d10453SEric Joyner device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n", 506*71d10453SEric Joyner ice_status_str(status), 507*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 508*71d10453SEric Joyner } 509*71d10453SEric Joyner goto free_pci_mapping; 510*71d10453SEric Joyner } 511*71d10453SEric Joyner 512*71d10453SEric Joyner /* Notify firmware of the device driver version */ 513*71d10453SEric Joyner err = ice_send_version(sc); 514*71d10453SEric Joyner if (err) 515*71d10453SEric Joyner goto deinit_hw; 516*71d10453SEric Joyner 517*71d10453SEric Joyner ice_load_pkg_file(sc); 518*71d10453SEric Joyner 519*71d10453SEric Joyner err = ice_init_link_events(sc); 520*71d10453SEric Joyner if (err) { 521*71d10453SEric Joyner device_printf(dev, "ice_init_link_events failed: %s\n", 522*71d10453SEric Joyner ice_err_str(err)); 523*71d10453SEric Joyner goto deinit_hw; 524*71d10453SEric Joyner } 525*71d10453SEric Joyner 526*71d10453SEric Joyner ice_print_nvm_version(sc); 527*71d10453SEric Joyner 528*71d10453SEric Joyner ice_init_device_features(sc); 529*71d10453SEric Joyner 530*71d10453SEric Joyner /* Setup the MAC address */ 531*71d10453SEric Joyner iflib_set_mac(ctx, hw->port_info->mac.lan_addr); 532*71d10453SEric Joyner 533*71d10453SEric Joyner /* Setup the iflib softc context structure */ 534*71d10453SEric Joyner ice_setup_scctx(sc); 535*71d10453SEric Joyner 536*71d10453SEric Joyner /* Initialize the Tx queue manager */ 537*71d10453SEric Joyner err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq); 538*71d10453SEric Joyner if (err) { 539*71d10453SEric Joyner device_printf(dev, "Unable to initialize Tx queue manager: %s\n", 540*71d10453SEric Joyner ice_err_str(err)); 541*71d10453SEric Joyner goto deinit_hw; 542*71d10453SEric Joyner } 543*71d10453SEric Joyner 544*71d10453SEric Joyner /* Initialize the Rx queue manager */ 545*71d10453SEric Joyner err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq); 546*71d10453SEric Joyner if (err) { 547*71d10453SEric Joyner device_printf(dev, "Unable to initialize Rx queue manager: %s\n", 548*71d10453SEric Joyner ice_err_str(err)); 549*71d10453SEric Joyner goto free_tx_qmgr; 550*71d10453SEric Joyner } 551*71d10453SEric Joyner 552*71d10453SEric Joyner /* Initialize the interrupt resource manager */ 553*71d10453SEric Joyner err = ice_alloc_intr_tracking(sc); 554*71d10453SEric Joyner if (err) 555*71d10453SEric Joyner /* Errors are already printed */ 556*71d10453SEric Joyner goto free_rx_qmgr; 557*71d10453SEric Joyner 558*71d10453SEric Joyner /* Determine maximum number of VSIs we'll prepare for */ 559*71d10453SEric Joyner sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE, 560*71d10453SEric Joyner hw->func_caps.guar_num_vsi); 561*71d10453SEric Joyner 562*71d10453SEric Joyner if (!sc->num_available_vsi) { 563*71d10453SEric Joyner err = EIO; 564*71d10453SEric Joyner device_printf(dev, "No VSIs allocated to host\n"); 565*71d10453SEric Joyner goto free_intr_tracking; 566*71d10453SEric Joyner } 567*71d10453SEric Joyner 568*71d10453SEric Joyner /* Allocate storage for the VSI pointers */ 569*71d10453SEric Joyner sc->all_vsi = (struct ice_vsi **) 570*71d10453SEric Joyner malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi, 571*71d10453SEric Joyner M_ICE, M_WAITOK | M_ZERO); 572*71d10453SEric Joyner if (!sc->all_vsi) { 573*71d10453SEric Joyner err = ENOMEM; 574*71d10453SEric Joyner device_printf(dev, "Unable to allocate VSI array\n"); 575*71d10453SEric Joyner goto free_intr_tracking; 576*71d10453SEric Joyner } 577*71d10453SEric Joyner 578*71d10453SEric Joyner /* 579*71d10453SEric Joyner * Prepare the statically allocated primary PF VSI in the softc 580*71d10453SEric Joyner * structure. Other VSIs will be dynamically allocated as needed. 581*71d10453SEric Joyner */ 582*71d10453SEric Joyner ice_setup_pf_vsi(sc); 583*71d10453SEric Joyner 584*71d10453SEric Joyner err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max, 585*71d10453SEric Joyner scctx->isc_nrxqsets_max); 586*71d10453SEric Joyner if (err) { 587*71d10453SEric Joyner device_printf(dev, "Unable to allocate VSI Queue maps\n"); 588*71d10453SEric Joyner goto free_main_vsi; 589*71d10453SEric Joyner } 590*71d10453SEric Joyner 591*71d10453SEric Joyner /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */ 592*71d10453SEric Joyner err = ice_allocate_msix(sc); 593*71d10453SEric Joyner if (err) 594*71d10453SEric Joyner goto free_main_vsi; 595*71d10453SEric Joyner 596*71d10453SEric Joyner return 0; 597*71d10453SEric Joyner 598*71d10453SEric Joyner free_main_vsi: 599*71d10453SEric Joyner /* ice_release_vsi will free the queue maps if they were allocated */ 600*71d10453SEric Joyner ice_release_vsi(&sc->pf_vsi); 601*71d10453SEric Joyner free(sc->all_vsi, M_ICE); 602*71d10453SEric Joyner sc->all_vsi = NULL; 603*71d10453SEric Joyner free_intr_tracking: 604*71d10453SEric Joyner ice_free_intr_tracking(sc); 605*71d10453SEric Joyner free_rx_qmgr: 606*71d10453SEric Joyner ice_resmgr_destroy(&sc->rx_qmgr); 607*71d10453SEric Joyner free_tx_qmgr: 608*71d10453SEric Joyner ice_resmgr_destroy(&sc->tx_qmgr); 609*71d10453SEric Joyner deinit_hw: 610*71d10453SEric Joyner ice_deinit_hw(hw); 611*71d10453SEric Joyner free_pci_mapping: 612*71d10453SEric Joyner ice_free_pci_mapping(sc); 613*71d10453SEric Joyner destroy_admin_timer: 614*71d10453SEric Joyner mtx_lock(&sc->admin_mtx); 615*71d10453SEric Joyner callout_stop(&sc->admin_timer); 616*71d10453SEric Joyner mtx_unlock(&sc->admin_mtx); 617*71d10453SEric Joyner mtx_destroy(&sc->admin_mtx); 618*71d10453SEric Joyner return err; 619*71d10453SEric Joyner } /* ice_if_attach_pre */ 620*71d10453SEric Joyner 621*71d10453SEric Joyner /** 622*71d10453SEric Joyner * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery 623*71d10453SEric Joyner * @sc: the device private softc 624*71d10453SEric Joyner * 625*71d10453SEric Joyner * Loads the device driver in limited Firmware Recovery mode, intended to 626*71d10453SEric Joyner * allow users to update the firmware to attempt to recover the device. 627*71d10453SEric Joyner * 628*71d10453SEric Joyner * @remark We may enter recovery mode in case either (a) the firmware is 629*71d10453SEric Joyner * detected to be in an invalid state and must be re-programmed, or (b) the 630*71d10453SEric Joyner * driver detects that the loaded firmware has a non-compatible API version 631*71d10453SEric Joyner * that the driver cannot operate with. 632*71d10453SEric Joyner */ 633*71d10453SEric Joyner static int 634*71d10453SEric Joyner ice_attach_pre_recovery_mode(struct ice_softc *sc) 635*71d10453SEric Joyner { 636*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE); 637*71d10453SEric Joyner 638*71d10453SEric Joyner /* Setup the iflib softc context */ 639*71d10453SEric Joyner ice_setup_scctx(sc); 640*71d10453SEric Joyner 641*71d10453SEric Joyner /* Setup the PF VSI back pointer */ 642*71d10453SEric Joyner sc->pf_vsi.sc = sc; 643*71d10453SEric Joyner 644*71d10453SEric Joyner /* 645*71d10453SEric Joyner * We still need to allocate MSI-X vectors since we need one vector to 646*71d10453SEric Joyner * run the administrative admin interrupt 647*71d10453SEric Joyner */ 648*71d10453SEric Joyner return ice_allocate_msix(sc); 649*71d10453SEric Joyner } 650*71d10453SEric Joyner 651*71d10453SEric Joyner /** 652*71d10453SEric Joyner * ice_update_link_status - notify OS of link state change 653*71d10453SEric Joyner * @sc: device private softc structure 654*71d10453SEric Joyner * @update_media: true if we should update media even if link didn't change 655*71d10453SEric Joyner * 656*71d10453SEric Joyner * Called to notify iflib core of link status changes. Should be called once 657*71d10453SEric Joyner * during attach_post, and whenever link status changes during runtime. 658*71d10453SEric Joyner * 659*71d10453SEric Joyner * This call only updates the currently supported media types if the link 660*71d10453SEric Joyner * status changed, or if update_media is set to true. 661*71d10453SEric Joyner */ 662*71d10453SEric Joyner static void 663*71d10453SEric Joyner ice_update_link_status(struct ice_softc *sc, bool update_media) 664*71d10453SEric Joyner { 665*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 666*71d10453SEric Joyner 667*71d10453SEric Joyner /* Never report link up when in recovery mode */ 668*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 669*71d10453SEric Joyner return; 670*71d10453SEric Joyner 671*71d10453SEric Joyner /* Report link status to iflib only once each time it changes */ 672*71d10453SEric Joyner if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) { 673*71d10453SEric Joyner if (sc->link_up) { /* link is up */ 674*71d10453SEric Joyner uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info); 675*71d10453SEric Joyner 676*71d10453SEric Joyner iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate); 677*71d10453SEric Joyner 678*71d10453SEric Joyner ice_link_up_msg(sc); 679*71d10453SEric Joyner 680*71d10453SEric Joyner update_media = true; 681*71d10453SEric Joyner } else { /* link is down */ 682*71d10453SEric Joyner iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 683*71d10453SEric Joyner 684*71d10453SEric Joyner update_media = true; 685*71d10453SEric Joyner } 686*71d10453SEric Joyner } 687*71d10453SEric Joyner 688*71d10453SEric Joyner /* Update the supported media types */ 689*71d10453SEric Joyner if (update_media) { 690*71d10453SEric Joyner enum ice_status status = ice_add_media_types(sc, sc->media); 691*71d10453SEric Joyner if (status) 692*71d10453SEric Joyner device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 693*71d10453SEric Joyner ice_status_str(status), 694*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 695*71d10453SEric Joyner } 696*71d10453SEric Joyner 697*71d10453SEric Joyner /* TODO: notify VFs of link state change */ 698*71d10453SEric Joyner } 699*71d10453SEric Joyner 700*71d10453SEric Joyner /** 701*71d10453SEric Joyner * ice_if_attach_post - Late device attach logic 702*71d10453SEric Joyner * @ctx: the iflib context structure 703*71d10453SEric Joyner * 704*71d10453SEric Joyner * Called by iflib to finish up attaching the device. Performs any attach 705*71d10453SEric Joyner * logic which must wait until after the Tx and Rx queues have been 706*71d10453SEric Joyner * allocated. 707*71d10453SEric Joyner */ 708*71d10453SEric Joyner static int 709*71d10453SEric Joyner ice_if_attach_post(if_ctx_t ctx) 710*71d10453SEric Joyner { 711*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 712*71d10453SEric Joyner if_t ifp = iflib_get_ifp(ctx); 713*71d10453SEric Joyner int err; 714*71d10453SEric Joyner 715*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 716*71d10453SEric Joyner 717*71d10453SEric Joyner /* We don't yet support loading if MSI-X is not supported */ 718*71d10453SEric Joyner if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) { 719*71d10453SEric Joyner device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n"); 720*71d10453SEric Joyner return (ENOTSUP); 721*71d10453SEric Joyner } 722*71d10453SEric Joyner 723*71d10453SEric Joyner /* The ifnet structure hasn't yet been initialized when the attach_pre 724*71d10453SEric Joyner * handler is called, so wait until attach_post to setup the 725*71d10453SEric Joyner * isc_max_frame_size. 726*71d10453SEric Joyner */ 727*71d10453SEric Joyner 728*71d10453SEric Joyner sc->ifp = ifp; 729*71d10453SEric Joyner sc->scctx->isc_max_frame_size = ifp->if_mtu + 730*71d10453SEric Joyner ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 731*71d10453SEric Joyner 732*71d10453SEric Joyner /* 733*71d10453SEric Joyner * If we are in recovery mode, only perform a limited subset of 734*71d10453SEric Joyner * initialization to support NVM recovery. 735*71d10453SEric Joyner */ 736*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 737*71d10453SEric Joyner ice_attach_post_recovery_mode(sc); 738*71d10453SEric Joyner return (0); 739*71d10453SEric Joyner } 740*71d10453SEric Joyner 741*71d10453SEric Joyner sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 742*71d10453SEric Joyner 743*71d10453SEric Joyner err = ice_initialize_vsi(&sc->pf_vsi); 744*71d10453SEric Joyner if (err) { 745*71d10453SEric Joyner device_printf(sc->dev, "Unable to initialize Main VSI: %s\n", 746*71d10453SEric Joyner ice_err_str(err)); 747*71d10453SEric Joyner return err; 748*71d10453SEric Joyner } 749*71d10453SEric Joyner 750*71d10453SEric Joyner /* Configure the main PF VSI for RSS */ 751*71d10453SEric Joyner err = ice_config_rss(&sc->pf_vsi); 752*71d10453SEric Joyner if (err) { 753*71d10453SEric Joyner device_printf(sc->dev, 754*71d10453SEric Joyner "Unable to configure RSS for the main VSI, err %s\n", 755*71d10453SEric Joyner ice_err_str(err)); 756*71d10453SEric Joyner return err; 757*71d10453SEric Joyner } 758*71d10453SEric Joyner 759*71d10453SEric Joyner /* Configure switch to drop transmitted LLDP and PAUSE frames */ 760*71d10453SEric Joyner err = ice_cfg_pf_ethertype_filters(sc); 761*71d10453SEric Joyner if (err) 762*71d10453SEric Joyner return err; 763*71d10453SEric Joyner 764*71d10453SEric Joyner ice_get_and_print_bus_info(sc); 765*71d10453SEric Joyner 766*71d10453SEric Joyner ice_set_link_management_mode(sc); 767*71d10453SEric Joyner 768*71d10453SEric Joyner ice_init_saved_phy_cfg(sc); 769*71d10453SEric Joyner 770*71d10453SEric Joyner ice_add_device_sysctls(sc); 771*71d10453SEric Joyner 772*71d10453SEric Joyner /* Get DCBX/LLDP state and start DCBX agent */ 773*71d10453SEric Joyner ice_init_dcb_setup(sc); 774*71d10453SEric Joyner 775*71d10453SEric Joyner /* Setup link configuration parameters */ 776*71d10453SEric Joyner ice_init_link_configuration(sc); 777*71d10453SEric Joyner ice_update_link_status(sc, true); 778*71d10453SEric Joyner 779*71d10453SEric Joyner /* Configure interrupt causes for the administrative interrupt */ 780*71d10453SEric Joyner ice_configure_misc_interrupts(sc); 781*71d10453SEric Joyner 782*71d10453SEric Joyner /* Enable ITR 0 right away, so that we can handle admin interrupts */ 783*71d10453SEric Joyner ice_enable_intr(&sc->hw, sc->irqvs[0].me); 784*71d10453SEric Joyner 785*71d10453SEric Joyner /* Start the admin timer */ 786*71d10453SEric Joyner mtx_lock(&sc->admin_mtx); 787*71d10453SEric Joyner callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 788*71d10453SEric Joyner mtx_unlock(&sc->admin_mtx); 789*71d10453SEric Joyner 790*71d10453SEric Joyner return 0; 791*71d10453SEric Joyner } /* ice_if_attach_post */ 792*71d10453SEric Joyner 793*71d10453SEric Joyner /** 794*71d10453SEric Joyner * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery 795*71d10453SEric Joyner * @sc: the device private softc 796*71d10453SEric Joyner * 797*71d10453SEric Joyner * Performs minimal work to prepare the driver to recover an NVM in case the 798*71d10453SEric Joyner * firmware is in recovery mode. 799*71d10453SEric Joyner */ 800*71d10453SEric Joyner static void 801*71d10453SEric Joyner ice_attach_post_recovery_mode(struct ice_softc *sc) 802*71d10453SEric Joyner { 803*71d10453SEric Joyner /* Configure interrupt causes for the administrative interrupt */ 804*71d10453SEric Joyner ice_configure_misc_interrupts(sc); 805*71d10453SEric Joyner 806*71d10453SEric Joyner /* Enable ITR 0 right away, so that we can handle admin interrupts */ 807*71d10453SEric Joyner ice_enable_intr(&sc->hw, sc->irqvs[0].me); 808*71d10453SEric Joyner 809*71d10453SEric Joyner /* Start the admin timer */ 810*71d10453SEric Joyner mtx_lock(&sc->admin_mtx); 811*71d10453SEric Joyner callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 812*71d10453SEric Joyner mtx_unlock(&sc->admin_mtx); 813*71d10453SEric Joyner } 814*71d10453SEric Joyner 815*71d10453SEric Joyner /** 816*71d10453SEric Joyner * ice_free_irqvs - Free IRQ vector memory 817*71d10453SEric Joyner * @sc: the device private softc structure 818*71d10453SEric Joyner * 819*71d10453SEric Joyner * Free IRQ vector memory allocated during ice_if_msix_intr_assign. 820*71d10453SEric Joyner */ 821*71d10453SEric Joyner static void 822*71d10453SEric Joyner ice_free_irqvs(struct ice_softc *sc) 823*71d10453SEric Joyner { 824*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 825*71d10453SEric Joyner if_ctx_t ctx = sc->ctx; 826*71d10453SEric Joyner int i; 827*71d10453SEric Joyner 828*71d10453SEric Joyner /* If the irqvs array is NULL, then there are no vectors to free */ 829*71d10453SEric Joyner if (sc->irqvs == NULL) 830*71d10453SEric Joyner return; 831*71d10453SEric Joyner 832*71d10453SEric Joyner /* Free the IRQ vectors */ 833*71d10453SEric Joyner for (i = 0; i < sc->num_irq_vectors; i++) 834*71d10453SEric Joyner iflib_irq_free(ctx, &sc->irqvs[i].irq); 835*71d10453SEric Joyner 836*71d10453SEric Joyner /* Clear the irqv pointers */ 837*71d10453SEric Joyner for (i = 0; i < vsi->num_rx_queues; i++) 838*71d10453SEric Joyner vsi->rx_queues[i].irqv = NULL; 839*71d10453SEric Joyner 840*71d10453SEric Joyner for (i = 0; i < vsi->num_tx_queues; i++) 841*71d10453SEric Joyner vsi->tx_queues[i].irqv = NULL; 842*71d10453SEric Joyner 843*71d10453SEric Joyner /* Release the vector array memory */ 844*71d10453SEric Joyner free(sc->irqvs, M_ICE); 845*71d10453SEric Joyner sc->irqvs = NULL; 846*71d10453SEric Joyner sc->num_irq_vectors = 0; 847*71d10453SEric Joyner } 848*71d10453SEric Joyner 849*71d10453SEric Joyner /** 850*71d10453SEric Joyner * ice_if_detach - Device driver detach logic 851*71d10453SEric Joyner * @ctx: iflib context structure 852*71d10453SEric Joyner * 853*71d10453SEric Joyner * Perform device shutdown logic to detach the device driver. 854*71d10453SEric Joyner * 855*71d10453SEric Joyner * Note that there is no guarantee of the ordering of ice_if_queues_free() and 856*71d10453SEric Joyner * ice_if_detach(). It is possible for the functions to be called in either 857*71d10453SEric Joyner * order, and they must not assume to have a strict ordering. 858*71d10453SEric Joyner */ 859*71d10453SEric Joyner static int 860*71d10453SEric Joyner ice_if_detach(if_ctx_t ctx) 861*71d10453SEric Joyner { 862*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 863*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 864*71d10453SEric Joyner int i; 865*71d10453SEric Joyner 866*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 867*71d10453SEric Joyner 868*71d10453SEric Joyner /* Indicate that we're detaching */ 869*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_DETACHING); 870*71d10453SEric Joyner 871*71d10453SEric Joyner /* Stop the admin timer */ 872*71d10453SEric Joyner mtx_lock(&sc->admin_mtx); 873*71d10453SEric Joyner callout_stop(&sc->admin_timer); 874*71d10453SEric Joyner mtx_unlock(&sc->admin_mtx); 875*71d10453SEric Joyner mtx_destroy(&sc->admin_mtx); 876*71d10453SEric Joyner 877*71d10453SEric Joyner /* Free allocated media types */ 878*71d10453SEric Joyner ifmedia_removeall(sc->media); 879*71d10453SEric Joyner 880*71d10453SEric Joyner /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 881*71d10453SEric Joyner * pointers. Note, the calls here and those in ice_if_queues_free() 882*71d10453SEric Joyner * are *BOTH* necessary, as we cannot guarantee which path will be 883*71d10453SEric Joyner * run first 884*71d10453SEric Joyner */ 885*71d10453SEric Joyner ice_vsi_del_txqs_ctx(vsi); 886*71d10453SEric Joyner ice_vsi_del_rxqs_ctx(vsi); 887*71d10453SEric Joyner 888*71d10453SEric Joyner /* Release MSI-X resources */ 889*71d10453SEric Joyner ice_free_irqvs(sc); 890*71d10453SEric Joyner 891*71d10453SEric Joyner for (i = 0; i < sc->num_available_vsi; i++) { 892*71d10453SEric Joyner if (sc->all_vsi[i]) 893*71d10453SEric Joyner ice_release_vsi(sc->all_vsi[i]); 894*71d10453SEric Joyner } 895*71d10453SEric Joyner 896*71d10453SEric Joyner if (sc->all_vsi) { 897*71d10453SEric Joyner free(sc->all_vsi, M_ICE); 898*71d10453SEric Joyner sc->all_vsi = NULL; 899*71d10453SEric Joyner } 900*71d10453SEric Joyner 901*71d10453SEric Joyner /* Release MSI-X memory */ 902*71d10453SEric Joyner pci_release_msi(sc->dev); 903*71d10453SEric Joyner 904*71d10453SEric Joyner if (sc->msix_table != NULL) { 905*71d10453SEric Joyner bus_release_resource(sc->dev, SYS_RES_MEMORY, 906*71d10453SEric Joyner rman_get_rid(sc->msix_table), 907*71d10453SEric Joyner sc->msix_table); 908*71d10453SEric Joyner sc->msix_table = NULL; 909*71d10453SEric Joyner } 910*71d10453SEric Joyner 911*71d10453SEric Joyner ice_free_intr_tracking(sc); 912*71d10453SEric Joyner 913*71d10453SEric Joyner /* Destroy the queue managers */ 914*71d10453SEric Joyner ice_resmgr_destroy(&sc->tx_qmgr); 915*71d10453SEric Joyner ice_resmgr_destroy(&sc->rx_qmgr); 916*71d10453SEric Joyner 917*71d10453SEric Joyner if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 918*71d10453SEric Joyner ice_deinit_hw(&sc->hw); 919*71d10453SEric Joyner 920*71d10453SEric Joyner ice_free_pci_mapping(sc); 921*71d10453SEric Joyner 922*71d10453SEric Joyner return 0; 923*71d10453SEric Joyner } /* ice_if_detach */ 924*71d10453SEric Joyner 925*71d10453SEric Joyner /** 926*71d10453SEric Joyner * ice_if_tx_queues_alloc - Allocate Tx queue memory 927*71d10453SEric Joyner * @ctx: iflib context structure 928*71d10453SEric Joyner * @vaddrs: virtual addresses for the queue memory 929*71d10453SEric Joyner * @paddrs: physical addresses for the queue memory 930*71d10453SEric Joyner * @ntxqs: the number of Tx queues per set (should always be 1) 931*71d10453SEric Joyner * @ntxqsets: the number of Tx queue sets to allocate 932*71d10453SEric Joyner * 933*71d10453SEric Joyner * Called by iflib to allocate Tx queues for the device. Allocates driver 934*71d10453SEric Joyner * memory to track each queue, the status arrays used for descriptor 935*71d10453SEric Joyner * status reporting, and Tx queue sysctls. 936*71d10453SEric Joyner */ 937*71d10453SEric Joyner static int 938*71d10453SEric Joyner ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 939*71d10453SEric Joyner int __invariant_only ntxqs, int ntxqsets) 940*71d10453SEric Joyner { 941*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 942*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 943*71d10453SEric Joyner struct ice_tx_queue *txq; 944*71d10453SEric Joyner int err, i, j; 945*71d10453SEric Joyner 946*71d10453SEric Joyner MPASS(ntxqs == 1); 947*71d10453SEric Joyner MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 948*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 949*71d10453SEric Joyner 950*71d10453SEric Joyner /* Do not bother allocating queues if we're in recovery mode */ 951*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 952*71d10453SEric Joyner return (0); 953*71d10453SEric Joyner 954*71d10453SEric Joyner /* Allocate queue structure memory */ 955*71d10453SEric Joyner if (!(vsi->tx_queues = 956*71d10453SEric Joyner (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) { 957*71d10453SEric Joyner device_printf(sc->dev, "Unable to allocate Tx queue memory\n"); 958*71d10453SEric Joyner return (ENOMEM); 959*71d10453SEric Joyner } 960*71d10453SEric Joyner 961*71d10453SEric Joyner /* Allocate report status arrays */ 962*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 963*71d10453SEric Joyner if (!(txq->tx_rsq = 964*71d10453SEric Joyner (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) { 965*71d10453SEric Joyner device_printf(sc->dev, "Unable to allocate tx_rsq memory\n"); 966*71d10453SEric Joyner err = ENOMEM; 967*71d10453SEric Joyner goto free_tx_queues; 968*71d10453SEric Joyner } 969*71d10453SEric Joyner /* Initialize report status array */ 970*71d10453SEric Joyner for (j = 0; j < sc->scctx->isc_ntxd[0]; j++) 971*71d10453SEric Joyner txq->tx_rsq[j] = QIDX_INVALID; 972*71d10453SEric Joyner } 973*71d10453SEric Joyner 974*71d10453SEric Joyner /* Assign queues from PF space to the main VSI */ 975*71d10453SEric Joyner err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets); 976*71d10453SEric Joyner if (err) { 977*71d10453SEric Joyner device_printf(sc->dev, "Unable to assign PF queues: %s\n", 978*71d10453SEric Joyner ice_err_str(err)); 979*71d10453SEric Joyner goto free_tx_queues; 980*71d10453SEric Joyner } 981*71d10453SEric Joyner vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 982*71d10453SEric Joyner 983*71d10453SEric Joyner /* Add Tx queue sysctls context */ 984*71d10453SEric Joyner ice_vsi_add_txqs_ctx(vsi); 985*71d10453SEric Joyner 986*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 987*71d10453SEric Joyner txq->me = i; 988*71d10453SEric Joyner txq->vsi = vsi; 989*71d10453SEric Joyner 990*71d10453SEric Joyner /* store the queue size for easier access */ 991*71d10453SEric Joyner txq->desc_count = sc->scctx->isc_ntxd[0]; 992*71d10453SEric Joyner 993*71d10453SEric Joyner /* get the virtual and physical address of the hardware queues */ 994*71d10453SEric Joyner txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 995*71d10453SEric Joyner txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 996*71d10453SEric Joyner txq->tx_paddr = paddrs[i]; 997*71d10453SEric Joyner 998*71d10453SEric Joyner ice_add_txq_sysctls(txq); 999*71d10453SEric Joyner } 1000*71d10453SEric Joyner 1001*71d10453SEric Joyner vsi->num_tx_queues = ntxqsets; 1002*71d10453SEric Joyner 1003*71d10453SEric Joyner return (0); 1004*71d10453SEric Joyner 1005*71d10453SEric Joyner free_tx_queues: 1006*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1007*71d10453SEric Joyner if (txq->tx_rsq != NULL) { 1008*71d10453SEric Joyner free(txq->tx_rsq, M_ICE); 1009*71d10453SEric Joyner txq->tx_rsq = NULL; 1010*71d10453SEric Joyner } 1011*71d10453SEric Joyner } 1012*71d10453SEric Joyner free(vsi->tx_queues, M_ICE); 1013*71d10453SEric Joyner vsi->tx_queues = NULL; 1014*71d10453SEric Joyner return err; 1015*71d10453SEric Joyner } 1016*71d10453SEric Joyner 1017*71d10453SEric Joyner /** 1018*71d10453SEric Joyner * ice_if_rx_queues_alloc - Allocate Rx queue memory 1019*71d10453SEric Joyner * @ctx: iflib context structure 1020*71d10453SEric Joyner * @vaddrs: virtual addresses for the queue memory 1021*71d10453SEric Joyner * @paddrs: physical addresses for the queue memory 1022*71d10453SEric Joyner * @nrxqs: number of Rx queues per set (should always be 1) 1023*71d10453SEric Joyner * @nrxqsets: number of Rx queue sets to allocate 1024*71d10453SEric Joyner * 1025*71d10453SEric Joyner * Called by iflib to allocate Rx queues for the device. Allocates driver 1026*71d10453SEric Joyner * memory to track each queue, as well as sets up the Rx queue sysctls. 1027*71d10453SEric Joyner */ 1028*71d10453SEric Joyner static int 1029*71d10453SEric Joyner ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1030*71d10453SEric Joyner int __invariant_only nrxqs, int nrxqsets) 1031*71d10453SEric Joyner { 1032*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1033*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1034*71d10453SEric Joyner struct ice_rx_queue *rxq; 1035*71d10453SEric Joyner int err, i; 1036*71d10453SEric Joyner 1037*71d10453SEric Joyner MPASS(nrxqs == 1); 1038*71d10453SEric Joyner MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 1039*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1040*71d10453SEric Joyner 1041*71d10453SEric Joyner /* Do not bother allocating queues if we're in recovery mode */ 1042*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1043*71d10453SEric Joyner return (0); 1044*71d10453SEric Joyner 1045*71d10453SEric Joyner /* Allocate queue structure memory */ 1046*71d10453SEric Joyner if (!(vsi->rx_queues = 1047*71d10453SEric Joyner (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) { 1048*71d10453SEric Joyner device_printf(sc->dev, "Unable to allocate Rx queue memory\n"); 1049*71d10453SEric Joyner return (ENOMEM); 1050*71d10453SEric Joyner } 1051*71d10453SEric Joyner 1052*71d10453SEric Joyner /* Assign queues from PF space to the main VSI */ 1053*71d10453SEric Joyner err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets); 1054*71d10453SEric Joyner if (err) { 1055*71d10453SEric Joyner device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1056*71d10453SEric Joyner ice_err_str(err)); 1057*71d10453SEric Joyner goto free_rx_queues; 1058*71d10453SEric Joyner } 1059*71d10453SEric Joyner vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1060*71d10453SEric Joyner 1061*71d10453SEric Joyner /* Add Rx queue sysctls context */ 1062*71d10453SEric Joyner ice_vsi_add_rxqs_ctx(vsi); 1063*71d10453SEric Joyner 1064*71d10453SEric Joyner for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 1065*71d10453SEric Joyner rxq->me = i; 1066*71d10453SEric Joyner rxq->vsi = vsi; 1067*71d10453SEric Joyner 1068*71d10453SEric Joyner /* store the queue size for easier access */ 1069*71d10453SEric Joyner rxq->desc_count = sc->scctx->isc_nrxd[0]; 1070*71d10453SEric Joyner 1071*71d10453SEric Joyner /* get the virtual and physical address of the hardware queues */ 1072*71d10453SEric Joyner rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 1073*71d10453SEric Joyner rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 1074*71d10453SEric Joyner rxq->rx_paddr = paddrs[i]; 1075*71d10453SEric Joyner 1076*71d10453SEric Joyner ice_add_rxq_sysctls(rxq); 1077*71d10453SEric Joyner } 1078*71d10453SEric Joyner 1079*71d10453SEric Joyner vsi->num_rx_queues = nrxqsets; 1080*71d10453SEric Joyner 1081*71d10453SEric Joyner return (0); 1082*71d10453SEric Joyner 1083*71d10453SEric Joyner free_rx_queues: 1084*71d10453SEric Joyner free(vsi->rx_queues, M_ICE); 1085*71d10453SEric Joyner vsi->rx_queues = NULL; 1086*71d10453SEric Joyner return err; 1087*71d10453SEric Joyner } 1088*71d10453SEric Joyner 1089*71d10453SEric Joyner /** 1090*71d10453SEric Joyner * ice_if_queues_free - Free queue memory 1091*71d10453SEric Joyner * @ctx: the iflib context structure 1092*71d10453SEric Joyner * 1093*71d10453SEric Joyner * Free queue memory allocated by ice_if_tx_queues_alloc() and 1094*71d10453SEric Joyner * ice_if_rx_queues_alloc(). 1095*71d10453SEric Joyner * 1096*71d10453SEric Joyner * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be 1097*71d10453SEric Joyner * called in the same order. It's possible for ice_if_queues_free() to be 1098*71d10453SEric Joyner * called prior to ice_if_detach(), and vice versa. 1099*71d10453SEric Joyner * 1100*71d10453SEric Joyner * For this reason, the main VSI is a static member of the ice_softc, which is 1101*71d10453SEric Joyner * not free'd until after iflib finishes calling both of these functions. 1102*71d10453SEric Joyner * 1103*71d10453SEric Joyner * Thus, care must be taken in how we manage the memory being freed by this 1104*71d10453SEric Joyner * function, and in what tasks it can and must perform. 1105*71d10453SEric Joyner */ 1106*71d10453SEric Joyner static void 1107*71d10453SEric Joyner ice_if_queues_free(if_ctx_t ctx) 1108*71d10453SEric Joyner { 1109*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1110*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1111*71d10453SEric Joyner struct ice_tx_queue *txq; 1112*71d10453SEric Joyner int i; 1113*71d10453SEric Joyner 1114*71d10453SEric Joyner /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1115*71d10453SEric Joyner * pointers. Note, the calls here and those in ice_if_detach() 1116*71d10453SEric Joyner * are *BOTH* necessary, as we cannot guarantee which path will be 1117*71d10453SEric Joyner * run first 1118*71d10453SEric Joyner */ 1119*71d10453SEric Joyner ice_vsi_del_txqs_ctx(vsi); 1120*71d10453SEric Joyner ice_vsi_del_rxqs_ctx(vsi); 1121*71d10453SEric Joyner 1122*71d10453SEric Joyner /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */ 1123*71d10453SEric Joyner ice_free_irqvs(sc); 1124*71d10453SEric Joyner 1125*71d10453SEric Joyner if (vsi->tx_queues != NULL) { 1126*71d10453SEric Joyner /* free the tx_rsq arrays */ 1127*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1128*71d10453SEric Joyner if (txq->tx_rsq != NULL) { 1129*71d10453SEric Joyner free(txq->tx_rsq, M_ICE); 1130*71d10453SEric Joyner txq->tx_rsq = NULL; 1131*71d10453SEric Joyner } 1132*71d10453SEric Joyner } 1133*71d10453SEric Joyner free(vsi->tx_queues, M_ICE); 1134*71d10453SEric Joyner vsi->tx_queues = NULL; 1135*71d10453SEric Joyner vsi->num_tx_queues = 0; 1136*71d10453SEric Joyner } 1137*71d10453SEric Joyner if (vsi->rx_queues != NULL) { 1138*71d10453SEric Joyner free(vsi->rx_queues, M_ICE); 1139*71d10453SEric Joyner vsi->rx_queues = NULL; 1140*71d10453SEric Joyner vsi->num_rx_queues = 0; 1141*71d10453SEric Joyner } 1142*71d10453SEric Joyner } 1143*71d10453SEric Joyner 1144*71d10453SEric Joyner /** 1145*71d10453SEric Joyner * ice_msix_que - Fast interrupt handler for MSI-X receive queues 1146*71d10453SEric Joyner * @arg: The Rx queue memory 1147*71d10453SEric Joyner * 1148*71d10453SEric Joyner * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when 1149*71d10453SEric Joyner * an MSI-X interrupt for a given queue is triggered. Currently this just asks 1150*71d10453SEric Joyner * iflib to schedule the main Rx thread. 1151*71d10453SEric Joyner */ 1152*71d10453SEric Joyner static int 1153*71d10453SEric Joyner ice_msix_que(void *arg) 1154*71d10453SEric Joyner { 1155*71d10453SEric Joyner struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg; 1156*71d10453SEric Joyner 1157*71d10453SEric Joyner /* TODO: dynamic ITR algorithm?? */ 1158*71d10453SEric Joyner 1159*71d10453SEric Joyner return (FILTER_SCHEDULE_THREAD); 1160*71d10453SEric Joyner } 1161*71d10453SEric Joyner 1162*71d10453SEric Joyner /** 1163*71d10453SEric Joyner * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt 1164*71d10453SEric Joyner * @arg: pointer to device softc memory 1165*71d10453SEric Joyner * 1166*71d10453SEric Joyner * Called by iflib when an administrative interrupt occurs. Should perform any 1167*71d10453SEric Joyner * fast logic for handling the interrupt cause, and then indicate whether the 1168*71d10453SEric Joyner * admin task needs to be queued. 1169*71d10453SEric Joyner */ 1170*71d10453SEric Joyner static int 1171*71d10453SEric Joyner ice_msix_admin(void *arg) 1172*71d10453SEric Joyner { 1173*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 1174*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1175*71d10453SEric Joyner device_t dev = sc->dev; 1176*71d10453SEric Joyner u32 oicr; 1177*71d10453SEric Joyner 1178*71d10453SEric Joyner /* There is no safe way to modify the enabled miscellaneous causes of 1179*71d10453SEric Joyner * the OICR vector at runtime, as doing so would be prone to race 1180*71d10453SEric Joyner * conditions. Reading PFINT_OICR will unmask the associated interrupt 1181*71d10453SEric Joyner * causes and allow future interrupts to occur. The admin interrupt 1182*71d10453SEric Joyner * vector will not be re-enabled until after we exit this function, 1183*71d10453SEric Joyner * but any delayed tasks must be resilient against possible "late 1184*71d10453SEric Joyner * arrival" interrupts that occur while we're already handling the 1185*71d10453SEric Joyner * task. This is done by using state bits and serializing these 1186*71d10453SEric Joyner * delayed tasks via the admin status task function. 1187*71d10453SEric Joyner */ 1188*71d10453SEric Joyner oicr = rd32(hw, PFINT_OICR); 1189*71d10453SEric Joyner 1190*71d10453SEric Joyner /* Processing multiple controlq interrupts on a single vector does not 1191*71d10453SEric Joyner * provide an indication of which controlq triggered the interrupt. 1192*71d10453SEric Joyner * We might try reading the INTEVENT bit of the respective PFINT_*_CTL 1193*71d10453SEric Joyner * registers. However, the INTEVENT bit is not guaranteed to be set as 1194*71d10453SEric Joyner * it gets automatically cleared when the hardware acknowledges the 1195*71d10453SEric Joyner * interrupt. 1196*71d10453SEric Joyner * 1197*71d10453SEric Joyner * This means we don't really have a good indication of whether or 1198*71d10453SEric Joyner * which controlq triggered this interrupt. We'll just notify the 1199*71d10453SEric Joyner * admin task that it should check all the controlqs. 1200*71d10453SEric Joyner */ 1201*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 1202*71d10453SEric Joyner 1203*71d10453SEric Joyner if (oicr & PFINT_OICR_VFLR_M) { 1204*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING); 1205*71d10453SEric Joyner } 1206*71d10453SEric Joyner 1207*71d10453SEric Joyner if (oicr & PFINT_OICR_MAL_DETECT_M) { 1208*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_MDD_PENDING); 1209*71d10453SEric Joyner } 1210*71d10453SEric Joyner 1211*71d10453SEric Joyner if (oicr & PFINT_OICR_GRST_M) { 1212*71d10453SEric Joyner u32 reset; 1213*71d10453SEric Joyner 1214*71d10453SEric Joyner reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1215*71d10453SEric Joyner GLGEN_RSTAT_RESET_TYPE_S; 1216*71d10453SEric Joyner 1217*71d10453SEric Joyner if (reset == ICE_RESET_CORER) 1218*71d10453SEric Joyner sc->soft_stats.corer_count++; 1219*71d10453SEric Joyner else if (reset == ICE_RESET_GLOBR) 1220*71d10453SEric Joyner sc->soft_stats.globr_count++; 1221*71d10453SEric Joyner else 1222*71d10453SEric Joyner sc->soft_stats.empr_count++; 1223*71d10453SEric Joyner 1224*71d10453SEric Joyner /* There are a couple of bits at play for handling resets. 1225*71d10453SEric Joyner * First, the ICE_STATE_RESET_OICR_RECV bit is used to 1226*71d10453SEric Joyner * indicate that the driver has received an OICR with a reset 1227*71d10453SEric Joyner * bit active, indicating that a CORER/GLOBR/EMPR is about to 1228*71d10453SEric Joyner * happen. Second, we set hw->reset_ongoing to indicate that 1229*71d10453SEric Joyner * the hardware is in reset. We will set this back to false as 1230*71d10453SEric Joyner * soon as the driver has determined that the hardware is out 1231*71d10453SEric Joyner * of reset. 1232*71d10453SEric Joyner * 1233*71d10453SEric Joyner * If the driver wishes to trigger a reqest, it can set one of 1234*71d10453SEric Joyner * the ICE_STATE_RESET_*_REQ bits, which will trigger the 1235*71d10453SEric Joyner * correct type of reset. 1236*71d10453SEric Joyner */ 1237*71d10453SEric Joyner if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 1238*71d10453SEric Joyner hw->reset_ongoing = true; 1239*71d10453SEric Joyner } 1240*71d10453SEric Joyner 1241*71d10453SEric Joyner if (oicr & PFINT_OICR_ECC_ERR_M) { 1242*71d10453SEric Joyner device_printf(dev, "ECC Error detected!\n"); 1243*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1244*71d10453SEric Joyner } 1245*71d10453SEric Joyner 1246*71d10453SEric Joyner if (oicr & PFINT_OICR_PE_CRITERR_M) { 1247*71d10453SEric Joyner device_printf(dev, "Critical Protocol Engine Error detected!\n"); 1248*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1249*71d10453SEric Joyner } 1250*71d10453SEric Joyner 1251*71d10453SEric Joyner if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { 1252*71d10453SEric Joyner device_printf(dev, "PCI Exception detected!\n"); 1253*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1254*71d10453SEric Joyner } 1255*71d10453SEric Joyner 1256*71d10453SEric Joyner if (oicr & PFINT_OICR_HMC_ERR_M) { 1257*71d10453SEric Joyner /* Log the HMC errors, but don't disable the interrupt cause */ 1258*71d10453SEric Joyner ice_log_hmc_error(hw, dev); 1259*71d10453SEric Joyner } 1260*71d10453SEric Joyner 1261*71d10453SEric Joyner return (FILTER_SCHEDULE_THREAD); 1262*71d10453SEric Joyner } 1263*71d10453SEric Joyner 1264*71d10453SEric Joyner /** 1265*71d10453SEric Joyner * ice_allocate_msix - Allocate MSI-X vectors for the interface 1266*71d10453SEric Joyner * @sc: the device private softc 1267*71d10453SEric Joyner * 1268*71d10453SEric Joyner * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process. 1269*71d10453SEric Joyner * 1270*71d10453SEric Joyner * First, determine a suitable total number of vectors based on the number 1271*71d10453SEric Joyner * of CPUs, RSS buckets, the administrative vector, and other demands such as 1272*71d10453SEric Joyner * RDMA. 1273*71d10453SEric Joyner * 1274*71d10453SEric Joyner * Request the desired amount of vectors, and see how many we obtain. If we 1275*71d10453SEric Joyner * don't obtain as many as desired, reduce the demands by lowering the number 1276*71d10453SEric Joyner * of requested queues or reducing the demand from other features such as 1277*71d10453SEric Joyner * RDMA. 1278*71d10453SEric Joyner * 1279*71d10453SEric Joyner * @remark This function is required because the driver sets the 1280*71d10453SEric Joyner * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors 1281*71d10453SEric Joyner * manually. 1282*71d10453SEric Joyner * 1283*71d10453SEric Joyner * @remark This driver will only use MSI-X vectors. If this is not possible, 1284*71d10453SEric Joyner * neither MSI or legacy interrupts will be tried. 1285*71d10453SEric Joyner * 1286*71d10453SEric Joyner * @post on success this function must set the following scctx parameters: 1287*71d10453SEric Joyner * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr. 1288*71d10453SEric Joyner * 1289*71d10453SEric Joyner * @returns zero on success or an error code on failure. 1290*71d10453SEric Joyner */ 1291*71d10453SEric Joyner static int 1292*71d10453SEric Joyner ice_allocate_msix(struct ice_softc *sc) 1293*71d10453SEric Joyner { 1294*71d10453SEric Joyner bool iflib_override_queue_count = false; 1295*71d10453SEric Joyner if_softc_ctx_t scctx = sc->scctx; 1296*71d10453SEric Joyner device_t dev = sc->dev; 1297*71d10453SEric Joyner cpuset_t cpus; 1298*71d10453SEric Joyner int bar, queues, vectors, requested; 1299*71d10453SEric Joyner int err = 0; 1300*71d10453SEric Joyner 1301*71d10453SEric Joyner /* Allocate the MSI-X bar */ 1302*71d10453SEric Joyner bar = scctx->isc_msix_bar; 1303*71d10453SEric Joyner sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); 1304*71d10453SEric Joyner if (!sc->msix_table) { 1305*71d10453SEric Joyner device_printf(dev, "Unable to map MSI-X table\n"); 1306*71d10453SEric Joyner return (ENOMEM); 1307*71d10453SEric Joyner } 1308*71d10453SEric Joyner 1309*71d10453SEric Joyner /* Check if the iflib queue count sysctls have been set */ 1310*71d10453SEric Joyner if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs) 1311*71d10453SEric Joyner iflib_override_queue_count = true; 1312*71d10453SEric Joyner 1313*71d10453SEric Joyner err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus); 1314*71d10453SEric Joyner if (err) { 1315*71d10453SEric Joyner device_printf(dev, "%s: Unable to fetch the CPU list: %s\n", 1316*71d10453SEric Joyner __func__, ice_err_str(err)); 1317*71d10453SEric Joyner CPU_COPY(&all_cpus, &cpus); 1318*71d10453SEric Joyner } 1319*71d10453SEric Joyner 1320*71d10453SEric Joyner /* Attempt to mimic behavior of iflib_msix_init */ 1321*71d10453SEric Joyner if (iflib_override_queue_count) { 1322*71d10453SEric Joyner /* 1323*71d10453SEric Joyner * If the override sysctls have been set, limit the queues to 1324*71d10453SEric Joyner * the number of logical CPUs. 1325*71d10453SEric Joyner */ 1326*71d10453SEric Joyner queues = mp_ncpus; 1327*71d10453SEric Joyner } else { 1328*71d10453SEric Joyner /* 1329*71d10453SEric Joyner * Otherwise, limit the queue count to the CPUs associated 1330*71d10453SEric Joyner * with the NUMA node the device is associated with. 1331*71d10453SEric Joyner */ 1332*71d10453SEric Joyner queues = CPU_COUNT(&cpus); 1333*71d10453SEric Joyner } 1334*71d10453SEric Joyner 1335*71d10453SEric Joyner /* Clamp to the number of RSS buckets */ 1336*71d10453SEric Joyner queues = imin(queues, rss_getnumbuckets()); 1337*71d10453SEric Joyner 1338*71d10453SEric Joyner /* 1339*71d10453SEric Joyner * Clamp the number of queue pairs to the minimum of the requested Tx 1340*71d10453SEric Joyner * and Rx queues. 1341*71d10453SEric Joyner */ 1342*71d10453SEric Joyner queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets); 1343*71d10453SEric Joyner queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets); 1344*71d10453SEric Joyner 1345*71d10453SEric Joyner /* 1346*71d10453SEric Joyner * Determine the number of vectors to request. Note that we also need 1347*71d10453SEric Joyner * to allocate one vector for administrative tasks. 1348*71d10453SEric Joyner */ 1349*71d10453SEric Joyner requested = queues + 1; 1350*71d10453SEric Joyner 1351*71d10453SEric Joyner vectors = requested; 1352*71d10453SEric Joyner 1353*71d10453SEric Joyner err = pci_alloc_msix(dev, &vectors); 1354*71d10453SEric Joyner if (err) { 1355*71d10453SEric Joyner device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n", 1356*71d10453SEric Joyner vectors, ice_err_str(err)); 1357*71d10453SEric Joyner goto err_free_msix_table; 1358*71d10453SEric Joyner } 1359*71d10453SEric Joyner 1360*71d10453SEric Joyner /* If we don't receive enough vectors, reduce demands */ 1361*71d10453SEric Joyner if (vectors < requested) { 1362*71d10453SEric Joyner int diff = requested - vectors; 1363*71d10453SEric Joyner 1364*71d10453SEric Joyner device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n", 1365*71d10453SEric Joyner requested, vectors); 1366*71d10453SEric Joyner 1367*71d10453SEric Joyner /* 1368*71d10453SEric Joyner * If we still have a difference, we need to reduce the number 1369*71d10453SEric Joyner * of queue pairs. 1370*71d10453SEric Joyner * 1371*71d10453SEric Joyner * However, we still need at least one vector for the admin 1372*71d10453SEric Joyner * interrupt and one queue pair. 1373*71d10453SEric Joyner */ 1374*71d10453SEric Joyner if (queues <= diff) { 1375*71d10453SEric Joyner device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n"); 1376*71d10453SEric Joyner err = (ERANGE); 1377*71d10453SEric Joyner goto err_pci_release_msi; 1378*71d10453SEric Joyner } 1379*71d10453SEric Joyner 1380*71d10453SEric Joyner queues -= diff; 1381*71d10453SEric Joyner } 1382*71d10453SEric Joyner 1383*71d10453SEric Joyner device_printf(dev, "Using %d Tx and Rx queues\n", queues); 1384*71d10453SEric Joyner device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 1385*71d10453SEric Joyner vectors); 1386*71d10453SEric Joyner 1387*71d10453SEric Joyner scctx->isc_vectors = vectors; 1388*71d10453SEric Joyner scctx->isc_nrxqsets = queues; 1389*71d10453SEric Joyner scctx->isc_ntxqsets = queues; 1390*71d10453SEric Joyner scctx->isc_intr = IFLIB_INTR_MSIX; 1391*71d10453SEric Joyner 1392*71d10453SEric Joyner /* Interrupt allocation tracking isn't required in recovery mode, 1393*71d10453SEric Joyner * since neither RDMA nor VFs are enabled. 1394*71d10453SEric Joyner */ 1395*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1396*71d10453SEric Joyner return (0); 1397*71d10453SEric Joyner 1398*71d10453SEric Joyner /* Keep track of which interrupt indices are being used for what */ 1399*71d10453SEric Joyner sc->lan_vectors = vectors; 1400*71d10453SEric Joyner err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors); 1401*71d10453SEric Joyner if (err) { 1402*71d10453SEric Joyner device_printf(dev, "Unable to assign PF interrupt mapping: %s\n", 1403*71d10453SEric Joyner ice_err_str(err)); 1404*71d10453SEric Joyner goto err_pci_release_msi; 1405*71d10453SEric Joyner } 1406*71d10453SEric Joyner 1407*71d10453SEric Joyner return (0); 1408*71d10453SEric Joyner 1409*71d10453SEric Joyner err_pci_release_msi: 1410*71d10453SEric Joyner pci_release_msi(dev); 1411*71d10453SEric Joyner err_free_msix_table: 1412*71d10453SEric Joyner if (sc->msix_table != NULL) { 1413*71d10453SEric Joyner bus_release_resource(sc->dev, SYS_RES_MEMORY, 1414*71d10453SEric Joyner rman_get_rid(sc->msix_table), 1415*71d10453SEric Joyner sc->msix_table); 1416*71d10453SEric Joyner sc->msix_table = NULL; 1417*71d10453SEric Joyner } 1418*71d10453SEric Joyner 1419*71d10453SEric Joyner return (err); 1420*71d10453SEric Joyner } 1421*71d10453SEric Joyner 1422*71d10453SEric Joyner /** 1423*71d10453SEric Joyner * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues 1424*71d10453SEric Joyner * @ctx: the iflib context structure 1425*71d10453SEric Joyner * @msix: the number of vectors we were assigned 1426*71d10453SEric Joyner * 1427*71d10453SEric Joyner * Called by iflib to assign MSI-X vectors to queues. Currently requires that 1428*71d10453SEric Joyner * we get at least the same number of vectors as we have queues, and that we 1429*71d10453SEric Joyner * always have the same number of Tx and Rx queues. 1430*71d10453SEric Joyner * 1431*71d10453SEric Joyner * Tx queues use a softirq instead of using their own hardware interrupt. 1432*71d10453SEric Joyner */ 1433*71d10453SEric Joyner static int 1434*71d10453SEric Joyner ice_if_msix_intr_assign(if_ctx_t ctx, int msix) 1435*71d10453SEric Joyner { 1436*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1437*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1438*71d10453SEric Joyner int err, i, vector; 1439*71d10453SEric Joyner 1440*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1441*71d10453SEric Joyner 1442*71d10453SEric Joyner if (vsi->num_rx_queues != vsi->num_tx_queues) { 1443*71d10453SEric Joyner device_printf(sc->dev, 1444*71d10453SEric Joyner "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 1445*71d10453SEric Joyner vsi->num_tx_queues, vsi->num_rx_queues); 1446*71d10453SEric Joyner return (EOPNOTSUPP); 1447*71d10453SEric Joyner } 1448*71d10453SEric Joyner 1449*71d10453SEric Joyner if (msix < (vsi->num_rx_queues + 1)) { 1450*71d10453SEric Joyner device_printf(sc->dev, 1451*71d10453SEric Joyner "Not enough MSI-X vectors to assign one vector to each queue pair\n"); 1452*71d10453SEric Joyner return (EOPNOTSUPP); 1453*71d10453SEric Joyner } 1454*71d10453SEric Joyner 1455*71d10453SEric Joyner /* Save the number of vectors for future use */ 1456*71d10453SEric Joyner sc->num_irq_vectors = vsi->num_rx_queues + 1; 1457*71d10453SEric Joyner 1458*71d10453SEric Joyner /* Allocate space to store the IRQ vector data */ 1459*71d10453SEric Joyner if (!(sc->irqvs = 1460*71d10453SEric Joyner (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors), 1461*71d10453SEric Joyner M_ICE, M_NOWAIT))) { 1462*71d10453SEric Joyner device_printf(sc->dev, 1463*71d10453SEric Joyner "Unable to allocate irqv memory\n"); 1464*71d10453SEric Joyner return (ENOMEM); 1465*71d10453SEric Joyner } 1466*71d10453SEric Joyner 1467*71d10453SEric Joyner /* Administrative interrupt events will use vector 0 */ 1468*71d10453SEric Joyner err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN, 1469*71d10453SEric Joyner ice_msix_admin, sc, 0, "admin"); 1470*71d10453SEric Joyner if (err) { 1471*71d10453SEric Joyner device_printf(sc->dev, 1472*71d10453SEric Joyner "Failed to register Admin queue handler: %s\n", 1473*71d10453SEric Joyner ice_err_str(err)); 1474*71d10453SEric Joyner goto free_irqvs; 1475*71d10453SEric Joyner } 1476*71d10453SEric Joyner sc->irqvs[0].me = 0; 1477*71d10453SEric Joyner 1478*71d10453SEric Joyner /* Do not allocate queue interrupts when in recovery mode */ 1479*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1480*71d10453SEric Joyner return (0); 1481*71d10453SEric Joyner 1482*71d10453SEric Joyner for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) { 1483*71d10453SEric Joyner struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 1484*71d10453SEric Joyner struct ice_tx_queue *txq = &vsi->tx_queues[i]; 1485*71d10453SEric Joyner int rid = vector + 1; 1486*71d10453SEric Joyner char irq_name[16]; 1487*71d10453SEric Joyner 1488*71d10453SEric Joyner snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1489*71d10453SEric Joyner err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid, 1490*71d10453SEric Joyner IFLIB_INTR_RX, ice_msix_que, 1491*71d10453SEric Joyner rxq, rxq->me, irq_name); 1492*71d10453SEric Joyner if (err) { 1493*71d10453SEric Joyner device_printf(sc->dev, 1494*71d10453SEric Joyner "Failed to allocate q int %d err: %s\n", 1495*71d10453SEric Joyner i, ice_err_str(err)); 1496*71d10453SEric Joyner vector--; 1497*71d10453SEric Joyner i--; 1498*71d10453SEric Joyner goto fail; 1499*71d10453SEric Joyner } 1500*71d10453SEric Joyner sc->irqvs[vector].me = vector; 1501*71d10453SEric Joyner rxq->irqv = &sc->irqvs[vector]; 1502*71d10453SEric Joyner 1503*71d10453SEric Joyner bzero(irq_name, sizeof(irq_name)); 1504*71d10453SEric Joyner 1505*71d10453SEric Joyner snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1506*71d10453SEric Joyner iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq, 1507*71d10453SEric Joyner IFLIB_INTR_TX, txq, 1508*71d10453SEric Joyner txq->me, irq_name); 1509*71d10453SEric Joyner txq->irqv = &sc->irqvs[vector]; 1510*71d10453SEric Joyner } 1511*71d10453SEric Joyner 1512*71d10453SEric Joyner return (0); 1513*71d10453SEric Joyner fail: 1514*71d10453SEric Joyner for (; i >= 0; i--, vector--) 1515*71d10453SEric Joyner iflib_irq_free(ctx, &sc->irqvs[vector].irq); 1516*71d10453SEric Joyner iflib_irq_free(ctx, &sc->irqvs[0].irq); 1517*71d10453SEric Joyner free_irqvs: 1518*71d10453SEric Joyner free(sc->irqvs, M_ICE); 1519*71d10453SEric Joyner sc->irqvs = NULL; 1520*71d10453SEric Joyner return err; 1521*71d10453SEric Joyner } 1522*71d10453SEric Joyner 1523*71d10453SEric Joyner /** 1524*71d10453SEric Joyner * ice_if_mtu_set - Set the device MTU 1525*71d10453SEric Joyner * @ctx: iflib context structure 1526*71d10453SEric Joyner * @mtu: the MTU requested 1527*71d10453SEric Joyner * 1528*71d10453SEric Joyner * Called by iflib to configure the device's Maximum Transmission Unit (MTU). 1529*71d10453SEric Joyner * 1530*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 1531*71d10453SEric Joyner */ 1532*71d10453SEric Joyner static int 1533*71d10453SEric Joyner ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1534*71d10453SEric Joyner { 1535*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1536*71d10453SEric Joyner 1537*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1538*71d10453SEric Joyner 1539*71d10453SEric Joyner /* Do not support configuration when in recovery mode */ 1540*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1541*71d10453SEric Joyner return (ENOSYS); 1542*71d10453SEric Joyner 1543*71d10453SEric Joyner if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU) 1544*71d10453SEric Joyner return (EINVAL); 1545*71d10453SEric Joyner 1546*71d10453SEric Joyner sc->scctx->isc_max_frame_size = mtu + 1547*71d10453SEric Joyner ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 1548*71d10453SEric Joyner 1549*71d10453SEric Joyner sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 1550*71d10453SEric Joyner 1551*71d10453SEric Joyner return (0); 1552*71d10453SEric Joyner } 1553*71d10453SEric Joyner 1554*71d10453SEric Joyner /** 1555*71d10453SEric Joyner * ice_if_intr_enable - Enable device interrupts 1556*71d10453SEric Joyner * @ctx: iflib context structure 1557*71d10453SEric Joyner * 1558*71d10453SEric Joyner * Called by iflib to request enabling device interrupts. 1559*71d10453SEric Joyner */ 1560*71d10453SEric Joyner static void 1561*71d10453SEric Joyner ice_if_intr_enable(if_ctx_t ctx) 1562*71d10453SEric Joyner { 1563*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1564*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1565*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1566*71d10453SEric Joyner 1567*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1568*71d10453SEric Joyner 1569*71d10453SEric Joyner /* Enable ITR 0 */ 1570*71d10453SEric Joyner ice_enable_intr(hw, sc->irqvs[0].me); 1571*71d10453SEric Joyner 1572*71d10453SEric Joyner /* Do not enable queue interrupts in recovery mode */ 1573*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1574*71d10453SEric Joyner return; 1575*71d10453SEric Joyner 1576*71d10453SEric Joyner /* Enable all queue interrupts */ 1577*71d10453SEric Joyner for (int i = 0; i < vsi->num_rx_queues; i++) 1578*71d10453SEric Joyner ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 1579*71d10453SEric Joyner } 1580*71d10453SEric Joyner 1581*71d10453SEric Joyner /** 1582*71d10453SEric Joyner * ice_if_intr_disable - Disable device interrupts 1583*71d10453SEric Joyner * @ctx: iflib context structure 1584*71d10453SEric Joyner * 1585*71d10453SEric Joyner * Called by iflib to request disabling device interrupts. 1586*71d10453SEric Joyner */ 1587*71d10453SEric Joyner static void 1588*71d10453SEric Joyner ice_if_intr_disable(if_ctx_t ctx) 1589*71d10453SEric Joyner { 1590*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1591*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1592*71d10453SEric Joyner unsigned int i; 1593*71d10453SEric Joyner 1594*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1595*71d10453SEric Joyner 1596*71d10453SEric Joyner /* IFDI_INTR_DISABLE may be called prior to interrupts actually being 1597*71d10453SEric Joyner * assigned to queues. Instead of assuming that the interrupt 1598*71d10453SEric Joyner * assignment in the rx_queues structure is valid, just disable all 1599*71d10453SEric Joyner * possible interrupts 1600*71d10453SEric Joyner * 1601*71d10453SEric Joyner * Note that we choose not to disable ITR 0 because this handles the 1602*71d10453SEric Joyner * AdminQ interrupts, and we want to keep processing these even when 1603*71d10453SEric Joyner * the interface is offline. 1604*71d10453SEric Joyner */ 1605*71d10453SEric Joyner for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++) 1606*71d10453SEric Joyner ice_disable_intr(hw, i); 1607*71d10453SEric Joyner } 1608*71d10453SEric Joyner 1609*71d10453SEric Joyner /** 1610*71d10453SEric Joyner * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 1611*71d10453SEric Joyner * @ctx: iflib context structure 1612*71d10453SEric Joyner * @rxqid: the Rx queue to enable 1613*71d10453SEric Joyner * 1614*71d10453SEric Joyner * Enable a specific Rx queue interrupt. 1615*71d10453SEric Joyner * 1616*71d10453SEric Joyner * This function is not protected by the iflib CTX lock. 1617*71d10453SEric Joyner */ 1618*71d10453SEric Joyner static int 1619*71d10453SEric Joyner ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1620*71d10453SEric Joyner { 1621*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1622*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1623*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1624*71d10453SEric Joyner 1625*71d10453SEric Joyner /* Do not enable queue interrupts in recovery mode */ 1626*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1627*71d10453SEric Joyner return (ENOSYS); 1628*71d10453SEric Joyner 1629*71d10453SEric Joyner ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 1630*71d10453SEric Joyner return (0); 1631*71d10453SEric Joyner } 1632*71d10453SEric Joyner 1633*71d10453SEric Joyner /** 1634*71d10453SEric Joyner * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 1635*71d10453SEric Joyner * @ctx: iflib context structure 1636*71d10453SEric Joyner * @txqid: the Tx queue to enable 1637*71d10453SEric Joyner * 1638*71d10453SEric Joyner * Enable a specific Tx queue interrupt. 1639*71d10453SEric Joyner * 1640*71d10453SEric Joyner * This function is not protected by the iflib CTX lock. 1641*71d10453SEric Joyner */ 1642*71d10453SEric Joyner static int 1643*71d10453SEric Joyner ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1644*71d10453SEric Joyner { 1645*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1646*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1647*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1648*71d10453SEric Joyner 1649*71d10453SEric Joyner /* Do not enable queue interrupts in recovery mode */ 1650*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1651*71d10453SEric Joyner return (ENOSYS); 1652*71d10453SEric Joyner 1653*71d10453SEric Joyner ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 1654*71d10453SEric Joyner return (0); 1655*71d10453SEric Joyner } 1656*71d10453SEric Joyner 1657*71d10453SEric Joyner /** 1658*71d10453SEric Joyner * ice_if_promisc_set - Set device promiscuous mode 1659*71d10453SEric Joyner * @ctx: iflib context structure 1660*71d10453SEric Joyner * @flags: promiscuous flags to configure 1661*71d10453SEric Joyner * 1662*71d10453SEric Joyner * Called by iflib to configure device promiscuous mode. 1663*71d10453SEric Joyner * 1664*71d10453SEric Joyner * @remark Calls to this function will always overwrite the previous setting 1665*71d10453SEric Joyner */ 1666*71d10453SEric Joyner static int 1667*71d10453SEric Joyner ice_if_promisc_set(if_ctx_t ctx, int flags) 1668*71d10453SEric Joyner { 1669*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1670*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1671*71d10453SEric Joyner device_t dev = sc->dev; 1672*71d10453SEric Joyner enum ice_status status; 1673*71d10453SEric Joyner bool promisc_enable = flags & IFF_PROMISC; 1674*71d10453SEric Joyner bool multi_enable = flags & IFF_ALLMULTI; 1675*71d10453SEric Joyner 1676*71d10453SEric Joyner /* Do not support configuration when in recovery mode */ 1677*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1678*71d10453SEric Joyner return (ENOSYS); 1679*71d10453SEric Joyner 1680*71d10453SEric Joyner if (multi_enable) 1681*71d10453SEric Joyner return (EOPNOTSUPP); 1682*71d10453SEric Joyner 1683*71d10453SEric Joyner if (promisc_enable) { 1684*71d10453SEric Joyner status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1685*71d10453SEric Joyner ICE_VSI_PROMISC_MASK, 0); 1686*71d10453SEric Joyner if (status && status != ICE_ERR_ALREADY_EXISTS) { 1687*71d10453SEric Joyner device_printf(dev, 1688*71d10453SEric Joyner "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n", 1689*71d10453SEric Joyner ice_status_str(status), 1690*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 1691*71d10453SEric Joyner return (EIO); 1692*71d10453SEric Joyner } 1693*71d10453SEric Joyner } else { 1694*71d10453SEric Joyner status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx, 1695*71d10453SEric Joyner ICE_VSI_PROMISC_MASK, 0); 1696*71d10453SEric Joyner if (status) { 1697*71d10453SEric Joyner device_printf(dev, 1698*71d10453SEric Joyner "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n", 1699*71d10453SEric Joyner ice_status_str(status), 1700*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 1701*71d10453SEric Joyner return (EIO); 1702*71d10453SEric Joyner } 1703*71d10453SEric Joyner } 1704*71d10453SEric Joyner 1705*71d10453SEric Joyner return (0); 1706*71d10453SEric Joyner } 1707*71d10453SEric Joyner 1708*71d10453SEric Joyner /** 1709*71d10453SEric Joyner * ice_if_media_change - Change device media 1710*71d10453SEric Joyner * @ctx: device ctx structure 1711*71d10453SEric Joyner * 1712*71d10453SEric Joyner * Called by iflib when a media change is requested. This operation is not 1713*71d10453SEric Joyner * supported by the hardware, so we just return an error code. 1714*71d10453SEric Joyner */ 1715*71d10453SEric Joyner static int 1716*71d10453SEric Joyner ice_if_media_change(if_ctx_t ctx) 1717*71d10453SEric Joyner { 1718*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1719*71d10453SEric Joyner 1720*71d10453SEric Joyner device_printf(sc->dev, "Media change is not supported.\n"); 1721*71d10453SEric Joyner return (ENODEV); 1722*71d10453SEric Joyner } 1723*71d10453SEric Joyner 1724*71d10453SEric Joyner /** 1725*71d10453SEric Joyner * ice_if_media_status - Report current device media 1726*71d10453SEric Joyner * @ctx: iflib context structure 1727*71d10453SEric Joyner * @ifmr: ifmedia request structure to update 1728*71d10453SEric Joyner * 1729*71d10453SEric Joyner * Updates the provided ifmr with current device media status, including link 1730*71d10453SEric Joyner * status and media type. 1731*71d10453SEric Joyner */ 1732*71d10453SEric Joyner static void 1733*71d10453SEric Joyner ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1734*71d10453SEric Joyner { 1735*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1736*71d10453SEric Joyner struct ice_link_status *li = &sc->hw.port_info->phy.link_info; 1737*71d10453SEric Joyner 1738*71d10453SEric Joyner ifmr->ifm_status = IFM_AVALID; 1739*71d10453SEric Joyner ifmr->ifm_active = IFM_ETHER; 1740*71d10453SEric Joyner 1741*71d10453SEric Joyner /* Never report link up or media types when in recovery mode */ 1742*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1743*71d10453SEric Joyner return; 1744*71d10453SEric Joyner 1745*71d10453SEric Joyner if (!sc->link_up) 1746*71d10453SEric Joyner return; 1747*71d10453SEric Joyner 1748*71d10453SEric Joyner ifmr->ifm_status |= IFM_ACTIVE; 1749*71d10453SEric Joyner ifmr->ifm_active |= IFM_FDX; 1750*71d10453SEric Joyner 1751*71d10453SEric Joyner if (li->phy_type_low) 1752*71d10453SEric Joyner ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low); 1753*71d10453SEric Joyner else if (li->phy_type_high) 1754*71d10453SEric Joyner ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high); 1755*71d10453SEric Joyner else 1756*71d10453SEric Joyner ifmr->ifm_active |= IFM_UNKNOWN; 1757*71d10453SEric Joyner 1758*71d10453SEric Joyner /* Report flow control status as well */ 1759*71d10453SEric Joyner if (li->an_info & ICE_AQ_LINK_PAUSE_TX) 1760*71d10453SEric Joyner ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1761*71d10453SEric Joyner if (li->an_info & ICE_AQ_LINK_PAUSE_RX) 1762*71d10453SEric Joyner ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1763*71d10453SEric Joyner } 1764*71d10453SEric Joyner 1765*71d10453SEric Joyner /** 1766*71d10453SEric Joyner * ice_init_tx_tracking - Initialize Tx queue software tracking values 1767*71d10453SEric Joyner * @vsi: the VSI to initialize 1768*71d10453SEric Joyner * 1769*71d10453SEric Joyner * Initialize Tx queue software tracking values, including the Report Status 1770*71d10453SEric Joyner * queue, and related software tracking values. 1771*71d10453SEric Joyner */ 1772*71d10453SEric Joyner static void 1773*71d10453SEric Joyner ice_init_tx_tracking(struct ice_vsi *vsi) 1774*71d10453SEric Joyner { 1775*71d10453SEric Joyner struct ice_tx_queue *txq; 1776*71d10453SEric Joyner size_t j; 1777*71d10453SEric Joyner int i; 1778*71d10453SEric Joyner 1779*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1780*71d10453SEric Joyner 1781*71d10453SEric Joyner txq->tx_rs_cidx = txq->tx_rs_pidx = 0; 1782*71d10453SEric Joyner 1783*71d10453SEric Joyner /* Initialize the last processed descriptor to be the end of 1784*71d10453SEric Joyner * the ring, rather than the start, so that we avoid an 1785*71d10453SEric Joyner * off-by-one error in ice_ift_txd_credits_update for the 1786*71d10453SEric Joyner * first packet. 1787*71d10453SEric Joyner */ 1788*71d10453SEric Joyner txq->tx_cidx_processed = txq->desc_count - 1; 1789*71d10453SEric Joyner 1790*71d10453SEric Joyner for (j = 0; j < txq->desc_count; j++) 1791*71d10453SEric Joyner txq->tx_rsq[j] = QIDX_INVALID; 1792*71d10453SEric Joyner } 1793*71d10453SEric Joyner } 1794*71d10453SEric Joyner 1795*71d10453SEric Joyner /** 1796*71d10453SEric Joyner * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues 1797*71d10453SEric Joyner * @sc: the device softc 1798*71d10453SEric Joyner * 1799*71d10453SEric Joyner * Called to update the Rx queue mbuf_sz parameter for configuring the receive 1800*71d10453SEric Joyner * buffer sizes when programming hardware. 1801*71d10453SEric Joyner */ 1802*71d10453SEric Joyner static void 1803*71d10453SEric Joyner ice_update_rx_mbuf_sz(struct ice_softc *sc) 1804*71d10453SEric Joyner { 1805*71d10453SEric Joyner uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx); 1806*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 1807*71d10453SEric Joyner 1808*71d10453SEric Joyner MPASS(mbuf_sz <= UINT16_MAX); 1809*71d10453SEric Joyner vsi->mbuf_sz = mbuf_sz; 1810*71d10453SEric Joyner } 1811*71d10453SEric Joyner 1812*71d10453SEric Joyner /** 1813*71d10453SEric Joyner * ice_if_init - Initialize the device 1814*71d10453SEric Joyner * @ctx: iflib ctx structure 1815*71d10453SEric Joyner * 1816*71d10453SEric Joyner * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes 1817*71d10453SEric Joyner * device filters and prepares the Tx and Rx engines. 1818*71d10453SEric Joyner * 1819*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 1820*71d10453SEric Joyner */ 1821*71d10453SEric Joyner static void 1822*71d10453SEric Joyner ice_if_init(if_ctx_t ctx) 1823*71d10453SEric Joyner { 1824*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1825*71d10453SEric Joyner device_t dev = sc->dev; 1826*71d10453SEric Joyner int err; 1827*71d10453SEric Joyner 1828*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 1829*71d10453SEric Joyner 1830*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1831*71d10453SEric Joyner return; 1832*71d10453SEric Joyner 1833*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 1834*71d10453SEric Joyner device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n"); 1835*71d10453SEric Joyner return; 1836*71d10453SEric Joyner } 1837*71d10453SEric Joyner 1838*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 1839*71d10453SEric Joyner device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n"); 1840*71d10453SEric Joyner return; 1841*71d10453SEric Joyner } 1842*71d10453SEric Joyner 1843*71d10453SEric Joyner ice_update_rx_mbuf_sz(sc); 1844*71d10453SEric Joyner 1845*71d10453SEric Joyner /* Update the MAC address... User might use a LAA */ 1846*71d10453SEric Joyner err = ice_update_laa_mac(sc); 1847*71d10453SEric Joyner if (err) { 1848*71d10453SEric Joyner device_printf(dev, 1849*71d10453SEric Joyner "LAA address change failed, err %s\n", 1850*71d10453SEric Joyner ice_err_str(err)); 1851*71d10453SEric Joyner return; 1852*71d10453SEric Joyner } 1853*71d10453SEric Joyner 1854*71d10453SEric Joyner /* Initialize software Tx tracking values */ 1855*71d10453SEric Joyner ice_init_tx_tracking(&sc->pf_vsi); 1856*71d10453SEric Joyner 1857*71d10453SEric Joyner err = ice_cfg_vsi_for_tx(&sc->pf_vsi); 1858*71d10453SEric Joyner if (err) { 1859*71d10453SEric Joyner device_printf(dev, 1860*71d10453SEric Joyner "Unable to configure the main VSI for Tx: %s\n", 1861*71d10453SEric Joyner ice_err_str(err)); 1862*71d10453SEric Joyner return; 1863*71d10453SEric Joyner } 1864*71d10453SEric Joyner 1865*71d10453SEric Joyner err = ice_cfg_vsi_for_rx(&sc->pf_vsi); 1866*71d10453SEric Joyner if (err) { 1867*71d10453SEric Joyner device_printf(dev, 1868*71d10453SEric Joyner "Unable to configure the main VSI for Rx: %s\n", 1869*71d10453SEric Joyner ice_err_str(err)); 1870*71d10453SEric Joyner goto err_cleanup_tx; 1871*71d10453SEric Joyner } 1872*71d10453SEric Joyner 1873*71d10453SEric Joyner err = ice_control_rx_queues(&sc->pf_vsi, true); 1874*71d10453SEric Joyner if (err) { 1875*71d10453SEric Joyner device_printf(dev, 1876*71d10453SEric Joyner "Unable to enable Rx rings for transmit: %s\n", 1877*71d10453SEric Joyner ice_err_str(err)); 1878*71d10453SEric Joyner goto err_cleanup_tx; 1879*71d10453SEric Joyner } 1880*71d10453SEric Joyner 1881*71d10453SEric Joyner err = ice_cfg_pf_default_mac_filters(sc); 1882*71d10453SEric Joyner if (err) { 1883*71d10453SEric Joyner device_printf(dev, 1884*71d10453SEric Joyner "Unable to configure default MAC filters: %s\n", 1885*71d10453SEric Joyner ice_err_str(err)); 1886*71d10453SEric Joyner goto err_stop_rx; 1887*71d10453SEric Joyner } 1888*71d10453SEric Joyner 1889*71d10453SEric Joyner /* We use software interrupts for Tx, so we only program the hardware 1890*71d10453SEric Joyner * interrupts for Rx. 1891*71d10453SEric Joyner */ 1892*71d10453SEric Joyner ice_configure_rxq_interrupts(&sc->pf_vsi); 1893*71d10453SEric Joyner ice_configure_rx_itr(&sc->pf_vsi); 1894*71d10453SEric Joyner 1895*71d10453SEric Joyner /* Configure promiscuous mode */ 1896*71d10453SEric Joyner ice_if_promisc_set(ctx, if_getflags(sc->ifp)); 1897*71d10453SEric Joyner 1898*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED); 1899*71d10453SEric Joyner return; 1900*71d10453SEric Joyner 1901*71d10453SEric Joyner err_stop_rx: 1902*71d10453SEric Joyner ice_control_rx_queues(&sc->pf_vsi, false); 1903*71d10453SEric Joyner err_cleanup_tx: 1904*71d10453SEric Joyner ice_vsi_disable_tx(&sc->pf_vsi); 1905*71d10453SEric Joyner } 1906*71d10453SEric Joyner 1907*71d10453SEric Joyner /** 1908*71d10453SEric Joyner * ice_poll_for_media_avail - Re-enable link if media is detected 1909*71d10453SEric Joyner * @sc: device private structure 1910*71d10453SEric Joyner * 1911*71d10453SEric Joyner * Intended to be called from the driver's timer function, this function 1912*71d10453SEric Joyner * sends the Get Link Status AQ command and re-enables HW link if the 1913*71d10453SEric Joyner * command says that media is available. 1914*71d10453SEric Joyner * 1915*71d10453SEric Joyner * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing, 1916*71d10453SEric Joyner * since media removal events are supposed to be sent to the driver through 1917*71d10453SEric Joyner * a link status event. 1918*71d10453SEric Joyner */ 1919*71d10453SEric Joyner static void 1920*71d10453SEric Joyner ice_poll_for_media_avail(struct ice_softc *sc) 1921*71d10453SEric Joyner { 1922*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 1923*71d10453SEric Joyner struct ice_port_info *pi = hw->port_info; 1924*71d10453SEric Joyner 1925*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) { 1926*71d10453SEric Joyner pi->phy.get_link_info = true; 1927*71d10453SEric Joyner ice_get_link_status(pi, &sc->link_up); 1928*71d10453SEric Joyner 1929*71d10453SEric Joyner if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 1930*71d10453SEric Joyner enum ice_status status; 1931*71d10453SEric Joyner 1932*71d10453SEric Joyner /* Re-enable link and re-apply user link settings */ 1933*71d10453SEric Joyner ice_apply_saved_phy_cfg(sc); 1934*71d10453SEric Joyner 1935*71d10453SEric Joyner /* Update the OS about changes in media capability */ 1936*71d10453SEric Joyner status = ice_add_media_types(sc, sc->media); 1937*71d10453SEric Joyner if (status) 1938*71d10453SEric Joyner device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 1939*71d10453SEric Joyner ice_status_str(status), 1940*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 1941*71d10453SEric Joyner 1942*71d10453SEric Joyner ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA); 1943*71d10453SEric Joyner } 1944*71d10453SEric Joyner } 1945*71d10453SEric Joyner } 1946*71d10453SEric Joyner 1947*71d10453SEric Joyner /** 1948*71d10453SEric Joyner * ice_if_timer - called by iflib periodically 1949*71d10453SEric Joyner * @ctx: iflib ctx structure 1950*71d10453SEric Joyner * @qid: the queue this timer was called for 1951*71d10453SEric Joyner * 1952*71d10453SEric Joyner * This callback is triggered by iflib periodically. We use it to update the 1953*71d10453SEric Joyner * hw statistics. 1954*71d10453SEric Joyner * 1955*71d10453SEric Joyner * @remark this function is not protected by the iflib CTX lock. 1956*71d10453SEric Joyner */ 1957*71d10453SEric Joyner static void 1958*71d10453SEric Joyner ice_if_timer(if_ctx_t ctx, uint16_t qid) 1959*71d10453SEric Joyner { 1960*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1961*71d10453SEric Joyner uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx; 1962*71d10453SEric Joyner 1963*71d10453SEric Joyner if (qid != 0) 1964*71d10453SEric Joyner return; 1965*71d10453SEric Joyner 1966*71d10453SEric Joyner /* Do not attempt to update stats when in recovery mode */ 1967*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1968*71d10453SEric Joyner return; 1969*71d10453SEric Joyner 1970*71d10453SEric Joyner /* Update device statistics */ 1971*71d10453SEric Joyner ice_update_pf_stats(sc); 1972*71d10453SEric Joyner 1973*71d10453SEric Joyner /* 1974*71d10453SEric Joyner * For proper watchdog management, the iflib stack needs to know if 1975*71d10453SEric Joyner * we've been paused during the last interval. Check if the 1976*71d10453SEric Joyner * link_xoff_rx stat changed, and set the isc_pause_frames, if so. 1977*71d10453SEric Joyner */ 1978*71d10453SEric Joyner if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx) 1979*71d10453SEric Joyner sc->scctx->isc_pause_frames = 1; 1980*71d10453SEric Joyner 1981*71d10453SEric Joyner /* Update the primary VSI stats */ 1982*71d10453SEric Joyner ice_update_vsi_hw_stats(&sc->pf_vsi); 1983*71d10453SEric Joyner } 1984*71d10453SEric Joyner 1985*71d10453SEric Joyner /** 1986*71d10453SEric Joyner * ice_admin_timer - called periodically to trigger the admin task 1987*71d10453SEric Joyner * @arg: callout(9) argument pointing to the device private softc structure 1988*71d10453SEric Joyner * 1989*71d10453SEric Joyner * Timer function used as part of a callout(9) timer that will periodically 1990*71d10453SEric Joyner * trigger the admin task, even when the interface is down. 1991*71d10453SEric Joyner * 1992*71d10453SEric Joyner * @remark this function is not called by iflib and is not protected by the 1993*71d10453SEric Joyner * iflib CTX lock. 1994*71d10453SEric Joyner * 1995*71d10453SEric Joyner * @remark because this is a callout function, it cannot sleep and should not 1996*71d10453SEric Joyner * attempt taking the iflib CTX lock. 1997*71d10453SEric Joyner */ 1998*71d10453SEric Joyner static void 1999*71d10453SEric Joyner ice_admin_timer(void *arg) 2000*71d10453SEric Joyner { 2001*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)arg; 2002*71d10453SEric Joyner 2003*71d10453SEric Joyner /* Fire off the admin task */ 2004*71d10453SEric Joyner iflib_admin_intr_deferred(sc->ctx); 2005*71d10453SEric Joyner 2006*71d10453SEric Joyner /* Reschedule the admin timer */ 2007*71d10453SEric Joyner callout_schedule(&sc->admin_timer, hz/2); 2008*71d10453SEric Joyner } 2009*71d10453SEric Joyner 2010*71d10453SEric Joyner /** 2011*71d10453SEric Joyner * ice_transition_recovery_mode - Transition to recovery mode 2012*71d10453SEric Joyner * @sc: the device private softc 2013*71d10453SEric Joyner * 2014*71d10453SEric Joyner * Called when the driver detects that the firmware has entered recovery mode 2015*71d10453SEric Joyner * at run time. 2016*71d10453SEric Joyner */ 2017*71d10453SEric Joyner static void 2018*71d10453SEric Joyner ice_transition_recovery_mode(struct ice_softc *sc) 2019*71d10453SEric Joyner { 2020*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 2021*71d10453SEric Joyner int i; 2022*71d10453SEric Joyner 2023*71d10453SEric Joyner device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 2024*71d10453SEric Joyner 2025*71d10453SEric Joyner /* Tell the stack that the link has gone down */ 2026*71d10453SEric Joyner iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 2027*71d10453SEric Joyner 2028*71d10453SEric Joyner /* Request that the device be re-initialized */ 2029*71d10453SEric Joyner ice_request_stack_reinit(sc); 2030*71d10453SEric Joyner 2031*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2032*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2033*71d10453SEric Joyner 2034*71d10453SEric Joyner ice_vsi_del_txqs_ctx(vsi); 2035*71d10453SEric Joyner ice_vsi_del_rxqs_ctx(vsi); 2036*71d10453SEric Joyner 2037*71d10453SEric Joyner for (i = 0; i < sc->num_available_vsi; i++) { 2038*71d10453SEric Joyner if (sc->all_vsi[i]) 2039*71d10453SEric Joyner ice_release_vsi(sc->all_vsi[i]); 2040*71d10453SEric Joyner } 2041*71d10453SEric Joyner sc->num_available_vsi = 0; 2042*71d10453SEric Joyner 2043*71d10453SEric Joyner if (sc->all_vsi) { 2044*71d10453SEric Joyner free(sc->all_vsi, M_ICE); 2045*71d10453SEric Joyner sc->all_vsi = NULL; 2046*71d10453SEric Joyner } 2047*71d10453SEric Joyner 2048*71d10453SEric Joyner /* Destroy the interrupt manager */ 2049*71d10453SEric Joyner ice_resmgr_destroy(&sc->imgr); 2050*71d10453SEric Joyner /* Destroy the queue managers */ 2051*71d10453SEric Joyner ice_resmgr_destroy(&sc->tx_qmgr); 2052*71d10453SEric Joyner ice_resmgr_destroy(&sc->rx_qmgr); 2053*71d10453SEric Joyner 2054*71d10453SEric Joyner ice_deinit_hw(&sc->hw); 2055*71d10453SEric Joyner } 2056*71d10453SEric Joyner 2057*71d10453SEric Joyner /** 2058*71d10453SEric Joyner * ice_transition_safe_mode - Transition to safe mode 2059*71d10453SEric Joyner * @sc: the device private softc 2060*71d10453SEric Joyner * 2061*71d10453SEric Joyner * Called when the driver attempts to reload the DDP package during a device 2062*71d10453SEric Joyner * reset, and the new download fails. If so, we must transition to safe mode 2063*71d10453SEric Joyner * at run time. 2064*71d10453SEric Joyner * 2065*71d10453SEric Joyner * @remark although safe mode normally allocates only a single queue, we can't 2066*71d10453SEric Joyner * change the number of queues dynamically when using iflib. Due to this, we 2067*71d10453SEric Joyner * do not attempt to reduce the number of queues. 2068*71d10453SEric Joyner */ 2069*71d10453SEric Joyner static void 2070*71d10453SEric Joyner ice_transition_safe_mode(struct ice_softc *sc) 2071*71d10453SEric Joyner { 2072*71d10453SEric Joyner /* Indicate that we are in Safe mode */ 2073*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap); 2074*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en); 2075*71d10453SEric Joyner 2076*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2077*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2078*71d10453SEric Joyner 2079*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2080*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en); 2081*71d10453SEric Joyner } 2082*71d10453SEric Joyner 2083*71d10453SEric Joyner /** 2084*71d10453SEric Joyner * ice_if_update_admin_status - update admin status 2085*71d10453SEric Joyner * @ctx: iflib ctx structure 2086*71d10453SEric Joyner * 2087*71d10453SEric Joyner * Called by iflib to update the admin status. For our purposes, this means 2088*71d10453SEric Joyner * check the adminq, and update the link status. It's ultimately triggered by 2089*71d10453SEric Joyner * our admin interrupt, or by the ice_if_timer periodically. 2090*71d10453SEric Joyner * 2091*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 2092*71d10453SEric Joyner */ 2093*71d10453SEric Joyner static void 2094*71d10453SEric Joyner ice_if_update_admin_status(if_ctx_t ctx) 2095*71d10453SEric Joyner { 2096*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2097*71d10453SEric Joyner enum ice_fw_modes fw_mode; 2098*71d10453SEric Joyner bool reschedule = false; 2099*71d10453SEric Joyner u16 pending = 0; 2100*71d10453SEric Joyner 2101*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2102*71d10453SEric Joyner 2103*71d10453SEric Joyner /* Check if the firmware entered recovery mode at run time */ 2104*71d10453SEric Joyner fw_mode = ice_get_fw_mode(&sc->hw); 2105*71d10453SEric Joyner if (fw_mode == ICE_FW_MODE_REC) { 2106*71d10453SEric Joyner if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2107*71d10453SEric Joyner /* If we just entered recovery mode, log a warning to 2108*71d10453SEric Joyner * the system administrator and deinit driver state 2109*71d10453SEric Joyner * that is no longer functional. 2110*71d10453SEric Joyner */ 2111*71d10453SEric Joyner ice_transition_recovery_mode(sc); 2112*71d10453SEric Joyner } 2113*71d10453SEric Joyner } else if (fw_mode == ICE_FW_MODE_ROLLBACK) { 2114*71d10453SEric Joyner if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) { 2115*71d10453SEric Joyner /* Rollback mode isn't fatal, but we don't want to 2116*71d10453SEric Joyner * repeatedly post a message about it. 2117*71d10453SEric Joyner */ 2118*71d10453SEric Joyner ice_print_rollback_msg(&sc->hw); 2119*71d10453SEric Joyner } 2120*71d10453SEric Joyner } 2121*71d10453SEric Joyner 2122*71d10453SEric Joyner /* Handle global reset events */ 2123*71d10453SEric Joyner ice_handle_reset_event(sc); 2124*71d10453SEric Joyner 2125*71d10453SEric Joyner /* Handle PF reset requests */ 2126*71d10453SEric Joyner ice_handle_pf_reset_request(sc); 2127*71d10453SEric Joyner 2128*71d10453SEric Joyner /* Handle MDD events */ 2129*71d10453SEric Joyner ice_handle_mdd_event(sc); 2130*71d10453SEric Joyner 2131*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) || 2132*71d10453SEric Joyner ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) || 2133*71d10453SEric Joyner ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2134*71d10453SEric Joyner /* 2135*71d10453SEric Joyner * If we know the control queues are disabled, skip processing 2136*71d10453SEric Joyner * the control queues entirely. 2137*71d10453SEric Joyner */ 2138*71d10453SEric Joyner ; 2139*71d10453SEric Joyner } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) { 2140*71d10453SEric Joyner ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending); 2141*71d10453SEric Joyner if (pending > 0) 2142*71d10453SEric Joyner reschedule = true; 2143*71d10453SEric Joyner 2144*71d10453SEric Joyner ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending); 2145*71d10453SEric Joyner if (pending > 0) 2146*71d10453SEric Joyner reschedule = true; 2147*71d10453SEric Joyner } 2148*71d10453SEric Joyner 2149*71d10453SEric Joyner /* Poll for link up */ 2150*71d10453SEric Joyner ice_poll_for_media_avail(sc); 2151*71d10453SEric Joyner 2152*71d10453SEric Joyner /* Check and update link status */ 2153*71d10453SEric Joyner ice_update_link_status(sc, false); 2154*71d10453SEric Joyner 2155*71d10453SEric Joyner /* 2156*71d10453SEric Joyner * If there are still messages to process, we need to reschedule 2157*71d10453SEric Joyner * ourselves. Otherwise, we can just re-enable the interrupt. We'll be 2158*71d10453SEric Joyner * woken up at the next interrupt or timer event. 2159*71d10453SEric Joyner */ 2160*71d10453SEric Joyner if (reschedule) { 2161*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 2162*71d10453SEric Joyner iflib_admin_intr_deferred(ctx); 2163*71d10453SEric Joyner } else { 2164*71d10453SEric Joyner ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2165*71d10453SEric Joyner } 2166*71d10453SEric Joyner } 2167*71d10453SEric Joyner 2168*71d10453SEric Joyner /** 2169*71d10453SEric Joyner * ice_prepare_for_reset - Prepare device for an impending reset 2170*71d10453SEric Joyner * @sc: The device private softc 2171*71d10453SEric Joyner * 2172*71d10453SEric Joyner * Prepare the driver for an impending reset, shutting down VSIs, clearing the 2173*71d10453SEric Joyner * scheduler setup, and shutting down controlqs. Uses the 2174*71d10453SEric Joyner * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the 2175*71d10453SEric Joyner * driver for reset or not. 2176*71d10453SEric Joyner */ 2177*71d10453SEric Joyner static void 2178*71d10453SEric Joyner ice_prepare_for_reset(struct ice_softc *sc) 2179*71d10453SEric Joyner { 2180*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 2181*71d10453SEric Joyner 2182*71d10453SEric Joyner /* If we're already prepared, there's nothing to do */ 2183*71d10453SEric Joyner if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) 2184*71d10453SEric Joyner return; 2185*71d10453SEric Joyner 2186*71d10453SEric Joyner log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname); 2187*71d10453SEric Joyner 2188*71d10453SEric Joyner /* In recovery mode, hardware is not initialized */ 2189*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2190*71d10453SEric Joyner return; 2191*71d10453SEric Joyner 2192*71d10453SEric Joyner /* Release the main PF VSI queue mappings */ 2193*71d10453SEric Joyner ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2194*71d10453SEric Joyner sc->pf_vsi.num_tx_queues); 2195*71d10453SEric Joyner ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2196*71d10453SEric Joyner sc->pf_vsi.num_rx_queues); 2197*71d10453SEric Joyner 2198*71d10453SEric Joyner ice_clear_hw_tbls(hw); 2199*71d10453SEric Joyner 2200*71d10453SEric Joyner if (hw->port_info) 2201*71d10453SEric Joyner ice_sched_clear_port(hw->port_info); 2202*71d10453SEric Joyner 2203*71d10453SEric Joyner ice_shutdown_all_ctrlq(hw); 2204*71d10453SEric Joyner } 2205*71d10453SEric Joyner 2206*71d10453SEric Joyner /** 2207*71d10453SEric Joyner * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping 2208*71d10453SEric Joyner * @sc: the device softc pointer 2209*71d10453SEric Joyner * 2210*71d10453SEric Joyner * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue 2211*71d10453SEric Joyner * mapping after a reset occurred. 2212*71d10453SEric Joyner */ 2213*71d10453SEric Joyner static int 2214*71d10453SEric Joyner ice_rebuild_pf_vsi_qmap(struct ice_softc *sc) 2215*71d10453SEric Joyner { 2216*71d10453SEric Joyner struct ice_vsi *vsi = &sc->pf_vsi; 2217*71d10453SEric Joyner struct ice_tx_queue *txq; 2218*71d10453SEric Joyner struct ice_rx_queue *rxq; 2219*71d10453SEric Joyner int err, i; 2220*71d10453SEric Joyner 2221*71d10453SEric Joyner /* Re-assign Tx queues from PF space to the main VSI */ 2222*71d10453SEric Joyner err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, 2223*71d10453SEric Joyner vsi->num_tx_queues); 2224*71d10453SEric Joyner if (err) { 2225*71d10453SEric Joyner device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n", 2226*71d10453SEric Joyner ice_err_str(err)); 2227*71d10453SEric Joyner return (err); 2228*71d10453SEric Joyner } 2229*71d10453SEric Joyner 2230*71d10453SEric Joyner /* Re-assign Rx queues from PF space to this VSI */ 2231*71d10453SEric Joyner err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, 2232*71d10453SEric Joyner vsi->num_rx_queues); 2233*71d10453SEric Joyner if (err) { 2234*71d10453SEric Joyner device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n", 2235*71d10453SEric Joyner ice_err_str(err)); 2236*71d10453SEric Joyner goto err_release_tx_queues; 2237*71d10453SEric Joyner } 2238*71d10453SEric Joyner 2239*71d10453SEric Joyner vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 2240*71d10453SEric Joyner 2241*71d10453SEric Joyner /* Re-assign Tx queue tail pointers */ 2242*71d10453SEric Joyner for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 2243*71d10453SEric Joyner txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 2244*71d10453SEric Joyner 2245*71d10453SEric Joyner /* Re-assign Rx queue tail pointers */ 2246*71d10453SEric Joyner for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 2247*71d10453SEric Joyner rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 2248*71d10453SEric Joyner 2249*71d10453SEric Joyner return (0); 2250*71d10453SEric Joyner 2251*71d10453SEric Joyner err_release_tx_queues: 2252*71d10453SEric Joyner ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2253*71d10453SEric Joyner sc->pf_vsi.num_tx_queues); 2254*71d10453SEric Joyner 2255*71d10453SEric Joyner return (err); 2256*71d10453SEric Joyner } 2257*71d10453SEric Joyner 2258*71d10453SEric Joyner /* determine if the iflib context is active */ 2259*71d10453SEric Joyner #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING)) 2260*71d10453SEric Joyner 2261*71d10453SEric Joyner /** 2262*71d10453SEric Joyner * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode 2263*71d10453SEric Joyner * @sc: The device private softc 2264*71d10453SEric Joyner * 2265*71d10453SEric Joyner * Handle a driver rebuild while in recovery mode. This will only rebuild the 2266*71d10453SEric Joyner * limited functionality supported while in recovery mode. 2267*71d10453SEric Joyner */ 2268*71d10453SEric Joyner static void 2269*71d10453SEric Joyner ice_rebuild_recovery_mode(struct ice_softc *sc) 2270*71d10453SEric Joyner { 2271*71d10453SEric Joyner device_t dev = sc->dev; 2272*71d10453SEric Joyner 2273*71d10453SEric Joyner /* enable PCIe bus master */ 2274*71d10453SEric Joyner pci_enable_busmaster(dev); 2275*71d10453SEric Joyner 2276*71d10453SEric Joyner /* Configure interrupt causes for the administrative interrupt */ 2277*71d10453SEric Joyner ice_configure_misc_interrupts(sc); 2278*71d10453SEric Joyner 2279*71d10453SEric Joyner /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2280*71d10453SEric Joyner ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2281*71d10453SEric Joyner 2282*71d10453SEric Joyner /* Now that the rebuild is finished, we're no longer prepared to reset */ 2283*71d10453SEric Joyner ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2284*71d10453SEric Joyner 2285*71d10453SEric Joyner log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2286*71d10453SEric Joyner 2287*71d10453SEric Joyner /* In order to completely restore device functionality, the iflib core 2288*71d10453SEric Joyner * needs to be reset. We need to request an iflib reset. Additionally, 2289*71d10453SEric Joyner * because the state of IFC_DO_RESET is cached within task_fn_admin in 2290*71d10453SEric Joyner * the iflib core, we also want re-run the admin task so that iflib 2291*71d10453SEric Joyner * resets immediately instead of waiting for the next interrupt. 2292*71d10453SEric Joyner */ 2293*71d10453SEric Joyner ice_request_stack_reinit(sc); 2294*71d10453SEric Joyner 2295*71d10453SEric Joyner return; 2296*71d10453SEric Joyner } 2297*71d10453SEric Joyner 2298*71d10453SEric Joyner /** 2299*71d10453SEric Joyner * ice_rebuild - Rebuild driver state post reset 2300*71d10453SEric Joyner * @sc: The device private softc 2301*71d10453SEric Joyner * 2302*71d10453SEric Joyner * Restore driver state after a reset occurred. Restart the controlqs, setup 2303*71d10453SEric Joyner * the hardware port, and re-enable the VSIs. 2304*71d10453SEric Joyner */ 2305*71d10453SEric Joyner static void 2306*71d10453SEric Joyner ice_rebuild(struct ice_softc *sc) 2307*71d10453SEric Joyner { 2308*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 2309*71d10453SEric Joyner device_t dev = sc->dev; 2310*71d10453SEric Joyner enum ice_status status; 2311*71d10453SEric Joyner int err; 2312*71d10453SEric Joyner 2313*71d10453SEric Joyner sc->rebuild_ticks = ticks; 2314*71d10453SEric Joyner 2315*71d10453SEric Joyner /* If we're rebuilding, then a reset has succeeded. */ 2316*71d10453SEric Joyner ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED); 2317*71d10453SEric Joyner 2318*71d10453SEric Joyner /* 2319*71d10453SEric Joyner * If the firmware is in recovery mode, only restore the limited 2320*71d10453SEric Joyner * functionality supported by recovery mode. 2321*71d10453SEric Joyner */ 2322*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2323*71d10453SEric Joyner ice_rebuild_recovery_mode(sc); 2324*71d10453SEric Joyner return; 2325*71d10453SEric Joyner } 2326*71d10453SEric Joyner 2327*71d10453SEric Joyner /* enable PCIe bus master */ 2328*71d10453SEric Joyner pci_enable_busmaster(dev); 2329*71d10453SEric Joyner 2330*71d10453SEric Joyner status = ice_init_all_ctrlq(hw); 2331*71d10453SEric Joyner if (status) { 2332*71d10453SEric Joyner device_printf(dev, "failed to re-init controlqs, err %s\n", 2333*71d10453SEric Joyner ice_status_str(status)); 2334*71d10453SEric Joyner goto err_shutdown_ctrlq; 2335*71d10453SEric Joyner } 2336*71d10453SEric Joyner 2337*71d10453SEric Joyner /* Query the allocated resources for Tx scheduler */ 2338*71d10453SEric Joyner status = ice_sched_query_res_alloc(hw); 2339*71d10453SEric Joyner if (status) { 2340*71d10453SEric Joyner device_printf(dev, 2341*71d10453SEric Joyner "Failed to query scheduler resources, err %s aq_err %s\n", 2342*71d10453SEric Joyner ice_status_str(status), 2343*71d10453SEric Joyner ice_aq_str(hw->adminq.sq_last_status)); 2344*71d10453SEric Joyner goto err_shutdown_ctrlq; 2345*71d10453SEric Joyner } 2346*71d10453SEric Joyner 2347*71d10453SEric Joyner err = ice_send_version(sc); 2348*71d10453SEric Joyner if (err) 2349*71d10453SEric Joyner goto err_shutdown_ctrlq; 2350*71d10453SEric Joyner 2351*71d10453SEric Joyner err = ice_init_link_events(sc); 2352*71d10453SEric Joyner if (err) { 2353*71d10453SEric Joyner device_printf(dev, "ice_init_link_events failed: %s\n", 2354*71d10453SEric Joyner ice_err_str(err)); 2355*71d10453SEric Joyner goto err_shutdown_ctrlq; 2356*71d10453SEric Joyner } 2357*71d10453SEric Joyner 2358*71d10453SEric Joyner status = ice_clear_pf_cfg(hw); 2359*71d10453SEric Joyner if (status) { 2360*71d10453SEric Joyner device_printf(dev, "failed to clear PF configuration, err %s\n", 2361*71d10453SEric Joyner ice_status_str(status)); 2362*71d10453SEric Joyner goto err_shutdown_ctrlq; 2363*71d10453SEric Joyner } 2364*71d10453SEric Joyner 2365*71d10453SEric Joyner ice_clear_pxe_mode(hw); 2366*71d10453SEric Joyner 2367*71d10453SEric Joyner status = ice_get_caps(hw); 2368*71d10453SEric Joyner if (status) { 2369*71d10453SEric Joyner device_printf(dev, "failed to get capabilities, err %s\n", 2370*71d10453SEric Joyner ice_status_str(status)); 2371*71d10453SEric Joyner goto err_shutdown_ctrlq; 2372*71d10453SEric Joyner } 2373*71d10453SEric Joyner 2374*71d10453SEric Joyner status = ice_sched_init_port(hw->port_info); 2375*71d10453SEric Joyner if (status) { 2376*71d10453SEric Joyner device_printf(dev, "failed to initialize port, err %s\n", 2377*71d10453SEric Joyner ice_status_str(status)); 2378*71d10453SEric Joyner goto err_sched_cleanup; 2379*71d10453SEric Joyner } 2380*71d10453SEric Joyner 2381*71d10453SEric Joyner /* If we previously loaded the package, it needs to be reloaded now */ 2382*71d10453SEric Joyner if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) { 2383*71d10453SEric Joyner status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 2384*71d10453SEric Joyner if (status) { 2385*71d10453SEric Joyner ice_log_pkg_init(sc, &status); 2386*71d10453SEric Joyner 2387*71d10453SEric Joyner ice_transition_safe_mode(sc); 2388*71d10453SEric Joyner } 2389*71d10453SEric Joyner } 2390*71d10453SEric Joyner 2391*71d10453SEric Joyner ice_reset_pf_stats(sc); 2392*71d10453SEric Joyner 2393*71d10453SEric Joyner err = ice_rebuild_pf_vsi_qmap(sc); 2394*71d10453SEric Joyner if (err) { 2395*71d10453SEric Joyner device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n", 2396*71d10453SEric Joyner ice_err_str(err)); 2397*71d10453SEric Joyner goto err_sched_cleanup; 2398*71d10453SEric Joyner } 2399*71d10453SEric Joyner err = ice_initialize_vsi(&sc->pf_vsi); 2400*71d10453SEric Joyner if (err) { 2401*71d10453SEric Joyner device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n", 2402*71d10453SEric Joyner ice_err_str(err)); 2403*71d10453SEric Joyner goto err_release_queue_allocations; 2404*71d10453SEric Joyner } 2405*71d10453SEric Joyner 2406*71d10453SEric Joyner /* Replay all VSI configuration */ 2407*71d10453SEric Joyner err = ice_replay_all_vsi_cfg(sc); 2408*71d10453SEric Joyner if (err) 2409*71d10453SEric Joyner goto err_deinit_pf_vsi; 2410*71d10453SEric Joyner 2411*71d10453SEric Joyner /* Reconfigure the main PF VSI for RSS */ 2412*71d10453SEric Joyner err = ice_config_rss(&sc->pf_vsi); 2413*71d10453SEric Joyner if (err) { 2414*71d10453SEric Joyner device_printf(sc->dev, 2415*71d10453SEric Joyner "Unable to reconfigure RSS for the main VSI, err %s\n", 2416*71d10453SEric Joyner ice_err_str(err)); 2417*71d10453SEric Joyner goto err_deinit_pf_vsi; 2418*71d10453SEric Joyner } 2419*71d10453SEric Joyner 2420*71d10453SEric Joyner /* Refresh link status */ 2421*71d10453SEric Joyner ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED); 2422*71d10453SEric Joyner sc->hw.port_info->phy.get_link_info = true; 2423*71d10453SEric Joyner ice_get_link_status(sc->hw.port_info, &sc->link_up); 2424*71d10453SEric Joyner ice_update_link_status(sc, true); 2425*71d10453SEric Joyner 2426*71d10453SEric Joyner /* Configure interrupt causes for the administrative interrupt */ 2427*71d10453SEric Joyner ice_configure_misc_interrupts(sc); 2428*71d10453SEric Joyner 2429*71d10453SEric Joyner /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2430*71d10453SEric Joyner ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2431*71d10453SEric Joyner 2432*71d10453SEric Joyner /* Now that the rebuild is finished, we're no longer prepared to reset */ 2433*71d10453SEric Joyner ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2434*71d10453SEric Joyner 2435*71d10453SEric Joyner log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2436*71d10453SEric Joyner 2437*71d10453SEric Joyner /* In order to completely restore device functionality, the iflib core 2438*71d10453SEric Joyner * needs to be reset. We need to request an iflib reset. Additionally, 2439*71d10453SEric Joyner * because the state of IFC_DO_RESET is cached within task_fn_admin in 2440*71d10453SEric Joyner * the iflib core, we also want re-run the admin task so that iflib 2441*71d10453SEric Joyner * resets immediately instead of waiting for the next interrupt. 2442*71d10453SEric Joyner */ 2443*71d10453SEric Joyner ice_request_stack_reinit(sc); 2444*71d10453SEric Joyner 2445*71d10453SEric Joyner return; 2446*71d10453SEric Joyner 2447*71d10453SEric Joyner err_deinit_pf_vsi: 2448*71d10453SEric Joyner ice_deinit_vsi(&sc->pf_vsi); 2449*71d10453SEric Joyner err_release_queue_allocations: 2450*71d10453SEric Joyner ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2451*71d10453SEric Joyner sc->pf_vsi.num_tx_queues); 2452*71d10453SEric Joyner ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2453*71d10453SEric Joyner sc->pf_vsi.num_rx_queues); 2454*71d10453SEric Joyner err_sched_cleanup: 2455*71d10453SEric Joyner ice_sched_cleanup_all(hw); 2456*71d10453SEric Joyner err_shutdown_ctrlq: 2457*71d10453SEric Joyner ice_shutdown_all_ctrlq(hw); 2458*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2459*71d10453SEric Joyner device_printf(dev, "Driver rebuild failed, please reload the device driver\n"); 2460*71d10453SEric Joyner } 2461*71d10453SEric Joyner 2462*71d10453SEric Joyner /** 2463*71d10453SEric Joyner * ice_handle_reset_event - Handle reset events triggered by OICR 2464*71d10453SEric Joyner * @sc: The device private softc 2465*71d10453SEric Joyner * 2466*71d10453SEric Joyner * Handle reset events triggered by an OICR notification. This includes CORER, 2467*71d10453SEric Joyner * GLOBR, and EMPR resets triggered by software on this or any other PF or by 2468*71d10453SEric Joyner * firmware. 2469*71d10453SEric Joyner * 2470*71d10453SEric Joyner * @pre assumes the iflib context lock is held, and will unlock it while 2471*71d10453SEric Joyner * waiting for the hardware to finish reset. 2472*71d10453SEric Joyner */ 2473*71d10453SEric Joyner static void 2474*71d10453SEric Joyner ice_handle_reset_event(struct ice_softc *sc) 2475*71d10453SEric Joyner { 2476*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 2477*71d10453SEric Joyner enum ice_status status; 2478*71d10453SEric Joyner device_t dev = sc->dev; 2479*71d10453SEric Joyner 2480*71d10453SEric Joyner /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will 2481*71d10453SEric Joyner * trigger an OICR interrupt. Our OICR handler will determine when 2482*71d10453SEric Joyner * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as 2483*71d10453SEric Joyner * appropriate. 2484*71d10453SEric Joyner */ 2485*71d10453SEric Joyner if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 2486*71d10453SEric Joyner return; 2487*71d10453SEric Joyner 2488*71d10453SEric Joyner ice_prepare_for_reset(sc); 2489*71d10453SEric Joyner 2490*71d10453SEric Joyner /* 2491*71d10453SEric Joyner * Release the iflib context lock and wait for the device to finish 2492*71d10453SEric Joyner * resetting. 2493*71d10453SEric Joyner */ 2494*71d10453SEric Joyner IFLIB_CTX_UNLOCK(sc); 2495*71d10453SEric Joyner status = ice_check_reset(hw); 2496*71d10453SEric Joyner IFLIB_CTX_LOCK(sc); 2497*71d10453SEric Joyner if (status) { 2498*71d10453SEric Joyner device_printf(dev, "Device never came out of reset, err %s\n", 2499*71d10453SEric Joyner ice_status_str(status)); 2500*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2501*71d10453SEric Joyner return; 2502*71d10453SEric Joyner } 2503*71d10453SEric Joyner 2504*71d10453SEric Joyner /* We're done with the reset, so we can rebuild driver state */ 2505*71d10453SEric Joyner sc->hw.reset_ongoing = false; 2506*71d10453SEric Joyner ice_rebuild(sc); 2507*71d10453SEric Joyner 2508*71d10453SEric Joyner /* In the unlikely event that a PF reset request occurs at the same 2509*71d10453SEric Joyner * time as a global reset, clear the request now. This avoids 2510*71d10453SEric Joyner * resetting a second time right after we reset due to a global event. 2511*71d10453SEric Joyner */ 2512*71d10453SEric Joyner if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2513*71d10453SEric Joyner device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n"); 2514*71d10453SEric Joyner } 2515*71d10453SEric Joyner 2516*71d10453SEric Joyner /** 2517*71d10453SEric Joyner * ice_handle_pf_reset_request - Initiate PF reset requested by software 2518*71d10453SEric Joyner * @sc: The device private softc 2519*71d10453SEric Joyner * 2520*71d10453SEric Joyner * Initiate a PF reset requested by software. We handle this in the admin task 2521*71d10453SEric Joyner * so that only one thread actually handles driver preparation and cleanup, 2522*71d10453SEric Joyner * rather than having multiple threads possibly attempt to run this code 2523*71d10453SEric Joyner * simultaneously. 2524*71d10453SEric Joyner * 2525*71d10453SEric Joyner * @pre assumes the iflib context lock is held and will unlock it while 2526*71d10453SEric Joyner * waiting for the PF reset to complete. 2527*71d10453SEric Joyner */ 2528*71d10453SEric Joyner static void 2529*71d10453SEric Joyner ice_handle_pf_reset_request(struct ice_softc *sc) 2530*71d10453SEric Joyner { 2531*71d10453SEric Joyner struct ice_hw *hw = &sc->hw; 2532*71d10453SEric Joyner enum ice_status status; 2533*71d10453SEric Joyner 2534*71d10453SEric Joyner /* Check for PF reset requests */ 2535*71d10453SEric Joyner if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2536*71d10453SEric Joyner return; 2537*71d10453SEric Joyner 2538*71d10453SEric Joyner /* Make sure we're prepared for reset */ 2539*71d10453SEric Joyner ice_prepare_for_reset(sc); 2540*71d10453SEric Joyner 2541*71d10453SEric Joyner /* 2542*71d10453SEric Joyner * Release the iflib context lock and wait for the device to finish 2543*71d10453SEric Joyner * resetting. 2544*71d10453SEric Joyner */ 2545*71d10453SEric Joyner IFLIB_CTX_UNLOCK(sc); 2546*71d10453SEric Joyner status = ice_reset(hw, ICE_RESET_PFR); 2547*71d10453SEric Joyner IFLIB_CTX_LOCK(sc); 2548*71d10453SEric Joyner if (status) { 2549*71d10453SEric Joyner device_printf(sc->dev, "device PF reset failed, err %s\n", 2550*71d10453SEric Joyner ice_status_str(status)); 2551*71d10453SEric Joyner ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2552*71d10453SEric Joyner return; 2553*71d10453SEric Joyner } 2554*71d10453SEric Joyner 2555*71d10453SEric Joyner sc->soft_stats.pfr_count++; 2556*71d10453SEric Joyner ice_rebuild(sc); 2557*71d10453SEric Joyner } 2558*71d10453SEric Joyner 2559*71d10453SEric Joyner /** 2560*71d10453SEric Joyner * ice_init_device_features - Init device driver features 2561*71d10453SEric Joyner * @sc: driver softc structure 2562*71d10453SEric Joyner * 2563*71d10453SEric Joyner * @pre assumes that the function capabilities bits have been set up by 2564*71d10453SEric Joyner * ice_init_hw(). 2565*71d10453SEric Joyner */ 2566*71d10453SEric Joyner static void 2567*71d10453SEric Joyner ice_init_device_features(struct ice_softc *sc) 2568*71d10453SEric Joyner { 2569*71d10453SEric Joyner /* 2570*71d10453SEric Joyner * A failed pkg file download triggers safe mode, disabling advanced 2571*71d10453SEric Joyner * device feature support 2572*71d10453SEric Joyner */ 2573*71d10453SEric Joyner if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) 2574*71d10453SEric Joyner return; 2575*71d10453SEric Joyner 2576*71d10453SEric Joyner /* Set capabilities that the driver supports */ 2577*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2578*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap); 2579*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap); 2580*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_DEFAULT_OVERRIDE, sc->feat_cap); 2581*71d10453SEric Joyner 2582*71d10453SEric Joyner /* Disable features due to hardware limitations... */ 2583*71d10453SEric Joyner if (!sc->hw.func_caps.common_cap.rss_table_size) 2584*71d10453SEric Joyner ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2585*71d10453SEric Joyner 2586*71d10453SEric Joyner /* Disable capabilities not supported by the OS */ 2587*71d10453SEric Joyner ice_disable_unsupported_features(sc->feat_cap); 2588*71d10453SEric Joyner 2589*71d10453SEric Joyner /* RSS is always enabled for iflib */ 2590*71d10453SEric Joyner if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS)) 2591*71d10453SEric Joyner ice_set_bit(ICE_FEATURE_RSS, sc->feat_en); 2592*71d10453SEric Joyner } 2593*71d10453SEric Joyner 2594*71d10453SEric Joyner /** 2595*71d10453SEric Joyner * ice_if_multi_set - Callback to update Multicast filters in HW 2596*71d10453SEric Joyner * @ctx: iflib ctx structure 2597*71d10453SEric Joyner * 2598*71d10453SEric Joyner * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search 2599*71d10453SEric Joyner * the if_multiaddrs list and determine which filters have been added or 2600*71d10453SEric Joyner * removed from the list, and update HW programming to reflect the new list. 2601*71d10453SEric Joyner * 2602*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 2603*71d10453SEric Joyner */ 2604*71d10453SEric Joyner static void 2605*71d10453SEric Joyner ice_if_multi_set(if_ctx_t ctx) 2606*71d10453SEric Joyner { 2607*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2608*71d10453SEric Joyner int err; 2609*71d10453SEric Joyner 2610*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2611*71d10453SEric Joyner 2612*71d10453SEric Joyner /* Do not handle multicast configuration in recovery mode */ 2613*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2614*71d10453SEric Joyner return; 2615*71d10453SEric Joyner 2616*71d10453SEric Joyner err = ice_sync_multicast_filters(sc); 2617*71d10453SEric Joyner if (err) { 2618*71d10453SEric Joyner device_printf(sc->dev, 2619*71d10453SEric Joyner "Failed to synchronize multicast filter list: %s\n", 2620*71d10453SEric Joyner ice_err_str(err)); 2621*71d10453SEric Joyner return; 2622*71d10453SEric Joyner } 2623*71d10453SEric Joyner } 2624*71d10453SEric Joyner 2625*71d10453SEric Joyner /** 2626*71d10453SEric Joyner * ice_if_vlan_register - Register a VLAN with the hardware 2627*71d10453SEric Joyner * @ctx: iflib ctx pointer 2628*71d10453SEric Joyner * @vtag: VLAN to add 2629*71d10453SEric Joyner * 2630*71d10453SEric Joyner * Programs the main PF VSI with a hardware filter for the given VLAN. 2631*71d10453SEric Joyner * 2632*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 2633*71d10453SEric Joyner */ 2634*71d10453SEric Joyner static void 2635*71d10453SEric Joyner ice_if_vlan_register(if_ctx_t ctx, u16 vtag) 2636*71d10453SEric Joyner { 2637*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2638*71d10453SEric Joyner enum ice_status status; 2639*71d10453SEric Joyner 2640*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2641*71d10453SEric Joyner 2642*71d10453SEric Joyner /* Do not handle VLAN configuration in recovery mode */ 2643*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2644*71d10453SEric Joyner return; 2645*71d10453SEric Joyner 2646*71d10453SEric Joyner status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag); 2647*71d10453SEric Joyner if (status) { 2648*71d10453SEric Joyner device_printf(sc->dev, 2649*71d10453SEric Joyner "Failure adding VLAN %d to main VSI, err %s aq_err %s\n", 2650*71d10453SEric Joyner vtag, ice_status_str(status), 2651*71d10453SEric Joyner ice_aq_str(sc->hw.adminq.sq_last_status)); 2652*71d10453SEric Joyner } 2653*71d10453SEric Joyner } 2654*71d10453SEric Joyner 2655*71d10453SEric Joyner /** 2656*71d10453SEric Joyner * ice_if_vlan_unregister - Remove a VLAN filter from the hardware 2657*71d10453SEric Joyner * @ctx: iflib ctx pointer 2658*71d10453SEric Joyner * @vtag: VLAN to add 2659*71d10453SEric Joyner * 2660*71d10453SEric Joyner * Removes the previously programmed VLAN filter from the main PF VSI. 2661*71d10453SEric Joyner * 2662*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 2663*71d10453SEric Joyner */ 2664*71d10453SEric Joyner static void 2665*71d10453SEric Joyner ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 2666*71d10453SEric Joyner { 2667*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2668*71d10453SEric Joyner enum ice_status status; 2669*71d10453SEric Joyner 2670*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2671*71d10453SEric Joyner 2672*71d10453SEric Joyner /* Do not handle VLAN configuration in recovery mode */ 2673*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2674*71d10453SEric Joyner return; 2675*71d10453SEric Joyner 2676*71d10453SEric Joyner status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag); 2677*71d10453SEric Joyner if (status) { 2678*71d10453SEric Joyner device_printf(sc->dev, 2679*71d10453SEric Joyner "Failure removing VLAN %d from main VSI, err %s aq_err %s\n", 2680*71d10453SEric Joyner vtag, ice_status_str(status), 2681*71d10453SEric Joyner ice_aq_str(sc->hw.adminq.sq_last_status)); 2682*71d10453SEric Joyner } 2683*71d10453SEric Joyner } 2684*71d10453SEric Joyner 2685*71d10453SEric Joyner /** 2686*71d10453SEric Joyner * ice_if_stop - Stop the device 2687*71d10453SEric Joyner * @ctx: iflib context structure 2688*71d10453SEric Joyner * 2689*71d10453SEric Joyner * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0 2690*71d10453SEric Joyner * down) 2691*71d10453SEric Joyner * 2692*71d10453SEric Joyner * @pre assumes the caller holds the iflib CTX lock 2693*71d10453SEric Joyner */ 2694*71d10453SEric Joyner static void 2695*71d10453SEric Joyner ice_if_stop(if_ctx_t ctx) 2696*71d10453SEric Joyner { 2697*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2698*71d10453SEric Joyner 2699*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2700*71d10453SEric Joyner 2701*71d10453SEric Joyner /* 2702*71d10453SEric Joyner * The iflib core may call IFDI_STOP prior to the first call to 2703*71d10453SEric Joyner * IFDI_INIT. This will cause us to attempt to remove MAC filters we 2704*71d10453SEric Joyner * don't have, and disable Tx queues which aren't yet configured. 2705*71d10453SEric Joyner * Although it is likely these extra operations are harmless, they do 2706*71d10453SEric Joyner * cause spurious warning messages to be displayed, which may confuse 2707*71d10453SEric Joyner * users. 2708*71d10453SEric Joyner * 2709*71d10453SEric Joyner * To avoid these messages, we use a state bit indicating if we've 2710*71d10453SEric Joyner * been initialized. It will be set when ice_if_init is called, and 2711*71d10453SEric Joyner * cleared here in ice_if_stop. 2712*71d10453SEric Joyner */ 2713*71d10453SEric Joyner if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED)) 2714*71d10453SEric Joyner return; 2715*71d10453SEric Joyner 2716*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 2717*71d10453SEric Joyner device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n"); 2718*71d10453SEric Joyner return; 2719*71d10453SEric Joyner } 2720*71d10453SEric Joyner 2721*71d10453SEric Joyner if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 2722*71d10453SEric Joyner device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n"); 2723*71d10453SEric Joyner return; 2724*71d10453SEric Joyner } 2725*71d10453SEric Joyner 2726*71d10453SEric Joyner /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the 2727*71d10453SEric Joyner * return of these functions because there's nothing we can really do 2728*71d10453SEric Joyner * if they fail, and the functions already print error messages. 2729*71d10453SEric Joyner * Just try to shut down as much as we can. 2730*71d10453SEric Joyner */ 2731*71d10453SEric Joyner ice_rm_pf_default_mac_filters(sc); 2732*71d10453SEric Joyner 2733*71d10453SEric Joyner /* Dissociate the Tx and Rx queues from the interrupts */ 2734*71d10453SEric Joyner ice_flush_txq_interrupts(&sc->pf_vsi); 2735*71d10453SEric Joyner ice_flush_rxq_interrupts(&sc->pf_vsi); 2736*71d10453SEric Joyner 2737*71d10453SEric Joyner /* Disable the Tx and Rx queues */ 2738*71d10453SEric Joyner ice_vsi_disable_tx(&sc->pf_vsi); 2739*71d10453SEric Joyner ice_control_rx_queues(&sc->pf_vsi, false); 2740*71d10453SEric Joyner } 2741*71d10453SEric Joyner 2742*71d10453SEric Joyner /** 2743*71d10453SEric Joyner * ice_if_get_counter - Get current value of an ifnet statistic 2744*71d10453SEric Joyner * @ctx: iflib context pointer 2745*71d10453SEric Joyner * @counter: ifnet counter to read 2746*71d10453SEric Joyner * 2747*71d10453SEric Joyner * Reads the current value of an ifnet counter for the device. 2748*71d10453SEric Joyner * 2749*71d10453SEric Joyner * This function is not protected by the iflib CTX lock. 2750*71d10453SEric Joyner */ 2751*71d10453SEric Joyner static uint64_t 2752*71d10453SEric Joyner ice_if_get_counter(if_ctx_t ctx, ift_counter counter) 2753*71d10453SEric Joyner { 2754*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2755*71d10453SEric Joyner 2756*71d10453SEric Joyner /* Return the counter for the main PF VSI */ 2757*71d10453SEric Joyner return ice_get_ifnet_counter(&sc->pf_vsi, counter); 2758*71d10453SEric Joyner } 2759*71d10453SEric Joyner 2760*71d10453SEric Joyner /** 2761*71d10453SEric Joyner * ice_request_stack_reinit - Request that iflib re-initialize 2762*71d10453SEric Joyner * @sc: the device private softc 2763*71d10453SEric Joyner * 2764*71d10453SEric Joyner * Request that the device be brought down and up, to re-initialize. For 2765*71d10453SEric Joyner * example, this may be called when a device reset occurs, or when Tx and Rx 2766*71d10453SEric Joyner * queues need to be re-initialized. 2767*71d10453SEric Joyner * 2768*71d10453SEric Joyner * This is required because the iflib state is outside the driver, and must be 2769*71d10453SEric Joyner * re-initialized if we need to resart Tx and Rx queues. 2770*71d10453SEric Joyner */ 2771*71d10453SEric Joyner void 2772*71d10453SEric Joyner ice_request_stack_reinit(struct ice_softc *sc) 2773*71d10453SEric Joyner { 2774*71d10453SEric Joyner if (CTX_ACTIVE(sc->ctx)) { 2775*71d10453SEric Joyner iflib_request_reset(sc->ctx); 2776*71d10453SEric Joyner iflib_admin_intr_deferred(sc->ctx); 2777*71d10453SEric Joyner } 2778*71d10453SEric Joyner } 2779*71d10453SEric Joyner 2780*71d10453SEric Joyner /** 2781*71d10453SEric Joyner * ice_driver_is_detaching - Check if the driver is detaching/unloading 2782*71d10453SEric Joyner * @sc: device private softc 2783*71d10453SEric Joyner * 2784*71d10453SEric Joyner * Returns true if the driver is detaching, false otherwise. 2785*71d10453SEric Joyner * 2786*71d10453SEric Joyner * @remark on newer kernels, take advantage of iflib_in_detach in order to 2787*71d10453SEric Joyner * report detachment correctly as early as possible. 2788*71d10453SEric Joyner * 2789*71d10453SEric Joyner * @remark this function is used by various code paths that want to avoid 2790*71d10453SEric Joyner * running if the driver is about to be removed. This includes sysctls and 2791*71d10453SEric Joyner * other driver access points. Note that it does not fully resolve 2792*71d10453SEric Joyner * detach-based race conditions as it is possible for a thread to race with 2793*71d10453SEric Joyner * iflib_in_detach. 2794*71d10453SEric Joyner */ 2795*71d10453SEric Joyner bool 2796*71d10453SEric Joyner ice_driver_is_detaching(struct ice_softc *sc) 2797*71d10453SEric Joyner { 2798*71d10453SEric Joyner return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || 2799*71d10453SEric Joyner iflib_in_detach(sc->ctx)); 2800*71d10453SEric Joyner } 2801*71d10453SEric Joyner 2802*71d10453SEric Joyner /** 2803*71d10453SEric Joyner * ice_if_priv_ioctl - Device private ioctl handler 2804*71d10453SEric Joyner * @ctx: iflib context pointer 2805*71d10453SEric Joyner * @command: The ioctl command issued 2806*71d10453SEric Joyner * @data: ioctl specific data 2807*71d10453SEric Joyner * 2808*71d10453SEric Joyner * iflib callback for handling custom driver specific ioctls. 2809*71d10453SEric Joyner * 2810*71d10453SEric Joyner * @pre Assumes that the iflib context lock is held. 2811*71d10453SEric Joyner */ 2812*71d10453SEric Joyner static int 2813*71d10453SEric Joyner ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) 2814*71d10453SEric Joyner { 2815*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2816*71d10453SEric Joyner struct ifdrv *ifd; 2817*71d10453SEric Joyner device_t dev = sc->dev; 2818*71d10453SEric Joyner 2819*71d10453SEric Joyner if (data == NULL) 2820*71d10453SEric Joyner return (EINVAL); 2821*71d10453SEric Joyner 2822*71d10453SEric Joyner ASSERT_CTX_LOCKED(sc); 2823*71d10453SEric Joyner 2824*71d10453SEric Joyner /* Make sure the command type is valid */ 2825*71d10453SEric Joyner switch (command) { 2826*71d10453SEric Joyner case SIOCSDRVSPEC: 2827*71d10453SEric Joyner case SIOCGDRVSPEC: 2828*71d10453SEric Joyner /* Accepted commands */ 2829*71d10453SEric Joyner break; 2830*71d10453SEric Joyner case SIOCGPRIVATE_0: 2831*71d10453SEric Joyner /* 2832*71d10453SEric Joyner * Although we do not support this ioctl command, it's 2833*71d10453SEric Joyner * expected that iflib will forward it to the IFDI_PRIV_IOCTL 2834*71d10453SEric Joyner * handler. Do not print a message in this case 2835*71d10453SEric Joyner */ 2836*71d10453SEric Joyner return (ENOTSUP); 2837*71d10453SEric Joyner default: 2838*71d10453SEric Joyner /* 2839*71d10453SEric Joyner * If we get a different command for this function, it's 2840*71d10453SEric Joyner * definitely unexpected, so log a message indicating what 2841*71d10453SEric Joyner * command we got for debugging purposes. 2842*71d10453SEric Joyner */ 2843*71d10453SEric Joyner device_printf(dev, "%s: unexpected ioctl command %08lx\n", 2844*71d10453SEric Joyner __func__, command); 2845*71d10453SEric Joyner return (EINVAL); 2846*71d10453SEric Joyner } 2847*71d10453SEric Joyner 2848*71d10453SEric Joyner ifd = (struct ifdrv *)data; 2849*71d10453SEric Joyner 2850*71d10453SEric Joyner switch (ifd->ifd_cmd) { 2851*71d10453SEric Joyner case ICE_NVM_ACCESS: 2852*71d10453SEric Joyner return ice_handle_nvm_access_ioctl(sc, ifd); 2853*71d10453SEric Joyner default: 2854*71d10453SEric Joyner return EINVAL; 2855*71d10453SEric Joyner } 2856*71d10453SEric Joyner } 2857*71d10453SEric Joyner 2858*71d10453SEric Joyner /** 2859*71d10453SEric Joyner * ice_if_i2c_req - I2C request handler for iflib 2860*71d10453SEric Joyner * @ctx: iflib context pointer 2861*71d10453SEric Joyner * @req: The I2C parameters to use 2862*71d10453SEric Joyner * 2863*71d10453SEric Joyner * Read from the port's I2C eeprom using the parameters from the ioctl. 2864*71d10453SEric Joyner * 2865*71d10453SEric Joyner * @remark The iflib-only part is pretty simple. 2866*71d10453SEric Joyner */ 2867*71d10453SEric Joyner static int 2868*71d10453SEric Joyner ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req) 2869*71d10453SEric Joyner { 2870*71d10453SEric Joyner struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2871*71d10453SEric Joyner 2872*71d10453SEric Joyner return ice_handle_i2c_req(sc, req); 2873*71d10453SEric Joyner } 2874*71d10453SEric Joyner 2875