1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2021, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*$FreeBSD$*/ 32 33 /** 34 * @file if_ice_iflib.c 35 * @brief iflib driver implementation 36 * 37 * Contains the main entry point for the iflib driver implementation. It 38 * implements the various ifdi driver methods, and sets up the module and 39 * driver values to load an iflib driver. 40 */ 41 42 #include "ice_iflib.h" 43 #include "ice_drv_info.h" 44 #include "ice_switch.h" 45 #include "ice_sched.h" 46 47 #include <sys/module.h> 48 #include <sys/sockio.h> 49 #include <sys/smp.h> 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pcireg.h> 52 53 /* 54 * Device method prototypes 55 */ 56 57 static void *ice_register(device_t); 58 static int ice_if_attach_pre(if_ctx_t); 59 static int ice_attach_pre_recovery_mode(struct ice_softc *sc); 60 static int ice_if_attach_post(if_ctx_t); 61 static void ice_attach_post_recovery_mode(struct ice_softc *sc); 62 static int ice_if_detach(if_ctx_t); 63 static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); 64 static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); 65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix); 66 static void ice_if_queues_free(if_ctx_t ctx); 67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 68 static void ice_if_intr_enable(if_ctx_t ctx); 69 static void ice_if_intr_disable(if_ctx_t ctx); 70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 72 static int ice_if_promisc_set(if_ctx_t ctx, int flags); 73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 74 static int ice_if_media_change(if_ctx_t ctx); 75 static void ice_if_init(if_ctx_t ctx); 76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid); 77 static void ice_if_update_admin_status(if_ctx_t ctx); 78 static void ice_if_multi_set(if_ctx_t ctx); 79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag); 80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag); 81 static void ice_if_stop(if_ctx_t ctx); 82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter); 83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); 84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req); 85 static int ice_if_suspend(if_ctx_t ctx); 86 static int ice_if_resume(if_ctx_t ctx); 87 88 static int ice_msix_que(void *arg); 89 static int ice_msix_admin(void *arg); 90 91 /* 92 * Helper function prototypes 93 */ 94 static int ice_pci_mapping(struct ice_softc *sc); 95 static void ice_free_pci_mapping(struct ice_softc *sc); 96 static void ice_update_link_status(struct ice_softc *sc, bool update_media); 97 static void ice_init_device_features(struct ice_softc *sc); 98 static void ice_init_tx_tracking(struct ice_vsi *vsi); 99 static void ice_handle_reset_event(struct ice_softc *sc); 100 static void ice_handle_pf_reset_request(struct ice_softc *sc); 101 static void ice_prepare_for_reset(struct ice_softc *sc); 102 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc); 103 static void ice_rebuild(struct ice_softc *sc); 104 static void ice_rebuild_recovery_mode(struct ice_softc *sc); 105 static void ice_free_irqvs(struct ice_softc *sc); 106 static void ice_update_rx_mbuf_sz(struct ice_softc *sc); 107 static void ice_poll_for_media_avail(struct ice_softc *sc); 108 static void ice_setup_scctx(struct ice_softc *sc); 109 static int ice_allocate_msix(struct ice_softc *sc); 110 static void ice_admin_timer(void *arg); 111 static void ice_transition_recovery_mode(struct ice_softc *sc); 112 static void ice_transition_safe_mode(struct ice_softc *sc); 113 114 /* 115 * Device Interface Declaration 116 */ 117 118 /** 119 * @var ice_methods 120 * @brief ice driver method entry points 121 * 122 * List of device methods implementing the generic device interface used by 123 * the device stack to interact with the ice driver. Since this is an iflib 124 * driver, most of the methods point to the generic iflib implementation. 125 */ 126 static device_method_t ice_methods[] = { 127 /* Device interface */ 128 DEVMETHOD(device_register, ice_register), 129 DEVMETHOD(device_probe, iflib_device_probe_vendor), 130 DEVMETHOD(device_attach, iflib_device_attach), 131 DEVMETHOD(device_detach, iflib_device_detach), 132 DEVMETHOD(device_shutdown, iflib_device_shutdown), 133 DEVMETHOD(device_suspend, iflib_device_suspend), 134 DEVMETHOD(device_resume, iflib_device_resume), 135 DEVMETHOD_END 136 }; 137 138 /** 139 * @var ice_iflib_methods 140 * @brief iflib method entry points 141 * 142 * List of device methods used by the iflib stack to interact with this 143 * driver. These are the real main entry points used to interact with this 144 * driver. 145 */ 146 static device_method_t ice_iflib_methods[] = { 147 DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre), 148 DEVMETHOD(ifdi_attach_post, ice_if_attach_post), 149 DEVMETHOD(ifdi_detach, ice_if_detach), 150 DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc), 151 DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc), 152 DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign), 153 DEVMETHOD(ifdi_queues_free, ice_if_queues_free), 154 DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set), 155 DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable), 156 DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable), 157 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable), 158 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable), 159 DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set), 160 DEVMETHOD(ifdi_media_status, ice_if_media_status), 161 DEVMETHOD(ifdi_media_change, ice_if_media_change), 162 DEVMETHOD(ifdi_init, ice_if_init), 163 DEVMETHOD(ifdi_stop, ice_if_stop), 164 DEVMETHOD(ifdi_timer, ice_if_timer), 165 DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status), 166 DEVMETHOD(ifdi_multi_set, ice_if_multi_set), 167 DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register), 168 DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister), 169 DEVMETHOD(ifdi_get_counter, ice_if_get_counter), 170 DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl), 171 DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req), 172 DEVMETHOD(ifdi_suspend, ice_if_suspend), 173 DEVMETHOD(ifdi_resume, ice_if_resume), 174 DEVMETHOD_END 175 }; 176 177 /** 178 * @var ice_driver 179 * @brief driver structure for the generic device stack 180 * 181 * driver_t definition used to setup the generic device methods. 182 */ 183 static driver_t ice_driver = { 184 .name = "ice", 185 .methods = ice_methods, 186 .size = sizeof(struct ice_softc), 187 }; 188 189 /** 190 * @var ice_iflib_driver 191 * @brief driver structure for the iflib stack 192 * 193 * driver_t definition used to setup the iflib device methods. 194 */ 195 static driver_t ice_iflib_driver = { 196 .name = "ice", 197 .methods = ice_iflib_methods, 198 .size = sizeof(struct ice_softc), 199 }; 200 201 extern struct if_txrx ice_txrx; 202 extern struct if_txrx ice_recovery_txrx; 203 204 /** 205 * @var ice_sctx 206 * @brief ice driver shared context 207 * 208 * Structure defining shared values (context) that is used by all instances of 209 * the device. Primarily used to setup details about how the iflib stack 210 * should treat this driver. Also defines the default, minimum, and maximum 211 * number of descriptors in each ring. 212 */ 213 static struct if_shared_ctx ice_sctx = { 214 .isc_magic = IFLIB_MAGIC, 215 .isc_q_align = PAGE_SIZE, 216 217 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 218 /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but 219 * that doesn't make sense since that would be larger than the maximum 220 * size of a single packet. 221 */ 222 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 223 224 /* XXX: This is only used by iflib to ensure that 225 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size. 226 */ 227 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 228 /* XXX: This is used by iflib to set the number of segments in the TSO 229 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the 230 * related ifnet parameter. 231 */ 232 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 233 234 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 235 .isc_rx_nsegments = ICE_MAX_RX_SEGS, 236 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 237 238 .isc_nfl = 1, 239 .isc_ntxqs = 1, 240 .isc_nrxqs = 1, 241 242 .isc_admin_intrcnt = 1, 243 .isc_vendor_info = ice_vendor_info_array, 244 .isc_driver_version = __DECONST(char *, ice_driver_version), 245 .isc_driver = &ice_iflib_driver, 246 247 /* 248 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available 249 * for hardware checksum offload 250 * 251 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the 252 * IP sum field, required by our hardware to calculate valid TSO 253 * checksums. 254 * 255 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs 256 * even when the interface is down. 257 * 258 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X 259 * vectors manually instead of relying on iflib code to do this. 260 */ 261 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 262 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX, 263 264 .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 265 .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 266 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 267 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 268 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 269 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 270 }; 271 272 /** 273 * @var ice_devclass 274 * @brief ice driver device class 275 * 276 * device class used to setup the ice driver module kobject class. 277 */ 278 devclass_t ice_devclass; 279 DRIVER_MODULE(ice, pci, ice_driver, ice_devclass, ice_module_event_handler, 0); 280 281 MODULE_VERSION(ice, 1); 282 MODULE_DEPEND(ice, pci, 1, 1, 1); 283 MODULE_DEPEND(ice, ether, 1, 1, 1); 284 MODULE_DEPEND(ice, iflib, 1, 1, 1); 285 286 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array); 287 288 /* Static driver-wide sysctls */ 289 #include "ice_iflib_sysctls.h" 290 291 /** 292 * ice_pci_mapping - Map PCI BAR memory 293 * @sc: device private softc 294 * 295 * Map PCI BAR 0 for device operation. 296 */ 297 static int 298 ice_pci_mapping(struct ice_softc *sc) 299 { 300 int rc; 301 302 /* Map BAR0 */ 303 rc = ice_map_bar(sc->dev, &sc->bar0, 0); 304 if (rc) 305 return rc; 306 307 return 0; 308 } 309 310 /** 311 * ice_free_pci_mapping - Release PCI BAR memory 312 * @sc: device private softc 313 * 314 * Release PCI BARs which were previously mapped by ice_pci_mapping(). 315 */ 316 static void 317 ice_free_pci_mapping(struct ice_softc *sc) 318 { 319 /* Free BAR0 */ 320 ice_free_bar(sc->dev, &sc->bar0); 321 } 322 323 /* 324 * Device methods 325 */ 326 327 /** 328 * ice_register - register device method callback 329 * @dev: the device being registered 330 * 331 * Returns a pointer to the shared context structure, which is used by iflib. 332 */ 333 static void * 334 ice_register(device_t dev __unused) 335 { 336 return &ice_sctx; 337 } /* ice_register */ 338 339 /** 340 * ice_setup_scctx - Setup the iflib softc context structure 341 * @sc: the device private structure 342 * 343 * Setup the parameters in if_softc_ctx_t structure used by the iflib stack 344 * when loading. 345 */ 346 static void 347 ice_setup_scctx(struct ice_softc *sc) 348 { 349 if_softc_ctx_t scctx = sc->scctx; 350 struct ice_hw *hw = &sc->hw; 351 bool safe_mode, recovery_mode; 352 353 safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE); 354 recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE); 355 356 /* 357 * If the driver loads in Safe mode or Recovery mode, limit iflib to 358 * a single queue pair. 359 */ 360 if (safe_mode || recovery_mode) { 361 scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1; 362 scctx->isc_ntxqsets_max = 1; 363 scctx->isc_nrxqsets_max = 1; 364 } else { 365 /* 366 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to 367 * the values of the override sysctls. Cache these initial 368 * values so that the driver can be aware of what the iflib 369 * sysctl value is when setting up MSI-X vectors. 370 */ 371 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets; 372 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets; 373 374 if (scctx->isc_ntxqsets == 0) 375 scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size; 376 if (scctx->isc_nrxqsets == 0) 377 scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size; 378 379 scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq; 380 scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq; 381 382 /* 383 * Sanity check that the iflib sysctl values are within the 384 * maximum supported range. 385 */ 386 if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max) 387 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max; 388 if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max) 389 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max; 390 } 391 392 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 393 * sizeof(struct ice_tx_desc), DBA_ALIGN); 394 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 395 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 396 397 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 398 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 399 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 400 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 401 402 scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR); 403 scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size; 404 405 /* 406 * If the driver loads in recovery mode, disable Tx/Rx functionality 407 */ 408 if (recovery_mode) 409 scctx->isc_txrx = &ice_recovery_txrx; 410 else 411 scctx->isc_txrx = &ice_txrx; 412 413 /* 414 * If the driver loads in Safe mode or Recovery mode, disable 415 * advanced features including hardware offloads. 416 */ 417 if (safe_mode || recovery_mode) { 418 scctx->isc_capenable = ICE_SAFE_CAPS; 419 scctx->isc_tx_csum_flags = 0; 420 } else { 421 scctx->isc_capenable = ICE_FULL_CAPS; 422 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 423 } 424 425 scctx->isc_capabilities = scctx->isc_capenable; 426 } /* ice_setup_scctx */ 427 428 /** 429 * ice_if_attach_pre - Early device attach logic 430 * @ctx: the iflib context structure 431 * 432 * Called by iflib during the attach process. Earliest main driver entry 433 * point which performs necessary hardware and driver initialization. Called 434 * before the Tx and Rx queues are allocated. 435 */ 436 static int 437 ice_if_attach_pre(if_ctx_t ctx) 438 { 439 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 440 enum ice_fw_modes fw_mode; 441 enum ice_status status; 442 if_softc_ctx_t scctx; 443 struct ice_hw *hw; 444 device_t dev; 445 int err; 446 447 device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n"); 448 449 sc->ctx = ctx; 450 sc->media = iflib_get_media(ctx); 451 sc->sctx = iflib_get_sctx(ctx); 452 sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx); 453 454 dev = sc->dev = iflib_get_dev(ctx); 455 scctx = sc->scctx = iflib_get_softc_ctx(ctx); 456 457 hw = &sc->hw; 458 hw->back = sc; 459 460 snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name), 461 "%s:admin", device_get_nameunit(dev)); 462 mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF); 463 callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0); 464 465 ASSERT_CTX_LOCKED(sc); 466 467 if (ice_pci_mapping(sc)) { 468 err = (ENXIO); 469 goto destroy_admin_timer; 470 } 471 472 /* Save off the PCI information */ 473 ice_save_pci_info(hw, dev); 474 475 /* create tunables as early as possible */ 476 ice_add_device_tunables(sc); 477 478 /* Setup ControlQ lengths */ 479 ice_set_ctrlq_len(hw); 480 481 fw_mode = ice_get_fw_mode(hw); 482 if (fw_mode == ICE_FW_MODE_REC) { 483 device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 484 485 err = ice_attach_pre_recovery_mode(sc); 486 if (err) 487 goto free_pci_mapping; 488 489 return (0); 490 } 491 492 /* Initialize the hw data structure */ 493 status = ice_init_hw(hw); 494 if (status) { 495 if (status == ICE_ERR_FW_API_VER) { 496 /* Enter recovery mode, so that the driver remains 497 * loaded. This way, if the system administrator 498 * cannot update the driver, they may still attempt to 499 * downgrade the NVM. 500 */ 501 err = ice_attach_pre_recovery_mode(sc); 502 if (err) 503 goto free_pci_mapping; 504 505 return (0); 506 } else { 507 err = EIO; 508 device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n", 509 ice_status_str(status), 510 ice_aq_str(hw->adminq.sq_last_status)); 511 } 512 goto free_pci_mapping; 513 } 514 515 /* Notify firmware of the device driver version */ 516 err = ice_send_version(sc); 517 if (err) 518 goto deinit_hw; 519 520 ice_load_pkg_file(sc); 521 522 err = ice_init_link_events(sc); 523 if (err) { 524 device_printf(dev, "ice_init_link_events failed: %s\n", 525 ice_err_str(err)); 526 goto deinit_hw; 527 } 528 529 ice_print_nvm_version(sc); 530 531 ice_init_device_features(sc); 532 533 /* Setup the MAC address */ 534 iflib_set_mac(ctx, hw->port_info->mac.lan_addr); 535 536 /* Setup the iflib softc context structure */ 537 ice_setup_scctx(sc); 538 539 /* Initialize the Tx queue manager */ 540 err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq); 541 if (err) { 542 device_printf(dev, "Unable to initialize Tx queue manager: %s\n", 543 ice_err_str(err)); 544 goto deinit_hw; 545 } 546 547 /* Initialize the Rx queue manager */ 548 err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq); 549 if (err) { 550 device_printf(dev, "Unable to initialize Rx queue manager: %s\n", 551 ice_err_str(err)); 552 goto free_tx_qmgr; 553 } 554 555 /* Initialize the interrupt resource manager */ 556 err = ice_alloc_intr_tracking(sc); 557 if (err) 558 /* Errors are already printed */ 559 goto free_rx_qmgr; 560 561 /* Determine maximum number of VSIs we'll prepare for */ 562 sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE, 563 hw->func_caps.guar_num_vsi); 564 565 if (!sc->num_available_vsi) { 566 err = EIO; 567 device_printf(dev, "No VSIs allocated to host\n"); 568 goto free_intr_tracking; 569 } 570 571 /* Allocate storage for the VSI pointers */ 572 sc->all_vsi = (struct ice_vsi **) 573 malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi, 574 M_ICE, M_WAITOK | M_ZERO); 575 if (!sc->all_vsi) { 576 err = ENOMEM; 577 device_printf(dev, "Unable to allocate VSI array\n"); 578 goto free_intr_tracking; 579 } 580 581 /* 582 * Prepare the statically allocated primary PF VSI in the softc 583 * structure. Other VSIs will be dynamically allocated as needed. 584 */ 585 ice_setup_pf_vsi(sc); 586 587 err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max, 588 scctx->isc_nrxqsets_max); 589 if (err) { 590 device_printf(dev, "Unable to allocate VSI Queue maps\n"); 591 goto free_main_vsi; 592 } 593 594 /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */ 595 err = ice_allocate_msix(sc); 596 if (err) 597 goto free_main_vsi; 598 599 return 0; 600 601 free_main_vsi: 602 /* ice_release_vsi will free the queue maps if they were allocated */ 603 ice_release_vsi(&sc->pf_vsi); 604 free(sc->all_vsi, M_ICE); 605 sc->all_vsi = NULL; 606 free_intr_tracking: 607 ice_free_intr_tracking(sc); 608 free_rx_qmgr: 609 ice_resmgr_destroy(&sc->rx_qmgr); 610 free_tx_qmgr: 611 ice_resmgr_destroy(&sc->tx_qmgr); 612 deinit_hw: 613 ice_deinit_hw(hw); 614 free_pci_mapping: 615 ice_free_pci_mapping(sc); 616 destroy_admin_timer: 617 mtx_lock(&sc->admin_mtx); 618 callout_stop(&sc->admin_timer); 619 mtx_unlock(&sc->admin_mtx); 620 mtx_destroy(&sc->admin_mtx); 621 return err; 622 } /* ice_if_attach_pre */ 623 624 /** 625 * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery 626 * @sc: the device private softc 627 * 628 * Loads the device driver in limited Firmware Recovery mode, intended to 629 * allow users to update the firmware to attempt to recover the device. 630 * 631 * @remark We may enter recovery mode in case either (a) the firmware is 632 * detected to be in an invalid state and must be re-programmed, or (b) the 633 * driver detects that the loaded firmware has a non-compatible API version 634 * that the driver cannot operate with. 635 */ 636 static int 637 ice_attach_pre_recovery_mode(struct ice_softc *sc) 638 { 639 ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE); 640 641 /* Setup the iflib softc context */ 642 ice_setup_scctx(sc); 643 644 /* Setup the PF VSI back pointer */ 645 sc->pf_vsi.sc = sc; 646 647 /* 648 * We still need to allocate MSI-X vectors since we need one vector to 649 * run the administrative admin interrupt 650 */ 651 return ice_allocate_msix(sc); 652 } 653 654 /** 655 * ice_update_link_status - notify OS of link state change 656 * @sc: device private softc structure 657 * @update_media: true if we should update media even if link didn't change 658 * 659 * Called to notify iflib core of link status changes. Should be called once 660 * during attach_post, and whenever link status changes during runtime. 661 * 662 * This call only updates the currently supported media types if the link 663 * status changed, or if update_media is set to true. 664 */ 665 static void 666 ice_update_link_status(struct ice_softc *sc, bool update_media) 667 { 668 struct ice_hw *hw = &sc->hw; 669 enum ice_status status; 670 671 /* Never report link up when in recovery mode */ 672 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 673 return; 674 675 /* Report link status to iflib only once each time it changes */ 676 if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) { 677 if (sc->link_up) { /* link is up */ 678 uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info); 679 680 ice_set_default_local_lldp_mib(sc); 681 682 iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate); 683 684 ice_link_up_msg(sc); 685 686 update_media = true; 687 } else { /* link is down */ 688 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 689 690 update_media = true; 691 } 692 } 693 694 /* Update the supported media types */ 695 if (update_media) { 696 status = ice_add_media_types(sc, sc->media); 697 if (status) 698 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 699 ice_status_str(status), 700 ice_aq_str(hw->adminq.sq_last_status)); 701 } 702 703 /* TODO: notify VFs of link state change */ 704 } 705 706 /** 707 * ice_if_attach_post - Late device attach logic 708 * @ctx: the iflib context structure 709 * 710 * Called by iflib to finish up attaching the device. Performs any attach 711 * logic which must wait until after the Tx and Rx queues have been 712 * allocated. 713 */ 714 static int 715 ice_if_attach_post(if_ctx_t ctx) 716 { 717 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 718 if_t ifp = iflib_get_ifp(ctx); 719 int err; 720 721 ASSERT_CTX_LOCKED(sc); 722 723 /* We don't yet support loading if MSI-X is not supported */ 724 if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) { 725 device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n"); 726 return (ENOTSUP); 727 } 728 729 /* The ifnet structure hasn't yet been initialized when the attach_pre 730 * handler is called, so wait until attach_post to setup the 731 * isc_max_frame_size. 732 */ 733 734 sc->ifp = ifp; 735 sc->scctx->isc_max_frame_size = ifp->if_mtu + 736 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 737 738 /* 739 * If we are in recovery mode, only perform a limited subset of 740 * initialization to support NVM recovery. 741 */ 742 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 743 ice_attach_post_recovery_mode(sc); 744 return (0); 745 } 746 747 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 748 749 err = ice_initialize_vsi(&sc->pf_vsi); 750 if (err) { 751 device_printf(sc->dev, "Unable to initialize Main VSI: %s\n", 752 ice_err_str(err)); 753 return err; 754 } 755 756 /* Enable FW health event reporting */ 757 ice_init_health_events(sc); 758 759 /* Configure the main PF VSI for RSS */ 760 err = ice_config_rss(&sc->pf_vsi); 761 if (err) { 762 device_printf(sc->dev, 763 "Unable to configure RSS for the main VSI, err %s\n", 764 ice_err_str(err)); 765 return err; 766 } 767 768 /* Configure switch to drop transmitted LLDP and PAUSE frames */ 769 err = ice_cfg_pf_ethertype_filters(sc); 770 if (err) 771 return err; 772 773 ice_get_and_print_bus_info(sc); 774 775 ice_set_link_management_mode(sc); 776 777 ice_init_saved_phy_cfg(sc); 778 779 ice_add_device_sysctls(sc); 780 781 /* Get DCBX/LLDP state and start DCBX agent */ 782 ice_init_dcb_setup(sc); 783 784 /* Setup link configuration parameters */ 785 ice_init_link_configuration(sc); 786 ice_update_link_status(sc, true); 787 788 /* Configure interrupt causes for the administrative interrupt */ 789 ice_configure_misc_interrupts(sc); 790 791 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 792 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 793 794 /* Start the admin timer */ 795 mtx_lock(&sc->admin_mtx); 796 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 797 mtx_unlock(&sc->admin_mtx); 798 799 return 0; 800 } /* ice_if_attach_post */ 801 802 /** 803 * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery 804 * @sc: the device private softc 805 * 806 * Performs minimal work to prepare the driver to recover an NVM in case the 807 * firmware is in recovery mode. 808 */ 809 static void 810 ice_attach_post_recovery_mode(struct ice_softc *sc) 811 { 812 /* Configure interrupt causes for the administrative interrupt */ 813 ice_configure_misc_interrupts(sc); 814 815 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 816 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 817 818 /* Start the admin timer */ 819 mtx_lock(&sc->admin_mtx); 820 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 821 mtx_unlock(&sc->admin_mtx); 822 } 823 824 /** 825 * ice_free_irqvs - Free IRQ vector memory 826 * @sc: the device private softc structure 827 * 828 * Free IRQ vector memory allocated during ice_if_msix_intr_assign. 829 */ 830 static void 831 ice_free_irqvs(struct ice_softc *sc) 832 { 833 struct ice_vsi *vsi = &sc->pf_vsi; 834 if_ctx_t ctx = sc->ctx; 835 int i; 836 837 /* If the irqvs array is NULL, then there are no vectors to free */ 838 if (sc->irqvs == NULL) 839 return; 840 841 /* Free the IRQ vectors */ 842 for (i = 0; i < sc->num_irq_vectors; i++) 843 iflib_irq_free(ctx, &sc->irqvs[i].irq); 844 845 /* Clear the irqv pointers */ 846 for (i = 0; i < vsi->num_rx_queues; i++) 847 vsi->rx_queues[i].irqv = NULL; 848 849 for (i = 0; i < vsi->num_tx_queues; i++) 850 vsi->tx_queues[i].irqv = NULL; 851 852 /* Release the vector array memory */ 853 free(sc->irqvs, M_ICE); 854 sc->irqvs = NULL; 855 sc->num_irq_vectors = 0; 856 } 857 858 /** 859 * ice_if_detach - Device driver detach logic 860 * @ctx: iflib context structure 861 * 862 * Perform device shutdown logic to detach the device driver. 863 * 864 * Note that there is no guarantee of the ordering of ice_if_queues_free() and 865 * ice_if_detach(). It is possible for the functions to be called in either 866 * order, and they must not assume to have a strict ordering. 867 */ 868 static int 869 ice_if_detach(if_ctx_t ctx) 870 { 871 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 872 struct ice_vsi *vsi = &sc->pf_vsi; 873 int i; 874 875 ASSERT_CTX_LOCKED(sc); 876 877 /* Indicate that we're detaching */ 878 ice_set_state(&sc->state, ICE_STATE_DETACHING); 879 880 /* Stop the admin timer */ 881 mtx_lock(&sc->admin_mtx); 882 callout_stop(&sc->admin_timer); 883 mtx_unlock(&sc->admin_mtx); 884 mtx_destroy(&sc->admin_mtx); 885 886 /* Free allocated media types */ 887 ifmedia_removeall(sc->media); 888 889 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 890 * pointers. Note, the calls here and those in ice_if_queues_free() 891 * are *BOTH* necessary, as we cannot guarantee which path will be 892 * run first 893 */ 894 ice_vsi_del_txqs_ctx(vsi); 895 ice_vsi_del_rxqs_ctx(vsi); 896 897 /* Release MSI-X resources */ 898 ice_free_irqvs(sc); 899 900 for (i = 0; i < sc->num_available_vsi; i++) { 901 if (sc->all_vsi[i]) 902 ice_release_vsi(sc->all_vsi[i]); 903 } 904 905 if (sc->all_vsi) { 906 free(sc->all_vsi, M_ICE); 907 sc->all_vsi = NULL; 908 } 909 910 /* Release MSI-X memory */ 911 pci_release_msi(sc->dev); 912 913 if (sc->msix_table != NULL) { 914 bus_release_resource(sc->dev, SYS_RES_MEMORY, 915 rman_get_rid(sc->msix_table), 916 sc->msix_table); 917 sc->msix_table = NULL; 918 } 919 920 ice_free_intr_tracking(sc); 921 922 /* Destroy the queue managers */ 923 ice_resmgr_destroy(&sc->tx_qmgr); 924 ice_resmgr_destroy(&sc->rx_qmgr); 925 926 if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 927 ice_deinit_hw(&sc->hw); 928 929 ice_free_pci_mapping(sc); 930 931 return 0; 932 } /* ice_if_detach */ 933 934 /** 935 * ice_if_tx_queues_alloc - Allocate Tx queue memory 936 * @ctx: iflib context structure 937 * @vaddrs: virtual addresses for the queue memory 938 * @paddrs: physical addresses for the queue memory 939 * @ntxqs: the number of Tx queues per set (should always be 1) 940 * @ntxqsets: the number of Tx queue sets to allocate 941 * 942 * Called by iflib to allocate Tx queues for the device. Allocates driver 943 * memory to track each queue, the status arrays used for descriptor 944 * status reporting, and Tx queue sysctls. 945 */ 946 static int 947 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 948 int __invariant_only ntxqs, int ntxqsets) 949 { 950 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 951 struct ice_vsi *vsi = &sc->pf_vsi; 952 struct ice_tx_queue *txq; 953 int err, i, j; 954 955 MPASS(ntxqs == 1); 956 MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 957 ASSERT_CTX_LOCKED(sc); 958 959 /* Do not bother allocating queues if we're in recovery mode */ 960 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 961 return (0); 962 963 /* Allocate queue structure memory */ 964 if (!(vsi->tx_queues = 965 (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) { 966 device_printf(sc->dev, "Unable to allocate Tx queue memory\n"); 967 return (ENOMEM); 968 } 969 970 /* Allocate report status arrays */ 971 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 972 if (!(txq->tx_rsq = 973 (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) { 974 device_printf(sc->dev, "Unable to allocate tx_rsq memory\n"); 975 err = ENOMEM; 976 goto free_tx_queues; 977 } 978 /* Initialize report status array */ 979 for (j = 0; j < sc->scctx->isc_ntxd[0]; j++) 980 txq->tx_rsq[j] = QIDX_INVALID; 981 } 982 983 /* Assign queues from PF space to the main VSI */ 984 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets); 985 if (err) { 986 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 987 ice_err_str(err)); 988 goto free_tx_queues; 989 } 990 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 991 992 /* Add Tx queue sysctls context */ 993 ice_vsi_add_txqs_ctx(vsi); 994 995 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 996 txq->me = i; 997 txq->vsi = vsi; 998 999 /* store the queue size for easier access */ 1000 txq->desc_count = sc->scctx->isc_ntxd[0]; 1001 1002 /* get the virtual and physical address of the hardware queues */ 1003 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 1004 txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 1005 txq->tx_paddr = paddrs[i]; 1006 1007 ice_add_txq_sysctls(txq); 1008 } 1009 1010 vsi->num_tx_queues = ntxqsets; 1011 1012 return (0); 1013 1014 free_tx_queues: 1015 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1016 if (txq->tx_rsq != NULL) { 1017 free(txq->tx_rsq, M_ICE); 1018 txq->tx_rsq = NULL; 1019 } 1020 } 1021 free(vsi->tx_queues, M_ICE); 1022 vsi->tx_queues = NULL; 1023 return err; 1024 } 1025 1026 /** 1027 * ice_if_rx_queues_alloc - Allocate Rx queue memory 1028 * @ctx: iflib context structure 1029 * @vaddrs: virtual addresses for the queue memory 1030 * @paddrs: physical addresses for the queue memory 1031 * @nrxqs: number of Rx queues per set (should always be 1) 1032 * @nrxqsets: number of Rx queue sets to allocate 1033 * 1034 * Called by iflib to allocate Rx queues for the device. Allocates driver 1035 * memory to track each queue, as well as sets up the Rx queue sysctls. 1036 */ 1037 static int 1038 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1039 int __invariant_only nrxqs, int nrxqsets) 1040 { 1041 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1042 struct ice_vsi *vsi = &sc->pf_vsi; 1043 struct ice_rx_queue *rxq; 1044 int err, i; 1045 1046 MPASS(nrxqs == 1); 1047 MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 1048 ASSERT_CTX_LOCKED(sc); 1049 1050 /* Do not bother allocating queues if we're in recovery mode */ 1051 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1052 return (0); 1053 1054 /* Allocate queue structure memory */ 1055 if (!(vsi->rx_queues = 1056 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) { 1057 device_printf(sc->dev, "Unable to allocate Rx queue memory\n"); 1058 return (ENOMEM); 1059 } 1060 1061 /* Assign queues from PF space to the main VSI */ 1062 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets); 1063 if (err) { 1064 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1065 ice_err_str(err)); 1066 goto free_rx_queues; 1067 } 1068 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1069 1070 /* Add Rx queue sysctls context */ 1071 ice_vsi_add_rxqs_ctx(vsi); 1072 1073 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 1074 rxq->me = i; 1075 rxq->vsi = vsi; 1076 1077 /* store the queue size for easier access */ 1078 rxq->desc_count = sc->scctx->isc_nrxd[0]; 1079 1080 /* get the virtual and physical address of the hardware queues */ 1081 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 1082 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 1083 rxq->rx_paddr = paddrs[i]; 1084 1085 ice_add_rxq_sysctls(rxq); 1086 } 1087 1088 vsi->num_rx_queues = nrxqsets; 1089 1090 return (0); 1091 1092 free_rx_queues: 1093 free(vsi->rx_queues, M_ICE); 1094 vsi->rx_queues = NULL; 1095 return err; 1096 } 1097 1098 /** 1099 * ice_if_queues_free - Free queue memory 1100 * @ctx: the iflib context structure 1101 * 1102 * Free queue memory allocated by ice_if_tx_queues_alloc() and 1103 * ice_if_rx_queues_alloc(). 1104 * 1105 * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be 1106 * called in the same order. It's possible for ice_if_queues_free() to be 1107 * called prior to ice_if_detach(), and vice versa. 1108 * 1109 * For this reason, the main VSI is a static member of the ice_softc, which is 1110 * not free'd until after iflib finishes calling both of these functions. 1111 * 1112 * Thus, care must be taken in how we manage the memory being freed by this 1113 * function, and in what tasks it can and must perform. 1114 */ 1115 static void 1116 ice_if_queues_free(if_ctx_t ctx) 1117 { 1118 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1119 struct ice_vsi *vsi = &sc->pf_vsi; 1120 struct ice_tx_queue *txq; 1121 int i; 1122 1123 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1124 * pointers. Note, the calls here and those in ice_if_detach() 1125 * are *BOTH* necessary, as we cannot guarantee which path will be 1126 * run first 1127 */ 1128 ice_vsi_del_txqs_ctx(vsi); 1129 ice_vsi_del_rxqs_ctx(vsi); 1130 1131 /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */ 1132 ice_free_irqvs(sc); 1133 1134 if (vsi->tx_queues != NULL) { 1135 /* free the tx_rsq arrays */ 1136 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1137 if (txq->tx_rsq != NULL) { 1138 free(txq->tx_rsq, M_ICE); 1139 txq->tx_rsq = NULL; 1140 } 1141 } 1142 free(vsi->tx_queues, M_ICE); 1143 vsi->tx_queues = NULL; 1144 vsi->num_tx_queues = 0; 1145 } 1146 if (vsi->rx_queues != NULL) { 1147 free(vsi->rx_queues, M_ICE); 1148 vsi->rx_queues = NULL; 1149 vsi->num_rx_queues = 0; 1150 } 1151 } 1152 1153 /** 1154 * ice_msix_que - Fast interrupt handler for MSI-X receive queues 1155 * @arg: The Rx queue memory 1156 * 1157 * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when 1158 * an MSI-X interrupt for a given queue is triggered. Currently this just asks 1159 * iflib to schedule the main Rx thread. 1160 */ 1161 static int 1162 ice_msix_que(void *arg) 1163 { 1164 struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg; 1165 1166 /* TODO: dynamic ITR algorithm?? */ 1167 1168 return (FILTER_SCHEDULE_THREAD); 1169 } 1170 1171 /** 1172 * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt 1173 * @arg: pointer to device softc memory 1174 * 1175 * Called by iflib when an administrative interrupt occurs. Should perform any 1176 * fast logic for handling the interrupt cause, and then indicate whether the 1177 * admin task needs to be queued. 1178 */ 1179 static int 1180 ice_msix_admin(void *arg) 1181 { 1182 struct ice_softc *sc = (struct ice_softc *)arg; 1183 struct ice_hw *hw = &sc->hw; 1184 device_t dev = sc->dev; 1185 u32 oicr; 1186 1187 /* There is no safe way to modify the enabled miscellaneous causes of 1188 * the OICR vector at runtime, as doing so would be prone to race 1189 * conditions. Reading PFINT_OICR will unmask the associated interrupt 1190 * causes and allow future interrupts to occur. The admin interrupt 1191 * vector will not be re-enabled until after we exit this function, 1192 * but any delayed tasks must be resilient against possible "late 1193 * arrival" interrupts that occur while we're already handling the 1194 * task. This is done by using state bits and serializing these 1195 * delayed tasks via the admin status task function. 1196 */ 1197 oicr = rd32(hw, PFINT_OICR); 1198 1199 /* Processing multiple controlq interrupts on a single vector does not 1200 * provide an indication of which controlq triggered the interrupt. 1201 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL 1202 * registers. However, the INTEVENT bit is not guaranteed to be set as 1203 * it gets automatically cleared when the hardware acknowledges the 1204 * interrupt. 1205 * 1206 * This means we don't really have a good indication of whether or 1207 * which controlq triggered this interrupt. We'll just notify the 1208 * admin task that it should check all the controlqs. 1209 */ 1210 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 1211 1212 if (oicr & PFINT_OICR_VFLR_M) { 1213 ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING); 1214 } 1215 1216 if (oicr & PFINT_OICR_MAL_DETECT_M) { 1217 ice_set_state(&sc->state, ICE_STATE_MDD_PENDING); 1218 } 1219 1220 if (oicr & PFINT_OICR_GRST_M) { 1221 u32 reset; 1222 1223 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1224 GLGEN_RSTAT_RESET_TYPE_S; 1225 1226 if (reset == ICE_RESET_CORER) 1227 sc->soft_stats.corer_count++; 1228 else if (reset == ICE_RESET_GLOBR) 1229 sc->soft_stats.globr_count++; 1230 else 1231 sc->soft_stats.empr_count++; 1232 1233 /* There are a couple of bits at play for handling resets. 1234 * First, the ICE_STATE_RESET_OICR_RECV bit is used to 1235 * indicate that the driver has received an OICR with a reset 1236 * bit active, indicating that a CORER/GLOBR/EMPR is about to 1237 * happen. Second, we set hw->reset_ongoing to indicate that 1238 * the hardware is in reset. We will set this back to false as 1239 * soon as the driver has determined that the hardware is out 1240 * of reset. 1241 * 1242 * If the driver wishes to trigger a reqest, it can set one of 1243 * the ICE_STATE_RESET_*_REQ bits, which will trigger the 1244 * correct type of reset. 1245 */ 1246 if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 1247 hw->reset_ongoing = true; 1248 } 1249 1250 if (oicr & PFINT_OICR_ECC_ERR_M) { 1251 device_printf(dev, "ECC Error detected!\n"); 1252 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1253 } 1254 1255 if (oicr & PFINT_OICR_PE_CRITERR_M) { 1256 device_printf(dev, "Critical Protocol Engine Error detected!\n"); 1257 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1258 } 1259 1260 if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { 1261 device_printf(dev, "PCI Exception detected!\n"); 1262 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1263 } 1264 1265 if (oicr & PFINT_OICR_HMC_ERR_M) { 1266 /* Log the HMC errors, but don't disable the interrupt cause */ 1267 ice_log_hmc_error(hw, dev); 1268 } 1269 1270 return (FILTER_SCHEDULE_THREAD); 1271 } 1272 1273 /** 1274 * ice_allocate_msix - Allocate MSI-X vectors for the interface 1275 * @sc: the device private softc 1276 * 1277 * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process. 1278 * 1279 * First, determine a suitable total number of vectors based on the number 1280 * of CPUs, RSS buckets, the administrative vector, and other demands such as 1281 * RDMA. 1282 * 1283 * Request the desired amount of vectors, and see how many we obtain. If we 1284 * don't obtain as many as desired, reduce the demands by lowering the number 1285 * of requested queues or reducing the demand from other features such as 1286 * RDMA. 1287 * 1288 * @remark This function is required because the driver sets the 1289 * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors 1290 * manually. 1291 * 1292 * @remark This driver will only use MSI-X vectors. If this is not possible, 1293 * neither MSI or legacy interrupts will be tried. 1294 * 1295 * @post on success this function must set the following scctx parameters: 1296 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr. 1297 * 1298 * @returns zero on success or an error code on failure. 1299 */ 1300 static int 1301 ice_allocate_msix(struct ice_softc *sc) 1302 { 1303 bool iflib_override_queue_count = false; 1304 if_softc_ctx_t scctx = sc->scctx; 1305 device_t dev = sc->dev; 1306 cpuset_t cpus; 1307 int bar, queues, vectors, requested; 1308 int err = 0; 1309 1310 /* Allocate the MSI-X bar */ 1311 bar = scctx->isc_msix_bar; 1312 sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); 1313 if (!sc->msix_table) { 1314 device_printf(dev, "Unable to map MSI-X table\n"); 1315 return (ENOMEM); 1316 } 1317 1318 /* Check if the iflib queue count sysctls have been set */ 1319 if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs) 1320 iflib_override_queue_count = true; 1321 1322 err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus); 1323 if (err) { 1324 device_printf(dev, "%s: Unable to fetch the CPU list: %s\n", 1325 __func__, ice_err_str(err)); 1326 CPU_COPY(&all_cpus, &cpus); 1327 } 1328 1329 /* Attempt to mimic behavior of iflib_msix_init */ 1330 if (iflib_override_queue_count) { 1331 /* 1332 * If the override sysctls have been set, limit the queues to 1333 * the number of logical CPUs. 1334 */ 1335 queues = mp_ncpus; 1336 } else { 1337 /* 1338 * Otherwise, limit the queue count to the CPUs associated 1339 * with the NUMA node the device is associated with. 1340 */ 1341 queues = CPU_COUNT(&cpus); 1342 } 1343 1344 /* Clamp to the number of RSS buckets */ 1345 queues = imin(queues, rss_getnumbuckets()); 1346 1347 /* 1348 * Clamp the number of queue pairs to the minimum of the requested Tx 1349 * and Rx queues. 1350 */ 1351 queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets); 1352 queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets); 1353 1354 /* 1355 * Determine the number of vectors to request. Note that we also need 1356 * to allocate one vector for administrative tasks. 1357 */ 1358 requested = queues + 1; 1359 1360 vectors = requested; 1361 1362 err = pci_alloc_msix(dev, &vectors); 1363 if (err) { 1364 device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n", 1365 vectors, ice_err_str(err)); 1366 goto err_free_msix_table; 1367 } 1368 1369 /* If we don't receive enough vectors, reduce demands */ 1370 if (vectors < requested) { 1371 int diff = requested - vectors; 1372 1373 device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n", 1374 requested, vectors); 1375 1376 /* 1377 * If we still have a difference, we need to reduce the number 1378 * of queue pairs. 1379 * 1380 * However, we still need at least one vector for the admin 1381 * interrupt and one queue pair. 1382 */ 1383 if (queues <= diff) { 1384 device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n"); 1385 err = (ERANGE); 1386 goto err_pci_release_msi; 1387 } 1388 1389 queues -= diff; 1390 } 1391 1392 device_printf(dev, "Using %d Tx and Rx queues\n", queues); 1393 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 1394 vectors); 1395 1396 scctx->isc_vectors = vectors; 1397 scctx->isc_nrxqsets = queues; 1398 scctx->isc_ntxqsets = queues; 1399 scctx->isc_intr = IFLIB_INTR_MSIX; 1400 1401 /* Interrupt allocation tracking isn't required in recovery mode, 1402 * since neither RDMA nor VFs are enabled. 1403 */ 1404 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1405 return (0); 1406 1407 /* Keep track of which interrupt indices are being used for what */ 1408 sc->lan_vectors = vectors; 1409 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors); 1410 if (err) { 1411 device_printf(dev, "Unable to assign PF interrupt mapping: %s\n", 1412 ice_err_str(err)); 1413 goto err_pci_release_msi; 1414 } 1415 1416 return (0); 1417 1418 err_pci_release_msi: 1419 pci_release_msi(dev); 1420 err_free_msix_table: 1421 if (sc->msix_table != NULL) { 1422 bus_release_resource(sc->dev, SYS_RES_MEMORY, 1423 rman_get_rid(sc->msix_table), 1424 sc->msix_table); 1425 sc->msix_table = NULL; 1426 } 1427 1428 return (err); 1429 } 1430 1431 /** 1432 * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues 1433 * @ctx: the iflib context structure 1434 * @msix: the number of vectors we were assigned 1435 * 1436 * Called by iflib to assign MSI-X vectors to queues. Currently requires that 1437 * we get at least the same number of vectors as we have queues, and that we 1438 * always have the same number of Tx and Rx queues. 1439 * 1440 * Tx queues use a softirq instead of using their own hardware interrupt. 1441 */ 1442 static int 1443 ice_if_msix_intr_assign(if_ctx_t ctx, int msix) 1444 { 1445 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1446 struct ice_vsi *vsi = &sc->pf_vsi; 1447 int err, i, vector; 1448 1449 ASSERT_CTX_LOCKED(sc); 1450 1451 if (vsi->num_rx_queues != vsi->num_tx_queues) { 1452 device_printf(sc->dev, 1453 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 1454 vsi->num_tx_queues, vsi->num_rx_queues); 1455 return (EOPNOTSUPP); 1456 } 1457 1458 if (msix < (vsi->num_rx_queues + 1)) { 1459 device_printf(sc->dev, 1460 "Not enough MSI-X vectors to assign one vector to each queue pair\n"); 1461 return (EOPNOTSUPP); 1462 } 1463 1464 /* Save the number of vectors for future use */ 1465 sc->num_irq_vectors = vsi->num_rx_queues + 1; 1466 1467 /* Allocate space to store the IRQ vector data */ 1468 if (!(sc->irqvs = 1469 (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors), 1470 M_ICE, M_NOWAIT))) { 1471 device_printf(sc->dev, 1472 "Unable to allocate irqv memory\n"); 1473 return (ENOMEM); 1474 } 1475 1476 /* Administrative interrupt events will use vector 0 */ 1477 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN, 1478 ice_msix_admin, sc, 0, "admin"); 1479 if (err) { 1480 device_printf(sc->dev, 1481 "Failed to register Admin queue handler: %s\n", 1482 ice_err_str(err)); 1483 goto free_irqvs; 1484 } 1485 sc->irqvs[0].me = 0; 1486 1487 /* Do not allocate queue interrupts when in recovery mode */ 1488 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1489 return (0); 1490 1491 for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) { 1492 struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 1493 struct ice_tx_queue *txq = &vsi->tx_queues[i]; 1494 int rid = vector + 1; 1495 char irq_name[16]; 1496 1497 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1498 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid, 1499 IFLIB_INTR_RXTX, ice_msix_que, 1500 rxq, rxq->me, irq_name); 1501 if (err) { 1502 device_printf(sc->dev, 1503 "Failed to allocate q int %d err: %s\n", 1504 i, ice_err_str(err)); 1505 vector--; 1506 i--; 1507 goto fail; 1508 } 1509 sc->irqvs[vector].me = vector; 1510 rxq->irqv = &sc->irqvs[vector]; 1511 1512 bzero(irq_name, sizeof(irq_name)); 1513 1514 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1515 iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq, 1516 IFLIB_INTR_TX, txq, 1517 txq->me, irq_name); 1518 txq->irqv = &sc->irqvs[vector]; 1519 } 1520 1521 return (0); 1522 fail: 1523 for (; i >= 0; i--, vector--) 1524 iflib_irq_free(ctx, &sc->irqvs[vector].irq); 1525 iflib_irq_free(ctx, &sc->irqvs[0].irq); 1526 free_irqvs: 1527 free(sc->irqvs, M_ICE); 1528 sc->irqvs = NULL; 1529 return err; 1530 } 1531 1532 /** 1533 * ice_if_mtu_set - Set the device MTU 1534 * @ctx: iflib context structure 1535 * @mtu: the MTU requested 1536 * 1537 * Called by iflib to configure the device's Maximum Transmission Unit (MTU). 1538 * 1539 * @pre assumes the caller holds the iflib CTX lock 1540 */ 1541 static int 1542 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1543 { 1544 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1545 1546 ASSERT_CTX_LOCKED(sc); 1547 1548 /* Do not support configuration when in recovery mode */ 1549 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1550 return (ENOSYS); 1551 1552 if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU) 1553 return (EINVAL); 1554 1555 sc->scctx->isc_max_frame_size = mtu + 1556 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 1557 1558 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 1559 1560 return (0); 1561 } 1562 1563 /** 1564 * ice_if_intr_enable - Enable device interrupts 1565 * @ctx: iflib context structure 1566 * 1567 * Called by iflib to request enabling device interrupts. 1568 */ 1569 static void 1570 ice_if_intr_enable(if_ctx_t ctx) 1571 { 1572 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1573 struct ice_vsi *vsi = &sc->pf_vsi; 1574 struct ice_hw *hw = &sc->hw; 1575 1576 ASSERT_CTX_LOCKED(sc); 1577 1578 /* Enable ITR 0 */ 1579 ice_enable_intr(hw, sc->irqvs[0].me); 1580 1581 /* Do not enable queue interrupts in recovery mode */ 1582 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1583 return; 1584 1585 /* Enable all queue interrupts */ 1586 for (int i = 0; i < vsi->num_rx_queues; i++) 1587 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 1588 } 1589 1590 /** 1591 * ice_if_intr_disable - Disable device interrupts 1592 * @ctx: iflib context structure 1593 * 1594 * Called by iflib to request disabling device interrupts. 1595 */ 1596 static void 1597 ice_if_intr_disable(if_ctx_t ctx) 1598 { 1599 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1600 struct ice_hw *hw = &sc->hw; 1601 unsigned int i; 1602 1603 ASSERT_CTX_LOCKED(sc); 1604 1605 /* IFDI_INTR_DISABLE may be called prior to interrupts actually being 1606 * assigned to queues. Instead of assuming that the interrupt 1607 * assignment in the rx_queues structure is valid, just disable all 1608 * possible interrupts 1609 * 1610 * Note that we choose not to disable ITR 0 because this handles the 1611 * AdminQ interrupts, and we want to keep processing these even when 1612 * the interface is offline. 1613 */ 1614 for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++) 1615 ice_disable_intr(hw, i); 1616 } 1617 1618 /** 1619 * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 1620 * @ctx: iflib context structure 1621 * @rxqid: the Rx queue to enable 1622 * 1623 * Enable a specific Rx queue interrupt. 1624 * 1625 * This function is not protected by the iflib CTX lock. 1626 */ 1627 static int 1628 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1629 { 1630 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1631 struct ice_vsi *vsi = &sc->pf_vsi; 1632 struct ice_hw *hw = &sc->hw; 1633 1634 /* Do not enable queue interrupts in recovery mode */ 1635 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1636 return (ENOSYS); 1637 1638 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 1639 return (0); 1640 } 1641 1642 /** 1643 * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 1644 * @ctx: iflib context structure 1645 * @txqid: the Tx queue to enable 1646 * 1647 * Enable a specific Tx queue interrupt. 1648 * 1649 * This function is not protected by the iflib CTX lock. 1650 */ 1651 static int 1652 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1653 { 1654 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1655 struct ice_vsi *vsi = &sc->pf_vsi; 1656 struct ice_hw *hw = &sc->hw; 1657 1658 /* Do not enable queue interrupts in recovery mode */ 1659 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1660 return (ENOSYS); 1661 1662 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 1663 return (0); 1664 } 1665 1666 /** 1667 * ice_if_promisc_set - Set device promiscuous mode 1668 * @ctx: iflib context structure 1669 * @flags: promiscuous flags to configure 1670 * 1671 * Called by iflib to configure device promiscuous mode. 1672 * 1673 * @remark Calls to this function will always overwrite the previous setting 1674 */ 1675 static int 1676 ice_if_promisc_set(if_ctx_t ctx, int flags) 1677 { 1678 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1679 struct ice_hw *hw = &sc->hw; 1680 device_t dev = sc->dev; 1681 enum ice_status status; 1682 bool promisc_enable = flags & IFF_PROMISC; 1683 bool multi_enable = flags & IFF_ALLMULTI; 1684 1685 /* Do not support configuration when in recovery mode */ 1686 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1687 return (ENOSYS); 1688 1689 if (multi_enable) 1690 return (EOPNOTSUPP); 1691 1692 if (promisc_enable) { 1693 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1694 ICE_VSI_PROMISC_MASK, 0); 1695 if (status && status != ICE_ERR_ALREADY_EXISTS) { 1696 device_printf(dev, 1697 "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n", 1698 ice_status_str(status), 1699 ice_aq_str(hw->adminq.sq_last_status)); 1700 return (EIO); 1701 } 1702 } else { 1703 status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx, 1704 ICE_VSI_PROMISC_MASK, 0); 1705 if (status) { 1706 device_printf(dev, 1707 "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n", 1708 ice_status_str(status), 1709 ice_aq_str(hw->adminq.sq_last_status)); 1710 return (EIO); 1711 } 1712 } 1713 1714 return (0); 1715 } 1716 1717 /** 1718 * ice_if_media_change - Change device media 1719 * @ctx: device ctx structure 1720 * 1721 * Called by iflib when a media change is requested. This operation is not 1722 * supported by the hardware, so we just return an error code. 1723 */ 1724 static int 1725 ice_if_media_change(if_ctx_t ctx) 1726 { 1727 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1728 1729 device_printf(sc->dev, "Media change is not supported.\n"); 1730 return (ENODEV); 1731 } 1732 1733 /** 1734 * ice_if_media_status - Report current device media 1735 * @ctx: iflib context structure 1736 * @ifmr: ifmedia request structure to update 1737 * 1738 * Updates the provided ifmr with current device media status, including link 1739 * status and media type. 1740 */ 1741 static void 1742 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1743 { 1744 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1745 struct ice_link_status *li = &sc->hw.port_info->phy.link_info; 1746 1747 ifmr->ifm_status = IFM_AVALID; 1748 ifmr->ifm_active = IFM_ETHER; 1749 1750 /* Never report link up or media types when in recovery mode */ 1751 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1752 return; 1753 1754 if (!sc->link_up) 1755 return; 1756 1757 ifmr->ifm_status |= IFM_ACTIVE; 1758 ifmr->ifm_active |= IFM_FDX; 1759 1760 if (li->phy_type_low) 1761 ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low); 1762 else if (li->phy_type_high) 1763 ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high); 1764 else 1765 ifmr->ifm_active |= IFM_UNKNOWN; 1766 1767 /* Report flow control status as well */ 1768 if (li->an_info & ICE_AQ_LINK_PAUSE_TX) 1769 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1770 if (li->an_info & ICE_AQ_LINK_PAUSE_RX) 1771 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1772 } 1773 1774 /** 1775 * ice_init_tx_tracking - Initialize Tx queue software tracking values 1776 * @vsi: the VSI to initialize 1777 * 1778 * Initialize Tx queue software tracking values, including the Report Status 1779 * queue, and related software tracking values. 1780 */ 1781 static void 1782 ice_init_tx_tracking(struct ice_vsi *vsi) 1783 { 1784 struct ice_tx_queue *txq; 1785 size_t j; 1786 int i; 1787 1788 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1789 1790 txq->tx_rs_cidx = txq->tx_rs_pidx = 0; 1791 1792 /* Initialize the last processed descriptor to be the end of 1793 * the ring, rather than the start, so that we avoid an 1794 * off-by-one error in ice_ift_txd_credits_update for the 1795 * first packet. 1796 */ 1797 txq->tx_cidx_processed = txq->desc_count - 1; 1798 1799 for (j = 0; j < txq->desc_count; j++) 1800 txq->tx_rsq[j] = QIDX_INVALID; 1801 } 1802 } 1803 1804 /** 1805 * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues 1806 * @sc: the device softc 1807 * 1808 * Called to update the Rx queue mbuf_sz parameter for configuring the receive 1809 * buffer sizes when programming hardware. 1810 */ 1811 static void 1812 ice_update_rx_mbuf_sz(struct ice_softc *sc) 1813 { 1814 uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx); 1815 struct ice_vsi *vsi = &sc->pf_vsi; 1816 1817 MPASS(mbuf_sz <= UINT16_MAX); 1818 vsi->mbuf_sz = mbuf_sz; 1819 } 1820 1821 /** 1822 * ice_if_init - Initialize the device 1823 * @ctx: iflib ctx structure 1824 * 1825 * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes 1826 * device filters and prepares the Tx and Rx engines. 1827 * 1828 * @pre assumes the caller holds the iflib CTX lock 1829 */ 1830 static void 1831 ice_if_init(if_ctx_t ctx) 1832 { 1833 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1834 device_t dev = sc->dev; 1835 int err; 1836 1837 ASSERT_CTX_LOCKED(sc); 1838 1839 /* 1840 * We've seen an issue with 11.3/12.1 where sideband routines are 1841 * called after detach is called. This would call routines after 1842 * if_stop, causing issues with the teardown process. This has 1843 * seemingly been fixed in STABLE snapshots, but it seems like a 1844 * good idea to have this guard here regardless. 1845 */ 1846 if (ice_driver_is_detaching(sc)) 1847 return; 1848 1849 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1850 return; 1851 1852 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 1853 device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n"); 1854 return; 1855 } 1856 1857 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 1858 device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n"); 1859 return; 1860 } 1861 1862 ice_update_rx_mbuf_sz(sc); 1863 1864 /* Update the MAC address... User might use a LAA */ 1865 err = ice_update_laa_mac(sc); 1866 if (err) { 1867 device_printf(dev, 1868 "LAA address change failed, err %s\n", 1869 ice_err_str(err)); 1870 return; 1871 } 1872 1873 /* Initialize software Tx tracking values */ 1874 ice_init_tx_tracking(&sc->pf_vsi); 1875 1876 err = ice_cfg_vsi_for_tx(&sc->pf_vsi); 1877 if (err) { 1878 device_printf(dev, 1879 "Unable to configure the main VSI for Tx: %s\n", 1880 ice_err_str(err)); 1881 return; 1882 } 1883 1884 err = ice_cfg_vsi_for_rx(&sc->pf_vsi); 1885 if (err) { 1886 device_printf(dev, 1887 "Unable to configure the main VSI for Rx: %s\n", 1888 ice_err_str(err)); 1889 goto err_cleanup_tx; 1890 } 1891 1892 err = ice_control_rx_queues(&sc->pf_vsi, true); 1893 if (err) { 1894 device_printf(dev, 1895 "Unable to enable Rx rings for transmit: %s\n", 1896 ice_err_str(err)); 1897 goto err_cleanup_tx; 1898 } 1899 1900 err = ice_cfg_pf_default_mac_filters(sc); 1901 if (err) { 1902 device_printf(dev, 1903 "Unable to configure default MAC filters: %s\n", 1904 ice_err_str(err)); 1905 goto err_stop_rx; 1906 } 1907 1908 /* We use software interrupts for Tx, so we only program the hardware 1909 * interrupts for Rx. 1910 */ 1911 ice_configure_rxq_interrupts(&sc->pf_vsi); 1912 ice_configure_rx_itr(&sc->pf_vsi); 1913 1914 /* Configure promiscuous mode */ 1915 ice_if_promisc_set(ctx, if_getflags(sc->ifp)); 1916 1917 ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED); 1918 return; 1919 1920 err_stop_rx: 1921 ice_control_rx_queues(&sc->pf_vsi, false); 1922 err_cleanup_tx: 1923 ice_vsi_disable_tx(&sc->pf_vsi); 1924 } 1925 1926 /** 1927 * ice_poll_for_media_avail - Re-enable link if media is detected 1928 * @sc: device private structure 1929 * 1930 * Intended to be called from the driver's timer function, this function 1931 * sends the Get Link Status AQ command and re-enables HW link if the 1932 * command says that media is available. 1933 * 1934 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing, 1935 * since media removal events are supposed to be sent to the driver through 1936 * a link status event. 1937 */ 1938 static void 1939 ice_poll_for_media_avail(struct ice_softc *sc) 1940 { 1941 struct ice_hw *hw = &sc->hw; 1942 struct ice_port_info *pi = hw->port_info; 1943 1944 if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) { 1945 pi->phy.get_link_info = true; 1946 ice_get_link_status(pi, &sc->link_up); 1947 1948 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 1949 enum ice_status status; 1950 1951 /* Re-enable link and re-apply user link settings */ 1952 ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC); 1953 1954 /* Update the OS about changes in media capability */ 1955 status = ice_add_media_types(sc, sc->media); 1956 if (status) 1957 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 1958 ice_status_str(status), 1959 ice_aq_str(hw->adminq.sq_last_status)); 1960 1961 ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA); 1962 } 1963 } 1964 } 1965 1966 /** 1967 * ice_if_timer - called by iflib periodically 1968 * @ctx: iflib ctx structure 1969 * @qid: the queue this timer was called for 1970 * 1971 * This callback is triggered by iflib periodically. We use it to update the 1972 * hw statistics. 1973 * 1974 * @remark this function is not protected by the iflib CTX lock. 1975 */ 1976 static void 1977 ice_if_timer(if_ctx_t ctx, uint16_t qid) 1978 { 1979 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1980 uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx; 1981 1982 if (qid != 0) 1983 return; 1984 1985 /* Do not attempt to update stats when in recovery mode */ 1986 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1987 return; 1988 1989 /* Update device statistics */ 1990 ice_update_pf_stats(sc); 1991 1992 /* 1993 * For proper watchdog management, the iflib stack needs to know if 1994 * we've been paused during the last interval. Check if the 1995 * link_xoff_rx stat changed, and set the isc_pause_frames, if so. 1996 */ 1997 if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx) 1998 sc->scctx->isc_pause_frames = 1; 1999 2000 /* Update the primary VSI stats */ 2001 ice_update_vsi_hw_stats(&sc->pf_vsi); 2002 } 2003 2004 /** 2005 * ice_admin_timer - called periodically to trigger the admin task 2006 * @arg: callout(9) argument pointing to the device private softc structure 2007 * 2008 * Timer function used as part of a callout(9) timer that will periodically 2009 * trigger the admin task, even when the interface is down. 2010 * 2011 * @remark this function is not called by iflib and is not protected by the 2012 * iflib CTX lock. 2013 * 2014 * @remark because this is a callout function, it cannot sleep and should not 2015 * attempt taking the iflib CTX lock. 2016 */ 2017 static void 2018 ice_admin_timer(void *arg) 2019 { 2020 struct ice_softc *sc = (struct ice_softc *)arg; 2021 2022 /* 2023 * There is a point where callout routines are no longer 2024 * cancelable. So there exists a window of time where the 2025 * driver enters detach() and tries to cancel the callout, but the 2026 * callout routine has passed the cancellation point. The detach() 2027 * routine is unaware of this and tries to free resources that the 2028 * callout routine needs. So we check for the detach state flag to 2029 * at least shrink the window of opportunity. 2030 */ 2031 if (ice_driver_is_detaching(sc)) 2032 return; 2033 2034 /* Fire off the admin task */ 2035 iflib_admin_intr_deferred(sc->ctx); 2036 2037 /* Reschedule the admin timer */ 2038 callout_schedule(&sc->admin_timer, hz/2); 2039 } 2040 2041 /** 2042 * ice_transition_recovery_mode - Transition to recovery mode 2043 * @sc: the device private softc 2044 * 2045 * Called when the driver detects that the firmware has entered recovery mode 2046 * at run time. 2047 */ 2048 static void 2049 ice_transition_recovery_mode(struct ice_softc *sc) 2050 { 2051 struct ice_vsi *vsi = &sc->pf_vsi; 2052 int i; 2053 2054 device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 2055 2056 /* Tell the stack that the link has gone down */ 2057 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 2058 2059 /* Request that the device be re-initialized */ 2060 ice_request_stack_reinit(sc); 2061 2062 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2063 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2064 2065 ice_vsi_del_txqs_ctx(vsi); 2066 ice_vsi_del_rxqs_ctx(vsi); 2067 2068 for (i = 0; i < sc->num_available_vsi; i++) { 2069 if (sc->all_vsi[i]) 2070 ice_release_vsi(sc->all_vsi[i]); 2071 } 2072 sc->num_available_vsi = 0; 2073 2074 if (sc->all_vsi) { 2075 free(sc->all_vsi, M_ICE); 2076 sc->all_vsi = NULL; 2077 } 2078 2079 /* Destroy the interrupt manager */ 2080 ice_resmgr_destroy(&sc->imgr); 2081 /* Destroy the queue managers */ 2082 ice_resmgr_destroy(&sc->tx_qmgr); 2083 ice_resmgr_destroy(&sc->rx_qmgr); 2084 2085 ice_deinit_hw(&sc->hw); 2086 } 2087 2088 /** 2089 * ice_transition_safe_mode - Transition to safe mode 2090 * @sc: the device private softc 2091 * 2092 * Called when the driver attempts to reload the DDP package during a device 2093 * reset, and the new download fails. If so, we must transition to safe mode 2094 * at run time. 2095 * 2096 * @remark although safe mode normally allocates only a single queue, we can't 2097 * change the number of queues dynamically when using iflib. Due to this, we 2098 * do not attempt to reduce the number of queues. 2099 */ 2100 static void 2101 ice_transition_safe_mode(struct ice_softc *sc) 2102 { 2103 /* Indicate that we are in Safe mode */ 2104 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap); 2105 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en); 2106 2107 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2108 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2109 2110 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2111 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en); 2112 } 2113 2114 /** 2115 * ice_if_update_admin_status - update admin status 2116 * @ctx: iflib ctx structure 2117 * 2118 * Called by iflib to update the admin status. For our purposes, this means 2119 * check the adminq, and update the link status. It's ultimately triggered by 2120 * our admin interrupt, or by the ice_if_timer periodically. 2121 * 2122 * @pre assumes the caller holds the iflib CTX lock 2123 */ 2124 static void 2125 ice_if_update_admin_status(if_ctx_t ctx) 2126 { 2127 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2128 enum ice_fw_modes fw_mode; 2129 bool reschedule = false; 2130 u16 pending = 0; 2131 2132 ASSERT_CTX_LOCKED(sc); 2133 2134 /* Check if the firmware entered recovery mode at run time */ 2135 fw_mode = ice_get_fw_mode(&sc->hw); 2136 if (fw_mode == ICE_FW_MODE_REC) { 2137 if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2138 /* If we just entered recovery mode, log a warning to 2139 * the system administrator and deinit driver state 2140 * that is no longer functional. 2141 */ 2142 ice_transition_recovery_mode(sc); 2143 } 2144 } else if (fw_mode == ICE_FW_MODE_ROLLBACK) { 2145 if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) { 2146 /* Rollback mode isn't fatal, but we don't want to 2147 * repeatedly post a message about it. 2148 */ 2149 ice_print_rollback_msg(&sc->hw); 2150 } 2151 } 2152 2153 /* Handle global reset events */ 2154 ice_handle_reset_event(sc); 2155 2156 /* Handle PF reset requests */ 2157 ice_handle_pf_reset_request(sc); 2158 2159 /* Handle MDD events */ 2160 ice_handle_mdd_event(sc); 2161 2162 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) || 2163 ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) || 2164 ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2165 /* 2166 * If we know the control queues are disabled, skip processing 2167 * the control queues entirely. 2168 */ 2169 ; 2170 } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) { 2171 ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending); 2172 if (pending > 0) 2173 reschedule = true; 2174 2175 ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending); 2176 if (pending > 0) 2177 reschedule = true; 2178 } 2179 2180 /* Poll for link up */ 2181 ice_poll_for_media_avail(sc); 2182 2183 /* Check and update link status */ 2184 ice_update_link_status(sc, false); 2185 2186 /* 2187 * If there are still messages to process, we need to reschedule 2188 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be 2189 * woken up at the next interrupt or timer event. 2190 */ 2191 if (reschedule) { 2192 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 2193 iflib_admin_intr_deferred(ctx); 2194 } else { 2195 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2196 } 2197 } 2198 2199 /** 2200 * ice_prepare_for_reset - Prepare device for an impending reset 2201 * @sc: The device private softc 2202 * 2203 * Prepare the driver for an impending reset, shutting down VSIs, clearing the 2204 * scheduler setup, and shutting down controlqs. Uses the 2205 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the 2206 * driver for reset or not. 2207 */ 2208 static void 2209 ice_prepare_for_reset(struct ice_softc *sc) 2210 { 2211 struct ice_hw *hw = &sc->hw; 2212 2213 /* If we're already prepared, there's nothing to do */ 2214 if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) 2215 return; 2216 2217 log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname); 2218 2219 /* In recovery mode, hardware is not initialized */ 2220 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2221 return; 2222 2223 /* Release the main PF VSI queue mappings */ 2224 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2225 sc->pf_vsi.num_tx_queues); 2226 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2227 sc->pf_vsi.num_rx_queues); 2228 2229 ice_clear_hw_tbls(hw); 2230 2231 if (hw->port_info) 2232 ice_sched_clear_port(hw->port_info); 2233 2234 ice_shutdown_all_ctrlq(hw); 2235 } 2236 2237 /** 2238 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping 2239 * @sc: the device softc pointer 2240 * 2241 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue 2242 * mapping after a reset occurred. 2243 */ 2244 static int 2245 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc) 2246 { 2247 struct ice_vsi *vsi = &sc->pf_vsi; 2248 struct ice_tx_queue *txq; 2249 struct ice_rx_queue *rxq; 2250 int err, i; 2251 2252 /* Re-assign Tx queues from PF space to the main VSI */ 2253 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, 2254 vsi->num_tx_queues); 2255 if (err) { 2256 device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n", 2257 ice_err_str(err)); 2258 return (err); 2259 } 2260 2261 /* Re-assign Rx queues from PF space to this VSI */ 2262 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, 2263 vsi->num_rx_queues); 2264 if (err) { 2265 device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n", 2266 ice_err_str(err)); 2267 goto err_release_tx_queues; 2268 } 2269 2270 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 2271 2272 /* Re-assign Tx queue tail pointers */ 2273 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 2274 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 2275 2276 /* Re-assign Rx queue tail pointers */ 2277 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 2278 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 2279 2280 return (0); 2281 2282 err_release_tx_queues: 2283 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2284 sc->pf_vsi.num_tx_queues); 2285 2286 return (err); 2287 } 2288 2289 /* determine if the iflib context is active */ 2290 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING)) 2291 2292 /** 2293 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode 2294 * @sc: The device private softc 2295 * 2296 * Handle a driver rebuild while in recovery mode. This will only rebuild the 2297 * limited functionality supported while in recovery mode. 2298 */ 2299 static void 2300 ice_rebuild_recovery_mode(struct ice_softc *sc) 2301 { 2302 device_t dev = sc->dev; 2303 2304 /* enable PCIe bus master */ 2305 pci_enable_busmaster(dev); 2306 2307 /* Configure interrupt causes for the administrative interrupt */ 2308 ice_configure_misc_interrupts(sc); 2309 2310 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2311 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2312 2313 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2314 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2315 2316 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2317 2318 /* In order to completely restore device functionality, the iflib core 2319 * needs to be reset. We need to request an iflib reset. Additionally, 2320 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2321 * the iflib core, we also want re-run the admin task so that iflib 2322 * resets immediately instead of waiting for the next interrupt. 2323 */ 2324 ice_request_stack_reinit(sc); 2325 2326 return; 2327 } 2328 2329 /** 2330 * ice_rebuild - Rebuild driver state post reset 2331 * @sc: The device private softc 2332 * 2333 * Restore driver state after a reset occurred. Restart the controlqs, setup 2334 * the hardware port, and re-enable the VSIs. 2335 */ 2336 static void 2337 ice_rebuild(struct ice_softc *sc) 2338 { 2339 struct ice_hw *hw = &sc->hw; 2340 device_t dev = sc->dev; 2341 enum ice_status status; 2342 int err; 2343 2344 sc->rebuild_ticks = ticks; 2345 2346 /* If we're rebuilding, then a reset has succeeded. */ 2347 ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED); 2348 2349 /* 2350 * If the firmware is in recovery mode, only restore the limited 2351 * functionality supported by recovery mode. 2352 */ 2353 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2354 ice_rebuild_recovery_mode(sc); 2355 return; 2356 } 2357 2358 /* enable PCIe bus master */ 2359 pci_enable_busmaster(dev); 2360 2361 status = ice_init_all_ctrlq(hw); 2362 if (status) { 2363 device_printf(dev, "failed to re-init controlqs, err %s\n", 2364 ice_status_str(status)); 2365 goto err_shutdown_ctrlq; 2366 } 2367 2368 /* Query the allocated resources for Tx scheduler */ 2369 status = ice_sched_query_res_alloc(hw); 2370 if (status) { 2371 device_printf(dev, 2372 "Failed to query scheduler resources, err %s aq_err %s\n", 2373 ice_status_str(status), 2374 ice_aq_str(hw->adminq.sq_last_status)); 2375 goto err_shutdown_ctrlq; 2376 } 2377 2378 err = ice_send_version(sc); 2379 if (err) 2380 goto err_shutdown_ctrlq; 2381 2382 err = ice_init_link_events(sc); 2383 if (err) { 2384 device_printf(dev, "ice_init_link_events failed: %s\n", 2385 ice_err_str(err)); 2386 goto err_shutdown_ctrlq; 2387 } 2388 2389 status = ice_clear_pf_cfg(hw); 2390 if (status) { 2391 device_printf(dev, "failed to clear PF configuration, err %s\n", 2392 ice_status_str(status)); 2393 goto err_shutdown_ctrlq; 2394 } 2395 2396 ice_clear_pxe_mode(hw); 2397 2398 status = ice_get_caps(hw); 2399 if (status) { 2400 device_printf(dev, "failed to get capabilities, err %s\n", 2401 ice_status_str(status)); 2402 goto err_shutdown_ctrlq; 2403 } 2404 2405 status = ice_sched_init_port(hw->port_info); 2406 if (status) { 2407 device_printf(dev, "failed to initialize port, err %s\n", 2408 ice_status_str(status)); 2409 goto err_sched_cleanup; 2410 } 2411 2412 /* If we previously loaded the package, it needs to be reloaded now */ 2413 if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) { 2414 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 2415 if (status) { 2416 ice_log_pkg_init(sc, &status); 2417 2418 ice_transition_safe_mode(sc); 2419 } 2420 } 2421 2422 ice_reset_pf_stats(sc); 2423 2424 err = ice_rebuild_pf_vsi_qmap(sc); 2425 if (err) { 2426 device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n", 2427 ice_err_str(err)); 2428 goto err_sched_cleanup; 2429 } 2430 err = ice_initialize_vsi(&sc->pf_vsi); 2431 if (err) { 2432 device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n", 2433 ice_err_str(err)); 2434 goto err_release_queue_allocations; 2435 } 2436 2437 /* Replay all VSI configuration */ 2438 err = ice_replay_all_vsi_cfg(sc); 2439 if (err) 2440 goto err_deinit_pf_vsi; 2441 2442 /* Re-enable FW health event reporting */ 2443 ice_init_health_events(sc); 2444 2445 /* Reconfigure the main PF VSI for RSS */ 2446 err = ice_config_rss(&sc->pf_vsi); 2447 if (err) { 2448 device_printf(sc->dev, 2449 "Unable to reconfigure RSS for the main VSI, err %s\n", 2450 ice_err_str(err)); 2451 goto err_deinit_pf_vsi; 2452 } 2453 2454 /* Refresh link status */ 2455 ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED); 2456 sc->hw.port_info->phy.get_link_info = true; 2457 ice_get_link_status(sc->hw.port_info, &sc->link_up); 2458 ice_update_link_status(sc, true); 2459 2460 /* Configure interrupt causes for the administrative interrupt */ 2461 ice_configure_misc_interrupts(sc); 2462 2463 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2464 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2465 2466 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2467 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2468 2469 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2470 2471 /* In order to completely restore device functionality, the iflib core 2472 * needs to be reset. We need to request an iflib reset. Additionally, 2473 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2474 * the iflib core, we also want re-run the admin task so that iflib 2475 * resets immediately instead of waiting for the next interrupt. 2476 */ 2477 ice_request_stack_reinit(sc); 2478 2479 return; 2480 2481 err_deinit_pf_vsi: 2482 ice_deinit_vsi(&sc->pf_vsi); 2483 err_release_queue_allocations: 2484 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2485 sc->pf_vsi.num_tx_queues); 2486 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2487 sc->pf_vsi.num_rx_queues); 2488 err_sched_cleanup: 2489 ice_sched_cleanup_all(hw); 2490 err_shutdown_ctrlq: 2491 ice_shutdown_all_ctrlq(hw); 2492 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2493 device_printf(dev, "Driver rebuild failed, please reload the device driver\n"); 2494 } 2495 2496 /** 2497 * ice_handle_reset_event - Handle reset events triggered by OICR 2498 * @sc: The device private softc 2499 * 2500 * Handle reset events triggered by an OICR notification. This includes CORER, 2501 * GLOBR, and EMPR resets triggered by software on this or any other PF or by 2502 * firmware. 2503 * 2504 * @pre assumes the iflib context lock is held, and will unlock it while 2505 * waiting for the hardware to finish reset. 2506 */ 2507 static void 2508 ice_handle_reset_event(struct ice_softc *sc) 2509 { 2510 struct ice_hw *hw = &sc->hw; 2511 enum ice_status status; 2512 device_t dev = sc->dev; 2513 2514 /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will 2515 * trigger an OICR interrupt. Our OICR handler will determine when 2516 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as 2517 * appropriate. 2518 */ 2519 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 2520 return; 2521 2522 ice_prepare_for_reset(sc); 2523 2524 /* 2525 * Release the iflib context lock and wait for the device to finish 2526 * resetting. 2527 */ 2528 IFLIB_CTX_UNLOCK(sc); 2529 status = ice_check_reset(hw); 2530 IFLIB_CTX_LOCK(sc); 2531 if (status) { 2532 device_printf(dev, "Device never came out of reset, err %s\n", 2533 ice_status_str(status)); 2534 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2535 return; 2536 } 2537 2538 /* We're done with the reset, so we can rebuild driver state */ 2539 sc->hw.reset_ongoing = false; 2540 ice_rebuild(sc); 2541 2542 /* In the unlikely event that a PF reset request occurs at the same 2543 * time as a global reset, clear the request now. This avoids 2544 * resetting a second time right after we reset due to a global event. 2545 */ 2546 if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2547 device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n"); 2548 } 2549 2550 /** 2551 * ice_handle_pf_reset_request - Initiate PF reset requested by software 2552 * @sc: The device private softc 2553 * 2554 * Initiate a PF reset requested by software. We handle this in the admin task 2555 * so that only one thread actually handles driver preparation and cleanup, 2556 * rather than having multiple threads possibly attempt to run this code 2557 * simultaneously. 2558 * 2559 * @pre assumes the iflib context lock is held and will unlock it while 2560 * waiting for the PF reset to complete. 2561 */ 2562 static void 2563 ice_handle_pf_reset_request(struct ice_softc *sc) 2564 { 2565 struct ice_hw *hw = &sc->hw; 2566 enum ice_status status; 2567 2568 /* Check for PF reset requests */ 2569 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2570 return; 2571 2572 /* Make sure we're prepared for reset */ 2573 ice_prepare_for_reset(sc); 2574 2575 /* 2576 * Release the iflib context lock and wait for the device to finish 2577 * resetting. 2578 */ 2579 IFLIB_CTX_UNLOCK(sc); 2580 status = ice_reset(hw, ICE_RESET_PFR); 2581 IFLIB_CTX_LOCK(sc); 2582 if (status) { 2583 device_printf(sc->dev, "device PF reset failed, err %s\n", 2584 ice_status_str(status)); 2585 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2586 return; 2587 } 2588 2589 sc->soft_stats.pfr_count++; 2590 ice_rebuild(sc); 2591 } 2592 2593 /** 2594 * ice_init_device_features - Init device driver features 2595 * @sc: driver softc structure 2596 * 2597 * @pre assumes that the function capabilities bits have been set up by 2598 * ice_init_hw(). 2599 */ 2600 static void 2601 ice_init_device_features(struct ice_softc *sc) 2602 { 2603 /* 2604 * A failed pkg file download triggers safe mode, disabling advanced 2605 * device feature support 2606 */ 2607 if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) 2608 return; 2609 2610 /* Set capabilities that all devices support */ 2611 ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2612 ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap); 2613 ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap); 2614 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap); 2615 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap); 2616 ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2617 2618 /* Disable features due to hardware limitations... */ 2619 if (!sc->hw.func_caps.common_cap.rss_table_size) 2620 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2621 /* Disable features due to firmware limitations... */ 2622 if (!ice_is_fw_health_report_supported(&sc->hw)) 2623 ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2624 2625 /* Disable capabilities not supported by the OS */ 2626 ice_disable_unsupported_features(sc->feat_cap); 2627 2628 /* RSS is always enabled for iflib */ 2629 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS)) 2630 ice_set_bit(ICE_FEATURE_RSS, sc->feat_en); 2631 } 2632 2633 /** 2634 * ice_if_multi_set - Callback to update Multicast filters in HW 2635 * @ctx: iflib ctx structure 2636 * 2637 * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search 2638 * the if_multiaddrs list and determine which filters have been added or 2639 * removed from the list, and update HW programming to reflect the new list. 2640 * 2641 * @pre assumes the caller holds the iflib CTX lock 2642 */ 2643 static void 2644 ice_if_multi_set(if_ctx_t ctx) 2645 { 2646 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2647 int err; 2648 2649 ASSERT_CTX_LOCKED(sc); 2650 2651 /* Do not handle multicast configuration in recovery mode */ 2652 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2653 return; 2654 2655 err = ice_sync_multicast_filters(sc); 2656 if (err) { 2657 device_printf(sc->dev, 2658 "Failed to synchronize multicast filter list: %s\n", 2659 ice_err_str(err)); 2660 return; 2661 } 2662 } 2663 2664 /** 2665 * ice_if_vlan_register - Register a VLAN with the hardware 2666 * @ctx: iflib ctx pointer 2667 * @vtag: VLAN to add 2668 * 2669 * Programs the main PF VSI with a hardware filter for the given VLAN. 2670 * 2671 * @pre assumes the caller holds the iflib CTX lock 2672 */ 2673 static void 2674 ice_if_vlan_register(if_ctx_t ctx, u16 vtag) 2675 { 2676 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2677 enum ice_status status; 2678 2679 ASSERT_CTX_LOCKED(sc); 2680 2681 /* Do not handle VLAN configuration in recovery mode */ 2682 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2683 return; 2684 2685 status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag); 2686 if (status) { 2687 device_printf(sc->dev, 2688 "Failure adding VLAN %d to main VSI, err %s aq_err %s\n", 2689 vtag, ice_status_str(status), 2690 ice_aq_str(sc->hw.adminq.sq_last_status)); 2691 } 2692 } 2693 2694 /** 2695 * ice_if_vlan_unregister - Remove a VLAN filter from the hardware 2696 * @ctx: iflib ctx pointer 2697 * @vtag: VLAN to add 2698 * 2699 * Removes the previously programmed VLAN filter from the main PF VSI. 2700 * 2701 * @pre assumes the caller holds the iflib CTX lock 2702 */ 2703 static void 2704 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 2705 { 2706 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2707 enum ice_status status; 2708 2709 ASSERT_CTX_LOCKED(sc); 2710 2711 /* Do not handle VLAN configuration in recovery mode */ 2712 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2713 return; 2714 2715 status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag); 2716 if (status) { 2717 device_printf(sc->dev, 2718 "Failure removing VLAN %d from main VSI, err %s aq_err %s\n", 2719 vtag, ice_status_str(status), 2720 ice_aq_str(sc->hw.adminq.sq_last_status)); 2721 } 2722 } 2723 2724 /** 2725 * ice_if_stop - Stop the device 2726 * @ctx: iflib context structure 2727 * 2728 * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0 2729 * down) 2730 * 2731 * @pre assumes the caller holds the iflib CTX lock 2732 */ 2733 static void 2734 ice_if_stop(if_ctx_t ctx) 2735 { 2736 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2737 2738 ASSERT_CTX_LOCKED(sc); 2739 2740 /* 2741 * The iflib core may call IFDI_STOP prior to the first call to 2742 * IFDI_INIT. This will cause us to attempt to remove MAC filters we 2743 * don't have, and disable Tx queues which aren't yet configured. 2744 * Although it is likely these extra operations are harmless, they do 2745 * cause spurious warning messages to be displayed, which may confuse 2746 * users. 2747 * 2748 * To avoid these messages, we use a state bit indicating if we've 2749 * been initialized. It will be set when ice_if_init is called, and 2750 * cleared here in ice_if_stop. 2751 */ 2752 if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED)) 2753 return; 2754 2755 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 2756 device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n"); 2757 return; 2758 } 2759 2760 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 2761 device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n"); 2762 return; 2763 } 2764 2765 /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the 2766 * return of these functions because there's nothing we can really do 2767 * if they fail, and the functions already print error messages. 2768 * Just try to shut down as much as we can. 2769 */ 2770 ice_rm_pf_default_mac_filters(sc); 2771 2772 /* Dissociate the Tx and Rx queues from the interrupts */ 2773 ice_flush_txq_interrupts(&sc->pf_vsi); 2774 ice_flush_rxq_interrupts(&sc->pf_vsi); 2775 2776 /* Disable the Tx and Rx queues */ 2777 ice_vsi_disable_tx(&sc->pf_vsi); 2778 ice_control_rx_queues(&sc->pf_vsi, false); 2779 } 2780 2781 /** 2782 * ice_if_get_counter - Get current value of an ifnet statistic 2783 * @ctx: iflib context pointer 2784 * @counter: ifnet counter to read 2785 * 2786 * Reads the current value of an ifnet counter for the device. 2787 * 2788 * This function is not protected by the iflib CTX lock. 2789 */ 2790 static uint64_t 2791 ice_if_get_counter(if_ctx_t ctx, ift_counter counter) 2792 { 2793 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2794 2795 /* Return the counter for the main PF VSI */ 2796 return ice_get_ifnet_counter(&sc->pf_vsi, counter); 2797 } 2798 2799 /** 2800 * ice_request_stack_reinit - Request that iflib re-initialize 2801 * @sc: the device private softc 2802 * 2803 * Request that the device be brought down and up, to re-initialize. For 2804 * example, this may be called when a device reset occurs, or when Tx and Rx 2805 * queues need to be re-initialized. 2806 * 2807 * This is required because the iflib state is outside the driver, and must be 2808 * re-initialized if we need to resart Tx and Rx queues. 2809 */ 2810 void 2811 ice_request_stack_reinit(struct ice_softc *sc) 2812 { 2813 if (CTX_ACTIVE(sc->ctx)) { 2814 iflib_request_reset(sc->ctx); 2815 iflib_admin_intr_deferred(sc->ctx); 2816 } 2817 } 2818 2819 /** 2820 * ice_driver_is_detaching - Check if the driver is detaching/unloading 2821 * @sc: device private softc 2822 * 2823 * Returns true if the driver is detaching, false otherwise. 2824 * 2825 * @remark on newer kernels, take advantage of iflib_in_detach in order to 2826 * report detachment correctly as early as possible. 2827 * 2828 * @remark this function is used by various code paths that want to avoid 2829 * running if the driver is about to be removed. This includes sysctls and 2830 * other driver access points. Note that it does not fully resolve 2831 * detach-based race conditions as it is possible for a thread to race with 2832 * iflib_in_detach. 2833 */ 2834 bool 2835 ice_driver_is_detaching(struct ice_softc *sc) 2836 { 2837 return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || 2838 iflib_in_detach(sc->ctx)); 2839 } 2840 2841 /** 2842 * ice_if_priv_ioctl - Device private ioctl handler 2843 * @ctx: iflib context pointer 2844 * @command: The ioctl command issued 2845 * @data: ioctl specific data 2846 * 2847 * iflib callback for handling custom driver specific ioctls. 2848 * 2849 * @pre Assumes that the iflib context lock is held. 2850 */ 2851 static int 2852 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) 2853 { 2854 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2855 struct ifdrv *ifd; 2856 device_t dev = sc->dev; 2857 2858 if (data == NULL) 2859 return (EINVAL); 2860 2861 ASSERT_CTX_LOCKED(sc); 2862 2863 /* Make sure the command type is valid */ 2864 switch (command) { 2865 case SIOCSDRVSPEC: 2866 case SIOCGDRVSPEC: 2867 /* Accepted commands */ 2868 break; 2869 case SIOCGPRIVATE_0: 2870 /* 2871 * Although we do not support this ioctl command, it's 2872 * expected that iflib will forward it to the IFDI_PRIV_IOCTL 2873 * handler. Do not print a message in this case 2874 */ 2875 return (ENOTSUP); 2876 default: 2877 /* 2878 * If we get a different command for this function, it's 2879 * definitely unexpected, so log a message indicating what 2880 * command we got for debugging purposes. 2881 */ 2882 device_printf(dev, "%s: unexpected ioctl command %08lx\n", 2883 __func__, command); 2884 return (EINVAL); 2885 } 2886 2887 ifd = (struct ifdrv *)data; 2888 2889 switch (ifd->ifd_cmd) { 2890 case ICE_NVM_ACCESS: 2891 return ice_handle_nvm_access_ioctl(sc, ifd); 2892 default: 2893 return EINVAL; 2894 } 2895 } 2896 2897 /** 2898 * ice_if_i2c_req - I2C request handler for iflib 2899 * @ctx: iflib context pointer 2900 * @req: The I2C parameters to use 2901 * 2902 * Read from the port's I2C eeprom using the parameters from the ioctl. 2903 * 2904 * @remark The iflib-only part is pretty simple. 2905 */ 2906 static int 2907 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req) 2908 { 2909 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2910 2911 return ice_handle_i2c_req(sc, req); 2912 } 2913 2914 /** 2915 * ice_if_suspend - PCI device suspend handler for iflib 2916 * @ctx: iflib context pointer 2917 * 2918 * Deinitializes the driver and clears HW resources in preparation for 2919 * suspend or an FLR. 2920 * 2921 * @returns 0; this return value is ignored 2922 */ 2923 static int 2924 ice_if_suspend(if_ctx_t ctx) 2925 { 2926 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2927 2928 /* At least a PFR is always going to happen after this; 2929 * either via FLR or during the D3->D0 transition. 2930 */ 2931 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 2932 2933 ice_prepare_for_reset(sc); 2934 2935 return (0); 2936 } 2937 2938 /** 2939 * ice_if_resume - PCI device resume handler for iflib 2940 * @ctx: iflib context pointer 2941 * 2942 * Reinitializes the driver and the HW after PCI resume or after 2943 * an FLR. An init is performed by iflib after this function is finished. 2944 * 2945 * @returns 0; this return value is ignored 2946 */ 2947 static int 2948 ice_if_resume(if_ctx_t ctx) 2949 { 2950 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2951 2952 ice_rebuild(sc); 2953 2954 return (0); 2955 } 2956 2957