1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2021, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*$FreeBSD$*/ 32 33 /** 34 * @file if_ice_iflib.c 35 * @brief iflib driver implementation 36 * 37 * Contains the main entry point for the iflib driver implementation. It 38 * implements the various ifdi driver methods, and sets up the module and 39 * driver values to load an iflib driver. 40 */ 41 42 #include "ice_iflib.h" 43 #include "ice_drv_info.h" 44 #include "ice_switch.h" 45 #include "ice_sched.h" 46 47 #include <sys/module.h> 48 #include <sys/sockio.h> 49 #include <sys/smp.h> 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pcireg.h> 52 53 /* 54 * Device method prototypes 55 */ 56 57 static void *ice_register(device_t); 58 static int ice_if_attach_pre(if_ctx_t); 59 static int ice_attach_pre_recovery_mode(struct ice_softc *sc); 60 static int ice_if_attach_post(if_ctx_t); 61 static void ice_attach_post_recovery_mode(struct ice_softc *sc); 62 static int ice_if_detach(if_ctx_t); 63 static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); 64 static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); 65 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix); 66 static void ice_if_queues_free(if_ctx_t ctx); 67 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 68 static void ice_if_intr_enable(if_ctx_t ctx); 69 static void ice_if_intr_disable(if_ctx_t ctx); 70 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 71 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 72 static int ice_if_promisc_set(if_ctx_t ctx, int flags); 73 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 74 static int ice_if_media_change(if_ctx_t ctx); 75 static void ice_if_init(if_ctx_t ctx); 76 static void ice_if_timer(if_ctx_t ctx, uint16_t qid); 77 static void ice_if_update_admin_status(if_ctx_t ctx); 78 static void ice_if_multi_set(if_ctx_t ctx); 79 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag); 80 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag); 81 static void ice_if_stop(if_ctx_t ctx); 82 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter); 83 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); 84 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req); 85 static int ice_if_suspend(if_ctx_t ctx); 86 static int ice_if_resume(if_ctx_t ctx); 87 88 static int ice_msix_que(void *arg); 89 static int ice_msix_admin(void *arg); 90 91 /* 92 * Helper function prototypes 93 */ 94 static int ice_pci_mapping(struct ice_softc *sc); 95 static void ice_free_pci_mapping(struct ice_softc *sc); 96 static void ice_update_link_status(struct ice_softc *sc, bool update_media); 97 static void ice_init_device_features(struct ice_softc *sc); 98 static void ice_init_tx_tracking(struct ice_vsi *vsi); 99 static void ice_handle_reset_event(struct ice_softc *sc); 100 static void ice_handle_pf_reset_request(struct ice_softc *sc); 101 static void ice_prepare_for_reset(struct ice_softc *sc); 102 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc); 103 static void ice_rebuild(struct ice_softc *sc); 104 static void ice_rebuild_recovery_mode(struct ice_softc *sc); 105 static void ice_free_irqvs(struct ice_softc *sc); 106 static void ice_update_rx_mbuf_sz(struct ice_softc *sc); 107 static void ice_poll_for_media_avail(struct ice_softc *sc); 108 static void ice_setup_scctx(struct ice_softc *sc); 109 static int ice_allocate_msix(struct ice_softc *sc); 110 static void ice_admin_timer(void *arg); 111 static void ice_transition_recovery_mode(struct ice_softc *sc); 112 static void ice_transition_safe_mode(struct ice_softc *sc); 113 114 /* 115 * Device Interface Declaration 116 */ 117 118 /** 119 * @var ice_methods 120 * @brief ice driver method entry points 121 * 122 * List of device methods implementing the generic device interface used by 123 * the device stack to interact with the ice driver. Since this is an iflib 124 * driver, most of the methods point to the generic iflib implementation. 125 */ 126 static device_method_t ice_methods[] = { 127 /* Device interface */ 128 DEVMETHOD(device_register, ice_register), 129 DEVMETHOD(device_probe, iflib_device_probe_vendor), 130 DEVMETHOD(device_attach, iflib_device_attach), 131 DEVMETHOD(device_detach, iflib_device_detach), 132 DEVMETHOD(device_shutdown, iflib_device_shutdown), 133 DEVMETHOD(device_suspend, iflib_device_suspend), 134 DEVMETHOD(device_resume, iflib_device_resume), 135 DEVMETHOD_END 136 }; 137 138 /** 139 * @var ice_iflib_methods 140 * @brief iflib method entry points 141 * 142 * List of device methods used by the iflib stack to interact with this 143 * driver. These are the real main entry points used to interact with this 144 * driver. 145 */ 146 static device_method_t ice_iflib_methods[] = { 147 DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre), 148 DEVMETHOD(ifdi_attach_post, ice_if_attach_post), 149 DEVMETHOD(ifdi_detach, ice_if_detach), 150 DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc), 151 DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc), 152 DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign), 153 DEVMETHOD(ifdi_queues_free, ice_if_queues_free), 154 DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set), 155 DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable), 156 DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable), 157 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable), 158 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable), 159 DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set), 160 DEVMETHOD(ifdi_media_status, ice_if_media_status), 161 DEVMETHOD(ifdi_media_change, ice_if_media_change), 162 DEVMETHOD(ifdi_init, ice_if_init), 163 DEVMETHOD(ifdi_stop, ice_if_stop), 164 DEVMETHOD(ifdi_timer, ice_if_timer), 165 DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status), 166 DEVMETHOD(ifdi_multi_set, ice_if_multi_set), 167 DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register), 168 DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister), 169 DEVMETHOD(ifdi_get_counter, ice_if_get_counter), 170 DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl), 171 DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req), 172 DEVMETHOD(ifdi_suspend, ice_if_suspend), 173 DEVMETHOD(ifdi_resume, ice_if_resume), 174 DEVMETHOD_END 175 }; 176 177 /** 178 * @var ice_driver 179 * @brief driver structure for the generic device stack 180 * 181 * driver_t definition used to setup the generic device methods. 182 */ 183 static driver_t ice_driver = { 184 .name = "ice", 185 .methods = ice_methods, 186 .size = sizeof(struct ice_softc), 187 }; 188 189 /** 190 * @var ice_iflib_driver 191 * @brief driver structure for the iflib stack 192 * 193 * driver_t definition used to setup the iflib device methods. 194 */ 195 static driver_t ice_iflib_driver = { 196 .name = "ice", 197 .methods = ice_iflib_methods, 198 .size = sizeof(struct ice_softc), 199 }; 200 201 extern struct if_txrx ice_txrx; 202 extern struct if_txrx ice_recovery_txrx; 203 204 /** 205 * @var ice_sctx 206 * @brief ice driver shared context 207 * 208 * Structure defining shared values (context) that is used by all instances of 209 * the device. Primarily used to setup details about how the iflib stack 210 * should treat this driver. Also defines the default, minimum, and maximum 211 * number of descriptors in each ring. 212 */ 213 static struct if_shared_ctx ice_sctx = { 214 .isc_magic = IFLIB_MAGIC, 215 .isc_q_align = PAGE_SIZE, 216 217 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 218 /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but 219 * that doesn't make sense since that would be larger than the maximum 220 * size of a single packet. 221 */ 222 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 223 224 /* XXX: This is only used by iflib to ensure that 225 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size. 226 */ 227 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 228 /* XXX: This is used by iflib to set the number of segments in the TSO 229 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the 230 * related ifnet parameter. 231 */ 232 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 233 234 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 235 .isc_rx_nsegments = ICE_MAX_RX_SEGS, 236 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 237 238 .isc_nfl = 1, 239 .isc_ntxqs = 1, 240 .isc_nrxqs = 1, 241 242 .isc_admin_intrcnt = 1, 243 .isc_vendor_info = ice_vendor_info_array, 244 .isc_driver_version = __DECONST(char *, ice_driver_version), 245 .isc_driver = &ice_iflib_driver, 246 247 /* 248 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available 249 * for hardware checksum offload 250 * 251 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the 252 * IP sum field, required by our hardware to calculate valid TSO 253 * checksums. 254 * 255 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs 256 * even when the interface is down. 257 * 258 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X 259 * vectors manually instead of relying on iflib code to do this. 260 */ 261 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 262 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX, 263 264 .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 265 .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 266 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 267 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 268 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 269 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 270 }; 271 272 DRIVER_MODULE(ice, pci, ice_driver, ice_module_event_handler, NULL); 273 274 MODULE_VERSION(ice, 1); 275 MODULE_DEPEND(ice, pci, 1, 1, 1); 276 MODULE_DEPEND(ice, ether, 1, 1, 1); 277 MODULE_DEPEND(ice, iflib, 1, 1, 1); 278 279 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array); 280 281 /* Static driver-wide sysctls */ 282 #include "ice_iflib_sysctls.h" 283 284 /** 285 * ice_pci_mapping - Map PCI BAR memory 286 * @sc: device private softc 287 * 288 * Map PCI BAR 0 for device operation. 289 */ 290 static int 291 ice_pci_mapping(struct ice_softc *sc) 292 { 293 int rc; 294 295 /* Map BAR0 */ 296 rc = ice_map_bar(sc->dev, &sc->bar0, 0); 297 if (rc) 298 return rc; 299 300 return 0; 301 } 302 303 /** 304 * ice_free_pci_mapping - Release PCI BAR memory 305 * @sc: device private softc 306 * 307 * Release PCI BARs which were previously mapped by ice_pci_mapping(). 308 */ 309 static void 310 ice_free_pci_mapping(struct ice_softc *sc) 311 { 312 /* Free BAR0 */ 313 ice_free_bar(sc->dev, &sc->bar0); 314 } 315 316 /* 317 * Device methods 318 */ 319 320 /** 321 * ice_register - register device method callback 322 * @dev: the device being registered 323 * 324 * Returns a pointer to the shared context structure, which is used by iflib. 325 */ 326 static void * 327 ice_register(device_t dev __unused) 328 { 329 return &ice_sctx; 330 } /* ice_register */ 331 332 /** 333 * ice_setup_scctx - Setup the iflib softc context structure 334 * @sc: the device private structure 335 * 336 * Setup the parameters in if_softc_ctx_t structure used by the iflib stack 337 * when loading. 338 */ 339 static void 340 ice_setup_scctx(struct ice_softc *sc) 341 { 342 if_softc_ctx_t scctx = sc->scctx; 343 struct ice_hw *hw = &sc->hw; 344 bool safe_mode, recovery_mode; 345 346 safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE); 347 recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE); 348 349 /* 350 * If the driver loads in Safe mode or Recovery mode, limit iflib to 351 * a single queue pair. 352 */ 353 if (safe_mode || recovery_mode) { 354 scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1; 355 scctx->isc_ntxqsets_max = 1; 356 scctx->isc_nrxqsets_max = 1; 357 } else { 358 /* 359 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to 360 * the values of the override sysctls. Cache these initial 361 * values so that the driver can be aware of what the iflib 362 * sysctl value is when setting up MSI-X vectors. 363 */ 364 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets; 365 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets; 366 367 if (scctx->isc_ntxqsets == 0) 368 scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size; 369 if (scctx->isc_nrxqsets == 0) 370 scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size; 371 372 scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq; 373 scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq; 374 375 /* 376 * Sanity check that the iflib sysctl values are within the 377 * maximum supported range. 378 */ 379 if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max) 380 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max; 381 if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max) 382 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max; 383 } 384 385 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 386 * sizeof(struct ice_tx_desc), DBA_ALIGN); 387 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 388 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 389 390 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 391 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 392 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 393 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 394 395 scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR); 396 scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size; 397 398 /* 399 * If the driver loads in recovery mode, disable Tx/Rx functionality 400 */ 401 if (recovery_mode) 402 scctx->isc_txrx = &ice_recovery_txrx; 403 else 404 scctx->isc_txrx = &ice_txrx; 405 406 /* 407 * If the driver loads in Safe mode or Recovery mode, disable 408 * advanced features including hardware offloads. 409 */ 410 if (safe_mode || recovery_mode) { 411 scctx->isc_capenable = ICE_SAFE_CAPS; 412 scctx->isc_tx_csum_flags = 0; 413 } else { 414 scctx->isc_capenable = ICE_FULL_CAPS; 415 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 416 } 417 418 scctx->isc_capabilities = scctx->isc_capenable; 419 } /* ice_setup_scctx */ 420 421 /** 422 * ice_if_attach_pre - Early device attach logic 423 * @ctx: the iflib context structure 424 * 425 * Called by iflib during the attach process. Earliest main driver entry 426 * point which performs necessary hardware and driver initialization. Called 427 * before the Tx and Rx queues are allocated. 428 */ 429 static int 430 ice_if_attach_pre(if_ctx_t ctx) 431 { 432 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 433 enum ice_fw_modes fw_mode; 434 enum ice_status status; 435 if_softc_ctx_t scctx; 436 struct ice_hw *hw; 437 device_t dev; 438 int err; 439 440 device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n"); 441 442 ice_set_state(&sc->state, ICE_STATE_ATTACHING); 443 444 sc->ctx = ctx; 445 sc->media = iflib_get_media(ctx); 446 sc->sctx = iflib_get_sctx(ctx); 447 sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx); 448 449 dev = sc->dev = iflib_get_dev(ctx); 450 scctx = sc->scctx = iflib_get_softc_ctx(ctx); 451 452 hw = &sc->hw; 453 hw->back = sc; 454 455 snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name), 456 "%s:admin", device_get_nameunit(dev)); 457 mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF); 458 callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0); 459 460 ASSERT_CTX_LOCKED(sc); 461 462 if (ice_pci_mapping(sc)) { 463 err = (ENXIO); 464 goto destroy_admin_timer; 465 } 466 467 /* Save off the PCI information */ 468 ice_save_pci_info(hw, dev); 469 470 /* create tunables as early as possible */ 471 ice_add_device_tunables(sc); 472 473 /* Setup ControlQ lengths */ 474 ice_set_ctrlq_len(hw); 475 476 fw_mode = ice_get_fw_mode(hw); 477 if (fw_mode == ICE_FW_MODE_REC) { 478 device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 479 480 err = ice_attach_pre_recovery_mode(sc); 481 if (err) 482 goto free_pci_mapping; 483 484 return (0); 485 } 486 487 /* Initialize the hw data structure */ 488 status = ice_init_hw(hw); 489 if (status) { 490 if (status == ICE_ERR_FW_API_VER) { 491 /* Enter recovery mode, so that the driver remains 492 * loaded. This way, if the system administrator 493 * cannot update the driver, they may still attempt to 494 * downgrade the NVM. 495 */ 496 err = ice_attach_pre_recovery_mode(sc); 497 if (err) 498 goto free_pci_mapping; 499 500 return (0); 501 } else { 502 err = EIO; 503 device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n", 504 ice_status_str(status), 505 ice_aq_str(hw->adminq.sq_last_status)); 506 } 507 goto free_pci_mapping; 508 } 509 510 /* Notify firmware of the device driver version */ 511 err = ice_send_version(sc); 512 if (err) 513 goto deinit_hw; 514 515 ice_load_pkg_file(sc); 516 517 err = ice_init_link_events(sc); 518 if (err) { 519 device_printf(dev, "ice_init_link_events failed: %s\n", 520 ice_err_str(err)); 521 goto deinit_hw; 522 } 523 524 ice_print_nvm_version(sc); 525 526 ice_init_device_features(sc); 527 528 /* Setup the MAC address */ 529 iflib_set_mac(ctx, hw->port_info->mac.lan_addr); 530 531 /* Setup the iflib softc context structure */ 532 ice_setup_scctx(sc); 533 534 /* Initialize the Tx queue manager */ 535 err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq); 536 if (err) { 537 device_printf(dev, "Unable to initialize Tx queue manager: %s\n", 538 ice_err_str(err)); 539 goto deinit_hw; 540 } 541 542 /* Initialize the Rx queue manager */ 543 err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq); 544 if (err) { 545 device_printf(dev, "Unable to initialize Rx queue manager: %s\n", 546 ice_err_str(err)); 547 goto free_tx_qmgr; 548 } 549 550 /* Initialize the interrupt resource manager */ 551 err = ice_alloc_intr_tracking(sc); 552 if (err) 553 /* Errors are already printed */ 554 goto free_rx_qmgr; 555 556 /* Determine maximum number of VSIs we'll prepare for */ 557 sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE, 558 hw->func_caps.guar_num_vsi); 559 560 if (!sc->num_available_vsi) { 561 err = EIO; 562 device_printf(dev, "No VSIs allocated to host\n"); 563 goto free_intr_tracking; 564 } 565 566 /* Allocate storage for the VSI pointers */ 567 sc->all_vsi = (struct ice_vsi **) 568 malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi, 569 M_ICE, M_WAITOK | M_ZERO); 570 if (!sc->all_vsi) { 571 err = ENOMEM; 572 device_printf(dev, "Unable to allocate VSI array\n"); 573 goto free_intr_tracking; 574 } 575 576 /* 577 * Prepare the statically allocated primary PF VSI in the softc 578 * structure. Other VSIs will be dynamically allocated as needed. 579 */ 580 ice_setup_pf_vsi(sc); 581 582 err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max, 583 scctx->isc_nrxqsets_max); 584 if (err) { 585 device_printf(dev, "Unable to allocate VSI Queue maps\n"); 586 goto free_main_vsi; 587 } 588 589 /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */ 590 err = ice_allocate_msix(sc); 591 if (err) 592 goto free_main_vsi; 593 594 return 0; 595 596 free_main_vsi: 597 /* ice_release_vsi will free the queue maps if they were allocated */ 598 ice_release_vsi(&sc->pf_vsi); 599 free(sc->all_vsi, M_ICE); 600 sc->all_vsi = NULL; 601 free_intr_tracking: 602 ice_free_intr_tracking(sc); 603 free_rx_qmgr: 604 ice_resmgr_destroy(&sc->rx_qmgr); 605 free_tx_qmgr: 606 ice_resmgr_destroy(&sc->tx_qmgr); 607 deinit_hw: 608 ice_deinit_hw(hw); 609 free_pci_mapping: 610 ice_free_pci_mapping(sc); 611 destroy_admin_timer: 612 mtx_lock(&sc->admin_mtx); 613 callout_stop(&sc->admin_timer); 614 mtx_unlock(&sc->admin_mtx); 615 mtx_destroy(&sc->admin_mtx); 616 return err; 617 } /* ice_if_attach_pre */ 618 619 /** 620 * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery 621 * @sc: the device private softc 622 * 623 * Loads the device driver in limited Firmware Recovery mode, intended to 624 * allow users to update the firmware to attempt to recover the device. 625 * 626 * @remark We may enter recovery mode in case either (a) the firmware is 627 * detected to be in an invalid state and must be re-programmed, or (b) the 628 * driver detects that the loaded firmware has a non-compatible API version 629 * that the driver cannot operate with. 630 */ 631 static int 632 ice_attach_pre_recovery_mode(struct ice_softc *sc) 633 { 634 ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE); 635 636 /* Setup the iflib softc context */ 637 ice_setup_scctx(sc); 638 639 /* Setup the PF VSI back pointer */ 640 sc->pf_vsi.sc = sc; 641 642 /* 643 * We still need to allocate MSI-X vectors since we need one vector to 644 * run the administrative admin interrupt 645 */ 646 return ice_allocate_msix(sc); 647 } 648 649 /** 650 * ice_update_link_status - notify OS of link state change 651 * @sc: device private softc structure 652 * @update_media: true if we should update media even if link didn't change 653 * 654 * Called to notify iflib core of link status changes. Should be called once 655 * during attach_post, and whenever link status changes during runtime. 656 * 657 * This call only updates the currently supported media types if the link 658 * status changed, or if update_media is set to true. 659 */ 660 static void 661 ice_update_link_status(struct ice_softc *sc, bool update_media) 662 { 663 struct ice_hw *hw = &sc->hw; 664 enum ice_status status; 665 666 /* Never report link up when in recovery mode */ 667 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 668 return; 669 670 /* Report link status to iflib only once each time it changes */ 671 if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) { 672 if (sc->link_up) { /* link is up */ 673 uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info); 674 675 ice_set_default_local_lldp_mib(sc); 676 677 iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate); 678 ice_rdma_link_change(sc, LINK_STATE_UP, baudrate); 679 680 ice_link_up_msg(sc); 681 682 update_media = true; 683 } else { /* link is down */ 684 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 685 ice_rdma_link_change(sc, LINK_STATE_DOWN, 0); 686 687 update_media = true; 688 } 689 } 690 691 /* Update the supported media types */ 692 if (update_media) { 693 status = ice_add_media_types(sc, sc->media); 694 if (status) 695 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 696 ice_status_str(status), 697 ice_aq_str(hw->adminq.sq_last_status)); 698 } 699 700 /* TODO: notify VFs of link state change */ 701 } 702 703 /** 704 * ice_if_attach_post - Late device attach logic 705 * @ctx: the iflib context structure 706 * 707 * Called by iflib to finish up attaching the device. Performs any attach 708 * logic which must wait until after the Tx and Rx queues have been 709 * allocated. 710 */ 711 static int 712 ice_if_attach_post(if_ctx_t ctx) 713 { 714 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 715 if_t ifp = iflib_get_ifp(ctx); 716 int err; 717 718 ASSERT_CTX_LOCKED(sc); 719 720 /* We don't yet support loading if MSI-X is not supported */ 721 if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) { 722 device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n"); 723 return (ENOTSUP); 724 } 725 726 /* The ifnet structure hasn't yet been initialized when the attach_pre 727 * handler is called, so wait until attach_post to setup the 728 * isc_max_frame_size. 729 */ 730 731 sc->ifp = ifp; 732 sc->scctx->isc_max_frame_size = ifp->if_mtu + 733 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 734 735 /* 736 * If we are in recovery mode, only perform a limited subset of 737 * initialization to support NVM recovery. 738 */ 739 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 740 ice_attach_post_recovery_mode(sc); 741 return (0); 742 } 743 744 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 745 746 err = ice_initialize_vsi(&sc->pf_vsi); 747 if (err) { 748 device_printf(sc->dev, "Unable to initialize Main VSI: %s\n", 749 ice_err_str(err)); 750 return err; 751 } 752 753 /* Enable FW health event reporting */ 754 ice_init_health_events(sc); 755 756 /* Configure the main PF VSI for RSS */ 757 err = ice_config_rss(&sc->pf_vsi); 758 if (err) { 759 device_printf(sc->dev, 760 "Unable to configure RSS for the main VSI, err %s\n", 761 ice_err_str(err)); 762 return err; 763 } 764 765 /* Configure switch to drop transmitted LLDP and PAUSE frames */ 766 err = ice_cfg_pf_ethertype_filters(sc); 767 if (err) 768 return err; 769 770 ice_get_and_print_bus_info(sc); 771 772 ice_set_link_management_mode(sc); 773 774 ice_init_saved_phy_cfg(sc); 775 776 ice_cfg_pba_num(sc); 777 778 ice_add_device_sysctls(sc); 779 780 /* Get DCBX/LLDP state and start DCBX agent */ 781 ice_init_dcb_setup(sc); 782 783 /* Setup link configuration parameters */ 784 ice_init_link_configuration(sc); 785 ice_update_link_status(sc, true); 786 787 /* Configure interrupt causes for the administrative interrupt */ 788 ice_configure_misc_interrupts(sc); 789 790 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 791 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 792 793 err = ice_rdma_pf_attach(sc); 794 if (err) 795 return (err); 796 797 /* Start the admin timer */ 798 mtx_lock(&sc->admin_mtx); 799 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 800 mtx_unlock(&sc->admin_mtx); 801 802 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 803 804 return 0; 805 } /* ice_if_attach_post */ 806 807 /** 808 * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery 809 * @sc: the device private softc 810 * 811 * Performs minimal work to prepare the driver to recover an NVM in case the 812 * firmware is in recovery mode. 813 */ 814 static void 815 ice_attach_post_recovery_mode(struct ice_softc *sc) 816 { 817 /* Configure interrupt causes for the administrative interrupt */ 818 ice_configure_misc_interrupts(sc); 819 820 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 821 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 822 823 /* Start the admin timer */ 824 mtx_lock(&sc->admin_mtx); 825 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 826 mtx_unlock(&sc->admin_mtx); 827 828 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 829 } 830 831 /** 832 * ice_free_irqvs - Free IRQ vector memory 833 * @sc: the device private softc structure 834 * 835 * Free IRQ vector memory allocated during ice_if_msix_intr_assign. 836 */ 837 static void 838 ice_free_irqvs(struct ice_softc *sc) 839 { 840 struct ice_vsi *vsi = &sc->pf_vsi; 841 if_ctx_t ctx = sc->ctx; 842 int i; 843 844 /* If the irqvs array is NULL, then there are no vectors to free */ 845 if (sc->irqvs == NULL) 846 return; 847 848 /* Free the IRQ vectors */ 849 for (i = 0; i < sc->num_irq_vectors; i++) 850 iflib_irq_free(ctx, &sc->irqvs[i].irq); 851 852 /* Clear the irqv pointers */ 853 for (i = 0; i < vsi->num_rx_queues; i++) 854 vsi->rx_queues[i].irqv = NULL; 855 856 for (i = 0; i < vsi->num_tx_queues; i++) 857 vsi->tx_queues[i].irqv = NULL; 858 859 /* Release the vector array memory */ 860 free(sc->irqvs, M_ICE); 861 sc->irqvs = NULL; 862 sc->num_irq_vectors = 0; 863 } 864 865 /** 866 * ice_if_detach - Device driver detach logic 867 * @ctx: iflib context structure 868 * 869 * Perform device shutdown logic to detach the device driver. 870 * 871 * Note that there is no guarantee of the ordering of ice_if_queues_free() and 872 * ice_if_detach(). It is possible for the functions to be called in either 873 * order, and they must not assume to have a strict ordering. 874 */ 875 static int 876 ice_if_detach(if_ctx_t ctx) 877 { 878 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 879 struct ice_vsi *vsi = &sc->pf_vsi; 880 int i; 881 882 ASSERT_CTX_LOCKED(sc); 883 884 /* Indicate that we're detaching */ 885 ice_set_state(&sc->state, ICE_STATE_DETACHING); 886 887 /* Stop the admin timer */ 888 mtx_lock(&sc->admin_mtx); 889 callout_stop(&sc->admin_timer); 890 mtx_unlock(&sc->admin_mtx); 891 mtx_destroy(&sc->admin_mtx); 892 893 ice_rdma_pf_detach(sc); 894 895 /* Free allocated media types */ 896 ifmedia_removeall(sc->media); 897 898 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 899 * pointers. Note, the calls here and those in ice_if_queues_free() 900 * are *BOTH* necessary, as we cannot guarantee which path will be 901 * run first 902 */ 903 ice_vsi_del_txqs_ctx(vsi); 904 ice_vsi_del_rxqs_ctx(vsi); 905 906 /* Release MSI-X resources */ 907 ice_free_irqvs(sc); 908 909 for (i = 0; i < sc->num_available_vsi; i++) { 910 if (sc->all_vsi[i]) 911 ice_release_vsi(sc->all_vsi[i]); 912 } 913 914 if (sc->all_vsi) { 915 free(sc->all_vsi, M_ICE); 916 sc->all_vsi = NULL; 917 } 918 919 /* Release MSI-X memory */ 920 pci_release_msi(sc->dev); 921 922 if (sc->msix_table != NULL) { 923 bus_release_resource(sc->dev, SYS_RES_MEMORY, 924 rman_get_rid(sc->msix_table), 925 sc->msix_table); 926 sc->msix_table = NULL; 927 } 928 929 ice_free_intr_tracking(sc); 930 931 /* Destroy the queue managers */ 932 ice_resmgr_destroy(&sc->tx_qmgr); 933 ice_resmgr_destroy(&sc->rx_qmgr); 934 935 if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 936 ice_deinit_hw(&sc->hw); 937 938 ice_free_pci_mapping(sc); 939 940 return 0; 941 } /* ice_if_detach */ 942 943 /** 944 * ice_if_tx_queues_alloc - Allocate Tx queue memory 945 * @ctx: iflib context structure 946 * @vaddrs: virtual addresses for the queue memory 947 * @paddrs: physical addresses for the queue memory 948 * @ntxqs: the number of Tx queues per set (should always be 1) 949 * @ntxqsets: the number of Tx queue sets to allocate 950 * 951 * Called by iflib to allocate Tx queues for the device. Allocates driver 952 * memory to track each queue, the status arrays used for descriptor 953 * status reporting, and Tx queue sysctls. 954 */ 955 static int 956 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 957 int __invariant_only ntxqs, int ntxqsets) 958 { 959 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 960 struct ice_vsi *vsi = &sc->pf_vsi; 961 struct ice_tx_queue *txq; 962 int err, i, j; 963 964 MPASS(ntxqs == 1); 965 MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 966 ASSERT_CTX_LOCKED(sc); 967 968 /* Do not bother allocating queues if we're in recovery mode */ 969 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 970 return (0); 971 972 /* Allocate queue structure memory */ 973 if (!(vsi->tx_queues = 974 (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_WAITOK | M_ZERO))) { 975 device_printf(sc->dev, "Unable to allocate Tx queue memory\n"); 976 return (ENOMEM); 977 } 978 979 /* Allocate report status arrays */ 980 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 981 if (!(txq->tx_rsq = 982 (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_WAITOK))) { 983 device_printf(sc->dev, "Unable to allocate tx_rsq memory\n"); 984 err = ENOMEM; 985 goto free_tx_queues; 986 } 987 /* Initialize report status array */ 988 for (j = 0; j < sc->scctx->isc_ntxd[0]; j++) 989 txq->tx_rsq[j] = QIDX_INVALID; 990 } 991 992 /* Assign queues from PF space to the main VSI */ 993 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets); 994 if (err) { 995 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 996 ice_err_str(err)); 997 goto free_tx_queues; 998 } 999 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1000 1001 /* Add Tx queue sysctls context */ 1002 ice_vsi_add_txqs_ctx(vsi); 1003 1004 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1005 /* q_handle == me when only one TC */ 1006 txq->me = txq->q_handle = i; 1007 txq->vsi = vsi; 1008 1009 /* store the queue size for easier access */ 1010 txq->desc_count = sc->scctx->isc_ntxd[0]; 1011 1012 /* get the virtual and physical address of the hardware queues */ 1013 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 1014 txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 1015 txq->tx_paddr = paddrs[i]; 1016 1017 ice_add_txq_sysctls(txq); 1018 } 1019 1020 vsi->num_tx_queues = ntxqsets; 1021 1022 return (0); 1023 1024 free_tx_queues: 1025 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1026 if (txq->tx_rsq != NULL) { 1027 free(txq->tx_rsq, M_ICE); 1028 txq->tx_rsq = NULL; 1029 } 1030 } 1031 free(vsi->tx_queues, M_ICE); 1032 vsi->tx_queues = NULL; 1033 return err; 1034 } 1035 1036 /** 1037 * ice_if_rx_queues_alloc - Allocate Rx queue memory 1038 * @ctx: iflib context structure 1039 * @vaddrs: virtual addresses for the queue memory 1040 * @paddrs: physical addresses for the queue memory 1041 * @nrxqs: number of Rx queues per set (should always be 1) 1042 * @nrxqsets: number of Rx queue sets to allocate 1043 * 1044 * Called by iflib to allocate Rx queues for the device. Allocates driver 1045 * memory to track each queue, as well as sets up the Rx queue sysctls. 1046 */ 1047 static int 1048 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1049 int __invariant_only nrxqs, int nrxqsets) 1050 { 1051 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1052 struct ice_vsi *vsi = &sc->pf_vsi; 1053 struct ice_rx_queue *rxq; 1054 int err, i; 1055 1056 MPASS(nrxqs == 1); 1057 MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 1058 ASSERT_CTX_LOCKED(sc); 1059 1060 /* Do not bother allocating queues if we're in recovery mode */ 1061 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1062 return (0); 1063 1064 /* Allocate queue structure memory */ 1065 if (!(vsi->rx_queues = 1066 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_WAITOK | M_ZERO))) { 1067 device_printf(sc->dev, "Unable to allocate Rx queue memory\n"); 1068 return (ENOMEM); 1069 } 1070 1071 /* Assign queues from PF space to the main VSI */ 1072 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets); 1073 if (err) { 1074 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1075 ice_err_str(err)); 1076 goto free_rx_queues; 1077 } 1078 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1079 1080 /* Add Rx queue sysctls context */ 1081 ice_vsi_add_rxqs_ctx(vsi); 1082 1083 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 1084 rxq->me = i; 1085 rxq->vsi = vsi; 1086 1087 /* store the queue size for easier access */ 1088 rxq->desc_count = sc->scctx->isc_nrxd[0]; 1089 1090 /* get the virtual and physical address of the hardware queues */ 1091 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 1092 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 1093 rxq->rx_paddr = paddrs[i]; 1094 1095 ice_add_rxq_sysctls(rxq); 1096 } 1097 1098 vsi->num_rx_queues = nrxqsets; 1099 1100 return (0); 1101 1102 free_rx_queues: 1103 free(vsi->rx_queues, M_ICE); 1104 vsi->rx_queues = NULL; 1105 return err; 1106 } 1107 1108 /** 1109 * ice_if_queues_free - Free queue memory 1110 * @ctx: the iflib context structure 1111 * 1112 * Free queue memory allocated by ice_if_tx_queues_alloc() and 1113 * ice_if_rx_queues_alloc(). 1114 * 1115 * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be 1116 * called in the same order. It's possible for ice_if_queues_free() to be 1117 * called prior to ice_if_detach(), and vice versa. 1118 * 1119 * For this reason, the main VSI is a static member of the ice_softc, which is 1120 * not free'd until after iflib finishes calling both of these functions. 1121 * 1122 * Thus, care must be taken in how we manage the memory being freed by this 1123 * function, and in what tasks it can and must perform. 1124 */ 1125 static void 1126 ice_if_queues_free(if_ctx_t ctx) 1127 { 1128 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1129 struct ice_vsi *vsi = &sc->pf_vsi; 1130 struct ice_tx_queue *txq; 1131 int i; 1132 1133 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1134 * pointers. Note, the calls here and those in ice_if_detach() 1135 * are *BOTH* necessary, as we cannot guarantee which path will be 1136 * run first 1137 */ 1138 ice_vsi_del_txqs_ctx(vsi); 1139 ice_vsi_del_rxqs_ctx(vsi); 1140 1141 /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */ 1142 ice_free_irqvs(sc); 1143 1144 if (vsi->tx_queues != NULL) { 1145 /* free the tx_rsq arrays */ 1146 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1147 if (txq->tx_rsq != NULL) { 1148 free(txq->tx_rsq, M_ICE); 1149 txq->tx_rsq = NULL; 1150 } 1151 } 1152 free(vsi->tx_queues, M_ICE); 1153 vsi->tx_queues = NULL; 1154 vsi->num_tx_queues = 0; 1155 } 1156 if (vsi->rx_queues != NULL) { 1157 free(vsi->rx_queues, M_ICE); 1158 vsi->rx_queues = NULL; 1159 vsi->num_rx_queues = 0; 1160 } 1161 } 1162 1163 /** 1164 * ice_msix_que - Fast interrupt handler for MSI-X receive queues 1165 * @arg: The Rx queue memory 1166 * 1167 * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when 1168 * an MSI-X interrupt for a given queue is triggered. Currently this just asks 1169 * iflib to schedule the main Rx thread. 1170 */ 1171 static int 1172 ice_msix_que(void *arg) 1173 { 1174 struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg; 1175 1176 /* TODO: dynamic ITR algorithm?? */ 1177 1178 return (FILTER_SCHEDULE_THREAD); 1179 } 1180 1181 /** 1182 * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt 1183 * @arg: pointer to device softc memory 1184 * 1185 * Called by iflib when an administrative interrupt occurs. Should perform any 1186 * fast logic for handling the interrupt cause, and then indicate whether the 1187 * admin task needs to be queued. 1188 */ 1189 static int 1190 ice_msix_admin(void *arg) 1191 { 1192 struct ice_softc *sc = (struct ice_softc *)arg; 1193 struct ice_hw *hw = &sc->hw; 1194 device_t dev = sc->dev; 1195 u32 oicr; 1196 1197 /* There is no safe way to modify the enabled miscellaneous causes of 1198 * the OICR vector at runtime, as doing so would be prone to race 1199 * conditions. Reading PFINT_OICR will unmask the associated interrupt 1200 * causes and allow future interrupts to occur. The admin interrupt 1201 * vector will not be re-enabled until after we exit this function, 1202 * but any delayed tasks must be resilient against possible "late 1203 * arrival" interrupts that occur while we're already handling the 1204 * task. This is done by using state bits and serializing these 1205 * delayed tasks via the admin status task function. 1206 */ 1207 oicr = rd32(hw, PFINT_OICR); 1208 1209 /* Processing multiple controlq interrupts on a single vector does not 1210 * provide an indication of which controlq triggered the interrupt. 1211 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL 1212 * registers. However, the INTEVENT bit is not guaranteed to be set as 1213 * it gets automatically cleared when the hardware acknowledges the 1214 * interrupt. 1215 * 1216 * This means we don't really have a good indication of whether or 1217 * which controlq triggered this interrupt. We'll just notify the 1218 * admin task that it should check all the controlqs. 1219 */ 1220 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 1221 1222 if (oicr & PFINT_OICR_VFLR_M) { 1223 ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING); 1224 } 1225 1226 if (oicr & PFINT_OICR_MAL_DETECT_M) { 1227 ice_set_state(&sc->state, ICE_STATE_MDD_PENDING); 1228 } 1229 1230 if (oicr & PFINT_OICR_GRST_M) { 1231 u32 reset; 1232 1233 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1234 GLGEN_RSTAT_RESET_TYPE_S; 1235 1236 if (reset == ICE_RESET_CORER) 1237 sc->soft_stats.corer_count++; 1238 else if (reset == ICE_RESET_GLOBR) 1239 sc->soft_stats.globr_count++; 1240 else 1241 sc->soft_stats.empr_count++; 1242 1243 /* There are a couple of bits at play for handling resets. 1244 * First, the ICE_STATE_RESET_OICR_RECV bit is used to 1245 * indicate that the driver has received an OICR with a reset 1246 * bit active, indicating that a CORER/GLOBR/EMPR is about to 1247 * happen. Second, we set hw->reset_ongoing to indicate that 1248 * the hardware is in reset. We will set this back to false as 1249 * soon as the driver has determined that the hardware is out 1250 * of reset. 1251 * 1252 * If the driver wishes to trigger a reqest, it can set one of 1253 * the ICE_STATE_RESET_*_REQ bits, which will trigger the 1254 * correct type of reset. 1255 */ 1256 if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 1257 hw->reset_ongoing = true; 1258 } 1259 1260 if (oicr & PFINT_OICR_ECC_ERR_M) { 1261 device_printf(dev, "ECC Error detected!\n"); 1262 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1263 } 1264 1265 if (oicr & PFINT_OICR_PE_CRITERR_M) { 1266 device_printf(dev, "Critical Protocol Engine Error detected!\n"); 1267 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1268 } 1269 1270 if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { 1271 device_printf(dev, "PCI Exception detected!\n"); 1272 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1273 } 1274 1275 if (oicr & PFINT_OICR_HMC_ERR_M) { 1276 /* Log the HMC errors, but don't disable the interrupt cause */ 1277 ice_log_hmc_error(hw, dev); 1278 } 1279 1280 return (FILTER_SCHEDULE_THREAD); 1281 } 1282 1283 /** 1284 * ice_allocate_msix - Allocate MSI-X vectors for the interface 1285 * @sc: the device private softc 1286 * 1287 * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process. 1288 * 1289 * First, determine a suitable total number of vectors based on the number 1290 * of CPUs, RSS buckets, the administrative vector, and other demands such as 1291 * RDMA. 1292 * 1293 * Request the desired amount of vectors, and see how many we obtain. If we 1294 * don't obtain as many as desired, reduce the demands by lowering the number 1295 * of requested queues or reducing the demand from other features such as 1296 * RDMA. 1297 * 1298 * @remark This function is required because the driver sets the 1299 * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors 1300 * manually. 1301 * 1302 * @remark This driver will only use MSI-X vectors. If this is not possible, 1303 * neither MSI or legacy interrupts will be tried. 1304 * 1305 * @post on success this function must set the following scctx parameters: 1306 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr. 1307 * 1308 * @returns zero on success or an error code on failure. 1309 */ 1310 static int 1311 ice_allocate_msix(struct ice_softc *sc) 1312 { 1313 bool iflib_override_queue_count = false; 1314 if_softc_ctx_t scctx = sc->scctx; 1315 device_t dev = sc->dev; 1316 cpuset_t cpus; 1317 int bar, queues, vectors, requested; 1318 int err = 0; 1319 int rdma; 1320 1321 /* Allocate the MSI-X bar */ 1322 bar = scctx->isc_msix_bar; 1323 sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); 1324 if (!sc->msix_table) { 1325 device_printf(dev, "Unable to map MSI-X table\n"); 1326 return (ENOMEM); 1327 } 1328 1329 /* Check if the iflib queue count sysctls have been set */ 1330 if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs) 1331 iflib_override_queue_count = true; 1332 1333 err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus); 1334 if (err) { 1335 device_printf(dev, "%s: Unable to fetch the CPU list: %s\n", 1336 __func__, ice_err_str(err)); 1337 CPU_COPY(&all_cpus, &cpus); 1338 } 1339 1340 /* Attempt to mimic behavior of iflib_msix_init */ 1341 if (iflib_override_queue_count) { 1342 /* 1343 * If the override sysctls have been set, limit the queues to 1344 * the number of logical CPUs. 1345 */ 1346 queues = mp_ncpus; 1347 } else { 1348 /* 1349 * Otherwise, limit the queue count to the CPUs associated 1350 * with the NUMA node the device is associated with. 1351 */ 1352 queues = CPU_COUNT(&cpus); 1353 } 1354 1355 /* Clamp to the number of RSS buckets */ 1356 queues = imin(queues, rss_getnumbuckets()); 1357 1358 /* 1359 * Clamp the number of queue pairs to the minimum of the requested Tx 1360 * and Rx queues. 1361 */ 1362 queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets); 1363 queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets); 1364 1365 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RDMA)) { 1366 /* 1367 * Choose a number of RDMA vectors based on the number of CPUs 1368 * up to a maximum 1369 */ 1370 rdma = min(CPU_COUNT(&cpus), ICE_RDMA_MAX_MSIX); 1371 1372 /* Further limit by the user configurable tunable */ 1373 rdma = min(rdma, ice_rdma_max_msix); 1374 } else { 1375 rdma = 0; 1376 } 1377 1378 /* 1379 * Determine the number of vectors to request. Note that we also need 1380 * to allocate one vector for administrative tasks. 1381 */ 1382 requested = rdma + queues + 1; 1383 1384 vectors = requested; 1385 1386 err = pci_alloc_msix(dev, &vectors); 1387 if (err) { 1388 device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n", 1389 vectors, ice_err_str(err)); 1390 goto err_free_msix_table; 1391 } 1392 1393 /* If we don't receive enough vectors, reduce demands */ 1394 if (vectors < requested) { 1395 int diff = requested - vectors; 1396 1397 device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n", 1398 requested, vectors); 1399 1400 /* 1401 * The OS didn't grant us the requested number of vectors. 1402 * Check to see if we can reduce demands by limiting the 1403 * number of vectors allocated to certain features. 1404 */ 1405 1406 if (rdma >= diff) { 1407 /* Reduce the number of RDMA vectors we reserve */ 1408 rdma -= diff; 1409 diff = 0; 1410 } else { 1411 /* Disable RDMA and reduce the difference */ 1412 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 1413 diff -= rdma; 1414 rdma = 0; 1415 } 1416 1417 /* 1418 * If we still have a difference, we need to reduce the number 1419 * of queue pairs. 1420 * 1421 * However, we still need at least one vector for the admin 1422 * interrupt and one queue pair. 1423 */ 1424 if (queues <= diff) { 1425 device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n"); 1426 err = (ERANGE); 1427 goto err_pci_release_msi; 1428 } 1429 1430 queues -= diff; 1431 } 1432 1433 device_printf(dev, "Using %d Tx and Rx queues\n", queues); 1434 if (rdma) 1435 device_printf(dev, "Reserving %d MSI-X interrupts for iRDMA\n", 1436 rdma); 1437 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 1438 vectors); 1439 1440 scctx->isc_vectors = vectors; 1441 scctx->isc_nrxqsets = queues; 1442 scctx->isc_ntxqsets = queues; 1443 scctx->isc_intr = IFLIB_INTR_MSIX; 1444 1445 sc->irdma_vectors = rdma; 1446 1447 /* Interrupt allocation tracking isn't required in recovery mode, 1448 * since neither RDMA nor VFs are enabled. 1449 */ 1450 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1451 return (0); 1452 1453 /* Keep track of which interrupt indices are being used for what */ 1454 sc->lan_vectors = vectors - rdma; 1455 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors); 1456 if (err) { 1457 device_printf(dev, "Unable to assign PF interrupt mapping: %s\n", 1458 ice_err_str(err)); 1459 goto err_pci_release_msi; 1460 } 1461 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->rdma_imap, rdma); 1462 if (err) { 1463 device_printf(dev, "Unable to assign PF RDMA interrupt mapping: %s\n", 1464 ice_err_str(err)); 1465 ice_resmgr_release_map(&sc->imgr, sc->pf_imap, 1466 sc->lan_vectors); 1467 goto err_pci_release_msi; 1468 } 1469 1470 return (0); 1471 1472 err_pci_release_msi: 1473 pci_release_msi(dev); 1474 err_free_msix_table: 1475 if (sc->msix_table != NULL) { 1476 bus_release_resource(sc->dev, SYS_RES_MEMORY, 1477 rman_get_rid(sc->msix_table), 1478 sc->msix_table); 1479 sc->msix_table = NULL; 1480 } 1481 1482 return (err); 1483 } 1484 1485 /** 1486 * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues 1487 * @ctx: the iflib context structure 1488 * @msix: the number of vectors we were assigned 1489 * 1490 * Called by iflib to assign MSI-X vectors to queues. Currently requires that 1491 * we get at least the same number of vectors as we have queues, and that we 1492 * always have the same number of Tx and Rx queues. 1493 * 1494 * Tx queues use a softirq instead of using their own hardware interrupt. 1495 */ 1496 static int 1497 ice_if_msix_intr_assign(if_ctx_t ctx, int msix) 1498 { 1499 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1500 struct ice_vsi *vsi = &sc->pf_vsi; 1501 int err, i, vector; 1502 1503 ASSERT_CTX_LOCKED(sc); 1504 1505 if (vsi->num_rx_queues != vsi->num_tx_queues) { 1506 device_printf(sc->dev, 1507 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 1508 vsi->num_tx_queues, vsi->num_rx_queues); 1509 return (EOPNOTSUPP); 1510 } 1511 1512 if (msix < (vsi->num_rx_queues + 1)) { 1513 device_printf(sc->dev, 1514 "Not enough MSI-X vectors to assign one vector to each queue pair\n"); 1515 return (EOPNOTSUPP); 1516 } 1517 1518 /* Save the number of vectors for future use */ 1519 sc->num_irq_vectors = vsi->num_rx_queues + 1; 1520 1521 /* Allocate space to store the IRQ vector data */ 1522 if (!(sc->irqvs = 1523 (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors), 1524 M_ICE, M_NOWAIT))) { 1525 device_printf(sc->dev, 1526 "Unable to allocate irqv memory\n"); 1527 return (ENOMEM); 1528 } 1529 1530 /* Administrative interrupt events will use vector 0 */ 1531 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN, 1532 ice_msix_admin, sc, 0, "admin"); 1533 if (err) { 1534 device_printf(sc->dev, 1535 "Failed to register Admin queue handler: %s\n", 1536 ice_err_str(err)); 1537 goto free_irqvs; 1538 } 1539 sc->irqvs[0].me = 0; 1540 1541 /* Do not allocate queue interrupts when in recovery mode */ 1542 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1543 return (0); 1544 1545 for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) { 1546 struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 1547 struct ice_tx_queue *txq = &vsi->tx_queues[i]; 1548 int rid = vector + 1; 1549 char irq_name[16]; 1550 1551 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1552 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid, 1553 IFLIB_INTR_RXTX, ice_msix_que, 1554 rxq, rxq->me, irq_name); 1555 if (err) { 1556 device_printf(sc->dev, 1557 "Failed to allocate q int %d err: %s\n", 1558 i, ice_err_str(err)); 1559 vector--; 1560 i--; 1561 goto fail; 1562 } 1563 sc->irqvs[vector].me = vector; 1564 rxq->irqv = &sc->irqvs[vector]; 1565 1566 bzero(irq_name, sizeof(irq_name)); 1567 1568 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1569 iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq, 1570 IFLIB_INTR_TX, txq, 1571 txq->me, irq_name); 1572 txq->irqv = &sc->irqvs[vector]; 1573 } 1574 1575 return (0); 1576 fail: 1577 for (; i >= 0; i--, vector--) 1578 iflib_irq_free(ctx, &sc->irqvs[vector].irq); 1579 iflib_irq_free(ctx, &sc->irqvs[0].irq); 1580 free_irqvs: 1581 free(sc->irqvs, M_ICE); 1582 sc->irqvs = NULL; 1583 return err; 1584 } 1585 1586 /** 1587 * ice_if_mtu_set - Set the device MTU 1588 * @ctx: iflib context structure 1589 * @mtu: the MTU requested 1590 * 1591 * Called by iflib to configure the device's Maximum Transmission Unit (MTU). 1592 * 1593 * @pre assumes the caller holds the iflib CTX lock 1594 */ 1595 static int 1596 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1597 { 1598 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1599 1600 ASSERT_CTX_LOCKED(sc); 1601 1602 /* Do not support configuration when in recovery mode */ 1603 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1604 return (ENOSYS); 1605 1606 if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU) 1607 return (EINVAL); 1608 1609 sc->scctx->isc_max_frame_size = mtu + 1610 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 1611 1612 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 1613 1614 return (0); 1615 } 1616 1617 /** 1618 * ice_if_intr_enable - Enable device interrupts 1619 * @ctx: iflib context structure 1620 * 1621 * Called by iflib to request enabling device interrupts. 1622 */ 1623 static void 1624 ice_if_intr_enable(if_ctx_t ctx) 1625 { 1626 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1627 struct ice_vsi *vsi = &sc->pf_vsi; 1628 struct ice_hw *hw = &sc->hw; 1629 1630 ASSERT_CTX_LOCKED(sc); 1631 1632 /* Enable ITR 0 */ 1633 ice_enable_intr(hw, sc->irqvs[0].me); 1634 1635 /* Do not enable queue interrupts in recovery mode */ 1636 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1637 return; 1638 1639 /* Enable all queue interrupts */ 1640 for (int i = 0; i < vsi->num_rx_queues; i++) 1641 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 1642 } 1643 1644 /** 1645 * ice_if_intr_disable - Disable device interrupts 1646 * @ctx: iflib context structure 1647 * 1648 * Called by iflib to request disabling device interrupts. 1649 */ 1650 static void 1651 ice_if_intr_disable(if_ctx_t ctx) 1652 { 1653 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1654 struct ice_hw *hw = &sc->hw; 1655 unsigned int i; 1656 1657 ASSERT_CTX_LOCKED(sc); 1658 1659 /* IFDI_INTR_DISABLE may be called prior to interrupts actually being 1660 * assigned to queues. Instead of assuming that the interrupt 1661 * assignment in the rx_queues structure is valid, just disable all 1662 * possible interrupts 1663 * 1664 * Note that we choose not to disable ITR 0 because this handles the 1665 * AdminQ interrupts, and we want to keep processing these even when 1666 * the interface is offline. 1667 */ 1668 for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++) 1669 ice_disable_intr(hw, i); 1670 } 1671 1672 /** 1673 * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 1674 * @ctx: iflib context structure 1675 * @rxqid: the Rx queue to enable 1676 * 1677 * Enable a specific Rx queue interrupt. 1678 * 1679 * This function is not protected by the iflib CTX lock. 1680 */ 1681 static int 1682 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1683 { 1684 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1685 struct ice_vsi *vsi = &sc->pf_vsi; 1686 struct ice_hw *hw = &sc->hw; 1687 1688 /* Do not enable queue interrupts in recovery mode */ 1689 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1690 return (ENOSYS); 1691 1692 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 1693 return (0); 1694 } 1695 1696 /** 1697 * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 1698 * @ctx: iflib context structure 1699 * @txqid: the Tx queue to enable 1700 * 1701 * Enable a specific Tx queue interrupt. 1702 * 1703 * This function is not protected by the iflib CTX lock. 1704 */ 1705 static int 1706 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1707 { 1708 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1709 struct ice_vsi *vsi = &sc->pf_vsi; 1710 struct ice_hw *hw = &sc->hw; 1711 1712 /* Do not enable queue interrupts in recovery mode */ 1713 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1714 return (ENOSYS); 1715 1716 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 1717 return (0); 1718 } 1719 1720 /** 1721 * ice_if_promisc_set - Set device promiscuous mode 1722 * @ctx: iflib context structure 1723 * @flags: promiscuous flags to configure 1724 * 1725 * Called by iflib to configure device promiscuous mode. 1726 * 1727 * @remark Calls to this function will always overwrite the previous setting 1728 */ 1729 static int 1730 ice_if_promisc_set(if_ctx_t ctx, int flags) 1731 { 1732 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1733 struct ice_hw *hw = &sc->hw; 1734 device_t dev = sc->dev; 1735 enum ice_status status; 1736 bool promisc_enable = flags & IFF_PROMISC; 1737 bool multi_enable = flags & IFF_ALLMULTI; 1738 1739 /* Do not support configuration when in recovery mode */ 1740 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1741 return (ENOSYS); 1742 1743 if (multi_enable) 1744 return (EOPNOTSUPP); 1745 1746 if (promisc_enable) { 1747 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1748 ICE_VSI_PROMISC_MASK, 0); 1749 if (status && status != ICE_ERR_ALREADY_EXISTS) { 1750 device_printf(dev, 1751 "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n", 1752 ice_status_str(status), 1753 ice_aq_str(hw->adminq.sq_last_status)); 1754 return (EIO); 1755 } 1756 } else { 1757 status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx, 1758 ICE_VSI_PROMISC_MASK, 0); 1759 if (status) { 1760 device_printf(dev, 1761 "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n", 1762 ice_status_str(status), 1763 ice_aq_str(hw->adminq.sq_last_status)); 1764 return (EIO); 1765 } 1766 } 1767 1768 return (0); 1769 } 1770 1771 /** 1772 * ice_if_media_change - Change device media 1773 * @ctx: device ctx structure 1774 * 1775 * Called by iflib when a media change is requested. This operation is not 1776 * supported by the hardware, so we just return an error code. 1777 */ 1778 static int 1779 ice_if_media_change(if_ctx_t ctx) 1780 { 1781 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1782 1783 device_printf(sc->dev, "Media change is not supported.\n"); 1784 return (ENODEV); 1785 } 1786 1787 /** 1788 * ice_if_media_status - Report current device media 1789 * @ctx: iflib context structure 1790 * @ifmr: ifmedia request structure to update 1791 * 1792 * Updates the provided ifmr with current device media status, including link 1793 * status and media type. 1794 */ 1795 static void 1796 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1797 { 1798 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1799 struct ice_link_status *li = &sc->hw.port_info->phy.link_info; 1800 1801 ifmr->ifm_status = IFM_AVALID; 1802 ifmr->ifm_active = IFM_ETHER; 1803 1804 /* Never report link up or media types when in recovery mode */ 1805 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1806 return; 1807 1808 if (!sc->link_up) 1809 return; 1810 1811 ifmr->ifm_status |= IFM_ACTIVE; 1812 ifmr->ifm_active |= IFM_FDX; 1813 1814 if (li->phy_type_low) 1815 ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low); 1816 else if (li->phy_type_high) 1817 ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high); 1818 else 1819 ifmr->ifm_active |= IFM_UNKNOWN; 1820 1821 /* Report flow control status as well */ 1822 if (li->an_info & ICE_AQ_LINK_PAUSE_TX) 1823 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1824 if (li->an_info & ICE_AQ_LINK_PAUSE_RX) 1825 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1826 } 1827 1828 /** 1829 * ice_init_tx_tracking - Initialize Tx queue software tracking values 1830 * @vsi: the VSI to initialize 1831 * 1832 * Initialize Tx queue software tracking values, including the Report Status 1833 * queue, and related software tracking values. 1834 */ 1835 static void 1836 ice_init_tx_tracking(struct ice_vsi *vsi) 1837 { 1838 struct ice_tx_queue *txq; 1839 size_t j; 1840 int i; 1841 1842 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1843 1844 txq->tx_rs_cidx = txq->tx_rs_pidx = 0; 1845 1846 /* Initialize the last processed descriptor to be the end of 1847 * the ring, rather than the start, so that we avoid an 1848 * off-by-one error in ice_ift_txd_credits_update for the 1849 * first packet. 1850 */ 1851 txq->tx_cidx_processed = txq->desc_count - 1; 1852 1853 for (j = 0; j < txq->desc_count; j++) 1854 txq->tx_rsq[j] = QIDX_INVALID; 1855 } 1856 } 1857 1858 /** 1859 * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues 1860 * @sc: the device softc 1861 * 1862 * Called to update the Rx queue mbuf_sz parameter for configuring the receive 1863 * buffer sizes when programming hardware. 1864 */ 1865 static void 1866 ice_update_rx_mbuf_sz(struct ice_softc *sc) 1867 { 1868 uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx); 1869 struct ice_vsi *vsi = &sc->pf_vsi; 1870 1871 MPASS(mbuf_sz <= UINT16_MAX); 1872 vsi->mbuf_sz = mbuf_sz; 1873 } 1874 1875 /** 1876 * ice_if_init - Initialize the device 1877 * @ctx: iflib ctx structure 1878 * 1879 * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes 1880 * device filters and prepares the Tx and Rx engines. 1881 * 1882 * @pre assumes the caller holds the iflib CTX lock 1883 */ 1884 static void 1885 ice_if_init(if_ctx_t ctx) 1886 { 1887 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1888 device_t dev = sc->dev; 1889 int err; 1890 1891 ASSERT_CTX_LOCKED(sc); 1892 1893 /* 1894 * We've seen an issue with 11.3/12.1 where sideband routines are 1895 * called after detach is called. This would call routines after 1896 * if_stop, causing issues with the teardown process. This has 1897 * seemingly been fixed in STABLE snapshots, but it seems like a 1898 * good idea to have this guard here regardless. 1899 */ 1900 if (ice_driver_is_detaching(sc)) 1901 return; 1902 1903 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1904 return; 1905 1906 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 1907 device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n"); 1908 return; 1909 } 1910 1911 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 1912 device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n"); 1913 return; 1914 } 1915 1916 ice_update_rx_mbuf_sz(sc); 1917 1918 /* Update the MAC address... User might use a LAA */ 1919 err = ice_update_laa_mac(sc); 1920 if (err) { 1921 device_printf(dev, 1922 "LAA address change failed, err %s\n", 1923 ice_err_str(err)); 1924 return; 1925 } 1926 1927 /* Initialize software Tx tracking values */ 1928 ice_init_tx_tracking(&sc->pf_vsi); 1929 1930 err = ice_cfg_vsi_for_tx(&sc->pf_vsi); 1931 if (err) { 1932 device_printf(dev, 1933 "Unable to configure the main VSI for Tx: %s\n", 1934 ice_err_str(err)); 1935 return; 1936 } 1937 1938 err = ice_cfg_vsi_for_rx(&sc->pf_vsi); 1939 if (err) { 1940 device_printf(dev, 1941 "Unable to configure the main VSI for Rx: %s\n", 1942 ice_err_str(err)); 1943 goto err_cleanup_tx; 1944 } 1945 1946 err = ice_control_rx_queues(&sc->pf_vsi, true); 1947 if (err) { 1948 device_printf(dev, 1949 "Unable to enable Rx rings for transmit: %s\n", 1950 ice_err_str(err)); 1951 goto err_cleanup_tx; 1952 } 1953 1954 err = ice_cfg_pf_default_mac_filters(sc); 1955 if (err) { 1956 device_printf(dev, 1957 "Unable to configure default MAC filters: %s\n", 1958 ice_err_str(err)); 1959 goto err_stop_rx; 1960 } 1961 1962 /* We use software interrupts for Tx, so we only program the hardware 1963 * interrupts for Rx. 1964 */ 1965 ice_configure_rxq_interrupts(&sc->pf_vsi); 1966 ice_configure_rx_itr(&sc->pf_vsi); 1967 1968 /* Configure promiscuous mode */ 1969 ice_if_promisc_set(ctx, if_getflags(sc->ifp)); 1970 1971 ice_rdma_pf_init(sc); 1972 1973 ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED); 1974 return; 1975 1976 err_stop_rx: 1977 ice_control_rx_queues(&sc->pf_vsi, false); 1978 err_cleanup_tx: 1979 ice_vsi_disable_tx(&sc->pf_vsi); 1980 } 1981 1982 /** 1983 * ice_poll_for_media_avail - Re-enable link if media is detected 1984 * @sc: device private structure 1985 * 1986 * Intended to be called from the driver's timer function, this function 1987 * sends the Get Link Status AQ command and re-enables HW link if the 1988 * command says that media is available. 1989 * 1990 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing, 1991 * since media removal events are supposed to be sent to the driver through 1992 * a link status event. 1993 */ 1994 static void 1995 ice_poll_for_media_avail(struct ice_softc *sc) 1996 { 1997 struct ice_hw *hw = &sc->hw; 1998 struct ice_port_info *pi = hw->port_info; 1999 2000 if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) { 2001 pi->phy.get_link_info = true; 2002 ice_get_link_status(pi, &sc->link_up); 2003 2004 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2005 enum ice_status status; 2006 2007 /* Re-enable link and re-apply user link settings */ 2008 ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC); 2009 2010 /* Update the OS about changes in media capability */ 2011 status = ice_add_media_types(sc, sc->media); 2012 if (status) 2013 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 2014 ice_status_str(status), 2015 ice_aq_str(hw->adminq.sq_last_status)); 2016 2017 ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA); 2018 } 2019 } 2020 } 2021 2022 /** 2023 * ice_if_timer - called by iflib periodically 2024 * @ctx: iflib ctx structure 2025 * @qid: the queue this timer was called for 2026 * 2027 * This callback is triggered by iflib periodically. We use it to update the 2028 * hw statistics. 2029 * 2030 * @remark this function is not protected by the iflib CTX lock. 2031 */ 2032 static void 2033 ice_if_timer(if_ctx_t ctx, uint16_t qid) 2034 { 2035 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2036 uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx; 2037 2038 if (qid != 0) 2039 return; 2040 2041 /* Do not attempt to update stats when in recovery mode */ 2042 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2043 return; 2044 2045 /* Update device statistics */ 2046 ice_update_pf_stats(sc); 2047 2048 /* 2049 * For proper watchdog management, the iflib stack needs to know if 2050 * we've been paused during the last interval. Check if the 2051 * link_xoff_rx stat changed, and set the isc_pause_frames, if so. 2052 */ 2053 if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx) 2054 sc->scctx->isc_pause_frames = 1; 2055 2056 /* Update the primary VSI stats */ 2057 ice_update_vsi_hw_stats(&sc->pf_vsi); 2058 } 2059 2060 /** 2061 * ice_admin_timer - called periodically to trigger the admin task 2062 * @arg: callout(9) argument pointing to the device private softc structure 2063 * 2064 * Timer function used as part of a callout(9) timer that will periodically 2065 * trigger the admin task, even when the interface is down. 2066 * 2067 * @remark this function is not called by iflib and is not protected by the 2068 * iflib CTX lock. 2069 * 2070 * @remark because this is a callout function, it cannot sleep and should not 2071 * attempt taking the iflib CTX lock. 2072 */ 2073 static void 2074 ice_admin_timer(void *arg) 2075 { 2076 struct ice_softc *sc = (struct ice_softc *)arg; 2077 2078 /* 2079 * There is a point where callout routines are no longer 2080 * cancelable. So there exists a window of time where the 2081 * driver enters detach() and tries to cancel the callout, but the 2082 * callout routine has passed the cancellation point. The detach() 2083 * routine is unaware of this and tries to free resources that the 2084 * callout routine needs. So we check for the detach state flag to 2085 * at least shrink the window of opportunity. 2086 */ 2087 if (ice_driver_is_detaching(sc)) 2088 return; 2089 2090 /* Fire off the admin task */ 2091 iflib_admin_intr_deferred(sc->ctx); 2092 2093 /* Reschedule the admin timer */ 2094 callout_schedule(&sc->admin_timer, hz/2); 2095 } 2096 2097 /** 2098 * ice_transition_recovery_mode - Transition to recovery mode 2099 * @sc: the device private softc 2100 * 2101 * Called when the driver detects that the firmware has entered recovery mode 2102 * at run time. 2103 */ 2104 static void 2105 ice_transition_recovery_mode(struct ice_softc *sc) 2106 { 2107 struct ice_vsi *vsi = &sc->pf_vsi; 2108 int i; 2109 2110 device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 2111 2112 /* Tell the stack that the link has gone down */ 2113 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 2114 2115 /* Request that the device be re-initialized */ 2116 ice_request_stack_reinit(sc); 2117 2118 ice_rdma_pf_detach(sc); 2119 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2120 2121 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2122 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2123 2124 ice_vsi_del_txqs_ctx(vsi); 2125 ice_vsi_del_rxqs_ctx(vsi); 2126 2127 for (i = 0; i < sc->num_available_vsi; i++) { 2128 if (sc->all_vsi[i]) 2129 ice_release_vsi(sc->all_vsi[i]); 2130 } 2131 sc->num_available_vsi = 0; 2132 2133 if (sc->all_vsi) { 2134 free(sc->all_vsi, M_ICE); 2135 sc->all_vsi = NULL; 2136 } 2137 2138 /* Destroy the interrupt manager */ 2139 ice_resmgr_destroy(&sc->imgr); 2140 /* Destroy the queue managers */ 2141 ice_resmgr_destroy(&sc->tx_qmgr); 2142 ice_resmgr_destroy(&sc->rx_qmgr); 2143 2144 ice_deinit_hw(&sc->hw); 2145 } 2146 2147 /** 2148 * ice_transition_safe_mode - Transition to safe mode 2149 * @sc: the device private softc 2150 * 2151 * Called when the driver attempts to reload the DDP package during a device 2152 * reset, and the new download fails. If so, we must transition to safe mode 2153 * at run time. 2154 * 2155 * @remark although safe mode normally allocates only a single queue, we can't 2156 * change the number of queues dynamically when using iflib. Due to this, we 2157 * do not attempt to reduce the number of queues. 2158 */ 2159 static void 2160 ice_transition_safe_mode(struct ice_softc *sc) 2161 { 2162 /* Indicate that we are in Safe mode */ 2163 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap); 2164 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en); 2165 2166 ice_rdma_pf_detach(sc); 2167 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2168 2169 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2170 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2171 2172 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2173 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en); 2174 } 2175 2176 /** 2177 * ice_if_update_admin_status - update admin status 2178 * @ctx: iflib ctx structure 2179 * 2180 * Called by iflib to update the admin status. For our purposes, this means 2181 * check the adminq, and update the link status. It's ultimately triggered by 2182 * our admin interrupt, or by the ice_if_timer periodically. 2183 * 2184 * @pre assumes the caller holds the iflib CTX lock 2185 */ 2186 static void 2187 ice_if_update_admin_status(if_ctx_t ctx) 2188 { 2189 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2190 enum ice_fw_modes fw_mode; 2191 bool reschedule = false; 2192 u16 pending = 0; 2193 2194 ASSERT_CTX_LOCKED(sc); 2195 2196 /* Check if the firmware entered recovery mode at run time */ 2197 fw_mode = ice_get_fw_mode(&sc->hw); 2198 if (fw_mode == ICE_FW_MODE_REC) { 2199 if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2200 /* If we just entered recovery mode, log a warning to 2201 * the system administrator and deinit driver state 2202 * that is no longer functional. 2203 */ 2204 ice_transition_recovery_mode(sc); 2205 } 2206 } else if (fw_mode == ICE_FW_MODE_ROLLBACK) { 2207 if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) { 2208 /* Rollback mode isn't fatal, but we don't want to 2209 * repeatedly post a message about it. 2210 */ 2211 ice_print_rollback_msg(&sc->hw); 2212 } 2213 } 2214 2215 /* Handle global reset events */ 2216 ice_handle_reset_event(sc); 2217 2218 /* Handle PF reset requests */ 2219 ice_handle_pf_reset_request(sc); 2220 2221 /* Handle MDD events */ 2222 ice_handle_mdd_event(sc); 2223 2224 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) || 2225 ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) || 2226 ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2227 /* 2228 * If we know the control queues are disabled, skip processing 2229 * the control queues entirely. 2230 */ 2231 ; 2232 } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) { 2233 ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending); 2234 if (pending > 0) 2235 reschedule = true; 2236 2237 ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending); 2238 if (pending > 0) 2239 reschedule = true; 2240 } 2241 2242 /* Poll for link up */ 2243 ice_poll_for_media_avail(sc); 2244 2245 /* Check and update link status */ 2246 ice_update_link_status(sc, false); 2247 2248 /* 2249 * If there are still messages to process, we need to reschedule 2250 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be 2251 * woken up at the next interrupt or timer event. 2252 */ 2253 if (reschedule) { 2254 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 2255 iflib_admin_intr_deferred(ctx); 2256 } else { 2257 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2258 } 2259 } 2260 2261 /** 2262 * ice_prepare_for_reset - Prepare device for an impending reset 2263 * @sc: The device private softc 2264 * 2265 * Prepare the driver for an impending reset, shutting down VSIs, clearing the 2266 * scheduler setup, and shutting down controlqs. Uses the 2267 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the 2268 * driver for reset or not. 2269 */ 2270 static void 2271 ice_prepare_for_reset(struct ice_softc *sc) 2272 { 2273 struct ice_hw *hw = &sc->hw; 2274 2275 /* If we're already prepared, there's nothing to do */ 2276 if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) 2277 return; 2278 2279 log(LOG_INFO, "%s: preparing to reset device logic\n", sc->ifp->if_xname); 2280 2281 /* In recovery mode, hardware is not initialized */ 2282 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2283 return; 2284 2285 /* stop the RDMA client */ 2286 ice_rdma_pf_stop(sc); 2287 2288 /* Release the main PF VSI queue mappings */ 2289 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2290 sc->pf_vsi.num_tx_queues); 2291 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2292 sc->pf_vsi.num_rx_queues); 2293 2294 ice_clear_hw_tbls(hw); 2295 2296 if (hw->port_info) 2297 ice_sched_clear_port(hw->port_info); 2298 2299 ice_shutdown_all_ctrlq(hw); 2300 } 2301 2302 /** 2303 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping 2304 * @sc: the device softc pointer 2305 * 2306 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue 2307 * mapping after a reset occurred. 2308 */ 2309 static int 2310 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc) 2311 { 2312 struct ice_vsi *vsi = &sc->pf_vsi; 2313 struct ice_tx_queue *txq; 2314 struct ice_rx_queue *rxq; 2315 int err, i; 2316 2317 /* Re-assign Tx queues from PF space to the main VSI */ 2318 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, 2319 vsi->num_tx_queues); 2320 if (err) { 2321 device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n", 2322 ice_err_str(err)); 2323 return (err); 2324 } 2325 2326 /* Re-assign Rx queues from PF space to this VSI */ 2327 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, 2328 vsi->num_rx_queues); 2329 if (err) { 2330 device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n", 2331 ice_err_str(err)); 2332 goto err_release_tx_queues; 2333 } 2334 2335 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 2336 2337 /* Re-assign Tx queue tail pointers */ 2338 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 2339 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 2340 2341 /* Re-assign Rx queue tail pointers */ 2342 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 2343 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 2344 2345 return (0); 2346 2347 err_release_tx_queues: 2348 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2349 sc->pf_vsi.num_tx_queues); 2350 2351 return (err); 2352 } 2353 2354 /* determine if the iflib context is active */ 2355 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING)) 2356 2357 /** 2358 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode 2359 * @sc: The device private softc 2360 * 2361 * Handle a driver rebuild while in recovery mode. This will only rebuild the 2362 * limited functionality supported while in recovery mode. 2363 */ 2364 static void 2365 ice_rebuild_recovery_mode(struct ice_softc *sc) 2366 { 2367 device_t dev = sc->dev; 2368 2369 /* enable PCIe bus master */ 2370 pci_enable_busmaster(dev); 2371 2372 /* Configure interrupt causes for the administrative interrupt */ 2373 ice_configure_misc_interrupts(sc); 2374 2375 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2376 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2377 2378 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2379 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2380 2381 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2382 2383 /* In order to completely restore device functionality, the iflib core 2384 * needs to be reset. We need to request an iflib reset. Additionally, 2385 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2386 * the iflib core, we also want re-run the admin task so that iflib 2387 * resets immediately instead of waiting for the next interrupt. 2388 */ 2389 ice_request_stack_reinit(sc); 2390 2391 return; 2392 } 2393 2394 /** 2395 * ice_rebuild - Rebuild driver state post reset 2396 * @sc: The device private softc 2397 * 2398 * Restore driver state after a reset occurred. Restart the controlqs, setup 2399 * the hardware port, and re-enable the VSIs. 2400 */ 2401 static void 2402 ice_rebuild(struct ice_softc *sc) 2403 { 2404 struct ice_hw *hw = &sc->hw; 2405 device_t dev = sc->dev; 2406 enum ice_status status; 2407 int err; 2408 2409 sc->rebuild_ticks = ticks; 2410 2411 /* If we're rebuilding, then a reset has succeeded. */ 2412 ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED); 2413 2414 /* 2415 * If the firmware is in recovery mode, only restore the limited 2416 * functionality supported by recovery mode. 2417 */ 2418 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2419 ice_rebuild_recovery_mode(sc); 2420 return; 2421 } 2422 2423 /* enable PCIe bus master */ 2424 pci_enable_busmaster(dev); 2425 2426 status = ice_init_all_ctrlq(hw); 2427 if (status) { 2428 device_printf(dev, "failed to re-init controlqs, err %s\n", 2429 ice_status_str(status)); 2430 goto err_shutdown_ctrlq; 2431 } 2432 2433 /* Query the allocated resources for Tx scheduler */ 2434 status = ice_sched_query_res_alloc(hw); 2435 if (status) { 2436 device_printf(dev, 2437 "Failed to query scheduler resources, err %s aq_err %s\n", 2438 ice_status_str(status), 2439 ice_aq_str(hw->adminq.sq_last_status)); 2440 goto err_shutdown_ctrlq; 2441 } 2442 2443 /* Re-enable FW logging. Keep going even if this fails */ 2444 status = ice_fwlog_set(hw, &hw->fwlog_cfg); 2445 if (!status) { 2446 /* 2447 * We should have the most updated cached copy of the 2448 * configuration, regardless of whether we're rebuilding 2449 * or not. So we'll simply check to see if logging was 2450 * enabled pre-rebuild. 2451 */ 2452 if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 2453 status = ice_fwlog_register(hw); 2454 if (status) 2455 device_printf(dev, "failed to re-register fw logging, err %s aq_err %s\n", 2456 ice_status_str(status), 2457 ice_aq_str(hw->adminq.sq_last_status)); 2458 } 2459 } else 2460 device_printf(dev, "failed to rebuild fw logging configuration, err %s aq_err %s\n", 2461 ice_status_str(status), 2462 ice_aq_str(hw->adminq.sq_last_status)); 2463 2464 err = ice_send_version(sc); 2465 if (err) 2466 goto err_shutdown_ctrlq; 2467 2468 err = ice_init_link_events(sc); 2469 if (err) { 2470 device_printf(dev, "ice_init_link_events failed: %s\n", 2471 ice_err_str(err)); 2472 goto err_shutdown_ctrlq; 2473 } 2474 2475 status = ice_clear_pf_cfg(hw); 2476 if (status) { 2477 device_printf(dev, "failed to clear PF configuration, err %s\n", 2478 ice_status_str(status)); 2479 goto err_shutdown_ctrlq; 2480 } 2481 2482 ice_clear_pxe_mode(hw); 2483 2484 status = ice_get_caps(hw); 2485 if (status) { 2486 device_printf(dev, "failed to get capabilities, err %s\n", 2487 ice_status_str(status)); 2488 goto err_shutdown_ctrlq; 2489 } 2490 2491 status = ice_sched_init_port(hw->port_info); 2492 if (status) { 2493 device_printf(dev, "failed to initialize port, err %s\n", 2494 ice_status_str(status)); 2495 goto err_sched_cleanup; 2496 } 2497 2498 /* If we previously loaded the package, it needs to be reloaded now */ 2499 if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) { 2500 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 2501 if (status) { 2502 ice_log_pkg_init(sc, &status); 2503 2504 ice_transition_safe_mode(sc); 2505 } 2506 } 2507 2508 ice_reset_pf_stats(sc); 2509 2510 err = ice_rebuild_pf_vsi_qmap(sc); 2511 if (err) { 2512 device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n", 2513 ice_err_str(err)); 2514 goto err_sched_cleanup; 2515 } 2516 err = ice_initialize_vsi(&sc->pf_vsi); 2517 if (err) { 2518 device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n", 2519 ice_err_str(err)); 2520 goto err_release_queue_allocations; 2521 } 2522 2523 /* Replay all VSI configuration */ 2524 err = ice_replay_all_vsi_cfg(sc); 2525 if (err) 2526 goto err_deinit_pf_vsi; 2527 2528 /* Re-enable FW health event reporting */ 2529 ice_init_health_events(sc); 2530 2531 /* Reconfigure the main PF VSI for RSS */ 2532 err = ice_config_rss(&sc->pf_vsi); 2533 if (err) { 2534 device_printf(sc->dev, 2535 "Unable to reconfigure RSS for the main VSI, err %s\n", 2536 ice_err_str(err)); 2537 goto err_deinit_pf_vsi; 2538 } 2539 2540 /* Refresh link status */ 2541 ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED); 2542 sc->hw.port_info->phy.get_link_info = true; 2543 ice_get_link_status(sc->hw.port_info, &sc->link_up); 2544 ice_update_link_status(sc, true); 2545 2546 /* RDMA interface will be restarted by the stack re-init */ 2547 2548 /* Configure interrupt causes for the administrative interrupt */ 2549 ice_configure_misc_interrupts(sc); 2550 2551 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2552 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2553 2554 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2555 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2556 2557 log(LOG_INFO, "%s: device rebuild successful\n", sc->ifp->if_xname); 2558 2559 /* In order to completely restore device functionality, the iflib core 2560 * needs to be reset. We need to request an iflib reset. Additionally, 2561 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2562 * the iflib core, we also want re-run the admin task so that iflib 2563 * resets immediately instead of waiting for the next interrupt. 2564 */ 2565 ice_request_stack_reinit(sc); 2566 2567 return; 2568 2569 err_deinit_pf_vsi: 2570 ice_deinit_vsi(&sc->pf_vsi); 2571 err_release_queue_allocations: 2572 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2573 sc->pf_vsi.num_tx_queues); 2574 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2575 sc->pf_vsi.num_rx_queues); 2576 err_sched_cleanup: 2577 ice_sched_cleanup_all(hw); 2578 err_shutdown_ctrlq: 2579 ice_shutdown_all_ctrlq(hw); 2580 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2581 device_printf(dev, "Driver rebuild failed, please reload the device driver\n"); 2582 } 2583 2584 /** 2585 * ice_handle_reset_event - Handle reset events triggered by OICR 2586 * @sc: The device private softc 2587 * 2588 * Handle reset events triggered by an OICR notification. This includes CORER, 2589 * GLOBR, and EMPR resets triggered by software on this or any other PF or by 2590 * firmware. 2591 * 2592 * @pre assumes the iflib context lock is held, and will unlock it while 2593 * waiting for the hardware to finish reset. 2594 */ 2595 static void 2596 ice_handle_reset_event(struct ice_softc *sc) 2597 { 2598 struct ice_hw *hw = &sc->hw; 2599 enum ice_status status; 2600 device_t dev = sc->dev; 2601 2602 /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will 2603 * trigger an OICR interrupt. Our OICR handler will determine when 2604 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as 2605 * appropriate. 2606 */ 2607 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 2608 return; 2609 2610 ice_prepare_for_reset(sc); 2611 2612 /* 2613 * Release the iflib context lock and wait for the device to finish 2614 * resetting. 2615 */ 2616 IFLIB_CTX_UNLOCK(sc); 2617 status = ice_check_reset(hw); 2618 IFLIB_CTX_LOCK(sc); 2619 if (status) { 2620 device_printf(dev, "Device never came out of reset, err %s\n", 2621 ice_status_str(status)); 2622 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2623 return; 2624 } 2625 2626 /* We're done with the reset, so we can rebuild driver state */ 2627 sc->hw.reset_ongoing = false; 2628 ice_rebuild(sc); 2629 2630 /* In the unlikely event that a PF reset request occurs at the same 2631 * time as a global reset, clear the request now. This avoids 2632 * resetting a second time right after we reset due to a global event. 2633 */ 2634 if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2635 device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n"); 2636 } 2637 2638 /** 2639 * ice_handle_pf_reset_request - Initiate PF reset requested by software 2640 * @sc: The device private softc 2641 * 2642 * Initiate a PF reset requested by software. We handle this in the admin task 2643 * so that only one thread actually handles driver preparation and cleanup, 2644 * rather than having multiple threads possibly attempt to run this code 2645 * simultaneously. 2646 * 2647 * @pre assumes the iflib context lock is held and will unlock it while 2648 * waiting for the PF reset to complete. 2649 */ 2650 static void 2651 ice_handle_pf_reset_request(struct ice_softc *sc) 2652 { 2653 struct ice_hw *hw = &sc->hw; 2654 enum ice_status status; 2655 2656 /* Check for PF reset requests */ 2657 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2658 return; 2659 2660 /* Make sure we're prepared for reset */ 2661 ice_prepare_for_reset(sc); 2662 2663 /* 2664 * Release the iflib context lock and wait for the device to finish 2665 * resetting. 2666 */ 2667 IFLIB_CTX_UNLOCK(sc); 2668 status = ice_reset(hw, ICE_RESET_PFR); 2669 IFLIB_CTX_LOCK(sc); 2670 if (status) { 2671 device_printf(sc->dev, "device PF reset failed, err %s\n", 2672 ice_status_str(status)); 2673 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2674 return; 2675 } 2676 2677 sc->soft_stats.pfr_count++; 2678 ice_rebuild(sc); 2679 } 2680 2681 /** 2682 * ice_init_device_features - Init device driver features 2683 * @sc: driver softc structure 2684 * 2685 * @pre assumes that the function capabilities bits have been set up by 2686 * ice_init_hw(). 2687 */ 2688 static void 2689 ice_init_device_features(struct ice_softc *sc) 2690 { 2691 /* 2692 * A failed pkg file download triggers safe mode, disabling advanced 2693 * device feature support 2694 */ 2695 if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) 2696 return; 2697 2698 /* Set capabilities that all devices support */ 2699 ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2700 ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap); 2701 ice_set_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2702 ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap); 2703 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap); 2704 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap); 2705 ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2706 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 2707 ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap); 2708 2709 /* Disable features due to hardware limitations... */ 2710 if (!sc->hw.func_caps.common_cap.rss_table_size) 2711 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2712 if (!sc->hw.func_caps.common_cap.iwarp || !ice_enable_irdma) 2713 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2714 /* Disable features due to firmware limitations... */ 2715 if (!ice_is_fw_health_report_supported(&sc->hw)) 2716 ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2717 if (!ice_fwlog_supported(&sc->hw)) 2718 ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 2719 if (sc->hw.fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 2720 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING)) 2721 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en); 2722 else 2723 ice_fwlog_unregister(&sc->hw); 2724 } 2725 2726 /* Disable capabilities not supported by the OS */ 2727 ice_disable_unsupported_features(sc->feat_cap); 2728 2729 /* RSS is always enabled for iflib */ 2730 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS)) 2731 ice_set_bit(ICE_FEATURE_RSS, sc->feat_en); 2732 } 2733 2734 /** 2735 * ice_if_multi_set - Callback to update Multicast filters in HW 2736 * @ctx: iflib ctx structure 2737 * 2738 * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search 2739 * the if_multiaddrs list and determine which filters have been added or 2740 * removed from the list, and update HW programming to reflect the new list. 2741 * 2742 * @pre assumes the caller holds the iflib CTX lock 2743 */ 2744 static void 2745 ice_if_multi_set(if_ctx_t ctx) 2746 { 2747 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2748 int err; 2749 2750 ASSERT_CTX_LOCKED(sc); 2751 2752 /* Do not handle multicast configuration in recovery mode */ 2753 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2754 return; 2755 2756 err = ice_sync_multicast_filters(sc); 2757 if (err) { 2758 device_printf(sc->dev, 2759 "Failed to synchronize multicast filter list: %s\n", 2760 ice_err_str(err)); 2761 return; 2762 } 2763 } 2764 2765 /** 2766 * ice_if_vlan_register - Register a VLAN with the hardware 2767 * @ctx: iflib ctx pointer 2768 * @vtag: VLAN to add 2769 * 2770 * Programs the main PF VSI with a hardware filter for the given VLAN. 2771 * 2772 * @pre assumes the caller holds the iflib CTX lock 2773 */ 2774 static void 2775 ice_if_vlan_register(if_ctx_t ctx, u16 vtag) 2776 { 2777 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2778 enum ice_status status; 2779 2780 ASSERT_CTX_LOCKED(sc); 2781 2782 /* Do not handle VLAN configuration in recovery mode */ 2783 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2784 return; 2785 2786 status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag); 2787 if (status) { 2788 device_printf(sc->dev, 2789 "Failure adding VLAN %d to main VSI, err %s aq_err %s\n", 2790 vtag, ice_status_str(status), 2791 ice_aq_str(sc->hw.adminq.sq_last_status)); 2792 } 2793 } 2794 2795 /** 2796 * ice_if_vlan_unregister - Remove a VLAN filter from the hardware 2797 * @ctx: iflib ctx pointer 2798 * @vtag: VLAN to add 2799 * 2800 * Removes the previously programmed VLAN filter from the main PF VSI. 2801 * 2802 * @pre assumes the caller holds the iflib CTX lock 2803 */ 2804 static void 2805 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 2806 { 2807 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2808 enum ice_status status; 2809 2810 ASSERT_CTX_LOCKED(sc); 2811 2812 /* Do not handle VLAN configuration in recovery mode */ 2813 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2814 return; 2815 2816 status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag); 2817 if (status) { 2818 device_printf(sc->dev, 2819 "Failure removing VLAN %d from main VSI, err %s aq_err %s\n", 2820 vtag, ice_status_str(status), 2821 ice_aq_str(sc->hw.adminq.sq_last_status)); 2822 } 2823 } 2824 2825 /** 2826 * ice_if_stop - Stop the device 2827 * @ctx: iflib context structure 2828 * 2829 * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0 2830 * down) 2831 * 2832 * @pre assumes the caller holds the iflib CTX lock 2833 */ 2834 static void 2835 ice_if_stop(if_ctx_t ctx) 2836 { 2837 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2838 2839 ASSERT_CTX_LOCKED(sc); 2840 2841 /* 2842 * The iflib core may call IFDI_STOP prior to the first call to 2843 * IFDI_INIT. This will cause us to attempt to remove MAC filters we 2844 * don't have, and disable Tx queues which aren't yet configured. 2845 * Although it is likely these extra operations are harmless, they do 2846 * cause spurious warning messages to be displayed, which may confuse 2847 * users. 2848 * 2849 * To avoid these messages, we use a state bit indicating if we've 2850 * been initialized. It will be set when ice_if_init is called, and 2851 * cleared here in ice_if_stop. 2852 */ 2853 if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED)) 2854 return; 2855 2856 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 2857 device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n"); 2858 return; 2859 } 2860 2861 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 2862 device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n"); 2863 return; 2864 } 2865 2866 ice_rdma_pf_stop(sc); 2867 2868 /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the 2869 * return of these functions because there's nothing we can really do 2870 * if they fail, and the functions already print error messages. 2871 * Just try to shut down as much as we can. 2872 */ 2873 ice_rm_pf_default_mac_filters(sc); 2874 2875 /* Dissociate the Tx and Rx queues from the interrupts */ 2876 ice_flush_txq_interrupts(&sc->pf_vsi); 2877 ice_flush_rxq_interrupts(&sc->pf_vsi); 2878 2879 /* Disable the Tx and Rx queues */ 2880 ice_vsi_disable_tx(&sc->pf_vsi); 2881 ice_control_rx_queues(&sc->pf_vsi, false); 2882 } 2883 2884 /** 2885 * ice_if_get_counter - Get current value of an ifnet statistic 2886 * @ctx: iflib context pointer 2887 * @counter: ifnet counter to read 2888 * 2889 * Reads the current value of an ifnet counter for the device. 2890 * 2891 * This function is not protected by the iflib CTX lock. 2892 */ 2893 static uint64_t 2894 ice_if_get_counter(if_ctx_t ctx, ift_counter counter) 2895 { 2896 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2897 2898 /* Return the counter for the main PF VSI */ 2899 return ice_get_ifnet_counter(&sc->pf_vsi, counter); 2900 } 2901 2902 /** 2903 * ice_request_stack_reinit - Request that iflib re-initialize 2904 * @sc: the device private softc 2905 * 2906 * Request that the device be brought down and up, to re-initialize. For 2907 * example, this may be called when a device reset occurs, or when Tx and Rx 2908 * queues need to be re-initialized. 2909 * 2910 * This is required because the iflib state is outside the driver, and must be 2911 * re-initialized if we need to resart Tx and Rx queues. 2912 */ 2913 void 2914 ice_request_stack_reinit(struct ice_softc *sc) 2915 { 2916 if (CTX_ACTIVE(sc->ctx)) { 2917 iflib_request_reset(sc->ctx); 2918 iflib_admin_intr_deferred(sc->ctx); 2919 } 2920 } 2921 2922 /** 2923 * ice_driver_is_detaching - Check if the driver is detaching/unloading 2924 * @sc: device private softc 2925 * 2926 * Returns true if the driver is detaching, false otherwise. 2927 * 2928 * @remark on newer kernels, take advantage of iflib_in_detach in order to 2929 * report detachment correctly as early as possible. 2930 * 2931 * @remark this function is used by various code paths that want to avoid 2932 * running if the driver is about to be removed. This includes sysctls and 2933 * other driver access points. Note that it does not fully resolve 2934 * detach-based race conditions as it is possible for a thread to race with 2935 * iflib_in_detach. 2936 */ 2937 bool 2938 ice_driver_is_detaching(struct ice_softc *sc) 2939 { 2940 return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || 2941 iflib_in_detach(sc->ctx)); 2942 } 2943 2944 /** 2945 * ice_if_priv_ioctl - Device private ioctl handler 2946 * @ctx: iflib context pointer 2947 * @command: The ioctl command issued 2948 * @data: ioctl specific data 2949 * 2950 * iflib callback for handling custom driver specific ioctls. 2951 * 2952 * @pre Assumes that the iflib context lock is held. 2953 */ 2954 static int 2955 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) 2956 { 2957 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2958 struct ifdrv *ifd; 2959 device_t dev = sc->dev; 2960 2961 if (data == NULL) 2962 return (EINVAL); 2963 2964 ASSERT_CTX_LOCKED(sc); 2965 2966 /* Make sure the command type is valid */ 2967 switch (command) { 2968 case SIOCSDRVSPEC: 2969 case SIOCGDRVSPEC: 2970 /* Accepted commands */ 2971 break; 2972 case SIOCGPRIVATE_0: 2973 /* 2974 * Although we do not support this ioctl command, it's 2975 * expected that iflib will forward it to the IFDI_PRIV_IOCTL 2976 * handler. Do not print a message in this case 2977 */ 2978 return (ENOTSUP); 2979 default: 2980 /* 2981 * If we get a different command for this function, it's 2982 * definitely unexpected, so log a message indicating what 2983 * command we got for debugging purposes. 2984 */ 2985 device_printf(dev, "%s: unexpected ioctl command %08lx\n", 2986 __func__, command); 2987 return (EINVAL); 2988 } 2989 2990 ifd = (struct ifdrv *)data; 2991 2992 switch (ifd->ifd_cmd) { 2993 case ICE_NVM_ACCESS: 2994 return ice_handle_nvm_access_ioctl(sc, ifd); 2995 default: 2996 return EINVAL; 2997 } 2998 } 2999 3000 /** 3001 * ice_if_i2c_req - I2C request handler for iflib 3002 * @ctx: iflib context pointer 3003 * @req: The I2C parameters to use 3004 * 3005 * Read from the port's I2C eeprom using the parameters from the ioctl. 3006 * 3007 * @remark The iflib-only part is pretty simple. 3008 */ 3009 static int 3010 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req) 3011 { 3012 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3013 3014 return ice_handle_i2c_req(sc, req); 3015 } 3016 3017 /** 3018 * ice_if_suspend - PCI device suspend handler for iflib 3019 * @ctx: iflib context pointer 3020 * 3021 * Deinitializes the driver and clears HW resources in preparation for 3022 * suspend or an FLR. 3023 * 3024 * @returns 0; this return value is ignored 3025 */ 3026 static int 3027 ice_if_suspend(if_ctx_t ctx) 3028 { 3029 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3030 3031 /* At least a PFR is always going to happen after this; 3032 * either via FLR or during the D3->D0 transition. 3033 */ 3034 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 3035 3036 ice_prepare_for_reset(sc); 3037 3038 return (0); 3039 } 3040 3041 /** 3042 * ice_if_resume - PCI device resume handler for iflib 3043 * @ctx: iflib context pointer 3044 * 3045 * Reinitializes the driver and the HW after PCI resume or after 3046 * an FLR. An init is performed by iflib after this function is finished. 3047 * 3048 * @returns 0; this return value is ignored 3049 */ 3050 static int 3051 ice_if_resume(if_ctx_t ctx) 3052 { 3053 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3054 3055 ice_rebuild(sc); 3056 3057 return (0); 3058 } 3059 3060