1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2023, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /** 33 * @file if_ice_iflib.c 34 * @brief iflib driver implementation 35 * 36 * Contains the main entry point for the iflib driver implementation. It 37 * implements the various ifdi driver methods, and sets up the module and 38 * driver values to load an iflib driver. 39 */ 40 41 #include "ice_iflib.h" 42 #include "ice_drv_info.h" 43 #include "ice_switch.h" 44 #include "ice_sched.h" 45 46 #include <sys/module.h> 47 #include <sys/sockio.h> 48 #include <sys/smp.h> 49 #include <dev/pci/pcivar.h> 50 #include <dev/pci/pcireg.h> 51 52 /* 53 * Device method prototypes 54 */ 55 56 static void *ice_register(device_t); 57 static int ice_if_attach_pre(if_ctx_t); 58 static int ice_attach_pre_recovery_mode(struct ice_softc *sc); 59 static int ice_if_attach_post(if_ctx_t); 60 static void ice_attach_post_recovery_mode(struct ice_softc *sc); 61 static int ice_if_detach(if_ctx_t); 62 static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); 63 static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); 64 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix); 65 static void ice_if_queues_free(if_ctx_t ctx); 66 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 67 static void ice_if_intr_enable(if_ctx_t ctx); 68 static void ice_if_intr_disable(if_ctx_t ctx); 69 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 70 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 71 static int ice_if_promisc_set(if_ctx_t ctx, int flags); 72 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 73 static int ice_if_media_change(if_ctx_t ctx); 74 static void ice_if_init(if_ctx_t ctx); 75 static void ice_if_timer(if_ctx_t ctx, uint16_t qid); 76 static void ice_if_update_admin_status(if_ctx_t ctx); 77 static void ice_if_multi_set(if_ctx_t ctx); 78 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag); 79 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag); 80 static void ice_if_stop(if_ctx_t ctx); 81 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter); 82 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); 83 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req); 84 static int ice_if_suspend(if_ctx_t ctx); 85 static int ice_if_resume(if_ctx_t ctx); 86 87 static int ice_msix_que(void *arg); 88 static int ice_msix_admin(void *arg); 89 90 /* 91 * Helper function prototypes 92 */ 93 static int ice_pci_mapping(struct ice_softc *sc); 94 static void ice_free_pci_mapping(struct ice_softc *sc); 95 static void ice_update_link_status(struct ice_softc *sc, bool update_media); 96 static void ice_init_device_features(struct ice_softc *sc); 97 static void ice_init_tx_tracking(struct ice_vsi *vsi); 98 static void ice_handle_reset_event(struct ice_softc *sc); 99 static void ice_handle_pf_reset_request(struct ice_softc *sc); 100 static void ice_prepare_for_reset(struct ice_softc *sc); 101 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc); 102 static void ice_rebuild(struct ice_softc *sc); 103 static void ice_rebuild_recovery_mode(struct ice_softc *sc); 104 static void ice_free_irqvs(struct ice_softc *sc); 105 static void ice_update_rx_mbuf_sz(struct ice_softc *sc); 106 static void ice_poll_for_media_avail(struct ice_softc *sc); 107 static void ice_setup_scctx(struct ice_softc *sc); 108 static int ice_allocate_msix(struct ice_softc *sc); 109 static void ice_admin_timer(void *arg); 110 static void ice_transition_recovery_mode(struct ice_softc *sc); 111 static void ice_transition_safe_mode(struct ice_softc *sc); 112 113 /* 114 * Device Interface Declaration 115 */ 116 117 /** 118 * @var ice_methods 119 * @brief ice driver method entry points 120 * 121 * List of device methods implementing the generic device interface used by 122 * the device stack to interact with the ice driver. Since this is an iflib 123 * driver, most of the methods point to the generic iflib implementation. 124 */ 125 static device_method_t ice_methods[] = { 126 /* Device interface */ 127 DEVMETHOD(device_register, ice_register), 128 DEVMETHOD(device_probe, iflib_device_probe_vendor), 129 DEVMETHOD(device_attach, iflib_device_attach), 130 DEVMETHOD(device_detach, iflib_device_detach), 131 DEVMETHOD(device_shutdown, iflib_device_shutdown), 132 DEVMETHOD(device_suspend, iflib_device_suspend), 133 DEVMETHOD(device_resume, iflib_device_resume), 134 DEVMETHOD_END 135 }; 136 137 /** 138 * @var ice_iflib_methods 139 * @brief iflib method entry points 140 * 141 * List of device methods used by the iflib stack to interact with this 142 * driver. These are the real main entry points used to interact with this 143 * driver. 144 */ 145 static device_method_t ice_iflib_methods[] = { 146 DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre), 147 DEVMETHOD(ifdi_attach_post, ice_if_attach_post), 148 DEVMETHOD(ifdi_detach, ice_if_detach), 149 DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc), 150 DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc), 151 DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign), 152 DEVMETHOD(ifdi_queues_free, ice_if_queues_free), 153 DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set), 154 DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable), 155 DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable), 156 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable), 157 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable), 158 DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set), 159 DEVMETHOD(ifdi_media_status, ice_if_media_status), 160 DEVMETHOD(ifdi_media_change, ice_if_media_change), 161 DEVMETHOD(ifdi_init, ice_if_init), 162 DEVMETHOD(ifdi_stop, ice_if_stop), 163 DEVMETHOD(ifdi_timer, ice_if_timer), 164 DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status), 165 DEVMETHOD(ifdi_multi_set, ice_if_multi_set), 166 DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register), 167 DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister), 168 DEVMETHOD(ifdi_get_counter, ice_if_get_counter), 169 DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl), 170 DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req), 171 DEVMETHOD(ifdi_suspend, ice_if_suspend), 172 DEVMETHOD(ifdi_resume, ice_if_resume), 173 DEVMETHOD_END 174 }; 175 176 /** 177 * @var ice_driver 178 * @brief driver structure for the generic device stack 179 * 180 * driver_t definition used to setup the generic device methods. 181 */ 182 static driver_t ice_driver = { 183 .name = "ice", 184 .methods = ice_methods, 185 .size = sizeof(struct ice_softc), 186 }; 187 188 /** 189 * @var ice_iflib_driver 190 * @brief driver structure for the iflib stack 191 * 192 * driver_t definition used to setup the iflib device methods. 193 */ 194 static driver_t ice_iflib_driver = { 195 .name = "ice", 196 .methods = ice_iflib_methods, 197 .size = sizeof(struct ice_softc), 198 }; 199 200 extern struct if_txrx ice_txrx; 201 extern struct if_txrx ice_recovery_txrx; 202 203 /** 204 * @var ice_sctx 205 * @brief ice driver shared context 206 * 207 * Structure defining shared values (context) that is used by all instances of 208 * the device. Primarily used to setup details about how the iflib stack 209 * should treat this driver. Also defines the default, minimum, and maximum 210 * number of descriptors in each ring. 211 */ 212 static struct if_shared_ctx ice_sctx = { 213 .isc_magic = IFLIB_MAGIC, 214 .isc_q_align = PAGE_SIZE, 215 216 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 217 /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but 218 * that doesn't make sense since that would be larger than the maximum 219 * size of a single packet. 220 */ 221 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 222 223 /* XXX: This is only used by iflib to ensure that 224 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size. 225 */ 226 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 227 /* XXX: This is used by iflib to set the number of segments in the TSO 228 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the 229 * related ifnet parameter. 230 */ 231 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 232 233 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 234 .isc_rx_nsegments = ICE_MAX_RX_SEGS, 235 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 236 237 .isc_nfl = 1, 238 .isc_ntxqs = 1, 239 .isc_nrxqs = 1, 240 241 .isc_admin_intrcnt = 1, 242 .isc_vendor_info = ice_vendor_info_array, 243 .isc_driver_version = __DECONST(char *, ice_driver_version), 244 .isc_driver = &ice_iflib_driver, 245 246 /* 247 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available 248 * for hardware checksum offload 249 * 250 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the 251 * IP sum field, required by our hardware to calculate valid TSO 252 * checksums. 253 * 254 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs 255 * even when the interface is down. 256 * 257 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X 258 * vectors manually instead of relying on iflib code to do this. 259 */ 260 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 261 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX, 262 263 .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 264 .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 265 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 266 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 267 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 268 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 269 }; 270 271 DRIVER_MODULE(ice, pci, ice_driver, ice_module_event_handler, NULL); 272 273 MODULE_VERSION(ice, 1); 274 MODULE_DEPEND(ice, pci, 1, 1, 1); 275 MODULE_DEPEND(ice, ether, 1, 1, 1); 276 MODULE_DEPEND(ice, iflib, 1, 1, 1); 277 278 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array); 279 280 /* Static driver-wide sysctls */ 281 #include "ice_iflib_sysctls.h" 282 283 /** 284 * ice_pci_mapping - Map PCI BAR memory 285 * @sc: device private softc 286 * 287 * Map PCI BAR 0 for device operation. 288 */ 289 static int 290 ice_pci_mapping(struct ice_softc *sc) 291 { 292 int rc; 293 294 /* Map BAR0 */ 295 rc = ice_map_bar(sc->dev, &sc->bar0, 0); 296 if (rc) 297 return rc; 298 299 return 0; 300 } 301 302 /** 303 * ice_free_pci_mapping - Release PCI BAR memory 304 * @sc: device private softc 305 * 306 * Release PCI BARs which were previously mapped by ice_pci_mapping(). 307 */ 308 static void 309 ice_free_pci_mapping(struct ice_softc *sc) 310 { 311 /* Free BAR0 */ 312 ice_free_bar(sc->dev, &sc->bar0); 313 } 314 315 /* 316 * Device methods 317 */ 318 319 /** 320 * ice_register - register device method callback 321 * @dev: the device being registered 322 * 323 * Returns a pointer to the shared context structure, which is used by iflib. 324 */ 325 static void * 326 ice_register(device_t dev __unused) 327 { 328 return &ice_sctx; 329 } /* ice_register */ 330 331 /** 332 * ice_setup_scctx - Setup the iflib softc context structure 333 * @sc: the device private structure 334 * 335 * Setup the parameters in if_softc_ctx_t structure used by the iflib stack 336 * when loading. 337 */ 338 static void 339 ice_setup_scctx(struct ice_softc *sc) 340 { 341 if_softc_ctx_t scctx = sc->scctx; 342 struct ice_hw *hw = &sc->hw; 343 bool safe_mode, recovery_mode; 344 345 safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE); 346 recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE); 347 348 /* 349 * If the driver loads in Safe mode or Recovery mode, limit iflib to 350 * a single queue pair. 351 */ 352 if (safe_mode || recovery_mode) { 353 scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1; 354 scctx->isc_ntxqsets_max = 1; 355 scctx->isc_nrxqsets_max = 1; 356 } else { 357 /* 358 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to 359 * the values of the override sysctls. Cache these initial 360 * values so that the driver can be aware of what the iflib 361 * sysctl value is when setting up MSI-X vectors. 362 */ 363 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets; 364 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets; 365 366 if (scctx->isc_ntxqsets == 0) 367 scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size; 368 if (scctx->isc_nrxqsets == 0) 369 scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size; 370 371 scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq; 372 scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq; 373 374 /* 375 * Sanity check that the iflib sysctl values are within the 376 * maximum supported range. 377 */ 378 if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max) 379 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max; 380 if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max) 381 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max; 382 } 383 384 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 385 * sizeof(struct ice_tx_desc), DBA_ALIGN); 386 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 387 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 388 389 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 390 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 391 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 392 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 393 394 scctx->isc_msix_bar = PCIR_BAR(ICE_MSIX_BAR); 395 scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size; 396 397 /* 398 * If the driver loads in recovery mode, disable Tx/Rx functionality 399 */ 400 if (recovery_mode) 401 scctx->isc_txrx = &ice_recovery_txrx; 402 else 403 scctx->isc_txrx = &ice_txrx; 404 405 /* 406 * If the driver loads in Safe mode or Recovery mode, disable 407 * advanced features including hardware offloads. 408 */ 409 if (safe_mode || recovery_mode) { 410 scctx->isc_capenable = ICE_SAFE_CAPS; 411 scctx->isc_tx_csum_flags = 0; 412 } else { 413 scctx->isc_capenable = ICE_FULL_CAPS; 414 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 415 } 416 417 scctx->isc_capabilities = scctx->isc_capenable; 418 } /* ice_setup_scctx */ 419 420 /** 421 * ice_if_attach_pre - Early device attach logic 422 * @ctx: the iflib context structure 423 * 424 * Called by iflib during the attach process. Earliest main driver entry 425 * point which performs necessary hardware and driver initialization. Called 426 * before the Tx and Rx queues are allocated. 427 */ 428 static int 429 ice_if_attach_pre(if_ctx_t ctx) 430 { 431 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 432 enum ice_fw_modes fw_mode; 433 enum ice_status status; 434 if_softc_ctx_t scctx; 435 struct ice_hw *hw; 436 device_t dev; 437 int err; 438 439 device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n"); 440 441 ice_set_state(&sc->state, ICE_STATE_ATTACHING); 442 443 sc->ctx = ctx; 444 sc->media = iflib_get_media(ctx); 445 sc->sctx = iflib_get_sctx(ctx); 446 sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx); 447 448 dev = sc->dev = iflib_get_dev(ctx); 449 scctx = sc->scctx = iflib_get_softc_ctx(ctx); 450 451 hw = &sc->hw; 452 hw->back = sc; 453 454 snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name), 455 "%s:admin", device_get_nameunit(dev)); 456 mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF); 457 callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0); 458 459 ASSERT_CTX_LOCKED(sc); 460 461 if (ice_pci_mapping(sc)) { 462 err = (ENXIO); 463 goto destroy_admin_timer; 464 } 465 466 /* Save off the PCI information */ 467 ice_save_pci_info(hw, dev); 468 469 /* create tunables as early as possible */ 470 ice_add_device_tunables(sc); 471 472 /* Setup ControlQ lengths */ 473 ice_set_ctrlq_len(hw); 474 475 reinit_hw: 476 477 fw_mode = ice_get_fw_mode(hw); 478 if (fw_mode == ICE_FW_MODE_REC) { 479 device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 480 481 err = ice_attach_pre_recovery_mode(sc); 482 if (err) 483 goto free_pci_mapping; 484 485 return (0); 486 } 487 488 /* Initialize the hw data structure */ 489 status = ice_init_hw(hw); 490 if (status) { 491 if (status == ICE_ERR_FW_API_VER) { 492 /* Enter recovery mode, so that the driver remains 493 * loaded. This way, if the system administrator 494 * cannot update the driver, they may still attempt to 495 * downgrade the NVM. 496 */ 497 err = ice_attach_pre_recovery_mode(sc); 498 if (err) 499 goto free_pci_mapping; 500 501 return (0); 502 } else { 503 err = EIO; 504 device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n", 505 ice_status_str(status), 506 ice_aq_str(hw->adminq.sq_last_status)); 507 } 508 goto free_pci_mapping; 509 } 510 511 ice_init_device_features(sc); 512 513 /* Notify firmware of the device driver version */ 514 err = ice_send_version(sc); 515 if (err) 516 goto deinit_hw; 517 518 /* 519 * Success indicates a change was made that requires a reinitialization 520 * of the hardware 521 */ 522 err = ice_load_pkg_file(sc); 523 if (err == ICE_SUCCESS) { 524 ice_deinit_hw(hw); 525 goto reinit_hw; 526 } 527 528 err = ice_init_link_events(sc); 529 if (err) { 530 device_printf(dev, "ice_init_link_events failed: %s\n", 531 ice_err_str(err)); 532 goto deinit_hw; 533 } 534 535 /* Initialize VLAN mode in FW; if dual VLAN mode is supported by the package 536 * and firmware, this will force them to use single VLAN mode. 537 */ 538 status = ice_set_vlan_mode(hw); 539 if (status) { 540 err = EIO; 541 device_printf(dev, "Unable to initialize VLAN mode, err %s aq_err %s\n", 542 ice_status_str(status), 543 ice_aq_str(hw->adminq.sq_last_status)); 544 goto deinit_hw; 545 } 546 547 ice_print_nvm_version(sc); 548 549 /* Setup the MAC address */ 550 iflib_set_mac(ctx, hw->port_info->mac.lan_addr); 551 552 /* Setup the iflib softc context structure */ 553 ice_setup_scctx(sc); 554 555 /* Initialize the Tx queue manager */ 556 err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq); 557 if (err) { 558 device_printf(dev, "Unable to initialize Tx queue manager: %s\n", 559 ice_err_str(err)); 560 goto deinit_hw; 561 } 562 563 /* Initialize the Rx queue manager */ 564 err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq); 565 if (err) { 566 device_printf(dev, "Unable to initialize Rx queue manager: %s\n", 567 ice_err_str(err)); 568 goto free_tx_qmgr; 569 } 570 571 /* Initialize the interrupt resource manager */ 572 err = ice_alloc_intr_tracking(sc); 573 if (err) 574 /* Errors are already printed */ 575 goto free_rx_qmgr; 576 577 /* Determine maximum number of VSIs we'll prepare for */ 578 sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE, 579 hw->func_caps.guar_num_vsi); 580 581 if (!sc->num_available_vsi) { 582 err = EIO; 583 device_printf(dev, "No VSIs allocated to host\n"); 584 goto free_intr_tracking; 585 } 586 587 /* Allocate storage for the VSI pointers */ 588 sc->all_vsi = (struct ice_vsi **) 589 malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi, 590 M_ICE, M_WAITOK | M_ZERO); 591 if (!sc->all_vsi) { 592 err = ENOMEM; 593 device_printf(dev, "Unable to allocate VSI array\n"); 594 goto free_intr_tracking; 595 } 596 597 /* 598 * Prepare the statically allocated primary PF VSI in the softc 599 * structure. Other VSIs will be dynamically allocated as needed. 600 */ 601 ice_setup_pf_vsi(sc); 602 603 err = ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max, 604 scctx->isc_nrxqsets_max); 605 if (err) { 606 device_printf(dev, "Unable to allocate VSI Queue maps\n"); 607 goto free_main_vsi; 608 } 609 610 /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */ 611 err = ice_allocate_msix(sc); 612 if (err) 613 goto free_main_vsi; 614 615 return 0; 616 617 free_main_vsi: 618 /* ice_release_vsi will free the queue maps if they were allocated */ 619 ice_release_vsi(&sc->pf_vsi); 620 free(sc->all_vsi, M_ICE); 621 sc->all_vsi = NULL; 622 free_intr_tracking: 623 ice_free_intr_tracking(sc); 624 free_rx_qmgr: 625 ice_resmgr_destroy(&sc->rx_qmgr); 626 free_tx_qmgr: 627 ice_resmgr_destroy(&sc->tx_qmgr); 628 deinit_hw: 629 ice_deinit_hw(hw); 630 free_pci_mapping: 631 ice_free_pci_mapping(sc); 632 destroy_admin_timer: 633 mtx_lock(&sc->admin_mtx); 634 callout_stop(&sc->admin_timer); 635 mtx_unlock(&sc->admin_mtx); 636 mtx_destroy(&sc->admin_mtx); 637 return err; 638 } /* ice_if_attach_pre */ 639 640 /** 641 * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery 642 * @sc: the device private softc 643 * 644 * Loads the device driver in limited Firmware Recovery mode, intended to 645 * allow users to update the firmware to attempt to recover the device. 646 * 647 * @remark We may enter recovery mode in case either (a) the firmware is 648 * detected to be in an invalid state and must be re-programmed, or (b) the 649 * driver detects that the loaded firmware has a non-compatible API version 650 * that the driver cannot operate with. 651 */ 652 static int 653 ice_attach_pre_recovery_mode(struct ice_softc *sc) 654 { 655 ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE); 656 657 /* Setup the iflib softc context */ 658 ice_setup_scctx(sc); 659 660 /* Setup the PF VSI back pointer */ 661 sc->pf_vsi.sc = sc; 662 663 /* 664 * We still need to allocate MSI-X vectors since we need one vector to 665 * run the administrative admin interrupt 666 */ 667 return ice_allocate_msix(sc); 668 } 669 670 /** 671 * ice_update_link_status - notify OS of link state change 672 * @sc: device private softc structure 673 * @update_media: true if we should update media even if link didn't change 674 * 675 * Called to notify iflib core of link status changes. Should be called once 676 * during attach_post, and whenever link status changes during runtime. 677 * 678 * This call only updates the currently supported media types if the link 679 * status changed, or if update_media is set to true. 680 */ 681 static void 682 ice_update_link_status(struct ice_softc *sc, bool update_media) 683 { 684 struct ice_hw *hw = &sc->hw; 685 enum ice_status status; 686 687 /* Never report link up when in recovery mode */ 688 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 689 return; 690 691 /* Report link status to iflib only once each time it changes */ 692 if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) { 693 if (sc->link_up) { /* link is up */ 694 uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info); 695 696 ice_set_default_local_lldp_mib(sc); 697 698 iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate); 699 ice_rdma_link_change(sc, LINK_STATE_UP, baudrate); 700 701 ice_link_up_msg(sc); 702 } else { /* link is down */ 703 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 704 ice_rdma_link_change(sc, LINK_STATE_DOWN, 0); 705 } 706 update_media = true; 707 } 708 709 /* Update the supported media types */ 710 if (update_media) { 711 status = ice_add_media_types(sc, sc->media); 712 if (status) 713 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 714 ice_status_str(status), 715 ice_aq_str(hw->adminq.sq_last_status)); 716 } 717 } 718 719 /** 720 * ice_if_attach_post - Late device attach logic 721 * @ctx: the iflib context structure 722 * 723 * Called by iflib to finish up attaching the device. Performs any attach 724 * logic which must wait until after the Tx and Rx queues have been 725 * allocated. 726 */ 727 static int 728 ice_if_attach_post(if_ctx_t ctx) 729 { 730 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 731 if_t ifp = iflib_get_ifp(ctx); 732 int err; 733 734 ASSERT_CTX_LOCKED(sc); 735 736 /* We don't yet support loading if MSI-X is not supported */ 737 if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) { 738 device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n"); 739 return (ENOTSUP); 740 } 741 742 /* The ifnet structure hasn't yet been initialized when the attach_pre 743 * handler is called, so wait until attach_post to setup the 744 * isc_max_frame_size. 745 */ 746 747 sc->ifp = ifp; 748 sc->scctx->isc_max_frame_size = if_getmtu(ifp) + 749 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 750 751 /* 752 * If we are in recovery mode, only perform a limited subset of 753 * initialization to support NVM recovery. 754 */ 755 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 756 ice_attach_post_recovery_mode(sc); 757 return (0); 758 } 759 760 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 761 762 err = ice_initialize_vsi(&sc->pf_vsi); 763 if (err) { 764 device_printf(sc->dev, "Unable to initialize Main VSI: %s\n", 765 ice_err_str(err)); 766 return err; 767 } 768 769 /* Enable FW health event reporting */ 770 ice_init_health_events(sc); 771 772 /* Configure the main PF VSI for RSS */ 773 err = ice_config_rss(&sc->pf_vsi); 774 if (err) { 775 device_printf(sc->dev, 776 "Unable to configure RSS for the main VSI, err %s\n", 777 ice_err_str(err)); 778 return err; 779 } 780 781 /* Configure switch to drop transmitted LLDP and PAUSE frames */ 782 err = ice_cfg_pf_ethertype_filters(sc); 783 if (err) 784 return err; 785 786 ice_get_and_print_bus_info(sc); 787 788 ice_set_link_management_mode(sc); 789 790 ice_init_saved_phy_cfg(sc); 791 792 ice_cfg_pba_num(sc); 793 794 ice_add_device_sysctls(sc); 795 796 /* Get DCBX/LLDP state and start DCBX agent */ 797 ice_init_dcb_setup(sc); 798 799 /* Setup link configuration parameters */ 800 ice_init_link_configuration(sc); 801 ice_update_link_status(sc, true); 802 803 /* Configure interrupt causes for the administrative interrupt */ 804 ice_configure_misc_interrupts(sc); 805 806 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 807 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 808 809 err = ice_rdma_pf_attach(sc); 810 if (err) 811 return (err); 812 813 /* Start the admin timer */ 814 mtx_lock(&sc->admin_mtx); 815 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 816 mtx_unlock(&sc->admin_mtx); 817 818 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 819 820 return 0; 821 } /* ice_if_attach_post */ 822 823 /** 824 * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery 825 * @sc: the device private softc 826 * 827 * Performs minimal work to prepare the driver to recover an NVM in case the 828 * firmware is in recovery mode. 829 */ 830 static void 831 ice_attach_post_recovery_mode(struct ice_softc *sc) 832 { 833 /* Configure interrupt causes for the administrative interrupt */ 834 ice_configure_misc_interrupts(sc); 835 836 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 837 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 838 839 /* Start the admin timer */ 840 mtx_lock(&sc->admin_mtx); 841 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 842 mtx_unlock(&sc->admin_mtx); 843 844 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 845 } 846 847 /** 848 * ice_free_irqvs - Free IRQ vector memory 849 * @sc: the device private softc structure 850 * 851 * Free IRQ vector memory allocated during ice_if_msix_intr_assign. 852 */ 853 static void 854 ice_free_irqvs(struct ice_softc *sc) 855 { 856 struct ice_vsi *vsi = &sc->pf_vsi; 857 if_ctx_t ctx = sc->ctx; 858 int i; 859 860 /* If the irqvs array is NULL, then there are no vectors to free */ 861 if (sc->irqvs == NULL) 862 return; 863 864 /* Free the IRQ vectors */ 865 for (i = 0; i < sc->num_irq_vectors; i++) 866 iflib_irq_free(ctx, &sc->irqvs[i].irq); 867 868 /* Clear the irqv pointers */ 869 for (i = 0; i < vsi->num_rx_queues; i++) 870 vsi->rx_queues[i].irqv = NULL; 871 872 for (i = 0; i < vsi->num_tx_queues; i++) 873 vsi->tx_queues[i].irqv = NULL; 874 875 /* Release the vector array memory */ 876 free(sc->irqvs, M_ICE); 877 sc->irqvs = NULL; 878 sc->num_irq_vectors = 0; 879 } 880 881 /** 882 * ice_if_detach - Device driver detach logic 883 * @ctx: iflib context structure 884 * 885 * Perform device shutdown logic to detach the device driver. 886 * 887 * Note that there is no guarantee of the ordering of ice_if_queues_free() and 888 * ice_if_detach(). It is possible for the functions to be called in either 889 * order, and they must not assume to have a strict ordering. 890 */ 891 static int 892 ice_if_detach(if_ctx_t ctx) 893 { 894 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 895 struct ice_vsi *vsi = &sc->pf_vsi; 896 int i; 897 898 ASSERT_CTX_LOCKED(sc); 899 900 /* Indicate that we're detaching */ 901 ice_set_state(&sc->state, ICE_STATE_DETACHING); 902 903 /* Stop the admin timer */ 904 mtx_lock(&sc->admin_mtx); 905 callout_stop(&sc->admin_timer); 906 mtx_unlock(&sc->admin_mtx); 907 mtx_destroy(&sc->admin_mtx); 908 909 ice_rdma_pf_detach(sc); 910 911 /* Free allocated media types */ 912 ifmedia_removeall(sc->media); 913 914 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 915 * pointers. Note, the calls here and those in ice_if_queues_free() 916 * are *BOTH* necessary, as we cannot guarantee which path will be 917 * run first 918 */ 919 ice_vsi_del_txqs_ctx(vsi); 920 ice_vsi_del_rxqs_ctx(vsi); 921 922 /* Release MSI-X resources */ 923 ice_free_irqvs(sc); 924 925 for (i = 0; i < sc->num_available_vsi; i++) { 926 if (sc->all_vsi[i]) 927 ice_release_vsi(sc->all_vsi[i]); 928 } 929 930 if (sc->all_vsi) { 931 free(sc->all_vsi, M_ICE); 932 sc->all_vsi = NULL; 933 } 934 935 /* Release MSI-X memory */ 936 pci_release_msi(sc->dev); 937 938 if (sc->msix_table != NULL) { 939 bus_release_resource(sc->dev, SYS_RES_MEMORY, 940 rman_get_rid(sc->msix_table), 941 sc->msix_table); 942 sc->msix_table = NULL; 943 } 944 945 ice_free_intr_tracking(sc); 946 947 /* Destroy the queue managers */ 948 ice_resmgr_destroy(&sc->tx_qmgr); 949 ice_resmgr_destroy(&sc->rx_qmgr); 950 951 if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 952 ice_deinit_hw(&sc->hw); 953 954 ice_free_pci_mapping(sc); 955 956 return 0; 957 } /* ice_if_detach */ 958 959 /** 960 * ice_if_tx_queues_alloc - Allocate Tx queue memory 961 * @ctx: iflib context structure 962 * @vaddrs: virtual addresses for the queue memory 963 * @paddrs: physical addresses for the queue memory 964 * @ntxqs: the number of Tx queues per set (should always be 1) 965 * @ntxqsets: the number of Tx queue sets to allocate 966 * 967 * Called by iflib to allocate Tx queues for the device. Allocates driver 968 * memory to track each queue, the status arrays used for descriptor 969 * status reporting, and Tx queue sysctls. 970 */ 971 static int 972 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 973 int __invariant_only ntxqs, int ntxqsets) 974 { 975 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 976 struct ice_vsi *vsi = &sc->pf_vsi; 977 struct ice_tx_queue *txq; 978 int err, i, j; 979 980 MPASS(ntxqs == 1); 981 MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 982 ASSERT_CTX_LOCKED(sc); 983 984 /* Do not bother allocating queues if we're in recovery mode */ 985 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 986 return (0); 987 988 /* Allocate queue structure memory */ 989 if (!(vsi->tx_queues = 990 (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 991 device_printf(sc->dev, "Unable to allocate Tx queue memory\n"); 992 return (ENOMEM); 993 } 994 995 /* Allocate report status arrays */ 996 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 997 if (!(txq->tx_rsq = 998 (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_NOWAIT))) { 999 device_printf(sc->dev, "Unable to allocate tx_rsq memory\n"); 1000 err = ENOMEM; 1001 goto free_tx_queues; 1002 } 1003 /* Initialize report status array */ 1004 for (j = 0; j < sc->scctx->isc_ntxd[0]; j++) 1005 txq->tx_rsq[j] = QIDX_INVALID; 1006 } 1007 1008 /* Assign queues from PF space to the main VSI */ 1009 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets); 1010 if (err) { 1011 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1012 ice_err_str(err)); 1013 goto free_tx_queues; 1014 } 1015 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1016 1017 /* Add Tx queue sysctls context */ 1018 ice_vsi_add_txqs_ctx(vsi); 1019 1020 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1021 /* q_handle == me when only one TC */ 1022 txq->me = txq->q_handle = i; 1023 txq->vsi = vsi; 1024 1025 /* store the queue size for easier access */ 1026 txq->desc_count = sc->scctx->isc_ntxd[0]; 1027 1028 /* get the virtual and physical address of the hardware queues */ 1029 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 1030 txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 1031 txq->tx_paddr = paddrs[i]; 1032 1033 ice_add_txq_sysctls(txq); 1034 } 1035 1036 vsi->num_tx_queues = ntxqsets; 1037 1038 return (0); 1039 1040 free_tx_queues: 1041 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1042 if (txq->tx_rsq != NULL) { 1043 free(txq->tx_rsq, M_ICE); 1044 txq->tx_rsq = NULL; 1045 } 1046 } 1047 free(vsi->tx_queues, M_ICE); 1048 vsi->tx_queues = NULL; 1049 return err; 1050 } 1051 1052 /** 1053 * ice_if_rx_queues_alloc - Allocate Rx queue memory 1054 * @ctx: iflib context structure 1055 * @vaddrs: virtual addresses for the queue memory 1056 * @paddrs: physical addresses for the queue memory 1057 * @nrxqs: number of Rx queues per set (should always be 1) 1058 * @nrxqsets: number of Rx queue sets to allocate 1059 * 1060 * Called by iflib to allocate Rx queues for the device. Allocates driver 1061 * memory to track each queue, as well as sets up the Rx queue sysctls. 1062 */ 1063 static int 1064 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1065 int __invariant_only nrxqs, int nrxqsets) 1066 { 1067 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1068 struct ice_vsi *vsi = &sc->pf_vsi; 1069 struct ice_rx_queue *rxq; 1070 int err, i; 1071 1072 MPASS(nrxqs == 1); 1073 MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 1074 ASSERT_CTX_LOCKED(sc); 1075 1076 /* Do not bother allocating queues if we're in recovery mode */ 1077 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1078 return (0); 1079 1080 /* Allocate queue structure memory */ 1081 if (!(vsi->rx_queues = 1082 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 1083 device_printf(sc->dev, "Unable to allocate Rx queue memory\n"); 1084 return (ENOMEM); 1085 } 1086 1087 /* Assign queues from PF space to the main VSI */ 1088 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets); 1089 if (err) { 1090 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1091 ice_err_str(err)); 1092 goto free_rx_queues; 1093 } 1094 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1095 1096 /* Add Rx queue sysctls context */ 1097 ice_vsi_add_rxqs_ctx(vsi); 1098 1099 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 1100 rxq->me = i; 1101 rxq->vsi = vsi; 1102 1103 /* store the queue size for easier access */ 1104 rxq->desc_count = sc->scctx->isc_nrxd[0]; 1105 1106 /* get the virtual and physical address of the hardware queues */ 1107 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 1108 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 1109 rxq->rx_paddr = paddrs[i]; 1110 1111 ice_add_rxq_sysctls(rxq); 1112 } 1113 1114 vsi->num_rx_queues = nrxqsets; 1115 1116 return (0); 1117 1118 free_rx_queues: 1119 free(vsi->rx_queues, M_ICE); 1120 vsi->rx_queues = NULL; 1121 return err; 1122 } 1123 1124 /** 1125 * ice_if_queues_free - Free queue memory 1126 * @ctx: the iflib context structure 1127 * 1128 * Free queue memory allocated by ice_if_tx_queues_alloc() and 1129 * ice_if_rx_queues_alloc(). 1130 * 1131 * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be 1132 * called in the same order. It's possible for ice_if_queues_free() to be 1133 * called prior to ice_if_detach(), and vice versa. 1134 * 1135 * For this reason, the main VSI is a static member of the ice_softc, which is 1136 * not free'd until after iflib finishes calling both of these functions. 1137 * 1138 * Thus, care must be taken in how we manage the memory being freed by this 1139 * function, and in what tasks it can and must perform. 1140 */ 1141 static void 1142 ice_if_queues_free(if_ctx_t ctx) 1143 { 1144 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1145 struct ice_vsi *vsi = &sc->pf_vsi; 1146 struct ice_tx_queue *txq; 1147 int i; 1148 1149 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1150 * pointers. Note, the calls here and those in ice_if_detach() 1151 * are *BOTH* necessary, as we cannot guarantee which path will be 1152 * run first 1153 */ 1154 ice_vsi_del_txqs_ctx(vsi); 1155 ice_vsi_del_rxqs_ctx(vsi); 1156 1157 /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */ 1158 ice_free_irqvs(sc); 1159 1160 if (vsi->tx_queues != NULL) { 1161 /* free the tx_rsq arrays */ 1162 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1163 if (txq->tx_rsq != NULL) { 1164 free(txq->tx_rsq, M_ICE); 1165 txq->tx_rsq = NULL; 1166 } 1167 } 1168 free(vsi->tx_queues, M_ICE); 1169 vsi->tx_queues = NULL; 1170 vsi->num_tx_queues = 0; 1171 } 1172 if (vsi->rx_queues != NULL) { 1173 free(vsi->rx_queues, M_ICE); 1174 vsi->rx_queues = NULL; 1175 vsi->num_rx_queues = 0; 1176 } 1177 } 1178 1179 /** 1180 * ice_msix_que - Fast interrupt handler for MSI-X receive queues 1181 * @arg: The Rx queue memory 1182 * 1183 * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when 1184 * an MSI-X interrupt for a given queue is triggered. Currently this just asks 1185 * iflib to schedule the main Rx thread. 1186 */ 1187 static int 1188 ice_msix_que(void *arg) 1189 { 1190 struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg; 1191 1192 /* TODO: dynamic ITR algorithm?? */ 1193 1194 return (FILTER_SCHEDULE_THREAD); 1195 } 1196 1197 /** 1198 * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt 1199 * @arg: pointer to device softc memory 1200 * 1201 * Called by iflib when an administrative interrupt occurs. Should perform any 1202 * fast logic for handling the interrupt cause, and then indicate whether the 1203 * admin task needs to be queued. 1204 */ 1205 static int 1206 ice_msix_admin(void *arg) 1207 { 1208 struct ice_softc *sc = (struct ice_softc *)arg; 1209 struct ice_hw *hw = &sc->hw; 1210 device_t dev = sc->dev; 1211 u32 oicr; 1212 1213 /* There is no safe way to modify the enabled miscellaneous causes of 1214 * the OICR vector at runtime, as doing so would be prone to race 1215 * conditions. Reading PFINT_OICR will unmask the associated interrupt 1216 * causes and allow future interrupts to occur. The admin interrupt 1217 * vector will not be re-enabled until after we exit this function, 1218 * but any delayed tasks must be resilient against possible "late 1219 * arrival" interrupts that occur while we're already handling the 1220 * task. This is done by using state bits and serializing these 1221 * delayed tasks via the admin status task function. 1222 */ 1223 oicr = rd32(hw, PFINT_OICR); 1224 1225 /* Processing multiple controlq interrupts on a single vector does not 1226 * provide an indication of which controlq triggered the interrupt. 1227 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL 1228 * registers. However, the INTEVENT bit is not guaranteed to be set as 1229 * it gets automatically cleared when the hardware acknowledges the 1230 * interrupt. 1231 * 1232 * This means we don't really have a good indication of whether or 1233 * which controlq triggered this interrupt. We'll just notify the 1234 * admin task that it should check all the controlqs. 1235 */ 1236 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 1237 1238 if (oicr & PFINT_OICR_VFLR_M) { 1239 ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING); 1240 } 1241 1242 if (oicr & PFINT_OICR_MAL_DETECT_M) { 1243 ice_set_state(&sc->state, ICE_STATE_MDD_PENDING); 1244 } 1245 1246 if (oicr & PFINT_OICR_GRST_M) { 1247 u32 reset; 1248 1249 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1250 GLGEN_RSTAT_RESET_TYPE_S; 1251 1252 if (reset == ICE_RESET_CORER) 1253 sc->soft_stats.corer_count++; 1254 else if (reset == ICE_RESET_GLOBR) 1255 sc->soft_stats.globr_count++; 1256 else 1257 sc->soft_stats.empr_count++; 1258 1259 /* There are a couple of bits at play for handling resets. 1260 * First, the ICE_STATE_RESET_OICR_RECV bit is used to 1261 * indicate that the driver has received an OICR with a reset 1262 * bit active, indicating that a CORER/GLOBR/EMPR is about to 1263 * happen. Second, we set hw->reset_ongoing to indicate that 1264 * the hardware is in reset. We will set this back to false as 1265 * soon as the driver has determined that the hardware is out 1266 * of reset. 1267 * 1268 * If the driver wishes to trigger a request, it can set one of 1269 * the ICE_STATE_RESET_*_REQ bits, which will trigger the 1270 * correct type of reset. 1271 */ 1272 if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 1273 hw->reset_ongoing = true; 1274 } 1275 1276 if (oicr & PFINT_OICR_ECC_ERR_M) { 1277 device_printf(dev, "ECC Error detected!\n"); 1278 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1279 } 1280 1281 if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) { 1282 if (oicr & PFINT_OICR_HMC_ERR_M) 1283 /* Log the HMC errors */ 1284 ice_log_hmc_error(hw, dev); 1285 ice_rdma_notify_pe_intr(sc, oicr); 1286 } 1287 1288 if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { 1289 device_printf(dev, "PCI Exception detected!\n"); 1290 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1291 } 1292 1293 return (FILTER_SCHEDULE_THREAD); 1294 } 1295 1296 /** 1297 * ice_allocate_msix - Allocate MSI-X vectors for the interface 1298 * @sc: the device private softc 1299 * 1300 * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process. 1301 * 1302 * First, determine a suitable total number of vectors based on the number 1303 * of CPUs, RSS buckets, the administrative vector, and other demands such as 1304 * RDMA. 1305 * 1306 * Request the desired amount of vectors, and see how many we obtain. If we 1307 * don't obtain as many as desired, reduce the demands by lowering the number 1308 * of requested queues or reducing the demand from other features such as 1309 * RDMA. 1310 * 1311 * @remark This function is required because the driver sets the 1312 * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors 1313 * manually. 1314 * 1315 * @remark This driver will only use MSI-X vectors. If this is not possible, 1316 * neither MSI or legacy interrupts will be tried. 1317 * 1318 * @post on success this function must set the following scctx parameters: 1319 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr. 1320 * 1321 * @returns zero on success or an error code on failure. 1322 */ 1323 static int 1324 ice_allocate_msix(struct ice_softc *sc) 1325 { 1326 bool iflib_override_queue_count = false; 1327 if_softc_ctx_t scctx = sc->scctx; 1328 device_t dev = sc->dev; 1329 cpuset_t cpus; 1330 int bar, queues, vectors, requested; 1331 int err = 0; 1332 int rdma; 1333 1334 /* Allocate the MSI-X bar */ 1335 bar = scctx->isc_msix_bar; 1336 sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); 1337 if (!sc->msix_table) { 1338 device_printf(dev, "Unable to map MSI-X table\n"); 1339 return (ENOMEM); 1340 } 1341 1342 /* Check if the iflib queue count sysctls have been set */ 1343 if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs) 1344 iflib_override_queue_count = true; 1345 1346 err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus); 1347 if (err) { 1348 device_printf(dev, "%s: Unable to fetch the CPU list: %s\n", 1349 __func__, ice_err_str(err)); 1350 CPU_COPY(&all_cpus, &cpus); 1351 } 1352 1353 /* Attempt to mimic behavior of iflib_msix_init */ 1354 if (iflib_override_queue_count) { 1355 /* 1356 * If the override sysctls have been set, limit the queues to 1357 * the number of logical CPUs. 1358 */ 1359 queues = mp_ncpus; 1360 } else { 1361 /* 1362 * Otherwise, limit the queue count to the CPUs associated 1363 * with the NUMA node the device is associated with. 1364 */ 1365 queues = CPU_COUNT(&cpus); 1366 } 1367 1368 /* Clamp to the number of RSS buckets */ 1369 queues = imin(queues, rss_getnumbuckets()); 1370 1371 /* 1372 * Clamp the number of queue pairs to the minimum of the requested Tx 1373 * and Rx queues. 1374 */ 1375 queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets); 1376 queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets); 1377 1378 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RDMA)) { 1379 /* 1380 * Choose a number of RDMA vectors based on the number of CPUs 1381 * up to a maximum 1382 */ 1383 rdma = min(CPU_COUNT(&cpus), ICE_RDMA_MAX_MSIX); 1384 1385 /* Further limit by the user configurable tunable */ 1386 rdma = min(rdma, ice_rdma_max_msix); 1387 } else { 1388 rdma = 0; 1389 } 1390 1391 /* 1392 * Determine the number of vectors to request. Note that we also need 1393 * to allocate one vector for administrative tasks. 1394 */ 1395 requested = rdma + queues + 1; 1396 1397 vectors = requested; 1398 1399 err = pci_alloc_msix(dev, &vectors); 1400 if (err) { 1401 device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n", 1402 vectors, ice_err_str(err)); 1403 goto err_free_msix_table; 1404 } 1405 1406 /* If we don't receive enough vectors, reduce demands */ 1407 if (vectors < requested) { 1408 int diff = requested - vectors; 1409 1410 device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n", 1411 requested, vectors); 1412 1413 /* 1414 * The OS didn't grant us the requested number of vectors. 1415 * Check to see if we can reduce demands by limiting the 1416 * number of vectors allocated to certain features. 1417 */ 1418 1419 if (rdma >= diff) { 1420 /* Reduce the number of RDMA vectors we reserve */ 1421 rdma -= diff; 1422 diff = 0; 1423 } else { 1424 /* Disable RDMA and reduce the difference */ 1425 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 1426 diff -= rdma; 1427 rdma = 0; 1428 } 1429 1430 /* 1431 * If we still have a difference, we need to reduce the number 1432 * of queue pairs. 1433 * 1434 * However, we still need at least one vector for the admin 1435 * interrupt and one queue pair. 1436 */ 1437 if (queues <= diff) { 1438 device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n"); 1439 err = (ERANGE); 1440 goto err_pci_release_msi; 1441 } 1442 1443 queues -= diff; 1444 } 1445 1446 device_printf(dev, "Using %d Tx and Rx queues\n", queues); 1447 if (rdma) 1448 device_printf(dev, "Reserving %d MSI-X interrupts for iRDMA\n", 1449 rdma); 1450 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 1451 vectors); 1452 1453 scctx->isc_vectors = vectors; 1454 scctx->isc_nrxqsets = queues; 1455 scctx->isc_ntxqsets = queues; 1456 scctx->isc_intr = IFLIB_INTR_MSIX; 1457 1458 sc->irdma_vectors = rdma; 1459 1460 /* Interrupt allocation tracking isn't required in recovery mode, 1461 * since neither RDMA nor VFs are enabled. 1462 */ 1463 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1464 return (0); 1465 1466 /* Keep track of which interrupt indices are being used for what */ 1467 sc->lan_vectors = vectors - rdma; 1468 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->pf_imap, sc->lan_vectors); 1469 if (err) { 1470 device_printf(dev, "Unable to assign PF interrupt mapping: %s\n", 1471 ice_err_str(err)); 1472 goto err_pci_release_msi; 1473 } 1474 err = ice_resmgr_assign_contiguous(&sc->imgr, sc->rdma_imap, rdma); 1475 if (err) { 1476 device_printf(dev, "Unable to assign PF RDMA interrupt mapping: %s\n", 1477 ice_err_str(err)); 1478 ice_resmgr_release_map(&sc->imgr, sc->pf_imap, 1479 sc->lan_vectors); 1480 goto err_pci_release_msi; 1481 } 1482 1483 return (0); 1484 1485 err_pci_release_msi: 1486 pci_release_msi(dev); 1487 err_free_msix_table: 1488 if (sc->msix_table != NULL) { 1489 bus_release_resource(sc->dev, SYS_RES_MEMORY, 1490 rman_get_rid(sc->msix_table), 1491 sc->msix_table); 1492 sc->msix_table = NULL; 1493 } 1494 1495 return (err); 1496 } 1497 1498 /** 1499 * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues 1500 * @ctx: the iflib context structure 1501 * @msix: the number of vectors we were assigned 1502 * 1503 * Called by iflib to assign MSI-X vectors to queues. Currently requires that 1504 * we get at least the same number of vectors as we have queues, and that we 1505 * always have the same number of Tx and Rx queues. 1506 * 1507 * Tx queues use a softirq instead of using their own hardware interrupt. 1508 */ 1509 static int 1510 ice_if_msix_intr_assign(if_ctx_t ctx, int msix) 1511 { 1512 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1513 struct ice_vsi *vsi = &sc->pf_vsi; 1514 int err, i, vector; 1515 1516 ASSERT_CTX_LOCKED(sc); 1517 1518 if (vsi->num_rx_queues != vsi->num_tx_queues) { 1519 device_printf(sc->dev, 1520 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 1521 vsi->num_tx_queues, vsi->num_rx_queues); 1522 return (EOPNOTSUPP); 1523 } 1524 1525 if (msix < (vsi->num_rx_queues + 1)) { 1526 device_printf(sc->dev, 1527 "Not enough MSI-X vectors to assign one vector to each queue pair\n"); 1528 return (EOPNOTSUPP); 1529 } 1530 1531 /* Save the number of vectors for future use */ 1532 sc->num_irq_vectors = vsi->num_rx_queues + 1; 1533 1534 /* Allocate space to store the IRQ vector data */ 1535 if (!(sc->irqvs = 1536 (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors), 1537 M_ICE, M_NOWAIT))) { 1538 device_printf(sc->dev, 1539 "Unable to allocate irqv memory\n"); 1540 return (ENOMEM); 1541 } 1542 1543 /* Administrative interrupt events will use vector 0 */ 1544 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN, 1545 ice_msix_admin, sc, 0, "admin"); 1546 if (err) { 1547 device_printf(sc->dev, 1548 "Failed to register Admin queue handler: %s\n", 1549 ice_err_str(err)); 1550 goto free_irqvs; 1551 } 1552 sc->irqvs[0].me = 0; 1553 1554 /* Do not allocate queue interrupts when in recovery mode */ 1555 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1556 return (0); 1557 1558 for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) { 1559 struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 1560 struct ice_tx_queue *txq = &vsi->tx_queues[i]; 1561 int rid = vector + 1; 1562 char irq_name[16]; 1563 1564 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1565 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid, 1566 IFLIB_INTR_RXTX, ice_msix_que, 1567 rxq, rxq->me, irq_name); 1568 if (err) { 1569 device_printf(sc->dev, 1570 "Failed to allocate q int %d err: %s\n", 1571 i, ice_err_str(err)); 1572 vector--; 1573 i--; 1574 goto fail; 1575 } 1576 sc->irqvs[vector].me = vector; 1577 rxq->irqv = &sc->irqvs[vector]; 1578 1579 bzero(irq_name, sizeof(irq_name)); 1580 1581 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1582 iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq, 1583 IFLIB_INTR_TX, txq, 1584 txq->me, irq_name); 1585 txq->irqv = &sc->irqvs[vector]; 1586 } 1587 1588 return (0); 1589 fail: 1590 for (; i >= 0; i--, vector--) 1591 iflib_irq_free(ctx, &sc->irqvs[vector].irq); 1592 iflib_irq_free(ctx, &sc->irqvs[0].irq); 1593 free_irqvs: 1594 free(sc->irqvs, M_ICE); 1595 sc->irqvs = NULL; 1596 return err; 1597 } 1598 1599 /** 1600 * ice_if_mtu_set - Set the device MTU 1601 * @ctx: iflib context structure 1602 * @mtu: the MTU requested 1603 * 1604 * Called by iflib to configure the device's Maximum Transmission Unit (MTU). 1605 * 1606 * @pre assumes the caller holds the iflib CTX lock 1607 */ 1608 static int 1609 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1610 { 1611 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1612 1613 ASSERT_CTX_LOCKED(sc); 1614 1615 /* Do not support configuration when in recovery mode */ 1616 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1617 return (ENOSYS); 1618 1619 if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU) 1620 return (EINVAL); 1621 1622 sc->scctx->isc_max_frame_size = mtu + 1623 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 1624 1625 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 1626 1627 return (0); 1628 } 1629 1630 /** 1631 * ice_if_intr_enable - Enable device interrupts 1632 * @ctx: iflib context structure 1633 * 1634 * Called by iflib to request enabling device interrupts. 1635 */ 1636 static void 1637 ice_if_intr_enable(if_ctx_t ctx) 1638 { 1639 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1640 struct ice_vsi *vsi = &sc->pf_vsi; 1641 struct ice_hw *hw = &sc->hw; 1642 1643 ASSERT_CTX_LOCKED(sc); 1644 1645 /* Enable ITR 0 */ 1646 ice_enable_intr(hw, sc->irqvs[0].me); 1647 1648 /* Do not enable queue interrupts in recovery mode */ 1649 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1650 return; 1651 1652 /* Enable all queue interrupts */ 1653 for (int i = 0; i < vsi->num_rx_queues; i++) 1654 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 1655 } 1656 1657 /** 1658 * ice_if_intr_disable - Disable device interrupts 1659 * @ctx: iflib context structure 1660 * 1661 * Called by iflib to request disabling device interrupts. 1662 */ 1663 static void 1664 ice_if_intr_disable(if_ctx_t ctx) 1665 { 1666 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1667 struct ice_hw *hw = &sc->hw; 1668 unsigned int i; 1669 1670 ASSERT_CTX_LOCKED(sc); 1671 1672 /* IFDI_INTR_DISABLE may be called prior to interrupts actually being 1673 * assigned to queues. Instead of assuming that the interrupt 1674 * assignment in the rx_queues structure is valid, just disable all 1675 * possible interrupts 1676 * 1677 * Note that we choose not to disable ITR 0 because this handles the 1678 * AdminQ interrupts, and we want to keep processing these even when 1679 * the interface is offline. 1680 */ 1681 for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++) 1682 ice_disable_intr(hw, i); 1683 } 1684 1685 /** 1686 * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 1687 * @ctx: iflib context structure 1688 * @rxqid: the Rx queue to enable 1689 * 1690 * Enable a specific Rx queue interrupt. 1691 * 1692 * This function is not protected by the iflib CTX lock. 1693 */ 1694 static int 1695 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1696 { 1697 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1698 struct ice_vsi *vsi = &sc->pf_vsi; 1699 struct ice_hw *hw = &sc->hw; 1700 1701 /* Do not enable queue interrupts in recovery mode */ 1702 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1703 return (ENOSYS); 1704 1705 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 1706 return (0); 1707 } 1708 1709 /** 1710 * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 1711 * @ctx: iflib context structure 1712 * @txqid: the Tx queue to enable 1713 * 1714 * Enable a specific Tx queue interrupt. 1715 * 1716 * This function is not protected by the iflib CTX lock. 1717 */ 1718 static int 1719 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1720 { 1721 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1722 struct ice_vsi *vsi = &sc->pf_vsi; 1723 struct ice_hw *hw = &sc->hw; 1724 1725 /* Do not enable queue interrupts in recovery mode */ 1726 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1727 return (ENOSYS); 1728 1729 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 1730 return (0); 1731 } 1732 1733 /** 1734 * ice_if_promisc_set - Set device promiscuous mode 1735 * @ctx: iflib context structure 1736 * @flags: promiscuous flags to configure 1737 * 1738 * Called by iflib to configure device promiscuous mode. 1739 * 1740 * @remark Calls to this function will always overwrite the previous setting 1741 */ 1742 static int 1743 ice_if_promisc_set(if_ctx_t ctx, int flags) 1744 { 1745 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1746 struct ice_hw *hw = &sc->hw; 1747 device_t dev = sc->dev; 1748 enum ice_status status; 1749 bool promisc_enable = flags & IFF_PROMISC; 1750 bool multi_enable = flags & IFF_ALLMULTI; 1751 1752 /* Do not support configuration when in recovery mode */ 1753 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1754 return (ENOSYS); 1755 1756 if (multi_enable) 1757 return (EOPNOTSUPP); 1758 1759 if (promisc_enable) { 1760 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1761 ICE_VSI_PROMISC_MASK, 0); 1762 if (status && status != ICE_ERR_ALREADY_EXISTS) { 1763 device_printf(dev, 1764 "Failed to enable promiscuous mode for PF VSI, err %s aq_err %s\n", 1765 ice_status_str(status), 1766 ice_aq_str(hw->adminq.sq_last_status)); 1767 return (EIO); 1768 } 1769 } else { 1770 status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx, 1771 ICE_VSI_PROMISC_MASK, 0); 1772 if (status) { 1773 device_printf(dev, 1774 "Failed to disable promiscuous mode for PF VSI, err %s aq_err %s\n", 1775 ice_status_str(status), 1776 ice_aq_str(hw->adminq.sq_last_status)); 1777 return (EIO); 1778 } 1779 } 1780 1781 return (0); 1782 } 1783 1784 /** 1785 * ice_if_media_change - Change device media 1786 * @ctx: device ctx structure 1787 * 1788 * Called by iflib when a media change is requested. This operation is not 1789 * supported by the hardware, so we just return an error code. 1790 */ 1791 static int 1792 ice_if_media_change(if_ctx_t ctx) 1793 { 1794 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1795 1796 device_printf(sc->dev, "Media change is not supported.\n"); 1797 return (ENODEV); 1798 } 1799 1800 /** 1801 * ice_if_media_status - Report current device media 1802 * @ctx: iflib context structure 1803 * @ifmr: ifmedia request structure to update 1804 * 1805 * Updates the provided ifmr with current device media status, including link 1806 * status and media type. 1807 */ 1808 static void 1809 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1810 { 1811 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1812 struct ice_link_status *li = &sc->hw.port_info->phy.link_info; 1813 1814 ifmr->ifm_status = IFM_AVALID; 1815 ifmr->ifm_active = IFM_ETHER; 1816 1817 /* Never report link up or media types when in recovery mode */ 1818 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1819 return; 1820 1821 if (!sc->link_up) 1822 return; 1823 1824 ifmr->ifm_status |= IFM_ACTIVE; 1825 ifmr->ifm_active |= IFM_FDX; 1826 1827 if (li->phy_type_low) 1828 ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low); 1829 else if (li->phy_type_high) 1830 ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high); 1831 else 1832 ifmr->ifm_active |= IFM_UNKNOWN; 1833 1834 /* Report flow control status as well */ 1835 if (li->an_info & ICE_AQ_LINK_PAUSE_TX) 1836 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1837 if (li->an_info & ICE_AQ_LINK_PAUSE_RX) 1838 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1839 } 1840 1841 /** 1842 * ice_init_tx_tracking - Initialize Tx queue software tracking values 1843 * @vsi: the VSI to initialize 1844 * 1845 * Initialize Tx queue software tracking values, including the Report Status 1846 * queue, and related software tracking values. 1847 */ 1848 static void 1849 ice_init_tx_tracking(struct ice_vsi *vsi) 1850 { 1851 struct ice_tx_queue *txq; 1852 size_t j; 1853 int i; 1854 1855 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1856 1857 txq->tx_rs_cidx = txq->tx_rs_pidx = 0; 1858 1859 /* Initialize the last processed descriptor to be the end of 1860 * the ring, rather than the start, so that we avoid an 1861 * off-by-one error in ice_ift_txd_credits_update for the 1862 * first packet. 1863 */ 1864 txq->tx_cidx_processed = txq->desc_count - 1; 1865 1866 for (j = 0; j < txq->desc_count; j++) 1867 txq->tx_rsq[j] = QIDX_INVALID; 1868 } 1869 } 1870 1871 /** 1872 * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues 1873 * @sc: the device softc 1874 * 1875 * Called to update the Rx queue mbuf_sz parameter for configuring the receive 1876 * buffer sizes when programming hardware. 1877 */ 1878 static void 1879 ice_update_rx_mbuf_sz(struct ice_softc *sc) 1880 { 1881 uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx); 1882 struct ice_vsi *vsi = &sc->pf_vsi; 1883 1884 MPASS(mbuf_sz <= UINT16_MAX); 1885 vsi->mbuf_sz = mbuf_sz; 1886 } 1887 1888 /** 1889 * ice_if_init - Initialize the device 1890 * @ctx: iflib ctx structure 1891 * 1892 * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes 1893 * device filters and prepares the Tx and Rx engines. 1894 * 1895 * @pre assumes the caller holds the iflib CTX lock 1896 */ 1897 static void 1898 ice_if_init(if_ctx_t ctx) 1899 { 1900 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1901 device_t dev = sc->dev; 1902 int err; 1903 1904 ASSERT_CTX_LOCKED(sc); 1905 1906 /* 1907 * We've seen an issue with 11.3/12.1 where sideband routines are 1908 * called after detach is called. This would call routines after 1909 * if_stop, causing issues with the teardown process. This has 1910 * seemingly been fixed in STABLE snapshots, but it seems like a 1911 * good idea to have this guard here regardless. 1912 */ 1913 if (ice_driver_is_detaching(sc)) 1914 return; 1915 1916 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1917 return; 1918 1919 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 1920 device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n"); 1921 return; 1922 } 1923 1924 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 1925 device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n"); 1926 return; 1927 } 1928 1929 ice_update_rx_mbuf_sz(sc); 1930 1931 /* Update the MAC address... User might use a LAA */ 1932 err = ice_update_laa_mac(sc); 1933 if (err) { 1934 device_printf(dev, 1935 "LAA address change failed, err %s\n", 1936 ice_err_str(err)); 1937 return; 1938 } 1939 1940 /* Initialize software Tx tracking values */ 1941 ice_init_tx_tracking(&sc->pf_vsi); 1942 1943 err = ice_cfg_vsi_for_tx(&sc->pf_vsi); 1944 if (err) { 1945 device_printf(dev, 1946 "Unable to configure the main VSI for Tx: %s\n", 1947 ice_err_str(err)); 1948 return; 1949 } 1950 1951 err = ice_cfg_vsi_for_rx(&sc->pf_vsi); 1952 if (err) { 1953 device_printf(dev, 1954 "Unable to configure the main VSI for Rx: %s\n", 1955 ice_err_str(err)); 1956 goto err_cleanup_tx; 1957 } 1958 1959 err = ice_control_all_rx_queues(&sc->pf_vsi, true); 1960 if (err) { 1961 device_printf(dev, 1962 "Unable to enable Rx rings for transmit: %s\n", 1963 ice_err_str(err)); 1964 goto err_cleanup_tx; 1965 } 1966 1967 err = ice_cfg_pf_default_mac_filters(sc); 1968 if (err) { 1969 device_printf(dev, 1970 "Unable to configure default MAC filters: %s\n", 1971 ice_err_str(err)); 1972 goto err_stop_rx; 1973 } 1974 1975 /* We use software interrupts for Tx, so we only program the hardware 1976 * interrupts for Rx. 1977 */ 1978 ice_configure_all_rxq_interrupts(&sc->pf_vsi); 1979 ice_configure_rx_itr(&sc->pf_vsi); 1980 1981 /* Configure promiscuous mode */ 1982 ice_if_promisc_set(ctx, if_getflags(sc->ifp)); 1983 1984 ice_rdma_pf_init(sc); 1985 1986 ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED); 1987 return; 1988 1989 err_stop_rx: 1990 ice_control_all_rx_queues(&sc->pf_vsi, false); 1991 err_cleanup_tx: 1992 ice_vsi_disable_tx(&sc->pf_vsi); 1993 } 1994 1995 /** 1996 * ice_poll_for_media_avail - Re-enable link if media is detected 1997 * @sc: device private structure 1998 * 1999 * Intended to be called from the driver's timer function, this function 2000 * sends the Get Link Status AQ command and re-enables HW link if the 2001 * command says that media is available. 2002 * 2003 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing, 2004 * since media removal events are supposed to be sent to the driver through 2005 * a link status event. 2006 */ 2007 static void 2008 ice_poll_for_media_avail(struct ice_softc *sc) 2009 { 2010 struct ice_hw *hw = &sc->hw; 2011 struct ice_port_info *pi = hw->port_info; 2012 2013 if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) { 2014 pi->phy.get_link_info = true; 2015 ice_get_link_status(pi, &sc->link_up); 2016 2017 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2018 enum ice_status status; 2019 2020 /* Re-enable link and re-apply user link settings */ 2021 ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC); 2022 2023 /* Update the OS about changes in media capability */ 2024 status = ice_add_media_types(sc, sc->media); 2025 if (status) 2026 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 2027 ice_status_str(status), 2028 ice_aq_str(hw->adminq.sq_last_status)); 2029 2030 ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA); 2031 } 2032 } 2033 } 2034 2035 /** 2036 * ice_if_timer - called by iflib periodically 2037 * @ctx: iflib ctx structure 2038 * @qid: the queue this timer was called for 2039 * 2040 * This callback is triggered by iflib periodically. We use it to update the 2041 * hw statistics. 2042 * 2043 * @remark this function is not protected by the iflib CTX lock. 2044 */ 2045 static void 2046 ice_if_timer(if_ctx_t ctx, uint16_t qid) 2047 { 2048 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2049 uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx; 2050 2051 if (qid != 0) 2052 return; 2053 2054 /* Do not attempt to update stats when in recovery mode */ 2055 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2056 return; 2057 2058 /* Update device statistics */ 2059 ice_update_pf_stats(sc); 2060 2061 /* 2062 * For proper watchdog management, the iflib stack needs to know if 2063 * we've been paused during the last interval. Check if the 2064 * link_xoff_rx stat changed, and set the isc_pause_frames, if so. 2065 */ 2066 if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx) 2067 sc->scctx->isc_pause_frames = 1; 2068 2069 /* Update the primary VSI stats */ 2070 ice_update_vsi_hw_stats(&sc->pf_vsi); 2071 } 2072 2073 /** 2074 * ice_admin_timer - called periodically to trigger the admin task 2075 * @arg: callout(9) argument pointing to the device private softc structure 2076 * 2077 * Timer function used as part of a callout(9) timer that will periodically 2078 * trigger the admin task, even when the interface is down. 2079 * 2080 * @remark this function is not called by iflib and is not protected by the 2081 * iflib CTX lock. 2082 * 2083 * @remark because this is a callout function, it cannot sleep and should not 2084 * attempt taking the iflib CTX lock. 2085 */ 2086 static void 2087 ice_admin_timer(void *arg) 2088 { 2089 struct ice_softc *sc = (struct ice_softc *)arg; 2090 2091 /* 2092 * There is a point where callout routines are no longer 2093 * cancelable. So there exists a window of time where the 2094 * driver enters detach() and tries to cancel the callout, but the 2095 * callout routine has passed the cancellation point. The detach() 2096 * routine is unaware of this and tries to free resources that the 2097 * callout routine needs. So we check for the detach state flag to 2098 * at least shrink the window of opportunity. 2099 */ 2100 if (ice_driver_is_detaching(sc)) 2101 return; 2102 2103 /* Fire off the admin task */ 2104 iflib_admin_intr_deferred(sc->ctx); 2105 2106 /* Reschedule the admin timer */ 2107 callout_schedule(&sc->admin_timer, hz/2); 2108 } 2109 2110 /** 2111 * ice_transition_recovery_mode - Transition to recovery mode 2112 * @sc: the device private softc 2113 * 2114 * Called when the driver detects that the firmware has entered recovery mode 2115 * at run time. 2116 */ 2117 static void 2118 ice_transition_recovery_mode(struct ice_softc *sc) 2119 { 2120 struct ice_vsi *vsi = &sc->pf_vsi; 2121 int i; 2122 2123 device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 2124 2125 /* Tell the stack that the link has gone down */ 2126 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 2127 2128 /* Request that the device be re-initialized */ 2129 ice_request_stack_reinit(sc); 2130 2131 ice_rdma_pf_detach(sc); 2132 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2133 2134 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2135 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2136 2137 ice_vsi_del_txqs_ctx(vsi); 2138 ice_vsi_del_rxqs_ctx(vsi); 2139 2140 for (i = 0; i < sc->num_available_vsi; i++) { 2141 if (sc->all_vsi[i]) 2142 ice_release_vsi(sc->all_vsi[i]); 2143 } 2144 sc->num_available_vsi = 0; 2145 2146 if (sc->all_vsi) { 2147 free(sc->all_vsi, M_ICE); 2148 sc->all_vsi = NULL; 2149 } 2150 2151 /* Destroy the interrupt manager */ 2152 ice_resmgr_destroy(&sc->imgr); 2153 /* Destroy the queue managers */ 2154 ice_resmgr_destroy(&sc->tx_qmgr); 2155 ice_resmgr_destroy(&sc->rx_qmgr); 2156 2157 ice_deinit_hw(&sc->hw); 2158 } 2159 2160 /** 2161 * ice_transition_safe_mode - Transition to safe mode 2162 * @sc: the device private softc 2163 * 2164 * Called when the driver attempts to reload the DDP package during a device 2165 * reset, and the new download fails. If so, we must transition to safe mode 2166 * at run time. 2167 * 2168 * @remark although safe mode normally allocates only a single queue, we can't 2169 * change the number of queues dynamically when using iflib. Due to this, we 2170 * do not attempt to reduce the number of queues. 2171 */ 2172 static void 2173 ice_transition_safe_mode(struct ice_softc *sc) 2174 { 2175 /* Indicate that we are in Safe mode */ 2176 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap); 2177 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en); 2178 2179 ice_rdma_pf_detach(sc); 2180 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2181 2182 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2183 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2184 2185 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2186 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en); 2187 } 2188 2189 /** 2190 * ice_if_update_admin_status - update admin status 2191 * @ctx: iflib ctx structure 2192 * 2193 * Called by iflib to update the admin status. For our purposes, this means 2194 * check the adminq, and update the link status. It's ultimately triggered by 2195 * our admin interrupt, or by the ice_if_timer periodically. 2196 * 2197 * @pre assumes the caller holds the iflib CTX lock 2198 */ 2199 static void 2200 ice_if_update_admin_status(if_ctx_t ctx) 2201 { 2202 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2203 enum ice_fw_modes fw_mode; 2204 bool reschedule = false; 2205 u16 pending = 0; 2206 2207 ASSERT_CTX_LOCKED(sc); 2208 2209 /* Check if the firmware entered recovery mode at run time */ 2210 fw_mode = ice_get_fw_mode(&sc->hw); 2211 if (fw_mode == ICE_FW_MODE_REC) { 2212 if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2213 /* If we just entered recovery mode, log a warning to 2214 * the system administrator and deinit driver state 2215 * that is no longer functional. 2216 */ 2217 ice_transition_recovery_mode(sc); 2218 } 2219 } else if (fw_mode == ICE_FW_MODE_ROLLBACK) { 2220 if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) { 2221 /* Rollback mode isn't fatal, but we don't want to 2222 * repeatedly post a message about it. 2223 */ 2224 ice_print_rollback_msg(&sc->hw); 2225 } 2226 } 2227 2228 /* Handle global reset events */ 2229 ice_handle_reset_event(sc); 2230 2231 /* Handle PF reset requests */ 2232 ice_handle_pf_reset_request(sc); 2233 2234 /* Handle MDD events */ 2235 ice_handle_mdd_event(sc); 2236 2237 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) || 2238 ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) || 2239 ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2240 /* 2241 * If we know the control queues are disabled, skip processing 2242 * the control queues entirely. 2243 */ 2244 ; 2245 } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) { 2246 ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending); 2247 if (pending > 0) 2248 reschedule = true; 2249 2250 ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending); 2251 if (pending > 0) 2252 reschedule = true; 2253 } 2254 2255 /* Poll for link up */ 2256 ice_poll_for_media_avail(sc); 2257 2258 /* Check and update link status */ 2259 ice_update_link_status(sc, false); 2260 2261 /* 2262 * If there are still messages to process, we need to reschedule 2263 * ourselves. Otherwise, we can just re-enable the interrupt. We'll be 2264 * woken up at the next interrupt or timer event. 2265 */ 2266 if (reschedule) { 2267 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 2268 iflib_admin_intr_deferred(ctx); 2269 } else { 2270 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2271 } 2272 } 2273 2274 /** 2275 * ice_prepare_for_reset - Prepare device for an impending reset 2276 * @sc: The device private softc 2277 * 2278 * Prepare the driver for an impending reset, shutting down VSIs, clearing the 2279 * scheduler setup, and shutting down controlqs. Uses the 2280 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the 2281 * driver for reset or not. 2282 */ 2283 static void 2284 ice_prepare_for_reset(struct ice_softc *sc) 2285 { 2286 struct ice_hw *hw = &sc->hw; 2287 2288 /* If we're already prepared, there's nothing to do */ 2289 if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) 2290 return; 2291 2292 log(LOG_INFO, "%s: preparing to reset device logic\n", if_name(sc->ifp)); 2293 2294 /* In recovery mode, hardware is not initialized */ 2295 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2296 return; 2297 2298 /* inform the RDMA client */ 2299 ice_rdma_notify_reset(sc); 2300 /* stop the RDMA client */ 2301 ice_rdma_pf_stop(sc); 2302 2303 /* Release the main PF VSI queue mappings */ 2304 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2305 sc->pf_vsi.num_tx_queues); 2306 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2307 sc->pf_vsi.num_rx_queues); 2308 2309 ice_clear_hw_tbls(hw); 2310 2311 if (hw->port_info) 2312 ice_sched_clear_port(hw->port_info); 2313 2314 ice_shutdown_all_ctrlq(hw, false); 2315 } 2316 2317 /** 2318 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping 2319 * @sc: the device softc pointer 2320 * 2321 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue 2322 * mapping after a reset occurred. 2323 */ 2324 static int 2325 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc) 2326 { 2327 struct ice_vsi *vsi = &sc->pf_vsi; 2328 struct ice_tx_queue *txq; 2329 struct ice_rx_queue *rxq; 2330 int err, i; 2331 2332 /* Re-assign Tx queues from PF space to the main VSI */ 2333 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, 2334 vsi->num_tx_queues); 2335 if (err) { 2336 device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n", 2337 ice_err_str(err)); 2338 return (err); 2339 } 2340 2341 /* Re-assign Rx queues from PF space to this VSI */ 2342 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, 2343 vsi->num_rx_queues); 2344 if (err) { 2345 device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n", 2346 ice_err_str(err)); 2347 goto err_release_tx_queues; 2348 } 2349 2350 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 2351 2352 /* Re-assign Tx queue tail pointers */ 2353 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 2354 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 2355 2356 /* Re-assign Rx queue tail pointers */ 2357 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 2358 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 2359 2360 return (0); 2361 2362 err_release_tx_queues: 2363 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2364 sc->pf_vsi.num_tx_queues); 2365 2366 return (err); 2367 } 2368 2369 /* determine if the iflib context is active */ 2370 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING)) 2371 2372 /** 2373 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode 2374 * @sc: The device private softc 2375 * 2376 * Handle a driver rebuild while in recovery mode. This will only rebuild the 2377 * limited functionality supported while in recovery mode. 2378 */ 2379 static void 2380 ice_rebuild_recovery_mode(struct ice_softc *sc) 2381 { 2382 device_t dev = sc->dev; 2383 2384 /* enable PCIe bus master */ 2385 pci_enable_busmaster(dev); 2386 2387 /* Configure interrupt causes for the administrative interrupt */ 2388 ice_configure_misc_interrupts(sc); 2389 2390 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2391 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2392 2393 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2394 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2395 2396 log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp)); 2397 2398 /* In order to completely restore device functionality, the iflib core 2399 * needs to be reset. We need to request an iflib reset. Additionally, 2400 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2401 * the iflib core, we also want re-run the admin task so that iflib 2402 * resets immediately instead of waiting for the next interrupt. 2403 */ 2404 ice_request_stack_reinit(sc); 2405 2406 return; 2407 } 2408 2409 /** 2410 * ice_rebuild - Rebuild driver state post reset 2411 * @sc: The device private softc 2412 * 2413 * Restore driver state after a reset occurred. Restart the controlqs, setup 2414 * the hardware port, and re-enable the VSIs. 2415 */ 2416 static void 2417 ice_rebuild(struct ice_softc *sc) 2418 { 2419 struct ice_hw *hw = &sc->hw; 2420 device_t dev = sc->dev; 2421 enum ice_ddp_state pkg_state; 2422 enum ice_status status; 2423 int err; 2424 2425 sc->rebuild_ticks = ticks; 2426 2427 /* If we're rebuilding, then a reset has succeeded. */ 2428 ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED); 2429 2430 /* 2431 * If the firmware is in recovery mode, only restore the limited 2432 * functionality supported by recovery mode. 2433 */ 2434 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2435 ice_rebuild_recovery_mode(sc); 2436 return; 2437 } 2438 2439 /* enable PCIe bus master */ 2440 pci_enable_busmaster(dev); 2441 2442 status = ice_init_all_ctrlq(hw); 2443 if (status) { 2444 device_printf(dev, "failed to re-init controlqs, err %s\n", 2445 ice_status_str(status)); 2446 goto err_shutdown_ctrlq; 2447 } 2448 2449 /* Query the allocated resources for Tx scheduler */ 2450 status = ice_sched_query_res_alloc(hw); 2451 if (status) { 2452 device_printf(dev, 2453 "Failed to query scheduler resources, err %s aq_err %s\n", 2454 ice_status_str(status), 2455 ice_aq_str(hw->adminq.sq_last_status)); 2456 goto err_shutdown_ctrlq; 2457 } 2458 2459 /* Re-enable FW logging. Keep going even if this fails */ 2460 status = ice_fwlog_set(hw, &hw->fwlog_cfg); 2461 if (!status) { 2462 /* 2463 * We should have the most updated cached copy of the 2464 * configuration, regardless of whether we're rebuilding 2465 * or not. So we'll simply check to see if logging was 2466 * enabled pre-rebuild. 2467 */ 2468 if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 2469 status = ice_fwlog_register(hw); 2470 if (status) 2471 device_printf(dev, "failed to re-register fw logging, err %s aq_err %s\n", 2472 ice_status_str(status), 2473 ice_aq_str(hw->adminq.sq_last_status)); 2474 } 2475 } else 2476 device_printf(dev, "failed to rebuild fw logging configuration, err %s aq_err %s\n", 2477 ice_status_str(status), 2478 ice_aq_str(hw->adminq.sq_last_status)); 2479 2480 err = ice_send_version(sc); 2481 if (err) 2482 goto err_shutdown_ctrlq; 2483 2484 err = ice_init_link_events(sc); 2485 if (err) { 2486 device_printf(dev, "ice_init_link_events failed: %s\n", 2487 ice_err_str(err)); 2488 goto err_shutdown_ctrlq; 2489 } 2490 2491 status = ice_clear_pf_cfg(hw); 2492 if (status) { 2493 device_printf(dev, "failed to clear PF configuration, err %s\n", 2494 ice_status_str(status)); 2495 goto err_shutdown_ctrlq; 2496 } 2497 2498 ice_clear_pxe_mode(hw); 2499 2500 status = ice_get_caps(hw); 2501 if (status) { 2502 device_printf(dev, "failed to get capabilities, err %s\n", 2503 ice_status_str(status)); 2504 goto err_shutdown_ctrlq; 2505 } 2506 2507 status = ice_sched_init_port(hw->port_info); 2508 if (status) { 2509 device_printf(dev, "failed to initialize port, err %s\n", 2510 ice_status_str(status)); 2511 goto err_sched_cleanup; 2512 } 2513 2514 /* If we previously loaded the package, it needs to be reloaded now */ 2515 if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) { 2516 pkg_state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 2517 if (!ice_is_init_pkg_successful(pkg_state)) { 2518 ice_log_pkg_init(sc, pkg_state); 2519 ice_transition_safe_mode(sc); 2520 } 2521 } 2522 2523 ice_reset_pf_stats(sc); 2524 2525 err = ice_rebuild_pf_vsi_qmap(sc); 2526 if (err) { 2527 device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n", 2528 ice_err_str(err)); 2529 goto err_sched_cleanup; 2530 } 2531 err = ice_initialize_vsi(&sc->pf_vsi); 2532 if (err) { 2533 device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n", 2534 ice_err_str(err)); 2535 goto err_release_queue_allocations; 2536 } 2537 2538 /* Replay all VSI configuration */ 2539 err = ice_replay_all_vsi_cfg(sc); 2540 if (err) 2541 goto err_deinit_pf_vsi; 2542 2543 /* Re-enable FW health event reporting */ 2544 ice_init_health_events(sc); 2545 2546 /* Reconfigure the main PF VSI for RSS */ 2547 err = ice_config_rss(&sc->pf_vsi); 2548 if (err) { 2549 device_printf(sc->dev, 2550 "Unable to reconfigure RSS for the main VSI, err %s\n", 2551 ice_err_str(err)); 2552 goto err_deinit_pf_vsi; 2553 } 2554 2555 /* Refresh link status */ 2556 ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED); 2557 sc->hw.port_info->phy.get_link_info = true; 2558 ice_get_link_status(sc->hw.port_info, &sc->link_up); 2559 ice_update_link_status(sc, true); 2560 2561 /* RDMA interface will be restarted by the stack re-init */ 2562 2563 /* Configure interrupt causes for the administrative interrupt */ 2564 ice_configure_misc_interrupts(sc); 2565 2566 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2567 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2568 2569 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2570 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2571 2572 log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp)); 2573 2574 /* In order to completely restore device functionality, the iflib core 2575 * needs to be reset. We need to request an iflib reset. Additionally, 2576 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2577 * the iflib core, we also want re-run the admin task so that iflib 2578 * resets immediately instead of waiting for the next interrupt. 2579 */ 2580 ice_request_stack_reinit(sc); 2581 2582 return; 2583 2584 err_deinit_pf_vsi: 2585 ice_deinit_vsi(&sc->pf_vsi); 2586 err_release_queue_allocations: 2587 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2588 sc->pf_vsi.num_tx_queues); 2589 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2590 sc->pf_vsi.num_rx_queues); 2591 err_sched_cleanup: 2592 ice_sched_cleanup_all(hw); 2593 err_shutdown_ctrlq: 2594 ice_shutdown_all_ctrlq(hw, false); 2595 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2596 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2597 device_printf(dev, "Driver rebuild failed, please reload the device driver\n"); 2598 } 2599 2600 /** 2601 * ice_handle_reset_event - Handle reset events triggered by OICR 2602 * @sc: The device private softc 2603 * 2604 * Handle reset events triggered by an OICR notification. This includes CORER, 2605 * GLOBR, and EMPR resets triggered by software on this or any other PF or by 2606 * firmware. 2607 * 2608 * @pre assumes the iflib context lock is held, and will unlock it while 2609 * waiting for the hardware to finish reset. 2610 */ 2611 static void 2612 ice_handle_reset_event(struct ice_softc *sc) 2613 { 2614 struct ice_hw *hw = &sc->hw; 2615 enum ice_status status; 2616 device_t dev = sc->dev; 2617 2618 /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will 2619 * trigger an OICR interrupt. Our OICR handler will determine when 2620 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as 2621 * appropriate. 2622 */ 2623 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 2624 return; 2625 2626 ice_prepare_for_reset(sc); 2627 2628 /* 2629 * Release the iflib context lock and wait for the device to finish 2630 * resetting. 2631 */ 2632 IFLIB_CTX_UNLOCK(sc); 2633 status = ice_check_reset(hw); 2634 IFLIB_CTX_LOCK(sc); 2635 if (status) { 2636 device_printf(dev, "Device never came out of reset, err %s\n", 2637 ice_status_str(status)); 2638 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2639 return; 2640 } 2641 2642 /* We're done with the reset, so we can rebuild driver state */ 2643 sc->hw.reset_ongoing = false; 2644 ice_rebuild(sc); 2645 2646 /* In the unlikely event that a PF reset request occurs at the same 2647 * time as a global reset, clear the request now. This avoids 2648 * resetting a second time right after we reset due to a global event. 2649 */ 2650 if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2651 device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n"); 2652 } 2653 2654 /** 2655 * ice_handle_pf_reset_request - Initiate PF reset requested by software 2656 * @sc: The device private softc 2657 * 2658 * Initiate a PF reset requested by software. We handle this in the admin task 2659 * so that only one thread actually handles driver preparation and cleanup, 2660 * rather than having multiple threads possibly attempt to run this code 2661 * simultaneously. 2662 * 2663 * @pre assumes the iflib context lock is held and will unlock it while 2664 * waiting for the PF reset to complete. 2665 */ 2666 static void 2667 ice_handle_pf_reset_request(struct ice_softc *sc) 2668 { 2669 struct ice_hw *hw = &sc->hw; 2670 enum ice_status status; 2671 2672 /* Check for PF reset requests */ 2673 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 2674 return; 2675 2676 /* Make sure we're prepared for reset */ 2677 ice_prepare_for_reset(sc); 2678 2679 /* 2680 * Release the iflib context lock and wait for the device to finish 2681 * resetting. 2682 */ 2683 IFLIB_CTX_UNLOCK(sc); 2684 status = ice_reset(hw, ICE_RESET_PFR); 2685 IFLIB_CTX_LOCK(sc); 2686 if (status) { 2687 device_printf(sc->dev, "device PF reset failed, err %s\n", 2688 ice_status_str(status)); 2689 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2690 return; 2691 } 2692 2693 sc->soft_stats.pfr_count++; 2694 ice_rebuild(sc); 2695 } 2696 2697 /** 2698 * ice_init_device_features - Init device driver features 2699 * @sc: driver softc structure 2700 * 2701 * @pre assumes that the function capabilities bits have been set up by 2702 * ice_init_hw(). 2703 */ 2704 static void 2705 ice_init_device_features(struct ice_softc *sc) 2706 { 2707 /* Set capabilities that all devices support */ 2708 ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2709 ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap); 2710 ice_set_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2711 ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap); 2712 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap); 2713 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap); 2714 ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2715 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 2716 ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap); 2717 ice_set_bit(ICE_FEATURE_DCB, sc->feat_cap); 2718 ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap); 2719 2720 /* Disable features due to hardware limitations... */ 2721 if (!sc->hw.func_caps.common_cap.rss_table_size) 2722 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2723 if (!sc->hw.func_caps.common_cap.iwarp || !ice_enable_irdma) 2724 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2725 if (!sc->hw.func_caps.common_cap.dcb) 2726 ice_clear_bit(ICE_FEATURE_DCB, sc->feat_cap); 2727 /* Disable features due to firmware limitations... */ 2728 if (!ice_is_fw_health_report_supported(&sc->hw)) 2729 ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 2730 if (!ice_fwlog_supported(&sc->hw)) 2731 ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 2732 if (sc->hw.fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 2733 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING)) 2734 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en); 2735 else 2736 ice_fwlog_unregister(&sc->hw); 2737 } 2738 2739 /* Disable capabilities not supported by the OS */ 2740 ice_disable_unsupported_features(sc->feat_cap); 2741 2742 /* RSS is always enabled for iflib */ 2743 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS)) 2744 ice_set_bit(ICE_FEATURE_RSS, sc->feat_en); 2745 2746 /* Disable features based on sysctl settings */ 2747 if (!ice_tx_balance_en) 2748 ice_clear_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap); 2749 } 2750 2751 /** 2752 * ice_if_multi_set - Callback to update Multicast filters in HW 2753 * @ctx: iflib ctx structure 2754 * 2755 * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search 2756 * the if_multiaddrs list and determine which filters have been added or 2757 * removed from the list, and update HW programming to reflect the new list. 2758 * 2759 * @pre assumes the caller holds the iflib CTX lock 2760 */ 2761 static void 2762 ice_if_multi_set(if_ctx_t ctx) 2763 { 2764 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2765 int err; 2766 2767 ASSERT_CTX_LOCKED(sc); 2768 2769 /* Do not handle multicast configuration in recovery mode */ 2770 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2771 return; 2772 2773 err = ice_sync_multicast_filters(sc); 2774 if (err) { 2775 device_printf(sc->dev, 2776 "Failed to synchronize multicast filter list: %s\n", 2777 ice_err_str(err)); 2778 return; 2779 } 2780 } 2781 2782 /** 2783 * ice_if_vlan_register - Register a VLAN with the hardware 2784 * @ctx: iflib ctx pointer 2785 * @vtag: VLAN to add 2786 * 2787 * Programs the main PF VSI with a hardware filter for the given VLAN. 2788 * 2789 * @pre assumes the caller holds the iflib CTX lock 2790 */ 2791 static void 2792 ice_if_vlan_register(if_ctx_t ctx, u16 vtag) 2793 { 2794 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2795 enum ice_status status; 2796 2797 ASSERT_CTX_LOCKED(sc); 2798 2799 /* Do not handle VLAN configuration in recovery mode */ 2800 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2801 return; 2802 2803 status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag); 2804 if (status) { 2805 device_printf(sc->dev, 2806 "Failure adding VLAN %d to main VSI, err %s aq_err %s\n", 2807 vtag, ice_status_str(status), 2808 ice_aq_str(sc->hw.adminq.sq_last_status)); 2809 } 2810 } 2811 2812 /** 2813 * ice_if_vlan_unregister - Remove a VLAN filter from the hardware 2814 * @ctx: iflib ctx pointer 2815 * @vtag: VLAN to add 2816 * 2817 * Removes the previously programmed VLAN filter from the main PF VSI. 2818 * 2819 * @pre assumes the caller holds the iflib CTX lock 2820 */ 2821 static void 2822 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 2823 { 2824 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2825 enum ice_status status; 2826 2827 ASSERT_CTX_LOCKED(sc); 2828 2829 /* Do not handle VLAN configuration in recovery mode */ 2830 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2831 return; 2832 2833 status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag); 2834 if (status) { 2835 device_printf(sc->dev, 2836 "Failure removing VLAN %d from main VSI, err %s aq_err %s\n", 2837 vtag, ice_status_str(status), 2838 ice_aq_str(sc->hw.adminq.sq_last_status)); 2839 } 2840 } 2841 2842 /** 2843 * ice_if_stop - Stop the device 2844 * @ctx: iflib context structure 2845 * 2846 * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0 2847 * down) 2848 * 2849 * @pre assumes the caller holds the iflib CTX lock 2850 */ 2851 static void 2852 ice_if_stop(if_ctx_t ctx) 2853 { 2854 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2855 2856 ASSERT_CTX_LOCKED(sc); 2857 2858 /* 2859 * The iflib core may call IFDI_STOP prior to the first call to 2860 * IFDI_INIT. This will cause us to attempt to remove MAC filters we 2861 * don't have, and disable Tx queues which aren't yet configured. 2862 * Although it is likely these extra operations are harmless, they do 2863 * cause spurious warning messages to be displayed, which may confuse 2864 * users. 2865 * 2866 * To avoid these messages, we use a state bit indicating if we've 2867 * been initialized. It will be set when ice_if_init is called, and 2868 * cleared here in ice_if_stop. 2869 */ 2870 if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED)) 2871 return; 2872 2873 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 2874 device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n"); 2875 return; 2876 } 2877 2878 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 2879 device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n"); 2880 return; 2881 } 2882 2883 ice_rdma_pf_stop(sc); 2884 2885 /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the 2886 * return of these functions because there's nothing we can really do 2887 * if they fail, and the functions already print error messages. 2888 * Just try to shut down as much as we can. 2889 */ 2890 ice_rm_pf_default_mac_filters(sc); 2891 2892 /* Dissociate the Tx and Rx queues from the interrupts */ 2893 ice_flush_txq_interrupts(&sc->pf_vsi); 2894 ice_flush_rxq_interrupts(&sc->pf_vsi); 2895 2896 /* Disable the Tx and Rx queues */ 2897 ice_vsi_disable_tx(&sc->pf_vsi); 2898 ice_control_all_rx_queues(&sc->pf_vsi, false); 2899 } 2900 2901 /** 2902 * ice_if_get_counter - Get current value of an ifnet statistic 2903 * @ctx: iflib context pointer 2904 * @counter: ifnet counter to read 2905 * 2906 * Reads the current value of an ifnet counter for the device. 2907 * 2908 * This function is not protected by the iflib CTX lock. 2909 */ 2910 static uint64_t 2911 ice_if_get_counter(if_ctx_t ctx, ift_counter counter) 2912 { 2913 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2914 2915 /* Return the counter for the main PF VSI */ 2916 return ice_get_ifnet_counter(&sc->pf_vsi, counter); 2917 } 2918 2919 /** 2920 * ice_request_stack_reinit - Request that iflib re-initialize 2921 * @sc: the device private softc 2922 * 2923 * Request that the device be brought down and up, to re-initialize. For 2924 * example, this may be called when a device reset occurs, or when Tx and Rx 2925 * queues need to be re-initialized. 2926 * 2927 * This is required because the iflib state is outside the driver, and must be 2928 * re-initialized if we need to resart Tx and Rx queues. 2929 */ 2930 void 2931 ice_request_stack_reinit(struct ice_softc *sc) 2932 { 2933 if (CTX_ACTIVE(sc->ctx)) { 2934 iflib_request_reset(sc->ctx); 2935 iflib_admin_intr_deferred(sc->ctx); 2936 } 2937 } 2938 2939 /** 2940 * ice_driver_is_detaching - Check if the driver is detaching/unloading 2941 * @sc: device private softc 2942 * 2943 * Returns true if the driver is detaching, false otherwise. 2944 * 2945 * @remark on newer kernels, take advantage of iflib_in_detach in order to 2946 * report detachment correctly as early as possible. 2947 * 2948 * @remark this function is used by various code paths that want to avoid 2949 * running if the driver is about to be removed. This includes sysctls and 2950 * other driver access points. Note that it does not fully resolve 2951 * detach-based race conditions as it is possible for a thread to race with 2952 * iflib_in_detach. 2953 */ 2954 bool 2955 ice_driver_is_detaching(struct ice_softc *sc) 2956 { 2957 return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || 2958 iflib_in_detach(sc->ctx)); 2959 } 2960 2961 /** 2962 * ice_if_priv_ioctl - Device private ioctl handler 2963 * @ctx: iflib context pointer 2964 * @command: The ioctl command issued 2965 * @data: ioctl specific data 2966 * 2967 * iflib callback for handling custom driver specific ioctls. 2968 * 2969 * @pre Assumes that the iflib context lock is held. 2970 */ 2971 static int 2972 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) 2973 { 2974 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2975 struct ifdrv *ifd; 2976 device_t dev = sc->dev; 2977 2978 if (data == NULL) 2979 return (EINVAL); 2980 2981 ASSERT_CTX_LOCKED(sc); 2982 2983 /* Make sure the command type is valid */ 2984 switch (command) { 2985 case SIOCSDRVSPEC: 2986 case SIOCGDRVSPEC: 2987 /* Accepted commands */ 2988 break; 2989 case SIOCGPRIVATE_0: 2990 /* 2991 * Although we do not support this ioctl command, it's 2992 * expected that iflib will forward it to the IFDI_PRIV_IOCTL 2993 * handler. Do not print a message in this case 2994 */ 2995 return (ENOTSUP); 2996 default: 2997 /* 2998 * If we get a different command for this function, it's 2999 * definitely unexpected, so log a message indicating what 3000 * command we got for debugging purposes. 3001 */ 3002 device_printf(dev, "%s: unexpected ioctl command %08lx\n", 3003 __func__, command); 3004 return (EINVAL); 3005 } 3006 3007 ifd = (struct ifdrv *)data; 3008 3009 switch (ifd->ifd_cmd) { 3010 case ICE_NVM_ACCESS: 3011 return ice_handle_nvm_access_ioctl(sc, ifd); 3012 case ICE_DEBUG_DUMP: 3013 return ice_handle_debug_dump_ioctl(sc, ifd); 3014 default: 3015 return EINVAL; 3016 } 3017 } 3018 3019 /** 3020 * ice_if_i2c_req - I2C request handler for iflib 3021 * @ctx: iflib context pointer 3022 * @req: The I2C parameters to use 3023 * 3024 * Read from the port's I2C eeprom using the parameters from the ioctl. 3025 * 3026 * @remark The iflib-only part is pretty simple. 3027 */ 3028 static int 3029 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req) 3030 { 3031 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3032 3033 return ice_handle_i2c_req(sc, req); 3034 } 3035 3036 /** 3037 * ice_if_suspend - PCI device suspend handler for iflib 3038 * @ctx: iflib context pointer 3039 * 3040 * Deinitializes the driver and clears HW resources in preparation for 3041 * suspend or an FLR. 3042 * 3043 * @returns 0; this return value is ignored 3044 */ 3045 static int 3046 ice_if_suspend(if_ctx_t ctx) 3047 { 3048 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3049 3050 /* At least a PFR is always going to happen after this; 3051 * either via FLR or during the D3->D0 transition. 3052 */ 3053 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 3054 3055 ice_prepare_for_reset(sc); 3056 3057 return (0); 3058 } 3059 3060 /** 3061 * ice_if_resume - PCI device resume handler for iflib 3062 * @ctx: iflib context pointer 3063 * 3064 * Reinitializes the driver and the HW after PCI resume or after 3065 * an FLR. An init is performed by iflib after this function is finished. 3066 * 3067 * @returns 0; this return value is ignored 3068 */ 3069 static int 3070 ice_if_resume(if_ctx_t ctx) 3071 { 3072 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3073 3074 ice_rebuild(sc); 3075 3076 return (0); 3077 } 3078 3079