1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2024, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /** 33 * @file if_ice_iflib.c 34 * @brief iflib driver implementation 35 * 36 * Contains the main entry point for the iflib driver implementation. It 37 * implements the various ifdi driver methods, and sets up the module and 38 * driver values to load an iflib driver. 39 */ 40 41 #include "ice_iflib.h" 42 #include "ice_drv_info.h" 43 #include "ice_switch.h" 44 #include "ice_sched.h" 45 #ifdef PCI_IOV 46 #include "ice_iov.h" 47 #endif 48 49 #include <sys/module.h> 50 #include <sys/sockio.h> 51 #include <sys/smp.h> 52 #include <dev/pci/pcivar.h> 53 #include <dev/pci/pcireg.h> 54 55 /* 56 * Device method prototypes 57 */ 58 59 static void *ice_register(device_t); 60 static int ice_if_attach_pre(if_ctx_t); 61 static int ice_attach_pre_recovery_mode(struct ice_softc *sc); 62 static int ice_if_attach_post(if_ctx_t); 63 static void ice_attach_post_recovery_mode(struct ice_softc *sc); 64 static int ice_if_detach(if_ctx_t); 65 static int ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); 66 static int ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); 67 static int ice_if_msix_intr_assign(if_ctx_t ctx, int msix); 68 static void ice_if_queues_free(if_ctx_t ctx); 69 static int ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 70 static void ice_if_intr_enable(if_ctx_t ctx); 71 static void ice_if_intr_disable(if_ctx_t ctx); 72 static int ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 73 static int ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 74 static int ice_if_promisc_set(if_ctx_t ctx, int flags); 75 static void ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 76 static int ice_if_media_change(if_ctx_t ctx); 77 static void ice_if_init(if_ctx_t ctx); 78 static void ice_if_timer(if_ctx_t ctx, uint16_t qid); 79 static void ice_if_update_admin_status(if_ctx_t ctx); 80 static void ice_if_multi_set(if_ctx_t ctx); 81 static void ice_if_vlan_register(if_ctx_t ctx, u16 vtag); 82 static void ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag); 83 static void ice_if_stop(if_ctx_t ctx); 84 static void ice_if_led_func(if_ctx_t ctx, int onoff); 85 static uint64_t ice_if_get_counter(if_ctx_t ctx, ift_counter counter); 86 static int ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); 87 static int ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req); 88 static int ice_if_suspend(if_ctx_t ctx); 89 static int ice_if_resume(if_ctx_t ctx); 90 static bool ice_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event); 91 static void ice_init_link(struct ice_softc *sc); 92 #ifdef PCI_IOV 93 static int ice_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params); 94 static void ice_if_iov_uninit(if_ctx_t ctx); 95 static int ice_if_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params); 96 static int ice_if_vf_status(if_ctx_t ctx, struct if_vf_status **statusp); 97 static void ice_if_vflr_handle(if_ctx_t ctx); 98 #endif 99 static int ice_setup_mirror_vsi(struct ice_mirr_if *mif); 100 static int ice_wire_mirror_intrs(struct ice_mirr_if *mif); 101 static void ice_free_irqvs_subif(struct ice_mirr_if *mif); 102 static void *ice_subif_register(device_t); 103 static void ice_subif_setup_scctx(struct ice_mirr_if *mif); 104 static int ice_subif_rebuild(struct ice_softc *sc); 105 static int ice_subif_rebuild_vsi_qmap(struct ice_softc *sc); 106 107 /* Iflib API */ 108 static int ice_subif_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, 109 uint64_t *paddrs, int ntxqs, int ntxqsets); 110 static int ice_subif_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, 111 uint64_t *paddrs, int nrxqs, int nrxqsets); 112 static int ice_subif_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 113 static int ice_subif_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 114 static void ice_subif_if_intr_enable(if_ctx_t ctx); 115 static int ice_subif_if_msix_intr_assign(if_ctx_t ctx, int msix); 116 static void ice_subif_if_init(if_ctx_t ctx); 117 static void ice_subif_if_stop(if_ctx_t ctx); 118 static void ice_subif_if_queues_free(if_ctx_t ctx); 119 static int ice_subif_if_attach_pre(if_ctx_t); 120 static int ice_subif_if_attach_post(if_ctx_t); 121 static void ice_subif_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 122 static int ice_subif_if_promisc_set(if_ctx_t ctx, int flags); 123 124 static int ice_msix_que(void *arg); 125 static int ice_msix_admin(void *arg); 126 127 /* 128 * Helper function prototypes 129 */ 130 static int ice_pci_mapping(struct ice_softc *sc); 131 static void ice_free_pci_mapping(struct ice_softc *sc); 132 static void ice_update_link_status(struct ice_softc *sc, bool update_media); 133 static void ice_init_device_features(struct ice_softc *sc); 134 static void ice_init_tx_tracking(struct ice_vsi *vsi); 135 static void ice_handle_rdma_pe_intr(struct ice_softc *sc); 136 static void ice_handle_reset_event(struct ice_softc *sc); 137 static void ice_handle_pf_reset_request(struct ice_softc *sc); 138 static void ice_prepare_for_reset(struct ice_softc *sc); 139 static int ice_rebuild_pf_vsi_qmap(struct ice_softc *sc); 140 static void ice_rebuild(struct ice_softc *sc); 141 static void ice_rebuild_recovery_mode(struct ice_softc *sc); 142 static void ice_free_irqvs(struct ice_softc *sc); 143 static void ice_update_rx_mbuf_sz(struct ice_softc *sc); 144 static void ice_poll_for_media_avail(struct ice_softc *sc); 145 static void ice_led_restore(struct ice_softc *sc); 146 static void ice_setup_scctx(struct ice_softc *sc); 147 static int ice_allocate_msix(struct ice_softc *sc); 148 static void ice_admin_timer(void *arg); 149 static void ice_transition_recovery_mode(struct ice_softc *sc); 150 static void ice_transition_safe_mode(struct ice_softc *sc); 151 static void ice_set_default_promisc_mask(ice_bitmap_t *promisc_mask); 152 153 /* 154 * Device Interface Declaration 155 */ 156 157 /** 158 * @var ice_methods 159 * @brief ice driver method entry points 160 * 161 * List of device methods implementing the generic device interface used by 162 * the device stack to interact with the ice driver. Since this is an iflib 163 * driver, most of the methods point to the generic iflib implementation. 164 */ 165 static device_method_t ice_methods[] = { 166 /* Device interface */ 167 DEVMETHOD(device_register, ice_register), 168 DEVMETHOD(device_probe, iflib_device_probe_vendor), 169 DEVMETHOD(device_attach, iflib_device_attach), 170 DEVMETHOD(device_detach, iflib_device_detach), 171 DEVMETHOD(device_shutdown, iflib_device_shutdown), 172 DEVMETHOD(device_suspend, iflib_device_suspend), 173 DEVMETHOD(device_resume, iflib_device_resume), 174 #ifdef PCI_IOV 175 DEVMETHOD(pci_iov_init, iflib_device_iov_init), 176 DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit), 177 DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), 178 #endif 179 DEVMETHOD_END 180 }; 181 182 /** 183 * @var ice_iflib_methods 184 * @brief iflib method entry points 185 * 186 * List of device methods used by the iflib stack to interact with this 187 * driver. These are the real main entry points used to interact with this 188 * driver. 189 */ 190 static device_method_t ice_iflib_methods[] = { 191 DEVMETHOD(ifdi_attach_pre, ice_if_attach_pre), 192 DEVMETHOD(ifdi_attach_post, ice_if_attach_post), 193 DEVMETHOD(ifdi_detach, ice_if_detach), 194 DEVMETHOD(ifdi_tx_queues_alloc, ice_if_tx_queues_alloc), 195 DEVMETHOD(ifdi_rx_queues_alloc, ice_if_rx_queues_alloc), 196 DEVMETHOD(ifdi_msix_intr_assign, ice_if_msix_intr_assign), 197 DEVMETHOD(ifdi_queues_free, ice_if_queues_free), 198 DEVMETHOD(ifdi_mtu_set, ice_if_mtu_set), 199 DEVMETHOD(ifdi_intr_enable, ice_if_intr_enable), 200 DEVMETHOD(ifdi_intr_disable, ice_if_intr_disable), 201 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_if_rx_queue_intr_enable), 202 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_if_tx_queue_intr_enable), 203 DEVMETHOD(ifdi_promisc_set, ice_if_promisc_set), 204 DEVMETHOD(ifdi_media_status, ice_if_media_status), 205 DEVMETHOD(ifdi_media_change, ice_if_media_change), 206 DEVMETHOD(ifdi_init, ice_if_init), 207 DEVMETHOD(ifdi_stop, ice_if_stop), 208 DEVMETHOD(ifdi_led_func, ice_if_led_func), 209 DEVMETHOD(ifdi_timer, ice_if_timer), 210 DEVMETHOD(ifdi_update_admin_status, ice_if_update_admin_status), 211 DEVMETHOD(ifdi_multi_set, ice_if_multi_set), 212 DEVMETHOD(ifdi_vlan_register, ice_if_vlan_register), 213 DEVMETHOD(ifdi_vlan_unregister, ice_if_vlan_unregister), 214 DEVMETHOD(ifdi_get_counter, ice_if_get_counter), 215 DEVMETHOD(ifdi_priv_ioctl, ice_if_priv_ioctl), 216 DEVMETHOD(ifdi_i2c_req, ice_if_i2c_req), 217 DEVMETHOD(ifdi_suspend, ice_if_suspend), 218 DEVMETHOD(ifdi_resume, ice_if_resume), 219 DEVMETHOD(ifdi_needs_restart, ice_if_needs_restart), 220 #ifdef PCI_IOV 221 DEVMETHOD(ifdi_iov_vf_add, ice_if_iov_vf_add), 222 DEVMETHOD(ifdi_iov_init, ice_if_iov_init), 223 DEVMETHOD(ifdi_iov_uninit, ice_if_iov_uninit), 224 DEVMETHOD(ifdi_vf_status, ice_if_vf_status), 225 DEVMETHOD(ifdi_vflr_handle, ice_if_vflr_handle), 226 #endif 227 DEVMETHOD_END 228 }; 229 230 /** 231 * @var ice_driver 232 * @brief driver structure for the generic device stack 233 * 234 * driver_t definition used to setup the generic device methods. 235 */ 236 static driver_t ice_driver = { 237 .name = "ice", 238 .methods = ice_methods, 239 .size = sizeof(struct ice_softc), 240 }; 241 242 /** 243 * @var ice_iflib_driver 244 * @brief driver structure for the iflib stack 245 * 246 * driver_t definition used to setup the iflib device methods. 247 */ 248 static driver_t ice_iflib_driver = { 249 .name = "ice", 250 .methods = ice_iflib_methods, 251 .size = sizeof(struct ice_softc), 252 }; 253 254 extern struct if_txrx ice_txrx; 255 extern struct if_txrx ice_recovery_txrx; 256 257 /** 258 * @var ice_sctx 259 * @brief ice driver shared context 260 * 261 * Structure defining shared values (context) that is used by all instances of 262 * the device. Primarily used to setup details about how the iflib stack 263 * should treat this driver. Also defines the default, minimum, and maximum 264 * number of descriptors in each ring. 265 */ 266 static struct if_shared_ctx ice_sctx = { 267 .isc_magic = IFLIB_MAGIC, 268 .isc_q_align = PAGE_SIZE, 269 270 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 271 /* We could technically set this as high as ICE_MAX_DMA_SEG_SIZE, but 272 * that doesn't make sense since that would be larger than the maximum 273 * size of a single packet. 274 */ 275 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 276 277 /* XXX: This is only used by iflib to ensure that 278 * scctx->isc_tx_tso_size_max + the VLAN header is a valid size. 279 */ 280 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 281 /* XXX: This is used by iflib to set the number of segments in the TSO 282 * DMA tag. However, scctx->isc_tx_tso_segsize_max is used to set the 283 * related ifnet parameter. 284 */ 285 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 286 287 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 288 .isc_rx_nsegments = ICE_MAX_RX_SEGS, 289 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 290 291 .isc_nfl = 1, 292 .isc_ntxqs = 1, 293 .isc_nrxqs = 1, 294 295 .isc_admin_intrcnt = 1, 296 .isc_vendor_info = ice_vendor_info_array, 297 .isc_driver_version = __DECONST(char *, ice_driver_version), 298 .isc_driver = &ice_iflib_driver, 299 300 /* 301 * IFLIB_NEED_SCRATCH ensures that mbufs have scratch space available 302 * for hardware checksum offload 303 * 304 * IFLIB_TSO_INIT_IP ensures that the TSO packets have zeroed out the 305 * IP sum field, required by our hardware to calculate valid TSO 306 * checksums. 307 * 308 * IFLIB_ADMIN_ALWAYS_RUN ensures that the administrative task runs 309 * even when the interface is down. 310 * 311 * IFLIB_SKIP_MSIX allows the driver to handle allocating MSI-X 312 * vectors manually instead of relying on iflib code to do this. 313 */ 314 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 315 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX, 316 317 .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 318 .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 319 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 320 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 321 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 322 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 323 }; 324 325 DRIVER_MODULE(ice, pci, ice_driver, ice_module_event_handler, NULL); 326 327 MODULE_VERSION(ice, 1); 328 MODULE_DEPEND(ice, pci, 1, 1, 1); 329 MODULE_DEPEND(ice, ether, 1, 1, 1); 330 MODULE_DEPEND(ice, iflib, 1, 1, 1); 331 332 IFLIB_PNP_INFO(pci, ice, ice_vendor_info_array); 333 334 /* Static driver-wide sysctls */ 335 #include "ice_iflib_sysctls.h" 336 337 /** 338 * ice_pci_mapping - Map PCI BAR memory 339 * @sc: device private softc 340 * 341 * Map PCI BAR 0 for device operation. 342 */ 343 static int 344 ice_pci_mapping(struct ice_softc *sc) 345 { 346 int rc; 347 348 /* Map BAR0 */ 349 rc = ice_map_bar(sc->dev, &sc->bar0, 0); 350 if (rc) 351 return rc; 352 353 return 0; 354 } 355 356 /** 357 * ice_free_pci_mapping - Release PCI BAR memory 358 * @sc: device private softc 359 * 360 * Release PCI BARs which were previously mapped by ice_pci_mapping(). 361 */ 362 static void 363 ice_free_pci_mapping(struct ice_softc *sc) 364 { 365 /* Free BAR0 */ 366 ice_free_bar(sc->dev, &sc->bar0); 367 } 368 369 /* 370 * Device methods 371 */ 372 373 /** 374 * ice_register - register device method callback 375 * @dev: the device being registered 376 * 377 * Returns a pointer to the shared context structure, which is used by iflib. 378 */ 379 static void * 380 ice_register(device_t dev __unused) 381 { 382 return &ice_sctx; 383 } /* ice_register */ 384 385 /** 386 * ice_setup_scctx - Setup the iflib softc context structure 387 * @sc: the device private structure 388 * 389 * Setup the parameters in if_softc_ctx_t structure used by the iflib stack 390 * when loading. 391 */ 392 static void 393 ice_setup_scctx(struct ice_softc *sc) 394 { 395 if_softc_ctx_t scctx = sc->scctx; 396 struct ice_hw *hw = &sc->hw; 397 device_t dev = sc->dev; 398 bool safe_mode, recovery_mode; 399 400 safe_mode = ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE); 401 recovery_mode = ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE); 402 403 /* 404 * If the driver loads in Safe mode or Recovery mode, limit iflib to 405 * a single queue pair. 406 */ 407 if (safe_mode || recovery_mode) { 408 scctx->isc_ntxqsets = scctx->isc_nrxqsets = 1; 409 scctx->isc_ntxqsets_max = 1; 410 scctx->isc_nrxqsets_max = 1; 411 } else { 412 /* 413 * iflib initially sets the isc_ntxqsets and isc_nrxqsets to 414 * the values of the override sysctls. Cache these initial 415 * values so that the driver can be aware of what the iflib 416 * sysctl value is when setting up MSI-X vectors. 417 */ 418 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets; 419 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets; 420 421 if (scctx->isc_ntxqsets == 0) 422 scctx->isc_ntxqsets = hw->func_caps.common_cap.rss_table_size; 423 if (scctx->isc_nrxqsets == 0) 424 scctx->isc_nrxqsets = hw->func_caps.common_cap.rss_table_size; 425 426 scctx->isc_ntxqsets_max = hw->func_caps.common_cap.num_txq; 427 scctx->isc_nrxqsets_max = hw->func_caps.common_cap.num_rxq; 428 429 /* 430 * Sanity check that the iflib sysctl values are within the 431 * maximum supported range. 432 */ 433 if (sc->ifc_sysctl_ntxqs > scctx->isc_ntxqsets_max) 434 sc->ifc_sysctl_ntxqs = scctx->isc_ntxqsets_max; 435 if (sc->ifc_sysctl_nrxqs > scctx->isc_nrxqsets_max) 436 sc->ifc_sysctl_nrxqs = scctx->isc_nrxqsets_max; 437 } 438 439 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 440 * sizeof(struct ice_tx_desc), DBA_ALIGN); 441 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 442 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 443 444 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 445 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 446 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 447 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 448 449 scctx->isc_msix_bar = pci_msix_table_bar(dev); 450 scctx->isc_rss_table_size = hw->func_caps.common_cap.rss_table_size; 451 452 /* 453 * If the driver loads in recovery mode, disable Tx/Rx functionality 454 */ 455 if (recovery_mode) 456 scctx->isc_txrx = &ice_recovery_txrx; 457 else 458 scctx->isc_txrx = &ice_txrx; 459 460 /* 461 * If the driver loads in Safe mode or Recovery mode, disable 462 * advanced features including hardware offloads. 463 */ 464 if (safe_mode || recovery_mode) { 465 scctx->isc_capenable = ICE_SAFE_CAPS; 466 scctx->isc_tx_csum_flags = 0; 467 } else { 468 scctx->isc_capenable = ICE_FULL_CAPS; 469 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 470 } 471 472 scctx->isc_capabilities = scctx->isc_capenable; 473 } /* ice_setup_scctx */ 474 475 /** 476 * ice_if_attach_pre - Early device attach logic 477 * @ctx: the iflib context structure 478 * 479 * Called by iflib during the attach process. Earliest main driver entry 480 * point which performs necessary hardware and driver initialization. Called 481 * before the Tx and Rx queues are allocated. 482 */ 483 static int 484 ice_if_attach_pre(if_ctx_t ctx) 485 { 486 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 487 enum ice_fw_modes fw_mode; 488 int status; 489 if_softc_ctx_t scctx; 490 struct ice_hw *hw; 491 device_t dev; 492 int err; 493 494 device_printf(iflib_get_dev(ctx), "Loading the iflib ice driver\n"); 495 496 ice_set_state(&sc->state, ICE_STATE_ATTACHING); 497 498 sc->ctx = ctx; 499 sc->media = iflib_get_media(ctx); 500 sc->sctx = iflib_get_sctx(ctx); 501 sc->iflib_ctx_lock = iflib_ctx_lock_get(ctx); 502 sc->ifp = iflib_get_ifp(ctx); 503 504 dev = sc->dev = iflib_get_dev(ctx); 505 scctx = sc->scctx = iflib_get_softc_ctx(ctx); 506 507 hw = &sc->hw; 508 hw->back = sc; 509 510 snprintf(sc->admin_mtx_name, sizeof(sc->admin_mtx_name), 511 "%s:admin", device_get_nameunit(dev)); 512 mtx_init(&sc->admin_mtx, sc->admin_mtx_name, NULL, MTX_DEF); 513 callout_init_mtx(&sc->admin_timer, &sc->admin_mtx, 0); 514 515 ASSERT_CTX_LOCKED(sc); 516 517 if (ice_pci_mapping(sc)) { 518 err = (ENXIO); 519 goto destroy_admin_timer; 520 } 521 522 /* Save off the PCI information */ 523 ice_save_pci_info(hw, dev); 524 525 /* create tunables as early as possible */ 526 ice_add_device_tunables(sc); 527 528 /* Setup ControlQ lengths */ 529 ice_set_ctrlq_len(hw); 530 531 reinit_hw: 532 533 fw_mode = ice_get_fw_mode(hw); 534 if (fw_mode == ICE_FW_MODE_REC) { 535 device_printf(dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 536 537 err = ice_attach_pre_recovery_mode(sc); 538 if (err) 539 goto free_pci_mapping; 540 541 return (0); 542 } 543 544 /* Initialize the hw data structure */ 545 status = ice_init_hw(hw); 546 if (status) { 547 if (status == ICE_ERR_FW_API_VER) { 548 /* Enter recovery mode, so that the driver remains 549 * loaded. This way, if the system administrator 550 * cannot update the driver, they may still attempt to 551 * downgrade the NVM. 552 */ 553 err = ice_attach_pre_recovery_mode(sc); 554 if (err) 555 goto free_pci_mapping; 556 557 return (0); 558 } else { 559 err = EIO; 560 device_printf(dev, "Unable to initialize hw, err %s aq_err %s\n", 561 ice_status_str(status), 562 ice_aq_str(hw->adminq.sq_last_status)); 563 } 564 goto free_pci_mapping; 565 } 566 567 ice_init_device_features(sc); 568 569 /* Keep flag set by default */ 570 ice_set_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN); 571 572 /* Notify firmware of the device driver version */ 573 err = ice_send_version(sc); 574 if (err) 575 goto deinit_hw; 576 577 /* 578 * Success indicates a change was made that requires a reinitialization 579 * of the hardware 580 */ 581 err = ice_load_pkg_file(sc); 582 if (!err) { 583 ice_deinit_hw(hw); 584 goto reinit_hw; 585 } 586 587 err = ice_init_link_events(sc); 588 if (err) { 589 device_printf(dev, "ice_init_link_events failed: %s\n", 590 ice_err_str(err)); 591 goto deinit_hw; 592 } 593 594 /* Initialize VLAN mode in FW; if dual VLAN mode is supported by the package 595 * and firmware, this will force them to use single VLAN mode. 596 */ 597 status = ice_set_vlan_mode(hw); 598 if (status) { 599 err = EIO; 600 device_printf(dev, "Unable to initialize VLAN mode, err %s aq_err %s\n", 601 ice_status_str(status), 602 ice_aq_str(hw->adminq.sq_last_status)); 603 goto deinit_hw; 604 } 605 606 ice_print_nvm_version(sc); 607 608 /* Setup the MAC address */ 609 iflib_set_mac(ctx, hw->port_info->mac.lan_addr); 610 611 /* Setup the iflib softc context structure */ 612 ice_setup_scctx(sc); 613 614 /* Initialize the Tx queue manager */ 615 err = ice_resmgr_init(&sc->tx_qmgr, hw->func_caps.common_cap.num_txq); 616 if (err) { 617 device_printf(dev, "Unable to initialize Tx queue manager: %s\n", 618 ice_err_str(err)); 619 goto deinit_hw; 620 } 621 622 /* Initialize the Rx queue manager */ 623 err = ice_resmgr_init(&sc->rx_qmgr, hw->func_caps.common_cap.num_rxq); 624 if (err) { 625 device_printf(dev, "Unable to initialize Rx queue manager: %s\n", 626 ice_err_str(err)); 627 goto free_tx_qmgr; 628 } 629 630 /* Initialize the PF device interrupt resource manager */ 631 err = ice_alloc_intr_tracking(sc); 632 if (err) 633 /* Errors are already printed */ 634 goto free_rx_qmgr; 635 636 /* Determine maximum number of VSIs we'll prepare for */ 637 sc->num_available_vsi = min(ICE_MAX_VSI_AVAILABLE, 638 hw->func_caps.guar_num_vsi); 639 640 if (!sc->num_available_vsi) { 641 err = EIO; 642 device_printf(dev, "No VSIs allocated to host\n"); 643 goto free_intr_tracking; 644 } 645 646 /* Allocate storage for the VSI pointers */ 647 sc->all_vsi = (struct ice_vsi **) 648 malloc(sizeof(struct ice_vsi *) * sc->num_available_vsi, 649 M_ICE, M_WAITOK | M_ZERO); 650 if (!sc->all_vsi) { 651 err = ENOMEM; 652 device_printf(dev, "Unable to allocate VSI array\n"); 653 goto free_intr_tracking; 654 } 655 656 /* 657 * Prepare the statically allocated primary PF VSI in the softc 658 * structure. Other VSIs will be dynamically allocated as needed. 659 */ 660 ice_setup_pf_vsi(sc); 661 662 ice_alloc_vsi_qmap(&sc->pf_vsi, scctx->isc_ntxqsets_max, 663 scctx->isc_nrxqsets_max); 664 665 /* Allocate MSI-X vectors (due to isc_flags IFLIB_SKIP_MSIX) */ 666 err = ice_allocate_msix(sc); 667 if (err) 668 goto free_main_vsi; 669 670 return 0; 671 672 free_main_vsi: 673 /* ice_release_vsi will free the queue maps if they were allocated */ 674 ice_release_vsi(&sc->pf_vsi); 675 free(sc->all_vsi, M_ICE); 676 sc->all_vsi = NULL; 677 free_intr_tracking: 678 ice_free_intr_tracking(sc); 679 free_rx_qmgr: 680 ice_resmgr_destroy(&sc->rx_qmgr); 681 free_tx_qmgr: 682 ice_resmgr_destroy(&sc->tx_qmgr); 683 deinit_hw: 684 ice_deinit_hw(hw); 685 free_pci_mapping: 686 ice_free_pci_mapping(sc); 687 destroy_admin_timer: 688 mtx_lock(&sc->admin_mtx); 689 callout_stop(&sc->admin_timer); 690 mtx_unlock(&sc->admin_mtx); 691 mtx_destroy(&sc->admin_mtx); 692 return err; 693 } /* ice_if_attach_pre */ 694 695 /** 696 * ice_attach_pre_recovery_mode - Limited driver attach_pre for FW recovery 697 * @sc: the device private softc 698 * 699 * Loads the device driver in limited Firmware Recovery mode, intended to 700 * allow users to update the firmware to attempt to recover the device. 701 * 702 * @remark We may enter recovery mode in case either (a) the firmware is 703 * detected to be in an invalid state and must be re-programmed, or (b) the 704 * driver detects that the loaded firmware has a non-compatible API version 705 * that the driver cannot operate with. 706 */ 707 static int 708 ice_attach_pre_recovery_mode(struct ice_softc *sc) 709 { 710 ice_set_state(&sc->state, ICE_STATE_RECOVERY_MODE); 711 712 /* Setup the iflib softc context */ 713 ice_setup_scctx(sc); 714 715 /* Setup the PF VSI back pointer */ 716 sc->pf_vsi.sc = sc; 717 718 /* 719 * We still need to allocate MSI-X vectors since we need one vector to 720 * run the administrative admin interrupt 721 */ 722 return ice_allocate_msix(sc); 723 } 724 725 /** 726 * ice_update_link_status - notify OS of link state change 727 * @sc: device private softc structure 728 * @update_media: true if we should update media even if link didn't change 729 * 730 * Called to notify iflib core of link status changes. Should be called once 731 * during attach_post, and whenever link status changes during runtime. 732 * 733 * This call only updates the currently supported media types if the link 734 * status changed, or if update_media is set to true. 735 */ 736 static void 737 ice_update_link_status(struct ice_softc *sc, bool update_media) 738 { 739 struct ice_hw *hw = &sc->hw; 740 int status; 741 742 /* Never report link up when in recovery mode */ 743 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 744 return; 745 746 /* Report link status to iflib only once each time it changes */ 747 if (!ice_testandset_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED)) { 748 if (sc->link_up) { /* link is up */ 749 uint64_t baudrate = ice_aq_speed_to_rate(sc->hw.port_info); 750 751 if (!(hw->port_info->phy.link_info_old.link_info & ICE_AQ_LINK_UP)) 752 ice_set_default_local_lldp_mib(sc); 753 754 iflib_link_state_change(sc->ctx, LINK_STATE_UP, baudrate); 755 ice_rdma_link_change(sc, LINK_STATE_UP, baudrate); 756 757 ice_link_up_msg(sc); 758 } else { /* link is down */ 759 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 760 ice_rdma_link_change(sc, LINK_STATE_DOWN, 0); 761 } 762 #ifdef PCI_IOV 763 ice_vc_notify_all_vfs_link_state(sc); 764 #endif 765 update_media = true; 766 } 767 768 /* Update the supported media types */ 769 if (update_media && !ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 770 status = ice_add_media_types(sc, sc->media); 771 if (status) 772 device_printf(sc->dev, "Error adding device media types: %s aq_err %s\n", 773 ice_status_str(status), 774 ice_aq_str(hw->adminq.sq_last_status)); 775 } 776 } 777 778 /** 779 * ice_if_attach_post - Late device attach logic 780 * @ctx: the iflib context structure 781 * 782 * Called by iflib to finish up attaching the device. Performs any attach 783 * logic which must wait until after the Tx and Rx queues have been 784 * allocated. 785 */ 786 static int 787 ice_if_attach_post(if_ctx_t ctx) 788 { 789 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 790 if_t ifp = iflib_get_ifp(ctx); 791 int status; 792 int err; 793 794 ASSERT_CTX_LOCKED(sc); 795 796 /* We don't yet support loading if MSI-X is not supported */ 797 if (sc->scctx->isc_intr != IFLIB_INTR_MSIX) { 798 device_printf(sc->dev, "The ice driver does not support loading without MSI-X\n"); 799 return (ENOTSUP); 800 } 801 802 /* The ifnet structure hasn't yet been initialized when the attach_pre 803 * handler is called, so wait until attach_post to setup the 804 * isc_max_frame_size. 805 */ 806 sc->scctx->isc_max_frame_size = if_getmtu(ifp) + 807 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 808 809 /* 810 * If we are in recovery mode, only perform a limited subset of 811 * initialization to support NVM recovery. 812 */ 813 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 814 ice_attach_post_recovery_mode(sc); 815 return (0); 816 } 817 818 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 819 820 err = ice_initialize_vsi(&sc->pf_vsi); 821 if (err) { 822 device_printf(sc->dev, "Unable to initialize Main VSI: %s\n", 823 ice_err_str(err)); 824 return err; 825 } 826 827 /* Enable FW health event reporting */ 828 ice_init_health_events(sc); 829 830 /* Configure the main PF VSI for RSS */ 831 err = ice_config_rss(&sc->pf_vsi); 832 if (err) { 833 device_printf(sc->dev, 834 "Unable to configure RSS for the main VSI, err %s\n", 835 ice_err_str(err)); 836 return err; 837 } 838 839 /* Configure switch to drop transmitted LLDP and PAUSE frames */ 840 err = ice_cfg_pf_ethertype_filters(sc); 841 if (err) 842 return err; 843 844 ice_get_and_print_bus_info(sc); 845 846 ice_set_link_management_mode(sc); 847 848 ice_init_saved_phy_cfg(sc); 849 850 ice_cfg_pba_num(sc); 851 852 /* Set a default value for PFC mode on attach since the FW state is unknown 853 * before sysctl tunables are executed and it can't be queried. This fixes an 854 * issue when loading the driver with the FW LLDP agent enabled but the FW 855 * was previously in DSCP PFC mode. 856 */ 857 status = ice_aq_set_pfc_mode(&sc->hw, ICE_AQC_PFC_VLAN_BASED_PFC, NULL); 858 if (status) 859 device_printf(sc->dev, "Setting pfc mode failed, status %s\n", ice_status_str(status)); 860 861 ice_add_device_sysctls(sc); 862 863 #ifdef PCI_IOV 864 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_SRIOV)) { 865 err = ice_iov_attach(sc); 866 if (err == ENOMEM) 867 return (err); 868 } 869 #endif /* PCI_IOV */ 870 871 /* Get DCBX/LLDP state and start DCBX agent */ 872 ice_init_dcb_setup(sc); 873 874 /* Setup link, if PHY FW is ready */ 875 ice_init_link(sc); 876 877 /* Configure interrupt causes for the administrative interrupt */ 878 ice_configure_misc_interrupts(sc); 879 880 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 881 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 882 883 err = ice_rdma_pf_attach(sc); 884 if (err) 885 return (err); 886 887 /* Start the admin timer */ 888 mtx_lock(&sc->admin_mtx); 889 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 890 mtx_unlock(&sc->admin_mtx); 891 892 if (ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && 893 !ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) 894 ice_set_state(&sc->state, ICE_STATE_FIRST_INIT_LINK); 895 896 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 897 898 return 0; 899 } /* ice_if_attach_post */ 900 901 /** 902 * ice_attach_post_recovery_mode - Limited driver attach_post for FW recovery 903 * @sc: the device private softc 904 * 905 * Performs minimal work to prepare the driver to recover an NVM in case the 906 * firmware is in recovery mode. 907 */ 908 static void 909 ice_attach_post_recovery_mode(struct ice_softc *sc) 910 { 911 /* Configure interrupt causes for the administrative interrupt */ 912 ice_configure_misc_interrupts(sc); 913 914 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 915 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 916 917 /* Start the admin timer */ 918 mtx_lock(&sc->admin_mtx); 919 callout_reset(&sc->admin_timer, hz/2, ice_admin_timer, sc); 920 mtx_unlock(&sc->admin_mtx); 921 922 ice_clear_state(&sc->state, ICE_STATE_ATTACHING); 923 } 924 925 /** 926 * ice_free_irqvs - Free IRQ vector memory 927 * @sc: the device private softc structure 928 * 929 * Free IRQ vector memory allocated during ice_if_msix_intr_assign. 930 */ 931 static void 932 ice_free_irqvs(struct ice_softc *sc) 933 { 934 struct ice_vsi *vsi = &sc->pf_vsi; 935 if_ctx_t ctx = sc->ctx; 936 int i; 937 938 /* If the irqvs array is NULL, then there are no vectors to free */ 939 if (sc->irqvs == NULL) 940 return; 941 942 /* Free the IRQ vectors */ 943 for (i = 0; i < sc->num_irq_vectors; i++) 944 iflib_irq_free(ctx, &sc->irqvs[i].irq); 945 946 /* Clear the irqv pointers */ 947 for (i = 0; i < vsi->num_rx_queues; i++) 948 vsi->rx_queues[i].irqv = NULL; 949 950 for (i = 0; i < vsi->num_tx_queues; i++) 951 vsi->tx_queues[i].irqv = NULL; 952 953 /* Release the vector array memory */ 954 free(sc->irqvs, M_ICE); 955 sc->irqvs = NULL; 956 sc->num_irq_vectors = 0; 957 } 958 959 /** 960 * ice_if_detach - Device driver detach logic 961 * @ctx: iflib context structure 962 * 963 * Perform device shutdown logic to detach the device driver. 964 * 965 * Note that there is no guarantee of the ordering of ice_if_queues_free() and 966 * ice_if_detach(). It is possible for the functions to be called in either 967 * order, and they must not assume to have a strict ordering. 968 */ 969 static int 970 ice_if_detach(if_ctx_t ctx) 971 { 972 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 973 struct ice_vsi *vsi = &sc->pf_vsi; 974 int status; 975 int i; 976 977 ASSERT_CTX_LOCKED(sc); 978 979 /* Indicate that we're detaching */ 980 ice_set_state(&sc->state, ICE_STATE_DETACHING); 981 982 /* Stop the admin timer */ 983 mtx_lock(&sc->admin_mtx); 984 callout_stop(&sc->admin_timer); 985 mtx_unlock(&sc->admin_mtx); 986 mtx_destroy(&sc->admin_mtx); 987 988 /* Remove additional interfaces if they exist */ 989 if (sc->mirr_if) 990 ice_destroy_mirror_interface(sc); 991 ice_rdma_pf_detach(sc); 992 993 #ifdef PCI_IOV 994 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_SRIOV)) 995 ice_iov_detach(sc); 996 #endif /* PCI_IOV */ 997 998 /* Free allocated media types */ 999 ifmedia_removeall(sc->media); 1000 1001 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1002 * pointers. Note, the calls here and those in ice_if_queues_free() 1003 * are *BOTH* necessary, as we cannot guarantee which path will be 1004 * run first 1005 */ 1006 ice_vsi_del_txqs_ctx(vsi); 1007 ice_vsi_del_rxqs_ctx(vsi); 1008 1009 /* Release MSI-X resources */ 1010 ice_free_irqvs(sc); 1011 1012 for (i = 0; i < sc->num_available_vsi; i++) { 1013 if (sc->all_vsi[i]) 1014 ice_release_vsi(sc->all_vsi[i]); 1015 } 1016 1017 if (sc->all_vsi) { 1018 free(sc->all_vsi, M_ICE); 1019 sc->all_vsi = NULL; 1020 } 1021 1022 /* Release MSI-X memory */ 1023 pci_release_msi(sc->dev); 1024 1025 if (sc->msix_table != NULL) { 1026 bus_release_resource(sc->dev, SYS_RES_MEMORY, 1027 rman_get_rid(sc->msix_table), 1028 sc->msix_table); 1029 sc->msix_table = NULL; 1030 } 1031 1032 ice_free_intr_tracking(sc); 1033 1034 /* Destroy the queue managers */ 1035 ice_resmgr_destroy(&sc->tx_qmgr); 1036 ice_resmgr_destroy(&sc->rx_qmgr); 1037 1038 if (!ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1039 ice_deinit_hw(&sc->hw); 1040 1041 IFLIB_CTX_UNLOCK(sc); 1042 status = ice_reset(&sc->hw, ICE_RESET_PFR); 1043 IFLIB_CTX_LOCK(sc); 1044 if (status) { 1045 device_printf(sc->dev, "device PF reset failed, err %s\n", 1046 ice_status_str(status)); 1047 } 1048 1049 ice_free_pci_mapping(sc); 1050 1051 return 0; 1052 } /* ice_if_detach */ 1053 1054 /** 1055 * ice_if_tx_queues_alloc - Allocate Tx queue memory 1056 * @ctx: iflib context structure 1057 * @vaddrs: virtual addresses for the queue memory 1058 * @paddrs: physical addresses for the queue memory 1059 * @ntxqs: the number of Tx queues per set (should always be 1) 1060 * @ntxqsets: the number of Tx queue sets to allocate 1061 * 1062 * Called by iflib to allocate Tx queues for the device. Allocates driver 1063 * memory to track each queue, the status arrays used for descriptor 1064 * status reporting, and Tx queue sysctls. 1065 */ 1066 static int 1067 ice_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1068 int __invariant_only ntxqs, int ntxqsets) 1069 { 1070 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1071 struct ice_vsi *vsi = &sc->pf_vsi; 1072 struct ice_tx_queue *txq; 1073 int err, i, j; 1074 1075 MPASS(ntxqs == 1); 1076 MPASS(sc->scctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 1077 ASSERT_CTX_LOCKED(sc); 1078 1079 /* Do not bother allocating queues if we're in recovery mode */ 1080 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1081 return (0); 1082 1083 /* Allocate queue structure memory */ 1084 if (!(vsi->tx_queues = 1085 (struct ice_tx_queue *) malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 1086 device_printf(sc->dev, "Unable to allocate Tx queue memory\n"); 1087 return (ENOMEM); 1088 } 1089 1090 /* Allocate report status arrays */ 1091 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1092 if (!(txq->tx_rsq = 1093 (uint16_t *) malloc(sizeof(uint16_t) * sc->scctx->isc_ntxd[0], M_ICE, M_NOWAIT))) { 1094 device_printf(sc->dev, "Unable to allocate tx_rsq memory\n"); 1095 err = ENOMEM; 1096 goto free_tx_queues; 1097 } 1098 /* Initialize report status array */ 1099 for (j = 0; j < sc->scctx->isc_ntxd[0]; j++) 1100 txq->tx_rsq[j] = QIDX_INVALID; 1101 } 1102 1103 /* Assign queues from PF space to the main VSI */ 1104 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, ntxqsets); 1105 if (err) { 1106 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1107 ice_err_str(err)); 1108 goto free_tx_queues; 1109 } 1110 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1111 1112 /* Add Tx queue sysctls context */ 1113 ice_vsi_add_txqs_ctx(vsi); 1114 1115 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1116 /* q_handle == me when only one TC */ 1117 txq->me = txq->q_handle = i; 1118 txq->vsi = vsi; 1119 1120 /* store the queue size for easier access */ 1121 txq->desc_count = sc->scctx->isc_ntxd[0]; 1122 1123 /* get the virtual and physical address of the hardware queues */ 1124 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 1125 txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 1126 txq->tx_paddr = paddrs[i]; 1127 1128 ice_add_txq_sysctls(txq); 1129 } 1130 1131 vsi->num_tx_queues = ntxqsets; 1132 1133 return (0); 1134 1135 free_tx_queues: 1136 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 1137 if (txq->tx_rsq != NULL) { 1138 free(txq->tx_rsq, M_ICE); 1139 txq->tx_rsq = NULL; 1140 } 1141 } 1142 free(vsi->tx_queues, M_ICE); 1143 vsi->tx_queues = NULL; 1144 return err; 1145 } 1146 1147 /** 1148 * ice_if_rx_queues_alloc - Allocate Rx queue memory 1149 * @ctx: iflib context structure 1150 * @vaddrs: virtual addresses for the queue memory 1151 * @paddrs: physical addresses for the queue memory 1152 * @nrxqs: number of Rx queues per set (should always be 1) 1153 * @nrxqsets: number of Rx queue sets to allocate 1154 * 1155 * Called by iflib to allocate Rx queues for the device. Allocates driver 1156 * memory to track each queue, as well as sets up the Rx queue sysctls. 1157 */ 1158 static int 1159 ice_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 1160 int __invariant_only nrxqs, int nrxqsets) 1161 { 1162 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1163 struct ice_vsi *vsi = &sc->pf_vsi; 1164 struct ice_rx_queue *rxq; 1165 int err, i; 1166 1167 MPASS(nrxqs == 1); 1168 MPASS(sc->scctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 1169 ASSERT_CTX_LOCKED(sc); 1170 1171 /* Do not bother allocating queues if we're in recovery mode */ 1172 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1173 return (0); 1174 1175 /* Allocate queue structure memory */ 1176 if (!(vsi->rx_queues = 1177 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 1178 device_printf(sc->dev, "Unable to allocate Rx queue memory\n"); 1179 return (ENOMEM); 1180 } 1181 1182 /* Assign queues from PF space to the main VSI */ 1183 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, nrxqsets); 1184 if (err) { 1185 device_printf(sc->dev, "Unable to assign PF queues: %s\n", 1186 ice_err_str(err)); 1187 goto free_rx_queues; 1188 } 1189 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 1190 1191 /* Add Rx queue sysctls context */ 1192 ice_vsi_add_rxqs_ctx(vsi); 1193 1194 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 1195 rxq->me = i; 1196 rxq->vsi = vsi; 1197 1198 /* store the queue size for easier access */ 1199 rxq->desc_count = sc->scctx->isc_nrxd[0]; 1200 1201 /* get the virtual and physical address of the hardware queues */ 1202 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 1203 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 1204 rxq->rx_paddr = paddrs[i]; 1205 1206 ice_add_rxq_sysctls(rxq); 1207 } 1208 1209 vsi->num_rx_queues = nrxqsets; 1210 1211 return (0); 1212 1213 free_rx_queues: 1214 free(vsi->rx_queues, M_ICE); 1215 vsi->rx_queues = NULL; 1216 return err; 1217 } 1218 1219 /** 1220 * ice_if_queues_free - Free queue memory 1221 * @ctx: the iflib context structure 1222 * 1223 * Free queue memory allocated by ice_if_tx_queues_alloc() and 1224 * ice_if_rx_queues_alloc(). 1225 * 1226 * There is no guarantee that ice_if_queues_free() and ice_if_detach() will be 1227 * called in the same order. It's possible for ice_if_queues_free() to be 1228 * called prior to ice_if_detach(), and vice versa. 1229 * 1230 * For this reason, the main VSI is a static member of the ice_softc, which is 1231 * not free'd until after iflib finishes calling both of these functions. 1232 * 1233 * Thus, care must be taken in how we manage the memory being freed by this 1234 * function, and in what tasks it can and must perform. 1235 */ 1236 static void 1237 ice_if_queues_free(if_ctx_t ctx) 1238 { 1239 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1240 struct ice_vsi *vsi = &sc->pf_vsi; 1241 struct ice_tx_queue *txq; 1242 int i; 1243 1244 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 1245 * pointers. Note, the calls here and those in ice_if_detach() 1246 * are *BOTH* necessary, as we cannot guarantee which path will be 1247 * run first 1248 */ 1249 ice_vsi_del_txqs_ctx(vsi); 1250 ice_vsi_del_rxqs_ctx(vsi); 1251 1252 /* Release MSI-X IRQ vectors, if not yet released in ice_if_detach */ 1253 ice_free_irqvs(sc); 1254 1255 if (vsi->tx_queues != NULL) { 1256 /* free the tx_rsq arrays */ 1257 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 1258 if (txq->tx_rsq != NULL) { 1259 free(txq->tx_rsq, M_ICE); 1260 txq->tx_rsq = NULL; 1261 } 1262 } 1263 free(vsi->tx_queues, M_ICE); 1264 vsi->tx_queues = NULL; 1265 vsi->num_tx_queues = 0; 1266 } 1267 if (vsi->rx_queues != NULL) { 1268 free(vsi->rx_queues, M_ICE); 1269 vsi->rx_queues = NULL; 1270 vsi->num_rx_queues = 0; 1271 } 1272 } 1273 1274 /** 1275 * ice_msix_que - Fast interrupt handler for MSI-X receive queues 1276 * @arg: The Rx queue memory 1277 * 1278 * Interrupt filter function for iflib MSI-X interrupts. Called by iflib when 1279 * an MSI-X interrupt for a given queue is triggered. Currently this just asks 1280 * iflib to schedule the main Rx thread. 1281 */ 1282 static int 1283 ice_msix_que(void *arg) 1284 { 1285 struct ice_rx_queue __unused *rxq = (struct ice_rx_queue *)arg; 1286 1287 /* TODO: dynamic ITR algorithm?? */ 1288 1289 return (FILTER_SCHEDULE_THREAD); 1290 } 1291 1292 /** 1293 * ice_msix_admin - Fast interrupt handler for MSI-X admin interrupt 1294 * @arg: pointer to device softc memory 1295 * 1296 * Called by iflib when an administrative interrupt occurs. Should perform any 1297 * fast logic for handling the interrupt cause, and then indicate whether the 1298 * admin task needs to be queued. 1299 */ 1300 static int 1301 ice_msix_admin(void *arg) 1302 { 1303 struct ice_softc *sc = (struct ice_softc *)arg; 1304 struct ice_hw *hw = &sc->hw; 1305 device_t dev = sc->dev; 1306 u32 oicr; 1307 1308 /* There is no safe way to modify the enabled miscellaneous causes of 1309 * the OICR vector at runtime, as doing so would be prone to race 1310 * conditions. Reading PFINT_OICR will unmask the associated interrupt 1311 * causes and allow future interrupts to occur. The admin interrupt 1312 * vector will not be re-enabled until after we exit this function, 1313 * but any delayed tasks must be resilient against possible "late 1314 * arrival" interrupts that occur while we're already handling the 1315 * task. This is done by using state bits and serializing these 1316 * delayed tasks via the admin status task function. 1317 */ 1318 oicr = rd32(hw, PFINT_OICR); 1319 1320 /* Processing multiple controlq interrupts on a single vector does not 1321 * provide an indication of which controlq triggered the interrupt. 1322 * We might try reading the INTEVENT bit of the respective PFINT_*_CTL 1323 * registers. However, the INTEVENT bit is not guaranteed to be set as 1324 * it gets automatically cleared when the hardware acknowledges the 1325 * interrupt. 1326 * 1327 * This means we don't really have a good indication of whether or 1328 * which controlq triggered this interrupt. We'll just notify the 1329 * admin task that it should check all the controlqs. 1330 */ 1331 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 1332 1333 if (oicr & PFINT_OICR_VFLR_M) { 1334 ice_set_state(&sc->state, ICE_STATE_VFLR_PENDING); 1335 } 1336 1337 if (oicr & PFINT_OICR_MAL_DETECT_M) { 1338 ice_set_state(&sc->state, ICE_STATE_MDD_PENDING); 1339 } 1340 1341 if (oicr & PFINT_OICR_GRST_M) { 1342 u32 reset; 1343 1344 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1345 GLGEN_RSTAT_RESET_TYPE_S; 1346 1347 if (reset == ICE_RESET_CORER) 1348 sc->soft_stats.corer_count++; 1349 else if (reset == ICE_RESET_GLOBR) 1350 sc->soft_stats.globr_count++; 1351 else 1352 sc->soft_stats.empr_count++; 1353 1354 /* There are a couple of bits at play for handling resets. 1355 * First, the ICE_STATE_RESET_OICR_RECV bit is used to 1356 * indicate that the driver has received an OICR with a reset 1357 * bit active, indicating that a CORER/GLOBR/EMPR is about to 1358 * happen. Second, we set hw->reset_ongoing to indicate that 1359 * the hardware is in reset. We will set this back to false as 1360 * soon as the driver has determined that the hardware is out 1361 * of reset. 1362 * 1363 * If the driver wishes to trigger a request, it can set one of 1364 * the ICE_STATE_RESET_*_REQ bits, which will trigger the 1365 * correct type of reset. 1366 */ 1367 if (!ice_testandset_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) { 1368 hw->reset_ongoing = true; 1369 /* 1370 * During the NVM update process, there is a driver reset and link 1371 * goes down and then up. The below if-statement prevents a second 1372 * link flap from occurring in ice_if_init(). 1373 */ 1374 if (if_getflags(sc->ifp) & IFF_UP) 1375 ice_set_state(&sc->state, ICE_STATE_FIRST_INIT_LINK); 1376 } 1377 } 1378 1379 if (oicr & PFINT_OICR_ECC_ERR_M) { 1380 device_printf(dev, "ECC Error detected!\n"); 1381 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1382 } 1383 1384 if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) { 1385 if (oicr & PFINT_OICR_HMC_ERR_M) 1386 /* Log the HMC errors */ 1387 ice_log_hmc_error(hw, dev); 1388 atomic_set_32(&sc->rdma_oicr, oicr); 1389 ice_set_state(&sc->state, ICE_STATE_RDMA_PE_INTR_PENDING); 1390 } 1391 1392 if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { 1393 device_printf(dev, "PCI Exception detected!\n"); 1394 ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 1395 } 1396 1397 return (FILTER_SCHEDULE_THREAD); 1398 } 1399 1400 /** 1401 * ice_allocate_msix - Allocate MSI-X vectors for the interface 1402 * @sc: the device private softc 1403 * 1404 * Map the MSI-X bar, and then request MSI-X vectors in a two-stage process. 1405 * 1406 * First, determine a suitable total number of vectors based on the number 1407 * of CPUs, RSS buckets, the administrative vector, and other demands such as 1408 * RDMA. 1409 * 1410 * Request the desired amount of vectors, and see how many we obtain. If we 1411 * don't obtain as many as desired, reduce the demands by lowering the number 1412 * of requested queues or reducing the demand from other features such as 1413 * RDMA. 1414 * 1415 * @remark This function is required because the driver sets the 1416 * IFLIB_SKIP_MSIX flag indicating that the driver will manage MSI-X vectors 1417 * manually. 1418 * 1419 * @remark This driver will only use MSI-X vectors. If this is not possible, 1420 * neither MSI or legacy interrupts will be tried. 1421 * 1422 * @remark if it exists, os_imgr is initialized here for keeping track of 1423 * the assignments of extra MSIX vectors. 1424 * 1425 * @post on success this function must set the following scctx parameters: 1426 * isc_vectors, isc_nrxqsets, isc_ntxqsets, and isc_intr. 1427 * 1428 * @returns zero on success or an error code on failure. 1429 */ 1430 static int 1431 ice_allocate_msix(struct ice_softc *sc) 1432 { 1433 bool iflib_override_queue_count = false; 1434 if_softc_ctx_t scctx = sc->scctx; 1435 device_t dev = sc->dev; 1436 cpuset_t cpus; 1437 int bar, queues, vectors, requested; 1438 int err = 0; 1439 int rdma; 1440 1441 /* Allocate the MSI-X bar */ 1442 bar = scctx->isc_msix_bar; 1443 sc->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); 1444 if (!sc->msix_table) { 1445 device_printf(dev, "Unable to map MSI-X table\n"); 1446 return (ENOMEM); 1447 } 1448 1449 /* Check if the iflib queue count sysctls have been set */ 1450 if (sc->ifc_sysctl_ntxqs || sc->ifc_sysctl_nrxqs) 1451 iflib_override_queue_count = true; 1452 1453 err = bus_get_cpus(dev, INTR_CPUS, sizeof(cpus), &cpus); 1454 if (err) { 1455 device_printf(dev, "%s: Unable to fetch the CPU list: %s\n", 1456 __func__, ice_err_str(err)); 1457 CPU_COPY(&all_cpus, &cpus); 1458 } 1459 1460 /* Attempt to mimic behavior of iflib_msix_init */ 1461 if (iflib_override_queue_count) { 1462 /* 1463 * If the override sysctls have been set, limit the queues to 1464 * the number of logical CPUs. 1465 */ 1466 queues = mp_ncpus; 1467 } else { 1468 /* 1469 * Otherwise, limit the queue count to the CPUs associated 1470 * with the NUMA node the device is associated with. 1471 */ 1472 queues = CPU_COUNT(&cpus); 1473 } 1474 1475 /* Clamp to the number of RSS buckets */ 1476 queues = imin(queues, rss_getnumbuckets()); 1477 1478 /* 1479 * Clamp the number of queue pairs to the minimum of the requested Tx 1480 * and Rx queues. 1481 */ 1482 queues = imin(queues, sc->ifc_sysctl_ntxqs ?: scctx->isc_ntxqsets); 1483 queues = imin(queues, sc->ifc_sysctl_nrxqs ?: scctx->isc_nrxqsets); 1484 1485 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RDMA)) { 1486 /* 1487 * Choose a number of RDMA vectors based on the number of CPUs 1488 * up to a maximum 1489 */ 1490 rdma = min(CPU_COUNT(&cpus), ICE_RDMA_MAX_MSIX); 1491 1492 /* Further limit by the user configurable tunable */ 1493 rdma = min(rdma, ice_rdma_max_msix); 1494 } else { 1495 rdma = 0; 1496 } 1497 1498 /* 1499 * Determine the number of vectors to request. Note that we also need 1500 * to allocate one vector for administrative tasks. 1501 */ 1502 requested = rdma + queues + 1; 1503 /* Add extra vectors requested by the user for later subinterface 1504 * creation. 1505 */ 1506 if_ctx_t ctx = sc->ctx; 1507 u32 extra_vectors = iflib_get_extra_msix_vectors_sysctl(ctx); 1508 requested += extra_vectors; 1509 1510 vectors = requested; 1511 err = pci_alloc_msix(dev, &vectors); 1512 if (err) { 1513 device_printf(dev, "Failed to allocate %d MSI-X vectors, err %s\n", 1514 vectors, ice_err_str(err)); 1515 goto err_free_msix_table; 1516 } 1517 1518 /* If we don't receive enough vectors, reduce demands */ 1519 if (vectors < requested) { 1520 int diff = requested - vectors; 1521 1522 device_printf(dev, "Requested %d MSI-X vectors, but got only %d\n", 1523 requested, vectors); 1524 1525 diff += extra_vectors; 1526 extra_vectors = 0; 1527 /* 1528 * The OS didn't grant us the requested number of vectors. 1529 * Check to see if we can reduce demands by limiting the 1530 * number of vectors allocated to certain features. 1531 */ 1532 1533 if (rdma >= diff) { 1534 /* Reduce the number of RDMA vectors we reserve */ 1535 rdma -= diff; 1536 diff = 0; 1537 } else { 1538 /* Disable RDMA and reduce the difference */ 1539 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 1540 diff -= rdma; 1541 rdma = 0; 1542 } 1543 1544 /* 1545 * If we still have a difference, we need to reduce the number 1546 * of queue pairs. 1547 * 1548 * However, we still need at least one vector for the admin 1549 * interrupt and one queue pair. 1550 */ 1551 if (queues <= diff) { 1552 device_printf(dev, "Unable to allocate sufficient MSI-X vectors\n"); 1553 err = (ERANGE); 1554 goto err_pci_release_msi; 1555 } 1556 1557 queues -= diff; 1558 } 1559 1560 device_printf(dev, "Using %d Tx and Rx queues\n", queues); 1561 if (rdma) 1562 device_printf(dev, "Reserving %d MSI-X interrupts for iRDMA\n", 1563 rdma); 1564 device_printf(dev, "Using MSI-X interrupts with %d vectors\n", 1565 vectors); 1566 1567 /* Split resulting vectors back into requested splits */ 1568 scctx->isc_vectors = vectors; 1569 scctx->isc_nrxqsets = queues; 1570 scctx->isc_ntxqsets = queues; 1571 scctx->isc_intr = IFLIB_INTR_MSIX; 1572 1573 sc->irdma_vectors = rdma; 1574 1575 /* Interrupt allocation tracking isn't required in recovery mode, 1576 * since neither RDMA nor VFs are enabled. 1577 */ 1578 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1579 return (0); 1580 1581 /* Keep track of which interrupt indices are being used for what */ 1582 sc->lan_vectors = vectors - rdma; 1583 sc->lan_vectors -= extra_vectors; 1584 err = ice_resmgr_assign_contiguous(&sc->dev_imgr, sc->pf_imap, sc->lan_vectors); 1585 if (err) { 1586 device_printf(dev, "Unable to assign PF interrupt mapping: %s\n", 1587 ice_err_str(err)); 1588 goto err_pci_release_msi; 1589 } 1590 err = ice_resmgr_assign_contiguous(&sc->dev_imgr, sc->rdma_imap, rdma); 1591 if (err) { 1592 device_printf(dev, "Unable to assign PF RDMA interrupt mapping: %s\n", 1593 ice_err_str(err)); 1594 goto err_release_pf_imap; 1595 } 1596 sc->extra_vectors = extra_vectors; 1597 /* Setup another resource manager to track the assignments of extra OS 1598 * vectors. These OS interrupt allocations don't need to be contiguous, 1599 * unlike the ones that come from the device. 1600 */ 1601 err = ice_resmgr_init(&sc->os_imgr, sc->extra_vectors); 1602 if (err) { 1603 device_printf(dev, "Unable to initialize OS extra interrupt manager: %s\n", 1604 ice_err_str(err)); 1605 ice_resmgr_release_map(&sc->dev_imgr, sc->rdma_imap, 1606 rdma); 1607 goto err_release_pf_imap; 1608 } 1609 return (0); 1610 1611 err_release_pf_imap: 1612 ice_resmgr_release_map(&sc->dev_imgr, sc->pf_imap, 1613 sc->lan_vectors); 1614 err_pci_release_msi: 1615 pci_release_msi(dev); 1616 err_free_msix_table: 1617 if (sc->msix_table != NULL) { 1618 bus_release_resource(sc->dev, SYS_RES_MEMORY, 1619 rman_get_rid(sc->msix_table), 1620 sc->msix_table); 1621 sc->msix_table = NULL; 1622 } 1623 1624 return (err); 1625 } 1626 1627 /** 1628 * ice_if_msix_intr_assign - Assign MSI-X interrupt vectors to queues 1629 * @ctx: the iflib context structure 1630 * @msix: the number of vectors we were assigned 1631 * 1632 * Called by iflib to assign MSI-X vectors to queues. Currently requires that 1633 * we get at least the same number of vectors as we have queues, and that we 1634 * always have the same number of Tx and Rx queues. 1635 * 1636 * Tx queues use a softirq instead of using their own hardware interrupt. 1637 */ 1638 static int 1639 ice_if_msix_intr_assign(if_ctx_t ctx, int msix) 1640 { 1641 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1642 struct ice_vsi *vsi = &sc->pf_vsi; 1643 int err, i, vector; 1644 1645 ASSERT_CTX_LOCKED(sc); 1646 1647 if (vsi->num_rx_queues != vsi->num_tx_queues) { 1648 device_printf(sc->dev, 1649 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 1650 vsi->num_tx_queues, vsi->num_rx_queues); 1651 return (EOPNOTSUPP); 1652 } 1653 1654 if (msix < (vsi->num_rx_queues + 1)) { 1655 device_printf(sc->dev, 1656 "Not enough MSI-X vectors to assign one vector to each queue pair\n"); 1657 return (EOPNOTSUPP); 1658 } 1659 1660 /* Save the number of vectors for future use */ 1661 sc->num_irq_vectors = vsi->num_rx_queues + 1; 1662 1663 /* Allocate space to store the IRQ vector data */ 1664 if (!(sc->irqvs = 1665 (struct ice_irq_vector *) malloc(sizeof(struct ice_irq_vector) * (sc->num_irq_vectors), 1666 M_ICE, M_NOWAIT))) { 1667 device_printf(sc->dev, 1668 "Unable to allocate irqv memory\n"); 1669 return (ENOMEM); 1670 } 1671 1672 /* Administrative interrupt events will use vector 0 */ 1673 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[0].irq, 1, IFLIB_INTR_ADMIN, 1674 ice_msix_admin, sc, 0, "admin"); 1675 if (err) { 1676 device_printf(sc->dev, 1677 "Failed to register Admin queue handler: %s\n", 1678 ice_err_str(err)); 1679 goto free_irqvs; 1680 } 1681 sc->irqvs[0].me = 0; 1682 1683 /* Do not allocate queue interrupts when in recovery mode */ 1684 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1685 return (0); 1686 1687 int rid; 1688 for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++) { 1689 struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 1690 struct ice_tx_queue *txq = &vsi->tx_queues[i]; 1691 char irq_name[16]; 1692 1693 rid = vector + 1; 1694 1695 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1696 err = iflib_irq_alloc_generic(ctx, &sc->irqvs[vector].irq, rid, 1697 IFLIB_INTR_RXTX, ice_msix_que, 1698 rxq, rxq->me, irq_name); 1699 if (err) { 1700 device_printf(sc->dev, 1701 "Failed to allocate q int %d err: %s\n", 1702 i, ice_err_str(err)); 1703 vector--; 1704 i--; 1705 goto fail; 1706 } 1707 sc->irqvs[vector].me = vector; 1708 rxq->irqv = &sc->irqvs[vector]; 1709 1710 bzero(irq_name, sizeof(irq_name)); 1711 1712 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1713 iflib_softirq_alloc_generic(ctx, &sc->irqvs[vector].irq, 1714 IFLIB_INTR_TX, txq, 1715 txq->me, irq_name); 1716 txq->irqv = &sc->irqvs[vector]; 1717 } 1718 1719 /* For future interrupt assignments */ 1720 sc->last_rid = rid + sc->irdma_vectors; 1721 1722 #ifdef PCI_IOV 1723 /* Create soft IRQ for handling VF resets */ 1724 iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_IOV, sc, 0, "iov"); 1725 #endif 1726 1727 return (0); 1728 fail: 1729 for (; i >= 0; i--, vector--) 1730 iflib_irq_free(ctx, &sc->irqvs[vector].irq); 1731 iflib_irq_free(ctx, &sc->irqvs[0].irq); 1732 free_irqvs: 1733 free(sc->irqvs, M_ICE); 1734 sc->irqvs = NULL; 1735 return err; 1736 } 1737 1738 /** 1739 * ice_if_mtu_set - Set the device MTU 1740 * @ctx: iflib context structure 1741 * @mtu: the MTU requested 1742 * 1743 * Called by iflib to configure the device's Maximum Transmission Unit (MTU). 1744 * 1745 * @pre assumes the caller holds the iflib CTX lock 1746 */ 1747 static int 1748 ice_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 1749 { 1750 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1751 1752 ASSERT_CTX_LOCKED(sc); 1753 1754 /* Do not support configuration when in recovery mode */ 1755 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1756 return (ENOSYS); 1757 1758 if (mtu < ICE_MIN_MTU || mtu > ICE_MAX_MTU) 1759 return (EINVAL); 1760 1761 sc->scctx->isc_max_frame_size = mtu + 1762 ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 1763 1764 sc->pf_vsi.max_frame_size = sc->scctx->isc_max_frame_size; 1765 1766 return (0); 1767 } 1768 1769 /** 1770 * ice_if_intr_enable - Enable device interrupts 1771 * @ctx: iflib context structure 1772 * 1773 * Called by iflib to request enabling device interrupts. 1774 */ 1775 static void 1776 ice_if_intr_enable(if_ctx_t ctx) 1777 { 1778 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1779 struct ice_vsi *vsi = &sc->pf_vsi; 1780 struct ice_hw *hw = &sc->hw; 1781 1782 ASSERT_CTX_LOCKED(sc); 1783 1784 /* Enable ITR 0 */ 1785 ice_enable_intr(hw, sc->irqvs[0].me); 1786 1787 /* Do not enable queue interrupts in recovery mode */ 1788 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1789 return; 1790 1791 /* Enable all queue interrupts */ 1792 for (int i = 0; i < vsi->num_rx_queues; i++) 1793 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 1794 } 1795 1796 /** 1797 * ice_if_intr_disable - Disable device interrupts 1798 * @ctx: iflib context structure 1799 * 1800 * Called by iflib to request disabling device interrupts. 1801 */ 1802 static void 1803 ice_if_intr_disable(if_ctx_t ctx) 1804 { 1805 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1806 struct ice_hw *hw = &sc->hw; 1807 unsigned int i; 1808 1809 ASSERT_CTX_LOCKED(sc); 1810 1811 /* IFDI_INTR_DISABLE may be called prior to interrupts actually being 1812 * assigned to queues. Instead of assuming that the interrupt 1813 * assignment in the rx_queues structure is valid, just disable all 1814 * possible interrupts 1815 * 1816 * Note that we choose not to disable ITR 0 because this handles the 1817 * AdminQ interrupts, and we want to keep processing these even when 1818 * the interface is offline. 1819 */ 1820 for (i = 1; i < hw->func_caps.common_cap.num_msix_vectors; i++) 1821 ice_disable_intr(hw, i); 1822 } 1823 1824 /** 1825 * ice_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 1826 * @ctx: iflib context structure 1827 * @rxqid: the Rx queue to enable 1828 * 1829 * Enable a specific Rx queue interrupt. 1830 * 1831 * This function is not protected by the iflib CTX lock. 1832 */ 1833 static int 1834 ice_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1835 { 1836 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1837 struct ice_vsi *vsi = &sc->pf_vsi; 1838 struct ice_hw *hw = &sc->hw; 1839 1840 /* Do not enable queue interrupts in recovery mode */ 1841 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1842 return (ENOSYS); 1843 1844 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 1845 return (0); 1846 } 1847 1848 /** 1849 * ice_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 1850 * @ctx: iflib context structure 1851 * @txqid: the Tx queue to enable 1852 * 1853 * Enable a specific Tx queue interrupt. 1854 * 1855 * This function is not protected by the iflib CTX lock. 1856 */ 1857 static int 1858 ice_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1859 { 1860 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1861 struct ice_vsi *vsi = &sc->pf_vsi; 1862 struct ice_hw *hw = &sc->hw; 1863 1864 /* Do not enable queue interrupts in recovery mode */ 1865 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1866 return (ENOSYS); 1867 1868 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 1869 return (0); 1870 } 1871 1872 /** 1873 * ice_set_default_promisc_mask - Set default config for promisc settings 1874 * @promisc_mask: bitmask to setup 1875 * 1876 * The ice_(set|clear)_vsi_promisc() function expects a mask of promiscuous 1877 * modes to operate on. The mask used in here is the default one for the 1878 * driver, where promiscuous is enabled/disabled for all types of 1879 * non-VLAN-tagged/VLAN 0 traffic. 1880 */ 1881 static void 1882 ice_set_default_promisc_mask(ice_bitmap_t *promisc_mask) 1883 { 1884 ice_zero_bitmap(promisc_mask, ICE_PROMISC_MAX); 1885 ice_set_bit(ICE_PROMISC_UCAST_TX, promisc_mask); 1886 ice_set_bit(ICE_PROMISC_UCAST_RX, promisc_mask); 1887 ice_set_bit(ICE_PROMISC_MCAST_TX, promisc_mask); 1888 ice_set_bit(ICE_PROMISC_MCAST_RX, promisc_mask); 1889 } 1890 1891 /** 1892 * ice_if_promisc_set - Set device promiscuous mode 1893 * @ctx: iflib context structure 1894 * @flags: promiscuous flags to configure 1895 * 1896 * Called by iflib to configure device promiscuous mode. 1897 * 1898 * @remark Calls to this function will always overwrite the previous setting 1899 */ 1900 static int 1901 ice_if_promisc_set(if_ctx_t ctx, int flags) 1902 { 1903 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1904 struct ice_hw *hw = &sc->hw; 1905 device_t dev = sc->dev; 1906 int status; 1907 bool promisc_enable = flags & IFF_PROMISC; 1908 bool multi_enable = flags & IFF_ALLMULTI; 1909 ice_declare_bitmap(promisc_mask, ICE_PROMISC_MAX); 1910 1911 /* Do not support configuration when in recovery mode */ 1912 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1913 return (ENOSYS); 1914 1915 ice_set_default_promisc_mask(promisc_mask); 1916 1917 if (promisc_enable) { 1918 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1919 promisc_mask, 0); 1920 if (status && status != ICE_ERR_ALREADY_EXISTS) { 1921 device_printf(dev, 1922 "Failed to enable promiscuous mode for " 1923 "PF VSI, err %s aq_err %s\n", 1924 ice_status_str(status), 1925 ice_aq_str(hw->adminq.sq_last_status)); 1926 return (EIO); 1927 } 1928 } else { 1929 status = ice_clear_vsi_promisc(hw, sc->pf_vsi.idx, 1930 promisc_mask, 0); 1931 if (status) { 1932 device_printf(dev, 1933 "Failed to disable promiscuous mode for" 1934 " PF VSI, err %s aq_err %s\n", 1935 ice_status_str(status), 1936 ice_aq_str(hw->adminq.sq_last_status)); 1937 return (EIO); 1938 } 1939 1940 if (multi_enable) { 1941 ice_clear_bit(ICE_PROMISC_UCAST_TX, promisc_mask); 1942 ice_clear_bit(ICE_PROMISC_UCAST_RX, promisc_mask); 1943 status = ice_set_vsi_promisc(hw, sc->pf_vsi.idx, 1944 promisc_mask, 0); 1945 if (status && status != ICE_ERR_ALREADY_EXISTS) { 1946 device_printf(dev, 1947 "Failed to enable allmulti mode " 1948 "for PF VSI, err %s aq_err %s\n", 1949 ice_status_str(status), 1950 ice_aq_str( 1951 hw->adminq.sq_last_status)); 1952 return (EIO); 1953 } 1954 } 1955 } 1956 1957 return (0); 1958 } 1959 1960 /** 1961 * ice_if_media_change - Change device media 1962 * @ctx: device ctx structure 1963 * 1964 * Called by iflib when a media change is requested. This operation is not 1965 * supported by the hardware, so we just return an error code. 1966 */ 1967 static int 1968 ice_if_media_change(if_ctx_t ctx) 1969 { 1970 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1971 1972 device_printf(sc->dev, "Media change is not supported.\n"); 1973 return (ENODEV); 1974 } 1975 1976 /** 1977 * ice_if_media_status - Report current device media 1978 * @ctx: iflib context structure 1979 * @ifmr: ifmedia request structure to update 1980 * 1981 * Updates the provided ifmr with current device media status, including link 1982 * status and media type. 1983 */ 1984 static void 1985 ice_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1986 { 1987 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 1988 struct ice_link_status *li = &sc->hw.port_info->phy.link_info; 1989 1990 ifmr->ifm_status = IFM_AVALID; 1991 ifmr->ifm_active = IFM_ETHER; 1992 1993 /* Never report link up or media types when in recovery mode */ 1994 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 1995 return; 1996 1997 if (!sc->link_up) 1998 return; 1999 2000 ifmr->ifm_status |= IFM_ACTIVE; 2001 ifmr->ifm_active |= IFM_FDX; 2002 2003 if (li->phy_type_low) 2004 ifmr->ifm_active |= ice_get_phy_type_low(li->phy_type_low); 2005 else if (li->phy_type_high) 2006 ifmr->ifm_active |= ice_get_phy_type_high(li->phy_type_high); 2007 else 2008 ifmr->ifm_active |= IFM_UNKNOWN; 2009 2010 /* Report flow control status as well */ 2011 if (li->an_info & ICE_AQ_LINK_PAUSE_TX) 2012 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 2013 if (li->an_info & ICE_AQ_LINK_PAUSE_RX) 2014 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 2015 } 2016 2017 /** 2018 * ice_init_tx_tracking - Initialize Tx queue software tracking values 2019 * @vsi: the VSI to initialize 2020 * 2021 * Initialize Tx queue software tracking values, including the Report Status 2022 * queue, and related software tracking values. 2023 */ 2024 static void 2025 ice_init_tx_tracking(struct ice_vsi *vsi) 2026 { 2027 struct ice_tx_queue *txq; 2028 size_t j; 2029 int i; 2030 2031 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 2032 2033 txq->tx_rs_cidx = txq->tx_rs_pidx = 0; 2034 2035 /* Initialize the last processed descriptor to be the end of 2036 * the ring, rather than the start, so that we avoid an 2037 * off-by-one error in ice_ift_txd_credits_update for the 2038 * first packet. 2039 */ 2040 txq->tx_cidx_processed = txq->desc_count - 1; 2041 2042 for (j = 0; j < txq->desc_count; j++) 2043 txq->tx_rsq[j] = QIDX_INVALID; 2044 } 2045 } 2046 2047 /** 2048 * ice_update_rx_mbuf_sz - Update the Rx buffer size for all queues 2049 * @sc: the device softc 2050 * 2051 * Called to update the Rx queue mbuf_sz parameter for configuring the receive 2052 * buffer sizes when programming hardware. 2053 */ 2054 static void 2055 ice_update_rx_mbuf_sz(struct ice_softc *sc) 2056 { 2057 uint32_t mbuf_sz = iflib_get_rx_mbuf_sz(sc->ctx); 2058 struct ice_vsi *vsi = &sc->pf_vsi; 2059 2060 MPASS(mbuf_sz <= UINT16_MAX); 2061 vsi->mbuf_sz = mbuf_sz; 2062 } 2063 2064 /** 2065 * ice_if_init - Initialize the device 2066 * @ctx: iflib ctx structure 2067 * 2068 * Called by iflib to bring the device up, i.e. ifconfig ice0 up. Initializes 2069 * device filters and prepares the Tx and Rx engines. 2070 * 2071 * @pre assumes the caller holds the iflib CTX lock 2072 */ 2073 static void 2074 ice_if_init(if_ctx_t ctx) 2075 { 2076 struct ice_mirr_if *mif; 2077 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2078 device_t dev = sc->dev; 2079 int err; 2080 2081 ASSERT_CTX_LOCKED(sc); 2082 mif = sc->mirr_if; 2083 2084 /* 2085 * We've seen an issue with 11.3/12.1 where sideband routines are 2086 * called after detach is called. This would call routines after 2087 * if_stop, causing issues with the teardown process. This has 2088 * seemingly been fixed in STABLE snapshots, but it seems like a 2089 * good idea to have this guard here regardless. 2090 */ 2091 if (ice_driver_is_detaching(sc)) 2092 return; 2093 2094 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2095 goto err_init_failed; 2096 2097 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 2098 device_printf(sc->dev, "request to start interface cannot be completed as the device failed to reset\n"); 2099 goto err_init_failed; 2100 } 2101 2102 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 2103 device_printf(sc->dev, "request to start interface while device is prepared for impending reset\n"); 2104 goto err_init_failed; 2105 } 2106 2107 ice_update_rx_mbuf_sz(sc); 2108 2109 /* Update the MAC address... User might use a LAA */ 2110 err = ice_update_laa_mac(sc); 2111 if (err) { 2112 device_printf(dev, 2113 "LAA address change failed, err %s\n", 2114 ice_err_str(err)); 2115 goto err_init_failed; 2116 } 2117 2118 /* Initialize software Tx tracking values */ 2119 ice_init_tx_tracking(&sc->pf_vsi); 2120 2121 err = ice_cfg_vsi_for_tx(&sc->pf_vsi); 2122 if (err) { 2123 device_printf(dev, 2124 "Unable to configure the main VSI for Tx: %s\n", 2125 ice_err_str(err)); 2126 goto err_cleanup_tx; 2127 } 2128 2129 err = ice_cfg_vsi_for_rx(&sc->pf_vsi); 2130 if (err) { 2131 device_printf(dev, 2132 "Unable to configure the main VSI for Rx: %s\n", 2133 ice_err_str(err)); 2134 goto err_cleanup_tx; 2135 } 2136 2137 err = ice_control_all_rx_queues(&sc->pf_vsi, true); 2138 if (err) { 2139 device_printf(dev, 2140 "Unable to enable Rx rings for receive: %s\n", 2141 ice_err_str(err)); 2142 goto err_stop_rx; 2143 } 2144 2145 err = ice_cfg_pf_default_mac_filters(sc); 2146 if (err) { 2147 device_printf(dev, 2148 "Unable to configure default MAC filters: %s\n", 2149 ice_err_str(err)); 2150 goto err_stop_rx; 2151 } 2152 2153 /* We use software interrupts for Tx, so we only program the hardware 2154 * interrupts for Rx. 2155 */ 2156 ice_configure_all_rxq_interrupts(&sc->pf_vsi); 2157 ice_configure_rx_itr(&sc->pf_vsi); 2158 2159 /* Configure promiscuous mode */ 2160 ice_if_promisc_set(ctx, if_getflags(sc->ifp)); 2161 2162 if (!ice_testandclear_state(&sc->state, ICE_STATE_FIRST_INIT_LINK)) 2163 if (!sc->link_up && ((if_getflags(sc->ifp) & IFF_UP) || 2164 ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN))) 2165 ice_set_link(sc, true); 2166 2167 ice_rdma_pf_init(sc); 2168 2169 ice_set_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED); 2170 2171 if (mif != NULL && ice_testandclear_state(&mif->state, 2172 ICE_STATE_SUBIF_NEEDS_REINIT)) { 2173 ice_clear_state(&mif->state, ICE_STATE_DRIVER_INITIALIZED); 2174 iflib_request_reset(mif->subctx); 2175 iflib_admin_intr_deferred(mif->subctx); 2176 } 2177 2178 return; 2179 2180 err_stop_rx: 2181 ice_control_all_rx_queues(&sc->pf_vsi, false); 2182 err_cleanup_tx: 2183 ice_vsi_disable_tx(&sc->pf_vsi); 2184 err_init_failed: 2185 iflib_init_failed(ctx); 2186 } 2187 2188 /** 2189 * ice_poll_for_media_avail - Re-enable link if media is detected 2190 * @sc: device private structure 2191 * 2192 * Intended to be called from the driver's timer function, this function 2193 * sends the Get Link Status AQ command and re-enables HW link if the 2194 * command says that media is available. 2195 * 2196 * If the driver doesn't have the "NO_MEDIA" state set, then this does nothing, 2197 * since media removal events are supposed to be sent to the driver through 2198 * a link status event. 2199 */ 2200 static void 2201 ice_poll_for_media_avail(struct ice_softc *sc) 2202 { 2203 struct ice_hw *hw = &sc->hw; 2204 struct ice_port_info *pi = hw->port_info; 2205 2206 /* E830 only: There's no interrupt for when the PHY FW has finished loading, 2207 * so poll for the status in the media task here if it's previously 2208 * been detected that it's still loading. 2209 */ 2210 if (ice_is_e830(hw) && 2211 ice_test_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING)) { 2212 if (rd32(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_LOADING_M) 2213 ice_clear_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING); 2214 else 2215 return; 2216 } 2217 2218 if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA)) { 2219 pi->phy.get_link_info = true; 2220 ice_get_link_status(pi, &sc->link_up); 2221 2222 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2223 int status; 2224 2225 /* Re-enable link and re-apply user link settings */ 2226 if (ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) || 2227 (if_getflags(sc->ifp) & IFF_UP)) { 2228 ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC); 2229 2230 /* Update the OS about changes in media capability */ 2231 status = ice_add_media_types(sc, sc->media); 2232 if (status) 2233 device_printf(sc->dev, 2234 "Error adding device media types: %s aq_err %s\n", 2235 ice_status_str(status), 2236 ice_aq_str(hw->adminq.sq_last_status)); 2237 } 2238 2239 ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA); 2240 } 2241 } 2242 } 2243 2244 /** 2245 * ice_if_timer - called by iflib periodically 2246 * @ctx: iflib ctx structure 2247 * @qid: the queue this timer was called for 2248 * 2249 * This callback is triggered by iflib periodically. We use it to update the 2250 * hw statistics. 2251 * 2252 * @remark this function is not protected by the iflib CTX lock. 2253 */ 2254 static void 2255 ice_if_timer(if_ctx_t ctx, uint16_t qid) 2256 { 2257 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2258 uint64_t prev_link_xoff_rx = sc->stats.cur.link_xoff_rx; 2259 2260 if (qid != 0) 2261 return; 2262 2263 /* Do not attempt to update stats when in recovery mode */ 2264 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2265 return; 2266 2267 /* Update device statistics */ 2268 ice_update_pf_stats(sc); 2269 2270 /* 2271 * For proper watchdog management, the iflib stack needs to know if 2272 * we've been paused during the last interval. Check if the 2273 * link_xoff_rx stat changed, and set the isc_pause_frames, if so. 2274 */ 2275 if (sc->stats.cur.link_xoff_rx != prev_link_xoff_rx) 2276 sc->scctx->isc_pause_frames = 1; 2277 2278 /* Update the primary VSI stats */ 2279 ice_update_vsi_hw_stats(&sc->pf_vsi); 2280 2281 /* Update mirror VSI stats */ 2282 if (sc->mirr_if && sc->mirr_if->if_attached) 2283 ice_update_vsi_hw_stats(sc->mirr_if->vsi); 2284 } 2285 2286 /** 2287 * ice_admin_timer - called periodically to trigger the admin task 2288 * @arg: callout(9) argument pointing to the device private softc structure 2289 * 2290 * Timer function used as part of a callout(9) timer that will periodically 2291 * trigger the admin task, even when the interface is down. 2292 * 2293 * @remark this function is not called by iflib and is not protected by the 2294 * iflib CTX lock. 2295 * 2296 * @remark because this is a callout function, it cannot sleep and should not 2297 * attempt taking the iflib CTX lock. 2298 */ 2299 static void 2300 ice_admin_timer(void *arg) 2301 { 2302 struct ice_softc *sc = (struct ice_softc *)arg; 2303 2304 /* 2305 * There is a point where callout routines are no longer 2306 * cancelable. So there exists a window of time where the 2307 * driver enters detach() and tries to cancel the callout, but the 2308 * callout routine has passed the cancellation point. The detach() 2309 * routine is unaware of this and tries to free resources that the 2310 * callout routine needs. So we check for the detach state flag to 2311 * at least shrink the window of opportunity. 2312 */ 2313 if (ice_driver_is_detaching(sc)) 2314 return; 2315 2316 /* Fire off the admin task */ 2317 iflib_admin_intr_deferred(sc->ctx); 2318 2319 /* Reschedule the admin timer */ 2320 callout_schedule(&sc->admin_timer, hz/2); 2321 } 2322 2323 /** 2324 * ice_transition_recovery_mode - Transition to recovery mode 2325 * @sc: the device private softc 2326 * 2327 * Called when the driver detects that the firmware has entered recovery mode 2328 * at run time. 2329 */ 2330 static void 2331 ice_transition_recovery_mode(struct ice_softc *sc) 2332 { 2333 struct ice_vsi *vsi = &sc->pf_vsi; 2334 int i; 2335 2336 device_printf(sc->dev, "Firmware recovery mode detected. Limiting functionality. Refer to Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 2337 2338 /* Tell the stack that the link has gone down */ 2339 iflib_link_state_change(sc->ctx, LINK_STATE_DOWN, 0); 2340 2341 /* Request that the device be re-initialized */ 2342 ice_request_stack_reinit(sc); 2343 2344 ice_rdma_pf_detach(sc); 2345 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2346 2347 #ifdef PCI_IOV 2348 if (ice_test_and_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en)) 2349 ice_iov_detach(sc); 2350 #else 2351 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2352 #endif /* PCI_IOV */ 2353 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2354 2355 ice_vsi_del_txqs_ctx(vsi); 2356 ice_vsi_del_rxqs_ctx(vsi); 2357 2358 for (i = 0; i < sc->num_available_vsi; i++) { 2359 if (sc->all_vsi[i]) 2360 ice_release_vsi(sc->all_vsi[i]); 2361 } 2362 sc->num_available_vsi = 0; 2363 2364 if (sc->all_vsi) { 2365 free(sc->all_vsi, M_ICE); 2366 sc->all_vsi = NULL; 2367 } 2368 2369 /* Destroy the interrupt manager */ 2370 ice_resmgr_destroy(&sc->dev_imgr); 2371 /* Destroy the queue managers */ 2372 ice_resmgr_destroy(&sc->tx_qmgr); 2373 ice_resmgr_destroy(&sc->rx_qmgr); 2374 2375 ice_deinit_hw(&sc->hw); 2376 } 2377 2378 /** 2379 * ice_transition_safe_mode - Transition to safe mode 2380 * @sc: the device private softc 2381 * 2382 * Called when the driver attempts to reload the DDP package during a device 2383 * reset, and the new download fails. If so, we must transition to safe mode 2384 * at run time. 2385 * 2386 * @remark although safe mode normally allocates only a single queue, we can't 2387 * change the number of queues dynamically when using iflib. Due to this, we 2388 * do not attempt to reduce the number of queues. 2389 */ 2390 static void 2391 ice_transition_safe_mode(struct ice_softc *sc) 2392 { 2393 /* Indicate that we are in Safe mode */ 2394 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap); 2395 ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en); 2396 2397 ice_rdma_pf_detach(sc); 2398 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 2399 2400 #ifdef PCI_IOV 2401 if (ice_test_and_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en)) 2402 ice_iov_detach(sc); 2403 #else 2404 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en); 2405 #endif /* PCI_IOV */ 2406 ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 2407 2408 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 2409 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_en); 2410 } 2411 2412 /** 2413 * ice_handle_rdma_pe_intr - Notify RDMA of deferred PE/HMC errors 2414 * @sc: device private softc 2415 * 2416 * Deliver PE and HMC error notifications from the admin task because the 2417 * RDMA notification path takes a sleepable lock. Multiple OICR causes which 2418 * arrive before the task runs are accumulated by the interrupt filter. 2419 */ 2420 static void 2421 ice_handle_rdma_pe_intr(struct ice_softc *sc) 2422 { 2423 u32 oicr; 2424 2425 if (!ice_testandclear_state(&sc->state, 2426 ICE_STATE_RDMA_PE_INTR_PENDING)) 2427 return; 2428 2429 oicr = atomic_readandclear_32(&sc->rdma_oicr); 2430 if (oicr != 0) 2431 ice_rdma_notify_pe_intr(sc, oicr); 2432 } 2433 2434 /** 2435 * ice_if_update_admin_status - update admin status 2436 * @ctx: iflib ctx structure 2437 * 2438 * Called by iflib to update the admin status. For our purposes, this means 2439 * check the adminq, and update the link status. It's ultimately triggered by 2440 * our admin interrupt, or by the ice_if_timer periodically. 2441 * 2442 * @pre assumes the caller holds the iflib CTX lock 2443 */ 2444 static void 2445 ice_if_update_admin_status(if_ctx_t ctx) 2446 { 2447 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 2448 enum ice_fw_modes fw_mode; 2449 bool defer_mailbox = false, reschedule = false; 2450 u32 reg; 2451 u16 pending = 0; 2452 int error; 2453 2454 ASSERT_CTX_LOCKED(sc); 2455 2456 /* Check if the firmware entered recovery mode at run time */ 2457 fw_mode = ice_get_fw_mode(&sc->hw); 2458 if (fw_mode == ICE_FW_MODE_REC) { 2459 if (!ice_testandset_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2460 /* If we just entered recovery mode, log a warning to 2461 * the system administrator and deinit driver state 2462 * that is no longer functional. 2463 */ 2464 ice_transition_recovery_mode(sc); 2465 } 2466 } else if (fw_mode == ICE_FW_MODE_ROLLBACK) { 2467 if (!ice_testandset_state(&sc->state, ICE_STATE_ROLLBACK_MODE)) { 2468 /* Rollback mode isn't fatal, but we don't want to 2469 * repeatedly post a message about it. 2470 */ 2471 ice_print_rollback_msg(&sc->hw); 2472 } 2473 } 2474 2475 /* Notify RDMA before handling a reset it may request. */ 2476 ice_handle_rdma_pe_intr(sc); 2477 2478 /* Handle global reset events */ 2479 ice_handle_reset_event(sc); 2480 2481 /* Handle PF reset requests */ 2482 ice_handle_pf_reset_request(sc); 2483 2484 /* Handle MDD events */ 2485 ice_handle_mdd_event(sc); 2486 2487 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED) || 2488 ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET) || 2489 ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2490 /* 2491 * If we know the control queues are disabled, skip processing 2492 * the control queues entirely. 2493 */ 2494 ; 2495 } else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) { 2496 pending = 0; 2497 ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending); 2498 if (pending > 0) 2499 reschedule = true; 2500 2501 if (ice_is_generic_mac(&sc->hw)) { 2502 pending = 0; 2503 ice_process_ctrlq(sc, ICE_CTL_Q_SB, &pending); 2504 if (pending > 0) 2505 reschedule = true; 2506 } 2507 2508 pending = 0; 2509 error = ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending); 2510 if (error == 0 && pending == 0) { 2511 reg = rd32(&sc->hw, PFINT_MBX_CTL); 2512 if ((reg & PFINT_MBX_CTL_CAUSE_ENA_M) == 0) { 2513 wr32(&sc->hw, PFINT_MBX_CTL, 2514 reg | PFINT_MBX_CTL_CAUSE_ENA_M); 2515 ice_flush(&sc->hw); 2516 /* Events received while masked may not interrupt. */ 2517 pending = (rd32(&sc->hw, sc->hw.mailboxq.rq.head) & 2518 sc->hw.mailboxq.rq.head_mask) != 2519 sc->hw.mailboxq.rq.next_to_clean; 2520 } 2521 } 2522 if (error != 0) { 2523 /* Retry a failed read on the timer, not in a task loop. */ 2524 defer_mailbox = true; 2525 } else if (pending > 0) { 2526 #ifdef PCI_IOV 2527 /* 2528 * Two passes drain one initially full 512-entry mailbox. 2529 * If it remains nonempty, a VF is replenishing it faster 2530 * than this task can drain it. Mask only the mailbox cause 2531 * and let the periodic admin timer schedule bounded work. 2532 */ 2533 if (sc->mbx_admin_passes < 2534 howmany(ICE_MBXQ_LEN, ICE_CTRLQ_WORK_LIMIT)) 2535 sc->mbx_admin_passes++; 2536 if (sc->mbx_admin_passes < 2537 howmany(ICE_MBXQ_LEN, ICE_CTRLQ_WORK_LIMIT)) 2538 reschedule = true; 2539 else 2540 defer_mailbox = true; 2541 #else 2542 reschedule = true; 2543 #endif 2544 } else { 2545 #ifdef PCI_IOV 2546 sc->mbx_admin_passes = 0; 2547 #endif 2548 } 2549 } 2550 2551 /* Poll for link up */ 2552 ice_poll_for_media_avail(sc); 2553 2554 /* Check and update link status */ 2555 ice_update_link_status(sc, false); 2556 2557 #ifdef PCI_IOV 2558 /* 2559 * Schedule VFs' reset handler after global resets 2560 * and other events were processed. 2561 */ 2562 if (ice_testandclear_state(&sc->state, ICE_STATE_VFLR_PENDING)) 2563 iflib_iov_intr_deferred(ctx); 2564 #endif 2565 2566 if (defer_mailbox) { 2567 reg = rd32(&sc->hw, PFINT_MBX_CTL); 2568 wr32(&sc->hw, PFINT_MBX_CTL, 2569 reg & ~PFINT_MBX_CTL_CAUSE_ENA_M); 2570 } 2571 if (reschedule || defer_mailbox) 2572 ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING); 2573 if (reschedule) 2574 iflib_admin_intr_deferred(ctx); 2575 /* Keep OICR and the other control queues live during mailbox deferral. */ 2576 if (!reschedule || defer_mailbox) 2577 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2578 } 2579 2580 /** 2581 * ice_prepare_for_reset - Prepare device for an impending reset 2582 * @sc: The device private softc 2583 * 2584 * Prepare the driver for an impending reset, shutting down VSIs, clearing the 2585 * scheduler setup, and shutting down controlqs. Uses the 2586 * ICE_STATE_PREPARED_FOR_RESET to indicate whether we've already prepared the 2587 * driver for reset or not. 2588 */ 2589 static void 2590 ice_prepare_for_reset(struct ice_softc *sc) 2591 { 2592 struct ice_hw *hw = &sc->hw; 2593 #ifdef PCI_IOV 2594 int error; 2595 #endif 2596 2597 /* If we're already prepared, there's nothing to do */ 2598 if (ice_testandset_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) 2599 return; 2600 2601 log(LOG_INFO, "%s: preparing to reset device logic\n", if_name(sc->ifp)); 2602 2603 /* In recovery mode, hardware is not initialized */ 2604 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 2605 return; 2606 2607 #ifdef PCI_IOV 2608 /* 2609 * A reset already reported by OICR gates DMA in hardware and rejects 2610 * new function resets. Otherwise notify and hold VFs while AdminQ is 2611 * still usable, before releasing any firmware topology. 2612 */ 2613 if (!hw->reset_ongoing) { 2614 error = ice_iov_quiesce_vfs_for_reset(sc); 2615 if (error != 0) 2616 device_printf(sc->dev, 2617 "Failed to quiesce one or more VFs: %d\n", error); 2618 } else { 2619 /* 2620 * Hardware has already gated the VFs. Invalidate their cached 2621 * handshake before dropping CTX_LOCK to wait for reset, even if 2622 * rebuilding later fails before reaching the VF VSIs. 2623 */ 2624 for (int i = 0; i < sc->num_vfs; i++) { 2625 struct ice_vf *vf = &sc->vfs[i]; 2626 2627 if ((atomic_load_acq_32(&vf->vf_flags) & 2628 VF_FLAG_ENABLED) == 0 || vf->vsi == NULL) 2629 continue; 2630 atomic_clear_32(&vf->vf_flags, VF_FLAG_INITIALIZED); 2631 atomic_set_32(&vf->vf_flags, VF_FLAG_REBUILD_REQUIRED); 2632 } 2633 } 2634 #endif 2635 2636 /* Restore identification while the control queues are still usable. */ 2637 ice_led_restore(sc); 2638 2639 /* inform the RDMA client */ 2640 ice_rdma_notify_reset(sc); 2641 /* stop the RDMA client */ 2642 ice_rdma_pf_stop(sc); 2643 2644 /* Release the main PF VSI queue mappings */ 2645 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2646 sc->pf_vsi.num_tx_queues); 2647 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2648 sc->pf_vsi.num_rx_queues); 2649 if (sc->mirr_if) { 2650 ice_resmgr_release_map(&sc->tx_qmgr, sc->mirr_if->vsi->tx_qmap, 2651 sc->mirr_if->num_irq_vectors); 2652 ice_resmgr_release_map(&sc->rx_qmgr, sc->mirr_if->vsi->rx_qmap, 2653 sc->mirr_if->num_irq_vectors); 2654 } 2655 2656 ice_clear_hw_tbls(hw); 2657 2658 if (hw->port_info) 2659 ice_sched_cleanup_all(hw); 2660 2661 ice_shutdown_all_ctrlq(hw, false); 2662 } 2663 2664 /** 2665 * ice_rebuild_pf_vsi_qmap - Rebuild the main PF VSI queue mapping 2666 * @sc: the device softc pointer 2667 * 2668 * Loops over the Tx and Rx queues for the main PF VSI and reassigns the queue 2669 * mapping after a reset occurred. 2670 */ 2671 static int 2672 ice_rebuild_pf_vsi_qmap(struct ice_softc *sc) 2673 { 2674 struct ice_vsi *vsi = &sc->pf_vsi; 2675 struct ice_tx_queue *txq; 2676 struct ice_rx_queue *rxq; 2677 int err, i; 2678 2679 /* Re-assign Tx queues from PF space to the main VSI */ 2680 err = ice_resmgr_assign_contiguous(&sc->tx_qmgr, vsi->tx_qmap, 2681 vsi->num_tx_queues); 2682 if (err) { 2683 device_printf(sc->dev, "Unable to re-assign PF Tx queues: %s\n", 2684 ice_err_str(err)); 2685 return (err); 2686 } 2687 2688 /* Re-assign Rx queues from PF space to this VSI */ 2689 err = ice_resmgr_assign_contiguous(&sc->rx_qmgr, vsi->rx_qmap, 2690 vsi->num_rx_queues); 2691 if (err) { 2692 device_printf(sc->dev, "Unable to re-assign PF Rx queues: %s\n", 2693 ice_err_str(err)); 2694 goto err_release_tx_queues; 2695 } 2696 2697 vsi->qmap_type = ICE_RESMGR_ALLOC_CONTIGUOUS; 2698 2699 /* Re-assign Tx queue tail pointers */ 2700 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 2701 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 2702 2703 /* Re-assign Rx queue tail pointers */ 2704 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 2705 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 2706 2707 return (0); 2708 2709 err_release_tx_queues: 2710 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2711 sc->pf_vsi.num_tx_queues); 2712 2713 return (err); 2714 } 2715 2716 /* determine if the iflib context is active */ 2717 #define CTX_ACTIVE(ctx) iflib_is_running(ctx) 2718 2719 /** 2720 * ice_rebuild_recovery_mode - Rebuild driver state while in recovery mode 2721 * @sc: The device private softc 2722 * 2723 * Handle a driver rebuild while in recovery mode. This will only rebuild the 2724 * limited functionality supported while in recovery mode. 2725 */ 2726 static void 2727 ice_rebuild_recovery_mode(struct ice_softc *sc) 2728 { 2729 device_t dev = sc->dev; 2730 2731 /* enable PCIe bus master */ 2732 pci_enable_busmaster(dev); 2733 2734 /* Configure interrupt causes for the administrative interrupt */ 2735 ice_configure_misc_interrupts(sc); 2736 2737 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2738 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2739 2740 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2741 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2742 2743 log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp)); 2744 2745 /* In order to completely restore device functionality, the iflib core 2746 * needs to be reset. We need to request an iflib reset. Additionally, 2747 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2748 * the iflib core, we also want re-run the admin task so that iflib 2749 * resets immediately instead of waiting for the next interrupt. 2750 */ 2751 ice_request_stack_reinit(sc); 2752 2753 return; 2754 } 2755 2756 /** 2757 * ice_rebuild - Rebuild driver state post reset 2758 * @sc: The device private softc 2759 * 2760 * Restore driver state after a reset occurred. Restart the controlqs, setup 2761 * the hardware port, and re-enable the VSIs. 2762 */ 2763 static void 2764 ice_rebuild(struct ice_softc *sc) 2765 { 2766 struct ice_hw *hw = &sc->hw; 2767 device_t dev = sc->dev; 2768 enum ice_ddp_state pkg_state; 2769 int status; 2770 int err; 2771 int i; 2772 2773 sc->rebuild_ticks = ticks; 2774 2775 /* If we're rebuilding, then a reset has succeeded. */ 2776 ice_clear_state(&sc->state, ICE_STATE_RESET_FAILED); 2777 /* The reset discarded every firmware VSI before reconstruction. */ 2778 for (i = 0; i < sc->num_available_vsi; i++) { 2779 if (sc->all_vsi[i] != NULL) 2780 sc->all_vsi[i]->hw_vsi_created = false; 2781 } 2782 2783 /* 2784 * If the firmware is in recovery mode, only restore the limited 2785 * functionality supported by recovery mode. 2786 */ 2787 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) { 2788 ice_rebuild_recovery_mode(sc); 2789 return; 2790 } 2791 2792 /* enable PCIe bus master */ 2793 pci_enable_busmaster(dev); 2794 2795 status = ice_init_all_ctrlq(hw); 2796 if (status) { 2797 device_printf(dev, "failed to re-init controlqs, err %s\n", 2798 ice_status_str(status)); 2799 goto err_shutdown_ctrlq; 2800 } 2801 2802 #ifdef PCI_IOV 2803 ice_iov_reconfigure_mbx(sc); 2804 #endif 2805 2806 /* Query the allocated resources for Tx scheduler */ 2807 status = ice_sched_query_res_alloc(hw); 2808 if (status) { 2809 device_printf(dev, 2810 "Failed to query scheduler resources, err %s aq_err %s\n", 2811 ice_status_str(status), 2812 ice_aq_str(hw->adminq.sq_last_status)); 2813 goto err_shutdown_ctrlq; 2814 } 2815 2816 /* Re-enable FW logging. Keep going even if this fails */ 2817 status = ICE_SUCCESS; 2818 if (hw->pf_id == 0) 2819 status = ice_fwlog_set(hw, &hw->fwlog_cfg); 2820 if (!status) { 2821 /* 2822 * We should have the most updated cached copy of the 2823 * configuration, regardless of whether we're rebuilding 2824 * or not. So we'll simply check to see if logging was 2825 * enabled pre-rebuild. 2826 */ 2827 if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 2828 status = ice_fwlog_register(hw); 2829 if (status) 2830 device_printf(dev, "failed to re-register fw logging, err %s aq_err %s\n", 2831 ice_status_str(status), 2832 ice_aq_str(hw->adminq.sq_last_status)); 2833 } 2834 } else 2835 device_printf(dev, "failed to rebuild fw logging configuration, err %s aq_err %s\n", 2836 ice_status_str(status), 2837 ice_aq_str(hw->adminq.sq_last_status)); 2838 2839 err = ice_send_version(sc); 2840 if (err) 2841 goto err_shutdown_ctrlq; 2842 2843 /* Retry a restore which could not complete while reset was pending. */ 2844 ice_led_restore(sc); 2845 2846 err = ice_init_link_events(sc); 2847 if (err) { 2848 device_printf(dev, "ice_init_link_events failed: %s\n", 2849 ice_err_str(err)); 2850 goto err_shutdown_ctrlq; 2851 } 2852 2853 status = ice_clear_pf_cfg(hw); 2854 if (status) { 2855 device_printf(dev, "failed to clear PF configuration, err %s\n", 2856 ice_status_str(status)); 2857 goto err_shutdown_ctrlq; 2858 } 2859 2860 ice_clean_all_vsi_rss_cfg(sc); 2861 2862 ice_clear_pxe_mode(hw); 2863 2864 status = ice_get_caps(hw); 2865 if (status) { 2866 device_printf(dev, "failed to get capabilities, err %s\n", 2867 ice_status_str(status)); 2868 goto err_shutdown_ctrlq; 2869 } 2870 2871 status = ice_sched_init_port(hw->port_info); 2872 if (status) { 2873 device_printf(dev, "failed to initialize port, err %s\n", 2874 ice_status_str(status)); 2875 goto err_sched_cleanup; 2876 } 2877 2878 /* If we previously loaded the package, it needs to be reloaded now */ 2879 if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_SAFE_MODE)) { 2880 pkg_state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 2881 if (!ice_is_init_pkg_successful(pkg_state)) { 2882 ice_log_pkg_init(sc, pkg_state); 2883 ice_transition_safe_mode(sc); 2884 } 2885 } 2886 2887 ice_reset_pf_stats(sc); 2888 2889 err = ice_rebuild_pf_vsi_qmap(sc); 2890 if (err) { 2891 device_printf(sc->dev, "Unable to re-assign main VSI queues, err %s\n", 2892 ice_err_str(err)); 2893 goto err_sched_cleanup; 2894 } 2895 err = ice_initialize_vsi(&sc->pf_vsi); 2896 if (err) { 2897 device_printf(sc->dev, "Unable to re-initialize Main VSI, err %s\n", 2898 ice_err_str(err)); 2899 goto err_release_queue_allocations; 2900 } 2901 2902 /* Replay all VSI configuration */ 2903 err = ice_replay_all_vsi_cfg(sc); 2904 if (err) 2905 goto err_deinit_pf_vsi; 2906 2907 /* Re-enable FW health event reporting */ 2908 ice_init_health_events(sc); 2909 2910 /* Reconfigure the main PF VSI for RSS */ 2911 err = ice_config_rss(&sc->pf_vsi); 2912 if (err) { 2913 device_printf(sc->dev, 2914 "Unable to reconfigure RSS for the main VSI, err %s\n", 2915 ice_err_str(err)); 2916 goto err_deinit_pf_vsi; 2917 } 2918 2919 if (hw->port_info->qos_cfg.is_sw_lldp) 2920 ice_add_rx_lldp_filter(sc); 2921 2922 /* Apply previous link settings and refresh link status, if PHY 2923 * FW is ready. 2924 */ 2925 ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED); 2926 ice_init_link(sc); 2927 2928 /* RDMA interface will be restarted by the stack re-init */ 2929 2930 /* Configure interrupt causes for the administrative interrupt */ 2931 ice_configure_misc_interrupts(sc); 2932 2933 /* Enable ITR 0 right away, so that we can handle admin interrupts */ 2934 ice_enable_intr(&sc->hw, sc->irqvs[0].me); 2935 2936 /* Now that the rebuild is finished, we're no longer prepared to reset */ 2937 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2938 2939 /* Reconfigure the subinterface */ 2940 if (sc->mirr_if) { 2941 err = ice_subif_rebuild(sc); 2942 if (err) 2943 goto err_deinit_pf_vsi; 2944 } 2945 2946 log(LOG_INFO, "%s: device rebuild successful\n", if_name(sc->ifp)); 2947 2948 /* In order to completely restore device functionality, the iflib core 2949 * needs to be reset. We need to request an iflib reset. Additionally, 2950 * because the state of IFC_DO_RESET is cached within task_fn_admin in 2951 * the iflib core, we also want re-run the admin task so that iflib 2952 * resets immediately instead of waiting for the next interrupt. 2953 * If LLDP is enabled we need to reconfig DCB to properly reinit all TC 2954 * queues, not only 0. It contains ice_request_stack_reinit as well. 2955 */ 2956 if (hw->port_info->qos_cfg.is_sw_lldp) 2957 ice_request_stack_reinit(sc); 2958 else 2959 ice_do_dcb_reconfig(sc, false); 2960 2961 return; 2962 2963 err_deinit_pf_vsi: 2964 ice_deinit_vsi(&sc->pf_vsi); 2965 err_release_queue_allocations: 2966 ice_resmgr_release_map(&sc->tx_qmgr, sc->pf_vsi.tx_qmap, 2967 sc->pf_vsi.num_tx_queues); 2968 ice_resmgr_release_map(&sc->rx_qmgr, sc->pf_vsi.rx_qmap, 2969 sc->pf_vsi.num_rx_queues); 2970 err_sched_cleanup: 2971 ice_sched_cleanup_all(hw); 2972 err_shutdown_ctrlq: 2973 ice_shutdown_all_ctrlq(hw, false); 2974 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 2975 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 2976 device_printf(dev, "Driver rebuild failed, please reload the device driver\n"); 2977 } 2978 2979 /** 2980 * ice_handle_reset_event - Handle reset events triggered by OICR 2981 * @sc: The device private softc 2982 * 2983 * Handle reset events triggered by an OICR notification. This includes CORER, 2984 * GLOBR, and EMPR resets triggered by software on this or any other PF or by 2985 * firmware. 2986 * 2987 * @pre assumes the iflib context lock is held, and will unlock it while 2988 * waiting for the hardware to finish reset. 2989 */ 2990 static void 2991 ice_handle_reset_event(struct ice_softc *sc) 2992 { 2993 struct ice_hw *hw = &sc->hw; 2994 int status; 2995 device_t dev = sc->dev; 2996 2997 /* When a CORER, GLOBR, or EMPR is about to happen, the hardware will 2998 * trigger an OICR interrupt. Our OICR handler will determine when 2999 * this occurs and set the ICE_STATE_RESET_OICR_RECV bit as 3000 * appropriate. 3001 */ 3002 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_OICR_RECV)) 3003 return; 3004 3005 ice_prepare_for_reset(sc); 3006 3007 /* 3008 * Release the iflib context lock and wait for the device to finish 3009 * resetting. 3010 */ 3011 IFLIB_CTX_UNLOCK(sc); 3012 3013 #define ICE_EMPR_ADDL_WAIT_MSEC_SLOW 20000 3014 if ((ice_is_e830(hw) || ice_is_e825c(hw)) && 3015 (((rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 3016 GLGEN_RSTAT_RESET_TYPE_S) == ICE_RESET_EMPR)) 3017 ice_msec_pause(ICE_EMPR_ADDL_WAIT_MSEC_SLOW); 3018 3019 status = ice_check_reset(hw); 3020 IFLIB_CTX_LOCK(sc); 3021 if (status) { 3022 device_printf(dev, "Device never came out of reset, err %s\n", 3023 ice_status_str(status)); 3024 3025 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 3026 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 3027 ice_clear_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET); 3028 device_printf(dev, "Reset failed; please reload the device driver\n"); 3029 return; 3030 } 3031 3032 /* We're done with the reset, so we can rebuild driver state */ 3033 sc->hw.reset_ongoing = false; 3034 ice_rebuild(sc); 3035 3036 /* In the unlikely event that a PF reset request occurs at the same 3037 * time as a global reset, clear the request now. This avoids 3038 * resetting a second time right after we reset due to a global event. 3039 */ 3040 if (ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 3041 device_printf(dev, "Ignoring PFR request that occurred while a reset was ongoing\n"); 3042 } 3043 3044 /** 3045 * ice_handle_pf_reset_request - Initiate PF reset requested by software 3046 * @sc: The device private softc 3047 * 3048 * Initiate a PF reset requested by software. We handle this in the admin task 3049 * so that only one thread actually handles driver preparation and cleanup, 3050 * rather than having multiple threads possibly attempt to run this code 3051 * simultaneously. 3052 * 3053 * @pre assumes the iflib context lock is held and will unlock it while 3054 * waiting for the PF reset to complete. 3055 */ 3056 static void 3057 ice_handle_pf_reset_request(struct ice_softc *sc) 3058 { 3059 struct ice_hw *hw = &sc->hw; 3060 int status; 3061 3062 /* Check for PF reset requests */ 3063 if (!ice_testandclear_state(&sc->state, ICE_STATE_RESET_PFR_REQ)) 3064 return; 3065 3066 /* Make sure we're prepared for reset */ 3067 ice_prepare_for_reset(sc); 3068 3069 /* 3070 * Release the iflib context lock and wait for the device to finish 3071 * resetting. 3072 */ 3073 IFLIB_CTX_UNLOCK(sc); 3074 status = ice_reset(hw, ICE_RESET_PFR); 3075 IFLIB_CTX_LOCK(sc); 3076 if (status) { 3077 device_printf(sc->dev, "device PF reset failed, err %s\n", 3078 ice_status_str(status)); 3079 ice_set_state(&sc->state, ICE_STATE_RESET_FAILED); 3080 return; 3081 } 3082 3083 sc->soft_stats.pfr_count++; 3084 ice_rebuild(sc); 3085 } 3086 3087 /** 3088 * ice_init_device_features - Init device driver features 3089 * @sc: driver softc structure 3090 * 3091 * @pre assumes that the function capabilities bits have been set up by 3092 * ice_init_hw(). 3093 */ 3094 static void 3095 ice_init_device_features(struct ice_softc *sc) 3096 { 3097 struct ice_hw *hw = &sc->hw; 3098 3099 /* Set capabilities that all devices support */ 3100 ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_cap); 3101 ice_set_bit(ICE_FEATURE_RSS, sc->feat_cap); 3102 ice_set_bit(ICE_FEATURE_RDMA, sc->feat_cap); 3103 ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_cap); 3104 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_cap); 3105 ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_cap); 3106 ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 3107 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 3108 ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_cap); 3109 ice_set_bit(ICE_FEATURE_DCB, sc->feat_cap); 3110 ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap); 3111 ice_set_bit(ICE_FEATURE_PHY_STATISTICS, sc->feat_cap); 3112 3113 if (ice_is_e810(hw)) 3114 ice_set_bit(ICE_FEATURE_PHY_STATISTICS, sc->feat_en); 3115 3116 if (ice_is_e825c(hw)) 3117 ice_set_bit(ICE_FEATURE_DUAL_NAC, sc->feat_cap); 3118 /* Disable features due to hardware limitations... */ 3119 if (!hw->func_caps.common_cap.rss_table_size) 3120 ice_clear_bit(ICE_FEATURE_RSS, sc->feat_cap); 3121 if (!hw->func_caps.common_cap.iwarp || !ice_enable_irdma) 3122 ice_clear_bit(ICE_FEATURE_RDMA, sc->feat_cap); 3123 if (!hw->func_caps.common_cap.dcb) 3124 ice_clear_bit(ICE_FEATURE_DCB, sc->feat_cap); 3125 /* Disable features due to firmware limitations... */ 3126 if (!ice_is_fw_health_report_supported(hw)) 3127 ice_clear_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_cap); 3128 if (!ice_fwlog_supported(hw)) 3129 ice_clear_bit(ICE_FEATURE_FW_LOGGING, sc->feat_cap); 3130 if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { 3131 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_FW_LOGGING)) 3132 ice_set_bit(ICE_FEATURE_FW_LOGGING, sc->feat_en); 3133 else 3134 ice_fwlog_unregister(hw); 3135 } 3136 3137 /* Disable capabilities not supported by the OS */ 3138 ice_disable_unsupported_features(sc->feat_cap); 3139 3140 /* RSS is always enabled for iflib */ 3141 if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_RSS)) 3142 ice_set_bit(ICE_FEATURE_RSS, sc->feat_en); 3143 3144 /* Disable features based on sysctl settings */ 3145 if (!ice_tx_balance_en) 3146 ice_clear_bit(ICE_FEATURE_TX_BALANCE, sc->feat_cap); 3147 3148 if (hw->dev_caps.supported_sensors & ICE_SENSOR_SUPPORT_E810_INT_TEMP) { 3149 ice_set_bit(ICE_FEATURE_TEMP_SENSOR, sc->feat_cap); 3150 ice_set_bit(ICE_FEATURE_TEMP_SENSOR, sc->feat_en); 3151 } 3152 3153 if (hw->func_caps.common_cap.next_cluster_id_support || 3154 hw->dev_caps.common_cap.next_cluster_id_support) { 3155 ice_set_bit(ICE_FEATURE_NEXT_CLUSTER_ID, sc->feat_cap); 3156 ice_set_bit(ICE_FEATURE_NEXT_CLUSTER_ID, sc->feat_en); 3157 } 3158 } 3159 3160 /** 3161 * ice_if_multi_set - Callback to update Multicast filters in HW 3162 * @ctx: iflib ctx structure 3163 * 3164 * Called by iflib in response to SIOCDELMULTI and SIOCADDMULTI. Must search 3165 * the if_multiaddrs list and determine which filters have been added or 3166 * removed from the list, and update HW programming to reflect the new list. 3167 * 3168 * @pre assumes the caller holds the iflib CTX lock 3169 */ 3170 static void 3171 ice_if_multi_set(if_ctx_t ctx) 3172 { 3173 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3174 int err; 3175 3176 ASSERT_CTX_LOCKED(sc); 3177 3178 /* Do not handle multicast configuration in recovery mode */ 3179 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 3180 return; 3181 3182 err = ice_sync_multicast_filters(sc); 3183 if (err) { 3184 device_printf(sc->dev, 3185 "Failed to synchronize multicast filter list: %s\n", 3186 ice_err_str(err)); 3187 return; 3188 } 3189 } 3190 3191 /** 3192 * ice_if_vlan_register - Register a VLAN with the hardware 3193 * @ctx: iflib ctx pointer 3194 * @vtag: VLAN to add 3195 * 3196 * Programs the main PF VSI with a hardware filter for the given VLAN. 3197 * 3198 * @pre assumes the caller holds the iflib CTX lock 3199 */ 3200 static void 3201 ice_if_vlan_register(if_ctx_t ctx, u16 vtag) 3202 { 3203 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3204 int status; 3205 3206 ASSERT_CTX_LOCKED(sc); 3207 3208 /* Do not handle VLAN configuration in recovery mode */ 3209 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 3210 return; 3211 3212 status = ice_add_vlan_hw_filter(&sc->pf_vsi, vtag); 3213 if (status) { 3214 device_printf(sc->dev, 3215 "Failure adding VLAN %d to main VSI, err %s aq_err %s\n", 3216 vtag, ice_status_str(status), 3217 ice_aq_str(sc->hw.adminq.sq_last_status)); 3218 } 3219 } 3220 3221 /** 3222 * ice_if_vlan_unregister - Remove a VLAN filter from the hardware 3223 * @ctx: iflib ctx pointer 3224 * @vtag: VLAN to add 3225 * 3226 * Removes the previously programmed VLAN filter from the main PF VSI. 3227 * 3228 * @pre assumes the caller holds the iflib CTX lock 3229 */ 3230 static void 3231 ice_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 3232 { 3233 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3234 int status; 3235 3236 ASSERT_CTX_LOCKED(sc); 3237 3238 /* Do not handle VLAN configuration in recovery mode */ 3239 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 3240 return; 3241 3242 status = ice_remove_vlan_hw_filter(&sc->pf_vsi, vtag); 3243 if (status) { 3244 device_printf(sc->dev, 3245 "Failure removing VLAN %d from main VSI, err %s aq_err %s\n", 3246 vtag, ice_status_str(status), 3247 ice_aq_str(sc->hw.adminq.sq_last_status)); 3248 } 3249 } 3250 3251 /** 3252 * ice_if_stop - Stop the device 3253 * @ctx: iflib context structure 3254 * 3255 * Called by iflib to stop the device and bring it down. (i.e. ifconfig ice0 3256 * down) 3257 * 3258 * @pre assumes the caller holds the iflib CTX lock 3259 */ 3260 static void 3261 ice_if_stop(if_ctx_t ctx) 3262 { 3263 struct ice_mirr_if *mif; 3264 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3265 3266 ASSERT_CTX_LOCKED(sc); 3267 mif = sc->mirr_if; 3268 ice_led_restore(sc); 3269 3270 /* 3271 * The iflib core may call IFDI_STOP prior to the first call to 3272 * IFDI_INIT. This will cause us to attempt to remove MAC filters we 3273 * don't have, and disable Tx queues which aren't yet configured. 3274 * Although it is likely these extra operations are harmless, they do 3275 * cause spurious warning messages to be displayed, which may confuse 3276 * users. 3277 * 3278 * To avoid these messages, we use a state bit indicating if we've 3279 * been initialized. It will be set when ice_if_init is called, and 3280 * cleared here in ice_if_stop. 3281 */ 3282 if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED)) 3283 return; 3284 3285 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 3286 device_printf(sc->dev, "request to stop interface cannot be completed as the device failed to reset\n"); 3287 return; 3288 } 3289 3290 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 3291 device_printf(sc->dev, "request to stop interface while device is prepared for impending reset\n"); 3292 return; 3293 } 3294 3295 ice_rdma_pf_stop(sc); 3296 3297 /* Remove the MAC filters, stop Tx, and stop Rx. We don't check the 3298 * return of these functions because there's nothing we can really do 3299 * if they fail, and the functions already print error messages. 3300 * Just try to shut down as much as we can. 3301 */ 3302 ice_rm_pf_default_mac_filters(sc); 3303 3304 /* Dissociate the Tx and Rx queues from the interrupts */ 3305 ice_flush_txq_interrupts(&sc->pf_vsi); 3306 ice_flush_rxq_interrupts(&sc->pf_vsi); 3307 3308 /* Disable the Tx and Rx queues */ 3309 ice_vsi_disable_tx(&sc->pf_vsi); 3310 ice_control_all_rx_queues(&sc->pf_vsi, false); 3311 3312 if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && 3313 !(if_getflags(sc->ifp) & IFF_UP) && sc->link_up) 3314 ice_set_link(sc, false); 3315 3316 if (mif != NULL && ice_test_state(&mif->state, 3317 ICE_STATE_SUBIF_NEEDS_REINIT)) { 3318 ice_subif_if_stop(mif->subctx); 3319 device_printf(sc->dev, "The subinterface also comes down and up after reset\n"); 3320 } 3321 } 3322 3323 /** 3324 * ice_if_led_func - Control the physical port identification LED 3325 * @ctx: iflib context structure 3326 * @onoff: non-zero to identify the port, zero to restore normal operation 3327 * 3328 * The firmware implements identification as a blinking mode and retains the 3329 * netlist-selected mode so it can be restored without a register snapshot. 3330 */ 3331 static void 3332 ice_if_led_func(if_ctx_t ctx, int onoff) 3333 { 3334 struct ice_softc *sc = iflib_get_softc(ctx); 3335 enum ice_status status; 3336 bool active; 3337 3338 active = onoff != 0; 3339 if (active == sc->led_active) 3340 return; 3341 3342 status = ice_aq_set_port_id_led(sc->hw.port_info, !active, NULL); 3343 if (status == ICE_SUCCESS) 3344 sc->led_active = active; 3345 } 3346 3347 static void 3348 ice_led_restore(struct ice_softc *sc) 3349 { 3350 3351 if (!sc->led_active) 3352 return; 3353 if (ice_aq_set_port_id_led(sc->hw.port_info, true, NULL) == ICE_SUCCESS) 3354 sc->led_active = false; 3355 } 3356 3357 /** 3358 * ice_if_get_counter - Get current value of an ifnet statistic 3359 * @ctx: iflib context pointer 3360 * @counter: ifnet counter to read 3361 * 3362 * Reads the current value of an ifnet counter for the device. 3363 * 3364 * This function is not protected by the iflib CTX lock. 3365 */ 3366 static uint64_t 3367 ice_if_get_counter(if_ctx_t ctx, ift_counter counter) 3368 { 3369 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3370 3371 /* Return the counter for the main PF VSI */ 3372 return ice_get_ifnet_counter(&sc->pf_vsi, counter); 3373 } 3374 3375 /** 3376 * ice_request_stack_reinit - Request that iflib re-initialize 3377 * @sc: the device private softc 3378 * 3379 * Request that the device be brought down and up, to re-initialize. For 3380 * example, this may be called when a device reset occurs, or when Tx and Rx 3381 * queues need to be re-initialized. 3382 * 3383 * This is required because the iflib state is outside the driver, and must be 3384 * re-initialized if we need to resart Tx and Rx queues. 3385 */ 3386 void 3387 ice_request_stack_reinit(struct ice_softc *sc) 3388 { 3389 if (CTX_ACTIVE(sc->ctx)) { 3390 iflib_request_reset(sc->ctx); 3391 iflib_admin_intr_deferred(sc->ctx); 3392 } 3393 } 3394 3395 /** 3396 * ice_driver_is_detaching - Check if the driver is detaching/unloading 3397 * @sc: device private softc 3398 * 3399 * Returns true if the driver is detaching, false otherwise. 3400 * 3401 * @remark on newer kernels, take advantage of iflib_in_detach in order to 3402 * report detachment correctly as early as possible. 3403 * 3404 * @remark this function is used by various code paths that want to avoid 3405 * running if the driver is about to be removed. This includes sysctls and 3406 * other driver access points. Note that it does not fully resolve 3407 * detach-based race conditions as it is possible for a thread to race with 3408 * iflib_in_detach. 3409 */ 3410 bool 3411 ice_driver_is_detaching(struct ice_softc *sc) 3412 { 3413 return (ice_test_state(&sc->state, ICE_STATE_DETACHING) || 3414 iflib_in_detach(sc->ctx)); 3415 } 3416 3417 /** 3418 * ice_if_priv_ioctl - Device private ioctl handler 3419 * @ctx: iflib context pointer 3420 * @command: The ioctl command issued 3421 * @data: ioctl specific data 3422 * 3423 * iflib callback for handling custom driver specific ioctls. 3424 * 3425 * @pre Assumes that the iflib context lock is held. 3426 */ 3427 static int 3428 ice_if_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) 3429 { 3430 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3431 struct ifdrv *ifd; 3432 device_t dev = sc->dev; 3433 3434 if (data == NULL) 3435 return (EINVAL); 3436 3437 ASSERT_CTX_LOCKED(sc); 3438 3439 /* Make sure the command type is valid */ 3440 switch (command) { 3441 case SIOCSDRVSPEC: 3442 case SIOCGDRVSPEC: 3443 /* Accepted commands */ 3444 break; 3445 case SIOCGPRIVATE_0: 3446 /* 3447 * Although we do not support this ioctl command, it's 3448 * expected that iflib will forward it to the IFDI_PRIV_IOCTL 3449 * handler. Do not print a message in this case 3450 */ 3451 return (ENOTSUP); 3452 default: 3453 /* 3454 * If we get a different command for this function, it's 3455 * definitely unexpected, so log a message indicating what 3456 * command we got for debugging purposes. 3457 */ 3458 device_printf(dev, "%s: unexpected ioctl command %08lx\n", 3459 __func__, command); 3460 return (EINVAL); 3461 } 3462 3463 ifd = (struct ifdrv *)data; 3464 3465 switch (ifd->ifd_cmd) { 3466 case ICE_NVM_ACCESS: 3467 return ice_handle_nvm_access_ioctl(sc, ifd); 3468 case ICE_DEBUG_DUMP: 3469 return ice_handle_debug_dump_ioctl(sc, ifd); 3470 default: 3471 return EINVAL; 3472 } 3473 } 3474 3475 /** 3476 * ice_if_i2c_req - I2C request handler for iflib 3477 * @ctx: iflib context pointer 3478 * @req: The I2C parameters to use 3479 * 3480 * Read from the port's I2C eeprom using the parameters from the ioctl. 3481 * 3482 * @remark The iflib-only part is pretty simple. 3483 */ 3484 static int 3485 ice_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req) 3486 { 3487 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3488 3489 return ice_handle_i2c_req(sc, req); 3490 } 3491 3492 /** 3493 * ice_if_suspend - PCI device suspend handler for iflib 3494 * @ctx: iflib context pointer 3495 * 3496 * Deinitializes the driver and clears HW resources in preparation for 3497 * suspend or an FLR. 3498 * 3499 * @returns 0 on success, or an error code on failure 3500 */ 3501 static int 3502 ice_if_suspend(if_ctx_t ctx) 3503 { 3504 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3505 3506 /* At least a PFR is always going to happen after this; 3507 * either via FLR or during the D3->D0 transition. 3508 */ 3509 ice_clear_state(&sc->state, ICE_STATE_RESET_PFR_REQ); 3510 3511 ice_prepare_for_reset(sc); 3512 3513 return (0); 3514 } 3515 3516 /** 3517 * ice_if_resume - PCI device resume handler for iflib 3518 * @ctx: iflib context pointer 3519 * 3520 * Reinitializes the driver and the HW after PCI resume or after 3521 * an FLR. An init is performed by iflib after this function is finished. 3522 * 3523 * @returns 0 on success, or an error code on failure 3524 */ 3525 static int 3526 ice_if_resume(if_ctx_t ctx) 3527 { 3528 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3529 3530 ice_rebuild(sc); 3531 3532 return (0); 3533 } 3534 3535 /** 3536 * ice_if_needs_restart - Tell iflib when the driver needs to be reinitialized 3537 * @ctx: iflib context pointer 3538 * @event: event code to check 3539 * 3540 * Defaults to returning true for unknown events. 3541 * 3542 * @returns true if iflib needs to reinit the interface 3543 */ 3544 static bool 3545 ice_if_needs_restart(if_ctx_t ctx, enum iflib_restart_event event) 3546 { 3547 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3548 3549 switch (event) { 3550 case IFLIB_RESTART_VLAN_CONFIG: 3551 if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && 3552 !(if_getflags(sc->ifp) & IFF_UP)) 3553 return false; 3554 default: 3555 return true; 3556 } 3557 } 3558 3559 /** 3560 * ice_init_link - Do link configuration and link status reporting 3561 * @sc: driver private structure 3562 * 3563 * Contains an extra check that skips link config when an E830 device 3564 * does not have the "FW_LOADING"/"PHYBUSY" bit set in GL_MNG_FWSM set. 3565 */ 3566 static void 3567 ice_init_link(struct ice_softc *sc) 3568 { 3569 struct ice_hw *hw = &sc->hw; 3570 device_t dev = sc->dev; 3571 3572 /* Check if FW is ready before setting up link; defer setup to the 3573 * admin task if it isn't. 3574 */ 3575 if (ice_is_e830(hw) && 3576 (rd32(hw, GL_MNG_FWSM) & GL_MNG_FWSM_FW_LOADING_M)) { 3577 ice_set_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING); 3578 device_printf(dev, 3579 "Link initialization is blocked by PHY FW initialization.\n"); 3580 device_printf(dev, 3581 "Link initialization will continue after PHY FW initialization completes.\n"); 3582 /* Do not access PHY config while PHY FW is busy initializing */ 3583 } else { 3584 ice_clear_state(&sc->state, ICE_STATE_PHY_FW_INIT_PENDING); 3585 3586 if (ice_is_e830(hw)) { 3587 if (!(sc->ldo_tlv.options & ICE_LINK_OVERRIDE_PORT_DIS)) 3588 return; 3589 3590 ice_set_state(&sc->state, ICE_STATE_TOTAL_PORT_SHUTDOWN); 3591 ice_clear_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN); 3592 } 3593 3594 ice_init_link_configuration(sc); 3595 ice_update_link_status(sc, true); 3596 } 3597 3598 } 3599 3600 #ifdef PCI_IOV 3601 /** 3602 * ice_if_iov_init - iov init handler for iflib 3603 * @ctx: iflib context pointer 3604 * @num_vfs: number of VFs to create 3605 * @params: configuration parameters for the PF 3606 * 3607 * Configure the driver for SR-IOV mode. Used to setup things like memory 3608 * before any VFs are created. 3609 * 3610 * @remark This is a wrapper for ice_iov_init 3611 */ 3612 static int 3613 ice_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params) 3614 { 3615 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3616 3617 return ice_iov_init(sc, num_vfs, params); 3618 } 3619 3620 /** 3621 * ice_if_iov_uninit - iov uninit handler for iflib 3622 * @ctx: iflib context pointer 3623 * 3624 * Destroys VFs and frees their memory and resources. 3625 * 3626 * @remark This is a wrapper for ice_iov_uninit 3627 */ 3628 static void 3629 ice_if_iov_uninit(if_ctx_t ctx) 3630 { 3631 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3632 3633 ice_iov_uninit(sc); 3634 } 3635 3636 /** 3637 * ice_if_iov_vf_add - iov add vf handler for iflib 3638 * @ctx: iflib context pointer 3639 * @vfnum: index of VF to configure 3640 * @params: configuration parameters for the VF 3641 * 3642 * Sets up the VF given by the vfnum index. This is called by the OS 3643 * for each VF created by the PF driver after it is spawned. 3644 * 3645 * @remark This is a wrapper for ice_iov_vf_add 3646 */ 3647 static int 3648 ice_if_iov_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params) 3649 { 3650 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3651 3652 return ice_iov_add_vf(sc, vfnum, params); 3653 } 3654 3655 /** 3656 * ice_if_vf_status - report configured VF state 3657 * @ctx: iflib context pointer 3658 * @statusp: returned VF status snapshot 3659 */ 3660 static int 3661 ice_if_vf_status(if_ctx_t ctx, struct if_vf_status **statusp) 3662 { 3663 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3664 3665 return (ice_iov_vf_status(sc, statusp)); 3666 } 3667 3668 /** 3669 * ice_if_vflr_handle - iov VFLR handler 3670 * @ctx: iflib context pointer 3671 * 3672 * Performs the necessar teardown or setup required for a VF after 3673 * a VFLR is initiated. 3674 * 3675 * @remark This is a wrapper for ice_iov_handle_vflr 3676 */ 3677 static void 3678 ice_if_vflr_handle(if_ctx_t ctx) 3679 { 3680 struct ice_softc *sc = (struct ice_softc *)iflib_get_softc(ctx); 3681 ice_iov_handle_vflr(sc); 3682 } 3683 #endif /* PCI_IOV */ 3684 3685 extern struct if_txrx ice_subif_txrx; 3686 3687 /** 3688 * @var ice_subif_methods 3689 * @brief ice driver method entry points 3690 */ 3691 static device_method_t ice_subif_methods[] = { 3692 /* Device interface */ 3693 DEVMETHOD(device_register, ice_subif_register), 3694 DEVMETHOD_END 3695 }; 3696 3697 /** 3698 * @var ice_subif_driver 3699 * @brief driver structure for the device API 3700 */ 3701 static driver_t ice_subif_driver = { 3702 .name = "ice_subif", 3703 .methods = ice_subif_methods, 3704 .size = sizeof(struct ice_mirr_if), 3705 }; 3706 3707 static device_method_t ice_iflib_subif_methods[] = { 3708 DEVMETHOD(ifdi_attach_pre, ice_subif_if_attach_pre), 3709 DEVMETHOD(ifdi_attach_post, ice_subif_if_attach_post), 3710 DEVMETHOD(ifdi_tx_queues_alloc, ice_subif_if_tx_queues_alloc), 3711 DEVMETHOD(ifdi_rx_queues_alloc, ice_subif_if_rx_queues_alloc), 3712 DEVMETHOD(ifdi_msix_intr_assign, ice_subif_if_msix_intr_assign), 3713 DEVMETHOD(ifdi_intr_enable, ice_subif_if_intr_enable), 3714 DEVMETHOD(ifdi_rx_queue_intr_enable, ice_subif_if_rx_queue_intr_enable), 3715 DEVMETHOD(ifdi_tx_queue_intr_enable, ice_subif_if_tx_queue_intr_enable), 3716 DEVMETHOD(ifdi_init, ice_subif_if_init), 3717 DEVMETHOD(ifdi_stop, ice_subif_if_stop), 3718 DEVMETHOD(ifdi_queues_free, ice_subif_if_queues_free), 3719 DEVMETHOD(ifdi_media_status, ice_subif_if_media_status), 3720 DEVMETHOD(ifdi_promisc_set, ice_subif_if_promisc_set), 3721 }; 3722 3723 /** 3724 * @var ice_iflib_subif_driver 3725 * @brief driver structure for the iflib stack 3726 * 3727 * driver_t definition used to setup the iflib device methods. 3728 */ 3729 static driver_t ice_iflib_subif_driver = { 3730 .name = "ice_subif", 3731 .methods = ice_iflib_subif_methods, 3732 .size = sizeof(struct ice_mirr_if), 3733 }; 3734 3735 /** 3736 * @var ice_subif_sctx 3737 * @brief ice driver shared context 3738 * 3739 * Similar to the existing ice_sctx, this structure has these differences: 3740 * - isc_admin_intrcnt is set to 0 3741 * - Uses subif iflib driver methods 3742 * - Flagged as a VF for iflib 3743 */ 3744 static struct if_shared_ctx ice_subif_sctx = { 3745 .isc_magic = IFLIB_MAGIC, 3746 .isc_q_align = PAGE_SIZE, 3747 3748 .isc_tx_maxsize = ICE_MAX_FRAME_SIZE, 3749 .isc_tx_maxsegsize = ICE_MAX_FRAME_SIZE, 3750 .isc_tso_maxsize = ICE_TSO_SIZE + sizeof(struct ether_vlan_header), 3751 .isc_tso_maxsegsize = ICE_MAX_DMA_SEG_SIZE, 3752 3753 .isc_rx_maxsize = ICE_MAX_FRAME_SIZE, 3754 .isc_rx_nsegments = ICE_MAX_RX_SEGS, 3755 .isc_rx_maxsegsize = ICE_MAX_FRAME_SIZE, 3756 3757 .isc_nfl = 1, 3758 .isc_ntxqs = 1, 3759 .isc_nrxqs = 1, 3760 3761 .isc_admin_intrcnt = 0, 3762 .isc_vendor_info = ice_vendor_info_array, 3763 .isc_driver_version = __DECONST(char *, ice_driver_version), 3764 .isc_driver = &ice_iflib_subif_driver, 3765 3766 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 3767 IFLIB_ADMIN_ALWAYS_RUN | IFLIB_SKIP_MSIX | 3768 IFLIB_IS_VF, 3769 3770 .isc_nrxd_min = {ICE_MIN_DESC_COUNT}, 3771 .isc_ntxd_min = {ICE_MIN_DESC_COUNT}, 3772 .isc_nrxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 3773 .isc_ntxd_max = {ICE_IFLIB_MAX_DESC_COUNT}, 3774 .isc_nrxd_default = {ICE_DEFAULT_DESC_COUNT}, 3775 .isc_ntxd_default = {ICE_DEFAULT_DESC_COUNT}, 3776 }; 3777 3778 static void * 3779 ice_subif_register(device_t dev __unused) 3780 { 3781 return (&ice_subif_sctx); 3782 } 3783 3784 static void 3785 ice_subif_setup_scctx(struct ice_mirr_if *mif) 3786 { 3787 if_softc_ctx_t scctx = mif->subscctx; 3788 3789 scctx->isc_txrx = &ice_subif_txrx; 3790 3791 scctx->isc_capenable = ICE_FULL_CAPS; 3792 scctx->isc_tx_csum_flags = ICE_CSUM_OFFLOAD; 3793 3794 scctx->isc_ntxqsets = 4; 3795 scctx->isc_nrxqsets = 4; 3796 scctx->isc_vectors = scctx->isc_nrxqsets; 3797 3798 scctx->isc_ntxqsets_max = 256; 3799 scctx->isc_nrxqsets_max = 256; 3800 3801 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] 3802 * sizeof(struct ice_tx_desc), DBA_ALIGN); 3803 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] 3804 * sizeof(union ice_32b_rx_flex_desc), DBA_ALIGN); 3805 3806 scctx->isc_tx_nsegments = ICE_MAX_TX_SEGS; 3807 scctx->isc_tx_tso_segments_max = ICE_MAX_TSO_SEGS; 3808 scctx->isc_tx_tso_size_max = ICE_TSO_SIZE; 3809 scctx->isc_tx_tso_segsize_max = ICE_MAX_DMA_SEG_SIZE; 3810 } 3811 3812 static int 3813 ice_subif_if_attach_pre(if_ctx_t ctx) 3814 { 3815 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 3816 device_t dev = iflib_get_dev(ctx); 3817 3818 mif->subctx = ctx; 3819 mif->subdev = dev; 3820 mif->subscctx = iflib_get_softc_ctx(ctx); 3821 3822 /* Setup the iflib softc context structure */ 3823 ice_subif_setup_scctx(mif); 3824 3825 return (0); 3826 } 3827 3828 static int 3829 ice_subif_if_attach_post(if_ctx_t ctx __unused) 3830 { 3831 return (0); 3832 } 3833 3834 /** 3835 * ice_destroy_mirror_interface - destroy mirror interface 3836 * @sc: driver private data 3837 * 3838 * Destroys all resources associated with the mirroring interface. 3839 * Will not exit early on failure. 3840 * 3841 * @pre: Mirror interface already exists and is initialized. 3842 */ 3843 void 3844 ice_destroy_mirror_interface(struct ice_softc *sc) 3845 { 3846 struct ice_mirr_if *mif = sc->mirr_if; 3847 struct ice_vsi *vsi = mif->vsi; 3848 bool is_locked = false; 3849 int ret; 3850 3851 is_locked = sx_xlocked(sc->iflib_ctx_lock); 3852 if (is_locked) 3853 IFLIB_CTX_UNLOCK(sc); 3854 3855 if (mif->ifp) { 3856 ret = iflib_device_deregister(mif->subctx); 3857 if (ret) { 3858 device_printf(sc->dev, 3859 "iflib_device_deregister for mirror interface failed: %d\n", 3860 ret); 3861 } 3862 } 3863 3864 bus_topo_lock(); 3865 ret = device_delete_child(sc->dev, mif->subdev); 3866 bus_topo_unlock(); 3867 if (ret) { 3868 device_printf(sc->dev, 3869 "device_delete_child for mirror interface failed: %d\n", 3870 ret); 3871 } 3872 3873 if (is_locked) 3874 IFLIB_CTX_LOCK(sc); 3875 3876 if (mif->if_imap) { 3877 free(mif->if_imap, M_ICE); 3878 mif->if_imap = NULL; 3879 } 3880 if (mif->os_imap) { 3881 free(mif->os_imap, M_ICE); 3882 mif->os_imap = NULL; 3883 } 3884 3885 /* These are freed via ice_subif_queues_free_subif 3886 * vsi: 3887 * - rx_irqvs 3888 * - tx_queues 3889 * - rx_queues 3890 */ 3891 ice_release_vsi(vsi); 3892 3893 free(mif, M_ICE); 3894 sc->mirr_if = NULL; 3895 3896 } 3897 3898 /** 3899 * ice_setup_mirror_vsi - Initialize mirror VSI 3900 * @mif: driver private data for mirror interface 3901 * 3902 * Allocates a VSI for a mirror interface, and sets that VSI up for use as a 3903 * mirror for the main PF VSI. 3904 * 3905 * Returns 0 on success, or a standard error code on failure. 3906 */ 3907 static int 3908 ice_setup_mirror_vsi(struct ice_mirr_if *mif) 3909 { 3910 struct ice_softc *sc = mif->back; 3911 device_t dev = sc->dev; 3912 struct ice_vsi *vsi; 3913 int ret = 0; 3914 3915 /* vsi is for the new mirror vsi, not the PF's main VSI */ 3916 vsi = ice_alloc_vsi(sc, ICE_VSI_VMDQ2); 3917 if (!vsi) { 3918 /* Already prints an error message */ 3919 return (ENOMEM); 3920 } 3921 mif->vsi = vsi; 3922 3923 /* Reserve VSI queue allocation from PF queues */ 3924 ice_alloc_vsi_qmap(vsi, ICE_DEFAULT_VF_QUEUES, ICE_DEFAULT_VF_QUEUES); 3925 vsi->num_tx_queues = vsi->num_rx_queues = ICE_DEFAULT_VF_QUEUES; 3926 3927 /* Assign Tx queues from PF space */ 3928 ret = ice_resmgr_assign_scattered(&sc->tx_qmgr, vsi->tx_qmap, 3929 vsi->num_tx_queues); 3930 if (ret) { 3931 device_printf(dev, "Unable to assign mirror VSI Tx queues: %s\n", 3932 ice_err_str(ret)); 3933 goto release_vsi; 3934 } 3935 /* Assign Rx queues from PF space */ 3936 ret = ice_resmgr_assign_scattered(&sc->rx_qmgr, vsi->rx_qmap, 3937 vsi->num_rx_queues); 3938 if (ret) { 3939 device_printf(dev, "Unable to assign mirror VSI Rx queues: %s\n", 3940 ice_err_str(ret)); 3941 goto release_vsi; 3942 } 3943 vsi->qmap_type = ICE_RESMGR_ALLOC_SCATTERED; 3944 vsi->max_frame_size = ICE_MAX_FRAME_SIZE; 3945 3946 ret = ice_initialize_vsi(vsi); 3947 if (ret) { 3948 device_printf(dev, "%s: Error in ice_initialize_vsi for mirror VSI: %s\n", 3949 __func__, ice_err_str(ret)); 3950 goto release_vsi; 3951 } 3952 3953 /* Setup this VSI for receiving traffic */ 3954 ret = ice_config_rss(vsi); 3955 if (ret) { 3956 device_printf(dev, 3957 "Unable to configure RSS for mirror VSI: %s\n", 3958 ice_err_str(ret)); 3959 goto release_vsi; 3960 } 3961 3962 /* Set HW rules for mirroring traffic */ 3963 vsi->mirror_src_vsi = sc->pf_vsi.idx; 3964 3965 ice_debug(&sc->hw, ICE_DBG_INIT, 3966 "Configuring mirroring from VSI %d to %d\n", 3967 vsi->mirror_src_vsi, vsi->idx); 3968 ice_debug(&sc->hw, ICE_DBG_INIT, "(HW num: VSI %d to %d)\n", 3969 ice_get_hw_vsi_num(&sc->hw, vsi->mirror_src_vsi), 3970 ice_get_hw_vsi_num(&sc->hw, vsi->idx)); 3971 3972 ret = ice_setup_vsi_mirroring(vsi); 3973 if (ret) { 3974 device_printf(dev, 3975 "Unable to configure mirroring for VSI: %s\n", 3976 ice_err_str(ret)); 3977 goto release_vsi; 3978 } 3979 3980 return (0); 3981 3982 release_vsi: 3983 ice_release_vsi(vsi); 3984 mif->vsi = NULL; 3985 return (ret); 3986 } 3987 3988 /** 3989 * ice_create_mirror_interface - Initialize mirror interface 3990 * @sc: driver private data 3991 * 3992 * Creates and sets up a mirror interface that will mirror traffic from 3993 * the main PF interface. Includes a call to iflib_device_register() in order 3994 * to setup necessary iflib structures for this new interface as well. 3995 * 3996 * If it returns successfully, a new interface will be created and will show 3997 * up in the ifconfig interface list. 3998 * 3999 * Returns 0 on success, or a standard error code on failure. 4000 */ 4001 int 4002 ice_create_mirror_interface(struct ice_softc *sc) 4003 { 4004 device_t dev = sc->dev; 4005 struct ice_mirr_if *mif; 4006 struct ifmedia *media; 4007 struct sbuf *sb; 4008 int ret = 0; 4009 4010 mif = (struct ice_mirr_if *)malloc(sizeof(*mif), M_ICE, M_ZERO | M_NOWAIT); 4011 if (!mif) { 4012 device_printf(dev, "malloc() error allocating mirror interface\n"); 4013 return (ENOMEM); 4014 } 4015 4016 /* Set pointers */ 4017 sc->mirr_if = mif; 4018 mif->back = sc; 4019 4020 /* Do early setup because these will be called during iflib_device_register(): 4021 * - ice_subif_if_tx_queues_alloc 4022 * - ice_subif_if_rx_queues_alloc 4023 */ 4024 ret = ice_setup_mirror_vsi(mif); 4025 if (ret) 4026 goto out; 4027 4028 /* Determine name for new interface: 4029 * (base interface name)(modifier name)(modifier unit number) 4030 * e.g. for ice0 with a new mirror interface (modifier m) 4031 * of index 0, this equals "ice0m0" 4032 */ 4033 sb = sbuf_new_auto(); 4034 MPASS(sb != NULL); 4035 sbuf_printf(sb, "%sm", device_get_nameunit(dev)); 4036 sbuf_finish(sb); 4037 4038 bus_topo_lock(); 4039 mif->subdev = device_add_child(dev, sbuf_data(sb), 0); 4040 bus_topo_unlock(); 4041 4042 if (!mif->subdev) { 4043 device_printf(dev, "device_add_child failed for %s0\n", sbuf_data(sb)); 4044 sbuf_delete(sb); 4045 free(mif, M_ICE); 4046 sc->mirr_if = NULL; 4047 return (ENOMEM); 4048 } 4049 sbuf_delete(sb); 4050 4051 device_set_driver(mif->subdev, &ice_subif_driver); 4052 4053 /* Use iflib_device_register() directly because the driver already 4054 * has an initialized softc to pass to iflib 4055 */ 4056 ret = iflib_device_register(mif->subdev, mif, &ice_subif_sctx, &mif->subctx); 4057 if (ret) 4058 goto out; 4059 4060 /* Indicate that created interface will be just for monitoring */ 4061 mif->ifp = iflib_get_ifp(mif->subctx); 4062 if_setflagbits(mif->ifp, IFF_MONITOR, 0); 4063 4064 /* Use autoselect media by default */ 4065 media = iflib_get_media(mif->subctx); 4066 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 4067 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 4068 4069 device_printf(dev, "Created dev %s and ifnet %s for mirroring\n", 4070 device_get_nameunit(mif->subdev), if_name(mif->ifp)); 4071 4072 ice_add_vsi_sysctls(mif->vsi); 4073 4074 ret = ice_wire_mirror_intrs(mif); 4075 if (ret) 4076 goto out; 4077 4078 mif->if_attached = true; 4079 return (0); 4080 4081 out: 4082 ice_destroy_mirror_interface(sc); 4083 return (ret); 4084 } 4085 4086 /** 4087 * ice_wire_mirror_intrs 4088 * @mif: driver private subinterface structure 4089 * 4090 * Helper function that sets up driver interrupt data and calls 4091 * into iflib in order to setup interrupts in its data structures as well. 4092 * 4093 * Like ice_if_msix_intr_assign, currently requires that we get at least the same 4094 * number of vectors as we have queues, and that we always have the same number 4095 * of Tx and Rx queues. Unlike that function, this calls a special 4096 * iflib_irq_alloc_generic_subif() function for RX interrupts because the 4097 * driver needs to get MSI-X resources from the parent device. 4098 * 4099 * Tx queues use a softirq instead of using their own hardware interrupt so that 4100 * remains unchanged. 4101 * 4102 * Returns 0 on success or an error code from iflib_irq_alloc_generic_subctx() 4103 * on failure. 4104 */ 4105 static int 4106 ice_wire_mirror_intrs(struct ice_mirr_if *mif) 4107 { 4108 struct ice_softc *sc = mif->back; 4109 struct ice_hw *hw = &sc->hw; 4110 struct ice_vsi *vsi = mif->vsi; 4111 device_t dev = mif->subdev; 4112 int err, i, rid; 4113 4114 if_ctx_t ctx = mif->subctx; 4115 4116 ice_debug(hw, ICE_DBG_INIT, "%s: Last rid: %d\n", __func__, sc->last_rid); 4117 4118 rid = sc->last_rid + 1; 4119 for (i = 0; i < vsi->num_rx_queues; i++, rid++) { 4120 struct ice_rx_queue *rxq = &vsi->rx_queues[i]; 4121 struct ice_tx_queue *txq = &vsi->tx_queues[i]; 4122 char irq_name[16]; 4123 4124 // TODO: Change to use dynamic interface number 4125 snprintf(irq_name, sizeof(irq_name), "m0rxq%d", i); 4126 /* First arg is parent device (physical port's) iflib ctx */ 4127 err = iflib_irq_alloc_generic_subctx(sc->ctx, ctx, 4128 &mif->rx_irqvs[i].irq, rid, IFLIB_INTR_RXTX, ice_msix_que, 4129 rxq, rxq->me, irq_name); 4130 if (err) { 4131 device_printf(dev, 4132 "Failed to allocate q int %d err: %s\n", 4133 i, ice_err_str(err)); 4134 i--; 4135 goto fail; 4136 } 4137 MPASS(rid - 1 > 0); 4138 /* Set vector number used in interrupt enable/disable functions */ 4139 mif->rx_irqvs[i].me = rid - 1; 4140 rxq->irqv = &mif->rx_irqvs[i]; 4141 4142 bzero(irq_name, sizeof(irq_name)); 4143 snprintf(irq_name, sizeof(irq_name), "m0txq%d", i); 4144 iflib_softirq_alloc_generic(ctx, &mif->rx_irqvs[i].irq, 4145 IFLIB_INTR_TX, txq, txq->me, irq_name); 4146 txq->irqv = &mif->rx_irqvs[i]; 4147 } 4148 4149 sc->last_rid = rid - 1; 4150 4151 ice_debug(hw, ICE_DBG_INIT, "%s: New last rid: %d\n", __func__, 4152 sc->last_rid); 4153 4154 return (0); 4155 4156 fail: 4157 for (; i >= 0; i--) 4158 iflib_irq_free(ctx, &mif->rx_irqvs[i].irq); 4159 return (err); 4160 } 4161 4162 /** 4163 * ice_subif_rebuild - Rebuild subinterface post reset 4164 * @sc: The device private softc 4165 * 4166 * Restore subinterface state after a reset occurred. 4167 * Restart the VSI and enable the mirroring. 4168 */ 4169 static int 4170 ice_subif_rebuild(struct ice_softc *sc) 4171 { 4172 struct ice_mirr_if *mif = sc->mirr_if; 4173 struct ice_vsi *vsi = sc->mirr_if->vsi; 4174 int err; 4175 4176 err = ice_subif_rebuild_vsi_qmap(sc); 4177 if (err) { 4178 device_printf(sc->dev, "Unable to re-assign mirror VSI queues, err %s\n", 4179 ice_err_str(err)); 4180 return (err); 4181 } 4182 4183 err = ice_initialize_vsi(vsi); 4184 if (err) { 4185 device_printf(sc->dev, "Unable to re-initialize mirror VSI, err %s\n", 4186 ice_err_str(err)); 4187 goto err_release_queue_allocations_subif; 4188 } 4189 4190 err = ice_config_rss(vsi); 4191 if (err) { 4192 device_printf(sc->dev, 4193 "Unable to reconfigure RSS for the mirror VSI, err %s\n", 4194 ice_err_str(err)); 4195 goto err_deinit_subif_vsi; 4196 } 4197 4198 vsi->mirror_src_vsi = sc->pf_vsi.idx; 4199 4200 err = ice_setup_vsi_mirroring(vsi); 4201 if (err) { 4202 device_printf(sc->dev, 4203 "Unable to configure mirroring for VSI: %s\n", 4204 ice_err_str(err)); 4205 goto err_deinit_subif_vsi; 4206 } 4207 4208 ice_set_state(&mif->state, ICE_STATE_SUBIF_NEEDS_REINIT); 4209 4210 return (0); 4211 4212 err_deinit_subif_vsi: 4213 ice_deinit_vsi(vsi); 4214 err_release_queue_allocations_subif: 4215 ice_resmgr_release_map(&sc->tx_qmgr, vsi->tx_qmap, 4216 sc->mirr_if->num_irq_vectors); 4217 ice_resmgr_release_map(&sc->rx_qmgr, vsi->rx_qmap, 4218 sc->mirr_if->num_irq_vectors); 4219 4220 return (err); 4221 } 4222 4223 /** 4224 * ice_subif_rebuild_vsi_qmap - Rebuild the mirror VSI queue mapping 4225 * @sc: the device softc pointer 4226 * 4227 * Loops over the Tx and Rx queues for the mirror VSI and reassigns the queue 4228 * mapping after a reset occurred. 4229 */ 4230 static int 4231 ice_subif_rebuild_vsi_qmap(struct ice_softc *sc) 4232 { 4233 struct ice_vsi *vsi = sc->mirr_if->vsi; 4234 struct ice_tx_queue *txq; 4235 struct ice_rx_queue *rxq; 4236 int err, i; 4237 4238 err = ice_resmgr_assign_scattered(&sc->tx_qmgr, vsi->tx_qmap, sc->mirr_if->num_irq_vectors); 4239 if (err) { 4240 device_printf(sc->dev, "Unable to assign mirror VSI Tx queues: %s\n", 4241 ice_err_str(err)); 4242 return (err); 4243 } 4244 4245 err = ice_resmgr_assign_scattered(&sc->rx_qmgr, vsi->rx_qmap, sc->mirr_if->num_irq_vectors); 4246 if (err) { 4247 device_printf(sc->dev, "Unable to assign mirror VSI Rx queues: %s\n", 4248 ice_err_str(err)); 4249 goto err_release_tx_queues; 4250 } 4251 4252 vsi->qmap_type = ICE_RESMGR_ALLOC_SCATTERED; 4253 4254 /* Re-assign Tx queue tail pointers */ 4255 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) 4256 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 4257 4258 /* Re-assign Rx queue tail pointers */ 4259 for (i = 0, rxq = vsi->rx_queues; i < vsi->num_rx_queues; i++, rxq++) 4260 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 4261 4262 return (0); 4263 4264 err_release_tx_queues: 4265 ice_resmgr_release_map(&sc->tx_qmgr, vsi->tx_qmap, vsi->num_tx_queues); 4266 4267 return (err); 4268 } 4269 4270 /** 4271 * ice_subif_if_tx_queues_alloc - Allocate Tx queue memory for subinterfaces 4272 * @ctx: iflib context structure 4273 * @vaddrs: virtual addresses for the queue memory 4274 * @paddrs: physical addresses for the queue memory 4275 * @ntxqs: the number of Tx queues per set (should always be 1) 4276 * @ntxqsets: the number of Tx queue sets to allocate 4277 * 4278 * See ice_if_tx_queues_alloc() description. Similar to that function, but 4279 * for subinterfaces instead. 4280 */ 4281 static int 4282 ice_subif_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 4283 int __invariant_only ntxqs, int ntxqsets) 4284 { 4285 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4286 struct ice_tx_queue *txq; 4287 device_t dev = mif->subdev; 4288 struct ice_vsi *vsi; 4289 int err, i, j; 4290 4291 MPASS(mif != NULL); 4292 MPASS(ntxqs == 1); 4293 MPASS(mif->subscctx->isc_ntxd[0] <= ICE_MAX_DESC_COUNT); 4294 4295 vsi = mif->vsi; 4296 4297 MPASS(vsi->num_tx_queues == ntxqsets); 4298 4299 /* Allocate queue structure memory */ 4300 if (!(vsi->tx_queues = 4301 (struct ice_tx_queue *)malloc(sizeof(struct ice_tx_queue) * ntxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 4302 device_printf(dev, "%s: Unable to allocate Tx queue memory for subfunction\n", 4303 __func__); 4304 return (ENOMEM); 4305 } 4306 4307 /* Allocate report status arrays */ 4308 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 4309 if (!(txq->tx_rsq = 4310 (uint16_t *)malloc(sizeof(uint16_t) * mif->subscctx->isc_ntxd[0], M_ICE, M_NOWAIT))) { 4311 device_printf(dev, 4312 "%s: Unable to allocate tx_rsq memory for subfunction\n", __func__); 4313 err = ENOMEM; 4314 goto free_tx_queues; 4315 } 4316 /* Initialize report status array */ 4317 for (j = 0; j < mif->subscctx->isc_ntxd[0]; j++) 4318 txq->tx_rsq[j] = QIDX_INVALID; 4319 } 4320 4321 /* Add Tx queue sysctls context */ 4322 ice_vsi_add_txqs_ctx(vsi); 4323 4324 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 4325 /* q_handle == me when only one TC */ 4326 txq->me = txq->q_handle = i; 4327 txq->vsi = vsi; 4328 4329 /* store the queue size for easier access */ 4330 txq->desc_count = mif->subscctx->isc_ntxd[0]; 4331 4332 /* get the virtual and physical address of the hardware queues */ 4333 txq->tail = QTX_COMM_DBELL(vsi->tx_qmap[i]); 4334 txq->tx_base = (struct ice_tx_desc *)vaddrs[i]; 4335 txq->tx_paddr = paddrs[i]; 4336 4337 ice_add_txq_sysctls(txq); 4338 } 4339 4340 return (0); 4341 4342 free_tx_queues: 4343 for (i = 0, txq = vsi->tx_queues; i < ntxqsets; i++, txq++) { 4344 if (txq->tx_rsq != NULL) { 4345 free(txq->tx_rsq, M_ICE); 4346 txq->tx_rsq = NULL; 4347 } 4348 } 4349 free(vsi->tx_queues, M_ICE); 4350 vsi->tx_queues = NULL; 4351 return (err); 4352 } 4353 4354 /** 4355 * ice_subif_if_rx_queues_alloc - Allocate Rx queue memory for subinterfaces 4356 * @ctx: iflib context structure 4357 * @vaddrs: virtual addresses for the queue memory 4358 * @paddrs: physical addresses for the queue memory 4359 * @nrxqs: number of Rx queues per set (should always be 1) 4360 * @nrxqsets: number of Rx queue sets to allocate 4361 * 4362 * See ice_if_rx_queues_alloc() for general summary; this is similar to that 4363 * but implemented for subinterfaces. 4364 */ 4365 static int 4366 ice_subif_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 4367 int __invariant_only nrxqs, int nrxqsets) 4368 { 4369 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4370 struct ice_rx_queue *rxq; 4371 device_t dev = mif->subdev; 4372 struct ice_vsi *vsi; 4373 int i; 4374 4375 MPASS(mif != NULL); 4376 MPASS(nrxqs == 1); 4377 MPASS(mif->subscctx->isc_nrxd[0] <= ICE_MAX_DESC_COUNT); 4378 4379 vsi = mif->vsi; 4380 4381 MPASS(vsi->num_rx_queues == nrxqsets); 4382 4383 /* Allocate queue structure memory */ 4384 if (!(vsi->rx_queues = 4385 (struct ice_rx_queue *) malloc(sizeof(struct ice_rx_queue) * nrxqsets, M_ICE, M_NOWAIT | M_ZERO))) { 4386 device_printf(dev, "%s: Unable to allocate Rx queue memory for subfunction\n", 4387 __func__); 4388 return (ENOMEM); 4389 } 4390 4391 /* Add Rx queue sysctls context */ 4392 ice_vsi_add_rxqs_ctx(vsi); 4393 4394 for (i = 0, rxq = vsi->rx_queues; i < nrxqsets; i++, rxq++) { 4395 rxq->me = i; 4396 rxq->vsi = vsi; 4397 4398 /* store the queue size for easier access */ 4399 rxq->desc_count = mif->subscctx->isc_nrxd[0]; 4400 4401 /* get the virtual and physical address of the hardware queues */ 4402 rxq->tail = QRX_TAIL(vsi->rx_qmap[i]); 4403 rxq->rx_base = (union ice_32b_rx_flex_desc *)vaddrs[i]; 4404 rxq->rx_paddr = paddrs[i]; 4405 4406 ice_add_rxq_sysctls(rxq); 4407 } 4408 4409 return (0); 4410 } 4411 4412 /** 4413 * ice_subif_if_msix_intr_assign - Assign MSI-X interrupts to new sub interface 4414 * @ctx: the iflib context structure 4415 * @msix: the number of vectors we were assigned 4416 * 4417 * Allocates and assigns driver private resources for MSI-X interrupt tracking. 4418 * 4419 * @pre OS MSI-X resources have been pre-allocated by parent interface. 4420 */ 4421 static int 4422 ice_subif_if_msix_intr_assign(if_ctx_t ctx, int msix) 4423 { 4424 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4425 struct ice_softc *sc = mif->back; 4426 struct ice_vsi *vsi = mif->vsi; 4427 4428 device_t dev = mif->subdev; 4429 int ret; 4430 4431 if (vsi->num_rx_queues != vsi->num_tx_queues) { 4432 device_printf(dev, 4433 "iflib requested %d Tx queues, and %d Rx queues, but the driver isn't able to support a differing number of Tx and Rx queues\n", 4434 vsi->num_tx_queues, vsi->num_rx_queues); 4435 return (EOPNOTSUPP); 4436 } 4437 4438 if (msix > sc->extra_vectors) { 4439 device_printf(dev, 4440 "%s: Not enough spare (%d) msix vectors for new sub-interface requested (%d)\n", 4441 __func__, sc->extra_vectors, msix); 4442 return (ENOSPC); 4443 } 4444 device_printf(dev, "%s: Using %d vectors for sub-interface\n", __func__, 4445 msix); 4446 4447 /* Allocate space to store the IRQ vector data */ 4448 mif->num_irq_vectors = vsi->num_rx_queues; 4449 mif->rx_irqvs = (struct ice_irq_vector *) 4450 malloc(sizeof(struct ice_irq_vector) * (mif->num_irq_vectors), 4451 M_ICE, M_NOWAIT); 4452 if (!mif->rx_irqvs) { 4453 device_printf(dev, 4454 "Unable to allocate RX irqv memory for mirror's %d vectors\n", 4455 mif->num_irq_vectors); 4456 return (ENOMEM); 4457 } 4458 4459 /* Assign mirror interface interrupts from PF device space */ 4460 if (!(mif->if_imap = 4461 (u16 *)malloc(sizeof(u16) * mif->num_irq_vectors, 4462 M_ICE, M_NOWAIT))) { 4463 device_printf(dev, "Unable to allocate mirror intfc if_imap memory\n"); 4464 ret = ENOMEM; 4465 goto free_irqvs; 4466 } 4467 ret = ice_resmgr_assign_contiguous(&sc->dev_imgr, mif->if_imap, mif->num_irq_vectors); 4468 if (ret) { 4469 device_printf(dev, "Unable to assign mirror intfc PF device interrupt mapping: %s\n", 4470 ice_err_str(ret)); 4471 goto free_if_imap; 4472 } 4473 /* Assign mirror interface interrupts from OS interrupt allocation space */ 4474 if (!(mif->os_imap = 4475 (u16 *)malloc(sizeof(u16) * mif->num_irq_vectors, 4476 M_ICE, M_NOWAIT))) { 4477 device_printf(dev, "Unable to allocate mirror intfc os_imap memory\n"); 4478 ret = ENOMEM; 4479 goto free_if_imap; 4480 } 4481 ret = ice_resmgr_assign_contiguous(&sc->os_imgr, mif->os_imap, mif->num_irq_vectors); 4482 if (ret) { 4483 device_printf(dev, "Unable to assign mirror intfc OS interrupt mapping: %s\n", 4484 ice_err_str(ret)); 4485 goto free_if_imap; 4486 } 4487 4488 return (0); 4489 4490 free_if_imap: 4491 free(mif->if_imap, M_ICE); 4492 mif->if_imap = NULL; 4493 free_irqvs: 4494 free(mif->rx_irqvs, M_ICE); 4495 mif->rx_irqvs = NULL; 4496 return (ret); 4497 } 4498 4499 /** 4500 * ice_subif_if_intr_enable - Enable device interrupts for a subinterface 4501 * @ctx: iflib context structure 4502 * 4503 * Called by iflib to request enabling all interrupts that belong to a 4504 * subinterface. 4505 */ 4506 static void 4507 ice_subif_if_intr_enable(if_ctx_t ctx) 4508 { 4509 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4510 struct ice_softc *sc = mif->back; 4511 struct ice_vsi *vsi = mif->vsi; 4512 struct ice_hw *hw = &sc->hw; 4513 4514 /* Do not enable queue interrupts in recovery mode */ 4515 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 4516 return; 4517 4518 /* Enable all queue interrupts */ 4519 for (int i = 0; i < vsi->num_rx_queues; i++) 4520 ice_enable_intr(hw, vsi->rx_queues[i].irqv->me); 4521 } 4522 4523 /** 4524 * ice_subif_if_rx_queue_intr_enable - Enable a specific Rx queue interrupt 4525 * @ctx: iflib context structure 4526 * @rxqid: the Rx queue to enable 4527 * 4528 * Enable a specific Rx queue interrupt. 4529 * 4530 * This function is not protected by the iflib CTX lock. 4531 */ 4532 static int 4533 ice_subif_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 4534 { 4535 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4536 struct ice_softc *sc = mif->back; 4537 struct ice_vsi *vsi = mif->vsi; 4538 struct ice_hw *hw = &sc->hw; 4539 4540 /* Do not enable queue interrupts in recovery mode */ 4541 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 4542 return (ENOSYS); 4543 4544 ice_enable_intr(hw, vsi->rx_queues[rxqid].irqv->me); 4545 return (0); 4546 } 4547 4548 /** 4549 * ice_subif_if_tx_queue_intr_enable - Enable a specific Tx queue interrupt 4550 * @ctx: iflib context structure 4551 * @txqid: the Tx queue to enable 4552 * 4553 * Enable a specific Tx queue interrupt. 4554 * 4555 * This function is not protected by the iflib CTX lock. 4556 */ 4557 static int 4558 ice_subif_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 4559 { 4560 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4561 struct ice_softc *sc = mif->back; 4562 struct ice_vsi *vsi = mif->vsi; 4563 struct ice_hw *hw = &sc->hw; 4564 4565 /* Do not enable queue interrupts in recovery mode */ 4566 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 4567 return (ENOSYS); 4568 4569 ice_enable_intr(hw, vsi->tx_queues[txqid].irqv->me); 4570 return (0); 4571 } 4572 4573 /** 4574 * ice_subif_if_init - Initialize the subinterface 4575 * @ctx: iflib ctx structure 4576 * 4577 * Called by iflib to bring the device up, i.e. ifconfig ice0m0 up. 4578 * Prepares the Tx and Rx engines and enables interrupts. 4579 * 4580 * @pre assumes the caller holds the iflib CTX lock 4581 */ 4582 static void 4583 ice_subif_if_init(if_ctx_t ctx) 4584 { 4585 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4586 struct ice_softc *sc = mif->back; 4587 struct ice_vsi *vsi = mif->vsi; 4588 device_t dev = mif->subdev; 4589 int err; 4590 4591 if (ice_driver_is_detaching(sc)) 4592 return; 4593 4594 if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) 4595 goto err_init_failed; 4596 4597 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 4598 device_printf(dev, 4599 "request to start interface cannot be completed as the parent device %s failed to reset\n", 4600 device_get_nameunit(sc->dev)); 4601 goto err_init_failed; 4602 } 4603 4604 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 4605 device_printf(dev, 4606 "request to start interface cannot be completed while parent device %s is prepared for impending reset\n", 4607 device_get_nameunit(sc->dev)); 4608 goto err_init_failed; 4609 } 4610 4611 /* XXX: Equiv to ice_update_rx_mbuf_sz */ 4612 vsi->mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 4613 4614 /* Initialize software Tx tracking values */ 4615 ice_init_tx_tracking(vsi); 4616 4617 err = ice_cfg_vsi_for_tx(vsi); 4618 if (err) { 4619 device_printf(dev, 4620 "Unable to configure subif VSI for Tx: %s\n", 4621 ice_err_str(err)); 4622 goto err_cleanup_tx; 4623 } 4624 4625 err = ice_cfg_vsi_for_rx(vsi); 4626 if (err) { 4627 device_printf(dev, 4628 "Unable to configure subif VSI for Rx: %s\n", 4629 ice_err_str(err)); 4630 goto err_cleanup_tx; 4631 } 4632 4633 err = ice_control_all_rx_queues(vsi, true); 4634 if (err) { 4635 device_printf(dev, 4636 "Unable to enable subif Rx rings for receive: %s\n", 4637 ice_err_str(err)); 4638 goto err_stop_rx; 4639 } 4640 4641 ice_configure_all_rxq_interrupts(vsi); 4642 ice_configure_rx_itr(vsi); 4643 4644 ice_set_state(&mif->state, ICE_STATE_DRIVER_INITIALIZED); 4645 return; 4646 4647 err_stop_rx: 4648 ice_control_all_rx_queues(vsi, false); 4649 err_cleanup_tx: 4650 ice_vsi_disable_tx(vsi); 4651 err_init_failed: 4652 iflib_init_failed(ctx); 4653 } 4654 4655 /** 4656 * ice_if_stop_subif - Stop the subinterface 4657 * @ctx: iflib context structure 4658 * @ifs: subinterface context structure 4659 * 4660 * Called by iflib to stop the subinterface and bring it down. 4661 * (e.g. ifconfig ice0m0 down) 4662 * 4663 * @pre assumes the caller holds the iflib CTX lock 4664 */ 4665 static void 4666 ice_subif_if_stop(if_ctx_t ctx) 4667 { 4668 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4669 struct ice_softc *sc = mif->back; 4670 struct ice_vsi *vsi = mif->vsi; 4671 device_t dev = mif->subdev; 4672 4673 if (!ice_testandclear_state(&mif->state, ICE_STATE_DRIVER_INITIALIZED)) 4674 return; 4675 4676 if (ice_test_state(&sc->state, ICE_STATE_RESET_FAILED)) { 4677 device_printf(dev, 4678 "request to stop interface cannot be completed as the parent device %s failed to reset\n", 4679 device_get_nameunit(sc->dev)); 4680 return; 4681 } 4682 4683 if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) { 4684 device_printf(dev, 4685 "request to stop interface cannot be completed while parent device %s is prepared for impending reset\n", 4686 device_get_nameunit(sc->dev)); 4687 return; 4688 } 4689 4690 /* Dissociate the Tx and Rx queues from the interrupts */ 4691 ice_flush_txq_interrupts(vsi); 4692 ice_flush_rxq_interrupts(vsi); 4693 4694 /* Disable the Tx and Rx queues */ 4695 ice_vsi_disable_tx(vsi); 4696 ice_control_all_rx_queues(vsi, false); 4697 } 4698 4699 /** 4700 * ice_free_irqvs_subif - Free IRQ vector memory for subinterfaces 4701 * @mif: Mirror interface private structure 4702 * 4703 * Free IRQ vector memory allocated during ice_subif_if_msix_intr_assign. 4704 */ 4705 static void 4706 ice_free_irqvs_subif(struct ice_mirr_if *mif) 4707 { 4708 struct ice_softc *sc = mif->back; 4709 struct ice_vsi *vsi = mif->vsi; 4710 if_ctx_t ctx = sc->ctx; 4711 int i; 4712 4713 /* If the irqvs array is NULL, then there are no vectors to free */ 4714 if (mif->rx_irqvs == NULL) 4715 return; 4716 4717 /* Free the IRQ vectors -- currently subinterfaces have number 4718 * of vectors equal to number of RX queues 4719 * 4720 * XXX: ctx is parent device's ctx, not the subinterface ctx 4721 */ 4722 for (i = 0; i < vsi->num_rx_queues; i++) 4723 iflib_irq_free(ctx, &mif->rx_irqvs[i].irq); 4724 4725 ice_resmgr_release_map(&sc->os_imgr, mif->os_imap, 4726 mif->num_irq_vectors); 4727 ice_resmgr_release_map(&sc->dev_imgr, mif->if_imap, 4728 mif->num_irq_vectors); 4729 4730 sc->last_rid -= vsi->num_rx_queues; 4731 4732 /* Clear the irqv pointers */ 4733 for (i = 0; i < vsi->num_rx_queues; i++) 4734 vsi->rx_queues[i].irqv = NULL; 4735 4736 for (i = 0; i < vsi->num_tx_queues; i++) 4737 vsi->tx_queues[i].irqv = NULL; 4738 4739 /* Release the vector array memory */ 4740 free(mif->rx_irqvs, M_ICE); 4741 mif->rx_irqvs = NULL; 4742 } 4743 4744 /** 4745 * ice_subif_if_queues_free - Free queue memory for subinterfaces 4746 * @ctx: the iflib context structure 4747 * 4748 * Free queue memory allocated by ice_subif_tx_queues_alloc() and 4749 * ice_subif_if_rx_queues_alloc(). 4750 */ 4751 static void 4752 ice_subif_if_queues_free(if_ctx_t ctx) 4753 { 4754 struct ice_mirr_if *mif = (struct ice_mirr_if *)iflib_get_softc(ctx); 4755 struct ice_vsi *vsi = mif->vsi; 4756 struct ice_tx_queue *txq; 4757 int i; 4758 4759 /* Free the Tx and Rx sysctl contexts, and assign NULL to the node 4760 * pointers. 4761 */ 4762 ice_vsi_del_txqs_ctx(vsi); 4763 ice_vsi_del_rxqs_ctx(vsi); 4764 4765 /* Release MSI-X IRQ vectors */ 4766 ice_free_irqvs_subif(mif); 4767 4768 if (vsi->tx_queues != NULL) { 4769 /* free the tx_rsq arrays */ 4770 for (i = 0, txq = vsi->tx_queues; i < vsi->num_tx_queues; i++, txq++) { 4771 if (txq->tx_rsq != NULL) { 4772 free(txq->tx_rsq, M_ICE); 4773 txq->tx_rsq = NULL; 4774 } 4775 } 4776 free(vsi->tx_queues, M_ICE); 4777 vsi->tx_queues = NULL; 4778 } 4779 if (vsi->rx_queues != NULL) { 4780 free(vsi->rx_queues, M_ICE); 4781 vsi->rx_queues = NULL; 4782 } 4783 } 4784 4785 /** 4786 * ice_subif_if_media_status - Report subinterface media 4787 * @ctx: iflib context structure 4788 * @ifmr: ifmedia request structure to update 4789 * 4790 * Updates the provided ifmr with something, in order to prevent a 4791 * "no media types?" message from ifconfig. 4792 * 4793 * Mirror interfaces are always up. 4794 */ 4795 static void 4796 ice_subif_if_media_status(if_ctx_t ctx __unused, struct ifmediareq *ifmr) 4797 { 4798 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 4799 ifmr->ifm_active = IFM_ETHER | IFM_AUTO; 4800 } 4801 4802 /** 4803 * ice_subif_if_promisc_set - Set subinterface promiscuous mode 4804 * @ctx: iflib context structure 4805 * @flags: promiscuous flags to configure 4806 * 4807 * Called by iflib to configure device promiscuous mode. 4808 * 4809 * @remark This does not need to be implemented for now. 4810 */ 4811 static int 4812 ice_subif_if_promisc_set(if_ctx_t ctx __unused, int flags __unused) 4813 { 4814 return (0); 4815 } 4816 4817