1 /***************************************************************************** 2 3 Copyright (c) 2001-2017, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 *****************************************************************************/ 33 34 35 #include "opt_inet.h" 36 #include "opt_inet6.h" 37 #include "opt_rss.h" 38 39 #include "ixgbe.h" 40 #include "ixgbe_hv_vf.h" 41 #include "ifdi_if.h" 42 43 #include <net/netmap.h> 44 #include <dev/netmap/netmap_kern.h> 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 static const char ixv_driver_version[] = "2.0.1-k"; 50 51 static const sbintime_t ixv_mbx_retry_delay[] = { 52 250 * SBT_1MS, 53 1 * SBT_1S, 54 4 * SBT_1S, 55 8 * SBT_1S, 56 }; 57 58 static const struct timeval ixv_mbx_log_interval = { 60, 0 }; 59 60 /* Bound stale carrier state without flapping on a busy PF mailbox. */ 61 #define IXV_LINK_MBX_FAILURE_LIMIT 3 62 /* Match Intel's two-second VF service timer and spread PF mailbox load. */ 63 #define IXV_LINK_POLL_TICKS 4 64 65 /************************************************************************ 66 * PCI Device ID Table 67 * 68 * Used by probe to select devices to load on 69 * Last field stores an index into ixv_strings 70 * Last entry must be all 0s 71 * 72 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 73 ************************************************************************/ 74 static const pci_vendor_info_t ixv_vendor_info_array[] = 75 { 76 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 77 "Intel(R) X520 82599 Virtual Function"), 78 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV, 79 "Intel(R) X520 82599 Hyper-V Virtual Function"), 80 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 81 "Intel(R) X540 Virtual Function"), 82 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV, 83 "Intel(R) X540 Hyper-V Virtual Function"), 84 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 85 "Intel(R) X550 Virtual Function"), 86 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV, 87 "Intel(R) X550 Hyper-V Virtual Function"), 88 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 89 "Intel(R) X552 Virtual Function"), 90 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV, 91 "Intel(R) X552 Hyper-V Virtual Function"), 92 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 93 "Intel(R) X553 Virtual Function"), 94 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV, 95 "Intel(R) X553 Hyper-V Virtual Function"), 96 PVID_OEM(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF, 0, 97 IXGBE_SUBDEV_ID_E610_VF_HV, 0, 98 "Intel(R) E610 Hyper-V Virtual Function"), 99 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF, 100 "Intel(R) E610 Virtual Function"), 101 /* required last entry */ 102 PVID_END 103 }; 104 105 /************************************************************************ 106 * Function prototypes 107 ************************************************************************/ 108 static void *ixv_register(device_t); 109 static int ixv_if_attach_pre(if_ctx_t); 110 static int ixv_if_attach_post(if_ctx_t); 111 static int ixv_if_detach(if_ctx_t); 112 113 static int ixv_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 114 static int ixv_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 115 int); 116 static int ixv_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 117 int); 118 static void ixv_if_queues_free(if_ctx_t); 119 static void ixv_identify_hardware(if_ctx_t); 120 static bool ixv_is_hyperv(const struct ixgbe_hw *); 121 static void ixv_init_device_features(struct ixgbe_softc *); 122 static int ixv_allocate_pci_resources(if_ctx_t); 123 static void ixv_free_pci_resources(if_ctx_t); 124 static int ixv_setup_interface(if_ctx_t); 125 static void ixv_if_media_status(if_ctx_t, struct ifmediareq *); 126 static int ixv_if_media_change(if_ctx_t); 127 static void ixv_if_update_admin_status(if_ctx_t); 128 static int ixv_if_msix_intr_assign(if_ctx_t, int); 129 130 static int ixv_if_mtu_set(if_ctx_t, uint32_t); 131 static int ixv_if_get_rss_key(if_ctx_t, struct ifrsskey *); 132 static int ixv_if_get_rss_hash(if_ctx_t, struct ifrsshash *); 133 static void ixv_reconcile_mac(struct ixgbe_softc *, if_t); 134 static void ixv_if_init(if_ctx_t); 135 static void ixv_if_local_timer(if_ctx_t, uint16_t); 136 static void ixv_if_stop(if_ctx_t); 137 static void ixv_log_negotiate_failure(struct ixgbe_softc *, bool); 138 static void ixv_log_reset_failure(struct ixgbe_softc *, s32, bool); 139 static void ixv_mbx_retry_detach(struct ixgbe_softc *); 140 static void ixv_mbx_retry_failed(if_ctx_t); 141 static void ixv_mbx_retry_prepare(struct ixgbe_softc *); 142 static void ixv_mbx_retry_stop(struct ixgbe_softc *); 143 static void ixv_mbx_retry_succeeded(struct ixgbe_softc *); 144 static int ixv_negotiate_api(struct ixgbe_softc *); 145 static int ixv_queue_limit(struct ixgbe_softc *, bool); 146 147 static void ixv_initialize_transmit_units(if_ctx_t); 148 static void ixv_initialize_receive_units(if_ctx_t); 149 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 150 151 static void ixv_setup_vlan_support(if_ctx_t); 152 static bool ixv_vlan_error_retryable(s32); 153 static void ixv_vlan_retry_add(struct ixgbe_softc *, u16); 154 static void ixv_vlan_retry_clear(struct ixgbe_softc *, u16); 155 static bool ixv_vlan_retry_pending(const struct ixgbe_softc *); 156 static void ixv_vlan_retry_tick(struct ixgbe_softc *); 157 static void ixv_configure_ivars(struct ixgbe_softc *); 158 static void ixv_if_enable_intr(if_ctx_t); 159 static void ixv_if_disable_intr(if_ctx_t); 160 static void ixv_if_multi_set(if_ctx_t); 161 static int ixv_if_promisc_set(if_ctx_t, int); 162 163 static void ixv_if_register_vlan(if_ctx_t, u16); 164 static void ixv_if_unregister_vlan(if_ctx_t, u16); 165 166 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter); 167 static bool ixv_if_needs_restart(if_ctx_t, enum iflib_restart_event); 168 169 static void ixv_init_stats(struct ixgbe_softc *); 170 static void ixv_update_stats(struct ixgbe_softc *); 171 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 172 173 static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 174 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 175 176 static u8 *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 177 178 /* The MSI-X Interrupt handlers */ 179 static int ixv_msix_que(void *); 180 static int ixv_msix_mbx(void *); 181 182 /************************************************************************ 183 * FreeBSD Device Interface Entry Points 184 ************************************************************************/ 185 static device_method_t ixv_methods[] = { 186 /* Device interface */ 187 DEVMETHOD(device_register, ixv_register), 188 DEVMETHOD(device_probe, iflib_device_probe), 189 DEVMETHOD(device_attach, iflib_device_attach), 190 DEVMETHOD(device_detach, iflib_device_detach), 191 DEVMETHOD(device_shutdown, iflib_device_shutdown), 192 DEVMETHOD(device_suspend, iflib_device_suspend), 193 DEVMETHOD(device_resume, iflib_device_resume), 194 DEVMETHOD_END 195 }; 196 197 static driver_t ixv_driver = { 198 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 199 }; 200 201 DRIVER_MODULE(ixv, pci, ixv_driver, 0, 0); 202 IFLIB_PNP_INFO(pci, ixv_driver, ixv_vendor_info_array); 203 MODULE_DEPEND(ixv, iflib, 1, 1, 1); 204 MODULE_DEPEND(ixv, pci, 1, 1, 1); 205 MODULE_DEPEND(ixv, ether, 1, 1, 1); 206 207 static device_method_t ixv_if_methods[] = { 208 DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre), 209 DEVMETHOD(ifdi_attach_post, ixv_if_attach_post), 210 DEVMETHOD(ifdi_detach, ixv_if_detach), 211 DEVMETHOD(ifdi_init, ixv_if_init), 212 DEVMETHOD(ifdi_stop, ixv_if_stop), 213 DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign), 214 DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr), 215 DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr), 216 DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable), 217 DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable), 218 DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc), 219 DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc), 220 DEVMETHOD(ifdi_queues_free, ixv_if_queues_free), 221 DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status), 222 DEVMETHOD(ifdi_multi_set, ixv_if_multi_set), 223 DEVMETHOD(ifdi_promisc_set, ixv_if_promisc_set), 224 DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set), 225 DEVMETHOD(ifdi_get_rss_key, ixv_if_get_rss_key), 226 DEVMETHOD(ifdi_get_rss_hash, ixv_if_get_rss_hash), 227 DEVMETHOD(ifdi_media_status, ixv_if_media_status), 228 DEVMETHOD(ifdi_media_change, ixv_if_media_change), 229 DEVMETHOD(ifdi_timer, ixv_if_local_timer), 230 DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan), 231 DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan), 232 DEVMETHOD(ifdi_get_counter, ixv_if_get_counter), 233 DEVMETHOD(ifdi_needs_restart, ixv_if_needs_restart), 234 DEVMETHOD_END 235 }; 236 237 static driver_t ixv_if_driver = { 238 "ixv_if", ixv_if_methods, sizeof(struct ixgbe_softc) 239 }; 240 241 #define IXV_VLAN_RETRY_BATCH 4 242 #define IXV_VLAN_RETRY_WINDOW (8 * SBT_1S) 243 244 extern struct if_txrx ixgbe_txrx; 245 246 static struct if_shared_ctx ixv_sctx_init = { 247 .isc_magic = IFLIB_MAGIC, 248 .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ 249 .isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header), 250 .isc_tx_maxsegsize = PAGE_SIZE, 251 .isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header), 252 .isc_tso_maxsegsize = PAGE_SIZE, 253 .isc_rx_maxsize = MJUM16BYTES, 254 .isc_rx_nsegments = 1, 255 .isc_rx_maxsegsize = MJUM16BYTES, 256 .isc_nfl = 1, 257 .isc_ntxqs = 1, 258 .isc_nrxqs = 1, 259 .isc_admin_intrcnt = 1, 260 .isc_vendor_info = ixv_vendor_info_array, 261 .isc_driver_version = ixv_driver_version, 262 .isc_driver = &ixv_if_driver, 263 .isc_flags = IFLIB_IS_VF | IFLIB_TSO_INIT_IP, 264 265 .isc_nrxd_min = {MIN_RXD}, 266 .isc_ntxd_min = {MIN_TXD}, 267 .isc_nrxd_max = {MAX_RXD}, 268 .isc_ntxd_max = {MAX_TXD}, 269 .isc_nrxd_default = {DEFAULT_RXD}, 270 .isc_ntxd_default = {DEFAULT_TXD}, 271 }; 272 273 static void * 274 ixv_register(device_t dev) 275 { 276 return (&ixv_sctx_init); 277 } 278 279 /************************************************************************ 280 * ixv_if_tx_queues_alloc 281 ************************************************************************/ 282 static int 283 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 284 int ntxqs, int ntxqsets) 285 { 286 struct ixgbe_softc *sc = iflib_get_softc(ctx); 287 if_softc_ctx_t scctx = sc->shared; 288 struct ix_tx_queue *que; 289 int i, j, error; 290 291 MPASS(sc->num_tx_queues == ntxqsets); 292 MPASS(ntxqs == 1); 293 294 /* Allocate queue structure memory */ 295 sc->tx_queues = 296 (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * 297 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 298 if (!sc->tx_queues) { 299 device_printf(iflib_get_dev(ctx), 300 "Unable to allocate TX ring memory\n"); 301 return (ENOMEM); 302 } 303 304 for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) { 305 struct tx_ring *txr = &que->txr; 306 307 txr->me = i; 308 txr->sc = que->sc = sc; 309 310 /* Allocate report status array */ 311 if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * 312 scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { 313 error = ENOMEM; 314 goto fail; 315 } 316 for (j = 0; j < scctx->isc_ntxd[0]; j++) 317 txr->tx_rsq[j] = QIDX_INVALID; 318 /* get virtual and physical address of the hardware queues */ 319 txr->tail = IXGBE_VFTDT(txr->me); 320 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs]; 321 txr->tx_paddr = paddrs[i*ntxqs]; 322 323 txr->bytes = 0; 324 txr->total_packets = 0; 325 326 } 327 328 device_printf(iflib_get_dev(ctx), "allocated for %d queues\n", 329 sc->num_tx_queues); 330 331 return (0); 332 333 fail: 334 ixv_if_queues_free(ctx); 335 336 return (error); 337 } /* ixv_if_tx_queues_alloc */ 338 339 /************************************************************************ 340 * ixv_if_rx_queues_alloc 341 ************************************************************************/ 342 static int 343 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 344 int nrxqs, int nrxqsets) 345 { 346 struct ixgbe_softc *sc = iflib_get_softc(ctx); 347 struct ix_rx_queue *que; 348 int i, error; 349 350 MPASS(sc->num_rx_queues == nrxqsets); 351 MPASS(nrxqs == 1); 352 353 /* Allocate queue structure memory */ 354 sc->rx_queues = 355 (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) * 356 nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 357 if (!sc->rx_queues) { 358 device_printf(iflib_get_dev(ctx), 359 "Unable to allocate TX ring memory\n"); 360 error = ENOMEM; 361 goto fail; 362 } 363 364 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 365 struct rx_ring *rxr = &que->rxr; 366 rxr->me = i; 367 rxr->sc = que->sc = sc; 368 369 370 /* get the virtual and physical address of the hw queues */ 371 rxr->tail = IXGBE_VFRDT(rxr->me); 372 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i]; 373 rxr->rx_paddr = paddrs[i*nrxqs]; 374 rxr->bytes = 0; 375 rxr->que = que; 376 } 377 378 device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n", 379 sc->num_rx_queues); 380 381 return (0); 382 383 fail: 384 ixv_if_queues_free(ctx); 385 386 return (error); 387 } /* ixv_if_rx_queues_alloc */ 388 389 /************************************************************************ 390 * ixv_if_queues_free 391 ************************************************************************/ 392 static void 393 ixv_if_queues_free(if_ctx_t ctx) 394 { 395 struct ixgbe_softc *sc = iflib_get_softc(ctx); 396 struct ix_tx_queue *que = sc->tx_queues; 397 int i; 398 399 if (que == NULL) 400 goto free; 401 402 for (i = 0; i < sc->num_tx_queues; i++, que++) { 403 struct tx_ring *txr = &que->txr; 404 if (txr->tx_rsq == NULL) 405 break; 406 407 free(txr->tx_rsq, M_DEVBUF); 408 txr->tx_rsq = NULL; 409 } 410 if (sc->tx_queues != NULL) 411 free(sc->tx_queues, M_DEVBUF); 412 free: 413 if (sc->rx_queues != NULL) 414 free(sc->rx_queues, M_DEVBUF); 415 sc->tx_queues = NULL; 416 sc->rx_queues = NULL; 417 } /* ixv_if_queues_free */ 418 419 /************************************************************************ 420 * ixv_if_attach_pre - Device initialization routine 421 * 422 * Called when the driver is being loaded. 423 * Identifies the type of hardware, allocates all resources 424 * and initializes the hardware. 425 * 426 * return 0 on success, positive on failure 427 ************************************************************************/ 428 static int 429 ixv_if_attach_pre(if_ctx_t ctx) 430 { 431 struct ixgbe_softc *sc; 432 device_t dev; 433 if_softc_ctx_t scctx; 434 struct ixgbe_hw *hw; 435 bool mailbox_ready; 436 int error = 0; 437 438 INIT_DEBUGOUT("ixv_attach: begin"); 439 440 /* Allocate, clear, and link in our sc structure */ 441 dev = iflib_get_dev(ctx); 442 sc = iflib_get_softc(ctx); 443 sc->dev = dev; 444 sc->ctx = ctx; 445 sc->hw.back = sc; 446 scctx = sc->shared = iflib_get_softc_ctx(ctx); 447 sc->media = iflib_get_media(ctx); 448 hw = &sc->hw; 449 450 /* Do base PCI setup - map BAR0 */ 451 if (ixv_allocate_pci_resources(ctx)) { 452 device_printf(dev, "ixv_allocate_pci_resources() failed!\n"); 453 error = ENXIO; 454 goto err_out; 455 } 456 457 /* SYSCTL APIs */ 458 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 459 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", 460 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 461 sc, 0, ixv_sysctl_debug, "I", "Debug Info"); 462 463 /* Determine hardware revision */ 464 ixv_identify_hardware(ctx); 465 ixv_init_device_features(sc); 466 sc->vf_link_poll_tick = device_get_unit(dev) % IXV_LINK_POLL_TICKS; 467 468 /* Initialize the shared code */ 469 if (ixv_is_hyperv(hw)) 470 error = ixgbevf_hv_init_ops_vf(hw); 471 else 472 error = ixgbe_init_ops_vf(hw); 473 if (error) { 474 device_printf(dev, "VF shared-code initialization failed\n"); 475 error = EIO; 476 goto err_out; 477 } 478 479 /* Setup the mailbox */ 480 ixgbe_init_mbx_params_vf(hw); 481 482 mailbox_ready = false; 483 error = hw->mac.ops.reset_hw(hw); 484 if (error != IXGBE_SUCCESS) { 485 /* 486 * A PF may be resetting or servicing a slow link event while its 487 * VFs are enumerated. Keep the VF attached so a later if_init can 488 * retry the mailbox handshake. 489 */ 490 ixv_log_reset_failure(sc, error, true); 491 } else { 492 error = hw->mac.ops.init_hw(hw); 493 if (error != IXGBE_SUCCESS) { 494 device_printf(dev, "...init_hw() failed with error %d\n", 495 error); 496 error = EIO; 497 goto err_out; 498 } 499 500 /* Negotiate mailbox API version. */ 501 error = ixv_negotiate_api(sc); 502 if (error != 0) { 503 ixv_log_negotiate_failure(sc, true); 504 hw->mac.ops.stop_adapter(hw); 505 } else 506 mailbox_ready = true; 507 } 508 509 /* Check if VF was disabled by PF. */ 510 if (!mailbox_ready || 511 hw->mac.ops.get_link_state(hw, &sc->link_enabled) != 0) { 512 /* PF is not capable of controlling VF state. Enable link. */ 513 sc->link_enabled = true; 514 } 515 516 /* If no mac address was assigned, make a random one */ 517 if (!ixv_check_ether_addr(hw->mac.addr)) { 518 ether_gen_addr(iflib_get_ifp(ctx), 519 (struct ether_addr *)hw->mac.addr); 520 bcopy(hw->mac.addr, hw->mac.perm_addr, 521 sizeof(hw->mac.perm_addr)); 522 } 523 524 /* Most of the iflib initialization... */ 525 526 iflib_set_mac(ctx, hw->mac.addr); 527 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 528 ixv_queue_limit(sc, mailbox_ready); 529 scctx->isc_txqsizes[0] = 530 roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) + 531 sizeof(u32), DBA_ALIGN); 532 scctx->isc_rxqsizes[0] = 533 roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc), 534 DBA_ALIGN); 535 /* XXX */ 536 scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO | 537 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO | CSUM_SCTP | 538 CSUM_IP6_SCTP; 539 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER; 540 scctx->isc_msix_bar = pci_msix_table_bar(dev); 541 scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; 542 scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE; 543 scctx->isc_tx_tso_segsize_max = PAGE_SIZE; 544 545 scctx->isc_txrx = &ixgbe_txrx; 546 547 /* We support everything the PF does; VFs do not do WoL. */ 548 scctx->isc_capabilities = IXGBE_CAPS; 549 scctx->isc_capenable = scctx->isc_capabilities; 550 atomic_store_rel_32(&sc->vf_mbx_ready, mailbox_ready); 551 callout_init(&sc->vf_mbx_retry, 1); 552 sc->vf_mbx_retry_initialized = true; 553 554 INIT_DEBUGOUT("ixv_if_attach_pre: end"); 555 556 return (0); 557 558 err_out: 559 ixv_free_pci_resources(ctx); 560 561 return (error); 562 } /* ixv_if_attach_pre */ 563 564 static int 565 ixv_if_attach_post(if_ctx_t ctx) 566 { 567 struct ixgbe_softc *sc = iflib_get_softc(ctx); 568 device_t dev = iflib_get_dev(ctx); 569 int error = 0; 570 571 /* Setup OS specific network interface */ 572 error = ixv_setup_interface(ctx); 573 if (error) { 574 device_printf(dev, "Interface setup failed: %d\n", error); 575 goto end; 576 } 577 578 /* Do the stats setup */ 579 if (atomic_load_acq_32(&sc->vf_mbx_ready) != 0) 580 ixv_init_stats(sc); 581 ixv_add_stats_sysctls(sc); 582 583 end: 584 return error; 585 } /* ixv_if_attach_post */ 586 587 /************************************************************************ 588 * ixv_detach - Device removal routine 589 * 590 * Called when the driver is being removed. 591 * Stops the adapter and deallocates all the resources 592 * that were allocated for driver operation. 593 * 594 * return 0 on success, positive on failure 595 ************************************************************************/ 596 static int 597 ixv_if_detach(if_ctx_t ctx) 598 { 599 struct ixgbe_softc *sc; 600 601 INIT_DEBUGOUT("ixv_detach: begin"); 602 603 sc = iflib_get_softc(ctx); 604 ixv_mbx_retry_detach(sc); 605 ixv_free_pci_resources(ctx); 606 607 return (0); 608 } /* ixv_if_detach */ 609 610 /************************************************************************ 611 * ixv_if_mtu_set 612 ************************************************************************/ 613 static int 614 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 615 { 616 struct ixgbe_softc *sc = iflib_get_softc(ctx); 617 if_t ifp = iflib_get_ifp(ctx); 618 uint32_t max_mtu; 619 int error = 0; 620 621 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 622 max_mtu = IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR; 623 /* The 82599 VF cannot enforce the X540 RLPML field. */ 624 if (ixv_is_hyperv(&sc->hw) && 625 sc->hw.mac.type == ixgbe_mac_82599_vf) 626 max_mtu = ETHERMTU + ETHER_CRC_LEN; 627 if (mtu > max_mtu) { 628 error = EINVAL; 629 } else { 630 if_setmtu(ifp, mtu); 631 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR; 632 } 633 634 return error; 635 } /* ixv_if_mtu_set */ 636 637 static int 638 ixv_rss_query_status(if_ctx_t ctx) 639 { 640 struct ixgbe_softc *sc = iflib_get_softc(ctx); 641 642 sx_assert(iflib_ctx_lock_get(ctx), SA_XLOCKED); 643 /* Older VFs share PF-controlled RSS settings which we cannot query. */ 644 if (sc->hw.mac.type < ixgbe_mac_X550_vf) 645 return (EOPNOTSUPP); 646 if (atomic_load_acq_32(&sc->vf_mbx_ready) == 0 || 647 sc->vf_rss_mrqc == 0) 648 return (ENXIO); 649 return (0); 650 } 651 652 static int 653 ixv_if_get_rss_key(if_ctx_t ctx, struct ifrsskey *ifrk) 654 { 655 struct ixgbe_softc *sc = iflib_get_softc(ctx); 656 int error; 657 658 error = ixv_rss_query_status(ctx); 659 if (error != 0) 660 return (error); 661 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; 662 ifrk->ifrk_keylen = sizeof(sc->vf_rss_key); 663 _Static_assert(sizeof(ifrk->ifrk_key) >= sizeof(sc->vf_rss_key), 664 "RSS query buffer too small"); 665 bzero(ifrk->ifrk_key, sizeof(ifrk->ifrk_key)); 666 /* Preserve the byte order of the programmed registers. */ 667 for (u_int i = 0; i < nitems(sc->vf_rss_key); i++) 668 le32enc(ifrk->ifrk_key + i * sizeof(u32), sc->vf_rss_key[i]); 669 return (0); 670 } 671 672 static int 673 ixv_if_get_rss_hash(if_ctx_t ctx, struct ifrsshash *ifrh) 674 { 675 struct ixgbe_softc *sc = iflib_get_softc(ctx); 676 u32 mrqc; 677 int error; 678 679 error = ixv_rss_query_status(ctx); 680 if (error != 0) 681 return (error); 682 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; 683 ifrh->ifrh_types = 0; 684 mrqc = sc->vf_rss_mrqc; 685 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4) 686 ifrh->ifrh_types |= RSS_TYPE_IPV4; 687 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP) 688 ifrh->ifrh_types |= RSS_TYPE_TCP_IPV4; 689 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP) 690 ifrh->ifrh_types |= RSS_TYPE_UDP_IPV4; 691 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6) 692 ifrh->ifrh_types |= RSS_TYPE_IPV6; 693 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP) 694 ifrh->ifrh_types |= RSS_TYPE_TCP_IPV6; 695 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP) 696 ifrh->ifrh_types |= RSS_TYPE_UDP_IPV6; 697 return (0); 698 } 699 700 static void 701 ixv_reconcile_mac(struct ixgbe_softc *sc, if_t ifp) 702 { 703 uint8_t *lladdr; 704 705 if (ixgbe_validate_mac_addr(sc->hw.mac.addr) != IXGBE_SUCCESS) 706 return; 707 lladdr = (uint8_t *)if_getlladdr(ifp); 708 if (bcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) 709 return; 710 711 device_printf(sc->dev, 712 "PF rejected or replaced the requested MAC; using %6D\n", 713 sc->hw.mac.addr, ":"); 714 /* 715 * Initialization holds the context lock; avoid re-entering the driver. 716 */ 717 bcopy(sc->hw.mac.addr, lladdr, ETHER_ADDR_LEN); 718 CURVNET_SET_QUIET(if_getvnet(ifp)); 719 EVENTHANDLER_INVOKE(iflladdr_event, ifp); 720 CURVNET_RESTORE(); 721 } /* ixv_reconcile_mac */ 722 723 /************************************************************************ 724 * ixv_if_init - Init entry point 725 * 726 * Used in two ways: It is used by the stack as an init entry 727 * point in network interface structure. It is also used 728 * by the driver as a hw/sw initialization routine to get 729 * to a consistent state. 730 * 731 * return 0 on success, positive on failure 732 ************************************************************************/ 733 static void 734 ixv_if_init(if_ctx_t ctx) 735 { 736 struct ixgbe_softc *sc = iflib_get_softc(ctx); 737 if_t ifp = iflib_get_ifp(ctx); 738 device_t dev = iflib_get_dev(ctx); 739 struct ixgbe_hw *hw = &sc->hw; 740 u8 requested_addr[IXGBE_ETH_LENGTH_OF_ADDRESS]; 741 int error = 0; 742 743 INIT_DEBUGOUT("ixv_if_init: begin"); 744 sc->vf_rss_mrqc = 0; 745 ixv_mbx_retry_prepare(sc); 746 hw->adapter_stopped = false; 747 hw->mac.ops.stop_adapter(hw); 748 749 /* Preserve a requested LAA across the reset handshake. */ 750 bcopy(if_getlladdr(ifp), requested_addr, sizeof(requested_addr)); 751 752 /* Reset VF and renegotiate mailbox API version. */ 753 error = hw->mac.ops.reset_hw(hw); 754 if (error != IXGBE_SUCCESS) { 755 sc->stats.vf.initialized = false; 756 ixv_log_reset_failure(sc, error, false); 757 hw->mac.ops.stop_adapter(hw); 758 ixv_mbx_retry_failed(ctx); 759 return; 760 } 761 hw->mac.ops.start_hw(hw); 762 ixv_init_stats(sc); 763 error = ixv_negotiate_api(sc); 764 if (error) { 765 /* 766 * Leave the adapter stopped until an explicit or deferred retry. 767 * Otherwise the admin-status callback immediately requests another 768 * reset and can keep its taskqueue in a tight loop while the PF is 769 * deliberately withholding mailbox CTS (for example, when the VF is 770 * quarantined). 771 */ 772 ixv_log_negotiate_failure(sc, false); 773 hw->mac.ops.stop_adapter(hw); 774 ixv_mbx_retry_failed(ctx); 775 return; 776 } 777 /* Program the address only after the PF mailbox is responsive. */ 778 error = hw->mac.ops.set_rar(hw, 0, requested_addr, 0, 1); 779 if (error == IXGBE_SUCCESS) 780 bcopy(requested_addr, hw->mac.addr, sizeof(requested_addr)); 781 else 782 hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 783 ixv_reconcile_mac(sc, ifp); 784 ixv_mbx_retry_succeeded(sc); 785 786 ixv_initialize_transmit_units(ctx); 787 788 /* Setup Multicast table */ 789 ixv_if_multi_set(ctx); 790 791 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 792 793 /* Configure RX settings */ 794 ixv_initialize_receive_units(ctx); 795 796 /* Set up VLAN offload and filter */ 797 ixv_setup_vlan_support(ctx); 798 799 /* Set up MSI-X routing */ 800 ixv_configure_ivars(sc); 801 802 /* Set up auto-mask */ 803 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 804 805 /* Set moderation on the Link interrupt */ 806 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(sc->vector), IXGBE_LINK_ITR); 807 808 /* Config/Enable Link */ 809 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 810 if (error) { 811 /* PF is not capable of controlling VF state. Enable the link. */ 812 sc->link_enabled = true; 813 } else if (sc->link_enabled == false) 814 device_printf(dev, "VF is disabled by PF\n"); 815 816 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 817 false); 818 819 /* And now turn on interrupts */ 820 ixv_if_enable_intr(ctx); 821 822 return; 823 } /* ixv_if_init */ 824 825 static const char * 826 ixv_reset_error_desc(s32 error) 827 { 828 829 switch (error) { 830 case IXGBE_ERR_RESET_FAILED: 831 return ("PF reset acknowledgement timed out"); 832 case IXGBE_ERR_INVALID_MAC_ADDR: 833 /* 834 * The shared VF reset code historically uses this error for an 835 * unexpected reset reply, before it validates or copies the MAC. 836 */ 837 return ("PF returned an invalid VF reset response"); 838 case IXGBE_ERR_MBX: 839 case IXGBE_ERR_MBX_NOMSG: 840 case IXGBE_ERR_TIMEOUT: 841 return ("PF mailbox reset exchange failed"); 842 default: 843 return ("VF reset handshake failed"); 844 } 845 } 846 847 /* 848 * Report each backoff stage, then limit the steady eight-second retry so a 849 * persistent PF outage does not spam the console. 850 */ 851 static bool 852 ixv_mbx_log_allowed(struct ixgbe_softc *sc) 853 { 854 855 if (sc->vf_mbx_retry_stage == nitems(ixv_mbx_retry_delay) - 1 && 856 !ratecheck(&sc->vf_mbx_last_log, &ixv_mbx_log_interval)) 857 return (false); 858 return (true); 859 } 860 861 static void 862 ixv_log_negotiate_failure(struct ixgbe_softc *sc, bool attaching) 863 { 864 865 if (!ixv_mbx_log_allowed(sc)) 866 return; 867 device_printf(sc->dev, "Mailbox API negotiation failed%s\n", 868 attaching ? "; continuing attach" : ""); 869 } 870 871 static void 872 ixv_log_reset_failure(struct ixgbe_softc *sc, s32 error, bool attaching) 873 { 874 875 if (!ixv_mbx_log_allowed(sc)) 876 return; 877 device_printf(sc->dev, "%s (%d)%s\n", ixv_reset_error_desc(error), 878 error, attaching ? "; continuing attach" : ""); 879 } 880 881 /* 882 * A missing PF can make the posted reset handshake wait for a full mailbox 883 * timeout. Keep that work out of stopped status paths. An administratively 884 * up VF retries complete initialization with an exponential delay capped at 885 * eight seconds, so it recovers without creating a tight mailbox poller. 886 */ 887 static void 888 ixv_mbx_retry_callout(void *arg) 889 { 890 struct ixgbe_softc *sc; 891 if_t ifp; 892 893 sc = arg; 894 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 || 895 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 || 896 iflib_in_detach(sc->ctx)) 897 return; 898 ifp = iflib_get_ifp(sc->ctx); 899 if ((if_getflags(ifp) & IFF_UP) == 0) 900 return; 901 902 iflib_request_reset_if_up(sc->ctx); 903 iflib_admin_intr_deferred(sc->ctx); 904 } 905 906 static void 907 ixv_mbx_retry_detach(struct ixgbe_softc *sc) 908 { 909 910 if (!sc->vf_mbx_retry_initialized) 911 return; 912 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 913 callout_drain(&sc->vf_mbx_retry); 914 sc->vf_mbx_retry_initialized = false; 915 } 916 917 static void 918 ixv_mbx_retry_prepare(struct ixgbe_softc *sc) 919 { 920 921 if (!sc->vf_mbx_retry_initialized) 922 return; 923 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 924 callout_drain(&sc->vf_mbx_retry); 925 } 926 927 static void 928 ixv_mbx_retry_stop(struct ixgbe_softc *sc) 929 { 930 if_t ifp; 931 932 if (!sc->vf_mbx_retry_initialized) 933 return; 934 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 935 callout_drain(&sc->vf_mbx_retry); 936 ifp = iflib_get_ifp(sc->ctx); 937 if ((if_getflags(ifp) & IFF_UP) == 0) 938 sc->vf_mbx_retry_stage = 0; 939 } 940 941 static void 942 ixv_mbx_retry_failed(if_ctx_t ctx) 943 { 944 struct ixgbe_softc *sc; 945 if_t ifp; 946 sbintime_t delay; 947 u_int stage; 948 949 sc = iflib_get_softc(ctx); 950 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 951 sc->link_up = false; 952 if (sc->link_active) { 953 sc->link_active = false; 954 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 955 } 956 iflib_init_failed(ctx); 957 958 ifp = iflib_get_ifp(ctx); 959 if (!sc->vf_mbx_retry_initialized || 960 (if_getflags(ifp) & IFF_UP) == 0) 961 return; 962 stage = sc->vf_mbx_retry_stage; 963 if (stage >= nitems(ixv_mbx_retry_delay)) 964 stage = nitems(ixv_mbx_retry_delay) - 1; 965 delay = ixv_mbx_retry_delay[stage]; 966 if (sc->vf_mbx_retry_stage + 1 < nitems(ixv_mbx_retry_delay)) 967 sc->vf_mbx_retry_stage++; 968 atomic_set_32(&sc->vf_mbx_retry_pending, 1); 969 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0, 970 ixv_mbx_retry_callout, sc, C_PREL(1)); 971 } 972 973 static void 974 ixv_mbx_retry_succeeded(struct ixgbe_softc *sc) 975 { 976 977 atomic_store_rel_32(&sc->vf_mbx_ready, 1); 978 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 979 if (sc->vf_mbx_retry_initialized) 980 callout_stop(&sc->vf_mbx_retry); 981 sc->vf_mbx_retry_stage = 0; 982 sc->vf_link_mbx_failures = 0; 983 sc->vf_link_poll_tick = 984 device_get_unit(sc->dev) % IXV_LINK_POLL_TICKS; 985 sc->vf_mbx_last_log.tv_sec = 0; 986 sc->vf_mbx_last_log.tv_usec = 0; 987 } 988 989 /************************************************************************ 990 * ixv_enable_queue 991 ************************************************************************/ 992 static inline void 993 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 994 { 995 struct ixgbe_hw *hw = &sc->hw; 996 u32 queue = 1 << vector; 997 u32 mask; 998 999 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1000 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1001 } /* ixv_enable_queue */ 1002 1003 /************************************************************************ 1004 * ixv_disable_queue 1005 ************************************************************************/ 1006 static inline void 1007 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 1008 { 1009 struct ixgbe_hw *hw = &sc->hw; 1010 u64 queue = (u64)(1 << vector); 1011 u32 mask; 1012 1013 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1014 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 1015 } /* ixv_disable_queue */ 1016 1017 1018 /************************************************************************ 1019 * ixv_msix_que - MSI-X Queue Interrupt Service routine 1020 ************************************************************************/ 1021 static int 1022 ixv_msix_que(void *arg) 1023 { 1024 struct ix_rx_queue *que = arg; 1025 struct ixgbe_softc *sc = que->sc; 1026 1027 ixv_disable_queue(sc, que->msix); 1028 ++que->irqs; 1029 1030 return (FILTER_SCHEDULE_THREAD); 1031 } /* ixv_msix_que */ 1032 1033 /************************************************************************ 1034 * ixv_msix_mbx 1035 ************************************************************************/ 1036 static int 1037 ixv_msix_mbx(void *arg) 1038 { 1039 struct ixgbe_softc *sc = arg; 1040 struct ixgbe_hw *hw = &sc->hw; 1041 u32 reg; 1042 1043 ++sc->link_irq; 1044 1045 /* First get the cause */ 1046 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1047 /* Clear interrupt with write */ 1048 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1049 1050 /* The admin vector also carries PF mailbox notifications. */ 1051 iflib_admin_intr_deferred(sc->ctx); 1052 1053 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1054 1055 return (FILTER_HANDLED); 1056 } /* ixv_msix_mbx */ 1057 1058 /************************************************************************ 1059 * ixv_media_status - Media Ioctl callback 1060 * 1061 * Called whenever the user queries the status of 1062 * the interface using ifconfig. 1063 ************************************************************************/ 1064 static void 1065 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) 1066 { 1067 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1068 1069 INIT_DEBUGOUT("ixv_media_status: begin"); 1070 1071 /* E610 link state is refreshed by the phased service timer. */ 1072 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 1073 sc->hw.api_version != ixgbe_mbox_api_16) { 1074 atomic_set_32(&sc->vf_link_update, 1); 1075 iflib_admin_intr_deferred(ctx); 1076 } 1077 1078 ifmr->ifm_status = IFM_AVALID; 1079 ifmr->ifm_active = IFM_ETHER; 1080 1081 if (!sc->link_active) 1082 return; 1083 1084 ifmr->ifm_status |= IFM_ACTIVE; 1085 1086 switch (sc->link_speed) { 1087 case IXGBE_LINK_SPEED_5GB_FULL: 1088 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1089 break; 1090 case IXGBE_LINK_SPEED_2_5GB_FULL: 1091 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1092 break; 1093 case IXGBE_LINK_SPEED_1GB_FULL: 1094 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1095 break; 1096 case IXGBE_LINK_SPEED_10GB_FULL: 1097 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1098 break; 1099 case IXGBE_LINK_SPEED_100_FULL: 1100 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1101 break; 1102 case IXGBE_LINK_SPEED_10_FULL: 1103 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1104 break; 1105 } 1106 } /* ixv_if_media_status */ 1107 1108 /************************************************************************ 1109 * ixv_if_media_change - Media Ioctl callback 1110 * 1111 * Called when the user changes speed/duplex using 1112 * media/mediopt option with ifconfig. 1113 ************************************************************************/ 1114 static int 1115 ixv_if_media_change(if_ctx_t ctx) 1116 { 1117 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1118 struct ifmedia *ifm = iflib_get_media(ctx); 1119 1120 INIT_DEBUGOUT("ixv_media_change: begin"); 1121 1122 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1123 return (EINVAL); 1124 1125 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1126 case IFM_AUTO: 1127 break; 1128 default: 1129 device_printf(sc->dev, "Only auto media type\n"); 1130 return (EINVAL); 1131 } 1132 1133 return (0); 1134 } /* ixv_if_media_change */ 1135 1136 1137 /************************************************************************ 1138 * ixv_negotiate_api 1139 * 1140 * Negotiate the Mailbox API with the PF; 1141 * start with the most featured API first. 1142 ************************************************************************/ 1143 static int 1144 ixv_negotiate_api(struct ixgbe_softc *sc) 1145 { 1146 struct ixgbe_hw *hw = &sc->hw; 1147 int mbx_api[] = { 1148 ixgbe_mbox_api_13, 1149 ixgbe_mbox_api_12, 1150 ixgbe_mbox_api_11, 1151 ixgbe_mbox_api_10, 1152 ixgbe_mbox_api_unknown 1153 }; 1154 int i = 0; 1155 1156 if (hw->mac.type == ixgbe_mac_E610_vf && !ixv_is_hyperv(hw) && 1157 hw->mac.ops.negotiate_api_version(hw, ixgbe_mbox_api_16) == 0) 1158 return (0); 1159 1160 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1161 if (hw->mac.ops.negotiate_api_version(hw, mbx_api[i]) == 0) 1162 return (0); 1163 i++; 1164 } 1165 1166 return (EINVAL); 1167 } /* ixv_negotiate_api */ 1168 1169 /************************************************************************ 1170 * ixv_queue_limit 1171 * 1172 * Discover the number of symmetric RSS queue sets available to iflib. 1173 ************************************************************************/ 1174 static int 1175 ixv_queue_limit(struct ixgbe_softc *sc, bool mailbox_ready) 1176 { 1177 struct ixgbe_hw *hw; 1178 unsigned int default_tc, num_tcs; 1179 int admin_vectors, limit, msix_vectors; 1180 1181 hw = &sc->hw; 1182 if (ixv_is_hyperv(hw)) { 1183 limit = ixgbevf_hv_get_queues(hw); 1184 } else { 1185 /* Preserve the current family limit as the mailbox fallback. */ 1186 switch (hw->mac.type) { 1187 case ixgbe_mac_82599_vf: 1188 case ixgbe_mac_X540_vf: 1189 limit = 1; 1190 break; 1191 case ixgbe_mac_X550_vf: 1192 case ixgbe_mac_X550EM_x_vf: 1193 case ixgbe_mac_X550EM_a_vf: 1194 limit = 2; 1195 break; 1196 case ixgbe_mac_E610_vf: 1197 limit = 1; 1198 break; 1199 default: 1200 return (1); 1201 } 1202 } 1203 1204 /* Replace the fallback with the queue grant reported by the PF. */ 1205 if (mailbox_ready) { 1206 switch (hw->api_version) { 1207 case ixgbe_mbox_api_11: 1208 case ixgbe_mbox_api_12: 1209 case ixgbe_mbox_api_13: 1210 case ixgbe_mbox_api_16: 1211 num_tcs = default_tc = 0; 1212 if (ixgbevf_get_queues(hw, &num_tcs, &default_tc) == 0) { 1213 limit = imin(hw->mac.max_tx_queues, 1214 hw->mac.max_rx_queues); 1215 limit = imin(limit, 2); 1216 } 1217 break; 1218 default: 1219 break; 1220 } 1221 } 1222 1223 /* 1224 * iflib assigns one data vector to each queue set. A VF has at most 1225 * three MSI-X vectors; reserve one of them for the mailbox interrupt. 1226 */ 1227 admin_vectors = iflib_get_sctx(sc->ctx)->isc_admin_intrcnt; 1228 msix_vectors = pci_msix_count(sc->dev); 1229 if (msix_vectors <= admin_vectors) 1230 return (1); 1231 1232 return (imax(1, imin(limit, msix_vectors - admin_vectors))); 1233 } /* ixv_queue_limit */ 1234 1235 static int 1236 ixv_update_xcast_mode(struct ixgbe_softc *sc, int flags) 1237 { 1238 if_t ifp; 1239 int mode; 1240 1241 ifp = iflib_get_ifp(sc->ctx); 1242 if (flags & IFF_PROMISC) 1243 mode = IXGBEVF_XCAST_MODE_PROMISC; 1244 else if ((flags & IFF_ALLMULTI) != 0 || 1245 if_llmaddr_count(ifp) > IXGBE_MAX_VF_MC) 1246 mode = IXGBEVF_XCAST_MODE_ALLMULTI; 1247 else if (if_llmaddr_count(ifp) != 0) 1248 mode = IXGBEVF_XCAST_MODE_MULTI; 1249 else 1250 mode = IXGBEVF_XCAST_MODE_NONE; 1251 return (sc->hw.mac.ops.update_xcast_mode(&sc->hw, mode)); 1252 } 1253 1254 static int 1255 ixv_if_promisc_set(if_ctx_t ctx, int flags) 1256 { 1257 struct ixgbe_softc *sc; 1258 1259 sc = iflib_get_softc(ctx); 1260 if (!iflib_is_running(ctx)) 1261 return (0); 1262 if (ixv_update_xcast_mode(sc, flags) != IXGBE_SUCCESS) 1263 return (EOPNOTSUPP); 1264 return (0); 1265 } /* ixv_if_promisc_set */ 1266 1267 1268 static u_int 1269 ixv_if_multi_set_cb(void *cb_arg, struct sockaddr_dl *addr, u_int cnt) 1270 { 1271 if (cnt >= MAX_NUM_MULTICAST_ADDRESSES) 1272 return (0); 1273 1274 bcopy(LLADDR(addr), 1275 &((u8 *)cb_arg)[cnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1276 IXGBE_ETH_LENGTH_OF_ADDRESS); 1277 1278 return (1); 1279 } 1280 1281 /************************************************************************ 1282 * ixv_if_multi_set - Multicast Update 1283 * 1284 * Called whenever multicast address list is updated. 1285 ************************************************************************/ 1286 static void 1287 ixv_if_multi_set(if_ctx_t ctx) 1288 { 1289 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1290 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1291 u8 *update_ptr; 1292 if_t ifp = iflib_get_ifp(ctx); 1293 int error, mcnt = 0; 1294 1295 IOCTL_DEBUGOUT("ixv_if_multi_set: begin"); 1296 1297 mcnt = if_foreach_llmaddr(ifp, ixv_if_multi_set_cb, mta); 1298 1299 update_ptr = mta; 1300 1301 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1302 ixv_mc_array_itr, true); 1303 error = ixv_update_xcast_mode(sc, if_getflags(ifp)); 1304 if (mcnt > IXGBE_MAX_VF_MC && error != IXGBE_SUCCESS) { 1305 if (!sc->vf_mcast_overflow_warned) 1306 device_printf(sc->dev, 1307 "PF rejected all-multicast fallback; only %d " 1308 "multicast addresses are active\n", 1309 IXGBE_MAX_VF_MC); 1310 sc->vf_mcast_overflow_warned = true; 1311 } else if (mcnt <= IXGBE_MAX_VF_MC) 1312 sc->vf_mcast_overflow_warned = false; 1313 } /* ixv_if_multi_set */ 1314 1315 /************************************************************************ 1316 * ixv_mc_array_itr 1317 * 1318 * An iterator function needed by the multicast shared code. 1319 * It feeds the shared code routine the addresses in the 1320 * array of ixv_set_multi() one by one. 1321 ************************************************************************/ 1322 static u8 * 1323 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1324 { 1325 u8 *addr = *update_ptr; 1326 u8 *newptr; 1327 1328 *vmdq = 0; 1329 1330 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1331 *update_ptr = newptr; 1332 1333 return addr; 1334 } /* ixv_mc_array_itr */ 1335 1336 /************************************************************************ 1337 * ixv_if_local_timer - Timer routine 1338 * 1339 * Checks for link status, updates statistics, 1340 * and runs the watchdog check. 1341 ************************************************************************/ 1342 static void 1343 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid) 1344 { 1345 struct ixgbe_softc *sc; 1346 1347 if (qid != 0) 1348 return; 1349 1350 sc = iflib_get_softc(ctx); 1351 atomic_set_32(&sc->vf_vlan_retry_tick, 1); 1352 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 1353 sc->hw.api_version != ixgbe_mbox_api_16 || 1354 ++sc->vf_link_poll_tick == IXV_LINK_POLL_TICKS) { 1355 sc->vf_link_poll_tick = 0; 1356 atomic_set_32(&sc->vf_link_update, 1); 1357 } 1358 1359 /* Fire off the adminq task */ 1360 iflib_admin_intr_deferred(ctx); 1361 } /* ixv_if_local_timer */ 1362 1363 /************************************************************************ 1364 * ixv_if_update_admin_status - Update OS on link state 1365 * 1366 * Note: Only updates the OS on the cached link state. 1367 * The real check of the hardware only happens with 1368 * a link interrupt. 1369 ************************************************************************/ 1370 static void 1371 ixv_if_update_admin_status(if_ctx_t ctx) 1372 { 1373 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1374 device_t dev = iflib_get_dev(ctx); 1375 bool check_link, reset_seen; 1376 s32 status; 1377 uint64_t baudrate; 1378 1379 if (!iflib_is_running(ctx) || 1380 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1381 if (sc->link_active) { 1382 sc->link_active = false; 1383 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1384 } 1385 return; 1386 } 1387 1388 check_link = atomic_readandclear_32(&sc->vf_link_update) != 0; 1389 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 1390 sc->hw.api_version != ixgbe_mbox_api_16) 1391 check_link = true; 1392 /* 1393 * Hyper-V uses this indication only to invalidate cached VFLINKS 1394 * state; it does not expose the mailbox handshake needed to turn it 1395 * into a driver reset. Its check_link operation samples the bit. 1396 */ 1397 reset_seen = !ixv_is_hyperv(&sc->hw) && 1398 ixgbe_check_for_rst(&sc->hw, 0) == IXGBE_SUCCESS; 1399 if (reset_seen) 1400 sc->hw.mac.get_link_status = true; 1401 if (check_link) { 1402 sc->hw.mac.get_link_status = true; 1403 status = ixgbe_check_link(&sc->hw, &sc->link_speed, 1404 &sc->link_up, false); 1405 } else 1406 status = IXGBE_SUCCESS; 1407 if (sc->hw.mac.type == ixgbe_mac_E610_vf && 1408 sc->hw.api_version == ixgbe_mbox_api_16 && 1409 status != IXGBE_SUCCESS && status != IXGBE_ERR_MBX) { 1410 /* 1411 * A busy PF can miss an individual link-state request. Preserve 1412 * the last confirmed state across brief transport failures, but 1413 * bound how long stale carrier can remain visible if the PF is gone. 1414 */ 1415 if (sc->vf_link_mbx_failures < IXV_LINK_MBX_FAILURE_LIMIT) 1416 sc->vf_link_mbx_failures++; 1417 if (sc->vf_link_mbx_failures == IXV_LINK_MBX_FAILURE_LIMIT) 1418 sc->link_up = false; 1419 status = IXGBE_SUCCESS; 1420 } else if (status == IXGBE_SUCCESS) 1421 sc->vf_link_mbx_failures = 0; 1422 /* Reinitialize after an unsolicited reset, even while link is down. */ 1423 if (reset_seen) 1424 status = IXGBE_ERR_MBX; 1425 1426 if (status != IXGBE_SUCCESS && sc->hw.adapter_stopped == false) { 1427 /* Mailbox's Clear To Send status is lost or timeout occurred. 1428 * We need reinitialization. */ 1429 sc->vf_rss_mrqc = 0; 1430 iflib_request_reset(ctx); 1431 iflib_admin_intr_deferred(ctx); 1432 } 1433 1434 if (sc->link_up && sc->link_enabled) { 1435 if (sc->link_active == false) { 1436 if (bootverbose) { 1437 baudrate = ixgbe_link_speed_to_baudrate( 1438 sc->link_speed); 1439 if (baudrate != 0) 1440 device_printf(dev, 1441 "Link is up %ju Mbps Full Duplex\n", 1442 (uintmax_t)(baudrate / IF_Mbps(1))); 1443 else 1444 device_printf(dev, 1445 "Link is up (speed unknown)\n"); 1446 } 1447 sc->link_active = true; 1448 iflib_link_state_change(ctx, LINK_STATE_UP, 1449 ixgbe_link_speed_to_baudrate(sc->link_speed)); 1450 } 1451 } else { /* Link down */ 1452 if (sc->link_active == true) { 1453 if (bootverbose) 1454 device_printf(dev, "Link is Down\n"); 1455 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1456 sc->link_active = false; 1457 } 1458 } 1459 1460 /* iflib closes admission before stop; do not replay after VF reset. */ 1461 if (iflib_is_running(ctx) && 1462 atomic_readandclear_32(&sc->vf_vlan_retry_tick) != 0) 1463 ixv_vlan_retry_tick(sc); 1464 1465 /* Do not treat a PF reset as a hardware-counter wrap. */ 1466 if (status == IXGBE_SUCCESS) 1467 ixv_update_stats(sc); 1468 else 1469 sc->stats.vf.initialized = false; 1470 } /* ixv_if_update_admin_status */ 1471 1472 1473 /************************************************************************ 1474 * ixv_if_stop - Stop the hardware 1475 * 1476 * Disables all traffic on the adapter by issuing a 1477 * global reset on the MAC and deallocates TX/RX buffers. 1478 ************************************************************************/ 1479 static void 1480 ixv_if_stop(if_ctx_t ctx) 1481 { 1482 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1483 struct ixgbe_hw *hw = &sc->hw; 1484 if_t ifp = iflib_get_ifp(ctx); 1485 bool mailbox_ready, reset_seen; 1486 1487 INIT_DEBUGOUT("ixv_stop: begin\n"); 1488 sc->vf_rss_mrqc = 0; 1489 1490 ixv_mbx_retry_stop(sc); 1491 ixv_if_disable_intr(ctx); 1492 1493 mailbox_ready = atomic_load_acq_32(&sc->vf_mbx_ready) != 0; 1494 reset_seen = mailbox_ready && 1495 ixgbe_check_for_rst(hw, 0) == IXGBE_SUCCESS; 1496 if (reset_seen) 1497 sc->stats.vf.initialized = false; 1498 else if (mailbox_ready && sc->stats.vf.initialized) 1499 ixv_update_stats(sc); 1500 if (mailbox_ready && (if_getflags(ifp) & IFF_UP) == 0) { 1501 if (hw->mac.ops.reset_hw(hw) == IXGBE_SUCCESS) 1502 ixv_init_stats(sc); 1503 else 1504 sc->stats.vf.initialized = false; 1505 } 1506 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 1507 sc->vf_link_mbx_failures = 0; 1508 sc->hw.adapter_stopped = false; 1509 hw->mac.ops.stop_adapter(hw); 1510 1511 /* Publish the stopped state without touching the PF mailbox. */ 1512 sc->link_up = false; 1513 if (sc->link_active) { 1514 sc->link_active = false; 1515 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1516 } 1517 } /* ixv_if_stop */ 1518 1519 1520 /************************************************************************ 1521 * ixv_identify_hardware - Determine hardware revision. 1522 ************************************************************************/ 1523 static void 1524 ixv_identify_hardware(if_ctx_t ctx) 1525 { 1526 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1527 device_t dev = iflib_get_dev(ctx); 1528 struct ixgbe_hw *hw = &sc->hw; 1529 1530 /* Save off the information about this board */ 1531 hw->vendor_id = pci_get_vendor(dev); 1532 hw->device_id = pci_get_device(dev); 1533 hw->revision_id = pci_get_revid(dev); 1534 hw->subsystem_vendor_id = pci_get_subvendor(dev); 1535 hw->subsystem_device_id = pci_get_subdevice(dev); 1536 1537 /* A subset of set_mac_type */ 1538 switch (hw->device_id) { 1539 case IXGBE_DEV_ID_82599_VF: 1540 case IXGBE_DEV_ID_82599_VF_HV: 1541 hw->mac.type = ixgbe_mac_82599_vf; 1542 break; 1543 case IXGBE_DEV_ID_X540_VF: 1544 case IXGBE_DEV_ID_X540_VF_HV: 1545 hw->mac.type = ixgbe_mac_X540_vf; 1546 break; 1547 case IXGBE_DEV_ID_X550_VF: 1548 case IXGBE_DEV_ID_X550_VF_HV: 1549 hw->mac.type = ixgbe_mac_X550_vf; 1550 break; 1551 case IXGBE_DEV_ID_X550EM_X_VF: 1552 case IXGBE_DEV_ID_X550EM_X_VF_HV: 1553 hw->mac.type = ixgbe_mac_X550EM_x_vf; 1554 break; 1555 case IXGBE_DEV_ID_X550EM_A_VF: 1556 case IXGBE_DEV_ID_X550EM_A_VF_HV: 1557 hw->mac.type = ixgbe_mac_X550EM_a_vf; 1558 break; 1559 case IXGBE_DEV_ID_E610_VF: 1560 hw->mac.type = ixgbe_mac_E610_vf; 1561 break; 1562 default: 1563 device_printf(dev, "unknown mac type\n"); 1564 hw->mac.type = ixgbe_mac_unknown; 1565 break; 1566 } 1567 } /* ixv_identify_hardware */ 1568 1569 static bool 1570 ixv_is_hyperv(const struct ixgbe_hw *hw) 1571 { 1572 switch (hw->device_id) { 1573 case IXGBE_DEV_ID_82599_VF_HV: 1574 case IXGBE_DEV_ID_X540_VF_HV: 1575 case IXGBE_DEV_ID_X550_VF_HV: 1576 case IXGBE_DEV_ID_X550EM_X_VF_HV: 1577 case IXGBE_DEV_ID_X550EM_A_VF_HV: 1578 return (true); 1579 case IXGBE_DEV_ID_E610_VF: 1580 return (hw->subsystem_device_id == 1581 IXGBE_SUBDEV_ID_E610_VF_HV); 1582 default: 1583 return (false); 1584 } 1585 } 1586 1587 /************************************************************************ 1588 * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers 1589 ************************************************************************/ 1590 static int 1591 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix) 1592 { 1593 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1594 device_t dev = iflib_get_dev(ctx); 1595 struct ix_rx_queue *rx_que = sc->rx_queues; 1596 struct ix_tx_queue *tx_que; 1597 int error, rid, vector = 0; 1598 char buf[16]; 1599 1600 for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) { 1601 rid = vector + 1; 1602 1603 snprintf(buf, sizeof(buf), "rxq%d", i); 1604 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 1605 IFLIB_INTR_RXTX, ixv_msix_que, rx_que, rx_que->rxr.me, 1606 buf); 1607 1608 if (error) { 1609 device_printf(iflib_get_dev(ctx), 1610 "Failed to allocate que int %d err: %d", 1611 i, error); 1612 sc->num_rx_queues = i + 1; 1613 goto fail; 1614 } 1615 1616 rx_que->msix = vector; 1617 } 1618 1619 for (int i = 0; i < sc->num_tx_queues; i++) { 1620 snprintf(buf, sizeof(buf), "txq%d", i); 1621 tx_que = &sc->tx_queues[i]; 1622 tx_que->msix = i % sc->num_rx_queues; 1623 iflib_softirq_alloc_generic(ctx, 1624 &sc->rx_queues[tx_que->msix].que_irq, 1625 IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); 1626 } 1627 rid = vector + 1; 1628 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, 1629 IFLIB_INTR_ADMIN, ixv_msix_mbx, sc, 0, "aq"); 1630 if (error) { 1631 device_printf(iflib_get_dev(ctx), 1632 "Failed to register admin handler"); 1633 return (error); 1634 } 1635 1636 sc->vector = vector; 1637 /* 1638 * Due to a broken design QEMU will fail to properly 1639 * enable the guest for MSIX unless the vectors in 1640 * the table are all set up, so we must rewrite the 1641 * ENABLE in the MSIX control register again at this 1642 * point to cause it to successfully initialize us. 1643 */ 1644 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 1645 int msix_ctrl; 1646 if (pci_find_cap(dev, PCIY_MSIX, &rid)) { 1647 device_printf(dev, 1648 "Finding MSIX capability failed\n"); 1649 } else { 1650 rid += PCIR_MSIX_CTRL; 1651 msix_ctrl = pci_read_config(dev, rid, 2); 1652 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1653 pci_write_config(dev, rid, msix_ctrl, 2); 1654 } 1655 } 1656 1657 return (0); 1658 1659 fail: 1660 iflib_irq_free(ctx, &sc->irq); 1661 rx_que = sc->rx_queues; 1662 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) 1663 iflib_irq_free(ctx, &rx_que->que_irq); 1664 1665 return (error); 1666 } /* ixv_if_msix_intr_assign */ 1667 1668 /************************************************************************ 1669 * ixv_allocate_pci_resources 1670 ************************************************************************/ 1671 static int 1672 ixv_allocate_pci_resources(if_ctx_t ctx) 1673 { 1674 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1675 device_t dev = iflib_get_dev(ctx); 1676 int rid; 1677 1678 rid = PCIR_BAR(0); 1679 sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1680 RF_ACTIVE); 1681 1682 if (!(sc->pci_mem)) { 1683 device_printf(dev, 1684 "Unable to allocate bus resource: memory\n"); 1685 return (ENXIO); 1686 } 1687 1688 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem); 1689 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->pci_mem); 1690 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 1691 1692 return (0); 1693 } /* ixv_allocate_pci_resources */ 1694 1695 /************************************************************************ 1696 * ixv_free_pci_resources 1697 ************************************************************************/ 1698 static void 1699 ixv_free_pci_resources(if_ctx_t ctx) 1700 { 1701 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1702 struct ix_rx_queue *que = sc->rx_queues; 1703 device_t dev = iflib_get_dev(ctx); 1704 1705 /* Release all MSI-X queue resources */ 1706 if (sc->intr_type == IFLIB_INTR_MSIX) 1707 iflib_irq_free(ctx, &sc->irq); 1708 1709 if (que != NULL) { 1710 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 1711 iflib_irq_free(ctx, &que->que_irq); 1712 } 1713 } 1714 1715 if (sc->pci_mem != NULL) 1716 bus_release_resource(dev, SYS_RES_MEMORY, 1717 rman_get_rid(sc->pci_mem), sc->pci_mem); 1718 } /* ixv_free_pci_resources */ 1719 1720 /************************************************************************ 1721 * ixv_setup_interface 1722 * 1723 * Setup networking device structure and register an interface. 1724 ************************************************************************/ 1725 static int 1726 ixv_setup_interface(if_ctx_t ctx) 1727 { 1728 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1729 if_softc_ctx_t scctx = sc->shared; 1730 if_t ifp = iflib_get_ifp(ctx); 1731 1732 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1733 1734 if_setbaudrate(ifp, IF_Gbps(10)); 1735 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 2); 1736 1737 1738 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR; 1739 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1740 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 1741 1742 return 0; 1743 } /* ixv_setup_interface */ 1744 1745 /************************************************************************ 1746 * ixv_if_get_counter 1747 ************************************************************************/ 1748 static uint64_t 1749 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt) 1750 { 1751 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1752 if_t ifp = iflib_get_ifp(ctx); 1753 1754 switch (cnt) { 1755 case IFCOUNTER_IPACKETS: 1756 return (sc->ipackets); 1757 case IFCOUNTER_OPACKETS: 1758 return (sc->opackets); 1759 case IFCOUNTER_IBYTES: 1760 return (sc->ibytes); 1761 case IFCOUNTER_OBYTES: 1762 return (sc->obytes); 1763 case IFCOUNTER_IMCASTS: 1764 return (sc->imcasts); 1765 default: 1766 return (if_get_counter_default(ifp, cnt)); 1767 } 1768 } /* ixv_if_get_counter */ 1769 1770 /* ixv_if_needs_restart - Tell iflib when the driver needs to be reinitialized 1771 * @ctx: iflib context 1772 * @event: event code to check 1773 * 1774 * Defaults to returning true for every event. 1775 * 1776 * @returns true if iflib needs to reinit the interface 1777 */ 1778 static bool 1779 ixv_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 1780 { 1781 switch (event) { 1782 case IFLIB_RESTART_VLAN_CONFIG: 1783 /* The callbacks update the PF directly and queue failed work. */ 1784 return (false); 1785 default: 1786 return (true); 1787 } 1788 } 1789 1790 /************************************************************************ 1791 * ixv_initialize_transmit_units - Enable transmit unit. 1792 ************************************************************************/ 1793 static void 1794 ixv_initialize_transmit_units(if_ctx_t ctx) 1795 { 1796 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1797 struct ixgbe_hw *hw = &sc->hw; 1798 if_softc_ctx_t scctx = sc->shared; 1799 struct ix_tx_queue *que = sc->tx_queues; 1800 int i; 1801 1802 for (i = 0; i < sc->num_tx_queues; i++, que++) { 1803 struct tx_ring *txr = &que->txr; 1804 u64 tdba = txr->tx_paddr; 1805 u32 txctrl, txdctl; 1806 int j = txr->me; 1807 1808 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1809 txdctl &= ~IXGBE_TXDCTL_THRESH_MASK; 1810 txdctl |= IXGBE_TXDCTL_THRESH_DEFAULT; 1811 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1812 1813 /* Set the HW Tx Head and Tail indices */ 1814 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1815 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1816 1817 /* Set Tx Tail register */ 1818 txr->tail = IXGBE_VFTDT(j); 1819 1820 txr->tx_rs_cidx = txr->tx_rs_pidx; 1821 /* Initialize the last processed descriptor to be the end of 1822 * the ring, rather than the start, so that we avoid an 1823 * off-by-one error when calculating how many descriptors are 1824 * done in the credits_update function. 1825 */ 1826 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 1827 for (int k = 0; k < scctx->isc_ntxd[0]; k++) 1828 txr->tx_rsq[k] = QIDX_INVALID; 1829 1830 /* Set Ring parameters */ 1831 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1832 (tdba & 0x00000000ffffffffULL)); 1833 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1834 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1835 scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc)); 1836 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1837 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1838 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1839 1840 /* Now enable */ 1841 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1842 txdctl |= IXGBE_TXDCTL_ENABLE; 1843 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1844 } 1845 1846 return; 1847 } /* ixv_initialize_transmit_units */ 1848 1849 /************************************************************************ 1850 * ixv_initialize_rss_mapping 1851 ************************************************************************/ 1852 static void 1853 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1854 { 1855 struct ixgbe_hw *hw = &sc->hw; 1856 u32 reta = 0, mrqc; 1857 int queue_id; 1858 int i, j; 1859 u32 rss_hash_config; 1860 1861 rss_getkey((uint8_t *)sc->vf_rss_key); 1862 1863 /* Now fill out hash function seeds */ 1864 for (u_int k = 0; k < nitems(sc->vf_rss_key); k++) 1865 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(k), sc->vf_rss_key[k]); 1866 1867 /* Set up the redirection table */ 1868 for (i = 0, j = 0; i < 64; i++, j++) { 1869 if (j == sc->num_rx_queues) 1870 j = 0; 1871 1872 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1873 /* 1874 * Fetch the RSS bucket id for the given indirection 1875 * entry. Cap it at the number of configured buckets 1876 * (which is num_rx_queues.) 1877 */ 1878 queue_id = rss_get_indirection_to_bucket(i); 1879 queue_id = queue_id % sc->num_rx_queues; 1880 } else 1881 queue_id = j; 1882 1883 /* 1884 * The low 8 bits are for hash value (n+0); 1885 * The next 8 bits are for hash value (n+1), etc. 1886 */ 1887 reta >>= 8; 1888 reta |= ((uint32_t)queue_id) << 24; 1889 if ((i & 3) == 3) { 1890 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1891 reta = 0; 1892 } 1893 } 1894 1895 /* Perform hash on these packet types */ 1896 if (sc->feat_en & IXGBE_FEATURE_RSS) 1897 rss_hash_config = rss_gethashconfig(); 1898 else { 1899 /* 1900 * Disable UDP - IP fragments aren't currently being handled 1901 * and so we end up with a mix of 2-tuple and 4-tuple 1902 * traffic. 1903 */ 1904 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1905 | RSS_HASHTYPE_RSS_TCP_IPV4 1906 | RSS_HASHTYPE_RSS_IPV6 1907 | RSS_HASHTYPE_RSS_TCP_IPV6; 1908 } 1909 1910 mrqc = IXGBE_MRQC_RSSEN; 1911 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1912 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1913 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1914 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1915 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1916 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1917 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1918 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1919 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1920 device_printf(sc->dev, 1921 "%s: RSS_HASHTYPE_RSS_IPV6_EX defined," 1922 " but not supported\n", __func__); 1923 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1924 device_printf(sc->dev, 1925 "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined," 1926 " but not supported\n", __func__); 1927 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1928 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1929 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1930 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1931 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1932 device_printf(sc->dev, 1933 "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined," 1934 " but not supported\n", __func__); 1935 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1936 sc->vf_rss_mrqc = mrqc; 1937 } /* ixv_initialize_rss_mapping */ 1938 1939 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 1940 /************************************************************************ 1941 * ixv_initialize_receive_units - Setup receive registers and features. 1942 ************************************************************************/ 1943 static void 1944 ixv_initialize_receive_units(if_ctx_t ctx) 1945 { 1946 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1947 if_softc_ctx_t scctx; 1948 struct ixgbe_hw *hw = &sc->hw; 1949 #ifdef DEV_NETMAP 1950 if_t ifp = iflib_get_ifp(ctx); 1951 #endif 1952 struct ix_rx_queue *que = sc->rx_queues; 1953 u32 bufsz, psrtype; 1954 1955 bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 1956 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1957 1958 psrtype = IXGBE_PSRTYPE_TCPHDR | 1959 IXGBE_PSRTYPE_UDPHDR | 1960 IXGBE_PSRTYPE_IPV4HDR | 1961 IXGBE_PSRTYPE_IPV6HDR | 1962 IXGBE_PSRTYPE_L2HDR; 1963 1964 if (sc->num_rx_queues > 1) 1965 psrtype |= 1 << 29; 1966 1967 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1968 1969 /* Tell PF our max_frame size */ 1970 if (hw->mac.ops.set_rlpml(hw, sc->max_frame_size) != 0) { 1971 device_printf(sc->dev, 1972 "There is a problem with the PF setup. It is likely the" 1973 " receive unit for this VF will not function correctly." 1974 "\n"); 1975 } 1976 scctx = sc->shared; 1977 1978 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 1979 struct rx_ring *rxr = &que->rxr; 1980 u64 rdba = rxr->rx_paddr; 1981 u32 reg, rxdctl; 1982 int j = rxr->me; 1983 1984 /* Disable the queue */ 1985 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1986 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1987 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1988 for (int k = 0; k < 10; k++) { 1989 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1990 IXGBE_RXDCTL_ENABLE) 1991 msec_delay(1); 1992 else 1993 break; 1994 } 1995 wmb(); 1996 /* Setup the Base and Length of the Rx Descriptor Ring */ 1997 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1998 (rdba & 0x00000000ffffffffULL)); 1999 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 2000 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 2001 scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc)); 2002 2003 /* Reset the ring indices */ 2004 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 2005 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 2006 2007 /* Set up the SRRCTL register */ 2008 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 2009 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 2010 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 2011 reg |= bufsz; 2012 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2013 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 2014 2015 /* Capture Rx Tail index */ 2016 rxr->tail = IXGBE_VFRDT(rxr->me); 2017 2018 /* Do the queue enabling last */ 2019 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 2020 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 2021 for (int l = 0; l < 10; l++) { 2022 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 2023 IXGBE_RXDCTL_ENABLE) 2024 break; 2025 msec_delay(1); 2026 } 2027 wmb(); 2028 2029 /* Set the Tail Pointer */ 2030 #ifdef DEV_NETMAP 2031 /* 2032 * In netmap mode, we must preserve the buffers made 2033 * available to userspace before the if_init() 2034 * (this is true by default on the TX side, because 2035 * init makes all buffers available to userspace). 2036 * 2037 * netmap_reset() and the device specific routines 2038 * (e.g. ixgbe_setup_receive_rings()) map these 2039 * buffers at the end of the NIC ring, so here we 2040 * must set the RDT (tail) register to make sure 2041 * they are not overwritten. 2042 * 2043 * In this driver the NIC ring starts at RDH = 0, 2044 * RDT points to the last slot available for reception (?), 2045 * so RDT = num_rx_desc - 1 means the whole ring is available. 2046 */ 2047 if (if_getcapenable(ifp) & IFCAP_NETMAP) { 2048 struct netmap_adapter *na = NA(ifp); 2049 struct netmap_kring *kring = na->rx_rings[j]; 2050 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 2051 2052 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 2053 } else 2054 #endif /* DEV_NETMAP */ 2055 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 2056 scctx->isc_nrxd[0] - 1); 2057 } 2058 2059 /* 2060 * Do not touch RSS and RETA settings for older hardware 2061 * as those are shared among PF and all VF. 2062 */ 2063 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 2064 ixv_initialize_rss_mapping(sc); 2065 } /* ixv_initialize_receive_units */ 2066 2067 /************************************************************************ 2068 * VF VLAN mailbox retry helpers 2069 ************************************************************************/ 2070 static bool 2071 ixv_vlan_error_retryable(s32 error) 2072 { 2073 return (error != IXGBE_ERR_FEATURE_NOT_SUPPORTED); 2074 } 2075 2076 static void 2077 ixv_vlan_retry_add(struct ixgbe_softc *sc, u16 vid) 2078 { 2079 bool pending; 2080 2081 pending = ixv_vlan_retry_pending(sc); 2082 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); 2083 /* Start a bounded no-progress window when work becomes pending. */ 2084 if (!pending || sc->vf_vlan_retry_deadline == 0) 2085 sc->vf_vlan_retry_deadline = 2086 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 2087 } 2088 2089 static void 2090 ixv_vlan_retry_clear(struct ixgbe_softc *sc, u16 vid) 2091 { 2092 u32 bit; 2093 bool pending; 2094 2095 bit = 1U << (vid & 0x1f); 2096 pending = (sc->vf_vfta_retry[vid >> 5] & bit) != 0; 2097 sc->vf_vfta_retry[vid >> 5] &= ~bit; 2098 if (!pending) { 2099 /* A successful mailbox operation proves the PF is responsive. */ 2100 if (sc->vf_vlan_retry_deadline == 0 && 2101 ixv_vlan_retry_pending(sc)) 2102 sc->vf_vlan_retry_deadline = 2103 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 2104 return; 2105 } 2106 if (ixv_vlan_retry_pending(sc)) 2107 sc->vf_vlan_retry_deadline = 2108 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 2109 else 2110 sc->vf_vlan_retry_deadline = 0; 2111 } 2112 2113 static bool 2114 ixv_vlan_retry_pending(const struct ixgbe_softc *sc) 2115 { 2116 int i; 2117 2118 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2119 if (sc->vf_vfta_retry[i] != 0) 2120 return (true); 2121 return (false); 2122 } 2123 2124 static void 2125 ixv_vlan_retry_tick(struct ixgbe_softc *sc) 2126 { 2127 struct ixgbe_hw *hw; 2128 bool enable; 2129 s32 error; 2130 u32 bit; 2131 u16 vid; 2132 int attempts, i, remaining; 2133 2134 if (!ixv_vlan_retry_pending(sc)) { 2135 sc->vf_vlan_retry_deadline = 0; 2136 return; 2137 } 2138 /* 2139 * Exhausted entries remain dormant until reset, a VLAN callback, or 2140 * another successful VLAN mailbox request. 2141 */ 2142 if (sc->vf_vlan_retry_deadline == 0) 2143 return; 2144 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { 2145 remaining = 0; 2146 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2147 remaining += bitcount32(sc->vf_vfta_retry[i]); 2148 sc->vf_vlan_retry_deadline = 0; 2149 device_printf(sc->dev, 2150 "VF VLAN retries exhausted for %d VIDs\n", remaining); 2151 return; 2152 } 2153 2154 /* 2155 * A mailbox NACK does not distinguish transient PF unavailability 2156 * from a permanent policy rejection or VLVF exhaustion. Reconcile a 2157 * bounded batch per timer tick so none of those cases creates a busy 2158 * mailbox poller. Stop after the first failure so a silent PF can 2159 * consume at most one mailbox timeout per pass, while a responsive PF 2160 * can drain several successful requests. 2161 */ 2162 hw = &sc->hw; 2163 for (attempts = 0, i = 0; 2164 attempts < IXV_VLAN_RETRY_BATCH && i < 4096; i++) { 2165 vid = sc->vf_vlan_retry_cursor; 2166 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; 2167 bit = 1U << (vid & 0x1f); 2168 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) 2169 continue; 2170 attempts++; 2171 enable = (sc->shadow_vfta[vid >> 5] & bit) != 0; 2172 error = hw->mac.ops.set_vfta(hw, vid, 0, enable, false); 2173 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error)) 2174 break; 2175 ixv_vlan_retry_clear(sc, vid); 2176 } 2177 } 2178 2179 /************************************************************************ 2180 * ixv_setup_vlan_support - Configure and restore VLAN support 2181 ************************************************************************/ 2182 static void 2183 ixv_setup_vlan_support(if_ctx_t ctx) 2184 { 2185 if_t ifp = iflib_get_ifp(ctx); 2186 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2187 struct ixgbe_hw *hw = &sc->hw; 2188 s32 error; 2189 u32 ctrl, vfta; 2190 u16 vid; 2191 int restore_failures; 2192 2193 for (int i = 0; i < sc->num_rx_queues; i++) { 2194 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 2195 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { 2196 ctrl |= IXGBE_RXDCTL_VME; 2197 sc->rx_queues[i].rxr.vtag_strip = true; 2198 } else { 2199 ctrl &= ~IXGBE_RXDCTL_VME; 2200 sc->rx_queues[i].rxr.vtag_strip = false; 2201 } 2202 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 2203 } 2204 2205 /* 2206 * The PF controls the pool membership independently of the VF's local 2207 * HWFILTER capability. A reset removes those memberships, so replay 2208 * every registered VLAN through the mailbox. 2209 * 2210 * Keep failed removal requests pending as well. They are harmless and 2211 * idempotent after a successful reset, and still needed if the reset 2212 * handshake did not reach the PF. 2213 */ 2214 sc->vf_vlan_retry_deadline = 0; 2215 sc->vf_vlan_retry_cursor = 0; 2216 restore_failures = 0; 2217 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2218 if (sc->shadow_vfta[i] == 0) 2219 continue; 2220 vfta = sc->shadow_vfta[i]; 2221 for (int j = 0; j < 32; j++) { 2222 if ((vfta & (1U << j)) == 0) 2223 continue; 2224 vid = (i * 32) + j; 2225 /* One timeout is enough to declare this replay deferred. */ 2226 if (restore_failures == 0) 2227 error = hw->mac.ops.set_vfta(hw, vid, 0, true, 2228 false); 2229 else 2230 error = IXGBE_ERR_MBX; 2231 if (error != IXGBE_SUCCESS && 2232 ixv_vlan_error_retryable(error)) { 2233 ixv_vlan_retry_add(sc, vid); 2234 restore_failures++; 2235 } else 2236 ixv_vlan_retry_clear(sc, vid); 2237 } 2238 } 2239 if (ixv_vlan_retry_pending(sc)) 2240 sc->vf_vlan_retry_deadline = 2241 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 2242 if (restore_failures != 0) 2243 device_printf(sc->dev, 2244 "VF VLAN restore failed for %d VIDs; retrying\n", 2245 restore_failures); 2246 } /* ixv_setup_vlan_support */ 2247 2248 /************************************************************************ 2249 * ixv_if_register_vlan 2250 * 2251 * Run via a vlan config EVENT, it enables us to use the 2252 * HW Filter table since we can get the vlan id. 2253 ************************************************************************/ 2254 static void 2255 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag) 2256 { 2257 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2258 bool pending, present; 2259 u32 index, mask; 2260 s32 error; 2261 2262 index = (vtag >> 5) & 0x7F; 2263 mask = 1U << (vtag & 0x1F); 2264 present = (sc->shadow_vfta[index] & mask) != 0; 2265 pending = (sc->vf_vfta_retry[index] & mask) != 0; 2266 sc->shadow_vfta[index] |= mask; 2267 if (!present) 2268 ++sc->num_vlans; 2269 if (present && !pending) 2270 return; 2271 2272 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, true, false); 2273 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error)) { 2274 ixv_vlan_retry_add(sc, vtag); 2275 if (!pending) 2276 device_printf(sc->dev, 2277 "VF VLAN %u add request failed; retrying\n", vtag); 2278 } else 2279 ixv_vlan_retry_clear(sc, vtag); 2280 } /* ixv_if_register_vlan */ 2281 2282 /************************************************************************ 2283 * ixv_if_unregister_vlan 2284 * 2285 * Run via a vlan unconfig EVENT, remove our entry 2286 * in the soft vfta. 2287 ************************************************************************/ 2288 static void 2289 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag) 2290 { 2291 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2292 bool pending, present; 2293 u32 index, mask; 2294 s32 error; 2295 2296 index = (vtag >> 5) & 0x7F; 2297 mask = 1U << (vtag & 0x1F); 2298 present = (sc->shadow_vfta[index] & mask) != 0; 2299 pending = (sc->vf_vfta_retry[index] & mask) != 0; 2300 sc->shadow_vfta[index] &= ~mask; 2301 if (present) 2302 --sc->num_vlans; 2303 if (!present && !pending) 2304 return; 2305 2306 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, false, false); 2307 if (error != IXGBE_SUCCESS && ixv_vlan_error_retryable(error)) { 2308 ixv_vlan_retry_add(sc, vtag); 2309 if (!pending) 2310 device_printf(sc->dev, 2311 "VF VLAN %u remove request failed; " 2312 "retrying\n", vtag); 2313 } else 2314 ixv_vlan_retry_clear(sc, vtag); 2315 } /* ixv_if_unregister_vlan */ 2316 2317 /************************************************************************ 2318 * ixv_if_enable_intr 2319 ************************************************************************/ 2320 static void 2321 ixv_if_enable_intr(if_ctx_t ctx) 2322 { 2323 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2324 struct ixgbe_hw *hw = &sc->hw; 2325 struct ix_rx_queue *que = sc->rx_queues; 2326 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2327 2328 if (hw->adapter_stopped || 2329 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) 2330 return; 2331 2332 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 2333 2334 mask = IXGBE_EIMS_ENABLE_MASK; 2335 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 2336 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2337 2338 for (int i = 0; i < sc->num_rx_queues; i++, que++) 2339 ixv_enable_queue(sc, que->msix); 2340 2341 IXGBE_WRITE_FLUSH(hw); 2342 } /* ixv_if_enable_intr */ 2343 2344 /************************************************************************ 2345 * ixv_if_disable_intr 2346 ************************************************************************/ 2347 static void 2348 ixv_if_disable_intr(if_ctx_t ctx) 2349 { 2350 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2351 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2352 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, ~0); 2353 IXGBE_WRITE_FLUSH(&sc->hw); 2354 } /* ixv_if_disable_intr */ 2355 2356 /************************************************************************ 2357 * ixv_if_rx_queue_intr_enable 2358 ************************************************************************/ 2359 static int 2360 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 2361 { 2362 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2363 struct ix_rx_queue *que = &sc->rx_queues[rxqid]; 2364 2365 ixv_enable_queue(sc, que->rxr.me); 2366 2367 return (0); 2368 } /* ixv_if_rx_queue_intr_enable */ 2369 2370 /************************************************************************ 2371 * ixv_set_ivar 2372 * 2373 * Setup the correct IVAR register for a particular MSI-X interrupt 2374 * - entry is the register array entry 2375 * - vector is the MSI-X vector for this queue 2376 * - type is RX/TX/MISC 2377 ************************************************************************/ 2378 static void 2379 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2380 { 2381 struct ixgbe_hw *hw = &sc->hw; 2382 u32 ivar, index; 2383 2384 vector |= IXGBE_IVAR_ALLOC_VAL; 2385 2386 if (type == -1) { /* MISC IVAR */ 2387 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2388 ivar &= ~0xFF; 2389 ivar |= vector; 2390 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2391 } else { /* RX/TX IVARS */ 2392 index = (16 * (entry & 1)) + (8 * type); 2393 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2394 ivar &= ~(0xFF << index); 2395 ivar |= (vector << index); 2396 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2397 } 2398 } /* ixv_set_ivar */ 2399 2400 /************************************************************************ 2401 * ixv_configure_ivars 2402 ************************************************************************/ 2403 static void 2404 ixv_configure_ivars(struct ixgbe_softc *sc) 2405 { 2406 struct ix_rx_queue *que = sc->rx_queues; 2407 2408 MPASS(sc->num_rx_queues == sc->num_tx_queues); 2409 2410 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 2411 /* First the RX queue entry */ 2412 ixv_set_ivar(sc, i, que->msix, 0); 2413 /* ... and the TX */ 2414 ixv_set_ivar(sc, i, que->msix, 1); 2415 /* Set an initial value in EITR */ 2416 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(que->msix), 2417 IXGBE_EITR_DEFAULT); 2418 } 2419 2420 /* For the mailbox interrupt */ 2421 ixv_set_ivar(sc, 1, sc->vector, -1); 2422 } /* ixv_configure_ivars */ 2423 2424 /************************************************************************ 2425 * ixv_init_stats 2426 ************************************************************************/ 2427 #define IXV_STAT_36_MASK 0xFFFFFFFFFULL 2428 2429 static void 2430 ixv_init_stats(struct ixgbe_softc *sc) 2431 { 2432 struct ixgbe_hw *hw = &sc->hw; 2433 2434 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2435 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2436 sc->stats.vf.last_vfgorc |= 2437 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2438 sc->stats.vf.last_vfgorc &= IXV_STAT_36_MASK; 2439 2440 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2441 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2442 sc->stats.vf.last_vfgotc |= 2443 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2444 sc->stats.vf.last_vfgotc &= IXV_STAT_36_MASK; 2445 2446 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2447 sc->stats.vf.initialized = true; 2448 } /* ixv_init_stats */ 2449 2450 #define UPDATE_STAT_32(reg, last, count) do { \ 2451 u32 current = IXGBE_READ_REG(hw, reg); \ 2452 count += (u32)(current - (u32)last); \ 2453 last = current; \ 2454 } while (0) 2455 2456 #define UPDATE_STAT_36(lsb, msb, last, count) do { \ 2457 u64 current = IXGBE_READ_REG(hw, lsb); \ 2458 current |= (u64)IXGBE_READ_REG(hw, msb) << 32; \ 2459 current &= IXV_STAT_36_MASK; \ 2460 count += (current - last) & IXV_STAT_36_MASK; \ 2461 last = current; \ 2462 } while (0) 2463 2464 /************************************************************************ 2465 * ixv_update_stats - Update the board statistics counters. 2466 ************************************************************************/ 2467 void 2468 ixv_update_stats(struct ixgbe_softc *sc) 2469 { 2470 struct ixgbe_hw *hw = &sc->hw; 2471 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2472 2473 if (!stats->initialized) 2474 return; 2475 2476 UPDATE_STAT_32(IXGBE_VFGPRC, sc->stats.vf.last_vfgprc, 2477 sc->stats.vf.vfgprc); 2478 UPDATE_STAT_32(IXGBE_VFGPTC, sc->stats.vf.last_vfgptc, 2479 sc->stats.vf.vfgptc); 2480 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2481 sc->stats.vf.last_vfgorc, sc->stats.vf.vfgorc); 2482 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2483 sc->stats.vf.last_vfgotc, sc->stats.vf.vfgotc); 2484 UPDATE_STAT_32(IXGBE_VFMPRC, sc->stats.vf.last_vfmprc, 2485 sc->stats.vf.vfmprc); 2486 2487 /* Fill out the OS statistics structure */ 2488 IXGBE_SET_IPACKETS(sc, stats->vfgprc); 2489 IXGBE_SET_OPACKETS(sc, stats->vfgptc); 2490 IXGBE_SET_IBYTES(sc, stats->vfgorc); 2491 IXGBE_SET_OBYTES(sc, stats->vfgotc); 2492 IXGBE_SET_IMCASTS(sc, stats->vfmprc); 2493 } /* ixv_update_stats */ 2494 2495 /************************************************************************ 2496 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2497 ************************************************************************/ 2498 static void 2499 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2500 { 2501 device_t dev = sc->dev; 2502 struct ix_tx_queue *tx_que = sc->tx_queues; 2503 struct ix_rx_queue *rx_que = sc->rx_queues; 2504 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 2505 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 2506 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 2507 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2508 struct sysctl_oid *stat_node, *queue_node; 2509 struct sysctl_oid_list *stat_list, *queue_list; 2510 2511 #define QUEUE_NAME_LEN 32 2512 char namebuf[QUEUE_NAME_LEN]; 2513 2514 /* Driver Statistics */ 2515 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 2516 CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled"); 2517 2518 for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) { 2519 struct tx_ring *txr = &tx_que->txr; 2520 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 2521 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2522 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); 2523 queue_list = SYSCTL_CHILDREN(queue_node); 2524 2525 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", 2526 CTLFLAG_RD, &(txr->tso_tx), "TSO Packets"); 2527 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 2528 CTLFLAG_RD, &(txr->total_packets), "TX Packets"); 2529 } 2530 2531 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) { 2532 struct rx_ring *rxr = &rx_que->rxr; 2533 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 2534 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2535 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); 2536 queue_list = SYSCTL_CHILDREN(queue_node); 2537 2538 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 2539 CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue"); 2540 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", 2541 CTLFLAG_RD, &(rxr->rx_packets), "RX packets"); 2542 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 2543 CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes"); 2544 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded", 2545 CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets"); 2546 } 2547 2548 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", 2549 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 2550 "VF Statistics (read from HW registers)"); 2551 stat_list = SYSCTL_CHILDREN(stat_node); 2552 2553 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 2554 CTLFLAG_RD, &stats->vfgprc, "Good Packets Received"); 2555 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 2556 CTLFLAG_RD, &stats->vfgorc, "Good Octets Received"); 2557 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 2558 CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received"); 2559 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 2560 CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted"); 2561 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 2562 CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted"); 2563 } /* ixv_add_stats_sysctls */ 2564 2565 /************************************************************************ 2566 * ixv_print_debug_info 2567 * 2568 * Called only when em_display_debug_stats is enabled. 2569 * Provides a way to take a look at important statistics 2570 * maintained by the driver and hardware. 2571 ************************************************************************/ 2572 static void 2573 ixv_print_debug_info(struct ixgbe_softc *sc) 2574 { 2575 device_t dev = sc->dev; 2576 struct ixgbe_hw *hw = &sc->hw; 2577 2578 device_printf(dev, "Error Byte Count = %u \n", 2579 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 2580 2581 device_printf(dev, "MBX IRQ Handled: %lu\n", (long)sc->link_irq); 2582 } /* ixv_print_debug_info */ 2583 2584 /************************************************************************ 2585 * ixv_sysctl_debug 2586 ************************************************************************/ 2587 static int 2588 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 2589 { 2590 struct ixgbe_softc *sc; 2591 int error, result; 2592 2593 result = -1; 2594 error = sysctl_handle_int(oidp, &result, 0, req); 2595 2596 if (error || !req->newptr) 2597 return (error); 2598 2599 if (result == 1) { 2600 sc = (struct ixgbe_softc *)arg1; 2601 ixv_print_debug_info(sc); 2602 } 2603 2604 return error; 2605 } /* ixv_sysctl_debug */ 2606 2607 /************************************************************************ 2608 * ixv_init_device_features 2609 ************************************************************************/ 2610 static void 2611 ixv_init_device_features(struct ixgbe_softc *sc) 2612 { 2613 sc->feat_cap = IXGBE_FEATURE_NETMAP | 2614 IXGBE_FEATURE_VF | 2615 IXGBE_FEATURE_LEGACY_TX; 2616 2617 /* A tad short on feature flags for VFs, atm. */ 2618 switch (sc->hw.mac.type) { 2619 case ixgbe_mac_82599_vf: 2620 break; 2621 case ixgbe_mac_X540_vf: 2622 break; 2623 case ixgbe_mac_X550_vf: 2624 case ixgbe_mac_X550EM_x_vf: 2625 case ixgbe_mac_X550EM_a_vf: 2626 case ixgbe_mac_E610_vf: 2627 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2628 sc->feat_cap |= IXGBE_FEATURE_RSS; 2629 break; 2630 default: 2631 break; 2632 } 2633 2634 /* Enabled by default... */ 2635 /* Is a virtual function (VF) */ 2636 if (sc->feat_cap & IXGBE_FEATURE_VF) 2637 sc->feat_en |= IXGBE_FEATURE_VF; 2638 /* Netmap */ 2639 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 2640 sc->feat_en |= IXGBE_FEATURE_NETMAP; 2641 /* Receive-Side Scaling (RSS) */ 2642 if (sc->feat_cap & IXGBE_FEATURE_RSS) 2643 sc->feat_en |= IXGBE_FEATURE_RSS; 2644 /* Needs advanced context descriptor regardless of offloads req'd */ 2645 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2646 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2647 } /* ixv_init_device_features */ 2648