1 /***************************************************************************** 2 3 Copyright (c) 2001-2017, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 *****************************************************************************/ 33 34 35 #include "opt_inet.h" 36 #include "opt_inet6.h" 37 #include "opt_rss.h" 38 39 #include "ixgbe.h" 40 #include "ifdi_if.h" 41 42 #include <net/netmap.h> 43 #include <dev/netmap/netmap_kern.h> 44 45 /************************************************************************ 46 * Driver version 47 ************************************************************************/ 48 static const char ixv_driver_version[] = "2.0.1-k"; 49 50 static const sbintime_t ixv_mbx_retry_delay[] = { 51 250 * SBT_1MS, 52 1 * SBT_1S, 53 4 * SBT_1S, 54 8 * SBT_1S, 55 }; 56 57 static const struct timeval ixv_mbx_log_interval = { 60, 0 }; 58 59 /* Bound stale carrier state without flapping on a busy PF mailbox. */ 60 #define IXV_LINK_MBX_FAILURE_LIMIT 3 61 /* Match Intel's two-second VF service timer and spread PF mailbox load. */ 62 #define IXV_LINK_POLL_TICKS 4 63 64 /************************************************************************ 65 * PCI Device ID Table 66 * 67 * Used by probe to select devices to load on 68 * Last field stores an index into ixv_strings 69 * Last entry must be all 0s 70 * 71 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 72 ************************************************************************/ 73 static const pci_vendor_info_t ixv_vendor_info_array[] = 74 { 75 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 76 "Intel(R) X520 82599 Virtual Function"), 77 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 78 "Intel(R) X540 Virtual Function"), 79 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 80 "Intel(R) X550 Virtual Function"), 81 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 82 "Intel(R) X552 Virtual Function"), 83 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 84 "Intel(R) X553 Virtual Function"), 85 PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_E610_VF, 86 "Intel(R) E610 Virtual Function"), 87 /* required last entry */ 88 PVID_END 89 }; 90 91 /************************************************************************ 92 * Function prototypes 93 ************************************************************************/ 94 static void *ixv_register(device_t); 95 static int ixv_probe(device_t); 96 static int ixv_if_attach_pre(if_ctx_t); 97 static int ixv_if_attach_post(if_ctx_t); 98 static int ixv_if_detach(if_ctx_t); 99 100 static int ixv_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 101 static int ixv_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 102 int); 103 static int ixv_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 104 int); 105 static void ixv_if_queues_free(if_ctx_t); 106 static void ixv_identify_hardware(if_ctx_t); 107 static void ixv_init_device_features(struct ixgbe_softc *); 108 static int ixv_allocate_pci_resources(if_ctx_t); 109 static void ixv_free_pci_resources(if_ctx_t); 110 static int ixv_setup_interface(if_ctx_t); 111 static void ixv_if_media_status(if_ctx_t, struct ifmediareq *); 112 static int ixv_if_media_change(if_ctx_t); 113 static void ixv_if_update_admin_status(if_ctx_t); 114 static int ixv_if_msix_intr_assign(if_ctx_t, int); 115 116 static int ixv_if_mtu_set(if_ctx_t, uint32_t); 117 static void ixv_reconcile_mac(struct ixgbe_softc *, if_t); 118 static void ixv_if_init(if_ctx_t); 119 static void ixv_if_local_timer(if_ctx_t, uint16_t); 120 static void ixv_if_stop(if_ctx_t); 121 static void ixv_log_negotiate_failure(struct ixgbe_softc *, bool); 122 static void ixv_log_reset_failure(struct ixgbe_softc *, s32, bool); 123 static void ixv_mbx_retry_detach(struct ixgbe_softc *); 124 static void ixv_mbx_retry_failed(if_ctx_t); 125 static void ixv_mbx_retry_prepare(struct ixgbe_softc *); 126 static void ixv_mbx_retry_stop(struct ixgbe_softc *); 127 static void ixv_mbx_retry_succeeded(struct ixgbe_softc *); 128 static int ixv_negotiate_api(struct ixgbe_softc *); 129 static int ixv_queue_limit(struct ixgbe_softc *, bool); 130 131 static void ixv_initialize_transmit_units(if_ctx_t); 132 static void ixv_initialize_receive_units(if_ctx_t); 133 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 134 135 static void ixv_setup_vlan_support(if_ctx_t); 136 static void ixv_vlan_retry_add(struct ixgbe_softc *, u16); 137 static void ixv_vlan_retry_clear(struct ixgbe_softc *, u16); 138 static bool ixv_vlan_retry_pending(const struct ixgbe_softc *); 139 static void ixv_vlan_retry_tick(struct ixgbe_softc *); 140 static void ixv_configure_ivars(struct ixgbe_softc *); 141 static void ixv_if_enable_intr(if_ctx_t); 142 static void ixv_if_disable_intr(if_ctx_t); 143 static void ixv_if_multi_set(if_ctx_t); 144 static int ixv_if_promisc_set(if_ctx_t, int); 145 146 static void ixv_if_register_vlan(if_ctx_t, u16); 147 static void ixv_if_unregister_vlan(if_ctx_t, u16); 148 149 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter); 150 static bool ixv_if_needs_restart(if_ctx_t, enum iflib_restart_event); 151 152 static void ixv_init_stats(struct ixgbe_softc *); 153 static void ixv_update_stats(struct ixgbe_softc *); 154 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 155 156 static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 157 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 158 159 static u8 *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 160 161 /* The MSI-X Interrupt handlers */ 162 static int ixv_msix_que(void *); 163 static int ixv_msix_mbx(void *); 164 165 /************************************************************************ 166 * FreeBSD Device Interface Entry Points 167 ************************************************************************/ 168 static device_method_t ixv_methods[] = { 169 /* Device interface */ 170 DEVMETHOD(device_register, ixv_register), 171 DEVMETHOD(device_probe, ixv_probe), 172 DEVMETHOD(device_attach, iflib_device_attach), 173 DEVMETHOD(device_detach, iflib_device_detach), 174 DEVMETHOD(device_shutdown, iflib_device_shutdown), 175 DEVMETHOD(device_suspend, iflib_device_suspend), 176 DEVMETHOD(device_resume, iflib_device_resume), 177 DEVMETHOD_END 178 }; 179 180 static driver_t ixv_driver = { 181 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 182 }; 183 184 DRIVER_MODULE(ixv, pci, ixv_driver, 0, 0); 185 IFLIB_PNP_INFO(pci, ixv_driver, ixv_vendor_info_array); 186 MODULE_DEPEND(ixv, iflib, 1, 1, 1); 187 MODULE_DEPEND(ixv, pci, 1, 1, 1); 188 MODULE_DEPEND(ixv, ether, 1, 1, 1); 189 190 static device_method_t ixv_if_methods[] = { 191 DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre), 192 DEVMETHOD(ifdi_attach_post, ixv_if_attach_post), 193 DEVMETHOD(ifdi_detach, ixv_if_detach), 194 DEVMETHOD(ifdi_init, ixv_if_init), 195 DEVMETHOD(ifdi_stop, ixv_if_stop), 196 DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign), 197 DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr), 198 DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr), 199 DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable), 200 DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable), 201 DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc), 202 DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc), 203 DEVMETHOD(ifdi_queues_free, ixv_if_queues_free), 204 DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status), 205 DEVMETHOD(ifdi_multi_set, ixv_if_multi_set), 206 DEVMETHOD(ifdi_promisc_set, ixv_if_promisc_set), 207 DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set), 208 DEVMETHOD(ifdi_media_status, ixv_if_media_status), 209 DEVMETHOD(ifdi_media_change, ixv_if_media_change), 210 DEVMETHOD(ifdi_timer, ixv_if_local_timer), 211 DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan), 212 DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan), 213 DEVMETHOD(ifdi_get_counter, ixv_if_get_counter), 214 DEVMETHOD(ifdi_needs_restart, ixv_if_needs_restart), 215 DEVMETHOD_END 216 }; 217 218 static driver_t ixv_if_driver = { 219 "ixv_if", ixv_if_methods, sizeof(struct ixgbe_softc) 220 }; 221 222 #define IXV_VLAN_RETRY_BATCH 4 223 #define IXV_VLAN_RETRY_WINDOW (8 * SBT_1S) 224 225 extern struct if_txrx ixgbe_txrx; 226 227 static struct if_shared_ctx ixv_sctx_init = { 228 .isc_magic = IFLIB_MAGIC, 229 .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ 230 .isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header), 231 .isc_tx_maxsegsize = PAGE_SIZE, 232 .isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header), 233 .isc_tso_maxsegsize = PAGE_SIZE, 234 .isc_rx_maxsize = MJUM16BYTES, 235 .isc_rx_nsegments = 1, 236 .isc_rx_maxsegsize = MJUM16BYTES, 237 .isc_nfl = 1, 238 .isc_ntxqs = 1, 239 .isc_nrxqs = 1, 240 .isc_admin_intrcnt = 1, 241 .isc_vendor_info = ixv_vendor_info_array, 242 .isc_driver_version = ixv_driver_version, 243 .isc_driver = &ixv_if_driver, 244 .isc_flags = IFLIB_IS_VF | IFLIB_TSO_INIT_IP, 245 246 .isc_nrxd_min = {MIN_RXD}, 247 .isc_ntxd_min = {MIN_TXD}, 248 .isc_nrxd_max = {MAX_RXD}, 249 .isc_ntxd_max = {MAX_TXD}, 250 .isc_nrxd_default = {DEFAULT_RXD}, 251 .isc_ntxd_default = {DEFAULT_TXD}, 252 }; 253 254 static void * 255 ixv_register(device_t dev) 256 { 257 return (&ixv_sctx_init); 258 } 259 260 static int 261 ixv_probe(device_t dev) 262 { 263 if (pci_get_device(dev) == IXGBE_DEV_ID_E610_VF && 264 pci_get_subdevice(dev) == IXGBE_SUBDEV_ID_E610_VF_HV) 265 return (ENXIO); 266 267 return (iflib_device_probe(dev)); 268 } 269 270 /************************************************************************ 271 * ixv_if_tx_queues_alloc 272 ************************************************************************/ 273 static int 274 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 275 int ntxqs, int ntxqsets) 276 { 277 struct ixgbe_softc *sc = iflib_get_softc(ctx); 278 if_softc_ctx_t scctx = sc->shared; 279 struct ix_tx_queue *que; 280 int i, j, error; 281 282 MPASS(sc->num_tx_queues == ntxqsets); 283 MPASS(ntxqs == 1); 284 285 /* Allocate queue structure memory */ 286 sc->tx_queues = 287 (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * 288 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 289 if (!sc->tx_queues) { 290 device_printf(iflib_get_dev(ctx), 291 "Unable to allocate TX ring memory\n"); 292 return (ENOMEM); 293 } 294 295 for (i = 0, que = sc->tx_queues; i < ntxqsets; i++, que++) { 296 struct tx_ring *txr = &que->txr; 297 298 txr->me = i; 299 txr->sc = que->sc = sc; 300 301 /* Allocate report status array */ 302 if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * 303 scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { 304 error = ENOMEM; 305 goto fail; 306 } 307 for (j = 0; j < scctx->isc_ntxd[0]; j++) 308 txr->tx_rsq[j] = QIDX_INVALID; 309 /* get virtual and physical address of the hardware queues */ 310 txr->tail = IXGBE_VFTDT(txr->me); 311 txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs]; 312 txr->tx_paddr = paddrs[i*ntxqs]; 313 314 txr->bytes = 0; 315 txr->total_packets = 0; 316 317 } 318 319 device_printf(iflib_get_dev(ctx), "allocated for %d queues\n", 320 sc->num_tx_queues); 321 322 return (0); 323 324 fail: 325 ixv_if_queues_free(ctx); 326 327 return (error); 328 } /* ixv_if_tx_queues_alloc */ 329 330 /************************************************************************ 331 * ixv_if_rx_queues_alloc 332 ************************************************************************/ 333 static int 334 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 335 int nrxqs, int nrxqsets) 336 { 337 struct ixgbe_softc *sc = iflib_get_softc(ctx); 338 struct ix_rx_queue *que; 339 int i, error; 340 341 MPASS(sc->num_rx_queues == nrxqsets); 342 MPASS(nrxqs == 1); 343 344 /* Allocate queue structure memory */ 345 sc->rx_queues = 346 (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) * 347 nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); 348 if (!sc->rx_queues) { 349 device_printf(iflib_get_dev(ctx), 350 "Unable to allocate TX ring memory\n"); 351 error = ENOMEM; 352 goto fail; 353 } 354 355 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 356 struct rx_ring *rxr = &que->rxr; 357 rxr->me = i; 358 rxr->sc = que->sc = sc; 359 360 361 /* get the virtual and physical address of the hw queues */ 362 rxr->tail = IXGBE_VFRDT(rxr->me); 363 rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i]; 364 rxr->rx_paddr = paddrs[i*nrxqs]; 365 rxr->bytes = 0; 366 rxr->que = que; 367 } 368 369 device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n", 370 sc->num_rx_queues); 371 372 return (0); 373 374 fail: 375 ixv_if_queues_free(ctx); 376 377 return (error); 378 } /* ixv_if_rx_queues_alloc */ 379 380 /************************************************************************ 381 * ixv_if_queues_free 382 ************************************************************************/ 383 static void 384 ixv_if_queues_free(if_ctx_t ctx) 385 { 386 struct ixgbe_softc *sc = iflib_get_softc(ctx); 387 struct ix_tx_queue *que = sc->tx_queues; 388 int i; 389 390 if (que == NULL) 391 goto free; 392 393 for (i = 0; i < sc->num_tx_queues; i++, que++) { 394 struct tx_ring *txr = &que->txr; 395 if (txr->tx_rsq == NULL) 396 break; 397 398 free(txr->tx_rsq, M_DEVBUF); 399 txr->tx_rsq = NULL; 400 } 401 if (sc->tx_queues != NULL) 402 free(sc->tx_queues, M_DEVBUF); 403 free: 404 if (sc->rx_queues != NULL) 405 free(sc->rx_queues, M_DEVBUF); 406 sc->tx_queues = NULL; 407 sc->rx_queues = NULL; 408 } /* ixv_if_queues_free */ 409 410 /************************************************************************ 411 * ixv_if_attach_pre - Device initialization routine 412 * 413 * Called when the driver is being loaded. 414 * Identifies the type of hardware, allocates all resources 415 * and initializes the hardware. 416 * 417 * return 0 on success, positive on failure 418 ************************************************************************/ 419 static int 420 ixv_if_attach_pre(if_ctx_t ctx) 421 { 422 struct ixgbe_softc *sc; 423 device_t dev; 424 if_softc_ctx_t scctx; 425 struct ixgbe_hw *hw; 426 bool mailbox_ready; 427 int error = 0; 428 429 INIT_DEBUGOUT("ixv_attach: begin"); 430 431 /* Allocate, clear, and link in our sc structure */ 432 dev = iflib_get_dev(ctx); 433 sc = iflib_get_softc(ctx); 434 sc->dev = dev; 435 sc->ctx = ctx; 436 sc->hw.back = sc; 437 scctx = sc->shared = iflib_get_softc_ctx(ctx); 438 sc->media = iflib_get_media(ctx); 439 hw = &sc->hw; 440 441 /* Do base PCI setup - map BAR0 */ 442 if (ixv_allocate_pci_resources(ctx)) { 443 device_printf(dev, "ixv_allocate_pci_resources() failed!\n"); 444 error = ENXIO; 445 goto err_out; 446 } 447 448 /* SYSCTL APIs */ 449 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 450 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", 451 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 452 sc, 0, ixv_sysctl_debug, "I", "Debug Info"); 453 454 /* Determine hardware revision */ 455 ixv_identify_hardware(ctx); 456 ixv_init_device_features(sc); 457 sc->vf_link_poll_tick = device_get_unit(dev) % IXV_LINK_POLL_TICKS; 458 459 /* Initialize the shared code */ 460 error = ixgbe_init_ops_vf(hw); 461 if (error) { 462 device_printf(dev, "ixgbe_init_ops_vf() failed!\n"); 463 error = EIO; 464 goto err_out; 465 } 466 467 /* Setup the mailbox */ 468 ixgbe_init_mbx_params_vf(hw); 469 470 mailbox_ready = false; 471 error = hw->mac.ops.reset_hw(hw); 472 if (error != IXGBE_SUCCESS) { 473 /* 474 * A PF may be resetting or servicing a slow link event while its 475 * VFs are enumerated. Keep the VF attached so a later if_init can 476 * retry the mailbox handshake. 477 */ 478 ixv_log_reset_failure(sc, error, true); 479 } else { 480 error = hw->mac.ops.init_hw(hw); 481 if (error != IXGBE_SUCCESS) { 482 device_printf(dev, "...init_hw() failed with error %d\n", 483 error); 484 error = EIO; 485 goto err_out; 486 } 487 488 /* Negotiate mailbox API version. */ 489 error = ixv_negotiate_api(sc); 490 if (error != 0) { 491 ixv_log_negotiate_failure(sc, true); 492 hw->mac.ops.stop_adapter(hw); 493 } else 494 mailbox_ready = true; 495 } 496 497 /* Check if VF was disabled by PF. */ 498 if (!mailbox_ready || 499 hw->mac.ops.get_link_state(hw, &sc->link_enabled) != 0) { 500 /* PF is not capable of controlling VF state. Enable link. */ 501 sc->link_enabled = true; 502 } 503 504 /* If no mac address was assigned, make a random one */ 505 if (!ixv_check_ether_addr(hw->mac.addr)) { 506 ether_gen_addr(iflib_get_ifp(ctx), 507 (struct ether_addr *)hw->mac.addr); 508 bcopy(hw->mac.addr, hw->mac.perm_addr, 509 sizeof(hw->mac.perm_addr)); 510 } 511 512 /* Most of the iflib initialization... */ 513 514 iflib_set_mac(ctx, hw->mac.addr); 515 scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 516 ixv_queue_limit(sc, mailbox_ready); 517 scctx->isc_txqsizes[0] = 518 roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) + 519 sizeof(u32), DBA_ALIGN); 520 scctx->isc_rxqsizes[0] = 521 roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc), 522 DBA_ALIGN); 523 /* XXX */ 524 scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO | 525 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO | CSUM_SCTP | 526 CSUM_IP6_SCTP; 527 scctx->isc_tx_nsegments = IXGBE_82599_SCATTER; 528 scctx->isc_msix_bar = pci_msix_table_bar(dev); 529 scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; 530 scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE; 531 scctx->isc_tx_tso_segsize_max = PAGE_SIZE; 532 533 scctx->isc_txrx = &ixgbe_txrx; 534 535 /* We support everything the PF does; VFs do not do WoL. */ 536 scctx->isc_capabilities = IXGBE_CAPS; 537 scctx->isc_capenable = scctx->isc_capabilities; 538 atomic_store_rel_32(&sc->vf_mbx_ready, mailbox_ready); 539 callout_init(&sc->vf_mbx_retry, 1); 540 sc->vf_mbx_retry_initialized = true; 541 542 INIT_DEBUGOUT("ixv_if_attach_pre: end"); 543 544 return (0); 545 546 err_out: 547 ixv_free_pci_resources(ctx); 548 549 return (error); 550 } /* ixv_if_attach_pre */ 551 552 static int 553 ixv_if_attach_post(if_ctx_t ctx) 554 { 555 struct ixgbe_softc *sc = iflib_get_softc(ctx); 556 device_t dev = iflib_get_dev(ctx); 557 int error = 0; 558 559 /* Setup OS specific network interface */ 560 error = ixv_setup_interface(ctx); 561 if (error) { 562 device_printf(dev, "Interface setup failed: %d\n", error); 563 goto end; 564 } 565 566 /* Do the stats setup */ 567 if (atomic_load_acq_32(&sc->vf_mbx_ready) != 0) 568 ixv_init_stats(sc); 569 ixv_add_stats_sysctls(sc); 570 571 end: 572 return error; 573 } /* ixv_if_attach_post */ 574 575 /************************************************************************ 576 * ixv_detach - Device removal routine 577 * 578 * Called when the driver is being removed. 579 * Stops the adapter and deallocates all the resources 580 * that were allocated for driver operation. 581 * 582 * return 0 on success, positive on failure 583 ************************************************************************/ 584 static int 585 ixv_if_detach(if_ctx_t ctx) 586 { 587 struct ixgbe_softc *sc; 588 589 INIT_DEBUGOUT("ixv_detach: begin"); 590 591 sc = iflib_get_softc(ctx); 592 ixv_mbx_retry_detach(sc); 593 ixv_free_pci_resources(ctx); 594 595 return (0); 596 } /* ixv_if_detach */ 597 598 /************************************************************************ 599 * ixv_if_mtu_set 600 ************************************************************************/ 601 static int 602 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 603 { 604 struct ixgbe_softc *sc = iflib_get_softc(ctx); 605 if_t ifp = iflib_get_ifp(ctx); 606 int error = 0; 607 608 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 609 if (mtu > IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR) { 610 error = EINVAL; 611 } else { 612 if_setmtu(ifp, mtu); 613 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR; 614 } 615 616 return error; 617 } /* ixv_if_mtu_set */ 618 619 static void 620 ixv_reconcile_mac(struct ixgbe_softc *sc, if_t ifp) 621 { 622 uint8_t *lladdr; 623 624 if (ixgbe_validate_mac_addr(sc->hw.mac.addr) != IXGBE_SUCCESS) 625 return; 626 lladdr = (uint8_t *)if_getlladdr(ifp); 627 if (bcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) 628 return; 629 630 device_printf(sc->dev, 631 "PF rejected or replaced the requested MAC; using %6D\n", 632 sc->hw.mac.addr, ":"); 633 /* 634 * Initialization holds the context lock; avoid re-entering the driver. 635 */ 636 bcopy(sc->hw.mac.addr, lladdr, ETHER_ADDR_LEN); 637 CURVNET_SET_QUIET(if_getvnet(ifp)); 638 EVENTHANDLER_INVOKE(iflladdr_event, ifp); 639 CURVNET_RESTORE(); 640 } /* ixv_reconcile_mac */ 641 642 /************************************************************************ 643 * ixv_if_init - Init entry point 644 * 645 * Used in two ways: It is used by the stack as an init entry 646 * point in network interface structure. It is also used 647 * by the driver as a hw/sw initialization routine to get 648 * to a consistent state. 649 * 650 * return 0 on success, positive on failure 651 ************************************************************************/ 652 static void 653 ixv_if_init(if_ctx_t ctx) 654 { 655 struct ixgbe_softc *sc = iflib_get_softc(ctx); 656 if_t ifp = iflib_get_ifp(ctx); 657 device_t dev = iflib_get_dev(ctx); 658 struct ixgbe_hw *hw = &sc->hw; 659 u8 requested_addr[IXGBE_ETH_LENGTH_OF_ADDRESS]; 660 int error = 0; 661 662 INIT_DEBUGOUT("ixv_if_init: begin"); 663 ixv_mbx_retry_prepare(sc); 664 hw->adapter_stopped = false; 665 hw->mac.ops.stop_adapter(hw); 666 667 /* Preserve a requested LAA across the reset handshake. */ 668 bcopy(if_getlladdr(ifp), requested_addr, sizeof(requested_addr)); 669 670 /* Reset VF and renegotiate mailbox API version. */ 671 error = hw->mac.ops.reset_hw(hw); 672 if (error != IXGBE_SUCCESS) { 673 sc->stats.vf.initialized = false; 674 ixv_log_reset_failure(sc, error, false); 675 hw->mac.ops.stop_adapter(hw); 676 ixv_mbx_retry_failed(ctx); 677 return; 678 } 679 hw->mac.ops.start_hw(hw); 680 ixv_init_stats(sc); 681 error = ixv_negotiate_api(sc); 682 if (error) { 683 /* 684 * Leave the adapter stopped until an explicit or deferred retry. 685 * Otherwise the admin-status callback immediately requests another 686 * reset and can keep its taskqueue in a tight loop while the PF is 687 * deliberately withholding mailbox CTS (for example, when the VF is 688 * quarantined). 689 */ 690 ixv_log_negotiate_failure(sc, false); 691 hw->mac.ops.stop_adapter(hw); 692 ixv_mbx_retry_failed(ctx); 693 return; 694 } 695 /* Program the address only after the PF mailbox is responsive. */ 696 error = hw->mac.ops.set_rar(hw, 0, requested_addr, 0, 1); 697 if (error == IXGBE_SUCCESS) 698 bcopy(requested_addr, hw->mac.addr, sizeof(requested_addr)); 699 else 700 hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 701 ixv_reconcile_mac(sc, ifp); 702 ixv_mbx_retry_succeeded(sc); 703 704 ixv_initialize_transmit_units(ctx); 705 706 /* Setup Multicast table */ 707 ixv_if_multi_set(ctx); 708 709 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 710 711 /* Configure RX settings */ 712 ixv_initialize_receive_units(ctx); 713 714 /* Set up VLAN offload and filter */ 715 ixv_setup_vlan_support(ctx); 716 717 /* Set up MSI-X routing */ 718 ixv_configure_ivars(sc); 719 720 /* Set up auto-mask */ 721 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 722 723 /* Set moderation on the Link interrupt */ 724 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(sc->vector), IXGBE_LINK_ITR); 725 726 /* Config/Enable Link */ 727 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 728 if (error) { 729 /* PF is not capable of controlling VF state. Enable the link. */ 730 sc->link_enabled = true; 731 } else if (sc->link_enabled == false) 732 device_printf(dev, "VF is disabled by PF\n"); 733 734 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 735 false); 736 737 /* And now turn on interrupts */ 738 ixv_if_enable_intr(ctx); 739 740 return; 741 } /* ixv_if_init */ 742 743 static const char * 744 ixv_reset_error_desc(s32 error) 745 { 746 747 switch (error) { 748 case IXGBE_ERR_RESET_FAILED: 749 return ("PF reset acknowledgement timed out"); 750 case IXGBE_ERR_INVALID_MAC_ADDR: 751 /* 752 * The shared VF reset code historically uses this error for an 753 * unexpected reset reply, before it validates or copies the MAC. 754 */ 755 return ("PF returned an invalid VF reset response"); 756 case IXGBE_ERR_MBX: 757 case IXGBE_ERR_MBX_NOMSG: 758 case IXGBE_ERR_TIMEOUT: 759 return ("PF mailbox reset exchange failed"); 760 default: 761 return ("VF reset handshake failed"); 762 } 763 } 764 765 /* 766 * Report each backoff stage, then limit the steady eight-second retry so a 767 * persistent PF outage does not spam the console. 768 */ 769 static bool 770 ixv_mbx_log_allowed(struct ixgbe_softc *sc) 771 { 772 773 if (sc->vf_mbx_retry_stage == nitems(ixv_mbx_retry_delay) - 1 && 774 !ratecheck(&sc->vf_mbx_last_log, &ixv_mbx_log_interval)) 775 return (false); 776 return (true); 777 } 778 779 static void 780 ixv_log_negotiate_failure(struct ixgbe_softc *sc, bool attaching) 781 { 782 783 if (!ixv_mbx_log_allowed(sc)) 784 return; 785 device_printf(sc->dev, "Mailbox API negotiation failed%s\n", 786 attaching ? "; continuing attach" : ""); 787 } 788 789 static void 790 ixv_log_reset_failure(struct ixgbe_softc *sc, s32 error, bool attaching) 791 { 792 793 if (!ixv_mbx_log_allowed(sc)) 794 return; 795 device_printf(sc->dev, "%s (%d)%s\n", ixv_reset_error_desc(error), 796 error, attaching ? "; continuing attach" : ""); 797 } 798 799 /* 800 * A missing PF can make the posted reset handshake wait for a full mailbox 801 * timeout. Keep that work out of stopped status paths. An administratively 802 * up VF retries complete initialization with an exponential delay capped at 803 * eight seconds, so it recovers without creating a tight mailbox poller. 804 */ 805 static void 806 ixv_mbx_retry_callout(void *arg) 807 { 808 struct ixgbe_softc *sc; 809 if_t ifp; 810 811 sc = arg; 812 if (atomic_readandclear_32(&sc->vf_mbx_retry_pending) == 0 || 813 atomic_load_acq_32(&sc->vf_mbx_ready) != 0 || 814 iflib_in_detach(sc->ctx)) 815 return; 816 ifp = iflib_get_ifp(sc->ctx); 817 if ((if_getflags(ifp) & IFF_UP) == 0) 818 return; 819 820 iflib_request_reset_if_up(sc->ctx); 821 iflib_admin_intr_deferred(sc->ctx); 822 } 823 824 static void 825 ixv_mbx_retry_detach(struct ixgbe_softc *sc) 826 { 827 828 if (!sc->vf_mbx_retry_initialized) 829 return; 830 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 831 callout_drain(&sc->vf_mbx_retry); 832 sc->vf_mbx_retry_initialized = false; 833 } 834 835 static void 836 ixv_mbx_retry_prepare(struct ixgbe_softc *sc) 837 { 838 839 if (!sc->vf_mbx_retry_initialized) 840 return; 841 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 842 callout_drain(&sc->vf_mbx_retry); 843 } 844 845 static void 846 ixv_mbx_retry_stop(struct ixgbe_softc *sc) 847 { 848 if_t ifp; 849 850 if (!sc->vf_mbx_retry_initialized) 851 return; 852 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 853 callout_drain(&sc->vf_mbx_retry); 854 ifp = iflib_get_ifp(sc->ctx); 855 if ((if_getflags(ifp) & IFF_UP) == 0) 856 sc->vf_mbx_retry_stage = 0; 857 } 858 859 static void 860 ixv_mbx_retry_failed(if_ctx_t ctx) 861 { 862 struct ixgbe_softc *sc; 863 if_t ifp; 864 sbintime_t delay; 865 u_int stage; 866 867 sc = iflib_get_softc(ctx); 868 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 869 sc->link_up = false; 870 if (sc->link_active) { 871 sc->link_active = false; 872 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 873 } 874 iflib_init_failed(ctx); 875 876 ifp = iflib_get_ifp(ctx); 877 if (!sc->vf_mbx_retry_initialized || 878 (if_getflags(ifp) & IFF_UP) == 0) 879 return; 880 stage = sc->vf_mbx_retry_stage; 881 if (stage >= nitems(ixv_mbx_retry_delay)) 882 stage = nitems(ixv_mbx_retry_delay) - 1; 883 delay = ixv_mbx_retry_delay[stage]; 884 if (sc->vf_mbx_retry_stage + 1 < nitems(ixv_mbx_retry_delay)) 885 sc->vf_mbx_retry_stage++; 886 atomic_set_32(&sc->vf_mbx_retry_pending, 1); 887 callout_reset_sbt(&sc->vf_mbx_retry, delay, 0, 888 ixv_mbx_retry_callout, sc, C_PREL(1)); 889 } 890 891 static void 892 ixv_mbx_retry_succeeded(struct ixgbe_softc *sc) 893 { 894 895 atomic_store_rel_32(&sc->vf_mbx_ready, 1); 896 atomic_readandclear_32(&sc->vf_mbx_retry_pending); 897 if (sc->vf_mbx_retry_initialized) 898 callout_stop(&sc->vf_mbx_retry); 899 sc->vf_mbx_retry_stage = 0; 900 sc->vf_link_mbx_failures = 0; 901 sc->vf_link_poll_tick = 902 device_get_unit(sc->dev) % IXV_LINK_POLL_TICKS; 903 sc->vf_mbx_last_log.tv_sec = 0; 904 sc->vf_mbx_last_log.tv_usec = 0; 905 } 906 907 /************************************************************************ 908 * ixv_enable_queue 909 ************************************************************************/ 910 static inline void 911 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 912 { 913 struct ixgbe_hw *hw = &sc->hw; 914 u32 queue = 1 << vector; 915 u32 mask; 916 917 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 918 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 919 } /* ixv_enable_queue */ 920 921 /************************************************************************ 922 * ixv_disable_queue 923 ************************************************************************/ 924 static inline void 925 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 926 { 927 struct ixgbe_hw *hw = &sc->hw; 928 u64 queue = (u64)(1 << vector); 929 u32 mask; 930 931 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 932 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 933 } /* ixv_disable_queue */ 934 935 936 /************************************************************************ 937 * ixv_msix_que - MSI-X Queue Interrupt Service routine 938 ************************************************************************/ 939 static int 940 ixv_msix_que(void *arg) 941 { 942 struct ix_rx_queue *que = arg; 943 struct ixgbe_softc *sc = que->sc; 944 945 ixv_disable_queue(sc, que->msix); 946 ++que->irqs; 947 948 return (FILTER_SCHEDULE_THREAD); 949 } /* ixv_msix_que */ 950 951 /************************************************************************ 952 * ixv_msix_mbx 953 ************************************************************************/ 954 static int 955 ixv_msix_mbx(void *arg) 956 { 957 struct ixgbe_softc *sc = arg; 958 struct ixgbe_hw *hw = &sc->hw; 959 u32 reg; 960 961 ++sc->link_irq; 962 963 /* First get the cause */ 964 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 965 /* Clear interrupt with write */ 966 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 967 968 /* The admin vector also carries PF mailbox notifications. */ 969 iflib_admin_intr_deferred(sc->ctx); 970 971 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 972 973 return (FILTER_HANDLED); 974 } /* ixv_msix_mbx */ 975 976 /************************************************************************ 977 * ixv_media_status - Media Ioctl callback 978 * 979 * Called whenever the user queries the status of 980 * the interface using ifconfig. 981 ************************************************************************/ 982 static void 983 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) 984 { 985 struct ixgbe_softc *sc = iflib_get_softc(ctx); 986 987 INIT_DEBUGOUT("ixv_media_status: begin"); 988 989 /* E610 link state is refreshed by the phased service timer. */ 990 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 991 sc->hw.api_version != ixgbe_mbox_api_16) { 992 atomic_set_32(&sc->vf_link_update, 1); 993 iflib_admin_intr_deferred(ctx); 994 } 995 996 ifmr->ifm_status = IFM_AVALID; 997 ifmr->ifm_active = IFM_ETHER; 998 999 if (!sc->link_active) 1000 return; 1001 1002 ifmr->ifm_status |= IFM_ACTIVE; 1003 1004 switch (sc->link_speed) { 1005 case IXGBE_LINK_SPEED_5GB_FULL: 1006 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1007 break; 1008 case IXGBE_LINK_SPEED_2_5GB_FULL: 1009 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1010 break; 1011 case IXGBE_LINK_SPEED_1GB_FULL: 1012 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1013 break; 1014 case IXGBE_LINK_SPEED_10GB_FULL: 1015 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1016 break; 1017 case IXGBE_LINK_SPEED_100_FULL: 1018 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1019 break; 1020 case IXGBE_LINK_SPEED_10_FULL: 1021 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1022 break; 1023 } 1024 } /* ixv_if_media_status */ 1025 1026 /************************************************************************ 1027 * ixv_if_media_change - Media Ioctl callback 1028 * 1029 * Called when the user changes speed/duplex using 1030 * media/mediopt option with ifconfig. 1031 ************************************************************************/ 1032 static int 1033 ixv_if_media_change(if_ctx_t ctx) 1034 { 1035 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1036 struct ifmedia *ifm = iflib_get_media(ctx); 1037 1038 INIT_DEBUGOUT("ixv_media_change: begin"); 1039 1040 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1041 return (EINVAL); 1042 1043 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1044 case IFM_AUTO: 1045 break; 1046 default: 1047 device_printf(sc->dev, "Only auto media type\n"); 1048 return (EINVAL); 1049 } 1050 1051 return (0); 1052 } /* ixv_if_media_change */ 1053 1054 1055 /************************************************************************ 1056 * ixv_negotiate_api 1057 * 1058 * Negotiate the Mailbox API with the PF; 1059 * start with the most featured API first. 1060 ************************************************************************/ 1061 static int 1062 ixv_negotiate_api(struct ixgbe_softc *sc) 1063 { 1064 struct ixgbe_hw *hw = &sc->hw; 1065 int mbx_api[] = { 1066 ixgbe_mbox_api_13, 1067 ixgbe_mbox_api_12, 1068 ixgbe_mbox_api_11, 1069 ixgbe_mbox_api_10, 1070 ixgbe_mbox_api_unknown 1071 }; 1072 int i = 0; 1073 1074 if (hw->mac.type == ixgbe_mac_E610_vf && 1075 ixgbevf_negotiate_api_version(hw, ixgbe_mbox_api_16) == 0) 1076 return (0); 1077 1078 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1079 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1080 return (0); 1081 i++; 1082 } 1083 1084 return (EINVAL); 1085 } /* ixv_negotiate_api */ 1086 1087 /************************************************************************ 1088 * ixv_queue_limit 1089 * 1090 * Discover the number of symmetric RSS queue sets available to iflib. 1091 ************************************************************************/ 1092 static int 1093 ixv_queue_limit(struct ixgbe_softc *sc, bool mailbox_ready) 1094 { 1095 struct ixgbe_hw *hw; 1096 unsigned int default_tc, num_tcs; 1097 int admin_vectors, limit, msix_vectors; 1098 1099 hw = &sc->hw; 1100 /* Preserve the current family limit as the mailbox fallback. */ 1101 switch (hw->mac.type) { 1102 case ixgbe_mac_82599_vf: 1103 case ixgbe_mac_X540_vf: 1104 limit = 1; 1105 break; 1106 case ixgbe_mac_X550_vf: 1107 case ixgbe_mac_X550EM_x_vf: 1108 case ixgbe_mac_X550EM_a_vf: 1109 limit = 2; 1110 break; 1111 case ixgbe_mac_E610_vf: 1112 limit = 1; 1113 break; 1114 default: 1115 return (1); 1116 } 1117 1118 /* Replace the fallback with the queue grant reported by the PF. */ 1119 if (mailbox_ready) { 1120 switch (hw->api_version) { 1121 case ixgbe_mbox_api_11: 1122 case ixgbe_mbox_api_12: 1123 case ixgbe_mbox_api_13: 1124 case ixgbe_mbox_api_16: 1125 num_tcs = default_tc = 0; 1126 if (ixgbevf_get_queues(hw, &num_tcs, &default_tc) == 0) { 1127 limit = imin(hw->mac.max_tx_queues, 1128 hw->mac.max_rx_queues); 1129 limit = imin(limit, 2); 1130 } 1131 break; 1132 default: 1133 break; 1134 } 1135 } 1136 1137 /* 1138 * iflib assigns one data vector to each queue set. A VF has at most 1139 * three MSI-X vectors; reserve one of them for the mailbox interrupt. 1140 */ 1141 admin_vectors = iflib_get_sctx(sc->ctx)->isc_admin_intrcnt; 1142 msix_vectors = pci_msix_count(sc->dev); 1143 if (msix_vectors <= admin_vectors) 1144 return (1); 1145 1146 return (imax(1, imin(limit, msix_vectors - admin_vectors))); 1147 } /* ixv_queue_limit */ 1148 1149 static int 1150 ixv_update_xcast_mode(struct ixgbe_softc *sc, int flags) 1151 { 1152 if_t ifp; 1153 int mode; 1154 1155 ifp = iflib_get_ifp(sc->ctx); 1156 if (flags & IFF_PROMISC) 1157 mode = IXGBEVF_XCAST_MODE_PROMISC; 1158 else if ((flags & IFF_ALLMULTI) != 0 || 1159 if_llmaddr_count(ifp) > IXGBE_MAX_VF_MC) 1160 mode = IXGBEVF_XCAST_MODE_ALLMULTI; 1161 else if (if_llmaddr_count(ifp) != 0) 1162 mode = IXGBEVF_XCAST_MODE_MULTI; 1163 else 1164 mode = IXGBEVF_XCAST_MODE_NONE; 1165 return (ixgbevf_update_xcast_mode(&sc->hw, mode)); 1166 } 1167 1168 static int 1169 ixv_if_promisc_set(if_ctx_t ctx, int flags) 1170 { 1171 struct ixgbe_softc *sc; 1172 if_t ifp; 1173 1174 sc = iflib_get_softc(ctx); 1175 ifp = iflib_get_ifp(ctx); 1176 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 1177 return (0); 1178 if (ixv_update_xcast_mode(sc, flags) != IXGBE_SUCCESS) 1179 return (EOPNOTSUPP); 1180 return (0); 1181 } /* ixv_if_promisc_set */ 1182 1183 1184 static u_int 1185 ixv_if_multi_set_cb(void *cb_arg, struct sockaddr_dl *addr, u_int cnt) 1186 { 1187 if (cnt >= MAX_NUM_MULTICAST_ADDRESSES) 1188 return (0); 1189 1190 bcopy(LLADDR(addr), 1191 &((u8 *)cb_arg)[cnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1192 IXGBE_ETH_LENGTH_OF_ADDRESS); 1193 1194 return (1); 1195 } 1196 1197 /************************************************************************ 1198 * ixv_if_multi_set - Multicast Update 1199 * 1200 * Called whenever multicast address list is updated. 1201 ************************************************************************/ 1202 static void 1203 ixv_if_multi_set(if_ctx_t ctx) 1204 { 1205 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1206 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1207 u8 *update_ptr; 1208 if_t ifp = iflib_get_ifp(ctx); 1209 int error, mcnt = 0; 1210 1211 IOCTL_DEBUGOUT("ixv_if_multi_set: begin"); 1212 1213 mcnt = if_foreach_llmaddr(ifp, ixv_if_multi_set_cb, mta); 1214 1215 update_ptr = mta; 1216 1217 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1218 ixv_mc_array_itr, true); 1219 error = ixv_update_xcast_mode(sc, if_getflags(ifp)); 1220 if (mcnt > IXGBE_MAX_VF_MC && error != IXGBE_SUCCESS) { 1221 if (!sc->vf_mcast_overflow_warned) 1222 device_printf(sc->dev, 1223 "PF rejected all-multicast fallback; only %d " 1224 "multicast addresses are active\n", 1225 IXGBE_MAX_VF_MC); 1226 sc->vf_mcast_overflow_warned = true; 1227 } else if (mcnt <= IXGBE_MAX_VF_MC) 1228 sc->vf_mcast_overflow_warned = false; 1229 } /* ixv_if_multi_set */ 1230 1231 /************************************************************************ 1232 * ixv_mc_array_itr 1233 * 1234 * An iterator function needed by the multicast shared code. 1235 * It feeds the shared code routine the addresses in the 1236 * array of ixv_set_multi() one by one. 1237 ************************************************************************/ 1238 static u8 * 1239 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1240 { 1241 u8 *addr = *update_ptr; 1242 u8 *newptr; 1243 1244 *vmdq = 0; 1245 1246 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1247 *update_ptr = newptr; 1248 1249 return addr; 1250 } /* ixv_mc_array_itr */ 1251 1252 /************************************************************************ 1253 * ixv_if_local_timer - Timer routine 1254 * 1255 * Checks for link status, updates statistics, 1256 * and runs the watchdog check. 1257 ************************************************************************/ 1258 static void 1259 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid) 1260 { 1261 struct ixgbe_softc *sc; 1262 1263 if (qid != 0) 1264 return; 1265 1266 sc = iflib_get_softc(ctx); 1267 atomic_set_32(&sc->vf_vlan_retry_tick, 1); 1268 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 1269 sc->hw.api_version != ixgbe_mbox_api_16 || 1270 ++sc->vf_link_poll_tick == IXV_LINK_POLL_TICKS) { 1271 sc->vf_link_poll_tick = 0; 1272 atomic_set_32(&sc->vf_link_update, 1); 1273 } 1274 1275 /* Fire off the adminq task */ 1276 iflib_admin_intr_deferred(ctx); 1277 } /* ixv_if_local_timer */ 1278 1279 /************************************************************************ 1280 * ixv_if_update_admin_status - Update OS on link state 1281 * 1282 * Note: Only updates the OS on the cached link state. 1283 * The real check of the hardware only happens with 1284 * a link interrupt. 1285 ************************************************************************/ 1286 static void 1287 ixv_if_update_admin_status(if_ctx_t ctx) 1288 { 1289 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1290 device_t dev = iflib_get_dev(ctx); 1291 if_t ifp = iflib_get_ifp(ctx); 1292 bool check_link, reset_seen; 1293 s32 status; 1294 uint64_t baudrate; 1295 1296 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || 1297 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) { 1298 if (sc->link_active) { 1299 sc->link_active = false; 1300 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1301 } 1302 return; 1303 } 1304 1305 check_link = atomic_readandclear_32(&sc->vf_link_update) != 0; 1306 if (sc->hw.mac.type != ixgbe_mac_E610_vf || 1307 sc->hw.api_version != ixgbe_mbox_api_16) 1308 check_link = true; 1309 reset_seen = ixgbe_check_for_rst(&sc->hw, 0) == IXGBE_SUCCESS; 1310 if (reset_seen) 1311 sc->hw.mac.get_link_status = true; 1312 if (check_link) { 1313 sc->hw.mac.get_link_status = true; 1314 status = ixgbe_check_link(&sc->hw, &sc->link_speed, 1315 &sc->link_up, false); 1316 } else 1317 status = IXGBE_SUCCESS; 1318 if (sc->hw.mac.type == ixgbe_mac_E610_vf && 1319 sc->hw.api_version == ixgbe_mbox_api_16 && 1320 status != IXGBE_SUCCESS && status != IXGBE_ERR_MBX) { 1321 /* 1322 * A busy PF can miss an individual link-state request. Preserve 1323 * the last confirmed state across brief transport failures, but 1324 * bound how long stale carrier can remain visible if the PF is gone. 1325 */ 1326 if (sc->vf_link_mbx_failures < IXV_LINK_MBX_FAILURE_LIMIT) 1327 sc->vf_link_mbx_failures++; 1328 if (sc->vf_link_mbx_failures == IXV_LINK_MBX_FAILURE_LIMIT) 1329 sc->link_up = false; 1330 status = IXGBE_SUCCESS; 1331 } else if (status == IXGBE_SUCCESS) 1332 sc->vf_link_mbx_failures = 0; 1333 /* Reinitialize after an unsolicited reset, even while link is down. */ 1334 if (reset_seen) 1335 status = IXGBE_ERR_MBX; 1336 1337 if (status != IXGBE_SUCCESS && sc->hw.adapter_stopped == false) { 1338 /* Mailbox's Clear To Send status is lost or timeout occurred. 1339 * We need reinitialization. */ 1340 iflib_request_reset(ctx); 1341 iflib_admin_intr_deferred(ctx); 1342 } 1343 1344 if (sc->link_up && sc->link_enabled) { 1345 if (sc->link_active == false) { 1346 if (bootverbose) { 1347 baudrate = ixgbe_link_speed_to_baudrate( 1348 sc->link_speed); 1349 device_printf(dev, 1350 "Link is up %ju Mbps Full Duplex\n", 1351 (uintmax_t)(baudrate / IF_Mbps(1))); 1352 } 1353 sc->link_active = true; 1354 iflib_link_state_change(ctx, LINK_STATE_UP, 1355 ixgbe_link_speed_to_baudrate(sc->link_speed)); 1356 } 1357 } else { /* Link down */ 1358 if (sc->link_active == true) { 1359 if (bootverbose) 1360 device_printf(dev, "Link is Down\n"); 1361 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1362 sc->link_active = false; 1363 } 1364 } 1365 1366 /* iflib clears RUNNING before stop; do not replay after VF reset. */ 1367 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 && 1368 atomic_readandclear_32(&sc->vf_vlan_retry_tick) != 0) 1369 ixv_vlan_retry_tick(sc); 1370 1371 /* Do not treat a PF reset as a hardware-counter wrap. */ 1372 if (status == IXGBE_SUCCESS) 1373 ixv_update_stats(sc); 1374 else 1375 sc->stats.vf.initialized = false; 1376 } /* ixv_if_update_admin_status */ 1377 1378 1379 /************************************************************************ 1380 * ixv_if_stop - Stop the hardware 1381 * 1382 * Disables all traffic on the adapter by issuing a 1383 * global reset on the MAC and deallocates TX/RX buffers. 1384 ************************************************************************/ 1385 static void 1386 ixv_if_stop(if_ctx_t ctx) 1387 { 1388 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1389 struct ixgbe_hw *hw = &sc->hw; 1390 if_t ifp = iflib_get_ifp(ctx); 1391 bool mailbox_ready, reset_seen; 1392 1393 INIT_DEBUGOUT("ixv_stop: begin\n"); 1394 1395 ixv_mbx_retry_stop(sc); 1396 ixv_if_disable_intr(ctx); 1397 1398 mailbox_ready = atomic_load_acq_32(&sc->vf_mbx_ready) != 0; 1399 reset_seen = mailbox_ready && 1400 ixgbe_check_for_rst(hw, 0) == IXGBE_SUCCESS; 1401 if (reset_seen) 1402 sc->stats.vf.initialized = false; 1403 else if (mailbox_ready && sc->stats.vf.initialized) 1404 ixv_update_stats(sc); 1405 if (mailbox_ready && (if_getflags(ifp) & IFF_UP) == 0) { 1406 if (hw->mac.ops.reset_hw(hw) == IXGBE_SUCCESS) 1407 ixv_init_stats(sc); 1408 else 1409 sc->stats.vf.initialized = false; 1410 } 1411 atomic_store_rel_32(&sc->vf_mbx_ready, 0); 1412 sc->vf_link_mbx_failures = 0; 1413 sc->hw.adapter_stopped = false; 1414 hw->mac.ops.stop_adapter(hw); 1415 1416 /* Publish the stopped state without touching the PF mailbox. */ 1417 sc->link_up = false; 1418 if (sc->link_active) { 1419 sc->link_active = false; 1420 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1421 } 1422 } /* ixv_if_stop */ 1423 1424 1425 /************************************************************************ 1426 * ixv_identify_hardware - Determine hardware revision. 1427 ************************************************************************/ 1428 static void 1429 ixv_identify_hardware(if_ctx_t ctx) 1430 { 1431 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1432 device_t dev = iflib_get_dev(ctx); 1433 struct ixgbe_hw *hw = &sc->hw; 1434 1435 /* Save off the information about this board */ 1436 hw->vendor_id = pci_get_vendor(dev); 1437 hw->device_id = pci_get_device(dev); 1438 hw->revision_id = pci_get_revid(dev); 1439 hw->subsystem_vendor_id = pci_get_subvendor(dev); 1440 hw->subsystem_device_id = pci_get_subdevice(dev); 1441 1442 /* A subset of set_mac_type */ 1443 switch (hw->device_id) { 1444 case IXGBE_DEV_ID_82599_VF: 1445 hw->mac.type = ixgbe_mac_82599_vf; 1446 break; 1447 case IXGBE_DEV_ID_X540_VF: 1448 hw->mac.type = ixgbe_mac_X540_vf; 1449 break; 1450 case IXGBE_DEV_ID_X550_VF: 1451 hw->mac.type = ixgbe_mac_X550_vf; 1452 break; 1453 case IXGBE_DEV_ID_X550EM_X_VF: 1454 hw->mac.type = ixgbe_mac_X550EM_x_vf; 1455 break; 1456 case IXGBE_DEV_ID_X550EM_A_VF: 1457 hw->mac.type = ixgbe_mac_X550EM_a_vf; 1458 break; 1459 case IXGBE_DEV_ID_E610_VF: 1460 hw->mac.type = ixgbe_mac_E610_vf; 1461 break; 1462 default: 1463 device_printf(dev, "unknown mac type\n"); 1464 hw->mac.type = ixgbe_mac_unknown; 1465 break; 1466 } 1467 } /* ixv_identify_hardware */ 1468 1469 /************************************************************************ 1470 * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers 1471 ************************************************************************/ 1472 static int 1473 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix) 1474 { 1475 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1476 device_t dev = iflib_get_dev(ctx); 1477 struct ix_rx_queue *rx_que = sc->rx_queues; 1478 struct ix_tx_queue *tx_que; 1479 int error, rid, vector = 0; 1480 char buf[16]; 1481 1482 for (int i = 0; i < sc->num_rx_queues; i++, vector++, rx_que++) { 1483 rid = vector + 1; 1484 1485 snprintf(buf, sizeof(buf), "rxq%d", i); 1486 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 1487 IFLIB_INTR_RXTX, ixv_msix_que, rx_que, rx_que->rxr.me, 1488 buf); 1489 1490 if (error) { 1491 device_printf(iflib_get_dev(ctx), 1492 "Failed to allocate que int %d err: %d", 1493 i, error); 1494 sc->num_rx_queues = i + 1; 1495 goto fail; 1496 } 1497 1498 rx_que->msix = vector; 1499 } 1500 1501 for (int i = 0; i < sc->num_tx_queues; i++) { 1502 snprintf(buf, sizeof(buf), "txq%d", i); 1503 tx_que = &sc->tx_queues[i]; 1504 tx_que->msix = i % sc->num_rx_queues; 1505 iflib_softirq_alloc_generic(ctx, 1506 &sc->rx_queues[tx_que->msix].que_irq, 1507 IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); 1508 } 1509 rid = vector + 1; 1510 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, 1511 IFLIB_INTR_ADMIN, ixv_msix_mbx, sc, 0, "aq"); 1512 if (error) { 1513 device_printf(iflib_get_dev(ctx), 1514 "Failed to register admin handler"); 1515 return (error); 1516 } 1517 1518 sc->vector = vector; 1519 /* 1520 * Due to a broken design QEMU will fail to properly 1521 * enable the guest for MSIX unless the vectors in 1522 * the table are all set up, so we must rewrite the 1523 * ENABLE in the MSIX control register again at this 1524 * point to cause it to successfully initialize us. 1525 */ 1526 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 1527 int msix_ctrl; 1528 if (pci_find_cap(dev, PCIY_MSIX, &rid)) { 1529 device_printf(dev, 1530 "Finding MSIX capability failed\n"); 1531 } else { 1532 rid += PCIR_MSIX_CTRL; 1533 msix_ctrl = pci_read_config(dev, rid, 2); 1534 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1535 pci_write_config(dev, rid, msix_ctrl, 2); 1536 } 1537 } 1538 1539 return (0); 1540 1541 fail: 1542 iflib_irq_free(ctx, &sc->irq); 1543 rx_que = sc->rx_queues; 1544 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) 1545 iflib_irq_free(ctx, &rx_que->que_irq); 1546 1547 return (error); 1548 } /* ixv_if_msix_intr_assign */ 1549 1550 /************************************************************************ 1551 * ixv_allocate_pci_resources 1552 ************************************************************************/ 1553 static int 1554 ixv_allocate_pci_resources(if_ctx_t ctx) 1555 { 1556 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1557 device_t dev = iflib_get_dev(ctx); 1558 int rid; 1559 1560 rid = PCIR_BAR(0); 1561 sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1562 RF_ACTIVE); 1563 1564 if (!(sc->pci_mem)) { 1565 device_printf(dev, 1566 "Unable to allocate bus resource: memory\n"); 1567 return (ENXIO); 1568 } 1569 1570 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem); 1571 sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->pci_mem); 1572 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 1573 1574 return (0); 1575 } /* ixv_allocate_pci_resources */ 1576 1577 /************************************************************************ 1578 * ixv_free_pci_resources 1579 ************************************************************************/ 1580 static void 1581 ixv_free_pci_resources(if_ctx_t ctx) 1582 { 1583 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1584 struct ix_rx_queue *que = sc->rx_queues; 1585 device_t dev = iflib_get_dev(ctx); 1586 1587 /* Release all MSI-X queue resources */ 1588 if (sc->intr_type == IFLIB_INTR_MSIX) 1589 iflib_irq_free(ctx, &sc->irq); 1590 1591 if (que != NULL) { 1592 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 1593 iflib_irq_free(ctx, &que->que_irq); 1594 } 1595 } 1596 1597 if (sc->pci_mem != NULL) 1598 bus_release_resource(dev, SYS_RES_MEMORY, 1599 rman_get_rid(sc->pci_mem), sc->pci_mem); 1600 } /* ixv_free_pci_resources */ 1601 1602 /************************************************************************ 1603 * ixv_setup_interface 1604 * 1605 * Setup networking device structure and register an interface. 1606 ************************************************************************/ 1607 static int 1608 ixv_setup_interface(if_ctx_t ctx) 1609 { 1610 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1611 if_softc_ctx_t scctx = sc->shared; 1612 if_t ifp = iflib_get_ifp(ctx); 1613 1614 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1615 1616 if_setbaudrate(ifp, IF_Gbps(10)); 1617 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 2); 1618 1619 1620 sc->max_frame_size = if_getmtu(ifp) + IXGBE_MTU_HDR; 1621 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1622 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 1623 1624 return 0; 1625 } /* ixv_setup_interface */ 1626 1627 /************************************************************************ 1628 * ixv_if_get_counter 1629 ************************************************************************/ 1630 static uint64_t 1631 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt) 1632 { 1633 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1634 if_t ifp = iflib_get_ifp(ctx); 1635 1636 switch (cnt) { 1637 case IFCOUNTER_IPACKETS: 1638 return (sc->ipackets); 1639 case IFCOUNTER_OPACKETS: 1640 return (sc->opackets); 1641 case IFCOUNTER_IBYTES: 1642 return (sc->ibytes); 1643 case IFCOUNTER_OBYTES: 1644 return (sc->obytes); 1645 case IFCOUNTER_IMCASTS: 1646 return (sc->imcasts); 1647 default: 1648 return (if_get_counter_default(ifp, cnt)); 1649 } 1650 } /* ixv_if_get_counter */ 1651 1652 /* ixv_if_needs_restart - Tell iflib when the driver needs to be reinitialized 1653 * @ctx: iflib context 1654 * @event: event code to check 1655 * 1656 * Defaults to returning true for every event. 1657 * 1658 * @returns true if iflib needs to reinit the interface 1659 */ 1660 static bool 1661 ixv_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 1662 { 1663 switch (event) { 1664 case IFLIB_RESTART_VLAN_CONFIG: 1665 /* The callbacks update the PF directly and queue failed work. */ 1666 return (false); 1667 default: 1668 return (true); 1669 } 1670 } 1671 1672 /************************************************************************ 1673 * ixv_initialize_transmit_units - Enable transmit unit. 1674 ************************************************************************/ 1675 static void 1676 ixv_initialize_transmit_units(if_ctx_t ctx) 1677 { 1678 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1679 struct ixgbe_hw *hw = &sc->hw; 1680 if_softc_ctx_t scctx = sc->shared; 1681 struct ix_tx_queue *que = sc->tx_queues; 1682 int i; 1683 1684 for (i = 0; i < sc->num_tx_queues; i++, que++) { 1685 struct tx_ring *txr = &que->txr; 1686 u64 tdba = txr->tx_paddr; 1687 u32 txctrl, txdctl; 1688 int j = txr->me; 1689 1690 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1691 txdctl &= ~IXGBE_TXDCTL_THRESH_MASK; 1692 txdctl |= IXGBE_TXDCTL_THRESH_DEFAULT; 1693 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1694 1695 /* Set the HW Tx Head and Tail indices */ 1696 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1697 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1698 1699 /* Set Tx Tail register */ 1700 txr->tail = IXGBE_VFTDT(j); 1701 1702 txr->tx_rs_cidx = txr->tx_rs_pidx; 1703 /* Initialize the last processed descriptor to be the end of 1704 * the ring, rather than the start, so that we avoid an 1705 * off-by-one error when calculating how many descriptors are 1706 * done in the credits_update function. 1707 */ 1708 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 1709 for (int k = 0; k < scctx->isc_ntxd[0]; k++) 1710 txr->tx_rsq[k] = QIDX_INVALID; 1711 1712 /* Set Ring parameters */ 1713 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1714 (tdba & 0x00000000ffffffffULL)); 1715 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1716 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1717 scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc)); 1718 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1719 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1720 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1721 1722 /* Now enable */ 1723 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1724 txdctl |= IXGBE_TXDCTL_ENABLE; 1725 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1726 } 1727 1728 return; 1729 } /* ixv_initialize_transmit_units */ 1730 1731 /************************************************************************ 1732 * ixv_initialize_rss_mapping 1733 ************************************************************************/ 1734 static void 1735 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1736 { 1737 struct ixgbe_hw *hw = &sc->hw; 1738 u32 reta = 0, mrqc, rss_key[10]; 1739 int queue_id; 1740 int i, j; 1741 u32 rss_hash_config; 1742 1743 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1744 /* Fetch the configured RSS key */ 1745 rss_getkey((uint8_t *)&rss_key); 1746 } else { 1747 /* set up random bits */ 1748 arc4rand(&rss_key, sizeof(rss_key), 0); 1749 } 1750 1751 /* Now fill out hash function seeds */ 1752 for (i = 0; i < 10; i++) 1753 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1754 1755 /* Set up the redirection table */ 1756 for (i = 0, j = 0; i < 64; i++, j++) { 1757 if (j == sc->num_rx_queues) 1758 j = 0; 1759 1760 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1761 /* 1762 * Fetch the RSS bucket id for the given indirection 1763 * entry. Cap it at the number of configured buckets 1764 * (which is num_rx_queues.) 1765 */ 1766 queue_id = rss_get_indirection_to_bucket(i); 1767 queue_id = queue_id % sc->num_rx_queues; 1768 } else 1769 queue_id = j; 1770 1771 /* 1772 * The low 8 bits are for hash value (n+0); 1773 * The next 8 bits are for hash value (n+1), etc. 1774 */ 1775 reta >>= 8; 1776 reta |= ((uint32_t)queue_id) << 24; 1777 if ((i & 3) == 3) { 1778 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1779 reta = 0; 1780 } 1781 } 1782 1783 /* Perform hash on these packet types */ 1784 if (sc->feat_en & IXGBE_FEATURE_RSS) 1785 rss_hash_config = rss_gethashconfig(); 1786 else { 1787 /* 1788 * Disable UDP - IP fragments aren't currently being handled 1789 * and so we end up with a mix of 2-tuple and 4-tuple 1790 * traffic. 1791 */ 1792 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1793 | RSS_HASHTYPE_RSS_TCP_IPV4 1794 | RSS_HASHTYPE_RSS_IPV6 1795 | RSS_HASHTYPE_RSS_TCP_IPV6; 1796 } 1797 1798 mrqc = IXGBE_MRQC_RSSEN; 1799 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1800 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1801 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1802 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1803 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1804 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1805 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1806 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1807 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1808 device_printf(sc->dev, 1809 "%s: RSS_HASHTYPE_RSS_IPV6_EX defined," 1810 " but not supported\n", __func__); 1811 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1812 device_printf(sc->dev, 1813 "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined," 1814 " but not supported\n", __func__); 1815 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1816 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1817 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1818 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1819 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1820 device_printf(sc->dev, 1821 "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined," 1822 " but not supported\n", __func__); 1823 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1824 } /* ixv_initialize_rss_mapping */ 1825 1826 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 1827 /************************************************************************ 1828 * ixv_initialize_receive_units - Setup receive registers and features. 1829 ************************************************************************/ 1830 static void 1831 ixv_initialize_receive_units(if_ctx_t ctx) 1832 { 1833 struct ixgbe_softc *sc = iflib_get_softc(ctx); 1834 if_softc_ctx_t scctx; 1835 struct ixgbe_hw *hw = &sc->hw; 1836 #ifdef DEV_NETMAP 1837 if_t ifp = iflib_get_ifp(ctx); 1838 #endif 1839 struct ix_rx_queue *que = sc->rx_queues; 1840 u32 bufsz, psrtype; 1841 1842 bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 1843 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1844 1845 psrtype = IXGBE_PSRTYPE_TCPHDR | 1846 IXGBE_PSRTYPE_UDPHDR | 1847 IXGBE_PSRTYPE_IPV4HDR | 1848 IXGBE_PSRTYPE_IPV6HDR | 1849 IXGBE_PSRTYPE_L2HDR; 1850 1851 if (sc->num_rx_queues > 1) 1852 psrtype |= 1 << 29; 1853 1854 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1855 1856 /* Tell PF our max_frame size */ 1857 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) { 1858 device_printf(sc->dev, 1859 "There is a problem with the PF setup. It is likely the" 1860 " receive unit for this VF will not function correctly." 1861 "\n"); 1862 } 1863 scctx = sc->shared; 1864 1865 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 1866 struct rx_ring *rxr = &que->rxr; 1867 u64 rdba = rxr->rx_paddr; 1868 u32 reg, rxdctl; 1869 int j = rxr->me; 1870 1871 /* Disable the queue */ 1872 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1873 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1874 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1875 for (int k = 0; k < 10; k++) { 1876 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1877 IXGBE_RXDCTL_ENABLE) 1878 msec_delay(1); 1879 else 1880 break; 1881 } 1882 wmb(); 1883 /* Setup the Base and Length of the Rx Descriptor Ring */ 1884 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1885 (rdba & 0x00000000ffffffffULL)); 1886 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1887 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1888 scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc)); 1889 1890 /* Reset the ring indices */ 1891 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1892 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1893 1894 /* Set up the SRRCTL register */ 1895 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1896 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1897 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1898 reg |= bufsz; 1899 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1900 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1901 1902 /* Capture Rx Tail index */ 1903 rxr->tail = IXGBE_VFRDT(rxr->me); 1904 1905 /* Do the queue enabling last */ 1906 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1907 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1908 for (int l = 0; l < 10; l++) { 1909 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1910 IXGBE_RXDCTL_ENABLE) 1911 break; 1912 msec_delay(1); 1913 } 1914 wmb(); 1915 1916 /* Set the Tail Pointer */ 1917 #ifdef DEV_NETMAP 1918 /* 1919 * In netmap mode, we must preserve the buffers made 1920 * available to userspace before the if_init() 1921 * (this is true by default on the TX side, because 1922 * init makes all buffers available to userspace). 1923 * 1924 * netmap_reset() and the device specific routines 1925 * (e.g. ixgbe_setup_receive_rings()) map these 1926 * buffers at the end of the NIC ring, so here we 1927 * must set the RDT (tail) register to make sure 1928 * they are not overwritten. 1929 * 1930 * In this driver the NIC ring starts at RDH = 0, 1931 * RDT points to the last slot available for reception (?), 1932 * so RDT = num_rx_desc - 1 means the whole ring is available. 1933 */ 1934 if (if_getcapenable(ifp) & IFCAP_NETMAP) { 1935 struct netmap_adapter *na = NA(ifp); 1936 struct netmap_kring *kring = na->rx_rings[j]; 1937 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1938 1939 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1940 } else 1941 #endif /* DEV_NETMAP */ 1942 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1943 scctx->isc_nrxd[0] - 1); 1944 } 1945 1946 /* 1947 * Do not touch RSS and RETA settings for older hardware 1948 * as those are shared among PF and all VF. 1949 */ 1950 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 1951 ixv_initialize_rss_mapping(sc); 1952 } /* ixv_initialize_receive_units */ 1953 1954 /************************************************************************ 1955 * VF VLAN mailbox retry helpers 1956 ************************************************************************/ 1957 static void 1958 ixv_vlan_retry_add(struct ixgbe_softc *sc, u16 vid) 1959 { 1960 bool pending; 1961 1962 pending = ixv_vlan_retry_pending(sc); 1963 sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); 1964 /* Start a bounded no-progress window when work becomes pending. */ 1965 if (!pending || sc->vf_vlan_retry_deadline == 0) 1966 sc->vf_vlan_retry_deadline = 1967 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 1968 } 1969 1970 static void 1971 ixv_vlan_retry_clear(struct ixgbe_softc *sc, u16 vid) 1972 { 1973 u32 bit; 1974 bool pending; 1975 1976 bit = 1U << (vid & 0x1f); 1977 pending = (sc->vf_vfta_retry[vid >> 5] & bit) != 0; 1978 sc->vf_vfta_retry[vid >> 5] &= ~bit; 1979 if (!pending) { 1980 /* A successful mailbox operation proves the PF is responsive. */ 1981 if (sc->vf_vlan_retry_deadline == 0 && 1982 ixv_vlan_retry_pending(sc)) 1983 sc->vf_vlan_retry_deadline = 1984 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 1985 return; 1986 } 1987 if (ixv_vlan_retry_pending(sc)) 1988 sc->vf_vlan_retry_deadline = 1989 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 1990 else 1991 sc->vf_vlan_retry_deadline = 0; 1992 } 1993 1994 static bool 1995 ixv_vlan_retry_pending(const struct ixgbe_softc *sc) 1996 { 1997 int i; 1998 1999 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2000 if (sc->vf_vfta_retry[i] != 0) 2001 return (true); 2002 return (false); 2003 } 2004 2005 static void 2006 ixv_vlan_retry_tick(struct ixgbe_softc *sc) 2007 { 2008 struct ixgbe_hw *hw; 2009 bool enable; 2010 s32 error; 2011 u32 bit; 2012 u16 vid; 2013 int attempts, i, remaining; 2014 2015 if (!ixv_vlan_retry_pending(sc)) { 2016 sc->vf_vlan_retry_deadline = 0; 2017 return; 2018 } 2019 /* 2020 * Exhausted entries remain dormant until reset, a VLAN callback, or 2021 * another successful VLAN mailbox request. 2022 */ 2023 if (sc->vf_vlan_retry_deadline == 0) 2024 return; 2025 if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { 2026 remaining = 0; 2027 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2028 remaining += bitcount32(sc->vf_vfta_retry[i]); 2029 sc->vf_vlan_retry_deadline = 0; 2030 device_printf(sc->dev, 2031 "VF VLAN retries exhausted for %d VIDs\n", remaining); 2032 return; 2033 } 2034 2035 /* 2036 * A mailbox NACK does not distinguish transient PF unavailability 2037 * from a permanent policy rejection or VLVF exhaustion. Reconcile a 2038 * bounded batch per timer tick so none of those cases creates a busy 2039 * mailbox poller. Stop after the first failure so a silent PF can 2040 * consume at most one mailbox timeout per pass, while a responsive PF 2041 * can drain several successful requests. 2042 */ 2043 hw = &sc->hw; 2044 for (attempts = 0, i = 0; 2045 attempts < IXV_VLAN_RETRY_BATCH && i < 4096; i++) { 2046 vid = sc->vf_vlan_retry_cursor; 2047 sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; 2048 bit = 1U << (vid & 0x1f); 2049 if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) 2050 continue; 2051 attempts++; 2052 enable = (sc->shadow_vfta[vid >> 5] & bit) != 0; 2053 error = hw->mac.ops.set_vfta(hw, vid, 0, enable, false); 2054 if (error != IXGBE_SUCCESS) 2055 break; 2056 ixv_vlan_retry_clear(sc, vid); 2057 } 2058 } 2059 2060 /************************************************************************ 2061 * ixv_setup_vlan_support - Configure and restore VLAN support 2062 ************************************************************************/ 2063 static void 2064 ixv_setup_vlan_support(if_ctx_t ctx) 2065 { 2066 if_t ifp = iflib_get_ifp(ctx); 2067 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2068 struct ixgbe_hw *hw = &sc->hw; 2069 s32 error; 2070 u32 ctrl, vfta; 2071 u16 vid; 2072 int restore_failures; 2073 2074 for (int i = 0; i < sc->num_rx_queues; i++) { 2075 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 2076 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { 2077 ctrl |= IXGBE_RXDCTL_VME; 2078 sc->rx_queues[i].rxr.vtag_strip = true; 2079 } else { 2080 ctrl &= ~IXGBE_RXDCTL_VME; 2081 sc->rx_queues[i].rxr.vtag_strip = false; 2082 } 2083 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 2084 } 2085 2086 /* 2087 * The PF controls the pool membership independently of the VF's local 2088 * HWFILTER capability. A reset removes those memberships, so replay 2089 * every registered VLAN through the mailbox. 2090 * 2091 * Keep failed removal requests pending as well. They are harmless and 2092 * idempotent after a successful reset, and still needed if the reset 2093 * handshake did not reach the PF. 2094 */ 2095 sc->vf_vlan_retry_deadline = 0; 2096 sc->vf_vlan_retry_cursor = 0; 2097 restore_failures = 0; 2098 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2099 if (sc->shadow_vfta[i] == 0) 2100 continue; 2101 vfta = sc->shadow_vfta[i]; 2102 for (int j = 0; j < 32; j++) { 2103 if ((vfta & (1U << j)) == 0) 2104 continue; 2105 vid = (i * 32) + j; 2106 /* One timeout is enough to declare this replay deferred. */ 2107 if (restore_failures == 0) 2108 error = hw->mac.ops.set_vfta(hw, vid, 0, true, 2109 false); 2110 else 2111 error = IXGBE_ERR_MBX; 2112 if (error != IXGBE_SUCCESS) { 2113 ixv_vlan_retry_add(sc, vid); 2114 restore_failures++; 2115 } else 2116 ixv_vlan_retry_clear(sc, vid); 2117 } 2118 } 2119 if (ixv_vlan_retry_pending(sc)) 2120 sc->vf_vlan_retry_deadline = 2121 getsbinuptime() + IXV_VLAN_RETRY_WINDOW; 2122 if (restore_failures != 0) 2123 device_printf(sc->dev, 2124 "VF VLAN restore failed for %d VIDs; retrying\n", 2125 restore_failures); 2126 } /* ixv_setup_vlan_support */ 2127 2128 /************************************************************************ 2129 * ixv_if_register_vlan 2130 * 2131 * Run via a vlan config EVENT, it enables us to use the 2132 * HW Filter table since we can get the vlan id. 2133 ************************************************************************/ 2134 static void 2135 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag) 2136 { 2137 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2138 bool pending, present; 2139 u32 index, mask; 2140 s32 error; 2141 2142 index = (vtag >> 5) & 0x7F; 2143 mask = 1U << (vtag & 0x1F); 2144 present = (sc->shadow_vfta[index] & mask) != 0; 2145 pending = (sc->vf_vfta_retry[index] & mask) != 0; 2146 sc->shadow_vfta[index] |= mask; 2147 if (!present) 2148 ++sc->num_vlans; 2149 if (present && !pending) 2150 return; 2151 2152 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, true, false); 2153 if (error != IXGBE_SUCCESS) { 2154 ixv_vlan_retry_add(sc, vtag); 2155 if (!pending) 2156 device_printf(sc->dev, 2157 "VF VLAN %u add request failed; retrying\n", vtag); 2158 } else 2159 ixv_vlan_retry_clear(sc, vtag); 2160 } /* ixv_if_register_vlan */ 2161 2162 /************************************************************************ 2163 * ixv_if_unregister_vlan 2164 * 2165 * Run via a vlan unconfig EVENT, remove our entry 2166 * in the soft vfta. 2167 ************************************************************************/ 2168 static void 2169 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag) 2170 { 2171 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2172 bool pending, present; 2173 u32 index, mask; 2174 s32 error; 2175 2176 index = (vtag >> 5) & 0x7F; 2177 mask = 1U << (vtag & 0x1F); 2178 present = (sc->shadow_vfta[index] & mask) != 0; 2179 pending = (sc->vf_vfta_retry[index] & mask) != 0; 2180 sc->shadow_vfta[index] &= ~mask; 2181 if (present) 2182 --sc->num_vlans; 2183 if (!present && !pending) 2184 return; 2185 2186 error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, false, false); 2187 if (error != IXGBE_SUCCESS) { 2188 ixv_vlan_retry_add(sc, vtag); 2189 if (!pending) 2190 device_printf(sc->dev, 2191 "VF VLAN %u remove request failed; " 2192 "retrying\n", vtag); 2193 } else 2194 ixv_vlan_retry_clear(sc, vtag); 2195 } /* ixv_if_unregister_vlan */ 2196 2197 /************************************************************************ 2198 * ixv_if_enable_intr 2199 ************************************************************************/ 2200 static void 2201 ixv_if_enable_intr(if_ctx_t ctx) 2202 { 2203 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2204 struct ixgbe_hw *hw = &sc->hw; 2205 struct ix_rx_queue *que = sc->rx_queues; 2206 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 2207 2208 if (hw->adapter_stopped || 2209 atomic_load_acq_32(&sc->vf_mbx_ready) == 0) 2210 return; 2211 2212 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 2213 2214 mask = IXGBE_EIMS_ENABLE_MASK; 2215 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 2216 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2217 2218 for (int i = 0; i < sc->num_rx_queues; i++, que++) 2219 ixv_enable_queue(sc, que->msix); 2220 2221 IXGBE_WRITE_FLUSH(hw); 2222 } /* ixv_if_enable_intr */ 2223 2224 /************************************************************************ 2225 * ixv_if_disable_intr 2226 ************************************************************************/ 2227 static void 2228 ixv_if_disable_intr(if_ctx_t ctx) 2229 { 2230 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2231 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2232 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, ~0); 2233 IXGBE_WRITE_FLUSH(&sc->hw); 2234 } /* ixv_if_disable_intr */ 2235 2236 /************************************************************************ 2237 * ixv_if_rx_queue_intr_enable 2238 ************************************************************************/ 2239 static int 2240 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 2241 { 2242 struct ixgbe_softc *sc = iflib_get_softc(ctx); 2243 struct ix_rx_queue *que = &sc->rx_queues[rxqid]; 2244 2245 ixv_enable_queue(sc, que->rxr.me); 2246 2247 return (0); 2248 } /* ixv_if_rx_queue_intr_enable */ 2249 2250 /************************************************************************ 2251 * ixv_set_ivar 2252 * 2253 * Setup the correct IVAR register for a particular MSI-X interrupt 2254 * - entry is the register array entry 2255 * - vector is the MSI-X vector for this queue 2256 * - type is RX/TX/MISC 2257 ************************************************************************/ 2258 static void 2259 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2260 { 2261 struct ixgbe_hw *hw = &sc->hw; 2262 u32 ivar, index; 2263 2264 vector |= IXGBE_IVAR_ALLOC_VAL; 2265 2266 if (type == -1) { /* MISC IVAR */ 2267 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2268 ivar &= ~0xFF; 2269 ivar |= vector; 2270 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2271 } else { /* RX/TX IVARS */ 2272 index = (16 * (entry & 1)) + (8 * type); 2273 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2274 ivar &= ~(0xFF << index); 2275 ivar |= (vector << index); 2276 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2277 } 2278 } /* ixv_set_ivar */ 2279 2280 /************************************************************************ 2281 * ixv_configure_ivars 2282 ************************************************************************/ 2283 static void 2284 ixv_configure_ivars(struct ixgbe_softc *sc) 2285 { 2286 struct ix_rx_queue *que = sc->rx_queues; 2287 2288 MPASS(sc->num_rx_queues == sc->num_tx_queues); 2289 2290 for (int i = 0; i < sc->num_rx_queues; i++, que++) { 2291 /* First the RX queue entry */ 2292 ixv_set_ivar(sc, i, que->msix, 0); 2293 /* ... and the TX */ 2294 ixv_set_ivar(sc, i, que->msix, 1); 2295 /* Set an initial value in EITR */ 2296 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(que->msix), 2297 IXGBE_EITR_DEFAULT); 2298 } 2299 2300 /* For the mailbox interrupt */ 2301 ixv_set_ivar(sc, 1, sc->vector, -1); 2302 } /* ixv_configure_ivars */ 2303 2304 /************************************************************************ 2305 * ixv_init_stats 2306 ************************************************************************/ 2307 #define IXV_STAT_36_MASK 0xFFFFFFFFFULL 2308 2309 static void 2310 ixv_init_stats(struct ixgbe_softc *sc) 2311 { 2312 struct ixgbe_hw *hw = &sc->hw; 2313 2314 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2315 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2316 sc->stats.vf.last_vfgorc |= 2317 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2318 sc->stats.vf.last_vfgorc &= IXV_STAT_36_MASK; 2319 2320 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2321 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2322 sc->stats.vf.last_vfgotc |= 2323 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2324 sc->stats.vf.last_vfgotc &= IXV_STAT_36_MASK; 2325 2326 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2327 sc->stats.vf.initialized = true; 2328 } /* ixv_init_stats */ 2329 2330 #define UPDATE_STAT_32(reg, last, count) do { \ 2331 u32 current = IXGBE_READ_REG(hw, reg); \ 2332 count += (u32)(current - (u32)last); \ 2333 last = current; \ 2334 } while (0) 2335 2336 #define UPDATE_STAT_36(lsb, msb, last, count) do { \ 2337 u64 current = IXGBE_READ_REG(hw, lsb); \ 2338 current |= (u64)IXGBE_READ_REG(hw, msb) << 32; \ 2339 current &= IXV_STAT_36_MASK; \ 2340 count += (current - last) & IXV_STAT_36_MASK; \ 2341 last = current; \ 2342 } while (0) 2343 2344 /************************************************************************ 2345 * ixv_update_stats - Update the board statistics counters. 2346 ************************************************************************/ 2347 void 2348 ixv_update_stats(struct ixgbe_softc *sc) 2349 { 2350 struct ixgbe_hw *hw = &sc->hw; 2351 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2352 2353 if (!stats->initialized) 2354 return; 2355 2356 UPDATE_STAT_32(IXGBE_VFGPRC, sc->stats.vf.last_vfgprc, 2357 sc->stats.vf.vfgprc); 2358 UPDATE_STAT_32(IXGBE_VFGPTC, sc->stats.vf.last_vfgptc, 2359 sc->stats.vf.vfgptc); 2360 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2361 sc->stats.vf.last_vfgorc, sc->stats.vf.vfgorc); 2362 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2363 sc->stats.vf.last_vfgotc, sc->stats.vf.vfgotc); 2364 UPDATE_STAT_32(IXGBE_VFMPRC, sc->stats.vf.last_vfmprc, 2365 sc->stats.vf.vfmprc); 2366 2367 /* Fill out the OS statistics structure */ 2368 IXGBE_SET_IPACKETS(sc, stats->vfgprc); 2369 IXGBE_SET_OPACKETS(sc, stats->vfgptc); 2370 IXGBE_SET_IBYTES(sc, stats->vfgorc); 2371 IXGBE_SET_OBYTES(sc, stats->vfgotc); 2372 IXGBE_SET_IMCASTS(sc, stats->vfmprc); 2373 } /* ixv_update_stats */ 2374 2375 /************************************************************************ 2376 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2377 ************************************************************************/ 2378 static void 2379 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2380 { 2381 device_t dev = sc->dev; 2382 struct ix_tx_queue *tx_que = sc->tx_queues; 2383 struct ix_rx_queue *rx_que = sc->rx_queues; 2384 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 2385 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 2386 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 2387 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2388 struct sysctl_oid *stat_node, *queue_node; 2389 struct sysctl_oid_list *stat_list, *queue_list; 2390 2391 #define QUEUE_NAME_LEN 32 2392 char namebuf[QUEUE_NAME_LEN]; 2393 2394 /* Driver Statistics */ 2395 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 2396 CTLFLAG_RD, &sc->link_irq, "Link MSI-X IRQ Handled"); 2397 2398 for (int i = 0; i < sc->num_tx_queues; i++, tx_que++) { 2399 struct tx_ring *txr = &tx_que->txr; 2400 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 2401 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2402 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); 2403 queue_list = SYSCTL_CHILDREN(queue_node); 2404 2405 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", 2406 CTLFLAG_RD, &(txr->tso_tx), "TSO Packets"); 2407 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 2408 CTLFLAG_RD, &(txr->total_packets), "TX Packets"); 2409 } 2410 2411 for (int i = 0; i < sc->num_rx_queues; i++, rx_que++) { 2412 struct rx_ring *rxr = &rx_que->rxr; 2413 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 2414 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2415 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); 2416 queue_list = SYSCTL_CHILDREN(queue_node); 2417 2418 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 2419 CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue"); 2420 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", 2421 CTLFLAG_RD, &(rxr->rx_packets), "RX packets"); 2422 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 2423 CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes"); 2424 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded", 2425 CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets"); 2426 } 2427 2428 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", 2429 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 2430 "VF Statistics (read from HW registers)"); 2431 stat_list = SYSCTL_CHILDREN(stat_node); 2432 2433 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 2434 CTLFLAG_RD, &stats->vfgprc, "Good Packets Received"); 2435 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 2436 CTLFLAG_RD, &stats->vfgorc, "Good Octets Received"); 2437 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 2438 CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received"); 2439 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 2440 CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted"); 2441 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 2442 CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted"); 2443 } /* ixv_add_stats_sysctls */ 2444 2445 /************************************************************************ 2446 * ixv_print_debug_info 2447 * 2448 * Called only when em_display_debug_stats is enabled. 2449 * Provides a way to take a look at important statistics 2450 * maintained by the driver and hardware. 2451 ************************************************************************/ 2452 static void 2453 ixv_print_debug_info(struct ixgbe_softc *sc) 2454 { 2455 device_t dev = sc->dev; 2456 struct ixgbe_hw *hw = &sc->hw; 2457 2458 device_printf(dev, "Error Byte Count = %u \n", 2459 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 2460 2461 device_printf(dev, "MBX IRQ Handled: %lu\n", (long)sc->link_irq); 2462 } /* ixv_print_debug_info */ 2463 2464 /************************************************************************ 2465 * ixv_sysctl_debug 2466 ************************************************************************/ 2467 static int 2468 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 2469 { 2470 struct ixgbe_softc *sc; 2471 int error, result; 2472 2473 result = -1; 2474 error = sysctl_handle_int(oidp, &result, 0, req); 2475 2476 if (error || !req->newptr) 2477 return (error); 2478 2479 if (result == 1) { 2480 sc = (struct ixgbe_softc *)arg1; 2481 ixv_print_debug_info(sc); 2482 } 2483 2484 return error; 2485 } /* ixv_sysctl_debug */ 2486 2487 /************************************************************************ 2488 * ixv_init_device_features 2489 ************************************************************************/ 2490 static void 2491 ixv_init_device_features(struct ixgbe_softc *sc) 2492 { 2493 sc->feat_cap = IXGBE_FEATURE_NETMAP | 2494 IXGBE_FEATURE_VF | 2495 IXGBE_FEATURE_LEGACY_TX; 2496 2497 /* A tad short on feature flags for VFs, atm. */ 2498 switch (sc->hw.mac.type) { 2499 case ixgbe_mac_82599_vf: 2500 break; 2501 case ixgbe_mac_X540_vf: 2502 break; 2503 case ixgbe_mac_X550_vf: 2504 case ixgbe_mac_X550EM_x_vf: 2505 case ixgbe_mac_X550EM_a_vf: 2506 case ixgbe_mac_E610_vf: 2507 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2508 sc->feat_cap |= IXGBE_FEATURE_RSS; 2509 break; 2510 default: 2511 break; 2512 } 2513 2514 /* Enabled by default... */ 2515 /* Is a virtual function (VF) */ 2516 if (sc->feat_cap & IXGBE_FEATURE_VF) 2517 sc->feat_en |= IXGBE_FEATURE_VF; 2518 /* Netmap */ 2519 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 2520 sc->feat_en |= IXGBE_FEATURE_NETMAP; 2521 /* Receive-Side Scaling (RSS) */ 2522 if (sc->feat_cap & IXGBE_FEATURE_RSS) 2523 sc->feat_en |= IXGBE_FEATURE_RSS; 2524 /* Needs advanced context descriptor regardless of offloads req'd */ 2525 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2526 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2527 } /* ixv_init_device_features */ 2528