1 /* 2 * aQuantia Corporation Network Driver 3 * Copyright (C) 2019 aQuantia Corporation. All rights reserved 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * (1) Redistributions of source code must retain the above 10 * copyright notice, this list of conditions and the following 11 * disclaimer. 12 * 13 * (2) Redistributions in binary form must reproduce the above 14 * copyright notice, this list of conditions and the following 15 * disclaimer in the documentation and/or other materials provided 16 * with the distribution. 17 * 18 * (3)The name of the author may not be used to endorse or promote 19 * products derived from this software without specific prior 20 * written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 23 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 26 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 28 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include "opt_inet.h" 36 #include "opt_inet6.h" 37 #include "opt_rss.h" 38 39 #include <sys/param.h> 40 #include <sys/bitstring.h> 41 #include <sys/bus.h> 42 #include <sys/kernel.h> 43 #include <sys/malloc.h> 44 #include <sys/module.h> 45 #include <sys/rman.h> 46 #include <sys/sbuf.h> 47 #include <sys/socket.h> 48 #include <sys/sysctl.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <dev/pci/pcireg.h> 54 #include <dev/pci/pcivar.h> 55 56 #include <net/ethernet.h> 57 #include <net/if.h> 58 #include <net/if_dl.h> 59 #include <net/if_media.h> 60 #include <net/if_var.h> 61 #include <net/iflib.h> 62 #include <net/rss_config.h> 63 64 #include "ifdi_if.h" 65 66 #include "aq_device.h" 67 #include "aq_fw.h" 68 #include "aq_hw.h" 69 #include "aq2_hw.h" 70 #include "aq_hw_llh.h" 71 #include "aq_ring.h" 72 #include "aq_dbg.h" 73 74 MALLOC_DEFINE(M_AQ, "aq", "Aquantia"); 75 76 static const char aq_driver_version[] = AQ_VER; 77 78 #define AQUANTIA_VENDOR_ID 0x1D6A 79 80 #define AQ_DEVICE_ID_0001 0x0001 81 #define AQ_DEVICE_ID_D100 0xD100 82 #define AQ_DEVICE_ID_D107 0xD107 83 #define AQ_DEVICE_ID_D108 0xD108 84 #define AQ_DEVICE_ID_D109 0xD109 85 86 #define AQ_DEVICE_ID_AQC100 0x00B1 87 #define AQ_DEVICE_ID_AQC107 0x07B1 88 #define AQ_DEVICE_ID_AQC108 0x08B1 89 #define AQ_DEVICE_ID_AQC109 0x09B1 90 #define AQ_DEVICE_ID_AQC111 0x11B1 91 #define AQ_DEVICE_ID_AQC112 0x12B1 92 93 #define AQ_DEVICE_ID_AQC100S 0x80B1 94 #define AQ_DEVICE_ID_AQC107S 0x87B1 95 #define AQ_DEVICE_ID_AQC108S 0x88B1 96 #define AQ_DEVICE_ID_AQC109S 0x89B1 97 #define AQ_DEVICE_ID_AQC111S 0x91B1 98 #define AQ_DEVICE_ID_AQC112S 0x92B1 99 100 static pci_vendor_info_t aq_vendor_info_array[] = { 101 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_0001, 102 "Aquantia AQtion 10Gbit Network Adapter"), 103 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_D107, 104 "Aquantia AQtion 10Gbit Network Adapter"), 105 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_D108, 106 "Aquantia AQtion 5Gbit Network Adapter"), 107 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_D109, 108 "Aquantia AQtion 2.5Gbit Network Adapter"), 109 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_D100, 110 "Aquantia AQtion 10Gbit Network Adapter"), 111 112 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC107, 113 "Aquantia AQtion 10Gbit Network Adapter"), 114 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC108, 115 "Aquantia AQtion 5Gbit Network Adapter"), 116 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC109, 117 "Aquantia AQtion 2.5Gbit Network Adapter"), 118 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC100, 119 "Aquantia AQtion 10Gbit Network Adapter"), 120 121 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC107S, 122 "Aquantia AQtion 10Gbit Network Adapter"), 123 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC108S, 124 "Aquantia AQtion 5Gbit Network Adapter"), 125 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC109S, 126 "Aquantia AQtion 2.5Gbit Network Adapter"), 127 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC100S, 128 "Aquantia AQtion 10Gbit Network Adapter"), 129 130 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC111, 131 "Aquantia AQtion 5Gbit Network Adapter"), 132 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC112, 133 "Aquantia AQtion 2.5Gbit Network Adapter"), 134 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC111S, 135 "Aquantia AQtion 5Gbit Network Adapter"), 136 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC112S, 137 "Aquantia AQtion 2.5Gbit Network Adapter"), 138 139 /* Atlantic 2 (Marvell AQtion) */ 140 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC113, 141 "Marvell AQtion 10Gbit Network Adapter"), 142 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC113C, 143 "Marvell AQtion 10Gbit Network Adapter"), 144 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC113CA, 145 "Marvell AQtion 10Gbit Network Adapter"), 146 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC113CS, 147 "Marvell AQtion 10Gbit Network Adapter"), 148 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC114CS, 149 "Marvell AQtion 5Gbit Network Adapter"), 150 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC115C, 151 "Marvell AQtion 2.5Gbit Network Adapter"), 152 PVID(AQUANTIA_VENDOR_ID, AQ_DEVICE_ID_AQC116C, 153 "Marvell AQtion 1Gbit Network Adapter"), 154 155 PVID_END 156 }; 157 158 159 /* Device setup, teardown, etc */ 160 static void *aq_register(device_t dev); 161 static int aq_if_attach_pre(if_ctx_t ctx); 162 static int aq_if_attach_post(if_ctx_t ctx); 163 static int aq_if_detach(if_ctx_t ctx); 164 static int aq_if_shutdown(if_ctx_t ctx); 165 static int aq_if_suspend(if_ctx_t ctx); 166 static int aq_if_resume(if_ctx_t ctx); 167 168 /* Soft queue setup and teardown */ 169 static int aq_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, 170 uint64_t *paddrs, int ntxqs, int ntxqsets); 171 static int aq_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, 172 uint64_t *paddrs, int nrxqs, int nrxqsets); 173 static void aq_if_queues_free(if_ctx_t ctx); 174 175 /* Device configuration */ 176 static void aq_if_init(if_ctx_t ctx); 177 static void aq_if_stop(if_ctx_t ctx); 178 static void aq_if_multi_set(if_ctx_t ctx); 179 static int aq_if_mtu_set(if_ctx_t ctx, uint32_t mtu); 180 static void aq_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); 181 static int aq_if_media_change(if_ctx_t ctx); 182 static int aq_if_promisc_set(if_ctx_t ctx, int flags); 183 static uint64_t aq_if_get_counter(if_ctx_t ctx, ift_counter cnt); 184 static void aq_if_timer(if_ctx_t ctx, uint16_t qid); 185 static int aq_hw_capabilities(struct aq_dev *softc); 186 static void aq_add_stats_sysctls(struct aq_dev *softc); 187 188 /* Interrupt enable / disable */ 189 static void aq_if_enable_intr(if_ctx_t ctx); 190 static void aq_if_disable_intr(if_ctx_t ctx); 191 static int aq_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); 192 static int aq_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); 193 static int aq_if_msix_intr_assign(if_ctx_t ctx, int msix); 194 195 /* VLAN support */ 196 static bool aq_is_vlan_promisc_required(struct aq_dev *softc); 197 static void aq_update_vlan_filters(struct aq_dev *softc); 198 static void aq_if_vlan_register(if_ctx_t ctx, uint16_t vtag); 199 static void aq_if_vlan_unregister(if_ctx_t ctx, uint16_t vtag); 200 201 /* Informational/diagnostic */ 202 static void aq_if_led_func(if_ctx_t ctx, int onoff); 203 204 static device_method_t aq_methods[] = { 205 DEVMETHOD(device_register, aq_register), 206 DEVMETHOD(device_probe, iflib_device_probe), 207 DEVMETHOD(device_attach, iflib_device_attach), 208 DEVMETHOD(device_detach, iflib_device_detach), 209 DEVMETHOD(device_shutdown, iflib_device_shutdown), 210 DEVMETHOD(device_suspend, iflib_device_suspend), 211 DEVMETHOD(device_resume, iflib_device_resume), 212 213 DEVMETHOD_END 214 }; 215 216 static driver_t aq_driver = { 217 "aq", aq_methods, sizeof(struct aq_dev), 218 }; 219 220 DRIVER_MODULE(atlantic, pci, aq_driver, 0, 0); 221 222 MODULE_VERSION(atlantic, 1); 223 MODULE_DEPEND(atlantic, pci, 1, 1, 1); 224 MODULE_DEPEND(atlantic, ether, 1, 1, 1); 225 MODULE_DEPEND(atlantic, iflib, 1, 1, 1); 226 227 IFLIB_PNP_INFO(pci, atlantic, aq_vendor_info_array); 228 229 static device_method_t aq_if_methods[] = { 230 /* Device setup, teardown, etc */ 231 DEVMETHOD(ifdi_attach_pre, aq_if_attach_pre), 232 DEVMETHOD(ifdi_attach_post, aq_if_attach_post), 233 DEVMETHOD(ifdi_detach, aq_if_detach), 234 235 DEVMETHOD(ifdi_shutdown, aq_if_shutdown), 236 DEVMETHOD(ifdi_suspend, aq_if_suspend), 237 DEVMETHOD(ifdi_resume, aq_if_resume), 238 239 /* Soft queue setup and teardown */ 240 DEVMETHOD(ifdi_tx_queues_alloc, aq_if_tx_queues_alloc), 241 DEVMETHOD(ifdi_rx_queues_alloc, aq_if_rx_queues_alloc), 242 DEVMETHOD(ifdi_queues_free, aq_if_queues_free), 243 244 /* Device configuration */ 245 DEVMETHOD(ifdi_init, aq_if_init), 246 DEVMETHOD(ifdi_stop, aq_if_stop), 247 DEVMETHOD(ifdi_multi_set, aq_if_multi_set), 248 DEVMETHOD(ifdi_mtu_set, aq_if_mtu_set), 249 DEVMETHOD(ifdi_media_status, aq_if_media_status), 250 DEVMETHOD(ifdi_media_change, aq_if_media_change), 251 DEVMETHOD(ifdi_promisc_set, aq_if_promisc_set), 252 DEVMETHOD(ifdi_get_counter, aq_if_get_counter), 253 DEVMETHOD(ifdi_update_admin_status, aq_if_update_admin_status), 254 DEVMETHOD(ifdi_timer, aq_if_timer), 255 256 /* Interrupt enable / disable */ 257 DEVMETHOD(ifdi_intr_enable, aq_if_enable_intr), 258 DEVMETHOD(ifdi_intr_disable, aq_if_disable_intr), 259 DEVMETHOD(ifdi_rx_queue_intr_enable, aq_if_rx_queue_intr_enable), 260 DEVMETHOD(ifdi_tx_queue_intr_enable, aq_if_tx_queue_intr_enable), 261 DEVMETHOD(ifdi_msix_intr_assign, aq_if_msix_intr_assign), 262 263 /* VLAN support */ 264 DEVMETHOD(ifdi_vlan_register, aq_if_vlan_register), 265 DEVMETHOD(ifdi_vlan_unregister, aq_if_vlan_unregister), 266 267 /* Informational/diagnostic */ 268 DEVMETHOD(ifdi_led_func, aq_if_led_func), 269 270 DEVMETHOD_END 271 }; 272 273 static driver_t aq_if_driver = { 274 "aq_if", aq_if_methods, sizeof(struct aq_dev) 275 }; 276 277 static struct if_shared_ctx aq_sctx_init = { 278 .isc_magic = IFLIB_MAGIC, 279 .isc_q_align = PAGE_SIZE, 280 .isc_tx_maxsize = HW_ATL_B0_TSO_SIZE, 281 .isc_tx_maxsegsize = HW_ATL_B0_MTU_JUMBO, 282 .isc_tso_maxsize = HW_ATL_B0_TSO_SIZE, 283 .isc_tso_maxsegsize = HW_ATL_B0_MTU_JUMBO, 284 .isc_rx_maxsize = HW_ATL_B0_MTU_JUMBO, 285 .isc_rx_nsegments = 16, 286 .isc_rx_maxsegsize = PAGE_SIZE, 287 .isc_nfl = 1, 288 .isc_nrxqs = 1, 289 .isc_ntxqs = 1, 290 .isc_admin_intrcnt = 1, 291 .isc_vendor_info = aq_vendor_info_array, 292 .isc_driver_version = aq_driver_version, 293 .isc_driver = &aq_if_driver, 294 .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | 295 IFLIB_NEED_ZERO_CSUM, 296 297 .isc_nrxd_min = {HW_ATL_B0_MIN_RXD}, 298 .isc_ntxd_min = {HW_ATL_B0_MIN_TXD}, 299 .isc_nrxd_max = {HW_ATL_B0_MAX_RXD}, 300 .isc_ntxd_max = {HW_ATL_B0_MAX_TXD}, 301 .isc_nrxd_default = {PAGE_SIZE / sizeof(volatile union aq_txc_desc) * 4}, 302 .isc_ntxd_default = {PAGE_SIZE / sizeof(volatile union aq_txc_desc) * 4}, 303 }; 304 305 /* RSS hash types; honor the kernel policy (UDP 4-tuple off by default). */ 306 u_int 307 aq_rss_hashconfig(void) 308 { 309 #ifdef RSS 310 return (rss_gethashconfig()); 311 #else 312 return (RSS_HASHTYPE_RSS_IPV4 | RSS_HASHTYPE_RSS_TCP_IPV4 | 313 RSS_HASHTYPE_RSS_IPV6 | RSS_HASHTYPE_RSS_TCP_IPV6 | 314 RSS_HASHTYPE_RSS_IPV6_EX | RSS_HASHTYPE_RSS_TCP_IPV6_EX); 315 #endif 316 } 317 318 319 /* 320 * Device Methods 321 */ 322 static void * 323 aq_register(device_t dev) 324 { 325 return (&aq_sctx_init); 326 } 327 328 static int 329 aq_if_attach_pre(if_ctx_t ctx) 330 { 331 struct aq_dev *softc; 332 struct aq_hw *hw; 333 if_softc_ctx_t scctx; 334 int dbg, rc; 335 336 AQ_DBG_ENTER(); 337 softc = iflib_get_softc(ctx); 338 rc = 0; 339 340 sysctl_ctx_init(&softc->aq_sysctl_ctx); 341 342 softc->ctx = ctx; 343 softc->dev = iflib_get_dev(ctx); 344 softc->media = iflib_get_media(ctx); 345 softc->scctx = iflib_get_softc_ctx(ctx); 346 softc->sctx = iflib_get_sctx(ctx); 347 scctx = softc->scctx; 348 349 mtx_init(&softc->hw.fw_mtx, device_get_nameunit(softc->dev), 350 "aq firmware", MTX_DEF); 351 352 softc->mmio_rid = PCIR_BAR(0); 353 softc->mmio_res = bus_alloc_resource_any(softc->dev, SYS_RES_MEMORY, 354 &softc->mmio_rid, RF_ACTIVE|RF_SHAREABLE); 355 if (softc->mmio_res == NULL) { 356 device_printf(softc->dev, 357 "failed to allocate MMIO resources\n"); 358 rc = ENXIO; 359 goto fail; 360 } 361 362 softc->mmio_tag = rman_get_bustag(softc->mmio_res); 363 softc->mmio_handle = rman_get_bushandle(softc->mmio_res); 364 softc->mmio_size = rman_get_size(softc->mmio_res); 365 softc->hw.hw_tag = softc->mmio_tag; 366 softc->hw.hw_handle = softc->mmio_handle; 367 softc->hw.dev = softc->dev; 368 softc->hw.aq_dev = softc; 369 softc->hw.device_id = pci_get_device(softc->dev); 370 if (aq_is_atlantic2(softc->hw.device_id)) 371 softc->hw.chip_features |= AQ_HW_CHIP_ATLANTIC2; 372 hw = &softc->hw; 373 hw->link_rate = aq_fw_speed_auto; 374 hw->itr = -1; 375 hw->fc.fc_rx = 1; 376 hw->fc.fc_tx = 1; 377 softc->linkup = 0U; 378 softc->thermal_shutdown_enabled = true; 379 /* Set here, not in aq_if_init(): a recovery re-init must not reset it. */ 380 softc->thermal_retry_ticks = ticks; 381 382 softc->dbg_level = AQ_DBG_LEVEL_DEFAULT; 383 softc->dbg_categories = AQ_DBG_CATEGORIES_DEFAULT; 384 if (resource_int_value(device_get_name(softc->dev), 385 device_get_unit(softc->dev), "debug", &dbg) == 0) 386 softc->dbg_level = dbg; 387 if (resource_int_value(device_get_name(softc->dev), 388 device_get_unit(softc->dev), "debug_categories", &dbg) == 0) 389 softc->dbg_categories = dbg; 390 391 /* Look up ops and caps. */ 392 rc = aq_hw_mpi_create(hw); 393 if (rc != 0) { 394 device_printf(softc->dev, 395 "%s: aq_hw_mpi_create failed, err=%d\n", __func__, rc); 396 goto fail; 397 } 398 399 if (hw->fast_start_enabled) 400 rc = hw->fw_ops->reset(hw); 401 else 402 rc = aq_hw_reset(&softc->hw, !IS_CHIP_FEATURE(hw, ATLANTIC2)); 403 if (rc != 0) { 404 device_printf(softc->dev, "%s: reset failed, err=%d\n", 405 __func__, rc); 406 goto fail; 407 } 408 rc = aq_hw_capabilities(softc); 409 if (rc != 0) { 410 device_printf(softc->dev, "unsupported device %04x:%04x\n", 411 pci_get_vendor(softc->dev), pci_get_device(softc->dev)); 412 goto fail; 413 } 414 415 rc = aq_hw_get_mac_permanent(hw, hw->mac_addr); 416 if (rc != 0) { 417 device_printf(softc->dev, "unable to get MAC address from HW\n"); 418 goto fail; 419 } 420 421 softc->admin_ticks = 0; 422 423 iflib_set_mac(ctx, hw->mac_addr); 424 scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO | 425 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO; 426 scctx->isc_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_HWCSUM | 427 IFCAP_HWCSUM_IPV6 | IFCAP_TSO | IFCAP_LRO | IFCAP_JUMBO_MTU | 428 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | 429 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO; 430 scctx->isc_capenable = scctx->isc_capabilities; 431 scctx->isc_tx_nsegments = 31; 432 scctx->isc_tx_tso_segments_max = 31; 433 scctx->isc_tx_tso_size_max = 434 HW_ATL_B0_TSO_SIZE - sizeof(struct ether_vlan_header); 435 scctx->isc_tx_tso_segsize_max = HW_ATL_B0_MTU_JUMBO; 436 scctx->isc_min_frame_size = 52; 437 scctx->isc_max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN + 438 ETHER_VLAN_ENCAP_LEN; 439 scctx->isc_txrx = &aq_txrx; 440 441 scctx->isc_txqsizes[0] = sizeof(volatile struct aq_tx_desc) * scctx->isc_ntxd[0]; 442 scctx->isc_rxqsizes[0] = sizeof(volatile struct aq_rx_desc) * scctx->isc_nrxd[0]; 443 444 scctx->isc_ntxqsets_max = HW_ATL_B0_RINGS_MAX; 445 scctx->isc_nrxqsets_max = HW_ATL_RSS_INDIRECTION_QUEUES_MAX; 446 447 /* iflib will map and release this bar */ 448 scctx->isc_msix_bar = pci_msix_table_bar(softc->dev); 449 450 softc->vlan_tags = bit_alloc(4096, M_AQ, M_NOWAIT); 451 if (softc->vlan_tags == NULL) { 452 rc = ENOMEM; 453 goto fail; 454 } 455 456 AQ_DBG_EXIT(rc); 457 return (rc); 458 459 fail: 460 if (softc->mmio_res != NULL) 461 bus_release_resource(softc->dev, SYS_RES_MEMORY, 462 softc->mmio_rid, softc->mmio_res); 463 /* iflib skips ifdi_detach when ifdi_attach_pre fails. */ 464 mtx_destroy(&softc->hw.fw_mtx); 465 466 AQ_DBG_EXIT(rc); 467 return (rc); 468 } 469 470 471 static int 472 aq_if_attach_post(if_ctx_t ctx) 473 { 474 struct aq_dev *softc; 475 int rc; 476 477 AQ_DBG_ENTER(); 478 479 softc = iflib_get_softc(ctx); 480 rc = 0; 481 482 aq_update_hw_stats(softc); 483 484 aq_initmedia(softc); 485 486 487 switch (softc->scctx->isc_intr) { 488 case IFLIB_INTR_LEGACY: 489 rc = EOPNOTSUPP; 490 goto exit; 491 goto exit; 492 break; 493 case IFLIB_INTR_MSI: 494 rc = EOPNOTSUPP; 495 goto exit; 496 case IFLIB_INTR_MSIX: 497 break; 498 default: 499 device_printf(softc->dev, "unknown interrupt mode\n"); 500 rc = EOPNOTSUPP; 501 goto exit; 502 } 503 504 aq_add_stats_sysctls(softc); 505 /* RSS */ 506 uint32_t rss_qs = MIN(softc->rx_rings_count, HW_ATL_RSS_INDIRECTION_QUEUES_MAX); 507 _Static_assert(sizeof(softc->rss_key) == RSS_KEYSIZE, 508 "RSS key size mismatch"); 509 rss_getkey(softc->rss_key); 510 #ifdef RSS 511 for (int i = nitems(softc->rss_table); i--;) 512 softc->rss_table[i] = rss_get_indirection_to_bucket(i) % rss_qs; 513 #else 514 for (int i = nitems(softc->rss_table); i--;) 515 softc->rss_table[i] = i % rss_qs; 516 #endif 517 exit: 518 AQ_DBG_EXIT(rc); 519 return (rc); 520 } 521 522 523 static int 524 aq_if_detach(if_ctx_t ctx) 525 { 526 struct aq_dev *softc; 527 int i; 528 529 AQ_DBG_ENTER(); 530 softc = iflib_get_softc(ctx); 531 532 sysctl_ctx_free(&softc->aq_sysctl_ctx); 533 534 if (aq_hw_deinit(&softc->hw) != 0) 535 device_printf(softc->dev, "could not shut the hardware down\n"); 536 537 for (i = 0; i < softc->rx_rings_count; i++) 538 iflib_irq_free(ctx, &softc->rx_rings[i]->irq); 539 iflib_irq_free(ctx, &softc->irq); 540 541 542 if (softc->mmio_res != NULL) 543 bus_release_resource(softc->dev, SYS_RES_MEMORY, 544 softc->mmio_rid, softc->mmio_res); 545 546 free(softc->vlan_tags, M_AQ); 547 548 mtx_destroy(&softc->hw.fw_mtx); 549 550 AQ_DBG_EXIT(0); 551 return (0); 552 } 553 554 static int 555 aq_if_shutdown(if_ctx_t ctx) 556 { 557 return (aq_if_suspend(ctx)); 558 } 559 560 static int 561 aq_if_suspend(if_ctx_t ctx) 562 { 563 struct aq_dev *softc = iflib_get_softc(ctx); 564 565 AQ_DBG_ENTER(); 566 567 if (aq_hw_deinit(&softc->hw) != 0) 568 device_printf(softc->dev, 569 "could not shut the hardware down for suspend\n"); 570 571 AQ_DBG_EXIT(0); 572 return (0); 573 } 574 575 static int 576 aq_if_resume(if_ctx_t ctx) 577 { 578 struct aq_dev *softc = iflib_get_softc(ctx); 579 int err; 580 581 AQ_DBG_ENTER(); 582 err = aq_hw_mpi_create(&softc->hw); 583 AQ_DBG_EXIT(err); 584 return (err); 585 } 586 587 _Static_assert(sizeof(struct aq_ring_stats) % sizeof(counter_u64_t) == 0, 588 "aq_ring_stats must contain only counter_u64_t fields"); 589 590 static int 591 aq_ring_stats_alloc(struct aq_ring *ring) 592 { 593 counter_u64_t *c = (counter_u64_t *)&ring->stats; 594 int i, n = sizeof(ring->stats) / sizeof(counter_u64_t); 595 596 for (i = 0; i < n; i++) { 597 c[i] = counter_u64_alloc(M_NOWAIT); 598 if (c[i] == NULL) { 599 while (i-- > 0) { 600 counter_u64_free(c[i]); 601 c[i] = NULL; 602 } 603 return (ENOMEM); 604 } 605 } 606 return (0); 607 } 608 609 static void 610 aq_ring_stats_free(struct aq_ring *ring) 611 { 612 counter_u64_t *c = (counter_u64_t *)&ring->stats; 613 int i, n = sizeof(ring->stats) / sizeof(counter_u64_t); 614 615 for (i = 0; i < n; i++) { 616 if (c[i] != NULL) { 617 counter_u64_free(c[i]); 618 c[i] = NULL; 619 } 620 } 621 } 622 623 /* Soft queue setup and teardown */ 624 static int 625 aq_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 626 int ntxqs, int ntxqsets) 627 { 628 struct aq_dev *softc; 629 struct aq_ring *ring; 630 int rc = 0, i; 631 632 AQ_DBG_ENTERA("ntxqs=%d, ntxqsets=%d", ntxqs, ntxqsets); 633 softc = iflib_get_softc(ctx); 634 AQ_DBG_PRINT("tx descriptors number %d", softc->scctx->isc_ntxd[0]); 635 636 for (i = 0; i < ntxqsets; i++) { 637 ring = softc->tx_rings[i] = malloc(sizeof(struct aq_ring), 638 M_AQ, M_NOWAIT | M_ZERO); 639 if (!ring){ 640 rc = ENOMEM; 641 device_printf(softc->dev, "tx_ring malloc fail\n"); 642 goto fail; 643 } 644 ring->tx_descs = (volatile struct aq_tx_desc*)vaddrs[i]; 645 ring->tx_size = softc->scctx->isc_ntxd[0]; 646 ring->tx_descs_phys = paddrs[i]; 647 ring->tx_head = ring->tx_tail = 0; 648 ring->index = i; 649 ring->dev = softc; 650 651 softc->tx_rings_count++; 652 653 rc = aq_ring_stats_alloc(ring); 654 if (rc != 0) { 655 device_printf(softc->dev, 656 "tx_ring stats alloc fail\n"); 657 goto fail; 658 } 659 } 660 661 AQ_DBG_EXIT(rc); 662 return (rc); 663 664 fail: 665 aq_if_queues_free(ctx); 666 AQ_DBG_EXIT(rc); 667 return (rc); 668 } 669 670 static int 671 aq_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 672 int nrxqs, int nrxqsets) 673 { 674 struct aq_dev *softc; 675 struct aq_ring *ring; 676 int rc = 0, i; 677 678 AQ_DBG_ENTERA("nrxqs=%d, nrxqsets=%d", nrxqs, nrxqsets); 679 softc = iflib_get_softc(ctx); 680 681 for (i = 0; i < nrxqsets; i++) { 682 ring = softc->rx_rings[i] = malloc(sizeof(struct aq_ring), 683 M_AQ, M_NOWAIT | M_ZERO); 684 if (!ring){ 685 rc = ENOMEM; 686 device_printf(softc->dev, 687 "rx_ring malloc fail\n"); 688 goto fail; 689 } 690 691 ring->rx_descs = (volatile struct aq_rx_desc*)vaddrs[i]; 692 ring->rx_descs_phys = paddrs[i]; 693 ring->rx_size = softc->scctx->isc_nrxd[0]; 694 ring->index = i; 695 ring->dev = softc; 696 697 softc->rx_rings_count++; 698 699 rc = aq_ring_stats_alloc(ring); 700 if (rc != 0) { 701 device_printf(softc->dev, 702 "rx_ring stats alloc fail\n"); 703 goto fail; 704 } 705 } 706 707 AQ_DBG_EXIT(rc); 708 return (rc); 709 710 fail: 711 aq_if_queues_free(ctx); 712 AQ_DBG_EXIT(rc); 713 return (rc); 714 } 715 716 static void 717 aq_if_queues_free(if_ctx_t ctx) 718 { 719 struct aq_dev *softc; 720 int i; 721 722 AQ_DBG_ENTER(); 723 softc = iflib_get_softc(ctx); 724 725 for (i = 0; i < softc->tx_rings_count; i++) { 726 if (softc->tx_rings[i]) { 727 aq_ring_stats_free(softc->tx_rings[i]); 728 free(softc->tx_rings[i], M_AQ); 729 softc->tx_rings[i] = NULL; 730 } 731 } 732 softc->tx_rings_count = 0; 733 for (i = 0; i < softc->rx_rings_count; i++) { 734 if (softc->rx_rings[i]){ 735 aq_ring_stats_free(softc->rx_rings[i]); 736 free(softc->rx_rings[i], M_AQ); 737 softc->rx_rings[i] = NULL; 738 } 739 } 740 softc->rx_rings_count = 0; 741 742 AQ_DBG_EXIT(0); 743 return; 744 } 745 746 /* Device configuration */ 747 static void 748 aq_if_init(if_ctx_t ctx) 749 { 750 struct aq_dev *softc; 751 struct aq_hw *hw; 752 struct ifmediareq ifmr; 753 int i, err; 754 755 AQ_DBG_ENTER(); 756 softc = iflib_get_softc(ctx); 757 hw = &softc->hw; 758 759 atomic_store_rel_long(&hw->flags, 0); 760 761 softc->phy_fault_last = 0; 762 softc->thermal_state = AQ_THERMAL_NORMAL; 763 softc->reset_pending = false; 764 hw->tx_rings_count = softc->tx_rings_count; 765 766 /* Pick up a locally administered address set since the last init. */ 767 bcopy(if_getlladdr(iflib_get_ifp(ctx)), hw->mac_addr, ETHER_ADDR_LEN); 768 769 err = aq_hw_init(&softc->hw, softc->hw.mac_addr, softc->msix, 770 softc->scctx->isc_intr == IFLIB_INTR_MSIX); 771 if (err != 0) { 772 device_printf(softc->dev, "aq_hw_init: %d\n", err); 773 goto fail; 774 } 775 softc->init_failed = false; 776 softc->init_retries = 0; 777 778 aq_if_media_status(ctx, &ifmr); 779 780 aq_update_vlan_filters(softc); 781 782 for (i = 0; i < softc->tx_rings_count; i++) { 783 struct aq_ring *ring = softc->tx_rings[i]; 784 err = aq_ring_tx_init(&softc->hw, ring); 785 if (err) { 786 device_printf(softc->dev, 787 "aq_ring_tx_init: %d\n", err); 788 goto fail; 789 } 790 err = aq_ring_tx_start(hw, ring); 791 if (err != 0) { 792 device_printf(softc->dev, 793 "aq_ring_tx_start: %d\n", err); 794 goto fail; 795 } 796 } 797 for (i = 0; i < softc->rx_rings_count; i++) { 798 struct aq_ring *ring = softc->rx_rings[i]; 799 ring->rx_buf_size = iflib_get_rx_mbuf_sz(ctx); 800 err = aq_ring_rx_init(&softc->hw, ring); 801 if (err) { 802 device_printf(softc->dev, 803 "aq_ring_rx_init: %d\n", err); 804 goto fail; 805 } 806 err = aq_ring_rx_start(hw, ring); 807 if (err != 0) { 808 device_printf(softc->dev, 809 "aq_ring_rx_start: %d\n", err); 810 goto fail; 811 } 812 aq_if_rx_queue_intr_enable(ctx, i); 813 } 814 815 err = aq_hw_start(hw); 816 if (err != 0) { 817 device_printf(softc->dev, "could not start the datapath: %d\n", 818 err); 819 goto fail; 820 } 821 aq_if_enable_intr(ctx); 822 err = aq_hw_rss_hash_set(&softc->hw, softc->rss_key); 823 if (err != 0) 824 device_printf(softc->dev, "could not set the RSS key: %d\n", 825 err); 826 err = aq_hw_rss_set(&softc->hw, softc->rss_table); 827 if (err != 0) 828 device_printf(softc->dev, 829 "could not set the RSS indirection table: %d\n", err); 830 /* A2 selects UDP hashing per-protocol in REDIR2; A1 uses the filter. */ 831 if (!IS_CHIP_FEATURE(hw, ATLANTIC2)) { 832 err = aq_hw_udp_rss_enable(hw, (aq_rss_hashconfig() & 833 (RSS_HASHTYPE_RSS_UDP_IPV4 | RSS_HASHTYPE_RSS_UDP_IPV6 | 834 RSS_HASHTYPE_RSS_UDP_IPV6_EX)) != 0); 835 if (err != 0) 836 device_printf(softc->dev, 837 "could not configure UDP RSS hashing: %d\n", err); 838 } 839 err = aq_hw_set_link_speed(hw, hw->link_rate); 840 if (err != 0) 841 device_printf(softc->dev, "could not set link speed: %d\n", err); 842 843 /* iflib does not replay filter state after init; aq_hw_init() clears it. */ 844 aq_if_multi_set(ctx); 845 if (aq_if_promisc_set(ctx, if_getflags(iflib_get_ifp(ctx))) != 0) 846 device_printf(softc->dev, "could not restore promiscuous mode\n"); 847 848 AQ_DBG_EXIT(0); 849 return; 850 851 fail: 852 aq_if_stop(ctx); 853 softc->init_failed = true; 854 iflib_init_failed(ctx); 855 AQ_DBG_EXIT(err); 856 } 857 858 859 static void 860 aq_if_stop(if_ctx_t ctx) 861 { 862 struct aq_dev *softc; 863 struct aq_hw *hw; 864 int i; 865 866 AQ_DBG_ENTER(); 867 868 softc = iflib_get_softc(ctx); 869 hw = &softc->hw; 870 871 /* disable interrupt */ 872 aq_if_disable_intr(ctx); 873 874 for (i = 0; i < softc->tx_rings_count; i++) { 875 if (aq_ring_tx_stop(hw, softc->tx_rings[i]) != 0) 876 device_printf(softc->dev, 877 "could not stop TX ring %d\n", i); 878 softc->tx_rings[i]->tx_head = 0; 879 softc->tx_rings[i]->tx_tail = 0; 880 } 881 for (i = 0; i < softc->rx_rings_count; i++) { 882 if (aq_ring_rx_stop(hw, softc->rx_rings[i]) != 0) 883 device_printf(softc->dev, 884 "could not stop RX ring %d\n", i); 885 } 886 887 if (aq_hw_invalidate_descriptor_cache(hw) != 0) 888 device_printf(softc->dev, 889 "could not invalidate the RX descriptor cache\n"); 890 891 if (aq_hw_reset(&softc->hw, true) != 0) 892 device_printf(softc->dev, "could not reset the MAC on stop\n"); 893 memset(&softc->last_stats, 0, sizeof(softc->last_stats)); 894 /* Each bring-up gets its own budget of re-init attempts. */ 895 softc->init_retries = 0; 896 if (softc->linkup) { 897 softc->linkup = false; 898 softc->link_speed = 0; 899 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 900 } 901 AQ_DBG_EXIT(0); 902 } 903 904 static uint64_t 905 aq_if_get_counter(if_ctx_t ctx, ift_counter cnt) 906 { 907 struct aq_dev *softc = iflib_get_softc(ctx); 908 if_t ifp = iflib_get_ifp(ctx); 909 910 switch (cnt) { 911 case IFCOUNTER_IERRORS: 912 return (softc->curr_stats.erpr); 913 case IFCOUNTER_IQDROPS: 914 return (softc->curr_stats.dpc); 915 case IFCOUNTER_OERRORS: 916 return (softc->curr_stats.erpt); 917 default: 918 return (if_get_counter_default(ifp, cnt)); 919 } 920 } 921 922 static u_int 923 aq_mc_filter_apply(void *arg, struct sockaddr_dl *dl, u_int count) 924 { 925 struct aq_dev *softc = arg; 926 struct aq_hw *hw = &softc->hw; 927 uint8_t *mac_addr = NULL; 928 929 if (count >= AQ_HW_MAC_MAX - 1) 930 return (0); 931 932 mac_addr = LLADDR(dl); 933 if (aq_hw_mac_addr_set(hw, mac_addr, count + 1) != 0) { 934 device_printf(softc->dev, 935 "could not program multicast address %6D\n", mac_addr, ":"); 936 return (0); 937 } 938 939 aq_log_detail(hw, "set %d mc address %6D", count + 1, mac_addr, ":"); 940 return (1); 941 } 942 943 static bool 944 aq_is_mc_promisc_required(struct aq_dev *softc) 945 { 946 return (softc->mcnt >= AQ_HW_MAC_MAX); 947 } 948 949 static void 950 aq_if_multi_set(if_ctx_t ctx) 951 { 952 struct aq_dev *softc = iflib_get_softc(ctx); 953 if_t ifp = iflib_get_ifp(ctx); 954 struct aq_hw *hw = &softc->hw; 955 AQ_DBG_ENTER(); 956 softc->mcnt = if_llmaddr_count(ifp); 957 958 /* Reconcile HW to the current list: clear stale slots, reprogram. */ 959 if (softc->mcnt < AQ_HW_MAC_MAX) { 960 for (int i = 1; i < AQ_HW_MAC_MAX; i++) 961 rpfl2_uc_flr_en_set(hw, 0U, i); 962 if_foreach_llmaddr(ifp, &aq_mc_filter_apply, softc); 963 } 964 965 if (aq_hw_set_promisc(hw, !!(if_getflags(ifp) & IFF_PROMISC), 966 aq_is_vlan_promisc_required(softc), 967 !!(if_getflags(ifp) & IFF_ALLMULTI) || 968 aq_is_mc_promisc_required(softc)) != 0) 969 device_printf(softc->dev, 970 "multicast filter update failed\n"); 971 AQ_DBG_EXIT(0); 972 } 973 974 static int 975 aq_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 976 { 977 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 978 uint32_t max_frame; 979 980 AQ_DBG_ENTERA("mtu %u", mtu); 981 982 max_frame = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; 983 if (max_frame > HW_ATL_B0_MTU_JUMBO) { 984 AQ_DBG_EXIT(EINVAL); 985 return (EINVAL); 986 } 987 988 scctx->isc_max_frame_size = max_frame; 989 990 AQ_DBG_EXIT(0); 991 return (0); 992 } 993 994 static void 995 aq_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 996 { 997 if_t ifp; 998 999 AQ_DBG_ENTER(); 1000 1001 ifp = iflib_get_ifp(ctx); 1002 1003 aq_mediastatus(ifp, ifmr); 1004 1005 AQ_DBG_EXIT(0); 1006 } 1007 1008 static int 1009 aq_if_media_change(if_ctx_t ctx) 1010 { 1011 struct aq_dev *softc = iflib_get_softc(ctx); 1012 if_t ifp = iflib_get_ifp(ctx); 1013 int rc = 0; 1014 1015 AQ_DBG_ENTER(); 1016 1017 /* Not allowd in UP state, since causes unsync of rings */ 1018 if ((if_getflags(ifp) & IFF_UP)){ 1019 rc = EPERM; 1020 goto exit; 1021 } 1022 1023 ifp = iflib_get_ifp(softc->ctx); 1024 1025 rc = aq_mediachange(ifp); 1026 1027 exit: 1028 AQ_DBG_EXIT(rc); 1029 return (rc); 1030 } 1031 1032 static int 1033 aq_if_promisc_set(if_ctx_t ctx, int flags) 1034 { 1035 struct aq_dev *softc; 1036 int err; 1037 1038 AQ_DBG_ENTER(); 1039 1040 softc = iflib_get_softc(ctx); 1041 1042 err = aq_hw_set_promisc(&softc->hw, !!(flags & IFF_PROMISC), 1043 aq_is_vlan_promisc_required(softc), 1044 !!(flags & IFF_ALLMULTI) || aq_is_mc_promisc_required(softc)); 1045 1046 AQ_DBG_EXIT(err); 1047 return (err); 1048 } 1049 1050 static void 1051 aq_if_timer(if_ctx_t ctx, uint16_t qid) 1052 { 1053 struct aq_dev *softc; 1054 uint64_t ticks_now; 1055 1056 1057 softc = iflib_get_softc(ctx); 1058 ticks_now = ticks; 1059 1060 /* Schedule aqc_if_update_admin_status() once per sec */ 1061 if (ticks_now - softc->admin_ticks >= hz) { 1062 softc->admin_ticks = ticks_now; 1063 iflib_admin_intr_deferred(ctx); 1064 } 1065 1066 return; 1067 1068 } 1069 1070 /* Interrupt enable / disable */ 1071 static void 1072 aq_if_enable_intr(if_ctx_t ctx) 1073 { 1074 struct aq_dev *softc = iflib_get_softc(ctx); 1075 struct aq_hw *hw = &softc->hw; 1076 1077 AQ_DBG_ENTER(); 1078 1079 /* Enable interrupts */ 1080 itr_irq_msk_setlsw_set(hw, BIT(softc->msix + 1) - 1); 1081 1082 AQ_DBG_EXIT(0); 1083 } 1084 1085 static void 1086 aq_if_disable_intr(if_ctx_t ctx) 1087 { 1088 struct aq_dev *softc = iflib_get_softc(ctx); 1089 struct aq_hw *hw = &softc->hw; 1090 1091 AQ_DBG_ENTER(); 1092 1093 /* Disable interrupts */ 1094 itr_irq_msk_clearlsw_set(hw, BIT(softc->msix + 1) - 1); 1095 1096 AQ_DBG_EXIT(0); 1097 } 1098 1099 static int 1100 aq_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1101 { 1102 struct aq_dev *softc = iflib_get_softc(ctx); 1103 struct aq_hw *hw = &softc->hw; 1104 1105 AQ_DBG_ENTER(); 1106 1107 itr_irq_msk_setlsw_set(hw, BIT(softc->rx_rings[rxqid]->msix)); 1108 1109 AQ_DBG_EXIT(0); 1110 return (0); 1111 } 1112 1113 static int 1114 aq_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1115 { 1116 struct aq_dev *softc = iflib_get_softc(ctx); 1117 struct aq_hw *hw = &softc->hw; 1118 1119 AQ_DBG_ENTER(); 1120 1121 itr_irq_msk_setlsw_set(hw, BIT(softc->tx_rings[txqid]->msix)); 1122 1123 AQ_DBG_EXIT(0); 1124 return (0); 1125 } 1126 1127 static int 1128 aq_if_msix_intr_assign(if_ctx_t ctx, int msix) 1129 { 1130 struct aq_dev *softc; 1131 int i, vector = 0, rc; 1132 char irq_name[16]; 1133 int rx_vectors; 1134 1135 AQ_DBG_ENTER(); 1136 softc = iflib_get_softc(ctx); 1137 1138 for (i = 0; i < softc->rx_rings_count; i++, vector++) { 1139 snprintf(irq_name, sizeof(irq_name), "rxq%d", i); 1140 rc = iflib_irq_alloc_generic(ctx, &softc->rx_rings[i]->irq, 1141 vector + 1, IFLIB_INTR_RXTX, aq_isr_rx, softc->rx_rings[i], 1142 softc->rx_rings[i]->index, irq_name); 1143 if (rc) { 1144 device_printf(softc->dev, "failed to set up RX handler\n"); 1145 goto fail; 1146 } 1147 if (bootverbose) 1148 device_printf(softc->dev, 1149 "Assign IRQ %u to rx ring %u\n", vector, 1150 softc->rx_rings[i]->index); 1151 1152 softc->rx_rings[i]->msix = vector; 1153 } 1154 1155 rx_vectors = vector; 1156 1157 for (i = 0; i < softc->tx_rings_count; i++) { 1158 snprintf(irq_name, sizeof(irq_name), "txq%d", i); 1159 softc->tx_rings[i]->msix = (i % softc->rx_rings_count); 1160 iflib_softirq_alloc_generic(ctx, 1161 &softc->rx_rings[softc->tx_rings[i]->msix]->irq, 1162 IFLIB_INTR_TX, softc->tx_rings[i], 1163 softc->tx_rings[i]->index, irq_name); 1164 if (bootverbose) 1165 device_printf(softc->dev, 1166 "tx ring %u shares IRQ %u\n", 1167 softc->tx_rings[i]->index, 1168 softc->tx_rings[i]->msix); 1169 } 1170 1171 rc = iflib_irq_alloc_generic(ctx, &softc->irq, rx_vectors + 1, 1172 IFLIB_INTR_ADMIN, aq_linkstat_isr, softc, 0, "aq"); 1173 if (rc) { 1174 device_printf(iflib_get_dev(ctx), 1175 "Failed to register admin handler\n"); 1176 goto fail; 1177 } 1178 softc->msix = rx_vectors; 1179 if (bootverbose) 1180 device_printf(softc->dev, "Assign IRQ %u to admin proc\n", 1181 rx_vectors); 1182 AQ_DBG_EXIT(0); 1183 return (0); 1184 1185 fail: 1186 AQ_DBG_EXIT(rc); 1187 return (rc); 1188 } 1189 1190 static bool 1191 aq_is_vlan_promisc_required(struct aq_dev *softc) 1192 { 1193 int vlan_tag_count; 1194 1195 bit_count(softc->vlan_tags, 0, 4096, &vlan_tag_count); 1196 1197 /* Filter only with 1..16 VLANs; 0 or >16 pass all tags. */ 1198 return (vlan_tag_count == 0 || 1199 vlan_tag_count > AQ_HW_VLAN_MAX_FILTERS); 1200 } 1201 1202 static void 1203 aq_update_vlan_filters(struct aq_dev *softc) 1204 { 1205 struct aq_rx_filter_vlan aq_vlans[AQ_HW_VLAN_MAX_FILTERS]; 1206 struct aq_hw *hw = &softc->hw; 1207 int bit_pos = 0; 1208 int vlan_tag = -1; 1209 int i; 1210 1211 if (hw_atl_b0_hw_vlan_promisc_set(hw, true) != 0) 1212 device_printf(softc->dev, 1213 "could not open the VLAN filter for update\n"); 1214 for (i = 0; i < AQ_HW_VLAN_MAX_FILTERS; i++) { 1215 bit_ffs_at(softc->vlan_tags, bit_pos, 4096, &vlan_tag); 1216 if (vlan_tag != -1) { 1217 aq_vlans[i].enable = true; 1218 aq_vlans[i].location = i; 1219 aq_vlans[i].queue = 0xFF; 1220 aq_vlans[i].vlan_id = vlan_tag; 1221 bit_pos = vlan_tag + 1; 1222 } else { 1223 aq_vlans[i].enable = false; 1224 } 1225 } 1226 1227 if (hw_atl_b0_hw_vlan_set(hw, aq_vlans) != 0) 1228 device_printf(softc->dev, 1229 "could not program the VLAN filter table\n"); 1230 if (hw_atl_b0_hw_vlan_promisc_set(hw, 1231 aq_is_vlan_promisc_required(softc) || 1232 (if_getflags(iflib_get_ifp(softc->ctx)) & IFF_PROMISC) != 0) != 0) 1233 device_printf(softc->dev, "VLAN filter update failed\n"); 1234 } 1235 1236 /* VLAN support */ 1237 static void 1238 aq_if_vlan_register(if_ctx_t ctx, uint16_t vtag) 1239 { 1240 struct aq_dev *softc = iflib_get_softc(ctx); 1241 1242 AQ_DBG_ENTERA("%d", vtag); 1243 1244 bit_set(softc->vlan_tags, vtag); 1245 1246 aq_update_vlan_filters(softc); 1247 1248 AQ_DBG_EXIT(0); 1249 } 1250 1251 static void 1252 aq_if_vlan_unregister(if_ctx_t ctx, uint16_t vtag) 1253 { 1254 struct aq_dev *softc = iflib_get_softc(ctx); 1255 1256 AQ_DBG_ENTERA("%d", vtag); 1257 1258 bit_clear(softc->vlan_tags, vtag); 1259 1260 aq_update_vlan_filters(softc); 1261 1262 AQ_DBG_EXIT(0); 1263 } 1264 1265 static void 1266 aq_if_led_func(if_ctx_t ctx, int onoff) 1267 { 1268 struct aq_dev *softc = iflib_get_softc(ctx); 1269 struct aq_hw *hw = &softc->hw; 1270 1271 AQ_DBG_ENTERA("%d", onoff); 1272 if (hw->fw_ops->led_control) 1273 hw->fw_ops->led_control(hw, onoff); 1274 1275 AQ_DBG_EXIT(0); 1276 } 1277 1278 static int 1279 aq_hw_capabilities(struct aq_dev *softc) 1280 { 1281 1282 if (pci_get_vendor(softc->dev) != AQUANTIA_VENDOR_ID) 1283 return (ENXIO); 1284 1285 switch (pci_get_device(softc->dev)) { 1286 case AQ_DEVICE_ID_D100: 1287 case AQ_DEVICE_ID_AQC100: 1288 case AQ_DEVICE_ID_AQC100S: 1289 softc->media_type = AQ_MEDIA_TYPE_FIBRE; 1290 softc->link_speeds = AQ_LINK_ALL_ATLANTIC1; 1291 break; 1292 1293 case AQ_DEVICE_ID_0001: 1294 case AQ_DEVICE_ID_D107: 1295 case AQ_DEVICE_ID_AQC107: 1296 case AQ_DEVICE_ID_AQC107S: 1297 softc->media_type = AQ_MEDIA_TYPE_TP; 1298 softc->link_speeds = AQ_LINK_ALL_ATLANTIC1; 1299 break; 1300 1301 case AQ_DEVICE_ID_D108: 1302 case AQ_DEVICE_ID_AQC108: 1303 case AQ_DEVICE_ID_AQC108S: 1304 case AQ_DEVICE_ID_AQC111: 1305 case AQ_DEVICE_ID_AQC111S: 1306 softc->media_type = AQ_MEDIA_TYPE_TP; 1307 softc->link_speeds = AQ_LINK_ALL_ATLANTIC1 & ~AQ_LINK_10G; 1308 break; 1309 1310 case AQ_DEVICE_ID_D109: 1311 case AQ_DEVICE_ID_AQC109: 1312 case AQ_DEVICE_ID_AQC109S: 1313 case AQ_DEVICE_ID_AQC112: 1314 case AQ_DEVICE_ID_AQC112S: 1315 softc->media_type = AQ_MEDIA_TYPE_TP; 1316 softc->link_speeds = AQ_LINK_ALL_ATLANTIC1 & 1317 ~(AQ_LINK_10G | AQ_LINK_5G); 1318 break; 1319 1320 case AQ_DEVICE_ID_AQC113: 1321 case AQ_DEVICE_ID_AQC113C: 1322 case AQ_DEVICE_ID_AQC113CA: 1323 case AQ_DEVICE_ID_AQC113CS: 1324 softc->media_type = AQ_MEDIA_TYPE_TP; 1325 softc->link_speeds = AQ_LINK_ALL; 1326 break; 1327 1328 case AQ_DEVICE_ID_AQC114CS: 1329 softc->media_type = AQ_MEDIA_TYPE_TP; 1330 softc->link_speeds = AQ_LINK_ALL & ~AQ_LINK_10G; 1331 break; 1332 1333 case AQ_DEVICE_ID_AQC115C: 1334 softc->media_type = AQ_MEDIA_TYPE_TP; 1335 softc->link_speeds = AQ_LINK_ALL & ~(AQ_LINK_10G | AQ_LINK_5G); 1336 break; 1337 1338 case AQ_DEVICE_ID_AQC116C: 1339 softc->media_type = AQ_MEDIA_TYPE_TP; 1340 softc->link_speeds = 1341 AQ_LINK_ALL & ~(AQ_LINK_10G | AQ_LINK_5G | AQ_LINK_2G5); 1342 break; 1343 1344 default: 1345 return (ENXIO); 1346 } 1347 1348 return (0); 1349 } 1350 1351 static int 1352 aq_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS) 1353 { 1354 struct aq_dev *softc = (struct aq_dev *)arg1; 1355 device_t dev = softc->dev; 1356 struct sbuf *buf; 1357 int error = 0; 1358 1359 buf = sbuf_new_for_sysctl(NULL, NULL, 256, req); 1360 if (!buf) { 1361 device_printf(dev, "Could not allocate sbuf for output.\n"); 1362 return (ENOMEM); 1363 } 1364 1365 /* Print out the redirection table */ 1366 sbuf_cat(buf, "\nRSS Indirection table:\n"); 1367 for (int i = 0; i < HW_ATL_RSS_INDIRECTION_TABLE_MAX; i++) { 1368 sbuf_printf(buf, "%d ", softc->rss_table[i]); 1369 if ((i+1) % 10 == 0) 1370 sbuf_printf(buf, "\n"); 1371 } 1372 1373 sbuf_cat(buf, "\nRSS Key:\n"); 1374 for (int i = 0; i < HW_ATL_RSS_HASHKEY_SIZE; i++) { 1375 sbuf_printf(buf, "0x%02x ", softc->rss_key[i]); 1376 } 1377 sbuf_printf(buf, "\n"); 1378 1379 error = sbuf_finish(buf); 1380 if (error) 1381 device_printf(dev, "Error finishing sbuf: %d\n", error); 1382 1383 sbuf_delete(buf); 1384 1385 return (0); 1386 } 1387 1388 enum aq_ring_ptr { 1389 AQ_RING_TX_HEAD, 1390 AQ_RING_TX_TAIL, 1391 AQ_RING_RX_HEAD, 1392 AQ_RING_RX_TAIL, 1393 }; 1394 1395 static int 1396 aq_sysctl_print_ring_ptr(SYSCTL_HANDLER_ARGS) 1397 { 1398 struct aq_ring *ring = arg1; 1399 unsigned int val; 1400 1401 if (ring == NULL) 1402 return (0); 1403 1404 switch (arg2) { 1405 case AQ_RING_TX_HEAD: 1406 val = tdm_tx_desc_head_ptr_get(&ring->dev->hw, ring->index); 1407 break; 1408 case AQ_RING_TX_TAIL: 1409 val = reg_tx_dma_desc_tail_ptr_get(&ring->dev->hw, ring->index); 1410 break; 1411 case AQ_RING_RX_HEAD: 1412 val = rdm_rx_desc_head_ptr_get(&ring->dev->hw, ring->index); 1413 break; 1414 default: /* AQ_RING_RX_TAIL */ 1415 val = reg_rx_dma_desc_tail_ptr_get(&ring->dev->hw, ring->index); 1416 break; 1417 } 1418 1419 return (sysctl_handle_int(oidp, &val, 0, req)); 1420 } 1421 1422 static int 1423 aq_sysctl_temperature(SYSCTL_HANDLER_ARGS) 1424 { 1425 struct aq_dev *softc = arg1; 1426 int error, temp_mc, val; 1427 1428 if (softc->hw.fw_ops == NULL || softc->hw.fw_ops->get_temp == NULL) 1429 return (ENOTSUP); 1430 1431 error = softc->hw.fw_ops->get_temp(&softc->hw, &temp_mc); 1432 if (error != 0) 1433 return (error); 1434 1435 /* millidegrees Celsius -> decikelvin */ 1436 val = temp_mc / 100 + 2732; 1437 1438 return (sysctl_handle_int(oidp, &val, 0, req)); 1439 } 1440 1441 /* arg2 selects the counter: 0 = link up, 1 = link down. */ 1442 static int 1443 aq_sysctl_fw_link_counter(SYSCTL_HANDLER_ARGS) 1444 { 1445 struct aq_dev *softc = arg1; 1446 uint32_t up, down; 1447 int error; 1448 1449 if (softc->hw.fw_ops == NULL || 1450 softc->hw.fw_ops->get_link_counters == NULL) 1451 return (ENOTSUP); 1452 1453 error = softc->hw.fw_ops->get_link_counters(&softc->hw, &up, &down); 1454 if (error != 0) 1455 return (error); 1456 1457 return (sysctl_handle_32(oidp, arg2 == 0 ? &up : &down, 0, req)); 1458 } 1459 1460 static void 1461 aq_add_stats_sysctls(struct aq_dev *softc) 1462 { 1463 device_t dev = softc->dev; 1464 struct sysctl_ctx_list *ctx = &softc->aq_sysctl_ctx; 1465 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 1466 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 1467 struct aq_stats *stats = &softc->curr_stats; 1468 struct sysctl_oid *stat_node, *queue_node, *thermal_node; 1469 struct sysctl_oid_list *stat_list, *queue_list; 1470 uint32_t link_up, link_down; 1471 int temp_mc; 1472 1473 #define QUEUE_NAME_LEN 32 1474 char namebuf[QUEUE_NAME_LEN]; 1475 1476 /* RSS configuration */ 1477 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "print_rss_config", 1478 CTLTYPE_STRING | CTLFLAG_RD, softc, 0, 1479 aq_sysctl_print_rss_config, "A", "Prints RSS Configuration"); 1480 1481 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug", 1482 CTLFLAG_RW, &softc->dbg_level, 0, 1483 "Trace verbosity: 0=off, 3=err, 4=+warn, 5=+trace, 6=+detail"); 1484 SYSCTL_ADD_U32(ctx, child, OID_AUTO, "debug_categories", 1485 CTLFLAG_RW, &softc->dbg_categories, 0, 1486 "Trace category mask: init=1 config=2 tx=4 rx=8 intr=16 fw=32"); 1487 1488 /* ENOTSUP means no sensor; other errors may just be a cold PHY. */ 1489 if (softc->hw.fw_ops != NULL && softc->hw.fw_ops->get_temp != NULL && 1490 softc->hw.fw_ops->get_temp(&softc->hw, &temp_mc) != ENOTSUP) { 1491 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "temperature", 1492 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 1493 aq_sysctl_temperature, "IK", "PHY temperature"); 1494 /* Without the sensor there is no shutdown to arm. */ 1495 if (softc->hw.fw_ops->thermal_arm != NULL) { 1496 thermal_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, 1497 "thermal_shutdown", CTLFLAG_RD | CTLFLAG_MPSAFE, 1498 NULL, "PHY thermal shutdown"); 1499 SYSCTL_ADD_BOOL(ctx, SYSCTL_CHILDREN(thermal_node), 1500 OID_AUTO, "enabled", CTLFLAG_RWTUN, 1501 &softc->thermal_shutdown_enabled, 0, 1502 "Arm the PHY's autonomous thermal shutdown"); 1503 } 1504 } 1505 1506 /* Only some firmware interface versions count link transitions. */ 1507 if (softc->hw.fw_ops != NULL && 1508 softc->hw.fw_ops->get_link_counters != NULL && 1509 softc->hw.fw_ops->get_link_counters(&softc->hw, &link_up, 1510 &link_down) != ENOTSUP) { 1511 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "fw_link_up", 1512 CTLTYPE_U32 | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 1513 aq_sysctl_fw_link_counter, "IU", 1514 "Link up transitions counted by the firmware"); 1515 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "fw_link_down", 1516 CTLTYPE_U32 | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 1517 aq_sysctl_fw_link_counter, "IU", 1518 "Link down transitions counted by the firmware"); 1519 } 1520 1521 /* Driver Statistics */ 1522 for (int i = 0; i < softc->tx_rings_count; i++) { 1523 struct aq_ring *ring = softc->tx_rings[i]; 1524 snprintf(namebuf, QUEUE_NAME_LEN, "tx_queue%d", i); 1525 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 1526 CTLFLAG_RD, NULL, "Queue Name"); 1527 queue_list = SYSCTL_CHILDREN(queue_node); 1528 1529 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "tx_pkts", 1530 CTLFLAG_RD, &(ring->stats.tx_pkts), "TX Packets"); 1531 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "tx_bytes", 1532 CTLFLAG_RD, &(ring->stats.tx_bytes), "TX Octets"); 1533 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "tx_head", 1534 CTLTYPE_UINT | CTLFLAG_RD, ring, AQ_RING_TX_HEAD, 1535 aq_sysctl_print_ring_ptr, "IU", "ring head pointer"); 1536 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "tx_tail", 1537 CTLTYPE_UINT | CTLFLAG_RD, ring, AQ_RING_TX_TAIL, 1538 aq_sysctl_print_ring_ptr, "IU", "ring tail pointer"); 1539 } 1540 1541 for (int i = 0; i < softc->rx_rings_count; i++) { 1542 struct aq_ring *ring = softc->rx_rings[i]; 1543 snprintf(namebuf, QUEUE_NAME_LEN, "rx_queue%d", i); 1544 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 1545 CTLFLAG_RD, NULL, "Queue Name"); 1546 queue_list = SYSCTL_CHILDREN(queue_node); 1547 1548 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "rx_pkts", 1549 CTLFLAG_RD, &(ring->stats.rx_pkts), "RX Packets"); 1550 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "rx_bytes", 1551 CTLFLAG_RD, &(ring->stats.rx_bytes), "RX Octets"); 1552 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "rx_err", 1553 CTLFLAG_RD, &(ring->stats.rx_err), "RX Errors"); 1554 SYSCTL_ADD_COUNTER_U64(ctx, queue_list, OID_AUTO, "irq", 1555 CTLFLAG_RD, &(ring->stats.irq), "RX interrupts"); 1556 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rx_head", 1557 CTLTYPE_UINT | CTLFLAG_RD, ring, AQ_RING_RX_HEAD, 1558 aq_sysctl_print_ring_ptr, "IU", "ring head pointer"); 1559 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rx_tail", 1560 CTLTYPE_UINT | CTLFLAG_RD, ring, AQ_RING_RX_TAIL, 1561 aq_sysctl_print_ring_ptr, "IU", "ring tail pointer"); 1562 } 1563 1564 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", 1565 CTLFLAG_RD, NULL, "Statistics (read from HW registers)"); 1566 stat_list = SYSCTL_CHILDREN(stat_node); 1567 1568 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd", 1569 CTLFLAG_RD, &stats->prc, "Good Packets Received"); 1570 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ucast_pkts_rcvd", 1571 CTLFLAG_RD, &stats->uprc, "Unicast Packets Received"); 1572 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd", 1573 CTLFLAG_RD, &stats->mprc, "Multicast Packets Received"); 1574 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd", 1575 CTLFLAG_RD, &stats->bprc, "Broadcast Packets Received"); 1576 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rsc_pkts_rcvd", 1577 CTLFLAG_RD, &stats->cprc, "Coalesced Packets Received"); 1578 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "err_pkts_rcvd", 1579 CTLFLAG_RD, &stats->erpr, "Errors of Packet Receive"); 1580 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "drop_pkts_dma", 1581 CTLFLAG_RD, &stats->dpc, "Dropped Packets in DMA"); 1582 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd", 1583 CTLFLAG_RD, &stats->brc, "Good Octets Received"); 1584 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ucast_octets_rcvd", 1585 CTLFLAG_RD, &stats->ubrc, "Unicast Octets Received"); 1586 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_octets_rcvd", 1587 CTLFLAG_RD, &stats->mbrc, "Multicast Octets Received"); 1588 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_octets_rcvd", 1589 CTLFLAG_RD, &stats->bbrc, "Broadcast Octets Received"); 1590 1591 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 1592 CTLFLAG_RD, &stats->ptc, "Good Packets Transmitted"); 1593 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ucast_pkts_txd", 1594 CTLFLAG_RD, &stats->uptc, "Unicast Packets Transmitted"); 1595 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 1596 CTLFLAG_RD, &stats->mptc, "Multicast Packets Transmitted"); 1597 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 1598 CTLFLAG_RD, &stats->bptc, "Broadcast Packets Transmitted"); 1599 1600 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "err_pkts_txd", 1601 CTLFLAG_RD, &stats->erpt, "Errors of Packet Transmit"); 1602 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 1603 CTLFLAG_RD, &stats->btc, "Good Octets Transmitted"); 1604 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ucast_octets_txd", 1605 CTLFLAG_RD, &stats->ubtc, "Unicast Octets Transmitted"); 1606 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_octets_txd", 1607 CTLFLAG_RD, &stats->mbtc, "Multicast Octets Transmitted"); 1608 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_octets_txd", 1609 CTLFLAG_RD, &stats->bbtc, "Broadcast Octets Transmitted"); 1610 } 1611