1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2021-2024 Rubicon Communications, LLC (Netgate) 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "if_igc.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #include <net/rss_config.h> 36 #include <netinet/in_rss.h> 37 38 39 /********************************************************************* 40 * PCI Device ID Table 41 * 42 * Used by probe to select devices to load on 43 * Last entry must be all 0s 44 * 45 * { Vendor ID, Device ID, String } 46 *********************************************************************/ 47 48 static const pci_vendor_info_t igc_vendor_info_array[] = 49 { 50 /* Intel(R) PRO/1000 Network Connection - igc */ 51 PVID(0x8086, IGC_DEV_ID_I225_LM, 52 "Intel(R) Ethernet Controller I225-LM"), 53 PVID(0x8086, IGC_DEV_ID_I225_V, 54 "Intel(R) Ethernet Controller I225-V"), 55 PVID(0x8086, IGC_DEV_ID_I225_K, 56 "Intel(R) Ethernet Controller I225-K"), 57 PVID(0x8086, IGC_DEV_ID_I225_I, 58 "Intel(R) Ethernet Controller I225-IT"), 59 PVID(0x8086, IGC_DEV_ID_I220_V, 60 "Intel(R) Ethernet Controller I220-V"), 61 PVID(0x8086, IGC_DEV_ID_I225_K2, 62 "Intel(R) Ethernet Controller I225-K(2)"), 63 PVID(0x8086, IGC_DEV_ID_I225_LMVP, 64 "Intel(R) Ethernet Controller I225-LMvP(2)"), 65 PVID(0x8086, IGC_DEV_ID_I226_K, 66 "Intel(R) Ethernet Controller I226-K"), 67 PVID(0x8086, IGC_DEV_ID_I226_LMVP, 68 "Intel(R) Ethernet Controller I226-LMvP"), 69 PVID(0x8086, IGC_DEV_ID_I225_IT, 70 "Intel(R) Ethernet Controller I225-IT(2)"), 71 PVID(0x8086, IGC_DEV_ID_I226_LM, 72 "Intel(R) Ethernet Controller I226-LM"), 73 PVID(0x8086, IGC_DEV_ID_I226_V, 74 "Intel(R) Ethernet Controller I226-V"), 75 PVID(0x8086, IGC_DEV_ID_I226_IT, 76 "Intel(R) Ethernet Controller I226-IT"), 77 PVID(0x8086, IGC_DEV_ID_I221_V, 78 "Intel(R) Ethernet Controller I221-V"), 79 PVID(0x8086, IGC_DEV_ID_I226_BLANK_NVM, 80 "Intel(R) Ethernet Controller I226(blankNVM)"), 81 PVID(0x8086, IGC_DEV_ID_I225_BLANK_NVM, 82 "Intel(R) Ethernet Controller I225(blankNVM)"), 83 /* required last entry */ 84 PVID_END 85 }; 86 87 /********************************************************************* 88 * Function prototypes 89 *********************************************************************/ 90 static void *igc_register(device_t); 91 static int igc_if_attach_pre(if_ctx_t); 92 static int igc_if_attach_post(if_ctx_t); 93 static int igc_if_detach(if_ctx_t); 94 static int igc_if_shutdown(if_ctx_t); 95 static int igc_if_suspend(if_ctx_t); 96 static int igc_if_resume(if_ctx_t); 97 98 static int igc_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 99 int); 100 static int igc_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 101 int); 102 static void igc_if_queues_free(if_ctx_t); 103 104 static uint64_t igc_if_get_counter(if_ctx_t, ift_counter); 105 static void igc_if_init(if_ctx_t); 106 static void igc_if_stop(if_ctx_t); 107 static void igc_if_media_status(if_ctx_t, struct ifmediareq *); 108 static int igc_if_media_change(if_ctx_t); 109 static int igc_if_mtu_set(if_ctx_t, uint32_t); 110 static void igc_if_timer(if_ctx_t, uint16_t); 111 static void igc_if_watchdog_reset(if_ctx_t); 112 static bool igc_if_needs_restart(if_ctx_t, enum iflib_restart_event); 113 114 static void igc_identify_hardware(if_ctx_t); 115 static int igc_allocate_pci_resources(if_ctx_t); 116 static void igc_free_pci_resources(if_ctx_t); 117 static void igc_reset(if_ctx_t); 118 static int igc_setup_interface(if_ctx_t); 119 static int igc_setup_msix(if_ctx_t); 120 121 static void igc_initialize_transmit_unit(if_ctx_t); 122 static void igc_initialize_receive_unit(if_ctx_t); 123 124 static void igc_if_intr_enable(if_ctx_t); 125 static void igc_if_intr_disable(if_ctx_t); 126 static int igc_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 127 static int igc_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 128 static void igc_if_multi_set(if_ctx_t); 129 static void igc_if_update_admin_status(if_ctx_t); 130 static void igc_if_debug(if_ctx_t); 131 static void igc_update_stats_counters(struct igc_softc *); 132 static void igc_add_hw_stats(struct igc_softc *); 133 static int igc_if_set_promisc(if_ctx_t, int); 134 static void igc_setup_vlan_hw_support(if_ctx_t); 135 static void igc_fw_version(struct igc_softc *); 136 static void igc_sbuf_fw_version(struct igc_fw_version *, struct sbuf *); 137 static void igc_print_fw_version(struct igc_softc *); 138 static int igc_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 139 static int igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 140 static void igc_print_nvm_info(struct igc_softc *); 141 static int igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 142 static int igc_get_rs(SYSCTL_HANDLER_ARGS); 143 static void igc_print_debug_info(struct igc_softc *); 144 static int igc_is_valid_ether_addr(u8 *); 145 static void igc_neweitr(struct igc_softc *, struct igc_rx_queue *, 146 struct rx_ring *); 147 static int igc_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 148 /* Management and WOL Support */ 149 static void igc_get_hw_control(struct igc_softc *); 150 static void igc_release_hw_control(struct igc_softc *); 151 static void igc_get_wakeup(if_ctx_t); 152 static void igc_enable_wakeup(if_ctx_t); 153 154 int igc_intr(void *); 155 156 /* MSI-X handlers */ 157 static int igc_if_msix_intr_assign(if_ctx_t, int); 158 static int igc_msix_link(void *); 159 static void igc_handle_link(void *context); 160 161 static int igc_set_flowcntl(SYSCTL_HANDLER_ARGS); 162 static int igc_sysctl_dmac(SYSCTL_HANDLER_ARGS); 163 static int igc_sysctl_eee(SYSCTL_HANDLER_ARGS); 164 165 static int igc_get_regs(SYSCTL_HANDLER_ARGS); 166 167 static void igc_configure_queues(struct igc_softc *); 168 static void igc_initialize_interrupt_rate(struct igc_softc *); 169 170 171 /********************************************************************* 172 * FreeBSD Device Interface Entry Points 173 *********************************************************************/ 174 static device_method_t igc_methods[] = { 175 /* Device interface */ 176 DEVMETHOD(device_register, igc_register), 177 DEVMETHOD(device_probe, iflib_device_probe), 178 DEVMETHOD(device_attach, iflib_device_attach), 179 DEVMETHOD(device_detach, iflib_device_detach), 180 DEVMETHOD(device_shutdown, iflib_device_shutdown), 181 DEVMETHOD(device_suspend, iflib_device_suspend), 182 DEVMETHOD(device_resume, iflib_device_resume), 183 DEVMETHOD_END 184 }; 185 186 static driver_t igc_driver = { 187 "igc", igc_methods, sizeof(struct igc_softc), 188 }; 189 190 DRIVER_MODULE(igc, pci, igc_driver, 0, 0); 191 192 MODULE_DEPEND(igc, pci, 1, 1, 1); 193 MODULE_DEPEND(igc, ether, 1, 1, 1); 194 MODULE_DEPEND(igc, iflib, 1, 1, 1); 195 196 IFLIB_PNP_INFO(pci, igc, igc_vendor_info_array); 197 198 static device_method_t igc_if_methods[] = { 199 DEVMETHOD(ifdi_attach_pre, igc_if_attach_pre), 200 DEVMETHOD(ifdi_attach_post, igc_if_attach_post), 201 DEVMETHOD(ifdi_detach, igc_if_detach), 202 DEVMETHOD(ifdi_shutdown, igc_if_shutdown), 203 DEVMETHOD(ifdi_suspend, igc_if_suspend), 204 DEVMETHOD(ifdi_resume, igc_if_resume), 205 DEVMETHOD(ifdi_init, igc_if_init), 206 DEVMETHOD(ifdi_stop, igc_if_stop), 207 DEVMETHOD(ifdi_msix_intr_assign, igc_if_msix_intr_assign), 208 DEVMETHOD(ifdi_intr_enable, igc_if_intr_enable), 209 DEVMETHOD(ifdi_intr_disable, igc_if_intr_disable), 210 DEVMETHOD(ifdi_tx_queues_alloc, igc_if_tx_queues_alloc), 211 DEVMETHOD(ifdi_rx_queues_alloc, igc_if_rx_queues_alloc), 212 DEVMETHOD(ifdi_queues_free, igc_if_queues_free), 213 DEVMETHOD(ifdi_update_admin_status, igc_if_update_admin_status), 214 DEVMETHOD(ifdi_multi_set, igc_if_multi_set), 215 DEVMETHOD(ifdi_media_status, igc_if_media_status), 216 DEVMETHOD(ifdi_media_change, igc_if_media_change), 217 DEVMETHOD(ifdi_mtu_set, igc_if_mtu_set), 218 DEVMETHOD(ifdi_promisc_set, igc_if_set_promisc), 219 DEVMETHOD(ifdi_timer, igc_if_timer), 220 DEVMETHOD(ifdi_watchdog_reset, igc_if_watchdog_reset), 221 DEVMETHOD(ifdi_get_counter, igc_if_get_counter), 222 DEVMETHOD(ifdi_rx_queue_intr_enable, igc_if_rx_queue_intr_enable), 223 DEVMETHOD(ifdi_tx_queue_intr_enable, igc_if_tx_queue_intr_enable), 224 DEVMETHOD(ifdi_debug, igc_if_debug), 225 DEVMETHOD(ifdi_needs_restart, igc_if_needs_restart), 226 DEVMETHOD_END 227 }; 228 229 static driver_t igc_if_driver = { 230 "igc_if", igc_if_methods, sizeof(struct igc_softc) 231 }; 232 233 /********************************************************************* 234 * Tunable default values. 235 *********************************************************************/ 236 237 /* Allow common code without TSO */ 238 #ifndef CSUM_TSO 239 #define CSUM_TSO 0 240 #endif 241 242 static SYSCTL_NODE(_hw, OID_AUTO, igc, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 243 "igc driver parameters"); 244 245 static int igc_disable_crc_stripping = 0; 246 SYSCTL_INT(_hw_igc, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 247 &igc_disable_crc_stripping, 0, "Disable CRC Stripping"); 248 249 static int igc_smart_pwr_down = false; 250 SYSCTL_INT(_hw_igc, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 251 &igc_smart_pwr_down, 252 0, "Set to true to leave smart power down enabled on newer adapters"); 253 254 /* Controls whether promiscuous also shows bad packets */ 255 static int igc_debug_sbp = false; 256 SYSCTL_INT(_hw_igc, OID_AUTO, sbp, CTLFLAG_RDTUN, &igc_debug_sbp, 0, 257 "Show bad packets in promiscuous mode"); 258 259 /* Energy efficient ethernet - default to OFF */ 260 static int igc_eee_setting = 1; 261 SYSCTL_INT(_hw_igc, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &igc_eee_setting, 0, 262 "Enable Energy Efficient Ethernet"); 263 264 /* 265 * AIM: Adaptive Interrupt Moderation 266 * which means that the interrupt rate is varied over time based on the 267 * traffic for that interrupt vector 268 */ 269 static int igc_enable_aim = 1; 270 SYSCTL_INT(_hw_igc, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igc_enable_aim, 271 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 272 273 /* 274 ** Tuneable Interrupt rate 275 */ 276 static int igc_max_interrupt_rate = IGC_INTS_DEFAULT; 277 SYSCTL_INT(_hw_igc, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 278 &igc_max_interrupt_rate, 0, "Maximum interrupts per second"); 279 280 extern struct if_txrx igc_txrx; 281 282 static struct if_shared_ctx igc_sctx_init = { 283 .isc_magic = IFLIB_MAGIC, 284 .isc_q_align = PAGE_SIZE, 285 .isc_tx_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header), 286 .isc_tx_maxsegsize = PAGE_SIZE, 287 .isc_tso_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header), 288 .isc_tso_maxsegsize = IGC_TSO_SEG_SIZE, 289 .isc_rx_maxsize = MAX_JUMBO_FRAME_SIZE, 290 .isc_rx_nsegments = 1, 291 .isc_rx_maxsegsize = MJUM9BYTES, 292 .isc_nfl = 1, 293 .isc_nrxqs = 1, 294 .isc_ntxqs = 1, 295 .isc_admin_intrcnt = 1, 296 .isc_vendor_info = igc_vendor_info_array, 297 .isc_driver_version = "1", 298 .isc_driver = &igc_if_driver, 299 .isc_flags = 300 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 301 302 .isc_nrxd_min = {IGC_MIN_RXD}, 303 .isc_ntxd_min = {IGC_MIN_TXD}, 304 .isc_nrxd_max = {IGC_MAX_RXD}, 305 .isc_ntxd_max = {IGC_MAX_TXD}, 306 .isc_nrxd_default = {IGC_DEFAULT_RXD}, 307 .isc_ntxd_default = {IGC_DEFAULT_TXD}, 308 }; 309 310 /***************************************************************** 311 * 312 * Dump Registers 313 * 314 ****************************************************************/ 315 #define IGC_REGS_LEN 739 316 317 static int igc_get_regs(SYSCTL_HANDLER_ARGS) 318 { 319 struct igc_softc *sc = (struct igc_softc *)arg1; 320 struct igc_hw *hw = &sc->hw; 321 struct sbuf *sb; 322 u32 *regs_buff; 323 int rc; 324 325 regs_buff = malloc(sizeof(u32) * IGC_REGS_LEN, M_DEVBUF, M_WAITOK); 326 memset(regs_buff, 0, IGC_REGS_LEN * sizeof(u32)); 327 328 rc = sysctl_wire_old_buffer(req, 0); 329 MPASS(rc == 0); 330 if (rc != 0) { 331 free(regs_buff, M_DEVBUF); 332 return (rc); 333 } 334 335 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 336 MPASS(sb != NULL); 337 if (sb == NULL) { 338 free(regs_buff, M_DEVBUF); 339 return (ENOMEM); 340 } 341 342 /* General Registers */ 343 regs_buff[0] = IGC_READ_REG(hw, IGC_CTRL); 344 regs_buff[1] = IGC_READ_REG(hw, IGC_STATUS); 345 regs_buff[2] = IGC_READ_REG(hw, IGC_CTRL_EXT); 346 regs_buff[3] = IGC_READ_REG(hw, IGC_ICR); 347 regs_buff[4] = IGC_READ_REG(hw, IGC_RCTL); 348 regs_buff[5] = IGC_READ_REG(hw, IGC_RDLEN(0)); 349 regs_buff[6] = IGC_READ_REG(hw, IGC_RDH(0)); 350 regs_buff[7] = IGC_READ_REG(hw, IGC_RDT(0)); 351 regs_buff[8] = IGC_READ_REG(hw, IGC_RXDCTL(0)); 352 regs_buff[9] = IGC_READ_REG(hw, IGC_RDBAL(0)); 353 regs_buff[10] = IGC_READ_REG(hw, IGC_RDBAH(0)); 354 regs_buff[11] = IGC_READ_REG(hw, IGC_TCTL); 355 regs_buff[12] = IGC_READ_REG(hw, IGC_TDBAL(0)); 356 regs_buff[13] = IGC_READ_REG(hw, IGC_TDBAH(0)); 357 regs_buff[14] = IGC_READ_REG(hw, IGC_TDLEN(0)); 358 regs_buff[15] = IGC_READ_REG(hw, IGC_TDH(0)); 359 regs_buff[16] = IGC_READ_REG(hw, IGC_TDT(0)); 360 regs_buff[17] = IGC_READ_REG(hw, IGC_TXDCTL(0)); 361 362 sbuf_printf(sb, "General Registers\n"); 363 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 364 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 365 sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]); 366 367 sbuf_printf(sb, "Interrupt Registers\n"); 368 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 369 370 sbuf_printf(sb, "RX Registers\n"); 371 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 372 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 373 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 374 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 375 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 376 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 377 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 378 379 sbuf_printf(sb, "TX Registers\n"); 380 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 381 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 382 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 383 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 384 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 385 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 386 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 387 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 388 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 389 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 390 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 391 392 free(regs_buff, M_DEVBUF); 393 394 #ifdef DUMP_DESCS 395 { 396 if_softc_ctx_t scctx = sc->shared; 397 struct rx_ring *rxr = &rx_que->rxr; 398 struct tx_ring *txr = &tx_que->txr; 399 int ntxd = scctx->isc_ntxd[0]; 400 int nrxd = scctx->isc_nrxd[0]; 401 int j; 402 403 for (j = 0; j < nrxd; j++) { 404 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 405 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 406 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 407 PRIx64 " Error:%d Length:%d\n", 408 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 409 } 410 411 for (j = 0; j < min(ntxd, 256); j++) { 412 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 413 414 sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x" 415 "[3]: %08x eop: %d DD=%d\n", 416 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 417 buf->eop != -1 ? 418 txr->tx_base[buf->eop].upper.fields.status & 419 IGC_TXD_STAT_DD : 0); 420 421 } 422 } 423 #endif 424 425 rc = sbuf_finish(sb); 426 sbuf_delete(sb); 427 return(rc); 428 } 429 430 static void * 431 igc_register(device_t dev) 432 { 433 return (&igc_sctx_init); 434 } 435 436 static int 437 igc_set_num_queues(if_ctx_t ctx) 438 { 439 int maxqueues; 440 441 maxqueues = 4; 442 443 return (maxqueues); 444 } 445 446 #define IGC_CAPS \ 447 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 448 IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_TSO4 | IFCAP_LRO | \ 449 IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 | IFCAP_TSO6 450 451 /********************************************************************* 452 * Device initialization routine 453 * 454 * The attach entry point is called when the driver is being loaded. 455 * This routine identifies the type of hardware, allocates all resources 456 * and initializes the hardware. 457 * 458 * return 0 on success, positive on failure 459 *********************************************************************/ 460 static int 461 igc_if_attach_pre(if_ctx_t ctx) 462 { 463 struct igc_softc *sc; 464 if_softc_ctx_t scctx; 465 device_t dev; 466 struct igc_hw *hw; 467 int error = 0; 468 469 INIT_DEBUGOUT("igc_if_attach_pre: begin"); 470 dev = iflib_get_dev(ctx); 471 sc = iflib_get_softc(ctx); 472 473 if (igc_max_interrupt_rate <= 0) { 474 device_printf(dev, 475 "Invalid max_interrupt_rate %d; using default %d\n", 476 igc_max_interrupt_rate, IGC_INTS_DEFAULT); 477 igc_max_interrupt_rate = IGC_INTS_DEFAULT; 478 } 479 480 sc->ctx = sc->osdep.ctx = ctx; 481 sc->dev = sc->osdep.dev = dev; 482 scctx = sc->shared = iflib_get_softc_ctx(ctx); 483 sc->media = iflib_get_media(ctx); 484 hw = &sc->hw; 485 486 /* SYSCTL stuff */ 487 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 488 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 489 OID_AUTO, "nvm", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 490 sc, 0, igc_sysctl_nvm_info, "I", "NVM Information"); 491 492 sc->enable_aim = igc_enable_aim; 493 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 494 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 495 OID_AUTO, "enable_aim", CTLFLAG_RW, 496 &sc->enable_aim, 0, 497 "Interrupt Moderation (1=normal, 2=lowlatency)"); 498 499 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 500 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 501 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, 502 sc, 0, igc_sysctl_print_fw_version, "A", 503 "Prints FW/NVM Versions"); 504 505 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 506 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 507 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 508 sc, 0, igc_sysctl_debug_info, "I", "Debug Information"); 509 510 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 511 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 512 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 513 sc, 0, igc_set_flowcntl, "I", "Flow Control"); 514 515 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 516 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 517 OID_AUTO, "reg_dump", 518 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 519 igc_get_regs, "A", "Dump Registers"); 520 521 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 522 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 523 OID_AUTO, "rs_dump", 524 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 525 igc_get_rs, "I", "Dump RS indexes"); 526 527 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 528 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 529 OID_AUTO, "dmac", 530 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 531 igc_sysctl_dmac, "I", "DMA Coalesce"); 532 533 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 534 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 535 OID_AUTO, "tso_tcp_flags_mask_first_segment", 536 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 537 sc, 0, igc_sysctl_tso_tcp_flags_mask, "IU", 538 "TSO TCP flags mask for first segment"); 539 540 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 541 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 542 OID_AUTO, "tso_tcp_flags_mask_middle_segment", 543 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 544 sc, 1, igc_sysctl_tso_tcp_flags_mask, "IU", 545 "TSO TCP flags mask for middle segment"); 546 547 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 548 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 549 OID_AUTO, "tso_tcp_flags_mask_last_segment", 550 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 551 sc, 2, igc_sysctl_tso_tcp_flags_mask, "IU", 552 "TSO TCP flags mask for last segment"); 553 554 /* Determine hardware and mac info */ 555 igc_identify_hardware(ctx); 556 557 scctx->isc_tx_nsegments = IGC_MAX_SCATTER; 558 scctx->isc_nrxqsets_max = 559 scctx->isc_ntxqsets_max = igc_set_num_queues(ctx); 560 if (bootverbose) 561 device_printf(dev, "attach_pre capping queues at %d\n", 562 scctx->isc_ntxqsets_max); 563 564 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 565 sizeof(union igc_adv_tx_desc), IGC_DBA_ALIGN); 566 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 567 sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 568 scctx->isc_txd_size[0] = sizeof(union igc_adv_tx_desc); 569 scctx->isc_rxd_size[0] = sizeof(union igc_adv_rx_desc); 570 scctx->isc_txrx = &igc_txrx; 571 scctx->isc_tx_tso_segments_max = IGC_MAX_SCATTER; 572 scctx->isc_tx_tso_size_max = IGC_TSO_SIZE; 573 scctx->isc_tx_tso_segsize_max = IGC_TSO_SEG_SIZE; 574 scctx->isc_capabilities = scctx->isc_capenable = IGC_CAPS; 575 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 576 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_SCTP | CSUM_IP6_SCTP; 577 578 /* 579 ** Some new devices, as with ixgbe, now may 580 ** use a different BAR, so we need to keep 581 ** track of which is used. 582 */ 583 scctx->isc_msix_bar = PCIR_BAR(IGC_MSIX_BAR); 584 if (pci_read_config(dev, scctx->isc_msix_bar, 4) == 0) 585 scctx->isc_msix_bar += 4; 586 587 /* Setup PCI resources */ 588 if (igc_allocate_pci_resources(ctx)) { 589 device_printf(dev, "Allocation of PCI resources failed\n"); 590 error = ENXIO; 591 goto err_pci; 592 } 593 594 /* Do Shared Code initialization */ 595 error = igc_setup_init_funcs(hw, true); 596 if (error) { 597 device_printf(dev, "Setup of Shared code failed, error %d\n", 598 error); 599 error = ENXIO; 600 goto err_pci; 601 } 602 603 igc_setup_msix(ctx); 604 igc_get_bus_info(hw); 605 606 hw->mac.autoneg = DO_AUTO_NEG; 607 hw->phy.autoneg_wait_to_complete = false; 608 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 609 610 /* Copper options */ 611 if (hw->phy.media_type == igc_media_type_copper) { 612 hw->phy.mdix = AUTO_ALL_MODES; 613 } 614 615 /* 616 * Set the frame limits assuming 617 * standard ethernet sized frames. 618 */ 619 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 620 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 621 622 /* Allocate multicast array memory. */ 623 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 624 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 625 if (sc->mta == NULL) { 626 device_printf(dev, 627 "Can not allocate multicast setup array\n"); 628 error = ENOMEM; 629 goto err_late; 630 } 631 632 /* Check SOL/IDER usage */ 633 if (igc_check_reset_block(hw)) 634 device_printf(dev, "PHY reset is blocked" 635 " due to SOL/IDER session.\n"); 636 637 /* Sysctl for setting Energy Efficient Ethernet */ 638 sc->hw.dev_spec._i225.eee_disable = igc_eee_setting; 639 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 640 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 641 OID_AUTO, "eee_control", 642 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 643 sc, 0, igc_sysctl_eee, "I", 644 "Disable Energy Efficient Ethernet"); 645 646 /* 647 ** Start from a known state, this is 648 ** important in reading the nvm and 649 ** mac from that. 650 */ 651 igc_reset_hw(hw); 652 653 /* Make sure we have a good EEPROM before we read from it */ 654 if (igc_validate_nvm_checksum(hw) < 0) { 655 /* 656 ** Some PCI-E parts fail the first check due to 657 ** the link being in sleep state, call it again, 658 ** if it fails a second time its a real issue. 659 */ 660 if (igc_validate_nvm_checksum(hw) < 0) { 661 device_printf(dev, 662 "The EEPROM Checksum Is Not Valid\n"); 663 error = EIO; 664 goto err_late; 665 } 666 } 667 668 /* Copy the permanent MAC address out of the EEPROM */ 669 if (igc_read_mac_addr(hw) < 0) { 670 device_printf(dev, "EEPROM read error while reading MAC" 671 " address\n"); 672 error = EIO; 673 goto err_late; 674 } 675 676 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 677 device_printf(dev, "Invalid MAC address\n"); 678 error = EIO; 679 goto err_late; 680 } 681 682 /* Save the EEPROM/NVM versions */ 683 igc_fw_version(sc); 684 685 igc_print_fw_version(sc); 686 687 /* 688 * Get Wake-on-Lan and Management info for later use 689 */ 690 igc_get_wakeup(ctx); 691 692 /* Enable only WOL MAGIC by default */ 693 scctx->isc_capenable &= ~IFCAP_WOL; 694 if (sc->wol != 0) 695 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 696 697 iflib_set_mac(ctx, hw->mac.addr); 698 699 return (0); 700 701 err_late: 702 igc_release_hw_control(sc); 703 err_pci: 704 igc_free_pci_resources(ctx); 705 free(sc->mta, M_DEVBUF); 706 707 return (error); 708 } 709 710 static int 711 igc_if_attach_post(if_ctx_t ctx) 712 { 713 struct igc_softc *sc = iflib_get_softc(ctx); 714 struct igc_hw *hw = &sc->hw; 715 int error = 0; 716 717 /* Setup OS specific network interface */ 718 error = igc_setup_interface(ctx); 719 if (error != 0) { 720 goto err_late; 721 } 722 723 igc_reset(ctx); 724 725 /* Initialize statistics */ 726 igc_update_stats_counters(sc); 727 hw->mac.get_link_status = true; 728 igc_if_update_admin_status(ctx); 729 igc_add_hw_stats(sc); 730 731 /* the driver can now take control from firmware */ 732 igc_get_hw_control(sc); 733 734 INIT_DEBUGOUT("igc_if_attach_post: end"); 735 736 return (error); 737 738 err_late: 739 igc_release_hw_control(sc); 740 igc_free_pci_resources(ctx); 741 igc_if_queues_free(ctx); 742 free(sc->mta, M_DEVBUF); 743 744 return (error); 745 } 746 747 /********************************************************************* 748 * Device removal routine 749 * 750 * The detach entry point is called when the driver is being removed. 751 * This routine stops the adapter and deallocates all the resources 752 * that were allocated for driver operation. 753 * 754 * return 0 on success, positive on failure 755 *********************************************************************/ 756 static int 757 igc_if_detach(if_ctx_t ctx) 758 { 759 struct igc_softc *sc = iflib_get_softc(ctx); 760 761 INIT_DEBUGOUT("igc_if_detach: begin"); 762 763 igc_phy_hw_reset(&sc->hw); 764 765 igc_release_hw_control(sc); 766 igc_free_pci_resources(ctx); 767 768 return (0); 769 } 770 771 /********************************************************************* 772 * 773 * Shutdown entry point 774 * 775 **********************************************************************/ 776 777 static int 778 igc_if_shutdown(if_ctx_t ctx) 779 { 780 return igc_if_suspend(ctx); 781 } 782 783 /* 784 * Suspend/resume device methods. 785 */ 786 static int 787 igc_if_suspend(if_ctx_t ctx) 788 { 789 struct igc_softc *sc = iflib_get_softc(ctx); 790 791 igc_release_hw_control(sc); 792 igc_enable_wakeup(ctx); 793 return (0); 794 } 795 796 static int 797 igc_if_resume(if_ctx_t ctx) 798 { 799 igc_if_init(ctx); 800 801 return(0); 802 } 803 804 static int 805 igc_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 806 { 807 int max_frame_size; 808 struct igc_softc *sc = iflib_get_softc(ctx); 809 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 810 811 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 812 813 /* 9K Jumbo Frame size */ 814 max_frame_size = 9234; 815 816 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 817 return (EINVAL); 818 } 819 820 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 821 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 822 return (0); 823 } 824 825 /********************************************************************* 826 * Init entry point 827 * 828 * This routine is used in two ways. It is used by the stack as 829 * init entry point in network interface structure. It is also used 830 * by the driver as a hw/sw initialization routine to get to a 831 * consistent state. 832 * 833 **********************************************************************/ 834 static void 835 igc_if_init(if_ctx_t ctx) 836 { 837 struct igc_softc *sc = iflib_get_softc(ctx); 838 if_softc_ctx_t scctx = sc->shared; 839 if_t ifp = iflib_get_ifp(ctx); 840 struct igc_tx_queue *tx_que; 841 int i; 842 843 INIT_DEBUGOUT("igc_if_init: begin"); 844 845 /* Get the latest mac address, User can use a LAA */ 846 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, 847 ETHER_ADDR_LEN); 848 849 /* Put the address into the Receive Address Array */ 850 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 851 852 /* Initialize the hardware */ 853 igc_reset(ctx); 854 igc_if_update_admin_status(ctx); 855 856 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 857 i++, tx_que++) { 858 struct tx_ring *txr = &tx_que->txr; 859 860 txr->tx_rs_cidx = txr->tx_rs_pidx; 861 862 /* Initialize the last processed descriptor to be the end of 863 * the ring, rather than the start, so that we avoid an 864 * off-by-one error when calculating how many descriptors are 865 * done in the credits_update function. 866 */ 867 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 868 } 869 870 /* Setup VLAN support, basic and offload if available */ 871 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 872 873 /* Prepare transmit descriptors and buffers */ 874 igc_initialize_transmit_unit(ctx); 875 876 /* Setup Multicast table */ 877 igc_if_multi_set(ctx); 878 879 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 880 igc_initialize_receive_unit(ctx); 881 882 /* Set up VLAN support */ 883 igc_setup_vlan_hw_support(ctx); 884 885 /* Don't lose promiscuous settings */ 886 igc_if_set_promisc(ctx, if_getflags(ifp)); 887 igc_clear_hw_cntrs_base_generic(&sc->hw); 888 889 if (sc->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */ 890 igc_configure_queues(sc); 891 igc_initialize_interrupt_rate(sc); 892 893 /* this clears any pending interrupts */ 894 IGC_READ_REG(&sc->hw, IGC_ICR); 895 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 896 897 /* the driver can now take control from firmware */ 898 igc_get_hw_control(sc); 899 900 /* Set Energy Efficient Ethernet */ 901 igc_set_eee_i225(&sc->hw, true, true, true); 902 } 903 904 /* 905 * RX publishes its byte and packet counters as one snapshot when iflib 906 * returns descriptors to hardware. This also covers watchdog-driven RX 907 * processing, which can run while the interrupt vector is unmasked. 908 */ 909 static __inline void 910 igc_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 911 { 912 uint64_t snapshot; 913 u32 now_bytes, now_packets; 914 915 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 916 now_bytes = snapshot >> 32; 917 now_packets = (u32)snapshot; 918 *bytes = now_bytes - rxr->rx_bytes_last; 919 *packets = now_packets - rxr->rx_packets_last; 920 rxr->rx_bytes_last = now_bytes; 921 rxr->rx_packets_last = now_packets; 922 } 923 924 /* 925 * TX publishes its byte and packet counters as one snapshot at the doorbell, 926 * because encapsulation can overlap the interrupt filter. The two halves 927 * remain independent free running u32 counters, so their deltas are correct 928 * across wrap. 929 */ 930 static __inline void 931 igc_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 932 { 933 uint64_t snapshot; 934 u32 now_bytes, now_packets; 935 936 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 937 now_bytes = snapshot >> 32; 938 now_packets = (u32)snapshot; 939 *bytes = now_bytes - txr->tx_bytes_last; 940 *packets = now_packets - txr->tx_packets_last; 941 txr->tx_bytes_last = now_bytes; 942 txr->tx_packets_last = now_packets; 943 } 944 945 /********************************************************************* 946 * 947 * Do Adaptive Interrupt Moderation: 948 * - Calculate based on average size over the last interval 949 * 950 * Returns interrupts per second rather than a register value, so that the 951 * caller's IGC_INTS_TO_EITR() conversion applies, or zero if the interval 952 * carried no packet to measure. 953 * 954 *********************************************************************/ 955 static u32 956 igc_ring_itr(struct igc_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 957 u32 txpackets) 958 { 959 u32 newitr = 0; 960 961 if (txbytes && txpackets) 962 newitr = txbytes / txpackets; 963 if (rxbytes && rxpackets) 964 newitr = max(newitr, rxbytes / rxpackets); 965 966 /* 967 * No packet was observed, so there is no size to work from. Report no 968 * observation and let the caller keep the rate it already has. 969 */ 970 if (newitr == 0) 971 return (0); 972 973 newitr += 24; /* account for hardware frame, crc */ 974 /* set an upper boundary */ 975 newitr = min(newitr, 3000); 976 /* Be nice to the mid range */ 977 if ((newitr > 300) && (newitr < 1200)) 978 newitr = (newitr / 3); 979 else 980 newitr = (newitr / 2); 981 982 /* The value above was written straight to EITR; make it a rate */ 983 newitr = IGC_AIM_DIVIDEND / newitr; 984 985 /* 986 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 987 * into the low latency end. The original was unbounded and would ask 988 * for ~95k ints/s on minimum sized frames. There is deliberately no 989 * floor, so jumbo traffic settles near 2.7k ints/s. 990 */ 991 if (sc->enable_aim == 1) 992 newitr = min(newitr, IGC_INTS_20K); 993 else 994 newitr = min(newitr, IGC_INTS_70K); 995 996 return (newitr); 997 } 998 999 /********************************************************************* 1000 * 1001 * Helper to calculate next EITR value for AIM 1002 * 1003 *********************************************************************/ 1004 static void 1005 igc_neweitr(struct igc_softc *sc, struct igc_rx_queue *que, 1006 struct rx_ring *rxr) 1007 { 1008 struct igc_hw *hw = &sc->hw; 1009 struct igc_tx_queue *tx_que; 1010 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 1011 u32 neweitr; 1012 int i; 1013 1014 igc_aim_rx_delta(rxr, &rxbytes, &rxpackets); 1015 1016 /* 1017 * A vector can service more than one TX ring when iflib is configured 1018 * with unequal RX and TX queue counts. Sample every ring routed to 1019 * this vector rather than treating the vector as a TX queue index. 1020 */ 1021 txbytes = txpackets = 0; 1022 for (i = 0; i < sc->tx_num_queues; i++) { 1023 tx_que = &sc->tx_queues[i]; 1024 if (tx_que->msix != que->msix) 1025 continue; 1026 igc_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 1027 txbytes += ringbytes; 1028 txpackets += ringpackets; 1029 } 1030 1031 /* Idle, do nothing */ 1032 if (txbytes == 0 && rxbytes == 0) 1033 return; 1034 1035 if (sc->enable_aim == 0) { 1036 neweitr = igc_max_interrupt_rate; 1037 } else if (sc->link_speed < SPEED_1000) { 1038 /* Use half default (4K) ITR if sub-gig */ 1039 neweitr = IGC_INTS_4K; 1040 } else if (sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 1041 /* Want at least enough packet buffer for two frames to AIM */ 1042 neweitr = igc_max_interrupt_rate; 1043 } else { 1044 neweitr = igc_ring_itr(sc, rxbytes, rxpackets, txbytes, 1045 txpackets); 1046 /* No usable observation; leave the rate where it is */ 1047 if (neweitr == 0) 1048 return; 1049 } 1050 1051 neweitr = IGC_INTS_TO_EITR(neweitr); 1052 1053 neweitr |= IGC_EITR_CNT_IGNR; 1054 1055 if (neweitr != que->eitr_setting) { 1056 que->eitr_setting = neweitr; 1057 IGC_WRITE_REG(hw, IGC_EITR(que->msix), que->eitr_setting); 1058 } 1059 } 1060 1061 /********************************************************************* 1062 * 1063 * Fast Legacy/MSI Combined Interrupt Service routine 1064 * 1065 *********************************************************************/ 1066 int 1067 igc_intr(void *arg) 1068 { 1069 struct igc_softc *sc = arg; 1070 struct igc_hw *hw = &sc->hw; 1071 struct igc_rx_queue *que = &sc->rx_queues[0]; 1072 struct rx_ring *rxr = &que->rxr; 1073 if_ctx_t ctx = sc->ctx; 1074 u32 reg_icr; 1075 1076 reg_icr = IGC_READ_REG(hw, IGC_ICR); 1077 1078 /* Hot eject? */ 1079 if (reg_icr == 0xffffffff) 1080 return FILTER_STRAY; 1081 1082 /* Definitely not our interrupt. */ 1083 if (reg_icr == 0x0) 1084 return FILTER_STRAY; 1085 1086 if ((reg_icr & IGC_ICR_INT_ASSERTED) == 0) 1087 return FILTER_STRAY; 1088 1089 /* 1090 * Only MSI-X interrupts have one-shot behavior by taking advantage 1091 * of the EIAC register. Thus, explicitly disable interrupts. This 1092 * also works around the MSI message reordering errata on certain 1093 * systems. 1094 */ 1095 IFDI_INTR_DISABLE(ctx); 1096 1097 /* Link status change */ 1098 if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) 1099 igc_handle_link(ctx); 1100 1101 if (reg_icr & IGC_ICR_RXO) 1102 sc->rx_overruns++; 1103 1104 igc_neweitr(sc, que, rxr); 1105 1106 return (FILTER_SCHEDULE_THREAD); 1107 } 1108 1109 static int 1110 igc_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1111 { 1112 struct igc_softc *sc = iflib_get_softc(ctx); 1113 struct igc_rx_queue *rxq = &sc->rx_queues[rxqid]; 1114 1115 IGC_WRITE_REG(&sc->hw, IGC_EIMS, rxq->eims); 1116 return (0); 1117 } 1118 1119 static int 1120 igc_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1121 { 1122 struct igc_softc *sc = iflib_get_softc(ctx); 1123 struct igc_tx_queue *txq = &sc->tx_queues[txqid]; 1124 1125 IGC_WRITE_REG(&sc->hw, IGC_EIMS, txq->eims); 1126 return (0); 1127 } 1128 1129 /********************************************************************* 1130 * 1131 * MSI-X RX Interrupt Service routine 1132 * 1133 **********************************************************************/ 1134 static int 1135 igc_msix_que(void *arg) 1136 { 1137 struct igc_rx_queue *que = arg; 1138 struct igc_softc *sc = que->sc; 1139 struct rx_ring *rxr = &que->rxr; 1140 1141 ++que->irqs; 1142 1143 igc_neweitr(sc, que, rxr); 1144 1145 return (FILTER_SCHEDULE_THREAD); 1146 } 1147 1148 /********************************************************************* 1149 * 1150 * MSI-X Link Fast Interrupt Service routine 1151 * 1152 **********************************************************************/ 1153 static int 1154 igc_msix_link(void *arg) 1155 { 1156 struct igc_softc *sc = arg; 1157 u32 reg_icr; 1158 1159 ++sc->link_irq; 1160 MPASS(sc->hw.back != NULL); 1161 reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 1162 1163 if (reg_icr & IGC_ICR_RXO) 1164 sc->rx_overruns++; 1165 1166 if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 1167 igc_handle_link(sc->ctx); 1168 } 1169 1170 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 1171 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->link_mask); 1172 1173 return (FILTER_HANDLED); 1174 } 1175 1176 static void 1177 igc_handle_link(void *context) 1178 { 1179 if_ctx_t ctx = context; 1180 struct igc_softc *sc = iflib_get_softc(ctx); 1181 1182 sc->hw.mac.get_link_status = true; 1183 iflib_admin_intr_deferred(ctx); 1184 } 1185 1186 /********************************************************************* 1187 * 1188 * Media Ioctl callback 1189 * 1190 * This routine is called whenever the user queries the status of 1191 * the interface using ifconfig. 1192 * 1193 **********************************************************************/ 1194 static void 1195 igc_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1196 { 1197 struct igc_softc *sc = iflib_get_softc(ctx); 1198 1199 INIT_DEBUGOUT("igc_if_media_status: begin"); 1200 1201 iflib_admin_intr_deferred(ctx); 1202 1203 ifmr->ifm_status = IFM_AVALID; 1204 ifmr->ifm_active = IFM_ETHER; 1205 1206 if (!sc->link_active) { 1207 return; 1208 } 1209 1210 ifmr->ifm_status |= IFM_ACTIVE; 1211 1212 switch (sc->link_speed) { 1213 case 10: 1214 ifmr->ifm_active |= IFM_10_T; 1215 break; 1216 case 100: 1217 ifmr->ifm_active |= IFM_100_TX; 1218 break; 1219 case 1000: 1220 ifmr->ifm_active |= IFM_1000_T; 1221 break; 1222 case 2500: 1223 ifmr->ifm_active |= IFM_2500_T; 1224 break; 1225 } 1226 1227 if (sc->link_duplex == FULL_DUPLEX) 1228 ifmr->ifm_active |= IFM_FDX; 1229 else 1230 ifmr->ifm_active |= IFM_HDX; 1231 } 1232 1233 /********************************************************************* 1234 * 1235 * Media Ioctl callback 1236 * 1237 * This routine is called when the user changes speed/duplex using 1238 * media/mediopt option with ifconfig. 1239 * 1240 **********************************************************************/ 1241 static int 1242 igc_if_media_change(if_ctx_t ctx) 1243 { 1244 struct igc_softc *sc = iflib_get_softc(ctx); 1245 struct ifmedia *ifm = iflib_get_media(ctx); 1246 1247 INIT_DEBUGOUT("igc_if_media_change: begin"); 1248 1249 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1250 return (EINVAL); 1251 1252 sc->hw.mac.autoneg = DO_AUTO_NEG; 1253 1254 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1255 case IFM_AUTO: 1256 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1257 break; 1258 case IFM_2500_T: 1259 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 1260 break; 1261 case IFM_1000_T: 1262 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1263 break; 1264 case IFM_100_TX: 1265 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1266 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 1267 else 1268 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 1269 break; 1270 case IFM_10_T: 1271 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1272 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 1273 else 1274 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 1275 break; 1276 default: 1277 device_printf(sc->dev, "Unsupported media type\n"); 1278 } 1279 1280 igc_if_init(ctx); 1281 1282 return (0); 1283 } 1284 1285 static int 1286 igc_if_set_promisc(if_ctx_t ctx, int flags) 1287 { 1288 struct igc_softc *sc = iflib_get_softc(ctx); 1289 if_t ifp = iflib_get_ifp(ctx); 1290 u32 reg_rctl; 1291 int mcnt = 0; 1292 1293 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1294 reg_rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_UPE); 1295 if (flags & IFF_ALLMULTI) 1296 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 1297 else 1298 mcnt = min(if_llmaddr_count(ifp), MAX_NUM_MULTICAST_ADDRESSES); 1299 1300 /* Don't disable if in MAX groups */ 1301 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 1302 reg_rctl &= (~IGC_RCTL_MPE); 1303 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1304 1305 if (flags & IFF_PROMISC) { 1306 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE); 1307 /* Turn this on if you want to see bad packets */ 1308 if (igc_debug_sbp) 1309 reg_rctl |= IGC_RCTL_SBP; 1310 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1311 } else if (flags & IFF_ALLMULTI) { 1312 reg_rctl |= IGC_RCTL_MPE; 1313 reg_rctl &= ~IGC_RCTL_UPE; 1314 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1315 } 1316 return (0); 1317 } 1318 1319 static u_int 1320 igc_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 1321 { 1322 u8 *mta = arg; 1323 1324 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 1325 return (0); 1326 1327 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1328 1329 return (1); 1330 } 1331 1332 /********************************************************************* 1333 * Multicast Update 1334 * 1335 * This routine is called whenever multicast address list is updated. 1336 * 1337 **********************************************************************/ 1338 1339 static void 1340 igc_if_multi_set(if_ctx_t ctx) 1341 { 1342 struct igc_softc *sc = iflib_get_softc(ctx); 1343 if_t ifp = iflib_get_ifp(ctx); 1344 u8 *mta; /* Multicast array memory */ 1345 u32 reg_rctl = 0; 1346 int mcnt = 0; 1347 1348 IOCTL_DEBUGOUT("igc_set_multi: begin"); 1349 1350 mta = sc->mta; 1351 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 1352 1353 mcnt = if_foreach_llmaddr(ifp, igc_copy_maddr, mta); 1354 1355 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1356 1357 if (if_getflags(ifp) & IFF_PROMISC) { 1358 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE); 1359 /* Turn this on if you want to see bad packets */ 1360 if (igc_debug_sbp) 1361 reg_rctl |= IGC_RCTL_SBP; 1362 } else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 1363 if_getflags(ifp) & IFF_ALLMULTI) { 1364 reg_rctl |= IGC_RCTL_MPE; 1365 reg_rctl &= ~IGC_RCTL_UPE; 1366 } else 1367 reg_rctl &= ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 1368 1369 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 1370 igc_update_mc_addr_list(&sc->hw, mta, mcnt); 1371 1372 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1373 } 1374 1375 /********************************************************************* 1376 * Timer routine 1377 * 1378 * This routine schedules igc_if_update_admin_status() to check for 1379 * link status and to gather statistics as well as to perform some 1380 * controller-specific hardware patting. 1381 * 1382 **********************************************************************/ 1383 static void 1384 igc_if_timer(if_ctx_t ctx, uint16_t qid) 1385 { 1386 1387 if (qid != 0) 1388 return; 1389 1390 iflib_admin_intr_deferred(ctx); 1391 } 1392 1393 static void 1394 igc_if_update_admin_status(if_ctx_t ctx) 1395 { 1396 struct igc_softc *sc = iflib_get_softc(ctx); 1397 struct igc_hw *hw = &sc->hw; 1398 device_t dev = iflib_get_dev(ctx); 1399 u32 link_check, thstat, ctrl; 1400 1401 link_check = thstat = ctrl = 0; 1402 /* Get the cached link value or read phy for real */ 1403 switch (hw->phy.media_type) { 1404 case igc_media_type_copper: 1405 if (hw->mac.get_link_status == true) { 1406 /* Do the work to read phy */ 1407 igc_check_for_link(hw); 1408 link_check = !hw->mac.get_link_status; 1409 } else 1410 link_check = true; 1411 break; 1412 case igc_media_type_unknown: 1413 igc_check_for_link(hw); 1414 link_check = !hw->mac.get_link_status; 1415 /* FALLTHROUGH */ 1416 default: 1417 break; 1418 } 1419 1420 /* Now check for a transition */ 1421 if (link_check && (sc->link_active == 0)) { 1422 igc_get_speed_and_duplex(hw, &sc->link_speed, 1423 &sc->link_duplex); 1424 if (bootverbose) 1425 device_printf(dev, "Link is up %d Mbps %s\n", 1426 sc->link_speed, 1427 ((sc->link_duplex == FULL_DUPLEX) ? 1428 "Full Duplex" : "Half Duplex")); 1429 sc->link_active = 1; 1430 iflib_link_state_change(ctx, LINK_STATE_UP, 1431 IF_Mbps(sc->link_speed)); 1432 } else if (!link_check && (sc->link_active == 1)) { 1433 sc->link_speed = 0; 1434 sc->link_duplex = 0; 1435 sc->link_active = 0; 1436 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1437 } 1438 igc_update_stats_counters(sc); 1439 } 1440 1441 static void 1442 igc_if_watchdog_reset(if_ctx_t ctx) 1443 { 1444 struct igc_softc *sc = iflib_get_softc(ctx); 1445 1446 /* 1447 * Just count the event; iflib(4) will already trigger a 1448 * sufficient reset of the controller. 1449 */ 1450 sc->watchdog_events++; 1451 } 1452 1453 /********************************************************************* 1454 * 1455 * This routine disables all traffic on the adapter by issuing a 1456 * global reset on the MAC. 1457 * 1458 **********************************************************************/ 1459 static void 1460 igc_if_stop(if_ctx_t ctx) 1461 { 1462 struct igc_softc *sc = iflib_get_softc(ctx); 1463 1464 INIT_DEBUGOUT("igc_if_stop: begin"); 1465 1466 igc_reset_hw(&sc->hw); 1467 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1468 } 1469 1470 /********************************************************************* 1471 * 1472 * Determine hardware revision. 1473 * 1474 **********************************************************************/ 1475 static void 1476 igc_identify_hardware(if_ctx_t ctx) 1477 { 1478 device_t dev = iflib_get_dev(ctx); 1479 struct igc_softc *sc = iflib_get_softc(ctx); 1480 1481 /* Make sure our PCI config space has the necessary stuff set */ 1482 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1483 1484 /* Save off the information about this board */ 1485 sc->hw.vendor_id = pci_get_vendor(dev); 1486 sc->hw.device_id = pci_get_device(dev); 1487 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1488 sc->hw.subsystem_vendor_id = 1489 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1490 sc->hw.subsystem_device_id = 1491 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1492 1493 /* Do Shared Code Init and Setup */ 1494 if (igc_set_mac_type(&sc->hw)) { 1495 device_printf(dev, "Setup init failure\n"); 1496 return; 1497 } 1498 } 1499 1500 static int 1501 igc_allocate_pci_resources(if_ctx_t ctx) 1502 { 1503 struct igc_softc *sc = iflib_get_softc(ctx); 1504 device_t dev = iflib_get_dev(ctx); 1505 int rid; 1506 1507 rid = PCIR_BAR(0); 1508 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1509 &rid, RF_ACTIVE); 1510 if (sc->memory == NULL) { 1511 device_printf(dev, 1512 "Unable to allocate bus resource: memory\n"); 1513 return (ENXIO); 1514 } 1515 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 1516 sc->osdep.mem_bus_space_handle = 1517 rman_get_bushandle(sc->memory); 1518 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 1519 1520 sc->hw.back = &sc->osdep; 1521 1522 return (0); 1523 } 1524 1525 /********************************************************************* 1526 * 1527 * Set up the MSI-X Interrupt handlers 1528 * 1529 **********************************************************************/ 1530 static int 1531 igc_if_msix_intr_assign(if_ctx_t ctx, int msix) 1532 { 1533 struct igc_softc *sc = iflib_get_softc(ctx); 1534 struct igc_rx_queue *rx_que = sc->rx_queues; 1535 struct igc_tx_queue *tx_que = sc->tx_queues; 1536 int error, rid, i, vector = 0, rx_vectors; 1537 char buf[16]; 1538 1539 /* First set up ring resources */ 1540 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 1541 rid = vector + 1; 1542 snprintf(buf, sizeof(buf), "rxq%d", i); 1543 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 1544 IFLIB_INTR_RXTX, igc_msix_que, rx_que, rx_que->me, buf); 1545 if (error) { 1546 device_printf(iflib_get_dev(ctx), 1547 "Failed to allocate que int %d err: %d", 1548 i, error); 1549 sc->rx_num_queues = i + 1; 1550 goto fail; 1551 } 1552 1553 rx_que->msix = vector; 1554 1555 /* 1556 * Set the bit to enable interrupt 1557 * in IGC_IMS -- bits 20 and 21 1558 * are for RX0 and RX1, note this has 1559 * NOTHING to do with the MSI-X vector 1560 */ 1561 rx_que->eims = 1 << vector; 1562 } 1563 rx_vectors = vector; 1564 1565 vector = 0; 1566 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 1567 snprintf(buf, sizeof(buf), "txq%d", i); 1568 tx_que = &sc->tx_queues[i]; 1569 iflib_softirq_alloc_generic(ctx, 1570 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 1571 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 1572 1573 tx_que->msix = (vector % sc->rx_num_queues); 1574 1575 /* 1576 * Set the bit to enable interrupt 1577 * in IGC_IMS -- bits 22 and 23 1578 * are for TX0 and TX1, note this has 1579 * NOTHING to do with the MSI-X vector 1580 */ 1581 tx_que->eims = 1 << i; 1582 } 1583 1584 /* Link interrupt */ 1585 rid = rx_vectors + 1; 1586 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 1587 igc_msix_link, sc, 0, "aq"); 1588 1589 if (error) { 1590 device_printf(iflib_get_dev(ctx), 1591 "Failed to register admin handler"); 1592 goto fail; 1593 } 1594 sc->linkvec = rx_vectors; 1595 return (0); 1596 fail: 1597 iflib_irq_free(ctx, &sc->irq); 1598 rx_que = sc->rx_queues; 1599 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 1600 iflib_irq_free(ctx, &rx_que->que_irq); 1601 return (error); 1602 } 1603 1604 static void 1605 igc_configure_queues(struct igc_softc *sc) 1606 { 1607 struct igc_hw *hw = &sc->hw; 1608 struct igc_rx_queue *rx_que; 1609 struct igc_tx_queue *tx_que; 1610 u32 ivar = 0; 1611 1612 /* First turn on RSS capability */ 1613 IGC_WRITE_REG(hw, IGC_GPIE, 1614 IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | IGC_GPIE_PBA | 1615 IGC_GPIE_NSICR); 1616 1617 /* Turn on MSI-X */ 1618 /* RX entries */ 1619 for (int i = 0; i < sc->rx_num_queues; i++) { 1620 u32 index = i >> 1; 1621 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1622 rx_que = &sc->rx_queues[i]; 1623 if (i & 1) { 1624 ivar &= 0xFF00FFFF; 1625 ivar |= (rx_que->msix | IGC_IVAR_VALID) << 16; 1626 } else { 1627 ivar &= 0xFFFFFF00; 1628 ivar |= rx_que->msix | IGC_IVAR_VALID; 1629 } 1630 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1631 } 1632 /* TX entries */ 1633 for (int i = 0; i < sc->tx_num_queues; i++) { 1634 u32 index = i >> 1; 1635 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1636 tx_que = &sc->tx_queues[i]; 1637 if (i & 1) { 1638 ivar &= 0x00FFFFFF; 1639 ivar |= (tx_que->msix | IGC_IVAR_VALID) << 24; 1640 } else { 1641 ivar &= 0xFFFF00FF; 1642 ivar |= (tx_que->msix | IGC_IVAR_VALID) << 8; 1643 } 1644 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1645 sc->que_mask |= tx_que->eims; 1646 } 1647 1648 /* And for the link interrupt */ 1649 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 1650 sc->link_mask = 1 << sc->linkvec; 1651 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 1652 1653 return; 1654 } 1655 1656 static void 1657 igc_initialize_interrupt_rate(struct igc_softc *sc) 1658 { 1659 struct igc_hw *hw = &sc->hw; 1660 struct igc_rx_queue *rx_que; 1661 u32 newitr; 1662 1663 newitr = IGC_INTS_TO_EITR(igc_max_interrupt_rate); 1664 newitr |= IGC_EITR_CNT_IGNR; 1665 1666 for (int i = 0; i < sc->rx_num_queues; i++) { 1667 rx_que = &sc->rx_queues[i]; 1668 rx_que->eitr_setting = newitr; 1669 IGC_WRITE_REG(hw, IGC_EITR(rx_que->msix), 1670 rx_que->eitr_setting); 1671 } 1672 } 1673 1674 static void 1675 igc_free_pci_resources(if_ctx_t ctx) 1676 { 1677 struct igc_softc *sc = iflib_get_softc(ctx); 1678 struct igc_rx_queue *que = sc->rx_queues; 1679 device_t dev = iflib_get_dev(ctx); 1680 1681 /* Release all MSI-X queue resources */ 1682 if (sc->intr_type == IFLIB_INTR_MSIX) 1683 iflib_irq_free(ctx, &sc->irq); 1684 1685 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 1686 iflib_irq_free(ctx, &que->que_irq); 1687 } 1688 1689 if (sc->memory != NULL) { 1690 bus_release_resource(dev, SYS_RES_MEMORY, 1691 rman_get_rid(sc->memory), sc->memory); 1692 sc->memory = NULL; 1693 } 1694 1695 if (sc->flash != NULL) { 1696 bus_release_resource(dev, SYS_RES_MEMORY, 1697 rman_get_rid(sc->flash), sc->flash); 1698 sc->flash = NULL; 1699 } 1700 1701 if (sc->ioport != NULL) { 1702 bus_release_resource(dev, SYS_RES_IOPORT, 1703 rman_get_rid(sc->ioport), sc->ioport); 1704 sc->ioport = NULL; 1705 } 1706 } 1707 1708 /* Set up MSI or MSI-X */ 1709 static int 1710 igc_setup_msix(if_ctx_t ctx) 1711 { 1712 return (0); 1713 } 1714 1715 /********************************************************************* 1716 * 1717 * Initialize the DMA Coalescing feature 1718 * 1719 **********************************************************************/ 1720 static void 1721 igc_init_dmac(struct igc_softc *sc, u32 pba) 1722 { 1723 device_t dev = sc->dev; 1724 struct igc_hw *hw = &sc->hw; 1725 u32 dmac, reg = ~IGC_DMACR_DMAC_EN; 1726 u16 hwm; 1727 u16 max_frame_size; 1728 int status; 1729 1730 max_frame_size = sc->shared->isc_max_frame_size; 1731 1732 if (sc->dmac == 0) { /* Disabling it */ 1733 IGC_WRITE_REG(hw, IGC_DMACR, reg); 1734 return; 1735 } else 1736 device_printf(dev, "DMA Coalescing enabled\n"); 1737 1738 /* Set starting threshold */ 1739 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 1740 1741 hwm = 64 * pba - max_frame_size / 16; 1742 if (hwm < 64 * (pba - 6)) 1743 hwm = 64 * (pba - 6); 1744 reg = IGC_READ_REG(hw, IGC_FCRTC); 1745 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 1746 reg |= ((hwm << IGC_FCRTC_RTH_COAL_SHIFT) 1747 & IGC_FCRTC_RTH_COAL_MASK); 1748 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 1749 1750 dmac = pba - max_frame_size / 512; 1751 if (dmac < pba - 10) 1752 dmac = pba - 10; 1753 reg = IGC_READ_REG(hw, IGC_DMACR); 1754 reg &= ~IGC_DMACR_DMACTHR_MASK; 1755 reg |= ((dmac << IGC_DMACR_DMACTHR_SHIFT) 1756 & IGC_DMACR_DMACTHR_MASK); 1757 1758 /* transition to L0x or L1 if available..*/ 1759 reg |= (IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK); 1760 1761 /* Check if status is 2.5Gb backplane connection 1762 * before configuration of watchdog timer, which is 1763 * in msec values in 12.8usec intervals 1764 * watchdog timer= msec values in 32usec intervals 1765 * for non 2.5Gb connection 1766 */ 1767 status = IGC_READ_REG(hw, IGC_STATUS); 1768 if ((status & IGC_STATUS_2P5_SKU) && 1769 (!(status & IGC_STATUS_2P5_SKU_OVER))) 1770 reg |= ((sc->dmac * 5) >> 6); 1771 else 1772 reg |= (sc->dmac >> 5); 1773 1774 IGC_WRITE_REG(hw, IGC_DMACR, reg); 1775 1776 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 1777 1778 /* Set the interval before transition */ 1779 reg = IGC_READ_REG(hw, IGC_DMCTLX); 1780 reg |= IGC_DMCTLX_DCFLUSH_DIS; 1781 1782 /* 1783 ** in 2.5Gb connection, TTLX unit is 0.4 usec 1784 ** which is 0x4*2 = 0xA. But delay is still 4 usec 1785 */ 1786 status = IGC_READ_REG(hw, IGC_STATUS); 1787 if ((status & IGC_STATUS_2P5_SKU) && 1788 (!(status & IGC_STATUS_2P5_SKU_OVER))) 1789 reg |= 0xA; 1790 else 1791 reg |= 0x4; 1792 1793 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 1794 1795 /* free space in tx packet buffer to wake from DMA coal */ 1796 IGC_WRITE_REG(hw, IGC_DMCTXTH, (IGC_TXPBSIZE - 1797 (2 * max_frame_size)) >> 6); 1798 1799 /* make low power state decision controlled by DMA coal */ 1800 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 1801 reg &= ~IGC_PCIEMISC_LX_DECISION; 1802 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 1803 } 1804 1805 /********************************************************************* 1806 * 1807 * Initialize the hardware to a configuration as specified by the 1808 * softc structure. 1809 * 1810 **********************************************************************/ 1811 static void 1812 igc_reset(if_ctx_t ctx) 1813 { 1814 device_t dev = iflib_get_dev(ctx); 1815 struct igc_softc *sc = iflib_get_softc(ctx); 1816 struct igc_hw *hw = &sc->hw; 1817 u32 rx_buffer_size; 1818 u32 pba; 1819 1820 INIT_DEBUGOUT("igc_reset: begin"); 1821 /* Let the firmware know the OS is in control */ 1822 igc_get_hw_control(sc); 1823 1824 /* 1825 * Packet Buffer Allocation (PBA) 1826 * Writing PBA sets the receive portion of the buffer 1827 * the remainder is used for the transmit buffer. 1828 */ 1829 pba = IGC_PBA_34K; 1830 1831 INIT_DEBUGOUT1("igc_reset: pba=%dK",pba); 1832 1833 /* 1834 * These parameters control the automatic generation (Tx) and 1835 * response (Rx) to Ethernet PAUSE frames. 1836 * - High water mark should allow for at least two frames to be 1837 * received after sending an XOFF. 1838 * - Low water mark works best when it is very near the high water 1839 * mark. 1840 * This allows the receiver to restart by sending XON when it has 1841 * drained a bit. Here we use an arbitrary value of 1500 which will 1842 * restart after one full frame is pulled from the buffer. There 1843 * could be several smaller frames in the buffer and if so they will 1844 * not trigger the XON until their total number reduces the buffer 1845 * by 1500. 1846 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 1847 */ 1848 rx_buffer_size = (pba & 0xffff) << 10; 1849 hw->fc.high_water = rx_buffer_size - 1850 roundup2(sc->hw.mac.max_frame_size, 1024); 1851 /* 16-byte granularity */ 1852 hw->fc.low_water = hw->fc.high_water - 16; 1853 1854 if (sc->fc) /* locally set flow control value? */ 1855 hw->fc.requested_mode = sc->fc; 1856 else 1857 hw->fc.requested_mode = igc_fc_full; 1858 1859 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 1860 1861 hw->fc.send_xon = true; 1862 1863 /* Issue a global reset */ 1864 igc_reset_hw(hw); 1865 IGC_WRITE_REG(hw, IGC_WUC, 0); 1866 1867 /* and a re-init */ 1868 if (igc_init_hw(hw) < 0) { 1869 device_printf(dev, "Hardware Initialization Failed\n"); 1870 return; 1871 } 1872 1873 /* Setup DMA Coalescing */ 1874 igc_init_dmac(sc, pba); 1875 1876 /* Save the final PBA off if it needs to be used elsewhere i.e. AIM */ 1877 sc->pba = pba; 1878 1879 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 1880 igc_get_phy_info(hw); 1881 igc_check_for_link(hw); 1882 } 1883 1884 /* 1885 * Initialise the RSS mapping for NICs that support multiple transmit/ 1886 * receive rings. 1887 */ 1888 1889 #define RSSKEYLEN 10 1890 static void 1891 igc_initialize_rss_mapping(struct igc_softc *sc) 1892 { 1893 struct igc_hw *hw = &sc->hw; 1894 int i; 1895 int queue_id; 1896 u32 reta; 1897 u32 rss_key[RSSKEYLEN], mrqc, shift = 0; 1898 1899 /* 1900 * The redirection table controls which destination 1901 * queue each bucket redirects traffic to. 1902 * Each DWORD represents four queues, with the LSB 1903 * being the first queue in the DWORD. 1904 * 1905 * This just allocates buckets to queues using round-robin 1906 * allocation. 1907 * 1908 * NOTE: It Just Happens to line up with the default 1909 * RSS allocation method. 1910 */ 1911 1912 /* Warning FM follows */ 1913 reta = 0; 1914 for (i = 0; i < 128; i++) { 1915 #ifdef RSS 1916 queue_id = rss_get_indirection_to_bucket(i); 1917 /* 1918 * If we have more queues than buckets, we'll 1919 * end up mapping buckets to a subset of the 1920 * queues. 1921 * 1922 * If we have more buckets than queues, we'll 1923 * end up instead assigning multiple buckets 1924 * to queues. 1925 * 1926 * Both are suboptimal, but we need to handle 1927 * the case so we don't go out of bounds 1928 * indexing arrays and such. 1929 */ 1930 queue_id = queue_id % sc->rx_num_queues; 1931 #else 1932 queue_id = (i % sc->rx_num_queues); 1933 #endif 1934 /* Adjust if required */ 1935 queue_id = queue_id << shift; 1936 1937 /* 1938 * The low 8 bits are for hash value (n+0); 1939 * The next 8 bits are for hash value (n+1), etc. 1940 */ 1941 reta = reta >> 8; 1942 reta = reta | ( ((uint32_t) queue_id) << 24); 1943 if ((i & 3) == 3) { 1944 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 1945 reta = 0; 1946 } 1947 } 1948 1949 /* Now fill in hash table */ 1950 1951 /* 1952 * MRQC: Multiple Receive Queues Command 1953 * Set queuing to RSS control, number depends on the device. 1954 */ 1955 mrqc = IGC_MRQC_ENABLE_RSS_4Q; 1956 1957 /* XXX ew typecasting */ 1958 rss_getkey((uint8_t *) &rss_key); 1959 for (i = 0; i < RSSKEYLEN; i++) 1960 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 1961 1962 /* 1963 * Configure the RSS fields to hash upon. 1964 */ 1965 mrqc |= (IGC_MRQC_RSS_FIELD_IPV4 | 1966 IGC_MRQC_RSS_FIELD_IPV4_TCP); 1967 mrqc |= (IGC_MRQC_RSS_FIELD_IPV6 | 1968 IGC_MRQC_RSS_FIELD_IPV6_TCP); 1969 mrqc |=( IGC_MRQC_RSS_FIELD_IPV4_UDP | 1970 IGC_MRQC_RSS_FIELD_IPV6_UDP); 1971 mrqc |=( IGC_MRQC_RSS_FIELD_IPV6_UDP_EX | 1972 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX); 1973 1974 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 1975 } 1976 1977 /********************************************************************* 1978 * 1979 * Setup networking device structure and register interface media. 1980 * 1981 **********************************************************************/ 1982 static int 1983 igc_setup_interface(if_ctx_t ctx) 1984 { 1985 if_t ifp = iflib_get_ifp(ctx); 1986 struct igc_softc *sc = iflib_get_softc(ctx); 1987 if_softc_ctx_t scctx = sc->shared; 1988 1989 INIT_DEBUGOUT("igc_setup_interface: begin"); 1990 1991 /* Single Queue */ 1992 if (sc->tx_num_queues == 1) { 1993 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 1994 if_setsendqready(ifp); 1995 } 1996 1997 /* 1998 * Specify the media types supported by this adapter and register 1999 * callbacks to update media and link information 2000 */ 2001 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 2002 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 2003 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2004 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 2005 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2006 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2007 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL); 2008 2009 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2010 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 2011 return (0); 2012 } 2013 2014 static int 2015 igc_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 2016 int ntxqs, int ntxqsets) 2017 { 2018 struct igc_softc *sc = iflib_get_softc(ctx); 2019 if_softc_ctx_t scctx = sc->shared; 2020 int error = IGC_SUCCESS; 2021 struct igc_tx_queue *que; 2022 int i, j; 2023 2024 MPASS(sc->tx_num_queues > 0); 2025 MPASS(sc->tx_num_queues == ntxqsets); 2026 2027 /* First allocate the top level queue structs */ 2028 if (!(sc->tx_queues = 2029 (struct igc_tx_queue *) malloc(sizeof(struct igc_tx_queue) * 2030 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2031 device_printf(iflib_get_dev(ctx), 2032 "Unable to allocate queue memory\n"); 2033 return(ENOMEM); 2034 } 2035 2036 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 2037 /* Set up some basics */ 2038 2039 struct tx_ring *txr = &que->txr; 2040 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 2041 ("%s: misaligned TX AIM snapshot %p", __func__, 2042 &txr->tx_aim_snapshot)); 2043 txr->sc = que->sc = sc; 2044 que->me = txr->me = i; 2045 2046 /* Allocate report status array */ 2047 if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * 2048 scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { 2049 device_printf(iflib_get_dev(ctx), 2050 "failed to allocate rs_idxs memory\n"); 2051 error = ENOMEM; 2052 goto fail; 2053 } 2054 for (j = 0; j < scctx->isc_ntxd[0]; j++) 2055 txr->tx_rsq[j] = QIDX_INVALID; 2056 /* get virtual and physical address of the hardware queues */ 2057 txr->tx_base = (struct igc_tx_desc *)vaddrs[i*ntxqs]; 2058 txr->tx_paddr = paddrs[i*ntxqs]; 2059 } 2060 2061 if (bootverbose) 2062 device_printf(iflib_get_dev(ctx), 2063 "allocated for %d tx_queues\n", sc->tx_num_queues); 2064 return (0); 2065 fail: 2066 igc_if_queues_free(ctx); 2067 return (error); 2068 } 2069 2070 static int 2071 igc_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 2072 int nrxqs, int nrxqsets) 2073 { 2074 struct igc_softc *sc = iflib_get_softc(ctx); 2075 int error = IGC_SUCCESS; 2076 struct igc_rx_queue *que; 2077 int i; 2078 2079 MPASS(sc->rx_num_queues > 0); 2080 MPASS(sc->rx_num_queues == nrxqsets); 2081 2082 /* First allocate the top level queue structs */ 2083 if (!(sc->rx_queues = 2084 (struct igc_rx_queue *) malloc(sizeof(struct igc_rx_queue) * 2085 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2086 device_printf(iflib_get_dev(ctx), 2087 "Unable to allocate queue memory\n"); 2088 error = ENOMEM; 2089 goto fail; 2090 } 2091 2092 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 2093 /* Set up some basics */ 2094 struct rx_ring *rxr = &que->rxr; 2095 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 2096 ("%s: misaligned RX AIM snapshot %p", __func__, 2097 &rxr->rx_aim_snapshot)); 2098 rxr->sc = que->sc = sc; 2099 rxr->que = que; 2100 que->me = rxr->me = i; 2101 2102 /* get virtual and physical address of the hardware queues */ 2103 rxr->rx_base = (union igc_rx_desc_extended *)vaddrs[i*nrxqs]; 2104 rxr->rx_paddr = paddrs[i*nrxqs]; 2105 } 2106 2107 if (bootverbose) 2108 device_printf(iflib_get_dev(ctx), 2109 "allocated for %d rx_queues\n", sc->rx_num_queues); 2110 2111 return (0); 2112 fail: 2113 igc_if_queues_free(ctx); 2114 return (error); 2115 } 2116 2117 static void 2118 igc_if_queues_free(if_ctx_t ctx) 2119 { 2120 struct igc_softc *sc = iflib_get_softc(ctx); 2121 struct igc_tx_queue *tx_que = sc->tx_queues; 2122 struct igc_rx_queue *rx_que = sc->rx_queues; 2123 2124 if (tx_que != NULL) { 2125 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 2126 struct tx_ring *txr = &tx_que->txr; 2127 if (txr->tx_rsq == NULL) 2128 break; 2129 2130 free(txr->tx_rsq, M_DEVBUF); 2131 txr->tx_rsq = NULL; 2132 } 2133 free(sc->tx_queues, M_DEVBUF); 2134 sc->tx_queues = NULL; 2135 } 2136 2137 if (rx_que != NULL) { 2138 free(sc->rx_queues, M_DEVBUF); 2139 sc->rx_queues = NULL; 2140 } 2141 2142 if (sc->mta != NULL) { 2143 free(sc->mta, M_DEVBUF); 2144 } 2145 } 2146 2147 /********************************************************************* 2148 * 2149 * Enable transmit unit. 2150 * 2151 **********************************************************************/ 2152 static void 2153 igc_initialize_transmit_unit(if_ctx_t ctx) 2154 { 2155 struct igc_softc *sc = iflib_get_softc(ctx); 2156 if_softc_ctx_t scctx = sc->shared; 2157 struct igc_tx_queue *que; 2158 struct tx_ring *txr; 2159 struct igc_hw *hw = &sc->hw; 2160 u32 tctl, txdctl = 0; 2161 2162 INIT_DEBUGOUT("igc_initialize_transmit_unit: begin"); 2163 2164 for (int i = 0; i < sc->tx_num_queues; i++, txr++) { 2165 u64 bus_addr; 2166 caddr_t offp, endp; 2167 2168 que = &sc->tx_queues[i]; 2169 txr = &que->txr; 2170 bus_addr = txr->tx_paddr; 2171 2172 /* Clear checksum offload context. */ 2173 offp = (caddr_t)&txr->csum_flags; 2174 endp = (caddr_t)(txr + 1); 2175 bzero(offp, endp - offp); 2176 2177 /* Base and Len of TX Ring */ 2178 IGC_WRITE_REG(hw, IGC_TDLEN(i), 2179 scctx->isc_ntxd[0] * sizeof(struct igc_tx_desc)); 2180 IGC_WRITE_REG(hw, IGC_TDBAH(i), 2181 (u32)(bus_addr >> 32)); 2182 IGC_WRITE_REG(hw, IGC_TDBAL(i), 2183 (u32)bus_addr); 2184 /* Init the HEAD/TAIL indices */ 2185 IGC_WRITE_REG(hw, IGC_TDT(i), 0); 2186 IGC_WRITE_REG(hw, IGC_TDH(i), 0); 2187 2188 HW_DEBUGOUT2("Base = %x, Length = %x\n", 2189 IGC_READ_REG(&sc->hw, IGC_TDBAL(i)), 2190 IGC_READ_REG(&sc->hw, IGC_TDLEN(i))); 2191 2192 txdctl = 0; /* clear txdctl */ 2193 txdctl |= 0x1f; /* PTHRESH */ 2194 txdctl |= 1 << 8; /* HTHRESH */ 2195 txdctl |= 1 << 16;/* WTHRESH */ 2196 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 2197 txdctl |= IGC_TXDCTL_GRAN; 2198 txdctl |= 1 << 25; /* LWTHRESH */ 2199 2200 IGC_WRITE_REG(hw, IGC_TXDCTL(i), txdctl); 2201 } 2202 2203 /* Program the Transmit Control Register */ 2204 tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 2205 tctl &= ~IGC_TCTL_CT; 2206 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 2207 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 2208 2209 /* This write will effectively turn on the transmit unit. */ 2210 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 2211 } 2212 2213 /********************************************************************* 2214 * 2215 * Enable receive unit. 2216 * 2217 **********************************************************************/ 2218 #define BSIZEPKT_ROUNDUP ((1<<IGC_SRRCTL_BSIZEPKT_SHIFT)-1) 2219 2220 static void 2221 igc_initialize_receive_unit(if_ctx_t ctx) 2222 { 2223 struct igc_softc *sc = iflib_get_softc(ctx); 2224 if_softc_ctx_t scctx = sc->shared; 2225 if_t ifp = iflib_get_ifp(ctx); 2226 struct igc_hw *hw = &sc->hw; 2227 struct igc_rx_queue *que; 2228 int i; 2229 u32 psize, rctl, rxcsum, srrctl = 0; 2230 2231 INIT_DEBUGOUT("igc_initialize_receive_units: begin"); 2232 2233 /* 2234 * Make sure receives are disabled while setting 2235 * up the descriptor ring 2236 */ 2237 rctl = IGC_READ_REG(hw, IGC_RCTL); 2238 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 2239 2240 /* Setup the Receive Control Register */ 2241 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 2242 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | 2243 IGC_RCTL_LBM_NO | IGC_RCTL_RDMTS_HALF | 2244 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 2245 2246 /* Do not store bad packets */ 2247 rctl &= ~IGC_RCTL_SBP; 2248 2249 /* Enable Long Packet receive */ 2250 if (if_getmtu(ifp) > ETHERMTU) 2251 rctl |= IGC_RCTL_LPE; 2252 else 2253 rctl &= ~IGC_RCTL_LPE; 2254 2255 /* Strip the CRC */ 2256 if (!igc_disable_crc_stripping) 2257 rctl |= IGC_RCTL_SECRC; 2258 2259 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 2260 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 2261 rxcsum |= IGC_RXCSUM_CRCOFL; 2262 if (sc->tx_num_queues > 1) 2263 rxcsum |= IGC_RXCSUM_PCSD; 2264 else 2265 rxcsum |= IGC_RXCSUM_IPPCSE; 2266 } else { 2267 if (sc->tx_num_queues > 1) 2268 rxcsum |= IGC_RXCSUM_PCSD; 2269 else 2270 rxcsum &= ~IGC_RXCSUM_TUOFL; 2271 } 2272 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 2273 2274 if (sc->rx_num_queues > 1) 2275 igc_initialize_rss_mapping(sc); 2276 2277 if (if_getmtu(ifp) > ETHERMTU) { 2278 psize = scctx->isc_max_frame_size; 2279 /* are we on a vlan? */ 2280 if (if_vlantrunkinuse(ifp)) 2281 psize += VLAN_TAG_SIZE; 2282 IGC_WRITE_REG(&sc->hw, IGC_RLPML, psize); 2283 } 2284 2285 /* Set maximum packet buffer len */ 2286 srrctl |= (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2287 IGC_SRRCTL_BSIZEPKT_SHIFT; 2288 /* srrctl above overrides this but set the register to a sane value */ 2289 rctl |= IGC_RCTL_SZ_2048; 2290 2291 /* 2292 * If TX flow control is disabled and there's >1 queue defined, 2293 * enable DROP. 2294 * 2295 * This drops frames rather than hanging the RX MAC for all queues. 2296 */ 2297 if ((sc->rx_num_queues > 1) && 2298 (sc->fc == igc_fc_none || 2299 sc->fc == igc_fc_rx_pause)) { 2300 srrctl |= IGC_SRRCTL_DROP_EN; 2301 } 2302 2303 /* Setup the Base and Length of the Rx Descriptor Rings */ 2304 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; i++, que++) { 2305 struct rx_ring *rxr = &que->rxr; 2306 u64 bus_addr = rxr->rx_paddr; 2307 u32 rxdctl; 2308 2309 #ifdef notyet 2310 /* Configure for header split? -- ignore for now */ 2311 rxr->hdr_split = igc_header_split; 2312 #else 2313 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 2314 #endif 2315 2316 IGC_WRITE_REG(hw, IGC_RDLEN(i), 2317 scctx->isc_nrxd[0] * sizeof(struct igc_rx_desc)); 2318 IGC_WRITE_REG(hw, IGC_RDBAH(i), (uint32_t)(bus_addr >> 32)); 2319 IGC_WRITE_REG(hw, IGC_RDBAL(i), (uint32_t)bus_addr); 2320 IGC_WRITE_REG(hw, IGC_SRRCTL(i), srrctl); 2321 /* Setup the Head and Tail Descriptor Pointers */ 2322 IGC_WRITE_REG(hw, IGC_RDH(i), 0); 2323 IGC_WRITE_REG(hw, IGC_RDT(i), 0); 2324 /* Enable this Queue */ 2325 rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(i)); 2326 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 2327 rxdctl &= 0xFFF00000; 2328 rxdctl |= IGC_RX_PTHRESH; 2329 rxdctl |= IGC_RX_HTHRESH << 8; 2330 rxdctl |= IGC_RX_WTHRESH << 16; 2331 IGC_WRITE_REG(hw, IGC_RXDCTL(i), rxdctl); 2332 } 2333 2334 /* Make sure VLAN Filters are off */ 2335 rctl &= ~IGC_RCTL_VFE; 2336 2337 /* Write out the settings */ 2338 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 2339 2340 return; 2341 } 2342 2343 static void 2344 igc_setup_vlan_hw_support(if_ctx_t ctx) 2345 { 2346 struct igc_softc *sc = iflib_get_softc(ctx); 2347 struct igc_hw *hw = &sc->hw; 2348 struct ifnet *ifp = iflib_get_ifp(ctx); 2349 u32 reg; 2350 2351 /* igc hardware doesn't seem to implement VFTA for HWFILTER */ 2352 2353 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 2354 !igc_disable_crc_stripping) { 2355 reg = IGC_READ_REG(hw, IGC_CTRL); 2356 reg |= IGC_CTRL_VME; 2357 IGC_WRITE_REG(hw, IGC_CTRL, reg); 2358 } else { 2359 reg = IGC_READ_REG(hw, IGC_CTRL); 2360 reg &= ~IGC_CTRL_VME; 2361 IGC_WRITE_REG(hw, IGC_CTRL, reg); 2362 } 2363 } 2364 2365 static void 2366 igc_if_intr_enable(if_ctx_t ctx) 2367 { 2368 struct igc_softc *sc = iflib_get_softc(ctx); 2369 struct igc_hw *hw = &sc->hw; 2370 u32 mask; 2371 2372 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 2373 mask = (sc->que_mask | sc->link_mask); 2374 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2375 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2376 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2377 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2378 } else 2379 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2380 IGC_WRITE_FLUSH(hw); 2381 } 2382 2383 static void 2384 igc_if_intr_disable(if_ctx_t ctx) 2385 { 2386 struct igc_softc *sc = iflib_get_softc(ctx); 2387 struct igc_hw *hw = &sc->hw; 2388 2389 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 2390 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2391 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2392 } 2393 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2394 IGC_WRITE_FLUSH(hw); 2395 } 2396 2397 /* 2398 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 2399 * For ASF and Pass Through versions of f/w this means 2400 * that the driver is loaded. For AMT version type f/w 2401 * this means that the network i/f is open. 2402 */ 2403 static void 2404 igc_get_hw_control(struct igc_softc *sc) 2405 { 2406 u32 ctrl_ext; 2407 2408 if (sc->vf_ifp) 2409 return; 2410 2411 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2412 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, 2413 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 2414 } 2415 2416 /* 2417 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 2418 * For ASF and Pass Through versions of f/w this means that 2419 * the driver is no longer loaded. For AMT versions of the 2420 * f/w this means that the network i/f is closed. 2421 */ 2422 static void 2423 igc_release_hw_control(struct igc_softc *sc) 2424 { 2425 u32 ctrl_ext; 2426 2427 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2428 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, 2429 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 2430 return; 2431 } 2432 2433 static int 2434 igc_is_valid_ether_addr(u8 *addr) 2435 { 2436 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 2437 2438 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 2439 return (false); 2440 } 2441 2442 return (true); 2443 } 2444 2445 /* 2446 ** Parse the interface capabilities with regard 2447 ** to both system management and wake-on-lan for 2448 ** later use. 2449 */ 2450 static void 2451 igc_get_wakeup(if_ctx_t ctx) 2452 { 2453 struct igc_softc *sc = iflib_get_softc(ctx); 2454 u16 eeprom_data = 0, apme_mask; 2455 2456 apme_mask = IGC_WUC_APME; 2457 eeprom_data = IGC_READ_REG(&sc->hw, IGC_WUC); 2458 2459 if (eeprom_data & apme_mask) 2460 sc->wol = IGC_WUFC_LNKC; 2461 } 2462 2463 2464 /* 2465 * Enable PCI Wake On Lan capability 2466 */ 2467 static void 2468 igc_enable_wakeup(if_ctx_t ctx) 2469 { 2470 struct igc_softc *sc = iflib_get_softc(ctx); 2471 device_t dev = iflib_get_dev(ctx); 2472 if_t ifp = iflib_get_ifp(ctx); 2473 int error = 0; 2474 u32 ctrl, rctl; 2475 2476 if (!pci_has_pm(dev)) 2477 return; 2478 2479 /* 2480 * Determine type of Wakeup: note that wol 2481 * is set with all bits on by default. 2482 */ 2483 if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) 2484 sc->wol &= ~IGC_WUFC_MAG; 2485 2486 if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0) 2487 sc->wol &= ~IGC_WUFC_EX; 2488 2489 if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) 2490 sc->wol &= ~IGC_WUFC_MC; 2491 else { 2492 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2493 rctl |= IGC_RCTL_MPE; 2494 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2495 } 2496 2497 if (!(sc->wol & (IGC_WUFC_EX | IGC_WUFC_MAG | IGC_WUFC_MC))) 2498 goto pme; 2499 2500 /* Advertise the wakeup capability */ 2501 ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 2502 ctrl |= IGC_CTRL_ADVD3WUC; 2503 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 2504 2505 /* Enable wakeup by the MAC */ 2506 IGC_WRITE_REG(&sc->hw, IGC_WUC, IGC_WUC_PME_EN); 2507 IGC_WRITE_REG(&sc->hw, IGC_WUFC, sc->wol); 2508 2509 pme: 2510 if (!error && (if_getcapenable(ifp) & IFCAP_WOL)) 2511 pci_enable_pme(dev); 2512 2513 return; 2514 } 2515 2516 /********************************************************************** 2517 * 2518 * Update the board statistics counters. 2519 * 2520 **********************************************************************/ 2521 static void 2522 igc_update_stats_counters(struct igc_softc *sc) 2523 { 2524 u64 prev_xoffrxc = sc->stats.xoffrxc; 2525 2526 sc->stats.crcerrs += IGC_READ_REG(&sc->hw, IGC_CRCERRS); 2527 sc->stats.mpc += IGC_READ_REG(&sc->hw, IGC_MPC); 2528 sc->stats.scc += IGC_READ_REG(&sc->hw, IGC_SCC); 2529 sc->stats.ecol += IGC_READ_REG(&sc->hw, IGC_ECOL); 2530 2531 sc->stats.mcc += IGC_READ_REG(&sc->hw, IGC_MCC); 2532 sc->stats.latecol += IGC_READ_REG(&sc->hw, IGC_LATECOL); 2533 sc->stats.colc += IGC_READ_REG(&sc->hw, IGC_COLC); 2534 sc->stats.colc += IGC_READ_REG(&sc->hw, IGC_RERC); 2535 sc->stats.dc += IGC_READ_REG(&sc->hw, IGC_DC); 2536 sc->stats.rlec += IGC_READ_REG(&sc->hw, IGC_RLEC); 2537 sc->stats.xonrxc += IGC_READ_REG(&sc->hw, IGC_XONRXC); 2538 sc->stats.xontxc += IGC_READ_REG(&sc->hw, IGC_XONTXC); 2539 sc->stats.xoffrxc += IGC_READ_REG(&sc->hw, IGC_XOFFRXC); 2540 /* 2541 * For watchdog management we need to know if we have been 2542 * paused during the last interval, so capture that here. 2543 */ 2544 if (sc->stats.xoffrxc != prev_xoffrxc) 2545 sc->shared->isc_pause_frames = 1; 2546 sc->stats.xofftxc += IGC_READ_REG(&sc->hw, IGC_XOFFTXC); 2547 sc->stats.fcruc += IGC_READ_REG(&sc->hw, IGC_FCRUC); 2548 sc->stats.prc64 += IGC_READ_REG(&sc->hw, IGC_PRC64); 2549 sc->stats.prc127 += IGC_READ_REG(&sc->hw, IGC_PRC127); 2550 sc->stats.prc255 += IGC_READ_REG(&sc->hw, IGC_PRC255); 2551 sc->stats.prc511 += IGC_READ_REG(&sc->hw, IGC_PRC511); 2552 sc->stats.prc1023 += IGC_READ_REG(&sc->hw, IGC_PRC1023); 2553 sc->stats.prc1522 += IGC_READ_REG(&sc->hw, IGC_PRC1522); 2554 sc->stats.tlpic += IGC_READ_REG(&sc->hw, IGC_TLPIC); 2555 sc->stats.rlpic += IGC_READ_REG(&sc->hw, IGC_RLPIC); 2556 sc->stats.gprc += IGC_READ_REG(&sc->hw, IGC_GPRC); 2557 sc->stats.bprc += IGC_READ_REG(&sc->hw, IGC_BPRC); 2558 sc->stats.mprc += IGC_READ_REG(&sc->hw, IGC_MPRC); 2559 sc->stats.gptc += IGC_READ_REG(&sc->hw, IGC_GPTC); 2560 2561 /* For the 64-bit byte counters the low dword must be read first. */ 2562 /* Both registers clear on the read of the high dword */ 2563 2564 sc->stats.gorc += IGC_READ_REG(&sc->hw, IGC_GORCL) + 2565 ((u64)IGC_READ_REG(&sc->hw, IGC_GORCH) << 32); 2566 sc->stats.gotc += IGC_READ_REG(&sc->hw, IGC_GOTCL) + 2567 ((u64)IGC_READ_REG(&sc->hw, IGC_GOTCH) << 32); 2568 2569 sc->stats.rnbc += IGC_READ_REG(&sc->hw, IGC_RNBC); 2570 sc->stats.ruc += IGC_READ_REG(&sc->hw, IGC_RUC); 2571 sc->stats.rfc += IGC_READ_REG(&sc->hw, IGC_RFC); 2572 sc->stats.roc += IGC_READ_REG(&sc->hw, IGC_ROC); 2573 sc->stats.rjc += IGC_READ_REG(&sc->hw, IGC_RJC); 2574 2575 sc->stats.mgprc += IGC_READ_REG(&sc->hw, IGC_MGTPRC); 2576 sc->stats.mgpdc += IGC_READ_REG(&sc->hw, IGC_MGTPDC); 2577 sc->stats.mgptc += IGC_READ_REG(&sc->hw, IGC_MGTPTC); 2578 2579 sc->stats.tor += IGC_READ_REG(&sc->hw, IGC_TORH); 2580 sc->stats.tot += IGC_READ_REG(&sc->hw, IGC_TOTH); 2581 2582 sc->stats.tpr += IGC_READ_REG(&sc->hw, IGC_TPR); 2583 sc->stats.tpt += IGC_READ_REG(&sc->hw, IGC_TPT); 2584 sc->stats.ptc64 += IGC_READ_REG(&sc->hw, IGC_PTC64); 2585 sc->stats.ptc127 += IGC_READ_REG(&sc->hw, IGC_PTC127); 2586 sc->stats.ptc255 += IGC_READ_REG(&sc->hw, IGC_PTC255); 2587 sc->stats.ptc511 += IGC_READ_REG(&sc->hw, IGC_PTC511); 2588 sc->stats.ptc1023 += IGC_READ_REG(&sc->hw, IGC_PTC1023); 2589 sc->stats.ptc1522 += IGC_READ_REG(&sc->hw, IGC_PTC1522); 2590 sc->stats.mptc += IGC_READ_REG(&sc->hw, IGC_MPTC); 2591 sc->stats.bptc += IGC_READ_REG(&sc->hw, IGC_BPTC); 2592 2593 /* Interrupt Counts */ 2594 sc->stats.iac += IGC_READ_REG(&sc->hw, IGC_IAC); 2595 sc->stats.rxdmtc += IGC_READ_REG(&sc->hw, IGC_RXDMTC); 2596 2597 sc->stats.algnerrc += IGC_READ_REG(&sc->hw, IGC_ALGNERRC); 2598 sc->stats.tncrs += IGC_READ_REG(&sc->hw, IGC_TNCRS); 2599 sc->stats.htdpmc += IGC_READ_REG(&sc->hw, IGC_HTDPMC); 2600 sc->stats.tsctc += IGC_READ_REG(&sc->hw, IGC_TSCTC); 2601 } 2602 2603 static uint64_t 2604 igc_if_get_counter(if_ctx_t ctx, ift_counter cnt) 2605 { 2606 struct igc_softc *sc = iflib_get_softc(ctx); 2607 if_t ifp = iflib_get_ifp(ctx); 2608 2609 switch (cnt) { 2610 case IFCOUNTER_COLLISIONS: 2611 return (sc->stats.colc); 2612 case IFCOUNTER_IERRORS: 2613 return (sc->dropped_pkts + sc->stats.rxerrc + 2614 sc->stats.crcerrs + sc->stats.algnerrc + 2615 sc->stats.ruc + sc->stats.roc + 2616 sc->stats.mpc + sc->stats.htdpmc); 2617 case IFCOUNTER_OERRORS: 2618 return (if_get_counter_default(ifp, cnt) + 2619 sc->stats.ecol + sc->stats.latecol + sc->watchdog_events); 2620 default: 2621 return (if_get_counter_default(ifp, cnt)); 2622 } 2623 } 2624 2625 /* igc_if_needs_restart - Tell iflib when the driver needs to be reinitialized 2626 * @ctx: iflib context 2627 * @event: event code to check 2628 * 2629 * Defaults to returning false for unknown events. 2630 * 2631 * @returns true if iflib needs to reinit the interface 2632 */ 2633 static bool 2634 igc_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 2635 { 2636 switch (event) { 2637 case IFLIB_RESTART_VLAN_CONFIG: 2638 default: 2639 return (false); 2640 } 2641 } 2642 2643 /* Export a single 32-bit register via a read-only sysctl. */ 2644 static int 2645 igc_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 2646 { 2647 struct igc_softc *sc; 2648 u_int val; 2649 2650 sc = oidp->oid_arg1; 2651 val = IGC_READ_REG(&sc->hw, oidp->oid_arg2); 2652 return (sysctl_handle_int(oidp, &val, 0, req)); 2653 } 2654 2655 /* Per queue holdoff interrupt rate handler */ 2656 static int 2657 igc_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 2658 { 2659 struct igc_rx_queue *rque; 2660 struct igc_tx_queue *tque; 2661 struct igc_hw *hw; 2662 int error; 2663 u32 reg, usec, rate; 2664 2665 bool tx = oidp->oid_arg2; 2666 2667 if (tx) { 2668 tque = oidp->oid_arg1; 2669 hw = &tque->sc->hw; 2670 reg = IGC_READ_REG(hw, IGC_EITR(tque->msix)); 2671 } else { 2672 rque = oidp->oid_arg1; 2673 hw = &rque->sc->hw; 2674 reg = IGC_READ_REG(hw, IGC_EITR(rque->msix)); 2675 } 2676 2677 usec = (reg & IGC_QVECTOR_MASK); 2678 if (usec > 0) 2679 rate = IGC_EITR_TO_INTS(usec); 2680 else 2681 rate = 0; 2682 2683 error = sysctl_handle_int(oidp, &rate, 0, req); 2684 if (error || !req->newptr) 2685 return error; 2686 return 0; 2687 } 2688 2689 /* 2690 * Add sysctl variables, one per statistic, to the system. 2691 */ 2692 static void 2693 igc_add_hw_stats(struct igc_softc *sc) 2694 { 2695 device_t dev = iflib_get_dev(sc->ctx); 2696 struct igc_tx_queue *tx_que = sc->tx_queues; 2697 struct igc_rx_queue *rx_que = sc->rx_queues; 2698 2699 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 2700 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 2701 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 2702 struct igc_hw_stats *stats = &sc->stats; 2703 2704 struct sysctl_oid *stat_node, *queue_node, *int_node; 2705 struct sysctl_oid_list *stat_list, *queue_list, *int_list; 2706 2707 #define QUEUE_NAME_LEN 32 2708 char namebuf[QUEUE_NAME_LEN]; 2709 2710 /* Driver Statistics */ 2711 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 2712 CTLFLAG_RD, &sc->dropped_pkts, 2713 "Driver dropped packets"); 2714 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 2715 CTLFLAG_RD, &sc->link_irq, 2716 "Link MSI-X IRQ Handled"); 2717 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 2718 CTLFLAG_RD, &sc->rx_overruns, 2719 "RX overruns"); 2720 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", 2721 CTLFLAG_RD, &sc->watchdog_events, 2722 "Watchdog timeouts"); 2723 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 2724 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 2725 sc, IGC_CTRL, igc_sysctl_reg_handler, "IU", 2726 "Device Control Register"); 2727 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 2728 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 2729 sc, IGC_RCTL, igc_sysctl_reg_handler, "IU", 2730 "Receiver Control Register"); 2731 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 2732 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 2733 "Flow Control High Watermark"); 2734 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 2735 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 2736 "Flow Control Low Watermark"); 2737 2738 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 2739 struct tx_ring *txr = &tx_que->txr; 2740 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 2741 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2742 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 2743 queue_list = SYSCTL_CHILDREN(queue_node); 2744 2745 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 2746 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 2747 true, igc_sysctl_interrupt_rate_handler, "IU", 2748 "Interrupt Rate"); 2749 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 2750 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 2751 IGC_TDH(txr->me), igc_sysctl_reg_handler, "IU", 2752 "Transmit Descriptor Head"); 2753 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 2754 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 2755 IGC_TDT(txr->me), igc_sysctl_reg_handler, "IU", 2756 "Transmit Descriptor Tail"); 2757 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 2758 CTLFLAG_RD, &txr->tx_irq, 2759 "Queue MSI-X Transmit Interrupts"); 2760 } 2761 2762 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 2763 struct rx_ring *rxr = &rx_que->rxr; 2764 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 2765 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 2766 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 2767 queue_list = SYSCTL_CHILDREN(queue_node); 2768 2769 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 2770 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 2771 false, igc_sysctl_interrupt_rate_handler, "IU", 2772 "Interrupt Rate"); 2773 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 2774 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 2775 IGC_RDH(rxr->me), igc_sysctl_reg_handler, "IU", 2776 "Receive Descriptor Head"); 2777 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 2778 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 2779 IGC_RDT(rxr->me), igc_sysctl_reg_handler, "IU", 2780 "Receive Descriptor Tail"); 2781 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 2782 CTLFLAG_RD, &rxr->rx_irq, 2783 "Queue MSI-X Receive Interrupts"); 2784 } 2785 2786 /* MAC stats get their own sub node */ 2787 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 2788 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 2789 stat_list = SYSCTL_CHILDREN(stat_node); 2790 2791 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 2792 CTLFLAG_RD, &stats->ecol, 2793 "Excessive collisions"); 2794 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 2795 CTLFLAG_RD, &stats->scc, 2796 "Single collisions"); 2797 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 2798 CTLFLAG_RD, &stats->mcc, 2799 "Multiple collisions"); 2800 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 2801 CTLFLAG_RD, &stats->latecol, 2802 "Late collisions"); 2803 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 2804 CTLFLAG_RD, &stats->colc, 2805 "Collision Count"); 2806 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 2807 CTLFLAG_RD, &sc->stats.symerrs, 2808 "Symbol Errors"); 2809 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 2810 CTLFLAG_RD, &sc->stats.sec, 2811 "Sequence Errors"); 2812 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 2813 CTLFLAG_RD, &sc->stats.dc, 2814 "Defer Count"); 2815 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 2816 CTLFLAG_RD, &sc->stats.mpc, 2817 "Missed Packets"); 2818 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 2819 CTLFLAG_RD, &sc->stats.rlec, 2820 "Receive Length Errors"); 2821 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 2822 CTLFLAG_RD, &sc->stats.rnbc, 2823 "Receive No Buffers"); 2824 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 2825 CTLFLAG_RD, &sc->stats.ruc, 2826 "Receive Undersize"); 2827 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 2828 CTLFLAG_RD, &sc->stats.rfc, 2829 "Fragmented Packets Received "); 2830 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 2831 CTLFLAG_RD, &sc->stats.roc, 2832 "Oversized Packets Received"); 2833 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 2834 CTLFLAG_RD, &sc->stats.rjc, 2835 "Received Jabber"); 2836 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 2837 CTLFLAG_RD, &sc->stats.rxerrc, 2838 "Receive Errors"); 2839 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 2840 CTLFLAG_RD, &sc->stats.crcerrs, 2841 "CRC errors"); 2842 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 2843 CTLFLAG_RD, &sc->stats.algnerrc, 2844 "Alignment Errors"); 2845 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 2846 CTLFLAG_RD, &sc->stats.xonrxc, 2847 "XON Received"); 2848 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 2849 CTLFLAG_RD, &sc->stats.xontxc, 2850 "XON Transmitted"); 2851 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 2852 CTLFLAG_RD, &sc->stats.xoffrxc, 2853 "XOFF Received"); 2854 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 2855 CTLFLAG_RD, &sc->stats.xofftxc, 2856 "XOFF Transmitted"); 2857 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 2858 CTLFLAG_RD, &sc->stats.fcruc, 2859 "Unsupported Flow Control Received"); 2860 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 2861 CTLFLAG_RD, &sc->stats.mgprc, 2862 "Management Packets Received"); 2863 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 2864 CTLFLAG_RD, &sc->stats.mgpdc, 2865 "Management Packets Dropped"); 2866 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 2867 CTLFLAG_RD, &sc->stats.mgptc, 2868 "Management Packets Transmitted"); 2869 2870 /* Packet Reception Stats */ 2871 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 2872 CTLFLAG_RD, &sc->stats.tpr, 2873 "Total Packets Received "); 2874 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 2875 CTLFLAG_RD, &sc->stats.gprc, 2876 "Good Packets Received"); 2877 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 2878 CTLFLAG_RD, &sc->stats.bprc, 2879 "Broadcast Packets Received"); 2880 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 2881 CTLFLAG_RD, &sc->stats.mprc, 2882 "Multicast Packets Received"); 2883 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 2884 CTLFLAG_RD, &sc->stats.prc64, 2885 "64 byte frames received "); 2886 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 2887 CTLFLAG_RD, &sc->stats.prc127, 2888 "65-127 byte frames received"); 2889 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 2890 CTLFLAG_RD, &sc->stats.prc255, 2891 "128-255 byte frames received"); 2892 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 2893 CTLFLAG_RD, &sc->stats.prc511, 2894 "256-511 byte frames received"); 2895 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 2896 CTLFLAG_RD, &sc->stats.prc1023, 2897 "512-1023 byte frames received"); 2898 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 2899 CTLFLAG_RD, &sc->stats.prc1522, 2900 "1023-1522 byte frames received"); 2901 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 2902 CTLFLAG_RD, &sc->stats.gorc, 2903 "Good Octets Received"); 2904 2905 /* Packet Transmission Stats */ 2906 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 2907 CTLFLAG_RD, &sc->stats.gotc, 2908 "Good Octets Transmitted"); 2909 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 2910 CTLFLAG_RD, &sc->stats.tpt, 2911 "Total Packets Transmitted"); 2912 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 2913 CTLFLAG_RD, &sc->stats.gptc, 2914 "Good Packets Transmitted"); 2915 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 2916 CTLFLAG_RD, &sc->stats.bptc, 2917 "Broadcast Packets Transmitted"); 2918 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 2919 CTLFLAG_RD, &sc->stats.mptc, 2920 "Multicast Packets Transmitted"); 2921 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 2922 CTLFLAG_RD, &sc->stats.ptc64, 2923 "64 byte frames transmitted "); 2924 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 2925 CTLFLAG_RD, &sc->stats.ptc127, 2926 "65-127 byte frames transmitted"); 2927 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 2928 CTLFLAG_RD, &sc->stats.ptc255, 2929 "128-255 byte frames transmitted"); 2930 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 2931 CTLFLAG_RD, &sc->stats.ptc511, 2932 "256-511 byte frames transmitted"); 2933 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 2934 CTLFLAG_RD, &sc->stats.ptc1023, 2935 "512-1023 byte frames transmitted"); 2936 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 2937 CTLFLAG_RD, &sc->stats.ptc1522, 2938 "1024-1522 byte frames transmitted"); 2939 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 2940 CTLFLAG_RD, &sc->stats.tsctc, 2941 "TSO Contexts Transmitted"); 2942 2943 /* Interrupt Stats */ 2944 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 2945 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 2946 int_list = SYSCTL_CHILDREN(int_node); 2947 2948 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 2949 CTLFLAG_RD, &sc->stats.iac, 2950 "Interrupt Assertion Count"); 2951 2952 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 2953 CTLFLAG_RD, &sc->stats.rxdmtc, 2954 "Rx Desc Min Thresh Count"); 2955 } 2956 2957 static void 2958 igc_fw_version(struct igc_softc *sc) 2959 { 2960 struct igc_hw *hw = &sc->hw; 2961 struct igc_fw_version *fw_ver = &sc->fw_ver; 2962 2963 *fw_ver = (struct igc_fw_version){0}; 2964 2965 igc_get_fw_version(hw, fw_ver); 2966 } 2967 2968 static void 2969 igc_sbuf_fw_version(struct igc_fw_version *fw_ver, struct sbuf *buf) 2970 { 2971 const char *space = ""; 2972 2973 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 2974 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 2975 fw_ver->eep_minor, fw_ver->eep_build); 2976 space = " "; 2977 } 2978 2979 if (fw_ver->invm_major || fw_ver->invm_minor || 2980 fw_ver->invm_img_type) { 2981 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 2982 space, fw_ver->invm_major, fw_ver->invm_minor, 2983 fw_ver->invm_img_type); 2984 space = " "; 2985 } 2986 2987 if (fw_ver->or_valid) { 2988 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 2989 space, fw_ver->or_major, fw_ver->or_build, 2990 fw_ver->or_patch); 2991 space = " "; 2992 } 2993 2994 if (fw_ver->etrack_id) 2995 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 2996 } 2997 2998 static void 2999 igc_print_fw_version(struct igc_softc *sc ) 3000 { 3001 device_t dev = sc->dev; 3002 struct sbuf *buf; 3003 int error = 0; 3004 3005 buf = sbuf_new_auto(); 3006 if (!buf) { 3007 device_printf(dev, "Could not allocate sbuf for output.\n"); 3008 return; 3009 } 3010 3011 igc_sbuf_fw_version(&sc->fw_ver, buf); 3012 3013 error = sbuf_finish(buf); 3014 if (error) 3015 device_printf(dev, "Error finishing sbuf: %d\n", error); 3016 else if (sbuf_len(buf)) 3017 device_printf(dev, "%s\n", sbuf_data(buf)); 3018 3019 sbuf_delete(buf); 3020 } 3021 3022 static int 3023 igc_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 3024 { 3025 struct igc_softc *sc = (struct igc_softc *)arg1; 3026 device_t dev = sc->dev; 3027 struct sbuf *buf; 3028 int error = 0; 3029 3030 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3031 if (!buf) { 3032 device_printf(dev, "Could not allocate sbuf for output.\n"); 3033 return (ENOMEM); 3034 } 3035 3036 igc_sbuf_fw_version(&sc->fw_ver, buf); 3037 3038 error = sbuf_finish(buf); 3039 if (error) 3040 device_printf(dev, "Error finishing sbuf: %d\n", error); 3041 3042 sbuf_delete(buf); 3043 3044 return (0); 3045 } 3046 3047 /********************************************************************** 3048 * 3049 * This routine provides a way to dump out the adapter eeprom, 3050 * often a useful debug/service tool. This only dumps the first 3051 * 32 words, stuff that matters is in that extent. 3052 * 3053 **********************************************************************/ 3054 static int 3055 igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 3056 { 3057 struct igc_softc *sc = (struct igc_softc *)arg1; 3058 int error; 3059 int result; 3060 3061 result = -1; 3062 error = sysctl_handle_int(oidp, &result, 0, req); 3063 3064 if (error || !req->newptr) 3065 return (error); 3066 3067 /* 3068 * This value will cause a hex dump of the 3069 * first 32 16-bit words of the EEPROM to 3070 * the screen. 3071 */ 3072 if (result == 1) 3073 igc_print_nvm_info(sc); 3074 3075 return (error); 3076 } 3077 3078 static void 3079 igc_print_nvm_info(struct igc_softc *sc) 3080 { 3081 u16 eeprom_data; 3082 int i, j, row = 0; 3083 3084 /* Its a bit crude, but it gets the job done */ 3085 printf("\nInterface EEPROM Dump:\n"); 3086 printf("Offset\n0x0000 "); 3087 for (i = 0, j = 0; i < 32; i++, j++) { 3088 if (j == 8) { /* Make the offset block */ 3089 j = 0; ++row; 3090 printf("\n0x00%x0 ",row); 3091 } 3092 igc_read_nvm(&sc->hw, i, 1, &eeprom_data); 3093 printf("%04x ", eeprom_data); 3094 } 3095 printf("\n"); 3096 } 3097 3098 static int 3099 igc_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 3100 { 3101 struct igc_softc *sc; 3102 u32 reg, val, shift; 3103 int error, mask; 3104 3105 sc = oidp->oid_arg1; 3106 switch (oidp->oid_arg2) { 3107 case 0: 3108 reg = IGC_DTXTCPFLGL; 3109 shift = 0; 3110 break; 3111 case 1: 3112 reg = IGC_DTXTCPFLGL; 3113 shift = 16; 3114 break; 3115 case 2: 3116 reg = IGC_DTXTCPFLGH; 3117 shift = 0; 3118 break; 3119 default: 3120 return (EINVAL); 3121 break; 3122 } 3123 val = IGC_READ_REG(&sc->hw, reg); 3124 mask = (val >> shift) & 0xfff; 3125 error = sysctl_handle_int(oidp, &mask, 0, req); 3126 if (error != 0 || req->newptr == NULL) 3127 return (error); 3128 if (mask < 0 || mask > 0xfff) 3129 return (EINVAL); 3130 val = (val & ~(0xfff << shift)) | (mask << shift); 3131 IGC_WRITE_REG(&sc->hw, reg, val); 3132 return (0); 3133 } 3134 3135 /* 3136 * Set flow control using sysctl: 3137 * Flow control values: 3138 * 0 - off 3139 * 1 - rx pause 3140 * 2 - tx pause 3141 * 3 - full 3142 */ 3143 static int 3144 igc_set_flowcntl(SYSCTL_HANDLER_ARGS) 3145 { 3146 int error; 3147 static int input = 3; /* default is full */ 3148 struct igc_softc *sc = (struct igc_softc *) arg1; 3149 3150 error = sysctl_handle_int(oidp, &input, 0, req); 3151 3152 if ((error) || (req->newptr == NULL)) 3153 return (error); 3154 3155 if (input == sc->fc) /* no change? */ 3156 return (error); 3157 3158 switch (input) { 3159 case igc_fc_rx_pause: 3160 case igc_fc_tx_pause: 3161 case igc_fc_full: 3162 case igc_fc_none: 3163 sc->hw.fc.requested_mode = input; 3164 sc->fc = input; 3165 break; 3166 default: 3167 /* Do nothing */ 3168 return (error); 3169 } 3170 3171 sc->hw.fc.current_mode = sc->hw.fc.requested_mode; 3172 igc_force_mac_fc(&sc->hw); 3173 return (error); 3174 } 3175 3176 /* 3177 * Manage DMA Coalesce: 3178 * Control values: 3179 * 0/1 - off/on 3180 * Legal timer values are: 3181 * 250,500,1000-10000 in thousands 3182 */ 3183 static int 3184 igc_sysctl_dmac(SYSCTL_HANDLER_ARGS) 3185 { 3186 struct igc_softc *sc = (struct igc_softc *) arg1; 3187 int error; 3188 3189 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 3190 3191 if ((error) || (req->newptr == NULL)) 3192 return (error); 3193 3194 switch (sc->dmac) { 3195 case 0: 3196 /* Disabling */ 3197 break; 3198 case 1: /* Just enable and use default */ 3199 sc->dmac = 1000; 3200 break; 3201 case 250: 3202 case 500: 3203 case 1000: 3204 case 2000: 3205 case 3000: 3206 case 4000: 3207 case 5000: 3208 case 6000: 3209 case 7000: 3210 case 8000: 3211 case 9000: 3212 case 10000: 3213 /* Legal values - allow */ 3214 break; 3215 default: 3216 /* Do nothing, illegal value */ 3217 sc->dmac = 0; 3218 return (EINVAL); 3219 } 3220 /* Reinit the interface */ 3221 igc_if_init(sc->ctx); 3222 return (error); 3223 } 3224 3225 /* 3226 * Manage Energy Efficient Ethernet: 3227 * Control values: 3228 * 0/1 - enabled/disabled 3229 */ 3230 static int 3231 igc_sysctl_eee(SYSCTL_HANDLER_ARGS) 3232 { 3233 struct igc_softc *sc = (struct igc_softc *) arg1; 3234 int error, value; 3235 3236 value = sc->hw.dev_spec._i225.eee_disable; 3237 error = sysctl_handle_int(oidp, &value, 0, req); 3238 if (error || req->newptr == NULL) 3239 return (error); 3240 3241 sc->hw.dev_spec._i225.eee_disable = (value != 0); 3242 igc_if_init(sc->ctx); 3243 3244 return (0); 3245 } 3246 3247 static int 3248 igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 3249 { 3250 struct igc_softc *sc; 3251 int error; 3252 int result; 3253 3254 result = -1; 3255 error = sysctl_handle_int(oidp, &result, 0, req); 3256 3257 if (error || !req->newptr) 3258 return (error); 3259 3260 if (result == 1) { 3261 sc = (struct igc_softc *) arg1; 3262 igc_print_debug_info(sc); 3263 } 3264 3265 return (error); 3266 } 3267 3268 static int 3269 igc_get_rs(SYSCTL_HANDLER_ARGS) 3270 { 3271 struct igc_softc *sc = (struct igc_softc *) arg1; 3272 int error; 3273 int result; 3274 3275 result = 0; 3276 error = sysctl_handle_int(oidp, &result, 0, req); 3277 3278 if (error || !req->newptr || result != 1) 3279 return (error); 3280 igc_dump_rs(sc); 3281 3282 return (error); 3283 } 3284 3285 static void 3286 igc_if_debug(if_ctx_t ctx) 3287 { 3288 igc_dump_rs(iflib_get_softc(ctx)); 3289 } 3290 3291 /* 3292 * This routine is meant to be fluid, add whatever is 3293 * needed for debugging a problem. -jfv 3294 */ 3295 static void 3296 igc_print_debug_info(struct igc_softc *sc) 3297 { 3298 device_t dev = iflib_get_dev(sc->ctx); 3299 if_t ifp = iflib_get_ifp(sc->ctx); 3300 struct tx_ring *txr = &sc->tx_queues->txr; 3301 struct rx_ring *rxr = &sc->rx_queues->rxr; 3302 3303 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3304 printf("Interface is RUNNING "); 3305 else 3306 printf("Interface is NOT RUNNING\n"); 3307 3308 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) 3309 printf("and INACTIVE\n"); 3310 else 3311 printf("and ACTIVE\n"); 3312 3313 for (int i = 0; i < sc->tx_num_queues; i++, txr++) { 3314 device_printf(dev, "TX Queue %d ------\n", i); 3315 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 3316 IGC_READ_REG(&sc->hw, IGC_TDH(i)), 3317 IGC_READ_REG(&sc->hw, IGC_TDT(i))); 3318 3319 } 3320 for (int j=0; j < sc->rx_num_queues; j++, rxr++) { 3321 device_printf(dev, "RX Queue %d ------\n", j); 3322 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 3323 IGC_READ_REG(&sc->hw, IGC_RDH(j)), 3324 IGC_READ_REG(&sc->hw, IGC_RDT(j))); 3325 } 3326 } 3327