1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2021-2024 Rubicon Communications, LLC (Netgate) 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "if_igc.h" 32 #include <sys/sbuf.h> 33 #include <machine/_inttypes.h> 34 35 #include <net/rss_config.h> 36 #include <netinet/in_rss.h> 37 38 39 /********************************************************************* 40 * PCI Device ID Table 41 * 42 * Used by probe to select devices to load on 43 * Last entry must be all 0s 44 * 45 * { Vendor ID, Device ID, String } 46 *********************************************************************/ 47 48 static const pci_vendor_info_t igc_vendor_info_array[] = 49 { 50 /* Intel(R) PRO/1000 Network Connection - igc */ 51 PVID(0x8086, IGC_DEV_ID_I225_LM, 52 "Intel(R) Ethernet Controller I225-LM"), 53 PVID(0x8086, IGC_DEV_ID_I225_V, 54 "Intel(R) Ethernet Controller I225-V"), 55 PVID(0x8086, IGC_DEV_ID_I225_K, 56 "Intel(R) Ethernet Controller I225-K"), 57 PVID(0x8086, IGC_DEV_ID_I225_I, 58 "Intel(R) Ethernet Controller I225-IT"), 59 PVID(0x8086, IGC_DEV_ID_I220_V, 60 "Intel(R) Ethernet Controller I220-V"), 61 PVID(0x8086, IGC_DEV_ID_I225_K2, 62 "Intel(R) Ethernet Controller I225-K(2)"), 63 PVID(0x8086, IGC_DEV_ID_I225_LMVP, 64 "Intel(R) Ethernet Controller I225-LMvP(2)"), 65 PVID(0x8086, IGC_DEV_ID_I226_K, 66 "Intel(R) Ethernet Controller I226-K"), 67 PVID(0x8086, IGC_DEV_ID_I226_LMVP, 68 "Intel(R) Ethernet Controller I226-LMvP"), 69 PVID(0x8086, IGC_DEV_ID_I225_IT, 70 "Intel(R) Ethernet Controller I225-IT(2)"), 71 PVID(0x8086, IGC_DEV_ID_I226_LM, 72 "Intel(R) Ethernet Controller I226-LM"), 73 PVID(0x8086, IGC_DEV_ID_I226_V, 74 "Intel(R) Ethernet Controller I226-V"), 75 PVID(0x8086, IGC_DEV_ID_I226_IT, 76 "Intel(R) Ethernet Controller I226-IT"), 77 PVID(0x8086, IGC_DEV_ID_I221_V, 78 "Intel(R) Ethernet Controller I221-V"), 79 PVID(0x8086, IGC_DEV_ID_I226_BLANK_NVM, 80 "Intel(R) Ethernet Controller I226(blankNVM)"), 81 PVID(0x8086, IGC_DEV_ID_I225_BLANK_NVM, 82 "Intel(R) Ethernet Controller I225(blankNVM)"), 83 /* required last entry */ 84 PVID_END 85 }; 86 87 /********************************************************************* 88 * Function prototypes 89 *********************************************************************/ 90 static void *igc_register(device_t); 91 static int igc_if_attach_pre(if_ctx_t); 92 static int igc_if_attach_post(if_ctx_t); 93 static int igc_if_detach(if_ctx_t); 94 static int igc_if_shutdown(if_ctx_t); 95 static int igc_if_suspend(if_ctx_t); 96 static int igc_if_resume(if_ctx_t); 97 98 static int igc_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 99 int); 100 static int igc_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, 101 int); 102 static void igc_if_queues_free(if_ctx_t); 103 104 static uint64_t igc_if_get_counter(if_ctx_t, ift_counter); 105 static void igc_if_init(if_ctx_t); 106 static void igc_if_stop(if_ctx_t); 107 static void igc_if_media_status(if_ctx_t, struct ifmediareq *); 108 static int igc_if_media_change(if_ctx_t); 109 static int igc_if_mtu_set(if_ctx_t, uint32_t); 110 static void igc_if_timer(if_ctx_t, uint16_t); 111 static void igc_if_vlan_register(if_ctx_t, u16); 112 static void igc_if_vlan_unregister(if_ctx_t, u16); 113 static bool igc_if_needs_restart(if_ctx_t, enum iflib_restart_event); 114 115 static void igc_identify_hardware(if_ctx_t); 116 static int igc_allocate_pci_resources(if_ctx_t); 117 static void igc_free_pci_resources(if_ctx_t); 118 static void igc_disable_broken_l1_2(if_ctx_t); 119 static void igc_reset(if_ctx_t); 120 static int igc_setup_interface(if_ctx_t); 121 static int igc_setup_msix(if_ctx_t); 122 123 static void igc_initialize_transmit_unit(if_ctx_t); 124 static void igc_initialize_receive_unit(if_ctx_t); 125 126 static void igc_if_intr_enable(if_ctx_t); 127 static void igc_if_intr_disable(if_ctx_t); 128 static int igc_if_rx_queue_intr_enable(if_ctx_t, uint16_t); 129 static int igc_if_tx_queue_intr_enable(if_ctx_t, uint16_t); 130 static void igc_handle_fatal_error_intr(struct igc_softc *, u32); 131 static bool igc_handle_fatal_error_admin(struct igc_softc *); 132 static void igc_prepare_fatal_error_reset(struct igc_softc *); 133 static void igc_finish_fatal_error_reset(struct igc_softc *); 134 static void igc_if_multi_set(if_ctx_t); 135 static void igc_if_update_admin_status(if_ctx_t); 136 static void igc_apply_i225_ipg_workaround(struct igc_softc *); 137 static void igc_if_debug(if_ctx_t); 138 static void igc_update_ecc_stats(struct igc_softc *); 139 static void igc_update_stats_counters(struct igc_softc *); 140 static void igc_add_hw_stats(struct igc_softc *); 141 static int igc_if_set_promisc(if_ctx_t, int); 142 static bool igc_if_vlan_filter_capable(if_ctx_t); 143 static bool igc_if_vlan_filter_used(if_ctx_t); 144 static void igc_if_vlan_filter_enable(struct igc_softc *); 145 static void igc_if_vlan_filter_disable(struct igc_softc *); 146 static void igc_setup_vlan_hw_support(if_ctx_t); 147 static void igc_if_led_func(if_ctx_t, int); 148 static void igc_led_restore(struct igc_softc *); 149 static void igc_fw_version(struct igc_softc *); 150 static void igc_sbuf_fw_version(struct igc_fw_version *, struct sbuf *); 151 static void igc_print_fw_version(struct igc_softc *); 152 static int igc_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS); 153 static int igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 154 static void igc_print_nvm_info(struct igc_softc *); 155 static int igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 156 static int igc_get_rs(SYSCTL_HANDLER_ARGS); 157 static void igc_print_debug_info(struct igc_softc *); 158 static int igc_is_valid_ether_addr(u8 *); 159 static void igc_neweitr(struct igc_softc *, struct igc_rx_queue *, 160 struct rx_ring *); 161 static int igc_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS); 162 /* Management and WOL Support */ 163 static void igc_get_hw_control(struct igc_softc *); 164 static void igc_release_hw_control(struct igc_softc *); 165 static void igc_get_wakeup(if_ctx_t); 166 static void igc_enable_wakeup(if_ctx_t); 167 168 int igc_intr(void *); 169 170 enum igc_fatal_error_state { 171 IGC_FATAL_ERROR_NONE, 172 IGC_FATAL_ERROR_CAPTURING, 173 IGC_FATAL_ERROR_DETECTED, 174 IGC_FATAL_ERROR_RESET_REQUESTED, 175 }; 176 177 /* MSI-X handlers */ 178 static int igc_if_msix_intr_assign(if_ctx_t, int); 179 static int igc_msix_link(void *); 180 static void igc_handle_link(void *context); 181 182 static int igc_set_flowcntl(SYSCTL_HANDLER_ARGS); 183 static int igc_sysctl_dmac(SYSCTL_HANDLER_ARGS); 184 static int igc_sysctl_eee(SYSCTL_HANDLER_ARGS); 185 186 static int igc_get_regs(SYSCTL_HANDLER_ARGS); 187 188 static void igc_configure_queues(struct igc_softc *); 189 static void igc_initialize_interrupt_rate(struct igc_softc *); 190 191 192 /********************************************************************* 193 * FreeBSD Device Interface Entry Points 194 *********************************************************************/ 195 static device_method_t igc_methods[] = { 196 /* Device interface */ 197 DEVMETHOD(device_register, igc_register), 198 DEVMETHOD(device_probe, iflib_device_probe), 199 DEVMETHOD(device_attach, iflib_device_attach), 200 DEVMETHOD(device_detach, iflib_device_detach), 201 DEVMETHOD(device_shutdown, iflib_device_shutdown), 202 DEVMETHOD(device_suspend, iflib_device_suspend), 203 DEVMETHOD(device_resume, iflib_device_resume), 204 DEVMETHOD_END 205 }; 206 207 static driver_t igc_driver = { 208 "igc", igc_methods, sizeof(struct igc_softc), 209 }; 210 211 DRIVER_MODULE(igc, pci, igc_driver, 0, 0); 212 213 MODULE_DEPEND(igc, pci, 1, 1, 1); 214 MODULE_DEPEND(igc, ether, 1, 1, 1); 215 MODULE_DEPEND(igc, iflib, 1, 1, 1); 216 217 IFLIB_PNP_INFO(pci, igc, igc_vendor_info_array); 218 219 static device_method_t igc_if_methods[] = { 220 DEVMETHOD(ifdi_attach_pre, igc_if_attach_pre), 221 DEVMETHOD(ifdi_attach_post, igc_if_attach_post), 222 DEVMETHOD(ifdi_detach, igc_if_detach), 223 DEVMETHOD(ifdi_shutdown, igc_if_shutdown), 224 DEVMETHOD(ifdi_suspend, igc_if_suspend), 225 DEVMETHOD(ifdi_resume, igc_if_resume), 226 DEVMETHOD(ifdi_init, igc_if_init), 227 DEVMETHOD(ifdi_stop, igc_if_stop), 228 DEVMETHOD(ifdi_msix_intr_assign, igc_if_msix_intr_assign), 229 DEVMETHOD(ifdi_intr_enable, igc_if_intr_enable), 230 DEVMETHOD(ifdi_intr_disable, igc_if_intr_disable), 231 DEVMETHOD(ifdi_tx_queues_alloc, igc_if_tx_queues_alloc), 232 DEVMETHOD(ifdi_rx_queues_alloc, igc_if_rx_queues_alloc), 233 DEVMETHOD(ifdi_queues_free, igc_if_queues_free), 234 DEVMETHOD(ifdi_update_admin_status, igc_if_update_admin_status), 235 DEVMETHOD(ifdi_multi_set, igc_if_multi_set), 236 DEVMETHOD(ifdi_media_status, igc_if_media_status), 237 DEVMETHOD(ifdi_media_change, igc_if_media_change), 238 DEVMETHOD(ifdi_mtu_set, igc_if_mtu_set), 239 DEVMETHOD(ifdi_promisc_set, igc_if_set_promisc), 240 DEVMETHOD(ifdi_timer, igc_if_timer), 241 DEVMETHOD(ifdi_vlan_register, igc_if_vlan_register), 242 DEVMETHOD(ifdi_vlan_unregister, igc_if_vlan_unregister), 243 DEVMETHOD(ifdi_get_counter, igc_if_get_counter), 244 DEVMETHOD(ifdi_rx_queue_intr_enable, igc_if_rx_queue_intr_enable), 245 DEVMETHOD(ifdi_tx_queue_intr_enable, igc_if_tx_queue_intr_enable), 246 DEVMETHOD(ifdi_debug, igc_if_debug), 247 DEVMETHOD(ifdi_needs_restart, igc_if_needs_restart), 248 DEVMETHOD(ifdi_led_func, igc_if_led_func), 249 DEVMETHOD_END 250 }; 251 252 static driver_t igc_if_driver = { 253 "igc_if", igc_if_methods, sizeof(struct igc_softc) 254 }; 255 256 /********************************************************************* 257 * Tunable default values. 258 *********************************************************************/ 259 260 /* Allow common code without TSO */ 261 #ifndef CSUM_TSO 262 #define CSUM_TSO 0 263 #endif 264 265 static SYSCTL_NODE(_hw, OID_AUTO, igc, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 266 "igc driver parameters"); 267 268 static int igc_disable_crc_stripping = 0; 269 SYSCTL_INT(_hw_igc, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN, 270 &igc_disable_crc_stripping, 0, "Disable CRC Stripping"); 271 272 static int igc_smart_pwr_down = false; 273 SYSCTL_INT(_hw_igc, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, 274 &igc_smart_pwr_down, 275 0, "Set to true to leave smart power down enabled on newer adapters"); 276 277 /* Controls whether promiscuous also shows bad packets */ 278 static int igc_debug_sbp = false; 279 SYSCTL_INT(_hw_igc, OID_AUTO, sbp, CTLFLAG_RDTUN, &igc_debug_sbp, 0, 280 "Show bad packets in promiscuous mode"); 281 282 /* Energy efficient ethernet - default to OFF */ 283 static int igc_eee_setting = 1; 284 SYSCTL_INT(_hw_igc, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &igc_eee_setting, 0, 285 "Enable Energy Efficient Ethernet"); 286 287 /* 288 * AIM: Adaptive Interrupt Moderation 289 * which means that the interrupt rate is varied over time based on the 290 * traffic for that interrupt vector 291 */ 292 static int igc_enable_aim = 1; 293 SYSCTL_INT(_hw_igc, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igc_enable_aim, 294 0, "Enable adaptive interrupt moderation (1=normal, 2=lowlatency)"); 295 296 /* 297 ** Tuneable Interrupt rate 298 */ 299 static int igc_max_interrupt_rate = IGC_INTS_DEFAULT; 300 SYSCTL_INT(_hw_igc, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 301 &igc_max_interrupt_rate, 0, "Maximum interrupts per second"); 302 303 extern struct if_txrx igc_txrx; 304 305 static struct if_shared_ctx igc_sctx_init = { 306 .isc_magic = IFLIB_MAGIC, 307 .isc_q_align = PAGE_SIZE, 308 .isc_tx_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header), 309 .isc_tx_maxsegsize = PAGE_SIZE, 310 .isc_tso_maxsize = IGC_TSO_SIZE + sizeof(struct ether_vlan_header), 311 .isc_tso_maxsegsize = IGC_TSO_SEG_SIZE, 312 .isc_rx_maxsize = MAX_JUMBO_FRAME_SIZE, 313 .isc_rx_nsegments = 1, 314 .isc_rx_maxsegsize = MJUM9BYTES, 315 .isc_nfl = 1, 316 .isc_nrxqs = 1, 317 .isc_ntxqs = 1, 318 .isc_admin_intrcnt = 1, 319 .isc_vendor_info = igc_vendor_info_array, 320 .isc_driver_version = "1", 321 .isc_driver = &igc_if_driver, 322 .isc_flags = 323 IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM, 324 325 .isc_nrxd_min = {IGC_MIN_RXD}, 326 .isc_ntxd_min = {IGC_MIN_TXD}, 327 .isc_nrxd_max = {IGC_MAX_RXD}, 328 .isc_ntxd_max = {IGC_MAX_TXD}, 329 .isc_nrxd_default = {IGC_DEFAULT_RXD}, 330 .isc_ntxd_default = {IGC_DEFAULT_TXD}, 331 }; 332 333 /***************************************************************** 334 * 335 * Dump Registers 336 * 337 ****************************************************************/ 338 #define IGC_REGS_LEN 739 339 340 static int igc_get_regs(SYSCTL_HANDLER_ARGS) 341 { 342 struct igc_softc *sc = (struct igc_softc *)arg1; 343 struct igc_hw *hw = &sc->hw; 344 struct sbuf *sb; 345 u32 *regs_buff; 346 int rc; 347 348 regs_buff = malloc(sizeof(u32) * IGC_REGS_LEN, M_DEVBUF, M_WAITOK); 349 memset(regs_buff, 0, IGC_REGS_LEN * sizeof(u32)); 350 351 rc = sysctl_wire_old_buffer(req, 0); 352 MPASS(rc == 0); 353 if (rc != 0) { 354 free(regs_buff, M_DEVBUF); 355 return (rc); 356 } 357 358 sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); 359 MPASS(sb != NULL); 360 if (sb == NULL) { 361 free(regs_buff, M_DEVBUF); 362 return (ENOMEM); 363 } 364 365 /* General Registers */ 366 regs_buff[0] = IGC_READ_REG(hw, IGC_CTRL); 367 regs_buff[1] = IGC_READ_REG(hw, IGC_STATUS); 368 regs_buff[2] = IGC_READ_REG(hw, IGC_CTRL_EXT); 369 regs_buff[3] = IGC_READ_REG(hw, IGC_ICR); 370 regs_buff[4] = IGC_READ_REG(hw, IGC_RCTL); 371 regs_buff[5] = IGC_READ_REG(hw, IGC_RDLEN(0)); 372 regs_buff[6] = IGC_READ_REG(hw, IGC_RDH(0)); 373 regs_buff[7] = IGC_READ_REG(hw, IGC_RDT(0)); 374 regs_buff[8] = IGC_READ_REG(hw, IGC_RXDCTL(0)); 375 regs_buff[9] = IGC_READ_REG(hw, IGC_RDBAL(0)); 376 regs_buff[10] = IGC_READ_REG(hw, IGC_RDBAH(0)); 377 regs_buff[11] = IGC_READ_REG(hw, IGC_TCTL); 378 regs_buff[12] = IGC_READ_REG(hw, IGC_TDBAL(0)); 379 regs_buff[13] = IGC_READ_REG(hw, IGC_TDBAH(0)); 380 regs_buff[14] = IGC_READ_REG(hw, IGC_TDLEN(0)); 381 regs_buff[15] = IGC_READ_REG(hw, IGC_TDH(0)); 382 regs_buff[16] = IGC_READ_REG(hw, IGC_TDT(0)); 383 regs_buff[17] = IGC_READ_REG(hw, IGC_TXDCTL(0)); 384 385 sbuf_printf(sb, "General Registers\n"); 386 sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); 387 sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); 388 sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]); 389 390 sbuf_printf(sb, "Interrupt Registers\n"); 391 sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); 392 393 sbuf_printf(sb, "RX Registers\n"); 394 sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); 395 sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); 396 sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); 397 sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); 398 sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); 399 sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); 400 sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); 401 402 sbuf_printf(sb, "TX Registers\n"); 403 sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); 404 sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); 405 sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); 406 sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); 407 sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); 408 sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); 409 sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); 410 sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); 411 sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); 412 sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); 413 sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); 414 415 free(regs_buff, M_DEVBUF); 416 417 #ifdef DUMP_DESCS 418 { 419 if_softc_ctx_t scctx = sc->shared; 420 struct rx_ring *rxr = &rx_que->rxr; 421 struct tx_ring *txr = &tx_que->txr; 422 int ntxd = scctx->isc_ntxd[0]; 423 int nrxd = scctx->isc_nrxd[0]; 424 int j; 425 426 for (j = 0; j < nrxd; j++) { 427 u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); 428 u32 length = le32toh(rxr->rx_base[j].wb.upper.length); 429 sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" 430 PRIx64 " Error:%d Length:%d\n", 431 j, rxr->rx_base[j].read.buffer_addr, staterr, length); 432 } 433 434 for (j = 0; j < min(ntxd, 256); j++) { 435 unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; 436 437 sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x" 438 "[3]: %08x eop: %d DD=%d\n", 439 j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, 440 buf->eop != -1 ? 441 txr->tx_base[buf->eop].upper.fields.status & 442 IGC_TXD_STAT_DD : 0); 443 444 } 445 } 446 #endif 447 448 rc = sbuf_finish(sb); 449 sbuf_delete(sb); 450 return(rc); 451 } 452 453 static void * 454 igc_register(device_t dev) 455 { 456 return (&igc_sctx_init); 457 } 458 459 static int 460 igc_set_num_queues(if_ctx_t ctx) 461 { 462 int maxqueues; 463 464 maxqueues = 4; 465 466 return (maxqueues); 467 } 468 469 #define IGC_CAPS \ 470 IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \ 471 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL | IFCAP_TSO4 | \ 472 IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | \ 473 IFCAP_HWCSUM_IPV6 | IFCAP_TSO6 474 475 /********************************************************************* 476 * Device initialization routine 477 * 478 * The attach entry point is called when the driver is being loaded. 479 * This routine identifies the type of hardware, allocates all resources 480 * and initializes the hardware. 481 * 482 * return 0 on success, positive on failure 483 *********************************************************************/ 484 static int 485 igc_if_attach_pre(if_ctx_t ctx) 486 { 487 struct igc_softc *sc; 488 if_softc_ctx_t scctx; 489 device_t dev; 490 struct igc_hw *hw; 491 int error = 0; 492 493 INIT_DEBUGOUT("igc_if_attach_pre: begin"); 494 dev = iflib_get_dev(ctx); 495 sc = iflib_get_softc(ctx); 496 497 if (igc_max_interrupt_rate <= 0) { 498 device_printf(dev, 499 "Invalid max_interrupt_rate %d; using default %d\n", 500 igc_max_interrupt_rate, IGC_INTS_DEFAULT); 501 igc_max_interrupt_rate = IGC_INTS_DEFAULT; 502 } 503 504 sc->ctx = sc->osdep.ctx = ctx; 505 sc->dev = sc->osdep.dev = dev; 506 scctx = sc->shared = iflib_get_softc_ctx(ctx); 507 sc->media = iflib_get_media(ctx); 508 hw = &sc->hw; 509 510 /* SYSCTL stuff */ 511 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 512 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 513 OID_AUTO, "nvm", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 514 sc, 0, igc_sysctl_nvm_info, "I", "NVM Information"); 515 516 sc->enable_aim = igc_enable_aim; 517 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 518 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 519 OID_AUTO, "enable_aim", CTLFLAG_RW, 520 &sc->enable_aim, 0, 521 "Interrupt Moderation (1=normal, 2=lowlatency)"); 522 523 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 524 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 525 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, 526 sc, 0, igc_sysctl_print_fw_version, "A", 527 "Prints FW/NVM Versions"); 528 529 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 530 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 531 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 532 sc, 0, igc_sysctl_debug_info, "I", "Debug Information"); 533 534 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 535 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 536 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 537 sc, 0, igc_set_flowcntl, "I", "Flow Control"); 538 539 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 540 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 541 OID_AUTO, "reg_dump", 542 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 543 igc_get_regs, "A", "Dump Registers"); 544 545 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 546 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 547 OID_AUTO, "rs_dump", 548 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, sc, 0, 549 igc_get_rs, "I", "Dump RS indexes"); 550 551 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 552 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 553 OID_AUTO, "dmac", 554 CTLTYPE_INT | CTLFLAG_RW, sc, 0, 555 igc_sysctl_dmac, "I", "DMA Coalesce"); 556 557 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 558 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 559 OID_AUTO, "tso_tcp_flags_mask_first_segment", 560 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 561 sc, 0, igc_sysctl_tso_tcp_flags_mask, "IU", 562 "TSO TCP flags mask for first segment"); 563 564 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 565 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 566 OID_AUTO, "tso_tcp_flags_mask_middle_segment", 567 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 568 sc, 1, igc_sysctl_tso_tcp_flags_mask, "IU", 569 "TSO TCP flags mask for middle segment"); 570 571 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 572 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 573 OID_AUTO, "tso_tcp_flags_mask_last_segment", 574 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 575 sc, 2, igc_sysctl_tso_tcp_flags_mask, "IU", 576 "TSO TCP flags mask for last segment"); 577 578 /* Determine hardware and mac info */ 579 igc_identify_hardware(ctx); 580 581 /* Apply device-specific PCIe L1.2 errata workarounds. */ 582 igc_disable_broken_l1_2(ctx); 583 584 scctx->isc_tx_nsegments = IGC_MAX_SCATTER; 585 scctx->isc_nrxqsets_max = 586 scctx->isc_ntxqsets_max = igc_set_num_queues(ctx); 587 if (bootverbose) 588 device_printf(dev, "attach_pre capping queues at %d\n", 589 scctx->isc_ntxqsets_max); 590 591 scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * 592 sizeof(union igc_adv_tx_desc), IGC_DBA_ALIGN); 593 scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * 594 sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 595 scctx->isc_txd_size[0] = sizeof(union igc_adv_tx_desc); 596 scctx->isc_rxd_size[0] = sizeof(union igc_adv_rx_desc); 597 scctx->isc_txrx = &igc_txrx; 598 scctx->isc_tx_tso_segments_max = IGC_MAX_SCATTER; 599 scctx->isc_tx_tso_size_max = IGC_TSO_SIZE; 600 scctx->isc_tx_tso_segsize_max = IGC_TSO_SEG_SIZE; 601 scctx->isc_capabilities = scctx->isc_capenable = IGC_CAPS; 602 scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | 603 CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_SCTP | CSUM_IP6_SCTP; 604 605 /* 606 ** Some new devices, as with ixgbe, now may 607 ** use a different BAR, so we need to keep 608 ** track of which is used. 609 */ 610 scctx->isc_msix_bar = PCIR_BAR(IGC_MSIX_BAR); 611 if (pci_read_config(dev, scctx->isc_msix_bar, 4) == 0) 612 scctx->isc_msix_bar += 4; 613 614 /* Setup PCI resources */ 615 if (igc_allocate_pci_resources(ctx)) { 616 device_printf(dev, "Allocation of PCI resources failed\n"); 617 error = ENXIO; 618 goto err_pci; 619 } 620 621 /* Do Shared Code initialization */ 622 error = igc_setup_init_funcs(hw, true); 623 if (error) { 624 device_printf(dev, "Setup of Shared code failed, error %d\n", 625 error); 626 error = ENXIO; 627 goto err_pci; 628 } 629 630 igc_setup_msix(ctx); 631 igc_get_bus_info(hw); 632 633 hw->mac.autoneg = DO_AUTO_NEG; 634 hw->phy.autoneg_wait_to_complete = false; 635 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 636 637 /* Copper options */ 638 if (hw->phy.media_type == igc_media_type_copper) { 639 hw->phy.mdix = AUTO_ALL_MODES; 640 } 641 642 /* 643 * Set the frame limits assuming 644 * standard ethernet sized frames. 645 */ 646 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 647 ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 648 649 /* Allocate multicast array memory. */ 650 sc->mta = malloc(sizeof(u8) * ETHER_ADDR_LEN * 651 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 652 if (sc->mta == NULL) { 653 device_printf(dev, 654 "Can not allocate multicast setup array\n"); 655 error = ENOMEM; 656 goto err_late; 657 } 658 659 /* Check SOL/IDER usage */ 660 if (igc_check_reset_block(hw)) 661 device_printf(dev, "PHY reset is blocked" 662 " due to SOL/IDER session.\n"); 663 664 /* Sysctl for setting Energy Efficient Ethernet */ 665 sc->hw.dev_spec._i225.eee_disable = igc_eee_setting; 666 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 667 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 668 OID_AUTO, "eee_control", 669 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 670 sc, 0, igc_sysctl_eee, "I", 671 "Disable Energy Efficient Ethernet"); 672 673 /* 674 ** Start from a known state, this is 675 ** important in reading the nvm and 676 ** mac from that. 677 */ 678 igc_reset_hw(hw); 679 680 /* Make sure we have a good EEPROM before we read from it */ 681 if (igc_validate_nvm_checksum(hw) < 0) { 682 /* 683 ** Some PCI-E parts fail the first check due to 684 ** the link being in sleep state, call it again, 685 ** if it fails a second time its a real issue. 686 */ 687 if (igc_validate_nvm_checksum(hw) < 0) { 688 device_printf(dev, 689 "The EEPROM Checksum Is Not Valid\n"); 690 error = EIO; 691 goto err_late; 692 } 693 } 694 695 /* Copy the permanent MAC address out of the EEPROM */ 696 if (igc_read_mac_addr(hw) < 0) { 697 device_printf(dev, "EEPROM read error while reading MAC" 698 " address\n"); 699 error = EIO; 700 goto err_late; 701 } 702 703 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 704 device_printf(dev, "Invalid MAC address\n"); 705 error = EIO; 706 goto err_late; 707 } 708 709 /* Save the EEPROM/NVM versions */ 710 igc_fw_version(sc); 711 712 igc_print_fw_version(sc); 713 714 /* 715 * Get Wake-on-Lan and Management info for later use 716 */ 717 igc_get_wakeup(ctx); 718 719 /* Enable only WOL MAGIC by default */ 720 scctx->isc_capenable &= ~IFCAP_WOL; 721 if (sc->wol != 0) 722 scctx->isc_capenable |= IFCAP_WOL_MAGIC; 723 724 iflib_set_mac(ctx, hw->mac.addr); 725 726 return (0); 727 728 err_late: 729 igc_release_hw_control(sc); 730 err_pci: 731 igc_free_pci_resources(ctx); 732 free(sc->mta, M_DEVBUF); 733 734 return (error); 735 } 736 737 static int 738 igc_if_attach_post(if_ctx_t ctx) 739 { 740 struct igc_softc *sc = iflib_get_softc(ctx); 741 struct igc_hw *hw = &sc->hw; 742 int error = 0; 743 744 /* Setup OS specific network interface */ 745 error = igc_setup_interface(ctx); 746 if (error != 0) { 747 goto err_late; 748 } 749 750 igc_reset(ctx); 751 752 /* Initialize statistics */ 753 igc_update_stats_counters(sc); 754 hw->mac.get_link_status = true; 755 igc_if_update_admin_status(ctx); 756 igc_add_hw_stats(sc); 757 758 /* the driver can now take control from firmware */ 759 igc_get_hw_control(sc); 760 761 INIT_DEBUGOUT("igc_if_attach_post: end"); 762 763 return (error); 764 765 err_late: 766 igc_release_hw_control(sc); 767 igc_free_pci_resources(ctx); 768 igc_if_queues_free(ctx); 769 free(sc->mta, M_DEVBUF); 770 771 return (error); 772 } 773 774 /********************************************************************* 775 * Device removal routine 776 * 777 * The detach entry point is called when the driver is being removed. 778 * This routine stops the adapter and deallocates all the resources 779 * that were allocated for driver operation. 780 * 781 * return 0 on success, positive on failure 782 *********************************************************************/ 783 static int 784 igc_if_detach(if_ctx_t ctx) 785 { 786 struct igc_softc *sc = iflib_get_softc(ctx); 787 788 INIT_DEBUGOUT("igc_if_detach: begin"); 789 790 igc_phy_hw_reset(&sc->hw); 791 792 igc_release_hw_control(sc); 793 igc_free_pci_resources(ctx); 794 795 return (0); 796 } 797 798 /********************************************************************* 799 * 800 * Shutdown entry point 801 * 802 **********************************************************************/ 803 804 static int 805 igc_if_shutdown(if_ctx_t ctx) 806 { 807 return igc_if_suspend(ctx); 808 } 809 810 /* 811 * Suspend/resume device methods. 812 */ 813 static int 814 igc_if_suspend(if_ctx_t ctx) 815 { 816 struct igc_softc *sc = iflib_get_softc(ctx); 817 818 igc_release_hw_control(sc); 819 igc_enable_wakeup(ctx); 820 return (0); 821 } 822 823 static int 824 igc_if_resume(if_ctx_t ctx) 825 { 826 /* 827 * PCIe config space, and with it L1.2, may have been reset 828 * across the suspend/resume cycle. 829 */ 830 igc_disable_broken_l1_2(ctx); 831 832 return(0); 833 } 834 835 static int 836 igc_if_mtu_set(if_ctx_t ctx, uint32_t mtu) 837 { 838 int max_frame_size; 839 struct igc_softc *sc = iflib_get_softc(ctx); 840 if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx); 841 842 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 843 844 /* 9K Jumbo Frame size */ 845 max_frame_size = 9234; 846 847 if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { 848 return (EINVAL); 849 } 850 851 scctx->isc_max_frame_size = sc->hw.mac.max_frame_size = 852 mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 853 return (0); 854 } 855 856 /********************************************************************* 857 * Init entry point 858 * 859 * This routine is used in two ways. It is used by the stack as 860 * init entry point in network interface structure. It is also used 861 * by the driver as a hw/sw initialization routine to get to a 862 * consistent state. 863 * 864 **********************************************************************/ 865 static void 866 igc_if_init(if_ctx_t ctx) 867 { 868 struct igc_softc *sc = iflib_get_softc(ctx); 869 if_softc_ctx_t scctx = sc->shared; 870 if_t ifp = iflib_get_ifp(ctx); 871 struct igc_tx_queue *tx_que; 872 int i; 873 874 INIT_DEBUGOUT("igc_if_init: begin"); 875 876 /* Get the latest mac address, User can use a LAA */ 877 bcopy(if_getlladdr(ifp), sc->hw.mac.addr, 878 ETHER_ADDR_LEN); 879 880 /* Put the address into the Receive Address Array */ 881 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 882 883 /* Initialize the hardware */ 884 igc_reset(ctx); 885 igc_if_update_admin_status(ctx); 886 887 for (i = 0, tx_que = sc->tx_queues; i < sc->tx_num_queues; 888 i++, tx_que++) { 889 struct tx_ring *txr = &tx_que->txr; 890 891 txr->tx_rs_cidx = txr->tx_rs_pidx; 892 893 /* Initialize the last processed descriptor to be the end of 894 * the ring, rather than the start, so that we avoid an 895 * off-by-one error when calculating how many descriptors are 896 * done in the credits_update function. 897 */ 898 txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; 899 } 900 901 /* Setup VLAN support, basic and offload if available */ 902 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 903 904 /* Prepare transmit descriptors and buffers */ 905 igc_initialize_transmit_unit(ctx); 906 907 /* Setup Multicast table */ 908 igc_if_multi_set(ctx); 909 910 sc->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx); 911 igc_initialize_receive_unit(ctx); 912 913 /* Set up VLAN support */ 914 igc_setup_vlan_hw_support(ctx); 915 916 /* Don't lose promiscuous settings */ 917 igc_if_set_promisc(ctx, if_getflags(ifp)); 918 igc_clear_hw_cntrs_base_generic(&sc->hw); 919 920 if (sc->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */ 921 igc_configure_queues(sc); 922 igc_initialize_interrupt_rate(sc); 923 924 /* this clears any pending interrupts */ 925 IGC_READ_REG(&sc->hw, IGC_ICR); 926 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 927 928 /* the driver can now take control from firmware */ 929 igc_get_hw_control(sc); 930 931 /* Set Energy Efficient Ethernet */ 932 igc_set_eee_i225(&sc->hw, true, true, true); 933 } 934 935 /* 936 * RX publishes its byte and packet counters as one snapshot when iflib 937 * returns descriptors to hardware. This also covers watchdog-driven RX 938 * processing, which can run while the interrupt vector is unmasked. 939 */ 940 static __inline void 941 igc_aim_rx_delta(struct rx_ring *rxr, u32 *bytes, u32 *packets) 942 { 943 uint64_t snapshot; 944 u32 now_bytes, now_packets; 945 946 snapshot = atomic_load_acq_64(&rxr->rx_aim_snapshot); 947 now_bytes = snapshot >> 32; 948 now_packets = (u32)snapshot; 949 *bytes = now_bytes - rxr->rx_bytes_last; 950 *packets = now_packets - rxr->rx_packets_last; 951 rxr->rx_bytes_last = now_bytes; 952 rxr->rx_packets_last = now_packets; 953 } 954 955 /* 956 * TX publishes its byte and packet counters as one snapshot at the doorbell, 957 * because encapsulation can overlap the interrupt filter. The two halves 958 * remain independent free running u32 counters, so their deltas are correct 959 * across wrap. 960 */ 961 static __inline void 962 igc_aim_tx_delta(struct tx_ring *txr, u32 *bytes, u32 *packets) 963 { 964 uint64_t snapshot; 965 u32 now_bytes, now_packets; 966 967 snapshot = atomic_load_acq_64(&txr->tx_aim_snapshot); 968 now_bytes = snapshot >> 32; 969 now_packets = (u32)snapshot; 970 *bytes = now_bytes - txr->tx_bytes_last; 971 *packets = now_packets - txr->tx_packets_last; 972 txr->tx_bytes_last = now_bytes; 973 txr->tx_packets_last = now_packets; 974 } 975 976 /********************************************************************* 977 * 978 * Do Adaptive Interrupt Moderation: 979 * - Calculate based on average size over the last interval 980 * 981 * Returns interrupts per second rather than a register value, so that the 982 * caller's IGC_INTS_TO_EITR() conversion applies, or zero if the interval 983 * carried no packet to measure. 984 * 985 *********************************************************************/ 986 static u32 987 igc_ring_itr(struct igc_softc *sc, u32 rxbytes, u32 rxpackets, u32 txbytes, 988 u32 txpackets) 989 { 990 u32 newitr = 0; 991 992 if (txbytes && txpackets) 993 newitr = txbytes / txpackets; 994 if (rxbytes && rxpackets) 995 newitr = max(newitr, rxbytes / rxpackets); 996 997 /* 998 * No packet was observed, so there is no size to work from. Report no 999 * observation and let the caller keep the rate it already has. 1000 */ 1001 if (newitr == 0) 1002 return (0); 1003 1004 newitr += 24; /* account for hardware frame, crc */ 1005 /* set an upper boundary */ 1006 newitr = min(newitr, 3000); 1007 /* Be nice to the mid range */ 1008 if ((newitr > 300) && (newitr < 1200)) 1009 newitr = (newitr / 3); 1010 else 1011 newitr = (newitr / 2); 1012 1013 /* The value above was written straight to EITR; make it a rate */ 1014 newitr = IGC_AIM_DIVIDEND / newitr; 1015 1016 /* 1017 * Cap the rate: enable_aim=1 is the normal setting, enable_aim=2 opts 1018 * into the low latency end. The original was unbounded and would ask 1019 * for ~95k ints/s on minimum sized frames. There is deliberately no 1020 * floor, so jumbo traffic settles near 2.7k ints/s. 1021 */ 1022 if (sc->enable_aim == 1) 1023 newitr = min(newitr, IGC_INTS_20K); 1024 else 1025 newitr = min(newitr, IGC_INTS_70K); 1026 1027 return (newitr); 1028 } 1029 1030 /********************************************************************* 1031 * 1032 * Helper to calculate next EITR value for AIM 1033 * 1034 *********************************************************************/ 1035 static void 1036 igc_neweitr(struct igc_softc *sc, struct igc_rx_queue *que, 1037 struct rx_ring *rxr) 1038 { 1039 struct igc_hw *hw = &sc->hw; 1040 struct igc_tx_queue *tx_que; 1041 u32 ringbytes, ringpackets, rxbytes, rxpackets, txbytes, txpackets; 1042 u32 neweitr; 1043 int i; 1044 1045 igc_aim_rx_delta(rxr, &rxbytes, &rxpackets); 1046 1047 /* 1048 * A vector can service more than one TX ring when iflib is configured 1049 * with unequal RX and TX queue counts. Sample every ring routed to 1050 * this vector rather than treating the vector as a TX queue index. 1051 */ 1052 txbytes = txpackets = 0; 1053 for (i = 0; i < sc->tx_num_queues; i++) { 1054 tx_que = &sc->tx_queues[i]; 1055 if (tx_que->msix != que->msix) 1056 continue; 1057 igc_aim_tx_delta(&tx_que->txr, &ringbytes, &ringpackets); 1058 txbytes += ringbytes; 1059 txpackets += ringpackets; 1060 } 1061 1062 /* Idle, do nothing */ 1063 if (txbytes == 0 && rxbytes == 0) 1064 return; 1065 1066 if (sc->enable_aim == 0) { 1067 neweitr = igc_max_interrupt_rate; 1068 } else if (sc->link_speed < SPEED_1000) { 1069 /* Use half default (4K) ITR if sub-gig */ 1070 neweitr = IGC_INTS_4K; 1071 } else if (sc->shared->isc_max_frame_size * 2 > (sc->pba << 10)) { 1072 /* Want at least enough packet buffer for two frames to AIM */ 1073 neweitr = igc_max_interrupt_rate; 1074 } else { 1075 neweitr = igc_ring_itr(sc, rxbytes, rxpackets, txbytes, 1076 txpackets); 1077 /* No usable observation; leave the rate where it is */ 1078 if (neweitr == 0) 1079 return; 1080 } 1081 1082 neweitr = IGC_INTS_TO_EITR(neweitr); 1083 1084 neweitr |= IGC_EITR_CNT_IGNR; 1085 1086 if (neweitr != que->eitr_setting) { 1087 que->eitr_setting = neweitr; 1088 IGC_WRITE_REG(hw, IGC_EITR(que->msix), que->eitr_setting); 1089 } 1090 } 1091 1092 /********************************************************************* 1093 * 1094 * Fast Legacy/MSI Combined Interrupt Service routine 1095 * 1096 *********************************************************************/ 1097 int 1098 igc_intr(void *arg) 1099 { 1100 struct igc_softc *sc = arg; 1101 struct igc_hw *hw = &sc->hw; 1102 struct igc_rx_queue *que = &sc->rx_queues[0]; 1103 struct rx_ring *rxr = &que->rxr; 1104 if_ctx_t ctx = sc->ctx; 1105 u32 reg_icr; 1106 1107 reg_icr = IGC_READ_REG(hw, IGC_ICR); 1108 1109 /* Hot eject? */ 1110 if (reg_icr == 0xffffffff) 1111 return FILTER_STRAY; 1112 1113 /* Definitely not our interrupt. */ 1114 if (reg_icr == 0x0) 1115 return FILTER_STRAY; 1116 1117 if ((reg_icr & IGC_ICR_INT_ASSERTED) == 0) 1118 return FILTER_STRAY; 1119 1120 /* 1121 * Only MSI-X interrupts have one-shot behavior by taking advantage 1122 * of the EIAC register. Thus, explicitly disable interrupts. This 1123 * also works around the MSI message reordering errata on certain 1124 * systems. 1125 */ 1126 IFDI_INTR_DISABLE(ctx); 1127 1128 /* Link status change */ 1129 if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) 1130 igc_handle_link(ctx); 1131 1132 if (reg_icr & IGC_ICR_RXO) 1133 sc->rx_overruns++; 1134 1135 igc_handle_fatal_error_intr(sc, reg_icr); 1136 1137 igc_neweitr(sc, que, rxr); 1138 1139 return (FILTER_SCHEDULE_THREAD); 1140 } 1141 1142 static int 1143 igc_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) 1144 { 1145 struct igc_softc *sc = iflib_get_softc(ctx); 1146 struct igc_rx_queue *rxq = &sc->rx_queues[rxqid]; 1147 1148 IGC_WRITE_REG(&sc->hw, IGC_EIMS, rxq->eims); 1149 return (0); 1150 } 1151 1152 static int 1153 igc_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) 1154 { 1155 struct igc_softc *sc = iflib_get_softc(ctx); 1156 struct igc_tx_queue *txq = &sc->tx_queues[txqid]; 1157 1158 IGC_WRITE_REG(&sc->hw, IGC_EIMS, txq->eims); 1159 return (0); 1160 } 1161 1162 /********************************************************************* 1163 * 1164 * MSI-X RX Interrupt Service routine 1165 * 1166 **********************************************************************/ 1167 static int 1168 igc_msix_que(void *arg) 1169 { 1170 struct igc_rx_queue *que = arg; 1171 struct igc_softc *sc = que->sc; 1172 struct rx_ring *rxr = &que->rxr; 1173 1174 ++que->irqs; 1175 1176 igc_neweitr(sc, que, rxr); 1177 1178 return (FILTER_SCHEDULE_THREAD); 1179 } 1180 1181 /********************************************************************* 1182 * 1183 * MSI-X Link Fast Interrupt Service routine 1184 * 1185 **********************************************************************/ 1186 static int 1187 igc_msix_link(void *arg) 1188 { 1189 struct igc_softc *sc = arg; 1190 u32 reg_icr; 1191 1192 ++sc->link_irq; 1193 MPASS(sc->hw.back != NULL); 1194 reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 1195 1196 if (reg_icr & IGC_ICR_RXO) 1197 sc->rx_overruns++; 1198 1199 if (reg_icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { 1200 igc_handle_link(sc->ctx); 1201 } 1202 igc_handle_fatal_error_intr(sc, reg_icr); 1203 1204 reg_icr = IGC_IMS_LSC; 1205 if (atomic_load_acq_32(&sc->fatal_error_state) == 1206 IGC_FATAL_ERROR_NONE) 1207 reg_icr |= IGC_IMS_FER; 1208 IGC_WRITE_REG(&sc->hw, IGC_IMS, reg_icr); 1209 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->link_mask); 1210 1211 return (FILTER_HANDLED); 1212 } 1213 1214 static void 1215 igc_handle_link(void *context) 1216 { 1217 if_ctx_t ctx = context; 1218 struct igc_softc *sc = iflib_get_softc(ctx); 1219 1220 sc->hw.mac.get_link_status = true; 1221 iflib_admin_intr_deferred(ctx); 1222 } 1223 1224 /* 1225 * Fatal internal memory errors stop some or all device traffic. Capture the 1226 * read-clear indication before handing recovery to the iflib admin task. 1227 */ 1228 static void 1229 igc_handle_fatal_error_intr(struct igc_softc *sc, u32 icr) 1230 { 1231 struct igc_hw *hw; 1232 u32 lanerr, mngerr, pcieerr, peind; 1233 1234 if ((icr & IGC_ICR_FER) == 0) 1235 return; 1236 1237 hw = &sc->hw; 1238 IGC_WRITE_REG(hw, IGC_IMC, IGC_IMS_FER); 1239 if (!atomic_cmpset_32(&sc->fatal_error_state, 1240 IGC_FATAL_ERROR_NONE, IGC_FATAL_ERROR_CAPTURING)) 1241 return; 1242 1243 peind = IGC_READ_REG(hw, IGC_PEIND) & IGC_PEIND_FATAL_MASK; 1244 pcieerr = IGC_READ_REG(hw, IGC_PCIEERRSTS) & 1245 IGC_PCIEERRSTS_FATAL_MASK; 1246 lanerr = IGC_READ_REG(hw, IGC_LANPERRSTS) & 1247 IGC_LANPERRSTS_RETX_BUF; 1248 mngerr = IGC_READ_REG(hw, IGC_MNGPARSTS) & 1249 IGC_MNGPARSTS_FATAL_MASK; 1250 if (pcieerr != 0) 1251 peind |= IGC_PEIND_PCIE_PARITY_FATAL; 1252 if (lanerr != 0) 1253 peind |= IGC_PEIND_LANPORT_PARITY_FATAL; 1254 1255 sc->fatal_error_peind = peind; 1256 sc->fatal_error_pcie = pcieerr; 1257 sc->fatal_error_lan = lanerr; 1258 sc->fatal_error_mng = mngerr; 1259 atomic_store_rel_32(&sc->fatal_error_state, 1260 IGC_FATAL_ERROR_DETECTED); 1261 iflib_admin_intr_deferred(sc->ctx); 1262 } 1263 1264 static bool 1265 igc_handle_fatal_error_admin(struct igc_softc *sc) 1266 { 1267 u32 peind; 1268 1269 if (!atomic_cmpset_acq_32(&sc->fatal_error_state, 1270 IGC_FATAL_ERROR_DETECTED, IGC_FATAL_ERROR_RESET_REQUESTED)) 1271 return (atomic_load_acq_32(&sc->fatal_error_state) != 1272 IGC_FATAL_ERROR_NONE); 1273 1274 peind = sc->fatal_error_peind; 1275 if (peind & IGC_PEIND_LANPORT_PARITY_FATAL) 1276 sc->fatal_error_lan_count++; 1277 if (peind & IGC_PEIND_MNG_PARITY_FATAL) 1278 sc->fatal_error_mng_count++; 1279 if (peind & IGC_PEIND_PCIE_PARITY_FATAL) 1280 sc->fatal_error_pcie_count++; 1281 if (peind & IGC_PEIND_DMA_PARITY_FATAL) 1282 sc->fatal_error_dma_count++; 1283 if (peind == 0) 1284 sc->fatal_error_unknown_count++; 1285 1286 device_printf(sc->dev, 1287 "fatal internal memory error: PEIND %#x, PCIEERRSTS %#x, " 1288 "LANPERRSTS %#x, MNGPARSTS %#x\n", 1289 peind, sc->fatal_error_pcie, sc->fatal_error_lan, 1290 sc->fatal_error_mng); 1291 /* Management-memory recovery is owned by management firmware. */ 1292 if (peind != 0 && (peind & IGC_PEIND_HOST_FATAL_MASK) == 0) { 1293 sc->fatal_error_peind = 0; 1294 sc->fatal_error_pcie = 0; 1295 sc->fatal_error_lan = 0; 1296 sc->fatal_error_mng = 0; 1297 atomic_store_rel_32(&sc->fatal_error_state, 1298 IGC_FATAL_ERROR_NONE); 1299 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_FER); 1300 IGC_WRITE_FLUSH(&sc->hw); 1301 return (true); 1302 } 1303 1304 device_printf(sc->dev, "requesting reset after memory error\n"); 1305 iflib_request_reset(sc->ctx); 1306 /* Re-enter the admin task so it observes the reset request. */ 1307 iflib_admin_intr_deferred(sc->ctx); 1308 return (true); 1309 } 1310 1311 /********************************************************************* 1312 * 1313 * Media Ioctl callback 1314 * 1315 * This routine is called whenever the user queries the status of 1316 * the interface using ifconfig. 1317 * 1318 **********************************************************************/ 1319 static void 1320 igc_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) 1321 { 1322 struct igc_softc *sc = iflib_get_softc(ctx); 1323 1324 INIT_DEBUGOUT("igc_if_media_status: begin"); 1325 1326 iflib_admin_intr_deferred(ctx); 1327 1328 ifmr->ifm_status = IFM_AVALID; 1329 ifmr->ifm_active = IFM_ETHER; 1330 1331 if (!sc->link_active) { 1332 return; 1333 } 1334 1335 ifmr->ifm_status |= IFM_ACTIVE; 1336 1337 switch (sc->link_speed) { 1338 case 10: 1339 ifmr->ifm_active |= IFM_10_T; 1340 break; 1341 case 100: 1342 ifmr->ifm_active |= IFM_100_TX; 1343 break; 1344 case 1000: 1345 ifmr->ifm_active |= IFM_1000_T; 1346 break; 1347 case 2500: 1348 ifmr->ifm_active |= IFM_2500_T; 1349 break; 1350 } 1351 1352 if (sc->link_duplex == FULL_DUPLEX) 1353 ifmr->ifm_active |= IFM_FDX; 1354 else 1355 ifmr->ifm_active |= IFM_HDX; 1356 } 1357 1358 /********************************************************************* 1359 * 1360 * Media Ioctl callback 1361 * 1362 * This routine is called when the user changes speed/duplex using 1363 * media/mediopt option with ifconfig. 1364 * 1365 **********************************************************************/ 1366 static int 1367 igc_if_media_change(if_ctx_t ctx) 1368 { 1369 struct igc_softc *sc = iflib_get_softc(ctx); 1370 struct ifmedia *ifm = iflib_get_media(ctx); 1371 1372 INIT_DEBUGOUT("igc_if_media_change: begin"); 1373 1374 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1375 return (EINVAL); 1376 1377 sc->hw.mac.autoneg = DO_AUTO_NEG; 1378 1379 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1380 case IFM_AUTO: 1381 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1382 break; 1383 case IFM_2500_T: 1384 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 1385 break; 1386 case IFM_1000_T: 1387 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1388 break; 1389 case IFM_100_TX: 1390 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1391 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 1392 else 1393 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 1394 break; 1395 case IFM_10_T: 1396 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1397 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 1398 else 1399 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 1400 break; 1401 default: 1402 device_printf(sc->dev, "Unsupported media type\n"); 1403 } 1404 1405 return (0); 1406 } 1407 1408 static int 1409 igc_if_set_promisc(if_ctx_t ctx, int flags) 1410 { 1411 struct igc_softc *sc = iflib_get_softc(ctx); 1412 if_t ifp = iflib_get_ifp(ctx); 1413 u32 reg_rctl; 1414 int mcnt = 0; 1415 1416 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1417 reg_rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_UPE); 1418 if (flags & IFF_ALLMULTI) 1419 mcnt = MAX_NUM_MULTICAST_ADDRESSES; 1420 else 1421 mcnt = min(if_llmaddr_count(ifp), MAX_NUM_MULTICAST_ADDRESSES); 1422 1423 /* Don't disable if in MAX groups */ 1424 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 1425 reg_rctl &= ~IGC_RCTL_MPE; 1426 1427 if (flags & IFF_PROMISC) { 1428 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE); 1429 /* Turn this on if you want to see bad packets */ 1430 if (igc_debug_sbp) 1431 reg_rctl |= IGC_RCTL_SBP; 1432 } else if (flags & IFF_ALLMULTI) { 1433 reg_rctl |= IGC_RCTL_MPE; 1434 reg_rctl &= ~IGC_RCTL_UPE; 1435 } 1436 1437 if ((flags & IFF_PROMISC) || !igc_if_vlan_filter_used(ctx)) 1438 reg_rctl &= ~IGC_RCTL_VFE; 1439 else 1440 reg_rctl |= IGC_RCTL_VFE; 1441 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1442 1443 return (0); 1444 } 1445 1446 static u_int 1447 igc_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int idx) 1448 { 1449 u8 *mta = arg; 1450 1451 if (idx == MAX_NUM_MULTICAST_ADDRESSES) 1452 return (0); 1453 1454 bcopy(LLADDR(sdl), &mta[idx * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 1455 1456 return (1); 1457 } 1458 1459 /********************************************************************* 1460 * Multicast Update 1461 * 1462 * This routine is called whenever multicast address list is updated. 1463 * 1464 **********************************************************************/ 1465 1466 static void 1467 igc_if_multi_set(if_ctx_t ctx) 1468 { 1469 struct igc_softc *sc = iflib_get_softc(ctx); 1470 if_t ifp = iflib_get_ifp(ctx); 1471 u8 *mta; /* Multicast array memory */ 1472 u32 reg_rctl = 0; 1473 int mcnt = 0; 1474 1475 IOCTL_DEBUGOUT("igc_set_multi: begin"); 1476 1477 mta = sc->mta; 1478 bzero(mta, sizeof(u8) * ETHER_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 1479 1480 mcnt = if_foreach_llmaddr(ifp, igc_copy_maddr, mta); 1481 1482 reg_rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 1483 1484 if (if_getflags(ifp) & IFF_PROMISC) { 1485 reg_rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE); 1486 /* Turn this on if you want to see bad packets */ 1487 if (igc_debug_sbp) 1488 reg_rctl |= IGC_RCTL_SBP; 1489 } else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES || 1490 if_getflags(ifp) & IFF_ALLMULTI) { 1491 reg_rctl |= IGC_RCTL_MPE; 1492 reg_rctl &= ~IGC_RCTL_UPE; 1493 } else 1494 reg_rctl &= ~(IGC_RCTL_UPE | IGC_RCTL_MPE); 1495 1496 if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) 1497 igc_update_mc_addr_list(&sc->hw, mta, mcnt); 1498 1499 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg_rctl); 1500 } 1501 1502 /********************************************************************* 1503 * Timer routine 1504 * 1505 * This routine schedules igc_if_update_admin_status() to check for 1506 * link status and to gather statistics as well as to perform some 1507 * controller-specific hardware patting. 1508 * 1509 **********************************************************************/ 1510 static void 1511 igc_if_timer(if_ctx_t ctx, uint16_t qid) 1512 { 1513 1514 if (qid != 0) 1515 return; 1516 1517 iflib_admin_intr_deferred(ctx); 1518 } 1519 1520 static void 1521 igc_apply_i225_ipg_workaround(struct igc_softc *sc) 1522 { 1523 struct igc_hw *hw = &sc->hw; 1524 u32 ipgt, tipg; 1525 1526 /* 1527 * I225 v1 cannot receive the minimum IPG required at 2.5 Gb/s. 1528 * Intel's documented back-to-back workaround is for the transmitter 1529 * to use a 15-byte IPG instead of 12 bytes. I225 v2 and later have 1530 * the receive-side fix and should retain the standard IPG. 1531 */ 1532 if (!igc_is_device_id_i225(hw) || 1533 hw->revision_id >= IGC_REVISION_2) 1534 return; 1535 1536 ipgt = sc->link_speed == SPEED_2500 ? IGC_I225_TIPG_IPGT_2P5 : 1537 DEFAULT_82543_TIPG_IPGT_COPPER; 1538 tipg = IGC_READ_REG(hw, IGC_TIPG); 1539 if ((tipg & IGC_TIPG_IPGT_MASK) == ipgt) 1540 return; 1541 1542 tipg &= ~IGC_TIPG_IPGT_MASK; 1543 tipg |= ipgt; 1544 IGC_WRITE_REG(hw, IGC_TIPG, tipg); 1545 } 1546 1547 static void 1548 igc_if_update_admin_status(if_ctx_t ctx) 1549 { 1550 struct igc_softc *sc = iflib_get_softc(ctx); 1551 struct igc_hw *hw = &sc->hw; 1552 device_t dev = iflib_get_dev(ctx); 1553 u32 link_check, thstat, ctrl; 1554 1555 if (igc_handle_fatal_error_admin(sc)) 1556 return; 1557 1558 link_check = thstat = ctrl = 0; 1559 /* Get the cached link value or read phy for real */ 1560 switch (hw->phy.media_type) { 1561 case igc_media_type_copper: 1562 if (hw->mac.get_link_status == true) { 1563 /* Do the work to read phy */ 1564 igc_check_for_link(hw); 1565 link_check = !hw->mac.get_link_status; 1566 } else 1567 link_check = true; 1568 break; 1569 case igc_media_type_unknown: 1570 igc_check_for_link(hw); 1571 link_check = !hw->mac.get_link_status; 1572 /* FALLTHROUGH */ 1573 default: 1574 break; 1575 } 1576 1577 /* Now check for a transition */ 1578 if (link_check && (sc->link_active == 0)) { 1579 igc_get_speed_and_duplex(hw, &sc->link_speed, 1580 &sc->link_duplex); 1581 if (bootverbose) 1582 device_printf(dev, "Link is up %d Mbps %s\n", 1583 sc->link_speed, 1584 ((sc->link_duplex == FULL_DUPLEX) ? 1585 "Full Duplex" : "Half Duplex")); 1586 sc->link_active = 1; 1587 iflib_link_state_change(ctx, LINK_STATE_UP, 1588 IF_Mbps(sc->link_speed)); 1589 } else if (!link_check && (sc->link_active == 1)) { 1590 sc->link_speed = 0; 1591 sc->link_duplex = 0; 1592 sc->link_active = 0; 1593 iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); 1594 } 1595 igc_apply_i225_ipg_workaround(sc); 1596 igc_update_stats_counters(sc); 1597 } 1598 1599 /********************************************************************* 1600 * 1601 * This routine disables all traffic on the adapter by issuing a 1602 * global reset on the MAC. 1603 * 1604 **********************************************************************/ 1605 static void 1606 igc_if_stop(if_ctx_t ctx) 1607 { 1608 struct igc_softc *sc = iflib_get_softc(ctx); 1609 1610 INIT_DEBUGOUT("igc_if_stop: begin"); 1611 1612 igc_led_restore(sc); 1613 igc_prepare_fatal_error_reset(sc); 1614 igc_reset_hw(&sc->hw); 1615 igc_finish_fatal_error_reset(sc); 1616 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1617 } 1618 1619 /* 1620 * A PCIe-region parity failure stops PCIe and DMA traffic. Intel requires a 1621 * device reset before master disable in this case, unlike the normal reset 1622 * path, which disables the bus master first. 1623 */ 1624 static void 1625 igc_prepare_fatal_error_reset(struct igc_softc *sc) 1626 { 1627 struct igc_hw *hw; 1628 s32 error; 1629 u32 ctrl, pcieerr; 1630 int i; 1631 1632 if (atomic_load_acq_32(&sc->fatal_error_state) == 1633 IGC_FATAL_ERROR_NONE) 1634 return; 1635 1636 hw = &sc->hw; 1637 pcieerr = sc->fatal_error_pcie | 1638 (IGC_READ_REG(hw, IGC_PCIEERRSTS) & IGC_PCIEERRSTS_FATAL_MASK); 1639 if ((sc->fatal_error_peind & IGC_PEIND_PCIE_PARITY_FATAL) == 0 && 1640 pcieerr == 0) 1641 return; 1642 1643 ctrl = IGC_READ_REG(hw, IGC_CTRL); 1644 IGC_WRITE_REG(hw, IGC_CTRL, ctrl | IGC_CTRL_DEV_RST); 1645 /* Do not access device registers for at least 3 ms after DEV_RST. */ 1646 msec_delay(3); 1647 for (i = 0; i < AUTO_READ_DONE_TIMEOUT; i++) { 1648 if ((IGC_READ_REG(hw, IGC_EECD) & IGC_EECD_AUTO_RD) != 0 && 1649 (IGC_READ_REG(hw, IGC_STATUS) & IGC_STATUS_RST_DONE) != 0) 1650 break; 1651 msec_delay(1); 1652 } 1653 if (i == AUTO_READ_DONE_TIMEOUT) 1654 device_printf(sc->dev, 1655 "device reset did not complete during parity recovery\n"); 1656 error = igc_disable_pcie_master_generic(hw); 1657 if (error != IGC_SUCCESS) 1658 device_printf(sc->dev, 1659 "PCIe master disable failed during parity recovery: %d\n", 1660 error); 1661 pcieerr |= IGC_READ_REG(hw, IGC_PCIEERRSTS) & 1662 IGC_PCIEERRSTS_FATAL_MASK; 1663 if (pcieerr != 0) 1664 IGC_WRITE_REG(hw, IGC_PCIEERRSTS, pcieerr); 1665 } 1666 1667 static void 1668 igc_finish_fatal_error_reset(struct igc_softc *sc) 1669 { 1670 struct igc_hw *hw; 1671 u32 lanerr, pcieerr; 1672 1673 if (atomic_load_acq_32(&sc->fatal_error_state) == 1674 IGC_FATAL_ERROR_NONE) 1675 return; 1676 1677 hw = &sc->hw; 1678 pcieerr = sc->fatal_error_pcie | 1679 (IGC_READ_REG(hw, IGC_PCIEERRSTS) & IGC_PCIEERRSTS_FATAL_MASK); 1680 if (pcieerr != 0) 1681 IGC_WRITE_REG(hw, IGC_PCIEERRSTS, pcieerr); 1682 lanerr = sc->fatal_error_lan | 1683 (IGC_READ_REG(hw, IGC_LANPERRSTS) & IGC_LANPERRSTS_RETX_BUF); 1684 if (lanerr != 0) 1685 IGC_WRITE_REG(hw, IGC_LANPERRSTS, lanerr); 1686 /* 1687 * DEV_RST can relatch PEIND from a subordinate status register 1688 * before that register is cleared. Drain the recovered indication 1689 * before unmasking FER so a later error is not misattributed. 1690 */ 1691 (void)IGC_READ_REG(hw, IGC_PEIND); 1692 1693 sc->fatal_error_peind = 0; 1694 sc->fatal_error_pcie = 0; 1695 sc->fatal_error_lan = 0; 1696 sc->fatal_error_mng = 0; 1697 atomic_store_rel_32(&sc->fatal_error_state, IGC_FATAL_ERROR_NONE); 1698 } 1699 1700 /* 1701 * I225/I226 have three configurable LED outputs. DPDK uses LED1 for 1702 * adapter identification; retain that convention and preserve the OEM's 1703 * configuration for normal link and activity indication. 1704 */ 1705 static void 1706 igc_if_led_func(if_ctx_t ctx, int onoff) 1707 { 1708 struct igc_softc *sc; 1709 struct igc_hw *hw; 1710 u32 ledctl; 1711 1712 sc = iflib_get_softc(ctx); 1713 hw = &sc->hw; 1714 if (onoff) { 1715 if (!sc->led_active) { 1716 sc->ledctl_default = IGC_READ_REG(hw, IGC_LEDCTL); 1717 sc->led_active = true; 1718 } 1719 ledctl = sc->ledctl_default; 1720 ledctl &= ~(IGC_LEDCTL_LED1_MODE_MASK | 1721 IGC_LEDCTL_LED1_BLINK); 1722 ledctl |= IGC_LEDCTL_MODE_LED_ON << 1723 IGC_LEDCTL_LED1_MODE_SHIFT; 1724 IGC_WRITE_REG(hw, IGC_LEDCTL, ledctl); 1725 } else { 1726 igc_led_restore(sc); 1727 } 1728 } 1729 1730 static void 1731 igc_led_restore(struct igc_softc *sc) 1732 { 1733 1734 if (!sc->led_active) 1735 return; 1736 IGC_WRITE_REG(&sc->hw, IGC_LEDCTL, sc->ledctl_default); 1737 sc->led_active = false; 1738 } 1739 1740 /********************************************************************* 1741 * 1742 * Determine hardware revision. 1743 * 1744 **********************************************************************/ 1745 static void 1746 igc_identify_hardware(if_ctx_t ctx) 1747 { 1748 device_t dev = iflib_get_dev(ctx); 1749 struct igc_softc *sc = iflib_get_softc(ctx); 1750 1751 /* Make sure our PCI config space has the necessary stuff set */ 1752 sc->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1753 1754 /* Save off the information about this board */ 1755 sc->hw.vendor_id = pci_get_vendor(dev); 1756 sc->hw.device_id = pci_get_device(dev); 1757 sc->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1758 sc->hw.subsystem_vendor_id = 1759 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1760 sc->hw.subsystem_device_id = 1761 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1762 1763 /* Do Shared Code Init and Setup */ 1764 if (igc_set_mac_type(&sc->hw)) { 1765 device_printf(dev, "Setup init failure\n"); 1766 return; 1767 } 1768 } 1769 1770 /********************************************************************* 1771 * 1772 * Intel's I225/I226 Specification Update, erratum 2, states that I225 1773 * devices can incorrectly enter L1 substates while CLKREQ# is asserted, 1774 * causing repeated L1-substate entry and exit. Disable both ASPM and 1775 * PCI-PM L1.2, as the erratum can occur while idle or in D3. 1776 * 1777 * I226 devices have a separate erratum where ASPM L1.2 exit latency can 1778 * exceed what the packet buffer can tolerate under load. Disabling ASPM 1779 * L1.2 on the device itself works around the issue. 1780 * 1781 **********************************************************************/ 1782 static void 1783 igc_disable_broken_l1_2(if_ctx_t ctx) 1784 { 1785 device_t dev = iflib_get_dev(ctx); 1786 struct igc_softc *sc = iflib_get_softc(ctx); 1787 int cap; 1788 uint32_t ctl1, mask; 1789 1790 if (igc_is_device_id_i225(&sc->hw)) 1791 mask = PCIM_L1PM_CTL1_ASPM_L1_2 | 1792 PCIM_L1PM_CTL1_PCIPM_L1_2; 1793 else if (igc_is_device_id_i226(&sc->hw)) 1794 mask = PCIM_L1PM_CTL1_ASPM_L1_2; 1795 else 1796 return; 1797 1798 if (pci_find_extcap(dev, PCIZ_L1PM, &cap) != 0) 1799 return; 1800 1801 ctl1 = pci_read_config(dev, cap + PCIR_L1PM_CTL1, 4); 1802 ctl1 &= ~mask; 1803 pci_write_config(dev, cap + PCIR_L1PM_CTL1, ctl1, 4); 1804 } 1805 1806 static int 1807 igc_allocate_pci_resources(if_ctx_t ctx) 1808 { 1809 struct igc_softc *sc = iflib_get_softc(ctx); 1810 device_t dev = iflib_get_dev(ctx); 1811 int rid; 1812 1813 rid = PCIR_BAR(0); 1814 sc->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1815 &rid, RF_ACTIVE); 1816 if (sc->memory == NULL) { 1817 device_printf(dev, 1818 "Unable to allocate bus resource: memory\n"); 1819 return (ENXIO); 1820 } 1821 sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->memory); 1822 sc->osdep.mem_bus_space_handle = 1823 rman_get_bushandle(sc->memory); 1824 sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; 1825 1826 sc->hw.back = &sc->osdep; 1827 1828 return (0); 1829 } 1830 1831 /********************************************************************* 1832 * 1833 * Set up the MSI-X Interrupt handlers 1834 * 1835 **********************************************************************/ 1836 static int 1837 igc_if_msix_intr_assign(if_ctx_t ctx, int msix) 1838 { 1839 struct igc_softc *sc = iflib_get_softc(ctx); 1840 struct igc_rx_queue *rx_que = sc->rx_queues; 1841 struct igc_tx_queue *tx_que = sc->tx_queues; 1842 int error, rid, i, vector = 0, rx_vectors; 1843 char buf[16]; 1844 1845 /* First set up ring resources */ 1846 for (i = 0; i < sc->rx_num_queues; i++, rx_que++, vector++) { 1847 rid = vector + 1; 1848 snprintf(buf, sizeof(buf), "rxq%d", i); 1849 error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, 1850 IFLIB_INTR_RXTX, igc_msix_que, rx_que, rx_que->me, buf); 1851 if (error) { 1852 device_printf(iflib_get_dev(ctx), 1853 "Failed to allocate que int %d err: %d", 1854 i, error); 1855 sc->rx_num_queues = i + 1; 1856 goto fail; 1857 } 1858 1859 rx_que->msix = vector; 1860 1861 /* 1862 * Set the bit to enable interrupt 1863 * in IGC_IMS -- bits 20 and 21 1864 * are for RX0 and RX1, note this has 1865 * NOTHING to do with the MSI-X vector 1866 */ 1867 rx_que->eims = 1 << vector; 1868 } 1869 rx_vectors = vector; 1870 1871 vector = 0; 1872 for (i = 0; i < sc->tx_num_queues; i++, tx_que++, vector++) { 1873 snprintf(buf, sizeof(buf), "txq%d", i); 1874 tx_que = &sc->tx_queues[i]; 1875 iflib_softirq_alloc_generic(ctx, 1876 &sc->rx_queues[i % sc->rx_num_queues].que_irq, 1877 IFLIB_INTR_TX, tx_que, tx_que->me, buf); 1878 1879 tx_que->msix = (vector % sc->rx_num_queues); 1880 1881 /* 1882 * Set the bit to enable interrupt 1883 * in IGC_IMS -- bits 22 and 23 1884 * are for TX0 and TX1, note this has 1885 * NOTHING to do with the MSI-X vector 1886 */ 1887 tx_que->eims = 1 << i; 1888 } 1889 1890 /* Link interrupt */ 1891 rid = rx_vectors + 1; 1892 error = iflib_irq_alloc_generic(ctx, &sc->irq, rid, IFLIB_INTR_ADMIN, 1893 igc_msix_link, sc, 0, "aq"); 1894 1895 if (error) { 1896 device_printf(iflib_get_dev(ctx), 1897 "Failed to register admin handler"); 1898 goto fail; 1899 } 1900 sc->linkvec = rx_vectors; 1901 return (0); 1902 fail: 1903 iflib_irq_free(ctx, &sc->irq); 1904 rx_que = sc->rx_queues; 1905 for (int i = 0; i < sc->rx_num_queues; i++, rx_que++) 1906 iflib_irq_free(ctx, &rx_que->que_irq); 1907 return (error); 1908 } 1909 1910 static void 1911 igc_configure_queues(struct igc_softc *sc) 1912 { 1913 struct igc_hw *hw = &sc->hw; 1914 struct igc_rx_queue *rx_que; 1915 struct igc_tx_queue *tx_que; 1916 u32 ivar = 0; 1917 1918 /* First turn on RSS capability */ 1919 IGC_WRITE_REG(hw, IGC_GPIE, 1920 IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | IGC_GPIE_PBA | 1921 IGC_GPIE_NSICR); 1922 1923 /* Turn on MSI-X */ 1924 /* RX entries */ 1925 for (int i = 0; i < sc->rx_num_queues; i++) { 1926 u32 index = i >> 1; 1927 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1928 rx_que = &sc->rx_queues[i]; 1929 if (i & 1) { 1930 ivar &= 0xFF00FFFF; 1931 ivar |= (rx_que->msix | IGC_IVAR_VALID) << 16; 1932 } else { 1933 ivar &= 0xFFFFFF00; 1934 ivar |= rx_que->msix | IGC_IVAR_VALID; 1935 } 1936 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1937 } 1938 /* TX entries */ 1939 for (int i = 0; i < sc->tx_num_queues; i++) { 1940 u32 index = i >> 1; 1941 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 1942 tx_que = &sc->tx_queues[i]; 1943 if (i & 1) { 1944 ivar &= 0x00FFFFFF; 1945 ivar |= (tx_que->msix | IGC_IVAR_VALID) << 24; 1946 } else { 1947 ivar &= 0xFFFF00FF; 1948 ivar |= (tx_que->msix | IGC_IVAR_VALID) << 8; 1949 } 1950 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 1951 sc->que_mask |= tx_que->eims; 1952 } 1953 1954 /* And for the link interrupt */ 1955 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 1956 sc->link_mask = 1 << sc->linkvec; 1957 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 1958 1959 return; 1960 } 1961 1962 static void 1963 igc_initialize_interrupt_rate(struct igc_softc *sc) 1964 { 1965 struct igc_hw *hw = &sc->hw; 1966 struct igc_rx_queue *rx_que; 1967 u32 newitr; 1968 1969 newitr = IGC_INTS_TO_EITR(igc_max_interrupt_rate); 1970 newitr |= IGC_EITR_CNT_IGNR; 1971 1972 for (int i = 0; i < sc->rx_num_queues; i++) { 1973 rx_que = &sc->rx_queues[i]; 1974 rx_que->eitr_setting = newitr; 1975 IGC_WRITE_REG(hw, IGC_EITR(rx_que->msix), 1976 rx_que->eitr_setting); 1977 } 1978 } 1979 1980 static void 1981 igc_free_pci_resources(if_ctx_t ctx) 1982 { 1983 struct igc_softc *sc = iflib_get_softc(ctx); 1984 struct igc_rx_queue *que = sc->rx_queues; 1985 device_t dev = iflib_get_dev(ctx); 1986 1987 /* Release all MSI-X queue resources */ 1988 if (sc->intr_type == IFLIB_INTR_MSIX) 1989 iflib_irq_free(ctx, &sc->irq); 1990 1991 for (int i = 0; i < sc->rx_num_queues; i++, que++) { 1992 iflib_irq_free(ctx, &que->que_irq); 1993 } 1994 1995 if (sc->memory != NULL) { 1996 bus_release_resource(dev, SYS_RES_MEMORY, 1997 rman_get_rid(sc->memory), sc->memory); 1998 sc->memory = NULL; 1999 } 2000 2001 if (sc->flash != NULL) { 2002 bus_release_resource(dev, SYS_RES_MEMORY, 2003 rman_get_rid(sc->flash), sc->flash); 2004 sc->flash = NULL; 2005 } 2006 2007 if (sc->ioport != NULL) { 2008 bus_release_resource(dev, SYS_RES_IOPORT, 2009 rman_get_rid(sc->ioport), sc->ioport); 2010 sc->ioport = NULL; 2011 } 2012 } 2013 2014 /* Set up MSI or MSI-X */ 2015 static int 2016 igc_setup_msix(if_ctx_t ctx) 2017 { 2018 return (0); 2019 } 2020 2021 /********************************************************************* 2022 * 2023 * Initialize the DMA Coalescing feature 2024 * 2025 **********************************************************************/ 2026 static void 2027 igc_init_dmac(struct igc_softc *sc, u32 pba) 2028 { 2029 device_t dev = sc->dev; 2030 struct igc_hw *hw = &sc->hw; 2031 u32 dmac, reg = ~IGC_DMACR_DMAC_EN; 2032 u16 hwm; 2033 u16 max_frame_size; 2034 int status; 2035 2036 max_frame_size = sc->shared->isc_max_frame_size; 2037 2038 if (sc->dmac == 0) { /* Disabling it */ 2039 IGC_WRITE_REG(hw, IGC_DMACR, reg); 2040 return; 2041 } else 2042 device_printf(dev, "DMA Coalescing enabled\n"); 2043 2044 /* Set starting threshold */ 2045 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 2046 2047 hwm = 64 * pba - max_frame_size / 16; 2048 if (hwm < 64 * (pba - 6)) 2049 hwm = 64 * (pba - 6); 2050 reg = IGC_READ_REG(hw, IGC_FCRTC); 2051 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 2052 reg |= ((hwm << IGC_FCRTC_RTH_COAL_SHIFT) 2053 & IGC_FCRTC_RTH_COAL_MASK); 2054 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 2055 2056 dmac = pba - max_frame_size / 512; 2057 if (dmac < pba - 10) 2058 dmac = pba - 10; 2059 reg = IGC_READ_REG(hw, IGC_DMACR); 2060 reg &= ~IGC_DMACR_DMACTHR_MASK; 2061 reg |= ((dmac << IGC_DMACR_DMACTHR_SHIFT) 2062 & IGC_DMACR_DMACTHR_MASK); 2063 2064 /* transition to L0x or L1 if available..*/ 2065 reg |= (IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK); 2066 2067 /* Check if status is 2.5Gb backplane connection 2068 * before configuration of watchdog timer, which is 2069 * in msec values in 12.8usec intervals 2070 * watchdog timer= msec values in 32usec intervals 2071 * for non 2.5Gb connection 2072 */ 2073 status = IGC_READ_REG(hw, IGC_STATUS); 2074 if ((status & IGC_STATUS_2P5_SKU) && 2075 (!(status & IGC_STATUS_2P5_SKU_OVER))) 2076 reg |= ((sc->dmac * 5) >> 6); 2077 else 2078 reg |= (sc->dmac >> 5); 2079 2080 IGC_WRITE_REG(hw, IGC_DMACR, reg); 2081 2082 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 2083 2084 /* Set the interval before transition */ 2085 reg = IGC_READ_REG(hw, IGC_DMCTLX); 2086 reg |= IGC_DMCTLX_DCFLUSH_DIS; 2087 2088 /* 2089 ** in 2.5Gb connection, TTLX unit is 0.4 usec 2090 ** which is 0x4*2 = 0xA. But delay is still 4 usec 2091 */ 2092 status = IGC_READ_REG(hw, IGC_STATUS); 2093 if ((status & IGC_STATUS_2P5_SKU) && 2094 (!(status & IGC_STATUS_2P5_SKU_OVER))) 2095 reg |= 0xA; 2096 else 2097 reg |= 0x4; 2098 2099 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 2100 2101 /* free space in tx packet buffer to wake from DMA coal */ 2102 IGC_WRITE_REG(hw, IGC_DMCTXTH, (IGC_TXPBSIZE - 2103 (2 * max_frame_size)) >> 6); 2104 2105 /* make low power state decision controlled by DMA coal */ 2106 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 2107 reg &= ~IGC_PCIEMISC_LX_DECISION; 2108 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 2109 } 2110 2111 /********************************************************************* 2112 * 2113 * Initialize the hardware to a configuration as specified by the 2114 * softc structure. 2115 * 2116 **********************************************************************/ 2117 static void 2118 igc_reset(if_ctx_t ctx) 2119 { 2120 device_t dev = iflib_get_dev(ctx); 2121 struct igc_softc *sc = iflib_get_softc(ctx); 2122 struct igc_hw *hw = &sc->hw; 2123 u32 rx_buffer_size; 2124 u32 pba; 2125 2126 INIT_DEBUGOUT("igc_reset: begin"); 2127 igc_led_restore(sc); 2128 /* Let the firmware know the OS is in control */ 2129 igc_get_hw_control(sc); 2130 2131 /* 2132 * Packet Buffer Allocation (PBA) 2133 * Writing PBA sets the receive portion of the buffer 2134 * the remainder is used for the transmit buffer. 2135 */ 2136 pba = IGC_PBA_34K; 2137 2138 INIT_DEBUGOUT1("igc_reset: pba=%dK",pba); 2139 2140 /* 2141 * These parameters control the automatic generation (Tx) and 2142 * response (Rx) to Ethernet PAUSE frames. 2143 * - High water mark should allow for at least two frames to be 2144 * received after sending an XOFF. 2145 * - Low water mark works best when it is very near the high water 2146 * mark. 2147 * This allows the receiver to restart by sending XON when it has 2148 * drained a bit. Here we use an arbitrary value of 1500 which will 2149 * restart after one full frame is pulled from the buffer. There 2150 * could be several smaller frames in the buffer and if so they will 2151 * not trigger the XON until their total number reduces the buffer 2152 * by 1500. 2153 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2154 */ 2155 rx_buffer_size = (pba & 0xffff) << 10; 2156 hw->fc.high_water = rx_buffer_size - 2157 roundup2(sc->hw.mac.max_frame_size, 1024); 2158 /* 16-byte granularity */ 2159 hw->fc.low_water = hw->fc.high_water - 16; 2160 2161 if (sc->fc) /* locally set flow control value? */ 2162 hw->fc.requested_mode = sc->fc; 2163 else 2164 hw->fc.requested_mode = igc_fc_full; 2165 2166 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 2167 2168 hw->fc.send_xon = true; 2169 2170 /* Issue a global reset */ 2171 igc_reset_hw(hw); 2172 IGC_WRITE_REG(hw, IGC_WUC, 0); 2173 2174 /* and a re-init */ 2175 if (igc_init_hw(hw) < 0) { 2176 device_printf(dev, "Hardware Initialization Failed\n"); 2177 return; 2178 } 2179 2180 /* Setup DMA Coalescing */ 2181 igc_init_dmac(sc, pba); 2182 2183 /* Save the final PBA off if it needs to be used elsewhere i.e. AIM */ 2184 sc->pba = pba; 2185 2186 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 2187 igc_get_phy_info(hw); 2188 igc_check_for_link(hw); 2189 } 2190 2191 /* 2192 * Initialise the RSS mapping for NICs that support multiple transmit/ 2193 * receive rings. 2194 */ 2195 2196 #define RSSKEYLEN 10 2197 static void 2198 igc_initialize_rss_mapping(struct igc_softc *sc) 2199 { 2200 struct igc_hw *hw = &sc->hw; 2201 int i; 2202 int queue_id; 2203 u32 reta; 2204 u32 rss_key[RSSKEYLEN], mrqc, shift = 0; 2205 2206 /* 2207 * The redirection table controls which destination 2208 * queue each bucket redirects traffic to. 2209 * Each DWORD represents four queues, with the LSB 2210 * being the first queue in the DWORD. 2211 * 2212 * This just allocates buckets to queues using round-robin 2213 * allocation. 2214 * 2215 * NOTE: It Just Happens to line up with the default 2216 * RSS allocation method. 2217 */ 2218 2219 /* Warning FM follows */ 2220 reta = 0; 2221 for (i = 0; i < 128; i++) { 2222 #ifdef RSS 2223 queue_id = rss_get_indirection_to_bucket(i); 2224 /* 2225 * If we have more queues than buckets, we'll 2226 * end up mapping buckets to a subset of the 2227 * queues. 2228 * 2229 * If we have more buckets than queues, we'll 2230 * end up instead assigning multiple buckets 2231 * to queues. 2232 * 2233 * Both are suboptimal, but we need to handle 2234 * the case so we don't go out of bounds 2235 * indexing arrays and such. 2236 */ 2237 queue_id = queue_id % sc->rx_num_queues; 2238 #else 2239 queue_id = (i % sc->rx_num_queues); 2240 #endif 2241 /* Adjust if required */ 2242 queue_id = queue_id << shift; 2243 2244 /* 2245 * The low 8 bits are for hash value (n+0); 2246 * The next 8 bits are for hash value (n+1), etc. 2247 */ 2248 reta = reta >> 8; 2249 reta = reta | ( ((uint32_t) queue_id) << 24); 2250 if ((i & 3) == 3) { 2251 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 2252 reta = 0; 2253 } 2254 } 2255 2256 /* Now fill in hash table */ 2257 2258 /* 2259 * MRQC: Multiple Receive Queues Command 2260 * Set queuing to RSS control, number depends on the device. 2261 */ 2262 mrqc = IGC_MRQC_ENABLE_RSS_4Q; 2263 2264 /* XXX ew typecasting */ 2265 rss_getkey((uint8_t *) &rss_key); 2266 for (i = 0; i < RSSKEYLEN; i++) 2267 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 2268 2269 /* 2270 * Configure the RSS fields to hash upon. 2271 */ 2272 mrqc |= (IGC_MRQC_RSS_FIELD_IPV4 | 2273 IGC_MRQC_RSS_FIELD_IPV4_TCP); 2274 mrqc |= (IGC_MRQC_RSS_FIELD_IPV6 | 2275 IGC_MRQC_RSS_FIELD_IPV6_TCP); 2276 mrqc |=( IGC_MRQC_RSS_FIELD_IPV4_UDP | 2277 IGC_MRQC_RSS_FIELD_IPV6_UDP); 2278 mrqc |=( IGC_MRQC_RSS_FIELD_IPV6_UDP_EX | 2279 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX); 2280 2281 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 2282 } 2283 2284 /********************************************************************* 2285 * 2286 * Setup networking device structure and register interface media. 2287 * 2288 **********************************************************************/ 2289 static int 2290 igc_setup_interface(if_ctx_t ctx) 2291 { 2292 if_t ifp = iflib_get_ifp(ctx); 2293 struct igc_softc *sc = iflib_get_softc(ctx); 2294 if_softc_ctx_t scctx = sc->shared; 2295 2296 INIT_DEBUGOUT("igc_setup_interface: begin"); 2297 2298 /* Single Queue */ 2299 if (sc->tx_num_queues == 1) { 2300 if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); 2301 if_setsendqready(ifp); 2302 } 2303 2304 /* 2305 * Specify the media types supported by this adapter and register 2306 * callbacks to update media and link information 2307 */ 2308 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 2309 ifmedia_add(sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 2310 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2311 ifmedia_add(sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 2312 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2313 ifmedia_add(sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2314 ifmedia_add(sc->media, IFM_ETHER | IFM_2500_T, 0, NULL); 2315 2316 ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2317 ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); 2318 return (0); 2319 } 2320 2321 static int 2322 igc_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 2323 int ntxqs, int ntxqsets) 2324 { 2325 struct igc_softc *sc = iflib_get_softc(ctx); 2326 if_softc_ctx_t scctx = sc->shared; 2327 int error = IGC_SUCCESS; 2328 struct igc_tx_queue *que; 2329 int i, j; 2330 2331 MPASS(sc->tx_num_queues > 0); 2332 MPASS(sc->tx_num_queues == ntxqsets); 2333 2334 /* First allocate the top level queue structs */ 2335 if (!(sc->tx_queues = 2336 (struct igc_tx_queue *) malloc(sizeof(struct igc_tx_queue) * 2337 sc->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2338 device_printf(iflib_get_dev(ctx), 2339 "Unable to allocate queue memory\n"); 2340 return(ENOMEM); 2341 } 2342 2343 for (i = 0, que = sc->tx_queues; i < sc->tx_num_queues; i++, que++) { 2344 /* Set up some basics */ 2345 2346 struct tx_ring *txr = &que->txr; 2347 KASSERT(__is_aligned(&txr->tx_aim_snapshot, sizeof(uint64_t)), 2348 ("%s: misaligned TX AIM snapshot %p", __func__, 2349 &txr->tx_aim_snapshot)); 2350 txr->sc = que->sc = sc; 2351 que->me = txr->me = i; 2352 2353 /* Allocate report status array */ 2354 if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * 2355 scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { 2356 device_printf(iflib_get_dev(ctx), 2357 "failed to allocate rs_idxs memory\n"); 2358 error = ENOMEM; 2359 goto fail; 2360 } 2361 for (j = 0; j < scctx->isc_ntxd[0]; j++) 2362 txr->tx_rsq[j] = QIDX_INVALID; 2363 /* get virtual and physical address of the hardware queues */ 2364 txr->tx_base = (struct igc_tx_desc *)vaddrs[i*ntxqs]; 2365 txr->tx_paddr = paddrs[i*ntxqs]; 2366 } 2367 2368 if (bootverbose) 2369 device_printf(iflib_get_dev(ctx), 2370 "allocated for %d tx_queues\n", sc->tx_num_queues); 2371 return (0); 2372 fail: 2373 igc_if_queues_free(ctx); 2374 return (error); 2375 } 2376 2377 static int 2378 igc_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, 2379 int nrxqs, int nrxqsets) 2380 { 2381 struct igc_softc *sc = iflib_get_softc(ctx); 2382 int error = IGC_SUCCESS; 2383 struct igc_rx_queue *que; 2384 int i; 2385 2386 MPASS(sc->rx_num_queues > 0); 2387 MPASS(sc->rx_num_queues == nrxqsets); 2388 2389 /* First allocate the top level queue structs */ 2390 if (!(sc->rx_queues = 2391 (struct igc_rx_queue *) malloc(sizeof(struct igc_rx_queue) * 2392 sc->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2393 device_printf(iflib_get_dev(ctx), 2394 "Unable to allocate queue memory\n"); 2395 error = ENOMEM; 2396 goto fail; 2397 } 2398 2399 for (i = 0, que = sc->rx_queues; i < nrxqsets; i++, que++) { 2400 /* Set up some basics */ 2401 struct rx_ring *rxr = &que->rxr; 2402 KASSERT(__is_aligned(&rxr->rx_aim_snapshot, sizeof(uint64_t)), 2403 ("%s: misaligned RX AIM snapshot %p", __func__, 2404 &rxr->rx_aim_snapshot)); 2405 rxr->sc = que->sc = sc; 2406 rxr->que = que; 2407 que->me = rxr->me = i; 2408 2409 /* get virtual and physical address of the hardware queues */ 2410 rxr->rx_base = (union igc_rx_desc_extended *)vaddrs[i*nrxqs]; 2411 rxr->rx_paddr = paddrs[i*nrxqs]; 2412 } 2413 2414 if (bootverbose) 2415 device_printf(iflib_get_dev(ctx), 2416 "allocated for %d rx_queues\n", sc->rx_num_queues); 2417 2418 return (0); 2419 fail: 2420 igc_if_queues_free(ctx); 2421 return (error); 2422 } 2423 2424 static void 2425 igc_if_queues_free(if_ctx_t ctx) 2426 { 2427 struct igc_softc *sc = iflib_get_softc(ctx); 2428 struct igc_tx_queue *tx_que = sc->tx_queues; 2429 struct igc_rx_queue *rx_que = sc->rx_queues; 2430 2431 if (tx_que != NULL) { 2432 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 2433 struct tx_ring *txr = &tx_que->txr; 2434 if (txr->tx_rsq == NULL) 2435 break; 2436 2437 free(txr->tx_rsq, M_DEVBUF); 2438 txr->tx_rsq = NULL; 2439 } 2440 free(sc->tx_queues, M_DEVBUF); 2441 sc->tx_queues = NULL; 2442 } 2443 2444 if (rx_que != NULL) { 2445 free(sc->rx_queues, M_DEVBUF); 2446 sc->rx_queues = NULL; 2447 } 2448 2449 if (sc->mta != NULL) { 2450 free(sc->mta, M_DEVBUF); 2451 } 2452 } 2453 2454 /********************************************************************* 2455 * 2456 * Enable transmit unit. 2457 * 2458 **********************************************************************/ 2459 static void 2460 igc_initialize_transmit_unit(if_ctx_t ctx) 2461 { 2462 struct igc_softc *sc = iflib_get_softc(ctx); 2463 if_softc_ctx_t scctx = sc->shared; 2464 struct igc_tx_queue *que; 2465 struct tx_ring *txr; 2466 struct igc_hw *hw = &sc->hw; 2467 u32 tctl, txdctl = 0; 2468 2469 INIT_DEBUGOUT("igc_initialize_transmit_unit: begin"); 2470 2471 for (int i = 0; i < sc->tx_num_queues; i++, txr++) { 2472 u64 bus_addr; 2473 caddr_t offp, endp; 2474 2475 que = &sc->tx_queues[i]; 2476 txr = &que->txr; 2477 bus_addr = txr->tx_paddr; 2478 2479 /* Clear checksum offload context. */ 2480 offp = (caddr_t)&txr->csum_flags; 2481 endp = (caddr_t)(txr + 1); 2482 bzero(offp, endp - offp); 2483 2484 /* Base and Len of TX Ring */ 2485 IGC_WRITE_REG(hw, IGC_TDLEN(i), 2486 scctx->isc_ntxd[0] * sizeof(struct igc_tx_desc)); 2487 IGC_WRITE_REG(hw, IGC_TDBAH(i), 2488 (u32)(bus_addr >> 32)); 2489 IGC_WRITE_REG(hw, IGC_TDBAL(i), 2490 (u32)bus_addr); 2491 /* Init the HEAD/TAIL indices */ 2492 IGC_WRITE_REG(hw, IGC_TDT(i), 0); 2493 IGC_WRITE_REG(hw, IGC_TDH(i), 0); 2494 2495 HW_DEBUGOUT2("Base = %x, Length = %x\n", 2496 IGC_READ_REG(&sc->hw, IGC_TDBAL(i)), 2497 IGC_READ_REG(&sc->hw, IGC_TDLEN(i))); 2498 2499 /* WTHRESH must be zero when iflib uses sparse RS. */ 2500 txdctl = IGC_TX_PTHRESH | (IGC_TX_HTHRESH << 8) | 2501 IGC_TXDCTL_QUEUE_ENABLE; 2502 2503 IGC_WRITE_REG(hw, IGC_TXDCTL(i), txdctl); 2504 } 2505 2506 /* Program the Transmit Control Register */ 2507 tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 2508 tctl &= ~IGC_TCTL_CT; 2509 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 2510 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 2511 2512 /* This write will effectively turn on the transmit unit. */ 2513 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 2514 } 2515 2516 /********************************************************************* 2517 * 2518 * Enable receive unit. 2519 * 2520 **********************************************************************/ 2521 #define BSIZEPKT_ROUNDUP ((1<<IGC_SRRCTL_BSIZEPKT_SHIFT)-1) 2522 2523 static void 2524 igc_initialize_receive_unit(if_ctx_t ctx) 2525 { 2526 struct igc_softc *sc = iflib_get_softc(ctx); 2527 if_softc_ctx_t scctx = sc->shared; 2528 if_t ifp = iflib_get_ifp(ctx); 2529 struct igc_hw *hw = &sc->hw; 2530 struct igc_rx_queue *que; 2531 int i; 2532 u32 psize, rctl, rxcsum, srrctl = 0; 2533 2534 INIT_DEBUGOUT("igc_initialize_receive_units: begin"); 2535 2536 /* 2537 * Make sure receives are disabled while setting 2538 * up the descriptor ring 2539 */ 2540 rctl = IGC_READ_REG(hw, IGC_RCTL); 2541 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 2542 2543 /* Setup the Receive Control Register */ 2544 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 2545 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | 2546 IGC_RCTL_LBM_NO | IGC_RCTL_RDMTS_HALF | 2547 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 2548 2549 /* Do not store bad packets */ 2550 rctl &= ~IGC_RCTL_SBP; 2551 2552 /* Enable Long Packet receive */ 2553 if (if_getmtu(ifp) > ETHERMTU) 2554 rctl |= IGC_RCTL_LPE; 2555 else 2556 rctl &= ~IGC_RCTL_LPE; 2557 2558 /* Strip the CRC */ 2559 if (!igc_disable_crc_stripping) 2560 rctl |= IGC_RCTL_SECRC; 2561 2562 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 2563 if (if_getcapenable(ifp) & IFCAP_RXCSUM) { 2564 rxcsum |= IGC_RXCSUM_CRCOFL; 2565 if (sc->tx_num_queues > 1) 2566 rxcsum |= IGC_RXCSUM_PCSD; 2567 else 2568 rxcsum |= IGC_RXCSUM_IPPCSE; 2569 } else { 2570 if (sc->tx_num_queues > 1) 2571 rxcsum |= IGC_RXCSUM_PCSD; 2572 else 2573 rxcsum &= ~IGC_RXCSUM_TUOFL; 2574 } 2575 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 2576 2577 if (sc->rx_num_queues > 1) 2578 igc_initialize_rss_mapping(sc); 2579 2580 if (if_getmtu(ifp) > ETHERMTU) { 2581 psize = scctx->isc_max_frame_size; 2582 /* are we on a vlan? */ 2583 if (if_vlantrunkinuse(ifp)) 2584 psize += VLAN_TAG_SIZE; 2585 IGC_WRITE_REG(&sc->hw, IGC_RLPML, psize); 2586 } 2587 2588 /* Set maximum packet buffer len */ 2589 srrctl |= (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 2590 IGC_SRRCTL_BSIZEPKT_SHIFT; 2591 /* srrctl above overrides this but set the register to a sane value */ 2592 rctl |= IGC_RCTL_SZ_2048; 2593 2594 /* 2595 * If TX flow control is disabled and there's >1 queue defined, 2596 * enable DROP. 2597 * 2598 * This drops frames rather than hanging the RX MAC for all queues. 2599 */ 2600 if ((sc->rx_num_queues > 1) && 2601 (sc->fc == igc_fc_none || 2602 sc->fc == igc_fc_rx_pause)) { 2603 srrctl |= IGC_SRRCTL_DROP_EN; 2604 } 2605 2606 /* Setup the Base and Length of the Rx Descriptor Rings */ 2607 for (i = 0, que = sc->rx_queues; i < sc->rx_num_queues; i++, que++) { 2608 struct rx_ring *rxr = &que->rxr; 2609 u64 bus_addr = rxr->rx_paddr; 2610 u32 rxdctl; 2611 2612 #ifdef notyet 2613 /* Configure for header split? -- ignore for now */ 2614 rxr->hdr_split = igc_header_split; 2615 #else 2616 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 2617 #endif 2618 2619 IGC_WRITE_REG(hw, IGC_RDLEN(i), 2620 scctx->isc_nrxd[0] * sizeof(struct igc_rx_desc)); 2621 IGC_WRITE_REG(hw, IGC_RDBAH(i), (uint32_t)(bus_addr >> 32)); 2622 IGC_WRITE_REG(hw, IGC_RDBAL(i), (uint32_t)bus_addr); 2623 IGC_WRITE_REG(hw, IGC_SRRCTL(i), srrctl); 2624 /* Setup the Head and Tail Descriptor Pointers */ 2625 IGC_WRITE_REG(hw, IGC_RDH(i), 0); 2626 IGC_WRITE_REG(hw, IGC_RDT(i), 0); 2627 /* Enable this Queue */ 2628 rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(i)); 2629 rxdctl &= ~(IGC_RXDCTL_PTHRESH | IGC_RXDCTL_HTHRESH | 2630 IGC_RXDCTL_WTHRESH); 2631 rxdctl |= IGC_RX_PTHRESH | (IGC_RX_HTHRESH << 8) | 2632 (IGC_RX_WTHRESH << 16) | IGC_RXDCTL_QUEUE_ENABLE; 2633 IGC_WRITE_REG(hw, IGC_RXDCTL(i), rxdctl); 2634 } 2635 2636 /* Make sure VLAN Filters are off */ 2637 rctl &= ~IGC_RCTL_VFE; 2638 2639 /* Write out the settings */ 2640 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 2641 2642 return; 2643 } 2644 2645 static void 2646 igc_if_vlan_register(if_ctx_t ctx, u16 vtag) 2647 { 2648 struct igc_softc *sc = iflib_get_softc(ctx); 2649 u32 index, mask; 2650 2651 index = (vtag >> 5) & 0x7f; 2652 mask = 1U << (vtag & 0x1f); 2653 if ((sc->shadow_vfta[index] & mask) != 0) 2654 return; 2655 sc->shadow_vfta[index] |= mask; 2656 igc_write_vfta(&sc->hw, index, sc->shadow_vfta[index]); 2657 } 2658 2659 static void 2660 igc_if_vlan_unregister(if_ctx_t ctx, u16 vtag) 2661 { 2662 struct igc_softc *sc = iflib_get_softc(ctx); 2663 u32 index, mask; 2664 2665 index = (vtag >> 5) & 0x7f; 2666 mask = 1U << (vtag & 0x1f); 2667 if ((sc->shadow_vfta[index] & mask) == 0) 2668 return; 2669 sc->shadow_vfta[index] &= ~mask; 2670 igc_write_vfta(&sc->hw, index, sc->shadow_vfta[index]); 2671 } 2672 2673 static bool 2674 igc_if_vlan_filter_capable(if_ctx_t ctx) 2675 { 2676 if_t ifp = iflib_get_ifp(ctx); 2677 2678 return ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) != 0 && 2679 !igc_disable_crc_stripping); 2680 } 2681 2682 static bool 2683 igc_if_vlan_filter_used(if_ctx_t ctx) 2684 { 2685 struct igc_softc *sc = iflib_get_softc(ctx); 2686 2687 if (!igc_if_vlan_filter_capable(ctx)) 2688 return (false); 2689 2690 for (int i = 0; i < IGC_VFTA_SIZE; i++) 2691 if (sc->shadow_vfta[i] != 0) 2692 return (true); 2693 2694 return (false); 2695 } 2696 2697 static void 2698 igc_if_vlan_filter_enable(struct igc_softc *sc) 2699 { 2700 u32 reg; 2701 2702 reg = IGC_READ_REG(&sc->hw, IGC_RCTL); 2703 reg &= ~IGC_RCTL_CFIEN; 2704 reg |= IGC_RCTL_VFE; 2705 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg); 2706 } 2707 2708 static void 2709 igc_if_vlan_filter_disable(struct igc_softc *sc) 2710 { 2711 u32 reg; 2712 2713 reg = IGC_READ_REG(&sc->hw, IGC_RCTL); 2714 reg &= ~(IGC_RCTL_VFE | IGC_RCTL_CFIEN); 2715 IGC_WRITE_REG(&sc->hw, IGC_RCTL, reg); 2716 } 2717 2718 static void 2719 igc_setup_vlan_hw_support(if_ctx_t ctx) 2720 { 2721 struct igc_softc *sc = iflib_get_softc(ctx); 2722 struct igc_hw *hw = &sc->hw; 2723 if_t ifp = iflib_get_ifp(ctx); 2724 u32 reg; 2725 2726 if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING && 2727 !igc_disable_crc_stripping) { 2728 reg = IGC_READ_REG(hw, IGC_CTRL); 2729 reg |= IGC_CTRL_VME; 2730 IGC_WRITE_REG(hw, IGC_CTRL, reg); 2731 } else { 2732 reg = IGC_READ_REG(hw, IGC_CTRL); 2733 reg &= ~IGC_CTRL_VME; 2734 IGC_WRITE_REG(hw, IGC_CTRL, reg); 2735 } 2736 2737 if (!igc_if_vlan_filter_capable(ctx)) { 2738 igc_if_vlan_filter_disable(sc); 2739 return; 2740 } 2741 2742 /* Always admit priority-tagged frames. */ 2743 sc->shadow_vfta[0] |= 1U; 2744 2745 /* A reset may clear the VFTA, so restore the complete desired table. */ 2746 for (int i = 0; i < IGC_VFTA_SIZE; i++) 2747 igc_write_vfta(hw, i, sc->shadow_vfta[i]); 2748 2749 igc_if_vlan_filter_enable(sc); 2750 } 2751 2752 static void 2753 igc_if_intr_enable(if_ctx_t ctx) 2754 { 2755 struct igc_softc *sc = iflib_get_softc(ctx); 2756 struct igc_hw *hw = &sc->hw; 2757 u32 mask; 2758 2759 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 2760 mask = (sc->que_mask | sc->link_mask); 2761 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2762 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2763 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2764 mask = IGC_IMS_LSC; 2765 } else 2766 mask = IMS_ENABLE_MASK; 2767 if (atomic_load_acq_32(&sc->fatal_error_state) == 2768 IGC_FATAL_ERROR_NONE) 2769 mask |= IGC_IMS_FER; 2770 IGC_WRITE_REG(hw, IGC_IMS, mask); 2771 IGC_WRITE_FLUSH(hw); 2772 } 2773 2774 static void 2775 igc_if_intr_disable(if_ctx_t ctx) 2776 { 2777 struct igc_softc *sc = iflib_get_softc(ctx); 2778 struct igc_hw *hw = &sc->hw; 2779 2780 if (__predict_true(sc->intr_type == IFLIB_INTR_MSIX)) { 2781 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2782 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2783 } 2784 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2785 IGC_WRITE_FLUSH(hw); 2786 } 2787 2788 /* 2789 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 2790 * For ASF and Pass Through versions of f/w this means 2791 * that the driver is loaded. For AMT version type f/w 2792 * this means that the network i/f is open. 2793 */ 2794 static void 2795 igc_get_hw_control(struct igc_softc *sc) 2796 { 2797 u32 ctrl_ext; 2798 2799 if (sc->vf_ifp) 2800 return; 2801 2802 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2803 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, 2804 ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 2805 } 2806 2807 /* 2808 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 2809 * For ASF and Pass Through versions of f/w this means that 2810 * the driver is no longer loaded. For AMT versions of the 2811 * f/w this means that the network i/f is closed. 2812 */ 2813 static void 2814 igc_release_hw_control(struct igc_softc *sc) 2815 { 2816 u32 ctrl_ext; 2817 2818 ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 2819 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, 2820 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 2821 return; 2822 } 2823 2824 static int 2825 igc_is_valid_ether_addr(u8 *addr) 2826 { 2827 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 2828 2829 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 2830 return (false); 2831 } 2832 2833 return (true); 2834 } 2835 2836 /* 2837 ** Parse the interface capabilities with regard 2838 ** to both system management and wake-on-lan for 2839 ** later use. 2840 */ 2841 static void 2842 igc_get_wakeup(if_ctx_t ctx) 2843 { 2844 struct igc_softc *sc = iflib_get_softc(ctx); 2845 u16 eeprom_data = 0, apme_mask; 2846 2847 apme_mask = IGC_WUC_APME; 2848 eeprom_data = IGC_READ_REG(&sc->hw, IGC_WUC); 2849 2850 if (eeprom_data & apme_mask) 2851 sc->wol = IGC_WUFC_LNKC; 2852 } 2853 2854 2855 /* 2856 * Enable PCI Wake On Lan capability 2857 */ 2858 static void 2859 igc_enable_wakeup(if_ctx_t ctx) 2860 { 2861 struct igc_softc *sc = iflib_get_softc(ctx); 2862 device_t dev = iflib_get_dev(ctx); 2863 if_t ifp = iflib_get_ifp(ctx); 2864 int error = 0; 2865 u32 ctrl, rctl; 2866 2867 if (!pci_has_pm(dev)) 2868 return; 2869 2870 /* 2871 * Determine type of Wakeup: note that wol 2872 * is set with all bits on by default. 2873 */ 2874 if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) 2875 sc->wol &= ~IGC_WUFC_MAG; 2876 2877 if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0) 2878 sc->wol &= ~IGC_WUFC_EX; 2879 2880 if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) 2881 sc->wol &= ~IGC_WUFC_MC; 2882 else { 2883 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2884 rctl |= IGC_RCTL_MPE; 2885 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2886 } 2887 2888 if (!(sc->wol & (IGC_WUFC_EX | IGC_WUFC_MAG | IGC_WUFC_MC))) 2889 goto pme; 2890 2891 /* Advertise the wakeup capability */ 2892 ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 2893 ctrl |= IGC_CTRL_ADVD3WUC; 2894 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 2895 2896 /* Enable wakeup by the MAC */ 2897 IGC_WRITE_REG(&sc->hw, IGC_WUC, IGC_WUC_PME_EN); 2898 IGC_WRITE_REG(&sc->hw, IGC_WUFC, sc->wol); 2899 2900 pme: 2901 if (!error && (if_getcapenable(ifp) & IFCAP_WOL)) 2902 pci_enable_pme(dev); 2903 2904 return; 2905 } 2906 2907 /********************************************************************** 2908 * 2909 * Update the board statistics counters. 2910 * 2911 **********************************************************************/ 2912 static void 2913 igc_update_ecc_stats(struct igc_softc *sc) 2914 { 2915 struct igc_hw *hw; 2916 u32 pbeccsts, pcieeccsts; 2917 2918 hw = &sc->hw; 2919 pbeccsts = IGC_READ_REG(hw, IGC_PBECCSTS); 2920 if (pbeccsts & IGC_PBECCSTS_CORR_ERR) { 2921 sc->corrected_error_dma_count++; 2922 /* Preserve the enable bit while clearing the RW1C status. */ 2923 IGC_WRITE_REG(hw, IGC_PBECCSTS, 2924 pbeccsts & (IGC_PBECCSTS_ECC_ENABLE | 2925 IGC_PBECCSTS_CORR_ERR)); 2926 } 2927 2928 pcieeccsts = IGC_READ_REG(hw, IGC_PCIEECCSTS) & 2929 IGC_PCIEECCSTS_CORR_MASK; 2930 if (pcieeccsts & IGC_PCIEECCSTS_TX_WR_DATA) 2931 sc->corrected_error_pcie_tx_data_count++; 2932 if (pcieeccsts & IGC_PCIEECCSTS_RETRY_BUF) 2933 sc->corrected_error_pcie_retry_count++; 2934 if (pcieeccsts != 0) 2935 IGC_WRITE_REG(hw, IGC_PCIEECCSTS, pcieeccsts); 2936 } 2937 2938 static void 2939 igc_update_stats_counters(struct igc_softc *sc) 2940 { 2941 u64 prev_xoffrxc = sc->stats.xoffrxc; 2942 2943 sc->stats.crcerrs += IGC_READ_REG(&sc->hw, IGC_CRCERRS); 2944 sc->stats.rxerrc += IGC_READ_REG(&sc->hw, IGC_RXERRC); 2945 sc->stats.mpc += IGC_READ_REG(&sc->hw, IGC_MPC); 2946 sc->stats.scc += IGC_READ_REG(&sc->hw, IGC_SCC); 2947 sc->stats.ecol += IGC_READ_REG(&sc->hw, IGC_ECOL); 2948 2949 sc->stats.mcc += IGC_READ_REG(&sc->hw, IGC_MCC); 2950 sc->stats.latecol += IGC_READ_REG(&sc->hw, IGC_LATECOL); 2951 sc->stats.colc += IGC_READ_REG(&sc->hw, IGC_COLC); 2952 sc->stats.rerc += IGC_READ_REG(&sc->hw, IGC_RERC); 2953 sc->stats.dc += IGC_READ_REG(&sc->hw, IGC_DC); 2954 sc->stats.rlec += IGC_READ_REG(&sc->hw, IGC_RLEC); 2955 sc->stats.xonrxc += IGC_READ_REG(&sc->hw, IGC_XONRXC); 2956 sc->stats.xontxc += IGC_READ_REG(&sc->hw, IGC_XONTXC); 2957 sc->stats.xoffrxc += IGC_READ_REG(&sc->hw, IGC_XOFFRXC); 2958 /* 2959 * For watchdog management we need to know if we have been 2960 * paused during the last interval, so capture that here. 2961 */ 2962 if (sc->stats.xoffrxc != prev_xoffrxc) 2963 sc->shared->isc_pause_frames = 1; 2964 sc->stats.xofftxc += IGC_READ_REG(&sc->hw, IGC_XOFFTXC); 2965 sc->stats.fcruc += IGC_READ_REG(&sc->hw, IGC_FCRUC); 2966 sc->stats.prc64 += IGC_READ_REG(&sc->hw, IGC_PRC64); 2967 sc->stats.prc127 += IGC_READ_REG(&sc->hw, IGC_PRC127); 2968 sc->stats.prc255 += IGC_READ_REG(&sc->hw, IGC_PRC255); 2969 sc->stats.prc511 += IGC_READ_REG(&sc->hw, IGC_PRC511); 2970 sc->stats.prc1023 += IGC_READ_REG(&sc->hw, IGC_PRC1023); 2971 sc->stats.prc1522 += IGC_READ_REG(&sc->hw, IGC_PRC1522); 2972 sc->stats.tlpic += IGC_READ_REG(&sc->hw, IGC_TLPIC); 2973 sc->stats.rlpic += IGC_READ_REG(&sc->hw, IGC_RLPIC); 2974 sc->stats.gprc += IGC_READ_REG(&sc->hw, IGC_GPRC); 2975 sc->stats.bprc += IGC_READ_REG(&sc->hw, IGC_BPRC); 2976 sc->stats.mprc += IGC_READ_REG(&sc->hw, IGC_MPRC); 2977 sc->stats.gptc += IGC_READ_REG(&sc->hw, IGC_GPTC); 2978 2979 /* For the 64-bit byte counters the low dword must be read first. */ 2980 /* Both registers clear on the read of the high dword */ 2981 2982 sc->stats.gorc += IGC_READ_REG(&sc->hw, IGC_GORCL) + 2983 ((u64)IGC_READ_REG(&sc->hw, IGC_GORCH) << 32); 2984 sc->stats.gotc += IGC_READ_REG(&sc->hw, IGC_GOTCL) + 2985 ((u64)IGC_READ_REG(&sc->hw, IGC_GOTCH) << 32); 2986 2987 sc->stats.rnbc += IGC_READ_REG(&sc->hw, IGC_RNBC); 2988 sc->stats.ruc += IGC_READ_REG(&sc->hw, IGC_RUC); 2989 sc->stats.rfc += IGC_READ_REG(&sc->hw, IGC_RFC); 2990 sc->stats.roc += IGC_READ_REG(&sc->hw, IGC_ROC); 2991 sc->stats.rjc += IGC_READ_REG(&sc->hw, IGC_RJC); 2992 2993 sc->stats.mgprc += IGC_READ_REG(&sc->hw, IGC_MGTPRC); 2994 sc->stats.mgpdc += IGC_READ_REG(&sc->hw, IGC_MGTPDC); 2995 sc->stats.mgptc += IGC_READ_REG(&sc->hw, IGC_MGTPTC); 2996 2997 sc->stats.tor += IGC_READ_REG(&sc->hw, IGC_TORH); 2998 sc->stats.tot += IGC_READ_REG(&sc->hw, IGC_TOTH); 2999 3000 sc->stats.tpr += IGC_READ_REG(&sc->hw, IGC_TPR); 3001 sc->stats.tpt += IGC_READ_REG(&sc->hw, IGC_TPT); 3002 sc->stats.ptc64 += IGC_READ_REG(&sc->hw, IGC_PTC64); 3003 sc->stats.ptc127 += IGC_READ_REG(&sc->hw, IGC_PTC127); 3004 sc->stats.ptc255 += IGC_READ_REG(&sc->hw, IGC_PTC255); 3005 sc->stats.ptc511 += IGC_READ_REG(&sc->hw, IGC_PTC511); 3006 sc->stats.ptc1023 += IGC_READ_REG(&sc->hw, IGC_PTC1023); 3007 sc->stats.ptc1522 += IGC_READ_REG(&sc->hw, IGC_PTC1522); 3008 sc->stats.mptc += IGC_READ_REG(&sc->hw, IGC_MPTC); 3009 sc->stats.bptc += IGC_READ_REG(&sc->hw, IGC_BPTC); 3010 3011 /* Interrupt Counts */ 3012 sc->stats.iac += IGC_READ_REG(&sc->hw, IGC_IAC); 3013 sc->stats.rxdmtc += IGC_READ_REG(&sc->hw, IGC_RXDMTC); 3014 3015 sc->stats.algnerrc += IGC_READ_REG(&sc->hw, IGC_ALGNERRC); 3016 sc->stats.tncrs += IGC_READ_REG(&sc->hw, IGC_TNCRS); 3017 sc->stats.htdpmc += IGC_READ_REG(&sc->hw, IGC_HTDPMC); 3018 sc->stats.tsctc += IGC_READ_REG(&sc->hw, IGC_TSCTC); 3019 3020 igc_update_ecc_stats(sc); 3021 } 3022 3023 static uint64_t 3024 igc_if_get_counter(if_ctx_t ctx, ift_counter cnt) 3025 { 3026 struct igc_softc *sc = iflib_get_softc(ctx); 3027 if_t ifp = iflib_get_ifp(ctx); 3028 3029 switch (cnt) { 3030 case IFCOUNTER_COLLISIONS: 3031 return (sc->stats.colc); 3032 case IFCOUNTER_IERRORS: 3033 /* 3034 * RERC overlaps the counters below and, on I225, omits length 3035 * errors. RFC covers bad-CRC runts that CRCERRS does not count. 3036 */ 3037 return (sc->dropped_pkts + sc->stats.rxerrc + 3038 sc->stats.crcerrs + sc->stats.algnerrc + 3039 sc->stats.ruc + sc->stats.rfc + sc->stats.roc + 3040 sc->stats.mpc); 3041 case IFCOUNTER_OERRORS: 3042 return (if_get_counter_default(ifp, cnt) + 3043 sc->stats.ecol + sc->stats.latecol); 3044 default: 3045 return (if_get_counter_default(ifp, cnt)); 3046 } 3047 } 3048 3049 /* igc_if_needs_restart - Tell iflib when the driver needs to be reinitialized 3050 * @ctx: iflib context 3051 * @event: event code to check 3052 * 3053 * Defaults to returning false for unknown events. 3054 * 3055 * @returns true if iflib needs to reinit the interface 3056 */ 3057 static bool 3058 igc_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) 3059 { 3060 switch (event) { 3061 case IFLIB_RESTART_VLAN_CONFIG: 3062 default: 3063 return (false); 3064 } 3065 } 3066 3067 /* Export a single 32-bit register via a read-only sysctl. */ 3068 static int 3069 igc_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) 3070 { 3071 struct igc_softc *sc; 3072 u_int val; 3073 3074 sc = oidp->oid_arg1; 3075 val = IGC_READ_REG(&sc->hw, oidp->oid_arg2); 3076 return (sysctl_handle_int(oidp, &val, 0, req)); 3077 } 3078 3079 /* Per queue holdoff interrupt rate handler */ 3080 static int 3081 igc_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) 3082 { 3083 struct igc_rx_queue *rque; 3084 struct igc_tx_queue *tque; 3085 struct igc_hw *hw; 3086 int error; 3087 u32 reg, usec, rate; 3088 3089 bool tx = oidp->oid_arg2; 3090 3091 if (tx) { 3092 tque = oidp->oid_arg1; 3093 hw = &tque->sc->hw; 3094 reg = IGC_READ_REG(hw, IGC_EITR(tque->msix)); 3095 } else { 3096 rque = oidp->oid_arg1; 3097 hw = &rque->sc->hw; 3098 reg = IGC_READ_REG(hw, IGC_EITR(rque->msix)); 3099 } 3100 3101 usec = (reg & IGC_QVECTOR_MASK); 3102 if (usec > 0) 3103 rate = IGC_EITR_TO_INTS(usec); 3104 else 3105 rate = 0; 3106 3107 error = sysctl_handle_int(oidp, &rate, 0, req); 3108 if (error || !req->newptr) 3109 return error; 3110 return 0; 3111 } 3112 3113 /* 3114 * Add sysctl variables, one per statistic, to the system. 3115 */ 3116 static void 3117 igc_add_hw_stats(struct igc_softc *sc) 3118 { 3119 device_t dev = iflib_get_dev(sc->ctx); 3120 struct igc_tx_queue *tx_que = sc->tx_queues; 3121 struct igc_rx_queue *rx_que = sc->rx_queues; 3122 3123 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3124 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3125 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3126 struct igc_hw_stats *stats = &sc->stats; 3127 3128 struct sysctl_oid *eee_node, *memerr_node, *stat_node, *queue_node, 3129 *int_node; 3130 struct sysctl_oid_list *eee_list, *memerr_list, *stat_list, *queue_list, 3131 *int_list; 3132 3133 #define QUEUE_NAME_LEN 32 3134 char namebuf[QUEUE_NAME_LEN]; 3135 3136 /* Driver Statistics */ 3137 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 3138 CTLFLAG_RD, &sc->dropped_pkts, 3139 "Driver dropped packets"); 3140 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 3141 CTLFLAG_RD, &sc->link_irq, 3142 "Link MSI-X IRQ Handled"); 3143 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", 3144 CTLFLAG_RD, &sc->rx_overruns, 3145 "RX overruns"); 3146 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", 3147 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 3148 sc, IGC_CTRL, igc_sysctl_reg_handler, "IU", 3149 "Device Control Register"); 3150 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", 3151 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 3152 sc, IGC_RCTL, igc_sysctl_reg_handler, "IU", 3153 "Receiver Control Register"); 3154 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 3155 CTLFLAG_RD, &sc->hw.fc.high_water, 0, 3156 "Flow Control High Watermark"); 3157 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 3158 CTLFLAG_RD, &sc->hw.fc.low_water, 0, 3159 "Flow Control Low Watermark"); 3160 memerr_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "memory_errors", 3161 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 3162 "Internal memory error indications"); 3163 memerr_list = SYSCTL_CHILDREN(memerr_node); 3164 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "fatal_lan", 3165 CTLFLAG_RD, &sc->fatal_error_lan_count, 3166 "Fatal LAN-port memory error indications"); 3167 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "fatal_management", 3168 CTLFLAG_RD, &sc->fatal_error_mng_count, 3169 "Fatal management-memory error indications"); 3170 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "fatal_pcie", 3171 CTLFLAG_RD, &sc->fatal_error_pcie_count, 3172 "Fatal PCIe memory error indications"); 3173 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "fatal_dma", 3174 CTLFLAG_RD, &sc->fatal_error_dma_count, 3175 "Fatal DMA memory error indications"); 3176 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "fatal_unknown", 3177 CTLFLAG_RD, &sc->fatal_error_unknown_count, 3178 "Fatal memory errors without a reported region"); 3179 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, "corrected_dma", 3180 CTLFLAG_RD, &sc->corrected_error_dma_count, 3181 "Corrected DMA memory error indications"); 3182 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 3183 "corrected_pcie_tx_data", CTLFLAG_RD, 3184 &sc->corrected_error_pcie_tx_data_count, 3185 "Corrected PCIe transmit-data memory error indications"); 3186 SYSCTL_ADD_UQUAD(ctx, memerr_list, OID_AUTO, 3187 "corrected_pcie_retry", CTLFLAG_RD, 3188 &sc->corrected_error_pcie_retry_count, 3189 "Corrected PCIe retry-buffer memory error indications"); 3190 eee_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "eee", 3191 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 3192 "Energy Efficient Ethernet statistics"); 3193 eee_list = SYSCTL_CHILDREN(eee_node); 3194 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "tx_lpi_count", 3195 CTLFLAG_RD, &stats->tlpic, "TX LPI event count"); 3196 SYSCTL_ADD_UQUAD(ctx, eee_list, OID_AUTO, "rx_lpi_count", 3197 CTLFLAG_RD, &stats->rlpic, "RX LPI event count"); 3198 3199 for (int i = 0; i < sc->tx_num_queues; i++, tx_que++) { 3200 struct tx_ring *txr = &tx_que->txr; 3201 snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); 3202 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3203 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TX Queue Name"); 3204 queue_list = SYSCTL_CHILDREN(queue_node); 3205 3206 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 3207 CTLTYPE_UINT | CTLFLAG_RD, tx_que, 3208 true, igc_sysctl_interrupt_rate_handler, "IU", 3209 "Interrupt Rate"); 3210 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 3211 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 3212 IGC_TDH(txr->me), igc_sysctl_reg_handler, "IU", 3213 "Transmit Descriptor Head"); 3214 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 3215 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 3216 IGC_TDT(txr->me), igc_sysctl_reg_handler, "IU", 3217 "Transmit Descriptor Tail"); 3218 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq", 3219 CTLFLAG_RD, &txr->tx_irq, 3220 "Queue MSI-X Transmit Interrupts"); 3221 } 3222 3223 for (int j = 0; j < sc->rx_num_queues; j++, rx_que++) { 3224 struct rx_ring *rxr = &rx_que->rxr; 3225 snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); 3226 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 3227 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "RX Queue Name"); 3228 queue_list = SYSCTL_CHILDREN(queue_node); 3229 3230 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 3231 CTLTYPE_UINT | CTLFLAG_RD, rx_que, 3232 false, igc_sysctl_interrupt_rate_handler, "IU", 3233 "Interrupt Rate"); 3234 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 3235 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 3236 IGC_RDH(rxr->me), igc_sysctl_reg_handler, "IU", 3237 "Receive Descriptor Head"); 3238 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 3239 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 3240 IGC_RDT(rxr->me), igc_sysctl_reg_handler, "IU", 3241 "Receive Descriptor Tail"); 3242 SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq", 3243 CTLFLAG_RD, &rxr->rx_irq, 3244 "Queue MSI-X Receive Interrupts"); 3245 } 3246 3247 /* MAC stats get their own sub node */ 3248 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 3249 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Statistics"); 3250 stat_list = SYSCTL_CHILDREN(stat_node); 3251 3252 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", 3253 CTLFLAG_RD, &stats->ecol, 3254 "Excessive collisions"); 3255 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", 3256 CTLFLAG_RD, &stats->scc, 3257 "Single collisions"); 3258 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 3259 CTLFLAG_RD, &stats->mcc, 3260 "Multiple collisions"); 3261 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", 3262 CTLFLAG_RD, &stats->latecol, 3263 "Late collisions"); 3264 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", 3265 CTLFLAG_RD, &stats->colc, 3266 "Collision Count"); 3267 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 3268 CTLFLAG_RD, &sc->stats.symerrs, 3269 "Symbol Errors"); 3270 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 3271 CTLFLAG_RD, &sc->stats.sec, 3272 "Sequence Errors"); 3273 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", 3274 CTLFLAG_RD, &sc->stats.dc, 3275 "Defer Count"); 3276 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", 3277 CTLFLAG_RD, &sc->stats.mpc, 3278 "Missed Packets"); 3279 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", 3280 CTLFLAG_RD, &sc->stats.rlec, 3281 "Receive Length Errors"); 3282 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 3283 CTLFLAG_RD, &sc->stats.rnbc, 3284 "Receive No Buffers"); 3285 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 3286 CTLFLAG_RD, &sc->stats.ruc, 3287 "Receive Undersize"); 3288 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 3289 CTLFLAG_RD, &sc->stats.rfc, 3290 "Fragmented Packets Received "); 3291 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 3292 CTLFLAG_RD, &sc->stats.roc, 3293 "Oversized Packets Received"); 3294 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 3295 CTLFLAG_RD, &sc->stats.rjc, 3296 "Received Jabber"); 3297 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", 3298 CTLFLAG_RD, &sc->stats.rxerrc, 3299 "Receive Errors"); 3300 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_error_count", 3301 CTLFLAG_RD, &sc->stats.rerc, 3302 "Receive Error Count (RERC)"); 3303 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", 3304 CTLFLAG_RD, &sc->stats.crcerrs, 3305 "CRC errors"); 3306 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 3307 CTLFLAG_RD, &sc->stats.algnerrc, 3308 "Alignment Errors"); 3309 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 3310 CTLFLAG_RD, &sc->stats.xonrxc, 3311 "XON Received"); 3312 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", 3313 CTLFLAG_RD, &sc->stats.xontxc, 3314 "XON Transmitted"); 3315 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 3316 CTLFLAG_RD, &sc->stats.xoffrxc, 3317 "XOFF Received"); 3318 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 3319 CTLFLAG_RD, &sc->stats.xofftxc, 3320 "XOFF Transmitted"); 3321 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", 3322 CTLFLAG_RD, &sc->stats.fcruc, 3323 "Unsupported Flow Control Received"); 3324 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", 3325 CTLFLAG_RD, &sc->stats.mgprc, 3326 "Management Packets Received"); 3327 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", 3328 CTLFLAG_RD, &sc->stats.mgpdc, 3329 "Management Packets Dropped"); 3330 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", 3331 CTLFLAG_RD, &sc->stats.mgptc, 3332 "Management Packets Transmitted"); 3333 3334 /* Packet Reception Stats */ 3335 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 3336 CTLFLAG_RD, &sc->stats.tpr, 3337 "Total Packets Received "); 3338 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 3339 CTLFLAG_RD, &sc->stats.gprc, 3340 "Good Packets Received"); 3341 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 3342 CTLFLAG_RD, &sc->stats.bprc, 3343 "Broadcast Packets Received"); 3344 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 3345 CTLFLAG_RD, &sc->stats.mprc, 3346 "Multicast Packets Received"); 3347 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 3348 CTLFLAG_RD, &sc->stats.prc64, 3349 "64 byte frames received "); 3350 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 3351 CTLFLAG_RD, &sc->stats.prc127, 3352 "65-127 byte frames received"); 3353 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 3354 CTLFLAG_RD, &sc->stats.prc255, 3355 "128-255 byte frames received"); 3356 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 3357 CTLFLAG_RD, &sc->stats.prc511, 3358 "256-511 byte frames received"); 3359 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 3360 CTLFLAG_RD, &sc->stats.prc1023, 3361 "512-1023 byte frames received"); 3362 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 3363 CTLFLAG_RD, &sc->stats.prc1522, 3364 "1023-1522 byte frames received"); 3365 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 3366 CTLFLAG_RD, &sc->stats.gorc, 3367 "Good Octets Received"); 3368 3369 /* Packet Transmission Stats */ 3370 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 3371 CTLFLAG_RD, &sc->stats.gotc, 3372 "Good Octets Transmitted"); 3373 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 3374 CTLFLAG_RD, &sc->stats.tpt, 3375 "Total Packets Transmitted"); 3376 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 3377 CTLFLAG_RD, &sc->stats.gptc, 3378 "Good Packets Transmitted"); 3379 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "host_tx_discarded", 3380 CTLFLAG_RD, &sc->stats.htdpmc, 3381 "Host Packets Discarded by Transmit MAC"); 3382 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 3383 CTLFLAG_RD, &sc->stats.bptc, 3384 "Broadcast Packets Transmitted"); 3385 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 3386 CTLFLAG_RD, &sc->stats.mptc, 3387 "Multicast Packets Transmitted"); 3388 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 3389 CTLFLAG_RD, &sc->stats.ptc64, 3390 "64 byte frames transmitted "); 3391 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 3392 CTLFLAG_RD, &sc->stats.ptc127, 3393 "65-127 byte frames transmitted"); 3394 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 3395 CTLFLAG_RD, &sc->stats.ptc255, 3396 "128-255 byte frames transmitted"); 3397 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 3398 CTLFLAG_RD, &sc->stats.ptc511, 3399 "256-511 byte frames transmitted"); 3400 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 3401 CTLFLAG_RD, &sc->stats.ptc1023, 3402 "512-1023 byte frames transmitted"); 3403 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 3404 CTLFLAG_RD, &sc->stats.ptc1522, 3405 "1024-1522 byte frames transmitted"); 3406 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", 3407 CTLFLAG_RD, &sc->stats.tsctc, 3408 "TSO Contexts Transmitted"); 3409 3410 /* Interrupt Stats */ 3411 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 3412 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Interrupt Statistics"); 3413 int_list = SYSCTL_CHILDREN(int_node); 3414 3415 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts", 3416 CTLFLAG_RD, &sc->stats.iac, 3417 "Interrupt Assertion Count"); 3418 3419 SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 3420 CTLFLAG_RD, &sc->stats.rxdmtc, 3421 "Rx Desc Min Thresh Count"); 3422 } 3423 3424 static void 3425 igc_fw_version(struct igc_softc *sc) 3426 { 3427 struct igc_hw *hw = &sc->hw; 3428 struct igc_fw_version *fw_ver = &sc->fw_ver; 3429 3430 *fw_ver = (struct igc_fw_version){0}; 3431 3432 igc_get_fw_version(hw, fw_ver); 3433 } 3434 3435 static void 3436 igc_sbuf_fw_version(struct igc_fw_version *fw_ver, struct sbuf *buf) 3437 { 3438 const char *space = ""; 3439 3440 if (fw_ver->eep_major || fw_ver->eep_minor || fw_ver->eep_build) { 3441 sbuf_printf(buf, "EEPROM V%d.%d-%d", fw_ver->eep_major, 3442 fw_ver->eep_minor, fw_ver->eep_build); 3443 space = " "; 3444 } 3445 3446 if (fw_ver->invm_major || fw_ver->invm_minor || 3447 fw_ver->invm_img_type) { 3448 sbuf_printf(buf, "%sNVM V%d.%d imgtype%d", 3449 space, fw_ver->invm_major, fw_ver->invm_minor, 3450 fw_ver->invm_img_type); 3451 space = " "; 3452 } 3453 3454 if (fw_ver->or_valid) { 3455 sbuf_printf(buf, "%sOption ROM V%d-b%d-p%d", 3456 space, fw_ver->or_major, fw_ver->or_build, 3457 fw_ver->or_patch); 3458 space = " "; 3459 } 3460 3461 if (fw_ver->etrack_id) 3462 sbuf_printf(buf, "%seTrack 0x%08x", space, fw_ver->etrack_id); 3463 } 3464 3465 static void 3466 igc_print_fw_version(struct igc_softc *sc ) 3467 { 3468 device_t dev = sc->dev; 3469 struct sbuf *buf; 3470 int error = 0; 3471 3472 buf = sbuf_new_auto(); 3473 if (!buf) { 3474 device_printf(dev, "Could not allocate sbuf for output.\n"); 3475 return; 3476 } 3477 3478 igc_sbuf_fw_version(&sc->fw_ver, buf); 3479 3480 error = sbuf_finish(buf); 3481 if (error) 3482 device_printf(dev, "Error finishing sbuf: %d\n", error); 3483 else if (sbuf_len(buf)) 3484 device_printf(dev, "%s\n", sbuf_data(buf)); 3485 3486 sbuf_delete(buf); 3487 } 3488 3489 static int 3490 igc_sysctl_print_fw_version(SYSCTL_HANDLER_ARGS) 3491 { 3492 struct igc_softc *sc = (struct igc_softc *)arg1; 3493 device_t dev = sc->dev; 3494 struct sbuf *buf; 3495 int error = 0; 3496 3497 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 3498 if (!buf) { 3499 device_printf(dev, "Could not allocate sbuf for output.\n"); 3500 return (ENOMEM); 3501 } 3502 3503 igc_sbuf_fw_version(&sc->fw_ver, buf); 3504 3505 error = sbuf_finish(buf); 3506 if (error) 3507 device_printf(dev, "Error finishing sbuf: %d\n", error); 3508 3509 sbuf_delete(buf); 3510 3511 return (0); 3512 } 3513 3514 /********************************************************************** 3515 * 3516 * This routine provides a way to dump out the adapter eeprom, 3517 * often a useful debug/service tool. This only dumps the first 3518 * 32 words, stuff that matters is in that extent. 3519 * 3520 **********************************************************************/ 3521 static int 3522 igc_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 3523 { 3524 struct igc_softc *sc = (struct igc_softc *)arg1; 3525 int error; 3526 int result; 3527 3528 result = -1; 3529 error = sysctl_handle_int(oidp, &result, 0, req); 3530 3531 if (error || !req->newptr) 3532 return (error); 3533 3534 /* 3535 * This value will cause a hex dump of the 3536 * first 32 16-bit words of the EEPROM to 3537 * the screen. 3538 */ 3539 if (result == 1) 3540 igc_print_nvm_info(sc); 3541 3542 return (error); 3543 } 3544 3545 static void 3546 igc_print_nvm_info(struct igc_softc *sc) 3547 { 3548 u16 eeprom_data; 3549 int i, j, row = 0; 3550 3551 /* Its a bit crude, but it gets the job done */ 3552 printf("\nInterface EEPROM Dump:\n"); 3553 printf("Offset\n0x0000 "); 3554 for (i = 0, j = 0; i < 32; i++, j++) { 3555 if (j == 8) { /* Make the offset block */ 3556 j = 0; ++row; 3557 printf("\n0x00%x0 ",row); 3558 } 3559 igc_read_nvm(&sc->hw, i, 1, &eeprom_data); 3560 printf("%04x ", eeprom_data); 3561 } 3562 printf("\n"); 3563 } 3564 3565 static int 3566 igc_sysctl_tso_tcp_flags_mask(SYSCTL_HANDLER_ARGS) 3567 { 3568 struct igc_softc *sc; 3569 u32 reg, val, shift; 3570 int error, mask; 3571 3572 sc = oidp->oid_arg1; 3573 switch (oidp->oid_arg2) { 3574 case 0: 3575 reg = IGC_DTXTCPFLGL; 3576 shift = 0; 3577 break; 3578 case 1: 3579 reg = IGC_DTXTCPFLGL; 3580 shift = 16; 3581 break; 3582 case 2: 3583 reg = IGC_DTXTCPFLGH; 3584 shift = 0; 3585 break; 3586 default: 3587 return (EINVAL); 3588 break; 3589 } 3590 val = IGC_READ_REG(&sc->hw, reg); 3591 mask = (val >> shift) & 0xfff; 3592 error = sysctl_handle_int(oidp, &mask, 0, req); 3593 if (error != 0 || req->newptr == NULL) 3594 return (error); 3595 if (mask < 0 || mask > 0xfff) 3596 return (EINVAL); 3597 val = (val & ~(0xfff << shift)) | (mask << shift); 3598 IGC_WRITE_REG(&sc->hw, reg, val); 3599 return (0); 3600 } 3601 3602 /* 3603 * Set flow control using sysctl: 3604 * Flow control values: 3605 * 0 - off 3606 * 1 - rx pause 3607 * 2 - tx pause 3608 * 3 - full 3609 */ 3610 static int 3611 igc_set_flowcntl(SYSCTL_HANDLER_ARGS) 3612 { 3613 int error; 3614 static int input = 3; /* default is full */ 3615 struct igc_softc *sc = (struct igc_softc *) arg1; 3616 3617 error = sysctl_handle_int(oidp, &input, 0, req); 3618 3619 if ((error) || (req->newptr == NULL)) 3620 return (error); 3621 3622 if (input == sc->fc) /* no change? */ 3623 return (error); 3624 3625 switch (input) { 3626 case igc_fc_rx_pause: 3627 case igc_fc_tx_pause: 3628 case igc_fc_full: 3629 case igc_fc_none: 3630 sc->hw.fc.requested_mode = input; 3631 sc->fc = input; 3632 break; 3633 default: 3634 /* Do nothing */ 3635 return (error); 3636 } 3637 3638 sc->hw.fc.current_mode = sc->hw.fc.requested_mode; 3639 igc_force_mac_fc(&sc->hw); 3640 return (error); 3641 } 3642 3643 static void 3644 igc_sysctl_request_reinit(struct igc_softc *sc) 3645 { 3646 if ((if_getflags(iflib_get_ifp(sc->ctx)) & IFF_UP) == 0) 3647 return; 3648 3649 iflib_request_reset(sc->ctx); 3650 iflib_admin_intr_deferred(sc->ctx); 3651 } 3652 3653 /* 3654 * Manage DMA Coalesce: 3655 * Control values: 3656 * 0/1 - off/on 3657 * Legal timer values are: 3658 * 250,500,1000-10000 in thousands 3659 */ 3660 static int 3661 igc_sysctl_dmac(SYSCTL_HANDLER_ARGS) 3662 { 3663 struct igc_softc *sc = (struct igc_softc *) arg1; 3664 int error; 3665 3666 error = sysctl_handle_int(oidp, &sc->dmac, 0, req); 3667 3668 if ((error) || (req->newptr == NULL)) 3669 return (error); 3670 3671 switch (sc->dmac) { 3672 case 0: 3673 /* Disabling */ 3674 break; 3675 case 1: /* Just enable and use default */ 3676 sc->dmac = 1000; 3677 break; 3678 case 250: 3679 case 500: 3680 case 1000: 3681 case 2000: 3682 case 3000: 3683 case 4000: 3684 case 5000: 3685 case 6000: 3686 case 7000: 3687 case 8000: 3688 case 9000: 3689 case 10000: 3690 /* Legal values - allow */ 3691 break; 3692 default: 3693 /* Do nothing, illegal value */ 3694 sc->dmac = 0; 3695 return (EINVAL); 3696 } 3697 /* Reinit the interface */ 3698 igc_sysctl_request_reinit(sc); 3699 return (error); 3700 } 3701 3702 /* 3703 * Manage Energy Efficient Ethernet: 3704 * Control values: 3705 * 0/1 - enabled/disabled 3706 */ 3707 static int 3708 igc_sysctl_eee(SYSCTL_HANDLER_ARGS) 3709 { 3710 struct igc_softc *sc = (struct igc_softc *) arg1; 3711 int error, value; 3712 3713 value = sc->hw.dev_spec._i225.eee_disable; 3714 error = sysctl_handle_int(oidp, &value, 0, req); 3715 if (error || req->newptr == NULL) 3716 return (error); 3717 3718 sc->hw.dev_spec._i225.eee_disable = (value != 0); 3719 igc_sysctl_request_reinit(sc); 3720 3721 return (0); 3722 } 3723 3724 static int 3725 igc_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 3726 { 3727 struct igc_softc *sc; 3728 int error; 3729 int result; 3730 3731 result = -1; 3732 error = sysctl_handle_int(oidp, &result, 0, req); 3733 3734 if (error || !req->newptr) 3735 return (error); 3736 3737 if (result == 1) { 3738 sc = (struct igc_softc *) arg1; 3739 igc_print_debug_info(sc); 3740 } 3741 3742 return (error); 3743 } 3744 3745 static int 3746 igc_get_rs(SYSCTL_HANDLER_ARGS) 3747 { 3748 struct igc_softc *sc = (struct igc_softc *) arg1; 3749 int error; 3750 int result; 3751 3752 result = 0; 3753 error = sysctl_handle_int(oidp, &result, 0, req); 3754 3755 if (error || !req->newptr || result != 1) 3756 return (error); 3757 igc_dump_rs(sc); 3758 3759 return (error); 3760 } 3761 3762 static void 3763 igc_if_debug(if_ctx_t ctx) 3764 { 3765 igc_dump_rs(iflib_get_softc(ctx)); 3766 } 3767 3768 /* 3769 * This routine is meant to be fluid, add whatever is 3770 * needed for debugging a problem. -jfv 3771 */ 3772 static void 3773 igc_print_debug_info(struct igc_softc *sc) 3774 { 3775 device_t dev = iflib_get_dev(sc->ctx); 3776 if_t ifp = iflib_get_ifp(sc->ctx); 3777 3778 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) 3779 printf("Interface is RUNNING "); 3780 else 3781 printf("Interface is NOT RUNNING\n"); 3782 3783 if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE) 3784 printf("and INACTIVE\n"); 3785 else 3786 printf("and ACTIVE\n"); 3787 3788 for (int i = 0; i < sc->tx_num_queues; i++) { 3789 device_printf(dev, "TX Queue %d ------\n", i); 3790 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 3791 IGC_READ_REG(&sc->hw, IGC_TDH(i)), 3792 IGC_READ_REG(&sc->hw, IGC_TDT(i))); 3793 3794 } 3795 for (int j = 0; j < sc->rx_num_queues; j++) { 3796 device_printf(dev, "RX Queue %d ------\n", j); 3797 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 3798 IGC_READ_REG(&sc->hw, IGC_RDH(j)), 3799 IGC_READ_REG(&sc->hw, IGC_RDT(j))); 3800 } 3801 } 3802