1 /****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #if __FreeBSD_version >= 800000 43 #include <sys/buf_ring.h> 44 #endif 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/kernel.h> 48 #include <sys/kthread.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/rman.h> 53 #include <sys/socket.h> 54 #include <sys/sockio.h> 55 #include <sys/sysctl.h> 56 #include <sys/taskqueue.h> 57 #include <sys/eventhandler.h> 58 #include <machine/bus.h> 59 #include <machine/resource.h> 60 61 #include <net/bpf.h> 62 #include <net/ethernet.h> 63 #include <net/if.h> 64 #include <net/if_arp.h> 65 #include <net/if_dl.h> 66 #include <net/if_media.h> 67 68 #include <net/if_types.h> 69 #include <net/if_vlan_var.h> 70 71 #include <netinet/in_systm.h> 72 #include <netinet/in.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip6.h> 76 #include <netinet/tcp.h> 77 #include <netinet/udp.h> 78 79 #include <machine/in_cksum.h> 80 #include <dev/led/led.h> 81 #include <dev/pci/pcivar.h> 82 #include <dev/pci/pcireg.h> 83 84 #include "e1000_api.h" 85 #include "e1000_82571.h" 86 #include "if_em.h" 87 88 /********************************************************************* 89 * Set this to one to display debug statistics 90 *********************************************************************/ 91 int em_display_debug_stats = 0; 92 93 /********************************************************************* 94 * Driver version: 95 *********************************************************************/ 96 char em_driver_version[] = "7.0.4"; 97 98 99 /********************************************************************* 100 * PCI Device ID Table 101 * 102 * Used by probe to select devices to load on 103 * Last field stores an index into e1000_strings 104 * Last entry must be all 0s 105 * 106 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 107 *********************************************************************/ 108 109 static em_vendor_info_t em_vendor_info_array[] = 110 { 111 /* Intel(R) PRO/1000 Network Connection */ 112 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 113 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 116 PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 118 PCI_ANY_ID, PCI_ANY_ID, 0}, 119 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 120 PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 122 PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 124 PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 126 PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 132 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 137 PCI_ANY_ID, PCI_ANY_ID, 0}, 138 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 139 PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 141 PCI_ANY_ID, PCI_ANY_ID, 0}, 142 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 143 PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, 152 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 153 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 154 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 158 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 162 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 168 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, 170 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, 172 /* required last entry */ 173 { 0, 0, 0, 0, 0} 174 }; 175 176 /********************************************************************* 177 * Table of branding strings for all supported NICs. 178 *********************************************************************/ 179 180 static char *em_strings[] = { 181 "Intel(R) PRO/1000 Network Connection" 182 }; 183 184 /********************************************************************* 185 * Function prototypes 186 *********************************************************************/ 187 static int em_probe(device_t); 188 static int em_attach(device_t); 189 static int em_detach(device_t); 190 static int em_shutdown(device_t); 191 static int em_suspend(device_t); 192 static int em_resume(device_t); 193 static void em_start(struct ifnet *); 194 static void em_start_locked(struct ifnet *, struct tx_ring *); 195 #ifdef EM_MULTIQUEUE 196 static int em_mq_start(struct ifnet *, struct mbuf *); 197 static int em_mq_start_locked(struct ifnet *, 198 struct tx_ring *, struct mbuf *); 199 static void em_qflush(struct ifnet *); 200 #endif 201 static int em_ioctl(struct ifnet *, u_long, caddr_t); 202 static void em_init(void *); 203 static void em_init_locked(struct adapter *); 204 static void em_stop(void *); 205 static void em_media_status(struct ifnet *, struct ifmediareq *); 206 static int em_media_change(struct ifnet *); 207 static void em_identify_hardware(struct adapter *); 208 static int em_allocate_pci_resources(struct adapter *); 209 static int em_allocate_legacy(struct adapter *); 210 static int em_allocate_msix(struct adapter *); 211 static int em_allocate_queues(struct adapter *); 212 static int em_setup_msix(struct adapter *); 213 static void em_free_pci_resources(struct adapter *); 214 static void em_local_timer(void *); 215 static void em_reset(struct adapter *); 216 static void em_setup_interface(device_t, struct adapter *); 217 218 static void em_setup_transmit_structures(struct adapter *); 219 static void em_initialize_transmit_unit(struct adapter *); 220 static int em_allocate_transmit_buffers(struct tx_ring *); 221 static void em_free_transmit_structures(struct adapter *); 222 static void em_free_transmit_buffers(struct tx_ring *); 223 224 static int em_setup_receive_structures(struct adapter *); 225 static int em_allocate_receive_buffers(struct rx_ring *); 226 static void em_initialize_receive_unit(struct adapter *); 227 static void em_free_receive_structures(struct adapter *); 228 static void em_free_receive_buffers(struct rx_ring *); 229 230 static void em_enable_intr(struct adapter *); 231 static void em_disable_intr(struct adapter *); 232 static void em_update_stats_counters(struct adapter *); 233 static bool em_txeof(struct tx_ring *); 234 static int em_rxeof(struct rx_ring *, int); 235 #ifndef __NO_STRICT_ALIGNMENT 236 static int em_fixup_rx(struct rx_ring *); 237 #endif 238 static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); 239 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, 240 u32 *, u32 *); 241 static bool em_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); 242 static void em_set_promisc(struct adapter *); 243 static void em_disable_promisc(struct adapter *); 244 static void em_set_multi(struct adapter *); 245 static void em_print_hw_stats(struct adapter *); 246 static void em_update_link_status(struct adapter *); 247 static void em_refresh_mbufs(struct rx_ring *, int); 248 static void em_register_vlan(void *, struct ifnet *, u16); 249 static void em_unregister_vlan(void *, struct ifnet *, u16); 250 static void em_setup_vlan_hw_support(struct adapter *); 251 static int em_xmit(struct tx_ring *, struct mbuf **); 252 static int em_dma_malloc(struct adapter *, bus_size_t, 253 struct em_dma_alloc *, int); 254 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 255 static void em_print_debug_info(struct adapter *); 256 static void em_print_nvm_info(struct adapter *); 257 static int em_is_valid_ether_addr(u8 *); 258 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 259 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 260 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 261 static void em_add_int_delay_sysctl(struct adapter *, const char *, 262 const char *, struct em_int_delay_info *, int, int); 263 /* Management and WOL Support */ 264 static void em_init_manageability(struct adapter *); 265 static void em_release_manageability(struct adapter *); 266 static void em_get_hw_control(struct adapter *); 267 static void em_release_hw_control(struct adapter *); 268 static void em_get_wakeup(device_t); 269 static void em_enable_wakeup(device_t); 270 static int em_enable_phy_wakeup(struct adapter *); 271 static void em_led_func(void *, int); 272 273 static int em_irq_fast(void *); 274 275 /* MSIX handlers */ 276 static void em_msix_tx(void *); 277 static void em_msix_rx(void *); 278 static void em_msix_link(void *); 279 static void em_handle_tx(void *context, int pending); 280 static void em_handle_rx(void *context, int pending); 281 static void em_handle_link(void *context, int pending); 282 283 static void em_add_rx_process_limit(struct adapter *, const char *, 284 const char *, int *, int); 285 286 #ifdef DEVICE_POLLING 287 static poll_handler_t em_poll; 288 #endif /* POLLING */ 289 290 /********************************************************************* 291 * FreeBSD Device Interface Entry Points 292 *********************************************************************/ 293 294 static device_method_t em_methods[] = { 295 /* Device interface */ 296 DEVMETHOD(device_probe, em_probe), 297 DEVMETHOD(device_attach, em_attach), 298 DEVMETHOD(device_detach, em_detach), 299 DEVMETHOD(device_shutdown, em_shutdown), 300 DEVMETHOD(device_suspend, em_suspend), 301 DEVMETHOD(device_resume, em_resume), 302 {0, 0} 303 }; 304 305 static driver_t em_driver = { 306 "em", em_methods, sizeof(struct adapter), 307 }; 308 309 devclass_t em_devclass; 310 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 311 MODULE_DEPEND(em, pci, 1, 1, 1); 312 MODULE_DEPEND(em, ether, 1, 1, 1); 313 314 /********************************************************************* 315 * Tunable default values. 316 *********************************************************************/ 317 318 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 319 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 320 #define M_TSO_LEN 66 321 322 /* Allow common code without TSO */ 323 #ifndef CSUM_TSO 324 #define CSUM_TSO 0 325 #endif 326 327 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 328 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 329 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 330 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 331 332 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 333 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 334 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 335 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 336 337 static int em_rxd = EM_DEFAULT_RXD; 338 static int em_txd = EM_DEFAULT_TXD; 339 TUNABLE_INT("hw.em.rxd", &em_rxd); 340 TUNABLE_INT("hw.em.txd", &em_txd); 341 342 static int em_smart_pwr_down = FALSE; 343 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 344 345 /* Controls whether promiscuous also shows bad packets */ 346 static int em_debug_sbp = FALSE; 347 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 348 349 /* Local controls for MSI/MSIX */ 350 static int em_enable_msix = TRUE; 351 static int em_msix_queues = 2; /* for 82574, can be 1 or 2 */ 352 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix); 353 TUNABLE_INT("hw.em.msix_queues", &em_msix_queues); 354 355 /* How many packets rxeof tries to clean at a time */ 356 static int em_rx_process_limit = 100; 357 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 358 359 /* Flow control setting - default to FULL */ 360 static int em_fc_setting = e1000_fc_full; 361 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); 362 363 /* 364 ** Shadow VFTA table, this is needed because 365 ** the real vlan filter table gets cleared during 366 ** a soft reset and the driver needs to be able 367 ** to repopulate it. 368 */ 369 static u32 em_shadow_vfta[EM_VFTA_SIZE]; 370 371 /* Global used in WOL setup with multiport cards */ 372 static int global_quad_port_a = 0; 373 374 /********************************************************************* 375 * Device identification routine 376 * 377 * em_probe determines if the driver should be loaded on 378 * adapter based on PCI vendor/device id of the adapter. 379 * 380 * return BUS_PROBE_DEFAULT on success, positive on failure 381 *********************************************************************/ 382 383 static int 384 em_probe(device_t dev) 385 { 386 char adapter_name[60]; 387 u16 pci_vendor_id = 0; 388 u16 pci_device_id = 0; 389 u16 pci_subvendor_id = 0; 390 u16 pci_subdevice_id = 0; 391 em_vendor_info_t *ent; 392 393 INIT_DEBUGOUT("em_probe: begin"); 394 395 pci_vendor_id = pci_get_vendor(dev); 396 if (pci_vendor_id != EM_VENDOR_ID) 397 return (ENXIO); 398 399 pci_device_id = pci_get_device(dev); 400 pci_subvendor_id = pci_get_subvendor(dev); 401 pci_subdevice_id = pci_get_subdevice(dev); 402 403 ent = em_vendor_info_array; 404 while (ent->vendor_id != 0) { 405 if ((pci_vendor_id == ent->vendor_id) && 406 (pci_device_id == ent->device_id) && 407 408 ((pci_subvendor_id == ent->subvendor_id) || 409 (ent->subvendor_id == PCI_ANY_ID)) && 410 411 ((pci_subdevice_id == ent->subdevice_id) || 412 (ent->subdevice_id == PCI_ANY_ID))) { 413 sprintf(adapter_name, "%s %s", 414 em_strings[ent->index], 415 em_driver_version); 416 device_set_desc_copy(dev, adapter_name); 417 return (BUS_PROBE_DEFAULT); 418 } 419 ent++; 420 } 421 422 return (ENXIO); 423 } 424 425 /********************************************************************* 426 * Device initialization routine 427 * 428 * The attach entry point is called when the driver is being loaded. 429 * This routine identifies the type of hardware, allocates all resources 430 * and initializes the hardware. 431 * 432 * return 0 on success, positive on failure 433 *********************************************************************/ 434 435 static int 436 em_attach(device_t dev) 437 { 438 struct adapter *adapter; 439 int error = 0; 440 441 INIT_DEBUGOUT("em_attach: begin"); 442 443 adapter = device_get_softc(dev); 444 adapter->dev = adapter->osdep.dev = dev; 445 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 446 447 /* SYSCTL stuff */ 448 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 449 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 450 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 451 em_sysctl_debug_info, "I", "Debug Information"); 452 453 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 454 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 455 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 456 em_sysctl_stats, "I", "Statistics"); 457 458 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 459 460 /* Determine hardware and mac info */ 461 em_identify_hardware(adapter); 462 463 /* Setup PCI resources */ 464 if (em_allocate_pci_resources(adapter)) { 465 device_printf(dev, "Allocation of PCI resources failed\n"); 466 error = ENXIO; 467 goto err_pci; 468 } 469 470 /* 471 ** For ICH8 and family we need to 472 ** map the flash memory, and this 473 ** must happen after the MAC is 474 ** identified 475 */ 476 if ((adapter->hw.mac.type == e1000_ich8lan) || 477 (adapter->hw.mac.type == e1000_pchlan) || 478 (adapter->hw.mac.type == e1000_ich9lan) || 479 (adapter->hw.mac.type == e1000_ich10lan)) { 480 int rid = EM_BAR_TYPE_FLASH; 481 adapter->flash = bus_alloc_resource_any(dev, 482 SYS_RES_MEMORY, &rid, RF_ACTIVE); 483 if (adapter->flash == NULL) { 484 device_printf(dev, "Mapping of Flash failed\n"); 485 error = ENXIO; 486 goto err_pci; 487 } 488 /* This is used in the shared code */ 489 adapter->hw.flash_address = (u8 *)adapter->flash; 490 adapter->osdep.flash_bus_space_tag = 491 rman_get_bustag(adapter->flash); 492 adapter->osdep.flash_bus_space_handle = 493 rman_get_bushandle(adapter->flash); 494 } 495 496 /* Do Shared Code initialization */ 497 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 498 device_printf(dev, "Setup of Shared code failed\n"); 499 error = ENXIO; 500 goto err_pci; 501 } 502 503 e1000_get_bus_info(&adapter->hw); 504 505 /* Set up some sysctls for the tunable interrupt delays */ 506 em_add_int_delay_sysctl(adapter, "rx_int_delay", 507 "receive interrupt delay in usecs", &adapter->rx_int_delay, 508 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 509 em_add_int_delay_sysctl(adapter, "tx_int_delay", 510 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 511 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 512 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 513 "receive interrupt delay limit in usecs", 514 &adapter->rx_abs_int_delay, 515 E1000_REGISTER(&adapter->hw, E1000_RADV), 516 em_rx_abs_int_delay_dflt); 517 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 518 "transmit interrupt delay limit in usecs", 519 &adapter->tx_abs_int_delay, 520 E1000_REGISTER(&adapter->hw, E1000_TADV), 521 em_tx_abs_int_delay_dflt); 522 523 /* Sysctls for limiting the amount of work done in the taskqueue */ 524 em_add_rx_process_limit(adapter, "rx_processing_limit", 525 "max number of rx packets to process", &adapter->rx_process_limit, 526 em_rx_process_limit); 527 528 /* 529 * Validate number of transmit and receive descriptors. It 530 * must not exceed hardware maximum, and must be multiple 531 * of E1000_DBA_ALIGN. 532 */ 533 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 534 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { 535 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 536 EM_DEFAULT_TXD, em_txd); 537 adapter->num_tx_desc = EM_DEFAULT_TXD; 538 } else 539 adapter->num_tx_desc = em_txd; 540 541 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 542 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { 543 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 544 EM_DEFAULT_RXD, em_rxd); 545 adapter->num_rx_desc = EM_DEFAULT_RXD; 546 } else 547 adapter->num_rx_desc = em_rxd; 548 549 adapter->hw.mac.autoneg = DO_AUTO_NEG; 550 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 551 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 552 553 /* Copper options */ 554 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 555 adapter->hw.phy.mdix = AUTO_ALL_MODES; 556 adapter->hw.phy.disable_polarity_correction = FALSE; 557 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 558 } 559 560 /* 561 * Set the frame limits assuming 562 * standard ethernet sized frames. 563 */ 564 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 565 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 566 567 /* 568 * This controls when hardware reports transmit completion 569 * status. 570 */ 571 adapter->hw.mac.report_tx_early = 1; 572 573 /* 574 ** Get queue/ring memory 575 */ 576 if (em_allocate_queues(adapter)) { 577 error = ENOMEM; 578 goto err_pci; 579 } 580 581 /* 582 ** Start from a known state, this is 583 ** important in reading the nvm and 584 ** mac from that. 585 */ 586 e1000_reset_hw(&adapter->hw); 587 588 /* Make sure we have a good EEPROM before we read from it */ 589 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 590 /* 591 ** Some PCI-E parts fail the first check due to 592 ** the link being in sleep state, call it again, 593 ** if it fails a second time its a real issue. 594 */ 595 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 596 device_printf(dev, 597 "The EEPROM Checksum Is Not Valid\n"); 598 error = EIO; 599 goto err_late; 600 } 601 } 602 603 /* Copy the permanent MAC address out of the EEPROM */ 604 if (e1000_read_mac_addr(&adapter->hw) < 0) { 605 device_printf(dev, "EEPROM read error while reading MAC" 606 " address\n"); 607 error = EIO; 608 goto err_late; 609 } 610 611 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 612 device_printf(dev, "Invalid MAC address\n"); 613 error = EIO; 614 goto err_late; 615 } 616 617 /* 618 ** Do interrupt configuration 619 */ 620 if (adapter->msix > 1) /* Do MSIX */ 621 error = em_allocate_msix(adapter); 622 else /* MSI or Legacy */ 623 error = em_allocate_legacy(adapter); 624 if (error) 625 goto err_late; 626 627 /* 628 * Get Wake-on-Lan and Management info for later use 629 */ 630 em_get_wakeup(dev); 631 632 /* Setup OS specific network interface */ 633 em_setup_interface(dev, adapter); 634 635 em_reset(adapter); 636 637 /* Initialize statistics */ 638 em_update_stats_counters(adapter); 639 640 adapter->hw.mac.get_link_status = 1; 641 em_update_link_status(adapter); 642 643 /* Indicate SOL/IDER usage */ 644 if (e1000_check_reset_block(&adapter->hw)) 645 device_printf(dev, 646 "PHY reset is blocked due to SOL/IDER session.\n"); 647 648 /* Register for VLAN events */ 649 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 650 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 651 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 652 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 653 654 /* Non-AMT based hardware can now take control from firmware */ 655 if (adapter->has_manage && !adapter->has_amt) 656 em_get_hw_control(adapter); 657 658 /* Tell the stack that the interface is not active */ 659 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 660 661 adapter->led_dev = led_create(em_led_func, adapter, 662 device_get_nameunit(dev)); 663 664 INIT_DEBUGOUT("em_attach: end"); 665 666 return (0); 667 668 err_late: 669 em_free_transmit_structures(adapter); 670 em_free_receive_structures(adapter); 671 em_release_hw_control(adapter); 672 err_pci: 673 em_free_pci_resources(adapter); 674 EM_CORE_LOCK_DESTROY(adapter); 675 676 return (error); 677 } 678 679 /********************************************************************* 680 * Device removal routine 681 * 682 * The detach entry point is called when the driver is being removed. 683 * This routine stops the adapter and deallocates all the resources 684 * that were allocated for driver operation. 685 * 686 * return 0 on success, positive on failure 687 *********************************************************************/ 688 689 static int 690 em_detach(device_t dev) 691 { 692 struct adapter *adapter = device_get_softc(dev); 693 struct ifnet *ifp = adapter->ifp; 694 695 INIT_DEBUGOUT("em_detach: begin"); 696 697 /* Make sure VLANS are not using driver */ 698 if (adapter->ifp->if_vlantrunk != NULL) { 699 device_printf(dev,"Vlan in use, detach first\n"); 700 return (EBUSY); 701 } 702 703 #ifdef DEVICE_POLLING 704 if (ifp->if_capenable & IFCAP_POLLING) 705 ether_poll_deregister(ifp); 706 #endif 707 708 EM_CORE_LOCK(adapter); 709 adapter->in_detach = 1; 710 em_stop(adapter); 711 EM_CORE_UNLOCK(adapter); 712 EM_CORE_LOCK_DESTROY(adapter); 713 714 e1000_phy_hw_reset(&adapter->hw); 715 716 em_release_manageability(adapter); 717 em_release_hw_control(adapter); 718 719 /* Unregister VLAN events */ 720 if (adapter->vlan_attach != NULL) 721 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 722 if (adapter->vlan_detach != NULL) 723 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 724 725 ether_ifdetach(adapter->ifp); 726 callout_drain(&adapter->timer); 727 728 em_free_pci_resources(adapter); 729 bus_generic_detach(dev); 730 if_free(ifp); 731 732 em_free_transmit_structures(adapter); 733 em_free_receive_structures(adapter); 734 735 em_release_hw_control(adapter); 736 737 return (0); 738 } 739 740 /********************************************************************* 741 * 742 * Shutdown entry point 743 * 744 **********************************************************************/ 745 746 static int 747 em_shutdown(device_t dev) 748 { 749 return em_suspend(dev); 750 } 751 752 /* 753 * Suspend/resume device methods. 754 */ 755 static int 756 em_suspend(device_t dev) 757 { 758 struct adapter *adapter = device_get_softc(dev); 759 760 EM_CORE_LOCK(adapter); 761 762 em_release_manageability(adapter); 763 em_release_hw_control(adapter); 764 em_enable_wakeup(dev); 765 766 EM_CORE_UNLOCK(adapter); 767 768 return bus_generic_suspend(dev); 769 } 770 771 static int 772 em_resume(device_t dev) 773 { 774 struct adapter *adapter = device_get_softc(dev); 775 struct ifnet *ifp = adapter->ifp; 776 777 if (adapter->led_dev != NULL) 778 led_destroy(adapter->led_dev); 779 780 EM_CORE_LOCK(adapter); 781 em_init_locked(adapter); 782 em_init_manageability(adapter); 783 EM_CORE_UNLOCK(adapter); 784 em_start(ifp); 785 786 return bus_generic_resume(dev); 787 } 788 789 790 /********************************************************************* 791 * Transmit entry point 792 * 793 * em_start is called by the stack to initiate a transmit. 794 * The driver will remain in this routine as long as there are 795 * packets to transmit and transmit resources are available. 796 * In case resources are not available stack is notified and 797 * the packet is requeued. 798 **********************************************************************/ 799 800 #ifdef EM_MULTIQUEUE 801 static int 802 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 803 { 804 struct adapter *adapter = txr->adapter; 805 struct mbuf *next; 806 int err = 0, enq = 0; 807 808 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 809 IFF_DRV_RUNNING || adapter->link_active == 0) { 810 if (m != NULL) 811 err = drbr_enqueue(ifp, txr->br, m); 812 return (err); 813 } 814 815 /* Call cleanup if number of TX descriptors low */ 816 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 817 em_txeof(txr); 818 819 enq = 0; 820 if (m == NULL) { 821 next = drbr_dequeue(ifp, txr->br); 822 } else if (drbr_needs_enqueue(ifp, txr->br)) { 823 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 824 return (err); 825 next = drbr_dequeue(ifp, txr->br); 826 } else 827 next = m; 828 829 /* Process the queue */ 830 while (next != NULL) { 831 if ((err = em_xmit(txr, &next)) != 0) { 832 if (next != NULL) 833 err = drbr_enqueue(ifp, txr->br, next); 834 break; 835 } 836 enq++; 837 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 838 ETHER_BPF_MTAP(ifp, next); 839 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 840 break; 841 if (txr->tx_avail < EM_MAX_SCATTER) { 842 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 843 break; 844 } 845 next = drbr_dequeue(ifp, txr->br); 846 } 847 848 if (enq > 0) { 849 /* Set the watchdog */ 850 txr->watchdog_check = TRUE; 851 txr->watchdog_time = ticks; 852 } 853 return (err); 854 } 855 856 /* 857 ** Multiqueue capable stack interface, this is not 858 ** yet truely multiqueue, but that is coming... 859 */ 860 static int 861 em_mq_start(struct ifnet *ifp, struct mbuf *m) 862 { 863 struct adapter *adapter = ifp->if_softc; 864 struct tx_ring *txr; 865 int i, error = 0; 866 867 /* Which queue to use */ 868 if ((m->m_flags & M_FLOWID) != 0) 869 i = m->m_pkthdr.flowid % adapter->num_queues; 870 else 871 i = curcpu % adapter->num_queues; 872 873 txr = &adapter->tx_rings[i]; 874 875 if (EM_TX_TRYLOCK(txr)) { 876 error = em_mq_start_locked(ifp, txr, m); 877 EM_TX_UNLOCK(txr); 878 } else 879 error = drbr_enqueue(ifp, txr->br, m); 880 881 return (error); 882 } 883 884 /* 885 ** Flush all ring buffers 886 */ 887 static void 888 em_qflush(struct ifnet *ifp) 889 { 890 struct adapter *adapter = ifp->if_softc; 891 struct tx_ring *txr = adapter->tx_rings; 892 struct mbuf *m; 893 894 for (int i = 0; i < adapter->num_queues; i++, txr++) { 895 EM_TX_LOCK(txr); 896 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 897 m_freem(m); 898 EM_TX_UNLOCK(txr); 899 } 900 if_qflush(ifp); 901 } 902 903 #endif /* EM_MULTIQUEUE */ 904 905 static void 906 em_start_locked(struct ifnet *ifp, struct tx_ring *txr) 907 { 908 struct adapter *adapter = ifp->if_softc; 909 struct mbuf *m_head; 910 911 EM_TX_LOCK_ASSERT(txr); 912 913 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 914 IFF_DRV_RUNNING) 915 return; 916 917 if (!adapter->link_active) 918 return; 919 920 /* Call cleanup if number of TX descriptors low */ 921 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 922 em_txeof(txr); 923 924 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 925 if (txr->tx_avail < EM_MAX_SCATTER) { 926 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 927 break; 928 } 929 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 930 if (m_head == NULL) 931 break; 932 /* 933 * Encapsulation can modify our pointer, and or make it 934 * NULL on failure. In that event, we can't requeue. 935 */ 936 if (em_xmit(txr, &m_head)) { 937 if (m_head == NULL) 938 break; 939 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 940 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 941 break; 942 } 943 944 /* Send a copy of the frame to the BPF listener */ 945 ETHER_BPF_MTAP(ifp, m_head); 946 947 /* Set timeout in case hardware has problems transmitting. */ 948 txr->watchdog_time = ticks; 949 txr->watchdog_check = TRUE; 950 } 951 952 return; 953 } 954 955 static void 956 em_start(struct ifnet *ifp) 957 { 958 struct adapter *adapter = ifp->if_softc; 959 struct tx_ring *txr = adapter->tx_rings; 960 961 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 962 EM_TX_LOCK(txr); 963 em_start_locked(ifp, txr); 964 EM_TX_UNLOCK(txr); 965 } 966 return; 967 } 968 969 /********************************************************************* 970 * Ioctl entry point 971 * 972 * em_ioctl is called when the user wants to configure the 973 * interface. 974 * 975 * return 0 on success, positive on failure 976 **********************************************************************/ 977 978 static int 979 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 980 { 981 struct adapter *adapter = ifp->if_softc; 982 struct ifreq *ifr = (struct ifreq *)data; 983 #ifdef INET 984 struct ifaddr *ifa = (struct ifaddr *)data; 985 #endif 986 int error = 0; 987 988 if (adapter->in_detach) 989 return (error); 990 991 switch (command) { 992 case SIOCSIFADDR: 993 #ifdef INET 994 if (ifa->ifa_addr->sa_family == AF_INET) { 995 /* 996 * XXX 997 * Since resetting hardware takes a very long time 998 * and results in link renegotiation we only 999 * initialize the hardware only when it is absolutely 1000 * required. 1001 */ 1002 ifp->if_flags |= IFF_UP; 1003 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1004 EM_CORE_LOCK(adapter); 1005 em_init_locked(adapter); 1006 EM_CORE_UNLOCK(adapter); 1007 } 1008 arp_ifinit(ifp, ifa); 1009 } else 1010 #endif 1011 error = ether_ioctl(ifp, command, data); 1012 break; 1013 case SIOCSIFMTU: 1014 { 1015 int max_frame_size; 1016 1017 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1018 1019 EM_CORE_LOCK(adapter); 1020 switch (adapter->hw.mac.type) { 1021 case e1000_82571: 1022 case e1000_82572: 1023 case e1000_ich9lan: 1024 case e1000_ich10lan: 1025 case e1000_82574: 1026 case e1000_80003es2lan: /* 9K Jumbo Frame size */ 1027 max_frame_size = 9234; 1028 break; 1029 case e1000_pchlan: 1030 max_frame_size = 4096; 1031 break; 1032 /* Adapters that do not support jumbo frames */ 1033 case e1000_82583: 1034 case e1000_ich8lan: 1035 max_frame_size = ETHER_MAX_LEN; 1036 break; 1037 default: 1038 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1039 } 1040 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1041 ETHER_CRC_LEN) { 1042 EM_CORE_UNLOCK(adapter); 1043 error = EINVAL; 1044 break; 1045 } 1046 1047 ifp->if_mtu = ifr->ifr_mtu; 1048 adapter->max_frame_size = 1049 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1050 em_init_locked(adapter); 1051 EM_CORE_UNLOCK(adapter); 1052 break; 1053 } 1054 case SIOCSIFFLAGS: 1055 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1056 SIOCSIFFLAGS (Set Interface Flags)"); 1057 EM_CORE_LOCK(adapter); 1058 if (ifp->if_flags & IFF_UP) { 1059 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1060 if ((ifp->if_flags ^ adapter->if_flags) & 1061 (IFF_PROMISC | IFF_ALLMULTI)) { 1062 em_disable_promisc(adapter); 1063 em_set_promisc(adapter); 1064 } 1065 } else 1066 em_init_locked(adapter); 1067 } else 1068 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1069 em_stop(adapter); 1070 adapter->if_flags = ifp->if_flags; 1071 EM_CORE_UNLOCK(adapter); 1072 break; 1073 case SIOCADDMULTI: 1074 case SIOCDELMULTI: 1075 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1076 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1077 EM_CORE_LOCK(adapter); 1078 em_disable_intr(adapter); 1079 em_set_multi(adapter); 1080 #ifdef DEVICE_POLLING 1081 if (!(ifp->if_capenable & IFCAP_POLLING)) 1082 #endif 1083 em_enable_intr(adapter); 1084 EM_CORE_UNLOCK(adapter); 1085 } 1086 break; 1087 case SIOCSIFMEDIA: 1088 /* Check SOL/IDER usage */ 1089 EM_CORE_LOCK(adapter); 1090 if (e1000_check_reset_block(&adapter->hw)) { 1091 EM_CORE_UNLOCK(adapter); 1092 device_printf(adapter->dev, "Media change is" 1093 " blocked due to SOL/IDER session.\n"); 1094 break; 1095 } 1096 EM_CORE_UNLOCK(adapter); 1097 case SIOCGIFMEDIA: 1098 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1099 SIOCxIFMEDIA (Get/Set Interface Media)"); 1100 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1101 break; 1102 case SIOCSIFCAP: 1103 { 1104 int mask, reinit; 1105 1106 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1107 reinit = 0; 1108 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1109 #ifdef DEVICE_POLLING 1110 if (mask & IFCAP_POLLING) { 1111 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1112 error = ether_poll_register(em_poll, ifp); 1113 if (error) 1114 return (error); 1115 EM_CORE_LOCK(adapter); 1116 em_disable_intr(adapter); 1117 ifp->if_capenable |= IFCAP_POLLING; 1118 EM_CORE_UNLOCK(adapter); 1119 } else { 1120 error = ether_poll_deregister(ifp); 1121 /* Enable interrupt even in error case */ 1122 EM_CORE_LOCK(adapter); 1123 em_enable_intr(adapter); 1124 ifp->if_capenable &= ~IFCAP_POLLING; 1125 EM_CORE_UNLOCK(adapter); 1126 } 1127 } 1128 #endif 1129 if (mask & IFCAP_HWCSUM) { 1130 ifp->if_capenable ^= IFCAP_HWCSUM; 1131 reinit = 1; 1132 } 1133 if (mask & IFCAP_TSO4) { 1134 ifp->if_capenable ^= IFCAP_TSO4; 1135 reinit = 1; 1136 } 1137 if (mask & IFCAP_VLAN_HWTAGGING) { 1138 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1139 reinit = 1; 1140 } 1141 if (mask & IFCAP_VLAN_HWFILTER) { 1142 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1143 reinit = 1; 1144 } 1145 if ((mask & IFCAP_WOL) && 1146 (ifp->if_capabilities & IFCAP_WOL) != 0) { 1147 if (mask & IFCAP_WOL_MCAST) 1148 ifp->if_capenable ^= IFCAP_WOL_MCAST; 1149 if (mask & IFCAP_WOL_MAGIC) 1150 ifp->if_capenable ^= IFCAP_WOL_MAGIC; 1151 } 1152 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1153 em_init(adapter); 1154 VLAN_CAPABILITIES(ifp); 1155 break; 1156 } 1157 1158 default: 1159 error = ether_ioctl(ifp, command, data); 1160 break; 1161 } 1162 1163 return (error); 1164 } 1165 1166 1167 /********************************************************************* 1168 * Init entry point 1169 * 1170 * This routine is used in two ways. It is used by the stack as 1171 * init entry point in network interface structure. It is also used 1172 * by the driver as a hw/sw initialization routine to get to a 1173 * consistent state. 1174 * 1175 * return 0 on success, positive on failure 1176 **********************************************************************/ 1177 1178 static void 1179 em_init_locked(struct adapter *adapter) 1180 { 1181 struct ifnet *ifp = adapter->ifp; 1182 device_t dev = adapter->dev; 1183 u32 pba; 1184 1185 INIT_DEBUGOUT("em_init: begin"); 1186 1187 EM_CORE_LOCK_ASSERT(adapter); 1188 1189 em_disable_intr(adapter); 1190 callout_stop(&adapter->timer); 1191 1192 /* 1193 * Packet Buffer Allocation (PBA) 1194 * Writing PBA sets the receive portion of the buffer 1195 * the remainder is used for the transmit buffer. 1196 */ 1197 switch (adapter->hw.mac.type) { 1198 /* Total Packet Buffer on these is 48K */ 1199 case e1000_82571: 1200 case e1000_82572: 1201 case e1000_80003es2lan: 1202 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1203 break; 1204 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1205 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1206 break; 1207 case e1000_82574: 1208 case e1000_82583: 1209 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1210 break; 1211 case e1000_ich9lan: 1212 case e1000_ich10lan: 1213 case e1000_pchlan: 1214 pba = E1000_PBA_10K; 1215 break; 1216 case e1000_ich8lan: 1217 pba = E1000_PBA_8K; 1218 break; 1219 default: 1220 if (adapter->max_frame_size > 8192) 1221 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1222 else 1223 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1224 } 1225 1226 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1227 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1228 1229 /* Get the latest mac address, User can use a LAA */ 1230 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1231 ETHER_ADDR_LEN); 1232 1233 /* Put the address into the Receive Address Array */ 1234 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1235 1236 /* 1237 * With the 82571 adapter, RAR[0] may be overwritten 1238 * when the other port is reset, we make a duplicate 1239 * in RAR[14] for that eventuality, this assures 1240 * the interface continues to function. 1241 */ 1242 if (adapter->hw.mac.type == e1000_82571) { 1243 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1244 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1245 E1000_RAR_ENTRIES - 1); 1246 } 1247 1248 /* Initialize the hardware */ 1249 em_reset(adapter); 1250 em_update_link_status(adapter); 1251 1252 /* Setup VLAN support, basic and offload if available */ 1253 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1254 1255 /* Use real VLAN Filter support? */ 1256 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1257 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1258 /* Use real VLAN Filter support */ 1259 em_setup_vlan_hw_support(adapter); 1260 else { 1261 u32 ctrl; 1262 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1263 ctrl |= E1000_CTRL_VME; 1264 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1265 } 1266 } 1267 1268 /* Set hardware offload abilities */ 1269 ifp->if_hwassist = 0; 1270 if (ifp->if_capenable & IFCAP_TXCSUM) 1271 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1272 if (ifp->if_capenable & IFCAP_TSO4) 1273 ifp->if_hwassist |= CSUM_TSO; 1274 1275 /* Configure for OS presence */ 1276 em_init_manageability(adapter); 1277 1278 /* Prepare transmit descriptors and buffers */ 1279 em_setup_transmit_structures(adapter); 1280 em_initialize_transmit_unit(adapter); 1281 1282 /* Setup Multicast table */ 1283 em_set_multi(adapter); 1284 1285 /* Prepare receive descriptors and buffers */ 1286 if (em_setup_receive_structures(adapter)) { 1287 device_printf(dev, "Could not setup receive structures\n"); 1288 em_stop(adapter); 1289 return; 1290 } 1291 em_initialize_receive_unit(adapter); 1292 1293 /* Don't lose promiscuous settings */ 1294 em_set_promisc(adapter); 1295 1296 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1297 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1298 1299 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1300 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1301 1302 /* MSI/X configuration for 82574 */ 1303 if (adapter->hw.mac.type == e1000_82574) { 1304 int tmp; 1305 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1306 tmp |= E1000_CTRL_EXT_PBA_CLR; 1307 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1308 /* Set the IVAR - interrupt vector routing. */ 1309 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); 1310 } 1311 1312 #ifdef DEVICE_POLLING 1313 /* 1314 * Only enable interrupts if we are not polling, make sure 1315 * they are off otherwise. 1316 */ 1317 if (ifp->if_capenable & IFCAP_POLLING) 1318 em_disable_intr(adapter); 1319 else 1320 #endif /* DEVICE_POLLING */ 1321 em_enable_intr(adapter); 1322 1323 /* AMT based hardware can now take control from firmware */ 1324 if (adapter->has_manage && adapter->has_amt) 1325 em_get_hw_control(adapter); 1326 1327 /* Don't reset the phy next time init gets called */ 1328 adapter->hw.phy.reset_disable = TRUE; 1329 } 1330 1331 static void 1332 em_init(void *arg) 1333 { 1334 struct adapter *adapter = arg; 1335 1336 EM_CORE_LOCK(adapter); 1337 em_init_locked(adapter); 1338 EM_CORE_UNLOCK(adapter); 1339 } 1340 1341 1342 #ifdef DEVICE_POLLING 1343 /********************************************************************* 1344 * 1345 * Legacy polling routine: note this only works with single queue 1346 * 1347 *********************************************************************/ 1348 static int 1349 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1350 { 1351 struct adapter *adapter = ifp->if_softc; 1352 struct tx_ring *txr = adapter->tx_rings; 1353 struct rx_ring *rxr = adapter->rx_rings; 1354 u32 reg_icr, rx_done = 0; 1355 1356 EM_CORE_LOCK(adapter); 1357 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1358 EM_CORE_UNLOCK(adapter); 1359 return (rx_done); 1360 } 1361 1362 if (cmd == POLL_AND_CHECK_STATUS) { 1363 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1364 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1365 callout_stop(&adapter->timer); 1366 adapter->hw.mac.get_link_status = 1; 1367 em_update_link_status(adapter); 1368 callout_reset(&adapter->timer, hz, 1369 em_local_timer, adapter); 1370 } 1371 } 1372 EM_CORE_UNLOCK(adapter); 1373 1374 rx_done = em_rxeof(rxr, count); 1375 1376 EM_TX_LOCK(txr); 1377 em_txeof(txr); 1378 #ifdef EM_MULTIQUEUE 1379 if (!drbr_empty(ifp, txr->br)) 1380 em_mq_start_locked(ifp, txr, NULL); 1381 #else 1382 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1383 em_start_locked(ifp, txr); 1384 #endif 1385 EM_TX_UNLOCK(txr); 1386 1387 return (rx_done); 1388 } 1389 #endif /* DEVICE_POLLING */ 1390 1391 1392 /********************************************************************* 1393 * 1394 * Fast Legacy/MSI Combined Interrupt Service routine 1395 * 1396 *********************************************************************/ 1397 static int 1398 em_irq_fast(void *arg) 1399 { 1400 struct adapter *adapter = arg; 1401 struct ifnet *ifp; 1402 u32 reg_icr; 1403 1404 ifp = adapter->ifp; 1405 1406 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1407 1408 /* Hot eject? */ 1409 if (reg_icr == 0xffffffff) 1410 return FILTER_STRAY; 1411 1412 /* Definitely not our interrupt. */ 1413 if (reg_icr == 0x0) 1414 return FILTER_STRAY; 1415 1416 /* 1417 * Starting with the 82571 chip, bit 31 should be used to 1418 * determine whether the interrupt belongs to us. 1419 */ 1420 if (adapter->hw.mac.type >= e1000_82571 && 1421 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1422 return FILTER_STRAY; 1423 1424 em_disable_intr(adapter); 1425 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1426 1427 /* Link status change */ 1428 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1429 adapter->hw.mac.get_link_status = 1; 1430 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1431 } 1432 1433 if (reg_icr & E1000_ICR_RXO) 1434 adapter->rx_overruns++; 1435 return FILTER_HANDLED; 1436 } 1437 1438 /* Combined RX/TX handler, used by Legacy and MSI */ 1439 static void 1440 em_handle_que(void *context, int pending) 1441 { 1442 struct adapter *adapter = context; 1443 struct ifnet *ifp = adapter->ifp; 1444 struct tx_ring *txr = adapter->tx_rings; 1445 struct rx_ring *rxr = adapter->rx_rings; 1446 bool more_rx; 1447 1448 1449 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1450 more_rx = em_rxeof(rxr, adapter->rx_process_limit); 1451 EM_TX_LOCK(txr); 1452 em_txeof(txr); 1453 #ifdef EM_MULTIQUEUE 1454 if (!drbr_empty(ifp, txr->br)) 1455 em_mq_start_locked(ifp, txr, NULL); 1456 #else 1457 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1458 em_start_locked(ifp, txr); 1459 #endif 1460 EM_TX_UNLOCK(txr); 1461 if (more_rx) { 1462 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1463 return; 1464 } 1465 } 1466 1467 em_enable_intr(adapter); 1468 return; 1469 } 1470 1471 1472 /********************************************************************* 1473 * 1474 * MSIX Interrupt Service Routines 1475 * 1476 **********************************************************************/ 1477 static void 1478 em_msix_tx(void *arg) 1479 { 1480 struct tx_ring *txr = arg; 1481 struct adapter *adapter = txr->adapter; 1482 1483 ++txr->tx_irq; 1484 EM_TX_LOCK(txr); 1485 em_txeof(txr); 1486 EM_TX_UNLOCK(txr); 1487 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1488 return; 1489 } 1490 1491 /********************************************************************* 1492 * 1493 * MSIX RX Interrupt Service routine 1494 * 1495 **********************************************************************/ 1496 1497 static void 1498 em_msix_rx(void *arg) 1499 { 1500 struct rx_ring *rxr = arg; 1501 struct adapter *adapter = rxr->adapter; 1502 bool more; 1503 1504 EM_RX_LOCK(rxr); 1505 ++rxr->rx_irq; 1506 more = em_rxeof(rxr, adapter->rx_process_limit); 1507 EM_RX_UNLOCK(rxr); 1508 if (more) 1509 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1510 else 1511 /* Reenable this interrupt */ 1512 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1513 return; 1514 } 1515 1516 /********************************************************************* 1517 * 1518 * MSIX Link Fast Interrupt Service routine 1519 * 1520 **********************************************************************/ 1521 static void 1522 em_msix_link(void *arg) 1523 { 1524 struct adapter *adapter = arg; 1525 u32 reg_icr; 1526 1527 ++adapter->link_irq; 1528 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1529 1530 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1531 adapter->hw.mac.get_link_status = 1; 1532 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1533 } else 1534 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1535 EM_MSIX_LINK | E1000_IMS_LSC); 1536 return; 1537 } 1538 1539 static void 1540 em_handle_rx(void *context, int pending) 1541 { 1542 struct rx_ring *rxr = context; 1543 struct adapter *adapter = rxr->adapter; 1544 bool more; 1545 1546 EM_RX_LOCK(rxr); 1547 more = em_rxeof(rxr, adapter->rx_process_limit); 1548 EM_RX_UNLOCK(rxr); 1549 if (more) 1550 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1551 else 1552 /* Reenable this interrupt */ 1553 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1554 } 1555 1556 static void 1557 em_handle_tx(void *context, int pending) 1558 { 1559 struct tx_ring *txr = context; 1560 struct adapter *adapter = txr->adapter; 1561 struct ifnet *ifp = adapter->ifp; 1562 1563 if (!EM_TX_TRYLOCK(txr)) 1564 return; 1565 1566 em_txeof(txr); 1567 1568 #ifdef EM_MULTIQUEUE 1569 if (!drbr_empty(ifp, txr->br)) 1570 em_mq_start_locked(ifp, txr, NULL); 1571 #else 1572 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1573 em_start_locked(ifp, txr); 1574 #endif 1575 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1576 EM_TX_UNLOCK(txr); 1577 } 1578 1579 static void 1580 em_handle_link(void *context, int pending) 1581 { 1582 struct adapter *adapter = context; 1583 struct ifnet *ifp = adapter->ifp; 1584 1585 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1586 return; 1587 1588 EM_CORE_LOCK(adapter); 1589 callout_stop(&adapter->timer); 1590 em_update_link_status(adapter); 1591 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1592 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1593 EM_MSIX_LINK | E1000_IMS_LSC); 1594 EM_CORE_UNLOCK(adapter); 1595 } 1596 1597 1598 /********************************************************************* 1599 * 1600 * Media Ioctl callback 1601 * 1602 * This routine is called whenever the user queries the status of 1603 * the interface using ifconfig. 1604 * 1605 **********************************************************************/ 1606 static void 1607 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1608 { 1609 struct adapter *adapter = ifp->if_softc; 1610 u_char fiber_type = IFM_1000_SX; 1611 1612 INIT_DEBUGOUT("em_media_status: begin"); 1613 1614 EM_CORE_LOCK(adapter); 1615 em_update_link_status(adapter); 1616 1617 ifmr->ifm_status = IFM_AVALID; 1618 ifmr->ifm_active = IFM_ETHER; 1619 1620 if (!adapter->link_active) { 1621 EM_CORE_UNLOCK(adapter); 1622 return; 1623 } 1624 1625 ifmr->ifm_status |= IFM_ACTIVE; 1626 1627 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1628 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1629 ifmr->ifm_active |= fiber_type | IFM_FDX; 1630 } else { 1631 switch (adapter->link_speed) { 1632 case 10: 1633 ifmr->ifm_active |= IFM_10_T; 1634 break; 1635 case 100: 1636 ifmr->ifm_active |= IFM_100_TX; 1637 break; 1638 case 1000: 1639 ifmr->ifm_active |= IFM_1000_T; 1640 break; 1641 } 1642 if (adapter->link_duplex == FULL_DUPLEX) 1643 ifmr->ifm_active |= IFM_FDX; 1644 else 1645 ifmr->ifm_active |= IFM_HDX; 1646 } 1647 EM_CORE_UNLOCK(adapter); 1648 } 1649 1650 /********************************************************************* 1651 * 1652 * Media Ioctl callback 1653 * 1654 * This routine is called when the user changes speed/duplex using 1655 * media/mediopt option with ifconfig. 1656 * 1657 **********************************************************************/ 1658 static int 1659 em_media_change(struct ifnet *ifp) 1660 { 1661 struct adapter *adapter = ifp->if_softc; 1662 struct ifmedia *ifm = &adapter->media; 1663 1664 INIT_DEBUGOUT("em_media_change: begin"); 1665 1666 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1667 return (EINVAL); 1668 1669 EM_CORE_LOCK(adapter); 1670 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1671 case IFM_AUTO: 1672 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1673 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1674 break; 1675 case IFM_1000_LX: 1676 case IFM_1000_SX: 1677 case IFM_1000_T: 1678 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1679 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1680 break; 1681 case IFM_100_TX: 1682 adapter->hw.mac.autoneg = FALSE; 1683 adapter->hw.phy.autoneg_advertised = 0; 1684 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1685 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1686 else 1687 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1688 break; 1689 case IFM_10_T: 1690 adapter->hw.mac.autoneg = FALSE; 1691 adapter->hw.phy.autoneg_advertised = 0; 1692 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1693 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1694 else 1695 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1696 break; 1697 default: 1698 device_printf(adapter->dev, "Unsupported media type\n"); 1699 } 1700 1701 /* As the speed/duplex settings my have changed we need to 1702 * reset the PHY. 1703 */ 1704 adapter->hw.phy.reset_disable = FALSE; 1705 1706 em_init_locked(adapter); 1707 EM_CORE_UNLOCK(adapter); 1708 1709 return (0); 1710 } 1711 1712 /********************************************************************* 1713 * 1714 * This routine maps the mbufs to tx descriptors. 1715 * 1716 * return 0 on success, positive on failure 1717 **********************************************************************/ 1718 1719 static int 1720 em_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1721 { 1722 struct adapter *adapter = txr->adapter; 1723 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1724 bus_dmamap_t map; 1725 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1726 struct e1000_tx_desc *ctxd = NULL; 1727 struct mbuf *m_head; 1728 u32 txd_upper, txd_lower, txd_used, txd_saved; 1729 int nsegs, i, j, first, last = 0; 1730 int error, do_tso, tso_desc = 0; 1731 1732 m_head = *m_headp; 1733 txd_upper = txd_lower = txd_used = txd_saved = 0; 1734 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1735 1736 /* 1737 * TSO workaround: 1738 * If an mbuf is only header we need 1739 * to pull 4 bytes of data into it. 1740 */ 1741 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 1742 m_head = m_pullup(m_head, M_TSO_LEN + 4); 1743 *m_headp = m_head; 1744 if (m_head == NULL) 1745 return (ENOBUFS); 1746 } 1747 1748 /* 1749 * Map the packet for DMA 1750 * 1751 * Capture the first descriptor index, 1752 * this descriptor will have the index 1753 * of the EOP which is the only one that 1754 * now gets a DONE bit writeback. 1755 */ 1756 first = txr->next_avail_desc; 1757 tx_buffer = &txr->tx_buffers[first]; 1758 tx_buffer_mapped = tx_buffer; 1759 map = tx_buffer->map; 1760 1761 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1762 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1763 1764 /* 1765 * There are two types of errors we can (try) to handle: 1766 * - EFBIG means the mbuf chain was too long and bus_dma ran 1767 * out of segments. Defragment the mbuf chain and try again. 1768 * - ENOMEM means bus_dma could not obtain enough bounce buffers 1769 * at this point in time. Defer sending and try again later. 1770 * All other errors, in particular EINVAL, are fatal and prevent the 1771 * mbuf chain from ever going through. Drop it and report error. 1772 */ 1773 if (error == EFBIG) { 1774 struct mbuf *m; 1775 1776 m = m_defrag(*m_headp, M_DONTWAIT); 1777 if (m == NULL) { 1778 adapter->mbuf_alloc_failed++; 1779 m_freem(*m_headp); 1780 *m_headp = NULL; 1781 return (ENOBUFS); 1782 } 1783 *m_headp = m; 1784 1785 /* Try it again */ 1786 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1787 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1788 1789 if (error) { 1790 adapter->no_tx_dma_setup++; 1791 m_freem(*m_headp); 1792 *m_headp = NULL; 1793 return (error); 1794 } 1795 } else if (error != 0) { 1796 adapter->no_tx_dma_setup++; 1797 return (error); 1798 } 1799 1800 /* 1801 * TSO Hardware workaround, if this packet is not 1802 * TSO, and is only a single descriptor long, and 1803 * it follows a TSO burst, then we need to add a 1804 * sentinel descriptor to prevent premature writeback. 1805 */ 1806 if ((do_tso == 0) && (txr->tx_tso == TRUE)) { 1807 if (nsegs == 1) 1808 tso_desc = TRUE; 1809 txr->tx_tso = FALSE; 1810 } 1811 1812 if (nsegs > (txr->tx_avail - 2)) { 1813 txr->no_desc_avail++; 1814 bus_dmamap_unload(txr->txtag, map); 1815 return (ENOBUFS); 1816 } 1817 m_head = *m_headp; 1818 1819 /* Do hardware assists */ 1820 #if __FreeBSD_version >= 700000 1821 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1822 error = em_tso_setup(txr, m_head, &txd_upper, &txd_lower); 1823 if (error != TRUE) 1824 return (ENXIO); /* something foobar */ 1825 /* we need to make a final sentinel transmit desc */ 1826 tso_desc = TRUE; 1827 } else 1828 #endif 1829 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 1830 em_transmit_checksum_setup(txr, m_head, 1831 &txd_upper, &txd_lower); 1832 1833 i = txr->next_avail_desc; 1834 1835 /* Set up our transmit descriptors */ 1836 for (j = 0; j < nsegs; j++) { 1837 bus_size_t seg_len; 1838 bus_addr_t seg_addr; 1839 1840 tx_buffer = &txr->tx_buffers[i]; 1841 ctxd = &txr->tx_base[i]; 1842 seg_addr = segs[j].ds_addr; 1843 seg_len = segs[j].ds_len; 1844 /* 1845 ** TSO Workaround: 1846 ** If this is the last descriptor, we want to 1847 ** split it so we have a small final sentinel 1848 */ 1849 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 1850 seg_len -= 4; 1851 ctxd->buffer_addr = htole64(seg_addr); 1852 ctxd->lower.data = htole32( 1853 adapter->txd_cmd | txd_lower | seg_len); 1854 ctxd->upper.data = 1855 htole32(txd_upper); 1856 if (++i == adapter->num_tx_desc) 1857 i = 0; 1858 /* Now make the sentinel */ 1859 ++txd_used; /* using an extra txd */ 1860 ctxd = &txr->tx_base[i]; 1861 tx_buffer = &txr->tx_buffers[i]; 1862 ctxd->buffer_addr = 1863 htole64(seg_addr + seg_len); 1864 ctxd->lower.data = htole32( 1865 adapter->txd_cmd | txd_lower | 4); 1866 ctxd->upper.data = 1867 htole32(txd_upper); 1868 last = i; 1869 if (++i == adapter->num_tx_desc) 1870 i = 0; 1871 } else { 1872 ctxd->buffer_addr = htole64(seg_addr); 1873 ctxd->lower.data = htole32( 1874 adapter->txd_cmd | txd_lower | seg_len); 1875 ctxd->upper.data = 1876 htole32(txd_upper); 1877 last = i; 1878 if (++i == adapter->num_tx_desc) 1879 i = 0; 1880 } 1881 tx_buffer->m_head = NULL; 1882 tx_buffer->next_eop = -1; 1883 } 1884 1885 txr->next_avail_desc = i; 1886 txr->tx_avail -= nsegs; 1887 if (tso_desc) /* TSO used an extra for sentinel */ 1888 txr->tx_avail -= txd_used; 1889 1890 if (m_head->m_flags & M_VLANTAG) { 1891 /* Set the vlan id. */ 1892 ctxd->upper.fields.special = 1893 htole16(m_head->m_pkthdr.ether_vtag); 1894 /* Tell hardware to add tag */ 1895 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 1896 } 1897 1898 tx_buffer->m_head = m_head; 1899 tx_buffer_mapped->map = tx_buffer->map; 1900 tx_buffer->map = map; 1901 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1902 1903 /* 1904 * Last Descriptor of Packet 1905 * needs End Of Packet (EOP) 1906 * and Report Status (RS) 1907 */ 1908 ctxd->lower.data |= 1909 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1910 /* 1911 * Keep track in the first buffer which 1912 * descriptor will be written back 1913 */ 1914 tx_buffer = &txr->tx_buffers[first]; 1915 tx_buffer->next_eop = last; 1916 1917 /* 1918 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1919 * that this frame is available to transmit. 1920 */ 1921 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1922 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1923 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1924 1925 return (0); 1926 } 1927 1928 static void 1929 em_set_promisc(struct adapter *adapter) 1930 { 1931 struct ifnet *ifp = adapter->ifp; 1932 u32 reg_rctl; 1933 1934 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1935 1936 if (ifp->if_flags & IFF_PROMISC) { 1937 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1938 /* Turn this on if you want to see bad packets */ 1939 if (em_debug_sbp) 1940 reg_rctl |= E1000_RCTL_SBP; 1941 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1942 } else if (ifp->if_flags & IFF_ALLMULTI) { 1943 reg_rctl |= E1000_RCTL_MPE; 1944 reg_rctl &= ~E1000_RCTL_UPE; 1945 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1946 } 1947 } 1948 1949 static void 1950 em_disable_promisc(struct adapter *adapter) 1951 { 1952 u32 reg_rctl; 1953 1954 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1955 1956 reg_rctl &= (~E1000_RCTL_UPE); 1957 reg_rctl &= (~E1000_RCTL_MPE); 1958 reg_rctl &= (~E1000_RCTL_SBP); 1959 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1960 } 1961 1962 1963 /********************************************************************* 1964 * Multicast Update 1965 * 1966 * This routine is called whenever multicast address list is updated. 1967 * 1968 **********************************************************************/ 1969 1970 static void 1971 em_set_multi(struct adapter *adapter) 1972 { 1973 struct ifnet *ifp = adapter->ifp; 1974 struct ifmultiaddr *ifma; 1975 u32 reg_rctl = 0; 1976 u8 *mta; /* Multicast array memory */ 1977 int mcnt = 0; 1978 1979 IOCTL_DEBUGOUT("em_set_multi: begin"); 1980 1981 if (adapter->hw.mac.type == e1000_82542 && 1982 adapter->hw.revision_id == E1000_REVISION_2) { 1983 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1984 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 1985 e1000_pci_clear_mwi(&adapter->hw); 1986 reg_rctl |= E1000_RCTL_RST; 1987 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1988 msec_delay(5); 1989 } 1990 1991 /* Allocate temporary memory to setup array */ 1992 mta = malloc(sizeof(u8) * 1993 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 1994 M_DEVBUF, M_NOWAIT | M_ZERO); 1995 if (mta == NULL) 1996 panic("em_set_multi memory failure\n"); 1997 1998 #if __FreeBSD_version < 800000 1999 IF_ADDR_LOCK(ifp); 2000 #else 2001 if_maddr_rlock(ifp); 2002 #endif 2003 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2004 if (ifma->ifma_addr->sa_family != AF_LINK) 2005 continue; 2006 2007 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2008 break; 2009 2010 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2011 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2012 mcnt++; 2013 } 2014 #if __FreeBSD_version < 800000 2015 IF_ADDR_UNLOCK(ifp); 2016 #else 2017 if_maddr_runlock(ifp); 2018 #endif 2019 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2020 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2021 reg_rctl |= E1000_RCTL_MPE; 2022 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2023 } else 2024 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2025 2026 if (adapter->hw.mac.type == e1000_82542 && 2027 adapter->hw.revision_id == E1000_REVISION_2) { 2028 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2029 reg_rctl &= ~E1000_RCTL_RST; 2030 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2031 msec_delay(5); 2032 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2033 e1000_pci_set_mwi(&adapter->hw); 2034 } 2035 free(mta, M_DEVBUF); 2036 } 2037 2038 2039 /********************************************************************* 2040 * Timer routine 2041 * 2042 * This routine checks for link status and updates statistics. 2043 * 2044 **********************************************************************/ 2045 2046 static void 2047 em_local_timer(void *arg) 2048 { 2049 struct adapter *adapter = arg; 2050 struct ifnet *ifp = adapter->ifp; 2051 struct tx_ring *txr = adapter->tx_rings; 2052 2053 EM_CORE_LOCK_ASSERT(adapter); 2054 2055 em_update_link_status(adapter); 2056 em_update_stats_counters(adapter); 2057 2058 /* Reset LAA into RAR[0] on 82571 */ 2059 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2060 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2061 2062 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2063 em_print_hw_stats(adapter); 2064 2065 /* 2066 ** Check for time since any descriptor was cleaned 2067 */ 2068 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2069 EM_TX_LOCK(txr); 2070 if (txr->watchdog_check == FALSE) { 2071 EM_TX_UNLOCK(txr); 2072 continue; 2073 } 2074 if ((ticks - txr->watchdog_time) > EM_WATCHDOG) 2075 goto hung; 2076 EM_TX_UNLOCK(txr); 2077 } 2078 2079 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2080 return; 2081 hung: 2082 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2083 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2084 adapter->watchdog_events++; 2085 EM_TX_UNLOCK(txr); 2086 em_init_locked(adapter); 2087 } 2088 2089 2090 static void 2091 em_update_link_status(struct adapter *adapter) 2092 { 2093 struct e1000_hw *hw = &adapter->hw; 2094 struct ifnet *ifp = adapter->ifp; 2095 device_t dev = adapter->dev; 2096 u32 link_check = 0; 2097 2098 /* Get the cached link value or read phy for real */ 2099 switch (hw->phy.media_type) { 2100 case e1000_media_type_copper: 2101 if (hw->mac.get_link_status) { 2102 /* Do the work to read phy */ 2103 e1000_check_for_link(hw); 2104 link_check = !hw->mac.get_link_status; 2105 if (link_check) /* ESB2 fix */ 2106 e1000_cfg_on_link_up(hw); 2107 } else 2108 link_check = TRUE; 2109 break; 2110 case e1000_media_type_fiber: 2111 e1000_check_for_link(hw); 2112 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2113 E1000_STATUS_LU); 2114 break; 2115 case e1000_media_type_internal_serdes: 2116 e1000_check_for_link(hw); 2117 link_check = adapter->hw.mac.serdes_has_link; 2118 break; 2119 default: 2120 case e1000_media_type_unknown: 2121 break; 2122 } 2123 2124 /* Now check for a transition */ 2125 if (link_check && (adapter->link_active == 0)) { 2126 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2127 &adapter->link_duplex); 2128 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2129 if ((adapter->link_speed != SPEED_1000) && 2130 ((hw->mac.type == e1000_82571) || 2131 (hw->mac.type == e1000_82572))) { 2132 int tarc0; 2133 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2134 tarc0 &= ~SPEED_MODE_BIT; 2135 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2136 } 2137 if (bootverbose) 2138 device_printf(dev, "Link is up %d Mbps %s\n", 2139 adapter->link_speed, 2140 ((adapter->link_duplex == FULL_DUPLEX) ? 2141 "Full Duplex" : "Half Duplex")); 2142 adapter->link_active = 1; 2143 adapter->smartspeed = 0; 2144 ifp->if_baudrate = adapter->link_speed * 1000000; 2145 if_link_state_change(ifp, LINK_STATE_UP); 2146 } else if (!link_check && (adapter->link_active == 1)) { 2147 ifp->if_baudrate = adapter->link_speed = 0; 2148 adapter->link_duplex = 0; 2149 if (bootverbose) 2150 device_printf(dev, "Link is Down\n"); 2151 adapter->link_active = 0; 2152 /* Link down, disable watchdog */ 2153 // JFV change later 2154 //adapter->watchdog_check = FALSE; 2155 if_link_state_change(ifp, LINK_STATE_DOWN); 2156 } 2157 } 2158 2159 /********************************************************************* 2160 * 2161 * This routine disables all traffic on the adapter by issuing a 2162 * global reset on the MAC and deallocates TX/RX buffers. 2163 * 2164 * This routine should always be called with BOTH the CORE 2165 * and TX locks. 2166 **********************************************************************/ 2167 2168 static void 2169 em_stop(void *arg) 2170 { 2171 struct adapter *adapter = arg; 2172 struct ifnet *ifp = adapter->ifp; 2173 struct tx_ring *txr = adapter->tx_rings; 2174 2175 EM_CORE_LOCK_ASSERT(adapter); 2176 2177 INIT_DEBUGOUT("em_stop: begin"); 2178 2179 em_disable_intr(adapter); 2180 callout_stop(&adapter->timer); 2181 2182 /* Tell the stack that the interface is no longer active */ 2183 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2184 2185 /* Unarm watchdog timer. */ 2186 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2187 EM_TX_LOCK(txr); 2188 txr->watchdog_check = FALSE; 2189 EM_TX_UNLOCK(txr); 2190 } 2191 2192 e1000_reset_hw(&adapter->hw); 2193 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2194 2195 e1000_led_off(&adapter->hw); 2196 e1000_cleanup_led(&adapter->hw); 2197 } 2198 2199 2200 /********************************************************************* 2201 * 2202 * Determine hardware revision. 2203 * 2204 **********************************************************************/ 2205 static void 2206 em_identify_hardware(struct adapter *adapter) 2207 { 2208 device_t dev = adapter->dev; 2209 2210 /* Make sure our PCI config space has the necessary stuff set */ 2211 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2212 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2213 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2214 device_printf(dev, "Memory Access and/or Bus Master bits " 2215 "were not set!\n"); 2216 adapter->hw.bus.pci_cmd_word |= 2217 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2218 pci_write_config(dev, PCIR_COMMAND, 2219 adapter->hw.bus.pci_cmd_word, 2); 2220 } 2221 2222 /* Save off the information about this board */ 2223 adapter->hw.vendor_id = pci_get_vendor(dev); 2224 adapter->hw.device_id = pci_get_device(dev); 2225 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2226 adapter->hw.subsystem_vendor_id = 2227 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2228 adapter->hw.subsystem_device_id = 2229 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2230 2231 /* Do Shared Code Init and Setup */ 2232 if (e1000_set_mac_type(&adapter->hw)) { 2233 device_printf(dev, "Setup init failure\n"); 2234 return; 2235 } 2236 } 2237 2238 static int 2239 em_allocate_pci_resources(struct adapter *adapter) 2240 { 2241 device_t dev = adapter->dev; 2242 int rid; 2243 2244 rid = PCIR_BAR(0); 2245 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2246 &rid, RF_ACTIVE); 2247 if (adapter->memory == NULL) { 2248 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2249 return (ENXIO); 2250 } 2251 adapter->osdep.mem_bus_space_tag = 2252 rman_get_bustag(adapter->memory); 2253 adapter->osdep.mem_bus_space_handle = 2254 rman_get_bushandle(adapter->memory); 2255 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2256 2257 /* Default to a single queue */ 2258 adapter->num_queues = 1; 2259 2260 /* 2261 * Setup MSI/X or MSI if PCI Express 2262 */ 2263 adapter->msix = em_setup_msix(adapter); 2264 2265 adapter->hw.back = &adapter->osdep; 2266 2267 return (0); 2268 } 2269 2270 /********************************************************************* 2271 * 2272 * Setup the Legacy or MSI Interrupt handler 2273 * 2274 **********************************************************************/ 2275 int 2276 em_allocate_legacy(struct adapter *adapter) 2277 { 2278 device_t dev = adapter->dev; 2279 int error, rid = 0; 2280 2281 /* Manually turn off all interrupts */ 2282 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2283 2284 if (adapter->msix == 1) /* using MSI */ 2285 rid = 1; 2286 /* We allocate a single interrupt resource */ 2287 adapter->res = bus_alloc_resource_any(dev, 2288 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2289 if (adapter->res == NULL) { 2290 device_printf(dev, "Unable to allocate bus resource: " 2291 "interrupt\n"); 2292 return (ENXIO); 2293 } 2294 2295 /* 2296 * Allocate a fast interrupt and the associated 2297 * deferred processing contexts. 2298 */ 2299 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); 2300 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2301 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2302 taskqueue_thread_enqueue, &adapter->tq); 2303 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2304 device_get_nameunit(adapter->dev)); 2305 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, 2306 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { 2307 device_printf(dev, "Failed to register fast interrupt " 2308 "handler: %d\n", error); 2309 taskqueue_free(adapter->tq); 2310 adapter->tq = NULL; 2311 return (error); 2312 } 2313 2314 return (0); 2315 } 2316 2317 /********************************************************************* 2318 * 2319 * Setup the MSIX Interrupt handlers 2320 * This is not really Multiqueue, rather 2321 * its just multiple interrupt vectors. 2322 * 2323 **********************************************************************/ 2324 int 2325 em_allocate_msix(struct adapter *adapter) 2326 { 2327 device_t dev = adapter->dev; 2328 struct tx_ring *txr = adapter->tx_rings; 2329 struct rx_ring *rxr = adapter->rx_rings; 2330 int error, rid, vector = 0; 2331 2332 2333 /* Make sure all interrupts are disabled */ 2334 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2335 2336 /* First set up ring resources */ 2337 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { 2338 2339 /* RX ring */ 2340 rid = vector + 1; 2341 2342 rxr->res = bus_alloc_resource_any(dev, 2343 SYS_RES_IRQ, &rid, RF_ACTIVE); 2344 if (rxr->res == NULL) { 2345 device_printf(dev, 2346 "Unable to allocate bus resource: " 2347 "RX MSIX Interrupt %d\n", i); 2348 return (ENXIO); 2349 } 2350 if ((error = bus_setup_intr(dev, rxr->res, 2351 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, 2352 rxr, &rxr->tag)) != 0) { 2353 device_printf(dev, "Failed to register RX handler"); 2354 return (error); 2355 } 2356 rxr->msix = vector++; /* NOTE increment vector for TX */ 2357 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); 2358 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, 2359 taskqueue_thread_enqueue, &rxr->tq); 2360 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 2361 device_get_nameunit(adapter->dev)); 2362 /* 2363 ** Set the bit to enable interrupt 2364 ** in E1000_IMS -- bits 20 and 21 2365 ** are for RX0 and RX1, note this has 2366 ** NOTHING to do with the MSIX vector 2367 */ 2368 rxr->ims = 1 << (20 + i); 2369 adapter->ivars |= (8 | rxr->msix) << (i * 4); 2370 2371 /* TX ring */ 2372 rid = vector + 1; 2373 txr->res = bus_alloc_resource_any(dev, 2374 SYS_RES_IRQ, &rid, RF_ACTIVE); 2375 if (txr->res == NULL) { 2376 device_printf(dev, 2377 "Unable to allocate bus resource: " 2378 "TX MSIX Interrupt %d\n", i); 2379 return (ENXIO); 2380 } 2381 if ((error = bus_setup_intr(dev, txr->res, 2382 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, 2383 txr, &txr->tag)) != 0) { 2384 device_printf(dev, "Failed to register TX handler"); 2385 return (error); 2386 } 2387 txr->msix = vector++; /* Increment vector for next pass */ 2388 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); 2389 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, 2390 taskqueue_thread_enqueue, &txr->tq); 2391 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 2392 device_get_nameunit(adapter->dev)); 2393 /* 2394 ** Set the bit to enable interrupt 2395 ** in E1000_IMS -- bits 22 and 23 2396 ** are for TX0 and TX1, note this has 2397 ** NOTHING to do with the MSIX vector 2398 */ 2399 txr->ims = 1 << (22 + i); 2400 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); 2401 } 2402 2403 /* Link interrupt */ 2404 ++rid; 2405 adapter->res = bus_alloc_resource_any(dev, 2406 SYS_RES_IRQ, &rid, RF_ACTIVE); 2407 if (!adapter->res) { 2408 device_printf(dev,"Unable to allocate " 2409 "bus resource: Link interrupt [%d]\n", rid); 2410 return (ENXIO); 2411 } 2412 /* Set the link handler function */ 2413 error = bus_setup_intr(dev, adapter->res, 2414 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2415 em_msix_link, adapter, &adapter->tag); 2416 if (error) { 2417 adapter->res = NULL; 2418 device_printf(dev, "Failed to register LINK handler"); 2419 return (error); 2420 } 2421 adapter->linkvec = vector; 2422 adapter->ivars |= (8 | vector) << 16; 2423 adapter->ivars |= 0x80000000; 2424 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2425 adapter->tq = taskqueue_create_fast("em_link", M_NOWAIT, 2426 taskqueue_thread_enqueue, &adapter->tq); 2427 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", 2428 device_get_nameunit(adapter->dev)); 2429 2430 return (0); 2431 } 2432 2433 2434 static void 2435 em_free_pci_resources(struct adapter *adapter) 2436 { 2437 device_t dev = adapter->dev; 2438 struct tx_ring *txr; 2439 struct rx_ring *rxr; 2440 int rid; 2441 2442 2443 /* 2444 ** Release all the queue interrupt resources: 2445 */ 2446 for (int i = 0; i < adapter->num_queues; i++) { 2447 txr = &adapter->tx_rings[i]; 2448 rxr = &adapter->rx_rings[i]; 2449 rid = txr->msix +1; 2450 if (txr->tag != NULL) { 2451 bus_teardown_intr(dev, txr->res, txr->tag); 2452 txr->tag = NULL; 2453 } 2454 if (txr->res != NULL) 2455 bus_release_resource(dev, SYS_RES_IRQ, 2456 rid, txr->res); 2457 rid = rxr->msix +1; 2458 if (rxr->tag != NULL) { 2459 bus_teardown_intr(dev, rxr->res, rxr->tag); 2460 rxr->tag = NULL; 2461 } 2462 if (rxr->res != NULL) 2463 bus_release_resource(dev, SYS_RES_IRQ, 2464 rid, rxr->res); 2465 } 2466 2467 if (adapter->linkvec) /* we are doing MSIX */ 2468 rid = adapter->linkvec + 1; 2469 else 2470 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2471 2472 if (adapter->tag != NULL) { 2473 bus_teardown_intr(dev, adapter->res, adapter->tag); 2474 adapter->tag = NULL; 2475 } 2476 2477 if (adapter->res != NULL) 2478 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2479 2480 2481 if (adapter->msix) 2482 pci_release_msi(dev); 2483 2484 if (adapter->msix_mem != NULL) 2485 bus_release_resource(dev, SYS_RES_MEMORY, 2486 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2487 2488 if (adapter->memory != NULL) 2489 bus_release_resource(dev, SYS_RES_MEMORY, 2490 PCIR_BAR(0), adapter->memory); 2491 2492 if (adapter->flash != NULL) 2493 bus_release_resource(dev, SYS_RES_MEMORY, 2494 EM_FLASH, adapter->flash); 2495 } 2496 2497 /* 2498 * Setup MSI or MSI/X 2499 */ 2500 static int 2501 em_setup_msix(struct adapter *adapter) 2502 { 2503 device_t dev = adapter->dev; 2504 int val = 0; 2505 2506 2507 /* Setup MSI/X for Hartwell */ 2508 if ((adapter->hw.mac.type == e1000_82574) && 2509 (em_enable_msix == TRUE)) { 2510 /* Map the MSIX BAR */ 2511 int rid = PCIR_BAR(EM_MSIX_BAR); 2512 adapter->msix_mem = bus_alloc_resource_any(dev, 2513 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2514 if (!adapter->msix_mem) { 2515 /* May not be enabled */ 2516 device_printf(adapter->dev, 2517 "Unable to map MSIX table \n"); 2518 goto msi; 2519 } 2520 val = pci_msix_count(dev); 2521 if (val != 5) { 2522 bus_release_resource(dev, SYS_RES_MEMORY, 2523 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2524 adapter->msix_mem = NULL; 2525 device_printf(adapter->dev, 2526 "MSIX vectors wrong, using MSI \n"); 2527 goto msi; 2528 } 2529 if (em_msix_queues == 2) { 2530 val = 5; 2531 adapter->num_queues = 2; 2532 } else { 2533 val = 3; 2534 adapter->num_queues = 1; 2535 } 2536 if (pci_alloc_msix(dev, &val) == 0) { 2537 device_printf(adapter->dev, 2538 "Using MSIX interrupts " 2539 "with %d vectors\n", val); 2540 } 2541 2542 return (val); 2543 } 2544 msi: 2545 val = pci_msi_count(dev); 2546 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2547 adapter->msix = 1; 2548 device_printf(adapter->dev,"Using MSI interrupt\n"); 2549 return (val); 2550 } 2551 /* Should only happen due to manual invention */ 2552 device_printf(adapter->dev,"Setup MSIX failure\n"); 2553 return (0); 2554 } 2555 2556 2557 /********************************************************************* 2558 * 2559 * Initialize the hardware to a configuration 2560 * as specified by the adapter structure. 2561 * 2562 **********************************************************************/ 2563 static void 2564 em_reset(struct adapter *adapter) 2565 { 2566 device_t dev = adapter->dev; 2567 struct e1000_hw *hw = &adapter->hw; 2568 u16 rx_buffer_size; 2569 2570 INIT_DEBUGOUT("em_reset: begin"); 2571 2572 /* Set up smart power down as default off on newer adapters. */ 2573 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 2574 hw->mac.type == e1000_82572)) { 2575 u16 phy_tmp = 0; 2576 2577 /* Speed up time to link by disabling smart power down. */ 2578 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2579 phy_tmp &= ~IGP02E1000_PM_SPD; 2580 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2581 } 2582 2583 /* 2584 * These parameters control the automatic generation (Tx) and 2585 * response (Rx) to Ethernet PAUSE frames. 2586 * - High water mark should allow for at least two frames to be 2587 * received after sending an XOFF. 2588 * - Low water mark works best when it is very near the high water mark. 2589 * This allows the receiver to restart by sending XON when it has 2590 * drained a bit. Here we use an arbitary value of 1500 which will 2591 * restart after one full frame is pulled from the buffer. There 2592 * could be several smaller frames in the buffer and if so they will 2593 * not trigger the XON until their total number reduces the buffer 2594 * by 1500. 2595 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2596 */ 2597 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); 2598 2599 hw->fc.high_water = rx_buffer_size - 2600 roundup2(adapter->max_frame_size, 1024); 2601 hw->fc.low_water = hw->fc.high_water - 1500; 2602 2603 if (hw->mac.type == e1000_80003es2lan) 2604 hw->fc.pause_time = 0xFFFF; 2605 else 2606 hw->fc.pause_time = EM_FC_PAUSE_TIME; 2607 2608 hw->fc.send_xon = TRUE; 2609 2610 /* Set Flow control, use the tunable location if sane */ 2611 if ((em_fc_setting >= 0) || (em_fc_setting < 4)) 2612 hw->fc.requested_mode = em_fc_setting; 2613 else 2614 hw->fc.requested_mode = e1000_fc_none; 2615 2616 /* Override - workaround for PCHLAN issue */ 2617 if (hw->mac.type == e1000_pchlan) 2618 hw->fc.requested_mode = e1000_fc_rx_pause; 2619 2620 /* Issue a global reset */ 2621 e1000_reset_hw(hw); 2622 E1000_WRITE_REG(hw, E1000_WUC, 0); 2623 2624 if (e1000_init_hw(hw) < 0) { 2625 device_printf(dev, "Hardware Initialization Failed\n"); 2626 return; 2627 } 2628 2629 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 2630 e1000_get_phy_info(hw); 2631 e1000_check_for_link(hw); 2632 return; 2633 } 2634 2635 /********************************************************************* 2636 * 2637 * Setup networking device structure and register an interface. 2638 * 2639 **********************************************************************/ 2640 static void 2641 em_setup_interface(device_t dev, struct adapter *adapter) 2642 { 2643 struct ifnet *ifp; 2644 2645 INIT_DEBUGOUT("em_setup_interface: begin"); 2646 2647 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2648 if (ifp == NULL) 2649 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2650 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2651 ifp->if_mtu = ETHERMTU; 2652 ifp->if_init = em_init; 2653 ifp->if_softc = adapter; 2654 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2655 ifp->if_ioctl = em_ioctl; 2656 ifp->if_start = em_start; 2657 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2658 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2659 IFQ_SET_READY(&ifp->if_snd); 2660 2661 ether_ifattach(ifp, adapter->hw.mac.addr); 2662 2663 ifp->if_capabilities = ifp->if_capenable = 0; 2664 2665 #ifdef EM_MULTIQUEUE 2666 /* Multiqueue tx functions */ 2667 ifp->if_transmit = em_mq_start; 2668 ifp->if_qflush = em_qflush; 2669 #endif 2670 2671 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2672 ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2673 2674 /* Enable TSO by default, can disable with ifconfig */ 2675 ifp->if_capabilities |= IFCAP_TSO4; 2676 ifp->if_capenable |= IFCAP_TSO4; 2677 2678 /* 2679 * Tell the upper layer(s) we 2680 * support full VLAN capability 2681 */ 2682 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2683 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2684 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2685 2686 /* 2687 ** Dont turn this on by default, if vlans are 2688 ** created on another pseudo device (eg. lagg) 2689 ** then vlan events are not passed thru, breaking 2690 ** operation, but with HW FILTER off it works. If 2691 ** using vlans directly on the em driver you can 2692 ** enable this and get full hardware tag filtering. 2693 */ 2694 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2695 2696 #ifdef DEVICE_POLLING 2697 ifp->if_capabilities |= IFCAP_POLLING; 2698 #endif 2699 2700 /* Enable All WOL methods by default */ 2701 if (adapter->wol) { 2702 ifp->if_capabilities |= IFCAP_WOL; 2703 ifp->if_capenable |= IFCAP_WOL; 2704 } 2705 2706 /* 2707 * Specify the media types supported by this adapter and register 2708 * callbacks to update media and link information 2709 */ 2710 ifmedia_init(&adapter->media, IFM_IMASK, 2711 em_media_change, em_media_status); 2712 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2713 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2714 u_char fiber_type = IFM_1000_SX; /* default type */ 2715 2716 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2717 0, NULL); 2718 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 2719 } else { 2720 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2721 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2722 0, NULL); 2723 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2724 0, NULL); 2725 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2726 0, NULL); 2727 if (adapter->hw.phy.type != e1000_phy_ife) { 2728 ifmedia_add(&adapter->media, 2729 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2730 ifmedia_add(&adapter->media, 2731 IFM_ETHER | IFM_1000_T, 0, NULL); 2732 } 2733 } 2734 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2735 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2736 } 2737 2738 2739 /* 2740 * Manage DMA'able memory. 2741 */ 2742 static void 2743 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2744 { 2745 if (error) 2746 return; 2747 *(bus_addr_t *) arg = segs[0].ds_addr; 2748 } 2749 2750 static int 2751 em_dma_malloc(struct adapter *adapter, bus_size_t size, 2752 struct em_dma_alloc *dma, int mapflags) 2753 { 2754 int error; 2755 2756 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2757 EM_DBA_ALIGN, 0, /* alignment, bounds */ 2758 BUS_SPACE_MAXADDR, /* lowaddr */ 2759 BUS_SPACE_MAXADDR, /* highaddr */ 2760 NULL, NULL, /* filter, filterarg */ 2761 size, /* maxsize */ 2762 1, /* nsegments */ 2763 size, /* maxsegsize */ 2764 0, /* flags */ 2765 NULL, /* lockfunc */ 2766 NULL, /* lockarg */ 2767 &dma->dma_tag); 2768 if (error) { 2769 device_printf(adapter->dev, 2770 "%s: bus_dma_tag_create failed: %d\n", 2771 __func__, error); 2772 goto fail_0; 2773 } 2774 2775 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2776 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 2777 if (error) { 2778 device_printf(adapter->dev, 2779 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2780 __func__, (uintmax_t)size, error); 2781 goto fail_2; 2782 } 2783 2784 dma->dma_paddr = 0; 2785 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2786 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2787 if (error || dma->dma_paddr == 0) { 2788 device_printf(adapter->dev, 2789 "%s: bus_dmamap_load failed: %d\n", 2790 __func__, error); 2791 goto fail_3; 2792 } 2793 2794 return (0); 2795 2796 fail_3: 2797 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2798 fail_2: 2799 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2800 bus_dma_tag_destroy(dma->dma_tag); 2801 fail_0: 2802 dma->dma_map = NULL; 2803 dma->dma_tag = NULL; 2804 2805 return (error); 2806 } 2807 2808 static void 2809 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 2810 { 2811 if (dma->dma_tag == NULL) 2812 return; 2813 if (dma->dma_map != NULL) { 2814 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2815 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2816 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2817 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2818 dma->dma_map = NULL; 2819 } 2820 bus_dma_tag_destroy(dma->dma_tag); 2821 dma->dma_tag = NULL; 2822 } 2823 2824 2825 /********************************************************************* 2826 * 2827 * Allocate memory for the transmit and receive rings, and then 2828 * the descriptors associated with each, called only once at attach. 2829 * 2830 **********************************************************************/ 2831 static int 2832 em_allocate_queues(struct adapter *adapter) 2833 { 2834 device_t dev = adapter->dev; 2835 struct tx_ring *txr = NULL; 2836 struct rx_ring *rxr = NULL; 2837 int rsize, tsize, error = E1000_SUCCESS; 2838 int txconf = 0, rxconf = 0; 2839 2840 2841 /* Allocate the TX ring struct memory */ 2842 if (!(adapter->tx_rings = 2843 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2844 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2845 device_printf(dev, "Unable to allocate TX ring memory\n"); 2846 error = ENOMEM; 2847 goto fail; 2848 } 2849 2850 /* Now allocate the RX */ 2851 if (!(adapter->rx_rings = 2852 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2853 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2854 device_printf(dev, "Unable to allocate RX ring memory\n"); 2855 error = ENOMEM; 2856 goto rx_fail; 2857 } 2858 2859 tsize = roundup2(adapter->num_tx_desc * 2860 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 2861 /* 2862 * Now set up the TX queues, txconf is needed to handle the 2863 * possibility that things fail midcourse and we need to 2864 * undo memory gracefully 2865 */ 2866 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2867 /* Set up some basics */ 2868 txr = &adapter->tx_rings[i]; 2869 txr->adapter = adapter; 2870 txr->me = i; 2871 2872 /* Initialize the TX lock */ 2873 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2874 device_get_nameunit(dev), txr->me); 2875 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2876 2877 if (em_dma_malloc(adapter, tsize, 2878 &txr->txdma, BUS_DMA_NOWAIT)) { 2879 device_printf(dev, 2880 "Unable to allocate TX Descriptor memory\n"); 2881 error = ENOMEM; 2882 goto err_tx_desc; 2883 } 2884 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2885 bzero((void *)txr->tx_base, tsize); 2886 2887 if (em_allocate_transmit_buffers(txr)) { 2888 device_printf(dev, 2889 "Critical Failure setting up transmit buffers\n"); 2890 error = ENOMEM; 2891 goto err_tx_desc; 2892 } 2893 #if __FreeBSD_version >= 800000 2894 /* Allocate a buf ring */ 2895 txr->br = buf_ring_alloc(4096, M_DEVBUF, 2896 M_WAITOK, &txr->tx_mtx); 2897 #endif 2898 } 2899 2900 /* 2901 * Next the RX queues... 2902 */ 2903 rsize = roundup2(adapter->num_rx_desc * 2904 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 2905 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2906 rxr = &adapter->rx_rings[i]; 2907 rxr->adapter = adapter; 2908 rxr->me = i; 2909 2910 /* Initialize the RX lock */ 2911 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2912 device_get_nameunit(dev), txr->me); 2913 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2914 2915 if (em_dma_malloc(adapter, rsize, 2916 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2917 device_printf(dev, 2918 "Unable to allocate RxDescriptor memory\n"); 2919 error = ENOMEM; 2920 goto err_rx_desc; 2921 } 2922 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; 2923 bzero((void *)rxr->rx_base, rsize); 2924 2925 /* Allocate receive buffers for the ring*/ 2926 if (em_allocate_receive_buffers(rxr)) { 2927 device_printf(dev, 2928 "Critical Failure setting up receive buffers\n"); 2929 error = ENOMEM; 2930 goto err_rx_desc; 2931 } 2932 } 2933 2934 return (0); 2935 2936 err_rx_desc: 2937 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2938 em_dma_free(adapter, &rxr->rxdma); 2939 err_tx_desc: 2940 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2941 em_dma_free(adapter, &txr->txdma); 2942 free(adapter->rx_rings, M_DEVBUF); 2943 rx_fail: 2944 buf_ring_free(txr->br, M_DEVBUF); 2945 free(adapter->tx_rings, M_DEVBUF); 2946 fail: 2947 return (error); 2948 } 2949 2950 2951 /********************************************************************* 2952 * 2953 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2954 * the information needed to transmit a packet on the wire. This is 2955 * called only once at attach, setup is done every reset. 2956 * 2957 **********************************************************************/ 2958 static int 2959 em_allocate_transmit_buffers(struct tx_ring *txr) 2960 { 2961 struct adapter *adapter = txr->adapter; 2962 device_t dev = adapter->dev; 2963 struct em_buffer *txbuf; 2964 int error, i; 2965 2966 /* 2967 * Setup DMA descriptor areas. 2968 */ 2969 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 2970 1, 0, /* alignment, bounds */ 2971 BUS_SPACE_MAXADDR, /* lowaddr */ 2972 BUS_SPACE_MAXADDR, /* highaddr */ 2973 NULL, NULL, /* filter, filterarg */ 2974 EM_TSO_SIZE, /* maxsize */ 2975 EM_MAX_SCATTER, /* nsegments */ 2976 PAGE_SIZE, /* maxsegsize */ 2977 0, /* flags */ 2978 NULL, /* lockfunc */ 2979 NULL, /* lockfuncarg */ 2980 &txr->txtag))) { 2981 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2982 goto fail; 2983 } 2984 2985 if (!(txr->tx_buffers = 2986 (struct em_buffer *) malloc(sizeof(struct em_buffer) * 2987 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2988 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2989 error = ENOMEM; 2990 goto fail; 2991 } 2992 2993 /* Create the descriptor buffer dma maps */ 2994 txbuf = txr->tx_buffers; 2995 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2996 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2997 if (error != 0) { 2998 device_printf(dev, "Unable to create TX DMA map\n"); 2999 goto fail; 3000 } 3001 } 3002 3003 return 0; 3004 fail: 3005 /* We free all, it handles case where we are in the middle */ 3006 em_free_transmit_structures(adapter); 3007 return (error); 3008 } 3009 3010 /********************************************************************* 3011 * 3012 * Initialize a transmit ring. 3013 * 3014 **********************************************************************/ 3015 static void 3016 em_setup_transmit_ring(struct tx_ring *txr) 3017 { 3018 struct adapter *adapter = txr->adapter; 3019 struct em_buffer *txbuf; 3020 int i; 3021 3022 /* Clear the old descriptor contents */ 3023 EM_TX_LOCK(txr); 3024 bzero((void *)txr->tx_base, 3025 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3026 /* Reset indices */ 3027 txr->next_avail_desc = 0; 3028 txr->next_to_clean = 0; 3029 3030 /* Free any existing tx buffers. */ 3031 txbuf = txr->tx_buffers; 3032 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3033 if (txbuf->m_head != NULL) { 3034 bus_dmamap_sync(txr->txtag, txbuf->map, 3035 BUS_DMASYNC_POSTWRITE); 3036 bus_dmamap_unload(txr->txtag, txbuf->map); 3037 m_freem(txbuf->m_head); 3038 txbuf->m_head = NULL; 3039 } 3040 /* clear the watch index */ 3041 txbuf->next_eop = -1; 3042 } 3043 3044 /* Set number of descriptors available */ 3045 txr->tx_avail = adapter->num_tx_desc; 3046 3047 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3048 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3049 EM_TX_UNLOCK(txr); 3050 } 3051 3052 /********************************************************************* 3053 * 3054 * Initialize all transmit rings. 3055 * 3056 **********************************************************************/ 3057 static void 3058 em_setup_transmit_structures(struct adapter *adapter) 3059 { 3060 struct tx_ring *txr = adapter->tx_rings; 3061 3062 for (int i = 0; i < adapter->num_queues; i++, txr++) 3063 em_setup_transmit_ring(txr); 3064 3065 return; 3066 } 3067 3068 /********************************************************************* 3069 * 3070 * Enable transmit unit. 3071 * 3072 **********************************************************************/ 3073 static void 3074 em_initialize_transmit_unit(struct adapter *adapter) 3075 { 3076 struct tx_ring *txr = adapter->tx_rings; 3077 struct e1000_hw *hw = &adapter->hw; 3078 u32 tctl, tarc, tipg = 0; 3079 3080 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3081 3082 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3083 u64 bus_addr = txr->txdma.dma_paddr; 3084 /* Base and Len of TX Ring */ 3085 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3086 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3087 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3088 (u32)(bus_addr >> 32)); 3089 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3090 (u32)bus_addr); 3091 /* Init the HEAD/TAIL indices */ 3092 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3093 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3094 3095 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3096 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 3097 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 3098 3099 txr->watchdog_check = FALSE; 3100 } 3101 3102 /* Set the default values for the Tx Inter Packet Gap timer */ 3103 switch (adapter->hw.mac.type) { 3104 case e1000_82542: 3105 tipg = DEFAULT_82542_TIPG_IPGT; 3106 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3107 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3108 break; 3109 case e1000_80003es2lan: 3110 tipg = DEFAULT_82543_TIPG_IPGR1; 3111 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3112 E1000_TIPG_IPGR2_SHIFT; 3113 break; 3114 default: 3115 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3116 (adapter->hw.phy.media_type == 3117 e1000_media_type_internal_serdes)) 3118 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3119 else 3120 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3121 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3122 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3123 } 3124 3125 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3126 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3127 3128 if(adapter->hw.mac.type >= e1000_82540) 3129 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3130 adapter->tx_abs_int_delay.value); 3131 3132 if ((adapter->hw.mac.type == e1000_82571) || 3133 (adapter->hw.mac.type == e1000_82572)) { 3134 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3135 tarc |= SPEED_MODE_BIT; 3136 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3137 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3138 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3139 tarc |= 1; 3140 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3141 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3142 tarc |= 1; 3143 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3144 } 3145 3146 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3147 if (adapter->tx_int_delay.value > 0) 3148 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3149 3150 /* Program the Transmit Control Register */ 3151 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3152 tctl &= ~E1000_TCTL_CT; 3153 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3154 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3155 3156 if (adapter->hw.mac.type >= e1000_82571) 3157 tctl |= E1000_TCTL_MULR; 3158 3159 /* This write will effectively turn on the transmit unit. */ 3160 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3161 3162 } 3163 3164 3165 /********************************************************************* 3166 * 3167 * Free all transmit rings. 3168 * 3169 **********************************************************************/ 3170 static void 3171 em_free_transmit_structures(struct adapter *adapter) 3172 { 3173 struct tx_ring *txr = adapter->tx_rings; 3174 3175 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3176 EM_TX_LOCK(txr); 3177 em_free_transmit_buffers(txr); 3178 em_dma_free(adapter, &txr->txdma); 3179 EM_TX_UNLOCK(txr); 3180 EM_TX_LOCK_DESTROY(txr); 3181 } 3182 3183 free(adapter->tx_rings, M_DEVBUF); 3184 } 3185 3186 /********************************************************************* 3187 * 3188 * Free transmit ring related data structures. 3189 * 3190 **********************************************************************/ 3191 static void 3192 em_free_transmit_buffers(struct tx_ring *txr) 3193 { 3194 struct adapter *adapter = txr->adapter; 3195 struct em_buffer *txbuf; 3196 3197 INIT_DEBUGOUT("free_transmit_ring: begin"); 3198 3199 if (txr->tx_buffers == NULL) 3200 return; 3201 3202 for (int i = 0; i < adapter->num_tx_desc; i++) { 3203 txbuf = &txr->tx_buffers[i]; 3204 if (txbuf->m_head != NULL) { 3205 bus_dmamap_sync(txr->txtag, txbuf->map, 3206 BUS_DMASYNC_POSTWRITE); 3207 bus_dmamap_unload(txr->txtag, 3208 txbuf->map); 3209 m_freem(txbuf->m_head); 3210 txbuf->m_head = NULL; 3211 if (txbuf->map != NULL) { 3212 bus_dmamap_destroy(txr->txtag, 3213 txbuf->map); 3214 txbuf->map = NULL; 3215 } 3216 } else if (txbuf->map != NULL) { 3217 bus_dmamap_unload(txr->txtag, 3218 txbuf->map); 3219 bus_dmamap_destroy(txr->txtag, 3220 txbuf->map); 3221 txbuf->map = NULL; 3222 } 3223 } 3224 #if __FreeBSD_version >= 800000 3225 if (txr->br != NULL) 3226 buf_ring_free(txr->br, M_DEVBUF); 3227 #endif 3228 if (txr->tx_buffers != NULL) { 3229 free(txr->tx_buffers, M_DEVBUF); 3230 txr->tx_buffers = NULL; 3231 } 3232 if (txr->txtag != NULL) { 3233 bus_dma_tag_destroy(txr->txtag); 3234 txr->txtag = NULL; 3235 } 3236 return; 3237 } 3238 3239 3240 /********************************************************************* 3241 * 3242 * The offload context needs to be set when we transfer the first 3243 * packet of a particular protocol (TCP/UDP). This routine has been 3244 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3245 * 3246 * Added back the old method of keeping the current context type 3247 * and not setting if unnecessary, as this is reported to be a 3248 * big performance win. -jfv 3249 **********************************************************************/ 3250 static void 3251 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, 3252 u32 *txd_upper, u32 *txd_lower) 3253 { 3254 struct adapter *adapter = txr->adapter; 3255 struct e1000_context_desc *TXD = NULL; 3256 struct em_buffer *tx_buffer; 3257 struct ether_vlan_header *eh; 3258 struct ip *ip = NULL; 3259 struct ip6_hdr *ip6; 3260 int cur, ehdrlen; 3261 u32 cmd, hdr_len, ip_hlen; 3262 u16 etype; 3263 u8 ipproto; 3264 3265 3266 cmd = hdr_len = ipproto = 0; 3267 cur = txr->next_avail_desc; 3268 3269 /* 3270 * Determine where frame payload starts. 3271 * Jump over vlan headers if already present, 3272 * helpful for QinQ too. 3273 */ 3274 eh = mtod(mp, struct ether_vlan_header *); 3275 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3276 etype = ntohs(eh->evl_proto); 3277 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3278 } else { 3279 etype = ntohs(eh->evl_encap_proto); 3280 ehdrlen = ETHER_HDR_LEN; 3281 } 3282 3283 /* 3284 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3285 * TODO: Support SCTP too when it hits the tree. 3286 */ 3287 switch (etype) { 3288 case ETHERTYPE_IP: 3289 ip = (struct ip *)(mp->m_data + ehdrlen); 3290 ip_hlen = ip->ip_hl << 2; 3291 3292 /* Setup of IP header checksum. */ 3293 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3294 /* 3295 * Start offset for header checksum calculation. 3296 * End offset for header checksum calculation. 3297 * Offset of place to put the checksum. 3298 */ 3299 TXD = (struct e1000_context_desc *) 3300 &txr->tx_base[cur]; 3301 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3302 TXD->lower_setup.ip_fields.ipcse = 3303 htole16(ehdrlen + ip_hlen); 3304 TXD->lower_setup.ip_fields.ipcso = 3305 ehdrlen + offsetof(struct ip, ip_sum); 3306 cmd |= E1000_TXD_CMD_IP; 3307 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3308 } 3309 3310 if (mp->m_len < ehdrlen + ip_hlen) 3311 return; /* failure */ 3312 3313 hdr_len = ehdrlen + ip_hlen; 3314 ipproto = ip->ip_p; 3315 3316 break; 3317 case ETHERTYPE_IPV6: 3318 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3319 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3320 3321 if (mp->m_len < ehdrlen + ip_hlen) 3322 return; /* failure */ 3323 3324 /* IPv6 doesn't have a header checksum. */ 3325 3326 hdr_len = ehdrlen + ip_hlen; 3327 ipproto = ip6->ip6_nxt; 3328 3329 break; 3330 default: 3331 *txd_upper = 0; 3332 *txd_lower = 0; 3333 return; 3334 } 3335 3336 switch (ipproto) { 3337 case IPPROTO_TCP: 3338 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3339 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3340 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3341 /* no need for context if already set */ 3342 if (txr->last_hw_offload == CSUM_TCP) 3343 return; 3344 txr->last_hw_offload = CSUM_TCP; 3345 /* 3346 * Start offset for payload checksum calculation. 3347 * End offset for payload checksum calculation. 3348 * Offset of place to put the checksum. 3349 */ 3350 TXD = (struct e1000_context_desc *) 3351 &txr->tx_base[cur]; 3352 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3353 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3354 TXD->upper_setup.tcp_fields.tucso = 3355 hdr_len + offsetof(struct tcphdr, th_sum); 3356 cmd |= E1000_TXD_CMD_TCP; 3357 } 3358 break; 3359 case IPPROTO_UDP: 3360 { 3361 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3362 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3363 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3364 /* no need for context if already set */ 3365 if (txr->last_hw_offload == CSUM_UDP) 3366 return; 3367 txr->last_hw_offload = CSUM_UDP; 3368 /* 3369 * Start offset for header checksum calculation. 3370 * End offset for header checksum calculation. 3371 * Offset of place to put the checksum. 3372 */ 3373 TXD = (struct e1000_context_desc *) 3374 &txr->tx_base[cur]; 3375 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3376 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3377 TXD->upper_setup.tcp_fields.tucso = 3378 hdr_len + offsetof(struct udphdr, uh_sum); 3379 } 3380 /* Fall Thru */ 3381 } 3382 default: 3383 break; 3384 } 3385 3386 TXD->tcp_seg_setup.data = htole32(0); 3387 TXD->cmd_and_length = 3388 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3389 tx_buffer = &txr->tx_buffers[cur]; 3390 tx_buffer->m_head = NULL; 3391 tx_buffer->next_eop = -1; 3392 3393 if (++cur == adapter->num_tx_desc) 3394 cur = 0; 3395 3396 txr->tx_avail--; 3397 txr->next_avail_desc = cur; 3398 } 3399 3400 3401 /********************************************************************** 3402 * 3403 * Setup work for hardware segmentation offload (TSO) 3404 * 3405 **********************************************************************/ 3406 static bool 3407 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *txd_upper, 3408 u32 *txd_lower) 3409 { 3410 struct adapter *adapter = txr->adapter; 3411 struct e1000_context_desc *TXD; 3412 struct em_buffer *tx_buffer; 3413 struct ether_vlan_header *eh; 3414 struct ip *ip; 3415 struct ip6_hdr *ip6; 3416 struct tcphdr *th; 3417 int cur, ehdrlen, hdr_len, ip_hlen, isip6; 3418 u16 etype; 3419 3420 /* 3421 * This function could/should be extended to support IP/IPv6 3422 * fragmentation as well. But as they say, one step at a time. 3423 */ 3424 3425 /* 3426 * Determine where frame payload starts. 3427 * Jump over vlan headers if already present, 3428 * helpful for QinQ too. 3429 */ 3430 eh = mtod(mp, struct ether_vlan_header *); 3431 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3432 etype = ntohs(eh->evl_proto); 3433 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3434 } else { 3435 etype = ntohs(eh->evl_encap_proto); 3436 ehdrlen = ETHER_HDR_LEN; 3437 } 3438 3439 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3440 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3441 return FALSE; /* -1 */ 3442 3443 /* 3444 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3445 * TODO: Support SCTP too when it hits the tree. 3446 */ 3447 switch (etype) { 3448 case ETHERTYPE_IP: 3449 isip6 = 0; 3450 ip = (struct ip *)(mp->m_data + ehdrlen); 3451 if (ip->ip_p != IPPROTO_TCP) 3452 return FALSE; /* 0 */ 3453 ip->ip_len = 0; 3454 ip->ip_sum = 0; 3455 ip_hlen = ip->ip_hl << 2; 3456 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3457 return FALSE; /* -1 */ 3458 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3459 #if 1 3460 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3461 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3462 #else 3463 th->th_sum = mp->m_pkthdr.csum_data; 3464 #endif 3465 break; 3466 case ETHERTYPE_IPV6: 3467 isip6 = 1; 3468 return FALSE; /* Not supported yet. */ 3469 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3470 if (ip6->ip6_nxt != IPPROTO_TCP) 3471 return FALSE; /* 0 */ 3472 ip6->ip6_plen = 0; 3473 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3474 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3475 return FALSE; /* -1 */ 3476 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3477 #if 0 3478 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3479 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3480 #else 3481 th->th_sum = mp->m_pkthdr.csum_data; 3482 #endif 3483 break; 3484 default: 3485 return FALSE; 3486 } 3487 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3488 3489 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3490 E1000_TXD_DTYP_D | /* Data descr type */ 3491 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3492 3493 /* IP and/or TCP header checksum calculation and insertion. */ 3494 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3495 E1000_TXD_POPTS_TXSM) << 8; 3496 3497 cur = txr->next_avail_desc; 3498 tx_buffer = &txr->tx_buffers[cur]; 3499 TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; 3500 3501 /* IPv6 doesn't have a header checksum. */ 3502 if (!isip6) { 3503 /* 3504 * Start offset for header checksum calculation. 3505 * End offset for header checksum calculation. 3506 * Offset of place put the checksum. 3507 */ 3508 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3509 TXD->lower_setup.ip_fields.ipcse = 3510 htole16(ehdrlen + ip_hlen - 1); 3511 TXD->lower_setup.ip_fields.ipcso = 3512 ehdrlen + offsetof(struct ip, ip_sum); 3513 } 3514 /* 3515 * Start offset for payload checksum calculation. 3516 * End offset for payload checksum calculation. 3517 * Offset of place to put the checksum. 3518 */ 3519 TXD->upper_setup.tcp_fields.tucss = 3520 ehdrlen + ip_hlen; 3521 TXD->upper_setup.tcp_fields.tucse = 0; 3522 TXD->upper_setup.tcp_fields.tucso = 3523 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3524 /* 3525 * Payload size per packet w/o any headers. 3526 * Length of all headers up to payload. 3527 */ 3528 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3529 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3530 3531 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3532 E1000_TXD_CMD_DEXT | /* Extended descr */ 3533 E1000_TXD_CMD_TSE | /* TSE context */ 3534 (isip6 ? 0 : E1000_TXD_CMD_IP) | 3535 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3536 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3537 3538 tx_buffer->m_head = NULL; 3539 tx_buffer->next_eop = -1; 3540 3541 if (++cur == adapter->num_tx_desc) 3542 cur = 0; 3543 3544 txr->tx_avail--; 3545 txr->next_avail_desc = cur; 3546 txr->tx_tso = TRUE; 3547 3548 return TRUE; 3549 } 3550 3551 3552 /********************************************************************** 3553 * 3554 * Examine each tx_buffer in the used queue. If the hardware is done 3555 * processing the packet then free associated resources. The 3556 * tx_buffer is put back on the free queue. 3557 * 3558 **********************************************************************/ 3559 static bool 3560 em_txeof(struct tx_ring *txr) 3561 { 3562 struct adapter *adapter = txr->adapter; 3563 int first, last, done, num_avail; 3564 struct em_buffer *tx_buffer; 3565 struct e1000_tx_desc *tx_desc, *eop_desc; 3566 struct ifnet *ifp = adapter->ifp; 3567 3568 EM_TX_LOCK_ASSERT(txr); 3569 3570 if (txr->tx_avail == adapter->num_tx_desc) 3571 return (FALSE); 3572 3573 num_avail = txr->tx_avail; 3574 first = txr->next_to_clean; 3575 tx_desc = &txr->tx_base[first]; 3576 tx_buffer = &txr->tx_buffers[first]; 3577 last = tx_buffer->next_eop; 3578 eop_desc = &txr->tx_base[last]; 3579 3580 /* 3581 * What this does is get the index of the 3582 * first descriptor AFTER the EOP of the 3583 * first packet, that way we can do the 3584 * simple comparison on the inner while loop. 3585 */ 3586 if (++last == adapter->num_tx_desc) 3587 last = 0; 3588 done = last; 3589 3590 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3591 BUS_DMASYNC_POSTREAD); 3592 3593 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3594 /* We clean the range of the packet */ 3595 while (first != done) { 3596 tx_desc->upper.data = 0; 3597 tx_desc->lower.data = 0; 3598 tx_desc->buffer_addr = 0; 3599 ++num_avail; 3600 3601 if (tx_buffer->m_head) { 3602 ifp->if_opackets++; 3603 bus_dmamap_sync(txr->txtag, 3604 tx_buffer->map, 3605 BUS_DMASYNC_POSTWRITE); 3606 bus_dmamap_unload(txr->txtag, 3607 tx_buffer->map); 3608 3609 m_freem(tx_buffer->m_head); 3610 tx_buffer->m_head = NULL; 3611 } 3612 tx_buffer->next_eop = -1; 3613 txr->watchdog_time = ticks; 3614 3615 if (++first == adapter->num_tx_desc) 3616 first = 0; 3617 3618 tx_buffer = &txr->tx_buffers[first]; 3619 tx_desc = &txr->tx_base[first]; 3620 } 3621 /* See if we can continue to the next packet */ 3622 last = tx_buffer->next_eop; 3623 if (last != -1) { 3624 eop_desc = &txr->tx_base[last]; 3625 /* Get new done point */ 3626 if (++last == adapter->num_tx_desc) last = 0; 3627 done = last; 3628 } else 3629 break; 3630 } 3631 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3632 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3633 3634 txr->next_to_clean = first; 3635 3636 /* 3637 * If we have enough room, clear IFF_DRV_OACTIVE to 3638 * tell the stack that it is OK to send packets. 3639 * If there are no pending descriptors, clear the watchdog. 3640 */ 3641 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 3642 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3643 if (num_avail == adapter->num_tx_desc) { 3644 txr->watchdog_check = FALSE; 3645 txr->tx_avail = num_avail; 3646 return (FALSE); 3647 } 3648 } 3649 3650 txr->tx_avail = num_avail; 3651 return (TRUE); 3652 } 3653 3654 3655 /********************************************************************* 3656 * 3657 * Refresh RX descriptor mbufs from system mbuf buffer pool. 3658 * 3659 **********************************************************************/ 3660 static void 3661 em_refresh_mbufs(struct rx_ring *rxr, int limit) 3662 { 3663 struct adapter *adapter = rxr->adapter; 3664 struct mbuf *m; 3665 bus_dma_segment_t segs[1]; 3666 bus_dmamap_t map; 3667 struct em_buffer *rxbuf; 3668 int i, error, nsegs, cleaned; 3669 3670 i = rxr->next_to_refresh; 3671 cleaned = -1; 3672 while (i != limit) { 3673 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3674 if (m == NULL) 3675 goto update; 3676 m->m_len = m->m_pkthdr.len = MCLBYTES; 3677 3678 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 3679 m_adj(m, ETHER_ALIGN); 3680 3681 /* 3682 * Using memory from the mbuf cluster pool, invoke the 3683 * bus_dma machinery to arrange the memory mapping. 3684 */ 3685 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxr->rx_sparemap, 3686 m, segs, &nsegs, BUS_DMA_NOWAIT); 3687 if (error != 0) { 3688 m_free(m); 3689 goto update; 3690 } 3691 3692 /* If nsegs is wrong then the stack is corrupt. */ 3693 KASSERT(nsegs == 1, ("Too many segments returned!")); 3694 3695 rxbuf = &rxr->rx_buffers[i]; 3696 if (rxbuf->m_head != NULL) 3697 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3698 3699 map = rxbuf->map; 3700 rxbuf->map = rxr->rx_sparemap; 3701 rxr->rx_sparemap = map; 3702 bus_dmamap_sync(rxr->rxtag, 3703 rxbuf->map, BUS_DMASYNC_PREREAD); 3704 rxbuf->m_head = m; 3705 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); 3706 3707 cleaned = i; 3708 /* Calculate next index */ 3709 if (++i == adapter->num_rx_desc) 3710 i = 0; 3711 /* This is the work marker for refresh */ 3712 rxr->next_to_refresh = i; 3713 } 3714 update: 3715 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3716 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3717 if (cleaned != -1) /* Update tail index */ 3718 E1000_WRITE_REG(&adapter->hw, 3719 E1000_RDT(rxr->me), cleaned); 3720 3721 return; 3722 } 3723 3724 3725 /********************************************************************* 3726 * 3727 * Allocate memory for rx_buffer structures. Since we use one 3728 * rx_buffer per received packet, the maximum number of rx_buffer's 3729 * that we'll need is equal to the number of receive descriptors 3730 * that we've allocated. 3731 * 3732 **********************************************************************/ 3733 static int 3734 em_allocate_receive_buffers(struct rx_ring *rxr) 3735 { 3736 struct adapter *adapter = rxr->adapter; 3737 device_t dev = adapter->dev; 3738 struct em_buffer *rxbuf; 3739 int error; 3740 3741 rxr->rx_buffers = malloc(sizeof(struct em_buffer) * 3742 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3743 if (rxr->rx_buffers == NULL) { 3744 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3745 return (ENOMEM); 3746 } 3747 3748 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3749 1, 0, /* alignment, bounds */ 3750 BUS_SPACE_MAXADDR, /* lowaddr */ 3751 BUS_SPACE_MAXADDR, /* highaddr */ 3752 NULL, NULL, /* filter, filterarg */ 3753 MCLBYTES, /* maxsize */ 3754 1, /* nsegments */ 3755 MCLBYTES, /* maxsegsize */ 3756 0, /* flags */ 3757 NULL, /* lockfunc */ 3758 NULL, /* lockarg */ 3759 &rxr->rxtag); 3760 if (error) { 3761 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 3762 __func__, error); 3763 goto fail; 3764 } 3765 3766 /* Create the spare map (used by getbuf) */ 3767 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3768 &rxr->rx_sparemap); 3769 if (error) { 3770 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3771 __func__, error); 3772 goto fail; 3773 } 3774 3775 rxbuf = rxr->rx_buffers; 3776 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3777 rxbuf = &rxr->rx_buffers[i]; 3778 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3779 &rxbuf->map); 3780 if (error) { 3781 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3782 __func__, error); 3783 goto fail; 3784 } 3785 } 3786 3787 return (0); 3788 3789 fail: 3790 em_free_receive_structures(adapter); 3791 return (error); 3792 } 3793 3794 3795 /********************************************************************* 3796 * 3797 * Initialize a receive ring and its buffers. 3798 * 3799 **********************************************************************/ 3800 static int 3801 em_setup_receive_ring(struct rx_ring *rxr) 3802 { 3803 struct adapter *adapter = rxr->adapter; 3804 struct em_buffer *rxbuf; 3805 bus_dma_segment_t seg[1]; 3806 int rsize, nsegs, error; 3807 3808 3809 /* Clear the ring contents */ 3810 EM_RX_LOCK(rxr); 3811 rsize = roundup2(adapter->num_rx_desc * 3812 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 3813 bzero((void *)rxr->rx_base, rsize); 3814 3815 /* 3816 ** Free current RX buffer structs and their mbufs 3817 */ 3818 for (int i = 0; i < adapter->num_rx_desc; i++) { 3819 rxbuf = &rxr->rx_buffers[i]; 3820 if (rxbuf->m_head != NULL) { 3821 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3822 BUS_DMASYNC_POSTREAD); 3823 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3824 m_freem(rxbuf->m_head); 3825 } 3826 } 3827 3828 /* Now replenish the mbufs */ 3829 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3830 3831 rxbuf = &rxr->rx_buffers[j]; 3832 rxbuf->m_head = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3833 if (rxbuf->m_head == NULL) 3834 panic("RX ring hdr initialization failed!\n"); 3835 rxbuf->m_head->m_len = MCLBYTES; 3836 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ 3837 rxbuf->m_head->m_pkthdr.len = MCLBYTES; 3838 3839 /* Get the memory mapping */ 3840 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3841 rxbuf->map, rxbuf->m_head, seg, 3842 &nsegs, BUS_DMA_NOWAIT); 3843 if (error != 0) 3844 panic("RX ring dma initialization failed!\n"); 3845 bus_dmamap_sync(rxr->rxtag, 3846 rxbuf->map, BUS_DMASYNC_PREREAD); 3847 3848 /* Update descriptor */ 3849 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); 3850 } 3851 3852 3853 /* Setup our descriptor indices */ 3854 rxr->next_to_check = 0; 3855 rxr->next_to_refresh = 0; 3856 3857 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3858 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3859 3860 EM_RX_UNLOCK(rxr); 3861 return (0); 3862 } 3863 3864 /********************************************************************* 3865 * 3866 * Initialize all receive rings. 3867 * 3868 **********************************************************************/ 3869 static int 3870 em_setup_receive_structures(struct adapter *adapter) 3871 { 3872 struct rx_ring *rxr = adapter->rx_rings; 3873 int j; 3874 3875 for (j = 0; j < adapter->num_queues; j++, rxr++) 3876 if (em_setup_receive_ring(rxr)) 3877 goto fail; 3878 3879 return (0); 3880 fail: 3881 /* 3882 * Free RX buffers allocated so far, we will only handle 3883 * the rings that completed, the failing case will have 3884 * cleaned up for itself. 'j' failed, so its the terminus. 3885 */ 3886 for (int i = 0; i < j; ++i) { 3887 rxr = &adapter->rx_rings[i]; 3888 for (int n = 0; n < adapter->num_rx_desc; n++) { 3889 struct em_buffer *rxbuf; 3890 rxbuf = &rxr->rx_buffers[n]; 3891 if (rxbuf->m_head != NULL) { 3892 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3893 BUS_DMASYNC_POSTREAD); 3894 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3895 m_freem(rxbuf->m_head); 3896 rxbuf->m_head = NULL; 3897 } 3898 } 3899 } 3900 3901 return (ENOBUFS); 3902 } 3903 3904 /********************************************************************* 3905 * 3906 * Free all receive rings. 3907 * 3908 **********************************************************************/ 3909 static void 3910 em_free_receive_structures(struct adapter *adapter) 3911 { 3912 struct rx_ring *rxr = adapter->rx_rings; 3913 3914 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3915 em_free_receive_buffers(rxr); 3916 /* Free the ring memory as well */ 3917 em_dma_free(adapter, &rxr->rxdma); 3918 EM_RX_LOCK_DESTROY(rxr); 3919 } 3920 3921 free(adapter->rx_rings, M_DEVBUF); 3922 } 3923 3924 3925 /********************************************************************* 3926 * 3927 * Free receive ring data structures 3928 * 3929 **********************************************************************/ 3930 static void 3931 em_free_receive_buffers(struct rx_ring *rxr) 3932 { 3933 struct adapter *adapter = rxr->adapter; 3934 struct em_buffer *rxbuf = NULL; 3935 3936 INIT_DEBUGOUT("free_receive_buffers: begin"); 3937 3938 if (rxr->rx_sparemap) { 3939 bus_dmamap_destroy(rxr->rxtag, rxr->rx_sparemap); 3940 rxr->rx_sparemap = NULL; 3941 } 3942 3943 if (rxr->rx_buffers != NULL) { 3944 for (int i = 0; i < adapter->num_rx_desc; i++) { 3945 rxbuf = &rxr->rx_buffers[i]; 3946 if (rxbuf->map != NULL) { 3947 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3948 BUS_DMASYNC_POSTREAD); 3949 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3950 bus_dmamap_destroy(rxr->rxtag, rxbuf->map); 3951 } 3952 if (rxbuf->m_head != NULL) { 3953 m_freem(rxbuf->m_head); 3954 rxbuf->m_head = NULL; 3955 } 3956 } 3957 free(rxr->rx_buffers, M_DEVBUF); 3958 rxr->rx_buffers = NULL; 3959 } 3960 3961 if (rxr->rxtag != NULL) { 3962 bus_dma_tag_destroy(rxr->rxtag); 3963 rxr->rxtag = NULL; 3964 } 3965 3966 return; 3967 } 3968 3969 3970 /********************************************************************* 3971 * 3972 * Enable receive unit. 3973 * 3974 **********************************************************************/ 3975 #define MAX_INTS_PER_SEC 8000 3976 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 3977 3978 static void 3979 em_initialize_receive_unit(struct adapter *adapter) 3980 { 3981 struct rx_ring *rxr = adapter->rx_rings; 3982 struct ifnet *ifp = adapter->ifp; 3983 struct e1000_hw *hw = &adapter->hw; 3984 u64 bus_addr; 3985 u32 rctl, rxcsum; 3986 3987 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 3988 3989 /* 3990 * Make sure receives are disabled while setting 3991 * up the descriptor ring 3992 */ 3993 rctl = E1000_READ_REG(hw, E1000_RCTL); 3994 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3995 3996 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 3997 adapter->rx_abs_int_delay.value); 3998 /* 3999 * Set the interrupt throttling rate. Value is calculated 4000 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4001 */ 4002 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); 4003 4004 /* 4005 ** When using MSIX interrupts we need to throttle 4006 ** using the EITR register (82574 only) 4007 */ 4008 if (hw->mac.type == e1000_82574) 4009 for (int i = 0; i < 4; i++) 4010 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 4011 DEFAULT_ITR); 4012 4013 /* Disable accelerated ackknowledge */ 4014 if (adapter->hw.mac.type == e1000_82574) 4015 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4016 4017 if (ifp->if_capenable & IFCAP_RXCSUM) { 4018 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 4019 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4020 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4021 } 4022 4023 /* 4024 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4025 ** long latencies are observed, like Lenovo X60. This 4026 ** change eliminates the problem, but since having positive 4027 ** values in RDTR is a known source of problems on other 4028 ** platforms another solution is being sought. 4029 */ 4030 if (hw->mac.type == e1000_82573) 4031 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 4032 4033 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4034 /* Setup the Base and Length of the Rx Descriptor Ring */ 4035 bus_addr = rxr->rxdma.dma_paddr; 4036 E1000_WRITE_REG(hw, E1000_RDLEN(i), 4037 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4038 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); 4039 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); 4040 /* Setup the Head and Tail Descriptor Pointers */ 4041 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4042 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1); 4043 } 4044 4045 /* Setup the Receive Control Register */ 4046 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4047 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 4048 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 4049 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4050 4051 /* Strip the CRC */ 4052 rctl |= E1000_RCTL_SECRC; 4053 4054 /* Make sure VLAN Filters are off */ 4055 rctl &= ~E1000_RCTL_VFE; 4056 rctl &= ~E1000_RCTL_SBP; 4057 rctl |= E1000_RCTL_SZ_2048; 4058 if (ifp->if_mtu > ETHERMTU) 4059 rctl |= E1000_RCTL_LPE; 4060 else 4061 rctl &= ~E1000_RCTL_LPE; 4062 4063 /* Write out the settings */ 4064 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4065 4066 return; 4067 } 4068 4069 4070 /********************************************************************* 4071 * 4072 * This routine executes in interrupt context. It replenishes 4073 * the mbufs in the descriptor and sends data which has been 4074 * dma'ed into host memory to upper layer. 4075 * 4076 * We loop at most count times if count is > 0, or until done if 4077 * count < 0. 4078 * 4079 * For polling we also now return the number of cleaned packets 4080 *********************************************************************/ 4081 static int 4082 em_rxeof(struct rx_ring *rxr, int count) 4083 { 4084 struct adapter *adapter = rxr->adapter; 4085 struct ifnet *ifp = adapter->ifp; 4086 struct mbuf *mp, *sendmp; 4087 u8 status = 0; 4088 u16 len; 4089 int i, processed, rxdone = 0; 4090 bool eop; 4091 struct e1000_rx_desc *cur; 4092 4093 EM_RX_LOCK_ASSERT(rxr); 4094 4095 for (i = rxr->next_to_check, processed = 0; count != 0;) { 4096 4097 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4098 break; 4099 4100 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4101 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4102 4103 cur = &rxr->rx_base[i]; 4104 status = cur->status; 4105 mp = sendmp = NULL; 4106 4107 if ((status & E1000_RXD_STAT_DD) == 0) 4108 break; 4109 4110 len = le16toh(cur->length); 4111 eop = (status & E1000_RXD_STAT_EOP) != 0; 4112 count--; 4113 4114 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) == 0) { 4115 4116 /* Assign correct length to the current fragment */ 4117 mp = rxr->rx_buffers[i].m_head; 4118 mp->m_len = len; 4119 4120 if (rxr->fmp == NULL) { 4121 mp->m_pkthdr.len = len; 4122 rxr->fmp = mp; /* Store the first mbuf */ 4123 rxr->lmp = mp; 4124 } else { 4125 /* Chain mbuf's together */ 4126 mp->m_flags &= ~M_PKTHDR; 4127 rxr->lmp->m_next = mp; 4128 rxr->lmp = rxr->lmp->m_next; 4129 rxr->fmp->m_pkthdr.len += len; 4130 } 4131 4132 if (eop) { 4133 rxr->fmp->m_pkthdr.rcvif = ifp; 4134 ifp->if_ipackets++; 4135 em_receive_checksum(cur, rxr->fmp); 4136 #ifndef __NO_STRICT_ALIGNMENT 4137 if (adapter->max_frame_size > 4138 (MCLBYTES - ETHER_ALIGN) && 4139 em_fixup_rx(rxr) != 0) 4140 goto skip; 4141 #endif 4142 if (status & E1000_RXD_STAT_VP) { 4143 rxr->fmp->m_pkthdr.ether_vtag = 4144 (le16toh(cur->special) & 4145 E1000_RXD_SPC_VLAN_MASK); 4146 rxr->fmp->m_flags |= M_VLANTAG; 4147 } 4148 #ifdef EM_MULTIQUEUE 4149 rxr->fmp->m_pkthdr.flowid = curcpu; 4150 rxr->fmp->m_flags |= M_FLOWID; 4151 #endif 4152 #ifndef __NO_STRICT_ALIGNMENT 4153 skip: 4154 #endif 4155 sendmp = rxr->fmp; 4156 rxr->fmp = NULL; 4157 rxr->lmp = NULL; 4158 } 4159 } else { 4160 ifp->if_ierrors++; 4161 /* Reuse loaded DMA map and just update mbuf chain */ 4162 mp = rxr->rx_buffers[i].m_head; 4163 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4164 mp->m_data = mp->m_ext.ext_buf; 4165 mp->m_next = NULL; 4166 if (adapter->max_frame_size <= 4167 (MCLBYTES - ETHER_ALIGN)) 4168 m_adj(mp, ETHER_ALIGN); 4169 if (rxr->fmp != NULL) { 4170 m_freem(rxr->fmp); 4171 rxr->fmp = NULL; 4172 rxr->lmp = NULL; 4173 } 4174 sendmp = NULL; 4175 } 4176 4177 /* Zero out the receive descriptors status. */ 4178 cur->status = 0; 4179 ++rxdone; /* cumulative for POLL */ 4180 ++processed; 4181 4182 /* Advance our pointers to the next descriptor. */ 4183 if (++i == adapter->num_rx_desc) 4184 i = 0; 4185 4186 /* Send to the stack */ 4187 if (sendmp != NULL) 4188 (*ifp->if_input)(ifp, sendmp); 4189 4190 /* Only refresh mbufs every 8 descriptors */ 4191 if (processed == 8) { 4192 em_refresh_mbufs(rxr, i); 4193 processed = 0; 4194 } 4195 } 4196 4197 /* Catch any remaining refresh work */ 4198 if (processed != 0) { 4199 em_refresh_mbufs(rxr, i); 4200 processed = 0; 4201 } 4202 4203 rxr->next_to_check = i; 4204 4205 #ifdef DEVICE_POLLING 4206 return (rxdone); 4207 #else 4208 return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); 4209 #endif 4210 } 4211 4212 #ifndef __NO_STRICT_ALIGNMENT 4213 /* 4214 * When jumbo frames are enabled we should realign entire payload on 4215 * architecures with strict alignment. This is serious design mistake of 8254x 4216 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4217 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4218 * payload. On architecures without strict alignment restrictions 8254x still 4219 * performs unaligned memory access which would reduce the performance too. 4220 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4221 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4222 * existing mbuf chain. 4223 * 4224 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4225 * not used at all on architectures with strict alignment. 4226 */ 4227 static int 4228 em_fixup_rx(struct rx_ring *rxr) 4229 { 4230 struct adapter *adapter = rxr->adapter; 4231 struct mbuf *m, *n; 4232 int error; 4233 4234 error = 0; 4235 m = rxr->fmp; 4236 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4237 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4238 m->m_data += ETHER_HDR_LEN; 4239 } else { 4240 MGETHDR(n, M_DONTWAIT, MT_DATA); 4241 if (n != NULL) { 4242 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4243 m->m_data += ETHER_HDR_LEN; 4244 m->m_len -= ETHER_HDR_LEN; 4245 n->m_len = ETHER_HDR_LEN; 4246 M_MOVE_PKTHDR(n, m); 4247 n->m_next = m; 4248 rxr->fmp = n; 4249 } else { 4250 adapter->dropped_pkts++; 4251 m_freem(rxr->fmp); 4252 rxr->fmp = NULL; 4253 error = ENOMEM; 4254 } 4255 } 4256 4257 return (error); 4258 } 4259 #endif 4260 4261 /********************************************************************* 4262 * 4263 * Verify that the hardware indicated that the checksum is valid. 4264 * Inform the stack about the status of checksum so that stack 4265 * doesn't spend time verifying the checksum. 4266 * 4267 *********************************************************************/ 4268 static void 4269 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4270 { 4271 /* Ignore Checksum bit is set */ 4272 if (rx_desc->status & E1000_RXD_STAT_IXSM) { 4273 mp->m_pkthdr.csum_flags = 0; 4274 return; 4275 } 4276 4277 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4278 /* Did it pass? */ 4279 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4280 /* IP Checksum Good */ 4281 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4282 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4283 4284 } else { 4285 mp->m_pkthdr.csum_flags = 0; 4286 } 4287 } 4288 4289 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4290 /* Did it pass? */ 4291 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4292 mp->m_pkthdr.csum_flags |= 4293 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4294 mp->m_pkthdr.csum_data = htons(0xffff); 4295 } 4296 } 4297 } 4298 4299 /* 4300 * This routine is run via an vlan 4301 * config EVENT 4302 */ 4303 static void 4304 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4305 { 4306 struct adapter *adapter = ifp->if_softc; 4307 u32 index, bit; 4308 4309 if (ifp->if_softc != arg) /* Not our event */ 4310 return; 4311 4312 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ 4313 return; 4314 4315 index = (vtag >> 5) & 0x7F; 4316 bit = vtag & 0x1F; 4317 em_shadow_vfta[index] |= (1 << bit); 4318 ++adapter->num_vlans; 4319 /* Re-init to load the changes */ 4320 em_init(adapter); 4321 } 4322 4323 /* 4324 * This routine is run via an vlan 4325 * unconfig EVENT 4326 */ 4327 static void 4328 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4329 { 4330 struct adapter *adapter = ifp->if_softc; 4331 u32 index, bit; 4332 4333 if (ifp->if_softc != arg) 4334 return; 4335 4336 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4337 return; 4338 4339 index = (vtag >> 5) & 0x7F; 4340 bit = vtag & 0x1F; 4341 em_shadow_vfta[index] &= ~(1 << bit); 4342 --adapter->num_vlans; 4343 /* Re-init to load the changes */ 4344 em_init(adapter); 4345 } 4346 4347 static void 4348 em_setup_vlan_hw_support(struct adapter *adapter) 4349 { 4350 struct e1000_hw *hw = &adapter->hw; 4351 u32 reg; 4352 4353 /* 4354 ** We get here thru init_locked, meaning 4355 ** a soft reset, this has already cleared 4356 ** the VFTA and other state, so if there 4357 ** have been no vlan's registered do nothing. 4358 */ 4359 if (adapter->num_vlans == 0) 4360 return; 4361 4362 /* 4363 ** A soft reset zero's out the VFTA, so 4364 ** we need to repopulate it now. 4365 */ 4366 for (int i = 0; i < EM_VFTA_SIZE; i++) 4367 if (em_shadow_vfta[i] != 0) 4368 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4369 i, em_shadow_vfta[i]); 4370 4371 reg = E1000_READ_REG(hw, E1000_CTRL); 4372 reg |= E1000_CTRL_VME; 4373 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4374 4375 /* Enable the Filter Table */ 4376 reg = E1000_READ_REG(hw, E1000_RCTL); 4377 reg &= ~E1000_RCTL_CFIEN; 4378 reg |= E1000_RCTL_VFE; 4379 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4380 4381 /* Update the frame size */ 4382 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4383 adapter->max_frame_size + VLAN_TAG_SIZE); 4384 } 4385 4386 static void 4387 em_enable_intr(struct adapter *adapter) 4388 { 4389 struct e1000_hw *hw = &adapter->hw; 4390 u32 ims_mask = IMS_ENABLE_MASK; 4391 4392 if (hw->mac.type == e1000_82574) 4393 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4394 ims_mask |= EM_MSIX_MASK; 4395 } 4396 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4397 } 4398 4399 static void 4400 em_disable_intr(struct adapter *adapter) 4401 { 4402 struct e1000_hw *hw = &adapter->hw; 4403 4404 if (hw->mac.type == e1000_82574) 4405 E1000_WRITE_REG(hw, EM_EIAC, 0); 4406 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4407 } 4408 4409 /* 4410 * Bit of a misnomer, what this really means is 4411 * to enable OS management of the system... aka 4412 * to disable special hardware management features 4413 */ 4414 static void 4415 em_init_manageability(struct adapter *adapter) 4416 { 4417 /* A shared code workaround */ 4418 #define E1000_82542_MANC2H E1000_MANC2H 4419 if (adapter->has_manage) { 4420 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4421 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4422 4423 /* disable hardware interception of ARP */ 4424 manc &= ~(E1000_MANC_ARP_EN); 4425 4426 /* enable receiving management packets to the host */ 4427 manc |= E1000_MANC_EN_MNG2HOST; 4428 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4429 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4430 manc2h |= E1000_MNG2HOST_PORT_623; 4431 manc2h |= E1000_MNG2HOST_PORT_664; 4432 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4433 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4434 } 4435 } 4436 4437 /* 4438 * Give control back to hardware management 4439 * controller if there is one. 4440 */ 4441 static void 4442 em_release_manageability(struct adapter *adapter) 4443 { 4444 if (adapter->has_manage) { 4445 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4446 4447 /* re-enable hardware interception of ARP */ 4448 manc |= E1000_MANC_ARP_EN; 4449 manc &= ~E1000_MANC_EN_MNG2HOST; 4450 4451 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4452 } 4453 } 4454 4455 /* 4456 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 4457 * For ASF and Pass Through versions of f/w this means 4458 * that the driver is loaded. For AMT version type f/w 4459 * this means that the network i/f is open. 4460 */ 4461 static void 4462 em_get_hw_control(struct adapter *adapter) 4463 { 4464 u32 ctrl_ext, swsm; 4465 4466 if (adapter->hw.mac.type == e1000_82573) { 4467 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4468 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4469 swsm | E1000_SWSM_DRV_LOAD); 4470 return; 4471 } 4472 /* else */ 4473 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4474 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4475 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4476 return; 4477 } 4478 4479 /* 4480 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4481 * For ASF and Pass Through versions of f/w this means that 4482 * the driver is no longer loaded. For AMT versions of the 4483 * f/w this means that the network i/f is closed. 4484 */ 4485 static void 4486 em_release_hw_control(struct adapter *adapter) 4487 { 4488 u32 ctrl_ext, swsm; 4489 4490 if (!adapter->has_manage) 4491 return; 4492 4493 if (adapter->hw.mac.type == e1000_82573) { 4494 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4495 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4496 swsm & ~E1000_SWSM_DRV_LOAD); 4497 return; 4498 } 4499 /* else */ 4500 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4501 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4502 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4503 return; 4504 } 4505 4506 static int 4507 em_is_valid_ether_addr(u8 *addr) 4508 { 4509 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4510 4511 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4512 return (FALSE); 4513 } 4514 4515 return (TRUE); 4516 } 4517 4518 /* 4519 ** Parse the interface capabilities with regard 4520 ** to both system management and wake-on-lan for 4521 ** later use. 4522 */ 4523 static void 4524 em_get_wakeup(device_t dev) 4525 { 4526 struct adapter *adapter = device_get_softc(dev); 4527 u16 eeprom_data = 0, device_id, apme_mask; 4528 4529 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 4530 apme_mask = EM_EEPROM_APME; 4531 4532 switch (adapter->hw.mac.type) { 4533 case e1000_82573: 4534 case e1000_82583: 4535 adapter->has_amt = TRUE; 4536 /* Falls thru */ 4537 case e1000_82571: 4538 case e1000_82572: 4539 case e1000_80003es2lan: 4540 if (adapter->hw.bus.func == 1) { 4541 e1000_read_nvm(&adapter->hw, 4542 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 4543 break; 4544 } else 4545 e1000_read_nvm(&adapter->hw, 4546 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4547 break; 4548 case e1000_ich8lan: 4549 case e1000_ich9lan: 4550 case e1000_ich10lan: 4551 case e1000_pchlan: 4552 apme_mask = E1000_WUC_APME; 4553 adapter->has_amt = TRUE; 4554 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 4555 break; 4556 default: 4557 e1000_read_nvm(&adapter->hw, 4558 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4559 break; 4560 } 4561 if (eeprom_data & apme_mask) 4562 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 4563 /* 4564 * We have the eeprom settings, now apply the special cases 4565 * where the eeprom may be wrong or the board won't support 4566 * wake on lan on a particular port 4567 */ 4568 device_id = pci_get_device(dev); 4569 switch (device_id) { 4570 case E1000_DEV_ID_82571EB_FIBER: 4571 /* Wake events only supported on port A for dual fiber 4572 * regardless of eeprom setting */ 4573 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 4574 E1000_STATUS_FUNC_1) 4575 adapter->wol = 0; 4576 break; 4577 case E1000_DEV_ID_82571EB_QUAD_COPPER: 4578 case E1000_DEV_ID_82571EB_QUAD_FIBER: 4579 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 4580 /* if quad port adapter, disable WoL on all but port A */ 4581 if (global_quad_port_a != 0) 4582 adapter->wol = 0; 4583 /* Reset for multiple quad port adapters */ 4584 if (++global_quad_port_a == 4) 4585 global_quad_port_a = 0; 4586 break; 4587 } 4588 return; 4589 } 4590 4591 4592 /* 4593 * Enable PCI Wake On Lan capability 4594 */ 4595 static void 4596 em_enable_wakeup(device_t dev) 4597 { 4598 struct adapter *adapter = device_get_softc(dev); 4599 struct ifnet *ifp = adapter->ifp; 4600 u32 pmc, ctrl, ctrl_ext, rctl; 4601 u16 status; 4602 4603 if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) 4604 return; 4605 4606 /* Advertise the wakeup capability */ 4607 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4608 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 4609 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4610 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4611 4612 if ((adapter->hw.mac.type == e1000_ich8lan) || 4613 (adapter->hw.mac.type == e1000_pchlan) || 4614 (adapter->hw.mac.type == e1000_ich9lan) || 4615 (adapter->hw.mac.type == e1000_ich10lan)) { 4616 e1000_disable_gig_wol_ich8lan(&adapter->hw); 4617 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); 4618 } 4619 4620 /* Keep the laser running on Fiber adapters */ 4621 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 4622 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 4623 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4624 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 4625 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); 4626 } 4627 4628 /* 4629 ** Determine type of Wakeup: note that wol 4630 ** is set with all bits on by default. 4631 */ 4632 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) 4633 adapter->wol &= ~E1000_WUFC_MAG; 4634 4635 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) 4636 adapter->wol &= ~E1000_WUFC_MC; 4637 else { 4638 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4639 rctl |= E1000_RCTL_MPE; 4640 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4641 } 4642 4643 if (adapter->hw.mac.type == e1000_pchlan) { 4644 if (em_enable_phy_wakeup(adapter)) 4645 return; 4646 } else { 4647 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4648 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 4649 } 4650 4651 if (adapter->hw.phy.type == e1000_phy_igp_3) 4652 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 4653 4654 /* Request PME */ 4655 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); 4656 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 4657 if (ifp->if_capenable & IFCAP_WOL) 4658 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4659 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); 4660 4661 return; 4662 } 4663 4664 /* 4665 ** WOL in the newer chipset interfaces (pchlan) 4666 ** require thing to be copied into the phy 4667 */ 4668 static int 4669 em_enable_phy_wakeup(struct adapter *adapter) 4670 { 4671 struct e1000_hw *hw = &adapter->hw; 4672 u32 mreg, ret = 0; 4673 u16 preg; 4674 4675 /* copy MAC RARs to PHY RARs */ 4676 for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { 4677 mreg = E1000_READ_REG(hw, E1000_RAL(i)); 4678 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); 4679 e1000_write_phy_reg(hw, BM_RAR_M(i), 4680 (u16)((mreg >> 16) & 0xFFFF)); 4681 mreg = E1000_READ_REG(hw, E1000_RAH(i)); 4682 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); 4683 e1000_write_phy_reg(hw, BM_RAR_CTRL(i), 4684 (u16)((mreg >> 16) & 0xFFFF)); 4685 } 4686 4687 /* copy MAC MTA to PHY MTA */ 4688 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 4689 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 4690 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 4691 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 4692 (u16)((mreg >> 16) & 0xFFFF)); 4693 } 4694 4695 /* configure PHY Rx Control register */ 4696 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); 4697 mreg = E1000_READ_REG(hw, E1000_RCTL); 4698 if (mreg & E1000_RCTL_UPE) 4699 preg |= BM_RCTL_UPE; 4700 if (mreg & E1000_RCTL_MPE) 4701 preg |= BM_RCTL_MPE; 4702 preg &= ~(BM_RCTL_MO_MASK); 4703 if (mreg & E1000_RCTL_MO_3) 4704 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 4705 << BM_RCTL_MO_SHIFT); 4706 if (mreg & E1000_RCTL_BAM) 4707 preg |= BM_RCTL_BAM; 4708 if (mreg & E1000_RCTL_PMCF) 4709 preg |= BM_RCTL_PMCF; 4710 mreg = E1000_READ_REG(hw, E1000_CTRL); 4711 if (mreg & E1000_CTRL_RFCE) 4712 preg |= BM_RCTL_RFCE; 4713 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); 4714 4715 /* enable PHY wakeup in MAC register */ 4716 E1000_WRITE_REG(hw, E1000_WUC, 4717 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); 4718 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); 4719 4720 /* configure and enable PHY wakeup in PHY registers */ 4721 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); 4722 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); 4723 4724 /* activate PHY wakeup */ 4725 ret = hw->phy.ops.acquire(hw); 4726 if (ret) { 4727 printf("Could not acquire PHY\n"); 4728 return ret; 4729 } 4730 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4731 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 4732 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 4733 if (ret) { 4734 printf("Could not read PHY page 769\n"); 4735 goto out; 4736 } 4737 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 4738 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 4739 if (ret) 4740 printf("Could not set PHY Host Wakeup bit\n"); 4741 out: 4742 hw->phy.ops.release(hw); 4743 4744 return ret; 4745 } 4746 4747 static void 4748 em_led_func(void *arg, int onoff) 4749 { 4750 struct adapter *adapter = arg; 4751 4752 EM_CORE_LOCK(adapter); 4753 if (onoff) { 4754 e1000_setup_led(&adapter->hw); 4755 e1000_led_on(&adapter->hw); 4756 } else { 4757 e1000_led_off(&adapter->hw); 4758 e1000_cleanup_led(&adapter->hw); 4759 } 4760 EM_CORE_UNLOCK(adapter); 4761 } 4762 4763 /********************************************************************** 4764 * 4765 * Update the board statistics counters. 4766 * 4767 **********************************************************************/ 4768 static void 4769 em_update_stats_counters(struct adapter *adapter) 4770 { 4771 struct ifnet *ifp; 4772 4773 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4774 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4775 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4776 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4777 } 4778 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4779 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4780 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4781 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4782 4783 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4784 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4785 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4786 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4787 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4788 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4789 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4790 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4791 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4792 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4793 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4794 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4795 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4796 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4797 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4798 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4799 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4800 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4801 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4802 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4803 4804 /* For the 64-bit byte counters the low dword must be read first. */ 4805 /* Both registers clear on the read of the high dword */ 4806 4807 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4808 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4809 4810 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4811 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4812 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4813 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4814 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4815 4816 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4817 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4818 4819 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4820 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4821 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4822 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4823 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4824 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4825 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4826 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4827 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4828 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4829 4830 if (adapter->hw.mac.type >= e1000_82543) { 4831 adapter->stats.algnerrc += 4832 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4833 adapter->stats.rxerrc += 4834 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4835 adapter->stats.tncrs += 4836 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4837 adapter->stats.cexterr += 4838 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4839 adapter->stats.tsctc += 4840 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4841 adapter->stats.tsctfc += 4842 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4843 } 4844 ifp = adapter->ifp; 4845 4846 ifp->if_collisions = adapter->stats.colc; 4847 4848 /* Rx Errors */ 4849 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4850 adapter->stats.crcerrs + adapter->stats.algnerrc + 4851 adapter->stats.ruc + adapter->stats.roc + 4852 adapter->stats.mpc + adapter->stats.cexterr; 4853 4854 /* Tx Errors */ 4855 ifp->if_oerrors = adapter->stats.ecol + 4856 adapter->stats.latecol + adapter->watchdog_events; 4857 } 4858 4859 4860 /********************************************************************** 4861 * 4862 * This routine is called only when em_display_debug_stats is enabled. 4863 * This routine provides a way to take a look at important statistics 4864 * maintained by the driver and hardware. 4865 * 4866 **********************************************************************/ 4867 static void 4868 em_print_debug_info(struct adapter *adapter) 4869 { 4870 device_t dev = adapter->dev; 4871 u8 *hw_addr = adapter->hw.hw_addr; 4872 struct rx_ring *rxr = adapter->rx_rings; 4873 struct tx_ring *txr = adapter->tx_rings; 4874 4875 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4876 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4877 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4878 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4879 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4880 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4881 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4882 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4883 adapter->hw.fc.high_water, 4884 adapter->hw.fc.low_water); 4885 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 4886 E1000_READ_REG(&adapter->hw, E1000_TIDV), 4887 E1000_READ_REG(&adapter->hw, E1000_TADV)); 4888 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 4889 E1000_READ_REG(&adapter->hw, E1000_RDTR), 4890 E1000_READ_REG(&adapter->hw, E1000_RADV)); 4891 4892 for (int i = 0; i < adapter->num_queues; i++, txr++) { 4893 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i, 4894 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4895 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4896 device_printf(dev, "TX(%d) no descriptors avail event = %ld\n", 4897 txr->me, txr->no_desc_avail); 4898 device_printf(dev, "TX(%d) MSIX IRQ Handled = %ld\n", 4899 txr->me, txr->tx_irq); 4900 device_printf(dev, "Num Tx descriptors avail = %d\n", 4901 txr->tx_avail); 4902 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 4903 txr->no_desc_avail); 4904 } 4905 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4906 device_printf(dev, "RX(%d) MSIX IRQ Handled = %ld\n", 4907 rxr->me, rxr->rx_irq); 4908 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 4909 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4910 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4911 } 4912 device_printf(dev, "Std mbuf failed = %ld\n", 4913 adapter->mbuf_alloc_failed); 4914 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4915 adapter->mbuf_cluster_failed); 4916 device_printf(dev, "Driver dropped packets = %ld\n", 4917 adapter->dropped_pkts); 4918 } 4919 4920 static void 4921 em_print_hw_stats(struct adapter *adapter) 4922 { 4923 device_t dev = adapter->dev; 4924 4925 device_printf(dev, "Excessive collisions = %lld\n", 4926 (long long)adapter->stats.ecol); 4927 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 4928 device_printf(dev, "Symbol errors = %lld\n", 4929 (long long)adapter->stats.symerrs); 4930 #endif 4931 device_printf(dev, "Sequence errors = %lld\n", 4932 (long long)adapter->stats.sec); 4933 device_printf(dev, "Defer count = %lld\n", 4934 (long long)adapter->stats.dc); 4935 device_printf(dev, "Missed Packets = %lld\n", 4936 (long long)adapter->stats.mpc); 4937 device_printf(dev, "Receive No Buffers = %lld\n", 4938 (long long)adapter->stats.rnbc); 4939 /* RLEC is inaccurate on some hardware, calculate our own. */ 4940 device_printf(dev, "Receive Length Errors = %lld\n", 4941 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4942 device_printf(dev, "Receive errors = %lld\n", 4943 (long long)adapter->stats.rxerrc); 4944 device_printf(dev, "Crc errors = %lld\n", 4945 (long long)adapter->stats.crcerrs); 4946 device_printf(dev, "Alignment errors = %lld\n", 4947 (long long)adapter->stats.algnerrc); 4948 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4949 (long long)adapter->stats.cexterr); 4950 device_printf(dev, "watchdog timeouts = %ld\n", 4951 adapter->watchdog_events); 4952 device_printf(dev, "XON Rcvd = %lld\n", 4953 (long long)adapter->stats.xonrxc); 4954 device_printf(dev, "XON Xmtd = %lld\n", 4955 (long long)adapter->stats.xontxc); 4956 device_printf(dev, "XOFF Rcvd = %lld\n", 4957 (long long)adapter->stats.xoffrxc); 4958 device_printf(dev, "XOFF Xmtd = %lld\n", 4959 (long long)adapter->stats.xofftxc); 4960 device_printf(dev, "Good Packets Rcvd = %lld\n", 4961 (long long)adapter->stats.gprc); 4962 device_printf(dev, "Good Packets Xmtd = %lld\n", 4963 (long long)adapter->stats.gptc); 4964 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4965 (long long)adapter->stats.tsctc); 4966 device_printf(dev, "TSO Contexts Failed = %lld\n", 4967 (long long)adapter->stats.tsctfc); 4968 } 4969 4970 /********************************************************************** 4971 * 4972 * This routine provides a way to dump out the adapter eeprom, 4973 * often a useful debug/service tool. This only dumps the first 4974 * 32 words, stuff that matters is in that extent. 4975 * 4976 **********************************************************************/ 4977 static void 4978 em_print_nvm_info(struct adapter *adapter) 4979 { 4980 u16 eeprom_data; 4981 int i, j, row = 0; 4982 4983 /* Its a bit crude, but it gets the job done */ 4984 printf("\nInterface EEPROM Dump:\n"); 4985 printf("Offset\n0x0000 "); 4986 for (i = 0, j = 0; i < 32; i++, j++) { 4987 if (j == 8) { /* Make the offset block */ 4988 j = 0; ++row; 4989 printf("\n0x00%x0 ",row); 4990 } 4991 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4992 printf("%04x ", eeprom_data); 4993 } 4994 printf("\n"); 4995 } 4996 4997 static int 4998 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4999 { 5000 struct adapter *adapter; 5001 int error; 5002 int result; 5003 5004 result = -1; 5005 error = sysctl_handle_int(oidp, &result, 0, req); 5006 5007 if (error || !req->newptr) 5008 return (error); 5009 5010 if (result == 1) { 5011 adapter = (struct adapter *)arg1; 5012 em_print_debug_info(adapter); 5013 } 5014 /* 5015 * This value will cause a hex dump of the 5016 * first 32 16-bit words of the EEPROM to 5017 * the screen. 5018 */ 5019 if (result == 2) { 5020 adapter = (struct adapter *)arg1; 5021 em_print_nvm_info(adapter); 5022 } 5023 5024 return (error); 5025 } 5026 5027 5028 static int 5029 em_sysctl_stats(SYSCTL_HANDLER_ARGS) 5030 { 5031 struct adapter *adapter; 5032 int error; 5033 int result; 5034 5035 result = -1; 5036 error = sysctl_handle_int(oidp, &result, 0, req); 5037 5038 if (error || !req->newptr) 5039 return (error); 5040 5041 if (result == 1) { 5042 adapter = (struct adapter *)arg1; 5043 em_print_hw_stats(adapter); 5044 } 5045 5046 return (error); 5047 } 5048 5049 static int 5050 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5051 { 5052 struct em_int_delay_info *info; 5053 struct adapter *adapter; 5054 u32 regval; 5055 int error, usecs, ticks; 5056 5057 info = (struct em_int_delay_info *)arg1; 5058 usecs = info->value; 5059 error = sysctl_handle_int(oidp, &usecs, 0, req); 5060 if (error != 0 || req->newptr == NULL) 5061 return (error); 5062 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5063 return (EINVAL); 5064 info->value = usecs; 5065 ticks = EM_USECS_TO_TICKS(usecs); 5066 5067 adapter = info->adapter; 5068 5069 EM_CORE_LOCK(adapter); 5070 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5071 regval = (regval & ~0xffff) | (ticks & 0xffff); 5072 /* Handle a few special cases. */ 5073 switch (info->offset) { 5074 case E1000_RDTR: 5075 break; 5076 case E1000_TIDV: 5077 if (ticks == 0) { 5078 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5079 /* Don't write 0 into the TIDV register. */ 5080 regval++; 5081 } else 5082 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5083 break; 5084 } 5085 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5086 EM_CORE_UNLOCK(adapter); 5087 return (0); 5088 } 5089 5090 static void 5091 em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5092 const char *description, struct em_int_delay_info *info, 5093 int offset, int value) 5094 { 5095 info->adapter = adapter; 5096 info->offset = offset; 5097 info->value = value; 5098 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5099 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5100 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5101 info, 0, em_sysctl_int_delay, "I", description); 5102 } 5103 5104 static void 5105 em_add_rx_process_limit(struct adapter *adapter, const char *name, 5106 const char *description, int *limit, int value) 5107 { 5108 *limit = value; 5109 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5110 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5111 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5112 } 5113 5114 5115