1 /****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #if __FreeBSD_version >= 800000 43 #include <sys/buf_ring.h> 44 #endif 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/kernel.h> 48 #include <sys/kthread.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/rman.h> 53 #include <sys/socket.h> 54 #include <sys/sockio.h> 55 #include <sys/sysctl.h> 56 #include <sys/taskqueue.h> 57 #include <sys/eventhandler.h> 58 #include <machine/bus.h> 59 #include <machine/resource.h> 60 61 #include <net/bpf.h> 62 #include <net/ethernet.h> 63 #include <net/if.h> 64 #include <net/if_arp.h> 65 #include <net/if_dl.h> 66 #include <net/if_media.h> 67 68 #include <net/if_types.h> 69 #include <net/if_vlan_var.h> 70 71 #include <netinet/in_systm.h> 72 #include <netinet/in.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip6.h> 76 #include <netinet/tcp.h> 77 #include <netinet/udp.h> 78 79 #include <machine/in_cksum.h> 80 #include <dev/led/led.h> 81 #include <dev/pci/pcivar.h> 82 #include <dev/pci/pcireg.h> 83 84 #include "e1000_api.h" 85 #include "e1000_82571.h" 86 #include "if_em.h" 87 88 /********************************************************************* 89 * Set this to one to display debug statistics 90 *********************************************************************/ 91 int em_display_debug_stats = 0; 92 93 /********************************************************************* 94 * Driver version: 95 *********************************************************************/ 96 char em_driver_version[] = "7.0.0"; 97 98 99 /********************************************************************* 100 * PCI Device ID Table 101 * 102 * Used by probe to select devices to load on 103 * Last field stores an index into e1000_strings 104 * Last entry must be all 0s 105 * 106 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 107 *********************************************************************/ 108 109 static em_vendor_info_t em_vendor_info_array[] = 110 { 111 /* Intel(R) PRO/1000 Network Connection */ 112 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 113 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 116 PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 118 PCI_ANY_ID, PCI_ANY_ID, 0}, 119 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 120 PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 122 PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 124 PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 126 PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 132 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 137 PCI_ANY_ID, PCI_ANY_ID, 0}, 138 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 139 PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 141 PCI_ANY_ID, PCI_ANY_ID, 0}, 142 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 143 PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, 152 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 153 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 154 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 158 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 162 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 168 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, 170 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, 172 /* required last entry */ 173 { 0, 0, 0, 0, 0} 174 }; 175 176 /********************************************************************* 177 * Table of branding strings for all supported NICs. 178 *********************************************************************/ 179 180 static char *em_strings[] = { 181 "Intel(R) PRO/1000 Network Connection" 182 }; 183 184 /********************************************************************* 185 * Function prototypes 186 *********************************************************************/ 187 static int em_probe(device_t); 188 static int em_attach(device_t); 189 static int em_detach(device_t); 190 static int em_shutdown(device_t); 191 static int em_suspend(device_t); 192 static int em_resume(device_t); 193 static void em_start(struct ifnet *); 194 static void em_start_locked(struct ifnet *, struct tx_ring *); 195 #if __FreeBSD_version >= 800000 196 static int em_mq_start(struct ifnet *, struct mbuf *); 197 static int em_mq_start_locked(struct ifnet *, 198 struct tx_ring *, struct mbuf *); 199 static void em_qflush(struct ifnet *); 200 #endif 201 static int em_ioctl(struct ifnet *, u_long, caddr_t); 202 static void em_init(void *); 203 static void em_init_locked(struct adapter *); 204 static void em_stop(void *); 205 static void em_media_status(struct ifnet *, struct ifmediareq *); 206 static int em_media_change(struct ifnet *); 207 static void em_identify_hardware(struct adapter *); 208 static int em_allocate_pci_resources(struct adapter *); 209 static int em_allocate_legacy(struct adapter *); 210 static int em_allocate_msix(struct adapter *); 211 static int em_allocate_queues(struct adapter *); 212 static int em_setup_msix(struct adapter *); 213 static void em_free_pci_resources(struct adapter *); 214 static void em_local_timer(void *); 215 static void em_reset(struct adapter *); 216 static void em_setup_interface(device_t, struct adapter *); 217 218 static void em_setup_transmit_structures(struct adapter *); 219 static void em_initialize_transmit_unit(struct adapter *); 220 static int em_allocate_transmit_buffers(struct tx_ring *); 221 static void em_free_transmit_structures(struct adapter *); 222 static void em_free_transmit_buffers(struct tx_ring *); 223 224 static int em_setup_receive_structures(struct adapter *); 225 static int em_allocate_receive_buffers(struct rx_ring *); 226 static void em_initialize_receive_unit(struct adapter *); 227 static void em_free_receive_structures(struct adapter *); 228 static void em_free_receive_buffers(struct rx_ring *); 229 230 static void em_enable_intr(struct adapter *); 231 static void em_disable_intr(struct adapter *); 232 static void em_update_stats_counters(struct adapter *); 233 static bool em_txeof(struct tx_ring *); 234 static int em_rxeof(struct rx_ring *, int); 235 #ifndef __NO_STRICT_ALIGNMENT 236 static int em_fixup_rx(struct rx_ring *); 237 #endif 238 static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); 239 static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, 240 u32 *, u32 *); 241 static bool em_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); 242 static void em_set_promisc(struct adapter *); 243 static void em_disable_promisc(struct adapter *); 244 static void em_set_multi(struct adapter *); 245 static void em_print_hw_stats(struct adapter *); 246 static void em_update_link_status(struct adapter *); 247 static void em_refresh_mbufs(struct rx_ring *, int); 248 static void em_register_vlan(void *, struct ifnet *, u16); 249 static void em_unregister_vlan(void *, struct ifnet *, u16); 250 static void em_setup_vlan_hw_support(struct adapter *); 251 static int em_xmit(struct tx_ring *, struct mbuf **); 252 static int em_dma_malloc(struct adapter *, bus_size_t, 253 struct em_dma_alloc *, int); 254 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 255 static void em_print_debug_info(struct adapter *); 256 static void em_print_nvm_info(struct adapter *); 257 static int em_is_valid_ether_addr(u8 *); 258 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 259 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 260 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 261 static void em_add_int_delay_sysctl(struct adapter *, const char *, 262 const char *, struct em_int_delay_info *, int, int); 263 /* Management and WOL Support */ 264 static void em_init_manageability(struct adapter *); 265 static void em_release_manageability(struct adapter *); 266 static void em_get_hw_control(struct adapter *); 267 static void em_release_hw_control(struct adapter *); 268 static void em_get_wakeup(device_t); 269 static void em_enable_wakeup(device_t); 270 static int em_enable_phy_wakeup(struct adapter *); 271 static void em_led_func(void *, int); 272 273 static int em_irq_fast(void *); 274 275 /* MSIX handlers */ 276 static void em_msix_tx(void *); 277 static void em_msix_rx(void *); 278 static void em_msix_link(void *); 279 static void em_handle_tx(void *context, int pending); 280 static void em_handle_rx(void *context, int pending); 281 static void em_handle_link(void *context, int pending); 282 283 static void em_add_rx_process_limit(struct adapter *, const char *, 284 const char *, int *, int); 285 286 #ifdef DEVICE_POLLING 287 static poll_handler_t em_poll; 288 #endif /* POLLING */ 289 290 /********************************************************************* 291 * FreeBSD Device Interface Entry Points 292 *********************************************************************/ 293 294 static device_method_t em_methods[] = { 295 /* Device interface */ 296 DEVMETHOD(device_probe, em_probe), 297 DEVMETHOD(device_attach, em_attach), 298 DEVMETHOD(device_detach, em_detach), 299 DEVMETHOD(device_shutdown, em_shutdown), 300 DEVMETHOD(device_suspend, em_suspend), 301 DEVMETHOD(device_resume, em_resume), 302 {0, 0} 303 }; 304 305 static driver_t em_driver = { 306 "em", em_methods, sizeof(struct adapter), 307 }; 308 309 devclass_t em_devclass; 310 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 311 MODULE_DEPEND(em, pci, 1, 1, 1); 312 MODULE_DEPEND(em, ether, 1, 1, 1); 313 314 /********************************************************************* 315 * Tunable default values. 316 *********************************************************************/ 317 318 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 319 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 320 #define M_TSO_LEN 66 321 322 /* Allow common code without TSO */ 323 #ifndef CSUM_TSO 324 #define CSUM_TSO 0 325 #endif 326 327 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 328 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 329 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 330 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 331 332 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 333 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 334 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 335 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 336 337 static int em_rxd = EM_DEFAULT_RXD; 338 static int em_txd = EM_DEFAULT_TXD; 339 TUNABLE_INT("hw.em.rxd", &em_rxd); 340 TUNABLE_INT("hw.em.txd", &em_txd); 341 342 static int em_smart_pwr_down = FALSE; 343 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 344 345 /* Controls whether promiscuous also shows bad packets */ 346 static int em_debug_sbp = FALSE; 347 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 348 349 /* Local controls for MSI/MSIX */ 350 static int em_enable_msix = TRUE; 351 static int em_msix_queues = 2; /* for 82574, can be 1 or 2 */ 352 TUNABLE_INT("hw.em.enable_msix", &em_enable_msix); 353 TUNABLE_INT("hw.em.msix_queues", &em_msix_queues); 354 355 /* How many packets rxeof tries to clean at a time */ 356 static int em_rx_process_limit = 100; 357 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 358 359 /* Flow control setting - default to FULL */ 360 static int em_fc_setting = e1000_fc_full; 361 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); 362 363 /* 364 ** Shadow VFTA table, this is needed because 365 ** the real vlan filter table gets cleared during 366 ** a soft reset and the driver needs to be able 367 ** to repopulate it. 368 */ 369 static u32 em_shadow_vfta[EM_VFTA_SIZE]; 370 371 /* Global used in WOL setup with multiport cards */ 372 static int global_quad_port_a = 0; 373 374 /********************************************************************* 375 * Device identification routine 376 * 377 * em_probe determines if the driver should be loaded on 378 * adapter based on PCI vendor/device id of the adapter. 379 * 380 * return BUS_PROBE_DEFAULT on success, positive on failure 381 *********************************************************************/ 382 383 static int 384 em_probe(device_t dev) 385 { 386 char adapter_name[60]; 387 u16 pci_vendor_id = 0; 388 u16 pci_device_id = 0; 389 u16 pci_subvendor_id = 0; 390 u16 pci_subdevice_id = 0; 391 em_vendor_info_t *ent; 392 393 INIT_DEBUGOUT("em_probe: begin"); 394 395 pci_vendor_id = pci_get_vendor(dev); 396 if (pci_vendor_id != EM_VENDOR_ID) 397 return (ENXIO); 398 399 pci_device_id = pci_get_device(dev); 400 pci_subvendor_id = pci_get_subvendor(dev); 401 pci_subdevice_id = pci_get_subdevice(dev); 402 403 ent = em_vendor_info_array; 404 while (ent->vendor_id != 0) { 405 if ((pci_vendor_id == ent->vendor_id) && 406 (pci_device_id == ent->device_id) && 407 408 ((pci_subvendor_id == ent->subvendor_id) || 409 (ent->subvendor_id == PCI_ANY_ID)) && 410 411 ((pci_subdevice_id == ent->subdevice_id) || 412 (ent->subdevice_id == PCI_ANY_ID))) { 413 sprintf(adapter_name, "%s %s", 414 em_strings[ent->index], 415 em_driver_version); 416 device_set_desc_copy(dev, adapter_name); 417 return (BUS_PROBE_DEFAULT); 418 } 419 ent++; 420 } 421 422 return (ENXIO); 423 } 424 425 /********************************************************************* 426 * Device initialization routine 427 * 428 * The attach entry point is called when the driver is being loaded. 429 * This routine identifies the type of hardware, allocates all resources 430 * and initializes the hardware. 431 * 432 * return 0 on success, positive on failure 433 *********************************************************************/ 434 435 static int 436 em_attach(device_t dev) 437 { 438 struct adapter *adapter; 439 int error = 0; 440 441 INIT_DEBUGOUT("em_attach: begin"); 442 443 adapter = device_get_softc(dev); 444 adapter->dev = adapter->osdep.dev = dev; 445 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 446 447 /* SYSCTL stuff */ 448 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 449 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 450 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 451 em_sysctl_debug_info, "I", "Debug Information"); 452 453 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 454 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 455 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 456 em_sysctl_stats, "I", "Statistics"); 457 458 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 459 460 /* Determine hardware and mac info */ 461 em_identify_hardware(adapter); 462 463 /* Setup PCI resources */ 464 if (em_allocate_pci_resources(adapter)) { 465 device_printf(dev, "Allocation of PCI resources failed\n"); 466 error = ENXIO; 467 goto err_pci; 468 } 469 470 /* 471 ** For ICH8 and family we need to 472 ** map the flash memory, and this 473 ** must happen after the MAC is 474 ** identified 475 */ 476 if ((adapter->hw.mac.type == e1000_ich8lan) || 477 (adapter->hw.mac.type == e1000_pchlan) || 478 (adapter->hw.mac.type == e1000_ich9lan) || 479 (adapter->hw.mac.type == e1000_ich10lan)) { 480 int rid = EM_BAR_TYPE_FLASH; 481 adapter->flash = bus_alloc_resource_any(dev, 482 SYS_RES_MEMORY, &rid, RF_ACTIVE); 483 if (adapter->flash == NULL) { 484 device_printf(dev, "Mapping of Flash failed\n"); 485 error = ENXIO; 486 goto err_pci; 487 } 488 /* This is used in the shared code */ 489 adapter->hw.flash_address = (u8 *)adapter->flash; 490 adapter->osdep.flash_bus_space_tag = 491 rman_get_bustag(adapter->flash); 492 adapter->osdep.flash_bus_space_handle = 493 rman_get_bushandle(adapter->flash); 494 } 495 496 /* Do Shared Code initialization */ 497 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 498 device_printf(dev, "Setup of Shared code failed\n"); 499 error = ENXIO; 500 goto err_pci; 501 } 502 503 e1000_get_bus_info(&adapter->hw); 504 505 /* Set up some sysctls for the tunable interrupt delays */ 506 em_add_int_delay_sysctl(adapter, "rx_int_delay", 507 "receive interrupt delay in usecs", &adapter->rx_int_delay, 508 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 509 em_add_int_delay_sysctl(adapter, "tx_int_delay", 510 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 511 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 512 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 513 "receive interrupt delay limit in usecs", 514 &adapter->rx_abs_int_delay, 515 E1000_REGISTER(&adapter->hw, E1000_RADV), 516 em_rx_abs_int_delay_dflt); 517 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 518 "transmit interrupt delay limit in usecs", 519 &adapter->tx_abs_int_delay, 520 E1000_REGISTER(&adapter->hw, E1000_TADV), 521 em_tx_abs_int_delay_dflt); 522 523 /* Sysctls for limiting the amount of work done in the taskqueue */ 524 em_add_rx_process_limit(adapter, "rx_processing_limit", 525 "max number of rx packets to process", &adapter->rx_process_limit, 526 em_rx_process_limit); 527 528 /* 529 * Validate number of transmit and receive descriptors. It 530 * must not exceed hardware maximum, and must be multiple 531 * of E1000_DBA_ALIGN. 532 */ 533 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 534 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { 535 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 536 EM_DEFAULT_TXD, em_txd); 537 adapter->num_tx_desc = EM_DEFAULT_TXD; 538 } else 539 adapter->num_tx_desc = em_txd; 540 541 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 542 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { 543 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 544 EM_DEFAULT_RXD, em_rxd); 545 adapter->num_rx_desc = EM_DEFAULT_RXD; 546 } else 547 adapter->num_rx_desc = em_rxd; 548 549 adapter->hw.mac.autoneg = DO_AUTO_NEG; 550 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 551 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 552 553 /* Copper options */ 554 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 555 adapter->hw.phy.mdix = AUTO_ALL_MODES; 556 adapter->hw.phy.disable_polarity_correction = FALSE; 557 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 558 } 559 560 /* 561 * Set the frame limits assuming 562 * standard ethernet sized frames. 563 */ 564 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 565 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 566 567 /* 568 * This controls when hardware reports transmit completion 569 * status. 570 */ 571 adapter->hw.mac.report_tx_early = 1; 572 573 /* 574 ** Get queue/ring memory 575 */ 576 if (em_allocate_queues(adapter)) { 577 error = ENOMEM; 578 goto err_pci; 579 } 580 581 /* 582 ** Start from a known state, this is 583 ** important in reading the nvm and 584 ** mac from that. 585 */ 586 e1000_reset_hw(&adapter->hw); 587 588 /* Make sure we have a good EEPROM before we read from it */ 589 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 590 /* 591 ** Some PCI-E parts fail the first check due to 592 ** the link being in sleep state, call it again, 593 ** if it fails a second time its a real issue. 594 */ 595 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 596 device_printf(dev, 597 "The EEPROM Checksum Is Not Valid\n"); 598 error = EIO; 599 goto err_late; 600 } 601 } 602 603 /* Copy the permanent MAC address out of the EEPROM */ 604 if (e1000_read_mac_addr(&adapter->hw) < 0) { 605 device_printf(dev, "EEPROM read error while reading MAC" 606 " address\n"); 607 error = EIO; 608 goto err_late; 609 } 610 611 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 612 device_printf(dev, "Invalid MAC address\n"); 613 error = EIO; 614 goto err_late; 615 } 616 617 /* 618 ** Do interrupt configuration 619 */ 620 if (adapter->msix > 1) /* Do MSIX */ 621 error = em_allocate_msix(adapter); 622 else /* MSI or Legacy */ 623 error = em_allocate_legacy(adapter); 624 if (error) 625 goto err_late; 626 627 /* 628 * Get Wake-on-Lan and Management info for later use 629 */ 630 em_get_wakeup(dev); 631 632 /* Setup OS specific network interface */ 633 em_setup_interface(dev, adapter); 634 635 em_reset(adapter); 636 637 /* Initialize statistics */ 638 em_update_stats_counters(adapter); 639 640 adapter->hw.mac.get_link_status = 1; 641 em_update_link_status(adapter); 642 643 /* Indicate SOL/IDER usage */ 644 if (e1000_check_reset_block(&adapter->hw)) 645 device_printf(dev, 646 "PHY reset is blocked due to SOL/IDER session.\n"); 647 648 /* Register for VLAN events */ 649 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 650 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 651 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 652 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 653 654 /* Non-AMT based hardware can now take control from firmware */ 655 if (adapter->has_manage && !adapter->has_amt) 656 em_get_hw_control(adapter); 657 658 /* Tell the stack that the interface is not active */ 659 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 660 661 adapter->led_dev = led_create(em_led_func, adapter, 662 device_get_nameunit(dev)); 663 664 INIT_DEBUGOUT("em_attach: end"); 665 666 return (0); 667 668 err_late: 669 em_free_transmit_structures(adapter); 670 em_free_receive_structures(adapter); 671 em_release_hw_control(adapter); 672 err_pci: 673 em_free_pci_resources(adapter); 674 EM_CORE_LOCK_DESTROY(adapter); 675 676 return (error); 677 } 678 679 /********************************************************************* 680 * Device removal routine 681 * 682 * The detach entry point is called when the driver is being removed. 683 * This routine stops the adapter and deallocates all the resources 684 * that were allocated for driver operation. 685 * 686 * return 0 on success, positive on failure 687 *********************************************************************/ 688 689 static int 690 em_detach(device_t dev) 691 { 692 struct adapter *adapter = device_get_softc(dev); 693 struct ifnet *ifp = adapter->ifp; 694 695 INIT_DEBUGOUT("em_detach: begin"); 696 697 /* Make sure VLANS are not using driver */ 698 if (adapter->ifp->if_vlantrunk != NULL) { 699 device_printf(dev,"Vlan in use, detach first\n"); 700 return (EBUSY); 701 } 702 703 #ifdef DEVICE_POLLING 704 if (ifp->if_capenable & IFCAP_POLLING) 705 ether_poll_deregister(ifp); 706 #endif 707 708 EM_CORE_LOCK(adapter); 709 adapter->in_detach = 1; 710 em_stop(adapter); 711 EM_CORE_UNLOCK(adapter); 712 EM_CORE_LOCK_DESTROY(adapter); 713 714 e1000_phy_hw_reset(&adapter->hw); 715 716 em_release_manageability(adapter); 717 em_release_hw_control(adapter); 718 719 /* Unregister VLAN events */ 720 if (adapter->vlan_attach != NULL) 721 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 722 if (adapter->vlan_detach != NULL) 723 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 724 725 ether_ifdetach(adapter->ifp); 726 callout_drain(&adapter->timer); 727 728 em_free_pci_resources(adapter); 729 bus_generic_detach(dev); 730 if_free(ifp); 731 732 em_free_transmit_structures(adapter); 733 em_free_receive_structures(adapter); 734 735 em_release_hw_control(adapter); 736 737 return (0); 738 } 739 740 /********************************************************************* 741 * 742 * Shutdown entry point 743 * 744 **********************************************************************/ 745 746 static int 747 em_shutdown(device_t dev) 748 { 749 return em_suspend(dev); 750 } 751 752 /* 753 * Suspend/resume device methods. 754 */ 755 static int 756 em_suspend(device_t dev) 757 { 758 struct adapter *adapter = device_get_softc(dev); 759 760 EM_CORE_LOCK(adapter); 761 762 em_release_manageability(adapter); 763 em_release_hw_control(adapter); 764 em_enable_wakeup(dev); 765 766 EM_CORE_UNLOCK(adapter); 767 768 return bus_generic_suspend(dev); 769 } 770 771 static int 772 em_resume(device_t dev) 773 { 774 struct adapter *adapter = device_get_softc(dev); 775 struct ifnet *ifp = adapter->ifp; 776 777 if (adapter->led_dev != NULL) 778 led_destroy(adapter->led_dev); 779 780 EM_CORE_LOCK(adapter); 781 em_init_locked(adapter); 782 em_init_manageability(adapter); 783 EM_CORE_UNLOCK(adapter); 784 em_start(ifp); 785 786 return bus_generic_resume(dev); 787 } 788 789 790 /********************************************************************* 791 * Transmit entry point 792 * 793 * em_start is called by the stack to initiate a transmit. 794 * The driver will remain in this routine as long as there are 795 * packets to transmit and transmit resources are available. 796 * In case resources are not available stack is notified and 797 * the packet is requeued. 798 **********************************************************************/ 799 800 #if __FreeBSD_version >= 800000 801 static int 802 em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 803 { 804 struct adapter *adapter = txr->adapter; 805 struct mbuf *next; 806 int err = 0, enq = 0; 807 808 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 809 IFF_DRV_RUNNING || adapter->link_active == 0) { 810 if (m != NULL) 811 err = drbr_enqueue(ifp, txr->br, m); 812 return (err); 813 } 814 815 enq = 0; 816 if (m == NULL) 817 next = drbr_dequeue(ifp, txr->br); 818 else 819 next = m; 820 821 /* Process the queue */ 822 while (next != NULL) { 823 if ((err = em_xmit(txr, &next)) != 0) { 824 if (next != NULL) 825 err = drbr_enqueue(ifp, txr->br, next); 826 break; 827 } 828 enq++; 829 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 830 ETHER_BPF_MTAP(ifp, next); 831 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 832 break; 833 next = drbr_dequeue(ifp, txr->br); 834 } 835 836 if (enq > 0) { 837 /* Set the watchdog */ 838 txr->watchdog_check = TRUE; 839 } 840 return (err); 841 } 842 843 /* 844 ** Multiqueue capable stack interface, this is not 845 ** yet truely multiqueue, but that is coming... 846 */ 847 static int 848 em_mq_start(struct ifnet *ifp, struct mbuf *m) 849 { 850 struct adapter *adapter = ifp->if_softc; 851 struct tx_ring *txr; 852 int i, error = 0; 853 854 /* Which queue to use */ 855 if ((m->m_flags & M_FLOWID) != 0) 856 i = m->m_pkthdr.flowid % adapter->num_queues; 857 else 858 i = curcpu % adapter->num_queues; 859 860 txr = &adapter->tx_rings[i]; 861 862 if (EM_TX_TRYLOCK(txr)) { 863 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 864 error = em_mq_start_locked(ifp, txr, m); 865 EM_TX_UNLOCK(txr); 866 } else 867 error = drbr_enqueue(ifp, txr->br, m); 868 869 return (error); 870 } 871 872 /* 873 ** Flush all ring buffers 874 */ 875 static void 876 em_qflush(struct ifnet *ifp) 877 { 878 struct adapter *adapter = ifp->if_softc; 879 struct tx_ring *txr = adapter->tx_rings; 880 struct mbuf *m; 881 882 for (int i = 0; i < adapter->num_queues; i++, txr++) { 883 EM_TX_LOCK(txr); 884 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 885 m_freem(m); 886 EM_TX_UNLOCK(txr); 887 } 888 if_qflush(ifp); 889 } 890 891 #endif /* FreeBSD_version */ 892 893 static void 894 em_start_locked(struct ifnet *ifp, struct tx_ring *txr) 895 { 896 struct adapter *adapter = ifp->if_softc; 897 struct mbuf *m_head; 898 899 EM_TX_LOCK_ASSERT(txr); 900 901 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 902 IFF_DRV_RUNNING) 903 return; 904 905 if (!adapter->link_active) 906 return; 907 908 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 909 910 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 911 if (m_head == NULL) 912 break; 913 /* 914 * Encapsulation can modify our pointer, and or make it 915 * NULL on failure. In that event, we can't requeue. 916 */ 917 if (em_xmit(txr, &m_head)) { 918 if (m_head == NULL) 919 break; 920 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 921 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 922 break; 923 } 924 925 /* Send a copy of the frame to the BPF listener */ 926 ETHER_BPF_MTAP(ifp, m_head); 927 928 /* Set timeout in case hardware has problems transmitting. */ 929 txr->watchdog_check = TRUE; 930 } 931 932 return; 933 } 934 935 static void 936 em_start(struct ifnet *ifp) 937 { 938 struct adapter *adapter = ifp->if_softc; 939 struct tx_ring *txr = adapter->tx_rings; 940 941 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 942 EM_TX_LOCK(txr); 943 em_start_locked(ifp, txr); 944 EM_TX_UNLOCK(txr); 945 } 946 return; 947 } 948 949 /********************************************************************* 950 * Ioctl entry point 951 * 952 * em_ioctl is called when the user wants to configure the 953 * interface. 954 * 955 * return 0 on success, positive on failure 956 **********************************************************************/ 957 958 static int 959 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 960 { 961 struct adapter *adapter = ifp->if_softc; 962 struct ifreq *ifr = (struct ifreq *)data; 963 #ifdef INET 964 struct ifaddr *ifa = (struct ifaddr *)data; 965 #endif 966 int error = 0; 967 968 if (adapter->in_detach) 969 return (error); 970 971 switch (command) { 972 case SIOCSIFADDR: 973 #ifdef INET 974 if (ifa->ifa_addr->sa_family == AF_INET) { 975 /* 976 * XXX 977 * Since resetting hardware takes a very long time 978 * and results in link renegotiation we only 979 * initialize the hardware only when it is absolutely 980 * required. 981 */ 982 ifp->if_flags |= IFF_UP; 983 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 984 EM_CORE_LOCK(adapter); 985 em_init_locked(adapter); 986 EM_CORE_UNLOCK(adapter); 987 } 988 arp_ifinit(ifp, ifa); 989 } else 990 #endif 991 error = ether_ioctl(ifp, command, data); 992 break; 993 case SIOCSIFMTU: 994 { 995 int max_frame_size; 996 997 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 998 999 EM_CORE_LOCK(adapter); 1000 switch (adapter->hw.mac.type) { 1001 case e1000_82571: 1002 case e1000_82572: 1003 case e1000_ich9lan: 1004 case e1000_ich10lan: 1005 case e1000_82574: 1006 case e1000_80003es2lan: /* 9K Jumbo Frame size */ 1007 max_frame_size = 9234; 1008 break; 1009 case e1000_pchlan: 1010 max_frame_size = 4096; 1011 break; 1012 /* Adapters that do not support jumbo frames */ 1013 case e1000_82583: 1014 case e1000_ich8lan: 1015 max_frame_size = ETHER_MAX_LEN; 1016 break; 1017 default: 1018 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1019 } 1020 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1021 ETHER_CRC_LEN) { 1022 EM_CORE_UNLOCK(adapter); 1023 error = EINVAL; 1024 break; 1025 } 1026 1027 ifp->if_mtu = ifr->ifr_mtu; 1028 adapter->max_frame_size = 1029 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1030 em_init_locked(adapter); 1031 EM_CORE_UNLOCK(adapter); 1032 break; 1033 } 1034 case SIOCSIFFLAGS: 1035 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1036 SIOCSIFFLAGS (Set Interface Flags)"); 1037 EM_CORE_LOCK(adapter); 1038 if (ifp->if_flags & IFF_UP) { 1039 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1040 if ((ifp->if_flags ^ adapter->if_flags) & 1041 (IFF_PROMISC | IFF_ALLMULTI)) { 1042 em_disable_promisc(adapter); 1043 em_set_promisc(adapter); 1044 } 1045 } else 1046 em_init_locked(adapter); 1047 } else 1048 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1049 em_stop(adapter); 1050 adapter->if_flags = ifp->if_flags; 1051 EM_CORE_UNLOCK(adapter); 1052 break; 1053 case SIOCADDMULTI: 1054 case SIOCDELMULTI: 1055 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1056 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1057 EM_CORE_LOCK(adapter); 1058 em_disable_intr(adapter); 1059 em_set_multi(adapter); 1060 #ifdef DEVICE_POLLING 1061 if (!(ifp->if_capenable & IFCAP_POLLING)) 1062 #endif 1063 em_enable_intr(adapter); 1064 EM_CORE_UNLOCK(adapter); 1065 } 1066 break; 1067 case SIOCSIFMEDIA: 1068 /* Check SOL/IDER usage */ 1069 EM_CORE_LOCK(adapter); 1070 if (e1000_check_reset_block(&adapter->hw)) { 1071 EM_CORE_UNLOCK(adapter); 1072 device_printf(adapter->dev, "Media change is" 1073 " blocked due to SOL/IDER session.\n"); 1074 break; 1075 } 1076 EM_CORE_UNLOCK(adapter); 1077 case SIOCGIFMEDIA: 1078 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1079 SIOCxIFMEDIA (Get/Set Interface Media)"); 1080 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1081 break; 1082 case SIOCSIFCAP: 1083 { 1084 int mask, reinit; 1085 1086 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1087 reinit = 0; 1088 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1089 #ifdef DEVICE_POLLING 1090 if (mask & IFCAP_POLLING) { 1091 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1092 error = ether_poll_register(em_poll, ifp); 1093 if (error) 1094 return (error); 1095 EM_CORE_LOCK(adapter); 1096 em_disable_intr(adapter); 1097 ifp->if_capenable |= IFCAP_POLLING; 1098 EM_CORE_UNLOCK(adapter); 1099 } else { 1100 error = ether_poll_deregister(ifp); 1101 /* Enable interrupt even in error case */ 1102 EM_CORE_LOCK(adapter); 1103 em_enable_intr(adapter); 1104 ifp->if_capenable &= ~IFCAP_POLLING; 1105 EM_CORE_UNLOCK(adapter); 1106 } 1107 } 1108 #endif 1109 if (mask & IFCAP_HWCSUM) { 1110 ifp->if_capenable ^= IFCAP_HWCSUM; 1111 reinit = 1; 1112 } 1113 if (mask & IFCAP_TSO4) { 1114 ifp->if_capenable ^= IFCAP_TSO4; 1115 reinit = 1; 1116 } 1117 if (mask & IFCAP_VLAN_HWTAGGING) { 1118 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1119 reinit = 1; 1120 } 1121 if ((mask & IFCAP_WOL) && 1122 (ifp->if_capabilities & IFCAP_WOL) != 0) { 1123 if (mask & IFCAP_WOL_MCAST) 1124 ifp->if_capenable ^= IFCAP_WOL_MCAST; 1125 if (mask & IFCAP_WOL_MAGIC) 1126 ifp->if_capenable ^= IFCAP_WOL_MAGIC; 1127 } 1128 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1129 em_init(adapter); 1130 VLAN_CAPABILITIES(ifp); 1131 break; 1132 } 1133 1134 default: 1135 error = ether_ioctl(ifp, command, data); 1136 break; 1137 } 1138 1139 return (error); 1140 } 1141 1142 1143 /********************************************************************* 1144 * Init entry point 1145 * 1146 * This routine is used in two ways. It is used by the stack as 1147 * init entry point in network interface structure. It is also used 1148 * by the driver as a hw/sw initialization routine to get to a 1149 * consistent state. 1150 * 1151 * return 0 on success, positive on failure 1152 **********************************************************************/ 1153 1154 static void 1155 em_init_locked(struct adapter *adapter) 1156 { 1157 struct ifnet *ifp = adapter->ifp; 1158 device_t dev = adapter->dev; 1159 u32 pba; 1160 1161 INIT_DEBUGOUT("em_init: begin"); 1162 1163 EM_CORE_LOCK_ASSERT(adapter); 1164 1165 em_disable_intr(adapter); 1166 callout_stop(&adapter->timer); 1167 1168 /* 1169 * Packet Buffer Allocation (PBA) 1170 * Writing PBA sets the receive portion of the buffer 1171 * the remainder is used for the transmit buffer. 1172 */ 1173 switch (adapter->hw.mac.type) { 1174 /* Total Packet Buffer on these is 48K */ 1175 case e1000_82571: 1176 case e1000_82572: 1177 case e1000_80003es2lan: 1178 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1179 break; 1180 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1181 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1182 break; 1183 case e1000_82574: 1184 case e1000_82583: 1185 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1186 break; 1187 case e1000_ich9lan: 1188 case e1000_ich10lan: 1189 case e1000_pchlan: 1190 pba = E1000_PBA_10K; 1191 break; 1192 case e1000_ich8lan: 1193 pba = E1000_PBA_8K; 1194 break; 1195 default: 1196 if (adapter->max_frame_size > 8192) 1197 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1198 else 1199 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1200 } 1201 1202 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1203 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1204 1205 /* Get the latest mac address, User can use a LAA */ 1206 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1207 ETHER_ADDR_LEN); 1208 1209 /* Put the address into the Receive Address Array */ 1210 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1211 1212 /* 1213 * With the 82571 adapter, RAR[0] may be overwritten 1214 * when the other port is reset, we make a duplicate 1215 * in RAR[14] for that eventuality, this assures 1216 * the interface continues to function. 1217 */ 1218 if (adapter->hw.mac.type == e1000_82571) { 1219 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1220 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1221 E1000_RAR_ENTRIES - 1); 1222 } 1223 1224 /* Initialize the hardware */ 1225 em_reset(adapter); 1226 em_update_link_status(adapter); 1227 1228 /* Setup VLAN support, basic and offload if available */ 1229 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1230 1231 /* Use real VLAN Filter support */ 1232 em_setup_vlan_hw_support(adapter); 1233 1234 /* Set hardware offload abilities */ 1235 ifp->if_hwassist = 0; 1236 if (ifp->if_capenable & IFCAP_TXCSUM) 1237 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1238 if (ifp->if_capenable & IFCAP_TSO4) 1239 ifp->if_hwassist |= CSUM_TSO; 1240 1241 /* Configure for OS presence */ 1242 em_init_manageability(adapter); 1243 1244 /* Prepare transmit descriptors and buffers */ 1245 em_setup_transmit_structures(adapter); 1246 em_initialize_transmit_unit(adapter); 1247 1248 /* Setup Multicast table */ 1249 em_set_multi(adapter); 1250 1251 /* Prepare receive descriptors and buffers */ 1252 if (em_setup_receive_structures(adapter)) { 1253 device_printf(dev, "Could not setup receive structures\n"); 1254 em_stop(adapter); 1255 return; 1256 } 1257 em_initialize_receive_unit(adapter); 1258 1259 /* Don't lose promiscuous settings */ 1260 em_set_promisc(adapter); 1261 1262 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1263 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1264 1265 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1266 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1267 1268 /* MSI/X configuration for 82574 */ 1269 if (adapter->hw.mac.type == e1000_82574) { 1270 int tmp; 1271 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1272 tmp |= E1000_CTRL_EXT_PBA_CLR; 1273 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1274 /* Set the IVAR - interrupt vector routing. */ 1275 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); 1276 } 1277 1278 #ifdef DEVICE_POLLING 1279 /* 1280 * Only enable interrupts if we are not polling, make sure 1281 * they are off otherwise. 1282 */ 1283 if (ifp->if_capenable & IFCAP_POLLING) 1284 em_disable_intr(adapter); 1285 else 1286 #endif /* DEVICE_POLLING */ 1287 em_enable_intr(adapter); 1288 1289 /* AMT based hardware can now take control from firmware */ 1290 if (adapter->has_manage && adapter->has_amt) 1291 em_get_hw_control(adapter); 1292 1293 /* Don't reset the phy next time init gets called */ 1294 adapter->hw.phy.reset_disable = TRUE; 1295 } 1296 1297 static void 1298 em_init(void *arg) 1299 { 1300 struct adapter *adapter = arg; 1301 1302 EM_CORE_LOCK(adapter); 1303 em_init_locked(adapter); 1304 EM_CORE_UNLOCK(adapter); 1305 } 1306 1307 1308 #ifdef DEVICE_POLLING 1309 /********************************************************************* 1310 * 1311 * Legacy polling routine: note this only works with single queue 1312 * 1313 *********************************************************************/ 1314 static int 1315 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1316 { 1317 struct adapter *adapter = ifp->if_softc; 1318 struct tx_ring *txr = adapter->tx_rings; 1319 struct rx_ring *rxr = adapter->rx_rings; 1320 u32 reg_icr, rx_done = 0; 1321 1322 EM_CORE_LOCK(adapter); 1323 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1324 EM_CORE_UNLOCK(adapter); 1325 return (rx_done); 1326 } 1327 1328 if (cmd == POLL_AND_CHECK_STATUS) { 1329 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1330 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1331 callout_stop(&adapter->timer); 1332 adapter->hw.mac.get_link_status = 1; 1333 em_update_link_status(adapter); 1334 callout_reset(&adapter->timer, hz, 1335 em_local_timer, adapter); 1336 } 1337 } 1338 EM_CORE_UNLOCK(adapter); 1339 1340 rx_done = em_rxeof(rxr, count); 1341 1342 EM_TX_LOCK(txr); 1343 em_txeof(txr); 1344 #if __FreeBSD_version >= 800000 1345 if (!drbr_empty(ifp, txr->br)) 1346 em_mq_start_locked(ifp, txr, NULL); 1347 #else 1348 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1349 em_start_locked(ifp, txr); 1350 #endif 1351 EM_TX_UNLOCK(txr); 1352 1353 return (rx_done); 1354 } 1355 #endif /* DEVICE_POLLING */ 1356 1357 1358 /********************************************************************* 1359 * 1360 * Fast Legacy/MSI Combined Interrupt Service routine 1361 * 1362 *********************************************************************/ 1363 static int 1364 em_irq_fast(void *arg) 1365 { 1366 struct adapter *adapter = arg; 1367 struct ifnet *ifp; 1368 u32 reg_icr; 1369 1370 ifp = adapter->ifp; 1371 1372 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1373 1374 /* Hot eject? */ 1375 if (reg_icr == 0xffffffff) 1376 return FILTER_STRAY; 1377 1378 /* Definitely not our interrupt. */ 1379 if (reg_icr == 0x0) 1380 return FILTER_STRAY; 1381 1382 /* 1383 * Starting with the 82571 chip, bit 31 should be used to 1384 * determine whether the interrupt belongs to us. 1385 */ 1386 if (adapter->hw.mac.type >= e1000_82571 && 1387 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1388 return FILTER_STRAY; 1389 1390 em_disable_intr(adapter); 1391 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1392 1393 /* Link status change */ 1394 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1395 adapter->hw.mac.get_link_status = 1; 1396 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1397 } 1398 1399 if (reg_icr & E1000_ICR_RXO) 1400 adapter->rx_overruns++; 1401 return FILTER_HANDLED; 1402 } 1403 1404 /* Combined RX/TX handler, used by Legacy and MSI */ 1405 static void 1406 em_handle_que(void *context, int pending) 1407 { 1408 struct adapter *adapter = context; 1409 struct ifnet *ifp = adapter->ifp; 1410 struct tx_ring *txr = adapter->tx_rings; 1411 struct rx_ring *rxr = adapter->rx_rings; 1412 u32 loop = EM_MAX_LOOP; 1413 bool more_rx, more_tx; 1414 1415 1416 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1417 EM_TX_LOCK(txr); 1418 do { 1419 more_rx = em_rxeof(rxr, adapter->rx_process_limit); 1420 more_tx = em_txeof(txr); 1421 } while (loop-- && (more_rx || more_tx)); 1422 1423 #if __FreeBSD_version >= 800000 1424 if (!drbr_empty(ifp, txr->br)) 1425 em_mq_start_locked(ifp, txr, NULL); 1426 #else 1427 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1428 em_start_locked(ifp, txr); 1429 #endif 1430 if (more_rx || more_tx) 1431 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1432 1433 EM_TX_UNLOCK(txr); 1434 } 1435 1436 em_enable_intr(adapter); 1437 return; 1438 } 1439 1440 1441 /********************************************************************* 1442 * 1443 * MSIX Interrupt Service Routines 1444 * 1445 **********************************************************************/ 1446 static void 1447 em_msix_tx(void *arg) 1448 { 1449 struct tx_ring *txr = arg; 1450 struct adapter *adapter = txr->adapter; 1451 bool more; 1452 1453 ++txr->tx_irq; 1454 EM_TX_LOCK(txr); 1455 more = em_txeof(txr); 1456 EM_TX_UNLOCK(txr); 1457 if (more) 1458 taskqueue_enqueue(txr->tq, &txr->tx_task); 1459 else 1460 /* Reenable this interrupt */ 1461 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1462 return; 1463 } 1464 1465 /********************************************************************* 1466 * 1467 * MSIX RX Interrupt Service routine 1468 * 1469 **********************************************************************/ 1470 1471 static void 1472 em_msix_rx(void *arg) 1473 { 1474 struct rx_ring *rxr = arg; 1475 struct adapter *adapter = rxr->adapter; 1476 bool more; 1477 1478 ++rxr->rx_irq; 1479 more = em_rxeof(rxr, adapter->rx_process_limit); 1480 if (more) 1481 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1482 else 1483 /* Reenable this interrupt */ 1484 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1485 return; 1486 } 1487 1488 /********************************************************************* 1489 * 1490 * MSIX Link Fast Interrupt Service routine 1491 * 1492 **********************************************************************/ 1493 static void 1494 em_msix_link(void *arg) 1495 { 1496 struct adapter *adapter = arg; 1497 u32 reg_icr; 1498 1499 ++adapter->link_irq; 1500 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1501 1502 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1503 adapter->hw.mac.get_link_status = 1; 1504 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1505 } else 1506 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1507 EM_MSIX_LINK | E1000_IMS_LSC); 1508 return; 1509 } 1510 1511 static void 1512 em_handle_rx(void *context, int pending) 1513 { 1514 struct rx_ring *rxr = context; 1515 struct adapter *adapter = rxr->adapter; 1516 u32 loop = EM_MAX_LOOP; 1517 bool more; 1518 1519 do { 1520 more = em_rxeof(rxr, adapter->rx_process_limit); 1521 } while (loop-- && more); 1522 /* Reenable this interrupt */ 1523 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1524 } 1525 1526 static void 1527 em_handle_tx(void *context, int pending) 1528 { 1529 struct tx_ring *txr = context; 1530 struct adapter *adapter = txr->adapter; 1531 struct ifnet *ifp = adapter->ifp; 1532 u32 loop = EM_MAX_LOOP; 1533 bool more; 1534 1535 if (!EM_TX_TRYLOCK(txr)) 1536 return; 1537 do { 1538 more = em_txeof(txr); 1539 } while (loop-- && more); 1540 1541 #if __FreeBSD_version >= 800000 1542 if (!drbr_empty(ifp, txr->br)) 1543 em_mq_start_locked(ifp, txr, NULL); 1544 #else 1545 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1546 em_start_locked(ifp, txr); 1547 #endif 1548 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1549 EM_TX_UNLOCK(txr); 1550 } 1551 1552 static void 1553 em_handle_link(void *context, int pending) 1554 { 1555 struct adapter *adapter = context; 1556 struct ifnet *ifp = adapter->ifp; 1557 1558 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1559 return; 1560 1561 EM_CORE_LOCK(adapter); 1562 callout_stop(&adapter->timer); 1563 em_update_link_status(adapter); 1564 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1565 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1566 EM_MSIX_LINK | E1000_IMS_LSC); 1567 EM_CORE_UNLOCK(adapter); 1568 } 1569 1570 1571 /********************************************************************* 1572 * 1573 * Media Ioctl callback 1574 * 1575 * This routine is called whenever the user queries the status of 1576 * the interface using ifconfig. 1577 * 1578 **********************************************************************/ 1579 static void 1580 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1581 { 1582 struct adapter *adapter = ifp->if_softc; 1583 u_char fiber_type = IFM_1000_SX; 1584 1585 INIT_DEBUGOUT("em_media_status: begin"); 1586 1587 EM_CORE_LOCK(adapter); 1588 em_update_link_status(adapter); 1589 1590 ifmr->ifm_status = IFM_AVALID; 1591 ifmr->ifm_active = IFM_ETHER; 1592 1593 if (!adapter->link_active) { 1594 EM_CORE_UNLOCK(adapter); 1595 return; 1596 } 1597 1598 ifmr->ifm_status |= IFM_ACTIVE; 1599 1600 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1601 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1602 ifmr->ifm_active |= fiber_type | IFM_FDX; 1603 } else { 1604 switch (adapter->link_speed) { 1605 case 10: 1606 ifmr->ifm_active |= IFM_10_T; 1607 break; 1608 case 100: 1609 ifmr->ifm_active |= IFM_100_TX; 1610 break; 1611 case 1000: 1612 ifmr->ifm_active |= IFM_1000_T; 1613 break; 1614 } 1615 if (adapter->link_duplex == FULL_DUPLEX) 1616 ifmr->ifm_active |= IFM_FDX; 1617 else 1618 ifmr->ifm_active |= IFM_HDX; 1619 } 1620 EM_CORE_UNLOCK(adapter); 1621 } 1622 1623 /********************************************************************* 1624 * 1625 * Media Ioctl callback 1626 * 1627 * This routine is called when the user changes speed/duplex using 1628 * media/mediopt option with ifconfig. 1629 * 1630 **********************************************************************/ 1631 static int 1632 em_media_change(struct ifnet *ifp) 1633 { 1634 struct adapter *adapter = ifp->if_softc; 1635 struct ifmedia *ifm = &adapter->media; 1636 1637 INIT_DEBUGOUT("em_media_change: begin"); 1638 1639 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1640 return (EINVAL); 1641 1642 EM_CORE_LOCK(adapter); 1643 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1644 case IFM_AUTO: 1645 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1646 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1647 break; 1648 case IFM_1000_LX: 1649 case IFM_1000_SX: 1650 case IFM_1000_T: 1651 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1652 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1653 break; 1654 case IFM_100_TX: 1655 adapter->hw.mac.autoneg = FALSE; 1656 adapter->hw.phy.autoneg_advertised = 0; 1657 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1658 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1659 else 1660 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1661 break; 1662 case IFM_10_T: 1663 adapter->hw.mac.autoneg = FALSE; 1664 adapter->hw.phy.autoneg_advertised = 0; 1665 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1666 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1667 else 1668 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1669 break; 1670 default: 1671 device_printf(adapter->dev, "Unsupported media type\n"); 1672 } 1673 1674 /* As the speed/duplex settings my have changed we need to 1675 * reset the PHY. 1676 */ 1677 adapter->hw.phy.reset_disable = FALSE; 1678 1679 em_init_locked(adapter); 1680 EM_CORE_UNLOCK(adapter); 1681 1682 return (0); 1683 } 1684 1685 /********************************************************************* 1686 * 1687 * This routine maps the mbufs to tx descriptors. 1688 * 1689 * return 0 on success, positive on failure 1690 **********************************************************************/ 1691 1692 static int 1693 em_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1694 { 1695 struct adapter *adapter = txr->adapter; 1696 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1697 bus_dmamap_t map; 1698 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1699 struct e1000_tx_desc *ctxd = NULL; 1700 struct mbuf *m_head; 1701 u32 txd_upper, txd_lower, txd_used, txd_saved; 1702 int nsegs, i, j, first, last = 0; 1703 int error, do_tso, tso_desc = 0; 1704 1705 m_head = *m_headp; 1706 txd_upper = txd_lower = txd_used = txd_saved = 0; 1707 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1708 1709 /* 1710 * Force a cleanup if number of TX descriptors 1711 * available hits the threshold 1712 */ 1713 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 1714 em_txeof(txr); 1715 1716 /* 1717 * TSO workaround: 1718 * If an mbuf is only header we need 1719 * to pull 4 bytes of data into it. 1720 */ 1721 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 1722 m_head = m_pullup(m_head, M_TSO_LEN + 4); 1723 *m_headp = m_head; 1724 if (m_head == NULL) 1725 return (ENOBUFS); 1726 } 1727 1728 /* 1729 * Map the packet for DMA 1730 * 1731 * Capture the first descriptor index, 1732 * this descriptor will have the index 1733 * of the EOP which is the only one that 1734 * now gets a DONE bit writeback. 1735 */ 1736 first = txr->next_avail_desc; 1737 tx_buffer = &txr->tx_buffers[first]; 1738 tx_buffer_mapped = tx_buffer; 1739 map = tx_buffer->map; 1740 1741 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1742 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1743 1744 /* 1745 * There are two types of errors we can (try) to handle: 1746 * - EFBIG means the mbuf chain was too long and bus_dma ran 1747 * out of segments. Defragment the mbuf chain and try again. 1748 * - ENOMEM means bus_dma could not obtain enough bounce buffers 1749 * at this point in time. Defer sending and try again later. 1750 * All other errors, in particular EINVAL, are fatal and prevent the 1751 * mbuf chain from ever going through. Drop it and report error. 1752 */ 1753 if (error == EFBIG) { 1754 struct mbuf *m; 1755 1756 m = m_defrag(*m_headp, M_DONTWAIT); 1757 if (m == NULL) { 1758 adapter->mbuf_alloc_failed++; 1759 m_freem(*m_headp); 1760 *m_headp = NULL; 1761 return (ENOBUFS); 1762 } 1763 *m_headp = m; 1764 1765 /* Try it again */ 1766 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1767 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1768 1769 if (error) { 1770 adapter->no_tx_dma_setup++; 1771 m_freem(*m_headp); 1772 *m_headp = NULL; 1773 return (error); 1774 } 1775 } else if (error != 0) { 1776 adapter->no_tx_dma_setup++; 1777 return (error); 1778 } 1779 1780 /* 1781 * TSO Hardware workaround, if this packet is not 1782 * TSO, and is only a single descriptor long, and 1783 * it follows a TSO burst, then we need to add a 1784 * sentinel descriptor to prevent premature writeback. 1785 */ 1786 if ((do_tso == 0) && (txr->tx_tso == TRUE)) { 1787 if (nsegs == 1) 1788 tso_desc = TRUE; 1789 txr->tx_tso = FALSE; 1790 } 1791 1792 if (nsegs > (txr->tx_avail - 2)) { 1793 txr->no_desc_avail++; 1794 bus_dmamap_unload(txr->txtag, map); 1795 return (ENOBUFS); 1796 } 1797 m_head = *m_headp; 1798 1799 /* Do hardware assists */ 1800 #if __FreeBSD_version >= 700000 1801 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1802 error = em_tso_setup(txr, m_head, &txd_upper, &txd_lower); 1803 if (error != TRUE) 1804 return (ENXIO); /* something foobar */ 1805 /* we need to make a final sentinel transmit desc */ 1806 tso_desc = TRUE; 1807 } else 1808 #endif 1809 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 1810 em_transmit_checksum_setup(txr, m_head, 1811 &txd_upper, &txd_lower); 1812 1813 i = txr->next_avail_desc; 1814 1815 /* Set up our transmit descriptors */ 1816 for (j = 0; j < nsegs; j++) { 1817 bus_size_t seg_len; 1818 bus_addr_t seg_addr; 1819 1820 tx_buffer = &txr->tx_buffers[i]; 1821 ctxd = &txr->tx_base[i]; 1822 seg_addr = segs[j].ds_addr; 1823 seg_len = segs[j].ds_len; 1824 /* 1825 ** TSO Workaround: 1826 ** If this is the last descriptor, we want to 1827 ** split it so we have a small final sentinel 1828 */ 1829 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 1830 seg_len -= 4; 1831 ctxd->buffer_addr = htole64(seg_addr); 1832 ctxd->lower.data = htole32( 1833 adapter->txd_cmd | txd_lower | seg_len); 1834 ctxd->upper.data = 1835 htole32(txd_upper); 1836 if (++i == adapter->num_tx_desc) 1837 i = 0; 1838 /* Now make the sentinel */ 1839 ++txd_used; /* using an extra txd */ 1840 ctxd = &txr->tx_base[i]; 1841 tx_buffer = &txr->tx_buffers[i]; 1842 ctxd->buffer_addr = 1843 htole64(seg_addr + seg_len); 1844 ctxd->lower.data = htole32( 1845 adapter->txd_cmd | txd_lower | 4); 1846 ctxd->upper.data = 1847 htole32(txd_upper); 1848 last = i; 1849 if (++i == adapter->num_tx_desc) 1850 i = 0; 1851 } else { 1852 ctxd->buffer_addr = htole64(seg_addr); 1853 ctxd->lower.data = htole32( 1854 adapter->txd_cmd | txd_lower | seg_len); 1855 ctxd->upper.data = 1856 htole32(txd_upper); 1857 last = i; 1858 if (++i == adapter->num_tx_desc) 1859 i = 0; 1860 } 1861 tx_buffer->m_head = NULL; 1862 tx_buffer->next_eop = -1; 1863 } 1864 1865 txr->next_avail_desc = i; 1866 txr->tx_avail -= nsegs; 1867 if (tso_desc) /* TSO used an extra for sentinel */ 1868 txr->tx_avail -= txd_used; 1869 1870 if (m_head->m_flags & M_VLANTAG) { 1871 /* Set the vlan id. */ 1872 ctxd->upper.fields.special = 1873 htole16(m_head->m_pkthdr.ether_vtag); 1874 /* Tell hardware to add tag */ 1875 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 1876 } 1877 1878 tx_buffer->m_head = m_head; 1879 tx_buffer_mapped->map = tx_buffer->map; 1880 tx_buffer->map = map; 1881 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1882 1883 /* 1884 * Last Descriptor of Packet 1885 * needs End Of Packet (EOP) 1886 * and Report Status (RS) 1887 */ 1888 ctxd->lower.data |= 1889 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1890 /* 1891 * Keep track in the first buffer which 1892 * descriptor will be written back 1893 */ 1894 tx_buffer = &txr->tx_buffers[first]; 1895 tx_buffer->next_eop = last; 1896 1897 /* 1898 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1899 * that this frame is available to transmit. 1900 */ 1901 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1902 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1903 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1904 1905 return (0); 1906 } 1907 1908 static void 1909 em_set_promisc(struct adapter *adapter) 1910 { 1911 struct ifnet *ifp = adapter->ifp; 1912 u32 reg_rctl; 1913 1914 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1915 1916 if (ifp->if_flags & IFF_PROMISC) { 1917 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1918 /* Turn this on if you want to see bad packets */ 1919 if (em_debug_sbp) 1920 reg_rctl |= E1000_RCTL_SBP; 1921 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1922 } else if (ifp->if_flags & IFF_ALLMULTI) { 1923 reg_rctl |= E1000_RCTL_MPE; 1924 reg_rctl &= ~E1000_RCTL_UPE; 1925 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1926 } 1927 } 1928 1929 static void 1930 em_disable_promisc(struct adapter *adapter) 1931 { 1932 u32 reg_rctl; 1933 1934 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1935 1936 reg_rctl &= (~E1000_RCTL_UPE); 1937 reg_rctl &= (~E1000_RCTL_MPE); 1938 reg_rctl &= (~E1000_RCTL_SBP); 1939 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1940 } 1941 1942 1943 /********************************************************************* 1944 * Multicast Update 1945 * 1946 * This routine is called whenever multicast address list is updated. 1947 * 1948 **********************************************************************/ 1949 1950 static void 1951 em_set_multi(struct adapter *adapter) 1952 { 1953 struct ifnet *ifp = adapter->ifp; 1954 struct ifmultiaddr *ifma; 1955 u32 reg_rctl = 0; 1956 u8 *mta; /* Multicast array memory */ 1957 int mcnt = 0; 1958 1959 IOCTL_DEBUGOUT("em_set_multi: begin"); 1960 1961 if (adapter->hw.mac.type == e1000_82542 && 1962 adapter->hw.revision_id == E1000_REVISION_2) { 1963 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1964 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 1965 e1000_pci_clear_mwi(&adapter->hw); 1966 reg_rctl |= E1000_RCTL_RST; 1967 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1968 msec_delay(5); 1969 } 1970 1971 /* Allocate temporary memory to setup array */ 1972 mta = malloc(sizeof(u8) * 1973 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 1974 M_DEVBUF, M_NOWAIT | M_ZERO); 1975 if (mta == NULL) 1976 panic("em_set_multi memory failure\n"); 1977 1978 #if __FreeBSD_version < 800000 1979 IF_ADDR_LOCK(ifp); 1980 #else 1981 if_maddr_rlock(ifp); 1982 #endif 1983 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1984 if (ifma->ifma_addr->sa_family != AF_LINK) 1985 continue; 1986 1987 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1988 break; 1989 1990 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1991 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1992 mcnt++; 1993 } 1994 #if __FreeBSD_version < 800000 1995 IF_ADDR_UNLOCK(ifp); 1996 #else 1997 if_maddr_runlock(ifp); 1998 #endif 1999 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2000 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2001 reg_rctl |= E1000_RCTL_MPE; 2002 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2003 } else 2004 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2005 2006 if (adapter->hw.mac.type == e1000_82542 && 2007 adapter->hw.revision_id == E1000_REVISION_2) { 2008 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2009 reg_rctl &= ~E1000_RCTL_RST; 2010 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2011 msec_delay(5); 2012 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2013 e1000_pci_set_mwi(&adapter->hw); 2014 } 2015 free(mta, M_DEVBUF); 2016 } 2017 2018 2019 /********************************************************************* 2020 * Timer routine 2021 * 2022 * This routine checks for link status and updates statistics. 2023 * 2024 **********************************************************************/ 2025 2026 static void 2027 em_local_timer(void *arg) 2028 { 2029 struct adapter *adapter = arg; 2030 struct ifnet *ifp = adapter->ifp; 2031 struct tx_ring *txr = adapter->tx_rings; 2032 2033 EM_CORE_LOCK_ASSERT(adapter); 2034 2035 em_update_link_status(adapter); 2036 em_update_stats_counters(adapter); 2037 2038 /* Reset LAA into RAR[0] on 82571 */ 2039 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2040 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2041 2042 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2043 em_print_hw_stats(adapter); 2044 2045 /* 2046 ** Check for time since any descriptor was cleaned 2047 */ 2048 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2049 EM_TX_LOCK(txr); 2050 if (txr->watchdog_check == FALSE) { 2051 EM_TX_UNLOCK(txr); 2052 continue; 2053 } 2054 if ((ticks - txr->watchdog_time) > EM_WATCHDOG) 2055 goto hung; 2056 EM_TX_UNLOCK(txr); 2057 } 2058 2059 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2060 return; 2061 hung: 2062 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2063 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2064 adapter->watchdog_events++; 2065 EM_TX_UNLOCK(txr); 2066 em_init_locked(adapter); 2067 } 2068 2069 2070 static void 2071 em_update_link_status(struct adapter *adapter) 2072 { 2073 struct e1000_hw *hw = &adapter->hw; 2074 struct ifnet *ifp = adapter->ifp; 2075 device_t dev = adapter->dev; 2076 u32 link_check = 0; 2077 2078 /* Get the cached link value or read phy for real */ 2079 switch (hw->phy.media_type) { 2080 case e1000_media_type_copper: 2081 if (hw->mac.get_link_status) { 2082 /* Do the work to read phy */ 2083 e1000_check_for_link(hw); 2084 link_check = !hw->mac.get_link_status; 2085 if (link_check) /* ESB2 fix */ 2086 e1000_cfg_on_link_up(hw); 2087 } else 2088 link_check = TRUE; 2089 break; 2090 case e1000_media_type_fiber: 2091 e1000_check_for_link(hw); 2092 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2093 E1000_STATUS_LU); 2094 break; 2095 case e1000_media_type_internal_serdes: 2096 e1000_check_for_link(hw); 2097 link_check = adapter->hw.mac.serdes_has_link; 2098 break; 2099 default: 2100 case e1000_media_type_unknown: 2101 break; 2102 } 2103 2104 /* Now check for a transition */ 2105 if (link_check && (adapter->link_active == 0)) { 2106 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2107 &adapter->link_duplex); 2108 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2109 if ((adapter->link_speed != SPEED_1000) && 2110 ((hw->mac.type == e1000_82571) || 2111 (hw->mac.type == e1000_82572))) { 2112 int tarc0; 2113 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2114 tarc0 &= ~SPEED_MODE_BIT; 2115 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2116 } 2117 if (bootverbose) 2118 device_printf(dev, "Link is up %d Mbps %s\n", 2119 adapter->link_speed, 2120 ((adapter->link_duplex == FULL_DUPLEX) ? 2121 "Full Duplex" : "Half Duplex")); 2122 adapter->link_active = 1; 2123 adapter->smartspeed = 0; 2124 ifp->if_baudrate = adapter->link_speed * 1000000; 2125 if_link_state_change(ifp, LINK_STATE_UP); 2126 } else if (!link_check && (adapter->link_active == 1)) { 2127 ifp->if_baudrate = adapter->link_speed = 0; 2128 adapter->link_duplex = 0; 2129 if (bootverbose) 2130 device_printf(dev, "Link is Down\n"); 2131 adapter->link_active = 0; 2132 /* Link down, disable watchdog */ 2133 // JFV change later 2134 //adapter->watchdog_check = FALSE; 2135 if_link_state_change(ifp, LINK_STATE_DOWN); 2136 } 2137 } 2138 2139 /********************************************************************* 2140 * 2141 * This routine disables all traffic on the adapter by issuing a 2142 * global reset on the MAC and deallocates TX/RX buffers. 2143 * 2144 * This routine should always be called with BOTH the CORE 2145 * and TX locks. 2146 **********************************************************************/ 2147 2148 static void 2149 em_stop(void *arg) 2150 { 2151 struct adapter *adapter = arg; 2152 struct ifnet *ifp = adapter->ifp; 2153 struct tx_ring *txr = adapter->tx_rings; 2154 2155 EM_CORE_LOCK_ASSERT(adapter); 2156 2157 INIT_DEBUGOUT("em_stop: begin"); 2158 2159 em_disable_intr(adapter); 2160 callout_stop(&adapter->timer); 2161 2162 /* Tell the stack that the interface is no longer active */ 2163 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2164 2165 /* Unarm watchdog timer. */ 2166 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2167 EM_TX_LOCK(txr); 2168 txr->watchdog_check = FALSE; 2169 EM_TX_UNLOCK(txr); 2170 } 2171 2172 e1000_reset_hw(&adapter->hw); 2173 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2174 2175 e1000_led_off(&adapter->hw); 2176 e1000_cleanup_led(&adapter->hw); 2177 } 2178 2179 2180 /********************************************************************* 2181 * 2182 * Determine hardware revision. 2183 * 2184 **********************************************************************/ 2185 static void 2186 em_identify_hardware(struct adapter *adapter) 2187 { 2188 device_t dev = adapter->dev; 2189 2190 /* Make sure our PCI config space has the necessary stuff set */ 2191 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2192 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2193 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2194 device_printf(dev, "Memory Access and/or Bus Master bits " 2195 "were not set!\n"); 2196 adapter->hw.bus.pci_cmd_word |= 2197 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2198 pci_write_config(dev, PCIR_COMMAND, 2199 adapter->hw.bus.pci_cmd_word, 2); 2200 } 2201 2202 /* Save off the information about this board */ 2203 adapter->hw.vendor_id = pci_get_vendor(dev); 2204 adapter->hw.device_id = pci_get_device(dev); 2205 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2206 adapter->hw.subsystem_vendor_id = 2207 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2208 adapter->hw.subsystem_device_id = 2209 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2210 2211 /* Do Shared Code Init and Setup */ 2212 if (e1000_set_mac_type(&adapter->hw)) { 2213 device_printf(dev, "Setup init failure\n"); 2214 return; 2215 } 2216 } 2217 2218 static int 2219 em_allocate_pci_resources(struct adapter *adapter) 2220 { 2221 device_t dev = adapter->dev; 2222 int rid; 2223 2224 rid = PCIR_BAR(0); 2225 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2226 &rid, RF_ACTIVE); 2227 if (adapter->memory == NULL) { 2228 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2229 return (ENXIO); 2230 } 2231 adapter->osdep.mem_bus_space_tag = 2232 rman_get_bustag(adapter->memory); 2233 adapter->osdep.mem_bus_space_handle = 2234 rman_get_bushandle(adapter->memory); 2235 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2236 2237 /* Default to a single queue */ 2238 adapter->num_queues = 1; 2239 2240 /* 2241 * Setup MSI/X or MSI if PCI Express 2242 */ 2243 adapter->msix = em_setup_msix(adapter); 2244 2245 adapter->hw.back = &adapter->osdep; 2246 2247 return (0); 2248 } 2249 2250 /********************************************************************* 2251 * 2252 * Setup the Legacy or MSI Interrupt handler 2253 * 2254 **********************************************************************/ 2255 int 2256 em_allocate_legacy(struct adapter *adapter) 2257 { 2258 device_t dev = adapter->dev; 2259 int error, rid = 0; 2260 2261 /* Manually turn off all interrupts */ 2262 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2263 2264 if (adapter->msix == 1) /* using MSI */ 2265 rid = 1; 2266 /* We allocate a single interrupt resource */ 2267 adapter->res = bus_alloc_resource_any(dev, 2268 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2269 if (adapter->res == NULL) { 2270 device_printf(dev, "Unable to allocate bus resource: " 2271 "interrupt\n"); 2272 return (ENXIO); 2273 } 2274 2275 /* 2276 * Allocate a fast interrupt and the associated 2277 * deferred processing contexts. 2278 */ 2279 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); 2280 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2281 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2282 taskqueue_thread_enqueue, &adapter->tq); 2283 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2284 device_get_nameunit(adapter->dev)); 2285 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, 2286 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { 2287 device_printf(dev, "Failed to register fast interrupt " 2288 "handler: %d\n", error); 2289 taskqueue_free(adapter->tq); 2290 adapter->tq = NULL; 2291 return (error); 2292 } 2293 2294 return (0); 2295 } 2296 2297 /********************************************************************* 2298 * 2299 * Setup the MSIX Interrupt handlers 2300 * This is not really Multiqueue, rather 2301 * its just multiple interrupt vectors. 2302 * 2303 **********************************************************************/ 2304 int 2305 em_allocate_msix(struct adapter *adapter) 2306 { 2307 device_t dev = adapter->dev; 2308 struct tx_ring *txr = adapter->tx_rings; 2309 struct rx_ring *rxr = adapter->rx_rings; 2310 int error, rid, vector = 0; 2311 2312 2313 /* Make sure all interrupts are disabled */ 2314 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2315 2316 /* First set up ring resources */ 2317 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { 2318 2319 /* RX ring */ 2320 rid = vector + 1; 2321 2322 rxr->res = bus_alloc_resource_any(dev, 2323 SYS_RES_IRQ, &rid, RF_ACTIVE); 2324 if (rxr->res == NULL) { 2325 device_printf(dev, 2326 "Unable to allocate bus resource: " 2327 "RX MSIX Interrupt %d\n", i); 2328 return (ENXIO); 2329 } 2330 if ((error = bus_setup_intr(dev, rxr->res, 2331 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, 2332 rxr, &rxr->tag)) != 0) { 2333 device_printf(dev, "Failed to register RX handler"); 2334 return (error); 2335 } 2336 rxr->msix = vector++; /* NOTE increment vector for TX */ 2337 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); 2338 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, 2339 taskqueue_thread_enqueue, &rxr->tq); 2340 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 2341 device_get_nameunit(adapter->dev)); 2342 /* 2343 ** Set the bit to enable interrupt 2344 ** in E1000_IMS -- bits 20 and 21 2345 ** are for RX0 and RX1, note this has 2346 ** NOTHING to do with the MSIX vector 2347 */ 2348 rxr->ims = 1 << (20 + i); 2349 adapter->ivars |= (8 | rxr->msix) << (i * 4); 2350 2351 /* TX ring */ 2352 rid = vector + 1; 2353 txr->res = bus_alloc_resource_any(dev, 2354 SYS_RES_IRQ, &rid, RF_ACTIVE); 2355 if (txr->res == NULL) { 2356 device_printf(dev, 2357 "Unable to allocate bus resource: " 2358 "TX MSIX Interrupt %d\n", i); 2359 return (ENXIO); 2360 } 2361 if ((error = bus_setup_intr(dev, txr->res, 2362 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, 2363 txr, &txr->tag)) != 0) { 2364 device_printf(dev, "Failed to register TX handler"); 2365 return (error); 2366 } 2367 txr->msix = vector++; /* Increment vector for next pass */ 2368 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); 2369 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, 2370 taskqueue_thread_enqueue, &txr->tq); 2371 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 2372 device_get_nameunit(adapter->dev)); 2373 /* 2374 ** Set the bit to enable interrupt 2375 ** in E1000_IMS -- bits 22 and 23 2376 ** are for TX0 and TX1, note this has 2377 ** NOTHING to do with the MSIX vector 2378 */ 2379 txr->ims = 1 << (22 + i); 2380 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); 2381 } 2382 2383 /* Link interrupt */ 2384 ++rid; 2385 adapter->res = bus_alloc_resource_any(dev, 2386 SYS_RES_IRQ, &rid, RF_ACTIVE); 2387 if (!adapter->res) { 2388 device_printf(dev,"Unable to allocate " 2389 "bus resource: Link interrupt [%d]\n", rid); 2390 return (ENXIO); 2391 } 2392 /* Set the link handler function */ 2393 error = bus_setup_intr(dev, adapter->res, 2394 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2395 em_msix_link, adapter, &adapter->tag); 2396 if (error) { 2397 adapter->res = NULL; 2398 device_printf(dev, "Failed to register LINK handler"); 2399 return (error); 2400 } 2401 adapter->linkvec = vector; 2402 adapter->ivars |= (8 | vector) << 16; 2403 adapter->ivars |= 0x80000000; 2404 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2405 adapter->tq = taskqueue_create_fast("em_link", M_NOWAIT, 2406 taskqueue_thread_enqueue, &adapter->tq); 2407 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", 2408 device_get_nameunit(adapter->dev)); 2409 2410 return (0); 2411 } 2412 2413 2414 static void 2415 em_free_pci_resources(struct adapter *adapter) 2416 { 2417 device_t dev = adapter->dev; 2418 struct tx_ring *txr; 2419 struct rx_ring *rxr; 2420 int rid; 2421 2422 2423 /* 2424 ** Release all the queue interrupt resources: 2425 */ 2426 for (int i = 0; i < adapter->num_queues; i++) { 2427 txr = &adapter->tx_rings[i]; 2428 rxr = &adapter->rx_rings[i]; 2429 rid = txr->msix +1; 2430 if (txr->tag != NULL) { 2431 bus_teardown_intr(dev, txr->res, txr->tag); 2432 txr->tag = NULL; 2433 } 2434 if (txr->res != NULL) 2435 bus_release_resource(dev, SYS_RES_IRQ, 2436 rid, txr->res); 2437 rid = rxr->msix +1; 2438 if (rxr->tag != NULL) { 2439 bus_teardown_intr(dev, rxr->res, rxr->tag); 2440 rxr->tag = NULL; 2441 } 2442 if (rxr->res != NULL) 2443 bus_release_resource(dev, SYS_RES_IRQ, 2444 rid, rxr->res); 2445 } 2446 2447 if (adapter->linkvec) /* we are doing MSIX */ 2448 rid = adapter->linkvec + 1; 2449 else 2450 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2451 2452 if (adapter->tag != NULL) { 2453 bus_teardown_intr(dev, adapter->res, adapter->tag); 2454 adapter->tag = NULL; 2455 } 2456 2457 if (adapter->res != NULL) 2458 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2459 2460 2461 if (adapter->msix) 2462 pci_release_msi(dev); 2463 2464 if (adapter->msix_mem != NULL) 2465 bus_release_resource(dev, SYS_RES_MEMORY, 2466 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2467 2468 if (adapter->memory != NULL) 2469 bus_release_resource(dev, SYS_RES_MEMORY, 2470 PCIR_BAR(0), adapter->memory); 2471 2472 if (adapter->flash != NULL) 2473 bus_release_resource(dev, SYS_RES_MEMORY, 2474 EM_FLASH, adapter->flash); 2475 } 2476 2477 /* 2478 * Setup MSI or MSI/X 2479 */ 2480 static int 2481 em_setup_msix(struct adapter *adapter) 2482 { 2483 device_t dev = adapter->dev; 2484 int val = 0; 2485 2486 2487 /* Setup MSI/X for Hartwell */ 2488 if ((adapter->hw.mac.type == e1000_82574) && 2489 (em_enable_msix == TRUE)) { 2490 /* Map the MSIX BAR */ 2491 int rid = PCIR_BAR(EM_MSIX_BAR); 2492 adapter->msix_mem = bus_alloc_resource_any(dev, 2493 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2494 if (!adapter->msix_mem) { 2495 /* May not be enabled */ 2496 device_printf(adapter->dev, 2497 "Unable to map MSIX table \n"); 2498 goto msi; 2499 } 2500 val = pci_msix_count(dev); 2501 if (val != 5) { 2502 bus_release_resource(dev, SYS_RES_MEMORY, 2503 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2504 adapter->msix_mem = NULL; 2505 device_printf(adapter->dev, 2506 "MSIX vectors wrong, using MSI \n"); 2507 goto msi; 2508 } 2509 if (em_msix_queues == 2) { 2510 val = 5; 2511 adapter->num_queues = 2; 2512 } else { 2513 val = 3; 2514 adapter->num_queues = 1; 2515 } 2516 if (pci_alloc_msix(dev, &val) == 0) { 2517 device_printf(adapter->dev, 2518 "Using MSIX interrupts " 2519 "with %d vectors\n", val); 2520 } 2521 2522 return (val); 2523 } 2524 msi: 2525 val = pci_msi_count(dev); 2526 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2527 adapter->msix = 1; 2528 device_printf(adapter->dev,"Using MSI interrupt\n"); 2529 return (val); 2530 } 2531 /* Should only happen due to manual invention */ 2532 device_printf(adapter->dev,"Setup MSIX failure\n"); 2533 return (0); 2534 } 2535 2536 2537 /********************************************************************* 2538 * 2539 * Initialize the hardware to a configuration 2540 * as specified by the adapter structure. 2541 * 2542 **********************************************************************/ 2543 static void 2544 em_reset(struct adapter *adapter) 2545 { 2546 device_t dev = adapter->dev; 2547 struct e1000_hw *hw = &adapter->hw; 2548 u16 rx_buffer_size; 2549 2550 INIT_DEBUGOUT("em_reset: begin"); 2551 2552 /* Set up smart power down as default off on newer adapters. */ 2553 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 2554 hw->mac.type == e1000_82572)) { 2555 u16 phy_tmp = 0; 2556 2557 /* Speed up time to link by disabling smart power down. */ 2558 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2559 phy_tmp &= ~IGP02E1000_PM_SPD; 2560 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2561 } 2562 2563 /* 2564 * These parameters control the automatic generation (Tx) and 2565 * response (Rx) to Ethernet PAUSE frames. 2566 * - High water mark should allow for at least two frames to be 2567 * received after sending an XOFF. 2568 * - Low water mark works best when it is very near the high water mark. 2569 * This allows the receiver to restart by sending XON when it has 2570 * drained a bit. Here we use an arbitary value of 1500 which will 2571 * restart after one full frame is pulled from the buffer. There 2572 * could be several smaller frames in the buffer and if so they will 2573 * not trigger the XON until their total number reduces the buffer 2574 * by 1500. 2575 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2576 */ 2577 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); 2578 2579 hw->fc.high_water = rx_buffer_size - 2580 roundup2(adapter->max_frame_size, 1024); 2581 hw->fc.low_water = hw->fc.high_water - 1500; 2582 2583 if (hw->mac.type == e1000_80003es2lan) 2584 hw->fc.pause_time = 0xFFFF; 2585 else 2586 hw->fc.pause_time = EM_FC_PAUSE_TIME; 2587 2588 hw->fc.send_xon = TRUE; 2589 2590 /* Set Flow control, use the tunable location if sane */ 2591 if ((em_fc_setting >= 0) || (em_fc_setting < 4)) 2592 hw->fc.requested_mode = em_fc_setting; 2593 else 2594 hw->fc.requested_mode = e1000_fc_none; 2595 2596 /* Override - workaround for PCHLAN issue */ 2597 if (hw->mac.type == e1000_pchlan) 2598 hw->fc.requested_mode = e1000_fc_rx_pause; 2599 2600 /* Issue a global reset */ 2601 e1000_reset_hw(hw); 2602 E1000_WRITE_REG(hw, E1000_WUC, 0); 2603 2604 if (e1000_init_hw(hw) < 0) { 2605 device_printf(dev, "Hardware Initialization Failed\n"); 2606 return; 2607 } 2608 2609 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 2610 e1000_get_phy_info(hw); 2611 e1000_check_for_link(hw); 2612 return; 2613 } 2614 2615 /********************************************************************* 2616 * 2617 * Setup networking device structure and register an interface. 2618 * 2619 **********************************************************************/ 2620 static void 2621 em_setup_interface(device_t dev, struct adapter *adapter) 2622 { 2623 struct ifnet *ifp; 2624 2625 INIT_DEBUGOUT("em_setup_interface: begin"); 2626 2627 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2628 if (ifp == NULL) 2629 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2630 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2631 ifp->if_mtu = ETHERMTU; 2632 ifp->if_init = em_init; 2633 ifp->if_softc = adapter; 2634 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2635 ifp->if_ioctl = em_ioctl; 2636 ifp->if_start = em_start; 2637 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2638 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2639 IFQ_SET_READY(&ifp->if_snd); 2640 2641 ether_ifattach(ifp, adapter->hw.mac.addr); 2642 2643 ifp->if_capabilities = ifp->if_capenable = 0; 2644 2645 #if __FreeBSD_version >= 800000 2646 /* Multiqueue tx functions */ 2647 ifp->if_transmit = em_mq_start; 2648 ifp->if_qflush = em_qflush; 2649 #endif 2650 2651 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2652 ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2653 2654 /* Enable TSO by default, can disable with ifconfig */ 2655 ifp->if_capabilities |= IFCAP_TSO4; 2656 ifp->if_capenable |= IFCAP_TSO4; 2657 2658 /* 2659 * Tell the upper layer(s) we support long frames. 2660 */ 2661 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2662 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2663 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2664 2665 #ifdef DEVICE_POLLING 2666 ifp->if_capabilities |= IFCAP_POLLING; 2667 #endif 2668 2669 /* Enable All WOL methods by default */ 2670 if (adapter->wol) { 2671 ifp->if_capabilities |= IFCAP_WOL; 2672 ifp->if_capenable |= IFCAP_WOL; 2673 } 2674 2675 /* 2676 * Specify the media types supported by this adapter and register 2677 * callbacks to update media and link information 2678 */ 2679 ifmedia_init(&adapter->media, IFM_IMASK, 2680 em_media_change, em_media_status); 2681 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2682 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2683 u_char fiber_type = IFM_1000_SX; /* default type */ 2684 2685 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2686 0, NULL); 2687 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 2688 } else { 2689 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2690 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2691 0, NULL); 2692 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2693 0, NULL); 2694 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2695 0, NULL); 2696 if (adapter->hw.phy.type != e1000_phy_ife) { 2697 ifmedia_add(&adapter->media, 2698 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2699 ifmedia_add(&adapter->media, 2700 IFM_ETHER | IFM_1000_T, 0, NULL); 2701 } 2702 } 2703 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2704 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2705 } 2706 2707 2708 /* 2709 * Manage DMA'able memory. 2710 */ 2711 static void 2712 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2713 { 2714 if (error) 2715 return; 2716 *(bus_addr_t *) arg = segs[0].ds_addr; 2717 } 2718 2719 static int 2720 em_dma_malloc(struct adapter *adapter, bus_size_t size, 2721 struct em_dma_alloc *dma, int mapflags) 2722 { 2723 int error; 2724 2725 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2726 EM_DBA_ALIGN, 0, /* alignment, bounds */ 2727 BUS_SPACE_MAXADDR, /* lowaddr */ 2728 BUS_SPACE_MAXADDR, /* highaddr */ 2729 NULL, NULL, /* filter, filterarg */ 2730 size, /* maxsize */ 2731 1, /* nsegments */ 2732 size, /* maxsegsize */ 2733 0, /* flags */ 2734 NULL, /* lockfunc */ 2735 NULL, /* lockarg */ 2736 &dma->dma_tag); 2737 if (error) { 2738 device_printf(adapter->dev, 2739 "%s: bus_dma_tag_create failed: %d\n", 2740 __func__, error); 2741 goto fail_0; 2742 } 2743 2744 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2745 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 2746 if (error) { 2747 device_printf(adapter->dev, 2748 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2749 __func__, (uintmax_t)size, error); 2750 goto fail_2; 2751 } 2752 2753 dma->dma_paddr = 0; 2754 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2755 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2756 if (error || dma->dma_paddr == 0) { 2757 device_printf(adapter->dev, 2758 "%s: bus_dmamap_load failed: %d\n", 2759 __func__, error); 2760 goto fail_3; 2761 } 2762 2763 return (0); 2764 2765 fail_3: 2766 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2767 fail_2: 2768 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2769 bus_dma_tag_destroy(dma->dma_tag); 2770 fail_0: 2771 dma->dma_map = NULL; 2772 dma->dma_tag = NULL; 2773 2774 return (error); 2775 } 2776 2777 static void 2778 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 2779 { 2780 if (dma->dma_tag == NULL) 2781 return; 2782 if (dma->dma_map != NULL) { 2783 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2784 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2785 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2786 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2787 dma->dma_map = NULL; 2788 } 2789 bus_dma_tag_destroy(dma->dma_tag); 2790 dma->dma_tag = NULL; 2791 } 2792 2793 2794 /********************************************************************* 2795 * 2796 * Allocate memory for the transmit and receive rings, and then 2797 * the descriptors associated with each, called only once at attach. 2798 * 2799 **********************************************************************/ 2800 static int 2801 em_allocate_queues(struct adapter *adapter) 2802 { 2803 device_t dev = adapter->dev; 2804 struct tx_ring *txr = NULL; 2805 struct rx_ring *rxr = NULL; 2806 int rsize, tsize, error = E1000_SUCCESS; 2807 int txconf = 0, rxconf = 0; 2808 2809 2810 /* Allocate the TX ring struct memory */ 2811 if (!(adapter->tx_rings = 2812 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2813 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2814 device_printf(dev, "Unable to allocate TX ring memory\n"); 2815 error = ENOMEM; 2816 goto fail; 2817 } 2818 2819 /* Now allocate the RX */ 2820 if (!(adapter->rx_rings = 2821 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2822 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2823 device_printf(dev, "Unable to allocate RX ring memory\n"); 2824 error = ENOMEM; 2825 goto rx_fail; 2826 } 2827 2828 tsize = roundup2(adapter->num_tx_desc * 2829 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 2830 /* 2831 * Now set up the TX queues, txconf is needed to handle the 2832 * possibility that things fail midcourse and we need to 2833 * undo memory gracefully 2834 */ 2835 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2836 /* Set up some basics */ 2837 txr = &adapter->tx_rings[i]; 2838 txr->adapter = adapter; 2839 txr->me = i; 2840 2841 /* Initialize the TX lock */ 2842 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2843 device_get_nameunit(dev), txr->me); 2844 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2845 2846 if (em_dma_malloc(adapter, tsize, 2847 &txr->txdma, BUS_DMA_NOWAIT)) { 2848 device_printf(dev, 2849 "Unable to allocate TX Descriptor memory\n"); 2850 error = ENOMEM; 2851 goto err_tx_desc; 2852 } 2853 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2854 bzero((void *)txr->tx_base, tsize); 2855 2856 if (em_allocate_transmit_buffers(txr)) { 2857 device_printf(dev, 2858 "Critical Failure setting up transmit buffers\n"); 2859 error = ENOMEM; 2860 goto err_tx_desc; 2861 } 2862 #if __FreeBSD_version >= 800000 2863 /* Allocate a buf ring */ 2864 txr->br = buf_ring_alloc(4096, M_DEVBUF, 2865 M_WAITOK, &txr->tx_mtx); 2866 #endif 2867 } 2868 2869 /* 2870 * Next the RX queues... 2871 */ 2872 rsize = roundup2(adapter->num_rx_desc * 2873 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 2874 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2875 rxr = &adapter->rx_rings[i]; 2876 rxr->adapter = adapter; 2877 rxr->me = i; 2878 2879 /* Initialize the RX lock */ 2880 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2881 device_get_nameunit(dev), txr->me); 2882 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2883 2884 if (em_dma_malloc(adapter, rsize, 2885 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2886 device_printf(dev, 2887 "Unable to allocate RxDescriptor memory\n"); 2888 error = ENOMEM; 2889 goto err_rx_desc; 2890 } 2891 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; 2892 bzero((void *)rxr->rx_base, rsize); 2893 2894 /* Allocate receive buffers for the ring*/ 2895 if (em_allocate_receive_buffers(rxr)) { 2896 device_printf(dev, 2897 "Critical Failure setting up receive buffers\n"); 2898 error = ENOMEM; 2899 goto err_rx_desc; 2900 } 2901 } 2902 2903 return (0); 2904 2905 err_rx_desc: 2906 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2907 em_dma_free(adapter, &rxr->rxdma); 2908 err_tx_desc: 2909 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2910 em_dma_free(adapter, &txr->txdma); 2911 free(adapter->rx_rings, M_DEVBUF); 2912 rx_fail: 2913 buf_ring_free(txr->br, M_DEVBUF); 2914 free(adapter->tx_rings, M_DEVBUF); 2915 fail: 2916 return (error); 2917 } 2918 2919 2920 /********************************************************************* 2921 * 2922 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2923 * the information needed to transmit a packet on the wire. This is 2924 * called only once at attach, setup is done every reset. 2925 * 2926 **********************************************************************/ 2927 static int 2928 em_allocate_transmit_buffers(struct tx_ring *txr) 2929 { 2930 struct adapter *adapter = txr->adapter; 2931 device_t dev = adapter->dev; 2932 struct em_buffer *txbuf; 2933 int error, i; 2934 2935 /* 2936 * Setup DMA descriptor areas. 2937 */ 2938 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 2939 1, 0, /* alignment, bounds */ 2940 BUS_SPACE_MAXADDR, /* lowaddr */ 2941 BUS_SPACE_MAXADDR, /* highaddr */ 2942 NULL, NULL, /* filter, filterarg */ 2943 EM_TSO_SIZE, /* maxsize */ 2944 EM_MAX_SCATTER, /* nsegments */ 2945 PAGE_SIZE, /* maxsegsize */ 2946 0, /* flags */ 2947 NULL, /* lockfunc */ 2948 NULL, /* lockfuncarg */ 2949 &txr->txtag))) { 2950 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2951 goto fail; 2952 } 2953 2954 if (!(txr->tx_buffers = 2955 (struct em_buffer *) malloc(sizeof(struct em_buffer) * 2956 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2957 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2958 error = ENOMEM; 2959 goto fail; 2960 } 2961 2962 /* Create the descriptor buffer dma maps */ 2963 txbuf = txr->tx_buffers; 2964 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2965 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2966 if (error != 0) { 2967 device_printf(dev, "Unable to create TX DMA map\n"); 2968 goto fail; 2969 } 2970 } 2971 2972 return 0; 2973 fail: 2974 /* We free all, it handles case where we are in the middle */ 2975 em_free_transmit_structures(adapter); 2976 return (error); 2977 } 2978 2979 /********************************************************************* 2980 * 2981 * Initialize a transmit ring. 2982 * 2983 **********************************************************************/ 2984 static void 2985 em_setup_transmit_ring(struct tx_ring *txr) 2986 { 2987 struct adapter *adapter = txr->adapter; 2988 struct em_buffer *txbuf; 2989 int i; 2990 2991 /* Clear the old descriptor contents */ 2992 EM_TX_LOCK(txr); 2993 bzero((void *)txr->tx_base, 2994 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 2995 /* Reset indices */ 2996 txr->next_avail_desc = 0; 2997 txr->next_to_clean = 0; 2998 2999 /* Free any existing tx buffers. */ 3000 txbuf = txr->tx_buffers; 3001 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3002 if (txbuf->m_head != NULL) { 3003 bus_dmamap_sync(txr->txtag, txbuf->map, 3004 BUS_DMASYNC_POSTWRITE); 3005 bus_dmamap_unload(txr->txtag, txbuf->map); 3006 m_freem(txbuf->m_head); 3007 txbuf->m_head = NULL; 3008 } 3009 /* clear the watch index */ 3010 txbuf->next_eop = -1; 3011 } 3012 3013 /* Set number of descriptors available */ 3014 txr->tx_avail = adapter->num_tx_desc; 3015 3016 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3017 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3018 EM_TX_UNLOCK(txr); 3019 } 3020 3021 /********************************************************************* 3022 * 3023 * Initialize all transmit rings. 3024 * 3025 **********************************************************************/ 3026 static void 3027 em_setup_transmit_structures(struct adapter *adapter) 3028 { 3029 struct tx_ring *txr = adapter->tx_rings; 3030 3031 for (int i = 0; i < adapter->num_queues; i++, txr++) 3032 em_setup_transmit_ring(txr); 3033 3034 return; 3035 } 3036 3037 /********************************************************************* 3038 * 3039 * Enable transmit unit. 3040 * 3041 **********************************************************************/ 3042 static void 3043 em_initialize_transmit_unit(struct adapter *adapter) 3044 { 3045 struct tx_ring *txr = adapter->tx_rings; 3046 struct e1000_hw *hw = &adapter->hw; 3047 u32 tctl, tarc, tipg = 0; 3048 3049 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3050 3051 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3052 u64 bus_addr = txr->txdma.dma_paddr; 3053 /* Base and Len of TX Ring */ 3054 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3055 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3056 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3057 (u32)(bus_addr >> 32)); 3058 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3059 (u32)bus_addr); 3060 /* Init the HEAD/TAIL indices */ 3061 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3062 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3063 3064 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3065 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 3066 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 3067 3068 txr->watchdog_check = FALSE; 3069 } 3070 3071 /* Set the default values for the Tx Inter Packet Gap timer */ 3072 switch (adapter->hw.mac.type) { 3073 case e1000_82542: 3074 tipg = DEFAULT_82542_TIPG_IPGT; 3075 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3076 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3077 break; 3078 case e1000_80003es2lan: 3079 tipg = DEFAULT_82543_TIPG_IPGR1; 3080 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3081 E1000_TIPG_IPGR2_SHIFT; 3082 break; 3083 default: 3084 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3085 (adapter->hw.phy.media_type == 3086 e1000_media_type_internal_serdes)) 3087 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3088 else 3089 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3090 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3091 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3092 } 3093 3094 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3095 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3096 3097 if(adapter->hw.mac.type >= e1000_82540) 3098 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3099 adapter->tx_abs_int_delay.value); 3100 3101 if ((adapter->hw.mac.type == e1000_82571) || 3102 (adapter->hw.mac.type == e1000_82572)) { 3103 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3104 tarc |= SPEED_MODE_BIT; 3105 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3106 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3107 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3108 tarc |= 1; 3109 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3110 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3111 tarc |= 1; 3112 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3113 } 3114 3115 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3116 if (adapter->tx_int_delay.value > 0) 3117 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3118 3119 /* Program the Transmit Control Register */ 3120 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3121 tctl &= ~E1000_TCTL_CT; 3122 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3123 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3124 3125 if (adapter->hw.mac.type >= e1000_82571) 3126 tctl |= E1000_TCTL_MULR; 3127 3128 /* This write will effectively turn on the transmit unit. */ 3129 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3130 3131 } 3132 3133 3134 /********************************************************************* 3135 * 3136 * Free all transmit rings. 3137 * 3138 **********************************************************************/ 3139 static void 3140 em_free_transmit_structures(struct adapter *adapter) 3141 { 3142 struct tx_ring *txr = adapter->tx_rings; 3143 3144 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3145 EM_TX_LOCK(txr); 3146 em_free_transmit_buffers(txr); 3147 em_dma_free(adapter, &txr->txdma); 3148 EM_TX_UNLOCK(txr); 3149 EM_TX_LOCK_DESTROY(txr); 3150 } 3151 3152 free(adapter->tx_rings, M_DEVBUF); 3153 } 3154 3155 /********************************************************************* 3156 * 3157 * Free transmit ring related data structures. 3158 * 3159 **********************************************************************/ 3160 static void 3161 em_free_transmit_buffers(struct tx_ring *txr) 3162 { 3163 struct adapter *adapter = txr->adapter; 3164 struct em_buffer *txbuf; 3165 3166 INIT_DEBUGOUT("free_transmit_ring: begin"); 3167 3168 if (txr->tx_buffers == NULL) 3169 return; 3170 3171 for (int i = 0; i < adapter->num_tx_desc; i++) { 3172 txbuf = &txr->tx_buffers[i]; 3173 if (txbuf->m_head != NULL) { 3174 bus_dmamap_sync(txr->txtag, txbuf->map, 3175 BUS_DMASYNC_POSTWRITE); 3176 bus_dmamap_unload(txr->txtag, 3177 txbuf->map); 3178 m_freem(txbuf->m_head); 3179 txbuf->m_head = NULL; 3180 if (txbuf->map != NULL) { 3181 bus_dmamap_destroy(txr->txtag, 3182 txbuf->map); 3183 txbuf->map = NULL; 3184 } 3185 } else if (txbuf->map != NULL) { 3186 bus_dmamap_unload(txr->txtag, 3187 txbuf->map); 3188 bus_dmamap_destroy(txr->txtag, 3189 txbuf->map); 3190 txbuf->map = NULL; 3191 } 3192 } 3193 #if __FreeBSD_version >= 800000 3194 if (txr->br != NULL) 3195 buf_ring_free(txr->br, M_DEVBUF); 3196 #endif 3197 if (txr->tx_buffers != NULL) { 3198 free(txr->tx_buffers, M_DEVBUF); 3199 txr->tx_buffers = NULL; 3200 } 3201 if (txr->txtag != NULL) { 3202 bus_dma_tag_destroy(txr->txtag); 3203 txr->txtag = NULL; 3204 } 3205 return; 3206 } 3207 3208 3209 /********************************************************************* 3210 * 3211 * The offload context needs to be set when we transfer the first 3212 * packet of a particular protocol (TCP/UDP). This routine has been 3213 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3214 * 3215 * Added back the old method of keeping the current context type 3216 * and not setting if unnecessary, as this is reported to be a 3217 * big performance win. -jfv 3218 **********************************************************************/ 3219 static void 3220 em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, 3221 u32 *txd_upper, u32 *txd_lower) 3222 { 3223 struct adapter *adapter = txr->adapter; 3224 struct e1000_context_desc *TXD = NULL; 3225 struct em_buffer *tx_buffer; 3226 struct ether_vlan_header *eh; 3227 struct ip *ip = NULL; 3228 struct ip6_hdr *ip6; 3229 int cur, ehdrlen; 3230 u32 cmd, hdr_len, ip_hlen; 3231 u16 etype; 3232 u8 ipproto; 3233 3234 3235 cmd = hdr_len = ipproto = 0; 3236 cur = txr->next_avail_desc; 3237 3238 /* 3239 * Determine where frame payload starts. 3240 * Jump over vlan headers if already present, 3241 * helpful for QinQ too. 3242 */ 3243 eh = mtod(mp, struct ether_vlan_header *); 3244 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3245 etype = ntohs(eh->evl_proto); 3246 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3247 } else { 3248 etype = ntohs(eh->evl_encap_proto); 3249 ehdrlen = ETHER_HDR_LEN; 3250 } 3251 3252 /* 3253 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3254 * TODO: Support SCTP too when it hits the tree. 3255 */ 3256 switch (etype) { 3257 case ETHERTYPE_IP: 3258 ip = (struct ip *)(mp->m_data + ehdrlen); 3259 ip_hlen = ip->ip_hl << 2; 3260 3261 /* Setup of IP header checksum. */ 3262 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3263 /* 3264 * Start offset for header checksum calculation. 3265 * End offset for header checksum calculation. 3266 * Offset of place to put the checksum. 3267 */ 3268 TXD = (struct e1000_context_desc *) 3269 &txr->tx_base[cur]; 3270 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3271 TXD->lower_setup.ip_fields.ipcse = 3272 htole16(ehdrlen + ip_hlen); 3273 TXD->lower_setup.ip_fields.ipcso = 3274 ehdrlen + offsetof(struct ip, ip_sum); 3275 cmd |= E1000_TXD_CMD_IP; 3276 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3277 } 3278 3279 if (mp->m_len < ehdrlen + ip_hlen) 3280 return; /* failure */ 3281 3282 hdr_len = ehdrlen + ip_hlen; 3283 ipproto = ip->ip_p; 3284 3285 break; 3286 case ETHERTYPE_IPV6: 3287 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3288 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3289 3290 if (mp->m_len < ehdrlen + ip_hlen) 3291 return; /* failure */ 3292 3293 /* IPv6 doesn't have a header checksum. */ 3294 3295 hdr_len = ehdrlen + ip_hlen; 3296 ipproto = ip6->ip6_nxt; 3297 3298 break; 3299 default: 3300 *txd_upper = 0; 3301 *txd_lower = 0; 3302 return; 3303 } 3304 3305 switch (ipproto) { 3306 case IPPROTO_TCP: 3307 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3308 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3309 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3310 /* no need for context if already set */ 3311 if (txr->last_hw_offload == CSUM_TCP) 3312 return; 3313 txr->last_hw_offload = CSUM_TCP; 3314 /* 3315 * Start offset for payload checksum calculation. 3316 * End offset for payload checksum calculation. 3317 * Offset of place to put the checksum. 3318 */ 3319 TXD = (struct e1000_context_desc *) 3320 &txr->tx_base[cur]; 3321 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3322 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3323 TXD->upper_setup.tcp_fields.tucso = 3324 hdr_len + offsetof(struct tcphdr, th_sum); 3325 cmd |= E1000_TXD_CMD_TCP; 3326 } 3327 break; 3328 case IPPROTO_UDP: 3329 { 3330 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3331 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3332 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3333 /* no need for context if already set */ 3334 if (txr->last_hw_offload == CSUM_UDP) 3335 return; 3336 txr->last_hw_offload = CSUM_UDP; 3337 /* 3338 * Start offset for header checksum calculation. 3339 * End offset for header checksum calculation. 3340 * Offset of place to put the checksum. 3341 */ 3342 TXD = (struct e1000_context_desc *) 3343 &txr->tx_base[cur]; 3344 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3345 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3346 TXD->upper_setup.tcp_fields.tucso = 3347 hdr_len + offsetof(struct udphdr, uh_sum); 3348 } 3349 /* Fall Thru */ 3350 } 3351 default: 3352 break; 3353 } 3354 3355 TXD->tcp_seg_setup.data = htole32(0); 3356 TXD->cmd_and_length = 3357 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3358 tx_buffer = &txr->tx_buffers[cur]; 3359 tx_buffer->m_head = NULL; 3360 tx_buffer->next_eop = -1; 3361 3362 if (++cur == adapter->num_tx_desc) 3363 cur = 0; 3364 3365 txr->tx_avail--; 3366 txr->next_avail_desc = cur; 3367 } 3368 3369 3370 /********************************************************************** 3371 * 3372 * Setup work for hardware segmentation offload (TSO) 3373 * 3374 **********************************************************************/ 3375 static bool 3376 em_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *txd_upper, 3377 u32 *txd_lower) 3378 { 3379 struct adapter *adapter = txr->adapter; 3380 struct e1000_context_desc *TXD; 3381 struct em_buffer *tx_buffer; 3382 struct ether_vlan_header *eh; 3383 struct ip *ip; 3384 struct ip6_hdr *ip6; 3385 struct tcphdr *th; 3386 int cur, ehdrlen, hdr_len, ip_hlen, isip6; 3387 u16 etype; 3388 3389 /* 3390 * This function could/should be extended to support IP/IPv6 3391 * fragmentation as well. But as they say, one step at a time. 3392 */ 3393 3394 /* 3395 * Determine where frame payload starts. 3396 * Jump over vlan headers if already present, 3397 * helpful for QinQ too. 3398 */ 3399 eh = mtod(mp, struct ether_vlan_header *); 3400 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3401 etype = ntohs(eh->evl_proto); 3402 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3403 } else { 3404 etype = ntohs(eh->evl_encap_proto); 3405 ehdrlen = ETHER_HDR_LEN; 3406 } 3407 3408 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3409 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3410 return FALSE; /* -1 */ 3411 3412 /* 3413 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3414 * TODO: Support SCTP too when it hits the tree. 3415 */ 3416 switch (etype) { 3417 case ETHERTYPE_IP: 3418 isip6 = 0; 3419 ip = (struct ip *)(mp->m_data + ehdrlen); 3420 if (ip->ip_p != IPPROTO_TCP) 3421 return FALSE; /* 0 */ 3422 ip->ip_len = 0; 3423 ip->ip_sum = 0; 3424 ip_hlen = ip->ip_hl << 2; 3425 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3426 return FALSE; /* -1 */ 3427 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3428 #if 1 3429 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3430 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3431 #else 3432 th->th_sum = mp->m_pkthdr.csum_data; 3433 #endif 3434 break; 3435 case ETHERTYPE_IPV6: 3436 isip6 = 1; 3437 return FALSE; /* Not supported yet. */ 3438 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3439 if (ip6->ip6_nxt != IPPROTO_TCP) 3440 return FALSE; /* 0 */ 3441 ip6->ip6_plen = 0; 3442 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3443 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3444 return FALSE; /* -1 */ 3445 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3446 #if 0 3447 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3448 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3449 #else 3450 th->th_sum = mp->m_pkthdr.csum_data; 3451 #endif 3452 break; 3453 default: 3454 return FALSE; 3455 } 3456 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3457 3458 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3459 E1000_TXD_DTYP_D | /* Data descr type */ 3460 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3461 3462 /* IP and/or TCP header checksum calculation and insertion. */ 3463 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3464 E1000_TXD_POPTS_TXSM) << 8; 3465 3466 cur = txr->next_avail_desc; 3467 tx_buffer = &txr->tx_buffers[cur]; 3468 TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; 3469 3470 /* IPv6 doesn't have a header checksum. */ 3471 if (!isip6) { 3472 /* 3473 * Start offset for header checksum calculation. 3474 * End offset for header checksum calculation. 3475 * Offset of place put the checksum. 3476 */ 3477 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3478 TXD->lower_setup.ip_fields.ipcse = 3479 htole16(ehdrlen + ip_hlen - 1); 3480 TXD->lower_setup.ip_fields.ipcso = 3481 ehdrlen + offsetof(struct ip, ip_sum); 3482 } 3483 /* 3484 * Start offset for payload checksum calculation. 3485 * End offset for payload checksum calculation. 3486 * Offset of place to put the checksum. 3487 */ 3488 TXD->upper_setup.tcp_fields.tucss = 3489 ehdrlen + ip_hlen; 3490 TXD->upper_setup.tcp_fields.tucse = 0; 3491 TXD->upper_setup.tcp_fields.tucso = 3492 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3493 /* 3494 * Payload size per packet w/o any headers. 3495 * Length of all headers up to payload. 3496 */ 3497 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3498 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3499 3500 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3501 E1000_TXD_CMD_DEXT | /* Extended descr */ 3502 E1000_TXD_CMD_TSE | /* TSE context */ 3503 (isip6 ? 0 : E1000_TXD_CMD_IP) | 3504 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3505 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3506 3507 tx_buffer->m_head = NULL; 3508 tx_buffer->next_eop = -1; 3509 3510 if (++cur == adapter->num_tx_desc) 3511 cur = 0; 3512 3513 txr->tx_avail--; 3514 txr->next_avail_desc = cur; 3515 txr->tx_tso = TRUE; 3516 3517 return TRUE; 3518 } 3519 3520 3521 /********************************************************************** 3522 * 3523 * Examine each tx_buffer in the used queue. If the hardware is done 3524 * processing the packet then free associated resources. The 3525 * tx_buffer is put back on the free queue. 3526 * 3527 **********************************************************************/ 3528 static bool 3529 em_txeof(struct tx_ring *txr) 3530 { 3531 struct adapter *adapter = txr->adapter; 3532 int first, last, done, num_avail; 3533 struct em_buffer *tx_buffer; 3534 struct e1000_tx_desc *tx_desc, *eop_desc; 3535 struct ifnet *ifp = adapter->ifp; 3536 3537 EM_TX_LOCK_ASSERT(txr); 3538 3539 if (txr->tx_avail == adapter->num_tx_desc) 3540 return (FALSE); 3541 3542 num_avail = txr->tx_avail; 3543 first = txr->next_to_clean; 3544 tx_desc = &txr->tx_base[first]; 3545 tx_buffer = &txr->tx_buffers[first]; 3546 last = tx_buffer->next_eop; 3547 eop_desc = &txr->tx_base[last]; 3548 3549 /* 3550 * What this does is get the index of the 3551 * first descriptor AFTER the EOP of the 3552 * first packet, that way we can do the 3553 * simple comparison on the inner while loop. 3554 */ 3555 if (++last == adapter->num_tx_desc) 3556 last = 0; 3557 done = last; 3558 3559 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3560 BUS_DMASYNC_POSTREAD); 3561 3562 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3563 /* We clean the range of the packet */ 3564 while (first != done) { 3565 tx_desc->upper.data = 0; 3566 tx_desc->lower.data = 0; 3567 tx_desc->buffer_addr = 0; 3568 ++num_avail; 3569 3570 if (tx_buffer->m_head) { 3571 ifp->if_opackets++; 3572 bus_dmamap_sync(txr->txtag, 3573 tx_buffer->map, 3574 BUS_DMASYNC_POSTWRITE); 3575 bus_dmamap_unload(txr->txtag, 3576 tx_buffer->map); 3577 3578 m_freem(tx_buffer->m_head); 3579 tx_buffer->m_head = NULL; 3580 } 3581 tx_buffer->next_eop = -1; 3582 txr->watchdog_time = ticks; 3583 3584 if (++first == adapter->num_tx_desc) 3585 first = 0; 3586 3587 tx_buffer = &txr->tx_buffers[first]; 3588 tx_desc = &txr->tx_base[first]; 3589 } 3590 /* See if we can continue to the next packet */ 3591 last = tx_buffer->next_eop; 3592 if (last != -1) { 3593 eop_desc = &txr->tx_base[last]; 3594 /* Get new done point */ 3595 if (++last == adapter->num_tx_desc) last = 0; 3596 done = last; 3597 } else 3598 break; 3599 } 3600 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3601 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3602 3603 txr->next_to_clean = first; 3604 3605 /* 3606 * If we have enough room, clear IFF_DRV_OACTIVE to 3607 * tell the stack that it is OK to send packets. 3608 * If there are no pending descriptors, clear the watchdog. 3609 */ 3610 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 3611 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3612 if (num_avail == adapter->num_tx_desc) { 3613 txr->watchdog_check = FALSE; 3614 txr->tx_avail = num_avail; 3615 return (FALSE); 3616 } 3617 } 3618 3619 txr->tx_avail = num_avail; 3620 return (TRUE); 3621 } 3622 3623 3624 /********************************************************************* 3625 * 3626 * Refresh RX descriptor mbufs from system mbuf buffer pool. 3627 * 3628 **********************************************************************/ 3629 static void 3630 em_refresh_mbufs(struct rx_ring *rxr, int limit) 3631 { 3632 struct adapter *adapter = rxr->adapter; 3633 struct mbuf *m; 3634 bus_dma_segment_t segs[1]; 3635 bus_dmamap_t map; 3636 struct em_buffer *rxbuf; 3637 int i, error, nsegs, cleaned; 3638 3639 i = rxr->next_to_refresh; 3640 cleaned = -1; 3641 while (i != limit) { 3642 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3643 if (m == NULL) 3644 goto update; 3645 m->m_len = m->m_pkthdr.len = MCLBYTES; 3646 3647 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 3648 m_adj(m, ETHER_ALIGN); 3649 3650 /* 3651 * Using memory from the mbuf cluster pool, invoke the 3652 * bus_dma machinery to arrange the memory mapping. 3653 */ 3654 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxr->rx_sparemap, 3655 m, segs, &nsegs, BUS_DMA_NOWAIT); 3656 if (error != 0) { 3657 m_free(m); 3658 goto update; 3659 } 3660 3661 /* If nsegs is wrong then the stack is corrupt. */ 3662 KASSERT(nsegs == 1, ("Too many segments returned!")); 3663 3664 rxbuf = &rxr->rx_buffers[i]; 3665 if (rxbuf->m_head != NULL) 3666 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3667 3668 map = rxbuf->map; 3669 rxbuf->map = rxr->rx_sparemap; 3670 rxr->rx_sparemap = map; 3671 bus_dmamap_sync(rxr->rxtag, 3672 rxbuf->map, BUS_DMASYNC_PREREAD); 3673 rxbuf->m_head = m; 3674 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); 3675 3676 cleaned = i; 3677 /* Calculate next index */ 3678 if (++i == adapter->num_rx_desc) 3679 i = 0; 3680 /* This is the work marker for refresh */ 3681 rxr->next_to_refresh = i; 3682 } 3683 update: 3684 if (cleaned != -1) /* Update tail index */ 3685 E1000_WRITE_REG(&adapter->hw, 3686 E1000_RDT(rxr->me), cleaned); 3687 3688 return; 3689 } 3690 3691 3692 /********************************************************************* 3693 * 3694 * Allocate memory for rx_buffer structures. Since we use one 3695 * rx_buffer per received packet, the maximum number of rx_buffer's 3696 * that we'll need is equal to the number of receive descriptors 3697 * that we've allocated. 3698 * 3699 **********************************************************************/ 3700 static int 3701 em_allocate_receive_buffers(struct rx_ring *rxr) 3702 { 3703 struct adapter *adapter = rxr->adapter; 3704 device_t dev = adapter->dev; 3705 struct em_buffer *rxbuf; 3706 int error; 3707 3708 rxr->rx_buffers = malloc(sizeof(struct em_buffer) * 3709 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3710 if (rxr->rx_buffers == NULL) { 3711 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3712 return (ENOMEM); 3713 } 3714 3715 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3716 1, 0, /* alignment, bounds */ 3717 BUS_SPACE_MAXADDR, /* lowaddr */ 3718 BUS_SPACE_MAXADDR, /* highaddr */ 3719 NULL, NULL, /* filter, filterarg */ 3720 MCLBYTES, /* maxsize */ 3721 1, /* nsegments */ 3722 MCLBYTES, /* maxsegsize */ 3723 0, /* flags */ 3724 NULL, /* lockfunc */ 3725 NULL, /* lockarg */ 3726 &rxr->rxtag); 3727 if (error) { 3728 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 3729 __func__, error); 3730 goto fail; 3731 } 3732 3733 /* Create the spare map (used by getbuf) */ 3734 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3735 &rxr->rx_sparemap); 3736 if (error) { 3737 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3738 __func__, error); 3739 goto fail; 3740 } 3741 3742 rxbuf = rxr->rx_buffers; 3743 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3744 rxbuf = &rxr->rx_buffers[i]; 3745 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3746 &rxbuf->map); 3747 if (error) { 3748 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3749 __func__, error); 3750 goto fail; 3751 } 3752 } 3753 3754 return (0); 3755 3756 fail: 3757 em_free_receive_structures(adapter); 3758 return (error); 3759 } 3760 3761 3762 /********************************************************************* 3763 * 3764 * Initialize a receive ring and its buffers. 3765 * 3766 **********************************************************************/ 3767 static int 3768 em_setup_receive_ring(struct rx_ring *rxr) 3769 { 3770 struct adapter *adapter = rxr->adapter; 3771 struct em_buffer *rxbuf; 3772 bus_dma_segment_t seg[1]; 3773 int rsize, nsegs, error; 3774 3775 3776 /* Clear the ring contents */ 3777 EM_RX_LOCK(rxr); 3778 rsize = roundup2(adapter->num_rx_desc * 3779 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 3780 bzero((void *)rxr->rx_base, rsize); 3781 3782 /* 3783 ** Free current RX buffer structs and their mbufs 3784 */ 3785 for (int i = 0; i < adapter->num_rx_desc; i++) { 3786 rxbuf = &rxr->rx_buffers[i]; 3787 if (rxbuf->m_head != NULL) { 3788 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3789 BUS_DMASYNC_POSTREAD); 3790 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3791 m_freem(rxbuf->m_head); 3792 } 3793 } 3794 3795 /* Now replenish the mbufs */ 3796 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3797 3798 rxbuf = &rxr->rx_buffers[j]; 3799 rxbuf->m_head = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3800 if (rxbuf->m_head == NULL) 3801 panic("RX ring hdr initialization failed!\n"); 3802 rxbuf->m_head->m_len = MCLBYTES; 3803 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ 3804 rxbuf->m_head->m_pkthdr.len = MCLBYTES; 3805 3806 /* Get the memory mapping */ 3807 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3808 rxbuf->map, rxbuf->m_head, seg, 3809 &nsegs, BUS_DMA_NOWAIT); 3810 if (error != 0) 3811 panic("RX ring dma initialization failed!\n"); 3812 bus_dmamap_sync(rxr->rxtag, 3813 rxbuf->map, BUS_DMASYNC_PREREAD); 3814 3815 /* Update descriptor */ 3816 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); 3817 } 3818 3819 3820 /* Setup our descriptor indices */ 3821 rxr->next_to_check = 0; 3822 rxr->next_to_refresh = 0; 3823 3824 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3825 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3826 3827 EM_RX_UNLOCK(rxr); 3828 return (0); 3829 } 3830 3831 /********************************************************************* 3832 * 3833 * Initialize all receive rings. 3834 * 3835 **********************************************************************/ 3836 static int 3837 em_setup_receive_structures(struct adapter *adapter) 3838 { 3839 struct rx_ring *rxr = adapter->rx_rings; 3840 int j; 3841 3842 for (j = 0; j < adapter->num_queues; j++, rxr++) 3843 if (em_setup_receive_ring(rxr)) 3844 goto fail; 3845 3846 return (0); 3847 fail: 3848 /* 3849 * Free RX buffers allocated so far, we will only handle 3850 * the rings that completed, the failing case will have 3851 * cleaned up for itself. 'j' failed, so its the terminus. 3852 */ 3853 for (int i = 0; i < j; ++i) { 3854 rxr = &adapter->rx_rings[i]; 3855 for (int n = 0; n < adapter->num_rx_desc; n++) { 3856 struct em_buffer *rxbuf; 3857 rxbuf = &rxr->rx_buffers[n]; 3858 if (rxbuf->m_head != NULL) { 3859 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3860 BUS_DMASYNC_POSTREAD); 3861 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3862 m_freem(rxbuf->m_head); 3863 rxbuf->m_head = NULL; 3864 } 3865 } 3866 } 3867 3868 return (ENOBUFS); 3869 } 3870 3871 /********************************************************************* 3872 * 3873 * Free all receive rings. 3874 * 3875 **********************************************************************/ 3876 static void 3877 em_free_receive_structures(struct adapter *adapter) 3878 { 3879 struct rx_ring *rxr = adapter->rx_rings; 3880 3881 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3882 em_free_receive_buffers(rxr); 3883 /* Free the ring memory as well */ 3884 em_dma_free(adapter, &rxr->rxdma); 3885 EM_RX_LOCK_DESTROY(rxr); 3886 } 3887 3888 free(adapter->rx_rings, M_DEVBUF); 3889 } 3890 3891 3892 /********************************************************************* 3893 * 3894 * Free receive ring data structures 3895 * 3896 **********************************************************************/ 3897 static void 3898 em_free_receive_buffers(struct rx_ring *rxr) 3899 { 3900 struct adapter *adapter = rxr->adapter; 3901 struct em_buffer *rxbuf = NULL; 3902 3903 INIT_DEBUGOUT("free_receive_buffers: begin"); 3904 3905 if (rxr->rx_sparemap) { 3906 bus_dmamap_destroy(rxr->rxtag, rxr->rx_sparemap); 3907 rxr->rx_sparemap = NULL; 3908 } 3909 3910 if (rxr->rx_buffers != NULL) { 3911 for (int i = 0; i < adapter->num_rx_desc; i++) { 3912 rxbuf = &rxr->rx_buffers[i]; 3913 if (rxbuf->map != NULL) { 3914 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3915 BUS_DMASYNC_POSTREAD); 3916 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3917 bus_dmamap_destroy(rxr->rxtag, rxbuf->map); 3918 } 3919 if (rxbuf->m_head != NULL) { 3920 m_freem(rxbuf->m_head); 3921 rxbuf->m_head = NULL; 3922 } 3923 } 3924 free(rxr->rx_buffers, M_DEVBUF); 3925 rxr->rx_buffers = NULL; 3926 } 3927 3928 if (rxr->rxtag != NULL) { 3929 bus_dma_tag_destroy(rxr->rxtag); 3930 rxr->rxtag = NULL; 3931 } 3932 3933 return; 3934 } 3935 3936 3937 /********************************************************************* 3938 * 3939 * Enable receive unit. 3940 * 3941 **********************************************************************/ 3942 #define MAX_INTS_PER_SEC 8000 3943 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 3944 3945 static void 3946 em_initialize_receive_unit(struct adapter *adapter) 3947 { 3948 struct rx_ring *rxr = adapter->rx_rings; 3949 struct ifnet *ifp = adapter->ifp; 3950 struct e1000_hw *hw = &adapter->hw; 3951 u64 bus_addr; 3952 u32 rctl, rxcsum; 3953 3954 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 3955 3956 /* 3957 * Make sure receives are disabled while setting 3958 * up the descriptor ring 3959 */ 3960 rctl = E1000_READ_REG(hw, E1000_RCTL); 3961 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3962 3963 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 3964 adapter->rx_abs_int_delay.value); 3965 /* 3966 * Set the interrupt throttling rate. Value is calculated 3967 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3968 */ 3969 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); 3970 3971 /* 3972 ** When using MSIX interrupts we need to throttle 3973 ** using the EITR register (82574 only) 3974 */ 3975 if (adapter->msix) 3976 for (int i = 0; i < 4; i++) 3977 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 3978 DEFAULT_ITR); 3979 3980 /* Disable accelerated ackknowledge */ 3981 if (adapter->hw.mac.type == e1000_82574) 3982 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); 3983 3984 if (ifp->if_capenable & IFCAP_RXCSUM) { 3985 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 3986 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 3987 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 3988 } 3989 3990 /* 3991 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 3992 ** long latencies are observed, like Lenovo X60. This 3993 ** change eliminates the problem, but since having positive 3994 ** values in RDTR is a known source of problems on other 3995 ** platforms another solution is being sought. 3996 */ 3997 if (hw->mac.type == e1000_82573) 3998 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 3999 4000 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4001 /* Setup the Base and Length of the Rx Descriptor Ring */ 4002 bus_addr = rxr->rxdma.dma_paddr; 4003 E1000_WRITE_REG(hw, E1000_RDLEN(i), 4004 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4005 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); 4006 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); 4007 /* Setup the Head and Tail Descriptor Pointers */ 4008 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4009 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1); 4010 } 4011 4012 /* Setup the Receive Control Register */ 4013 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4014 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 4015 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 4016 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4017 4018 /* Make sure VLAN Filters are off */ 4019 rctl &= ~E1000_RCTL_VFE; 4020 rctl &= ~E1000_RCTL_SBP; 4021 rctl |= E1000_RCTL_SZ_2048; 4022 if (ifp->if_mtu > ETHERMTU) 4023 rctl |= E1000_RCTL_LPE; 4024 else 4025 rctl &= ~E1000_RCTL_LPE; 4026 4027 /* Write out the settings */ 4028 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4029 4030 return; 4031 } 4032 4033 4034 /********************************************************************* 4035 * 4036 * This routine executes in interrupt context. It replenishes 4037 * the mbufs in the descriptor and sends data which has been 4038 * dma'ed into host memory to upper layer. 4039 * 4040 * We loop at most count times if count is > 0, or until done if 4041 * count < 0. 4042 * 4043 * For polling we also now return the number of cleaned packets 4044 *********************************************************************/ 4045 static int 4046 em_rxeof(struct rx_ring *rxr, int count) 4047 { 4048 struct adapter *adapter = rxr->adapter; 4049 struct ifnet *ifp = adapter->ifp;; 4050 struct mbuf *mp, *sendmp; 4051 u8 status; 4052 u16 len; 4053 int i, processed, rxdone = 0; 4054 bool eop; 4055 struct e1000_rx_desc *cur; 4056 4057 EM_RX_LOCK(rxr); 4058 4059 for (i = rxr->next_to_check, processed = 0; count != 0;) { 4060 4061 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4062 break; 4063 4064 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4065 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4066 4067 cur = &rxr->rx_base[i]; 4068 status = cur->status; 4069 mp = sendmp = NULL; 4070 4071 if ((status & E1000_RXD_STAT_DD) == 0) 4072 break; 4073 4074 len = le16toh(cur->length); 4075 eop = (status & E1000_RXD_STAT_EOP) != 0; 4076 count--; 4077 4078 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) == 0) { 4079 4080 /* Assign correct length to the current fragment */ 4081 mp = rxr->rx_buffers[i].m_head; 4082 mp->m_len = len; 4083 4084 if (rxr->fmp == NULL) { 4085 mp->m_pkthdr.len = len; 4086 rxr->fmp = mp; /* Store the first mbuf */ 4087 rxr->lmp = mp; 4088 } else { 4089 /* Chain mbuf's together */ 4090 mp->m_flags &= ~M_PKTHDR; 4091 rxr->lmp->m_next = mp; 4092 rxr->lmp = rxr->lmp->m_next; 4093 rxr->fmp->m_pkthdr.len += len; 4094 } 4095 4096 if (eop) { 4097 rxr->fmp->m_pkthdr.rcvif = ifp; 4098 ifp->if_ipackets++; 4099 em_receive_checksum(cur, rxr->fmp); 4100 #ifndef __NO_STRICT_ALIGNMENT 4101 if (adapter->max_frame_size > 4102 (MCLBYTES - ETHER_ALIGN) && 4103 em_fixup_rx(rxr) != 0) 4104 goto skip; 4105 #endif 4106 if (status & E1000_RXD_STAT_VP) { 4107 rxr->fmp->m_pkthdr.ether_vtag = 4108 (le16toh(cur->special) & 4109 E1000_RXD_SPC_VLAN_MASK); 4110 rxr->fmp->m_flags |= M_VLANTAG; 4111 } 4112 #ifndef __NO_STRICT_ALIGNMENT 4113 skip: 4114 #endif 4115 sendmp = rxr->fmp; 4116 rxr->fmp = NULL; 4117 rxr->lmp = NULL; 4118 } 4119 } else { 4120 ifp->if_ierrors++; 4121 /* Reuse loaded DMA map and just update mbuf chain */ 4122 mp = rxr->rx_buffers[i].m_head; 4123 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4124 mp->m_data = mp->m_ext.ext_buf; 4125 mp->m_next = NULL; 4126 if (adapter->max_frame_size <= 4127 (MCLBYTES - ETHER_ALIGN)) 4128 m_adj(mp, ETHER_ALIGN); 4129 if (rxr->fmp != NULL) { 4130 m_freem(rxr->fmp); 4131 rxr->fmp = NULL; 4132 rxr->lmp = NULL; 4133 } 4134 sendmp = NULL; 4135 } 4136 4137 /* Zero out the receive descriptors status. */ 4138 cur->status = 0; 4139 ++rxdone; /* cumulative for POLL */ 4140 ++processed; 4141 4142 /* Advance our pointers to the next descriptor. */ 4143 if (++i == adapter->num_rx_desc) 4144 i = 0; 4145 4146 /* Send to the stack */ 4147 if (sendmp != NULL) 4148 (*ifp->if_input)(ifp, sendmp); 4149 4150 /* Only refresh mbufs every 8 descriptors */ 4151 if (processed == 8) { 4152 em_refresh_mbufs(rxr, i); 4153 processed = 0; 4154 } 4155 } 4156 4157 /* Catch any remaining refresh work */ 4158 if (processed != 0) { 4159 em_refresh_mbufs(rxr, i); 4160 processed = 0; 4161 } 4162 4163 rxr->next_to_check = i; 4164 4165 EM_RX_UNLOCK(rxr); 4166 return (rxdone); 4167 } 4168 4169 #ifndef __NO_STRICT_ALIGNMENT 4170 /* 4171 * When jumbo frames are enabled we should realign entire payload on 4172 * architecures with strict alignment. This is serious design mistake of 8254x 4173 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4174 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4175 * payload. On architecures without strict alignment restrictions 8254x still 4176 * performs unaligned memory access which would reduce the performance too. 4177 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4178 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4179 * existing mbuf chain. 4180 * 4181 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4182 * not used at all on architectures with strict alignment. 4183 */ 4184 static int 4185 em_fixup_rx(struct rx_ring *rxr) 4186 { 4187 struct adapter *adapter = rxr->adapter; 4188 struct mbuf *m, *n; 4189 int error; 4190 4191 error = 0; 4192 m = rxr->fmp; 4193 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4194 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4195 m->m_data += ETHER_HDR_LEN; 4196 } else { 4197 MGETHDR(n, M_DONTWAIT, MT_DATA); 4198 if (n != NULL) { 4199 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4200 m->m_data += ETHER_HDR_LEN; 4201 m->m_len -= ETHER_HDR_LEN; 4202 n->m_len = ETHER_HDR_LEN; 4203 M_MOVE_PKTHDR(n, m); 4204 n->m_next = m; 4205 rxr->fmp = n; 4206 } else { 4207 adapter->dropped_pkts++; 4208 m_freem(rxr->fmp); 4209 rxr->fmp = NULL; 4210 error = ENOMEM; 4211 } 4212 } 4213 4214 return (error); 4215 } 4216 #endif 4217 4218 /********************************************************************* 4219 * 4220 * Verify that the hardware indicated that the checksum is valid. 4221 * Inform the stack about the status of checksum so that stack 4222 * doesn't spend time verifying the checksum. 4223 * 4224 *********************************************************************/ 4225 static void 4226 em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4227 { 4228 /* Ignore Checksum bit is set */ 4229 if (rx_desc->status & E1000_RXD_STAT_IXSM) { 4230 mp->m_pkthdr.csum_flags = 0; 4231 return; 4232 } 4233 4234 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4235 /* Did it pass? */ 4236 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4237 /* IP Checksum Good */ 4238 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4239 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4240 4241 } else { 4242 mp->m_pkthdr.csum_flags = 0; 4243 } 4244 } 4245 4246 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4247 /* Did it pass? */ 4248 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4249 mp->m_pkthdr.csum_flags |= 4250 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4251 mp->m_pkthdr.csum_data = htons(0xffff); 4252 } 4253 } 4254 } 4255 4256 /* 4257 * This routine is run via an vlan 4258 * config EVENT 4259 */ 4260 static void 4261 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4262 { 4263 struct adapter *adapter = ifp->if_softc; 4264 u32 index, bit; 4265 4266 if (ifp->if_softc != arg) /* Not our event */ 4267 return; 4268 4269 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ 4270 return; 4271 4272 index = (vtag >> 5) & 0x7F; 4273 bit = vtag & 0x1F; 4274 em_shadow_vfta[index] |= (1 << bit); 4275 ++adapter->num_vlans; 4276 /* Re-init to load the changes */ 4277 em_init(adapter); 4278 } 4279 4280 /* 4281 * This routine is run via an vlan 4282 * unconfig EVENT 4283 */ 4284 static void 4285 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4286 { 4287 struct adapter *adapter = ifp->if_softc; 4288 u32 index, bit; 4289 4290 if (ifp->if_softc != arg) 4291 return; 4292 4293 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4294 return; 4295 4296 index = (vtag >> 5) & 0x7F; 4297 bit = vtag & 0x1F; 4298 em_shadow_vfta[index] &= ~(1 << bit); 4299 --adapter->num_vlans; 4300 /* Re-init to load the changes */ 4301 em_init(adapter); 4302 } 4303 4304 static void 4305 em_setup_vlan_hw_support(struct adapter *adapter) 4306 { 4307 struct e1000_hw *hw = &adapter->hw; 4308 u32 reg; 4309 4310 /* 4311 ** We get here thru init_locked, meaning 4312 ** a soft reset, this has already cleared 4313 ** the VFTA and other state, so if there 4314 ** have been no vlan's registered do nothing. 4315 */ 4316 if (adapter->num_vlans == 0) 4317 return; 4318 4319 /* 4320 ** A soft reset zero's out the VFTA, so 4321 ** we need to repopulate it now. 4322 */ 4323 for (int i = 0; i < EM_VFTA_SIZE; i++) 4324 if (em_shadow_vfta[i] != 0) 4325 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4326 i, em_shadow_vfta[i]); 4327 4328 reg = E1000_READ_REG(hw, E1000_CTRL); 4329 reg |= E1000_CTRL_VME; 4330 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4331 4332 /* Enable the Filter Table */ 4333 reg = E1000_READ_REG(hw, E1000_RCTL); 4334 reg &= ~E1000_RCTL_CFIEN; 4335 reg |= E1000_RCTL_VFE; 4336 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4337 4338 /* Update the frame size */ 4339 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4340 adapter->max_frame_size + VLAN_TAG_SIZE); 4341 } 4342 4343 static void 4344 em_enable_intr(struct adapter *adapter) 4345 { 4346 struct e1000_hw *hw = &adapter->hw; 4347 u32 ims_mask = IMS_ENABLE_MASK; 4348 4349 if (adapter->msix) { 4350 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4351 ims_mask |= EM_MSIX_MASK; 4352 } 4353 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4354 } 4355 4356 static void 4357 em_disable_intr(struct adapter *adapter) 4358 { 4359 struct e1000_hw *hw = &adapter->hw; 4360 4361 if (adapter->msix) 4362 E1000_WRITE_REG(hw, EM_EIAC, 0); 4363 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4364 } 4365 4366 /* 4367 * Bit of a misnomer, what this really means is 4368 * to enable OS management of the system... aka 4369 * to disable special hardware management features 4370 */ 4371 static void 4372 em_init_manageability(struct adapter *adapter) 4373 { 4374 /* A shared code workaround */ 4375 #define E1000_82542_MANC2H E1000_MANC2H 4376 if (adapter->has_manage) { 4377 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4378 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4379 4380 /* disable hardware interception of ARP */ 4381 manc &= ~(E1000_MANC_ARP_EN); 4382 4383 /* enable receiving management packets to the host */ 4384 manc |= E1000_MANC_EN_MNG2HOST; 4385 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4386 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4387 manc2h |= E1000_MNG2HOST_PORT_623; 4388 manc2h |= E1000_MNG2HOST_PORT_664; 4389 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4390 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4391 } 4392 } 4393 4394 /* 4395 * Give control back to hardware management 4396 * controller if there is one. 4397 */ 4398 static void 4399 em_release_manageability(struct adapter *adapter) 4400 { 4401 if (adapter->has_manage) { 4402 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4403 4404 /* re-enable hardware interception of ARP */ 4405 manc |= E1000_MANC_ARP_EN; 4406 manc &= ~E1000_MANC_EN_MNG2HOST; 4407 4408 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4409 } 4410 } 4411 4412 /* 4413 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 4414 * For ASF and Pass Through versions of f/w this means 4415 * that the driver is loaded. For AMT version type f/w 4416 * this means that the network i/f is open. 4417 */ 4418 static void 4419 em_get_hw_control(struct adapter *adapter) 4420 { 4421 u32 ctrl_ext, swsm; 4422 4423 if (adapter->hw.mac.type == e1000_82573) { 4424 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4425 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4426 swsm | E1000_SWSM_DRV_LOAD); 4427 return; 4428 } 4429 /* else */ 4430 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4431 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4432 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4433 return; 4434 } 4435 4436 /* 4437 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4438 * For ASF and Pass Through versions of f/w this means that 4439 * the driver is no longer loaded. For AMT versions of the 4440 * f/w this means that the network i/f is closed. 4441 */ 4442 static void 4443 em_release_hw_control(struct adapter *adapter) 4444 { 4445 u32 ctrl_ext, swsm; 4446 4447 if (!adapter->has_manage) 4448 return; 4449 4450 if (adapter->hw.mac.type == e1000_82573) { 4451 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4452 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4453 swsm & ~E1000_SWSM_DRV_LOAD); 4454 return; 4455 } 4456 /* else */ 4457 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4458 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4459 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4460 return; 4461 } 4462 4463 static int 4464 em_is_valid_ether_addr(u8 *addr) 4465 { 4466 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4467 4468 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4469 return (FALSE); 4470 } 4471 4472 return (TRUE); 4473 } 4474 4475 /* 4476 ** Parse the interface capabilities with regard 4477 ** to both system management and wake-on-lan for 4478 ** later use. 4479 */ 4480 static void 4481 em_get_wakeup(device_t dev) 4482 { 4483 struct adapter *adapter = device_get_softc(dev); 4484 u16 eeprom_data = 0, device_id, apme_mask; 4485 4486 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 4487 apme_mask = EM_EEPROM_APME; 4488 4489 switch (adapter->hw.mac.type) { 4490 case e1000_82573: 4491 case e1000_82583: 4492 adapter->has_amt = TRUE; 4493 /* Falls thru */ 4494 case e1000_82571: 4495 case e1000_82572: 4496 case e1000_80003es2lan: 4497 if (adapter->hw.bus.func == 1) { 4498 e1000_read_nvm(&adapter->hw, 4499 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 4500 break; 4501 } else 4502 e1000_read_nvm(&adapter->hw, 4503 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4504 break; 4505 case e1000_ich8lan: 4506 case e1000_ich9lan: 4507 case e1000_ich10lan: 4508 case e1000_pchlan: 4509 apme_mask = E1000_WUC_APME; 4510 adapter->has_amt = TRUE; 4511 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 4512 break; 4513 default: 4514 e1000_read_nvm(&adapter->hw, 4515 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4516 break; 4517 } 4518 if (eeprom_data & apme_mask) 4519 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 4520 /* 4521 * We have the eeprom settings, now apply the special cases 4522 * where the eeprom may be wrong or the board won't support 4523 * wake on lan on a particular port 4524 */ 4525 device_id = pci_get_device(dev); 4526 switch (device_id) { 4527 case E1000_DEV_ID_82571EB_FIBER: 4528 /* Wake events only supported on port A for dual fiber 4529 * regardless of eeprom setting */ 4530 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 4531 E1000_STATUS_FUNC_1) 4532 adapter->wol = 0; 4533 break; 4534 case E1000_DEV_ID_82571EB_QUAD_COPPER: 4535 case E1000_DEV_ID_82571EB_QUAD_FIBER: 4536 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 4537 /* if quad port adapter, disable WoL on all but port A */ 4538 if (global_quad_port_a != 0) 4539 adapter->wol = 0; 4540 /* Reset for multiple quad port adapters */ 4541 if (++global_quad_port_a == 4) 4542 global_quad_port_a = 0; 4543 break; 4544 } 4545 return; 4546 } 4547 4548 4549 /* 4550 * Enable PCI Wake On Lan capability 4551 */ 4552 static void 4553 em_enable_wakeup(device_t dev) 4554 { 4555 struct adapter *adapter = device_get_softc(dev); 4556 struct ifnet *ifp = adapter->ifp; 4557 u32 pmc, ctrl, ctrl_ext, rctl; 4558 u16 status; 4559 4560 if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) 4561 return; 4562 4563 /* Advertise the wakeup capability */ 4564 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4565 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 4566 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4567 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4568 4569 if ((adapter->hw.mac.type == e1000_ich8lan) || 4570 (adapter->hw.mac.type == e1000_pchlan) || 4571 (adapter->hw.mac.type == e1000_ich9lan) || 4572 (adapter->hw.mac.type == e1000_ich10lan)) { 4573 e1000_disable_gig_wol_ich8lan(&adapter->hw); 4574 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); 4575 } 4576 4577 /* Keep the laser running on Fiber adapters */ 4578 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 4579 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 4580 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4581 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 4582 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); 4583 } 4584 4585 /* 4586 ** Determine type of Wakeup: note that wol 4587 ** is set with all bits on by default. 4588 */ 4589 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) 4590 adapter->wol &= ~E1000_WUFC_MAG; 4591 4592 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) 4593 adapter->wol &= ~E1000_WUFC_MC; 4594 else { 4595 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4596 rctl |= E1000_RCTL_MPE; 4597 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4598 } 4599 4600 if (adapter->hw.mac.type == e1000_pchlan) { 4601 if (em_enable_phy_wakeup(adapter)) 4602 return; 4603 } else { 4604 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4605 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 4606 } 4607 4608 if (adapter->hw.phy.type == e1000_phy_igp_3) 4609 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 4610 4611 /* Request PME */ 4612 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); 4613 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 4614 if (ifp->if_capenable & IFCAP_WOL) 4615 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4616 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); 4617 4618 return; 4619 } 4620 4621 /* 4622 ** WOL in the newer chipset interfaces (pchlan) 4623 ** require thing to be copied into the phy 4624 */ 4625 static int 4626 em_enable_phy_wakeup(struct adapter *adapter) 4627 { 4628 struct e1000_hw *hw = &adapter->hw; 4629 u32 mreg, ret = 0; 4630 u16 preg; 4631 4632 /* copy MAC RARs to PHY RARs */ 4633 for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { 4634 mreg = E1000_READ_REG(hw, E1000_RAL(i)); 4635 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); 4636 e1000_write_phy_reg(hw, BM_RAR_M(i), 4637 (u16)((mreg >> 16) & 0xFFFF)); 4638 mreg = E1000_READ_REG(hw, E1000_RAH(i)); 4639 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); 4640 e1000_write_phy_reg(hw, BM_RAR_CTRL(i), 4641 (u16)((mreg >> 16) & 0xFFFF)); 4642 } 4643 4644 /* copy MAC MTA to PHY MTA */ 4645 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 4646 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 4647 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 4648 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 4649 (u16)((mreg >> 16) & 0xFFFF)); 4650 } 4651 4652 /* configure PHY Rx Control register */ 4653 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); 4654 mreg = E1000_READ_REG(hw, E1000_RCTL); 4655 if (mreg & E1000_RCTL_UPE) 4656 preg |= BM_RCTL_UPE; 4657 if (mreg & E1000_RCTL_MPE) 4658 preg |= BM_RCTL_MPE; 4659 preg &= ~(BM_RCTL_MO_MASK); 4660 if (mreg & E1000_RCTL_MO_3) 4661 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 4662 << BM_RCTL_MO_SHIFT); 4663 if (mreg & E1000_RCTL_BAM) 4664 preg |= BM_RCTL_BAM; 4665 if (mreg & E1000_RCTL_PMCF) 4666 preg |= BM_RCTL_PMCF; 4667 mreg = E1000_READ_REG(hw, E1000_CTRL); 4668 if (mreg & E1000_CTRL_RFCE) 4669 preg |= BM_RCTL_RFCE; 4670 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); 4671 4672 /* enable PHY wakeup in MAC register */ 4673 E1000_WRITE_REG(hw, E1000_WUC, 4674 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); 4675 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); 4676 4677 /* configure and enable PHY wakeup in PHY registers */ 4678 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); 4679 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); 4680 4681 /* activate PHY wakeup */ 4682 ret = hw->phy.ops.acquire(hw); 4683 if (ret) { 4684 printf("Could not acquire PHY\n"); 4685 return ret; 4686 } 4687 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4688 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 4689 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 4690 if (ret) { 4691 printf("Could not read PHY page 769\n"); 4692 goto out; 4693 } 4694 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 4695 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 4696 if (ret) 4697 printf("Could not set PHY Host Wakeup bit\n"); 4698 out: 4699 hw->phy.ops.release(hw); 4700 4701 return ret; 4702 } 4703 4704 static void 4705 em_led_func(void *arg, int onoff) 4706 { 4707 struct adapter *adapter = arg; 4708 4709 EM_CORE_LOCK(adapter); 4710 if (onoff) { 4711 e1000_setup_led(&adapter->hw); 4712 e1000_led_on(&adapter->hw); 4713 } else { 4714 e1000_led_off(&adapter->hw); 4715 e1000_cleanup_led(&adapter->hw); 4716 } 4717 EM_CORE_UNLOCK(adapter); 4718 } 4719 4720 /********************************************************************** 4721 * 4722 * Update the board statistics counters. 4723 * 4724 **********************************************************************/ 4725 static void 4726 em_update_stats_counters(struct adapter *adapter) 4727 { 4728 struct ifnet *ifp; 4729 4730 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4731 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4732 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4733 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4734 } 4735 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4736 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4737 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4738 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4739 4740 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4741 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4742 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4743 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4744 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4745 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4746 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4747 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4748 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4749 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4750 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4751 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4752 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4753 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4754 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4755 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4756 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4757 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4758 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4759 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4760 4761 /* For the 64-bit byte counters the low dword must be read first. */ 4762 /* Both registers clear on the read of the high dword */ 4763 4764 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4765 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4766 4767 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4768 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4769 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4770 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4771 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4772 4773 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4774 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4775 4776 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4777 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4778 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4779 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4780 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4781 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4782 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4783 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4784 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4785 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4786 4787 if (adapter->hw.mac.type >= e1000_82543) { 4788 adapter->stats.algnerrc += 4789 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4790 adapter->stats.rxerrc += 4791 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4792 adapter->stats.tncrs += 4793 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4794 adapter->stats.cexterr += 4795 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4796 adapter->stats.tsctc += 4797 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4798 adapter->stats.tsctfc += 4799 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4800 } 4801 ifp = adapter->ifp; 4802 4803 ifp->if_collisions = adapter->stats.colc; 4804 4805 /* Rx Errors */ 4806 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4807 adapter->stats.crcerrs + adapter->stats.algnerrc + 4808 adapter->stats.ruc + adapter->stats.roc + 4809 adapter->stats.mpc + adapter->stats.cexterr; 4810 4811 /* Tx Errors */ 4812 ifp->if_oerrors = adapter->stats.ecol + 4813 adapter->stats.latecol + adapter->watchdog_events; 4814 } 4815 4816 4817 /********************************************************************** 4818 * 4819 * This routine is called only when em_display_debug_stats is enabled. 4820 * This routine provides a way to take a look at important statistics 4821 * maintained by the driver and hardware. 4822 * 4823 **********************************************************************/ 4824 static void 4825 em_print_debug_info(struct adapter *adapter) 4826 { 4827 device_t dev = adapter->dev; 4828 u8 *hw_addr = adapter->hw.hw_addr; 4829 struct rx_ring *rxr = adapter->rx_rings; 4830 struct tx_ring *txr = adapter->tx_rings; 4831 4832 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4833 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4834 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4835 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4836 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4837 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4838 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4839 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4840 adapter->hw.fc.high_water, 4841 adapter->hw.fc.low_water); 4842 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 4843 E1000_READ_REG(&adapter->hw, E1000_TIDV), 4844 E1000_READ_REG(&adapter->hw, E1000_TADV)); 4845 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 4846 E1000_READ_REG(&adapter->hw, E1000_RDTR), 4847 E1000_READ_REG(&adapter->hw, E1000_RADV)); 4848 4849 for (int i = 0; i < adapter->num_queues; i++, txr++) { 4850 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i, 4851 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4852 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4853 device_printf(dev, "TX(%d) no descriptors avail event = %ld\n", 4854 txr->me, txr->no_desc_avail); 4855 device_printf(dev, "TX(%d) MSIX IRQ Handled = %ld\n", 4856 txr->me, txr->tx_irq); 4857 device_printf(dev, "Num Tx descriptors avail = %d\n", 4858 txr->tx_avail); 4859 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 4860 txr->no_desc_avail); 4861 } 4862 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4863 device_printf(dev, "RX(%d) MSIX IRQ Handled = %ld\n", 4864 rxr->me, rxr->rx_irq); 4865 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 4866 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4867 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4868 } 4869 device_printf(dev, "Std mbuf failed = %ld\n", 4870 adapter->mbuf_alloc_failed); 4871 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4872 adapter->mbuf_cluster_failed); 4873 device_printf(dev, "Driver dropped packets = %ld\n", 4874 adapter->dropped_pkts); 4875 } 4876 4877 static void 4878 em_print_hw_stats(struct adapter *adapter) 4879 { 4880 device_t dev = adapter->dev; 4881 4882 device_printf(dev, "Excessive collisions = %lld\n", 4883 (long long)adapter->stats.ecol); 4884 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 4885 device_printf(dev, "Symbol errors = %lld\n", 4886 (long long)adapter->stats.symerrs); 4887 #endif 4888 device_printf(dev, "Sequence errors = %lld\n", 4889 (long long)adapter->stats.sec); 4890 device_printf(dev, "Defer count = %lld\n", 4891 (long long)adapter->stats.dc); 4892 device_printf(dev, "Missed Packets = %lld\n", 4893 (long long)adapter->stats.mpc); 4894 device_printf(dev, "Receive No Buffers = %lld\n", 4895 (long long)adapter->stats.rnbc); 4896 /* RLEC is inaccurate on some hardware, calculate our own. */ 4897 device_printf(dev, "Receive Length Errors = %lld\n", 4898 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4899 device_printf(dev, "Receive errors = %lld\n", 4900 (long long)adapter->stats.rxerrc); 4901 device_printf(dev, "Crc errors = %lld\n", 4902 (long long)adapter->stats.crcerrs); 4903 device_printf(dev, "Alignment errors = %lld\n", 4904 (long long)adapter->stats.algnerrc); 4905 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4906 (long long)adapter->stats.cexterr); 4907 device_printf(dev, "watchdog timeouts = %ld\n", 4908 adapter->watchdog_events); 4909 device_printf(dev, "XON Rcvd = %lld\n", 4910 (long long)adapter->stats.xonrxc); 4911 device_printf(dev, "XON Xmtd = %lld\n", 4912 (long long)adapter->stats.xontxc); 4913 device_printf(dev, "XOFF Rcvd = %lld\n", 4914 (long long)adapter->stats.xoffrxc); 4915 device_printf(dev, "XOFF Xmtd = %lld\n", 4916 (long long)adapter->stats.xofftxc); 4917 device_printf(dev, "Good Packets Rcvd = %lld\n", 4918 (long long)adapter->stats.gprc); 4919 device_printf(dev, "Good Packets Xmtd = %lld\n", 4920 (long long)adapter->stats.gptc); 4921 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4922 (long long)adapter->stats.tsctc); 4923 device_printf(dev, "TSO Contexts Failed = %lld\n", 4924 (long long)adapter->stats.tsctfc); 4925 } 4926 4927 /********************************************************************** 4928 * 4929 * This routine provides a way to dump out the adapter eeprom, 4930 * often a useful debug/service tool. This only dumps the first 4931 * 32 words, stuff that matters is in that extent. 4932 * 4933 **********************************************************************/ 4934 static void 4935 em_print_nvm_info(struct adapter *adapter) 4936 { 4937 u16 eeprom_data; 4938 int i, j, row = 0; 4939 4940 /* Its a bit crude, but it gets the job done */ 4941 printf("\nInterface EEPROM Dump:\n"); 4942 printf("Offset\n0x0000 "); 4943 for (i = 0, j = 0; i < 32; i++, j++) { 4944 if (j == 8) { /* Make the offset block */ 4945 j = 0; ++row; 4946 printf("\n0x00%x0 ",row); 4947 } 4948 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4949 printf("%04x ", eeprom_data); 4950 } 4951 printf("\n"); 4952 } 4953 4954 static int 4955 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4956 { 4957 struct adapter *adapter; 4958 int error; 4959 int result; 4960 4961 result = -1; 4962 error = sysctl_handle_int(oidp, &result, 0, req); 4963 4964 if (error || !req->newptr) 4965 return (error); 4966 4967 if (result == 1) { 4968 adapter = (struct adapter *)arg1; 4969 em_print_debug_info(adapter); 4970 } 4971 /* 4972 * This value will cause a hex dump of the 4973 * first 32 16-bit words of the EEPROM to 4974 * the screen. 4975 */ 4976 if (result == 2) { 4977 adapter = (struct adapter *)arg1; 4978 em_print_nvm_info(adapter); 4979 } 4980 4981 return (error); 4982 } 4983 4984 4985 static int 4986 em_sysctl_stats(SYSCTL_HANDLER_ARGS) 4987 { 4988 struct adapter *adapter; 4989 int error; 4990 int result; 4991 4992 result = -1; 4993 error = sysctl_handle_int(oidp, &result, 0, req); 4994 4995 if (error || !req->newptr) 4996 return (error); 4997 4998 if (result == 1) { 4999 adapter = (struct adapter *)arg1; 5000 em_print_hw_stats(adapter); 5001 } 5002 5003 return (error); 5004 } 5005 5006 static int 5007 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5008 { 5009 struct em_int_delay_info *info; 5010 struct adapter *adapter; 5011 u32 regval; 5012 int error, usecs, ticks; 5013 5014 info = (struct em_int_delay_info *)arg1; 5015 usecs = info->value; 5016 error = sysctl_handle_int(oidp, &usecs, 0, req); 5017 if (error != 0 || req->newptr == NULL) 5018 return (error); 5019 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5020 return (EINVAL); 5021 info->value = usecs; 5022 ticks = EM_USECS_TO_TICKS(usecs); 5023 5024 adapter = info->adapter; 5025 5026 EM_CORE_LOCK(adapter); 5027 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5028 regval = (regval & ~0xffff) | (ticks & 0xffff); 5029 /* Handle a few special cases. */ 5030 switch (info->offset) { 5031 case E1000_RDTR: 5032 break; 5033 case E1000_TIDV: 5034 if (ticks == 0) { 5035 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5036 /* Don't write 0 into the TIDV register. */ 5037 regval++; 5038 } else 5039 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5040 break; 5041 } 5042 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5043 EM_CORE_UNLOCK(adapter); 5044 return (0); 5045 } 5046 5047 static void 5048 em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5049 const char *description, struct em_int_delay_info *info, 5050 int offset, int value) 5051 { 5052 info->adapter = adapter; 5053 info->offset = offset; 5054 info->value = value; 5055 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5056 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5057 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5058 info, 0, em_sysctl_int_delay, "I", description); 5059 } 5060 5061 static void 5062 em_add_rx_process_limit(struct adapter *adapter, const char *name, 5063 const char *description, int *limit, int value) 5064 { 5065 *limit = value; 5066 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5067 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5068 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5069 } 5070 5071 5072