1 /****************************************************************************** 2 3 Copyright (c) 2001-2009, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/buf_ring.h> 43 #include <sys/bus.h> 44 #include <sys/endian.h> 45 #include <sys/kernel.h> 46 #include <sys/kthread.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/module.h> 50 #include <sys/rman.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <sys/taskqueue.h> 55 #include <sys/eventhandler.h> 56 #ifdef EM_TIMESYNC 57 #include <sys/ioccom.h> 58 #include <sys/time.h> 59 #endif 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <net/bpf.h> 64 #include <net/ethernet.h> 65 #include <net/if.h> 66 #include <net/if_arp.h> 67 #include <net/if_dl.h> 68 #include <net/if_media.h> 69 70 #include <net/if_types.h> 71 #include <net/if_vlan_var.h> 72 73 #include <netinet/in_systm.h> 74 #include <netinet/in.h> 75 #include <netinet/if_ether.h> 76 #include <netinet/ip.h> 77 #include <netinet/ip6.h> 78 #include <netinet/tcp.h> 79 #include <netinet/udp.h> 80 81 #include <machine/in_cksum.h> 82 #include <dev/pci/pcivar.h> 83 #include <dev/pci/pcireg.h> 84 85 #include "e1000_api.h" 86 #include "e1000_82571.h" 87 #include "if_em.h" 88 89 /********************************************************************* 90 * Set this to one to display debug statistics 91 *********************************************************************/ 92 int em_display_debug_stats = 0; 93 94 /********************************************************************* 95 * Driver version: 96 *********************************************************************/ 97 char em_driver_version[] = "6.9.9"; 98 99 100 /********************************************************************* 101 * PCI Device ID Table 102 * 103 * Used by probe to select devices to load on 104 * Last field stores an index into e1000_strings 105 * Last entry must be all 0s 106 * 107 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 108 *********************************************************************/ 109 110 static em_vendor_info_t em_vendor_info_array[] = 111 { 112 /* Intel(R) PRO/1000 Network Connection */ 113 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 119 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 127 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 129 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 132 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 137 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 138 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 142 143 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 152 PCI_ANY_ID, PCI_ANY_ID, 0}, 153 154 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 158 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 162 PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 164 PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 166 PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 168 PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 170 PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 172 PCI_ANY_ID, PCI_ANY_ID, 0}, 173 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 174 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 175 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 176 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 177 178 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 179 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 180 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 181 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 182 PCI_ANY_ID, PCI_ANY_ID, 0}, 183 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 184 PCI_ANY_ID, PCI_ANY_ID, 0}, 185 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 186 PCI_ANY_ID, PCI_ANY_ID, 0}, 187 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 188 PCI_ANY_ID, PCI_ANY_ID, 0}, 189 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 190 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 191 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 192 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 193 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 194 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 195 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 196 197 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 198 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 199 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 200 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 201 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 202 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 203 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 204 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 205 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 206 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 207 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 208 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 209 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 210 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 211 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 212 /* required last entry */ 213 { 0, 0, 0, 0, 0} 214 }; 215 216 /********************************************************************* 217 * Table of branding strings for all supported NICs. 218 *********************************************************************/ 219 220 static char *em_strings[] = { 221 "Intel(R) PRO/1000 Network Connection" 222 }; 223 224 /********************************************************************* 225 * Function prototypes 226 *********************************************************************/ 227 static int em_probe(device_t); 228 static int em_attach(device_t); 229 static int em_detach(device_t); 230 static int em_shutdown(device_t); 231 static int em_suspend(device_t); 232 static int em_resume(device_t); 233 static void em_start(struct ifnet *); 234 static void em_start_locked(struct ifnet *ifp); 235 static int em_ioctl(struct ifnet *, u_long, caddr_t); 236 static void em_watchdog(struct adapter *); 237 static void em_init(void *); 238 static void em_init_locked(struct adapter *); 239 static void em_stop(void *); 240 static void em_media_status(struct ifnet *, struct ifmediareq *); 241 static int em_media_change(struct ifnet *); 242 static void em_identify_hardware(struct adapter *); 243 static int em_allocate_pci_resources(struct adapter *); 244 static int em_allocate_legacy(struct adapter *adapter); 245 static int em_allocate_msix(struct adapter *adapter); 246 static int em_setup_msix(struct adapter *); 247 static void em_free_pci_resources(struct adapter *); 248 static void em_local_timer(void *); 249 static int em_hardware_init(struct adapter *); 250 static void em_setup_interface(device_t, struct adapter *); 251 static void em_setup_transmit_structures(struct adapter *); 252 static void em_initialize_transmit_unit(struct adapter *); 253 static int em_setup_receive_structures(struct adapter *); 254 static void em_initialize_receive_unit(struct adapter *); 255 static void em_enable_intr(struct adapter *); 256 static void em_disable_intr(struct adapter *); 257 static void em_free_transmit_structures(struct adapter *); 258 static void em_free_receive_structures(struct adapter *); 259 static void em_update_stats_counters(struct adapter *); 260 static void em_txeof(struct adapter *); 261 static void em_tx_purge(struct adapter *); 262 static int em_allocate_receive_structures(struct adapter *); 263 static int em_allocate_transmit_structures(struct adapter *); 264 static int em_rxeof(struct adapter *, int); 265 #ifndef __NO_STRICT_ALIGNMENT 266 static int em_fixup_rx(struct adapter *); 267 #endif 268 static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, 269 struct mbuf *); 270 static void em_transmit_checksum_setup(struct adapter *, struct mbuf *, 271 u32 *, u32 *); 272 #if __FreeBSD_version >= 700000 273 static bool em_tso_setup(struct adapter *, struct mbuf *, 274 u32 *, u32 *); 275 #endif /* FreeBSD_version >= 700000 */ 276 static void em_set_promisc(struct adapter *); 277 static void em_disable_promisc(struct adapter *); 278 static void em_set_multi(struct adapter *); 279 static void em_print_hw_stats(struct adapter *); 280 static void em_update_link_status(struct adapter *); 281 static int em_get_buf(struct adapter *, int); 282 static void em_register_vlan(void *, struct ifnet *, u16); 283 static void em_unregister_vlan(void *, struct ifnet *, u16); 284 static int em_xmit(struct adapter *, struct mbuf **); 285 static void em_smartspeed(struct adapter *); 286 static int em_82547_fifo_workaround(struct adapter *, int); 287 static void em_82547_update_fifo_head(struct adapter *, int); 288 static int em_82547_tx_fifo_reset(struct adapter *); 289 static void em_82547_move_tail(void *); 290 static int em_dma_malloc(struct adapter *, bus_size_t, 291 struct em_dma_alloc *, int); 292 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 293 static void em_print_debug_info(struct adapter *); 294 static void em_print_nvm_info(struct adapter *); 295 static int em_is_valid_ether_addr(u8 *); 296 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 297 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 298 static u32 em_fill_descriptors (bus_addr_t address, u32 length, 299 PDESC_ARRAY desc_array); 300 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 301 static void em_add_int_delay_sysctl(struct adapter *, const char *, 302 const char *, struct em_int_delay_info *, int, int); 303 /* Management and WOL Support */ 304 static void em_init_manageability(struct adapter *); 305 static void em_release_manageability(struct adapter *); 306 static void em_get_hw_control(struct adapter *); 307 static void em_release_hw_control(struct adapter *); 308 static void em_enable_wakeup(device_t); 309 310 #ifdef EM_TIMESYNC 311 /* Precision Time sync support */ 312 static int em_tsync_init(struct adapter *); 313 static void em_tsync_disable(struct adapter *); 314 #endif 315 316 #ifdef EM_LEGACY_IRQ 317 static void em_intr(void *); 318 #else /* FAST IRQ */ 319 #if __FreeBSD_version < 700000 320 static void em_irq_fast(void *); 321 #else 322 static int em_irq_fast(void *); 323 #endif 324 325 /* MSIX handlers */ 326 static void em_msix_tx(void *); 327 static void em_msix_rx(void *); 328 static void em_msix_link(void *); 329 static void em_handle_rx(void *context, int pending); 330 static void em_handle_tx(void *context, int pending); 331 332 static void em_handle_rxtx(void *context, int pending); 333 static void em_handle_link(void *context, int pending); 334 static void em_add_rx_process_limit(struct adapter *, const char *, 335 const char *, int *, int); 336 #endif /* ~EM_LEGACY_IRQ */ 337 338 #ifdef DEVICE_POLLING 339 static poll_handler_t em_poll; 340 #endif /* POLLING */ 341 342 /********************************************************************* 343 * FreeBSD Device Interface Entry Points 344 *********************************************************************/ 345 346 static device_method_t em_methods[] = { 347 /* Device interface */ 348 DEVMETHOD(device_probe, em_probe), 349 DEVMETHOD(device_attach, em_attach), 350 DEVMETHOD(device_detach, em_detach), 351 DEVMETHOD(device_shutdown, em_shutdown), 352 DEVMETHOD(device_suspend, em_suspend), 353 DEVMETHOD(device_resume, em_resume), 354 {0, 0} 355 }; 356 357 static driver_t em_driver = { 358 "em", em_methods, sizeof(struct adapter), 359 }; 360 361 static devclass_t em_devclass; 362 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 363 MODULE_DEPEND(em, pci, 1, 1, 1); 364 MODULE_DEPEND(em, ether, 1, 1, 1); 365 366 /********************************************************************* 367 * Tunable default values. 368 *********************************************************************/ 369 370 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 371 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 372 #define M_TSO_LEN 66 373 374 /* Allow common code without TSO */ 375 #ifndef CSUM_TSO 376 #define CSUM_TSO 0 377 #endif 378 379 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 380 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 381 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 382 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 383 static int em_rxd = EM_DEFAULT_RXD; 384 static int em_txd = EM_DEFAULT_TXD; 385 static int em_smart_pwr_down = FALSE; 386 /* Controls whether promiscuous also shows bad packets */ 387 static int em_debug_sbp = FALSE; 388 /* Local switch for MSI/MSIX */ 389 static int em_enable_msi = TRUE; 390 391 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 392 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 393 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 394 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 395 TUNABLE_INT("hw.em.rxd", &em_rxd); 396 TUNABLE_INT("hw.em.txd", &em_txd); 397 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 398 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 399 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi); 400 401 #ifndef EM_LEGACY_IRQ 402 /* How many packets rxeof tries to clean at a time */ 403 static int em_rx_process_limit = 100; 404 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 405 #endif 406 407 /* Global used in WOL setup with multiport cards */ 408 static int global_quad_port_a = 0; 409 410 /********************************************************************* 411 * Device identification routine 412 * 413 * em_probe determines if the driver should be loaded on 414 * adapter based on PCI vendor/device id of the adapter. 415 * 416 * return BUS_PROBE_DEFAULT on success, positive on failure 417 *********************************************************************/ 418 419 static int 420 em_probe(device_t dev) 421 { 422 char adapter_name[60]; 423 u16 pci_vendor_id = 0; 424 u16 pci_device_id = 0; 425 u16 pci_subvendor_id = 0; 426 u16 pci_subdevice_id = 0; 427 em_vendor_info_t *ent; 428 429 INIT_DEBUGOUT("em_probe: begin"); 430 431 pci_vendor_id = pci_get_vendor(dev); 432 if (pci_vendor_id != EM_VENDOR_ID) 433 return (ENXIO); 434 435 pci_device_id = pci_get_device(dev); 436 pci_subvendor_id = pci_get_subvendor(dev); 437 pci_subdevice_id = pci_get_subdevice(dev); 438 439 ent = em_vendor_info_array; 440 while (ent->vendor_id != 0) { 441 if ((pci_vendor_id == ent->vendor_id) && 442 (pci_device_id == ent->device_id) && 443 444 ((pci_subvendor_id == ent->subvendor_id) || 445 (ent->subvendor_id == PCI_ANY_ID)) && 446 447 ((pci_subdevice_id == ent->subdevice_id) || 448 (ent->subdevice_id == PCI_ANY_ID))) { 449 sprintf(adapter_name, "%s %s", 450 em_strings[ent->index], 451 em_driver_version); 452 device_set_desc_copy(dev, adapter_name); 453 return (BUS_PROBE_DEFAULT); 454 } 455 ent++; 456 } 457 458 return (ENXIO); 459 } 460 461 /********************************************************************* 462 * Device initialization routine 463 * 464 * The attach entry point is called when the driver is being loaded. 465 * This routine identifies the type of hardware, allocates all resources 466 * and initializes the hardware. 467 * 468 * return 0 on success, positive on failure 469 *********************************************************************/ 470 471 static int 472 em_attach(device_t dev) 473 { 474 struct adapter *adapter; 475 int tsize, rsize; 476 int error = 0; 477 u16 eeprom_data, device_id; 478 479 INIT_DEBUGOUT("em_attach: begin"); 480 481 adapter = device_get_softc(dev); 482 adapter->dev = adapter->osdep.dev = dev; 483 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 484 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); 485 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); 486 487 /* SYSCTL stuff */ 488 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 489 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 490 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 491 em_sysctl_debug_info, "I", "Debug Information"); 492 493 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 494 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 495 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 496 em_sysctl_stats, "I", "Statistics"); 497 498 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 499 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); 500 501 /* Determine hardware and mac info */ 502 em_identify_hardware(adapter); 503 504 /* Setup PCI resources */ 505 if (em_allocate_pci_resources(adapter)) { 506 device_printf(dev, "Allocation of PCI resources failed\n"); 507 error = ENXIO; 508 goto err_pci; 509 } 510 511 /* 512 ** For ICH8 and family we need to 513 ** map the flash memory, and this 514 ** must happen after the MAC is 515 ** identified 516 */ 517 if ((adapter->hw.mac.type == e1000_ich8lan) || 518 (adapter->hw.mac.type == e1000_ich9lan) || 519 (adapter->hw.mac.type == e1000_ich10lan)) { 520 int rid = EM_BAR_TYPE_FLASH; 521 adapter->flash = bus_alloc_resource_any(dev, 522 SYS_RES_MEMORY, &rid, RF_ACTIVE); 523 if (adapter->flash == NULL) { 524 device_printf(dev, "Mapping of Flash failed\n"); 525 error = ENXIO; 526 goto err_pci; 527 } 528 /* This is used in the shared code */ 529 adapter->hw.flash_address = (u8 *)adapter->flash; 530 adapter->osdep.flash_bus_space_tag = 531 rman_get_bustag(adapter->flash); 532 adapter->osdep.flash_bus_space_handle = 533 rman_get_bushandle(adapter->flash); 534 } 535 536 /* Do Shared Code initialization */ 537 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 538 device_printf(dev, "Setup of Shared code failed\n"); 539 error = ENXIO; 540 goto err_pci; 541 } 542 543 e1000_get_bus_info(&adapter->hw); 544 545 /* Set up some sysctls for the tunable interrupt delays */ 546 em_add_int_delay_sysctl(adapter, "rx_int_delay", 547 "receive interrupt delay in usecs", &adapter->rx_int_delay, 548 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 549 em_add_int_delay_sysctl(adapter, "tx_int_delay", 550 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 551 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 552 if (adapter->hw.mac.type >= e1000_82540) { 553 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 554 "receive interrupt delay limit in usecs", 555 &adapter->rx_abs_int_delay, 556 E1000_REGISTER(&adapter->hw, E1000_RADV), 557 em_rx_abs_int_delay_dflt); 558 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 559 "transmit interrupt delay limit in usecs", 560 &adapter->tx_abs_int_delay, 561 E1000_REGISTER(&adapter->hw, E1000_TADV), 562 em_tx_abs_int_delay_dflt); 563 } 564 565 #ifndef EM_LEGACY_IRQ 566 /* Sysctls for limiting the amount of work done in the taskqueue */ 567 em_add_rx_process_limit(adapter, "rx_processing_limit", 568 "max number of rx packets to process", &adapter->rx_process_limit, 569 em_rx_process_limit); 570 #endif 571 572 /* 573 * Validate number of transmit and receive descriptors. It 574 * must not exceed hardware maximum, and must be multiple 575 * of E1000_DBA_ALIGN. 576 */ 577 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 578 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 579 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 580 (em_txd < EM_MIN_TXD)) { 581 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 582 EM_DEFAULT_TXD, em_txd); 583 adapter->num_tx_desc = EM_DEFAULT_TXD; 584 } else 585 adapter->num_tx_desc = em_txd; 586 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 587 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 588 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 589 (em_rxd < EM_MIN_RXD)) { 590 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 591 EM_DEFAULT_RXD, em_rxd); 592 adapter->num_rx_desc = EM_DEFAULT_RXD; 593 } else 594 adapter->num_rx_desc = em_rxd; 595 596 adapter->hw.mac.autoneg = DO_AUTO_NEG; 597 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 598 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 599 adapter->rx_buffer_len = 2048; 600 601 e1000_init_script_state_82541(&adapter->hw, TRUE); 602 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 603 604 /* Copper options */ 605 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 606 adapter->hw.phy.mdix = AUTO_ALL_MODES; 607 adapter->hw.phy.disable_polarity_correction = FALSE; 608 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 609 } 610 611 /* 612 * Set the frame limits assuming 613 * standard ethernet sized frames. 614 */ 615 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 616 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 617 618 /* 619 * This controls when hardware reports transmit completion 620 * status. 621 */ 622 adapter->hw.mac.report_tx_early = 1; 623 624 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 625 EM_DBA_ALIGN); 626 627 /* Allocate Transmit Descriptor ring */ 628 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { 629 device_printf(dev, "Unable to allocate tx_desc memory\n"); 630 error = ENOMEM; 631 goto err_tx_desc; 632 } 633 adapter->tx_desc_base = 634 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; 635 636 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 637 EM_DBA_ALIGN); 638 639 /* Allocate Receive Descriptor ring */ 640 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { 641 device_printf(dev, "Unable to allocate rx_desc memory\n"); 642 error = ENOMEM; 643 goto err_rx_desc; 644 } 645 adapter->rx_desc_base = 646 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; 647 648 /* 649 ** Start from a known state, this is 650 ** important in reading the nvm and 651 ** mac from that. 652 */ 653 e1000_reset_hw(&adapter->hw); 654 655 /* Make sure we have a good EEPROM before we read from it */ 656 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 657 /* 658 ** Some PCI-E parts fail the first check due to 659 ** the link being in sleep state, call it again, 660 ** if it fails a second time its a real issue. 661 */ 662 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 663 device_printf(dev, 664 "The EEPROM Checksum Is Not Valid\n"); 665 error = EIO; 666 goto err_hw_init; 667 } 668 } 669 670 /* Copy the permanent MAC address out of the EEPROM */ 671 if (e1000_read_mac_addr(&adapter->hw) < 0) { 672 device_printf(dev, "EEPROM read error while reading MAC" 673 " address\n"); 674 error = EIO; 675 goto err_hw_init; 676 } 677 678 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 679 device_printf(dev, "Invalid MAC address\n"); 680 error = EIO; 681 goto err_hw_init; 682 } 683 684 /* Initialize the hardware */ 685 if (em_hardware_init(adapter)) { 686 device_printf(dev, "Unable to initialize the hardware\n"); 687 error = EIO; 688 goto err_hw_init; 689 } 690 691 /* Allocate transmit descriptors and buffers */ 692 if (em_allocate_transmit_structures(adapter)) { 693 device_printf(dev, "Could not setup transmit structures\n"); 694 error = ENOMEM; 695 goto err_tx_struct; 696 } 697 698 /* Allocate receive descriptors and buffers */ 699 if (em_allocate_receive_structures(adapter)) { 700 device_printf(dev, "Could not setup receive structures\n"); 701 error = ENOMEM; 702 goto err_rx_struct; 703 } 704 705 /* 706 ** Do interrupt configuration 707 */ 708 if (adapter->msi > 1) /* Do MSI/X */ 709 error = em_allocate_msix(adapter); 710 else /* MSI or Legacy */ 711 error = em_allocate_legacy(adapter); 712 if (error) 713 goto err_rx_struct; 714 715 /* Setup OS specific network interface */ 716 em_setup_interface(dev, adapter); 717 718 /* Initialize statistics */ 719 em_update_stats_counters(adapter); 720 721 adapter->hw.mac.get_link_status = 1; 722 em_update_link_status(adapter); 723 724 /* Indicate SOL/IDER usage */ 725 if (e1000_check_reset_block(&adapter->hw)) 726 device_printf(dev, 727 "PHY reset is blocked due to SOL/IDER session.\n"); 728 729 /* Determine if we have to control management hardware */ 730 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 731 732 /* 733 * Setup Wake-on-Lan 734 */ 735 switch (adapter->hw.mac.type) { 736 737 case e1000_82542: 738 case e1000_82543: 739 break; 740 case e1000_82546: 741 case e1000_82546_rev_3: 742 case e1000_82571: 743 case e1000_80003es2lan: 744 if (adapter->hw.bus.func == 1) 745 e1000_read_nvm(&adapter->hw, 746 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 747 else 748 e1000_read_nvm(&adapter->hw, 749 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 750 eeprom_data &= EM_EEPROM_APME; 751 break; 752 default: 753 /* APME bit in EEPROM is mapped to WUC.APME */ 754 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & 755 E1000_WUC_APME; 756 break; 757 } 758 if (eeprom_data) 759 adapter->wol = E1000_WUFC_MAG; 760 /* 761 * We have the eeprom settings, now apply the special cases 762 * where the eeprom may be wrong or the board won't support 763 * wake on lan on a particular port 764 */ 765 device_id = pci_get_device(dev); 766 switch (device_id) { 767 case E1000_DEV_ID_82546GB_PCIE: 768 adapter->wol = 0; 769 break; 770 case E1000_DEV_ID_82546EB_FIBER: 771 case E1000_DEV_ID_82546GB_FIBER: 772 case E1000_DEV_ID_82571EB_FIBER: 773 /* Wake events only supported on port A for dual fiber 774 * regardless of eeprom setting */ 775 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 776 E1000_STATUS_FUNC_1) 777 adapter->wol = 0; 778 break; 779 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 780 case E1000_DEV_ID_82571EB_QUAD_COPPER: 781 case E1000_DEV_ID_82571EB_QUAD_FIBER: 782 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 783 /* if quad port adapter, disable WoL on all but port A */ 784 if (global_quad_port_a != 0) 785 adapter->wol = 0; 786 /* Reset for multiple quad port adapters */ 787 if (++global_quad_port_a == 4) 788 global_quad_port_a = 0; 789 break; 790 } 791 792 /* Do we need workaround for 82544 PCI-X adapter? */ 793 if (adapter->hw.bus.type == e1000_bus_type_pcix && 794 adapter->hw.mac.type == e1000_82544) 795 adapter->pcix_82544 = TRUE; 796 else 797 adapter->pcix_82544 = FALSE; 798 799 /* Register for VLAN events */ 800 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 801 em_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 802 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 803 em_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 804 805 /* Tell the stack that the interface is not active */ 806 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 807 808 INIT_DEBUGOUT("em_attach: end"); 809 810 return (0); 811 812 err_rx_struct: 813 em_free_transmit_structures(adapter); 814 err_tx_struct: 815 err_hw_init: 816 em_release_hw_control(adapter); 817 em_dma_free(adapter, &adapter->rxdma); 818 err_rx_desc: 819 em_dma_free(adapter, &adapter->txdma); 820 err_tx_desc: 821 err_pci: 822 em_free_pci_resources(adapter); 823 EM_TX_LOCK_DESTROY(adapter); 824 EM_RX_LOCK_DESTROY(adapter); 825 EM_CORE_LOCK_DESTROY(adapter); 826 827 return (error); 828 } 829 830 /********************************************************************* 831 * Device removal routine 832 * 833 * The detach entry point is called when the driver is being removed. 834 * This routine stops the adapter and deallocates all the resources 835 * that were allocated for driver operation. 836 * 837 * return 0 on success, positive on failure 838 *********************************************************************/ 839 840 static int 841 em_detach(device_t dev) 842 { 843 struct adapter *adapter = device_get_softc(dev); 844 struct ifnet *ifp = adapter->ifp; 845 846 INIT_DEBUGOUT("em_detach: begin"); 847 848 /* Make sure VLANS are not using driver */ 849 #if __FreeBSD_version >= 700000 850 if (adapter->ifp->if_vlantrunk != NULL) { 851 #else 852 if (adapter->ifp->if_nvlans != 0) { 853 #endif 854 device_printf(dev,"Vlan in use, detach first\n"); 855 return (EBUSY); 856 } 857 858 #ifdef DEVICE_POLLING 859 if (ifp->if_capenable & IFCAP_POLLING) 860 ether_poll_deregister(ifp); 861 #endif 862 863 EM_CORE_LOCK(adapter); 864 EM_TX_LOCK(adapter); 865 adapter->in_detach = 1; 866 em_stop(adapter); 867 e1000_phy_hw_reset(&adapter->hw); 868 869 em_release_manageability(adapter); 870 871 if (((adapter->hw.mac.type == e1000_82573) || 872 (adapter->hw.mac.type == e1000_ich8lan) || 873 (adapter->hw.mac.type == e1000_ich10lan) || 874 (adapter->hw.mac.type == e1000_ich9lan)) && 875 e1000_check_mng_mode(&adapter->hw)) 876 em_release_hw_control(adapter); 877 878 if (adapter->wol) { 879 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 880 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 881 em_enable_wakeup(dev); 882 } 883 884 EM_TX_UNLOCK(adapter); 885 EM_CORE_UNLOCK(adapter); 886 887 /* Unregister VLAN events */ 888 if (adapter->vlan_attach != NULL) 889 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 890 if (adapter->vlan_detach != NULL) 891 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 892 893 ether_ifdetach(adapter->ifp); 894 callout_drain(&adapter->timer); 895 callout_drain(&adapter->tx_fifo_timer); 896 897 em_free_pci_resources(adapter); 898 bus_generic_detach(dev); 899 if_free(ifp); 900 901 drbr_free(adapter->br, M_DEVBUF); 902 em_free_transmit_structures(adapter); 903 em_free_receive_structures(adapter); 904 905 /* Free Transmit Descriptor ring */ 906 if (adapter->tx_desc_base) { 907 em_dma_free(adapter, &adapter->txdma); 908 adapter->tx_desc_base = NULL; 909 } 910 911 /* Free Receive Descriptor ring */ 912 if (adapter->rx_desc_base) { 913 em_dma_free(adapter, &adapter->rxdma); 914 adapter->rx_desc_base = NULL; 915 } 916 917 EM_TX_LOCK_DESTROY(adapter); 918 EM_RX_LOCK_DESTROY(adapter); 919 EM_CORE_LOCK_DESTROY(adapter); 920 921 return (0); 922 } 923 924 /********************************************************************* 925 * 926 * Shutdown entry point 927 * 928 **********************************************************************/ 929 930 static int 931 em_shutdown(device_t dev) 932 { 933 return em_suspend(dev); 934 } 935 936 /* 937 * Suspend/resume device methods. 938 */ 939 static int 940 em_suspend(device_t dev) 941 { 942 struct adapter *adapter = device_get_softc(dev); 943 944 EM_CORE_LOCK(adapter); 945 946 EM_TX_LOCK(adapter); 947 em_stop(adapter); 948 EM_TX_UNLOCK(adapter); 949 950 em_release_manageability(adapter); 951 952 if (((adapter->hw.mac.type == e1000_82573) || 953 (adapter->hw.mac.type == e1000_ich8lan) || 954 (adapter->hw.mac.type == e1000_ich10lan) || 955 (adapter->hw.mac.type == e1000_ich9lan)) && 956 e1000_check_mng_mode(&adapter->hw)) 957 em_release_hw_control(adapter); 958 959 if (adapter->wol) { 960 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 961 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 962 em_enable_wakeup(dev); 963 } 964 965 EM_CORE_UNLOCK(adapter); 966 967 return bus_generic_suspend(dev); 968 } 969 970 static int 971 em_resume(device_t dev) 972 { 973 struct adapter *adapter = device_get_softc(dev); 974 struct ifnet *ifp = adapter->ifp; 975 976 EM_CORE_LOCK(adapter); 977 em_init_locked(adapter); 978 em_init_manageability(adapter); 979 EM_CORE_UNLOCK(adapter); 980 em_start(ifp); 981 982 return bus_generic_resume(dev); 983 } 984 985 986 /********************************************************************* 987 * Transmit entry point 988 * 989 * em_start is called by the stack to initiate a transmit. 990 * The driver will remain in this routine as long as there are 991 * packets to transmit and transmit resources are available. 992 * In case resources are not available stack is notified and 993 * the packet is requeued. 994 **********************************************************************/ 995 996 #ifdef IFNET_BUF_RING 997 static int 998 em_transmit_locked(struct ifnet *ifp, struct mbuf *m) 999 { 1000 struct adapter *adapter = ifp->if_softc; 1001 int error; 1002 1003 EM_TX_LOCK_ASSERT(adapter); 1004 if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1005 IFF_DRV_RUNNING) 1006 || (!adapter->link_active)) { 1007 error = drbr_enqueue(ifp, adapter->br, m); 1008 return (error); 1009 } 1010 1011 if (ADAPTER_RING_EMPTY(adapter) && 1012 (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { 1013 if (em_xmit(adapter, &m)) { 1014 if (m && (error = drbr_enqueue(ifp, adapter->br, m)) != 0) 1015 return (error); 1016 } else { 1017 /* 1018 ** Send a copy of the frame to the BPF 1019 ** listener and set the watchdog on. 1020 */ 1021 ETHER_BPF_MTAP(ifp, m); 1022 adapter->watchdog_timer = EM_TX_TIMEOUT; 1023 } 1024 } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) 1025 return (error); 1026 1027 if (!ADAPTER_RING_EMPTY(adapter)) 1028 em_start_locked(ifp); 1029 1030 return (0); 1031 } 1032 1033 static int 1034 em_transmit(struct ifnet *ifp, struct mbuf *m) 1035 { 1036 1037 struct adapter *adapter = ifp->if_softc; 1038 int error = 0; 1039 1040 if(EM_TX_TRYLOCK(adapter)) { 1041 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1042 error = em_transmit_locked(ifp, m); 1043 EM_TX_UNLOCK(adapter); 1044 } else 1045 error = drbr_enqueue(ifp, adapter->br, m); 1046 1047 return (error); 1048 } 1049 1050 static void 1051 em_qflush(struct ifnet *ifp) 1052 { 1053 struct mbuf *m; 1054 struct adapter *adapter = (struct adapter *)ifp->if_softc; 1055 1056 EM_TX_LOCK(adapter); 1057 while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL) 1058 m_freem(m); 1059 if_qflush(ifp); 1060 EM_TX_UNLOCK(adapter); 1061 } 1062 #endif 1063 1064 static void 1065 em_start_locked(struct ifnet *ifp) 1066 { 1067 struct adapter *adapter = ifp->if_softc; 1068 struct mbuf *m_head; 1069 1070 EM_TX_LOCK_ASSERT(adapter); 1071 1072 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1073 IFF_DRV_RUNNING) 1074 return; 1075 if (!adapter->link_active) 1076 return; 1077 1078 while ((adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD) 1079 && (!ADAPTER_RING_EMPTY(adapter))) { 1080 1081 m_head = em_dequeue(ifp, adapter->br); 1082 if (m_head == NULL) 1083 break; 1084 /* 1085 * Encapsulation can modify our pointer, and or make it 1086 * NULL on failure. In that event, we can't requeue. 1087 */ 1088 if (em_xmit(adapter, &m_head)) { 1089 if (m_head == NULL) 1090 break; 1091 #ifndef IFNET_BUF_RING 1092 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1093 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 1094 #endif 1095 break; 1096 } 1097 1098 /* Send a copy of the frame to the BPF listener */ 1099 ETHER_BPF_MTAP(ifp, m_head); 1100 1101 /* Set timeout in case hardware has problems transmitting. */ 1102 adapter->watchdog_timer = EM_TX_TIMEOUT; 1103 } 1104 if ((adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)) 1105 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1106 1107 } 1108 1109 static void 1110 em_start(struct ifnet *ifp) 1111 { 1112 struct adapter *adapter = ifp->if_softc; 1113 1114 EM_TX_LOCK(adapter); 1115 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1116 em_start_locked(ifp); 1117 EM_TX_UNLOCK(adapter); 1118 } 1119 1120 /********************************************************************* 1121 * Ioctl entry point 1122 * 1123 * em_ioctl is called when the user wants to configure the 1124 * interface. 1125 * 1126 * return 0 on success, positive on failure 1127 **********************************************************************/ 1128 1129 static int 1130 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1131 { 1132 struct adapter *adapter = ifp->if_softc; 1133 struct ifreq *ifr = (struct ifreq *)data; 1134 #ifdef INET 1135 struct ifaddr *ifa = (struct ifaddr *)data; 1136 #endif 1137 int error = 0; 1138 1139 if (adapter->in_detach) 1140 return (error); 1141 1142 switch (command) { 1143 case SIOCSIFADDR: 1144 #ifdef INET 1145 if (ifa->ifa_addr->sa_family == AF_INET) { 1146 /* 1147 * XXX 1148 * Since resetting hardware takes a very long time 1149 * and results in link renegotiation we only 1150 * initialize the hardware only when it is absolutely 1151 * required. 1152 */ 1153 ifp->if_flags |= IFF_UP; 1154 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1155 EM_CORE_LOCK(adapter); 1156 em_init_locked(adapter); 1157 EM_CORE_UNLOCK(adapter); 1158 } 1159 arp_ifinit(ifp, ifa); 1160 } else 1161 #endif 1162 error = ether_ioctl(ifp, command, data); 1163 break; 1164 case SIOCSIFMTU: 1165 { 1166 int max_frame_size; 1167 u16 eeprom_data = 0; 1168 1169 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1170 1171 EM_CORE_LOCK(adapter); 1172 switch (adapter->hw.mac.type) { 1173 case e1000_82573: 1174 /* 1175 * 82573 only supports jumbo frames 1176 * if ASPM is disabled. 1177 */ 1178 e1000_read_nvm(&adapter->hw, 1179 NVM_INIT_3GIO_3, 1, &eeprom_data); 1180 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1181 max_frame_size = ETHER_MAX_LEN; 1182 break; 1183 } 1184 /* Allow Jumbo frames - fall thru */ 1185 case e1000_82571: 1186 case e1000_82572: 1187 case e1000_ich9lan: 1188 case e1000_ich10lan: 1189 case e1000_82574: 1190 case e1000_80003es2lan: /* Limit Jumbo Frame size */ 1191 max_frame_size = 9234; 1192 break; 1193 /* Adapters that do not support jumbo frames */ 1194 case e1000_82542: 1195 case e1000_ich8lan: 1196 max_frame_size = ETHER_MAX_LEN; 1197 break; 1198 default: 1199 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1200 } 1201 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1202 ETHER_CRC_LEN) { 1203 EM_CORE_UNLOCK(adapter); 1204 error = EINVAL; 1205 break; 1206 } 1207 1208 ifp->if_mtu = ifr->ifr_mtu; 1209 adapter->max_frame_size = 1210 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1211 em_init_locked(adapter); 1212 EM_CORE_UNLOCK(adapter); 1213 break; 1214 } 1215 case SIOCSIFFLAGS: 1216 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1217 SIOCSIFFLAGS (Set Interface Flags)"); 1218 EM_CORE_LOCK(adapter); 1219 if (ifp->if_flags & IFF_UP) { 1220 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1221 if ((ifp->if_flags ^ adapter->if_flags) & 1222 (IFF_PROMISC | IFF_ALLMULTI)) { 1223 em_disable_promisc(adapter); 1224 em_set_promisc(adapter); 1225 } 1226 } else 1227 em_init_locked(adapter); 1228 } else 1229 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1230 EM_TX_LOCK(adapter); 1231 em_stop(adapter); 1232 EM_TX_UNLOCK(adapter); 1233 } 1234 adapter->if_flags = ifp->if_flags; 1235 EM_CORE_UNLOCK(adapter); 1236 break; 1237 case SIOCADDMULTI: 1238 case SIOCDELMULTI: 1239 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1240 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1241 EM_CORE_LOCK(adapter); 1242 em_disable_intr(adapter); 1243 em_set_multi(adapter); 1244 if (adapter->hw.mac.type == e1000_82542 && 1245 adapter->hw.revision_id == E1000_REVISION_2) { 1246 em_initialize_receive_unit(adapter); 1247 } 1248 #ifdef DEVICE_POLLING 1249 if (!(ifp->if_capenable & IFCAP_POLLING)) 1250 #endif 1251 em_enable_intr(adapter); 1252 EM_CORE_UNLOCK(adapter); 1253 } 1254 break; 1255 case SIOCSIFMEDIA: 1256 /* Check SOL/IDER usage */ 1257 EM_CORE_LOCK(adapter); 1258 if (e1000_check_reset_block(&adapter->hw)) { 1259 EM_CORE_UNLOCK(adapter); 1260 device_printf(adapter->dev, "Media change is" 1261 " blocked due to SOL/IDER session.\n"); 1262 break; 1263 } 1264 EM_CORE_UNLOCK(adapter); 1265 case SIOCGIFMEDIA: 1266 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1267 SIOCxIFMEDIA (Get/Set Interface Media)"); 1268 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1269 break; 1270 case SIOCSIFCAP: 1271 { 1272 int mask, reinit; 1273 1274 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1275 reinit = 0; 1276 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1277 #ifdef DEVICE_POLLING 1278 if (mask & IFCAP_POLLING) { 1279 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1280 error = ether_poll_register(em_poll, ifp); 1281 if (error) 1282 return (error); 1283 EM_CORE_LOCK(adapter); 1284 em_disable_intr(adapter); 1285 ifp->if_capenable |= IFCAP_POLLING; 1286 EM_CORE_UNLOCK(adapter); 1287 } else { 1288 error = ether_poll_deregister(ifp); 1289 /* Enable interrupt even in error case */ 1290 EM_CORE_LOCK(adapter); 1291 em_enable_intr(adapter); 1292 ifp->if_capenable &= ~IFCAP_POLLING; 1293 EM_CORE_UNLOCK(adapter); 1294 } 1295 } 1296 #endif 1297 if (mask & IFCAP_HWCSUM) { 1298 ifp->if_capenable ^= IFCAP_HWCSUM; 1299 reinit = 1; 1300 } 1301 #if __FreeBSD_version >= 700000 1302 if (mask & IFCAP_TSO4) { 1303 ifp->if_capenable ^= IFCAP_TSO4; 1304 reinit = 1; 1305 } 1306 #endif 1307 1308 if (mask & IFCAP_VLAN_HWTAGGING) { 1309 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1310 reinit = 1; 1311 } 1312 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1313 em_init(adapter); 1314 #if __FreeBSD_version >= 700000 1315 VLAN_CAPABILITIES(ifp); 1316 #endif 1317 break; 1318 } 1319 1320 #ifdef EM_TIMESYNC 1321 /* 1322 ** IOCTL support for Precision Time (IEEE 1588) Support 1323 */ 1324 case EM_TIMESYNC_READTS: 1325 { 1326 u32 rx_ctl, tx_ctl; 1327 struct em_tsync_read *tdata; 1328 1329 tdata = (struct em_tsync_read *) ifr->ifr_data; 1330 1331 IOCTL_DEBUGOUT("Reading Timestamp\n"); 1332 1333 if (tdata->read_current_time) { 1334 getnanotime(&tdata->system_time); 1335 tdata->network_time = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 1336 tdata->network_time |= 1337 (u64)E1000_READ_REG(&adapter->hw, E1000_SYSTIMH ) << 32; 1338 } 1339 1340 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 1341 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 1342 1343 IOCTL_DEBUGOUT1("RX_CTL value = %u\n", rx_ctl); 1344 IOCTL_DEBUGOUT1("TX_CTL value = %u\n", tx_ctl); 1345 1346 if (rx_ctl & 0x1) { 1347 IOCTL_DEBUGOUT("RX timestamp is valid\n"); 1348 u32 tmp; 1349 unsigned char *tmp_cp; 1350 1351 tdata->rx_valid = 1; 1352 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL); 1353 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw, 1354 E1000_RXSTMPH) << 32; 1355 1356 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL); 1357 tmp_cp = (unsigned char *) &tmp; 1358 tdata->srcid[0] = tmp_cp[0]; 1359 tdata->srcid[1] = tmp_cp[1]; 1360 tdata->srcid[2] = tmp_cp[2]; 1361 tdata->srcid[3] = tmp_cp[3]; 1362 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 1363 tmp_cp = (unsigned char *) &tmp; 1364 tdata->srcid[4] = tmp_cp[0]; 1365 tdata->srcid[5] = tmp_cp[1]; 1366 tdata->seqid = tmp >> 16; 1367 tdata->seqid = htons(tdata->seqid); 1368 } else 1369 tdata->rx_valid = 0; 1370 1371 if (tx_ctl & 0x1) { 1372 IOCTL_DEBUGOUT("TX timestamp is valid\n"); 1373 tdata->tx_valid = 1; 1374 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL); 1375 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw, 1376 E1000_TXSTMPH) << 32; 1377 } else 1378 tdata->tx_valid = 0; 1379 1380 return (0); 1381 } 1382 #endif /* EM_TIMESYNC */ 1383 1384 default: 1385 error = ether_ioctl(ifp, command, data); 1386 break; 1387 } 1388 1389 return (error); 1390 } 1391 1392 /********************************************************************* 1393 * Watchdog timer: 1394 * 1395 * This routine is called from the local timer every second. 1396 * As long as transmit descriptors are being cleaned the value 1397 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1398 * and we then reset the device. 1399 * 1400 **********************************************************************/ 1401 1402 static void 1403 em_watchdog(struct adapter *adapter) 1404 { 1405 1406 EM_CORE_LOCK_ASSERT(adapter); 1407 1408 /* 1409 ** The timer is set to 5 every time start queues a packet. 1410 ** Then txeof keeps resetting it as long as it cleans at 1411 ** least one descriptor. 1412 ** Finally, anytime all descriptors are clean the timer is 1413 ** set to 0. 1414 */ 1415 EM_TX_LOCK(adapter); 1416 if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) { 1417 EM_TX_UNLOCK(adapter); 1418 return; 1419 } 1420 1421 /* If we are in this routine because of pause frames, then 1422 * don't reset the hardware. 1423 */ 1424 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1425 E1000_STATUS_TXOFF) { 1426 adapter->watchdog_timer = EM_TX_TIMEOUT; 1427 EM_TX_UNLOCK(adapter); 1428 return; 1429 } 1430 1431 if (e1000_check_for_link(&adapter->hw) == 0) 1432 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1433 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1434 adapter->watchdog_events++; 1435 EM_TX_UNLOCK(adapter); 1436 1437 em_init_locked(adapter); 1438 } 1439 1440 /********************************************************************* 1441 * Init entry point 1442 * 1443 * This routine is used in two ways. It is used by the stack as 1444 * init entry point in network interface structure. It is also used 1445 * by the driver as a hw/sw initialization routine to get to a 1446 * consistent state. 1447 * 1448 * return 0 on success, positive on failure 1449 **********************************************************************/ 1450 1451 static void 1452 em_init_locked(struct adapter *adapter) 1453 { 1454 struct ifnet *ifp = adapter->ifp; 1455 device_t dev = adapter->dev; 1456 u32 pba; 1457 1458 INIT_DEBUGOUT("em_init: begin"); 1459 1460 EM_CORE_LOCK_ASSERT(adapter); 1461 1462 EM_TX_LOCK(adapter); 1463 em_stop(adapter); 1464 EM_TX_UNLOCK(adapter); 1465 1466 /* 1467 * Packet Buffer Allocation (PBA) 1468 * Writing PBA sets the receive portion of the buffer 1469 * the remainder is used for the transmit buffer. 1470 * 1471 * Devices before the 82547 had a Packet Buffer of 64K. 1472 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 1473 * After the 82547 the buffer was reduced to 40K. 1474 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 1475 * Note: default does not leave enough room for Jumbo Frame >10k. 1476 */ 1477 switch (adapter->hw.mac.type) { 1478 case e1000_82547: 1479 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 1480 if (adapter->max_frame_size > 8192) 1481 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 1482 else 1483 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 1484 adapter->tx_fifo_head = 0; 1485 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 1486 adapter->tx_fifo_size = 1487 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 1488 break; 1489 /* Total Packet Buffer on these is 48K */ 1490 case e1000_82571: 1491 case e1000_82572: 1492 case e1000_80003es2lan: 1493 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1494 break; 1495 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1496 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1497 break; 1498 case e1000_82574: 1499 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1500 break; 1501 case e1000_ich9lan: 1502 case e1000_ich10lan: 1503 #define E1000_PBA_10K 0x000A 1504 pba = E1000_PBA_10K; 1505 break; 1506 case e1000_ich8lan: 1507 pba = E1000_PBA_8K; 1508 break; 1509 default: 1510 /* Devices before 82547 had a Packet Buffer of 64K. */ 1511 if (adapter->max_frame_size > 8192) 1512 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1513 else 1514 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1515 } 1516 1517 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1518 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1519 1520 /* Get the latest mac address, User can use a LAA */ 1521 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1522 ETHER_ADDR_LEN); 1523 1524 /* Put the address into the Receive Address Array */ 1525 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1526 1527 /* 1528 * With the 82571 adapter, RAR[0] may be overwritten 1529 * when the other port is reset, we make a duplicate 1530 * in RAR[14] for that eventuality, this assures 1531 * the interface continues to function. 1532 */ 1533 if (adapter->hw.mac.type == e1000_82571) { 1534 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1535 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1536 E1000_RAR_ENTRIES - 1); 1537 } 1538 1539 /* Initialize the hardware */ 1540 if (em_hardware_init(adapter)) { 1541 device_printf(dev, "Unable to initialize the hardware\n"); 1542 return; 1543 } 1544 em_update_link_status(adapter); 1545 1546 /* Setup VLAN support, basic and offload if available */ 1547 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1548 1549 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) && 1550 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) { 1551 u32 ctrl; 1552 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1553 ctrl |= E1000_CTRL_VME; 1554 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1555 } 1556 1557 1558 /* Set hardware offload abilities */ 1559 ifp->if_hwassist = 0; 1560 if (adapter->hw.mac.type >= e1000_82543) { 1561 if (ifp->if_capenable & IFCAP_TXCSUM) 1562 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1563 #if __FreeBSD_version >= 700000 1564 if (ifp->if_capenable & IFCAP_TSO4) 1565 ifp->if_hwassist |= CSUM_TSO; 1566 #endif 1567 } 1568 1569 /* Configure for OS presence */ 1570 em_init_manageability(adapter); 1571 1572 /* Prepare transmit descriptors and buffers */ 1573 em_setup_transmit_structures(adapter); 1574 em_initialize_transmit_unit(adapter); 1575 1576 /* Setup Multicast table */ 1577 em_set_multi(adapter); 1578 1579 /* Prepare receive descriptors and buffers */ 1580 if (em_setup_receive_structures(adapter)) { 1581 device_printf(dev, "Could not setup receive structures\n"); 1582 EM_TX_LOCK(adapter); 1583 em_stop(adapter); 1584 EM_TX_UNLOCK(adapter); 1585 return; 1586 } 1587 em_initialize_receive_unit(adapter); 1588 1589 /* Don't lose promiscuous settings */ 1590 em_set_promisc(adapter); 1591 1592 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1593 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1594 1595 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1596 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1597 1598 /* MSI/X configuration for 82574 */ 1599 if (adapter->hw.mac.type == e1000_82574) { 1600 int tmp; 1601 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1602 tmp |= E1000_CTRL_EXT_PBA_CLR; 1603 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1604 /* 1605 ** Set the IVAR - interrupt vector routing. 1606 ** Each nibble represents a vector, high bit 1607 ** is enable, other 3 bits are the MSIX table 1608 ** entry, we map RXQ0 to 0, TXQ0 to 1, and 1609 ** Link (other) to 2, hence the magic number. 1610 */ 1611 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); 1612 } 1613 1614 #ifdef DEVICE_POLLING 1615 /* 1616 * Only enable interrupts if we are not polling, make sure 1617 * they are off otherwise. 1618 */ 1619 if (ifp->if_capenable & IFCAP_POLLING) 1620 em_disable_intr(adapter); 1621 else 1622 #endif /* DEVICE_POLLING */ 1623 em_enable_intr(adapter); 1624 1625 #ifdef EM_TIMESYNC 1626 /* Initializae IEEE 1588 Precision Time hardware */ 1627 if ((adapter->hw.mac.type == e1000_82574) || 1628 (adapter->hw.mac.type == e1000_ich10lan)) 1629 em_tsync_init(adapter); 1630 #endif 1631 1632 /* Don't reset the phy next time init gets called */ 1633 adapter->hw.phy.reset_disable = TRUE; 1634 } 1635 1636 static void 1637 em_init(void *arg) 1638 { 1639 struct adapter *adapter = arg; 1640 1641 EM_CORE_LOCK(adapter); 1642 em_init_locked(adapter); 1643 EM_CORE_UNLOCK(adapter); 1644 } 1645 1646 1647 #ifdef DEVICE_POLLING 1648 /********************************************************************* 1649 * 1650 * Legacy polling routine 1651 * 1652 *********************************************************************/ 1653 static void 1654 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1655 { 1656 struct adapter *adapter = ifp->if_softc; 1657 u32 reg_icr; 1658 1659 EM_CORE_LOCK(adapter); 1660 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1661 EM_CORE_UNLOCK(adapter); 1662 return; 1663 } 1664 1665 if (cmd == POLL_AND_CHECK_STATUS) { 1666 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1667 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1668 callout_stop(&adapter->timer); 1669 adapter->hw.mac.get_link_status = 1; 1670 em_update_link_status(adapter); 1671 callout_reset(&adapter->timer, hz, 1672 em_local_timer, adapter); 1673 } 1674 } 1675 EM_CORE_UNLOCK(adapter); 1676 1677 em_rxeof(adapter, count); 1678 1679 EM_TX_LOCK(adapter); 1680 em_txeof(adapter); 1681 1682 if (!ADAPTER_RING_EMPTY(adapter)) 1683 em_start_locked(ifp); 1684 EM_TX_UNLOCK(adapter); 1685 } 1686 #endif /* DEVICE_POLLING */ 1687 1688 #ifdef EM_LEGACY_IRQ 1689 /********************************************************************* 1690 * 1691 * Legacy Interrupt Service routine 1692 * 1693 *********************************************************************/ 1694 1695 static void 1696 em_intr(void *arg) 1697 { 1698 struct adapter *adapter = arg; 1699 struct ifnet *ifp = adapter->ifp; 1700 u32 reg_icr; 1701 1702 1703 if (ifp->if_capenable & IFCAP_POLLING) 1704 return; 1705 1706 EM_CORE_LOCK(adapter); 1707 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1708 if ((reg_icr == 0xffffffff) || (reg_icr == 0)|| 1709 (adapter->hw.mac.type >= e1000_82571 && 1710 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)) 1711 goto out; 1712 1713 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1714 goto out; 1715 1716 EM_TX_LOCK(adapter); 1717 em_txeof(adapter); 1718 em_rxeof(adapter, -1); 1719 em_txeof(adapter); 1720 EM_TX_UNLOCK(adapter); 1721 1722 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1723 callout_stop(&adapter->timer); 1724 adapter->hw.mac.get_link_status = 1; 1725 em_update_link_status(adapter); 1726 /* Deal with TX cruft when link lost */ 1727 em_tx_purge(adapter); 1728 callout_reset(&adapter->timer, hz, 1729 em_local_timer, adapter); 1730 } 1731 1732 if (reg_icr & E1000_ICR_RXO) 1733 adapter->rx_overruns++; 1734 out: 1735 EM_CORE_UNLOCK(adapter); 1736 if (ifp->if_drv_flags & IFF_DRV_RUNNING && 1737 !ADAPTER_RING_EMPTY(adapter)) 1738 em_start(ifp); 1739 } 1740 1741 #else /* EM_FAST_IRQ, then fast interrupt routines only */ 1742 1743 static void 1744 em_handle_link(void *context, int pending) 1745 { 1746 struct adapter *adapter = context; 1747 struct ifnet *ifp = adapter->ifp; 1748 1749 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1750 return; 1751 1752 EM_CORE_LOCK(adapter); 1753 callout_stop(&adapter->timer); 1754 em_update_link_status(adapter); 1755 /* Deal with TX cruft when link lost */ 1756 em_tx_purge(adapter); 1757 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1758 EM_CORE_UNLOCK(adapter); 1759 } 1760 1761 1762 /* Combined RX/TX handler, used by Legacy and MSI */ 1763 static void 1764 em_handle_rxtx(void *context, int pending) 1765 { 1766 struct adapter *adapter = context; 1767 struct ifnet *ifp = adapter->ifp; 1768 1769 1770 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1771 if (em_rxeof(adapter, adapter->rx_process_limit) != 0) 1772 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1773 EM_TX_LOCK(adapter); 1774 em_txeof(adapter); 1775 1776 if (!ADAPTER_RING_EMPTY(adapter)) 1777 em_start_locked(ifp); 1778 EM_TX_UNLOCK(adapter); 1779 } 1780 1781 em_enable_intr(adapter); 1782 } 1783 1784 /********************************************************************* 1785 * 1786 * Fast Legacy/MSI Combined Interrupt Service routine 1787 * 1788 *********************************************************************/ 1789 #if __FreeBSD_version < 700000 1790 #define FILTER_STRAY 1791 #define FILTER_HANDLED 1792 static void 1793 #else 1794 static int 1795 #endif 1796 em_irq_fast(void *arg) 1797 { 1798 struct adapter *adapter = arg; 1799 struct ifnet *ifp; 1800 u32 reg_icr; 1801 1802 ifp = adapter->ifp; 1803 1804 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1805 1806 /* Hot eject? */ 1807 if (reg_icr == 0xffffffff) 1808 return FILTER_STRAY; 1809 1810 /* Definitely not our interrupt. */ 1811 if (reg_icr == 0x0) 1812 return FILTER_STRAY; 1813 1814 /* 1815 * Starting with the 82571 chip, bit 31 should be used to 1816 * determine whether the interrupt belongs to us. 1817 */ 1818 if (adapter->hw.mac.type >= e1000_82571 && 1819 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1820 return FILTER_STRAY; 1821 1822 /* 1823 * Mask interrupts until the taskqueue is finished running. This is 1824 * cheap, just assume that it is needed. This also works around the 1825 * MSI message reordering errata on certain systems. 1826 */ 1827 em_disable_intr(adapter); 1828 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1829 1830 /* Link status change */ 1831 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1832 adapter->hw.mac.get_link_status = 1; 1833 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1834 } 1835 1836 if (reg_icr & E1000_ICR_RXO) 1837 adapter->rx_overruns++; 1838 return FILTER_HANDLED; 1839 } 1840 1841 /********************************************************************* 1842 * 1843 * MSIX Interrupt Service Routines 1844 * 1845 **********************************************************************/ 1846 #define EM_MSIX_TX 0x00040000 1847 #define EM_MSIX_RX 0x00010000 1848 #define EM_MSIX_LINK 0x00100000 1849 1850 static void 1851 em_msix_tx(void *arg) 1852 { 1853 struct adapter *adapter = arg; 1854 struct ifnet *ifp = adapter->ifp; 1855 1856 ++adapter->tx_irq; 1857 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1858 EM_TX_LOCK(adapter); 1859 em_txeof(adapter); 1860 EM_TX_UNLOCK(adapter); 1861 taskqueue_enqueue(adapter->tq, &adapter->tx_task); 1862 } 1863 /* Reenable this interrupt */ 1864 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX); 1865 return; 1866 } 1867 1868 /********************************************************************* 1869 * 1870 * MSIX RX Interrupt Service routine 1871 * 1872 **********************************************************************/ 1873 1874 static void 1875 em_msix_rx(void *arg) 1876 { 1877 struct adapter *adapter = arg; 1878 struct ifnet *ifp = adapter->ifp; 1879 1880 ++adapter->rx_irq; 1881 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1882 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1883 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1884 /* Reenable this interrupt */ 1885 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX); 1886 return; 1887 } 1888 1889 /********************************************************************* 1890 * 1891 * MSIX Link Fast Interrupt Service routine 1892 * 1893 **********************************************************************/ 1894 1895 static void 1896 em_msix_link(void *arg) 1897 { 1898 struct adapter *adapter = arg; 1899 u32 reg_icr; 1900 1901 ++adapter->link_irq; 1902 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1903 1904 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1905 adapter->hw.mac.get_link_status = 1; 1906 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1907 } 1908 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1909 EM_MSIX_LINK | E1000_IMS_LSC); 1910 return; 1911 } 1912 1913 static void 1914 em_handle_rx(void *context, int pending) 1915 { 1916 struct adapter *adapter = context; 1917 struct ifnet *ifp = adapter->ifp; 1918 1919 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1920 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1921 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1922 1923 } 1924 1925 static void 1926 em_handle_tx(void *context, int pending) 1927 { 1928 struct adapter *adapter = context; 1929 struct ifnet *ifp = adapter->ifp; 1930 1931 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1932 if (!EM_TX_TRYLOCK(adapter)) 1933 return; 1934 1935 em_txeof(adapter); 1936 if (!ADAPTER_RING_EMPTY(adapter)) 1937 em_start_locked(ifp); 1938 EM_TX_UNLOCK(adapter); 1939 } 1940 } 1941 #endif /* EM_FAST_IRQ */ 1942 1943 /********************************************************************* 1944 * 1945 * Media Ioctl callback 1946 * 1947 * This routine is called whenever the user queries the status of 1948 * the interface using ifconfig. 1949 * 1950 **********************************************************************/ 1951 static void 1952 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1953 { 1954 struct adapter *adapter = ifp->if_softc; 1955 u_char fiber_type = IFM_1000_SX; 1956 1957 INIT_DEBUGOUT("em_media_status: begin"); 1958 1959 EM_CORE_LOCK(adapter); 1960 em_update_link_status(adapter); 1961 1962 ifmr->ifm_status = IFM_AVALID; 1963 ifmr->ifm_active = IFM_ETHER; 1964 1965 if (!adapter->link_active) { 1966 EM_CORE_UNLOCK(adapter); 1967 return; 1968 } 1969 1970 ifmr->ifm_status |= IFM_ACTIVE; 1971 1972 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1973 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1974 if (adapter->hw.mac.type == e1000_82545) 1975 fiber_type = IFM_1000_LX; 1976 ifmr->ifm_active |= fiber_type | IFM_FDX; 1977 } else { 1978 switch (adapter->link_speed) { 1979 case 10: 1980 ifmr->ifm_active |= IFM_10_T; 1981 break; 1982 case 100: 1983 ifmr->ifm_active |= IFM_100_TX; 1984 break; 1985 case 1000: 1986 ifmr->ifm_active |= IFM_1000_T; 1987 break; 1988 } 1989 if (adapter->link_duplex == FULL_DUPLEX) 1990 ifmr->ifm_active |= IFM_FDX; 1991 else 1992 ifmr->ifm_active |= IFM_HDX; 1993 } 1994 EM_CORE_UNLOCK(adapter); 1995 } 1996 1997 /********************************************************************* 1998 * 1999 * Media Ioctl callback 2000 * 2001 * This routine is called when the user changes speed/duplex using 2002 * media/mediopt option with ifconfig. 2003 * 2004 **********************************************************************/ 2005 static int 2006 em_media_change(struct ifnet *ifp) 2007 { 2008 struct adapter *adapter = ifp->if_softc; 2009 struct ifmedia *ifm = &adapter->media; 2010 2011 INIT_DEBUGOUT("em_media_change: begin"); 2012 2013 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2014 return (EINVAL); 2015 2016 EM_CORE_LOCK(adapter); 2017 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2018 case IFM_AUTO: 2019 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2020 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2021 break; 2022 case IFM_1000_LX: 2023 case IFM_1000_SX: 2024 case IFM_1000_T: 2025 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2026 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2027 break; 2028 case IFM_100_TX: 2029 adapter->hw.mac.autoneg = FALSE; 2030 adapter->hw.phy.autoneg_advertised = 0; 2031 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2032 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 2033 else 2034 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 2035 break; 2036 case IFM_10_T: 2037 adapter->hw.mac.autoneg = FALSE; 2038 adapter->hw.phy.autoneg_advertised = 0; 2039 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2040 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 2041 else 2042 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 2043 break; 2044 default: 2045 device_printf(adapter->dev, "Unsupported media type\n"); 2046 } 2047 2048 /* As the speed/duplex settings my have changed we need to 2049 * reset the PHY. 2050 */ 2051 adapter->hw.phy.reset_disable = FALSE; 2052 2053 em_init_locked(adapter); 2054 EM_CORE_UNLOCK(adapter); 2055 2056 return (0); 2057 } 2058 2059 /********************************************************************* 2060 * 2061 * This routine maps the mbufs to tx descriptors. 2062 * 2063 * return 0 on success, positive on failure 2064 **********************************************************************/ 2065 2066 static int 2067 em_xmit(struct adapter *adapter, struct mbuf **m_headp) 2068 { 2069 bus_dma_segment_t segs[EM_MAX_SCATTER]; 2070 bus_dmamap_t map; 2071 struct em_buffer *tx_buffer, *tx_buffer_mapped; 2072 struct e1000_tx_desc *ctxd = NULL; 2073 struct mbuf *m_head; 2074 u32 txd_upper, txd_lower, txd_used, txd_saved; 2075 int nsegs, i, j, first, last = 0; 2076 int error, do_tso, tso_desc = 0; 2077 #if __FreeBSD_version < 700000 2078 struct m_tag *mtag; 2079 #endif 2080 m_head = *m_headp; 2081 txd_upper = txd_lower = txd_used = txd_saved = 0; 2082 2083 #if __FreeBSD_version >= 700000 2084 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 2085 #else 2086 do_tso = 0; 2087 #endif 2088 2089 /* 2090 * Force a cleanup if number of TX descriptors 2091 * available hits the threshold 2092 */ 2093 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { 2094 em_txeof(adapter); 2095 /* Now do we at least have a minimal? */ 2096 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { 2097 adapter->no_tx_desc_avail1++; 2098 return (ENOBUFS); 2099 } 2100 } 2101 2102 2103 /* 2104 * TSO workaround: 2105 * If an mbuf is only header we need 2106 * to pull 4 bytes of data into it. 2107 */ 2108 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 2109 m_head = m_pullup(m_head, M_TSO_LEN + 4); 2110 *m_headp = m_head; 2111 if (m_head == NULL) 2112 return (ENOBUFS); 2113 } 2114 2115 /* 2116 * Map the packet for DMA 2117 * 2118 * Capture the first descriptor index, 2119 * this descriptor will have the index 2120 * of the EOP which is the only one that 2121 * now gets a DONE bit writeback. 2122 */ 2123 first = adapter->next_avail_tx_desc; 2124 tx_buffer = &adapter->tx_buffer_area[first]; 2125 tx_buffer_mapped = tx_buffer; 2126 map = tx_buffer->map; 2127 2128 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2129 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2130 2131 /* 2132 * There are two types of errors we can (try) to handle: 2133 * - EFBIG means the mbuf chain was too long and bus_dma ran 2134 * out of segments. Defragment the mbuf chain and try again. 2135 * - ENOMEM means bus_dma could not obtain enough bounce buffers 2136 * at this point in time. Defer sending and try again later. 2137 * All other errors, in particular EINVAL, are fatal and prevent the 2138 * mbuf chain from ever going through. Drop it and report error. 2139 */ 2140 if (error == EFBIG) { 2141 struct mbuf *m; 2142 2143 m = m_defrag(*m_headp, M_DONTWAIT); 2144 if (m == NULL) { 2145 adapter->mbuf_alloc_failed++; 2146 m_freem(*m_headp); 2147 *m_headp = NULL; 2148 return (ENOBUFS); 2149 } 2150 *m_headp = m; 2151 2152 /* Try it again */ 2153 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2154 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2155 2156 if (error) { 2157 adapter->no_tx_dma_setup++; 2158 m_freem(*m_headp); 2159 *m_headp = NULL; 2160 return (error); 2161 } 2162 } else if (error != 0) { 2163 adapter->no_tx_dma_setup++; 2164 return (error); 2165 } 2166 2167 /* 2168 * TSO Hardware workaround, if this packet is not 2169 * TSO, and is only a single descriptor long, and 2170 * it follows a TSO burst, then we need to add a 2171 * sentinel descriptor to prevent premature writeback. 2172 */ 2173 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { 2174 if (nsegs == 1) 2175 tso_desc = TRUE; 2176 adapter->tx_tso = FALSE; 2177 } 2178 2179 if (nsegs > (adapter->num_tx_desc_avail - 2)) { 2180 adapter->no_tx_desc_avail2++; 2181 bus_dmamap_unload(adapter->txtag, map); 2182 return (ENOBUFS); 2183 } 2184 m_head = *m_headp; 2185 2186 /* Do hardware assists */ 2187 #if __FreeBSD_version >= 700000 2188 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 2189 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 2190 if (error != TRUE) 2191 return (ENXIO); /* something foobar */ 2192 /* we need to make a final sentinel transmit desc */ 2193 tso_desc = TRUE; 2194 } else 2195 #endif 2196 #ifndef EM_TIMESYNC 2197 /* 2198 ** Timesync needs to check the packet header 2199 ** so call checksum code to do so, but don't 2200 ** penalize the code if not defined. 2201 */ 2202 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 2203 #endif 2204 em_transmit_checksum_setup(adapter, m_head, 2205 &txd_upper, &txd_lower); 2206 2207 i = adapter->next_avail_tx_desc; 2208 if (adapter->pcix_82544) 2209 txd_saved = i; 2210 2211 /* Set up our transmit descriptors */ 2212 for (j = 0; j < nsegs; j++) { 2213 bus_size_t seg_len; 2214 bus_addr_t seg_addr; 2215 /* If adapter is 82544 and on PCIX bus */ 2216 if(adapter->pcix_82544) { 2217 DESC_ARRAY desc_array; 2218 u32 array_elements, counter; 2219 /* 2220 * Check the Address and Length combination and 2221 * split the data accordingly 2222 */ 2223 array_elements = em_fill_descriptors(segs[j].ds_addr, 2224 segs[j].ds_len, &desc_array); 2225 for (counter = 0; counter < array_elements; counter++) { 2226 if (txd_used == adapter->num_tx_desc_avail) { 2227 adapter->next_avail_tx_desc = txd_saved; 2228 adapter->no_tx_desc_avail2++; 2229 bus_dmamap_unload(adapter->txtag, map); 2230 return (ENOBUFS); 2231 } 2232 tx_buffer = &adapter->tx_buffer_area[i]; 2233 ctxd = &adapter->tx_desc_base[i]; 2234 ctxd->buffer_addr = htole64( 2235 desc_array.descriptor[counter].address); 2236 ctxd->lower.data = htole32( 2237 (adapter->txd_cmd | txd_lower | (u16) 2238 desc_array.descriptor[counter].length)); 2239 ctxd->upper.data = 2240 htole32((txd_upper)); 2241 last = i; 2242 if (++i == adapter->num_tx_desc) 2243 i = 0; 2244 tx_buffer->m_head = NULL; 2245 tx_buffer->next_eop = -1; 2246 txd_used++; 2247 } 2248 } else { 2249 tx_buffer = &adapter->tx_buffer_area[i]; 2250 ctxd = &adapter->tx_desc_base[i]; 2251 seg_addr = segs[j].ds_addr; 2252 seg_len = segs[j].ds_len; 2253 /* 2254 ** TSO Workaround: 2255 ** If this is the last descriptor, we want to 2256 ** split it so we have a small final sentinel 2257 */ 2258 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 2259 seg_len -= 4; 2260 ctxd->buffer_addr = htole64(seg_addr); 2261 ctxd->lower.data = htole32( 2262 adapter->txd_cmd | txd_lower | seg_len); 2263 ctxd->upper.data = 2264 htole32(txd_upper); 2265 if (++i == adapter->num_tx_desc) 2266 i = 0; 2267 /* Now make the sentinel */ 2268 ++txd_used; /* using an extra txd */ 2269 ctxd = &adapter->tx_desc_base[i]; 2270 tx_buffer = &adapter->tx_buffer_area[i]; 2271 ctxd->buffer_addr = 2272 htole64(seg_addr + seg_len); 2273 ctxd->lower.data = htole32( 2274 adapter->txd_cmd | txd_lower | 4); 2275 ctxd->upper.data = 2276 htole32(txd_upper); 2277 last = i; 2278 if (++i == adapter->num_tx_desc) 2279 i = 0; 2280 } else { 2281 ctxd->buffer_addr = htole64(seg_addr); 2282 ctxd->lower.data = htole32( 2283 adapter->txd_cmd | txd_lower | seg_len); 2284 ctxd->upper.data = 2285 htole32(txd_upper); 2286 last = i; 2287 if (++i == adapter->num_tx_desc) 2288 i = 0; 2289 } 2290 tx_buffer->m_head = NULL; 2291 tx_buffer->next_eop = -1; 2292 } 2293 } 2294 2295 adapter->next_avail_tx_desc = i; 2296 if (adapter->pcix_82544) 2297 adapter->num_tx_desc_avail -= txd_used; 2298 else { 2299 adapter->num_tx_desc_avail -= nsegs; 2300 if (tso_desc) /* TSO used an extra for sentinel */ 2301 adapter->num_tx_desc_avail -= txd_used; 2302 } 2303 2304 /* 2305 ** Handle VLAN tag, this is the 2306 ** biggest difference between 2307 ** 6.x and 7 2308 */ 2309 #if __FreeBSD_version < 700000 2310 /* Find out if we are in vlan mode. */ 2311 mtag = VLAN_OUTPUT_TAG(ifp, m_head); 2312 if (mtag != NULL) { 2313 ctxd->upper.fields.special = 2314 htole16(VLAN_TAG_VALUE(mtag)); 2315 #else /* FreeBSD 7 */ 2316 if (m_head->m_flags & M_VLANTAG) { 2317 /* Set the vlan id. */ 2318 ctxd->upper.fields.special = 2319 htole16(m_head->m_pkthdr.ether_vtag); 2320 #endif 2321 /* Tell hardware to add tag */ 2322 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2323 } 2324 2325 tx_buffer->m_head = m_head; 2326 tx_buffer_mapped->map = tx_buffer->map; 2327 tx_buffer->map = map; 2328 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 2329 2330 /* 2331 * Last Descriptor of Packet 2332 * needs End Of Packet (EOP) 2333 * and Report Status (RS) 2334 */ 2335 ctxd->lower.data |= 2336 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2337 /* 2338 * Keep track in the first buffer which 2339 * descriptor will be written back 2340 */ 2341 tx_buffer = &adapter->tx_buffer_area[first]; 2342 tx_buffer->next_eop = last; 2343 2344 /* 2345 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2346 * that this frame is available to transmit. 2347 */ 2348 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 2349 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2350 if (adapter->hw.mac.type == e1000_82547 && 2351 adapter->link_duplex == HALF_DUPLEX) 2352 em_82547_move_tail(adapter); 2353 else { 2354 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 2355 if (adapter->hw.mac.type == e1000_82547) 2356 em_82547_update_fifo_head(adapter, 2357 m_head->m_pkthdr.len); 2358 } 2359 2360 #ifdef EM_TIMESYNC 2361 if (ctxd->upper.data & E1000_TXD_EXTCMD_TSTAMP) { 2362 HW_DEBUGOUT( "@@@ Timestamp bit is set in transmit descriptor\n" ); 2363 } 2364 #endif 2365 return (0); 2366 } 2367 2368 /********************************************************************* 2369 * 2370 * 82547 workaround to avoid controller hang in half-duplex environment. 2371 * The workaround is to avoid queuing a large packet that would span 2372 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 2373 * in this case. We do that only when FIFO is quiescent. 2374 * 2375 **********************************************************************/ 2376 static void 2377 em_82547_move_tail(void *arg) 2378 { 2379 struct adapter *adapter = arg; 2380 struct e1000_tx_desc *tx_desc; 2381 u16 hw_tdt, sw_tdt, length = 0; 2382 bool eop = 0; 2383 2384 EM_TX_LOCK_ASSERT(adapter); 2385 2386 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 2387 sw_tdt = adapter->next_avail_tx_desc; 2388 2389 while (hw_tdt != sw_tdt) { 2390 tx_desc = &adapter->tx_desc_base[hw_tdt]; 2391 length += tx_desc->lower.flags.length; 2392 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 2393 if (++hw_tdt == adapter->num_tx_desc) 2394 hw_tdt = 0; 2395 2396 if (eop) { 2397 if (em_82547_fifo_workaround(adapter, length)) { 2398 adapter->tx_fifo_wrk_cnt++; 2399 callout_reset(&adapter->tx_fifo_timer, 1, 2400 em_82547_move_tail, adapter); 2401 break; 2402 } 2403 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 2404 em_82547_update_fifo_head(adapter, length); 2405 length = 0; 2406 } 2407 } 2408 } 2409 2410 static int 2411 em_82547_fifo_workaround(struct adapter *adapter, int len) 2412 { 2413 int fifo_space, fifo_pkt_len; 2414 2415 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2416 2417 if (adapter->link_duplex == HALF_DUPLEX) { 2418 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 2419 2420 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 2421 if (em_82547_tx_fifo_reset(adapter)) 2422 return (0); 2423 else 2424 return (1); 2425 } 2426 } 2427 2428 return (0); 2429 } 2430 2431 static void 2432 em_82547_update_fifo_head(struct adapter *adapter, int len) 2433 { 2434 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2435 2436 /* tx_fifo_head is always 16 byte aligned */ 2437 adapter->tx_fifo_head += fifo_pkt_len; 2438 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { 2439 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2440 } 2441 } 2442 2443 2444 static int 2445 em_82547_tx_fifo_reset(struct adapter *adapter) 2446 { 2447 u32 tctl; 2448 2449 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2450 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2451 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2452 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2453 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2454 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2455 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2456 /* Disable TX unit */ 2457 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2458 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2459 tctl & ~E1000_TCTL_EN); 2460 2461 /* Reset FIFO pointers */ 2462 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2463 adapter->tx_head_addr); 2464 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2465 adapter->tx_head_addr); 2466 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2467 adapter->tx_head_addr); 2468 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2469 adapter->tx_head_addr); 2470 2471 /* Re-enable TX unit */ 2472 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2473 E1000_WRITE_FLUSH(&adapter->hw); 2474 2475 adapter->tx_fifo_head = 0; 2476 adapter->tx_fifo_reset_cnt++; 2477 2478 return (TRUE); 2479 } 2480 else { 2481 return (FALSE); 2482 } 2483 } 2484 2485 static void 2486 em_set_promisc(struct adapter *adapter) 2487 { 2488 struct ifnet *ifp = adapter->ifp; 2489 u32 reg_rctl; 2490 2491 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2492 2493 if (ifp->if_flags & IFF_PROMISC) { 2494 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2495 /* Turn this on if you want to see bad packets */ 2496 if (em_debug_sbp) 2497 reg_rctl |= E1000_RCTL_SBP; 2498 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2499 } else if (ifp->if_flags & IFF_ALLMULTI) { 2500 reg_rctl |= E1000_RCTL_MPE; 2501 reg_rctl &= ~E1000_RCTL_UPE; 2502 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2503 } 2504 } 2505 2506 static void 2507 em_disable_promisc(struct adapter *adapter) 2508 { 2509 u32 reg_rctl; 2510 2511 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2512 2513 reg_rctl &= (~E1000_RCTL_UPE); 2514 reg_rctl &= (~E1000_RCTL_MPE); 2515 reg_rctl &= (~E1000_RCTL_SBP); 2516 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2517 } 2518 2519 2520 /********************************************************************* 2521 * Multicast Update 2522 * 2523 * This routine is called whenever multicast address list is updated. 2524 * 2525 **********************************************************************/ 2526 2527 static void 2528 em_set_multi(struct adapter *adapter) 2529 { 2530 struct ifnet *ifp = adapter->ifp; 2531 struct ifmultiaddr *ifma; 2532 u32 reg_rctl = 0; 2533 u8 *mta; /* Multicast array memory */ 2534 int mcnt = 0; 2535 2536 IOCTL_DEBUGOUT("em_set_multi: begin"); 2537 2538 if (adapter->hw.mac.type == e1000_82542 && 2539 adapter->hw.revision_id == E1000_REVISION_2) { 2540 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2541 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2542 e1000_pci_clear_mwi(&adapter->hw); 2543 reg_rctl |= E1000_RCTL_RST; 2544 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2545 msec_delay(5); 2546 } 2547 2548 /* Allocate temporary memory to setup array */ 2549 mta = malloc(sizeof(u8) * 2550 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 2551 M_DEVBUF, M_NOWAIT | M_ZERO); 2552 if (mta == NULL) 2553 panic("em_set_multi memory failure\n"); 2554 2555 IF_ADDR_LOCK(ifp); 2556 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2557 if (ifma->ifma_addr->sa_family != AF_LINK) 2558 continue; 2559 2560 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2561 break; 2562 2563 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2564 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2565 mcnt++; 2566 } 2567 IF_ADDR_UNLOCK(ifp); 2568 2569 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2570 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2571 reg_rctl |= E1000_RCTL_MPE; 2572 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2573 } else 2574 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2575 2576 if (adapter->hw.mac.type == e1000_82542 && 2577 adapter->hw.revision_id == E1000_REVISION_2) { 2578 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2579 reg_rctl &= ~E1000_RCTL_RST; 2580 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2581 msec_delay(5); 2582 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2583 e1000_pci_set_mwi(&adapter->hw); 2584 } 2585 free(mta, M_DEVBUF); 2586 } 2587 2588 2589 /********************************************************************* 2590 * Timer routine 2591 * 2592 * This routine checks for link status and updates statistics. 2593 * 2594 **********************************************************************/ 2595 2596 static void 2597 em_local_timer(void *arg) 2598 { 2599 struct adapter *adapter = arg; 2600 struct ifnet *ifp = adapter->ifp; 2601 2602 EM_CORE_LOCK_ASSERT(adapter); 2603 2604 taskqueue_enqueue(adapter->tq, 2605 &adapter->rxtx_task); 2606 em_update_link_status(adapter); 2607 em_update_stats_counters(adapter); 2608 2609 /* Reset LAA into RAR[0] on 82571 */ 2610 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2611 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2612 2613 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2614 em_print_hw_stats(adapter); 2615 2616 em_smartspeed(adapter); 2617 2618 /* 2619 * Each second we check the watchdog to 2620 * protect against hardware hangs. 2621 */ 2622 em_watchdog(adapter); 2623 2624 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2625 2626 } 2627 2628 static void 2629 em_update_link_status(struct adapter *adapter) 2630 { 2631 struct e1000_hw *hw = &adapter->hw; 2632 struct ifnet *ifp = adapter->ifp; 2633 device_t dev = adapter->dev; 2634 u32 link_check = 0; 2635 2636 /* Get the cached link value or read phy for real */ 2637 switch (hw->phy.media_type) { 2638 case e1000_media_type_copper: 2639 if (hw->mac.get_link_status) { 2640 /* Do the work to read phy */ 2641 e1000_check_for_link(hw); 2642 link_check = !hw->mac.get_link_status; 2643 if (link_check) /* ESB2 fix */ 2644 e1000_cfg_on_link_up(hw); 2645 } else 2646 link_check = TRUE; 2647 break; 2648 case e1000_media_type_fiber: 2649 e1000_check_for_link(hw); 2650 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2651 E1000_STATUS_LU); 2652 break; 2653 case e1000_media_type_internal_serdes: 2654 e1000_check_for_link(hw); 2655 link_check = adapter->hw.mac.serdes_has_link; 2656 break; 2657 default: 2658 case e1000_media_type_unknown: 2659 break; 2660 } 2661 2662 /* Now check for a transition */ 2663 if (link_check && (adapter->link_active == 0)) { 2664 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2665 &adapter->link_duplex); 2666 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2667 if ((adapter->link_speed != SPEED_1000) && 2668 ((hw->mac.type == e1000_82571) || 2669 (hw->mac.type == e1000_82572))) { 2670 int tarc0; 2671 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2672 tarc0 &= ~SPEED_MODE_BIT; 2673 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2674 } 2675 if (bootverbose) 2676 device_printf(dev, "Link is up %d Mbps %s\n", 2677 adapter->link_speed, 2678 ((adapter->link_duplex == FULL_DUPLEX) ? 2679 "Full Duplex" : "Half Duplex")); 2680 adapter->link_active = 1; 2681 adapter->smartspeed = 0; 2682 ifp->if_baudrate = adapter->link_speed * 1000000; 2683 if_link_state_change(ifp, LINK_STATE_UP); 2684 } else if (!link_check && (adapter->link_active == 1)) { 2685 ifp->if_baudrate = adapter->link_speed = 0; 2686 adapter->link_duplex = 0; 2687 if (bootverbose) 2688 device_printf(dev, "Link is Down\n"); 2689 adapter->link_active = 0; 2690 /* Link down, disable watchdog */ 2691 adapter->watchdog_timer = FALSE; 2692 if_link_state_change(ifp, LINK_STATE_DOWN); 2693 } 2694 } 2695 2696 /********************************************************************* 2697 * 2698 * This routine disables all traffic on the adapter by issuing a 2699 * global reset on the MAC and deallocates TX/RX buffers. 2700 * 2701 * This routine should always be called with BOTH the CORE 2702 * and TX locks. 2703 **********************************************************************/ 2704 2705 static void 2706 em_stop(void *arg) 2707 { 2708 struct adapter *adapter = arg; 2709 struct ifnet *ifp = adapter->ifp; 2710 2711 EM_CORE_LOCK_ASSERT(adapter); 2712 EM_TX_LOCK_ASSERT(adapter); 2713 2714 INIT_DEBUGOUT("em_stop: begin"); 2715 2716 em_disable_intr(adapter); 2717 callout_stop(&adapter->timer); 2718 callout_stop(&adapter->tx_fifo_timer); 2719 2720 /* Tell the stack that the interface is no longer active */ 2721 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2722 2723 #ifdef EM_TIMESYNC 2724 /* Disable IEEE 1588 Time hardware */ 2725 if ((adapter->hw.mac.type == e1000_82574) || 2726 (adapter->hw.mac.type == e1000_ich10lan)) 2727 em_tsync_disable(adapter); 2728 #endif 2729 2730 e1000_reset_hw(&adapter->hw); 2731 if (adapter->hw.mac.type >= e1000_82544) 2732 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2733 } 2734 2735 2736 /********************************************************************* 2737 * 2738 * Determine hardware revision. 2739 * 2740 **********************************************************************/ 2741 static void 2742 em_identify_hardware(struct adapter *adapter) 2743 { 2744 device_t dev = adapter->dev; 2745 2746 /* Make sure our PCI config space has the necessary stuff set */ 2747 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2748 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2749 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2750 device_printf(dev, "Memory Access and/or Bus Master bits " 2751 "were not set!\n"); 2752 adapter->hw.bus.pci_cmd_word |= 2753 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2754 pci_write_config(dev, PCIR_COMMAND, 2755 adapter->hw.bus.pci_cmd_word, 2); 2756 } 2757 2758 /* Save off the information about this board */ 2759 adapter->hw.vendor_id = pci_get_vendor(dev); 2760 adapter->hw.device_id = pci_get_device(dev); 2761 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2762 adapter->hw.subsystem_vendor_id = 2763 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2764 adapter->hw.subsystem_device_id = 2765 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2766 2767 /* Do Shared Code Init and Setup */ 2768 if (e1000_set_mac_type(&adapter->hw)) { 2769 device_printf(dev, "Setup init failure\n"); 2770 return; 2771 } 2772 } 2773 2774 static int 2775 em_allocate_pci_resources(struct adapter *adapter) 2776 { 2777 device_t dev = adapter->dev; 2778 int val, rid, error = E1000_SUCCESS; 2779 2780 rid = PCIR_BAR(0); 2781 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2782 &rid, RF_ACTIVE); 2783 if (adapter->memory == NULL) { 2784 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2785 return (ENXIO); 2786 } 2787 adapter->osdep.mem_bus_space_tag = 2788 rman_get_bustag(adapter->memory); 2789 adapter->osdep.mem_bus_space_handle = 2790 rman_get_bushandle(adapter->memory); 2791 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2792 2793 /* Only older adapters use IO mapping */ 2794 if ((adapter->hw.mac.type > e1000_82543) && 2795 (adapter->hw.mac.type < e1000_82571)) { 2796 /* Figure our where our IO BAR is ? */ 2797 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2798 val = pci_read_config(dev, rid, 4); 2799 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2800 adapter->io_rid = rid; 2801 break; 2802 } 2803 rid += 4; 2804 /* check for 64bit BAR */ 2805 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2806 rid += 4; 2807 } 2808 if (rid >= PCIR_CIS) { 2809 device_printf(dev, "Unable to locate IO BAR\n"); 2810 return (ENXIO); 2811 } 2812 adapter->ioport = bus_alloc_resource_any(dev, 2813 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); 2814 if (adapter->ioport == NULL) { 2815 device_printf(dev, "Unable to allocate bus resource: " 2816 "ioport\n"); 2817 return (ENXIO); 2818 } 2819 adapter->hw.io_base = 0; 2820 adapter->osdep.io_bus_space_tag = 2821 rman_get_bustag(adapter->ioport); 2822 adapter->osdep.io_bus_space_handle = 2823 rman_get_bushandle(adapter->ioport); 2824 } 2825 2826 /* 2827 ** Init the resource arrays 2828 ** used by MSIX setup 2829 */ 2830 for (int i = 0; i < 3; i++) { 2831 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2832 adapter->tag[i] = NULL; 2833 adapter->res[i] = NULL; 2834 } 2835 2836 /* 2837 * Setup MSI/X or MSI if PCI Express 2838 */ 2839 if (em_enable_msi) 2840 adapter->msi = em_setup_msix(adapter); 2841 2842 adapter->hw.back = &adapter->osdep; 2843 2844 return (error); 2845 } 2846 2847 /********************************************************************* 2848 * 2849 * Setup the Legacy or MSI Interrupt handler 2850 * 2851 **********************************************************************/ 2852 int 2853 em_allocate_legacy(struct adapter *adapter) 2854 { 2855 device_t dev = adapter->dev; 2856 int error; 2857 2858 /* Manually turn off all interrupts */ 2859 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2860 2861 /* Legacy RID is 0 */ 2862 if (adapter->msi == 0) 2863 adapter->rid[0] = 0; 2864 2865 /* We allocate a single interrupt resource */ 2866 adapter->res[0] = bus_alloc_resource_any(dev, 2867 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2868 if (adapter->res[0] == NULL) { 2869 device_printf(dev, "Unable to allocate bus resource: " 2870 "interrupt\n"); 2871 return (ENXIO); 2872 } 2873 2874 #ifdef EM_LEGACY_IRQ 2875 /* We do Legacy setup */ 2876 if ((error = bus_setup_intr(dev, adapter->res[0], 2877 #if __FreeBSD_version > 700000 2878 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, 2879 #else /* 6.X */ 2880 INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, 2881 #endif 2882 &adapter->tag[0])) != 0) { 2883 device_printf(dev, "Failed to register interrupt handler"); 2884 return (error); 2885 } 2886 2887 #else /* FAST_IRQ */ 2888 /* 2889 * Try allocating a fast interrupt and the associated deferred 2890 * processing contexts. 2891 */ 2892 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); 2893 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2894 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2895 taskqueue_thread_enqueue, &adapter->tq); 2896 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2897 device_get_nameunit(adapter->dev)); 2898 #if __FreeBSD_version < 700000 2899 if ((error = bus_setup_intr(dev, adapter->res[0], 2900 INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, 2901 #else 2902 if ((error = bus_setup_intr(dev, adapter->res[0], 2903 INTR_TYPE_NET, em_irq_fast, NULL, adapter, 2904 #endif 2905 &adapter->tag[0])) != 0) { 2906 device_printf(dev, "Failed to register fast interrupt " 2907 "handler: %d\n", error); 2908 taskqueue_free(adapter->tq); 2909 adapter->tq = NULL; 2910 return (error); 2911 } 2912 #endif /* EM_LEGACY_IRQ */ 2913 2914 return (0); 2915 } 2916 2917 /********************************************************************* 2918 * 2919 * Setup the MSIX Interrupt handlers 2920 * This is not really Multiqueue, rather 2921 * its just multiple interrupt vectors. 2922 * 2923 **********************************************************************/ 2924 int 2925 em_allocate_msix(struct adapter *adapter) 2926 { 2927 device_t dev = adapter->dev; 2928 int error; 2929 2930 /* Make sure all interrupts are disabled */ 2931 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2932 2933 /* First get the resources */ 2934 for (int i = 0; i < adapter->msi; i++) { 2935 adapter->res[i] = bus_alloc_resource_any(dev, 2936 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); 2937 if (adapter->res[i] == NULL) { 2938 device_printf(dev, 2939 "Unable to allocate bus resource: " 2940 "MSIX Interrupt\n"); 2941 return (ENXIO); 2942 } 2943 } 2944 2945 /* 2946 * Now allocate deferred processing contexts. 2947 */ 2948 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); 2949 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); 2950 /* 2951 * Handle compatibility for msi case for deferral due to 2952 * trylock failure 2953 */ 2954 TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter); 2955 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2956 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2957 taskqueue_thread_enqueue, &adapter->tq); 2958 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2959 device_get_nameunit(adapter->dev)); 2960 2961 /* 2962 * And setup the interrupt handlers 2963 */ 2964 2965 /* First slot to RX */ 2966 if ((error = bus_setup_intr(dev, adapter->res[0], 2967 #if __FreeBSD_version > 700000 2968 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter, 2969 #else /* 6.X */ 2970 INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter, 2971 #endif 2972 &adapter->tag[0])) != 0) { 2973 device_printf(dev, "Failed to register RX handler"); 2974 return (error); 2975 } 2976 2977 /* Next TX */ 2978 if ((error = bus_setup_intr(dev, adapter->res[1], 2979 #if __FreeBSD_version > 700000 2980 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter, 2981 #else /* 6.X */ 2982 INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter, 2983 #endif 2984 &adapter->tag[1])) != 0) { 2985 device_printf(dev, "Failed to register TX handler"); 2986 return (error); 2987 } 2988 2989 /* And Link */ 2990 if ((error = bus_setup_intr(dev, adapter->res[2], 2991 #if __FreeBSD_version > 700000 2992 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter, 2993 #else /* 6.X */ 2994 INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter, 2995 #endif 2996 &adapter->tag[2])) != 0) { 2997 device_printf(dev, "Failed to register TX handler"); 2998 return (error); 2999 } 3000 3001 return (0); 3002 } 3003 3004 3005 static void 3006 em_free_pci_resources(struct adapter *adapter) 3007 { 3008 device_t dev = adapter->dev; 3009 3010 /* Make sure the for loop below runs once */ 3011 if (adapter->msi == 0) 3012 adapter->msi = 1; 3013 3014 /* 3015 * First release all the interrupt resources: 3016 * notice that since these are just kept 3017 * in an array we can do the same logic 3018 * whether its MSIX or just legacy. 3019 */ 3020 for (int i = 0; i < adapter->msi; i++) { 3021 if (adapter->tag[i] != NULL) { 3022 bus_teardown_intr(dev, adapter->res[i], 3023 adapter->tag[i]); 3024 adapter->tag[i] = NULL; 3025 } 3026 if (adapter->res[i] != NULL) { 3027 bus_release_resource(dev, SYS_RES_IRQ, 3028 adapter->rid[i], adapter->res[i]); 3029 } 3030 } 3031 3032 if (adapter->msi) 3033 pci_release_msi(dev); 3034 3035 if (adapter->msix != NULL) 3036 bus_release_resource(dev, SYS_RES_MEMORY, 3037 PCIR_BAR(EM_MSIX_BAR), adapter->msix); 3038 3039 if (adapter->memory != NULL) 3040 bus_release_resource(dev, SYS_RES_MEMORY, 3041 PCIR_BAR(0), adapter->memory); 3042 3043 if (adapter->flash != NULL) 3044 bus_release_resource(dev, SYS_RES_MEMORY, 3045 EM_FLASH, adapter->flash); 3046 3047 if (adapter->ioport != NULL) 3048 bus_release_resource(dev, SYS_RES_IOPORT, 3049 adapter->io_rid, adapter->ioport); 3050 } 3051 3052 /* 3053 * Setup MSI or MSI/X 3054 */ 3055 static int 3056 em_setup_msix(struct adapter *adapter) 3057 { 3058 device_t dev = adapter->dev; 3059 int val = 0; 3060 3061 if (adapter->hw.mac.type < e1000_82571) 3062 return (0); 3063 3064 /* Setup MSI/X for Hartwell */ 3065 if (adapter->hw.mac.type == e1000_82574) { 3066 /* Map the MSIX BAR */ 3067 int rid = PCIR_BAR(EM_MSIX_BAR); 3068 adapter->msix = bus_alloc_resource_any(dev, 3069 SYS_RES_MEMORY, &rid, RF_ACTIVE); 3070 if (!adapter->msix) { 3071 /* May not be enabled */ 3072 device_printf(adapter->dev, 3073 "Unable to map MSIX table \n"); 3074 goto msi; 3075 } 3076 val = pci_msix_count(dev); 3077 /* 3078 ** 82574 can be configured for 5 but 3079 ** we limit use to 3. 3080 */ 3081 if (val > 3) val = 3; 3082 if ((val) && pci_alloc_msix(dev, &val) == 0) { 3083 device_printf(adapter->dev,"Using MSIX interrupts\n"); 3084 return (val); 3085 } 3086 } 3087 msi: 3088 val = pci_msi_count(dev); 3089 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 3090 adapter->msi = 1; 3091 device_printf(adapter->dev,"Using MSI interrupt\n"); 3092 return (val); 3093 } 3094 return (0); 3095 } 3096 3097 /********************************************************************* 3098 * 3099 * Initialize the hardware to a configuration 3100 * as specified by the adapter structure. 3101 * 3102 **********************************************************************/ 3103 static int 3104 em_hardware_init(struct adapter *adapter) 3105 { 3106 device_t dev = adapter->dev; 3107 u16 rx_buffer_size; 3108 3109 INIT_DEBUGOUT("em_hardware_init: begin"); 3110 3111 /* Issue a global reset */ 3112 e1000_reset_hw(&adapter->hw); 3113 3114 /* Get control from any management/hw control */ 3115 if (((adapter->hw.mac.type == e1000_82573) || 3116 (adapter->hw.mac.type == e1000_ich8lan) || 3117 (adapter->hw.mac.type == e1000_ich10lan) || 3118 (adapter->hw.mac.type == e1000_ich9lan)) && 3119 e1000_check_mng_mode(&adapter->hw)) 3120 em_get_hw_control(adapter); 3121 3122 /* When hardware is reset, fifo_head is also reset */ 3123 adapter->tx_fifo_head = 0; 3124 3125 /* Set up smart power down as default off on newer adapters. */ 3126 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || 3127 adapter->hw.mac.type == e1000_82572)) { 3128 u16 phy_tmp = 0; 3129 3130 /* Speed up time to link by disabling smart power down. */ 3131 e1000_read_phy_reg(&adapter->hw, 3132 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 3133 phy_tmp &= ~IGP02E1000_PM_SPD; 3134 e1000_write_phy_reg(&adapter->hw, 3135 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 3136 } 3137 3138 /* 3139 * These parameters control the automatic generation (Tx) and 3140 * response (Rx) to Ethernet PAUSE frames. 3141 * - High water mark should allow for at least two frames to be 3142 * received after sending an XOFF. 3143 * - Low water mark works best when it is very near the high water mark. 3144 * This allows the receiver to restart by sending XON when it has 3145 * drained a bit. Here we use an arbitary value of 1500 which will 3146 * restart after one full frame is pulled from the buffer. There 3147 * could be several smaller frames in the buffer and if so they will 3148 * not trigger the XON until their total number reduces the buffer 3149 * by 1500. 3150 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 3151 */ 3152 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 3153 0xffff) << 10 ); 3154 3155 adapter->hw.fc.high_water = rx_buffer_size - 3156 roundup2(adapter->max_frame_size, 1024); 3157 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 3158 3159 if (adapter->hw.mac.type == e1000_80003es2lan) 3160 adapter->hw.fc.pause_time = 0xFFFF; 3161 else 3162 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 3163 adapter->hw.fc.send_xon = TRUE; 3164 adapter->hw.fc.requested_mode = e1000_fc_full; 3165 3166 if (e1000_init_hw(&adapter->hw) < 0) { 3167 device_printf(dev, "Hardware Initialization Failed\n"); 3168 return (EIO); 3169 } 3170 3171 e1000_check_for_link(&adapter->hw); 3172 3173 return (0); 3174 } 3175 3176 /********************************************************************* 3177 * 3178 * Setup networking device structure and register an interface. 3179 * 3180 **********************************************************************/ 3181 static void 3182 em_setup_interface(device_t dev, struct adapter *adapter) 3183 { 3184 struct ifnet *ifp; 3185 3186 INIT_DEBUGOUT("em_setup_interface: begin"); 3187 3188 ifp = adapter->ifp = if_alloc(IFT_ETHER); 3189 if (ifp == NULL) 3190 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 3191 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 3192 ifp->if_mtu = ETHERMTU; 3193 ifp->if_init = em_init; 3194 ifp->if_softc = adapter; 3195 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 3196 ifp->if_ioctl = em_ioctl; 3197 ifp->if_start = em_start; 3198 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 3199 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 3200 IFQ_SET_READY(&ifp->if_snd); 3201 3202 ether_ifattach(ifp, adapter->hw.mac.addr); 3203 3204 ifp->if_capabilities = ifp->if_capenable = 0; 3205 3206 #ifdef IFNET_BUF_RING 3207 ifp->if_transmit = em_transmit; 3208 ifp->if_qflush = em_qflush; 3209 adapter->br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); 3210 #endif 3211 if (adapter->hw.mac.type >= e1000_82543) { 3212 int version_cap; 3213 #if __FreeBSD_version < 700000 3214 version_cap = IFCAP_HWCSUM; 3215 #else 3216 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 3217 #endif 3218 ifp->if_capabilities |= version_cap; 3219 ifp->if_capenable |= version_cap; 3220 } 3221 3222 #if __FreeBSD_version >= 700000 3223 /* Identify TSO capable adapters */ 3224 if ((adapter->hw.mac.type > e1000_82544) && 3225 (adapter->hw.mac.type != e1000_82547)) 3226 ifp->if_capabilities |= IFCAP_TSO4; 3227 /* 3228 * By default only enable on PCI-E, this 3229 * can be overriden by ifconfig. 3230 */ 3231 if (adapter->hw.mac.type >= e1000_82571) 3232 ifp->if_capenable |= IFCAP_TSO4; 3233 #endif 3234 3235 /* 3236 * Tell the upper layer(s) we support long frames. 3237 */ 3238 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 3239 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3240 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3241 3242 #ifdef DEVICE_POLLING 3243 ifp->if_capabilities |= IFCAP_POLLING; 3244 #endif 3245 3246 /* 3247 * Specify the media types supported by this adapter and register 3248 * callbacks to update media and link information 3249 */ 3250 ifmedia_init(&adapter->media, IFM_IMASK, 3251 em_media_change, em_media_status); 3252 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3253 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3254 u_char fiber_type = IFM_1000_SX; /* default type */ 3255 3256 if (adapter->hw.mac.type == e1000_82545) 3257 fiber_type = IFM_1000_LX; 3258 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 3259 0, NULL); 3260 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 3261 } else { 3262 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 3263 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 3264 0, NULL); 3265 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 3266 0, NULL); 3267 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 3268 0, NULL); 3269 if (adapter->hw.phy.type != e1000_phy_ife) { 3270 ifmedia_add(&adapter->media, 3271 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 3272 ifmedia_add(&adapter->media, 3273 IFM_ETHER | IFM_1000_T, 0, NULL); 3274 } 3275 } 3276 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 3277 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 3278 } 3279 3280 3281 /********************************************************************* 3282 * 3283 * Workaround for SmartSpeed on 82541 and 82547 controllers 3284 * 3285 **********************************************************************/ 3286 static void 3287 em_smartspeed(struct adapter *adapter) 3288 { 3289 u16 phy_tmp; 3290 3291 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || 3292 adapter->hw.mac.autoneg == 0 || 3293 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3294 return; 3295 3296 if (adapter->smartspeed == 0) { 3297 /* If Master/Slave config fault is asserted twice, 3298 * we assume back-to-back */ 3299 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3300 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3301 return; 3302 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3303 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3304 e1000_read_phy_reg(&adapter->hw, 3305 PHY_1000T_CTRL, &phy_tmp); 3306 if(phy_tmp & CR_1000T_MS_ENABLE) { 3307 phy_tmp &= ~CR_1000T_MS_ENABLE; 3308 e1000_write_phy_reg(&adapter->hw, 3309 PHY_1000T_CTRL, phy_tmp); 3310 adapter->smartspeed++; 3311 if(adapter->hw.mac.autoneg && 3312 !e1000_phy_setup_autoneg(&adapter->hw) && 3313 !e1000_read_phy_reg(&adapter->hw, 3314 PHY_CONTROL, &phy_tmp)) { 3315 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3316 MII_CR_RESTART_AUTO_NEG); 3317 e1000_write_phy_reg(&adapter->hw, 3318 PHY_CONTROL, phy_tmp); 3319 } 3320 } 3321 } 3322 return; 3323 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3324 /* If still no link, perhaps using 2/3 pair cable */ 3325 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 3326 phy_tmp |= CR_1000T_MS_ENABLE; 3327 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 3328 if(adapter->hw.mac.autoneg && 3329 !e1000_phy_setup_autoneg(&adapter->hw) && 3330 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 3331 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3332 MII_CR_RESTART_AUTO_NEG); 3333 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 3334 } 3335 } 3336 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3337 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) 3338 adapter->smartspeed = 0; 3339 } 3340 3341 3342 /* 3343 * Manage DMA'able memory. 3344 */ 3345 static void 3346 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3347 { 3348 if (error) 3349 return; 3350 *(bus_addr_t *) arg = segs[0].ds_addr; 3351 } 3352 3353 static int 3354 em_dma_malloc(struct adapter *adapter, bus_size_t size, 3355 struct em_dma_alloc *dma, int mapflags) 3356 { 3357 int error; 3358 3359 #if __FreeBSD_version >= 700000 3360 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3361 #else 3362 error = bus_dma_tag_create(NULL, /* parent */ 3363 #endif 3364 EM_DBA_ALIGN, 0, /* alignment, bounds */ 3365 BUS_SPACE_MAXADDR, /* lowaddr */ 3366 BUS_SPACE_MAXADDR, /* highaddr */ 3367 NULL, NULL, /* filter, filterarg */ 3368 size, /* maxsize */ 3369 1, /* nsegments */ 3370 size, /* maxsegsize */ 3371 0, /* flags */ 3372 NULL, /* lockfunc */ 3373 NULL, /* lockarg */ 3374 &dma->dma_tag); 3375 if (error) { 3376 device_printf(adapter->dev, 3377 "%s: bus_dma_tag_create failed: %d\n", 3378 __func__, error); 3379 goto fail_0; 3380 } 3381 3382 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3383 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3384 if (error) { 3385 device_printf(adapter->dev, 3386 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3387 __func__, (uintmax_t)size, error); 3388 goto fail_2; 3389 } 3390 3391 dma->dma_paddr = 0; 3392 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3393 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3394 if (error || dma->dma_paddr == 0) { 3395 device_printf(adapter->dev, 3396 "%s: bus_dmamap_load failed: %d\n", 3397 __func__, error); 3398 goto fail_3; 3399 } 3400 3401 return (0); 3402 3403 fail_3: 3404 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3405 fail_2: 3406 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3407 bus_dma_tag_destroy(dma->dma_tag); 3408 fail_0: 3409 dma->dma_map = NULL; 3410 dma->dma_tag = NULL; 3411 3412 return (error); 3413 } 3414 3415 static void 3416 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 3417 { 3418 if (dma->dma_tag == NULL) 3419 return; 3420 if (dma->dma_map != NULL) { 3421 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3422 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3423 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3424 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3425 dma->dma_map = NULL; 3426 } 3427 bus_dma_tag_destroy(dma->dma_tag); 3428 dma->dma_tag = NULL; 3429 } 3430 3431 3432 /********************************************************************* 3433 * 3434 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3435 * the information needed to transmit a packet on the wire. 3436 * 3437 **********************************************************************/ 3438 static int 3439 em_allocate_transmit_structures(struct adapter *adapter) 3440 { 3441 device_t dev = adapter->dev; 3442 struct em_buffer *tx_buffer; 3443 int error; 3444 3445 /* 3446 * Create DMA tags for tx descriptors 3447 */ 3448 #if __FreeBSD_version >= 700000 3449 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3450 #else 3451 if ((error = bus_dma_tag_create(NULL, /* parent */ 3452 #endif 3453 1, 0, /* alignment, bounds */ 3454 BUS_SPACE_MAXADDR, /* lowaddr */ 3455 BUS_SPACE_MAXADDR, /* highaddr */ 3456 NULL, NULL, /* filter, filterarg */ 3457 EM_TSO_SIZE, /* maxsize */ 3458 EM_MAX_SCATTER, /* nsegments */ 3459 EM_TSO_SEG_SIZE, /* maxsegsize */ 3460 0, /* flags */ 3461 NULL, /* lockfunc */ 3462 NULL, /* lockarg */ 3463 &adapter->txtag)) != 0) { 3464 device_printf(dev, "Unable to allocate TX DMA tag\n"); 3465 goto fail; 3466 } 3467 3468 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * 3469 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3470 if (adapter->tx_buffer_area == NULL) { 3471 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3472 error = ENOMEM; 3473 goto fail; 3474 } 3475 3476 /* Create the descriptor buffer dma maps */ 3477 for (int i = 0; i < adapter->num_tx_desc; i++) { 3478 tx_buffer = &adapter->tx_buffer_area[i]; 3479 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); 3480 if (error != 0) { 3481 device_printf(dev, "Unable to create TX DMA map\n"); 3482 goto fail; 3483 } 3484 tx_buffer->next_eop = -1; 3485 } 3486 3487 return (0); 3488 fail: 3489 em_free_transmit_structures(adapter); 3490 return (error); 3491 } 3492 3493 /********************************************************************* 3494 * 3495 * (Re)Initialize transmit structures. 3496 * 3497 **********************************************************************/ 3498 static void 3499 em_setup_transmit_structures(struct adapter *adapter) 3500 { 3501 struct em_buffer *tx_buffer; 3502 3503 /* Clear the old ring contents */ 3504 bzero(adapter->tx_desc_base, 3505 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3506 3507 /* Free any existing TX buffers */ 3508 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3509 tx_buffer = &adapter->tx_buffer_area[i]; 3510 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3511 BUS_DMASYNC_POSTWRITE); 3512 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 3513 m_freem(tx_buffer->m_head); 3514 tx_buffer->m_head = NULL; 3515 tx_buffer->next_eop = -1; 3516 } 3517 3518 /* Reset state */ 3519 adapter->next_avail_tx_desc = 0; 3520 adapter->next_tx_to_clean = 0; 3521 adapter->num_tx_desc_avail = adapter->num_tx_desc; 3522 3523 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3524 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3525 3526 return; 3527 } 3528 3529 /********************************************************************* 3530 * 3531 * Enable transmit unit. 3532 * 3533 **********************************************************************/ 3534 static void 3535 em_initialize_transmit_unit(struct adapter *adapter) 3536 { 3537 u32 tctl, tarc, tipg = 0; 3538 u64 bus_addr; 3539 3540 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3541 /* Setup the Base and Length of the Tx Descriptor Ring */ 3542 bus_addr = adapter->txdma.dma_paddr; 3543 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 3544 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3545 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 3546 (u32)(bus_addr >> 32)); 3547 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 3548 (u32)bus_addr); 3549 /* Setup the HW Tx Head and Tail descriptor pointers */ 3550 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 3551 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 3552 3553 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3554 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), 3555 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); 3556 3557 /* Set the default values for the Tx Inter Packet Gap timer */ 3558 switch (adapter->hw.mac.type) { 3559 case e1000_82542: 3560 tipg = DEFAULT_82542_TIPG_IPGT; 3561 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3562 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3563 break; 3564 case e1000_80003es2lan: 3565 tipg = DEFAULT_82543_TIPG_IPGR1; 3566 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3567 E1000_TIPG_IPGR2_SHIFT; 3568 break; 3569 default: 3570 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3571 (adapter->hw.phy.media_type == 3572 e1000_media_type_internal_serdes)) 3573 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3574 else 3575 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3576 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3577 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3578 } 3579 3580 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3581 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3582 if(adapter->hw.mac.type >= e1000_82540) 3583 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3584 adapter->tx_abs_int_delay.value); 3585 3586 if ((adapter->hw.mac.type == e1000_82571) || 3587 (adapter->hw.mac.type == e1000_82572)) { 3588 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3589 tarc |= SPEED_MODE_BIT; 3590 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3591 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3592 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3593 tarc |= 1; 3594 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3595 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3596 tarc |= 1; 3597 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3598 } 3599 3600 /* Program the Transmit Control Register */ 3601 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3602 tctl &= ~E1000_TCTL_CT; 3603 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3604 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3605 3606 if (adapter->hw.mac.type >= e1000_82571) 3607 tctl |= E1000_TCTL_MULR; 3608 3609 /* This write will effectively turn on the transmit unit. */ 3610 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3611 3612 /* Setup Transmit Descriptor Base Settings */ 3613 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3614 3615 if (adapter->tx_int_delay.value > 0) 3616 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3617 } 3618 3619 /********************************************************************* 3620 * 3621 * Free all transmit related data structures. 3622 * 3623 **********************************************************************/ 3624 static void 3625 em_free_transmit_structures(struct adapter *adapter) 3626 { 3627 struct em_buffer *tx_buffer; 3628 3629 INIT_DEBUGOUT("free_transmit_structures: begin"); 3630 3631 if (adapter->tx_buffer_area != NULL) { 3632 for (int i = 0; i < adapter->num_tx_desc; i++) { 3633 tx_buffer = &adapter->tx_buffer_area[i]; 3634 if (tx_buffer->m_head != NULL) { 3635 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3636 BUS_DMASYNC_POSTWRITE); 3637 bus_dmamap_unload(adapter->txtag, 3638 tx_buffer->map); 3639 m_freem(tx_buffer->m_head); 3640 tx_buffer->m_head = NULL; 3641 } else if (tx_buffer->map != NULL) 3642 bus_dmamap_unload(adapter->txtag, 3643 tx_buffer->map); 3644 if (tx_buffer->map != NULL) { 3645 bus_dmamap_destroy(adapter->txtag, 3646 tx_buffer->map); 3647 tx_buffer->map = NULL; 3648 } 3649 } 3650 } 3651 if (adapter->tx_buffer_area != NULL) { 3652 free(adapter->tx_buffer_area, M_DEVBUF); 3653 adapter->tx_buffer_area = NULL; 3654 } 3655 if (adapter->txtag != NULL) { 3656 bus_dma_tag_destroy(adapter->txtag); 3657 adapter->txtag = NULL; 3658 } 3659 } 3660 3661 /********************************************************************* 3662 * 3663 * The offload context needs to be set when we transfer the first 3664 * packet of a particular protocol (TCP/UDP). This routine has been 3665 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3666 * 3667 **********************************************************************/ 3668 static void 3669 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, 3670 u32 *txd_upper, u32 *txd_lower) 3671 { 3672 struct e1000_context_desc *TXD; 3673 struct em_buffer *tx_buffer; 3674 struct ether_vlan_header *eh; 3675 struct ip *ip = NULL; 3676 struct ip6_hdr *ip6; 3677 struct tcp_hdr *th; 3678 int curr_txd, ehdrlen; 3679 u32 cmd, hdr_len, ip_hlen; 3680 u16 etype; 3681 u8 ipproto; 3682 3683 cmd = hdr_len = ipproto = 0; 3684 /* Setup checksum offload context. */ 3685 curr_txd = adapter->next_avail_tx_desc; 3686 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3687 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3688 3689 /* 3690 * Determine where frame payload starts. 3691 * Jump over vlan headers if already present, 3692 * helpful for QinQ too. 3693 */ 3694 eh = mtod(mp, struct ether_vlan_header *); 3695 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3696 etype = ntohs(eh->evl_proto); 3697 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3698 } else { 3699 etype = ntohs(eh->evl_encap_proto); 3700 ehdrlen = ETHER_HDR_LEN; 3701 } 3702 3703 /* 3704 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3705 * TODO: Support SCTP too when it hits the tree. 3706 */ 3707 switch (etype) { 3708 case ETHERTYPE_IP: 3709 ip = (struct ip *)(mp->m_data + ehdrlen); 3710 ip_hlen = ip->ip_hl << 2; 3711 3712 /* Setup of IP header checksum. */ 3713 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3714 /* 3715 * Start offset for header checksum calculation. 3716 * End offset for header checksum calculation. 3717 * Offset of place to put the checksum. 3718 */ 3719 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3720 TXD->lower_setup.ip_fields.ipcse = 3721 htole16(ehdrlen + ip_hlen); 3722 TXD->lower_setup.ip_fields.ipcso = 3723 ehdrlen + offsetof(struct ip, ip_sum); 3724 cmd |= E1000_TXD_CMD_IP; 3725 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3726 } 3727 3728 if (mp->m_len < ehdrlen + ip_hlen) 3729 return; /* failure */ 3730 3731 hdr_len = ehdrlen + ip_hlen; 3732 ipproto = ip->ip_p; 3733 3734 break; 3735 case ETHERTYPE_IPV6: 3736 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3737 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3738 3739 if (mp->m_len < ehdrlen + ip_hlen) 3740 return; /* failure */ 3741 3742 /* IPv6 doesn't have a header checksum. */ 3743 3744 hdr_len = ehdrlen + ip_hlen; 3745 ipproto = ip6->ip6_nxt; 3746 3747 break; 3748 #ifdef EM_TIMESYNC 3749 case ETHERTYPE_IEEE1588: 3750 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3751 break; 3752 #endif 3753 default: 3754 *txd_upper = 0; 3755 *txd_lower = 0; 3756 return; 3757 } 3758 3759 switch (ipproto) { 3760 case IPPROTO_TCP: 3761 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3762 /* 3763 * Start offset for payload checksum calculation. 3764 * End offset for payload checksum calculation. 3765 * Offset of place to put the checksum. 3766 */ 3767 th = (struct tcp_hdr *)(mp->m_data + hdr_len); 3768 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3769 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3770 TXD->upper_setup.tcp_fields.tucso = 3771 hdr_len + offsetof(struct tcphdr, th_sum); 3772 cmd |= E1000_TXD_CMD_TCP; 3773 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3774 } 3775 break; 3776 case IPPROTO_UDP: 3777 { 3778 #ifdef EM_TIMESYNC 3779 void *hdr = (caddr_t) ip + ip_hlen; 3780 struct udphdr *uh = (struct udphdr *)hdr; 3781 3782 if (uh->uh_dport == htons(TSYNC_PORT)) { 3783 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3784 IOCTL_DEBUGOUT("@@@ Sending Event Packet\n"); 3785 } 3786 #endif 3787 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3788 /* 3789 * Start offset for header checksum calculation. 3790 * End offset for header checksum calculation. 3791 * Offset of place to put the checksum. 3792 */ 3793 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3794 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3795 TXD->upper_setup.tcp_fields.tucso = 3796 hdr_len + offsetof(struct udphdr, uh_sum); 3797 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3798 } 3799 /* Fall Thru */ 3800 } 3801 default: 3802 break; 3803 } 3804 3805 #ifdef EM_TIMESYNC 3806 /* 3807 ** We might be here just for TIMESYNC 3808 ** which means we don't need the context 3809 ** descriptor. 3810 */ 3811 if (!mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) 3812 return; 3813 #endif 3814 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 3815 E1000_TXD_DTYP_D; /* Data descr */ 3816 TXD->tcp_seg_setup.data = htole32(0); 3817 TXD->cmd_and_length = 3818 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3819 tx_buffer->m_head = NULL; 3820 tx_buffer->next_eop = -1; 3821 3822 if (++curr_txd == adapter->num_tx_desc) 3823 curr_txd = 0; 3824 3825 adapter->num_tx_desc_avail--; 3826 adapter->next_avail_tx_desc = curr_txd; 3827 } 3828 3829 3830 #if __FreeBSD_version >= 700000 3831 /********************************************************************** 3832 * 3833 * Setup work for hardware segmentation offload (TSO) 3834 * 3835 **********************************************************************/ 3836 static bool 3837 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, 3838 u32 *txd_lower) 3839 { 3840 struct e1000_context_desc *TXD; 3841 struct em_buffer *tx_buffer; 3842 struct ether_vlan_header *eh; 3843 struct ip *ip; 3844 struct ip6_hdr *ip6; 3845 struct tcphdr *th; 3846 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; 3847 u16 etype; 3848 3849 /* 3850 * This function could/should be extended to support IP/IPv6 3851 * fragmentation as well. But as they say, one step at a time. 3852 */ 3853 3854 /* 3855 * Determine where frame payload starts. 3856 * Jump over vlan headers if already present, 3857 * helpful for QinQ too. 3858 */ 3859 eh = mtod(mp, struct ether_vlan_header *); 3860 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3861 etype = ntohs(eh->evl_proto); 3862 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3863 } else { 3864 etype = ntohs(eh->evl_encap_proto); 3865 ehdrlen = ETHER_HDR_LEN; 3866 } 3867 3868 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3869 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3870 return FALSE; /* -1 */ 3871 3872 /* 3873 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3874 * TODO: Support SCTP too when it hits the tree. 3875 */ 3876 switch (etype) { 3877 case ETHERTYPE_IP: 3878 isip6 = 0; 3879 ip = (struct ip *)(mp->m_data + ehdrlen); 3880 if (ip->ip_p != IPPROTO_TCP) 3881 return FALSE; /* 0 */ 3882 ip->ip_len = 0; 3883 ip->ip_sum = 0; 3884 ip_hlen = ip->ip_hl << 2; 3885 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3886 return FALSE; /* -1 */ 3887 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3888 #if 1 3889 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3890 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3891 #else 3892 th->th_sum = mp->m_pkthdr.csum_data; 3893 #endif 3894 break; 3895 case ETHERTYPE_IPV6: 3896 isip6 = 1; 3897 return FALSE; /* Not supported yet. */ 3898 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3899 if (ip6->ip6_nxt != IPPROTO_TCP) 3900 return FALSE; /* 0 */ 3901 ip6->ip6_plen = 0; 3902 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3903 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3904 return FALSE; /* -1 */ 3905 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3906 #if 0 3907 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3908 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3909 #else 3910 th->th_sum = mp->m_pkthdr.csum_data; 3911 #endif 3912 break; 3913 default: 3914 return FALSE; 3915 } 3916 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3917 3918 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3919 E1000_TXD_DTYP_D | /* Data descr type */ 3920 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3921 3922 /* IP and/or TCP header checksum calculation and insertion. */ 3923 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3924 E1000_TXD_POPTS_TXSM) << 8; 3925 3926 curr_txd = adapter->next_avail_tx_desc; 3927 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3928 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3929 3930 /* IPv6 doesn't have a header checksum. */ 3931 if (!isip6) { 3932 /* 3933 * Start offset for header checksum calculation. 3934 * End offset for header checksum calculation. 3935 * Offset of place put the checksum. 3936 */ 3937 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3938 TXD->lower_setup.ip_fields.ipcse = 3939 htole16(ehdrlen + ip_hlen - 1); 3940 TXD->lower_setup.ip_fields.ipcso = 3941 ehdrlen + offsetof(struct ip, ip_sum); 3942 } 3943 /* 3944 * Start offset for payload checksum calculation. 3945 * End offset for payload checksum calculation. 3946 * Offset of place to put the checksum. 3947 */ 3948 TXD->upper_setup.tcp_fields.tucss = 3949 ehdrlen + ip_hlen; 3950 TXD->upper_setup.tcp_fields.tucse = 0; 3951 TXD->upper_setup.tcp_fields.tucso = 3952 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3953 /* 3954 * Payload size per packet w/o any headers. 3955 * Length of all headers up to payload. 3956 */ 3957 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3958 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3959 3960 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3961 E1000_TXD_CMD_DEXT | /* Extended descr */ 3962 E1000_TXD_CMD_TSE | /* TSE context */ 3963 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ 3964 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3965 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3966 3967 tx_buffer->m_head = NULL; 3968 tx_buffer->next_eop = -1; 3969 3970 if (++curr_txd == adapter->num_tx_desc) 3971 curr_txd = 0; 3972 3973 adapter->num_tx_desc_avail--; 3974 adapter->next_avail_tx_desc = curr_txd; 3975 adapter->tx_tso = TRUE; 3976 3977 return TRUE; 3978 } 3979 3980 #endif /* __FreeBSD_version >= 700000 */ 3981 3982 /********************************************************************** 3983 * 3984 * Examine each tx_buffer in the used queue. If the hardware is done 3985 * processing the packet then free associated resources. The 3986 * tx_buffer is put back on the free queue. 3987 * 3988 **********************************************************************/ 3989 static void 3990 em_txeof(struct adapter *adapter) 3991 { 3992 int first, last, done, num_avail; 3993 u32 cleaned = 0; 3994 struct em_buffer *tx_buffer; 3995 struct e1000_tx_desc *tx_desc, *eop_desc; 3996 struct ifnet *ifp = adapter->ifp; 3997 3998 EM_TX_LOCK_ASSERT(adapter); 3999 4000 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 4001 return; 4002 4003 num_avail = adapter->num_tx_desc_avail; 4004 first = adapter->next_tx_to_clean; 4005 tx_desc = &adapter->tx_desc_base[first]; 4006 tx_buffer = &adapter->tx_buffer_area[first]; 4007 last = tx_buffer->next_eop; 4008 eop_desc = &adapter->tx_desc_base[last]; 4009 4010 /* 4011 * What this does is get the index of the 4012 * first descriptor AFTER the EOP of the 4013 * first packet, that way we can do the 4014 * simple comparison on the inner while loop. 4015 */ 4016 if (++last == adapter->num_tx_desc) 4017 last = 0; 4018 done = last; 4019 4020 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4021 BUS_DMASYNC_POSTREAD); 4022 4023 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 4024 /* We clean the range of the packet */ 4025 while (first != done) { 4026 tx_desc->upper.data = 0; 4027 tx_desc->lower.data = 0; 4028 tx_desc->buffer_addr = 0; 4029 ++num_avail; ++cleaned; 4030 4031 if (tx_buffer->m_head) { 4032 ifp->if_opackets++; 4033 bus_dmamap_sync(adapter->txtag, 4034 tx_buffer->map, 4035 BUS_DMASYNC_POSTWRITE); 4036 bus_dmamap_unload(adapter->txtag, 4037 tx_buffer->map); 4038 4039 m_freem(tx_buffer->m_head); 4040 tx_buffer->m_head = NULL; 4041 } 4042 tx_buffer->next_eop = -1; 4043 4044 if (++first == adapter->num_tx_desc) 4045 first = 0; 4046 4047 tx_buffer = &adapter->tx_buffer_area[first]; 4048 tx_desc = &adapter->tx_desc_base[first]; 4049 } 4050 /* See if we can continue to the next packet */ 4051 last = tx_buffer->next_eop; 4052 if (last != -1) { 4053 eop_desc = &adapter->tx_desc_base[last]; 4054 /* Get new done point */ 4055 if (++last == adapter->num_tx_desc) last = 0; 4056 done = last; 4057 } else 4058 break; 4059 } 4060 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4061 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4062 4063 adapter->next_tx_to_clean = first; 4064 4065 /* 4066 * If we have enough room, clear IFF_DRV_OACTIVE to 4067 * tell the stack that it is OK to send packets. 4068 * If there are no pending descriptors, clear the timeout. 4069 */ 4070 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 4071 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 4072 if (num_avail == adapter->num_tx_desc) { 4073 adapter->watchdog_timer = 0; 4074 adapter->num_tx_desc_avail = num_avail; 4075 return; 4076 } 4077 } 4078 4079 /* If any descriptors cleaned, reset the watchdog */ 4080 if (cleaned) 4081 adapter->watchdog_timer = EM_TX_TIMEOUT; 4082 adapter->num_tx_desc_avail = num_avail; 4083 return; 4084 } 4085 4086 /********************************************************************* 4087 * 4088 * When Link is lost sometimes there is work still in the TX ring 4089 * which will result in a watchdog, rather than allow that do an 4090 * attempted cleanup and then reinit here. Note that this has been 4091 * seens mostly with fiber adapters. 4092 * 4093 **********************************************************************/ 4094 static void 4095 em_tx_purge(struct adapter *adapter) 4096 { 4097 if ((!adapter->link_active) && (adapter->watchdog_timer)) { 4098 EM_TX_LOCK(adapter); 4099 em_txeof(adapter); 4100 EM_TX_UNLOCK(adapter); 4101 if (adapter->watchdog_timer) { /* Still not clean? */ 4102 adapter->watchdog_timer = 0; 4103 em_init_locked(adapter); 4104 } 4105 } 4106 } 4107 4108 /********************************************************************* 4109 * 4110 * Get a buffer from system mbuf buffer pool. 4111 * 4112 **********************************************************************/ 4113 static int 4114 em_get_buf(struct adapter *adapter, int i) 4115 { 4116 struct mbuf *m; 4117 bus_dma_segment_t segs[1]; 4118 bus_dmamap_t map; 4119 struct em_buffer *rx_buffer; 4120 int error, nsegs; 4121 4122 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 4123 if (m == NULL) { 4124 adapter->mbuf_cluster_failed++; 4125 return (ENOBUFS); 4126 } 4127 m->m_len = m->m_pkthdr.len = MCLBYTES; 4128 4129 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 4130 m_adj(m, ETHER_ALIGN); 4131 4132 /* 4133 * Using memory from the mbuf cluster pool, invoke the 4134 * bus_dma machinery to arrange the memory mapping. 4135 */ 4136 error = bus_dmamap_load_mbuf_sg(adapter->rxtag, 4137 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); 4138 if (error != 0) { 4139 m_free(m); 4140 return (error); 4141 } 4142 4143 /* If nsegs is wrong then the stack is corrupt. */ 4144 KASSERT(nsegs == 1, ("Too many segments returned!")); 4145 4146 rx_buffer = &adapter->rx_buffer_area[i]; 4147 if (rx_buffer->m_head != NULL) 4148 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4149 4150 map = rx_buffer->map; 4151 rx_buffer->map = adapter->rx_sparemap; 4152 adapter->rx_sparemap = map; 4153 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 4154 rx_buffer->m_head = m; 4155 4156 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); 4157 return (0); 4158 } 4159 4160 /********************************************************************* 4161 * 4162 * Allocate memory for rx_buffer structures. Since we use one 4163 * rx_buffer per received packet, the maximum number of rx_buffer's 4164 * that we'll need is equal to the number of receive descriptors 4165 * that we've allocated. 4166 * 4167 **********************************************************************/ 4168 static int 4169 em_allocate_receive_structures(struct adapter *adapter) 4170 { 4171 device_t dev = adapter->dev; 4172 struct em_buffer *rx_buffer; 4173 int i, error; 4174 4175 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * 4176 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 4177 if (adapter->rx_buffer_area == NULL) { 4178 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 4179 return (ENOMEM); 4180 } 4181 4182 #if __FreeBSD_version >= 700000 4183 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4184 #else 4185 error = bus_dma_tag_create(NULL, /* parent */ 4186 #endif 4187 1, 0, /* alignment, bounds */ 4188 BUS_SPACE_MAXADDR, /* lowaddr */ 4189 BUS_SPACE_MAXADDR, /* highaddr */ 4190 NULL, NULL, /* filter, filterarg */ 4191 MCLBYTES, /* maxsize */ 4192 1, /* nsegments */ 4193 MCLBYTES, /* maxsegsize */ 4194 0, /* flags */ 4195 NULL, /* lockfunc */ 4196 NULL, /* lockarg */ 4197 &adapter->rxtag); 4198 if (error) { 4199 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 4200 __func__, error); 4201 goto fail; 4202 } 4203 4204 /* Create the spare map (used by getbuf) */ 4205 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4206 &adapter->rx_sparemap); 4207 if (error) { 4208 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4209 __func__, error); 4210 goto fail; 4211 } 4212 4213 rx_buffer = adapter->rx_buffer_area; 4214 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4215 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4216 &rx_buffer->map); 4217 if (error) { 4218 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4219 __func__, error); 4220 goto fail; 4221 } 4222 } 4223 4224 return (0); 4225 4226 fail: 4227 em_free_receive_structures(adapter); 4228 return (error); 4229 } 4230 4231 /********************************************************************* 4232 * 4233 * (Re)initialize receive structures. 4234 * 4235 **********************************************************************/ 4236 static int 4237 em_setup_receive_structures(struct adapter *adapter) 4238 { 4239 struct em_buffer *rx_buffer; 4240 int i, error; 4241 4242 /* Reset descriptor ring */ 4243 bzero(adapter->rx_desc_base, 4244 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 4245 4246 /* Free current RX buffers. */ 4247 rx_buffer = adapter->rx_buffer_area; 4248 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4249 if (rx_buffer->m_head != NULL) { 4250 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4251 BUS_DMASYNC_POSTREAD); 4252 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4253 m_freem(rx_buffer->m_head); 4254 rx_buffer->m_head = NULL; 4255 } 4256 } 4257 4258 /* Allocate new ones. */ 4259 for (i = 0; i < adapter->num_rx_desc; i++) { 4260 error = em_get_buf(adapter, i); 4261 if (error) 4262 return (error); 4263 } 4264 4265 /* Setup our descriptor pointers */ 4266 adapter->next_rx_desc_to_check = 0; 4267 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4268 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4269 4270 return (0); 4271 } 4272 4273 /********************************************************************* 4274 * 4275 * Enable receive unit. 4276 * 4277 **********************************************************************/ 4278 #define MAX_INTS_PER_SEC 8000 4279 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 4280 4281 static void 4282 em_initialize_receive_unit(struct adapter *adapter) 4283 { 4284 struct ifnet *ifp = adapter->ifp; 4285 u64 bus_addr; 4286 u32 rctl, rxcsum; 4287 4288 INIT_DEBUGOUT("em_initialize_receive_unit: begin"); 4289 4290 /* 4291 * Make sure receives are disabled while setting 4292 * up the descriptor ring 4293 */ 4294 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4295 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4296 4297 if (adapter->hw.mac.type >= e1000_82540) { 4298 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 4299 adapter->rx_abs_int_delay.value); 4300 /* 4301 * Set the interrupt throttling rate. Value is calculated 4302 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4303 */ 4304 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); 4305 } 4306 4307 /* 4308 ** When using MSIX interrupts we need to throttle 4309 ** using the EITR register (82574 only) 4310 */ 4311 if (adapter->msix) 4312 for (int i = 0; i < 4; i++) 4313 E1000_WRITE_REG(&adapter->hw, 4314 E1000_EITR_82574(i), DEFAULT_ITR); 4315 4316 /* Disable accelerated ackknowledge */ 4317 if (adapter->hw.mac.type == e1000_82574) 4318 E1000_WRITE_REG(&adapter->hw, 4319 E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4320 4321 /* Setup the Base and Length of the Rx Descriptor Ring */ 4322 bus_addr = adapter->rxdma.dma_paddr; 4323 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 4324 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4325 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 4326 (u32)(bus_addr >> 32)); 4327 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 4328 (u32)bus_addr); 4329 4330 /* Setup the Receive Control Register */ 4331 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4332 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4333 E1000_RCTL_RDMTS_HALF | 4334 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4335 4336 /* Make sure VLAN Filters are off */ 4337 rctl &= ~E1000_RCTL_VFE; 4338 4339 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 4340 rctl |= E1000_RCTL_SBP; 4341 else 4342 rctl &= ~E1000_RCTL_SBP; 4343 4344 switch (adapter->rx_buffer_len) { 4345 default: 4346 case 2048: 4347 rctl |= E1000_RCTL_SZ_2048; 4348 break; 4349 case 4096: 4350 rctl |= E1000_RCTL_SZ_4096 | 4351 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4352 break; 4353 case 8192: 4354 rctl |= E1000_RCTL_SZ_8192 | 4355 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4356 break; 4357 case 16384: 4358 rctl |= E1000_RCTL_SZ_16384 | 4359 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4360 break; 4361 } 4362 4363 if (ifp->if_mtu > ETHERMTU) 4364 rctl |= E1000_RCTL_LPE; 4365 else 4366 rctl &= ~E1000_RCTL_LPE; 4367 4368 /* Enable 82543 Receive Checksum Offload for TCP and UDP */ 4369 if ((adapter->hw.mac.type >= e1000_82543) && 4370 (ifp->if_capenable & IFCAP_RXCSUM)) { 4371 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 4372 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4373 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 4374 } 4375 4376 /* 4377 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4378 ** long latencies are observed, like Lenovo X60. This 4379 ** change eliminates the problem, but since having positive 4380 ** values in RDTR is a known source of problems on other 4381 ** platforms another solution is being sought. 4382 */ 4383 if (adapter->hw.mac.type == e1000_82573) 4384 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); 4385 4386 /* Enable Receives */ 4387 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4388 4389 /* 4390 * Setup the HW Rx Head and 4391 * Tail Descriptor Pointers 4392 */ 4393 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 4394 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 4395 4396 return; 4397 } 4398 4399 /********************************************************************* 4400 * 4401 * Free receive related data structures. 4402 * 4403 **********************************************************************/ 4404 static void 4405 em_free_receive_structures(struct adapter *adapter) 4406 { 4407 struct em_buffer *rx_buffer; 4408 int i; 4409 4410 INIT_DEBUGOUT("free_receive_structures: begin"); 4411 4412 if (adapter->rx_sparemap) { 4413 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 4414 adapter->rx_sparemap = NULL; 4415 } 4416 4417 /* Cleanup any existing buffers */ 4418 if (adapter->rx_buffer_area != NULL) { 4419 rx_buffer = adapter->rx_buffer_area; 4420 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4421 if (rx_buffer->m_head != NULL) { 4422 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4423 BUS_DMASYNC_POSTREAD); 4424 bus_dmamap_unload(adapter->rxtag, 4425 rx_buffer->map); 4426 m_freem(rx_buffer->m_head); 4427 rx_buffer->m_head = NULL; 4428 } else if (rx_buffer->map != NULL) 4429 bus_dmamap_unload(adapter->rxtag, 4430 rx_buffer->map); 4431 if (rx_buffer->map != NULL) { 4432 bus_dmamap_destroy(adapter->rxtag, 4433 rx_buffer->map); 4434 rx_buffer->map = NULL; 4435 } 4436 } 4437 } 4438 4439 if (adapter->rx_buffer_area != NULL) { 4440 free(adapter->rx_buffer_area, M_DEVBUF); 4441 adapter->rx_buffer_area = NULL; 4442 } 4443 4444 if (adapter->rxtag != NULL) { 4445 bus_dma_tag_destroy(adapter->rxtag); 4446 adapter->rxtag = NULL; 4447 } 4448 } 4449 4450 /********************************************************************* 4451 * 4452 * This routine executes in interrupt context. It replenishes 4453 * the mbufs in the descriptor and sends data which has been 4454 * dma'ed into host memory to upper layer. 4455 * 4456 * We loop at most count times if count is > 0, or until done if 4457 * count < 0. 4458 * 4459 *********************************************************************/ 4460 static int 4461 em_rxeof(struct adapter *adapter, int count) 4462 { 4463 struct ifnet *ifp = adapter->ifp;; 4464 struct mbuf *mp; 4465 u8 status, accept_frame = 0, eop = 0; 4466 u16 len, desc_len, prev_len_adj; 4467 int i; 4468 struct e1000_rx_desc *current_desc; 4469 4470 EM_RX_LOCK(adapter); 4471 i = adapter->next_rx_desc_to_check; 4472 current_desc = &adapter->rx_desc_base[i]; 4473 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4474 BUS_DMASYNC_POSTREAD); 4475 4476 if (!((current_desc->status) & E1000_RXD_STAT_DD)) { 4477 EM_RX_UNLOCK(adapter); 4478 return (0); 4479 } 4480 4481 while ((current_desc->status & E1000_RXD_STAT_DD) && 4482 (count != 0) && 4483 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4484 struct mbuf *m = NULL; 4485 4486 mp = adapter->rx_buffer_area[i].m_head; 4487 /* 4488 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 4489 * needs to access the last received byte in the mbuf. 4490 */ 4491 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 4492 BUS_DMASYNC_POSTREAD); 4493 4494 accept_frame = 1; 4495 prev_len_adj = 0; 4496 desc_len = le16toh(current_desc->length); 4497 status = current_desc->status; 4498 if (status & E1000_RXD_STAT_EOP) { 4499 count--; 4500 eop = 1; 4501 if (desc_len < ETHER_CRC_LEN) { 4502 len = 0; 4503 prev_len_adj = ETHER_CRC_LEN - desc_len; 4504 } else 4505 len = desc_len - ETHER_CRC_LEN; 4506 } else { 4507 eop = 0; 4508 len = desc_len; 4509 } 4510 4511 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 4512 u8 last_byte; 4513 u32 pkt_len = desc_len; 4514 4515 if (adapter->fmp != NULL) 4516 pkt_len += adapter->fmp->m_pkthdr.len; 4517 4518 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 4519 if (TBI_ACCEPT(&adapter->hw, status, 4520 current_desc->errors, pkt_len, last_byte, 4521 adapter->min_frame_size, adapter->max_frame_size)) { 4522 e1000_tbi_adjust_stats_82543(&adapter->hw, 4523 &adapter->stats, pkt_len, 4524 adapter->hw.mac.addr, 4525 adapter->max_frame_size); 4526 if (len > 0) 4527 len--; 4528 } else 4529 accept_frame = 0; 4530 } 4531 4532 if (accept_frame) { 4533 if (em_get_buf(adapter, i) != 0) { 4534 ifp->if_iqdrops++; 4535 goto discard; 4536 } 4537 4538 /* Assign correct length to the current fragment */ 4539 mp->m_len = len; 4540 4541 if (adapter->fmp == NULL) { 4542 mp->m_pkthdr.len = len; 4543 adapter->fmp = mp; /* Store the first mbuf */ 4544 adapter->lmp = mp; 4545 } else { 4546 /* Chain mbuf's together */ 4547 mp->m_flags &= ~M_PKTHDR; 4548 /* 4549 * Adjust length of previous mbuf in chain if 4550 * we received less than 4 bytes in the last 4551 * descriptor. 4552 */ 4553 if (prev_len_adj > 0) { 4554 adapter->lmp->m_len -= prev_len_adj; 4555 adapter->fmp->m_pkthdr.len -= 4556 prev_len_adj; 4557 } 4558 adapter->lmp->m_next = mp; 4559 adapter->lmp = adapter->lmp->m_next; 4560 adapter->fmp->m_pkthdr.len += len; 4561 } 4562 4563 if (eop) { 4564 adapter->fmp->m_pkthdr.rcvif = ifp; 4565 ifp->if_ipackets++; 4566 em_receive_checksum(adapter, current_desc, 4567 adapter->fmp); 4568 #ifndef __NO_STRICT_ALIGNMENT 4569 if (adapter->max_frame_size > 4570 (MCLBYTES - ETHER_ALIGN) && 4571 em_fixup_rx(adapter) != 0) 4572 goto skip; 4573 #endif 4574 if (status & E1000_RXD_STAT_VP) { 4575 #if __FreeBSD_version < 700000 4576 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, 4577 (le16toh(current_desc->special) & 4578 E1000_RXD_SPC_VLAN_MASK)); 4579 #else 4580 adapter->fmp->m_pkthdr.ether_vtag = 4581 (le16toh(current_desc->special) & 4582 E1000_RXD_SPC_VLAN_MASK); 4583 adapter->fmp->m_flags |= M_VLANTAG; 4584 #endif 4585 } 4586 #ifndef __NO_STRICT_ALIGNMENT 4587 skip: 4588 #endif 4589 m = adapter->fmp; 4590 adapter->fmp = NULL; 4591 adapter->lmp = NULL; 4592 } 4593 } else { 4594 ifp->if_ierrors++; 4595 discard: 4596 /* Reuse loaded DMA map and just update mbuf chain */ 4597 mp = adapter->rx_buffer_area[i].m_head; 4598 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4599 mp->m_data = mp->m_ext.ext_buf; 4600 mp->m_next = NULL; 4601 if (adapter->max_frame_size <= 4602 (MCLBYTES - ETHER_ALIGN)) 4603 m_adj(mp, ETHER_ALIGN); 4604 if (adapter->fmp != NULL) { 4605 m_freem(adapter->fmp); 4606 adapter->fmp = NULL; 4607 adapter->lmp = NULL; 4608 } 4609 m = NULL; 4610 } 4611 4612 /* Zero out the receive descriptors status. */ 4613 current_desc->status = 0; 4614 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4615 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4616 4617 /* Advance our pointers to the next descriptor. */ 4618 if (++i == adapter->num_rx_desc) 4619 i = 0; 4620 if (m != NULL) { 4621 adapter->next_rx_desc_to_check = i; 4622 /* Unlock for call into stack */ 4623 EM_RX_UNLOCK(adapter); 4624 (*ifp->if_input)(ifp, m); 4625 EM_RX_LOCK(adapter); 4626 i = adapter->next_rx_desc_to_check; 4627 } 4628 current_desc = &adapter->rx_desc_base[i]; 4629 } 4630 adapter->next_rx_desc_to_check = i; 4631 4632 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4633 if (--i < 0) 4634 i = adapter->num_rx_desc - 1; 4635 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 4636 EM_RX_UNLOCK(adapter); 4637 if (!((current_desc->status) & E1000_RXD_STAT_DD)) 4638 return (0); 4639 4640 return (1); 4641 } 4642 4643 #ifndef __NO_STRICT_ALIGNMENT 4644 /* 4645 * When jumbo frames are enabled we should realign entire payload on 4646 * architecures with strict alignment. This is serious design mistake of 8254x 4647 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4648 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4649 * payload. On architecures without strict alignment restrictions 8254x still 4650 * performs unaligned memory access which would reduce the performance too. 4651 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4652 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4653 * existing mbuf chain. 4654 * 4655 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4656 * not used at all on architectures with strict alignment. 4657 */ 4658 static int 4659 em_fixup_rx(struct adapter *adapter) 4660 { 4661 struct mbuf *m, *n; 4662 int error; 4663 4664 error = 0; 4665 m = adapter->fmp; 4666 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4667 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4668 m->m_data += ETHER_HDR_LEN; 4669 } else { 4670 MGETHDR(n, M_DONTWAIT, MT_DATA); 4671 if (n != NULL) { 4672 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4673 m->m_data += ETHER_HDR_LEN; 4674 m->m_len -= ETHER_HDR_LEN; 4675 n->m_len = ETHER_HDR_LEN; 4676 M_MOVE_PKTHDR(n, m); 4677 n->m_next = m; 4678 adapter->fmp = n; 4679 } else { 4680 adapter->dropped_pkts++; 4681 m_freem(adapter->fmp); 4682 adapter->fmp = NULL; 4683 error = ENOMEM; 4684 } 4685 } 4686 4687 return (error); 4688 } 4689 #endif 4690 4691 /********************************************************************* 4692 * 4693 * Verify that the hardware indicated that the checksum is valid. 4694 * Inform the stack about the status of checksum so that stack 4695 * doesn't spend time verifying the checksum. 4696 * 4697 *********************************************************************/ 4698 static void 4699 em_receive_checksum(struct adapter *adapter, 4700 struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4701 { 4702 /* 82543 or newer only */ 4703 if ((adapter->hw.mac.type < e1000_82543) || 4704 /* Ignore Checksum bit is set */ 4705 (rx_desc->status & E1000_RXD_STAT_IXSM)) { 4706 mp->m_pkthdr.csum_flags = 0; 4707 return; 4708 } 4709 4710 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4711 /* Did it pass? */ 4712 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4713 /* IP Checksum Good */ 4714 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4715 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4716 4717 } else { 4718 mp->m_pkthdr.csum_flags = 0; 4719 } 4720 } 4721 4722 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4723 /* Did it pass? */ 4724 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4725 mp->m_pkthdr.csum_flags |= 4726 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4727 mp->m_pkthdr.csum_data = htons(0xffff); 4728 } 4729 } 4730 } 4731 4732 /* 4733 * This routine is run via an vlan 4734 * config EVENT 4735 */ 4736 static void 4737 em_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4738 { 4739 struct adapter *adapter = ifp->if_softc; 4740 u32 ctrl, rctl, index, vfta; 4741 4742 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4743 ctrl |= E1000_CTRL_VME; 4744 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4745 4746 /* Setup for Hardware Filter */ 4747 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4748 rctl |= E1000_RCTL_VFE; 4749 rctl &= ~E1000_RCTL_CFIEN; 4750 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4751 4752 /* Make entry in the hardware filter table */ 4753 index = ((vtag >> 5) & 0x7F); 4754 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4755 vfta |= (1 << (vtag & 0x1F)); 4756 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4757 4758 /* Update the frame size */ 4759 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4760 adapter->max_frame_size + VLAN_TAG_SIZE); 4761 4762 } 4763 4764 /* 4765 * This routine is run via an vlan 4766 * unconfig EVENT 4767 */ 4768 static void 4769 em_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4770 { 4771 struct adapter *adapter = ifp->if_softc; 4772 u32 index, vfta; 4773 4774 /* Remove entry in the hardware filter table */ 4775 index = ((vtag >> 5) & 0x7F); 4776 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4777 vfta &= ~(1 << (vtag & 0x1F)); 4778 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4779 /* Have all vlans unregistered? */ 4780 if (adapter->ifp->if_vlantrunk == NULL) { 4781 u32 rctl; 4782 /* Turn off the filter table */ 4783 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4784 rctl &= ~E1000_RCTL_VFE; 4785 rctl |= E1000_RCTL_CFIEN; 4786 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4787 /* Reset the frame size */ 4788 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4789 adapter->max_frame_size); 4790 } 4791 } 4792 4793 static void 4794 em_enable_intr(struct adapter *adapter) 4795 { 4796 struct e1000_hw *hw = &adapter->hw; 4797 u32 ims_mask = IMS_ENABLE_MASK; 4798 4799 if (adapter->msix) { 4800 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4801 ims_mask |= EM_MSIX_MASK; 4802 } 4803 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4804 } 4805 4806 static void 4807 em_disable_intr(struct adapter *adapter) 4808 { 4809 struct e1000_hw *hw = &adapter->hw; 4810 4811 if (adapter->msix) 4812 E1000_WRITE_REG(hw, EM_EIAC, 0); 4813 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4814 } 4815 4816 /* 4817 * Bit of a misnomer, what this really means is 4818 * to enable OS management of the system... aka 4819 * to disable special hardware management features 4820 */ 4821 static void 4822 em_init_manageability(struct adapter *adapter) 4823 { 4824 /* A shared code workaround */ 4825 #define E1000_82542_MANC2H E1000_MANC2H 4826 if (adapter->has_manage) { 4827 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4828 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4829 4830 /* disable hardware interception of ARP */ 4831 manc &= ~(E1000_MANC_ARP_EN); 4832 4833 /* enable receiving management packets to the host */ 4834 if (adapter->hw.mac.type >= e1000_82571) { 4835 manc |= E1000_MANC_EN_MNG2HOST; 4836 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4837 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4838 manc2h |= E1000_MNG2HOST_PORT_623; 4839 manc2h |= E1000_MNG2HOST_PORT_664; 4840 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4841 } 4842 4843 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4844 } 4845 } 4846 4847 /* 4848 * Give control back to hardware management 4849 * controller if there is one. 4850 */ 4851 static void 4852 em_release_manageability(struct adapter *adapter) 4853 { 4854 if (adapter->has_manage) { 4855 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4856 4857 /* re-enable hardware interception of ARP */ 4858 manc |= E1000_MANC_ARP_EN; 4859 4860 if (adapter->hw.mac.type >= e1000_82571) 4861 manc &= ~E1000_MANC_EN_MNG2HOST; 4862 4863 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4864 } 4865 } 4866 4867 /* 4868 * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4869 * For ASF and Pass Through versions of f/w this means that 4870 * the driver is loaded. For AMT version (only with 82573) 4871 * of the f/w this means that the network i/f is open. 4872 * 4873 */ 4874 static void 4875 em_get_hw_control(struct adapter *adapter) 4876 { 4877 u32 ctrl_ext, swsm; 4878 4879 /* Let firmware know the driver has taken over */ 4880 switch (adapter->hw.mac.type) { 4881 case e1000_82573: 4882 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4883 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4884 swsm | E1000_SWSM_DRV_LOAD); 4885 break; 4886 case e1000_82571: 4887 case e1000_82572: 4888 case e1000_80003es2lan: 4889 case e1000_ich8lan: 4890 case e1000_ich9lan: 4891 case e1000_ich10lan: 4892 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4893 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4894 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4895 break; 4896 default: 4897 break; 4898 } 4899 } 4900 4901 /* 4902 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4903 * For ASF and Pass Through versions of f/w this means that the 4904 * driver is no longer loaded. For AMT version (only with 82573) i 4905 * of the f/w this means that the network i/f is closed. 4906 * 4907 */ 4908 static void 4909 em_release_hw_control(struct adapter *adapter) 4910 { 4911 u32 ctrl_ext, swsm; 4912 4913 /* Let firmware taken over control of h/w */ 4914 switch (adapter->hw.mac.type) { 4915 case e1000_82573: 4916 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4917 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4918 swsm & ~E1000_SWSM_DRV_LOAD); 4919 break; 4920 case e1000_82571: 4921 case e1000_82572: 4922 case e1000_80003es2lan: 4923 case e1000_ich8lan: 4924 case e1000_ich9lan: 4925 case e1000_ich10lan: 4926 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4927 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4928 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4929 break; 4930 default: 4931 break; 4932 4933 } 4934 } 4935 4936 static int 4937 em_is_valid_ether_addr(u8 *addr) 4938 { 4939 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4940 4941 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4942 return (FALSE); 4943 } 4944 4945 return (TRUE); 4946 } 4947 4948 /* 4949 * Enable PCI Wake On Lan capability 4950 */ 4951 void 4952 em_enable_wakeup(device_t dev) 4953 { 4954 u16 cap, status; 4955 u8 id; 4956 4957 /* First find the capabilities pointer*/ 4958 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4959 /* Read the PM Capabilities */ 4960 id = pci_read_config(dev, cap, 1); 4961 if (id != PCIY_PMG) /* Something wrong */ 4962 return; 4963 /* OK, we have the power capabilities, so 4964 now get the status register */ 4965 cap += PCIR_POWER_STATUS; 4966 status = pci_read_config(dev, cap, 2); 4967 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4968 pci_write_config(dev, cap, status, 2); 4969 return; 4970 } 4971 4972 4973 /********************************************************************* 4974 * 82544 Coexistence issue workaround. 4975 * There are 2 issues. 4976 * 1. Transmit Hang issue. 4977 * To detect this issue, following equation can be used... 4978 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 4979 * If SUM[3:0] is in between 1 to 4, we will have this issue. 4980 * 4981 * 2. DAC issue. 4982 * To detect this issue, following equation can be used... 4983 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 4984 * If SUM[3:0] is in between 9 to c, we will have this issue. 4985 * 4986 * 4987 * WORKAROUND: 4988 * Make sure we do not have ending address 4989 * as 1,2,3,4(Hang) or 9,a,b,c (DAC) 4990 * 4991 *************************************************************************/ 4992 static u32 4993 em_fill_descriptors (bus_addr_t address, u32 length, 4994 PDESC_ARRAY desc_array) 4995 { 4996 u32 safe_terminator; 4997 4998 /* Since issue is sensitive to length and address.*/ 4999 /* Let us first check the address...*/ 5000 if (length <= 4) { 5001 desc_array->descriptor[0].address = address; 5002 desc_array->descriptor[0].length = length; 5003 desc_array->elements = 1; 5004 return (desc_array->elements); 5005 } 5006 safe_terminator = (u32)((((u32)address & 0x7) + 5007 (length & 0xF)) & 0xF); 5008 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 5009 if (safe_terminator == 0 || 5010 (safe_terminator > 4 && 5011 safe_terminator < 9) || 5012 (safe_terminator > 0xC && 5013 safe_terminator <= 0xF)) { 5014 desc_array->descriptor[0].address = address; 5015 desc_array->descriptor[0].length = length; 5016 desc_array->elements = 1; 5017 return (desc_array->elements); 5018 } 5019 5020 desc_array->descriptor[0].address = address; 5021 desc_array->descriptor[0].length = length - 4; 5022 desc_array->descriptor[1].address = address + (length - 4); 5023 desc_array->descriptor[1].length = 4; 5024 desc_array->elements = 2; 5025 return (desc_array->elements); 5026 } 5027 5028 /********************************************************************** 5029 * 5030 * Update the board statistics counters. 5031 * 5032 **********************************************************************/ 5033 static void 5034 em_update_stats_counters(struct adapter *adapter) 5035 { 5036 struct ifnet *ifp; 5037 5038 if(adapter->hw.phy.media_type == e1000_media_type_copper || 5039 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 5040 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 5041 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 5042 } 5043 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 5044 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 5045 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 5046 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 5047 5048 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 5049 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 5050 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 5051 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 5052 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 5053 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 5054 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 5055 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 5056 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 5057 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 5058 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 5059 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 5060 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 5061 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 5062 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 5063 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 5064 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 5065 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 5066 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 5067 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 5068 5069 /* For the 64-bit byte counters the low dword must be read first. */ 5070 /* Both registers clear on the read of the high dword */ 5071 5072 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 5073 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 5074 5075 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 5076 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 5077 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 5078 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 5079 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 5080 5081 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 5082 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 5083 5084 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 5085 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 5086 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 5087 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 5088 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 5089 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 5090 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 5091 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 5092 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 5093 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 5094 5095 if (adapter->hw.mac.type >= e1000_82543) { 5096 adapter->stats.algnerrc += 5097 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 5098 adapter->stats.rxerrc += 5099 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 5100 adapter->stats.tncrs += 5101 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 5102 adapter->stats.cexterr += 5103 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 5104 adapter->stats.tsctc += 5105 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 5106 adapter->stats.tsctfc += 5107 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 5108 } 5109 ifp = adapter->ifp; 5110 5111 ifp->if_collisions = adapter->stats.colc; 5112 5113 /* Rx Errors */ 5114 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 5115 adapter->stats.crcerrs + adapter->stats.algnerrc + 5116 adapter->stats.ruc + adapter->stats.roc + 5117 adapter->stats.mpc + adapter->stats.cexterr; 5118 5119 /* Tx Errors */ 5120 ifp->if_oerrors = adapter->stats.ecol + 5121 adapter->stats.latecol + adapter->watchdog_events; 5122 } 5123 5124 5125 /********************************************************************** 5126 * 5127 * This routine is called only when em_display_debug_stats is enabled. 5128 * This routine provides a way to take a look at important statistics 5129 * maintained by the driver and hardware. 5130 * 5131 **********************************************************************/ 5132 static void 5133 em_print_debug_info(struct adapter *adapter) 5134 { 5135 device_t dev = adapter->dev; 5136 u8 *hw_addr = adapter->hw.hw_addr; 5137 5138 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 5139 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 5140 E1000_READ_REG(&adapter->hw, E1000_CTRL), 5141 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 5142 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 5143 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 5144 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 5145 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 5146 adapter->hw.fc.high_water, 5147 adapter->hw.fc.low_water); 5148 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 5149 E1000_READ_REG(&adapter->hw, E1000_TIDV), 5150 E1000_READ_REG(&adapter->hw, E1000_TADV)); 5151 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 5152 E1000_READ_REG(&adapter->hw, E1000_RDTR), 5153 E1000_READ_REG(&adapter->hw, E1000_RADV)); 5154 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 5155 (long long)adapter->tx_fifo_wrk_cnt, 5156 (long long)adapter->tx_fifo_reset_cnt); 5157 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 5158 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 5159 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 5160 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 5161 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 5162 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 5163 device_printf(dev, "Num Tx descriptors avail = %d\n", 5164 adapter->num_tx_desc_avail); 5165 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 5166 adapter->no_tx_desc_avail1); 5167 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 5168 adapter->no_tx_desc_avail2); 5169 device_printf(dev, "Std mbuf failed = %ld\n", 5170 adapter->mbuf_alloc_failed); 5171 device_printf(dev, "Std mbuf cluster failed = %ld\n", 5172 adapter->mbuf_cluster_failed); 5173 device_printf(dev, "Driver dropped packets = %ld\n", 5174 adapter->dropped_pkts); 5175 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 5176 adapter->no_tx_dma_setup); 5177 } 5178 5179 static void 5180 em_print_hw_stats(struct adapter *adapter) 5181 { 5182 device_t dev = adapter->dev; 5183 5184 device_printf(dev, "Excessive collisions = %lld\n", 5185 (long long)adapter->stats.ecol); 5186 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 5187 device_printf(dev, "Symbol errors = %lld\n", 5188 (long long)adapter->stats.symerrs); 5189 #endif 5190 device_printf(dev, "Sequence errors = %lld\n", 5191 (long long)adapter->stats.sec); 5192 device_printf(dev, "Defer count = %lld\n", 5193 (long long)adapter->stats.dc); 5194 device_printf(dev, "Missed Packets = %lld\n", 5195 (long long)adapter->stats.mpc); 5196 device_printf(dev, "Receive No Buffers = %lld\n", 5197 (long long)adapter->stats.rnbc); 5198 /* RLEC is inaccurate on some hardware, calculate our own. */ 5199 device_printf(dev, "Receive Length Errors = %lld\n", 5200 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 5201 device_printf(dev, "Receive errors = %lld\n", 5202 (long long)adapter->stats.rxerrc); 5203 device_printf(dev, "Crc errors = %lld\n", 5204 (long long)adapter->stats.crcerrs); 5205 device_printf(dev, "Alignment errors = %lld\n", 5206 (long long)adapter->stats.algnerrc); 5207 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 5208 (long long)adapter->stats.cexterr); 5209 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 5210 device_printf(dev, "watchdog timeouts = %ld\n", 5211 adapter->watchdog_events); 5212 device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld" 5213 " LINK MSIX IRQ = %ld\n", adapter->rx_irq, 5214 adapter->tx_irq , adapter->link_irq); 5215 device_printf(dev, "XON Rcvd = %lld\n", 5216 (long long)adapter->stats.xonrxc); 5217 device_printf(dev, "XON Xmtd = %lld\n", 5218 (long long)adapter->stats.xontxc); 5219 device_printf(dev, "XOFF Rcvd = %lld\n", 5220 (long long)adapter->stats.xoffrxc); 5221 device_printf(dev, "XOFF Xmtd = %lld\n", 5222 (long long)adapter->stats.xofftxc); 5223 device_printf(dev, "Good Packets Rcvd = %lld\n", 5224 (long long)adapter->stats.gprc); 5225 device_printf(dev, "Good Packets Xmtd = %lld\n", 5226 (long long)adapter->stats.gptc); 5227 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 5228 (long long)adapter->stats.tsctc); 5229 device_printf(dev, "TSO Contexts Failed = %lld\n", 5230 (long long)adapter->stats.tsctfc); 5231 } 5232 5233 /********************************************************************** 5234 * 5235 * This routine provides a way to dump out the adapter eeprom, 5236 * often a useful debug/service tool. This only dumps the first 5237 * 32 words, stuff that matters is in that extent. 5238 * 5239 **********************************************************************/ 5240 static void 5241 em_print_nvm_info(struct adapter *adapter) 5242 { 5243 u16 eeprom_data; 5244 int i, j, row = 0; 5245 5246 /* Its a bit crude, but it gets the job done */ 5247 printf("\nInterface EEPROM Dump:\n"); 5248 printf("Offset\n0x0000 "); 5249 for (i = 0, j = 0; i < 32; i++, j++) { 5250 if (j == 8) { /* Make the offset block */ 5251 j = 0; ++row; 5252 printf("\n0x00%x0 ",row); 5253 } 5254 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5255 printf("%04x ", eeprom_data); 5256 } 5257 printf("\n"); 5258 } 5259 5260 static int 5261 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 5262 { 5263 struct adapter *adapter; 5264 int error; 5265 int result; 5266 5267 result = -1; 5268 error = sysctl_handle_int(oidp, &result, 0, req); 5269 5270 if (error || !req->newptr) 5271 return (error); 5272 5273 if (result == 1) { 5274 adapter = (struct adapter *)arg1; 5275 em_print_debug_info(adapter); 5276 } 5277 /* 5278 * This value will cause a hex dump of the 5279 * first 32 16-bit words of the EEPROM to 5280 * the screen. 5281 */ 5282 if (result == 2) { 5283 adapter = (struct adapter *)arg1; 5284 em_print_nvm_info(adapter); 5285 } 5286 5287 return (error); 5288 } 5289 5290 5291 static int 5292 em_sysctl_stats(SYSCTL_HANDLER_ARGS) 5293 { 5294 struct adapter *adapter; 5295 int error; 5296 int result; 5297 5298 result = -1; 5299 error = sysctl_handle_int(oidp, &result, 0, req); 5300 5301 if (error || !req->newptr) 5302 return (error); 5303 5304 if (result == 1) { 5305 adapter = (struct adapter *)arg1; 5306 em_print_hw_stats(adapter); 5307 } 5308 5309 return (error); 5310 } 5311 5312 static int 5313 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5314 { 5315 struct em_int_delay_info *info; 5316 struct adapter *adapter; 5317 u32 regval; 5318 int error; 5319 int usecs; 5320 int ticks; 5321 5322 info = (struct em_int_delay_info *)arg1; 5323 usecs = info->value; 5324 error = sysctl_handle_int(oidp, &usecs, 0, req); 5325 if (error != 0 || req->newptr == NULL) 5326 return (error); 5327 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5328 return (EINVAL); 5329 info->value = usecs; 5330 ticks = EM_USECS_TO_TICKS(usecs); 5331 5332 adapter = info->adapter; 5333 5334 EM_CORE_LOCK(adapter); 5335 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5336 regval = (regval & ~0xffff) | (ticks & 0xffff); 5337 /* Handle a few special cases. */ 5338 switch (info->offset) { 5339 case E1000_RDTR: 5340 break; 5341 case E1000_TIDV: 5342 if (ticks == 0) { 5343 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5344 /* Don't write 0 into the TIDV register. */ 5345 regval++; 5346 } else 5347 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5348 break; 5349 } 5350 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5351 EM_CORE_UNLOCK(adapter); 5352 return (0); 5353 } 5354 5355 static void 5356 em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5357 const char *description, struct em_int_delay_info *info, 5358 int offset, int value) 5359 { 5360 info->adapter = adapter; 5361 info->offset = offset; 5362 info->value = value; 5363 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5364 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5365 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5366 info, 0, em_sysctl_int_delay, "I", description); 5367 } 5368 5369 #ifndef EM_LEGACY_IRQ 5370 static void 5371 em_add_rx_process_limit(struct adapter *adapter, const char *name, 5372 const char *description, int *limit, int value) 5373 { 5374 *limit = value; 5375 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5376 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5377 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5378 } 5379 #endif 5380 5381 #ifdef EM_TIMESYNC 5382 /* 5383 * Initialize the Time Sync Feature 5384 */ 5385 static int 5386 em_tsync_init(struct adapter *adapter) 5387 { 5388 device_t dev = adapter->dev; 5389 u32 tx_ctl, rx_ctl; 5390 5391 5392 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) | 5393 20833/PICOSECS_PER_TICK); 5394 5395 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 5396 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw, 5397 E1000_SYSTIMH) << 32ULL; 5398 5399 /* Enable the TX side */ 5400 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5401 tx_ctl |= 0x10; 5402 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5403 E1000_WRITE_FLUSH(&adapter->hw); 5404 5405 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5406 if ((tx_ctl & 0x10) == 0) { 5407 device_printf(dev, "Failed to enable TX timestamping\n"); 5408 return (ENXIO); 5409 } 5410 5411 /* Enable RX */ 5412 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5413 rx_ctl |= 0x10; /* Enable the feature */ 5414 rx_ctl |= 0x0a; /* This value turns on Ver 1 and 2 */ 5415 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5416 5417 /* 5418 * Ethertype Stamping (Ethertype = 0x88F7) 5419 */ 5420 E1000_WRITE_REG(&adapter->hw, E1000_RXMTRL, htonl(0x440088f7)); 5421 5422 /* 5423 * Source Port Queue Filter Setup: 5424 * this is for UDP port filtering 5425 */ 5426 E1000_WRITE_REG(&adapter->hw, E1000_RXUDP, htons(TSYNC_PORT)); 5427 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */ 5428 5429 E1000_WRITE_FLUSH(&adapter->hw); 5430 5431 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5432 if ((rx_ctl & 0x10) == 0) { 5433 device_printf(dev, "Failed to enable RX timestamping\n"); 5434 return (ENXIO); 5435 } 5436 5437 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n"); 5438 5439 return (0); 5440 } 5441 5442 /* 5443 * Disable the Time Sync Feature 5444 */ 5445 static void 5446 em_tsync_disable(struct adapter *adapter) 5447 { 5448 u32 tx_ctl, rx_ctl; 5449 5450 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5451 tx_ctl &= ~0x10; 5452 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5453 E1000_WRITE_FLUSH(&adapter->hw); 5454 5455 /* Invalidate TX Timestamp */ 5456 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH); 5457 5458 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5459 if (tx_ctl & 0x10) 5460 HW_DEBUGOUT("Failed to disable TX timestamping\n"); 5461 5462 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5463 rx_ctl &= ~0x10; 5464 5465 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5466 E1000_WRITE_FLUSH(&adapter->hw); 5467 5468 /* Invalidate RX Timestamp */ 5469 E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 5470 5471 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5472 if (rx_ctl & 0x10) 5473 HW_DEBUGOUT("Failed to disable RX timestamping\n"); 5474 5475 return; 5476 } 5477 #endif /* EM_TIMESYNC */ 5478