1 /****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #include "opt_altq.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #if __FreeBSD_version >= 800000 44 #include <sys/buf_ring.h> 45 #endif 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/kthread.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/module.h> 53 #include <sys/rman.h> 54 #include <sys/socket.h> 55 #include <sys/sockio.h> 56 #include <sys/sysctl.h> 57 #include <sys/taskqueue.h> 58 #if __FreeBSD_version >= 700029 59 #include <sys/eventhandler.h> 60 #endif 61 #include <machine/bus.h> 62 #include <machine/resource.h> 63 64 #include <net/bpf.h> 65 #include <net/ethernet.h> 66 #include <net/if.h> 67 #include <net/if_arp.h> 68 #include <net/if_dl.h> 69 #include <net/if_media.h> 70 71 #include <net/if_types.h> 72 #include <net/if_vlan_var.h> 73 74 #include <netinet/in_systm.h> 75 #include <netinet/in.h> 76 #include <netinet/if_ether.h> 77 #include <netinet/ip.h> 78 #include <netinet/ip6.h> 79 #include <netinet/tcp.h> 80 #include <netinet/udp.h> 81 82 #include <machine/in_cksum.h> 83 #include <dev/pci/pcivar.h> 84 #include <dev/pci/pcireg.h> 85 86 #include "e1000_api.h" 87 #include "e1000_82571.h" 88 #include "if_em.h" 89 90 /********************************************************************* 91 * Set this to one to display debug statistics 92 *********************************************************************/ 93 int em_display_debug_stats = 0; 94 95 /********************************************************************* 96 * Driver version: 97 *********************************************************************/ 98 char em_driver_version[] = "6.9.25"; 99 100 101 /********************************************************************* 102 * PCI Device ID Table 103 * 104 * Used by probe to select devices to load on 105 * Last field stores an index into e1000_strings 106 * Last entry must be all 0s 107 * 108 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 109 *********************************************************************/ 110 111 static em_vendor_info_t em_vendor_info_array[] = 112 { 113 /* Intel(R) PRO/1000 Network Connection */ 114 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, 119 120 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 128 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, 129 130 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 133 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 137 138 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 142 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 143 144 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 152 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 153 PCI_ANY_ID, PCI_ANY_ID, 0}, 154 155 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 158 159 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 162 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 163 PCI_ANY_ID, PCI_ANY_ID, 0}, 164 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 165 PCI_ANY_ID, PCI_ANY_ID, 0}, 166 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 167 PCI_ANY_ID, PCI_ANY_ID, 0}, 168 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 169 PCI_ANY_ID, PCI_ANY_ID, 0}, 170 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 171 PCI_ANY_ID, PCI_ANY_ID, 0}, 172 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 173 PCI_ANY_ID, PCI_ANY_ID, 0}, 174 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 175 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 176 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 177 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 178 179 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 180 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 181 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 182 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, 183 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 184 PCI_ANY_ID, PCI_ANY_ID, 0}, 185 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 186 PCI_ANY_ID, PCI_ANY_ID, 0}, 187 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 188 PCI_ANY_ID, PCI_ANY_ID, 0}, 189 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 190 PCI_ANY_ID, PCI_ANY_ID, 0}, 191 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 192 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 193 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 194 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 195 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 196 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 197 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 198 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, 199 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 200 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 201 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 202 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 203 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 204 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 205 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 206 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 207 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 208 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 209 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, 210 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 211 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 212 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 213 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 214 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 215 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 216 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, 217 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, 218 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, 219 /* required last entry */ 220 { 0, 0, 0, 0, 0} 221 }; 222 223 /********************************************************************* 224 * Table of branding strings for all supported NICs. 225 *********************************************************************/ 226 227 static char *em_strings[] = { 228 "Intel(R) PRO/1000 Network Connection" 229 }; 230 231 /********************************************************************* 232 * Function prototypes 233 *********************************************************************/ 234 static int em_probe(device_t); 235 static int em_attach(device_t); 236 static int em_detach(device_t); 237 static int em_shutdown(device_t); 238 static int em_suspend(device_t); 239 static int em_resume(device_t); 240 static void em_start(struct ifnet *); 241 static void em_start_locked(struct ifnet *ifp); 242 #if __FreeBSD_version >= 800000 243 static int em_mq_start(struct ifnet *, struct mbuf *); 244 static int em_mq_start_locked(struct ifnet *, struct mbuf *); 245 static void em_qflush(struct ifnet *); 246 #endif 247 static int em_ioctl(struct ifnet *, u_long, caddr_t); 248 static void em_init(void *); 249 static void em_init_locked(struct adapter *); 250 static void em_stop(void *); 251 static void em_media_status(struct ifnet *, struct ifmediareq *); 252 static int em_media_change(struct ifnet *); 253 static void em_identify_hardware(struct adapter *); 254 static int em_allocate_pci_resources(struct adapter *); 255 static int em_allocate_legacy(struct adapter *adapter); 256 static int em_allocate_msix(struct adapter *adapter); 257 static int em_setup_msix(struct adapter *); 258 static void em_free_pci_resources(struct adapter *); 259 static void em_local_timer(void *); 260 static int em_hardware_init(struct adapter *); 261 static void em_setup_interface(device_t, struct adapter *); 262 static void em_setup_transmit_structures(struct adapter *); 263 static void em_initialize_transmit_unit(struct adapter *); 264 static int em_setup_receive_structures(struct adapter *); 265 static void em_initialize_receive_unit(struct adapter *); 266 static void em_enable_intr(struct adapter *); 267 static void em_disable_intr(struct adapter *); 268 static void em_free_transmit_structures(struct adapter *); 269 static void em_free_receive_structures(struct adapter *); 270 static void em_update_stats_counters(struct adapter *); 271 static void em_txeof(struct adapter *); 272 static void em_tx_purge(struct adapter *); 273 static int em_allocate_receive_structures(struct adapter *); 274 static int em_allocate_transmit_structures(struct adapter *); 275 static int em_rxeof(struct adapter *, int); 276 #ifndef __NO_STRICT_ALIGNMENT 277 static int em_fixup_rx(struct adapter *); 278 #endif 279 static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, 280 struct mbuf *); 281 static void em_transmit_checksum_setup(struct adapter *, struct mbuf *, 282 u32 *, u32 *); 283 #if __FreeBSD_version >= 700000 284 static bool em_tso_setup(struct adapter *, struct mbuf *, 285 u32 *, u32 *); 286 #endif /* FreeBSD_version >= 700000 */ 287 static void em_set_promisc(struct adapter *); 288 static void em_disable_promisc(struct adapter *); 289 static void em_set_multi(struct adapter *); 290 static void em_print_hw_stats(struct adapter *); 291 static void em_update_link_status(struct adapter *); 292 static int em_get_buf(struct adapter *, int); 293 #if __FreeBSD_version >= 700029 294 static void em_register_vlan(void *, struct ifnet *, u16); 295 static void em_unregister_vlan(void *, struct ifnet *, u16); 296 static void em_setup_vlan_hw_support(struct adapter *); 297 #endif 298 static int em_xmit(struct adapter *, struct mbuf **); 299 static void em_smartspeed(struct adapter *); 300 static int em_82547_fifo_workaround(struct adapter *, int); 301 static void em_82547_update_fifo_head(struct adapter *, int); 302 static int em_82547_tx_fifo_reset(struct adapter *); 303 static void em_82547_move_tail(void *); 304 static int em_dma_malloc(struct adapter *, bus_size_t, 305 struct em_dma_alloc *, int); 306 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 307 static void em_print_debug_info(struct adapter *); 308 static void em_print_nvm_info(struct adapter *); 309 static int em_is_valid_ether_addr(u8 *); 310 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 311 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 312 static u32 em_fill_descriptors (bus_addr_t address, u32 length, 313 PDESC_ARRAY desc_array); 314 static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 315 static void em_add_int_delay_sysctl(struct adapter *, const char *, 316 const char *, struct em_int_delay_info *, int, int); 317 /* Management and WOL Support */ 318 static void em_init_manageability(struct adapter *); 319 static void em_release_manageability(struct adapter *); 320 static void em_get_hw_control(struct adapter *); 321 static void em_release_hw_control(struct adapter *); 322 static void em_get_wakeup(device_t); 323 static void em_enable_wakeup(device_t); 324 static int em_enable_phy_wakeup(struct adapter *); 325 326 #ifdef EM_LEGACY_IRQ 327 static void em_intr(void *); 328 #else /* FAST IRQ */ 329 #if __FreeBSD_version < 700000 330 static void em_irq_fast(void *); 331 #else 332 static int em_irq_fast(void *); 333 #endif 334 335 /* MSIX handlers */ 336 static void em_msix_tx(void *); 337 static void em_msix_rx(void *); 338 static void em_msix_link(void *); 339 static void em_handle_rx(void *context, int pending); 340 static void em_handle_tx(void *context, int pending); 341 342 static void em_handle_rxtx(void *context, int pending); 343 static void em_handle_link(void *context, int pending); 344 static void em_add_rx_process_limit(struct adapter *, const char *, 345 const char *, int *, int); 346 #endif /* ~EM_LEGACY_IRQ */ 347 348 #ifdef DEVICE_POLLING 349 static poll_handler_t em_poll; 350 #endif /* POLLING */ 351 352 /********************************************************************* 353 * FreeBSD Device Interface Entry Points 354 *********************************************************************/ 355 356 static device_method_t em_methods[] = { 357 /* Device interface */ 358 DEVMETHOD(device_probe, em_probe), 359 DEVMETHOD(device_attach, em_attach), 360 DEVMETHOD(device_detach, em_detach), 361 DEVMETHOD(device_shutdown, em_shutdown), 362 DEVMETHOD(device_suspend, em_suspend), 363 DEVMETHOD(device_resume, em_resume), 364 {0, 0} 365 }; 366 367 static driver_t em_driver = { 368 "em", em_methods, sizeof(struct adapter), 369 }; 370 371 static devclass_t em_devclass; 372 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 373 MODULE_DEPEND(em, pci, 1, 1, 1); 374 MODULE_DEPEND(em, ether, 1, 1, 1); 375 376 /********************************************************************* 377 * Tunable default values. 378 *********************************************************************/ 379 380 #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 381 #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 382 #define M_TSO_LEN 66 383 384 /* Allow common code without TSO */ 385 #ifndef CSUM_TSO 386 #define CSUM_TSO 0 387 #endif 388 389 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 390 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 391 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 392 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 393 static int em_rxd = EM_DEFAULT_RXD; 394 static int em_txd = EM_DEFAULT_TXD; 395 static int em_smart_pwr_down = FALSE; 396 /* Controls whether promiscuous also shows bad packets */ 397 static int em_debug_sbp = FALSE; 398 /* Local switch for MSI/MSIX */ 399 static int em_enable_msi = TRUE; 400 401 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 402 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 403 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 404 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 405 TUNABLE_INT("hw.em.rxd", &em_rxd); 406 TUNABLE_INT("hw.em.txd", &em_txd); 407 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 408 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 409 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi); 410 411 #ifndef EM_LEGACY_IRQ 412 /* How many packets rxeof tries to clean at a time */ 413 static int em_rx_process_limit = 100; 414 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 415 #endif 416 417 /* Flow control setting - default to FULL */ 418 static int em_fc_setting = e1000_fc_full; 419 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); 420 421 /* 422 ** Shadow VFTA table, this is needed because 423 ** the real vlan filter table gets cleared during 424 ** a soft reset and the driver needs to be able 425 ** to repopulate it. 426 */ 427 static u32 em_shadow_vfta[EM_VFTA_SIZE]; 428 429 /* Global used in WOL setup with multiport cards */ 430 static int global_quad_port_a = 0; 431 432 /********************************************************************* 433 * Device identification routine 434 * 435 * em_probe determines if the driver should be loaded on 436 * adapter based on PCI vendor/device id of the adapter. 437 * 438 * return BUS_PROBE_DEFAULT on success, positive on failure 439 *********************************************************************/ 440 441 static int 442 em_probe(device_t dev) 443 { 444 char adapter_name[60]; 445 u16 pci_vendor_id = 0; 446 u16 pci_device_id = 0; 447 u16 pci_subvendor_id = 0; 448 u16 pci_subdevice_id = 0; 449 em_vendor_info_t *ent; 450 451 INIT_DEBUGOUT("em_probe: begin"); 452 453 pci_vendor_id = pci_get_vendor(dev); 454 if (pci_vendor_id != EM_VENDOR_ID) 455 return (ENXIO); 456 457 pci_device_id = pci_get_device(dev); 458 pci_subvendor_id = pci_get_subvendor(dev); 459 pci_subdevice_id = pci_get_subdevice(dev); 460 461 ent = em_vendor_info_array; 462 while (ent->vendor_id != 0) { 463 if ((pci_vendor_id == ent->vendor_id) && 464 (pci_device_id == ent->device_id) && 465 466 ((pci_subvendor_id == ent->subvendor_id) || 467 (ent->subvendor_id == PCI_ANY_ID)) && 468 469 ((pci_subdevice_id == ent->subdevice_id) || 470 (ent->subdevice_id == PCI_ANY_ID))) { 471 sprintf(adapter_name, "%s %s", 472 em_strings[ent->index], 473 em_driver_version); 474 device_set_desc_copy(dev, adapter_name); 475 return (BUS_PROBE_DEFAULT); 476 } 477 ent++; 478 } 479 480 return (ENXIO); 481 } 482 483 /********************************************************************* 484 * Device initialization routine 485 * 486 * The attach entry point is called when the driver is being loaded. 487 * This routine identifies the type of hardware, allocates all resources 488 * and initializes the hardware. 489 * 490 * return 0 on success, positive on failure 491 *********************************************************************/ 492 493 static int 494 em_attach(device_t dev) 495 { 496 struct adapter *adapter; 497 int tsize, rsize; 498 int error = 0; 499 500 INIT_DEBUGOUT("em_attach: begin"); 501 502 adapter = device_get_softc(dev); 503 adapter->dev = adapter->osdep.dev = dev; 504 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 505 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); 506 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); 507 508 /* SYSCTL stuff */ 509 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 510 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 511 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 512 em_sysctl_debug_info, "I", "Debug Information"); 513 514 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 515 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 516 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 517 em_sysctl_stats, "I", "Statistics"); 518 519 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 520 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); 521 522 /* Determine hardware and mac info */ 523 em_identify_hardware(adapter); 524 525 /* Setup PCI resources */ 526 if (em_allocate_pci_resources(adapter)) { 527 device_printf(dev, "Allocation of PCI resources failed\n"); 528 error = ENXIO; 529 goto err_pci; 530 } 531 532 /* 533 ** For ICH8 and family we need to 534 ** map the flash memory, and this 535 ** must happen after the MAC is 536 ** identified 537 */ 538 if ((adapter->hw.mac.type == e1000_ich8lan) || 539 (adapter->hw.mac.type == e1000_pchlan) || 540 (adapter->hw.mac.type == e1000_ich9lan) || 541 (adapter->hw.mac.type == e1000_ich10lan)) { 542 int rid = EM_BAR_TYPE_FLASH; 543 adapter->flash = bus_alloc_resource_any(dev, 544 SYS_RES_MEMORY, &rid, RF_ACTIVE); 545 if (adapter->flash == NULL) { 546 device_printf(dev, "Mapping of Flash failed\n"); 547 error = ENXIO; 548 goto err_pci; 549 } 550 /* This is used in the shared code */ 551 adapter->hw.flash_address = (u8 *)adapter->flash; 552 adapter->osdep.flash_bus_space_tag = 553 rman_get_bustag(adapter->flash); 554 adapter->osdep.flash_bus_space_handle = 555 rman_get_bushandle(adapter->flash); 556 } 557 558 /* Do Shared Code initialization */ 559 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 560 device_printf(dev, "Setup of Shared code failed\n"); 561 error = ENXIO; 562 goto err_pci; 563 } 564 565 e1000_get_bus_info(&adapter->hw); 566 567 /* Set up some sysctls for the tunable interrupt delays */ 568 em_add_int_delay_sysctl(adapter, "rx_int_delay", 569 "receive interrupt delay in usecs", &adapter->rx_int_delay, 570 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 571 em_add_int_delay_sysctl(adapter, "tx_int_delay", 572 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 573 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 574 if (adapter->hw.mac.type >= e1000_82540) { 575 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 576 "receive interrupt delay limit in usecs", 577 &adapter->rx_abs_int_delay, 578 E1000_REGISTER(&adapter->hw, E1000_RADV), 579 em_rx_abs_int_delay_dflt); 580 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 581 "transmit interrupt delay limit in usecs", 582 &adapter->tx_abs_int_delay, 583 E1000_REGISTER(&adapter->hw, E1000_TADV), 584 em_tx_abs_int_delay_dflt); 585 } 586 587 #ifndef EM_LEGACY_IRQ 588 /* Sysctls for limiting the amount of work done in the taskqueue */ 589 em_add_rx_process_limit(adapter, "rx_processing_limit", 590 "max number of rx packets to process", &adapter->rx_process_limit, 591 em_rx_process_limit); 592 #endif 593 594 /* 595 * Validate number of transmit and receive descriptors. It 596 * must not exceed hardware maximum, and must be multiple 597 * of E1000_DBA_ALIGN. 598 */ 599 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 600 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 601 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 602 (em_txd < EM_MIN_TXD)) { 603 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 604 EM_DEFAULT_TXD, em_txd); 605 adapter->num_tx_desc = EM_DEFAULT_TXD; 606 } else 607 adapter->num_tx_desc = em_txd; 608 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 609 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 610 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 611 (em_rxd < EM_MIN_RXD)) { 612 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 613 EM_DEFAULT_RXD, em_rxd); 614 adapter->num_rx_desc = EM_DEFAULT_RXD; 615 } else 616 adapter->num_rx_desc = em_rxd; 617 618 adapter->hw.mac.autoneg = DO_AUTO_NEG; 619 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 620 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 621 adapter->rx_buffer_len = 2048; 622 623 e1000_init_script_state_82541(&adapter->hw, TRUE); 624 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 625 626 /* Copper options */ 627 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 628 adapter->hw.phy.mdix = AUTO_ALL_MODES; 629 adapter->hw.phy.disable_polarity_correction = FALSE; 630 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 631 } 632 633 /* 634 * Set the frame limits assuming 635 * standard ethernet sized frames. 636 */ 637 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 638 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 639 640 /* 641 * This controls when hardware reports transmit completion 642 * status. 643 */ 644 adapter->hw.mac.report_tx_early = 1; 645 646 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 647 EM_DBA_ALIGN); 648 649 /* Allocate Transmit Descriptor ring */ 650 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { 651 device_printf(dev, "Unable to allocate tx_desc memory\n"); 652 error = ENOMEM; 653 goto err_tx_desc; 654 } 655 adapter->tx_desc_base = 656 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; 657 658 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 659 EM_DBA_ALIGN); 660 661 /* Allocate Receive Descriptor ring */ 662 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { 663 device_printf(dev, "Unable to allocate rx_desc memory\n"); 664 error = ENOMEM; 665 goto err_rx_desc; 666 } 667 adapter->rx_desc_base = 668 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; 669 670 /* 671 ** Start from a known state, this is 672 ** important in reading the nvm and 673 ** mac from that. 674 */ 675 e1000_reset_hw(&adapter->hw); 676 677 /* Make sure we have a good EEPROM before we read from it */ 678 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 679 /* 680 ** Some PCI-E parts fail the first check due to 681 ** the link being in sleep state, call it again, 682 ** if it fails a second time its a real issue. 683 */ 684 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 685 device_printf(dev, 686 "The EEPROM Checksum Is Not Valid\n"); 687 error = EIO; 688 goto err_hw_init; 689 } 690 } 691 692 /* Copy the permanent MAC address out of the EEPROM */ 693 if (e1000_read_mac_addr(&adapter->hw) < 0) { 694 device_printf(dev, "EEPROM read error while reading MAC" 695 " address\n"); 696 error = EIO; 697 goto err_hw_init; 698 } 699 700 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 701 device_printf(dev, "Invalid MAC address\n"); 702 error = EIO; 703 goto err_hw_init; 704 } 705 706 /* Initialize the hardware */ 707 if (em_hardware_init(adapter)) { 708 device_printf(dev, "Unable to initialize the hardware\n"); 709 error = EIO; 710 goto err_hw_init; 711 } 712 713 /* Allocate transmit descriptors and buffers */ 714 if (em_allocate_transmit_structures(adapter)) { 715 device_printf(dev, "Could not setup transmit structures\n"); 716 error = ENOMEM; 717 goto err_tx_struct; 718 } 719 720 /* Allocate receive descriptors and buffers */ 721 if (em_allocate_receive_structures(adapter)) { 722 device_printf(dev, "Could not setup receive structures\n"); 723 error = ENOMEM; 724 goto err_rx_struct; 725 } 726 727 /* 728 ** Do interrupt configuration 729 */ 730 if (adapter->msi > 1) /* Do MSI/X */ 731 error = em_allocate_msix(adapter); 732 else /* MSI or Legacy */ 733 error = em_allocate_legacy(adapter); 734 if (error) 735 goto err_rx_struct; 736 737 /* 738 * Get Wake-on-Lan and Management info for later use 739 */ 740 em_get_wakeup(dev); 741 742 /* Setup OS specific network interface */ 743 em_setup_interface(dev, adapter); 744 745 /* Initialize statistics */ 746 em_update_stats_counters(adapter); 747 748 adapter->hw.mac.get_link_status = 1; 749 em_update_link_status(adapter); 750 751 /* Indicate SOL/IDER usage */ 752 if (e1000_check_reset_block(&adapter->hw)) 753 device_printf(dev, 754 "PHY reset is blocked due to SOL/IDER session.\n"); 755 756 /* Do we need workaround for 82544 PCI-X adapter? */ 757 if (adapter->hw.bus.type == e1000_bus_type_pcix && 758 adapter->hw.mac.type == e1000_82544) 759 adapter->pcix_82544 = TRUE; 760 else 761 adapter->pcix_82544 = FALSE; 762 763 #if __FreeBSD_version >= 700029 764 /* Register for VLAN events */ 765 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 766 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 767 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 768 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 769 #endif 770 771 /* Non-AMT based hardware can now take control from firmware */ 772 if (adapter->has_manage && !adapter->has_amt) 773 em_get_hw_control(adapter); 774 775 /* Tell the stack that the interface is not active */ 776 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 777 778 INIT_DEBUGOUT("em_attach: end"); 779 780 return (0); 781 782 err_rx_struct: 783 em_free_transmit_structures(adapter); 784 err_tx_struct: 785 err_hw_init: 786 em_release_hw_control(adapter); 787 em_dma_free(adapter, &adapter->rxdma); 788 err_rx_desc: 789 em_dma_free(adapter, &adapter->txdma); 790 err_tx_desc: 791 err_pci: 792 em_free_pci_resources(adapter); 793 EM_TX_LOCK_DESTROY(adapter); 794 EM_RX_LOCK_DESTROY(adapter); 795 EM_CORE_LOCK_DESTROY(adapter); 796 797 return (error); 798 } 799 800 /********************************************************************* 801 * Device removal routine 802 * 803 * The detach entry point is called when the driver is being removed. 804 * This routine stops the adapter and deallocates all the resources 805 * that were allocated for driver operation. 806 * 807 * return 0 on success, positive on failure 808 *********************************************************************/ 809 810 static int 811 em_detach(device_t dev) 812 { 813 struct adapter *adapter = device_get_softc(dev); 814 struct ifnet *ifp = adapter->ifp; 815 816 INIT_DEBUGOUT("em_detach: begin"); 817 818 /* Make sure VLANS are not using driver */ 819 #if __FreeBSD_version >= 700000 820 if (adapter->ifp->if_vlantrunk != NULL) { 821 #else 822 if (adapter->ifp->if_nvlans != 0) { 823 #endif 824 device_printf(dev,"Vlan in use, detach first\n"); 825 return (EBUSY); 826 } 827 828 #ifdef DEVICE_POLLING 829 if (ifp->if_capenable & IFCAP_POLLING) 830 ether_poll_deregister(ifp); 831 #endif 832 833 EM_CORE_LOCK(adapter); 834 EM_TX_LOCK(adapter); 835 adapter->in_detach = 1; 836 em_stop(adapter); 837 e1000_phy_hw_reset(&adapter->hw); 838 839 em_release_manageability(adapter); 840 841 EM_TX_UNLOCK(adapter); 842 EM_CORE_UNLOCK(adapter); 843 844 #if __FreeBSD_version >= 700029 845 /* Unregister VLAN events */ 846 if (adapter->vlan_attach != NULL) 847 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 848 if (adapter->vlan_detach != NULL) 849 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 850 #endif 851 852 ether_ifdetach(adapter->ifp); 853 callout_drain(&adapter->timer); 854 callout_drain(&adapter->tx_fifo_timer); 855 856 em_free_pci_resources(adapter); 857 bus_generic_detach(dev); 858 if_free(ifp); 859 860 em_free_transmit_structures(adapter); 861 em_free_receive_structures(adapter); 862 863 /* Free Transmit Descriptor ring */ 864 if (adapter->tx_desc_base) { 865 em_dma_free(adapter, &adapter->txdma); 866 adapter->tx_desc_base = NULL; 867 } 868 869 /* Free Receive Descriptor ring */ 870 if (adapter->rx_desc_base) { 871 em_dma_free(adapter, &adapter->rxdma); 872 adapter->rx_desc_base = NULL; 873 } 874 875 em_release_hw_control(adapter); 876 EM_TX_LOCK_DESTROY(adapter); 877 EM_RX_LOCK_DESTROY(adapter); 878 EM_CORE_LOCK_DESTROY(adapter); 879 880 return (0); 881 } 882 883 /********************************************************************* 884 * 885 * Shutdown entry point 886 * 887 **********************************************************************/ 888 889 static int 890 em_shutdown(device_t dev) 891 { 892 return em_suspend(dev); 893 } 894 895 /* 896 * Suspend/resume device methods. 897 */ 898 static int 899 em_suspend(device_t dev) 900 { 901 struct adapter *adapter = device_get_softc(dev); 902 903 EM_CORE_LOCK(adapter); 904 905 em_release_manageability(adapter); 906 em_release_hw_control(adapter); 907 em_enable_wakeup(dev); 908 909 EM_CORE_UNLOCK(adapter); 910 911 return bus_generic_suspend(dev); 912 } 913 914 static int 915 em_resume(device_t dev) 916 { 917 struct adapter *adapter = device_get_softc(dev); 918 struct ifnet *ifp = adapter->ifp; 919 920 EM_CORE_LOCK(adapter); 921 em_init_locked(adapter); 922 em_init_manageability(adapter); 923 EM_CORE_UNLOCK(adapter); 924 em_start(ifp); 925 926 return bus_generic_resume(dev); 927 } 928 929 930 /********************************************************************* 931 * Transmit entry point 932 * 933 * em_start is called by the stack to initiate a transmit. 934 * The driver will remain in this routine as long as there are 935 * packets to transmit and transmit resources are available. 936 * In case resources are not available stack is notified and 937 * the packet is requeued. 938 **********************************************************************/ 939 940 #if __FreeBSD_version >= 800000 941 static int 942 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m) 943 { 944 struct adapter *adapter = ifp->if_softc; 945 struct mbuf *next; 946 int error = E1000_SUCCESS; 947 948 EM_TX_LOCK_ASSERT(adapter); 949 /* To allow being called from a tasklet */ 950 if (m == NULL) 951 goto process; 952 953 if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 954 IFF_DRV_RUNNING) 955 || (!adapter->link_active)) { 956 error = drbr_enqueue(ifp, adapter->br, m); 957 return (error); 958 } else if (!drbr_needs_enqueue(ifp, adapter->br) && 959 (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { 960 if ((error = em_xmit(adapter, &m)) != 0) { 961 if (m) 962 error = drbr_enqueue(ifp, adapter->br, m); 963 return (error); 964 } else { 965 /* 966 * We've bypassed the buf ring so we need to update 967 * ifp directly 968 */ 969 drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags); 970 /* 971 ** Send a copy of the frame to the BPF 972 ** listener and set the watchdog on. 973 */ 974 ETHER_BPF_MTAP(ifp, m); 975 adapter->watchdog_check = TRUE; 976 } 977 } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) 978 return (error); 979 980 process: 981 if (drbr_empty(ifp, adapter->br)) 982 return(error); 983 /* Process the queue */ 984 while (TRUE) { 985 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 986 break; 987 next = drbr_dequeue(ifp, adapter->br); 988 if (next == NULL) 989 break; 990 if ((error = em_xmit(adapter, &next)) != 0) { 991 if (next != NULL) 992 error = drbr_enqueue(ifp, adapter->br, next); 993 break; 994 } 995 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 996 ETHER_BPF_MTAP(ifp, next); 997 /* Set the watchdog */ 998 adapter->watchdog_check = TRUE; 999 } 1000 1001 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) 1002 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1003 1004 return (error); 1005 } 1006 1007 /* 1008 ** Multiqueue capable stack interface, this is not 1009 ** yet truely multiqueue, but that is coming... 1010 */ 1011 static int 1012 em_mq_start(struct ifnet *ifp, struct mbuf *m) 1013 { 1014 1015 struct adapter *adapter = ifp->if_softc; 1016 int error = 0; 1017 1018 if (EM_TX_TRYLOCK(adapter)) { 1019 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1020 error = em_mq_start_locked(ifp, m); 1021 EM_TX_UNLOCK(adapter); 1022 } else 1023 error = drbr_enqueue(ifp, adapter->br, m); 1024 1025 return (error); 1026 } 1027 1028 static void 1029 em_qflush(struct ifnet *ifp) 1030 { 1031 struct mbuf *m; 1032 struct adapter *adapter = (struct adapter *)ifp->if_softc; 1033 1034 EM_TX_LOCK(adapter); 1035 while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL) 1036 m_freem(m); 1037 if_qflush(ifp); 1038 EM_TX_UNLOCK(adapter); 1039 } 1040 #endif /* FreeBSD_version */ 1041 1042 static void 1043 em_start_locked(struct ifnet *ifp) 1044 { 1045 struct adapter *adapter = ifp->if_softc; 1046 struct mbuf *m_head; 1047 1048 EM_TX_LOCK_ASSERT(adapter); 1049 1050 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1051 IFF_DRV_RUNNING) 1052 return; 1053 if (!adapter->link_active) 1054 return; 1055 1056 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 1057 1058 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 1059 if (m_head == NULL) 1060 break; 1061 /* 1062 * Encapsulation can modify our pointer, and or make it 1063 * NULL on failure. In that event, we can't requeue. 1064 */ 1065 if (em_xmit(adapter, &m_head)) { 1066 if (m_head == NULL) 1067 break; 1068 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1069 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 1070 break; 1071 } 1072 1073 /* Send a copy of the frame to the BPF listener */ 1074 ETHER_BPF_MTAP(ifp, m_head); 1075 1076 /* Set timeout in case hardware has problems transmitting. */ 1077 adapter->watchdog_check = TRUE; 1078 } 1079 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) 1080 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1081 1082 return; 1083 } 1084 1085 static void 1086 em_start(struct ifnet *ifp) 1087 { 1088 struct adapter *adapter = ifp->if_softc; 1089 1090 EM_TX_LOCK(adapter); 1091 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1092 em_start_locked(ifp); 1093 EM_TX_UNLOCK(adapter); 1094 } 1095 1096 /********************************************************************* 1097 * Ioctl entry point 1098 * 1099 * em_ioctl is called when the user wants to configure the 1100 * interface. 1101 * 1102 * return 0 on success, positive on failure 1103 **********************************************************************/ 1104 1105 static int 1106 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1107 { 1108 struct adapter *adapter = ifp->if_softc; 1109 struct ifreq *ifr = (struct ifreq *)data; 1110 #ifdef INET 1111 struct ifaddr *ifa = (struct ifaddr *)data; 1112 #endif 1113 int error = 0; 1114 1115 if (adapter->in_detach) 1116 return (error); 1117 1118 switch (command) { 1119 case SIOCSIFADDR: 1120 #ifdef INET 1121 if (ifa->ifa_addr->sa_family == AF_INET) { 1122 /* 1123 * XXX 1124 * Since resetting hardware takes a very long time 1125 * and results in link renegotiation we only 1126 * initialize the hardware only when it is absolutely 1127 * required. 1128 */ 1129 ifp->if_flags |= IFF_UP; 1130 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1131 EM_CORE_LOCK(adapter); 1132 em_init_locked(adapter); 1133 EM_CORE_UNLOCK(adapter); 1134 } 1135 arp_ifinit(ifp, ifa); 1136 } else 1137 #endif 1138 error = ether_ioctl(ifp, command, data); 1139 break; 1140 case SIOCSIFMTU: 1141 { 1142 int max_frame_size; 1143 u16 eeprom_data = 0; 1144 1145 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1146 1147 EM_CORE_LOCK(adapter); 1148 switch (adapter->hw.mac.type) { 1149 case e1000_82573: 1150 /* 1151 * 82573 only supports jumbo frames 1152 * if ASPM is disabled. 1153 */ 1154 e1000_read_nvm(&adapter->hw, 1155 NVM_INIT_3GIO_3, 1, &eeprom_data); 1156 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1157 max_frame_size = ETHER_MAX_LEN; 1158 break; 1159 } 1160 /* Allow Jumbo frames - fall thru */ 1161 case e1000_82571: 1162 case e1000_82572: 1163 case e1000_ich9lan: 1164 case e1000_ich10lan: 1165 case e1000_82574: 1166 case e1000_80003es2lan: /* Limit Jumbo Frame size */ 1167 max_frame_size = 9234; 1168 break; 1169 case e1000_pchlan: 1170 max_frame_size = 4096; 1171 break; 1172 /* Adapters that do not support jumbo frames */ 1173 case e1000_82542: 1174 case e1000_82583: 1175 case e1000_ich8lan: 1176 max_frame_size = ETHER_MAX_LEN; 1177 break; 1178 default: 1179 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1180 } 1181 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1182 ETHER_CRC_LEN) { 1183 EM_CORE_UNLOCK(adapter); 1184 error = EINVAL; 1185 break; 1186 } 1187 1188 ifp->if_mtu = ifr->ifr_mtu; 1189 adapter->max_frame_size = 1190 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1191 em_init_locked(adapter); 1192 EM_CORE_UNLOCK(adapter); 1193 break; 1194 } 1195 case SIOCSIFFLAGS: 1196 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1197 SIOCSIFFLAGS (Set Interface Flags)"); 1198 EM_CORE_LOCK(adapter); 1199 if (ifp->if_flags & IFF_UP) { 1200 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1201 if ((ifp->if_flags ^ adapter->if_flags) & 1202 (IFF_PROMISC | IFF_ALLMULTI)) { 1203 em_disable_promisc(adapter); 1204 em_set_promisc(adapter); 1205 } 1206 } else 1207 em_init_locked(adapter); 1208 } else 1209 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1210 EM_TX_LOCK(adapter); 1211 em_stop(adapter); 1212 EM_TX_UNLOCK(adapter); 1213 } 1214 adapter->if_flags = ifp->if_flags; 1215 EM_CORE_UNLOCK(adapter); 1216 break; 1217 case SIOCADDMULTI: 1218 case SIOCDELMULTI: 1219 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1220 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1221 EM_CORE_LOCK(adapter); 1222 em_disable_intr(adapter); 1223 em_set_multi(adapter); 1224 if (adapter->hw.mac.type == e1000_82542 && 1225 adapter->hw.revision_id == E1000_REVISION_2) { 1226 em_initialize_receive_unit(adapter); 1227 } 1228 #ifdef DEVICE_POLLING 1229 if (!(ifp->if_capenable & IFCAP_POLLING)) 1230 #endif 1231 em_enable_intr(adapter); 1232 EM_CORE_UNLOCK(adapter); 1233 } 1234 break; 1235 case SIOCSIFMEDIA: 1236 /* Check SOL/IDER usage */ 1237 EM_CORE_LOCK(adapter); 1238 if (e1000_check_reset_block(&adapter->hw)) { 1239 EM_CORE_UNLOCK(adapter); 1240 device_printf(adapter->dev, "Media change is" 1241 " blocked due to SOL/IDER session.\n"); 1242 break; 1243 } 1244 EM_CORE_UNLOCK(adapter); 1245 case SIOCGIFMEDIA: 1246 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1247 SIOCxIFMEDIA (Get/Set Interface Media)"); 1248 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1249 break; 1250 case SIOCSIFCAP: 1251 { 1252 int mask, reinit; 1253 1254 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1255 reinit = 0; 1256 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1257 #ifdef DEVICE_POLLING 1258 if (mask & IFCAP_POLLING) { 1259 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1260 error = ether_poll_register(em_poll, ifp); 1261 if (error) 1262 return (error); 1263 EM_CORE_LOCK(adapter); 1264 em_disable_intr(adapter); 1265 ifp->if_capenable |= IFCAP_POLLING; 1266 EM_CORE_UNLOCK(adapter); 1267 } else { 1268 error = ether_poll_deregister(ifp); 1269 /* Enable interrupt even in error case */ 1270 EM_CORE_LOCK(adapter); 1271 em_enable_intr(adapter); 1272 ifp->if_capenable &= ~IFCAP_POLLING; 1273 EM_CORE_UNLOCK(adapter); 1274 } 1275 } 1276 #endif 1277 if (mask & IFCAP_HWCSUM) { 1278 ifp->if_capenable ^= IFCAP_HWCSUM; 1279 reinit = 1; 1280 } 1281 #if __FreeBSD_version >= 700000 1282 if (mask & IFCAP_TSO4) { 1283 ifp->if_capenable ^= IFCAP_TSO4; 1284 reinit = 1; 1285 } 1286 #endif 1287 if (mask & IFCAP_VLAN_HWTAGGING) { 1288 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1289 reinit = 1; 1290 } 1291 1292 if (mask & IFCAP_VLAN_HWFILTER) { 1293 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1294 reinit = 1; 1295 } 1296 1297 if ((mask & IFCAP_WOL) && 1298 (ifp->if_capabilities & IFCAP_WOL) != 0) { 1299 if (mask & IFCAP_WOL_MCAST) 1300 ifp->if_capenable ^= IFCAP_WOL_MCAST; 1301 if (mask & IFCAP_WOL_MAGIC) 1302 ifp->if_capenable ^= IFCAP_WOL_MAGIC; 1303 } 1304 1305 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1306 em_init(adapter); 1307 #if __FreeBSD_version >= 700000 1308 VLAN_CAPABILITIES(ifp); 1309 #endif 1310 break; 1311 } 1312 1313 default: 1314 error = ether_ioctl(ifp, command, data); 1315 break; 1316 } 1317 1318 return (error); 1319 } 1320 1321 1322 /********************************************************************* 1323 * Init entry point 1324 * 1325 * This routine is used in two ways. It is used by the stack as 1326 * init entry point in network interface structure. It is also used 1327 * by the driver as a hw/sw initialization routine to get to a 1328 * consistent state. 1329 * 1330 * return 0 on success, positive on failure 1331 **********************************************************************/ 1332 1333 static void 1334 em_init_locked(struct adapter *adapter) 1335 { 1336 struct ifnet *ifp = adapter->ifp; 1337 device_t dev = adapter->dev; 1338 u32 pba; 1339 1340 INIT_DEBUGOUT("em_init: begin"); 1341 1342 EM_CORE_LOCK_ASSERT(adapter); 1343 1344 EM_TX_LOCK(adapter); 1345 em_stop(adapter); 1346 EM_TX_UNLOCK(adapter); 1347 1348 /* 1349 * Packet Buffer Allocation (PBA) 1350 * Writing PBA sets the receive portion of the buffer 1351 * the remainder is used for the transmit buffer. 1352 * 1353 * Devices before the 82547 had a Packet Buffer of 64K. 1354 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 1355 * After the 82547 the buffer was reduced to 40K. 1356 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 1357 * Note: default does not leave enough room for Jumbo Frame >10k. 1358 */ 1359 switch (adapter->hw.mac.type) { 1360 case e1000_82547: 1361 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 1362 if (adapter->max_frame_size > 8192) 1363 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 1364 else 1365 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 1366 adapter->tx_fifo_head = 0; 1367 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 1368 adapter->tx_fifo_size = 1369 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 1370 break; 1371 /* Total Packet Buffer on these is 48K */ 1372 case e1000_82571: 1373 case e1000_82572: 1374 case e1000_80003es2lan: 1375 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1376 break; 1377 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1378 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1379 break; 1380 case e1000_82574: 1381 case e1000_82583: 1382 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1383 break; 1384 case e1000_ich9lan: 1385 case e1000_ich10lan: 1386 case e1000_pchlan: 1387 pba = E1000_PBA_10K; 1388 break; 1389 case e1000_ich8lan: 1390 pba = E1000_PBA_8K; 1391 break; 1392 default: 1393 /* Devices before 82547 had a Packet Buffer of 64K. */ 1394 if (adapter->max_frame_size > 8192) 1395 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1396 else 1397 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1398 } 1399 1400 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1401 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1402 1403 /* Get the latest mac address, User can use a LAA */ 1404 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1405 ETHER_ADDR_LEN); 1406 1407 /* Put the address into the Receive Address Array */ 1408 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1409 1410 /* 1411 * With the 82571 adapter, RAR[0] may be overwritten 1412 * when the other port is reset, we make a duplicate 1413 * in RAR[14] for that eventuality, this assures 1414 * the interface continues to function. 1415 */ 1416 if (adapter->hw.mac.type == e1000_82571) { 1417 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1418 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1419 E1000_RAR_ENTRIES - 1); 1420 } 1421 1422 /* Initialize the hardware */ 1423 if (em_hardware_init(adapter)) { 1424 device_printf(dev, "Unable to initialize the hardware\n"); 1425 return; 1426 } 1427 em_update_link_status(adapter); 1428 1429 /* Setup VLAN support, basic and offload if available */ 1430 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1431 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1432 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1433 /* Use real VLAN Filter support */ 1434 em_setup_vlan_hw_support(adapter); 1435 else { 1436 u32 ctrl; 1437 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1438 ctrl |= E1000_CTRL_VME; 1439 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1440 } 1441 } 1442 1443 /* Set hardware offload abilities */ 1444 ifp->if_hwassist = 0; 1445 if (adapter->hw.mac.type >= e1000_82543) { 1446 if (ifp->if_capenable & IFCAP_TXCSUM) 1447 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1448 #if __FreeBSD_version >= 700000 1449 if (ifp->if_capenable & IFCAP_TSO4) 1450 ifp->if_hwassist |= CSUM_TSO; 1451 #endif 1452 } 1453 1454 /* Configure for OS presence */ 1455 em_init_manageability(adapter); 1456 1457 /* Prepare transmit descriptors and buffers */ 1458 em_setup_transmit_structures(adapter); 1459 em_initialize_transmit_unit(adapter); 1460 1461 /* Setup Multicast table */ 1462 em_set_multi(adapter); 1463 1464 /* Prepare receive descriptors and buffers */ 1465 if (em_setup_receive_structures(adapter)) { 1466 device_printf(dev, "Could not setup receive structures\n"); 1467 EM_TX_LOCK(adapter); 1468 em_stop(adapter); 1469 EM_TX_UNLOCK(adapter); 1470 return; 1471 } 1472 em_initialize_receive_unit(adapter); 1473 1474 /* Don't lose promiscuous settings */ 1475 em_set_promisc(adapter); 1476 1477 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1478 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1479 1480 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1481 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1482 1483 /* MSI/X configuration for 82574 */ 1484 if (adapter->hw.mac.type == e1000_82574) { 1485 int tmp; 1486 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1487 tmp |= E1000_CTRL_EXT_PBA_CLR; 1488 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1489 /* 1490 ** Set the IVAR - interrupt vector routing. 1491 ** Each nibble represents a vector, high bit 1492 ** is enable, other 3 bits are the MSIX table 1493 ** entry, we map RXQ0 to 0, TXQ0 to 1, and 1494 ** Link (other) to 2, hence the magic number. 1495 */ 1496 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); 1497 } 1498 1499 #ifdef DEVICE_POLLING 1500 /* 1501 * Only enable interrupts if we are not polling, make sure 1502 * they are off otherwise. 1503 */ 1504 if (ifp->if_capenable & IFCAP_POLLING) 1505 em_disable_intr(adapter); 1506 else 1507 #endif /* DEVICE_POLLING */ 1508 em_enable_intr(adapter); 1509 1510 /* AMT based hardware can now take control from firmware */ 1511 if (adapter->has_manage && adapter->has_amt) 1512 em_get_hw_control(adapter); 1513 1514 /* Don't reset the phy next time init gets called */ 1515 adapter->hw.phy.reset_disable = TRUE; 1516 } 1517 1518 static void 1519 em_init(void *arg) 1520 { 1521 struct adapter *adapter = arg; 1522 1523 EM_CORE_LOCK(adapter); 1524 em_init_locked(adapter); 1525 EM_CORE_UNLOCK(adapter); 1526 } 1527 1528 1529 #ifdef DEVICE_POLLING 1530 /********************************************************************* 1531 * 1532 * Legacy polling routine 1533 * 1534 *********************************************************************/ 1535 static int 1536 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1537 { 1538 struct adapter *adapter = ifp->if_softc; 1539 u32 reg_icr, rx_done = 0; 1540 1541 EM_CORE_LOCK(adapter); 1542 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1543 EM_CORE_UNLOCK(adapter); 1544 return (rx_done); 1545 } 1546 1547 if (cmd == POLL_AND_CHECK_STATUS) { 1548 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1549 /* Link status change */ 1550 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1551 adapter->hw.mac.get_link_status = 1; 1552 em_update_link_status(adapter); 1553 } 1554 if (reg_icr & E1000_ICR_RXO) 1555 adapter->rx_overruns++; 1556 } 1557 EM_CORE_UNLOCK(adapter); 1558 1559 rx_done = em_rxeof(adapter, count); 1560 1561 EM_TX_LOCK(adapter); 1562 em_txeof(adapter); 1563 #if __FreeBSD_version >= 800000 1564 if (!drbr_empty(ifp, adapter->br)) 1565 em_mq_start_locked(ifp, NULL); 1566 #else 1567 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1568 em_start_locked(ifp); 1569 #endif 1570 EM_TX_UNLOCK(adapter); 1571 return (rx_done); 1572 } 1573 #endif /* DEVICE_POLLING */ 1574 1575 #ifdef EM_LEGACY_IRQ 1576 /********************************************************************* 1577 * 1578 * Legacy Interrupt Service routine 1579 * 1580 *********************************************************************/ 1581 1582 static void 1583 em_intr(void *arg) 1584 { 1585 struct adapter *adapter = arg; 1586 struct ifnet *ifp = adapter->ifp; 1587 u32 reg_icr; 1588 1589 1590 if (ifp->if_capenable & IFCAP_POLLING) 1591 return; 1592 1593 EM_CORE_LOCK(adapter); 1594 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1595 if (reg_icr & E1000_ICR_RXO) 1596 adapter->rx_overruns++; 1597 if ((reg_icr == 0xffffffff) || (reg_icr == 0)|| 1598 (adapter->hw.mac.type >= e1000_82571 && 1599 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)) 1600 goto out; 1601 1602 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1603 goto out; 1604 1605 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1606 callout_stop(&adapter->timer); 1607 adapter->hw.mac.get_link_status = 1; 1608 em_update_link_status(adapter); 1609 /* Deal with TX cruft when link lost */ 1610 em_tx_purge(adapter); 1611 callout_reset(&adapter->timer, hz, 1612 em_local_timer, adapter); 1613 goto out; 1614 } 1615 1616 EM_TX_LOCK(adapter); 1617 em_txeof(adapter); 1618 em_rxeof(adapter, -1); 1619 em_txeof(adapter); 1620 if (ifp->if_drv_flags & IFF_DRV_RUNNING && 1621 !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1622 em_start_locked(ifp); 1623 EM_TX_UNLOCK(adapter); 1624 1625 out: 1626 EM_CORE_UNLOCK(adapter); 1627 return; 1628 } 1629 1630 #else /* EM_FAST_IRQ, then fast interrupt routines only */ 1631 1632 static void 1633 em_handle_link(void *context, int pending) 1634 { 1635 struct adapter *adapter = context; 1636 struct ifnet *ifp = adapter->ifp; 1637 1638 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1639 return; 1640 1641 EM_CORE_LOCK(adapter); 1642 callout_stop(&adapter->timer); 1643 em_update_link_status(adapter); 1644 /* Deal with TX cruft when link lost */ 1645 em_tx_purge(adapter); 1646 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1647 EM_CORE_UNLOCK(adapter); 1648 } 1649 1650 1651 /* Combined RX/TX handler, used by Legacy and MSI */ 1652 static void 1653 em_handle_rxtx(void *context, int pending) 1654 { 1655 struct adapter *adapter = context; 1656 struct ifnet *ifp = adapter->ifp; 1657 1658 1659 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1660 if (em_rxeof(adapter, adapter->rx_process_limit) != 0) 1661 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1662 EM_TX_LOCK(adapter); 1663 em_txeof(adapter); 1664 1665 #if __FreeBSD_version >= 800000 1666 if (!drbr_empty(ifp, adapter->br)) 1667 em_mq_start_locked(ifp, NULL); 1668 #else 1669 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1670 em_start_locked(ifp); 1671 #endif 1672 EM_TX_UNLOCK(adapter); 1673 } 1674 1675 em_enable_intr(adapter); 1676 } 1677 1678 /********************************************************************* 1679 * 1680 * Fast Legacy/MSI Combined Interrupt Service routine 1681 * 1682 *********************************************************************/ 1683 #if __FreeBSD_version < 700000 1684 #define FILTER_STRAY 1685 #define FILTER_HANDLED 1686 static void 1687 #else 1688 static int 1689 #endif 1690 em_irq_fast(void *arg) 1691 { 1692 struct adapter *adapter = arg; 1693 struct ifnet *ifp; 1694 u32 reg_icr; 1695 1696 ifp = adapter->ifp; 1697 1698 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1699 1700 /* Hot eject? */ 1701 if (reg_icr == 0xffffffff) 1702 return FILTER_STRAY; 1703 1704 /* Definitely not our interrupt. */ 1705 if (reg_icr == 0x0) 1706 return FILTER_STRAY; 1707 1708 /* 1709 * Starting with the 82571 chip, bit 31 should be used to 1710 * determine whether the interrupt belongs to us. 1711 */ 1712 if (adapter->hw.mac.type >= e1000_82571 && 1713 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1714 return FILTER_STRAY; 1715 1716 /* 1717 * Mask interrupts until the taskqueue is finished running. This is 1718 * cheap, just assume that it is needed. This also works around the 1719 * MSI message reordering errata on certain systems. 1720 */ 1721 em_disable_intr(adapter); 1722 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1723 1724 /* Link status change */ 1725 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1726 adapter->hw.mac.get_link_status = 1; 1727 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1728 } 1729 1730 if (reg_icr & E1000_ICR_RXO) 1731 adapter->rx_overruns++; 1732 return FILTER_HANDLED; 1733 } 1734 1735 /********************************************************************* 1736 * 1737 * MSIX Interrupt Service Routines 1738 * 1739 **********************************************************************/ 1740 #define EM_MSIX_TX 0x00040000 1741 #define EM_MSIX_RX 0x00010000 1742 #define EM_MSIX_LINK 0x00100000 1743 1744 static void 1745 em_msix_tx(void *arg) 1746 { 1747 struct adapter *adapter = arg; 1748 struct ifnet *ifp = adapter->ifp; 1749 1750 ++adapter->tx_irq; 1751 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1752 EM_TX_LOCK(adapter); 1753 em_txeof(adapter); 1754 EM_TX_UNLOCK(adapter); 1755 taskqueue_enqueue(adapter->tq, &adapter->tx_task); 1756 } 1757 /* Reenable this interrupt */ 1758 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX); 1759 return; 1760 } 1761 1762 /********************************************************************* 1763 * 1764 * MSIX RX Interrupt Service routine 1765 * 1766 **********************************************************************/ 1767 1768 static void 1769 em_msix_rx(void *arg) 1770 { 1771 struct adapter *adapter = arg; 1772 struct ifnet *ifp = adapter->ifp; 1773 1774 ++adapter->rx_irq; 1775 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1776 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1777 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1778 /* Reenable this interrupt */ 1779 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX); 1780 return; 1781 } 1782 1783 /********************************************************************* 1784 * 1785 * MSIX Link Fast Interrupt Service routine 1786 * 1787 **********************************************************************/ 1788 1789 static void 1790 em_msix_link(void *arg) 1791 { 1792 struct adapter *adapter = arg; 1793 u32 reg_icr; 1794 1795 ++adapter->link_irq; 1796 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1797 1798 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1799 adapter->hw.mac.get_link_status = 1; 1800 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1801 } 1802 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1803 EM_MSIX_LINK | E1000_IMS_LSC); 1804 return; 1805 } 1806 1807 static void 1808 em_handle_rx(void *context, int pending) 1809 { 1810 struct adapter *adapter = context; 1811 struct ifnet *ifp = adapter->ifp; 1812 1813 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1814 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1815 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1816 1817 } 1818 1819 static void 1820 em_handle_tx(void *context, int pending) 1821 { 1822 struct adapter *adapter = context; 1823 struct ifnet *ifp = adapter->ifp; 1824 1825 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1826 if (!EM_TX_TRYLOCK(adapter)) 1827 return; 1828 em_txeof(adapter); 1829 #if __FreeBSD_version >= 800000 1830 if (!drbr_empty(ifp, adapter->br)) 1831 em_mq_start_locked(ifp, NULL); 1832 #else 1833 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1834 em_start_locked(ifp); 1835 #endif 1836 EM_TX_UNLOCK(adapter); 1837 } 1838 } 1839 #endif /* EM_FAST_IRQ */ 1840 1841 /********************************************************************* 1842 * 1843 * Media Ioctl callback 1844 * 1845 * This routine is called whenever the user queries the status of 1846 * the interface using ifconfig. 1847 * 1848 **********************************************************************/ 1849 static void 1850 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1851 { 1852 struct adapter *adapter = ifp->if_softc; 1853 u_char fiber_type = IFM_1000_SX; 1854 1855 INIT_DEBUGOUT("em_media_status: begin"); 1856 1857 EM_CORE_LOCK(adapter); 1858 em_update_link_status(adapter); 1859 1860 ifmr->ifm_status = IFM_AVALID; 1861 ifmr->ifm_active = IFM_ETHER; 1862 1863 if (!adapter->link_active) { 1864 EM_CORE_UNLOCK(adapter); 1865 return; 1866 } 1867 1868 ifmr->ifm_status |= IFM_ACTIVE; 1869 1870 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1871 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1872 if (adapter->hw.mac.type == e1000_82545) 1873 fiber_type = IFM_1000_LX; 1874 ifmr->ifm_active |= fiber_type | IFM_FDX; 1875 } else { 1876 switch (adapter->link_speed) { 1877 case 10: 1878 ifmr->ifm_active |= IFM_10_T; 1879 break; 1880 case 100: 1881 ifmr->ifm_active |= IFM_100_TX; 1882 break; 1883 case 1000: 1884 ifmr->ifm_active |= IFM_1000_T; 1885 break; 1886 } 1887 if (adapter->link_duplex == FULL_DUPLEX) 1888 ifmr->ifm_active |= IFM_FDX; 1889 else 1890 ifmr->ifm_active |= IFM_HDX; 1891 } 1892 EM_CORE_UNLOCK(adapter); 1893 } 1894 1895 /********************************************************************* 1896 * 1897 * Media Ioctl callback 1898 * 1899 * This routine is called when the user changes speed/duplex using 1900 * media/mediopt option with ifconfig. 1901 * 1902 **********************************************************************/ 1903 static int 1904 em_media_change(struct ifnet *ifp) 1905 { 1906 struct adapter *adapter = ifp->if_softc; 1907 struct ifmedia *ifm = &adapter->media; 1908 1909 INIT_DEBUGOUT("em_media_change: begin"); 1910 1911 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1912 return (EINVAL); 1913 1914 EM_CORE_LOCK(adapter); 1915 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1916 case IFM_AUTO: 1917 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1918 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1919 break; 1920 case IFM_1000_LX: 1921 case IFM_1000_SX: 1922 case IFM_1000_T: 1923 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1924 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1925 break; 1926 case IFM_100_TX: 1927 adapter->hw.mac.autoneg = FALSE; 1928 adapter->hw.phy.autoneg_advertised = 0; 1929 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1930 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1931 else 1932 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1933 break; 1934 case IFM_10_T: 1935 adapter->hw.mac.autoneg = FALSE; 1936 adapter->hw.phy.autoneg_advertised = 0; 1937 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1938 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1939 else 1940 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1941 break; 1942 default: 1943 device_printf(adapter->dev, "Unsupported media type\n"); 1944 } 1945 1946 /* As the speed/duplex settings my have changed we need to 1947 * reset the PHY. 1948 */ 1949 adapter->hw.phy.reset_disable = FALSE; 1950 1951 em_init_locked(adapter); 1952 EM_CORE_UNLOCK(adapter); 1953 1954 return (0); 1955 } 1956 1957 /********************************************************************* 1958 * 1959 * This routine maps the mbufs to tx descriptors. 1960 * 1961 * return 0 on success, positive on failure 1962 **********************************************************************/ 1963 1964 static int 1965 em_xmit(struct adapter *adapter, struct mbuf **m_headp) 1966 { 1967 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1968 bus_dmamap_t map; 1969 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1970 struct e1000_tx_desc *ctxd = NULL; 1971 struct mbuf *m_head; 1972 u32 txd_upper, txd_lower, txd_used, txd_saved; 1973 int nsegs, i, j, first, last = 0; 1974 int error, do_tso, tso_desc = 0; 1975 #if __FreeBSD_version < 700000 1976 struct m_tag *mtag; 1977 #endif 1978 m_head = *m_headp; 1979 txd_upper = txd_lower = txd_used = txd_saved = 0; 1980 1981 #if __FreeBSD_version >= 700000 1982 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1983 #else 1984 do_tso = 0; 1985 #endif 1986 1987 /* 1988 * Force a cleanup if number of TX descriptors 1989 * available hits the threshold 1990 */ 1991 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { 1992 em_txeof(adapter); 1993 /* Now do we at least have a minimal? */ 1994 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { 1995 adapter->no_tx_desc_avail1++; 1996 return (ENOBUFS); 1997 } 1998 } 1999 2000 2001 /* 2002 * TSO workaround: 2003 * If an mbuf is only header we need 2004 * to pull 4 bytes of data into it. 2005 */ 2006 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 2007 m_head = m_pullup(m_head, M_TSO_LEN + 4); 2008 *m_headp = m_head; 2009 if (m_head == NULL) 2010 return (ENOBUFS); 2011 } 2012 2013 /* 2014 * Map the packet for DMA 2015 * 2016 * Capture the first descriptor index, 2017 * this descriptor will have the index 2018 * of the EOP which is the only one that 2019 * now gets a DONE bit writeback. 2020 */ 2021 first = adapter->next_avail_tx_desc; 2022 tx_buffer = &adapter->tx_buffer_area[first]; 2023 tx_buffer_mapped = tx_buffer; 2024 map = tx_buffer->map; 2025 2026 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2027 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2028 2029 /* 2030 * There are two types of errors we can (try) to handle: 2031 * - EFBIG means the mbuf chain was too long and bus_dma ran 2032 * out of segments. Defragment the mbuf chain and try again. 2033 * - ENOMEM means bus_dma could not obtain enough bounce buffers 2034 * at this point in time. Defer sending and try again later. 2035 * All other errors, in particular EINVAL, are fatal and prevent the 2036 * mbuf chain from ever going through. Drop it and report error. 2037 */ 2038 if (error == EFBIG) { 2039 struct mbuf *m; 2040 2041 m = m_defrag(*m_headp, M_DONTWAIT); 2042 if (m == NULL) { 2043 adapter->mbuf_alloc_failed++; 2044 m_freem(*m_headp); 2045 *m_headp = NULL; 2046 return (ENOBUFS); 2047 } 2048 *m_headp = m; 2049 2050 /* Try it again */ 2051 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2052 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2053 2054 if (error) { 2055 adapter->no_tx_dma_setup++; 2056 m_freem(*m_headp); 2057 *m_headp = NULL; 2058 return (error); 2059 } 2060 } else if (error != 0) { 2061 adapter->no_tx_dma_setup++; 2062 return (error); 2063 } 2064 2065 /* 2066 * TSO Hardware workaround, if this packet is not 2067 * TSO, and is only a single descriptor long, and 2068 * it follows a TSO burst, then we need to add a 2069 * sentinel descriptor to prevent premature writeback. 2070 */ 2071 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { 2072 if (nsegs == 1) 2073 tso_desc = TRUE; 2074 adapter->tx_tso = FALSE; 2075 } 2076 2077 if (nsegs > (adapter->num_tx_desc_avail - 2)) { 2078 adapter->no_tx_desc_avail2++; 2079 bus_dmamap_unload(adapter->txtag, map); 2080 return (ENOBUFS); 2081 } 2082 m_head = *m_headp; 2083 2084 /* Do hardware assists */ 2085 #if __FreeBSD_version >= 700000 2086 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 2087 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 2088 if (error != TRUE) 2089 return (ENXIO); /* something foobar */ 2090 /* we need to make a final sentinel transmit desc */ 2091 tso_desc = TRUE; 2092 } else 2093 #endif 2094 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 2095 em_transmit_checksum_setup(adapter, m_head, 2096 &txd_upper, &txd_lower); 2097 2098 i = adapter->next_avail_tx_desc; 2099 if (adapter->pcix_82544) 2100 txd_saved = i; 2101 2102 /* Set up our transmit descriptors */ 2103 for (j = 0; j < nsegs; j++) { 2104 bus_size_t seg_len; 2105 bus_addr_t seg_addr; 2106 /* If adapter is 82544 and on PCIX bus */ 2107 if(adapter->pcix_82544) { 2108 DESC_ARRAY desc_array; 2109 u32 array_elements, counter; 2110 /* 2111 * Check the Address and Length combination and 2112 * split the data accordingly 2113 */ 2114 array_elements = em_fill_descriptors(segs[j].ds_addr, 2115 segs[j].ds_len, &desc_array); 2116 for (counter = 0; counter < array_elements; counter++) { 2117 if (txd_used == adapter->num_tx_desc_avail) { 2118 adapter->next_avail_tx_desc = txd_saved; 2119 adapter->no_tx_desc_avail2++; 2120 bus_dmamap_unload(adapter->txtag, map); 2121 return (ENOBUFS); 2122 } 2123 tx_buffer = &adapter->tx_buffer_area[i]; 2124 ctxd = &adapter->tx_desc_base[i]; 2125 ctxd->buffer_addr = htole64( 2126 desc_array.descriptor[counter].address); 2127 ctxd->lower.data = htole32( 2128 (adapter->txd_cmd | txd_lower | (u16) 2129 desc_array.descriptor[counter].length)); 2130 ctxd->upper.data = 2131 htole32((txd_upper)); 2132 last = i; 2133 if (++i == adapter->num_tx_desc) 2134 i = 0; 2135 tx_buffer->m_head = NULL; 2136 tx_buffer->next_eop = -1; 2137 txd_used++; 2138 } 2139 } else { 2140 tx_buffer = &adapter->tx_buffer_area[i]; 2141 ctxd = &adapter->tx_desc_base[i]; 2142 seg_addr = segs[j].ds_addr; 2143 seg_len = segs[j].ds_len; 2144 /* 2145 ** TSO Workaround: 2146 ** If this is the last descriptor, we want to 2147 ** split it so we have a small final sentinel 2148 */ 2149 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 2150 seg_len -= 4; 2151 ctxd->buffer_addr = htole64(seg_addr); 2152 ctxd->lower.data = htole32( 2153 adapter->txd_cmd | txd_lower | seg_len); 2154 ctxd->upper.data = 2155 htole32(txd_upper); 2156 if (++i == adapter->num_tx_desc) 2157 i = 0; 2158 /* Now make the sentinel */ 2159 ++txd_used; /* using an extra txd */ 2160 ctxd = &adapter->tx_desc_base[i]; 2161 tx_buffer = &adapter->tx_buffer_area[i]; 2162 ctxd->buffer_addr = 2163 htole64(seg_addr + seg_len); 2164 ctxd->lower.data = htole32( 2165 adapter->txd_cmd | txd_lower | 4); 2166 ctxd->upper.data = 2167 htole32(txd_upper); 2168 last = i; 2169 if (++i == adapter->num_tx_desc) 2170 i = 0; 2171 } else { 2172 ctxd->buffer_addr = htole64(seg_addr); 2173 ctxd->lower.data = htole32( 2174 adapter->txd_cmd | txd_lower | seg_len); 2175 ctxd->upper.data = 2176 htole32(txd_upper); 2177 last = i; 2178 if (++i == adapter->num_tx_desc) 2179 i = 0; 2180 } 2181 tx_buffer->m_head = NULL; 2182 tx_buffer->next_eop = -1; 2183 } 2184 } 2185 2186 adapter->next_avail_tx_desc = i; 2187 if (adapter->pcix_82544) 2188 adapter->num_tx_desc_avail -= txd_used; 2189 else { 2190 adapter->num_tx_desc_avail -= nsegs; 2191 if (tso_desc) /* TSO used an extra for sentinel */ 2192 adapter->num_tx_desc_avail -= txd_used; 2193 } 2194 2195 /* 2196 ** Handle VLAN tag, this is the 2197 ** biggest difference between 2198 ** 6.x and 7 2199 */ 2200 #if __FreeBSD_version < 700000 2201 /* Find out if we are in vlan mode. */ 2202 mtag = VLAN_OUTPUT_TAG(ifp, m_head); 2203 if (mtag != NULL) { 2204 ctxd->upper.fields.special = 2205 htole16(VLAN_TAG_VALUE(mtag)); 2206 #else /* FreeBSD 7 */ 2207 if (m_head->m_flags & M_VLANTAG) { 2208 /* Set the vlan id. */ 2209 ctxd->upper.fields.special = 2210 htole16(m_head->m_pkthdr.ether_vtag); 2211 #endif 2212 /* Tell hardware to add tag */ 2213 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2214 } 2215 2216 tx_buffer->m_head = m_head; 2217 tx_buffer_mapped->map = tx_buffer->map; 2218 tx_buffer->map = map; 2219 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 2220 2221 /* 2222 * Last Descriptor of Packet 2223 * needs End Of Packet (EOP) 2224 * and Report Status (RS) 2225 */ 2226 ctxd->lower.data |= 2227 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2228 /* 2229 * Keep track in the first buffer which 2230 * descriptor will be written back 2231 */ 2232 tx_buffer = &adapter->tx_buffer_area[first]; 2233 tx_buffer->next_eop = last; 2234 adapter->watchdog_time = ticks; 2235 2236 /* 2237 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2238 * that this frame is available to transmit. 2239 */ 2240 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 2241 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2242 if (adapter->hw.mac.type == e1000_82547 && 2243 adapter->link_duplex == HALF_DUPLEX) 2244 em_82547_move_tail(adapter); 2245 else { 2246 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 2247 if (adapter->hw.mac.type == e1000_82547) 2248 em_82547_update_fifo_head(adapter, 2249 m_head->m_pkthdr.len); 2250 } 2251 2252 return (0); 2253 } 2254 2255 /********************************************************************* 2256 * 2257 * 82547 workaround to avoid controller hang in half-duplex environment. 2258 * The workaround is to avoid queuing a large packet that would span 2259 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 2260 * in this case. We do that only when FIFO is quiescent. 2261 * 2262 **********************************************************************/ 2263 static void 2264 em_82547_move_tail(void *arg) 2265 { 2266 struct adapter *adapter = arg; 2267 struct e1000_tx_desc *tx_desc; 2268 u16 hw_tdt, sw_tdt, length = 0; 2269 bool eop = 0; 2270 2271 EM_TX_LOCK_ASSERT(adapter); 2272 2273 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 2274 sw_tdt = adapter->next_avail_tx_desc; 2275 2276 while (hw_tdt != sw_tdt) { 2277 tx_desc = &adapter->tx_desc_base[hw_tdt]; 2278 length += tx_desc->lower.flags.length; 2279 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 2280 if (++hw_tdt == adapter->num_tx_desc) 2281 hw_tdt = 0; 2282 2283 if (eop) { 2284 if (em_82547_fifo_workaround(adapter, length)) { 2285 adapter->tx_fifo_wrk_cnt++; 2286 callout_reset(&adapter->tx_fifo_timer, 1, 2287 em_82547_move_tail, adapter); 2288 break; 2289 } 2290 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 2291 em_82547_update_fifo_head(adapter, length); 2292 length = 0; 2293 } 2294 } 2295 } 2296 2297 static int 2298 em_82547_fifo_workaround(struct adapter *adapter, int len) 2299 { 2300 int fifo_space, fifo_pkt_len; 2301 2302 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2303 2304 if (adapter->link_duplex == HALF_DUPLEX) { 2305 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 2306 2307 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 2308 if (em_82547_tx_fifo_reset(adapter)) 2309 return (0); 2310 else 2311 return (1); 2312 } 2313 } 2314 2315 return (0); 2316 } 2317 2318 static void 2319 em_82547_update_fifo_head(struct adapter *adapter, int len) 2320 { 2321 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2322 2323 /* tx_fifo_head is always 16 byte aligned */ 2324 adapter->tx_fifo_head += fifo_pkt_len; 2325 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { 2326 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2327 } 2328 } 2329 2330 2331 static int 2332 em_82547_tx_fifo_reset(struct adapter *adapter) 2333 { 2334 u32 tctl; 2335 2336 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2337 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2338 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2339 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2340 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2341 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2342 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2343 /* Disable TX unit */ 2344 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2345 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2346 tctl & ~E1000_TCTL_EN); 2347 2348 /* Reset FIFO pointers */ 2349 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2350 adapter->tx_head_addr); 2351 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2352 adapter->tx_head_addr); 2353 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2354 adapter->tx_head_addr); 2355 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2356 adapter->tx_head_addr); 2357 2358 /* Re-enable TX unit */ 2359 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2360 E1000_WRITE_FLUSH(&adapter->hw); 2361 2362 adapter->tx_fifo_head = 0; 2363 adapter->tx_fifo_reset_cnt++; 2364 2365 return (TRUE); 2366 } 2367 else { 2368 return (FALSE); 2369 } 2370 } 2371 2372 static void 2373 em_set_promisc(struct adapter *adapter) 2374 { 2375 struct ifnet *ifp = adapter->ifp; 2376 u32 reg_rctl; 2377 2378 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2379 2380 if (ifp->if_flags & IFF_PROMISC) { 2381 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2382 /* Turn this on if you want to see bad packets */ 2383 if (em_debug_sbp) 2384 reg_rctl |= E1000_RCTL_SBP; 2385 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2386 } else if (ifp->if_flags & IFF_ALLMULTI) { 2387 reg_rctl |= E1000_RCTL_MPE; 2388 reg_rctl &= ~E1000_RCTL_UPE; 2389 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2390 } 2391 } 2392 2393 static void 2394 em_disable_promisc(struct adapter *adapter) 2395 { 2396 u32 reg_rctl; 2397 2398 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2399 2400 reg_rctl &= (~E1000_RCTL_UPE); 2401 reg_rctl &= (~E1000_RCTL_MPE); 2402 reg_rctl &= (~E1000_RCTL_SBP); 2403 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2404 } 2405 2406 2407 /********************************************************************* 2408 * Multicast Update 2409 * 2410 * This routine is called whenever multicast address list is updated. 2411 * 2412 **********************************************************************/ 2413 2414 static void 2415 em_set_multi(struct adapter *adapter) 2416 { 2417 struct ifnet *ifp = adapter->ifp; 2418 struct ifmultiaddr *ifma; 2419 u32 reg_rctl = 0; 2420 u8 *mta; /* Multicast array memory */ 2421 int mcnt = 0; 2422 2423 IOCTL_DEBUGOUT("em_set_multi: begin"); 2424 2425 if (adapter->hw.mac.type == e1000_82542 && 2426 adapter->hw.revision_id == E1000_REVISION_2) { 2427 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2428 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2429 e1000_pci_clear_mwi(&adapter->hw); 2430 reg_rctl |= E1000_RCTL_RST; 2431 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2432 msec_delay(5); 2433 } 2434 2435 /* Allocate temporary memory to setup array */ 2436 mta = malloc(sizeof(u8) * 2437 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 2438 M_DEVBUF, M_NOWAIT | M_ZERO); 2439 if (mta == NULL) 2440 panic("em_set_multi memory failure\n"); 2441 2442 #if __FreeBSD_version < 800000 2443 IF_ADDR_LOCK(ifp); 2444 #else 2445 if_maddr_rlock(ifp); 2446 #endif 2447 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2448 if (ifma->ifma_addr->sa_family != AF_LINK) 2449 continue; 2450 2451 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2452 break; 2453 2454 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2455 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2456 mcnt++; 2457 } 2458 #if __FreeBSD_version < 800000 2459 IF_ADDR_UNLOCK(ifp); 2460 #else 2461 if_maddr_runlock(ifp); 2462 #endif 2463 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2464 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2465 reg_rctl |= E1000_RCTL_MPE; 2466 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2467 } else 2468 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2469 2470 if (adapter->hw.mac.type == e1000_82542 && 2471 adapter->hw.revision_id == E1000_REVISION_2) { 2472 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2473 reg_rctl &= ~E1000_RCTL_RST; 2474 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2475 msec_delay(5); 2476 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2477 e1000_pci_set_mwi(&adapter->hw); 2478 } 2479 free(mta, M_DEVBUF); 2480 } 2481 2482 2483 /********************************************************************* 2484 * Timer routine 2485 * 2486 * This routine checks for link status and updates statistics. 2487 * 2488 **********************************************************************/ 2489 2490 static void 2491 em_local_timer(void *arg) 2492 { 2493 struct adapter *adapter = arg; 2494 struct ifnet *ifp = adapter->ifp; 2495 2496 EM_CORE_LOCK_ASSERT(adapter); 2497 2498 #ifndef DEVICE_POLLING 2499 taskqueue_enqueue(adapter->tq, 2500 &adapter->rxtx_task); 2501 #endif 2502 em_update_link_status(adapter); 2503 em_update_stats_counters(adapter); 2504 2505 /* Reset LAA into RAR[0] on 82571 */ 2506 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2507 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2508 2509 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2510 em_print_hw_stats(adapter); 2511 2512 em_smartspeed(adapter); 2513 2514 /* 2515 * We check the watchdog: the time since 2516 * the last TX descriptor was cleaned. 2517 * This implies a functional TX engine. 2518 */ 2519 if ((adapter->watchdog_check == TRUE) && 2520 (ticks - adapter->watchdog_time > EM_WATCHDOG)) 2521 goto hung; 2522 2523 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2524 return; 2525 hung: 2526 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2527 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2528 adapter->watchdog_events++; 2529 em_init_locked(adapter); 2530 } 2531 2532 static void 2533 em_update_link_status(struct adapter *adapter) 2534 { 2535 struct e1000_hw *hw = &adapter->hw; 2536 struct ifnet *ifp = adapter->ifp; 2537 device_t dev = adapter->dev; 2538 u32 link_check = 0; 2539 2540 /* Get the cached link value or read phy for real */ 2541 switch (hw->phy.media_type) { 2542 case e1000_media_type_copper: 2543 if (hw->mac.get_link_status) { 2544 /* Do the work to read phy */ 2545 e1000_check_for_link(hw); 2546 link_check = !hw->mac.get_link_status; 2547 if (link_check) /* ESB2 fix */ 2548 e1000_cfg_on_link_up(hw); 2549 } else 2550 link_check = TRUE; 2551 break; 2552 case e1000_media_type_fiber: 2553 e1000_check_for_link(hw); 2554 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2555 E1000_STATUS_LU); 2556 break; 2557 case e1000_media_type_internal_serdes: 2558 e1000_check_for_link(hw); 2559 link_check = adapter->hw.mac.serdes_has_link; 2560 break; 2561 default: 2562 case e1000_media_type_unknown: 2563 break; 2564 } 2565 2566 /* Now check for a transition */ 2567 if (link_check && (adapter->link_active == 0)) { 2568 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2569 &adapter->link_duplex); 2570 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2571 if ((adapter->link_speed != SPEED_1000) && 2572 ((hw->mac.type == e1000_82571) || 2573 (hw->mac.type == e1000_82572))) { 2574 int tarc0; 2575 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2576 tarc0 &= ~SPEED_MODE_BIT; 2577 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2578 } 2579 if (bootverbose) 2580 device_printf(dev, "Link is up %d Mbps %s\n", 2581 adapter->link_speed, 2582 ((adapter->link_duplex == FULL_DUPLEX) ? 2583 "Full Duplex" : "Half Duplex")); 2584 adapter->link_active = 1; 2585 adapter->smartspeed = 0; 2586 ifp->if_baudrate = adapter->link_speed * 1000000; 2587 if_link_state_change(ifp, LINK_STATE_UP); 2588 } else if (!link_check && (adapter->link_active == 1)) { 2589 ifp->if_baudrate = adapter->link_speed = 0; 2590 adapter->link_duplex = 0; 2591 if (bootverbose) 2592 device_printf(dev, "Link is Down\n"); 2593 adapter->link_active = 0; 2594 /* Link down, disable watchdog */ 2595 adapter->watchdog_check = FALSE; 2596 if_link_state_change(ifp, LINK_STATE_DOWN); 2597 } 2598 } 2599 2600 /********************************************************************* 2601 * 2602 * This routine disables all traffic on the adapter by issuing a 2603 * global reset on the MAC and deallocates TX/RX buffers. 2604 * 2605 * This routine should always be called with BOTH the CORE 2606 * and TX locks. 2607 **********************************************************************/ 2608 2609 static void 2610 em_stop(void *arg) 2611 { 2612 struct adapter *adapter = arg; 2613 struct ifnet *ifp = adapter->ifp; 2614 2615 EM_CORE_LOCK_ASSERT(adapter); 2616 EM_TX_LOCK_ASSERT(adapter); 2617 2618 INIT_DEBUGOUT("em_stop: begin"); 2619 2620 em_disable_intr(adapter); 2621 callout_stop(&adapter->timer); 2622 callout_stop(&adapter->tx_fifo_timer); 2623 2624 /* Tell the stack that the interface is no longer active */ 2625 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2626 2627 e1000_reset_hw(&adapter->hw); 2628 if (adapter->hw.mac.type >= e1000_82544) 2629 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2630 } 2631 2632 2633 /********************************************************************* 2634 * 2635 * Determine hardware revision. 2636 * 2637 **********************************************************************/ 2638 static void 2639 em_identify_hardware(struct adapter *adapter) 2640 { 2641 device_t dev = adapter->dev; 2642 2643 /* Make sure our PCI config space has the necessary stuff set */ 2644 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2645 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2646 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2647 device_printf(dev, "Memory Access and/or Bus Master bits " 2648 "were not set!\n"); 2649 adapter->hw.bus.pci_cmd_word |= 2650 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2651 pci_write_config(dev, PCIR_COMMAND, 2652 adapter->hw.bus.pci_cmd_word, 2); 2653 } 2654 2655 /* Save off the information about this board */ 2656 adapter->hw.vendor_id = pci_get_vendor(dev); 2657 adapter->hw.device_id = pci_get_device(dev); 2658 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2659 adapter->hw.subsystem_vendor_id = 2660 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2661 adapter->hw.subsystem_device_id = 2662 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2663 2664 /* Do Shared Code Init and Setup */ 2665 if (e1000_set_mac_type(&adapter->hw)) { 2666 device_printf(dev, "Setup init failure\n"); 2667 return; 2668 } 2669 } 2670 2671 static int 2672 em_allocate_pci_resources(struct adapter *adapter) 2673 { 2674 device_t dev = adapter->dev; 2675 int val, rid, error = E1000_SUCCESS; 2676 2677 rid = PCIR_BAR(0); 2678 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2679 &rid, RF_ACTIVE); 2680 if (adapter->memory == NULL) { 2681 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2682 return (ENXIO); 2683 } 2684 adapter->osdep.mem_bus_space_tag = 2685 rman_get_bustag(adapter->memory); 2686 adapter->osdep.mem_bus_space_handle = 2687 rman_get_bushandle(adapter->memory); 2688 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2689 2690 /* Only older adapters use IO mapping */ 2691 if ((adapter->hw.mac.type > e1000_82543) && 2692 (adapter->hw.mac.type < e1000_82571)) { 2693 /* Figure our where our IO BAR is ? */ 2694 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2695 val = pci_read_config(dev, rid, 4); 2696 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2697 adapter->io_rid = rid; 2698 break; 2699 } 2700 rid += 4; 2701 /* check for 64bit BAR */ 2702 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2703 rid += 4; 2704 } 2705 if (rid >= PCIR_CIS) { 2706 device_printf(dev, "Unable to locate IO BAR\n"); 2707 return (ENXIO); 2708 } 2709 adapter->ioport = bus_alloc_resource_any(dev, 2710 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); 2711 if (adapter->ioport == NULL) { 2712 device_printf(dev, "Unable to allocate bus resource: " 2713 "ioport\n"); 2714 return (ENXIO); 2715 } 2716 adapter->hw.io_base = 0; 2717 adapter->osdep.io_bus_space_tag = 2718 rman_get_bustag(adapter->ioport); 2719 adapter->osdep.io_bus_space_handle = 2720 rman_get_bushandle(adapter->ioport); 2721 } 2722 2723 /* 2724 ** Init the resource arrays 2725 ** used by MSIX setup 2726 */ 2727 for (int i = 0; i < 3; i++) { 2728 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2729 adapter->tag[i] = NULL; 2730 adapter->res[i] = NULL; 2731 } 2732 2733 /* 2734 * Setup MSI/X or MSI if PCI Express 2735 */ 2736 if (em_enable_msi) 2737 adapter->msi = em_setup_msix(adapter); 2738 2739 adapter->hw.back = &adapter->osdep; 2740 2741 return (error); 2742 } 2743 2744 /********************************************************************* 2745 * 2746 * Setup the Legacy or MSI Interrupt handler 2747 * 2748 **********************************************************************/ 2749 int 2750 em_allocate_legacy(struct adapter *adapter) 2751 { 2752 device_t dev = adapter->dev; 2753 int error; 2754 2755 /* Manually turn off all interrupts */ 2756 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2757 2758 /* Legacy RID is 0 */ 2759 if (adapter->msi == 0) 2760 adapter->rid[0] = 0; 2761 2762 /* We allocate a single interrupt resource */ 2763 adapter->res[0] = bus_alloc_resource_any(dev, 2764 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2765 if (adapter->res[0] == NULL) { 2766 device_printf(dev, "Unable to allocate bus resource: " 2767 "interrupt\n"); 2768 return (ENXIO); 2769 } 2770 2771 #ifdef EM_LEGACY_IRQ 2772 /* We do Legacy setup */ 2773 if ((error = bus_setup_intr(dev, adapter->res[0], 2774 #if __FreeBSD_version > 700000 2775 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, 2776 #else /* 6.X */ 2777 INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, 2778 #endif 2779 &adapter->tag[0])) != 0) { 2780 device_printf(dev, "Failed to register interrupt handler"); 2781 return (error); 2782 } 2783 2784 #else /* FAST_IRQ */ 2785 /* 2786 * Try allocating a fast interrupt and the associated deferred 2787 * processing contexts. 2788 */ 2789 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); 2790 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2791 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2792 taskqueue_thread_enqueue, &adapter->tq); 2793 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2794 device_get_nameunit(adapter->dev)); 2795 #if __FreeBSD_version < 700000 2796 if ((error = bus_setup_intr(dev, adapter->res[0], 2797 INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, 2798 #else 2799 if ((error = bus_setup_intr(dev, adapter->res[0], 2800 INTR_TYPE_NET, em_irq_fast, NULL, adapter, 2801 #endif 2802 &adapter->tag[0])) != 0) { 2803 device_printf(dev, "Failed to register fast interrupt " 2804 "handler: %d\n", error); 2805 taskqueue_free(adapter->tq); 2806 adapter->tq = NULL; 2807 return (error); 2808 } 2809 #endif /* EM_LEGACY_IRQ */ 2810 2811 return (0); 2812 } 2813 2814 /********************************************************************* 2815 * 2816 * Setup the MSIX Interrupt handlers 2817 * This is not really Multiqueue, rather 2818 * its just multiple interrupt vectors. 2819 * 2820 **********************************************************************/ 2821 int 2822 em_allocate_msix(struct adapter *adapter) 2823 { 2824 device_t dev = adapter->dev; 2825 int error; 2826 2827 /* Make sure all interrupts are disabled */ 2828 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2829 2830 /* First get the resources */ 2831 for (int i = 0; i < adapter->msi; i++) { 2832 adapter->res[i] = bus_alloc_resource_any(dev, 2833 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); 2834 if (adapter->res[i] == NULL) { 2835 device_printf(dev, 2836 "Unable to allocate bus resource: " 2837 "MSIX Interrupt\n"); 2838 return (ENXIO); 2839 } 2840 } 2841 2842 /* 2843 * Now allocate deferred processing contexts. 2844 */ 2845 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); 2846 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); 2847 /* 2848 * Handle compatibility for msi case for deferral due to 2849 * trylock failure 2850 */ 2851 TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter); 2852 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2853 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2854 taskqueue_thread_enqueue, &adapter->tq); 2855 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2856 device_get_nameunit(adapter->dev)); 2857 2858 /* 2859 * And setup the interrupt handlers 2860 */ 2861 2862 /* First slot to RX */ 2863 if ((error = bus_setup_intr(dev, adapter->res[0], 2864 #if __FreeBSD_version > 700000 2865 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter, 2866 #else /* 6.X */ 2867 INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter, 2868 #endif 2869 &adapter->tag[0])) != 0) { 2870 device_printf(dev, "Failed to register RX handler"); 2871 return (error); 2872 } 2873 2874 /* Next TX */ 2875 if ((error = bus_setup_intr(dev, adapter->res[1], 2876 #if __FreeBSD_version > 700000 2877 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter, 2878 #else /* 6.X */ 2879 INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter, 2880 #endif 2881 &adapter->tag[1])) != 0) { 2882 device_printf(dev, "Failed to register TX handler"); 2883 return (error); 2884 } 2885 2886 /* And Link */ 2887 if ((error = bus_setup_intr(dev, adapter->res[2], 2888 #if __FreeBSD_version > 700000 2889 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter, 2890 #else /* 6.X */ 2891 INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter, 2892 #endif 2893 &adapter->tag[2])) != 0) { 2894 device_printf(dev, "Failed to register TX handler"); 2895 return (error); 2896 } 2897 2898 return (0); 2899 } 2900 2901 2902 static void 2903 em_free_pci_resources(struct adapter *adapter) 2904 { 2905 device_t dev = adapter->dev; 2906 2907 /* Make sure the for loop below runs once */ 2908 if (adapter->msi == 0) 2909 adapter->msi = 1; 2910 2911 /* 2912 * First release all the interrupt resources: 2913 * notice that since these are just kept 2914 * in an array we can do the same logic 2915 * whether its MSIX or just legacy. 2916 */ 2917 for (int i = 0; i < adapter->msi; i++) { 2918 if (adapter->tag[i] != NULL) { 2919 bus_teardown_intr(dev, adapter->res[i], 2920 adapter->tag[i]); 2921 adapter->tag[i] = NULL; 2922 } 2923 if (adapter->res[i] != NULL) { 2924 bus_release_resource(dev, SYS_RES_IRQ, 2925 adapter->rid[i], adapter->res[i]); 2926 } 2927 } 2928 2929 if (adapter->msi) 2930 pci_release_msi(dev); 2931 2932 if (adapter->msix != NULL) 2933 bus_release_resource(dev, SYS_RES_MEMORY, 2934 PCIR_BAR(EM_MSIX_BAR), adapter->msix); 2935 2936 if (adapter->memory != NULL) 2937 bus_release_resource(dev, SYS_RES_MEMORY, 2938 PCIR_BAR(0), adapter->memory); 2939 2940 if (adapter->flash != NULL) 2941 bus_release_resource(dev, SYS_RES_MEMORY, 2942 EM_FLASH, adapter->flash); 2943 2944 if (adapter->ioport != NULL) 2945 bus_release_resource(dev, SYS_RES_IOPORT, 2946 adapter->io_rid, adapter->ioport); 2947 } 2948 2949 /* 2950 * Setup MSI or MSI/X 2951 */ 2952 static int 2953 em_setup_msix(struct adapter *adapter) 2954 { 2955 device_t dev = adapter->dev; 2956 int val = 0; 2957 2958 if (adapter->hw.mac.type < e1000_82571) 2959 return (0); 2960 2961 /* Setup MSI/X for Hartwell */ 2962 if (adapter->hw.mac.type == e1000_82574) { 2963 /* Map the MSIX BAR */ 2964 int rid = PCIR_BAR(EM_MSIX_BAR); 2965 adapter->msix = bus_alloc_resource_any(dev, 2966 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2967 if (!adapter->msix) { 2968 /* May not be enabled */ 2969 device_printf(adapter->dev, 2970 "Unable to map MSIX table \n"); 2971 goto msi; 2972 } 2973 val = pci_msix_count(dev); 2974 /* 2975 ** 82574 can be configured for 5 but 2976 ** we limit use to 3. 2977 */ 2978 if (val > 3) val = 3; 2979 if ((val) && pci_alloc_msix(dev, &val) == 0) { 2980 device_printf(adapter->dev,"Using MSIX interrupts\n"); 2981 return (val); 2982 } 2983 } 2984 msi: 2985 val = pci_msi_count(dev); 2986 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2987 adapter->msi = 1; 2988 device_printf(adapter->dev,"Using MSI interrupt\n"); 2989 return (val); 2990 } 2991 return (0); 2992 } 2993 2994 /********************************************************************* 2995 * 2996 * Initialize the hardware to a configuration 2997 * as specified by the adapter structure. 2998 * 2999 **********************************************************************/ 3000 static int 3001 em_hardware_init(struct adapter *adapter) 3002 { 3003 device_t dev = adapter->dev; 3004 u16 rx_buffer_size; 3005 3006 INIT_DEBUGOUT("em_hardware_init: begin"); 3007 3008 /* Issue a global reset */ 3009 e1000_reset_hw(&adapter->hw); 3010 3011 /* When hardware is reset, fifo_head is also reset */ 3012 adapter->tx_fifo_head = 0; 3013 3014 /* Set up smart power down as default off on newer adapters. */ 3015 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || 3016 adapter->hw.mac.type == e1000_82572)) { 3017 u16 phy_tmp = 0; 3018 3019 /* Speed up time to link by disabling smart power down. */ 3020 e1000_read_phy_reg(&adapter->hw, 3021 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 3022 phy_tmp &= ~IGP02E1000_PM_SPD; 3023 e1000_write_phy_reg(&adapter->hw, 3024 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 3025 } 3026 3027 /* 3028 * These parameters control the automatic generation (Tx) and 3029 * response (Rx) to Ethernet PAUSE frames. 3030 * - High water mark should allow for at least two frames to be 3031 * received after sending an XOFF. 3032 * - Low water mark works best when it is very near the high water mark. 3033 * This allows the receiver to restart by sending XON when it has 3034 * drained a bit. Here we use an arbitary value of 1500 which will 3035 * restart after one full frame is pulled from the buffer. There 3036 * could be several smaller frames in the buffer and if so they will 3037 * not trigger the XON until their total number reduces the buffer 3038 * by 1500. 3039 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 3040 */ 3041 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 3042 0xffff) << 10 ); 3043 3044 adapter->hw.fc.high_water = rx_buffer_size - 3045 roundup2(adapter->max_frame_size, 1024); 3046 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 3047 3048 if (adapter->hw.mac.type == e1000_80003es2lan) 3049 adapter->hw.fc.pause_time = 0xFFFF; 3050 else 3051 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 3052 adapter->hw.fc.send_xon = TRUE; 3053 3054 /* Set Flow control, use the tunable location if sane */ 3055 if ((em_fc_setting >= 0) || (em_fc_setting < 4)) 3056 adapter->hw.fc.requested_mode = em_fc_setting; 3057 else 3058 adapter->hw.fc.requested_mode = e1000_fc_none; 3059 3060 /* Override - workaround for PCHLAN issue */ 3061 if (adapter->hw.mac.type == e1000_pchlan) 3062 adapter->hw.fc.requested_mode = e1000_fc_rx_pause; 3063 3064 if (e1000_init_hw(&adapter->hw) < 0) { 3065 device_printf(dev, "Hardware Initialization Failed\n"); 3066 return (EIO); 3067 } 3068 3069 e1000_check_for_link(&adapter->hw); 3070 3071 return (0); 3072 } 3073 3074 /********************************************************************* 3075 * 3076 * Setup networking device structure and register an interface. 3077 * 3078 **********************************************************************/ 3079 static void 3080 em_setup_interface(device_t dev, struct adapter *adapter) 3081 { 3082 struct ifnet *ifp; 3083 3084 INIT_DEBUGOUT("em_setup_interface: begin"); 3085 3086 ifp = adapter->ifp = if_alloc(IFT_ETHER); 3087 if (ifp == NULL) 3088 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 3089 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 3090 ifp->if_mtu = ETHERMTU; 3091 ifp->if_init = em_init; 3092 ifp->if_softc = adapter; 3093 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 3094 ifp->if_ioctl = em_ioctl; 3095 ifp->if_start = em_start; 3096 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 3097 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 3098 IFQ_SET_READY(&ifp->if_snd); 3099 3100 ether_ifattach(ifp, adapter->hw.mac.addr); 3101 3102 ifp->if_capabilities = ifp->if_capenable = 0; 3103 3104 #if __FreeBSD_version >= 800000 3105 /* Multiqueue tx functions */ 3106 ifp->if_transmit = em_mq_start; 3107 ifp->if_qflush = em_qflush; 3108 adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); 3109 #endif 3110 if (adapter->hw.mac.type >= e1000_82543) { 3111 int version_cap; 3112 #if __FreeBSD_version < 700000 3113 version_cap = IFCAP_HWCSUM; 3114 #else 3115 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 3116 #endif 3117 ifp->if_capabilities |= version_cap; 3118 ifp->if_capenable |= version_cap; 3119 } 3120 3121 #if __FreeBSD_version >= 700000 3122 /* Identify TSO capable adapters */ 3123 if ((adapter->hw.mac.type > e1000_82544) && 3124 (adapter->hw.mac.type != e1000_82547)) 3125 ifp->if_capabilities |= IFCAP_TSO4; 3126 /* 3127 * By default only enable on PCI-E, this 3128 * can be overriden by ifconfig. 3129 */ 3130 if (adapter->hw.mac.type >= e1000_82571) 3131 ifp->if_capenable |= IFCAP_TSO4; 3132 #endif 3133 /* 3134 * Tell the upper layer(s) we 3135 * support full VLAN capability 3136 */ 3137 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 3138 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3139 ifp->if_capenable |= (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING); 3140 3141 /* 3142 ** Dont turn this on by default, if vlans are 3143 ** created on another pseudo device (eg. lagg) 3144 ** then vlan events are not passed thru, breaking 3145 ** operation, but with HW FILTER off it works. If 3146 ** using vlans directly on the em driver you can 3147 ** enable this and get full hardware tag filtering. 3148 */ 3149 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 3150 3151 #ifdef DEVICE_POLLING 3152 ifp->if_capabilities |= IFCAP_POLLING; 3153 #endif 3154 3155 /* Limit WOL to MAGIC, not clear others are used */ 3156 if (adapter->wol) { 3157 ifp->if_capabilities |= IFCAP_WOL_MAGIC; 3158 ifp->if_capenable |= IFCAP_WOL_MAGIC; 3159 } 3160 3161 /* 3162 * Specify the media types supported by this adapter and register 3163 * callbacks to update media and link information 3164 */ 3165 ifmedia_init(&adapter->media, IFM_IMASK, 3166 em_media_change, em_media_status); 3167 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3168 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3169 u_char fiber_type = IFM_1000_SX; /* default type */ 3170 3171 if (adapter->hw.mac.type == e1000_82545) 3172 fiber_type = IFM_1000_LX; 3173 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 3174 0, NULL); 3175 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 3176 } else { 3177 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 3178 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 3179 0, NULL); 3180 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 3181 0, NULL); 3182 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 3183 0, NULL); 3184 if (adapter->hw.phy.type != e1000_phy_ife) { 3185 ifmedia_add(&adapter->media, 3186 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 3187 ifmedia_add(&adapter->media, 3188 IFM_ETHER | IFM_1000_T, 0, NULL); 3189 } 3190 } 3191 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 3192 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 3193 } 3194 3195 3196 /********************************************************************* 3197 * 3198 * Workaround for SmartSpeed on 82541 and 82547 controllers 3199 * 3200 **********************************************************************/ 3201 static void 3202 em_smartspeed(struct adapter *adapter) 3203 { 3204 u16 phy_tmp; 3205 3206 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || 3207 adapter->hw.mac.autoneg == 0 || 3208 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3209 return; 3210 3211 if (adapter->smartspeed == 0) { 3212 /* If Master/Slave config fault is asserted twice, 3213 * we assume back-to-back */ 3214 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3215 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3216 return; 3217 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3218 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3219 e1000_read_phy_reg(&adapter->hw, 3220 PHY_1000T_CTRL, &phy_tmp); 3221 if(phy_tmp & CR_1000T_MS_ENABLE) { 3222 phy_tmp &= ~CR_1000T_MS_ENABLE; 3223 e1000_write_phy_reg(&adapter->hw, 3224 PHY_1000T_CTRL, phy_tmp); 3225 adapter->smartspeed++; 3226 if(adapter->hw.mac.autoneg && 3227 !e1000_copper_link_autoneg(&adapter->hw) && 3228 !e1000_read_phy_reg(&adapter->hw, 3229 PHY_CONTROL, &phy_tmp)) { 3230 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3231 MII_CR_RESTART_AUTO_NEG); 3232 e1000_write_phy_reg(&adapter->hw, 3233 PHY_CONTROL, phy_tmp); 3234 } 3235 } 3236 } 3237 return; 3238 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3239 /* If still no link, perhaps using 2/3 pair cable */ 3240 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 3241 phy_tmp |= CR_1000T_MS_ENABLE; 3242 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 3243 if(adapter->hw.mac.autoneg && 3244 !e1000_copper_link_autoneg(&adapter->hw) && 3245 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 3246 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3247 MII_CR_RESTART_AUTO_NEG); 3248 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 3249 } 3250 } 3251 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3252 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) 3253 adapter->smartspeed = 0; 3254 } 3255 3256 3257 /* 3258 * Manage DMA'able memory. 3259 */ 3260 static void 3261 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3262 { 3263 if (error) 3264 return; 3265 *(bus_addr_t *) arg = segs[0].ds_addr; 3266 } 3267 3268 static int 3269 em_dma_malloc(struct adapter *adapter, bus_size_t size, 3270 struct em_dma_alloc *dma, int mapflags) 3271 { 3272 int error; 3273 3274 #if __FreeBSD_version >= 700000 3275 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3276 #else 3277 error = bus_dma_tag_create(NULL, /* parent */ 3278 #endif 3279 EM_DBA_ALIGN, 0, /* alignment, bounds */ 3280 BUS_SPACE_MAXADDR, /* lowaddr */ 3281 BUS_SPACE_MAXADDR, /* highaddr */ 3282 NULL, NULL, /* filter, filterarg */ 3283 size, /* maxsize */ 3284 1, /* nsegments */ 3285 size, /* maxsegsize */ 3286 0, /* flags */ 3287 NULL, /* lockfunc */ 3288 NULL, /* lockarg */ 3289 &dma->dma_tag); 3290 if (error) { 3291 device_printf(adapter->dev, 3292 "%s: bus_dma_tag_create failed: %d\n", 3293 __func__, error); 3294 goto fail_0; 3295 } 3296 3297 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3298 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3299 if (error) { 3300 device_printf(adapter->dev, 3301 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3302 __func__, (uintmax_t)size, error); 3303 goto fail_2; 3304 } 3305 3306 dma->dma_paddr = 0; 3307 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3308 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3309 if (error || dma->dma_paddr == 0) { 3310 device_printf(adapter->dev, 3311 "%s: bus_dmamap_load failed: %d\n", 3312 __func__, error); 3313 goto fail_3; 3314 } 3315 3316 return (0); 3317 3318 fail_3: 3319 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3320 fail_2: 3321 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3322 bus_dma_tag_destroy(dma->dma_tag); 3323 fail_0: 3324 dma->dma_map = NULL; 3325 dma->dma_tag = NULL; 3326 3327 return (error); 3328 } 3329 3330 static void 3331 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 3332 { 3333 if (dma->dma_tag == NULL) 3334 return; 3335 if (dma->dma_map != NULL) { 3336 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3337 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3338 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3339 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3340 dma->dma_map = NULL; 3341 } 3342 bus_dma_tag_destroy(dma->dma_tag); 3343 dma->dma_tag = NULL; 3344 } 3345 3346 3347 /********************************************************************* 3348 * 3349 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3350 * the information needed to transmit a packet on the wire. 3351 * 3352 **********************************************************************/ 3353 static int 3354 em_allocate_transmit_structures(struct adapter *adapter) 3355 { 3356 device_t dev = adapter->dev; 3357 struct em_buffer *tx_buffer; 3358 int error; 3359 3360 /* 3361 * Create DMA tags for tx descriptors 3362 */ 3363 #if __FreeBSD_version >= 700000 3364 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3365 #else 3366 if ((error = bus_dma_tag_create(NULL, /* parent */ 3367 #endif 3368 1, 0, /* alignment, bounds */ 3369 BUS_SPACE_MAXADDR, /* lowaddr */ 3370 BUS_SPACE_MAXADDR, /* highaddr */ 3371 NULL, NULL, /* filter, filterarg */ 3372 EM_TSO_SIZE, /* maxsize */ 3373 EM_MAX_SCATTER, /* nsegments */ 3374 EM_TSO_SEG_SIZE, /* maxsegsize */ 3375 0, /* flags */ 3376 NULL, /* lockfunc */ 3377 NULL, /* lockarg */ 3378 &adapter->txtag)) != 0) { 3379 device_printf(dev, "Unable to allocate TX DMA tag\n"); 3380 goto fail; 3381 } 3382 3383 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * 3384 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3385 if (adapter->tx_buffer_area == NULL) { 3386 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3387 error = ENOMEM; 3388 goto fail; 3389 } 3390 3391 /* Create the descriptor buffer dma maps */ 3392 for (int i = 0; i < adapter->num_tx_desc; i++) { 3393 tx_buffer = &adapter->tx_buffer_area[i]; 3394 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); 3395 if (error != 0) { 3396 device_printf(dev, "Unable to create TX DMA map\n"); 3397 goto fail; 3398 } 3399 tx_buffer->next_eop = -1; 3400 } 3401 3402 return (0); 3403 fail: 3404 em_free_transmit_structures(adapter); 3405 return (error); 3406 } 3407 3408 /********************************************************************* 3409 * 3410 * (Re)Initialize transmit structures. 3411 * 3412 **********************************************************************/ 3413 static void 3414 em_setup_transmit_structures(struct adapter *adapter) 3415 { 3416 struct em_buffer *tx_buffer; 3417 3418 /* Clear the old ring contents */ 3419 bzero(adapter->tx_desc_base, 3420 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3421 3422 /* Free any existing TX buffers */ 3423 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3424 tx_buffer = &adapter->tx_buffer_area[i]; 3425 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3426 BUS_DMASYNC_POSTWRITE); 3427 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 3428 m_freem(tx_buffer->m_head); 3429 tx_buffer->m_head = NULL; 3430 tx_buffer->next_eop = -1; 3431 } 3432 3433 /* Reset state */ 3434 adapter->next_avail_tx_desc = 0; 3435 adapter->next_tx_to_clean = 0; 3436 adapter->num_tx_desc_avail = adapter->num_tx_desc; 3437 3438 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3439 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3440 3441 return; 3442 } 3443 3444 /********************************************************************* 3445 * 3446 * Enable transmit unit. 3447 * 3448 **********************************************************************/ 3449 static void 3450 em_initialize_transmit_unit(struct adapter *adapter) 3451 { 3452 u32 tctl, tarc, tipg = 0; 3453 u64 bus_addr; 3454 3455 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3456 /* Setup the Base and Length of the Tx Descriptor Ring */ 3457 bus_addr = adapter->txdma.dma_paddr; 3458 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 3459 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3460 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 3461 (u32)(bus_addr >> 32)); 3462 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 3463 (u32)bus_addr); 3464 /* Setup the HW Tx Head and Tail descriptor pointers */ 3465 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 3466 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 3467 3468 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3469 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), 3470 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); 3471 3472 /* Set the default values for the Tx Inter Packet Gap timer */ 3473 switch (adapter->hw.mac.type) { 3474 case e1000_82542: 3475 tipg = DEFAULT_82542_TIPG_IPGT; 3476 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3477 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3478 break; 3479 case e1000_80003es2lan: 3480 tipg = DEFAULT_82543_TIPG_IPGR1; 3481 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3482 E1000_TIPG_IPGR2_SHIFT; 3483 break; 3484 default: 3485 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3486 (adapter->hw.phy.media_type == 3487 e1000_media_type_internal_serdes)) 3488 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3489 else 3490 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3491 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3492 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3493 } 3494 3495 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3496 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3497 if(adapter->hw.mac.type >= e1000_82540) 3498 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3499 adapter->tx_abs_int_delay.value); 3500 3501 if ((adapter->hw.mac.type == e1000_82571) || 3502 (adapter->hw.mac.type == e1000_82572)) { 3503 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3504 tarc |= SPEED_MODE_BIT; 3505 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3506 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3507 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3508 tarc |= 1; 3509 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3510 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3511 tarc |= 1; 3512 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3513 } 3514 3515 /* Program the Transmit Control Register */ 3516 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3517 tctl &= ~E1000_TCTL_CT; 3518 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3519 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3520 3521 if (adapter->hw.mac.type >= e1000_82571) 3522 tctl |= E1000_TCTL_MULR; 3523 3524 /* This write will effectively turn on the transmit unit. */ 3525 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3526 3527 /* Setup Transmit Descriptor Base Settings */ 3528 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3529 3530 if (adapter->tx_int_delay.value > 0) 3531 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3532 } 3533 3534 /********************************************************************* 3535 * 3536 * Free all transmit related data structures. 3537 * 3538 **********************************************************************/ 3539 static void 3540 em_free_transmit_structures(struct adapter *adapter) 3541 { 3542 struct em_buffer *tx_buffer; 3543 3544 INIT_DEBUGOUT("free_transmit_structures: begin"); 3545 3546 if (adapter->tx_buffer_area != NULL) { 3547 for (int i = 0; i < adapter->num_tx_desc; i++) { 3548 tx_buffer = &adapter->tx_buffer_area[i]; 3549 if (tx_buffer->m_head != NULL) { 3550 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3551 BUS_DMASYNC_POSTWRITE); 3552 bus_dmamap_unload(adapter->txtag, 3553 tx_buffer->map); 3554 m_freem(tx_buffer->m_head); 3555 tx_buffer->m_head = NULL; 3556 } else if (tx_buffer->map != NULL) 3557 bus_dmamap_unload(adapter->txtag, 3558 tx_buffer->map); 3559 if (tx_buffer->map != NULL) { 3560 bus_dmamap_destroy(adapter->txtag, 3561 tx_buffer->map); 3562 tx_buffer->map = NULL; 3563 } 3564 } 3565 } 3566 if (adapter->tx_buffer_area != NULL) { 3567 free(adapter->tx_buffer_area, M_DEVBUF); 3568 adapter->tx_buffer_area = NULL; 3569 } 3570 if (adapter->txtag != NULL) { 3571 bus_dma_tag_destroy(adapter->txtag); 3572 adapter->txtag = NULL; 3573 } 3574 #if __FreeBSD_version >= 800000 3575 if (adapter->br != NULL) 3576 buf_ring_free(adapter->br, M_DEVBUF); 3577 #endif 3578 } 3579 3580 /********************************************************************* 3581 * 3582 * The offload context needs to be set when we transfer the first 3583 * packet of a particular protocol (TCP/UDP). This routine has been 3584 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3585 * 3586 * Added back the old method of keeping the current context type 3587 * and not setting if unnecessary, as this is reported to be a 3588 * big performance win. -jfv 3589 **********************************************************************/ 3590 static void 3591 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, 3592 u32 *txd_upper, u32 *txd_lower) 3593 { 3594 struct e1000_context_desc *TXD = NULL; 3595 struct em_buffer *tx_buffer; 3596 struct ether_vlan_header *eh; 3597 struct ip *ip = NULL; 3598 struct ip6_hdr *ip6; 3599 int curr_txd, ehdrlen; 3600 u32 cmd, hdr_len, ip_hlen; 3601 u16 etype; 3602 u8 ipproto; 3603 3604 3605 cmd = hdr_len = ipproto = 0; 3606 curr_txd = adapter->next_avail_tx_desc; 3607 3608 /* 3609 * Determine where frame payload starts. 3610 * Jump over vlan headers if already present, 3611 * helpful for QinQ too. 3612 */ 3613 eh = mtod(mp, struct ether_vlan_header *); 3614 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3615 etype = ntohs(eh->evl_proto); 3616 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3617 } else { 3618 etype = ntohs(eh->evl_encap_proto); 3619 ehdrlen = ETHER_HDR_LEN; 3620 } 3621 3622 /* 3623 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3624 * TODO: Support SCTP too when it hits the tree. 3625 */ 3626 switch (etype) { 3627 case ETHERTYPE_IP: 3628 ip = (struct ip *)(mp->m_data + ehdrlen); 3629 ip_hlen = ip->ip_hl << 2; 3630 3631 /* Setup of IP header checksum. */ 3632 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3633 /* 3634 * Start offset for header checksum calculation. 3635 * End offset for header checksum calculation. 3636 * Offset of place to put the checksum. 3637 */ 3638 TXD = (struct e1000_context_desc *) 3639 &adapter->tx_desc_base[curr_txd]; 3640 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3641 TXD->lower_setup.ip_fields.ipcse = 3642 htole16(ehdrlen + ip_hlen); 3643 TXD->lower_setup.ip_fields.ipcso = 3644 ehdrlen + offsetof(struct ip, ip_sum); 3645 cmd |= E1000_TXD_CMD_IP; 3646 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3647 } 3648 3649 if (mp->m_len < ehdrlen + ip_hlen) 3650 return; /* failure */ 3651 3652 hdr_len = ehdrlen + ip_hlen; 3653 ipproto = ip->ip_p; 3654 3655 break; 3656 case ETHERTYPE_IPV6: 3657 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3658 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3659 3660 if (mp->m_len < ehdrlen + ip_hlen) 3661 return; /* failure */ 3662 3663 /* IPv6 doesn't have a header checksum. */ 3664 3665 hdr_len = ehdrlen + ip_hlen; 3666 ipproto = ip6->ip6_nxt; 3667 3668 break; 3669 default: 3670 *txd_upper = 0; 3671 *txd_lower = 0; 3672 return; 3673 } 3674 3675 switch (ipproto) { 3676 case IPPROTO_TCP: 3677 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3678 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3679 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3680 /* no need for context if already set */ 3681 if (adapter->last_hw_offload == CSUM_TCP) 3682 return; 3683 adapter->last_hw_offload = CSUM_TCP; 3684 /* 3685 * Start offset for payload checksum calculation. 3686 * End offset for payload checksum calculation. 3687 * Offset of place to put the checksum. 3688 */ 3689 TXD = (struct e1000_context_desc *) 3690 &adapter->tx_desc_base[curr_txd]; 3691 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3692 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3693 TXD->upper_setup.tcp_fields.tucso = 3694 hdr_len + offsetof(struct tcphdr, th_sum); 3695 cmd |= E1000_TXD_CMD_TCP; 3696 } 3697 break; 3698 case IPPROTO_UDP: 3699 { 3700 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3701 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3702 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3703 /* no need for context if already set */ 3704 if (adapter->last_hw_offload == CSUM_UDP) 3705 return; 3706 adapter->last_hw_offload = CSUM_UDP; 3707 /* 3708 * Start offset for header checksum calculation. 3709 * End offset for header checksum calculation. 3710 * Offset of place to put the checksum. 3711 */ 3712 TXD = (struct e1000_context_desc *) 3713 &adapter->tx_desc_base[curr_txd]; 3714 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3715 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3716 TXD->upper_setup.tcp_fields.tucso = 3717 hdr_len + offsetof(struct udphdr, uh_sum); 3718 } 3719 /* Fall Thru */ 3720 } 3721 default: 3722 break; 3723 } 3724 3725 TXD->tcp_seg_setup.data = htole32(0); 3726 TXD->cmd_and_length = 3727 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3728 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3729 tx_buffer->m_head = NULL; 3730 tx_buffer->next_eop = -1; 3731 3732 if (++curr_txd == adapter->num_tx_desc) 3733 curr_txd = 0; 3734 3735 adapter->num_tx_desc_avail--; 3736 adapter->next_avail_tx_desc = curr_txd; 3737 } 3738 3739 3740 #if __FreeBSD_version >= 700000 3741 /********************************************************************** 3742 * 3743 * Setup work for hardware segmentation offload (TSO) 3744 * 3745 **********************************************************************/ 3746 static bool 3747 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, 3748 u32 *txd_lower) 3749 { 3750 struct e1000_context_desc *TXD; 3751 struct em_buffer *tx_buffer; 3752 struct ether_vlan_header *eh; 3753 struct ip *ip; 3754 struct ip6_hdr *ip6; 3755 struct tcphdr *th; 3756 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; 3757 u16 etype; 3758 3759 /* 3760 * This function could/should be extended to support IP/IPv6 3761 * fragmentation as well. But as they say, one step at a time. 3762 */ 3763 3764 /* 3765 * Determine where frame payload starts. 3766 * Jump over vlan headers if already present, 3767 * helpful for QinQ too. 3768 */ 3769 eh = mtod(mp, struct ether_vlan_header *); 3770 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3771 etype = ntohs(eh->evl_proto); 3772 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3773 } else { 3774 etype = ntohs(eh->evl_encap_proto); 3775 ehdrlen = ETHER_HDR_LEN; 3776 } 3777 3778 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3779 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3780 return FALSE; /* -1 */ 3781 3782 /* 3783 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3784 * TODO: Support SCTP too when it hits the tree. 3785 */ 3786 switch (etype) { 3787 case ETHERTYPE_IP: 3788 isip6 = 0; 3789 ip = (struct ip *)(mp->m_data + ehdrlen); 3790 if (ip->ip_p != IPPROTO_TCP) 3791 return FALSE; /* 0 */ 3792 ip->ip_len = 0; 3793 ip->ip_sum = 0; 3794 ip_hlen = ip->ip_hl << 2; 3795 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3796 return FALSE; /* -1 */ 3797 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3798 #if 1 3799 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3800 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3801 #else 3802 th->th_sum = mp->m_pkthdr.csum_data; 3803 #endif 3804 break; 3805 case ETHERTYPE_IPV6: 3806 isip6 = 1; 3807 return FALSE; /* Not supported yet. */ 3808 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3809 if (ip6->ip6_nxt != IPPROTO_TCP) 3810 return FALSE; /* 0 */ 3811 ip6->ip6_plen = 0; 3812 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3813 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3814 return FALSE; /* -1 */ 3815 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3816 #if 0 3817 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3818 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3819 #else 3820 th->th_sum = mp->m_pkthdr.csum_data; 3821 #endif 3822 break; 3823 default: 3824 return FALSE; 3825 } 3826 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3827 3828 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3829 E1000_TXD_DTYP_D | /* Data descr type */ 3830 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3831 3832 /* IP and/or TCP header checksum calculation and insertion. */ 3833 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3834 E1000_TXD_POPTS_TXSM) << 8; 3835 3836 curr_txd = adapter->next_avail_tx_desc; 3837 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3838 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3839 3840 /* IPv6 doesn't have a header checksum. */ 3841 if (!isip6) { 3842 /* 3843 * Start offset for header checksum calculation. 3844 * End offset for header checksum calculation. 3845 * Offset of place put the checksum. 3846 */ 3847 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3848 TXD->lower_setup.ip_fields.ipcse = 3849 htole16(ehdrlen + ip_hlen - 1); 3850 TXD->lower_setup.ip_fields.ipcso = 3851 ehdrlen + offsetof(struct ip, ip_sum); 3852 } 3853 /* 3854 * Start offset for payload checksum calculation. 3855 * End offset for payload checksum calculation. 3856 * Offset of place to put the checksum. 3857 */ 3858 TXD->upper_setup.tcp_fields.tucss = 3859 ehdrlen + ip_hlen; 3860 TXD->upper_setup.tcp_fields.tucse = 0; 3861 TXD->upper_setup.tcp_fields.tucso = 3862 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3863 /* 3864 * Payload size per packet w/o any headers. 3865 * Length of all headers up to payload. 3866 */ 3867 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3868 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3869 3870 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3871 E1000_TXD_CMD_DEXT | /* Extended descr */ 3872 E1000_TXD_CMD_TSE | /* TSE context */ 3873 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ 3874 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3875 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3876 3877 tx_buffer->m_head = NULL; 3878 tx_buffer->next_eop = -1; 3879 3880 if (++curr_txd == adapter->num_tx_desc) 3881 curr_txd = 0; 3882 3883 adapter->num_tx_desc_avail--; 3884 adapter->next_avail_tx_desc = curr_txd; 3885 adapter->tx_tso = TRUE; 3886 3887 return TRUE; 3888 } 3889 3890 #endif /* __FreeBSD_version >= 700000 */ 3891 3892 /********************************************************************** 3893 * 3894 * Examine each tx_buffer in the used queue. If the hardware is done 3895 * processing the packet then free associated resources. The 3896 * tx_buffer is put back on the free queue. 3897 * 3898 **********************************************************************/ 3899 static void 3900 em_txeof(struct adapter *adapter) 3901 { 3902 int first, last, done, num_avail; 3903 struct em_buffer *tx_buffer; 3904 struct e1000_tx_desc *tx_desc, *eop_desc; 3905 struct ifnet *ifp = adapter->ifp; 3906 3907 EM_TX_LOCK_ASSERT(adapter); 3908 3909 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3910 return; 3911 3912 num_avail = adapter->num_tx_desc_avail; 3913 first = adapter->next_tx_to_clean; 3914 tx_desc = &adapter->tx_desc_base[first]; 3915 tx_buffer = &adapter->tx_buffer_area[first]; 3916 last = tx_buffer->next_eop; 3917 eop_desc = &adapter->tx_desc_base[last]; 3918 3919 /* 3920 * What this does is get the index of the 3921 * first descriptor AFTER the EOP of the 3922 * first packet, that way we can do the 3923 * simple comparison on the inner while loop. 3924 */ 3925 if (++last == adapter->num_tx_desc) 3926 last = 0; 3927 done = last; 3928 3929 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3930 BUS_DMASYNC_POSTREAD); 3931 3932 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3933 /* We clean the range of the packet */ 3934 while (first != done) { 3935 tx_desc->upper.data = 0; 3936 tx_desc->lower.data = 0; 3937 tx_desc->buffer_addr = 0; 3938 ++num_avail; 3939 3940 if (tx_buffer->m_head) { 3941 ifp->if_opackets++; 3942 bus_dmamap_sync(adapter->txtag, 3943 tx_buffer->map, 3944 BUS_DMASYNC_POSTWRITE); 3945 bus_dmamap_unload(adapter->txtag, 3946 tx_buffer->map); 3947 3948 m_freem(tx_buffer->m_head); 3949 tx_buffer->m_head = NULL; 3950 } 3951 tx_buffer->next_eop = -1; 3952 adapter->watchdog_time = ticks; 3953 3954 if (++first == adapter->num_tx_desc) 3955 first = 0; 3956 3957 tx_buffer = &adapter->tx_buffer_area[first]; 3958 tx_desc = &adapter->tx_desc_base[first]; 3959 } 3960 /* See if we can continue to the next packet */ 3961 last = tx_buffer->next_eop; 3962 if (last != -1) { 3963 eop_desc = &adapter->tx_desc_base[last]; 3964 /* Get new done point */ 3965 if (++last == adapter->num_tx_desc) last = 0; 3966 done = last; 3967 } else 3968 break; 3969 } 3970 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3971 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3972 3973 adapter->next_tx_to_clean = first; 3974 3975 /* 3976 * If we have enough room, clear IFF_DRV_OACTIVE to 3977 * tell the stack that it is OK to send packets. 3978 * If there are no pending descriptors, clear the watchdog. 3979 */ 3980 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 3981 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3982 if (num_avail == adapter->num_tx_desc) { 3983 adapter->watchdog_check = FALSE; 3984 adapter->num_tx_desc_avail = num_avail; 3985 return; 3986 } 3987 } 3988 3989 adapter->num_tx_desc_avail = num_avail; 3990 return; 3991 } 3992 3993 /********************************************************************* 3994 * 3995 * When Link is lost sometimes there is work still in the TX ring 3996 * which may result in a watchdog, rather than allow that we do an 3997 * attempted cleanup and then reinit here. Note that this has been 3998 * seens mostly with fiber adapters. 3999 * 4000 **********************************************************************/ 4001 static void 4002 em_tx_purge(struct adapter *adapter) 4003 { 4004 if ((!adapter->link_active) && (adapter->watchdog_check)) { 4005 EM_TX_LOCK(adapter); 4006 em_txeof(adapter); 4007 EM_TX_UNLOCK(adapter); 4008 if (adapter->watchdog_check) /* Still outstanding? */ 4009 em_init_locked(adapter); 4010 } 4011 } 4012 4013 /********************************************************************* 4014 * 4015 * Get a buffer from system mbuf buffer pool. 4016 * 4017 **********************************************************************/ 4018 static int 4019 em_get_buf(struct adapter *adapter, int i) 4020 { 4021 struct mbuf *m; 4022 bus_dma_segment_t segs[1]; 4023 bus_dmamap_t map; 4024 struct em_buffer *rx_buffer; 4025 int error, nsegs; 4026 4027 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 4028 if (m == NULL) { 4029 adapter->mbuf_cluster_failed++; 4030 return (ENOBUFS); 4031 } 4032 m->m_len = m->m_pkthdr.len = MCLBYTES; 4033 4034 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 4035 m_adj(m, ETHER_ALIGN); 4036 4037 /* 4038 * Using memory from the mbuf cluster pool, invoke the 4039 * bus_dma machinery to arrange the memory mapping. 4040 */ 4041 error = bus_dmamap_load_mbuf_sg(adapter->rxtag, 4042 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); 4043 if (error != 0) { 4044 m_free(m); 4045 return (error); 4046 } 4047 4048 /* If nsegs is wrong then the stack is corrupt. */ 4049 KASSERT(nsegs == 1, ("Too many segments returned!")); 4050 4051 rx_buffer = &adapter->rx_buffer_area[i]; 4052 if (rx_buffer->m_head != NULL) 4053 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4054 4055 map = rx_buffer->map; 4056 rx_buffer->map = adapter->rx_sparemap; 4057 adapter->rx_sparemap = map; 4058 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 4059 rx_buffer->m_head = m; 4060 4061 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); 4062 return (0); 4063 } 4064 4065 /********************************************************************* 4066 * 4067 * Allocate memory for rx_buffer structures. Since we use one 4068 * rx_buffer per received packet, the maximum number of rx_buffer's 4069 * that we'll need is equal to the number of receive descriptors 4070 * that we've allocated. 4071 * 4072 **********************************************************************/ 4073 static int 4074 em_allocate_receive_structures(struct adapter *adapter) 4075 { 4076 device_t dev = adapter->dev; 4077 struct em_buffer *rx_buffer; 4078 int i, error; 4079 4080 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * 4081 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 4082 if (adapter->rx_buffer_area == NULL) { 4083 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 4084 return (ENOMEM); 4085 } 4086 4087 #if __FreeBSD_version >= 700000 4088 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4089 #else 4090 error = bus_dma_tag_create(NULL, /* parent */ 4091 #endif 4092 1, 0, /* alignment, bounds */ 4093 BUS_SPACE_MAXADDR, /* lowaddr */ 4094 BUS_SPACE_MAXADDR, /* highaddr */ 4095 NULL, NULL, /* filter, filterarg */ 4096 MCLBYTES, /* maxsize */ 4097 1, /* nsegments */ 4098 MCLBYTES, /* maxsegsize */ 4099 0, /* flags */ 4100 NULL, /* lockfunc */ 4101 NULL, /* lockarg */ 4102 &adapter->rxtag); 4103 if (error) { 4104 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 4105 __func__, error); 4106 goto fail; 4107 } 4108 4109 /* Create the spare map (used by getbuf) */ 4110 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4111 &adapter->rx_sparemap); 4112 if (error) { 4113 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4114 __func__, error); 4115 goto fail; 4116 } 4117 4118 rx_buffer = adapter->rx_buffer_area; 4119 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4120 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4121 &rx_buffer->map); 4122 if (error) { 4123 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4124 __func__, error); 4125 goto fail; 4126 } 4127 } 4128 4129 return (0); 4130 4131 fail: 4132 em_free_receive_structures(adapter); 4133 return (error); 4134 } 4135 4136 /********************************************************************* 4137 * 4138 * (Re)initialize receive structures. 4139 * 4140 **********************************************************************/ 4141 static int 4142 em_setup_receive_structures(struct adapter *adapter) 4143 { 4144 struct em_buffer *rx_buffer; 4145 int i, error; 4146 4147 /* Reset descriptor ring */ 4148 bzero(adapter->rx_desc_base, 4149 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 4150 4151 /* Free current RX buffers. */ 4152 rx_buffer = adapter->rx_buffer_area; 4153 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4154 if (rx_buffer->m_head != NULL) { 4155 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4156 BUS_DMASYNC_POSTREAD); 4157 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4158 m_freem(rx_buffer->m_head); 4159 rx_buffer->m_head = NULL; 4160 } 4161 } 4162 4163 /* Allocate new ones. */ 4164 for (i = 0; i < adapter->num_rx_desc; i++) { 4165 error = em_get_buf(adapter, i); 4166 if (error) 4167 return (error); 4168 } 4169 4170 /* Setup our descriptor pointers */ 4171 adapter->next_rx_desc_to_check = 0; 4172 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4173 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4174 4175 return (0); 4176 } 4177 4178 /********************************************************************* 4179 * 4180 * Enable receive unit. 4181 * 4182 **********************************************************************/ 4183 #define MAX_INTS_PER_SEC 8000 4184 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 4185 4186 static void 4187 em_initialize_receive_unit(struct adapter *adapter) 4188 { 4189 struct ifnet *ifp = adapter->ifp; 4190 u64 bus_addr; 4191 u32 rctl, rxcsum; 4192 4193 INIT_DEBUGOUT("em_initialize_receive_unit: begin"); 4194 4195 /* 4196 * Make sure receives are disabled while setting 4197 * up the descriptor ring 4198 */ 4199 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4200 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4201 4202 if (adapter->hw.mac.type >= e1000_82540) { 4203 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 4204 adapter->rx_abs_int_delay.value); 4205 /* 4206 * Set the interrupt throttling rate. Value is calculated 4207 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4208 */ 4209 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); 4210 } 4211 4212 /* 4213 ** When using MSIX interrupts we need to throttle 4214 ** using the EITR register (82574 only) 4215 */ 4216 if (adapter->msix) 4217 for (int i = 0; i < 4; i++) 4218 E1000_WRITE_REG(&adapter->hw, 4219 E1000_EITR_82574(i), DEFAULT_ITR); 4220 4221 /* Disable accelerated ackknowledge */ 4222 if (adapter->hw.mac.type == e1000_82574) 4223 E1000_WRITE_REG(&adapter->hw, 4224 E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4225 4226 /* Setup the Base and Length of the Rx Descriptor Ring */ 4227 bus_addr = adapter->rxdma.dma_paddr; 4228 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 4229 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4230 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 4231 (u32)(bus_addr >> 32)); 4232 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 4233 (u32)bus_addr); 4234 4235 /* Setup the Receive Control Register */ 4236 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4237 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4238 E1000_RCTL_RDMTS_HALF | 4239 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4240 4241 /* Make sure VLAN Filters are off */ 4242 rctl &= ~E1000_RCTL_VFE; 4243 4244 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 4245 rctl |= E1000_RCTL_SBP; 4246 else 4247 rctl &= ~E1000_RCTL_SBP; 4248 4249 switch (adapter->rx_buffer_len) { 4250 default: 4251 case 2048: 4252 rctl |= E1000_RCTL_SZ_2048; 4253 break; 4254 case 4096: 4255 rctl |= E1000_RCTL_SZ_4096 | 4256 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4257 break; 4258 case 8192: 4259 rctl |= E1000_RCTL_SZ_8192 | 4260 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4261 break; 4262 case 16384: 4263 rctl |= E1000_RCTL_SZ_16384 | 4264 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4265 break; 4266 } 4267 4268 if (ifp->if_mtu > ETHERMTU) 4269 rctl |= E1000_RCTL_LPE; 4270 else 4271 rctl &= ~E1000_RCTL_LPE; 4272 4273 /* Enable 82543 Receive Checksum Offload for TCP and UDP */ 4274 if ((adapter->hw.mac.type >= e1000_82543) && 4275 (ifp->if_capenable & IFCAP_RXCSUM)) { 4276 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 4277 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4278 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 4279 } 4280 4281 /* 4282 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4283 ** long latencies are observed, like Lenovo X60. This 4284 ** change eliminates the problem, but since having positive 4285 ** values in RDTR is a known source of problems on other 4286 ** platforms another solution is being sought. 4287 */ 4288 if (adapter->hw.mac.type == e1000_82573) 4289 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); 4290 4291 /* Enable Receives */ 4292 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4293 4294 /* 4295 * Setup the HW Rx Head and 4296 * Tail Descriptor Pointers 4297 */ 4298 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 4299 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 4300 4301 return; 4302 } 4303 4304 /********************************************************************* 4305 * 4306 * Free receive related data structures. 4307 * 4308 **********************************************************************/ 4309 static void 4310 em_free_receive_structures(struct adapter *adapter) 4311 { 4312 struct em_buffer *rx_buffer; 4313 int i; 4314 4315 INIT_DEBUGOUT("free_receive_structures: begin"); 4316 4317 if (adapter->rx_sparemap) { 4318 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 4319 adapter->rx_sparemap = NULL; 4320 } 4321 4322 /* Cleanup any existing buffers */ 4323 if (adapter->rx_buffer_area != NULL) { 4324 rx_buffer = adapter->rx_buffer_area; 4325 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4326 if (rx_buffer->m_head != NULL) { 4327 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4328 BUS_DMASYNC_POSTREAD); 4329 bus_dmamap_unload(adapter->rxtag, 4330 rx_buffer->map); 4331 m_freem(rx_buffer->m_head); 4332 rx_buffer->m_head = NULL; 4333 } else if (rx_buffer->map != NULL) 4334 bus_dmamap_unload(adapter->rxtag, 4335 rx_buffer->map); 4336 if (rx_buffer->map != NULL) { 4337 bus_dmamap_destroy(adapter->rxtag, 4338 rx_buffer->map); 4339 rx_buffer->map = NULL; 4340 } 4341 } 4342 } 4343 4344 if (adapter->rx_buffer_area != NULL) { 4345 free(adapter->rx_buffer_area, M_DEVBUF); 4346 adapter->rx_buffer_area = NULL; 4347 } 4348 4349 if (adapter->rxtag != NULL) { 4350 bus_dma_tag_destroy(adapter->rxtag); 4351 adapter->rxtag = NULL; 4352 } 4353 } 4354 4355 /********************************************************************* 4356 * 4357 * This routine executes in interrupt context. It replenishes 4358 * the mbufs in the descriptor and sends data which has been 4359 * dma'ed into host memory to upper layer. 4360 * 4361 * We loop at most count times if count is > 0, or until done if 4362 * count < 0. 4363 * 4364 * For polling we also now return the number of cleaned packets 4365 *********************************************************************/ 4366 static int 4367 em_rxeof(struct adapter *adapter, int count) 4368 { 4369 struct ifnet *ifp = adapter->ifp;; 4370 struct mbuf *mp; 4371 u8 status, accept_frame = 0, eop = 0; 4372 u16 len, desc_len, prev_len_adj; 4373 int i, rx_sent = 0; 4374 struct e1000_rx_desc *current_desc; 4375 4376 EM_RX_LOCK(adapter); 4377 i = adapter->next_rx_desc_to_check; 4378 current_desc = &adapter->rx_desc_base[i]; 4379 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4380 BUS_DMASYNC_POSTREAD); 4381 4382 if (!((current_desc->status) & E1000_RXD_STAT_DD)) { 4383 EM_RX_UNLOCK(adapter); 4384 return (rx_sent); 4385 } 4386 4387 while ((current_desc->status & E1000_RXD_STAT_DD) && 4388 (count != 0) && 4389 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4390 struct mbuf *m = NULL; 4391 4392 mp = adapter->rx_buffer_area[i].m_head; 4393 /* 4394 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 4395 * needs to access the last received byte in the mbuf. 4396 */ 4397 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 4398 BUS_DMASYNC_POSTREAD); 4399 4400 accept_frame = 1; 4401 prev_len_adj = 0; 4402 desc_len = le16toh(current_desc->length); 4403 status = current_desc->status; 4404 if (status & E1000_RXD_STAT_EOP) { 4405 count--; 4406 eop = 1; 4407 if (desc_len < ETHER_CRC_LEN) { 4408 len = 0; 4409 prev_len_adj = ETHER_CRC_LEN - desc_len; 4410 } else 4411 len = desc_len - ETHER_CRC_LEN; 4412 } else { 4413 eop = 0; 4414 len = desc_len; 4415 } 4416 4417 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 4418 u8 last_byte; 4419 u32 pkt_len = desc_len; 4420 4421 if (adapter->fmp != NULL) 4422 pkt_len += adapter->fmp->m_pkthdr.len; 4423 4424 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 4425 if (TBI_ACCEPT(&adapter->hw, status, 4426 current_desc->errors, pkt_len, last_byte, 4427 adapter->min_frame_size, adapter->max_frame_size)) { 4428 e1000_tbi_adjust_stats_82543(&adapter->hw, 4429 &adapter->stats, pkt_len, 4430 adapter->hw.mac.addr, 4431 adapter->max_frame_size); 4432 if (len > 0) 4433 len--; 4434 } else 4435 accept_frame = 0; 4436 } 4437 4438 if (accept_frame) { 4439 if (em_get_buf(adapter, i) != 0) { 4440 ifp->if_iqdrops++; 4441 goto discard; 4442 } 4443 4444 /* Assign correct length to the current fragment */ 4445 mp->m_len = len; 4446 4447 if (adapter->fmp == NULL) { 4448 mp->m_pkthdr.len = len; 4449 adapter->fmp = mp; /* Store the first mbuf */ 4450 adapter->lmp = mp; 4451 } else { 4452 /* Chain mbuf's together */ 4453 mp->m_flags &= ~M_PKTHDR; 4454 /* 4455 * Adjust length of previous mbuf in chain if 4456 * we received less than 4 bytes in the last 4457 * descriptor. 4458 */ 4459 if (prev_len_adj > 0) { 4460 adapter->lmp->m_len -= prev_len_adj; 4461 adapter->fmp->m_pkthdr.len -= 4462 prev_len_adj; 4463 } 4464 adapter->lmp->m_next = mp; 4465 adapter->lmp = adapter->lmp->m_next; 4466 adapter->fmp->m_pkthdr.len += len; 4467 } 4468 4469 if (eop) { 4470 adapter->fmp->m_pkthdr.rcvif = ifp; 4471 ifp->if_ipackets++; 4472 em_receive_checksum(adapter, current_desc, 4473 adapter->fmp); 4474 #ifndef __NO_STRICT_ALIGNMENT 4475 if (adapter->max_frame_size > 4476 (MCLBYTES - ETHER_ALIGN) && 4477 em_fixup_rx(adapter) != 0) 4478 goto skip; 4479 #endif 4480 if (status & E1000_RXD_STAT_VP) { 4481 #if __FreeBSD_version < 700000 4482 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, 4483 (le16toh(current_desc->special) & 4484 E1000_RXD_SPC_VLAN_MASK)); 4485 #else 4486 adapter->fmp->m_pkthdr.ether_vtag = 4487 (le16toh(current_desc->special) & 4488 E1000_RXD_SPC_VLAN_MASK); 4489 adapter->fmp->m_flags |= M_VLANTAG; 4490 #endif 4491 } 4492 #ifndef __NO_STRICT_ALIGNMENT 4493 skip: 4494 #endif 4495 m = adapter->fmp; 4496 adapter->fmp = NULL; 4497 adapter->lmp = NULL; 4498 } 4499 } else { 4500 ifp->if_ierrors++; 4501 discard: 4502 /* Reuse loaded DMA map and just update mbuf chain */ 4503 mp = adapter->rx_buffer_area[i].m_head; 4504 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4505 mp->m_data = mp->m_ext.ext_buf; 4506 mp->m_next = NULL; 4507 if (adapter->max_frame_size <= 4508 (MCLBYTES - ETHER_ALIGN)) 4509 m_adj(mp, ETHER_ALIGN); 4510 if (adapter->fmp != NULL) { 4511 m_freem(adapter->fmp); 4512 adapter->fmp = NULL; 4513 adapter->lmp = NULL; 4514 } 4515 m = NULL; 4516 } 4517 4518 /* Zero out the receive descriptors status. */ 4519 current_desc->status = 0; 4520 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4521 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4522 4523 /* Advance our pointers to the next descriptor. */ 4524 if (++i == adapter->num_rx_desc) 4525 i = 0; 4526 /* Call into the stack */ 4527 if (m != NULL) { 4528 adapter->next_rx_desc_to_check = i; 4529 EM_RX_UNLOCK(adapter); 4530 (*ifp->if_input)(ifp, m); 4531 EM_RX_LOCK(adapter); 4532 rx_sent++; 4533 i = adapter->next_rx_desc_to_check; 4534 } 4535 current_desc = &adapter->rx_desc_base[i]; 4536 } 4537 adapter->next_rx_desc_to_check = i; 4538 4539 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4540 if (--i < 0) 4541 i = adapter->num_rx_desc - 1; 4542 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 4543 EM_RX_UNLOCK(adapter); 4544 return (rx_sent); 4545 } 4546 4547 #ifndef __NO_STRICT_ALIGNMENT 4548 /* 4549 * When jumbo frames are enabled we should realign entire payload on 4550 * architecures with strict alignment. This is serious design mistake of 8254x 4551 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4552 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4553 * payload. On architecures without strict alignment restrictions 8254x still 4554 * performs unaligned memory access which would reduce the performance too. 4555 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4556 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4557 * existing mbuf chain. 4558 * 4559 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4560 * not used at all on architectures with strict alignment. 4561 */ 4562 static int 4563 em_fixup_rx(struct adapter *adapter) 4564 { 4565 struct mbuf *m, *n; 4566 int error; 4567 4568 error = 0; 4569 m = adapter->fmp; 4570 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4571 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4572 m->m_data += ETHER_HDR_LEN; 4573 } else { 4574 MGETHDR(n, M_DONTWAIT, MT_DATA); 4575 if (n != NULL) { 4576 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4577 m->m_data += ETHER_HDR_LEN; 4578 m->m_len -= ETHER_HDR_LEN; 4579 n->m_len = ETHER_HDR_LEN; 4580 M_MOVE_PKTHDR(n, m); 4581 n->m_next = m; 4582 adapter->fmp = n; 4583 } else { 4584 adapter->dropped_pkts++; 4585 m_freem(adapter->fmp); 4586 adapter->fmp = NULL; 4587 error = ENOMEM; 4588 } 4589 } 4590 4591 return (error); 4592 } 4593 #endif 4594 4595 /********************************************************************* 4596 * 4597 * Verify that the hardware indicated that the checksum is valid. 4598 * Inform the stack about the status of checksum so that stack 4599 * doesn't spend time verifying the checksum. 4600 * 4601 *********************************************************************/ 4602 static void 4603 em_receive_checksum(struct adapter *adapter, 4604 struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4605 { 4606 /* 82543 or newer only */ 4607 if ((adapter->hw.mac.type < e1000_82543) || 4608 /* Ignore Checksum bit is set */ 4609 (rx_desc->status & E1000_RXD_STAT_IXSM)) { 4610 mp->m_pkthdr.csum_flags = 0; 4611 return; 4612 } 4613 4614 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4615 /* Did it pass? */ 4616 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4617 /* IP Checksum Good */ 4618 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4619 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4620 4621 } else { 4622 mp->m_pkthdr.csum_flags = 0; 4623 } 4624 } 4625 4626 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4627 /* Did it pass? */ 4628 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4629 mp->m_pkthdr.csum_flags |= 4630 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4631 mp->m_pkthdr.csum_data = htons(0xffff); 4632 } 4633 } 4634 } 4635 4636 #if __FreeBSD_version >= 700029 4637 /* 4638 * This routine is run via an vlan 4639 * config EVENT 4640 */ 4641 static void 4642 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4643 { 4644 struct adapter *adapter = ifp->if_softc; 4645 u32 index, bit; 4646 4647 if (ifp->if_softc != arg) /* Not our event */ 4648 return; 4649 4650 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ 4651 return; 4652 4653 index = (vtag >> 5) & 0x7F; 4654 bit = vtag & 0x1F; 4655 em_shadow_vfta[index] |= (1 << bit); 4656 ++adapter->num_vlans; 4657 /* Re-init to load the changes */ 4658 em_init(adapter); 4659 } 4660 4661 /* 4662 * This routine is run via an vlan 4663 * unconfig EVENT 4664 */ 4665 static void 4666 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4667 { 4668 struct adapter *adapter = ifp->if_softc; 4669 u32 index, bit; 4670 4671 if (ifp->if_softc != arg) 4672 return; 4673 4674 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4675 return; 4676 4677 index = (vtag >> 5) & 0x7F; 4678 bit = vtag & 0x1F; 4679 em_shadow_vfta[index] &= ~(1 << bit); 4680 --adapter->num_vlans; 4681 /* Re-init to load the changes */ 4682 em_init(adapter); 4683 } 4684 4685 static void 4686 em_setup_vlan_hw_support(struct adapter *adapter) 4687 { 4688 struct e1000_hw *hw = &adapter->hw; 4689 u32 reg; 4690 4691 /* 4692 ** We get here thru init_locked, meaning 4693 ** a soft reset, this has already cleared 4694 ** the VFTA and other state, so if there 4695 ** have been no vlan's registered do nothing. 4696 */ 4697 if (adapter->num_vlans == 0) 4698 return; 4699 4700 /* 4701 ** A soft reset zero's out the VFTA, so 4702 ** we need to repopulate it now. 4703 */ 4704 for (int i = 0; i < EM_VFTA_SIZE; i++) 4705 if (em_shadow_vfta[i] != 0) 4706 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4707 i, em_shadow_vfta[i]); 4708 4709 reg = E1000_READ_REG(hw, E1000_CTRL); 4710 reg |= E1000_CTRL_VME; 4711 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4712 4713 /* Enable the Filter Table */ 4714 reg = E1000_READ_REG(hw, E1000_RCTL); 4715 reg &= ~E1000_RCTL_CFIEN; 4716 reg |= E1000_RCTL_VFE; 4717 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4718 4719 /* Update the frame size */ 4720 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4721 adapter->max_frame_size + VLAN_TAG_SIZE); 4722 } 4723 #endif 4724 4725 static void 4726 em_enable_intr(struct adapter *adapter) 4727 { 4728 struct e1000_hw *hw = &adapter->hw; 4729 u32 ims_mask = IMS_ENABLE_MASK; 4730 4731 if (adapter->msix) { 4732 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4733 ims_mask |= EM_MSIX_MASK; 4734 } 4735 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4736 } 4737 4738 static void 4739 em_disable_intr(struct adapter *adapter) 4740 { 4741 struct e1000_hw *hw = &adapter->hw; 4742 4743 if (adapter->msix) 4744 E1000_WRITE_REG(hw, EM_EIAC, 0); 4745 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4746 } 4747 4748 /* 4749 * Bit of a misnomer, what this really means is 4750 * to enable OS management of the system... aka 4751 * to disable special hardware management features 4752 */ 4753 static void 4754 em_init_manageability(struct adapter *adapter) 4755 { 4756 /* A shared code workaround */ 4757 #define E1000_82542_MANC2H E1000_MANC2H 4758 if (adapter->has_manage) { 4759 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4760 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4761 4762 /* disable hardware interception of ARP */ 4763 manc &= ~(E1000_MANC_ARP_EN); 4764 4765 /* enable receiving management packets to the host */ 4766 if (adapter->hw.mac.type >= e1000_82571) { 4767 manc |= E1000_MANC_EN_MNG2HOST; 4768 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4769 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4770 manc2h |= E1000_MNG2HOST_PORT_623; 4771 manc2h |= E1000_MNG2HOST_PORT_664; 4772 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4773 } 4774 4775 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4776 } 4777 } 4778 4779 /* 4780 * Give control back to hardware management 4781 * controller if there is one. 4782 */ 4783 static void 4784 em_release_manageability(struct adapter *adapter) 4785 { 4786 if (adapter->has_manage) { 4787 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4788 4789 /* re-enable hardware interception of ARP */ 4790 manc |= E1000_MANC_ARP_EN; 4791 4792 if (adapter->hw.mac.type >= e1000_82571) 4793 manc &= ~E1000_MANC_EN_MNG2HOST; 4794 4795 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4796 } 4797 } 4798 4799 /* 4800 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 4801 * For ASF and Pass Through versions of f/w this means 4802 * that the driver is loaded. For AMT version type f/w 4803 * this means that the network i/f is open. 4804 */ 4805 static void 4806 em_get_hw_control(struct adapter *adapter) 4807 { 4808 u32 ctrl_ext, swsm; 4809 4810 if (adapter->hw.mac.type == e1000_82573) { 4811 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4812 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4813 swsm | E1000_SWSM_DRV_LOAD); 4814 return; 4815 } 4816 /* else */ 4817 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4818 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4819 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4820 return; 4821 } 4822 4823 /* 4824 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4825 * For ASF and Pass Through versions of f/w this means that 4826 * the driver is no longer loaded. For AMT versions of the 4827 * f/w this means that the network i/f is closed. 4828 */ 4829 static void 4830 em_release_hw_control(struct adapter *adapter) 4831 { 4832 u32 ctrl_ext, swsm; 4833 4834 if (!adapter->has_manage) 4835 return; 4836 4837 if (adapter->hw.mac.type == e1000_82573) { 4838 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4839 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4840 swsm & ~E1000_SWSM_DRV_LOAD); 4841 return; 4842 } 4843 /* else */ 4844 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4845 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4846 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4847 return; 4848 } 4849 4850 static int 4851 em_is_valid_ether_addr(u8 *addr) 4852 { 4853 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4854 4855 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4856 return (FALSE); 4857 } 4858 4859 return (TRUE); 4860 } 4861 4862 /* 4863 ** Parse the interface capabilities with regard 4864 ** to both system management and wake-on-lan for 4865 ** later use. 4866 */ 4867 static void 4868 em_get_wakeup(device_t dev) 4869 { 4870 struct adapter *adapter = device_get_softc(dev); 4871 u16 eeprom_data = 0, device_id, apme_mask; 4872 4873 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 4874 apme_mask = EM_EEPROM_APME; 4875 4876 switch (adapter->hw.mac.type) { 4877 case e1000_82542: 4878 case e1000_82543: 4879 break; 4880 case e1000_82544: 4881 e1000_read_nvm(&adapter->hw, 4882 NVM_INIT_CONTROL2_REG, 1, &eeprom_data); 4883 apme_mask = EM_82544_APME; 4884 break; 4885 case e1000_82573: 4886 case e1000_82583: 4887 adapter->has_amt = TRUE; 4888 /* Falls thru */ 4889 case e1000_82546: 4890 case e1000_82546_rev_3: 4891 case e1000_82571: 4892 case e1000_82572: 4893 case e1000_80003es2lan: 4894 if (adapter->hw.bus.func == 1) { 4895 e1000_read_nvm(&adapter->hw, 4896 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 4897 break; 4898 } else 4899 e1000_read_nvm(&adapter->hw, 4900 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4901 break; 4902 case e1000_ich8lan: 4903 case e1000_ich9lan: 4904 case e1000_ich10lan: 4905 case e1000_pchlan: 4906 apme_mask = E1000_WUC_APME; 4907 adapter->has_amt = TRUE; 4908 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 4909 break; 4910 default: 4911 e1000_read_nvm(&adapter->hw, 4912 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4913 break; 4914 } 4915 if (eeprom_data & apme_mask) 4916 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 4917 /* 4918 * We have the eeprom settings, now apply the special cases 4919 * where the eeprom may be wrong or the board won't support 4920 * wake on lan on a particular port 4921 */ 4922 device_id = pci_get_device(dev); 4923 switch (device_id) { 4924 case E1000_DEV_ID_82546GB_PCIE: 4925 adapter->wol = 0; 4926 break; 4927 case E1000_DEV_ID_82546EB_FIBER: 4928 case E1000_DEV_ID_82546GB_FIBER: 4929 case E1000_DEV_ID_82571EB_FIBER: 4930 /* Wake events only supported on port A for dual fiber 4931 * regardless of eeprom setting */ 4932 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 4933 E1000_STATUS_FUNC_1) 4934 adapter->wol = 0; 4935 break; 4936 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 4937 case E1000_DEV_ID_82571EB_QUAD_COPPER: 4938 case E1000_DEV_ID_82571EB_QUAD_FIBER: 4939 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 4940 /* if quad port adapter, disable WoL on all but port A */ 4941 if (global_quad_port_a != 0) 4942 adapter->wol = 0; 4943 /* Reset for multiple quad port adapters */ 4944 if (++global_quad_port_a == 4) 4945 global_quad_port_a = 0; 4946 break; 4947 } 4948 return; 4949 } 4950 4951 4952 /* 4953 * Enable PCI Wake On Lan capability 4954 */ 4955 void 4956 em_enable_wakeup(device_t dev) 4957 { 4958 struct adapter *adapter = device_get_softc(dev); 4959 struct ifnet *ifp = adapter->ifp; 4960 u32 pmc, ctrl, ctrl_ext, rctl; 4961 u16 status; 4962 4963 if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) 4964 return; 4965 4966 /* Advertise the wakeup capability */ 4967 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4968 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 4969 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4970 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4971 4972 /* ICH workaround code */ 4973 if ((adapter->hw.mac.type == e1000_ich8lan) || 4974 (adapter->hw.mac.type == e1000_pchlan) || 4975 (adapter->hw.mac.type == e1000_ich9lan) || 4976 (adapter->hw.mac.type == e1000_ich10lan)) { 4977 e1000_disable_gig_wol_ich8lan(&adapter->hw); 4978 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); 4979 } 4980 4981 /* Keep the laser running on Fiber adapters */ 4982 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 4983 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 4984 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4985 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 4986 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); 4987 } 4988 4989 /* 4990 ** Determine type of Wakeup: note that wol 4991 ** is set with all bits on by default. 4992 */ 4993 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) 4994 adapter->wol &= ~E1000_WUFC_MAG; 4995 4996 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) 4997 adapter->wol &= ~E1000_WUFC_MC; 4998 else { 4999 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 5000 rctl |= E1000_RCTL_MPE; 5001 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 5002 } 5003 5004 if (adapter->hw.mac.type == e1000_pchlan) { 5005 if (em_enable_phy_wakeup(adapter)) 5006 return; 5007 } else { 5008 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 5009 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 5010 } 5011 5012 if (adapter->hw.phy.type == e1000_phy_igp_3) 5013 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 5014 5015 /* Request PME */ 5016 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); 5017 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 5018 if (ifp->if_capenable & IFCAP_WOL) 5019 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 5020 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); 5021 5022 return; 5023 } 5024 5025 /* 5026 ** WOL in the newer chipset interfaces (pchlan) 5027 ** require thing to be copied into the phy 5028 */ 5029 static int 5030 em_enable_phy_wakeup(struct adapter *adapter) 5031 { 5032 struct e1000_hw *hw = &adapter->hw; 5033 u32 mreg, ret = 0; 5034 u16 preg; 5035 5036 /* copy MAC RARs to PHY RARs */ 5037 for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { 5038 mreg = E1000_READ_REG(hw, E1000_RAL(i)); 5039 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); 5040 e1000_write_phy_reg(hw, BM_RAR_M(i), 5041 (u16)((mreg >> 16) & 0xFFFF)); 5042 mreg = E1000_READ_REG(hw, E1000_RAH(i)); 5043 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); 5044 e1000_write_phy_reg(hw, BM_RAR_CTRL(i), 5045 (u16)((mreg >> 16) & 0xFFFF)); 5046 } 5047 5048 /* copy MAC MTA to PHY MTA */ 5049 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 5050 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 5051 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 5052 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 5053 (u16)((mreg >> 16) & 0xFFFF)); 5054 } 5055 5056 /* configure PHY Rx Control register */ 5057 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); 5058 mreg = E1000_READ_REG(hw, E1000_RCTL); 5059 if (mreg & E1000_RCTL_UPE) 5060 preg |= BM_RCTL_UPE; 5061 if (mreg & E1000_RCTL_MPE) 5062 preg |= BM_RCTL_MPE; 5063 preg &= ~(BM_RCTL_MO_MASK); 5064 if (mreg & E1000_RCTL_MO_3) 5065 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 5066 << BM_RCTL_MO_SHIFT); 5067 if (mreg & E1000_RCTL_BAM) 5068 preg |= BM_RCTL_BAM; 5069 if (mreg & E1000_RCTL_PMCF) 5070 preg |= BM_RCTL_PMCF; 5071 mreg = E1000_READ_REG(hw, E1000_CTRL); 5072 if (mreg & E1000_CTRL_RFCE) 5073 preg |= BM_RCTL_RFCE; 5074 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); 5075 5076 /* enable PHY wakeup in MAC register */ 5077 E1000_WRITE_REG(hw, E1000_WUC, 5078 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); 5079 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); 5080 5081 /* configure and enable PHY wakeup in PHY registers */ 5082 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); 5083 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); 5084 5085 /* activate PHY wakeup */ 5086 ret = hw->phy.ops.acquire(hw); 5087 if (ret) { 5088 printf("Could not acquire PHY\n"); 5089 return ret; 5090 } 5091 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 5092 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 5093 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 5094 if (ret) { 5095 printf("Could not read PHY page 769\n"); 5096 goto out; 5097 } 5098 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 5099 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 5100 if (ret) 5101 printf("Could not set PHY Host Wakeup bit\n"); 5102 out: 5103 hw->phy.ops.release(hw); 5104 5105 return ret; 5106 } 5107 5108 5109 /********************************************************************* 5110 * 82544 Coexistence issue workaround. 5111 * There are 2 issues. 5112 * 1. Transmit Hang issue. 5113 * To detect this issue, following equation can be used... 5114 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5115 * If SUM[3:0] is in between 1 to 4, we will have this issue. 5116 * 5117 * 2. DAC issue. 5118 * To detect this issue, following equation can be used... 5119 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5120 * If SUM[3:0] is in between 9 to c, we will have this issue. 5121 * 5122 * 5123 * WORKAROUND: 5124 * Make sure we do not have ending address 5125 * as 1,2,3,4(Hang) or 9,a,b,c (DAC) 5126 * 5127 *************************************************************************/ 5128 static u32 5129 em_fill_descriptors (bus_addr_t address, u32 length, 5130 PDESC_ARRAY desc_array) 5131 { 5132 u32 safe_terminator; 5133 5134 /* Since issue is sensitive to length and address.*/ 5135 /* Let us first check the address...*/ 5136 if (length <= 4) { 5137 desc_array->descriptor[0].address = address; 5138 desc_array->descriptor[0].length = length; 5139 desc_array->elements = 1; 5140 return (desc_array->elements); 5141 } 5142 safe_terminator = (u32)((((u32)address & 0x7) + 5143 (length & 0xF)) & 0xF); 5144 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 5145 if (safe_terminator == 0 || 5146 (safe_terminator > 4 && 5147 safe_terminator < 9) || 5148 (safe_terminator > 0xC && 5149 safe_terminator <= 0xF)) { 5150 desc_array->descriptor[0].address = address; 5151 desc_array->descriptor[0].length = length; 5152 desc_array->elements = 1; 5153 return (desc_array->elements); 5154 } 5155 5156 desc_array->descriptor[0].address = address; 5157 desc_array->descriptor[0].length = length - 4; 5158 desc_array->descriptor[1].address = address + (length - 4); 5159 desc_array->descriptor[1].length = 4; 5160 desc_array->elements = 2; 5161 return (desc_array->elements); 5162 } 5163 5164 /********************************************************************** 5165 * 5166 * Update the board statistics counters. 5167 * 5168 **********************************************************************/ 5169 static void 5170 em_update_stats_counters(struct adapter *adapter) 5171 { 5172 struct ifnet *ifp; 5173 5174 if(adapter->hw.phy.media_type == e1000_media_type_copper || 5175 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 5176 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 5177 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 5178 } 5179 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 5180 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 5181 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 5182 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 5183 5184 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 5185 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 5186 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 5187 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 5188 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 5189 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 5190 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 5191 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 5192 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 5193 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 5194 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 5195 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 5196 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 5197 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 5198 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 5199 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 5200 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 5201 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 5202 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 5203 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 5204 5205 /* For the 64-bit byte counters the low dword must be read first. */ 5206 /* Both registers clear on the read of the high dword */ 5207 5208 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 5209 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 5210 5211 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 5212 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 5213 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 5214 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 5215 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 5216 5217 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 5218 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 5219 5220 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 5221 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 5222 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 5223 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 5224 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 5225 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 5226 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 5227 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 5228 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 5229 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 5230 5231 if (adapter->hw.mac.type >= e1000_82543) { 5232 adapter->stats.algnerrc += 5233 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 5234 adapter->stats.rxerrc += 5235 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 5236 adapter->stats.tncrs += 5237 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 5238 adapter->stats.cexterr += 5239 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 5240 adapter->stats.tsctc += 5241 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 5242 adapter->stats.tsctfc += 5243 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 5244 } 5245 ifp = adapter->ifp; 5246 5247 ifp->if_collisions = adapter->stats.colc; 5248 5249 /* Rx Errors */ 5250 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 5251 adapter->stats.crcerrs + adapter->stats.algnerrc + 5252 adapter->stats.ruc + adapter->stats.roc + 5253 adapter->stats.mpc + adapter->stats.cexterr; 5254 5255 /* Tx Errors */ 5256 ifp->if_oerrors = adapter->stats.ecol + 5257 adapter->stats.latecol + adapter->watchdog_events; 5258 } 5259 5260 5261 /********************************************************************** 5262 * 5263 * This routine is called only when em_display_debug_stats is enabled. 5264 * This routine provides a way to take a look at important statistics 5265 * maintained by the driver and hardware. 5266 * 5267 **********************************************************************/ 5268 static void 5269 em_print_debug_info(struct adapter *adapter) 5270 { 5271 device_t dev = adapter->dev; 5272 u8 *hw_addr = adapter->hw.hw_addr; 5273 5274 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 5275 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 5276 E1000_READ_REG(&adapter->hw, E1000_CTRL), 5277 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 5278 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 5279 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 5280 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 5281 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 5282 adapter->hw.fc.high_water, 5283 adapter->hw.fc.low_water); 5284 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 5285 E1000_READ_REG(&adapter->hw, E1000_TIDV), 5286 E1000_READ_REG(&adapter->hw, E1000_TADV)); 5287 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 5288 E1000_READ_REG(&adapter->hw, E1000_RDTR), 5289 E1000_READ_REG(&adapter->hw, E1000_RADV)); 5290 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 5291 (long long)adapter->tx_fifo_wrk_cnt, 5292 (long long)adapter->tx_fifo_reset_cnt); 5293 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 5294 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 5295 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 5296 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 5297 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 5298 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 5299 device_printf(dev, "Num Tx descriptors avail = %d\n", 5300 adapter->num_tx_desc_avail); 5301 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 5302 adapter->no_tx_desc_avail1); 5303 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 5304 adapter->no_tx_desc_avail2); 5305 device_printf(dev, "Std mbuf failed = %ld\n", 5306 adapter->mbuf_alloc_failed); 5307 device_printf(dev, "Std mbuf cluster failed = %ld\n", 5308 adapter->mbuf_cluster_failed); 5309 device_printf(dev, "Driver dropped packets = %ld\n", 5310 adapter->dropped_pkts); 5311 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 5312 adapter->no_tx_dma_setup); 5313 } 5314 5315 static void 5316 em_print_hw_stats(struct adapter *adapter) 5317 { 5318 device_t dev = adapter->dev; 5319 5320 device_printf(dev, "Excessive collisions = %lld\n", 5321 (long long)adapter->stats.ecol); 5322 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 5323 device_printf(dev, "Symbol errors = %lld\n", 5324 (long long)adapter->stats.symerrs); 5325 #endif 5326 device_printf(dev, "Sequence errors = %lld\n", 5327 (long long)adapter->stats.sec); 5328 device_printf(dev, "Defer count = %lld\n", 5329 (long long)adapter->stats.dc); 5330 device_printf(dev, "Missed Packets = %lld\n", 5331 (long long)adapter->stats.mpc); 5332 device_printf(dev, "Receive No Buffers = %lld\n", 5333 (long long)adapter->stats.rnbc); 5334 /* RLEC is inaccurate on some hardware, calculate our own. */ 5335 device_printf(dev, "Receive Length Errors = %lld\n", 5336 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 5337 device_printf(dev, "Receive errors = %lld\n", 5338 (long long)adapter->stats.rxerrc); 5339 device_printf(dev, "Crc errors = %lld\n", 5340 (long long)adapter->stats.crcerrs); 5341 device_printf(dev, "Alignment errors = %lld\n", 5342 (long long)adapter->stats.algnerrc); 5343 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 5344 (long long)adapter->stats.cexterr); 5345 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 5346 device_printf(dev, "watchdog timeouts = %ld\n", 5347 adapter->watchdog_events); 5348 device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld" 5349 " LINK MSIX IRQ = %ld\n", adapter->rx_irq, 5350 adapter->tx_irq , adapter->link_irq); 5351 device_printf(dev, "XON Rcvd = %lld\n", 5352 (long long)adapter->stats.xonrxc); 5353 device_printf(dev, "XON Xmtd = %lld\n", 5354 (long long)adapter->stats.xontxc); 5355 device_printf(dev, "XOFF Rcvd = %lld\n", 5356 (long long)adapter->stats.xoffrxc); 5357 device_printf(dev, "XOFF Xmtd = %lld\n", 5358 (long long)adapter->stats.xofftxc); 5359 device_printf(dev, "Good Packets Rcvd = %lld\n", 5360 (long long)adapter->stats.gprc); 5361 device_printf(dev, "Good Packets Xmtd = %lld\n", 5362 (long long)adapter->stats.gptc); 5363 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 5364 (long long)adapter->stats.tsctc); 5365 device_printf(dev, "TSO Contexts Failed = %lld\n", 5366 (long long)adapter->stats.tsctfc); 5367 } 5368 5369 /********************************************************************** 5370 * 5371 * This routine provides a way to dump out the adapter eeprom, 5372 * often a useful debug/service tool. This only dumps the first 5373 * 32 words, stuff that matters is in that extent. 5374 * 5375 **********************************************************************/ 5376 static void 5377 em_print_nvm_info(struct adapter *adapter) 5378 { 5379 u16 eeprom_data; 5380 int i, j, row = 0; 5381 5382 /* Its a bit crude, but it gets the job done */ 5383 printf("\nInterface EEPROM Dump:\n"); 5384 printf("Offset\n0x0000 "); 5385 for (i = 0, j = 0; i < 32; i++, j++) { 5386 if (j == 8) { /* Make the offset block */ 5387 j = 0; ++row; 5388 printf("\n0x00%x0 ",row); 5389 } 5390 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5391 printf("%04x ", eeprom_data); 5392 } 5393 printf("\n"); 5394 } 5395 5396 static int 5397 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 5398 { 5399 struct adapter *adapter; 5400 int error; 5401 int result; 5402 5403 result = -1; 5404 error = sysctl_handle_int(oidp, &result, 0, req); 5405 5406 if (error || !req->newptr) 5407 return (error); 5408 5409 if (result == 1) { 5410 adapter = (struct adapter *)arg1; 5411 em_print_debug_info(adapter); 5412 } 5413 /* 5414 * This value will cause a hex dump of the 5415 * first 32 16-bit words of the EEPROM to 5416 * the screen. 5417 */ 5418 if (result == 2) { 5419 adapter = (struct adapter *)arg1; 5420 em_print_nvm_info(adapter); 5421 } 5422 5423 return (error); 5424 } 5425 5426 5427 static int 5428 em_sysctl_stats(SYSCTL_HANDLER_ARGS) 5429 { 5430 struct adapter *adapter; 5431 int error; 5432 int result; 5433 5434 result = -1; 5435 error = sysctl_handle_int(oidp, &result, 0, req); 5436 5437 if (error || !req->newptr) 5438 return (error); 5439 5440 if (result == 1) { 5441 adapter = (struct adapter *)arg1; 5442 em_print_hw_stats(adapter); 5443 } 5444 5445 return (error); 5446 } 5447 5448 static int 5449 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5450 { 5451 struct em_int_delay_info *info; 5452 struct adapter *adapter; 5453 u32 regval; 5454 int error; 5455 int usecs; 5456 int ticks; 5457 5458 info = (struct em_int_delay_info *)arg1; 5459 usecs = info->value; 5460 error = sysctl_handle_int(oidp, &usecs, 0, req); 5461 if (error != 0 || req->newptr == NULL) 5462 return (error); 5463 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5464 return (EINVAL); 5465 info->value = usecs; 5466 ticks = EM_USECS_TO_TICKS(usecs); 5467 5468 adapter = info->adapter; 5469 5470 EM_CORE_LOCK(adapter); 5471 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5472 regval = (regval & ~0xffff) | (ticks & 0xffff); 5473 /* Handle a few special cases. */ 5474 switch (info->offset) { 5475 case E1000_RDTR: 5476 break; 5477 case E1000_TIDV: 5478 if (ticks == 0) { 5479 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5480 /* Don't write 0 into the TIDV register. */ 5481 regval++; 5482 } else 5483 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5484 break; 5485 } 5486 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5487 EM_CORE_UNLOCK(adapter); 5488 return (0); 5489 } 5490 5491 static void 5492 em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5493 const char *description, struct em_int_delay_info *info, 5494 int offset, int value) 5495 { 5496 info->adapter = adapter; 5497 info->offset = offset; 5498 info->value = value; 5499 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5500 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5501 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5502 info, 0, em_sysctl_int_delay, "I", description); 5503 } 5504 5505 #ifndef EM_LEGACY_IRQ 5506 static void 5507 em_add_rx_process_limit(struct adapter *adapter, const char *name, 5508 const char *description, int *limit, int value) 5509 { 5510 *limit = value; 5511 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5512 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5513 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5514 } 5515 #endif 5516 5517 5518