1 /*- 2 * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /*$FreeBSD$*/ 28 #include "opt_em.h" 29 #include "opt_ddb.h" 30 #include "opt_inet.h" 31 #include "opt_inet6.h" 32 33 #ifdef HAVE_KERNEL_OPTION_HEADERS 34 #include "opt_device_polling.h" 35 #endif 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #ifdef DDB 40 #include <sys/types.h> 41 #include <ddb/ddb.h> 42 #endif 43 #if __FreeBSD_version >= 800000 44 #include <sys/buf_ring.h> 45 #endif 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/kthread.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/module.h> 53 #include <sys/rman.h> 54 #include <sys/smp.h> 55 #include <sys/socket.h> 56 #include <sys/sockio.h> 57 #include <sys/sysctl.h> 58 #include <sys/taskqueue.h> 59 #include <sys/eventhandler.h> 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <net/bpf.h> 64 #include <net/ethernet.h> 65 #include <net/if.h> 66 #include <net/if_var.h> 67 #include <net/if_arp.h> 68 #include <net/if_dl.h> 69 #include <net/if_media.h> 70 #include <net/iflib.h> 71 72 #include <net/if_types.h> 73 #include <net/if_vlan_var.h> 74 75 #include <netinet/in_systm.h> 76 #include <netinet/in.h> 77 #include <netinet/if_ether.h> 78 #include <netinet/ip.h> 79 #include <netinet/ip6.h> 80 #include <netinet/tcp.h> 81 #include <netinet/udp.h> 82 83 #include <machine/in_cksum.h> 84 #include <dev/led/led.h> 85 #include <dev/pci/pcivar.h> 86 #include <dev/pci/pcireg.h> 87 88 #include "e1000_api.h" 89 #include "e1000_82571.h" 90 #include "ifdi_if.h" 91 92 93 #ifndef _EM_H_DEFINED_ 94 #define _EM_H_DEFINED_ 95 96 97 /* Tunables */ 98 99 /* 100 * EM_TXD: Maximum number of Transmit Descriptors 101 * Valid Range: 80-256 for 82542 and 82543-based adapters 102 * 80-4096 for others 103 * Default Value: 256 104 * This value is the number of transmit descriptors allocated by the driver. 105 * Increasing this value allows the driver to queue more transmits. Each 106 * descriptor is 16 bytes. 107 * Since TDLEN should be multiple of 128bytes, the number of transmit 108 * desscriptors should meet the following condition. 109 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 110 */ 111 #define EM_MIN_TXD 128 112 #define EM_MAX_TXD 4096 113 #define EM_DEFAULT_TXD 1024 114 #define EM_DEFAULT_MULTI_TXD 4096 115 116 /* 117 * EM_RXD - Maximum number of receive Descriptors 118 * Valid Range: 80-256 for 82542 and 82543-based adapters 119 * 80-4096 for others 120 * Default Value: 256 121 * This value is the number of receive descriptors allocated by the driver. 122 * Increasing this value allows the driver to buffer more incoming packets. 123 * Each descriptor is 16 bytes. A receive buffer is also allocated for each 124 * descriptor. The maximum MTU size is 16110. 125 * Since TDLEN should be multiple of 128bytes, the number of transmit 126 * desscriptors should meet the following condition. 127 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 128 */ 129 #define EM_MIN_RXD 128 130 #define EM_MAX_RXD 4096 131 #define EM_DEFAULT_RXD 1024 132 #define EM_DEFAULT_MULTI_RXD 4096 133 134 /* 135 * EM_TIDV - Transmit Interrupt Delay Value 136 * Valid Range: 0-65535 (0=off) 137 * Default Value: 64 138 * This value delays the generation of transmit interrupts in units of 139 * 1.024 microseconds. Transmit interrupt reduction can improve CPU 140 * efficiency if properly tuned for specific network traffic. If the 141 * system is reporting dropped transmits, this value may be set too high 142 * causing the driver to run out of available transmit descriptors. 143 */ 144 #define EM_TIDV 64 145 146 /* 147 * EM_TADV - Transmit Absolute Interrupt Delay Value 148 * (Not valid for 82542/82543/82544) 149 * Valid Range: 0-65535 (0=off) 150 * Default Value: 64 151 * This value, in units of 1.024 microseconds, limits the delay in which a 152 * transmit interrupt is generated. Useful only if EM_TIDV is non-zero, 153 * this value ensures that an interrupt is generated after the initial 154 * packet is sent on the wire within the set amount of time. Proper tuning, 155 * along with EM_TIDV, may improve traffic throughput in specific 156 * network conditions. 157 */ 158 #define EM_TADV 64 159 160 /* 161 * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer) 162 * Valid Range: 0-65535 (0=off) 163 * Default Value: 0 164 * This value delays the generation of receive interrupts in units of 1.024 165 * microseconds. Receive interrupt reduction can improve CPU efficiency if 166 * properly tuned for specific network traffic. Increasing this value adds 167 * extra latency to frame reception and can end up decreasing the throughput 168 * of TCP traffic. If the system is reporting dropped receives, this value 169 * may be set too high, causing the driver to run out of available receive 170 * descriptors. 171 * 172 * CAUTION: When setting EM_RDTR to a value other than 0, adapters 173 * may hang (stop transmitting) under certain network conditions. 174 * If this occurs a WATCHDOG message is logged in the system 175 * event log. In addition, the controller is automatically reset, 176 * restoring the network connection. To eliminate the potential 177 * for the hang ensure that EM_RDTR is set to 0. 178 */ 179 #ifdef EM_MULTIQUEUE 180 #define EM_RDTR 64 181 #else 182 #define EM_RDTR 0 183 #endif 184 185 /* 186 * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) 187 * Valid Range: 0-65535 (0=off) 188 * Default Value: 64 189 * This value, in units of 1.024 microseconds, limits the delay in which a 190 * receive interrupt is generated. Useful only if EM_RDTR is non-zero, 191 * this value ensures that an interrupt is generated after the initial 192 * packet is received within the set amount of time. Proper tuning, 193 * along with EM_RDTR, may improve traffic throughput in specific network 194 * conditions. 195 */ 196 #ifdef EM_MULTIQUEUE 197 #define EM_RADV 128 198 #else 199 #define EM_RADV 64 200 #endif 201 202 /* 203 * This parameter controls whether or not autonegotation is enabled. 204 * 0 - Disable autonegotiation 205 * 1 - Enable autonegotiation 206 */ 207 #define DO_AUTO_NEG 1 208 209 /* 210 * This parameter control whether or not the driver will wait for 211 * autonegotiation to complete. 212 * 1 - Wait for autonegotiation to complete 213 * 0 - Don't wait for autonegotiation to complete 214 */ 215 #define WAIT_FOR_AUTO_NEG_DEFAULT 0 216 217 /* Tunables -- End */ 218 219 #define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ 220 ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ 221 ADVERTISE_1000_FULL) 222 223 #define AUTO_ALL_MODES 0 224 225 /* PHY master/slave setting */ 226 #define EM_MASTER_SLAVE e1000_ms_hw_default 227 228 /* 229 * Micellaneous constants 230 */ 231 #define EM_VENDOR_ID 0x8086 232 #define EM_FLASH 0x0014 233 234 #define EM_JUMBO_PBA 0x00000028 235 #define EM_DEFAULT_PBA 0x00000030 236 #define EM_SMARTSPEED_DOWNSHIFT 3 237 #define EM_SMARTSPEED_MAX 15 238 #define EM_MAX_LOOP 10 239 240 #define MAX_NUM_MULTICAST_ADDRESSES 128 241 #define PCI_ANY_ID (~0U) 242 #define ETHER_ALIGN 2 243 #define EM_FC_PAUSE_TIME 0x0680 244 #define EM_EEPROM_APME 0x400; 245 #define EM_82544_APME 0x0004; 246 247 /* 248 * Driver state logic for the detection of a hung state 249 * in hardware. Set TX_HUNG whenever a TX packet is used 250 * (data is sent) and clear it when txeof() is invoked if 251 * any descriptors from the ring are cleaned/reclaimed. 252 * Increment internal counter if no descriptors are cleaned 253 * and compare to TX_MAXTRIES. When counter > TX_MAXTRIES, 254 * reset adapter. 255 */ 256 #define EM_TX_IDLE 0x00000000 257 #define EM_TX_BUSY 0x00000001 258 #define EM_TX_HUNG 0x80000000 259 #define EM_TX_MAXTRIES 10 260 261 #define PCICFG_DESC_RING_STATUS 0xe4 262 #define FLUSH_DESC_REQUIRED 0x100 263 264 265 #define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ 266 ((hw->mac.type <= e1000_82576) ? 16 : 8)) 267 #define IGB_RX_HTHRESH 8 268 #define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ 269 (adapter->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) 270 271 #define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 272 #define IGB_TX_HTHRESH 1 273 #define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ 274 (adapter->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) 275 276 /* 277 * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be 278 * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will 279 * also optimize cache line size effect. H/W supports up to cache line size 128. 280 */ 281 #define EM_DBA_ALIGN 128 282 283 /* 284 * See Intel 82574 Driver Programming Interface Manual, Section 10.2.6.9 285 */ 286 #define TARC_COMPENSATION_MODE (1 << 7) /* Compensation Mode */ 287 #define TARC_SPEED_MODE_BIT (1 << 21) /* On PCI-E MACs only */ 288 #define TARC_MQ_FIX (1 << 23) | \ 289 (1 << 24) | \ 290 (1 << 25) /* Handle errata in MQ mode */ 291 #define TARC_ERRATA_BIT (1 << 26) /* Note from errata on 82574 */ 292 293 /* PCI Config defines */ 294 #define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) 295 #define EM_BAR_TYPE_MASK 0x00000001 296 #define EM_BAR_TYPE_MMEM 0x00000000 297 #define EM_BAR_TYPE_IO 0x00000001 298 #define EM_BAR_TYPE_FLASH 0x0014 299 #define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) 300 #define EM_BAR_MEM_TYPE_MASK 0x00000006 301 #define EM_BAR_MEM_TYPE_32BIT 0x00000000 302 #define EM_BAR_MEM_TYPE_64BIT 0x00000004 303 #define EM_MSIX_BAR 3 /* On 82575 */ 304 305 /* More backward compatibility */ 306 #if __FreeBSD_version < 900000 307 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 308 #endif 309 310 /* Defines for printing debug information */ 311 #define DEBUG_INIT 0 312 #define DEBUG_IOCTL 0 313 #define DEBUG_HW 0 314 315 #define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") 316 #define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) 317 #define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) 318 #define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") 319 #define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) 320 #define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) 321 #define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") 322 #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) 323 #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) 324 325 #define EM_MAX_SCATTER 40 326 #define EM_VFTA_SIZE 128 327 #define EM_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) 328 #define EM_TSO_SEG_SIZE 4096 /* Max dma segment size */ 329 #define EM_MSIX_MASK 0x01F00000 /* For 82574 use */ 330 #define EM_MSIX_LINK 0x01000000 /* For 82574 use */ 331 #define ETH_ZLEN 60 332 #define ETH_ADDR_LEN 6 333 #define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ 334 335 #define IGB_PKTTYPE_MASK 0x0000FFF0 336 #define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 337 338 /* 339 * 82574 has a nonstandard address for EIAC 340 * and since its only used in MSIX, and in 341 * the em driver only 82574 uses MSIX we can 342 * solve it just using this define. 343 */ 344 #define EM_EIAC 0x000DC 345 /* 346 * 82574 only reports 3 MSI-X vectors by default; 347 * defines assisting with making it report 5 are 348 * located here. 349 */ 350 #define EM_NVM_PCIE_CTRL 0x1B 351 #define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) 352 #define EM_NVM_MSIX_N_SHIFT 7 353 354 struct adapter; 355 356 struct em_int_delay_info { 357 struct adapter *adapter; /* Back-pointer to the adapter struct */ 358 int offset; /* Register offset to read/write */ 359 int value; /* Current value in usecs */ 360 }; 361 362 /* 363 * The transmit ring, one per tx queue 364 */ 365 struct tx_ring { 366 struct adapter *adapter; 367 struct em_tx_queue *que; 368 u32 me; 369 int busy; 370 struct e1000_tx_desc *tx_base; 371 uint64_t tx_paddr; 372 struct em_txbuffer *tx_buffers; 373 u32 tx_tso; /* last tx was tso */ 374 375 /* Interrupt resources */ 376 void *tag; 377 struct resource *res; 378 unsigned long tx_irq; 379 unsigned long no_desc_avail; 380 381 /* Saved csum offloading context information */ 382 int csum_flags; 383 int csum_lhlen; 384 int csum_iphlen; 385 386 int csum_thlen; 387 int csum_mss; 388 int csum_pktlen; 389 390 uint32_t csum_txd_upper; 391 uint32_t csum_txd_lower; /* last field */ 392 }; 393 394 /* 395 * The Receive ring, one per rx queue 396 */ 397 struct rx_ring { 398 struct adapter *adapter; 399 struct em_rx_queue *que; 400 u32 me; 401 u32 payload; 402 union e1000_rx_desc_extended *rx_base; 403 uint64_t rx_paddr; 404 405 /* Interrupt resources */ 406 void *tag; 407 struct resource *res; 408 bool discard; 409 410 /* Soft stats */ 411 unsigned long rx_irq; 412 unsigned long rx_discarded; 413 unsigned long rx_packets; 414 unsigned long rx_bytes; 415 }; 416 417 struct em_tx_queue { 418 struct adapter *adapter; 419 u32 msix; 420 u32 eims; /* This queue's EIMS bit */ 421 u32 me; 422 struct tx_ring txr; 423 }; 424 425 struct em_rx_queue { 426 struct adapter *adapter; 427 u32 me; 428 u32 msix; 429 u32 eims; 430 struct rx_ring rxr; 431 u64 irqs; 432 struct if_irq que_irq; 433 }; 434 435 /* Our adapter structure */ 436 struct adapter { 437 struct ifnet *ifp; 438 struct e1000_hw hw; 439 440 if_softc_ctx_t shared; 441 if_ctx_t ctx; 442 #define tx_num_queues shared->isc_ntxqsets 443 #define rx_num_queues shared->isc_nrxqsets 444 #define intr_type shared->isc_intr 445 /* FreeBSD operating-system-specific structures. */ 446 struct e1000_osdep osdep; 447 struct device *dev; 448 struct cdev *led_dev; 449 450 struct em_tx_queue *tx_queues; 451 struct em_rx_queue *rx_queues; 452 struct if_irq irq; 453 454 struct resource *memory; 455 struct resource *flash; 456 struct resource *ioport; 457 int io_rid; 458 459 struct resource *res; 460 void *tag; 461 u32 linkvec; 462 u32 ivars; 463 464 struct ifmedia *media; 465 int msix; 466 int if_flags; 467 int min_frame_size; 468 int em_insert_vlan_header; 469 u32 ims; 470 bool in_detach; 471 472 /* Task for FAST handling */ 473 struct grouptask link_task; 474 475 u16 num_vlans; 476 u32 txd_cmd; 477 478 u32 tx_process_limit; 479 u32 rx_process_limit; 480 u32 rx_mbuf_sz; 481 482 /* Management and WOL features */ 483 u32 wol; 484 bool has_manage; 485 bool has_amt; 486 487 /* Multicast array memory */ 488 u8 *mta; 489 490 /* 491 ** Shadow VFTA table, this is needed because 492 ** the real vlan filter table gets cleared during 493 ** a soft reset and the driver needs to be able 494 ** to repopulate it. 495 */ 496 u32 shadow_vfta[EM_VFTA_SIZE]; 497 498 /* Info about the interface */ 499 u16 link_active; 500 u16 fc; 501 u16 link_speed; 502 u16 link_duplex; 503 u32 smartspeed; 504 u32 dmac; 505 int link_mask; 506 507 u64 que_mask; 508 509 510 struct em_int_delay_info tx_int_delay; 511 struct em_int_delay_info tx_abs_int_delay; 512 struct em_int_delay_info rx_int_delay; 513 struct em_int_delay_info rx_abs_int_delay; 514 struct em_int_delay_info tx_itr; 515 516 /* Misc stats maintained by the driver */ 517 unsigned long dropped_pkts; 518 unsigned long link_irq; 519 unsigned long mbuf_defrag_failed; 520 unsigned long no_tx_dma_setup; 521 unsigned long no_tx_map_avail; 522 unsigned long rx_overruns; 523 unsigned long watchdog_events; 524 525 struct e1000_hw_stats stats; 526 }; 527 528 /******************************************************************************** 529 * vendor_info_array 530 * 531 * This array contains the list of Subvendor/Subdevice IDs on which the driver 532 * should load. 533 * 534 ********************************************************************************/ 535 typedef struct _em_vendor_info_t { 536 unsigned int vendor_id; 537 unsigned int device_id; 538 unsigned int subvendor_id; 539 unsigned int subdevice_id; 540 unsigned int index; 541 } em_vendor_info_t; 542 543 struct em_txbuffer { 544 int eop; 545 }; 546 547 548 #define EM_CORE_LOCK_INIT(_sc, _name) \ 549 mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF) 550 #define EM_TX_LOCK_INIT(_sc, _name) \ 551 mtx_init(&(_sc)->tx_mtx, _name, "EM TX Lock", MTX_DEF) 552 #define EM_RX_LOCK_INIT(_sc, _name) \ 553 mtx_init(&(_sc)->rx_mtx, _name, "EM RX Lock", MTX_DEF) 554 #define EM_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx) 555 #define EM_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx) 556 #define EM_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) 557 #define EM_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) 558 #define EM_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) 559 #define EM_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) 560 #define EM_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) 561 #define EM_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) 562 #define EM_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) 563 #define EM_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx) 564 #define EM_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED) 565 #define EM_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) 566 #define EM_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_mtx, MA_OWNED) 567 568 #define EM_RSSRK_SIZE 4 569 #define EM_RSSRK_VAL(key, i) (key[(i) * EM_RSSRK_SIZE] | \ 570 key[(i) * EM_RSSRK_SIZE + 1] << 8 | \ 571 key[(i) * EM_RSSRK_SIZE + 2] << 16 | \ 572 key[(i) * EM_RSSRK_SIZE + 3] << 24) 573 #endif /* _EM_H_DEFINED_ */ 574