1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2024 Kevin Bowling <kbowling@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 31 #ifndef _EM_H_DEFINED_ 32 #define _EM_H_DEFINED_ 33 34 #include "opt_ddb.h" 35 #include "opt_inet.h" 36 #include "opt_inet6.h" 37 #include "opt_rss.h" 38 39 #ifdef HAVE_KERNEL_OPTION_HEADERS 40 #include "opt_device_polling.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #ifdef DDB 46 #include <sys/types.h> 47 #include <ddb/ddb.h> 48 #endif 49 #include <sys/buf_ring.h> 50 #include <sys/bus.h> 51 #include <sys/endian.h> 52 #include <sys/kernel.h> 53 #include <sys/kthread.h> 54 #include <sys/malloc.h> 55 #include <sys/mbuf.h> 56 #include <sys/module.h> 57 #include <sys/rman.h> 58 #include <sys/smp.h> 59 #include <sys/socket.h> 60 #include <sys/sockio.h> 61 #include <sys/sysctl.h> 62 #include <sys/taskqueue.h> 63 #include <sys/eventhandler.h> 64 #include <machine/bus.h> 65 #include <machine/resource.h> 66 67 #include <net/bpf.h> 68 #include <net/ethernet.h> 69 #include <net/if.h> 70 #include <net/if_var.h> 71 #include <net/if_arp.h> 72 #include <net/if_dl.h> 73 #include <net/if_media.h> 74 #include <net/iflib.h> 75 #ifdef RSS 76 #include <net/rss_config.h> 77 #include <netinet/in_rss.h> 78 #endif 79 80 #include <net/if_types.h> 81 #include <net/if_vlan_var.h> 82 83 #include <netinet/in_systm.h> 84 #include <netinet/in.h> 85 #include <netinet/if_ether.h> 86 #include <netinet/ip.h> 87 #include <netinet/ip6.h> 88 #include <netinet/tcp.h> 89 #include <netinet/udp.h> 90 91 #include <machine/in_cksum.h> 92 #include <dev/led/led.h> 93 #include <dev/pci/pcivar.h> 94 #include <dev/pci/pcireg.h> 95 96 #include "e1000_api.h" 97 #include "e1000_82571.h" 98 #include "ifdi_if.h" 99 100 /* Tunables */ 101 102 /* 103 * EM_MAX_TXD: Maximum number of Transmit Descriptors 104 * Valid Range: 80-256 for 82542 and 82543-based adapters 105 * 80-4096 for others 106 * Default Value: 1024 107 * This value is the number of transmit descriptors allocated by the driver. 108 * Increasing this value allows the driver to queue more transmits. Each 109 * descriptor is 16 bytes. 110 * Since TDLEN should be multiple of 128bytes, the number of transmit 111 * desscriptors should meet the following condition. 112 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 113 */ 114 #define EM_MIN_TXD 128 115 #define EM_MAX_TXD 4096 116 #define EM_DEFAULT_TXD 1024 117 #define EM_DEFAULT_MULTI_TXD 4096 118 #define IGB_MAX_TXD 4096 119 120 /* 121 * EM_MAX_RXD - Maximum number of receive Descriptors 122 * Valid Range: 80-256 for 82542 and 82543-based adapters 123 * 80-4096 for others 124 * Default Value: 1024 125 * This value is the number of receive descriptors allocated by the driver. 126 * Increasing this value allows the driver to buffer more incoming packets. 127 * Each descriptor is 16 bytes. A receive buffer is also allocated for each 128 * descriptor. The maximum MTU size is 16110. 129 * Since TDLEN should be multiple of 128bytes, the number of transmit 130 * desscriptors should meet the following condition. 131 * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 132 */ 133 #define EM_MIN_RXD 128 134 #define EM_MAX_RXD 4096 135 #define EM_DEFAULT_RXD 1024 136 #define EM_DEFAULT_MULTI_RXD 4096 137 #define IGB_MAX_RXD 4096 138 139 /* 140 * EM_TIDV - Transmit Interrupt Delay Value 141 * Valid Range: 0-65535 (0=off) 142 * Default Value: 64 143 * This value delays the generation of transmit interrupts in units of 144 * 1.024 microseconds. Transmit interrupt reduction can improve CPU 145 * efficiency if properly tuned for specific network traffic. If the 146 * system is reporting dropped transmits, this value may be set too high 147 * causing the driver to run out of available transmit descriptors. 148 */ 149 #define EM_TIDV 64 150 151 /* 152 * EM_TADV - Transmit Absolute Interrupt Delay Value 153 * (Not valid for 82542/82543/82544) 154 * Valid Range: 0-65535 (0=off) 155 * Default Value: 64 156 * This value, in units of 1.024 microseconds, limits the delay in which a 157 * transmit interrupt is generated. Useful only if EM_TIDV is non-zero, 158 * this value ensures that an interrupt is generated after the initial 159 * packet is sent on the wire within the set amount of time. Proper tuning, 160 * along with EM_TIDV, may improve traffic throughput in specific 161 * network conditions. 162 */ 163 #define EM_TADV 64 164 165 /* 166 * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer) 167 * Valid Range: 0-65535 (0=off) 168 * Default Value: 0 169 * This value delays the generation of receive interrupts in units of 1.024 170 * microseconds. Receive interrupt reduction can improve CPU efficiency if 171 * properly tuned for specific network traffic. Increasing this value adds 172 * extra latency to frame reception and can end up decreasing the throughput 173 * of TCP traffic. If the system is reporting dropped receives, this value 174 * may be set too high, causing the driver to run out of available receive 175 * descriptors. 176 * 177 * CAUTION: When setting EM_RDTR to a value other than 0, adapters 178 * may hang (stop transmitting) under certain network conditions. 179 * If this occurs a WATCHDOG message is logged in the system 180 * event log. In addition, the controller is automatically reset, 181 * restoring the network connection. To eliminate the potential 182 * for the hang ensure that EM_RDTR is set to 0. 183 */ 184 #define EM_RDTR 0 185 186 /* 187 * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) 188 * Valid Range: 0-65535 (0=off) 189 * Default Value: 64 190 * This value, in units of 1.024 microseconds, limits the delay in which a 191 * receive interrupt is generated. Useful only if EM_RDTR is non-zero, 192 * this value ensures that an interrupt is generated after the initial 193 * packet is received within the set amount of time. Proper tuning, 194 * along with EM_RDTR, may improve traffic throughput in specific network 195 * conditions. 196 */ 197 #define EM_RADV 64 198 199 /* 200 * This parameter controls whether or not autonegotiation is enabled. 201 * 0 - Disable autonegotiation 202 * 1 - Enable autonegotiation 203 */ 204 #define DO_AUTO_NEG 1 205 206 /* 207 * This parameter control whether or not the driver will wait for 208 * autonegotiation to complete. 209 * 1 - Wait for autonegotiation to complete 210 * 0 - Don't wait for autonegotiation to complete 211 */ 212 #define WAIT_FOR_AUTO_NEG_DEFAULT 0 213 214 /* Tunables -- End */ 215 216 #define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ 217 ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ 218 ADVERTISE_1000_FULL) 219 220 #define AUTO_ALL_MODES 0 221 222 /* PHY master/slave setting */ 223 #define EM_MASTER_SLAVE e1000_ms_hw_default 224 225 /* 226 * Miscellaneous constants 227 */ 228 #define EM_VENDOR_ID 0x8086 229 #define EM_FLASH 0x0014 230 231 #define EM_JUMBO_PBA 0x00000028 232 #define EM_DEFAULT_PBA 0x00000030 233 #define EM_SMARTSPEED_DOWNSHIFT 3 234 #define EM_SMARTSPEED_MAX 15 235 #define EM_MAX_LOOP 10 236 237 #define MAX_NUM_MULTICAST_ADDRESSES 128 238 #define PCI_ANY_ID (~0U) 239 #define ETHER_ALIGN 2 240 #define EM_FC_PAUSE_TIME 0x0680 241 #define EM_EEPROM_APME 0x400; 242 #define EM_82544_APME 0x0004; 243 244 /* Support AutoMediaDetect for Marvell M88 PHY in i354 */ 245 #define IGB_MEDIA_RESET (1 << 0) 246 247 /* Define the interrupt rates and ITR helpers */ 248 #define EM_INTS_4K 4000 249 #define EM_INTS_20K 20000 250 #define EM_INTS_70K 70000 251 #define EM_INTS_DEFAULT 8000 252 #define EM_INTS_MULTIPLIER 256 253 #define EM_ITR_DIVIDEND 1000000000 254 #define EM_INTS_TO_ITR(i) (EM_ITR_DIVIDEND/(i * EM_INTS_MULTIPLIER)) 255 #define IGB_EITR_DIVIDEND 1000000 256 #define IGB_EITR_SHIFT 2 257 #define IGB_QVECTOR_MASK 0x7FFC 258 #define IGB_INTS_TO_EITR(i) (((IGB_EITR_DIVIDEND/i) & IGB_QVECTOR_MASK) << \ 259 IGB_EITR_SHIFT) 260 261 #define IGB_LINK_ITR 2000 262 #define I210_LINK_DELAY 1000 263 264 #define IGB_TXPBSIZE 20408 265 #define IGB_HDR_BUF 128 266 #define IGB_PKTTYPE_MASK 0x0000FFF0 267 #define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 268 269 /* 270 * Driver state logic for the detection of a hung state 271 * in hardware. Set TX_HUNG whenever a TX packet is used 272 * (data is sent) and clear it when txeof() is invoked if 273 * any descriptors from the ring are cleaned/reclaimed. 274 * Increment internal counter if no descriptors are cleaned 275 * and compare to TX_MAXTRIES. When counter > TX_MAXTRIES, 276 * reset adapter. 277 */ 278 #define EM_TX_IDLE 0x00000000 279 #define EM_TX_BUSY 0x00000001 280 #define EM_TX_HUNG 0x80000000 281 #define EM_TX_MAXTRIES 10 282 283 #define PCICFG_DESC_RING_STATUS 0xe4 284 #define FLUSH_DESC_REQUIRED 0x100 285 286 287 #define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ 288 ((hw->mac.type <= e1000_82576) ? 16 : 8)) 289 #define IGB_RX_HTHRESH 8 290 #define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ 291 (sc->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) 292 293 #define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) 294 #define IGB_TX_HTHRESH 1 295 #define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ 296 sc->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) 297 298 /* 299 * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be 300 * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will 301 * also optimize cache line size effect. H/W supports up to cache line size 128. 302 */ 303 #define EM_DBA_ALIGN 128 304 305 /* 306 * See Intel 82574 Driver Programming Interface Manual, Section 10.2.6.9 307 */ 308 #define TARC_COMPENSATION_MODE (1 << 7) /* Compensation Mode */ 309 #define TARC_SPEED_MODE_BIT (1 << 21) /* On PCI-E MACs only */ 310 #define TARC_MQ_FIX (1 << 23) | \ 311 (1 << 24) | \ 312 (1 << 25) /* Handle errata in MQ mode */ 313 #define TARC_ERRATA_BIT (1 << 26) /* Note from errata on 82574 */ 314 315 /* PCI Config defines */ 316 #define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) 317 #define EM_BAR_TYPE_MASK 0x00000001 318 #define EM_BAR_TYPE_MMEM 0x00000000 319 #define EM_BAR_TYPE_IO 0x00000001 320 #define EM_BAR_TYPE_FLASH 0x0014 321 #define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) 322 #define EM_BAR_MEM_TYPE_MASK 0x00000006 323 #define EM_BAR_MEM_TYPE_32BIT 0x00000000 324 #define EM_BAR_MEM_TYPE_64BIT 0x00000004 325 326 /* Defines for printing debug information */ 327 #define DEBUG_INIT 0 328 #define DEBUG_IOCTL 0 329 #define DEBUG_HW 0 330 331 #define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") 332 #define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) 333 #define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) 334 #define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") 335 #define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) 336 #define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) 337 #define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") 338 #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) 339 #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) 340 341 #define EM_MAX_SCATTER 40 342 #define EM_VFTA_SIZE 128 343 #define EM_TSO_SIZE 65535 344 #define EM_TSO_SEG_SIZE 4096 /* Max dma segment size */ 345 #define ETH_ZLEN 60 346 347 /* Offload bits in mbuf flag */ 348 #define EM_CSUM_OFFLOAD (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | \ 349 CSUM_IP6_UDP | CSUM_IP6_TCP) 350 #define IGB_CSUM_OFFLOAD (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | \ 351 CSUM_IP_SCTP | CSUM_IP6_UDP | CSUM_IP6_TCP | \ 352 CSUM_IP6_SCTP) 353 354 #define IGB_PKTTYPE_MASK 0x0000FFF0 355 #define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 356 357 /* 358 * 82574 has a nonstandard address for EIAC 359 * and since its only used in MSI-X, and in 360 * the em driver only 82574 uses MSI-X we can 361 * solve it just using this define. 362 */ 363 #define EM_EIAC 0x000DC 364 /* 365 * 82574 only reports 3 MSI-X vectors by default; 366 * defines assisting with making it report 5 are 367 * located here. 368 */ 369 #define EM_NVM_PCIE_CTRL 0x1B 370 #define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) 371 #define EM_NVM_MSIX_N_SHIFT 7 372 373 /* 374 * VFs use 32-bit counter that rolls over. 375 */ 376 #define UPDATE_VF_REG(reg, last, cur) \ 377 do { \ 378 u32 new = E1000_READ_REG(&sc->hw, reg); \ 379 if (new < last) \ 380 cur += 0x100000000LL; \ 381 last = new; \ 382 cur &= 0xFFFFFFFF00000000LL; \ 383 cur |= new; \ 384 } while (0) 385 386 struct e1000_softc; 387 388 struct em_int_delay_info { 389 struct e1000_softc *sc; /* Back-pointer to the sc struct */ 390 int offset; /* Register offset to read/write */ 391 int value; /* Current value in usecs */ 392 }; 393 394 /* 395 * The transmit ring, one per tx queue 396 */ 397 struct tx_ring { 398 struct e1000_softc *sc; 399 struct e1000_tx_desc *tx_base; 400 uint64_t tx_paddr; 401 qidx_t *tx_rsq; 402 bool tx_tso; /* last tx was tso */ 403 uint8_t me; 404 qidx_t tx_rs_cidx; 405 qidx_t tx_rs_pidx; 406 qidx_t tx_cidx_processed; 407 /* Interrupt resources */ 408 void *tag; 409 struct resource *res; 410 411 /* Soft stats */ 412 unsigned long tx_irq; 413 unsigned long tx_packets; 414 unsigned long tx_bytes; 415 416 /* Saved csum offloading context information */ 417 int csum_flags; 418 int csum_lhlen; 419 int csum_iphlen; 420 421 int csum_thlen; 422 int csum_mss; 423 int csum_pktlen; 424 425 uint32_t csum_txd_upper; 426 uint32_t csum_txd_lower; /* last field */ 427 }; 428 429 /* 430 * The Receive ring, one per rx queue 431 */ 432 struct rx_ring { 433 struct e1000_softc *sc; 434 struct em_rx_queue *que; 435 u32 me; 436 u32 payload; 437 union e1000_rx_desc_extended *rx_base; 438 uint64_t rx_paddr; 439 440 /* Interrupt resources */ 441 void *tag; 442 struct resource *res; 443 bool discard; 444 445 /* Soft stats */ 446 unsigned long rx_irq; 447 unsigned long rx_discarded; 448 unsigned long rx_packets; 449 unsigned long rx_bytes; 450 451 /* Next requested ITR latency */ 452 u8 rx_nextlatency; 453 }; 454 455 struct em_tx_queue { 456 struct e1000_softc *sc; 457 u32 msix; 458 u32 eims; /* This queue's EIMS bit */ 459 u32 me; 460 struct tx_ring txr; 461 }; 462 463 struct em_rx_queue { 464 struct e1000_softc *sc; 465 u32 me; 466 u32 msix; 467 u32 eims; 468 u32 itr_setting; 469 struct rx_ring rxr; 470 u64 irqs; 471 struct if_irq que_irq; 472 }; 473 474 /* Our softc structure */ 475 struct e1000_softc { 476 struct e1000_hw hw; 477 478 if_softc_ctx_t shared; 479 if_ctx_t ctx; 480 #define tx_num_queues shared->isc_ntxqsets 481 #define rx_num_queues shared->isc_nrxqsets 482 #define intr_type shared->isc_intr 483 /* FreeBSD operating-system-specific structures. */ 484 struct e1000_osdep osdep; 485 device_t dev; 486 struct cdev *led_dev; 487 488 struct em_tx_queue *tx_queues; 489 struct em_rx_queue *rx_queues; 490 struct if_irq irq; 491 492 struct resource *memory; 493 struct resource *flash; 494 struct resource *ioport; 495 496 struct resource *res; 497 void *tag; 498 u32 linkvec; 499 u32 ivars; 500 501 struct ifmedia *media; 502 int msix; 503 int if_flags; 504 int em_insert_vlan_header; 505 u32 ims; 506 bool in_detach; 507 508 u32 flags; 509 /* Task for FAST handling */ 510 struct grouptask link_task; 511 512 u16 num_vlans; 513 u32 txd_cmd; 514 515 u32 rx_mbuf_sz; 516 517 int enable_aim; 518 /* Management and WOL features */ 519 u32 wol; 520 bool has_manage; 521 bool has_amt; 522 523 /* Multicast array memory */ 524 u8 *mta; 525 526 /* 527 ** Shadow VFTA table, this is needed because 528 ** the real vlan filter table gets cleared during 529 ** a soft reset and the driver needs to be able 530 ** to repopulate it. 531 */ 532 u32 shadow_vfta[EM_VFTA_SIZE]; 533 534 /* Info about the interface */ 535 u16 link_active; 536 u16 fc; 537 u16 link_speed; 538 u16 link_duplex; 539 u32 smartspeed; 540 u32 dmac; 541 u32 pba; 542 int link_mask; 543 int tso_automasked; 544 545 u64 que_mask; 546 547 /* We need to store this at attach due to e1000 hw/sw locking model */ 548 struct e1000_fw_version fw_ver; 549 550 struct em_int_delay_info tx_int_delay; 551 struct em_int_delay_info tx_abs_int_delay; 552 struct em_int_delay_info rx_int_delay; 553 struct em_int_delay_info rx_abs_int_delay; 554 struct em_int_delay_info tx_itr; 555 556 /* Misc stats maintained by the driver */ 557 unsigned long dropped_pkts; 558 unsigned long link_irq; 559 unsigned long rx_overruns; 560 unsigned long watchdog_events; 561 562 union { 563 struct e1000_hw_stats stats; /* !sc->vf_ifp */ 564 struct e1000_vf_stats vf_stats; /* sc->vf_ifp */ 565 } ustats; 566 567 u16 vf_ifp; 568 }; 569 570 /******************************************************************************** 571 * vendor_info_array 572 * 573 * This array contains the list of Subvendor/Subdevice IDs on which the driver 574 * should load. 575 * 576 ********************************************************************************/ 577 typedef struct _em_vendor_info_t { 578 unsigned int vendor_id; 579 unsigned int device_id; 580 unsigned int subvendor_id; 581 unsigned int subdevice_id; 582 unsigned int index; 583 } em_vendor_info_t; 584 585 void em_dump_rs(struct e1000_softc *); 586 587 #define EM_RSSRK_SIZE 4 588 #define EM_RSSRK_VAL(key, i) (key[(i) * EM_RSSRK_SIZE] | \ 589 key[(i) * EM_RSSRK_SIZE + 1] << 8 | \ 590 key[(i) * EM_RSSRK_SIZE + 2] << 16 | \ 591 key[(i) * EM_RSSRK_SIZE + 3] << 24) 592 #endif /* _EM_H_DEFINED_ */ 593