1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2001-2024, Intel Corporation 5 * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 6 * Copyright (c) 2021-2024 Rubicon Communications, LLC (Netgate) 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include "opt_ddb.h" 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 #include "opt_rss.h" 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #ifdef DDB 42 #include <sys/types.h> 43 #include <ddb/ddb.h> 44 #endif 45 #include <sys/buf_ring.h> 46 #include <sys/bus.h> 47 #include <sys/endian.h> 48 #include <sys/kernel.h> 49 #include <sys/kthread.h> 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/module.h> 53 #include <sys/rman.h> 54 #include <sys/smp.h> 55 #include <sys/socket.h> 56 #include <sys/sockio.h> 57 #include <sys/sysctl.h> 58 #include <sys/taskqueue.h> 59 #include <sys/eventhandler.h> 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <net/bpf.h> 64 #include <net/ethernet.h> 65 #include <net/if.h> 66 #include <net/if_var.h> 67 #include <net/if_arp.h> 68 #include <net/if_dl.h> 69 #include <net/if_media.h> 70 #include <net/iflib.h> 71 72 #include <net/if_types.h> 73 #include <net/if_vlan_var.h> 74 75 #include <netinet/in_systm.h> 76 #include <netinet/in.h> 77 #include <netinet/if_ether.h> 78 #include <netinet/ip.h> 79 #include <netinet/ip6.h> 80 #include <netinet/tcp.h> 81 #include <netinet/udp.h> 82 83 #include <machine/in_cksum.h> 84 #include <dev/led/led.h> 85 #include <dev/pci/pcivar.h> 86 #include <dev/pci/pcireg.h> 87 88 #include "igc_api.h" 89 #include "igc_i225.h" 90 #include "ifdi_if.h" 91 92 93 #ifndef _IGC_H_DEFINED_ 94 #define _IGC_H_DEFINED_ 95 96 97 /* Tunables */ 98 99 /* 100 * IGC_MAX_TXD: Maximum number of Transmit Descriptors 101 * Valid Range: 128-4096 102 * Default Value: 1024 103 * This value is the number of transmit descriptors allocated by the driver. 104 * Increasing this value allows the driver to queue more transmits. Each 105 * descriptor is 16 bytes. 106 * Since TDLEN should be multiple of 128bytes, the number of transmit 107 * desscriptors should meet the following condition. 108 * (num_tx_desc * sizeof(struct igc_tx_desc)) % 128 == 0 109 */ 110 #define IGC_MIN_TXD 128 111 #define IGC_MAX_TXD 4096 112 #define IGC_DEFAULT_TXD 1024 113 #define IGC_DEFAULT_MULTI_TXD 4096 114 #define IGC_MAX_TXD 4096 115 116 /* 117 * IGC_MAX_RXD - Maximum number of receive Descriptors 118 * Valid Range: 128-4096 119 * Default Value: 1024 120 * This value is the number of receive descriptors allocated by the driver. 121 * Increasing this value allows the driver to buffer more incoming packets. 122 * Each descriptor is 16 bytes. A receive buffer is also allocated for each 123 * descriptor. The maximum MTU size is 16110. 124 * Since TDLEN should be multiple of 128bytes, the number of transmit 125 * desscriptors should meet the following condition. 126 * (num_tx_desc * sizeof(struct igc_tx_desc)) % 128 == 0 127 */ 128 #define IGC_MIN_RXD 128 129 #define IGC_MAX_RXD 4096 130 #define IGC_DEFAULT_RXD 1024 131 #define IGC_DEFAULT_MULTI_RXD 4096 132 #define IGC_MAX_RXD 4096 133 134 /* 135 * This parameter controls whether or not autonegotation is enabled. 136 * 0 - Disable autonegotiation 137 * 1 - Enable autonegotiation 138 */ 139 #define DO_AUTO_NEG true 140 141 /* Tunables -- End */ 142 143 #define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ 144 ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ 145 ADVERTISE_1000_FULL | ADVERTISE_2500_FULL) 146 147 #define AUTO_ALL_MODES 0 148 149 /* 150 * Micellaneous constants 151 */ 152 #define MAX_NUM_MULTICAST_ADDRESSES 128 153 #define IGC_FC_PAUSE_TIME 0x0680 154 155 #define IGC_TXPBSIZE 20408 156 #define IGC_PKTTYPE_MASK 0x0000FFF0 157 #define IGC_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ 158 159 #define IGC_RX_PTHRESH 8 160 #define IGC_RX_HTHRESH 8 161 #define IGC_RX_WTHRESH 4 162 163 #define IGC_TX_PTHRESH 8 164 #define IGC_TX_HTHRESH 1 165 166 /* Define the interrupt rates and EITR helpers */ 167 #define IGC_INTS_4K 4000 168 #define IGC_INTS_20K 20000 169 #define IGC_INTS_70K 70000 170 #define IGC_INTS_DEFAULT 8000 171 #define IGC_EITR_DIVIDEND 1000000 172 #define IGC_EITR_SHIFT 2 173 #define IGC_QVECTOR_MASK 0x7FFC 174 #define IGC_INTS_TO_EITR(i) \ 175 (((IGC_EITR_DIVIDEND / (i)) << IGC_EITR_SHIFT) & IGC_QVECTOR_MASK) 176 #define IGC_EITR_TO_INTS(i) ((IGC_EITR_DIVIDEND << IGC_EITR_SHIFT) / \ 177 ((i) & IGC_QVECTOR_MASK)) 178 179 /* 180 * The average packet size calculation in igc_ring_itr() yields an EITR 181 * interval field value. That field is quarter microsecond granular (see 182 * IGC_EITR_SHIFT), so an interval of V is 1000000 / (V / 4) interrupts per 183 * second. 184 */ 185 #define IGC_AIM_DIVIDEND (IGC_EITR_DIVIDEND << IGC_EITR_SHIFT) 186 187 /* 188 * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be 189 * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will 190 * also optimize cache line size effect. H/W supports up to cache line size 128. 191 */ 192 #define IGC_DBA_ALIGN 128 193 194 #define IGC_MSIX_BAR 3 195 196 /* Defines for printing debug information */ 197 #define DEBUG_INIT 0 198 #define DEBUG_IOCTL 0 199 #define DEBUG_HW 0 200 201 #define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") 202 #define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) 203 #define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) 204 #define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") 205 #define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) 206 #define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) 207 #define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") 208 #define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) 209 #define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) 210 211 #define IGC_MAX_SCATTER 40 212 #define IGC_VFTA_SIZE 128 213 #define IGC_TSO_SIZE 65535 214 #define IGC_TSO_SEG_SIZE 4096 /* Max dma segment size */ 215 #define IGC_CSUM_OFFLOAD (CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | \ 216 CSUM_IP_SCTP | CSUM_IP6_UDP | CSUM_IP6_TCP | \ 217 CSUM_IP6_SCTP) /* Offload bits in mbuf flag */ 218 219 struct igc_softc; 220 221 struct igc_int_delay_info { 222 struct igc_softc *sc; /* Back-pointer to the softc struct */ 223 int offset; /* Register offset to read/write */ 224 int value; /* Current value in usecs */ 225 }; 226 227 /* 228 * The transmit ring, one per tx queue 229 */ 230 struct tx_ring { 231 struct igc_softc *sc; 232 struct igc_tx_desc *tx_base; 233 uint64_t tx_paddr; 234 qidx_t *tx_rsq; 235 uint8_t me; 236 qidx_t tx_rs_cidx; 237 qidx_t tx_rs_pidx; 238 qidx_t tx_cidx_processed; 239 /* Interrupt resources */ 240 void *tag; 241 struct resource *res; 242 243 /* Soft stats */ 244 unsigned long tx_irq; 245 246 /* 247 * Free running AIM counters. The producer updates these while 248 * encapsulating packets, then publishes both together at the TX 249 * doorbell. The interrupt handler samples only the published value, 250 * so it cannot observe one counter without the other. 251 */ 252 u32 tx_packets; 253 u32 tx_bytes; 254 uint64_t tx_aim_snapshot __aligned(8); 255 u32 tx_packets_last; 256 u32 tx_bytes_last; 257 258 /* Saved csum offloading context information */ 259 int csum_flags; 260 int csum_lhlen; 261 int csum_iphlen; 262 263 int csum_thlen; 264 int csum_mss; 265 int csum_pktlen; 266 267 uint32_t csum_txd_upper; 268 uint32_t csum_txd_lower; /* last field */ 269 }; 270 271 static __inline void 272 igc_aim_publish(struct tx_ring *txr) 273 { 274 uint64_t snapshot; 275 276 snapshot = ((uint64_t)txr->tx_bytes << 32) | txr->tx_packets; 277 atomic_store_rel_64(&txr->tx_aim_snapshot, snapshot); 278 } 279 280 /* 281 * The Receive ring, one per rx queue 282 */ 283 struct rx_ring { 284 struct igc_softc *sc; 285 struct igc_rx_queue *que; 286 u32 me; 287 u32 payload; 288 union igc_rx_desc_extended *rx_base; 289 uint64_t rx_paddr; 290 291 /* Interrupt resources */ 292 void *tag; 293 struct resource *res; 294 295 /* Soft stats */ 296 unsigned long rx_irq; 297 unsigned long rx_discarded; 298 299 /* 300 * Free running AIM counters. RX publishes both together when iflib 301 * returns descriptors to hardware. The interrupt handler samples only 302 * the published value, so watchdog-driven RX processing cannot expose 303 * one counter without the other. 304 */ 305 u32 rx_packets; 306 u32 rx_bytes; 307 uint64_t rx_aim_snapshot __aligned(8); 308 u32 rx_packets_last; 309 u32 rx_bytes_last; 310 }; 311 312 static __inline void 313 igc_aim_publish_rx(struct rx_ring *rxr) 314 { 315 uint64_t snapshot; 316 317 snapshot = ((uint64_t)rxr->rx_bytes << 32) | rxr->rx_packets; 318 atomic_store_rel_64(&rxr->rx_aim_snapshot, snapshot); 319 } 320 321 struct igc_tx_queue { 322 struct igc_softc *sc; 323 u32 msix; 324 u32 eims; /* This queue's EIMS bit */ 325 u32 me; 326 struct tx_ring txr; 327 }; 328 329 struct igc_rx_queue { 330 struct igc_softc *sc; 331 u32 me; 332 u32 msix; 333 u32 eims; 334 u32 eitr_setting; 335 struct rx_ring rxr; 336 u64 irqs; 337 struct if_irq que_irq; 338 }; 339 340 /* Our softc structure */ 341 struct igc_softc { 342 if_t ifp; 343 struct igc_hw hw; 344 345 if_softc_ctx_t shared; 346 if_ctx_t ctx; 347 #define tx_num_queues shared->isc_ntxqsets 348 #define rx_num_queues shared->isc_nrxqsets 349 #define intr_type shared->isc_intr 350 /* FreeBSD operating-system-specific structures. */ 351 struct igc_osdep osdep; 352 device_t dev; 353 struct cdev *led_dev; 354 355 struct igc_tx_queue *tx_queues; 356 struct igc_rx_queue *rx_queues; 357 struct if_irq irq; 358 359 struct resource *memory; 360 struct resource *flash; 361 struct resource *ioport; 362 363 struct resource *res; 364 void *tag; 365 u32 linkvec; 366 u32 ivars; 367 368 struct ifmedia *media; 369 int msix; 370 int if_flags; 371 int igc_insert_vlan_header; 372 u32 ims; 373 374 u32 flags; 375 /* Task for FAST handling */ 376 struct grouptask link_task; 377 378 u32 txd_cmd; 379 380 u32 rx_mbuf_sz; 381 382 int enable_aim; 383 384 /* Management and WOL features */ 385 u32 wol; 386 387 /* Multicast array memory */ 388 u8 *mta; 389 390 /* Info about the interface */ 391 u16 link_active; 392 u16 fc; 393 u16 link_speed; 394 u16 link_duplex; 395 u32 smartspeed; 396 u32 dmac; 397 u32 pba; 398 int link_mask; 399 400 u64 que_mask; 401 402 struct igc_fw_version fw_ver; 403 404 struct igc_int_delay_info tx_int_delay; 405 struct igc_int_delay_info tx_abs_int_delay; 406 struct igc_int_delay_info rx_int_delay; 407 struct igc_int_delay_info rx_abs_int_delay; 408 struct igc_int_delay_info tx_itr; 409 410 /* Misc stats maintained by the driver */ 411 unsigned long dropped_pkts; 412 unsigned long link_irq; 413 unsigned long rx_overruns; 414 unsigned long watchdog_events; 415 416 struct igc_hw_stats stats; 417 u16 vf_ifp; 418 }; 419 420 void igc_dump_rs(struct igc_softc *); 421 422 #define IGC_RSSRK_SIZE 4 423 #define IGC_RSSRK_VAL(key, i) (key[(i) * IGC_RSSRK_SIZE] | \ 424 key[(i) * IGC_RSSRK_SIZE + 1] << 8 | \ 425 key[(i) * IGC_RSSRK_SIZE + 2] << 16 | \ 426 key[(i) * IGC_RSSRK_SIZE + 3] << 24) 427 #endif /* _IGC_H_DEFINED_ */ 428