1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 34 #ifndef _IXL_H_ 35 #define _IXL_H_ 36 37 #include "opt_inet.h" 38 #include "opt_inet6.h" 39 #include "opt_rss.h" 40 #include "opt_ixl.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf_ring.h> 45 #include <sys/mbuf.h> 46 #include <sys/protosw.h> 47 #include <sys/socket.h> 48 #include <sys/malloc.h> 49 #include <sys/kernel.h> 50 #include <sys/module.h> 51 #include <sys/sockio.h> 52 #include <sys/eventhandler.h> 53 #include <sys/syslog.h> 54 #include <sys/priv.h> 55 #include <sys/bitstring.h> 56 57 #include <net/if.h> 58 #include <net/if_var.h> 59 #include <net/if_arp.h> 60 #include <net/bpf.h> 61 #include <net/ethernet.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/iflib.h> 65 66 #include <net/bpf.h> 67 #include <net/if_types.h> 68 #include <net/if_vlan_var.h> 69 70 #include <netinet/in_systm.h> 71 #include <netinet/in.h> 72 #include <netinet/if_ether.h> 73 #include <netinet/ip.h> 74 #include <netinet/ip6.h> 75 #include <netinet/tcp.h> 76 #include <netinet/tcp_lro.h> 77 #include <netinet/udp.h> 78 #include <netinet/sctp.h> 79 80 #include <machine/in_cksum.h> 81 82 #include <sys/bus.h> 83 #include <machine/bus.h> 84 #include <sys/rman.h> 85 #include <machine/resource.h> 86 #include <vm/vm.h> 87 #include <vm/pmap.h> 88 #include <machine/clock.h> 89 #include <dev/pci/pcivar.h> 90 #include <dev/pci/pcireg.h> 91 #include <sys/proc.h> 92 #include <sys/sysctl.h> 93 #include <sys/endian.h> 94 #include <sys/taskqueue.h> 95 #include <sys/pcpu.h> 96 #include <sys/smp.h> 97 #include <sys/sbuf.h> 98 #include <machine/smp.h> 99 #include <machine/stdarg.h> 100 101 #ifdef RSS 102 #include <net/rss_config.h> 103 #include <netinet/in_rss.h> 104 #endif 105 106 #include "ifdi_if.h" 107 #include "i40e_type.h" 108 #include "i40e_prototype.h" 109 #include "ixl_debug.h" 110 111 #define PVIDV(vendor, devid, name) \ 112 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING) 113 114 /* Tunables */ 115 116 /* 117 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the 118 * number of tx/rx descriptors allocated by the driver. Increasing this 119 * value allows the driver to queue more operations. 120 * 121 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. 122 * The driver currently always uses 32 byte Rx descriptors. 123 */ 124 #define IXL_DEFAULT_RING 1024 125 #define IXL_MAX_RING 4096 126 #define IXL_MIN_RING 64 127 #define IXL_RING_INCREMENT 32 128 129 #define IXL_AQ_LEN 256 130 #define IXL_AQ_LEN_MAX 1024 131 132 /* Alignment for rings */ 133 #define DBA_ALIGN 128 134 135 #define MAX_MULTICAST_ADDR 128 136 137 #define IXL_MSIX_BAR 3 138 #define IXL_ADM_LIMIT 2 139 #define IXL_TSO_SIZE ((255*1024)-1) 140 #define IXL_TX_BUF_SZ ((u32) 1514) 141 #define IXL_AQ_BUF_SZ ((u32) 4096) 142 #define IXL_RX_ITR 0 143 #define IXL_TX_ITR 1 144 #define IXL_ITR_NONE 3 145 #define IXL_QUEUE_EOL 0x7FF 146 #define IXL_MIN_FRAME 17 147 #define IXL_MAX_FRAME 9728 148 #define IXL_MAX_TX_SEGS 8 149 #define IXL_MAX_RX_SEGS 5 150 #define IXL_MAX_TSO_SEGS 128 151 #define IXL_SPARSE_CHAIN 7 152 #define IXL_MIN_TSO_MSS 64 153 #define IXL_MAX_TSO_MSS 9668 154 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1) 155 156 #define IXL_RSS_KEY_SIZE_REG 13 157 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4) 158 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ 159 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F 160 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF 161 162 #define IXL_VF_MAX_BUFFER 0x3F80 163 #define IXL_VF_MAX_HDR_BUFFER 0x840 164 #define IXL_VF_MAX_FRAME 0x3FFF 165 166 #define IXL_NVM_VERSION_LO_SHIFT 0 167 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) 168 #define IXL_NVM_VERSION_HI_SHIFT 12 169 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) 170 171 /* 172 * Interrupt Moderation parameters 173 * Multiply ITR values by 2 for real ITR value 174 */ 175 #define IXL_MAX_ITR 0x0FF0 176 #define IXL_ITR_100K 0x0005 177 #define IXL_ITR_20K 0x0019 178 #define IXL_ITR_8K 0x003E 179 #define IXL_ITR_4K 0x007A 180 #define IXL_ITR_1K 0x01F4 181 #define IXL_ITR_DYNAMIC 0x8000 182 #define IXL_LOW_LATENCY 0 183 #define IXL_AVE_LATENCY 1 184 #define IXL_BULK_LATENCY 2 185 186 /* MacVlan Flags */ 187 #define IXL_FILTER_VLAN (u16)(1 << 0) 188 #define IXL_FILTER_MC (u16)(1 << 1) 189 190 /* used in the vlan field of the filter when not a vlan */ 191 #define IXL_VLAN_ANY -1 192 193 /* Maximum number of MAC/VLAN filters supported by HW */ 194 #define IXL_MAX_VLAN_FILTERS 255 195 196 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) 197 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) 198 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) 199 200 /* Misc flags for ixl_vsi.flags */ 201 #define IXL_FLAGS_KEEP_TSO4 (1 << 0) 202 #define IXL_FLAGS_KEEP_TSO6 (1 << 1) 203 #define IXL_FLAGS_USES_MSIX (1 << 2) 204 #define IXL_FLAGS_IS_VF (1 << 3) 205 206 #define IXL_VSI_IS_PF(v) ((v->flags & IXL_FLAGS_IS_VF) == 0) 207 #define IXL_VSI_IS_VF(v) ((v->flags & IXL_FLAGS_IS_VF) != 0) 208 209 #define IXL_VF_RESET_TIMEOUT 100 210 211 #define IXL_VSI_DATA_PORT 0x01 212 213 #define IAVF_MAX_QUEUES 16 214 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) 215 216 #define IXL_RX_CTX_BASE_UNITS 128 217 #define IXL_TX_CTX_BASE_UNITS 128 218 219 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA 220 221 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 222 223 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) 224 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) 225 226 #define IXL_MAX_ITR_IDX 3 227 228 #define IXL_END_OF_INTR_LNKLST 0x7FF 229 230 #define IXL_DEFAULT_RSS_HENA_BASE (\ 231 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 232 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 233 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 234 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 235 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 236 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 237 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 238 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 239 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 240 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 241 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 242 243 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE 244 245 #define IXL_DEFAULT_RSS_HENA_X722 (\ 246 IXL_DEFAULT_RSS_HENA_BASE | \ 247 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 248 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 249 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 250 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 251 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 252 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 253 254 #define IXL_CAPS \ 255 (IFCAP_TSO4 | IFCAP_TSO6 | \ 256 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \ 257 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \ 258 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \ 259 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ 260 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO) 261 262 #define IXL_CSUM_TCP \ 263 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP) 264 #define IXL_CSUM_UDP \ 265 (CSUM_IP_UDP|CSUM_IP6_UDP) 266 #define IXL_CSUM_SCTP \ 267 (CSUM_IP_SCTP|CSUM_IP6_SCTP) 268 #define IXL_CSUM_IPV4 \ 269 (CSUM_IP|CSUM_IP_TSO) 270 271 /* Pre-11 counter(9) compatibility */ 272 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) 273 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) 274 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) 275 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) 276 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ 277 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) 278 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) 279 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) 280 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) 281 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) 282 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) 283 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 284 285 /* For stats sysctl naming */ 286 #define IXL_QUEUE_NAME_LEN 32 287 288 #define IXL_PF_MAX_LINK_POLL SBT_1S * 5 289 290 MALLOC_DECLARE(M_IXL); 291 292 #define IXL_DEV_ERR(_dev, _format, ...) \ 293 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__) 294 295 /* 296 ***************************************************************************** 297 * vendor_info_array 298 * 299 * This array contains the list of Subvendor/Subdevice IDs on which the driver 300 * should load. 301 * 302 ***************************************************************************** 303 */ 304 typedef struct _ixl_vendor_info_t { 305 unsigned int vendor_id; 306 unsigned int device_id; 307 unsigned int subvendor_id; 308 unsigned int subdevice_id; 309 unsigned int index; 310 } ixl_vendor_info_t; 311 312 /* 313 ** This struct has multiple uses, multicast 314 ** addresses, vlans, and mac filters all use it. 315 */ 316 struct ixl_mac_filter { 317 LIST_ENTRY(ixl_mac_filter) ftle; 318 u8 macaddr[ETHER_ADDR_LEN]; 319 s16 vlan; 320 u16 flags; 321 }; 322 323 /* 324 * The Transmit ring control struct 325 */ 326 struct tx_ring { 327 struct ixl_tx_queue *que; 328 u32 tail; 329 struct i40e_tx_desc *tx_base; 330 u64 tx_paddr; 331 u32 latency; 332 u32 packets; 333 u32 me; 334 /* 335 * For reporting completed packet status 336 * in descriptor writeback mode 337 */ 338 qidx_t *tx_rsq; 339 qidx_t tx_rs_cidx; 340 qidx_t tx_rs_pidx; 341 qidx_t tx_cidx_processed; 342 343 /* Used for Dynamic ITR calculation */ 344 u32 itr; 345 u32 bytes; 346 347 /* Soft Stats */ 348 u64 tx_bytes; 349 u64 tx_packets; 350 u64 mss_too_small; 351 }; 352 353 354 /* 355 * The Receive ring control struct 356 */ 357 struct rx_ring { 358 struct ixl_rx_queue *que; 359 union i40e_rx_desc *rx_base; 360 uint64_t rx_paddr; 361 bool discard; 362 u32 itr; 363 u32 latency; 364 u32 mbuf_sz; 365 u32 tail; 366 u32 me; 367 368 /* Used for Dynamic ITR calculation */ 369 u32 packets; 370 u32 bytes; 371 372 /* Soft stats */ 373 u64 rx_packets; 374 u64 rx_bytes; 375 u64 desc_errs; 376 u64 csum_errs; 377 }; 378 379 /* 380 ** Driver queue structs 381 */ 382 struct ixl_tx_queue { 383 struct ixl_vsi *vsi; 384 struct tx_ring txr; 385 struct if_irq que_irq; 386 u32 msix; 387 /* Stats */ 388 u64 irqs; 389 u64 tso; 390 }; 391 392 struct ixl_rx_queue { 393 struct ixl_vsi *vsi; 394 struct rx_ring rxr; 395 struct if_irq que_irq; 396 u32 msix; /* This queue's MSIX vector */ 397 /* Stats */ 398 u64 irqs; 399 }; 400 401 /* 402 ** Virtual Station Interface 403 */ 404 LIST_HEAD(ixl_ftl_head, ixl_mac_filter); 405 struct ixl_vsi { 406 if_ctx_t ctx; 407 if_softc_ctx_t shared; 408 if_t ifp; 409 device_t dev; 410 struct i40e_hw *hw; 411 struct ifmedia *media; 412 413 int num_rx_queues; 414 int num_tx_queues; 415 416 void *back; 417 enum i40e_vsi_type type; 418 int id; 419 u32 rx_itr_setting; 420 u32 tx_itr_setting; 421 bool enable_head_writeback; 422 423 u16 vsi_num; 424 bool link_active; 425 u16 seid; 426 u16 uplink_seid; 427 u16 downlink_seid; 428 429 struct ixl_tx_queue *tx_queues; /* TX queue array */ 430 struct ixl_rx_queue *rx_queues; /* RX queue array */ 431 struct if_irq irq; 432 u32 link_speed; 433 434 /* MAC/VLAN Filter list */ 435 struct ixl_ftl_head ftl; 436 u16 num_macs; 437 u64 num_hw_filters; 438 439 /* Contains readylist & stat counter id */ 440 struct i40e_aqc_vsi_properties_data info; 441 442 #define IXL_VLANS_MAP_LEN EVL_VLID_MASK + 1 443 bitstr_t bit_decl(vlans_map, IXL_VLANS_MAP_LEN); 444 u16 num_vlans; 445 446 /* Per-VSI stats from hardware */ 447 struct i40e_eth_stats eth_stats; 448 struct i40e_eth_stats eth_stats_offsets; 449 bool stat_offsets_loaded; 450 /* VSI stat counters */ 451 u64 ipackets; 452 u64 ierrors; 453 u64 opackets; 454 u64 oerrors; 455 u64 ibytes; 456 u64 obytes; 457 u64 imcasts; 458 u64 omcasts; 459 u64 iqdrops; 460 u64 oqdrops; 461 u64 noproto; 462 463 /* Misc. */ 464 u64 flags; 465 /* Stats sysctls for this VSI */ 466 struct sysctl_oid *vsi_node; 467 struct sysctl_ctx_list sysctl_ctx; 468 }; 469 470 struct ixl_add_maddr_arg { 471 struct ixl_ftl_head to_add; 472 struct ixl_vsi *vsi; 473 }; 474 475 /* 476 ** Compare two ethernet addresses 477 */ 478 static inline bool 479 ixl_ether_is_equal(const u8 *ea1, const u8 *ea2) 480 { 481 return (bcmp(ea1, ea2, ETHER_ADDR_LEN) == 0); 482 } 483 484 /* 485 * Return next largest power of 2, unsigned 486 * 487 * Public domain, from Bit Twiddling Hacks 488 */ 489 static inline u32 490 next_power_of_two(u32 n) 491 { 492 n--; 493 n |= n >> 1; 494 n |= n >> 2; 495 n |= n >> 4; 496 n |= n >> 8; 497 n |= n >> 16; 498 n++; 499 500 /* Next power of two > 0 is 1 */ 501 n += (n == 0); 502 503 return (n); 504 } 505 506 /* 507 * Info for stats sysctls 508 */ 509 struct ixl_sysctl_info { 510 u64 *stat; 511 char *name; 512 char *description; 513 }; 514 515 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN]; 516 517 /* Common function prototypes between PF/VF driver */ 518 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...); 519 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que); 520 void ixl_get_default_rss_key(u32 *); 521 const char * i40e_vc_stat_str(struct i40e_hw *hw, 522 enum virtchnl_status_code stat_err); 523 void ixl_init_tx_rsqs(struct ixl_vsi *vsi); 524 void ixl_init_tx_cidx(struct ixl_vsi *vsi); 525 u64 ixl_max_vc_speed_to_value(u8 link_speeds); 526 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi, 527 struct sysctl_ctx_list *ctx, const char *sysctl_name); 528 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 529 struct sysctl_oid_list *child, 530 struct i40e_eth_stats *eth_stats); 531 void ixl_vsi_add_queues_stats(struct ixl_vsi *vsi, 532 struct sysctl_ctx_list *ctx); 533 #endif /* _IXL_H_ */ 534