1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifndef _IXL_H_ 36 #define _IXL_H_ 37 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_rss.h" 41 #include "opt_ixl.h" 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/buf_ring.h> 46 #include <sys/mbuf.h> 47 #include <sys/protosw.h> 48 #include <sys/socket.h> 49 #include <sys/malloc.h> 50 #include <sys/kernel.h> 51 #include <sys/module.h> 52 #include <sys/sockio.h> 53 #include <sys/eventhandler.h> 54 #include <sys/syslog.h> 55 56 #include <net/if.h> 57 #include <net/if_var.h> 58 #include <net/if_arp.h> 59 #include <net/bpf.h> 60 #include <net/ethernet.h> 61 #include <net/if_dl.h> 62 #include <net/if_media.h> 63 #include <net/iflib.h> 64 65 #include <net/bpf.h> 66 #include <net/if_types.h> 67 #include <net/if_vlan_var.h> 68 69 #include <netinet/in_systm.h> 70 #include <netinet/in.h> 71 #include <netinet/if_ether.h> 72 #include <netinet/ip.h> 73 #include <netinet/ip6.h> 74 #include <netinet/tcp.h> 75 #include <netinet/tcp_lro.h> 76 #include <netinet/udp.h> 77 #include <netinet/sctp.h> 78 79 #include <machine/in_cksum.h> 80 81 #include <sys/bus.h> 82 #include <machine/bus.h> 83 #include <sys/rman.h> 84 #include <machine/resource.h> 85 #include <vm/vm.h> 86 #include <vm/pmap.h> 87 #include <machine/clock.h> 88 #include <dev/pci/pcivar.h> 89 #include <dev/pci/pcireg.h> 90 #include <sys/proc.h> 91 #include <sys/sysctl.h> 92 #include <sys/endian.h> 93 #include <sys/taskqueue.h> 94 #include <sys/pcpu.h> 95 #include <sys/smp.h> 96 #include <sys/sbuf.h> 97 #include <machine/smp.h> 98 #include <machine/stdarg.h> 99 100 #ifdef RSS 101 #include <net/rss_config.h> 102 #include <netinet/in_rss.h> 103 #endif 104 105 #include "ifdi_if.h" 106 #include "i40e_type.h" 107 #include "i40e_prototype.h" 108 #include "ixl_debug.h" 109 110 #define PVIDV(vendor, devid, name) \ 111 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING) 112 113 /* Tunables */ 114 115 /* 116 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the 117 * number of tx/rx descriptors allocated by the driver. Increasing this 118 * value allows the driver to queue more operations. 119 * 120 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. 121 * The driver currently always uses 32 byte Rx descriptors. 122 */ 123 #define IXL_DEFAULT_RING 1024 124 #define IXL_MAX_RING 4096 125 #define IXL_MIN_RING 64 126 #define IXL_RING_INCREMENT 32 127 128 #define IXL_AQ_LEN 256 129 #define IXL_AQ_LEN_MAX 1024 130 131 /* Alignment for rings */ 132 #define DBA_ALIGN 128 133 134 #define MAX_MULTICAST_ADDR 128 135 136 #define IXL_MSIX_BAR 3 137 #define IXL_ADM_LIMIT 2 138 #define IXL_TSO_SIZE ((255*1024)-1) 139 #define IXL_TX_BUF_SZ ((u32) 1514) 140 #define IXL_AQ_BUF_SZ ((u32) 4096) 141 #define IXL_RX_ITR 0 142 #define IXL_TX_ITR 1 143 #define IXL_ITR_NONE 3 144 #define IXL_QUEUE_EOL 0x7FF 145 #define IXL_MIN_FRAME 17 146 #define IXL_MAX_FRAME 9728 147 #define IXL_MAX_TX_SEGS 8 148 #define IXL_MAX_RX_SEGS 5 149 #define IXL_MAX_TSO_SEGS 128 150 #define IXL_SPARSE_CHAIN 7 151 #define IXL_MIN_TSO_MSS 64 152 #define IXL_MAX_TSO_MSS 9668 153 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1) 154 155 #define IXL_RSS_KEY_SIZE_REG 13 156 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4) 157 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ 158 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F 159 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF 160 161 #define IXL_VF_MAX_BUFFER 0x3F80 162 #define IXL_VF_MAX_HDR_BUFFER 0x840 163 #define IXL_VF_MAX_FRAME 0x3FFF 164 165 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */ 166 #define IXL_MAX_FILTERS 256 167 168 #define IXL_NVM_VERSION_LO_SHIFT 0 169 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) 170 #define IXL_NVM_VERSION_HI_SHIFT 12 171 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) 172 173 /* 174 * Interrupt Moderation parameters 175 * Multiply ITR values by 2 for real ITR value 176 */ 177 #define IXL_MAX_ITR 0x0FF0 178 #define IXL_ITR_100K 0x0005 179 #define IXL_ITR_20K 0x0019 180 #define IXL_ITR_8K 0x003E 181 #define IXL_ITR_4K 0x007A 182 #define IXL_ITR_1K 0x01F4 183 #define IXL_ITR_DYNAMIC 0x8000 184 #define IXL_LOW_LATENCY 0 185 #define IXL_AVE_LATENCY 1 186 #define IXL_BULK_LATENCY 2 187 188 /* MacVlan Flags */ 189 #define IXL_FILTER_USED (u16)(1 << 0) 190 #define IXL_FILTER_VLAN (u16)(1 << 1) 191 #define IXL_FILTER_ADD (u16)(1 << 2) 192 #define IXL_FILTER_DEL (u16)(1 << 3) 193 #define IXL_FILTER_MC (u16)(1 << 4) 194 195 /* used in the vlan field of the filter when not a vlan */ 196 #define IXL_VLAN_ANY -1 197 198 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) 199 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) 200 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) 201 202 #define IXL_VF_RESET_TIMEOUT 100 203 204 #define IXL_VSI_DATA_PORT 0x01 205 206 #define IAVF_MAX_QUEUES 16 207 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) 208 209 #define IXL_RX_CTX_BASE_UNITS 128 210 #define IXL_TX_CTX_BASE_UNITS 128 211 212 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA 213 214 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 215 216 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) 217 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) 218 219 #define IXL_MAX_ITR_IDX 3 220 221 #define IXL_END_OF_INTR_LNKLST 0x7FF 222 223 #define IXL_DEFAULT_RSS_HENA_BASE (\ 224 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 225 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 226 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 227 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 228 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 229 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 230 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 231 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 232 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 233 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 234 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 235 236 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE 237 238 #define IXL_DEFAULT_RSS_HENA_X722 (\ 239 IXL_DEFAULT_RSS_HENA_BASE | \ 240 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 241 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 242 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 243 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 244 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 245 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 246 247 #define IXL_CAPS \ 248 (IFCAP_TSO4 | IFCAP_TSO6 | \ 249 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \ 250 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \ 251 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \ 252 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ 253 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO) 254 255 #define IXL_CSUM_TCP \ 256 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP) 257 #define IXL_CSUM_UDP \ 258 (CSUM_IP_UDP|CSUM_IP6_UDP) 259 #define IXL_CSUM_SCTP \ 260 (CSUM_IP_SCTP|CSUM_IP6_SCTP) 261 #define IXL_CSUM_IPV4 \ 262 (CSUM_IP|CSUM_IP_TSO) 263 264 /* Pre-11 counter(9) compatibility */ 265 #if __FreeBSD_version >= 1100036 266 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) 267 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) 268 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) 269 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) 270 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ 271 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) 272 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) 273 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) 274 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) 275 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) 276 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) 277 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 278 #else 279 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) 280 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) 281 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) 282 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) 283 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) 284 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) 285 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) 286 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) 287 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) 288 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) 289 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) 290 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 291 #endif 292 293 /* For stats sysctl naming */ 294 #define QUEUE_NAME_LEN 32 295 296 #define IXL_DEV_ERR(_dev, _format, ...) \ 297 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__) 298 299 /* 300 ***************************************************************************** 301 * vendor_info_array 302 * 303 * This array contains the list of Subvendor/Subdevice IDs on which the driver 304 * should load. 305 * 306 ***************************************************************************** 307 */ 308 typedef struct _ixl_vendor_info_t { 309 unsigned int vendor_id; 310 unsigned int device_id; 311 unsigned int subvendor_id; 312 unsigned int subdevice_id; 313 unsigned int index; 314 } ixl_vendor_info_t; 315 316 /* 317 ** This struct has multiple uses, multicast 318 ** addresses, vlans, and mac filters all use it. 319 */ 320 struct ixl_mac_filter { 321 SLIST_ENTRY(ixl_mac_filter) next; 322 u8 macaddr[ETHER_ADDR_LEN]; 323 s16 vlan; 324 u16 flags; 325 }; 326 327 /* 328 * The Transmit ring control struct 329 */ 330 struct tx_ring { 331 struct ixl_tx_queue *que; 332 u32 tail; 333 struct i40e_tx_desc *tx_base; 334 u64 tx_paddr; 335 u32 latency; 336 u32 packets; 337 u32 me; 338 /* 339 * For reporting completed packet status 340 * in descriptor writeback mode 341 */ 342 qidx_t *tx_rsq; 343 qidx_t tx_rs_cidx; 344 qidx_t tx_rs_pidx; 345 qidx_t tx_cidx_processed; 346 347 /* Used for Dynamic ITR calculation */ 348 u32 itr; 349 u32 bytes; 350 351 /* Soft Stats */ 352 u64 tx_bytes; 353 u64 tx_packets; 354 u64 mss_too_small; 355 }; 356 357 358 /* 359 * The Receive ring control struct 360 */ 361 struct rx_ring { 362 struct ixl_rx_queue *que; 363 union i40e_rx_desc *rx_base; 364 uint64_t rx_paddr; 365 bool discard; 366 u32 itr; 367 u32 latency; 368 u32 mbuf_sz; 369 u32 tail; 370 u32 me; 371 372 /* Used for Dynamic ITR calculation */ 373 u32 packets; 374 u32 bytes; 375 376 /* Soft stats */ 377 u64 rx_packets; 378 u64 rx_bytes; 379 u64 desc_errs; 380 }; 381 382 /* 383 ** Driver queue structs 384 */ 385 struct ixl_tx_queue { 386 struct ixl_vsi *vsi; 387 struct tx_ring txr; 388 struct if_irq que_irq; 389 u32 msix; 390 /* Stats */ 391 u64 irqs; 392 u64 tso; 393 }; 394 395 struct ixl_rx_queue { 396 struct ixl_vsi *vsi; 397 struct rx_ring rxr; 398 struct if_irq que_irq; 399 u32 msix; /* This queue's MSIX vector */ 400 /* Stats */ 401 u64 irqs; 402 }; 403 404 /* 405 ** Virtual Station Interface 406 */ 407 SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); 408 struct ixl_vsi { 409 if_ctx_t ctx; 410 if_softc_ctx_t shared; 411 struct ifnet *ifp; 412 device_t dev; 413 struct i40e_hw *hw; 414 struct ifmedia *media; 415 416 int num_rx_queues; 417 int num_tx_queues; 418 419 void *back; 420 enum i40e_vsi_type type; 421 int id; 422 u32 rx_itr_setting; 423 u32 tx_itr_setting; 424 bool enable_head_writeback; 425 426 u16 vsi_num; 427 bool link_active; 428 u16 seid; 429 u16 uplink_seid; 430 u16 downlink_seid; 431 432 struct ixl_tx_queue *tx_queues; /* TX queue array */ 433 struct ixl_rx_queue *rx_queues; /* RX queue array */ 434 struct if_irq irq; 435 u32 link_speed; 436 437 /* MAC/VLAN Filter list */ 438 struct ixl_ftl_head ftl; 439 u16 num_macs; 440 441 /* Contains readylist & stat counter id */ 442 struct i40e_aqc_vsi_properties_data info; 443 444 u16 num_vlans; 445 446 /* Per-VSI stats from hardware */ 447 struct i40e_eth_stats eth_stats; 448 struct i40e_eth_stats eth_stats_offsets; 449 bool stat_offsets_loaded; 450 /* VSI stat counters */ 451 u64 ipackets; 452 u64 ierrors; 453 u64 opackets; 454 u64 oerrors; 455 u64 ibytes; 456 u64 obytes; 457 u64 imcasts; 458 u64 omcasts; 459 u64 iqdrops; 460 u64 oqdrops; 461 u64 noproto; 462 463 /* Driver statistics */ 464 u64 hw_filters_del; 465 u64 hw_filters_add; 466 467 /* Misc. */ 468 u64 flags; 469 /* Stats sysctls for this VSI */ 470 struct sysctl_oid *vsi_node; 471 }; 472 473 /* 474 ** Creates new filter with given MAC address and VLAN ID 475 */ 476 static inline struct ixl_mac_filter * 477 ixl_new_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan) 478 { 479 struct ixl_mac_filter *f; 480 481 /* create a new empty filter */ 482 f = malloc(sizeof(struct ixl_mac_filter), 483 M_DEVBUF, M_NOWAIT | M_ZERO); 484 if (f) { 485 SLIST_INSERT_HEAD(&vsi->ftl, f, next); 486 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 487 f->vlan = vlan; 488 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 489 } 490 491 return (f); 492 } 493 494 /* 495 ** Compare two ethernet addresses 496 */ 497 static inline bool 498 cmp_etheraddr(const u8 *ea1, const u8 *ea2) 499 { 500 return (bcmp(ea1, ea2, 6) == 0); 501 } 502 503 /* 504 * Return next largest power of 2, unsigned 505 * 506 * Public domain, from Bit Twiddling Hacks 507 */ 508 static inline u32 509 next_power_of_two(u32 n) 510 { 511 n--; 512 n |= n >> 1; 513 n |= n >> 2; 514 n |= n >> 4; 515 n |= n >> 8; 516 n |= n >> 16; 517 n++; 518 519 /* Next power of two > 0 is 1 */ 520 n += (n == 0); 521 522 return (n); 523 } 524 525 /* 526 * Info for stats sysctls 527 */ 528 struct ixl_sysctl_info { 529 u64 *stat; 530 char *name; 531 char *description; 532 }; 533 534 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN]; 535 536 /* Common function prototypes between PF/VF driver */ 537 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...); 538 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que); 539 void ixl_get_default_rss_key(u32 *); 540 const char * i40e_vc_stat_str(struct i40e_hw *hw, 541 enum virtchnl_status_code stat_err); 542 void ixl_init_tx_rsqs(struct ixl_vsi *vsi); 543 void ixl_init_tx_cidx(struct ixl_vsi *vsi); 544 u64 ixl_max_vc_speed_to_value(u8 link_speeds); 545 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi, 546 struct sysctl_ctx_list *ctx, const char *sysctl_name); 547 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 548 struct sysctl_oid_list *child, 549 struct i40e_eth_stats *eth_stats); 550 void ixl_add_queues_sysctls(device_t dev, struct ixl_vsi *vsi); 551 #endif /* _IXL_H_ */ 552