1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 36 #ifndef _IXL_H_ 37 #define _IXL_H_ 38 39 #include "opt_inet.h" 40 #include "opt_inet6.h" 41 #include "opt_rss.h" 42 #include "opt_ixl.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/buf_ring.h> 47 #include <sys/mbuf.h> 48 #include <sys/protosw.h> 49 #include <sys/socket.h> 50 #include <sys/malloc.h> 51 #include <sys/kernel.h> 52 #include <sys/module.h> 53 #include <sys/sockio.h> 54 #include <sys/eventhandler.h> 55 #include <sys/syslog.h> 56 57 #include <net/if.h> 58 #include <net/if_var.h> 59 #include <net/if_arp.h> 60 #include <net/bpf.h> 61 #include <net/ethernet.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/iflib.h> 65 66 #include <net/bpf.h> 67 #include <net/if_types.h> 68 #include <net/if_vlan_var.h> 69 70 #include <netinet/in_systm.h> 71 #include <netinet/in.h> 72 #include <netinet/if_ether.h> 73 #include <netinet/ip.h> 74 #include <netinet/ip6.h> 75 #include <netinet/tcp.h> 76 #include <netinet/tcp_lro.h> 77 #include <netinet/udp.h> 78 #include <netinet/sctp.h> 79 80 #include <machine/in_cksum.h> 81 82 #include <sys/bus.h> 83 #include <machine/bus.h> 84 #include <sys/rman.h> 85 #include <machine/resource.h> 86 #include <vm/vm.h> 87 #include <vm/pmap.h> 88 #include <machine/clock.h> 89 #include <dev/pci/pcivar.h> 90 #include <dev/pci/pcireg.h> 91 #include <sys/proc.h> 92 #include <sys/sysctl.h> 93 #include <sys/endian.h> 94 #include <sys/taskqueue.h> 95 #include <sys/pcpu.h> 96 #include <sys/smp.h> 97 #include <sys/sbuf.h> 98 #include <machine/smp.h> 99 #include <machine/stdarg.h> 100 101 #ifdef RSS 102 #include <net/rss_config.h> 103 #include <netinet/in_rss.h> 104 #endif 105 106 #include "ifdi_if.h" 107 #include "i40e_type.h" 108 #include "i40e_prototype.h" 109 #include "ixl_debug.h" 110 111 #define PVIDV(vendor, devid, name) \ 112 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING) 113 114 /* Tunables */ 115 116 /* 117 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the 118 * number of tx/rx descriptors allocated by the driver. Increasing this 119 * value allows the driver to queue more operations. 120 * 121 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. 122 * The driver currently always uses 32 byte Rx descriptors. 123 */ 124 #define IXL_DEFAULT_RING 1024 125 #define IXL_MAX_RING 4096 126 #define IXL_MIN_RING 64 127 #define IXL_RING_INCREMENT 32 128 129 #define IXL_AQ_LEN 256 130 #define IXL_AQ_LEN_MAX 1024 131 132 /* Alignment for rings */ 133 #define DBA_ALIGN 128 134 135 #define MAX_MULTICAST_ADDR 128 136 137 #define IXL_MSIX_BAR 3 138 #define IXL_ADM_LIMIT 2 139 // TODO: Find out which TSO_SIZE to use 140 //#define IXL_TSO_SIZE 65535 141 #define IXL_TSO_SIZE ((255*1024)-1) 142 #define IXL_TX_BUF_SZ ((u32) 1514) 143 #define IXL_AQ_BUF_SZ ((u32) 4096) 144 #define IXL_RX_ITR 0 145 #define IXL_TX_ITR 1 146 #define IXL_ITR_NONE 3 147 #define IXL_QUEUE_EOL 0x7FF 148 #define IXL_MAX_FRAME 9728 149 #define IXL_MAX_TX_SEGS 8 150 #define IXL_MAX_RX_SEGS 5 151 #define IXL_MAX_TSO_SEGS 128 152 #define IXL_SPARSE_CHAIN 7 153 #define IXL_MIN_TSO_MSS 64 154 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1) 155 156 #define IXL_RSS_KEY_SIZE_REG 13 157 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4) 158 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ 159 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F 160 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF 161 162 #define IXL_VF_MAX_BUFFER 0x3F80 163 #define IXL_VF_MAX_HDR_BUFFER 0x840 164 #define IXL_VF_MAX_FRAME 0x3FFF 165 166 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */ 167 #define IXL_MAX_FILTERS 256 168 169 #define IXL_NVM_VERSION_LO_SHIFT 0 170 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) 171 #define IXL_NVM_VERSION_HI_SHIFT 12 172 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) 173 174 /* 175 * Interrupt Moderation parameters 176 * Multiply ITR values by 2 for real ITR value 177 */ 178 #define IXL_MAX_ITR 0x0FF0 179 #define IXL_ITR_100K 0x0005 180 #define IXL_ITR_20K 0x0019 181 #define IXL_ITR_8K 0x003E 182 #define IXL_ITR_4K 0x007A 183 #define IXL_ITR_1K 0x01F4 184 #define IXL_ITR_DYNAMIC 0x8000 185 #define IXL_LOW_LATENCY 0 186 #define IXL_AVE_LATENCY 1 187 #define IXL_BULK_LATENCY 2 188 189 /* MacVlan Flags */ 190 #define IXL_FILTER_USED (u16)(1 << 0) 191 #define IXL_FILTER_VLAN (u16)(1 << 1) 192 #define IXL_FILTER_ADD (u16)(1 << 2) 193 #define IXL_FILTER_DEL (u16)(1 << 3) 194 #define IXL_FILTER_MC (u16)(1 << 4) 195 196 /* used in the vlan field of the filter when not a vlan */ 197 #define IXL_VLAN_ANY -1 198 199 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) 200 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) 201 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) 202 203 #define IXL_VF_RESET_TIMEOUT 100 204 205 #define IXL_VSI_DATA_PORT 0x01 206 207 #define IXLV_MAX_QUEUES 16 208 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) 209 210 #define IXL_RX_CTX_BASE_UNITS 128 211 #define IXL_TX_CTX_BASE_UNITS 128 212 213 #if 0 214 #define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ 215 I40E_VPINT_LNKLSTN(((vector) - 1) + \ 216 (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) 217 218 #define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ 219 I40E_VFINT_DYN_CTLN(((vector) - 1) + \ 220 (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) 221 #endif 222 223 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA 224 225 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 226 227 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) 228 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) 229 230 #define IXL_MAX_ITR_IDX 3 231 232 #define IXL_END_OF_INTR_LNKLST 0x7FF 233 234 #define IXL_DEFAULT_RSS_HENA_BASE (\ 235 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 236 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 237 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 238 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 239 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 240 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 241 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 242 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 243 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 244 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 245 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 246 247 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE 248 249 #define IXL_DEFAULT_RSS_HENA_X722 (\ 250 IXL_DEFAULT_RSS_HENA_BASE | \ 251 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 252 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 253 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 254 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 255 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 256 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 257 258 #define IXL_CAPS \ 259 (IFCAP_TSO4 | IFCAP_TSO6 | \ 260 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \ 261 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \ 262 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \ 263 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | \ 264 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO) 265 266 #define IXL_CSUM_TCP \ 267 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP) 268 #define IXL_CSUM_UDP \ 269 (CSUM_IP_UDP|CSUM_IP6_UDP) 270 #define IXL_CSUM_SCTP \ 271 (CSUM_IP_SCTP|CSUM_IP6_SCTP) 272 273 /* Pre-11 counter(9) compatibility */ 274 #if __FreeBSD_version >= 1100036 275 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) 276 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) 277 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) 278 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) 279 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ 280 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) 281 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) 282 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) 283 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) 284 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) 285 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) 286 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 287 #else 288 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) 289 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) 290 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) 291 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) 292 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) 293 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) 294 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) 295 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) 296 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) 297 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) 298 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) 299 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 300 #endif 301 302 #define IXL_DEV_ERR(_dev, _format, ...) \ 303 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__) 304 305 /* 306 ***************************************************************************** 307 * vendor_info_array 308 * 309 * This array contains the list of Subvendor/Subdevice IDs on which the driver 310 * should load. 311 * 312 ***************************************************************************** 313 */ 314 typedef struct _ixl_vendor_info_t { 315 unsigned int vendor_id; 316 unsigned int device_id; 317 unsigned int subvendor_id; 318 unsigned int subdevice_id; 319 unsigned int index; 320 } ixl_vendor_info_t; 321 322 /* 323 ** This struct has multiple uses, multicast 324 ** addresses, vlans, and mac filters all use it. 325 */ 326 struct ixl_mac_filter { 327 SLIST_ENTRY(ixl_mac_filter) next; 328 u8 macaddr[ETHER_ADDR_LEN]; 329 s16 vlan; 330 u16 flags; 331 }; 332 333 /* 334 * The Transmit ring control struct 335 */ 336 struct tx_ring { 337 struct ixl_tx_queue *que; 338 u32 tail; 339 struct i40e_tx_desc *tx_base; 340 u64 tx_paddr; 341 u32 latency; 342 u32 packets; 343 u32 me; 344 /* 345 * For reporting completed packet status 346 * in descriptor writeback mode 347 */ 348 qidx_t *tx_rsq; 349 qidx_t tx_rs_cidx; 350 qidx_t tx_rs_pidx; 351 qidx_t tx_cidx_processed; 352 353 /* Used for Dynamic ITR calculation */ 354 u32 itr; 355 u32 bytes; 356 357 /* Soft Stats */ 358 u64 tx_bytes; 359 u64 tx_packets; 360 u64 mss_too_small; 361 }; 362 363 364 /* 365 * The Receive ring control struct 366 */ 367 struct rx_ring { 368 struct ixl_rx_queue *que; 369 union i40e_rx_desc *rx_base; 370 uint64_t rx_paddr; 371 bool discard; 372 u32 itr; 373 u32 latency; 374 u32 mbuf_sz; 375 u32 tail; 376 u32 me; 377 378 /* Used for Dynamic ITR calculation */ 379 u32 packets; 380 u32 bytes; 381 382 /* Soft stats */ 383 u64 rx_packets; 384 u64 rx_bytes; 385 u64 desc_errs; 386 }; 387 388 /* 389 ** Driver queue structs 390 */ 391 struct ixl_tx_queue { 392 struct ixl_vsi *vsi; 393 struct tx_ring txr; 394 struct if_irq que_irq; 395 u32 msix; 396 /* Stats */ 397 u64 irqs; 398 u64 tso; 399 }; 400 401 struct ixl_rx_queue { 402 struct ixl_vsi *vsi; 403 struct rx_ring rxr; 404 struct if_irq que_irq; 405 u32 msix; /* This queue's MSIX vector */ 406 /* Stats */ 407 u64 irqs; 408 }; 409 410 /* 411 ** Virtual Station Interface 412 */ 413 SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); 414 struct ixl_vsi { 415 if_ctx_t ctx; 416 if_softc_ctx_t shared; 417 struct ifnet *ifp; 418 //device_t dev; 419 struct i40e_hw *hw; 420 struct ifmedia *media; 421 #define num_rx_queues shared->isc_nrxqsets 422 #define num_tx_queues shared->isc_ntxqsets 423 424 void *back; 425 enum i40e_vsi_type type; 426 // TODO: Remove? 427 u64 que_mask; 428 int id; 429 u32 rx_itr_setting; 430 u32 tx_itr_setting; 431 bool enable_head_writeback; 432 433 u16 vsi_num; 434 bool link_active; 435 u16 seid; 436 u16 uplink_seid; 437 u16 downlink_seid; 438 439 struct ixl_tx_queue *tx_queues; /* TX queue array */ 440 struct ixl_rx_queue *rx_queues; /* RX queue array */ 441 struct if_irq irq; 442 u32 link_speed; 443 444 /* MAC/VLAN Filter list */ 445 struct ixl_ftl_head ftl; 446 u16 num_macs; 447 448 /* Contains readylist & stat counter id */ 449 struct i40e_aqc_vsi_properties_data info; 450 451 u16 num_vlans; 452 453 /* Per-VSI stats from hardware */ 454 struct i40e_eth_stats eth_stats; 455 struct i40e_eth_stats eth_stats_offsets; 456 bool stat_offsets_loaded; 457 /* VSI stat counters */ 458 u64 ipackets; 459 u64 ierrors; 460 u64 opackets; 461 u64 oerrors; 462 u64 ibytes; 463 u64 obytes; 464 u64 imcasts; 465 u64 omcasts; 466 u64 iqdrops; 467 u64 oqdrops; 468 u64 noproto; 469 470 /* Driver statistics */ 471 u64 hw_filters_del; 472 u64 hw_filters_add; 473 474 /* Misc. */ 475 u64 flags; 476 /* Stats sysctls for this VSI */ 477 struct sysctl_oid *vsi_node; 478 }; 479 480 /* 481 ** Creates new filter with given MAC address and VLAN ID 482 */ 483 static inline struct ixl_mac_filter * 484 ixl_new_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan) 485 { 486 struct ixl_mac_filter *f; 487 488 /* create a new empty filter */ 489 f = malloc(sizeof(struct ixl_mac_filter), 490 M_DEVBUF, M_NOWAIT | M_ZERO); 491 if (f) { 492 SLIST_INSERT_HEAD(&vsi->ftl, f, next); 493 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 494 f->vlan = vlan; 495 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 496 } 497 498 return (f); 499 } 500 501 /* 502 ** Compare two ethernet addresses 503 */ 504 static inline bool 505 cmp_etheraddr(const u8 *ea1, const u8 *ea2) 506 { 507 return (bcmp(ea1, ea2, 6) == 0); 508 } 509 510 /* 511 * Return next largest power of 2, unsigned 512 * 513 * Public domain, from Bit Twiddling Hacks 514 */ 515 static inline u32 516 next_power_of_two(u32 n) 517 { 518 n--; 519 n |= n >> 1; 520 n |= n >> 2; 521 n |= n >> 4; 522 n |= n >> 8; 523 n |= n >> 16; 524 n++; 525 526 /* Next power of two > 0 is 1 */ 527 n += (n == 0); 528 529 return (n); 530 } 531 532 /* 533 * Info for stats sysctls 534 */ 535 struct ixl_sysctl_info { 536 u64 *stat; 537 char *name; 538 char *description; 539 }; 540 541 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN]; 542 543 /* Common function prototypes between PF/VF driver */ 544 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que); 545 void ixl_set_queue_rx_itr(struct ixl_rx_queue *que); 546 void ixl_get_default_rss_key(u32 *); 547 const char * i40e_vc_stat_str(struct i40e_hw *hw, 548 enum virtchnl_status_code stat_err); 549 u64 ixl_max_aq_speed_to_value(u8); 550 #endif /* _IXL_H_ */ 551