1 /****************************************************************************** 2 3 Copyright (c) 2013-2014, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 36 #ifndef _IXL_H_ 37 #define _IXL_H_ 38 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/buf_ring.h> 43 #include <sys/mbuf.h> 44 #include <sys/protosw.h> 45 #include <sys/socket.h> 46 #include <sys/malloc.h> 47 #include <sys/kernel.h> 48 #include <sys/module.h> 49 #include <sys/sockio.h> 50 #include <sys/eventhandler.h> 51 52 #include <net/if.h> 53 #include <net/if_var.h> 54 #include <net/if_arp.h> 55 #include <net/bpf.h> 56 #include <net/ethernet.h> 57 #include <net/if_dl.h> 58 #include <net/if_media.h> 59 60 #include <net/bpf.h> 61 #include <net/if_types.h> 62 #include <net/if_vlan_var.h> 63 64 #include <netinet/in_systm.h> 65 #include <netinet/in.h> 66 #include <netinet/if_ether.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 #include <netinet/tcp_lro.h> 71 #include <netinet/udp.h> 72 #include <netinet/sctp.h> 73 74 #include <machine/in_cksum.h> 75 76 #include <sys/bus.h> 77 #include <machine/bus.h> 78 #include <sys/rman.h> 79 #include <machine/resource.h> 80 #include <vm/vm.h> 81 #include <vm/pmap.h> 82 #include <machine/clock.h> 83 #include <dev/pci/pcivar.h> 84 #include <dev/pci/pcireg.h> 85 #include <sys/proc.h> 86 #include <sys/sysctl.h> 87 #include <sys/endian.h> 88 #include <sys/taskqueue.h> 89 #include <sys/pcpu.h> 90 #include <sys/smp.h> 91 #include <machine/smp.h> 92 93 #include "i40e_type.h" 94 #include "i40e_prototype.h" 95 96 #ifdef IXL_DEBUG 97 #include <sys/sbuf.h> 98 99 #define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" 100 #define MAC_FORMAT_ARGS(mac_addr) \ 101 (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ 102 (mac_addr)[4], (mac_addr)[5] 103 #define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") 104 105 106 #define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) 107 #define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) 108 #define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) 109 110 /* Defines for printing generic debug information */ 111 #define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) 112 #define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) 113 #define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) 114 115 /* Defines for printing specific debug information */ 116 #define DEBUG_INIT 1 117 #define DEBUG_IOCTL 1 118 #define DEBUG_HW 1 119 120 #define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) 121 #define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) 122 #define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) 123 124 #define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) 125 #define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ 126 if_printf(ifp, S "\n", ##__VA_ARGS__) 127 #define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) 128 129 #define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) 130 131 #else 132 #define DEBUG_INIT 0 133 #define DEBUG_IOCTL 0 134 #define DEBUG_HW 0 135 136 #define DPRINTF(...) 137 #define DDPRINTF(...) 138 #define IDPRINTF(...) 139 140 #define INIT_DEBUGOUT(...) 141 #define INIT_DBG_DEV(...) 142 #define INIT_DBG_IF(...) 143 #define IOCTL_DEBUGOUT(...) 144 #define IOCTL_DBG_IF2(...) 145 #define IOCTL_DBG_IF(...) 146 #define HW_DEBUGOUT(...) 147 #endif 148 149 /* Tunables */ 150 151 /* 152 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the 153 * number of tx/rx descriptors allocated by the driver. Increasing this 154 * value allows the driver to queue more operations. Each descriptor is 16 155 * or 32 bytes (configurable in FVL) 156 */ 157 #define DEFAULT_RING 1024 158 #define PERFORM_RING 2048 159 #define MAX_RING 4096 160 #define MIN_RING 32 161 162 /* 163 ** Default number of entries in Tx queue buf_ring. 164 */ 165 #define DEFAULT_TXBRSZ (4096 * 4096) 166 167 /* Alignment for rings */ 168 #define DBA_ALIGN 128 169 170 /* 171 * This parameter controls the maximum no of times the driver will loop in 172 * the isr. Minimum Value = 1 173 */ 174 #define MAX_LOOP 10 175 176 /* 177 * This is the max watchdog interval, ie. the time that can 178 * pass between any two TX clean operations, such only happening 179 * when the TX hardware is functioning. 180 */ 181 #define IXL_WATCHDOG (10 * hz) 182 183 /* 184 * This parameters control when the driver calls the routine to reclaim 185 * transmit descriptors. 186 */ 187 #define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8) 188 #define IXL_TX_OP_THRESHOLD (que->num_desc / 32) 189 190 /* Flow control constants */ 191 #define IXL_FC_PAUSE 0xFFFF 192 #define IXL_FC_HI 0x20000 193 #define IXL_FC_LO 0x10000 194 195 #define MAX_MULTICAST_ADDR 128 196 197 #define IXL_BAR 3 198 #define IXL_ADM_LIMIT 2 199 #define IXL_TSO_SIZE 65535 200 #define IXL_TX_BUF_SZ ((u32) 1514) 201 #define IXL_AQ_BUF_SZ ((u32) 4096) 202 #define IXL_RX_HDR 128 203 #define IXL_AQ_LEN 256 204 #define IXL_AQ_BUFSZ 4096 205 #define IXL_RX_LIMIT 512 206 #define IXL_RX_ITR 0 207 #define IXL_TX_ITR 1 208 #define IXL_ITR_NONE 3 209 #define IXL_QUEUE_EOL 0x7FF 210 #define IXL_MAX_FRAME 0x2600 211 #define IXL_MAX_TX_SEGS 8 212 #define IXL_MAX_TSO_SEGS 66 213 #define IXL_SPARSE_CHAIN 6 214 #define IXL_QUEUE_HUNG 0x80000000 215 216 /* ERJ: hardware can support ~1.5k filters between all functions */ 217 #define IXL_MAX_FILTERS 256 218 #define IXL_MAX_TX_BUSY 10 219 220 #define IXL_NVM_VERSION_LO_SHIFT 0 221 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) 222 #define IXL_NVM_VERSION_HI_SHIFT 12 223 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) 224 225 226 /* 227 * Interrupt Moderation parameters 228 */ 229 #define IXL_MAX_ITR 0x07FF 230 #define IXL_ITR_100K 0x0005 231 #define IXL_ITR_20K 0x0019 232 #define IXL_ITR_8K 0x003E 233 #define IXL_ITR_4K 0x007A 234 #define IXL_ITR_DYNAMIC 0x8000 235 #define IXL_LOW_LATENCY 0 236 #define IXL_AVE_LATENCY 1 237 #define IXL_BULK_LATENCY 2 238 239 /* MacVlan Flags */ 240 #define IXL_FILTER_USED (u16)(1 << 0) 241 #define IXL_FILTER_VLAN (u16)(1 << 1) 242 #define IXL_FILTER_ADD (u16)(1 << 2) 243 #define IXL_FILTER_DEL (u16)(1 << 3) 244 #define IXL_FILTER_MC (u16)(1 << 4) 245 246 /* used in the vlan field of the filter when not a vlan */ 247 #define IXL_VLAN_ANY -1 248 249 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) 250 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) 251 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) 252 253 /* Misc flags for ixl_vsi.flags */ 254 #define IXL_FLAGS_KEEP_TSO4 (1 << 0) 255 #define IXL_FLAGS_KEEP_TSO6 (1 << 1) 256 257 #define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx) 258 #define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) 259 #define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) 260 #define IXL_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->mtx) 261 #define IXL_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->mtx, MA_OWNED) 262 263 #define IXL_RX_LOCK(_sc) mtx_lock(&(_sc)->mtx) 264 #define IXL_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) 265 #define IXL_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) 266 267 #if __FreeBSD_version >= 1100000 268 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) 269 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) 270 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) 271 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) 272 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ 273 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) 274 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) 275 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) 276 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) 277 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) 278 #define IXL_SET_OQDROPS(vsi, count) (vsi)->iqdrops = (count) 279 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 280 #else 281 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) 282 #define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) 283 #define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) 284 #define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) 285 #define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) 286 #define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) 287 #define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) 288 #define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) 289 #define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) 290 #define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) 291 #define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) 292 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 293 #endif 294 295 296 /* 297 ***************************************************************************** 298 * vendor_info_array 299 * 300 * This array contains the list of Subvendor/Subdevice IDs on which the driver 301 * should load. 302 * 303 ***************************************************************************** 304 */ 305 typedef struct _ixl_vendor_info_t { 306 unsigned int vendor_id; 307 unsigned int device_id; 308 unsigned int subvendor_id; 309 unsigned int subdevice_id; 310 unsigned int index; 311 } ixl_vendor_info_t; 312 313 314 struct ixl_tx_buf { 315 u32 eop_index; 316 struct mbuf *m_head; 317 bus_dmamap_t map; 318 bus_dma_tag_t tag; 319 }; 320 321 struct ixl_rx_buf { 322 struct mbuf *m_head; 323 struct mbuf *m_pack; 324 struct mbuf *fmp; 325 bus_dmamap_t hmap; 326 bus_dmamap_t pmap; 327 }; 328 329 /* 330 ** This struct has multiple uses, multicast 331 ** addresses, vlans, and mac filters all use it. 332 */ 333 struct ixl_mac_filter { 334 SLIST_ENTRY(ixl_mac_filter) next; 335 u8 macaddr[ETHER_ADDR_LEN]; 336 s16 vlan; 337 u16 flags; 338 }; 339 340 341 /* 342 * The Transmit ring control struct 343 */ 344 struct tx_ring { 345 struct ixl_queue *que; 346 struct mtx mtx; 347 u32 tail; 348 struct i40e_tx_desc *base; 349 struct i40e_dma_mem dma; 350 u16 next_avail; 351 u16 next_to_clean; 352 u16 atr_rate; 353 u16 atr_count; 354 u16 itr; 355 u16 latency; 356 struct ixl_tx_buf *buffers; 357 volatile u16 avail; 358 u32 cmd; 359 bus_dma_tag_t tx_tag; 360 bus_dma_tag_t tso_tag; 361 char mtx_name[16]; 362 struct buf_ring *br; 363 364 /* Used for Dynamic ITR calculation */ 365 u32 packets; 366 u32 bytes; 367 368 /* Soft Stats */ 369 u64 tx_bytes; 370 u64 no_desc; 371 u64 total_packets; 372 }; 373 374 375 /* 376 * The Receive ring control struct 377 */ 378 struct rx_ring { 379 struct ixl_queue *que; 380 struct mtx mtx; 381 union i40e_rx_desc *base; 382 struct i40e_dma_mem dma; 383 struct lro_ctrl lro; 384 bool lro_enabled; 385 bool hdr_split; 386 bool discard; 387 u16 next_refresh; 388 u16 next_check; 389 u16 itr; 390 u16 latency; 391 char mtx_name[16]; 392 struct ixl_rx_buf *buffers; 393 u32 mbuf_sz; 394 u32 tail; 395 bus_dma_tag_t htag; 396 bus_dma_tag_t ptag; 397 398 /* Used for Dynamic ITR calculation */ 399 u32 packets; 400 u32 bytes; 401 402 /* Soft stats */ 403 u64 split; 404 u64 rx_packets; 405 u64 rx_bytes; 406 u64 discarded; 407 u64 not_done; 408 }; 409 410 /* 411 ** Driver queue struct: this is the interrupt container 412 ** for the associated tx and rx ring pair. 413 */ 414 struct ixl_queue { 415 struct ixl_vsi *vsi; 416 u32 me; 417 u32 msix; /* This queue's MSIX vector */ 418 u32 eims; /* This queue's EIMS bit */ 419 struct resource *res; 420 void *tag; 421 int num_desc; /* both tx and rx */ 422 int busy; 423 struct tx_ring txr; 424 struct rx_ring rxr; 425 struct task task; 426 struct task tx_task; 427 struct taskqueue *tq; 428 429 /* Queue stats */ 430 u64 irqs; 431 u64 tso; 432 u64 mbuf_defrag_failed; 433 u64 mbuf_hdr_failed; 434 u64 mbuf_pkt_failed; 435 u64 tx_map_avail; 436 u64 tx_dma_setup; 437 u64 dropped_pkts; 438 }; 439 440 /* 441 ** Virtual Station interface: 442 ** there would be one of these per traffic class/type 443 ** for now just one, and its embedded in the pf 444 */ 445 SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); 446 struct ixl_vsi { 447 void *back; 448 struct ifnet *ifp; 449 struct device *dev; 450 struct i40e_hw *hw; 451 struct ifmedia media; 452 u64 que_mask; 453 int id; 454 u16 msix_base; /* station base MSIX vector */ 455 u16 num_queues; 456 u16 rx_itr_setting; 457 u16 tx_itr_setting; 458 struct ixl_queue *queues; /* head of queues */ 459 bool link_active; 460 u16 seid; 461 u16 max_frame_size; 462 u32 link_speed; 463 bool link_up; 464 u32 fc; /* local flow ctrl setting */ 465 466 /* MAC/VLAN Filter list */ 467 struct ixl_ftl_head ftl; 468 469 struct i40e_aqc_vsi_properties_data info; 470 471 eventhandler_tag vlan_attach; 472 eventhandler_tag vlan_detach; 473 u16 num_vlans; 474 475 /* Per-VSI stats from hardware */ 476 struct i40e_eth_stats eth_stats; 477 struct i40e_eth_stats eth_stats_offsets; 478 bool stat_offsets_loaded; 479 u64 ipackets; 480 u64 ierrors; 481 u64 opackets; 482 u64 oerrors; 483 u64 ibytes; 484 u64 obytes; 485 u64 imcasts; 486 u64 omcasts; 487 u64 iqdrops; 488 u64 oqdrops; 489 u64 noproto; 490 491 /* Driver statistics */ 492 u64 hw_filters_del; 493 u64 hw_filters_add; 494 495 /* Misc. */ 496 u64 active_queues; 497 u64 flags; 498 }; 499 500 /* 501 ** Find the number of unrefreshed RX descriptors 502 */ 503 static inline u16 504 ixl_rx_unrefreshed(struct ixl_queue *que) 505 { 506 struct rx_ring *rxr = &que->rxr; 507 508 if (rxr->next_check > rxr->next_refresh) 509 return (rxr->next_check - rxr->next_refresh - 1); 510 else 511 return ((que->num_desc + rxr->next_check) - 512 rxr->next_refresh - 1); 513 } 514 515 /* 516 ** Find the next available unused filter 517 */ 518 static inline struct ixl_mac_filter * 519 ixl_get_filter(struct ixl_vsi *vsi) 520 { 521 struct ixl_mac_filter *f; 522 523 /* create a new empty filter */ 524 f = malloc(sizeof(struct ixl_mac_filter), 525 M_DEVBUF, M_NOWAIT | M_ZERO); 526 SLIST_INSERT_HEAD(&vsi->ftl, f, next); 527 528 return (f); 529 } 530 531 /* 532 ** Compare two ethernet addresses 533 */ 534 static inline bool 535 cmp_etheraddr(u8 *ea1, u8 *ea2) 536 { 537 bool cmp = FALSE; 538 539 if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) && 540 (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) && 541 (ea1[4] == ea2[4]) && (ea1[5] == ea2[5])) 542 cmp = TRUE; 543 544 return (cmp); 545 } 546 547 /* 548 * Info for stats sysctls 549 */ 550 struct ixl_sysctl_info { 551 u64 *stat; 552 char *name; 553 char *description; 554 }; 555 556 extern int ixl_atr_rate; 557 558 /* 559 ** ixl_fw_version_str - format the FW and NVM version strings 560 */ 561 static inline char * 562 ixl_fw_version_str(struct i40e_hw *hw) 563 { 564 static char buf[32]; 565 566 snprintf(buf, sizeof(buf), 567 "f%d.%d a%d.%d n%02x.%02x e%08x", 568 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, 569 hw->aq.api_maj_ver, hw->aq.api_min_ver, 570 (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> 571 IXL_NVM_VERSION_HI_SHIFT, 572 (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> 573 IXL_NVM_VERSION_LO_SHIFT, 574 hw->nvm.eetrack); 575 return buf; 576 } 577 578 /********************************************************************* 579 * TXRX Function prototypes 580 *********************************************************************/ 581 int ixl_allocate_tx_data(struct ixl_queue *); 582 int ixl_allocate_rx_data(struct ixl_queue *); 583 void ixl_init_tx_ring(struct ixl_queue *); 584 int ixl_init_rx_ring(struct ixl_queue *); 585 bool ixl_rxeof(struct ixl_queue *, int); 586 bool ixl_txeof(struct ixl_queue *); 587 int ixl_mq_start(struct ifnet *, struct mbuf *); 588 int ixl_mq_start_locked(struct ifnet *, struct tx_ring *); 589 void ixl_deferred_mq_start(void *, int); 590 void ixl_qflush(struct ifnet *); 591 void ixl_free_vsi(struct ixl_vsi *); 592 void ixl_free_que_tx(struct ixl_queue *); 593 void ixl_free_que_rx(struct ixl_queue *); 594 #ifdef IXL_FDIR 595 void ixl_atr(struct ixl_queue *, struct tcphdr *, int); 596 #endif 597 #if __FreeBSD_version >= 1100000 598 uint64_t ixl_get_counter(if_t ifp, ift_counter cnt); 599 #endif 600 601 #endif /* _IXL_H_ */ 602