1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2018 Intel Corporation */ 3 4 #ifndef _IGC_H_ 5 #define _IGC_H_ 6 7 #include <linux/kobject.h> 8 #include <linux/pci.h> 9 #include <linux/netdevice.h> 10 #include <linux/vmalloc.h> 11 #include <linux/ethtool.h> 12 #include <linux/sctp.h> 13 14 #include "igc_hw.h" 15 16 /* forward declaration */ 17 void igc_set_ethtool_ops(struct net_device *); 18 19 struct igc_adapter; 20 struct igc_ring; 21 22 void igc_up(struct igc_adapter *adapter); 23 void igc_down(struct igc_adapter *adapter); 24 int igc_setup_tx_resources(struct igc_ring *ring); 25 int igc_setup_rx_resources(struct igc_ring *ring); 26 void igc_free_tx_resources(struct igc_ring *ring); 27 void igc_free_rx_resources(struct igc_ring *ring); 28 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter); 29 void igc_set_flag_queue_pairs(struct igc_adapter *adapter, 30 const u32 max_rss_queues); 31 int igc_reinit_queues(struct igc_adapter *adapter); 32 void igc_write_rss_indir_tbl(struct igc_adapter *adapter); 33 bool igc_has_link(struct igc_adapter *adapter); 34 void igc_reset(struct igc_adapter *adapter); 35 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx); 36 int igc_add_mac_steering_filter(struct igc_adapter *adapter, 37 const u8 *addr, u8 queue, u8 flags); 38 int igc_del_mac_steering_filter(struct igc_adapter *adapter, 39 const u8 *addr, u8 queue, u8 flags); 40 void igc_update_stats(struct igc_adapter *adapter); 41 42 extern char igc_driver_name[]; 43 extern char igc_driver_version[]; 44 45 #define IGC_REGS_LEN 740 46 #define IGC_RETA_SIZE 128 47 48 /* Interrupt defines */ 49 #define IGC_START_ITR 648 /* ~6000 ints/sec */ 50 #define IGC_FLAG_HAS_MSI BIT(0) 51 #define IGC_FLAG_QUEUE_PAIRS BIT(3) 52 #define IGC_FLAG_DMAC BIT(4) 53 #define IGC_FLAG_NEED_LINK_UPDATE BIT(9) 54 #define IGC_FLAG_MEDIA_RESET BIT(10) 55 #define IGC_FLAG_MAS_ENABLE BIT(12) 56 #define IGC_FLAG_HAS_MSIX BIT(13) 57 #define IGC_FLAG_VLAN_PROMISC BIT(15) 58 #define IGC_FLAG_RX_LEGACY BIT(16) 59 60 #define IGC_FLAG_RSS_FIELD_IPV4_UDP BIT(6) 61 #define IGC_FLAG_RSS_FIELD_IPV6_UDP BIT(7) 62 63 #define IGC_MRQC_ENABLE_RSS_MQ 0x00000002 64 #define IGC_MRQC_RSS_FIELD_IPV4_UDP 0x00400000 65 #define IGC_MRQC_RSS_FIELD_IPV6_UDP 0x00800000 66 67 #define IGC_START_ITR 648 /* ~6000 ints/sec */ 68 #define IGC_4K_ITR 980 69 #define IGC_20K_ITR 196 70 #define IGC_70K_ITR 56 71 72 #define IGC_DEFAULT_ITR 3 /* dynamic */ 73 #define IGC_MAX_ITR_USECS 10000 74 #define IGC_MIN_ITR_USECS 10 75 #define NON_Q_VECTORS 1 76 #define MAX_MSIX_ENTRIES 10 77 78 /* TX/RX descriptor defines */ 79 #define IGC_DEFAULT_TXD 256 80 #define IGC_DEFAULT_TX_WORK 128 81 #define IGC_MIN_TXD 80 82 #define IGC_MAX_TXD 4096 83 84 #define IGC_DEFAULT_RXD 256 85 #define IGC_MIN_RXD 80 86 #define IGC_MAX_RXD 4096 87 88 /* Transmit and receive queues */ 89 #define IGC_MAX_RX_QUEUES 4 90 #define IGC_MAX_TX_QUEUES 4 91 92 #define MAX_Q_VECTORS 8 93 #define MAX_STD_JUMBO_FRAME_SIZE 9216 94 95 /* Supported Rx Buffer Sizes */ 96 #define IGC_RXBUFFER_256 256 97 #define IGC_RXBUFFER_2048 2048 98 #define IGC_RXBUFFER_3072 3072 99 100 #define AUTO_ALL_MODES 0 101 #define IGC_RX_HDR_LEN IGC_RXBUFFER_256 102 103 /* RX and TX descriptor control thresholds. 104 * PTHRESH - MAC will consider prefetch if it has fewer than this number of 105 * descriptors available in its onboard memory. 106 * Setting this to 0 disables RX descriptor prefetch. 107 * HTHRESH - MAC will only prefetch if there are at least this many descriptors 108 * available in host memory. 109 * If PTHRESH is 0, this should also be 0. 110 * WTHRESH - RX descriptor writeback threshold - MAC will delay writing back 111 * descriptors until either it has this many to write back, or the 112 * ITR timer expires. 113 */ 114 #define IGC_RX_PTHRESH 8 115 #define IGC_RX_HTHRESH 8 116 #define IGC_TX_PTHRESH 8 117 #define IGC_TX_HTHRESH 1 118 #define IGC_RX_WTHRESH 4 119 #define IGC_TX_WTHRESH 16 120 121 #define IGC_RX_DMA_ATTR \ 122 (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) 123 124 #define IGC_TS_HDR_LEN 16 125 126 #define IGC_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) 127 128 #if (PAGE_SIZE < 8192) 129 #define IGC_MAX_FRAME_BUILD_SKB \ 130 (SKB_WITH_OVERHEAD(IGC_RXBUFFER_2048) - IGC_SKB_PAD - IGC_TS_HDR_LEN) 131 #else 132 #define IGC_MAX_FRAME_BUILD_SKB (IGC_RXBUFFER_2048 - IGC_TS_HDR_LEN) 133 #endif 134 135 /* How many Rx Buffers do we bundle into one write to the hardware ? */ 136 #define IGC_RX_BUFFER_WRITE 16 /* Must be power of 2 */ 137 138 /* VLAN info */ 139 #define IGC_TX_FLAGS_VLAN_MASK 0xffff0000 140 141 /* igc_test_staterr - tests bits within Rx descriptor status and error fields */ 142 static inline __le32 igc_test_staterr(union igc_adv_rx_desc *rx_desc, 143 const u32 stat_err_bits) 144 { 145 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits); 146 } 147 148 enum igc_state_t { 149 __IGC_TESTING, 150 __IGC_RESETTING, 151 __IGC_DOWN, 152 __IGC_PTP_TX_IN_PROGRESS, 153 }; 154 155 enum igc_tx_flags { 156 /* cmd_type flags */ 157 IGC_TX_FLAGS_VLAN = 0x01, 158 IGC_TX_FLAGS_TSO = 0x02, 159 IGC_TX_FLAGS_TSTAMP = 0x04, 160 161 /* olinfo flags */ 162 IGC_TX_FLAGS_IPV4 = 0x10, 163 IGC_TX_FLAGS_CSUM = 0x20, 164 }; 165 166 enum igc_boards { 167 board_base, 168 }; 169 170 /* The largest size we can write to the descriptor is 65535. In order to 171 * maintain a power of two alignment we have to limit ourselves to 32K. 172 */ 173 #define IGC_MAX_TXD_PWR 15 174 #define IGC_MAX_DATA_PER_TXD BIT(IGC_MAX_TXD_PWR) 175 176 /* Tx Descriptors needed, worst case */ 177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IGC_MAX_DATA_PER_TXD) 178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4) 179 180 /* wrapper around a pointer to a socket buffer, 181 * so a DMA handle can be stored along with the buffer 182 */ 183 struct igc_tx_buffer { 184 union igc_adv_tx_desc *next_to_watch; 185 unsigned long time_stamp; 186 struct sk_buff *skb; 187 unsigned int bytecount; 188 u16 gso_segs; 189 __be16 protocol; 190 191 DEFINE_DMA_UNMAP_ADDR(dma); 192 DEFINE_DMA_UNMAP_LEN(len); 193 u32 tx_flags; 194 }; 195 196 struct igc_rx_buffer { 197 dma_addr_t dma; 198 struct page *page; 199 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) 200 __u32 page_offset; 201 #else 202 __u16 page_offset; 203 #endif 204 __u16 pagecnt_bias; 205 }; 206 207 struct igc_tx_queue_stats { 208 u64 packets; 209 u64 bytes; 210 u64 restart_queue; 211 u64 restart_queue2; 212 }; 213 214 struct igc_rx_queue_stats { 215 u64 packets; 216 u64 bytes; 217 u64 drops; 218 u64 csum_err; 219 u64 alloc_failed; 220 }; 221 222 struct igc_rx_packet_stats { 223 u64 ipv4_packets; /* IPv4 headers processed */ 224 u64 ipv4e_packets; /* IPv4E headers with extensions processed */ 225 u64 ipv6_packets; /* IPv6 headers processed */ 226 u64 ipv6e_packets; /* IPv6E headers with extensions processed */ 227 u64 tcp_packets; /* TCP headers processed */ 228 u64 udp_packets; /* UDP headers processed */ 229 u64 sctp_packets; /* SCTP headers processed */ 230 u64 nfs_packets; /* NFS headers processe */ 231 u64 other_packets; 232 }; 233 234 struct igc_ring_container { 235 struct igc_ring *ring; /* pointer to linked list of rings */ 236 unsigned int total_bytes; /* total bytes processed this int */ 237 unsigned int total_packets; /* total packets processed this int */ 238 u16 work_limit; /* total work allowed per interrupt */ 239 u8 count; /* total number of rings in vector */ 240 u8 itr; /* current ITR setting for ring */ 241 }; 242 243 struct igc_ring { 244 struct igc_q_vector *q_vector; /* backlink to q_vector */ 245 struct net_device *netdev; /* back pointer to net_device */ 246 struct device *dev; /* device for dma mapping */ 247 union { /* array of buffer info structs */ 248 struct igc_tx_buffer *tx_buffer_info; 249 struct igc_rx_buffer *rx_buffer_info; 250 }; 251 void *desc; /* descriptor ring memory */ 252 unsigned long flags; /* ring specific flags */ 253 void __iomem *tail; /* pointer to ring tail register */ 254 dma_addr_t dma; /* phys address of the ring */ 255 unsigned int size; /* length of desc. ring in bytes */ 256 257 u16 count; /* number of desc. in the ring */ 258 u8 queue_index; /* logical index of the ring*/ 259 u8 reg_idx; /* physical index of the ring */ 260 bool launchtime_enable; /* true if LaunchTime is enabled */ 261 262 /* everything past this point are written often */ 263 u16 next_to_clean; 264 u16 next_to_use; 265 u16 next_to_alloc; 266 267 union { 268 /* TX */ 269 struct { 270 struct igc_tx_queue_stats tx_stats; 271 struct u64_stats_sync tx_syncp; 272 struct u64_stats_sync tx_syncp2; 273 }; 274 /* RX */ 275 struct { 276 struct igc_rx_queue_stats rx_stats; 277 struct igc_rx_packet_stats pkt_stats; 278 struct u64_stats_sync rx_syncp; 279 struct sk_buff *skb; 280 }; 281 }; 282 } ____cacheline_internodealigned_in_smp; 283 284 struct igc_q_vector { 285 struct igc_adapter *adapter; /* backlink */ 286 void __iomem *itr_register; 287 u32 eims_value; /* EIMS mask value */ 288 289 u16 itr_val; 290 u8 set_itr; 291 292 struct igc_ring_container rx, tx; 293 294 struct napi_struct napi; 295 296 struct rcu_head rcu; /* to avoid race with update stats on free */ 297 char name[IFNAMSIZ + 9]; 298 struct net_device poll_dev; 299 300 /* for dynamic allocation of rings associated with this q_vector */ 301 struct igc_ring ring[0] ____cacheline_internodealigned_in_smp; 302 }; 303 304 #define MAX_ETYPE_FILTER (4 - 1) 305 306 enum igc_filter_match_flags { 307 IGC_FILTER_FLAG_ETHER_TYPE = 0x1, 308 IGC_FILTER_FLAG_VLAN_TCI = 0x2, 309 IGC_FILTER_FLAG_SRC_MAC_ADDR = 0x4, 310 IGC_FILTER_FLAG_DST_MAC_ADDR = 0x8, 311 }; 312 313 /* RX network flow classification data structure */ 314 struct igc_nfc_input { 315 /* Byte layout in order, all values with MSB first: 316 * match_flags - 1 byte 317 * etype - 2 bytes 318 * vlan_tci - 2 bytes 319 */ 320 u8 match_flags; 321 __be16 etype; 322 __be16 vlan_tci; 323 u8 src_addr[ETH_ALEN]; 324 u8 dst_addr[ETH_ALEN]; 325 }; 326 327 struct igc_nfc_filter { 328 struct hlist_node nfc_node; 329 struct igc_nfc_input filter; 330 unsigned long cookie; 331 u16 etype_reg_index; 332 u16 sw_idx; 333 u16 action; 334 }; 335 336 struct igc_mac_addr { 337 u8 addr[ETH_ALEN]; 338 u8 queue; 339 u8 state; /* bitmask */ 340 }; 341 342 #define IGC_MAC_STATE_DEFAULT 0x1 343 #define IGC_MAC_STATE_IN_USE 0x2 344 #define IGC_MAC_STATE_SRC_ADDR 0x4 345 #define IGC_MAC_STATE_QUEUE_STEERING 0x8 346 347 #define IGC_MAX_RXNFC_FILTERS 16 348 349 /* Board specific private data structure */ 350 struct igc_adapter { 351 struct net_device *netdev; 352 353 unsigned long state; 354 unsigned int flags; 355 unsigned int num_q_vectors; 356 357 struct msix_entry *msix_entries; 358 359 /* TX */ 360 u16 tx_work_limit; 361 u32 tx_timeout_count; 362 int num_tx_queues; 363 struct igc_ring *tx_ring[IGC_MAX_TX_QUEUES]; 364 365 /* RX */ 366 int num_rx_queues; 367 struct igc_ring *rx_ring[IGC_MAX_RX_QUEUES]; 368 369 struct timer_list watchdog_timer; 370 struct timer_list dma_err_timer; 371 struct timer_list phy_info_timer; 372 373 u16 link_speed; 374 u16 link_duplex; 375 376 u8 port_num; 377 378 u8 __iomem *io_addr; 379 /* Interrupt Throttle Rate */ 380 u32 rx_itr_setting; 381 u32 tx_itr_setting; 382 383 struct work_struct reset_task; 384 struct work_struct watchdog_task; 385 struct work_struct dma_err_task; 386 bool fc_autoneg; 387 388 u8 tx_timeout_factor; 389 390 int msg_enable; 391 u32 max_frame_size; 392 u32 min_frame_size; 393 394 /* OS defined structs */ 395 struct pci_dev *pdev; 396 /* lock for statistics */ 397 spinlock_t stats64_lock; 398 struct rtnl_link_stats64 stats64; 399 400 /* structs defined in igc_hw.h */ 401 struct igc_hw hw; 402 struct igc_hw_stats stats; 403 404 struct igc_q_vector *q_vector[MAX_Q_VECTORS]; 405 u32 eims_enable_mask; 406 u32 eims_other; 407 408 u16 tx_ring_count; 409 u16 rx_ring_count; 410 411 u32 tx_hwtstamp_timeouts; 412 u32 tx_hwtstamp_skipped; 413 u32 rx_hwtstamp_cleared; 414 u32 *shadow_vfta; 415 416 u32 rss_queues; 417 u32 rss_indir_tbl_init; 418 419 /* RX network flow classification support */ 420 struct hlist_head nfc_filter_list; 421 struct hlist_head cls_flower_list; 422 unsigned int nfc_filter_count; 423 424 /* lock for RX network flow classification filter */ 425 spinlock_t nfc_lock; 426 bool etype_bitmap[MAX_ETYPE_FILTER]; 427 428 struct igc_mac_addr *mac_table; 429 430 u8 rss_indir_tbl[IGC_RETA_SIZE]; 431 432 unsigned long link_check_timeout; 433 struct igc_info ei; 434 }; 435 436 /* igc_desc_unused - calculate if we have unused descriptors */ 437 static inline u16 igc_desc_unused(const struct igc_ring *ring) 438 { 439 u16 ntc = ring->next_to_clean; 440 u16 ntu = ring->next_to_use; 441 442 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1; 443 } 444 445 static inline s32 igc_get_phy_info(struct igc_hw *hw) 446 { 447 if (hw->phy.ops.get_phy_info) 448 return hw->phy.ops.get_phy_info(hw); 449 450 return 0; 451 } 452 453 static inline s32 igc_reset_phy(struct igc_hw *hw) 454 { 455 if (hw->phy.ops.reset) 456 return hw->phy.ops.reset(hw); 457 458 return 0; 459 } 460 461 static inline struct netdev_queue *txring_txq(const struct igc_ring *tx_ring) 462 { 463 return netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index); 464 } 465 466 enum igc_ring_flags_t { 467 IGC_RING_FLAG_RX_3K_BUFFER, 468 IGC_RING_FLAG_RX_BUILD_SKB_ENABLED, 469 IGC_RING_FLAG_RX_SCTP_CSUM, 470 IGC_RING_FLAG_RX_LB_VLAN_BSWAP, 471 IGC_RING_FLAG_TX_CTX_IDX, 472 IGC_RING_FLAG_TX_DETECT_HANG 473 }; 474 475 #define ring_uses_large_buffer(ring) \ 476 test_bit(IGC_RING_FLAG_RX_3K_BUFFER, &(ring)->flags) 477 478 #define ring_uses_build_skb(ring) \ 479 test_bit(IGC_RING_FLAG_RX_BUILD_SKB_ENABLED, &(ring)->flags) 480 481 static inline unsigned int igc_rx_bufsz(struct igc_ring *ring) 482 { 483 #if (PAGE_SIZE < 8192) 484 if (ring_uses_large_buffer(ring)) 485 return IGC_RXBUFFER_3072; 486 487 if (ring_uses_build_skb(ring)) 488 return IGC_MAX_FRAME_BUILD_SKB + IGC_TS_HDR_LEN; 489 #endif 490 return IGC_RXBUFFER_2048; 491 } 492 493 static inline unsigned int igc_rx_pg_order(struct igc_ring *ring) 494 { 495 #if (PAGE_SIZE < 8192) 496 if (ring_uses_large_buffer(ring)) 497 return 1; 498 #endif 499 return 0; 500 } 501 502 static inline s32 igc_read_phy_reg(struct igc_hw *hw, u32 offset, u16 *data) 503 { 504 if (hw->phy.ops.read_reg) 505 return hw->phy.ops.read_reg(hw, offset, data); 506 507 return 0; 508 } 509 510 /* forward declaration */ 511 void igc_reinit_locked(struct igc_adapter *); 512 int igc_add_filter(struct igc_adapter *adapter, 513 struct igc_nfc_filter *input); 514 int igc_erase_filter(struct igc_adapter *adapter, 515 struct igc_nfc_filter *input); 516 517 #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) 518 519 #define IGC_TXD_DCMD (IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS) 520 521 #define IGC_RX_DESC(R, i) \ 522 (&(((union igc_adv_rx_desc *)((R)->desc))[i])) 523 #define IGC_TX_DESC(R, i) \ 524 (&(((union igc_adv_tx_desc *)((R)->desc))[i])) 525 #define IGC_TX_CTXTDESC(R, i) \ 526 (&(((struct igc_adv_tx_context_desc *)((R)->desc))[i])) 527 528 #endif /* _IGC_H_ */ 529