1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. 4 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 5 */ 6 7 #ifndef _ENIC_H_ 8 #define _ENIC_H_ 9 10 #include "vnic_enet.h" 11 #include "vnic_dev.h" 12 #include "vnic_wq.h" 13 #include "vnic_rq.h" 14 #include "vnic_cq.h" 15 #include "vnic_intr.h" 16 #include "vnic_stats.h" 17 #include "vnic_nic.h" 18 #include "vnic_rss.h" 19 #include <linux/irq.h> 20 #include <net/page_pool/helpers.h> 21 22 #define DRV_NAME "enic" 23 #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Driver" 24 25 #define PCI_SUBDEV_ID_CISCO_VIC_1225 0x085 26 #define PCI_SUBDEV_ID_CISCO_VIC_1225T 0x0CE 27 #define PCI_SUBDEV_ID_CISCO_VIC_1227 0x12E 28 #define PCI_SUBDEV_ID_CISCO_VIC_1227T 0x139 29 #define PCI_SUBDEV_ID_CISCO_VIC_1240 0x084 30 #define PCI_SUBDEV_ID_CISCO_VIC_1280 0x04F 31 #define PCI_SUBDEV_ID_CISCO_VIC_1285 0x0CD 32 33 #define PCI_SUBDEV_ID_CISCO_VIC_1340 0x12C 34 #define PCI_SUBDEV_ID_CISCO_VIC_1380 0x137 35 #define PCI_SUBDEV_ID_CISCO_VIC_1385 0x14D 36 #define PCI_SUBDEV_ID_CISCO_VIC_1387 0x15D 37 38 #define PCI_SUBDEV_ID_CISCO_VIC_1440 0x0215 39 #define PCI_SUBDEV_ID_CISCO_VIC_1455 0x0217 40 #define PCI_SUBDEV_ID_CISCO_VIC_1457 0x0218 41 #define PCI_SUBDEV_ID_CISCO_VIC_1467 0x02AF 42 #define PCI_SUBDEV_ID_CISCO_VIC_1477 0x2B0 43 #define PCI_SUBDEV_ID_CISCO_VIC_1480 0x0216 44 #define PCI_SUBDEV_ID_CISCO_VIC_1485 0x0219 45 #define PCI_SUBDEV_ID_CISCO_VIC_1487 0x021A 46 #define PCI_SUBDEV_ID_CISCO_VIC_1495 0x024A 47 #define PCI_SUBDEV_ID_CISCO_VIC_1497 0x024B 48 #define PCI_SUBDEV_ID_CISCO_VIC_14425 0x02CF 49 #define PCI_SUBDEV_ID_CISCO_VIC_14825 0x02D0 50 51 #define PCI_SUBDEV_ID_CISCO_VIC_15230 0x02DF 52 #define PCI_SUBDEV_ID_CISCO_VIC_15231 0x02DB 53 #define PCI_SUBDEV_ID_CISCO_VIC_15235 0x02E4 54 #define PCI_SUBDEV_ID_CISCO_VIC_15237 0x02F3 55 #define PCI_SUBDEV_ID_CISCO_VIC_15238 0x02E8 56 #define PCI_SUBDEV_ID_CISCO_VIC_15411 0x02DC 57 #define PCI_SUBDEV_ID_CISCO_VIC_15412 0x02E2 58 #define PCI_SUBDEV_ID_CISCO_VIC_15420 0x02DE 59 #define PCI_SUBDEV_ID_CISCO_VIC_15422 0x02E1 60 #define PCI_SUBDEV_ID_CISCO_VIC_15425 0x02F2 61 #define PCI_SUBDEV_ID_CISCO_VIC_15427 0x02E0 62 #define PCI_SUBDEV_ID_CISCO_VIC_15428 0x02DD 63 64 #define ENIC_BARS_MAX 6 65 66 #define ENIC_WQ_MAX 256 67 #define ENIC_RQ_MAX 256 68 #define ENIC_RQ_MIN_DEFAULT 8 69 70 #define ENIC_WQ_NAPI_BUDGET 256 71 72 #define ENIC_AIC_LARGE_PKT_DIFF 3 73 74 enum ext_cq { 75 ENIC_RQ_CQ_ENTRY_SIZE_16, 76 ENIC_RQ_CQ_ENTRY_SIZE_32, 77 ENIC_RQ_CQ_ENTRY_SIZE_64, 78 ENIC_RQ_CQ_ENTRY_SIZE_MAX, 79 }; 80 81 struct enic_msix_entry { 82 int requested; 83 char devname[IFNAMSIZ + 8]; 84 irqreturn_t (*isr)(int, void *); 85 void *devid; 86 cpumask_var_t affinity_mask; 87 }; 88 89 /* Store only the lower range. Higher range is given by fw. */ 90 struct enic_intr_mod_range { 91 u32 small_pkt_range_start; 92 u32 large_pkt_range_start; 93 }; 94 95 struct enic_intr_mod_table { 96 u32 rx_rate; 97 u32 range_percent; 98 }; 99 100 #define ENIC_MAX_LINK_SPEEDS 3 101 #define ENIC_LINK_SPEED_10G 10000 102 #define ENIC_LINK_SPEED_4G 4000 103 #define ENIC_LINK_40G_INDEX 2 104 #define ENIC_LINK_10G_INDEX 1 105 #define ENIC_LINK_4G_INDEX 0 106 #define ENIC_RX_COALESCE_RANGE_END 125 107 #define ENIC_AIC_TS_BREAK 100 108 109 struct enic_rx_coal { 110 u32 small_pkt_range_start; 111 u32 large_pkt_range_start; 112 u32 range_end; 113 u32 use_adaptive_rx_coalesce; 114 }; 115 116 /* priv_flags */ 117 #define ENIC_SRIOV_ENABLED (1 << 0) 118 119 /* enic port profile set flags */ 120 #define ENIC_PORT_REQUEST_APPLIED (1 << 0) 121 #define ENIC_SET_REQUEST (1 << 1) 122 #define ENIC_SET_NAME (1 << 2) 123 #define ENIC_SET_INSTANCE (1 << 3) 124 #define ENIC_SET_HOST (1 << 4) 125 126 #define MAX_TSO BIT(16) 127 #define WQ_ENET_MAX_DESC_LEN BIT(WQ_ENET_LEN_BITS) 128 #define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1) 129 130 struct enic_port_profile { 131 u32 set; 132 u8 request; 133 char name[PORT_PROFILE_MAX]; 134 u8 instance_uuid[PORT_UUID_MAX]; 135 u8 host_uuid[PORT_UUID_MAX]; 136 u8 vf_mac[ETH_ALEN]; 137 u8 mac_addr[ETH_ALEN]; 138 }; 139 140 enum enic_vf_link_state { 141 ENIC_VF_LINK_STATE_UNKNOWN, 142 ENIC_VF_LINK_STATE_DOWN, 143 ENIC_VF_LINK_STATE_UP, 144 }; 145 146 /* enic_rfs_fltr_node - rfs filter node in hash table 147 * @@keys: IPv4 5 tuple 148 * @flow_id: flow_id of clsf filter provided by kernel 149 * @fltr_id: filter id of clsf filter returned by adaptor 150 * @rq_id: desired rq index 151 * @node: hlist_node 152 */ 153 struct enic_rfs_fltr_node { 154 struct flow_keys keys; 155 u32 flow_id; 156 u16 fltr_id; 157 u16 rq_id; 158 struct hlist_node node; 159 }; 160 161 /* enic_rfs_flw_tbl - rfs flow table 162 * @max: Maximum number of filters vNIC supports 163 * @free: Number of free filters available 164 * @toclean: hash table index to clean next 165 * @ht_head: hash table list head 166 * @lock: spin lock 167 * @rfs_may_expire: timer function for enic_rps_may_expire_flow 168 */ 169 struct enic_rfs_flw_tbl { 170 u16 max; 171 int free; 172 173 #define ENIC_RFS_FLW_BITSHIFT (10) 174 #define ENIC_RFS_FLW_MASK ((1 << ENIC_RFS_FLW_BITSHIFT) - 1) 175 u16 toclean:ENIC_RFS_FLW_BITSHIFT; 176 struct hlist_head ht_head[1 << ENIC_RFS_FLW_BITSHIFT]; 177 spinlock_t lock; 178 struct timer_list rfs_may_expire; 179 }; 180 181 struct vxlan_offload { 182 u16 vxlan_udp_port_number; 183 u8 patch_level; 184 u8 flags; 185 }; 186 187 struct enic_wq_stats { 188 u64 packets; /* pkts queued for Tx */ 189 u64 stopped; /* Tx ring almost full, queue stopped */ 190 u64 wake; /* Tx ring no longer full, queue woken up*/ 191 u64 tso; /* non-encap tso pkt */ 192 u64 encap_tso; /* encap tso pkt */ 193 u64 encap_csum; /* encap HW csum */ 194 u64 csum_partial; /* skb->ip_summed = CHECKSUM_PARTIAL */ 195 u64 csum_none; /* HW csum not required */ 196 u64 bytes; /* bytes queued for Tx */ 197 u64 add_vlan; /* HW adds vlan tag */ 198 u64 cq_work; /* Tx completions processed */ 199 u64 cq_bytes; /* Tx bytes processed */ 200 u64 null_pkt; /* skb length <= 0 */ 201 u64 skb_linear_fail; /* linearize failures */ 202 u64 desc_full_awake; /* TX ring full while queue awake */ 203 }; 204 205 struct enic_rq_stats { 206 u64 packets; /* pkts received */ 207 u64 bytes; /* bytes received */ 208 u64 l4_rss_hash; /* hashed on l4 */ 209 u64 l3_rss_hash; /* hashed on l3 */ 210 u64 csum_unnecessary; /* HW verified csum */ 211 u64 csum_unnecessary_encap; /* HW verified csum on encap packet */ 212 u64 vlan_stripped; /* HW stripped vlan */ 213 u64 napi_complete; /* napi complete intr reenabled */ 214 u64 napi_repoll; /* napi poll again */ 215 u64 bad_fcs; /* bad pkts */ 216 u64 pkt_truncated; /* truncated pkts */ 217 u64 no_skb; /* out of skbs */ 218 u64 desc_skip; /* Rx pkt went into later buffer */ 219 u64 pp_alloc_fail; /* page pool alloc failure */ 220 }; 221 222 struct enic_wq { 223 spinlock_t lock; /* spinlock for wq */ 224 struct vnic_wq vwq; 225 struct enic_wq_stats stats; 226 } ____cacheline_aligned; 227 228 struct enic_rq { 229 struct vnic_rq vrq; 230 struct enic_rq_stats stats; 231 struct page_pool *pool; 232 } ____cacheline_aligned; 233 234 enum enic_vf_type { 235 ENIC_VF_TYPE_NONE, 236 ENIC_VF_TYPE_V1, 237 ENIC_VF_TYPE_USNIC, 238 ENIC_VF_TYPE_V2, 239 }; 240 241 /* Per-instance private data structure */ 242 struct enic { 243 struct net_device *netdev; 244 struct pci_dev *pdev; 245 struct vnic_enet_config config; 246 struct vnic_dev_bar bar[ENIC_BARS_MAX]; 247 struct vnic_dev *vdev; 248 struct timer_list notify_timer; 249 struct work_struct reset; 250 struct work_struct tx_hang_reset; 251 struct work_struct change_mtu_work; 252 struct msix_entry *msix_entry; 253 struct enic_msix_entry *msix; 254 u32 msg_enable; 255 spinlock_t devcmd_lock; 256 u8 mac_addr[ETH_ALEN]; 257 unsigned int flags; 258 unsigned int priv_flags; 259 unsigned int mc_count; 260 unsigned int uc_count; 261 u32 port_mtu; 262 struct enic_rx_coal rx_coalesce_setting; 263 u32 rx_coalesce_usecs; 264 u32 tx_coalesce_usecs; 265 u16 num_vfs; 266 enum enic_vf_type vf_type; 267 bool vf_registered; 268 u32 pf_cap_version; 269 unsigned int enable_count; 270 spinlock_t enic_api_lock; 271 bool enic_api_busy; 272 struct enic_port_profile *pp; 273 274 struct enic_wq *wq; 275 unsigned int wq_avail; 276 unsigned int wq_count; 277 u16 loop_enable; 278 u16 loop_tag; 279 280 struct enic_rq *rq; 281 unsigned int rq_avail; 282 unsigned int rq_count; 283 struct vxlan_offload vxlan; 284 struct napi_struct *napi; 285 286 struct vnic_intr *intr; 287 unsigned int intr_avail; 288 unsigned int intr_count; 289 u32 __iomem *legacy_pba; /* memory-mapped */ 290 291 struct vnic_cq *cq; 292 unsigned int cq_avail; 293 unsigned int cq_count; 294 struct enic_rfs_flw_tbl rfs_h; 295 u8 rss_key[ENIC_RSS_LEN]; 296 struct vnic_gen_stats gen_stats; 297 enum ext_cq ext_cq; 298 299 /* Admin channel resources for SR-IOV MBOX */ 300 bool has_admin_channel; 301 /* true only while the admin WQ/RQ/CQ are allocated and enabled; gates 302 * enic_admin_channel_close() so it is a no-op after a failed (re)open 303 * left the resources freed. 304 */ 305 bool admin_chan_up; 306 /* set on send timeout; cleared on channel re-open */ 307 bool mbox_send_disabled; 308 struct vnic_wq admin_wq; 309 struct vnic_rq admin_rq; 310 struct vnic_cq admin_cq[2]; 311 struct vnic_intr admin_intr; 312 struct delayed_work admin_poll_work; 313 unsigned int admin_intr_index; 314 struct work_struct link_notify_work; 315 struct work_struct admin_msg_work; 316 spinlock_t admin_msg_lock; /* protects admin_msg_list */ 317 struct list_head admin_msg_list; 318 unsigned int admin_msg_count; /* current depth of admin_msg_list */ 319 void (*admin_rq_handler)(struct enic *enic, void *buf, 320 unsigned int len); 321 /* The PF is authoritative for a V2 VF's carrier. Keep the last 322 * notification across an ordinary netdev close/open and serialize it 323 * against the open/stop carrier transition. 324 */ 325 spinlock_t vf_link_state_lock; 326 enum enic_vf_link_state vf_link_state; 327 bool vf_link_running; 328 329 /* MBOX protocol state — mbox_lock serializes admin WQ sends */ 330 struct mutex mbox_lock; 331 u64 mbox_msg_num; 332 /* MBOX request-reply state. Existing request callers allow only one 333 * request in flight. The state lock arbitrates reply acceptance against 334 * timeout invalidation, while mbox_comp publishes the accepted result to 335 * the requester. 336 */ 337 struct completion mbox_comp; 338 spinlock_t mbox_state_lock; /* protects expected reply state */ 339 u64 mbox_expected_msg_num; 340 u8 mbox_expected_reply; 341 bool mbox_initialized; 342 343 /* PF: per-VF MBOX state, allocated when SRIOV V2 is enabled */ 344 struct enic_vf_state { 345 bool registered; 346 } *vf_state; 347 }; 348 349 static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev) 350 { 351 struct enic *enic = vdev->priv; 352 353 return enic->netdev; 354 } 355 356 /* wrappers function for kernel log 357 */ 358 #define vdev_err(vdev, fmt, ...) \ 359 dev_err(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) 360 #define vdev_warn(vdev, fmt, ...) \ 361 dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) 362 #define vdev_info(vdev, fmt, ...) \ 363 dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) 364 #define vdev_dbg(vdev, fmt, ...) \ 365 dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) 366 367 #define vdev_neterr(vdev, fmt, ...) \ 368 netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__) 369 #define vdev_netwarn(vdev, fmt, ...) \ 370 netdev_warn(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__) 371 #define vdev_netinfo(vdev, fmt, ...) \ 372 netdev_info(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__) 373 374 static inline struct device *enic_get_dev(struct enic *enic) 375 { 376 return &(enic->pdev->dev); 377 } 378 379 static inline unsigned int enic_cq_rq(struct enic *enic, unsigned int rq) 380 { 381 return rq; 382 } 383 384 static inline unsigned int enic_cq_wq(struct enic *enic, unsigned int wq) 385 { 386 return enic->rq_count + wq; 387 } 388 389 static inline unsigned int enic_msix_rq_intr(struct enic *enic, 390 unsigned int rq) 391 { 392 return enic->cq[enic_cq_rq(enic, rq)].interrupt_offset; 393 } 394 395 static inline unsigned int enic_msix_wq_intr(struct enic *enic, 396 unsigned int wq) 397 { 398 return enic->cq[enic_cq_wq(enic, wq)].interrupt_offset; 399 } 400 401 /* MSIX interrupts are organized as the error interrupt, then the notify 402 * interrupt followed by all the I/O interrupts. The error interrupt needs 403 * to fit in 7 bits due to hardware constraints 404 */ 405 #define ENIC_MSIX_RESERVED_INTR 2 406 #define ENIC_MSIX_ERR_INTR 0 407 #define ENIC_MSIX_NOTIFY_INTR 1 408 #define ENIC_MSIX_IO_INTR_BASE ENIC_MSIX_RESERVED_INTR 409 #define ENIC_MSIX_MIN_INTR (ENIC_MSIX_RESERVED_INTR + 2) 410 411 #define ENIC_LEGACY_IO_INTR 0 412 #define ENIC_LEGACY_ERR_INTR 1 413 #define ENIC_LEGACY_NOTIFY_INTR 2 414 415 static inline unsigned int enic_msix_err_intr(struct enic *enic) 416 { 417 return ENIC_MSIX_ERR_INTR; 418 } 419 420 static inline unsigned int enic_msix_notify_intr(struct enic *enic) 421 { 422 return ENIC_MSIX_NOTIFY_INTR; 423 } 424 425 static inline bool enic_is_err_intr(struct enic *enic, int intr) 426 { 427 switch (vnic_dev_get_intr_mode(enic->vdev)) { 428 case VNIC_DEV_INTR_MODE_INTX: 429 return intr == ENIC_LEGACY_ERR_INTR; 430 case VNIC_DEV_INTR_MODE_MSIX: 431 return intr == enic_msix_err_intr(enic); 432 case VNIC_DEV_INTR_MODE_MSI: 433 default: 434 return false; 435 } 436 } 437 438 static inline bool enic_is_notify_intr(struct enic *enic, int intr) 439 { 440 switch (vnic_dev_get_intr_mode(enic->vdev)) { 441 case VNIC_DEV_INTR_MODE_INTX: 442 return intr == ENIC_LEGACY_NOTIFY_INTR; 443 case VNIC_DEV_INTR_MODE_MSIX: 444 return intr == enic_msix_notify_intr(enic); 445 case VNIC_DEV_INTR_MODE_MSI: 446 default: 447 return false; 448 } 449 } 450 451 static inline int enic_dma_map_check(struct enic *enic, dma_addr_t dma_addr) 452 { 453 if (unlikely(dma_mapping_error(&enic->pdev->dev, dma_addr))) { 454 net_warn_ratelimited("%s: PCI dma mapping failed!\n", 455 enic->netdev->name); 456 enic->gen_stats.dma_map_error++; 457 458 return -ENOMEM; 459 } 460 461 return 0; 462 } 463 464 void enic_reset_addr_lists(struct enic *enic); 465 int enic_sriov_enabled(struct enic *enic); 466 int enic_is_valid_vf(struct enic *enic, int vf); 467 int enic_is_dynamic(struct enic *enic); 468 int enic_is_sriov_vf_v2(struct enic *enic); 469 void enic_set_ethtool_ops(struct net_device *netdev); 470 int __enic_set_rsskey(struct enic *enic); 471 void enic_ext_cq(struct enic *enic); 472 473 #endif /* _ENIC_H_ */ 474