1 /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ 2 /* QLogic qede NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #ifndef _QEDE_H_ 8 #define _QEDE_H_ 9 #include <linux/compiler.h> 10 #include <linux/version.h> 11 #include <linux/workqueue.h> 12 #include <linux/netdevice.h> 13 #include <linux/interrupt.h> 14 #include <linux/bitmap.h> 15 #include <linux/kernel.h> 16 #include <linux/mutex.h> 17 #include <linux/bpf.h> 18 #include <net/xdp.h> 19 #include <linux/qed/qede_rdma.h> 20 #include <linux/io.h> 21 #ifdef CONFIG_RFS_ACCEL 22 #include <linux/cpu_rmap.h> 23 #endif 24 #include <linux/qed/common_hsi.h> 25 #include <linux/qed/eth_common.h> 26 #include <linux/qed/qed_if.h> 27 #include <linux/qed/qed_chain.h> 28 #include <linux/qed/qed_eth_if.h> 29 30 #include <net/pkt_cls.h> 31 #include <net/tc_act/tc_gact.h> 32 33 #define QEDE_MAJOR_VERSION 8 34 #define QEDE_MINOR_VERSION 37 35 #define QEDE_REVISION_VERSION 0 36 #define QEDE_ENGINEERING_VERSION 20 37 #define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "." \ 38 __stringify(QEDE_MINOR_VERSION) "." \ 39 __stringify(QEDE_REVISION_VERSION) "." \ 40 __stringify(QEDE_ENGINEERING_VERSION) 41 42 #define DRV_MODULE_SYM qede 43 44 struct qede_stats_common { 45 u64 no_buff_discards; 46 u64 packet_too_big_discard; 47 u64 ttl0_discard; 48 u64 rx_ucast_bytes; 49 u64 rx_mcast_bytes; 50 u64 rx_bcast_bytes; 51 u64 rx_ucast_pkts; 52 u64 rx_mcast_pkts; 53 u64 rx_bcast_pkts; 54 u64 mftag_filter_discards; 55 u64 mac_filter_discards; 56 u64 gft_filter_drop; 57 u64 tx_ucast_bytes; 58 u64 tx_mcast_bytes; 59 u64 tx_bcast_bytes; 60 u64 tx_ucast_pkts; 61 u64 tx_mcast_pkts; 62 u64 tx_bcast_pkts; 63 u64 tx_err_drop_pkts; 64 u64 coalesced_pkts; 65 u64 coalesced_events; 66 u64 coalesced_aborts_num; 67 u64 non_coalesced_pkts; 68 u64 coalesced_bytes; 69 u64 link_change_count; 70 u64 ptp_skip_txts; 71 72 /* port */ 73 u64 rx_64_byte_packets; 74 u64 rx_65_to_127_byte_packets; 75 u64 rx_128_to_255_byte_packets; 76 u64 rx_256_to_511_byte_packets; 77 u64 rx_512_to_1023_byte_packets; 78 u64 rx_1024_to_1518_byte_packets; 79 u64 rx_crc_errors; 80 u64 rx_mac_crtl_frames; 81 u64 rx_pause_frames; 82 u64 rx_pfc_frames; 83 u64 rx_align_errors; 84 u64 rx_carrier_errors; 85 u64 rx_oversize_packets; 86 u64 rx_jabbers; 87 u64 rx_undersize_packets; 88 u64 rx_fragments; 89 u64 tx_64_byte_packets; 90 u64 tx_65_to_127_byte_packets; 91 u64 tx_128_to_255_byte_packets; 92 u64 tx_256_to_511_byte_packets; 93 u64 tx_512_to_1023_byte_packets; 94 u64 tx_1024_to_1518_byte_packets; 95 u64 tx_pause_frames; 96 u64 tx_pfc_frames; 97 u64 brb_truncates; 98 u64 brb_discards; 99 u64 tx_mac_ctrl_frames; 100 }; 101 102 struct qede_stats_bb { 103 u64 rx_1519_to_1522_byte_packets; 104 u64 rx_1519_to_2047_byte_packets; 105 u64 rx_2048_to_4095_byte_packets; 106 u64 rx_4096_to_9216_byte_packets; 107 u64 rx_9217_to_16383_byte_packets; 108 u64 tx_1519_to_2047_byte_packets; 109 u64 tx_2048_to_4095_byte_packets; 110 u64 tx_4096_to_9216_byte_packets; 111 u64 tx_9217_to_16383_byte_packets; 112 u64 tx_lpi_entry_count; 113 u64 tx_total_collisions; 114 }; 115 116 struct qede_stats_ah { 117 u64 rx_1519_to_max_byte_packets; 118 u64 tx_1519_to_max_byte_packets; 119 }; 120 121 struct qede_stats { 122 struct qede_stats_common common; 123 124 union { 125 struct qede_stats_bb bb; 126 struct qede_stats_ah ah; 127 }; 128 }; 129 130 struct qede_vlan { 131 struct list_head list; 132 u16 vid; 133 bool configured; 134 }; 135 136 struct qede_rdma_dev { 137 struct qedr_dev *qedr_dev; 138 struct list_head entry; 139 struct list_head rdma_event_list; 140 struct workqueue_struct *rdma_wq; 141 struct kref refcnt; 142 struct completion event_comp; 143 bool exp_recovery; 144 }; 145 146 struct qede_ptp; 147 148 #define QEDE_RFS_MAX_FLTR 256 149 150 enum qede_flags_bit { 151 QEDE_FLAGS_IS_VF = 0, 152 QEDE_FLAGS_LINK_REQUESTED, 153 QEDE_FLAGS_PTP_TX_IN_PRORGESS, 154 QEDE_FLAGS_TX_TIMESTAMPING_EN 155 }; 156 157 #define QEDE_DUMP_MAX_ARGS 4 158 enum qede_dump_cmd { 159 QEDE_DUMP_CMD_NONE = 0, 160 QEDE_DUMP_CMD_NVM_CFG, 161 QEDE_DUMP_CMD_GRCDUMP, 162 QEDE_DUMP_CMD_MAX 163 }; 164 165 struct qede_dump_info { 166 enum qede_dump_cmd cmd; 167 u8 num_args; 168 u32 args[QEDE_DUMP_MAX_ARGS]; 169 }; 170 171 struct qede_dev { 172 struct qed_dev *cdev; 173 struct net_device *ndev; 174 struct pci_dev *pdev; 175 176 u32 dp_module; 177 u8 dp_level; 178 179 unsigned long flags; 180 #define IS_VF(edev) (test_bit(QEDE_FLAGS_IS_VF, &(edev)->flags)) 181 182 const struct qed_eth_ops *ops; 183 struct qede_ptp *ptp; 184 u64 ptp_skip_txts; 185 186 struct qed_dev_eth_info dev_info; 187 #define QEDE_MAX_RSS_CNT(edev) ((edev)->dev_info.num_queues) 188 #define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues) 189 #define QEDE_IS_BB(edev) \ 190 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_BB) 191 #define QEDE_IS_AH(edev) \ 192 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_AH) 193 194 struct qede_fastpath *fp_array; 195 u8 req_num_tx; 196 u8 fp_num_tx; 197 u8 req_num_rx; 198 u8 fp_num_rx; 199 u16 req_queues; 200 u16 num_queues; 201 #define QEDE_QUEUE_CNT(edev) ((edev)->num_queues) 202 #define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx) 203 #define QEDE_RX_QUEUE_IDX(edev, i) (i) 204 #define QEDE_TSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_rx) 205 206 struct qed_int_info int_info; 207 208 /* Smaller private varaiant of the RTNL lock */ 209 struct mutex qede_lock; 210 u32 state; /* Protected by qede_lock */ 211 u16 rx_buf_size; 212 u32 rx_copybreak; 213 214 /* L2 header size + 2*VLANs (8 bytes) + LLC SNAP (8 bytes) */ 215 #define ETH_OVERHEAD (ETH_HLEN + 8 + 8) 216 /* Max supported alignment is 256 (8 shift) 217 * minimal alignment shift 6 is optimal for 57xxx HW performance 218 */ 219 #define QEDE_RX_ALIGN_SHIFT max(6, min(8, L1_CACHE_SHIFT)) 220 /* We assume skb_build() uses sizeof(struct skb_shared_info) bytes 221 * at the end of skb->data, to avoid wasting a full cache line. 222 * This reduces memory use (skb->truesize). 223 */ 224 #define QEDE_FW_RX_ALIGN_END \ 225 max_t(u64, 1UL << QEDE_RX_ALIGN_SHIFT, \ 226 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 227 228 struct qede_stats stats; 229 #define QEDE_RSS_INDIR_INITED BIT(0) 230 #define QEDE_RSS_KEY_INITED BIT(1) 231 #define QEDE_RSS_CAPS_INITED BIT(2) 232 u32 rss_params_inited; /* bit-field to track initialized rss params */ 233 u16 rss_ind_table[128]; 234 u32 rss_key[10]; 235 u8 rss_caps; 236 237 u16 q_num_rx_buffers; /* Must be a power of two */ 238 u16 q_num_tx_buffers; /* Must be a power of two */ 239 240 bool gro_disable; 241 struct list_head vlan_list; 242 u16 configured_vlans; 243 u16 non_configured_vlans; 244 bool accept_any_vlan; 245 struct delayed_work sp_task; 246 unsigned long sp_flags; 247 u16 vxlan_dst_port; 248 u16 geneve_dst_port; 249 250 struct qede_arfs *arfs; 251 bool wol_enabled; 252 253 struct qede_rdma_dev rdma_info; 254 255 struct bpf_prog *xdp_prog; 256 257 unsigned long err_flags; 258 #define QEDE_ERR_IS_HANDLED 31 259 #define QEDE_ERR_ATTN_CLR_EN 0 260 #define QEDE_ERR_GET_DBG_INFO 1 261 #define QEDE_ERR_IS_RECOVERABLE 2 262 #define QEDE_ERR_WARN 3 263 264 struct qede_dump_info dump_info; 265 }; 266 267 enum QEDE_STATE { 268 QEDE_STATE_CLOSED, 269 QEDE_STATE_OPEN, 270 QEDE_STATE_RECOVERY, 271 }; 272 273 #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) 274 275 #define MAX_NUM_TC 8 276 #define MAX_NUM_PRI 8 277 278 /* The driver supports the new build_skb() API: 279 * RX ring buffer contains pointer to kmalloc() data only, 280 * skb are built only after the frame was DMA-ed. 281 */ 282 struct sw_rx_data { 283 struct page *data; 284 dma_addr_t mapping; 285 unsigned int page_offset; 286 }; 287 288 enum qede_agg_state { 289 QEDE_AGG_STATE_NONE = 0, 290 QEDE_AGG_STATE_START = 1, 291 QEDE_AGG_STATE_ERROR = 2 292 }; 293 294 struct qede_agg_info { 295 /* rx_buf is a data buffer that can be placed / consumed from rx bd 296 * chain. It has two purposes: We will preallocate the data buffer 297 * for each aggregation when we open the interface and will place this 298 * buffer on the rx-bd-ring when we receive TPA_START. We don't want 299 * to be in a state where allocation fails, as we can't reuse the 300 * consumer buffer in the rx-chain since FW may still be writing to it 301 * (since header needs to be modified for TPA). 302 * The second purpose is to keep a pointer to the bd buffer during 303 * aggregation. 304 */ 305 struct sw_rx_data buffer; 306 struct sk_buff *skb; 307 308 /* We need some structs from the start cookie until termination */ 309 u16 vlan_tag; 310 311 bool tpa_start_fail; 312 u8 state; 313 u8 frag_id; 314 315 u8 tunnel_type; 316 }; 317 318 struct qede_rx_queue { 319 __le16 *hw_cons_ptr; 320 void __iomem *hw_rxq_prod_addr; 321 322 /* Required for the allocation of replacement buffers */ 323 struct device *dev; 324 325 struct bpf_prog *xdp_prog; 326 327 u16 sw_rx_cons; 328 u16 sw_rx_prod; 329 330 u16 filled_buffers; 331 u8 data_direction; 332 u8 rxq_id; 333 334 /* Used once per each NAPI run */ 335 u16 num_rx_buffers; 336 337 u16 rx_headroom; 338 339 u32 rx_buf_size; 340 u32 rx_buf_seg_size; 341 342 struct sw_rx_data *sw_rx_ring; 343 struct qed_chain rx_bd_ring; 344 struct qed_chain rx_comp_ring ____cacheline_aligned; 345 346 /* GRO */ 347 struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM]; 348 349 /* Used once per each NAPI run */ 350 u64 rcv_pkts; 351 352 u64 rx_hw_errors; 353 u64 rx_alloc_errors; 354 u64 rx_ip_frags; 355 356 u64 xdp_no_pass; 357 358 void *handle; 359 struct xdp_rxq_info xdp_rxq; 360 }; 361 362 union db_prod { 363 struct eth_db_data data; 364 u32 raw; 365 }; 366 367 struct sw_tx_bd { 368 struct sk_buff *skb; 369 u8 flags; 370 /* Set on the first BD descriptor when there is a split BD */ 371 #define QEDE_TSO_SPLIT_BD BIT(0) 372 }; 373 374 struct sw_tx_xdp { 375 struct page *page; 376 dma_addr_t mapping; 377 }; 378 379 struct qede_tx_queue { 380 u8 is_xdp; 381 bool is_legacy; 382 u16 sw_tx_cons; 383 u16 sw_tx_prod; 384 u16 num_tx_buffers; /* Slowpath only */ 385 386 u64 xmit_pkts; 387 u64 stopped_cnt; 388 u64 tx_mem_alloc_err; 389 390 __le16 *hw_cons_ptr; 391 392 /* Needed for the mapping of packets */ 393 struct device *dev; 394 395 void __iomem *doorbell_addr; 396 union db_prod tx_db; 397 int index; /* Slowpath only */ 398 #define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \ 399 QEDE_MAX_TSS_CNT(edev)) 400 #define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev)) 401 #define QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx) ((edev)->fp_num_rx + \ 402 ((idx) % QEDE_TSS_COUNT(edev))) 403 #define QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx) ((idx) / QEDE_TSS_COUNT(edev)) 404 #define QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq) ((QEDE_TSS_COUNT(edev) * \ 405 (txq)->cos) + (txq)->index) 406 #define QEDE_NDEV_TXQ_ID_TO_TXQ(edev, idx) \ 407 (&((edev)->fp_array[QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx)].txq \ 408 [QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx)])) 409 #define QEDE_FP_TC0_TXQ(fp) (&((fp)->txq[0])) 410 411 /* Regular Tx requires skb + metadata for release purpose, 412 * while XDP requires the pages and the mapped address. 413 */ 414 union { 415 struct sw_tx_bd *skbs; 416 struct sw_tx_xdp *xdp; 417 } sw_tx_ring; 418 419 struct qed_chain tx_pbl; 420 421 /* Slowpath; Should be kept in end [unless missing padding] */ 422 void *handle; 423 u16 cos; 424 u16 ndev_txq_id; 425 }; 426 427 #define BD_UNMAP_ADDR(bd) HILO_U64(le32_to_cpu((bd)->addr.hi), \ 428 le32_to_cpu((bd)->addr.lo)) 429 #define BD_SET_UNMAP_ADDR_LEN(bd, maddr, len) \ 430 do { \ 431 (bd)->addr.hi = cpu_to_le32(upper_32_bits(maddr)); \ 432 (bd)->addr.lo = cpu_to_le32(lower_32_bits(maddr)); \ 433 (bd)->nbytes = cpu_to_le16(len); \ 434 } while (0) 435 #define BD_UNMAP_LEN(bd) (le16_to_cpu((bd)->nbytes)) 436 437 struct qede_fastpath { 438 struct qede_dev *edev; 439 #define QEDE_FASTPATH_TX BIT(0) 440 #define QEDE_FASTPATH_RX BIT(1) 441 #define QEDE_FASTPATH_XDP BIT(2) 442 #define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX) 443 u8 type; 444 u8 id; 445 u8 xdp_xmit; 446 struct napi_struct napi; 447 struct qed_sb_info *sb_info; 448 struct qede_rx_queue *rxq; 449 struct qede_tx_queue *txq; 450 struct qede_tx_queue *xdp_tx; 451 452 #define VEC_NAME_SIZE (sizeof_field(struct net_device, name) + 8) 453 char name[VEC_NAME_SIZE]; 454 }; 455 456 /* Debug print definitions */ 457 #define DP_NAME(edev) ((edev)->ndev->name) 458 459 #define XMIT_PLAIN 0 460 #define XMIT_L4_CSUM BIT(0) 461 #define XMIT_LSO BIT(1) 462 #define XMIT_ENC BIT(2) 463 #define XMIT_ENC_GSO_L4_CSUM BIT(3) 464 465 #define QEDE_CSUM_ERROR BIT(0) 466 #define QEDE_CSUM_UNNECESSARY BIT(1) 467 #define QEDE_TUNN_CSUM_UNNECESSARY BIT(2) 468 469 #define QEDE_SP_RECOVERY 0 470 #define QEDE_SP_RX_MODE 1 471 #define QEDE_SP_RSVD1 2 472 #define QEDE_SP_RSVD2 3 473 #define QEDE_SP_HW_ERR 4 474 #define QEDE_SP_ARFS_CONFIG 5 475 #define QEDE_SP_AER 7 476 477 #ifdef CONFIG_RFS_ACCEL 478 int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 479 u16 rxq_index, u32 flow_id); 480 #define QEDE_SP_TASK_POLL_DELAY (5 * HZ) 481 #endif 482 483 void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr); 484 void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev); 485 void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc); 486 void qede_free_arfs(struct qede_dev *edev); 487 int qede_alloc_arfs(struct qede_dev *edev); 488 int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info); 489 int qede_delete_flow_filter(struct qede_dev *edev, u64 cookie); 490 int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd); 491 int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info, 492 u32 *rule_locs); 493 int qede_get_arfs_filter_count(struct qede_dev *edev); 494 495 struct qede_reload_args { 496 void (*func)(struct qede_dev *edev, struct qede_reload_args *args); 497 union { 498 netdev_features_t features; 499 struct bpf_prog *new_prog; 500 u16 mtu; 501 } u; 502 }; 503 504 /* Datapath functions definition */ 505 netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev); 506 u16 qede_select_queue(struct net_device *dev, struct sk_buff *skb, 507 struct net_device *sb_dev); 508 netdev_features_t qede_features_check(struct sk_buff *skb, 509 struct net_device *dev, 510 netdev_features_t features); 511 int qede_alloc_rx_buffer(struct qede_rx_queue *rxq, bool allow_lazy); 512 int qede_free_tx_pkt(struct qede_dev *edev, 513 struct qede_tx_queue *txq, int *len); 514 int qede_poll(struct napi_struct *napi, int budget); 515 irqreturn_t qede_msix_fp_int(int irq, void *fp_cookie); 516 517 /* Filtering function definitions */ 518 void qede_force_mac(void *dev, u8 *mac, bool forced); 519 void qede_udp_ports_update(void *dev, u16 vxlan_port, u16 geneve_port); 520 int qede_set_mac_addr(struct net_device *ndev, void *p); 521 522 int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid); 523 int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid); 524 void qede_vlan_mark_nonconfigured(struct qede_dev *edev); 525 int qede_configure_vlan_filters(struct qede_dev *edev); 526 527 netdev_features_t qede_fix_features(struct net_device *dev, 528 netdev_features_t features); 529 int qede_set_features(struct net_device *dev, netdev_features_t features); 530 void qede_set_rx_mode(struct net_device *ndev); 531 void qede_config_rx_mode(struct net_device *ndev); 532 void qede_fill_rss_params(struct qede_dev *edev, 533 struct qed_update_vport_rss_params *rss, u8 *update); 534 535 void qede_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti); 536 void qede_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti); 537 538 int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp); 539 540 #ifdef CONFIG_DCB 541 void qede_set_dcbnl_ops(struct net_device *ndev); 542 #endif 543 544 void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level); 545 void qede_set_ethtool_ops(struct net_device *netdev); 546 void qede_set_udp_tunnels(struct qede_dev *edev); 547 void qede_reload(struct qede_dev *edev, 548 struct qede_reload_args *args, bool is_locked); 549 int qede_change_mtu(struct net_device *dev, int new_mtu); 550 void qede_fill_by_demand_stats(struct qede_dev *edev); 551 void __qede_lock(struct qede_dev *edev); 552 void __qede_unlock(struct qede_dev *edev); 553 bool qede_has_rx_work(struct qede_rx_queue *rxq); 554 int qede_txq_has_work(struct qede_tx_queue *txq); 555 void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count); 556 void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq); 557 int qede_add_tc_flower_fltr(struct qede_dev *edev, __be16 proto, 558 struct flow_cls_offload *f); 559 560 #define RX_RING_SIZE_POW 13 561 #define RX_RING_SIZE ((u16)BIT(RX_RING_SIZE_POW)) 562 #define NUM_RX_BDS_MAX (RX_RING_SIZE - 1) 563 #define NUM_RX_BDS_MIN 128 564 #define NUM_RX_BDS_KDUMP_MIN 63 565 #define NUM_RX_BDS_DEF ((u16)BIT(10) - 1) 566 567 #define TX_RING_SIZE_POW 13 568 #define TX_RING_SIZE ((u16)BIT(TX_RING_SIZE_POW)) 569 #define NUM_TX_BDS_MAX (TX_RING_SIZE - 1) 570 #define NUM_TX_BDS_MIN 128 571 #define NUM_TX_BDS_KDUMP_MIN 63 572 #define NUM_TX_BDS_DEF NUM_TX_BDS_MAX 573 574 #define QEDE_MIN_PKT_LEN 64 575 #define QEDE_RX_HDR_SIZE 256 576 #define QEDE_MAX_JUMBO_PACKET_SIZE 9600 577 #define for_each_queue(i) for (i = 0; i < edev->num_queues; i++) 578 #define for_each_cos_in_txq(edev, var) \ 579 for ((var) = 0; (var) < (edev)->dev_info.num_tc; (var)++) 580 581 #endif /* _QEDE_H_ */ 582