1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #ifndef _MANA_H 5 #define _MANA_H 6 7 #include <linux/dim.h> 8 #include <net/xdp.h> 9 #include <net/net_shaper.h> 10 11 #include "gdma.h" 12 #include "hw_channel.h" 13 14 /* Microsoft Azure Network Adapter (MANA)'s definitions 15 * 16 * Structures labeled with "HW DATA" are exchanged with the hardware. All of 17 * them are naturally aligned and hence don't need __packed. 18 */ 19 20 /* MANA protocol version */ 21 #define MANA_MAJOR_VERSION 0 22 #define MANA_MINOR_VERSION 1 23 #define MANA_MICRO_VERSION 1 24 25 typedef u64 mana_handle_t; 26 #define INVALID_MANA_HANDLE ((mana_handle_t)-1) 27 28 enum TRI_STATE { 29 TRI_STATE_UNKNOWN = -1, 30 TRI_STATE_FALSE = 0, 31 TRI_STATE_TRUE = 1 32 }; 33 34 /* MANA ethtool private flag bit positions */ 35 enum mana_priv_flag_bits { 36 MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF = 0, 37 MANA_PRIV_FLAG_MAX, 38 }; 39 40 /* Number of entries for hardware indirection table must be in power of 2 */ 41 #define MANA_INDIRECT_TABLE_MAX_SIZE 512 42 #define MANA_INDIRECT_TABLE_DEF_SIZE 64 43 44 /* The Toeplitz hash key's length in bytes: should be multiple of 8 */ 45 #define MANA_HASH_KEY_SIZE 40 46 47 #define COMP_ENTRY_SIZE 64 48 49 /* This Max value for RX buffers is derived from __alloc_page()'s max page 50 * allocation calculation. It allows maximum 2^MAX_PAGE_ORDER pages. RX 51 * buffer size beyond this value gets rejected by __alloc_page() call. 52 */ 53 #define MAX_RX_BUFFERS_PER_QUEUE 8192 54 #define DEF_RX_BUFFERS_PER_QUEUE 1024 55 #define MIN_RX_BUFFERS_PER_QUEUE 128 56 57 /* This max value for TX buffers is derived as the maximum allocatable 58 * pages supported on host per guest through testing. TX buffer size beyond 59 * this value is rejected by the hardware. 60 */ 61 #define MAX_TX_BUFFERS_PER_QUEUE 16384 62 #define DEF_TX_BUFFERS_PER_QUEUE 256 63 #define MIN_TX_BUFFERS_PER_QUEUE 128 64 65 #define EQ_SIZE (8 * MANA_PAGE_SIZE) 66 67 #define LOG2_EQ_THROTTLE 3 68 69 #define MAX_PORTS_IN_MANA_DEV 256 70 71 /* Maximum number of PPIs per coalesced CQE */ 72 #define MANA_RXCOMP_OOB_NUM_PPI 4 73 74 /* 8-pkt mode packs up to 2 packets per PPI entry */ 75 #define MANA_CQE_COAL_PKTS_8 8 76 77 /* Default/max interrupt moderation settings */ 78 #define MANA_INTR_MODR_USEC_DEF 0 79 #define MANA_INTR_MODR_COMP_DEF 0 80 81 #define MANA_ADAPTIVE_RX_DEF true 82 #define MANA_ADAPTIVE_TX_DEF true 83 84 /* DIM doorbell value field layout */ 85 #define MANA_INTR_MODR_USEC_MAX GENMASK(9, 0) 86 #define MANA_INTR_MODR_USEC_VLD BIT(15) 87 #define MANA_INTR_MODR_COMP_MAX GENMASK(7, 0) 88 #define MANA_INTR_MODR_COMP_MASK GENMASK(23, 16) 89 90 /* Update this count whenever the respective structures are changed */ 91 #define MANA_STATS_RX_COUNT (6 + MANA_CQE_COAL_PKTS_8 - 1) 92 #define MANA_STATS_TX_COUNT 11 93 94 #define MANA_RX_FRAG_ALIGNMENT 64 95 96 struct mana_stats_rx { 97 u64 packets; 98 u64 bytes; 99 u64 xdp_drop; 100 u64 xdp_tx; 101 u64 xdp_redirect; 102 u64 pkt_len0_err; 103 u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1]; 104 struct u64_stats_sync syncp; 105 }; 106 107 struct mana_stats_tx { 108 u64 packets; 109 u64 bytes; 110 u64 xdp_xmit; 111 u64 tso_packets; 112 u64 tso_bytes; 113 u64 tso_inner_packets; 114 u64 tso_inner_bytes; 115 u64 short_pkt_fmt; 116 u64 long_pkt_fmt; 117 u64 csum_partial; 118 u64 mana_map_err; 119 struct u64_stats_sync syncp; 120 }; 121 122 struct mana_txq { 123 struct gdma_queue *gdma_sq; 124 125 union { 126 u32 gdma_txq_id; 127 struct { 128 u32 reserved1 : 10; 129 u32 vsq_frame : 14; 130 u32 reserved2 : 8; 131 }; 132 }; 133 134 u16 vp_offset; 135 136 struct net_device *ndev; 137 138 /* The SKBs are sent to the HW and we are waiting for the CQEs. */ 139 struct sk_buff_head pending_skbs; 140 struct netdev_queue *net_txq; 141 142 atomic_t pending_sends; 143 144 bool napi_initialized; 145 146 struct mana_stats_tx stats; 147 }; 148 149 /* skb data and frags dma mappings */ 150 struct mana_skb_head { 151 /* GSO pkts may have 2 SGEs for the linear part*/ 152 dma_addr_t dma_handle[MAX_SKB_FRAGS + 2]; 153 154 u32 size[MAX_SKB_FRAGS + 2]; 155 }; 156 157 #define MANA_HEADROOM sizeof(struct mana_skb_head) 158 159 enum mana_tx_pkt_format { 160 MANA_SHORT_PKT_FMT = 0, 161 MANA_LONG_PKT_FMT = 1, 162 }; 163 164 struct mana_tx_short_oob { 165 u32 pkt_fmt : 2; 166 u32 is_outer_ipv4 : 1; 167 u32 is_outer_ipv6 : 1; 168 u32 comp_iphdr_csum : 1; 169 u32 comp_tcp_csum : 1; 170 u32 comp_udp_csum : 1; 171 u32 supress_txcqe_gen : 1; 172 u32 vcq_num : 24; 173 174 u32 trans_off : 10; /* Transport header offset */ 175 u32 vsq_frame : 14; 176 u32 short_vp_offset : 8; 177 }; /* HW DATA */ 178 179 struct mana_tx_long_oob { 180 u32 is_encap : 1; 181 u32 inner_is_ipv6 : 1; 182 u32 inner_tcp_opt : 1; 183 u32 inject_vlan_pri_tag : 1; 184 u32 reserved1 : 12; 185 u32 pcp : 3; /* 802.1Q */ 186 u32 dei : 1; /* 802.1Q */ 187 u32 vlan_id : 12; /* 802.1Q */ 188 189 u32 inner_frame_offset : 10; 190 u32 inner_ip_rel_offset : 6; 191 u32 long_vp_offset : 12; 192 u32 reserved2 : 4; 193 194 u32 reserved3; 195 u32 reserved4; 196 }; /* HW DATA */ 197 198 struct mana_tx_oob { 199 struct mana_tx_short_oob s_oob; 200 struct mana_tx_long_oob l_oob; 201 }; /* HW DATA */ 202 203 enum mana_cq_type { 204 MANA_CQ_TYPE_RX, 205 MANA_CQ_TYPE_TX, 206 }; 207 208 enum mana_cqe_type { 209 CQE_INVALID = 0, 210 CQE_RX_OKAY = 1, 211 CQE_RX_COALESCED_4 = 2, 212 CQE_RX_OBJECT_FENCE = 3, 213 CQE_RX_TRUNCATED = 4, 214 CQE_RX_COALESCED_8 = 7, 215 216 CQE_TX_OKAY = 32, 217 CQE_TX_SA_DROP = 33, 218 CQE_TX_MTU_DROP = 34, 219 CQE_TX_INVALID_OOB = 35, 220 CQE_TX_INVALID_ETH_TYPE = 36, 221 CQE_TX_HDR_PROCESSING_ERROR = 37, 222 CQE_TX_VF_DISABLED = 38, 223 CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39, 224 CQE_TX_VPORT_DISABLED = 40, 225 CQE_TX_VLAN_TAGGING_VIOLATION = 41, 226 }; 227 228 #define MANA_CQE_COMPLETION 1 229 230 struct mana_cqe_header { 231 u32 cqe_type : 6; 232 u32 client_type : 2; 233 u32 vendor_err : 24; 234 }; /* HW DATA */ 235 236 /* NDIS HASH Types */ 237 #define NDIS_HASH_IPV4 BIT(0) 238 #define NDIS_HASH_TCP_IPV4 BIT(1) 239 #define NDIS_HASH_UDP_IPV4 BIT(2) 240 #define NDIS_HASH_IPV6 BIT(3) 241 #define NDIS_HASH_TCP_IPV6 BIT(4) 242 #define NDIS_HASH_UDP_IPV6 BIT(5) 243 #define NDIS_HASH_IPV6_EX BIT(6) 244 #define NDIS_HASH_TCP_IPV6_EX BIT(7) 245 #define NDIS_HASH_UDP_IPV6_EX BIT(8) 246 247 #define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX) 248 #define MANA_HASH_L4 \ 249 (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \ 250 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX) 251 #define MANA_HASH_ENABLE_SUPPORTED \ 252 (NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | \ 253 NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6 | NDIS_HASH_UDP_IPV6) 254 255 /* Read PPI in two different layouts based on cqe_type */ 256 union mana_rxcomp_perpkt_info { 257 struct { 258 u32 pkt_len : 16; 259 u32 reserved1 : 16; 260 u32 reserved2; 261 u32 pkt_hash; 262 }; 263 264 /* Up to two pkts per PPI entry */ 265 struct { 266 u32 pkt_hash0; 267 u16 pkt_len0; 268 u16 pkt_len1; 269 u32 pkt_hash1; 270 }; 271 }; /* HW DATA */ 272 273 /* Receive completion OOB */ 274 struct mana_rxcomp_oob { 275 struct mana_cqe_header cqe_hdr; 276 277 u32 rx_vlan_id : 12; 278 u32 rx_vlantag_present : 1; 279 u32 rx_outer_iphdr_csum_succeed : 1; 280 u32 rx_outer_iphdr_csum_fail : 1; 281 u32 reserved1 : 1; 282 u32 rx_hashtype : 9; 283 u32 rx_iphdr_csum_succeed : 1; 284 u32 rx_iphdr_csum_fail : 1; 285 u32 rx_tcp_csum_succeed : 1; 286 u32 rx_tcp_csum_fail : 1; 287 u32 rx_udp_csum_succeed : 1; 288 u32 rx_udp_csum_fail : 1; 289 u32 reserved2 : 1; 290 291 union mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI]; 292 293 u32 rx_wqe_offset; 294 }; /* HW DATA */ 295 296 struct mana_tx_comp_oob { 297 struct mana_cqe_header cqe_hdr; 298 299 u32 tx_data_offset; 300 301 u32 tx_sgl_offset : 5; 302 u32 tx_wqe_offset : 27; 303 304 u32 reserved[12]; 305 }; /* HW DATA */ 306 307 struct mana_rxq; 308 309 #define CQE_POLLING_BUFFER 512 310 311 struct mana_cq { 312 struct gdma_queue *gdma_cq; 313 314 /* Cache the CQ id (used to verify if each CQE comes to the right CQ. */ 315 u32 gdma_id; 316 317 /* Type of the CQ: TX or RX */ 318 enum mana_cq_type type; 319 320 /* Pointer to the mana_rxq that is pushing RX CQEs to the queue. 321 * Only and must be non-NULL if type is MANA_CQ_TYPE_RX. 322 */ 323 struct mana_rxq *rxq; 324 325 /* Pointer to the mana_txq that is pushing TX CQEs to the queue. 326 * Only and must be non-NULL if type is MANA_CQ_TYPE_TX. 327 */ 328 struct mana_txq *txq; 329 330 /* Buffer which the CQ handler can copy the CQE's into. */ 331 struct gdma_comp gdma_comp_buf[CQE_POLLING_BUFFER]; 332 333 /* NAPI data */ 334 struct napi_struct napi; 335 int work_done; 336 int work_done_since_doorbell; 337 int budget; 338 339 /* DIM - Dynamic Interrupt Moderation */ 340 struct dim dim; 341 u16 dim_event_ctr; 342 343 /* Cumulative TX completions fed to DIM. Updated and read only in 344 * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they 345 * measure the hardware completion rate and need no u64_stats_sync. 346 */ 347 u64 tx_dim_pkts; 348 u64 tx_dim_bytes; 349 }; 350 351 struct mana_recv_buf_oob { 352 /* A valid GDMA work request representing the data buffer. */ 353 struct gdma_wqe_request wqe_req; 354 355 void *buf_va; 356 bool from_pool; /* allocated from a page pool */ 357 /* head page of the page_pool fragment; valid only when 358 * from_pool && frag_count > 1. 359 */ 360 struct page *pp_page; 361 /* Fragment offset plus rxq->headroom, passed to 362 * page_pool_dma_sync_for_cpu(). 363 */ 364 u32 dma_sync_offset; 365 366 /* SGL of the buffer going to be sent as part of the work request. */ 367 u32 num_sge; 368 struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES]; 369 370 /* Required to store the result of mana_gd_post_work_request. 371 * gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the 372 * work queue when the WQE is consumed. 373 */ 374 struct gdma_posted_wqe_info wqe_inf; 375 }; 376 377 #define MANA_RXBUF_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) \ 378 + ETH_HLEN) 379 380 /* Headroom an RX buffer has to reserve while XDP is attached: the XDP program 381 * needs XDP_PACKET_HEADROOM, and the TX path needs LL_RESERVED_SPACE() (see 382 * mana_get_rxbuf_headroom()). LL_RESERVED_SPACE() grows with MAX_SKB_FRAGS and 383 * exceeds XDP_PACKET_HEADROOM once CONFIG_MAX_SKB_FRAGS is 19 or more. 384 */ 385 static inline u32 mana_xdp_headroom(struct net_device *ndev) 386 { 387 return max_t(u32, LL_RESERVED_SPACE(ndev), XDP_PACKET_HEADROOM); 388 } 389 390 #define MANA_XDP_MTU_MAX(ndev) \ 391 (PAGE_SIZE - MANA_RXBUF_PAD - mana_xdp_headroom(ndev)) 392 393 struct mana_rxq { 394 struct gdma_queue *gdma_rq; 395 /* Cache the gdma receive queue id */ 396 u32 gdma_id; 397 398 /* Index of RQ in the vPort, not gdma receive queue id */ 399 u32 rxq_idx; 400 401 u32 datasize; 402 u32 alloc_size; 403 u32 headroom; 404 u32 frag_count; 405 406 mana_handle_t rxobj; 407 408 struct mana_cq rx_cq; 409 410 struct completion fence_event; 411 412 struct net_device *ndev; 413 414 /* Total number of receive buffers to be allocated */ 415 u32 num_rx_buf; 416 417 u32 buf_index; 418 419 struct mana_stats_rx stats; 420 421 struct bpf_prog __rcu *bpf_prog; 422 struct xdp_rxq_info xdp_rxq; 423 void *xdp_save_va; /* for reusing */ 424 bool xdp_flush; 425 int xdp_rc; /* XDP redirect return code */ 426 427 struct page_pool *page_pool; 428 struct dentry *mana_rx_debugfs; 429 430 /* MUST BE THE LAST MEMBER: 431 * Each receive buffer has an associated mana_recv_buf_oob. 432 */ 433 struct mana_recv_buf_oob rx_oobs[] __counted_by(num_rx_buf); 434 }; 435 436 struct mana_tx_qp { 437 struct mana_txq txq; 438 439 struct mana_cq tx_cq; 440 441 mana_handle_t tx_object; 442 443 struct dentry *mana_tx_debugfs; 444 }; 445 446 struct mana_ethtool_stats { 447 u64 stop_queue; 448 u64 wake_queue; 449 u64 tx_cqe_err; 450 u64 tx_cqe_unknown_type; 451 u64 tx_linear_pkt_cnt; 452 u64 rx_cqe_unknown_type; 453 }; 454 455 struct mana_ethtool_hc_stats { 456 u64 hc_rx_discards_no_wqe; 457 u64 hc_rx_err_vport_disabled; 458 u64 hc_rx_bytes; 459 u64 hc_rx_ucast_pkts; 460 u64 hc_rx_ucast_bytes; 461 u64 hc_rx_bcast_pkts; 462 u64 hc_rx_bcast_bytes; 463 u64 hc_rx_mcast_pkts; 464 u64 hc_rx_mcast_bytes; 465 u64 hc_tx_err_gf_disabled; 466 u64 hc_tx_err_vport_disabled; 467 u64 hc_tx_err_inval_vportoffset_pkt; 468 u64 hc_tx_err_vlan_enforcement; 469 u64 hc_tx_err_eth_type_enforcement; 470 u64 hc_tx_err_sa_enforcement; 471 u64 hc_tx_err_sqpdid_enforcement; 472 u64 hc_tx_err_cqpdid_enforcement; 473 u64 hc_tx_err_mtu_violation; 474 u64 hc_tx_err_inval_oob; 475 u64 hc_tx_bytes; 476 u64 hc_tx_ucast_pkts; 477 u64 hc_tx_ucast_bytes; 478 u64 hc_tx_bcast_pkts; 479 u64 hc_tx_bcast_bytes; 480 u64 hc_tx_mcast_pkts; 481 u64 hc_tx_mcast_bytes; 482 u64 hc_tx_err_gdma; 483 }; 484 485 struct mana_ethtool_phy_stats { 486 /* Drop Counters */ 487 u64 rx_pkt_drop_phy; 488 u64 tx_pkt_drop_phy; 489 490 /* Per TC traffic Counters */ 491 u64 rx_pkt_tc0_phy; 492 u64 tx_pkt_tc0_phy; 493 u64 rx_pkt_tc1_phy; 494 u64 tx_pkt_tc1_phy; 495 u64 rx_pkt_tc2_phy; 496 u64 tx_pkt_tc2_phy; 497 u64 rx_pkt_tc3_phy; 498 u64 tx_pkt_tc3_phy; 499 u64 rx_pkt_tc4_phy; 500 u64 tx_pkt_tc4_phy; 501 u64 rx_pkt_tc5_phy; 502 u64 tx_pkt_tc5_phy; 503 u64 rx_pkt_tc6_phy; 504 u64 tx_pkt_tc6_phy; 505 u64 rx_pkt_tc7_phy; 506 u64 tx_pkt_tc7_phy; 507 508 u64 rx_byte_tc0_phy; 509 u64 tx_byte_tc0_phy; 510 u64 rx_byte_tc1_phy; 511 u64 tx_byte_tc1_phy; 512 u64 rx_byte_tc2_phy; 513 u64 tx_byte_tc2_phy; 514 u64 rx_byte_tc3_phy; 515 u64 tx_byte_tc3_phy; 516 u64 rx_byte_tc4_phy; 517 u64 tx_byte_tc4_phy; 518 u64 rx_byte_tc5_phy; 519 u64 tx_byte_tc5_phy; 520 u64 rx_byte_tc6_phy; 521 u64 tx_byte_tc6_phy; 522 u64 rx_byte_tc7_phy; 523 u64 tx_byte_tc7_phy; 524 525 /* Per TC pause Counters */ 526 u64 rx_pause_tc0_phy; 527 u64 tx_pause_tc0_phy; 528 u64 rx_pause_tc1_phy; 529 u64 tx_pause_tc1_phy; 530 u64 rx_pause_tc2_phy; 531 u64 tx_pause_tc2_phy; 532 u64 rx_pause_tc3_phy; 533 u64 tx_pause_tc3_phy; 534 u64 rx_pause_tc4_phy; 535 u64 tx_pause_tc4_phy; 536 u64 rx_pause_tc5_phy; 537 u64 tx_pause_tc5_phy; 538 u64 rx_pause_tc6_phy; 539 u64 tx_pause_tc6_phy; 540 u64 rx_pause_tc7_phy; 541 u64 tx_pause_tc7_phy; 542 }; 543 544 struct mana_context { 545 struct gdma_dev *gdma_dev; 546 547 u16 num_ports; 548 u8 bm_hostmode; 549 550 struct mana_ethtool_hc_stats hc_stats; 551 struct workqueue_struct *per_port_queue_reset_wq; 552 /* Workqueue for querying hardware stats */ 553 struct delayed_work gf_stats_work; 554 bool hwc_timeout_occurred; 555 556 struct net_device *ports[MAX_PORTS_IN_MANA_DEV]; 557 558 /* Link state change work */ 559 struct work_struct link_change_work; 560 u32 link_event; 561 }; 562 563 struct mana_port_context { 564 struct mana_context *ac; 565 struct net_device *ndev; 566 struct work_struct queue_reset_work; 567 568 /* Debug knob to log TX timeout but skip recovery reset */ 569 bool tx_timeout_skip_reset; 570 571 u8 mac_addr[ETH_ALEN]; 572 573 struct mana_eq *eqs; 574 struct dentry *mana_eqs_debugfs; 575 576 enum TRI_STATE rss_state; 577 578 mana_handle_t default_rxobj; 579 bool tx_shortform_allowed; 580 u16 tx_vp_offset; 581 582 struct mana_tx_qp **tx_qp; 583 584 /* Indirection Table for RX & TX. The values are queue indexes */ 585 u32 *indir_table; 586 u32 indir_table_sz; 587 588 /* Indirection table containing RxObject Handles */ 589 mana_handle_t *rxobj_table; 590 591 /* Hash key used by the NIC */ 592 u8 hashkey[MANA_HASH_KEY_SIZE]; 593 594 /* This points to an array of num_queues of RQ pointers. */ 595 struct mana_rxq **rxqs; 596 597 /* pre-allocated rx buffer array */ 598 void **rxbufs_pre; 599 dma_addr_t *das_pre; 600 int rxbpre_total; 601 u32 rxbpre_datasize; 602 u32 rxbpre_alloc_size; 603 u32 rxbpre_headroom; 604 u32 rxbpre_frag_count; 605 606 u32 priv_flags; 607 608 struct bpf_prog *bpf_prog; 609 610 /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */ 611 unsigned int max_queues; 612 unsigned int num_queues; 613 614 unsigned int rx_queue_size; 615 unsigned int tx_queue_size; 616 617 mana_handle_t port_handle; 618 mana_handle_t pf_filter_handle; 619 620 /* Mutex for sharing access to vport_use_count */ 621 struct mutex vport_mutex; 622 int vport_use_count; 623 624 /* Set by mana_set_channels() under vport_mutex to block RDMA 625 * from grabbing the vport during the detach/attach window. 626 * Checked by mana_cfg_vport() when called from the RDMA path. 627 */ 628 bool channel_changing; 629 630 /* Net shaper handle*/ 631 struct net_shaper_handle handle; 632 633 u16 port_idx; 634 /* Currently configured speed (mbps) */ 635 u32 speed; 636 /* Maximum speed supported by the SKU (mbps) */ 637 u32 max_speed; 638 /* 1 = not queried, 0 = cached success, negative = permanent error. 639 * Protected by the netdev instance lock. 640 */ 641 int link_cfg_error; 642 643 bool port_is_up; 644 bool port_st_save; /* Saved port state */ 645 646 u8 cqe_coalescing_enable; 647 u8 cqe8_coalescing_enable; 648 u32 cqe_coalescing_timeout_ns; 649 650 /* Interrupt moderation settings */ 651 u16 intr_modr_rx_usec; 652 u16 intr_modr_rx_comp; 653 u16 intr_modr_tx_usec; 654 u16 intr_modr_tx_comp; 655 656 bool rx_dim_enabled; 657 bool tx_dim_enabled; 658 659 struct mana_ethtool_stats eth_stats; 660 661 struct mana_ethtool_phy_stats phy_stats; 662 663 /* Debugfs */ 664 struct dentry *mana_port_debugfs; 665 666 /* Cached vport/steering config for debugfs */ 667 u32 vport_max_sq; 668 u32 vport_max_rq; 669 u32 steer_rx; 670 u32 steer_rss; 671 bool steer_update_tab; 672 u32 steer_cqe_coalescing; 673 }; 674 675 netdev_tx_t __mana_start_xmit(struct sk_buff *skb, struct net_device *ndev, 676 bool xdp); 677 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); 678 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, 679 bool update_hash, bool update_tab); 680 int mana_disable_vport_rx(struct mana_port_context *apc); 681 682 int mana_alloc_queues(struct net_device *ndev); 683 int mana_attach(struct net_device *ndev); 684 int mana_detach(struct net_device *ndev, bool from_close); 685 686 void mana_dim_change(struct mana_cq *cq, bool enable); 687 688 int mana_probe(struct gdma_dev *gd, bool resuming); 689 void mana_remove(struct gdma_dev *gd, bool suspending); 690 691 int mana_rdma_probe(struct gdma_dev *gd); 692 void mana_rdma_remove(struct gdma_dev *gd); 693 694 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); 695 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, 696 u32 flags); 697 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, 698 struct xdp_buff *xdp, void *buf_va, uint pkt_len); 699 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc); 700 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog); 701 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf); 702 int mana_query_gf_stats(struct mana_context *ac); 703 int mana_query_link_cfg(struct mana_port_context *apc); 704 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 705 int enable_clamping); 706 void mana_query_phy_stats(struct mana_port_context *apc); 707 bool mana_single_rxbuf_per_page_forced(struct mana_port_context *apc, u32 mtu); 708 int mana_pre_alloc_rxbufs(struct mana_port_context *apc, int mtu, int num_queues); 709 void mana_pre_dealloc_rxbufs(struct mana_port_context *apc); 710 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc); 711 712 extern const struct ethtool_ops mana_ethtool_ops; 713 extern struct dentry *mana_debugfs_root; 714 715 /* A CQ can be created not associated with any EQ */ 716 #define GDMA_CQ_NO_EQ 0xffff 717 718 struct mana_obj_spec { 719 u32 queue_index; 720 u64 gdma_region; 721 u32 queue_size; 722 u32 attached_eq; 723 u32 modr_ctx_id; 724 u8 req_cq_moderation; 725 u16 cq_moderation_comp; 726 u16 cq_moderation_usec; 727 }; 728 729 enum mana_command_code { 730 MANA_QUERY_DEV_CONFIG = 0x20001, 731 MANA_QUERY_GF_STAT = 0x20002, 732 MANA_CONFIG_VPORT_TX = 0x20003, 733 MANA_CREATE_WQ_OBJ = 0x20004, 734 MANA_DESTROY_WQ_OBJ = 0x20005, 735 MANA_FENCE_RQ = 0x20006, 736 MANA_CONFIG_VPORT_RX = 0x20007, 737 MANA_QUERY_VPORT_CONFIG = 0x20008, 738 MANA_QUERY_LINK_CONFIG = 0x2000A, 739 MANA_SET_BW_CLAMP = 0x2000B, 740 MANA_QUERY_PHY_STAT = 0x2000c, 741 742 /* Privileged commands for the PF mode */ 743 MANA_REGISTER_FILTER = 0x28000, 744 MANA_DEREGISTER_FILTER = 0x28001, 745 MANA_REGISTER_HW_PORT = 0x28003, 746 MANA_DEREGISTER_HW_PORT = 0x28004, 747 }; 748 749 /* Query Link Configuration*/ 750 struct mana_query_link_config_req { 751 struct gdma_req_hdr hdr; 752 mana_handle_t vport; 753 }; /* HW DATA */ 754 755 struct mana_query_link_config_resp { 756 struct gdma_resp_hdr hdr; 757 u32 qos_speed_mbps; 758 u8 qos_unconfigured; 759 u8 reserved1[3]; 760 u32 link_speed_mbps; 761 u8 reserved2[4]; 762 }; /* HW DATA */ 763 764 /* Set Bandwidth Clamp*/ 765 struct mana_set_bw_clamp_req { 766 struct gdma_req_hdr hdr; 767 mana_handle_t vport; 768 enum TRI_STATE enable_clamping; 769 u32 link_speed_mbps; 770 }; /* HW DATA */ 771 772 struct mana_set_bw_clamp_resp { 773 struct gdma_resp_hdr hdr; 774 u8 qos_unconfigured; 775 u8 reserved[7]; 776 }; /* HW DATA */ 777 778 /* Query Device Configuration */ 779 struct mana_query_device_cfg_req { 780 struct gdma_req_hdr hdr; 781 782 /* MANA Nic Driver Capability flags */ 783 u64 mn_drv_cap_flags1; 784 u64 mn_drv_cap_flags2; 785 u64 mn_drv_cap_flags3; 786 u64 mn_drv_cap_flags4; 787 788 u32 proto_major_ver; 789 u32 proto_minor_ver; 790 u32 proto_micro_ver; 791 792 u32 reserved; 793 }; /* HW DATA */ 794 795 #define MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED BIT(5) 796 797 struct mana_query_device_cfg_resp { 798 struct gdma_resp_hdr hdr; 799 800 u64 pf_cap_flags1; 801 u64 pf_cap_flags2; 802 u64 pf_cap_flags3; 803 u64 pf_cap_flags4; 804 805 u16 max_num_vports; 806 u8 bm_hostmode; /* response v3: Bare Metal Host Mode */ 807 u8 reserved; 808 u32 max_num_eqs; 809 810 /* response v2: */ 811 u16 adapter_mtu; 812 u16 reserved2; 813 u32 reserved3; 814 }; /* HW DATA */ 815 816 /* Query vPort Configuration */ 817 struct mana_query_vport_cfg_req { 818 struct gdma_req_hdr hdr; 819 u32 vport_index; 820 }; /* HW DATA */ 821 822 struct mana_query_vport_cfg_resp { 823 struct gdma_resp_hdr hdr; 824 u32 max_num_sq; 825 u32 max_num_rq; 826 u32 num_indirection_ent; 827 u32 reserved1; 828 u8 mac_addr[6]; 829 u8 reserved2[2]; 830 mana_handle_t vport; 831 }; /* HW DATA */ 832 833 /* Configure vPort */ 834 struct mana_config_vport_req { 835 struct gdma_req_hdr hdr; 836 mana_handle_t vport; 837 u32 pdid; 838 u32 doorbell_pageid; 839 }; /* HW DATA */ 840 841 struct mana_config_vport_resp { 842 struct gdma_resp_hdr hdr; 843 u16 tx_vport_offset; 844 u8 short_form_allowed; 845 u8 reserved; 846 }; /* HW DATA */ 847 848 /* Create WQ Object */ 849 struct mana_create_wqobj_req { 850 struct gdma_req_hdr hdr; 851 mana_handle_t vport; 852 u32 wq_type; 853 u32 reserved; 854 u64 wq_gdma_region; 855 u64 cq_gdma_region; 856 u32 wq_size; 857 u32 cq_size; 858 u32 cq_moderation_ctx_id; 859 u32 cq_parent_qid; 860 861 /* V2 */ 862 u8 allow_rqwqe_chain; 863 864 /* V3 */ 865 u8 req_cq_moderation; 866 u16 cq_moderation_comp; 867 u16 cq_moderation_usec; 868 u8 reserved2[2]; 869 }; /* HW DATA */ 870 871 struct mana_create_wqobj_resp { 872 struct gdma_resp_hdr hdr; 873 u32 wq_id; 874 u32 cq_id; 875 mana_handle_t wq_obj; 876 877 /* V2 */ 878 u16 cq_moderation_comp; 879 u16 cq_moderation_usec; 880 u8 cq_moderation_enabled; 881 u8 reserved1[3]; 882 }; /* HW DATA */ 883 884 /* Destroy WQ Object */ 885 struct mana_destroy_wqobj_req { 886 struct gdma_req_hdr hdr; 887 u32 wq_type; 888 u32 reserved; 889 mana_handle_t wq_obj_handle; 890 }; /* HW DATA */ 891 892 struct mana_destroy_wqobj_resp { 893 struct gdma_resp_hdr hdr; 894 }; /* HW DATA */ 895 896 /* Fence RQ */ 897 struct mana_fence_rq_req { 898 struct gdma_req_hdr hdr; 899 mana_handle_t wq_obj_handle; 900 }; /* HW DATA */ 901 902 struct mana_fence_rq_resp { 903 struct gdma_resp_hdr hdr; 904 }; /* HW DATA */ 905 906 /* Query stats RQ */ 907 struct mana_query_gf_stat_req { 908 struct gdma_req_hdr hdr; 909 u64 req_stats; 910 }; /* HW DATA */ 911 912 struct mana_query_gf_stat_resp { 913 struct gdma_resp_hdr hdr; 914 u64 reported_stats; 915 /* rx errors/discards */ 916 u64 rx_discards_nowqe; 917 u64 rx_err_vport_disabled; 918 /* rx bytes/packets */ 919 u64 hc_rx_bytes; 920 u64 hc_rx_ucast_pkts; 921 u64 hc_rx_ucast_bytes; 922 u64 hc_rx_bcast_pkts; 923 u64 hc_rx_bcast_bytes; 924 u64 hc_rx_mcast_pkts; 925 u64 hc_rx_mcast_bytes; 926 /* tx errors */ 927 u64 tx_err_gf_disabled; 928 u64 tx_err_vport_disabled; 929 u64 tx_err_inval_vport_offset_pkt; 930 u64 tx_err_vlan_enforcement; 931 u64 tx_err_ethtype_enforcement; 932 u64 tx_err_SA_enforcement; 933 u64 tx_err_SQPDID_enforcement; 934 u64 tx_err_CQPDID_enforcement; 935 u64 tx_err_mtu_violation; 936 u64 tx_err_inval_oob; 937 /* tx bytes/packets */ 938 u64 hc_tx_bytes; 939 u64 hc_tx_ucast_pkts; 940 u64 hc_tx_ucast_bytes; 941 u64 hc_tx_bcast_pkts; 942 u64 hc_tx_bcast_bytes; 943 u64 hc_tx_mcast_pkts; 944 u64 hc_tx_mcast_bytes; 945 /* tx error */ 946 u64 tx_err_gdma; 947 }; /* HW DATA */ 948 949 /* Query phy stats */ 950 struct mana_query_phy_stat_req { 951 struct gdma_req_hdr hdr; 952 u64 req_stats; 953 }; /* HW DATA */ 954 955 struct mana_query_phy_stat_resp { 956 struct gdma_resp_hdr hdr; 957 u64 reported_stats; 958 959 /* Aggregate Drop Counters */ 960 u64 rx_pkt_drop_phy; 961 u64 tx_pkt_drop_phy; 962 963 /* Per TC(Traffic class) traffic Counters */ 964 u64 rx_pkt_tc0_phy; 965 u64 tx_pkt_tc0_phy; 966 u64 rx_pkt_tc1_phy; 967 u64 tx_pkt_tc1_phy; 968 u64 rx_pkt_tc2_phy; 969 u64 tx_pkt_tc2_phy; 970 u64 rx_pkt_tc3_phy; 971 u64 tx_pkt_tc3_phy; 972 u64 rx_pkt_tc4_phy; 973 u64 tx_pkt_tc4_phy; 974 u64 rx_pkt_tc5_phy; 975 u64 tx_pkt_tc5_phy; 976 u64 rx_pkt_tc6_phy; 977 u64 tx_pkt_tc6_phy; 978 u64 rx_pkt_tc7_phy; 979 u64 tx_pkt_tc7_phy; 980 981 u64 rx_byte_tc0_phy; 982 u64 tx_byte_tc0_phy; 983 u64 rx_byte_tc1_phy; 984 u64 tx_byte_tc1_phy; 985 u64 rx_byte_tc2_phy; 986 u64 tx_byte_tc2_phy; 987 u64 rx_byte_tc3_phy; 988 u64 tx_byte_tc3_phy; 989 u64 rx_byte_tc4_phy; 990 u64 tx_byte_tc4_phy; 991 u64 rx_byte_tc5_phy; 992 u64 tx_byte_tc5_phy; 993 u64 rx_byte_tc6_phy; 994 u64 tx_byte_tc6_phy; 995 u64 rx_byte_tc7_phy; 996 u64 tx_byte_tc7_phy; 997 998 /* Per TC(Traffic Class) pause Counters */ 999 u64 rx_pause_tc0_phy; 1000 u64 tx_pause_tc0_phy; 1001 u64 rx_pause_tc1_phy; 1002 u64 tx_pause_tc1_phy; 1003 u64 rx_pause_tc2_phy; 1004 u64 tx_pause_tc2_phy; 1005 u64 rx_pause_tc3_phy; 1006 u64 tx_pause_tc3_phy; 1007 u64 rx_pause_tc4_phy; 1008 u64 tx_pause_tc4_phy; 1009 u64 rx_pause_tc5_phy; 1010 u64 tx_pause_tc5_phy; 1011 u64 rx_pause_tc6_phy; 1012 u64 tx_pause_tc6_phy; 1013 u64 rx_pause_tc7_phy; 1014 u64 tx_pause_tc7_phy; 1015 }; /* HW DATA */ 1016 1017 /* Configure vPort Rx Steering */ 1018 struct mana_cfg_rx_steer_req_v2 { 1019 struct gdma_req_hdr hdr; 1020 mana_handle_t vport; 1021 u16 num_indir_entries; 1022 u16 indir_tab_offset; 1023 u32 rx_enable; 1024 u32 rss_enable; 1025 u8 update_default_rxobj; 1026 u8 update_hashkey; 1027 u8 update_indir_tab; 1028 u8 reserved; 1029 mana_handle_t default_rxobj; 1030 u8 hashkey[MANA_HASH_KEY_SIZE]; 1031 u8 cqe_coalescing_enable; 1032 u8 reserved2[3]; 1033 u16 rss_hash_types; 1034 u8 cqe8_coalescing_enable; /* v5 message */ 1035 u8 reserved3; 1036 mana_handle_t indir_tab[] __counted_by(num_indir_entries); 1037 }; /* HW DATA */ 1038 1039 struct mana_cfg_rx_steer_resp { 1040 struct gdma_resp_hdr hdr; 1041 1042 /* V2 */ 1043 u32 cqe_coalescing_timeout_ns; 1044 u32 reserved1; 1045 }; /* HW DATA */ 1046 1047 /* Register HW vPort */ 1048 struct mana_register_hw_vport_req { 1049 struct gdma_req_hdr hdr; 1050 u16 attached_gfid; 1051 u8 is_pf_default_vport; 1052 u8 reserved1; 1053 u8 allow_all_ether_types; 1054 u8 reserved2; 1055 u8 reserved3; 1056 u8 reserved4; 1057 }; /* HW DATA */ 1058 1059 struct mana_register_hw_vport_resp { 1060 struct gdma_resp_hdr hdr; 1061 mana_handle_t hw_vport_handle; 1062 }; /* HW DATA */ 1063 1064 /* Deregister HW vPort */ 1065 struct mana_deregister_hw_vport_req { 1066 struct gdma_req_hdr hdr; 1067 mana_handle_t hw_vport_handle; 1068 }; /* HW DATA */ 1069 1070 struct mana_deregister_hw_vport_resp { 1071 struct gdma_resp_hdr hdr; 1072 }; /* HW DATA */ 1073 1074 /* Register filter */ 1075 struct mana_register_filter_req { 1076 struct gdma_req_hdr hdr; 1077 mana_handle_t vport; 1078 u8 mac_addr[6]; 1079 u8 reserved1; 1080 u8 reserved2; 1081 u8 reserved3; 1082 u8 reserved4; 1083 u16 reserved5; 1084 u32 reserved6; 1085 u32 reserved7; 1086 u32 reserved8; 1087 }; /* HW DATA */ 1088 1089 struct mana_register_filter_resp { 1090 struct gdma_resp_hdr hdr; 1091 mana_handle_t filter_handle; 1092 }; /* HW DATA */ 1093 1094 /* Deregister filter */ 1095 struct mana_deregister_filter_req { 1096 struct gdma_req_hdr hdr; 1097 mana_handle_t filter_handle; 1098 }; /* HW DATA */ 1099 1100 struct mana_deregister_filter_resp { 1101 struct gdma_resp_hdr hdr; 1102 }; /* HW DATA */ 1103 1104 /* Requested GF stats Flags */ 1105 /* Rx discards/Errors */ 1106 #define STATISTICS_FLAGS_RX_DISCARDS_NO_WQE 0x0000000000000001 1107 #define STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED 0x0000000000000002 1108 /* Rx bytes/pkts */ 1109 #define STATISTICS_FLAGS_HC_RX_BYTES 0x0000000000000004 1110 #define STATISTICS_FLAGS_HC_RX_UCAST_PACKETS 0x0000000000000008 1111 #define STATISTICS_FLAGS_HC_RX_UCAST_BYTES 0x0000000000000010 1112 #define STATISTICS_FLAGS_HC_RX_MCAST_PACKETS 0x0000000000000020 1113 #define STATISTICS_FLAGS_HC_RX_MCAST_BYTES 0x0000000000000040 1114 #define STATISTICS_FLAGS_HC_RX_BCAST_PACKETS 0x0000000000000080 1115 #define STATISTICS_FLAGS_HC_RX_BCAST_BYTES 0x0000000000000100 1116 /* Tx errors */ 1117 #define STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED 0x0000000000000200 1118 #define STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED 0x0000000000000400 1119 #define STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS \ 1120 0x0000000000000800 1121 #define STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT 0x0000000000001000 1122 #define STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT \ 1123 0x0000000000002000 1124 #define STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT 0x0000000000004000 1125 #define STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT 0x0000000000008000 1126 #define STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT 0x0000000000010000 1127 #define STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION 0x0000000000020000 1128 #define STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB 0x0000000000040000 1129 /* Tx bytes/pkts */ 1130 #define STATISTICS_FLAGS_HC_TX_BYTES 0x0000000000080000 1131 #define STATISTICS_FLAGS_HC_TX_UCAST_PACKETS 0x0000000000100000 1132 #define STATISTICS_FLAGS_HC_TX_UCAST_BYTES 0x0000000000200000 1133 #define STATISTICS_FLAGS_HC_TX_MCAST_PACKETS 0x0000000000400000 1134 #define STATISTICS_FLAGS_HC_TX_MCAST_BYTES 0x0000000000800000 1135 #define STATISTICS_FLAGS_HC_TX_BCAST_PACKETS 0x0000000001000000 1136 #define STATISTICS_FLAGS_HC_TX_BCAST_BYTES 0x0000000002000000 1137 /* Tx error */ 1138 #define STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR 0x0000000004000000 1139 1140 #define MANA_MAX_NUM_QUEUES 64 1141 #define MANA_DEF_NUM_QUEUES 16 1142 1143 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1) 1144 1145 struct mana_tx_package { 1146 struct gdma_wqe_request wqe_req; 1147 struct gdma_sge sgl_array[5]; 1148 struct gdma_sge *sgl_ptr; 1149 1150 struct mana_tx_oob tx_oob; 1151 1152 struct gdma_posted_wqe_info wqe_info; 1153 }; 1154 1155 int mana_create_wq_obj(struct mana_port_context *apc, 1156 mana_handle_t vport, 1157 u32 wq_type, struct mana_obj_spec *wq_spec, 1158 struct mana_obj_spec *cq_spec, 1159 mana_handle_t *wq_obj); 1160 1161 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1162 mana_handle_t wq_obj); 1163 1164 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1165 u32 doorbell_pg_id, bool check_channel_changing); 1166 void mana_uncfg_vport(struct mana_port_context *apc); 1167 int mana_create_eq(struct mana_port_context *apc); 1168 void mana_destroy_eq(struct mana_port_context *apc); 1169 1170 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 1171 u32 port_index, 1172 netdevice_tracker *tracker); 1173 #endif /* _MANA_H */ 1174