1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #ifndef _MANA_H 5 #define _MANA_H 6 7 #include <linux/dim.h> 8 #include <net/xdp.h> 9 #include <net/net_shaper.h> 10 11 #include "gdma.h" 12 #include "hw_channel.h" 13 14 /* Microsoft Azure Network Adapter (MANA)'s definitions 15 * 16 * Structures labeled with "HW DATA" are exchanged with the hardware. All of 17 * them are naturally aligned and hence don't need __packed. 18 */ 19 20 /* MANA protocol version */ 21 #define MANA_MAJOR_VERSION 0 22 #define MANA_MINOR_VERSION 1 23 #define MANA_MICRO_VERSION 1 24 25 typedef u64 mana_handle_t; 26 #define INVALID_MANA_HANDLE ((mana_handle_t)-1) 27 28 enum TRI_STATE { 29 TRI_STATE_UNKNOWN = -1, 30 TRI_STATE_FALSE = 0, 31 TRI_STATE_TRUE = 1 32 }; 33 34 /* Number of entries for hardware indirection table must be in power of 2 */ 35 #define MANA_INDIRECT_TABLE_MAX_SIZE 512 36 #define MANA_INDIRECT_TABLE_DEF_SIZE 64 37 38 /* The Toeplitz hash key's length in bytes: should be multiple of 8 */ 39 #define MANA_HASH_KEY_SIZE 40 40 41 #define COMP_ENTRY_SIZE 64 42 43 /* This Max value for RX buffers is derived from __alloc_page()'s max page 44 * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer 45 * size beyond this value gets rejected by __alloc_page() call. 46 */ 47 #define MAX_RX_BUFFERS_PER_QUEUE 8192 48 #define DEF_RX_BUFFERS_PER_QUEUE 1024 49 #define MIN_RX_BUFFERS_PER_QUEUE 128 50 51 /* This max value for TX buffers is derived as the maximum allocatable 52 * pages supported on host per guest through testing. TX buffer size beyond 53 * this value is rejected by the hardware. 54 */ 55 #define MAX_TX_BUFFERS_PER_QUEUE 16384 56 #define DEF_TX_BUFFERS_PER_QUEUE 256 57 #define MIN_TX_BUFFERS_PER_QUEUE 128 58 59 #define EQ_SIZE (8 * MANA_PAGE_SIZE) 60 61 #define LOG2_EQ_THROTTLE 3 62 63 #define MAX_PORTS_IN_MANA_DEV 256 64 65 /* Maximum number of packets per coalesced CQE */ 66 #define MANA_RXCOMP_OOB_NUM_PPI 4 67 68 /* Default/max interrupt moderation settings */ 69 #define MANA_INTR_MODR_USEC_DEF 0 70 #define MANA_INTR_MODR_COMP_DEF 0 71 72 #define MANA_ADAPTIVE_RX_DEF true 73 #define MANA_ADAPTIVE_TX_DEF true 74 75 /* DIM doorbell value field layout */ 76 #define MANA_INTR_MODR_USEC_MAX GENMASK(9, 0) 77 #define MANA_INTR_MODR_USEC_VLD BIT(15) 78 #define MANA_INTR_MODR_COMP_MAX GENMASK(7, 0) 79 #define MANA_INTR_MODR_COMP_MASK GENMASK(23, 16) 80 81 /* Update this count whenever the respective structures are changed */ 82 #define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1) 83 #define MANA_STATS_TX_COUNT 11 84 85 #define MANA_RX_FRAG_ALIGNMENT 64 86 87 struct mana_stats_rx { 88 u64 packets; 89 u64 bytes; 90 u64 xdp_drop; 91 u64 xdp_tx; 92 u64 xdp_redirect; 93 u64 pkt_len0_err; 94 u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1]; 95 struct u64_stats_sync syncp; 96 }; 97 98 struct mana_stats_tx { 99 u64 packets; 100 u64 bytes; 101 u64 xdp_xmit; 102 u64 tso_packets; 103 u64 tso_bytes; 104 u64 tso_inner_packets; 105 u64 tso_inner_bytes; 106 u64 short_pkt_fmt; 107 u64 long_pkt_fmt; 108 u64 csum_partial; 109 u64 mana_map_err; 110 struct u64_stats_sync syncp; 111 }; 112 113 struct mana_txq { 114 struct gdma_queue *gdma_sq; 115 116 union { 117 u32 gdma_txq_id; 118 struct { 119 u32 reserved1 : 10; 120 u32 vsq_frame : 14; 121 u32 reserved2 : 8; 122 }; 123 }; 124 125 u16 vp_offset; 126 127 struct net_device *ndev; 128 129 /* The SKBs are sent to the HW and we are waiting for the CQEs. */ 130 struct sk_buff_head pending_skbs; 131 struct netdev_queue *net_txq; 132 133 atomic_t pending_sends; 134 135 bool napi_initialized; 136 137 struct mana_stats_tx stats; 138 }; 139 140 /* skb data and frags dma mappings */ 141 struct mana_skb_head { 142 /* GSO pkts may have 2 SGEs for the linear part*/ 143 dma_addr_t dma_handle[MAX_SKB_FRAGS + 2]; 144 145 u32 size[MAX_SKB_FRAGS + 2]; 146 }; 147 148 #define MANA_HEADROOM sizeof(struct mana_skb_head) 149 150 enum mana_tx_pkt_format { 151 MANA_SHORT_PKT_FMT = 0, 152 MANA_LONG_PKT_FMT = 1, 153 }; 154 155 struct mana_tx_short_oob { 156 u32 pkt_fmt : 2; 157 u32 is_outer_ipv4 : 1; 158 u32 is_outer_ipv6 : 1; 159 u32 comp_iphdr_csum : 1; 160 u32 comp_tcp_csum : 1; 161 u32 comp_udp_csum : 1; 162 u32 supress_txcqe_gen : 1; 163 u32 vcq_num : 24; 164 165 u32 trans_off : 10; /* Transport header offset */ 166 u32 vsq_frame : 14; 167 u32 short_vp_offset : 8; 168 }; /* HW DATA */ 169 170 struct mana_tx_long_oob { 171 u32 is_encap : 1; 172 u32 inner_is_ipv6 : 1; 173 u32 inner_tcp_opt : 1; 174 u32 inject_vlan_pri_tag : 1; 175 u32 reserved1 : 12; 176 u32 pcp : 3; /* 802.1Q */ 177 u32 dei : 1; /* 802.1Q */ 178 u32 vlan_id : 12; /* 802.1Q */ 179 180 u32 inner_frame_offset : 10; 181 u32 inner_ip_rel_offset : 6; 182 u32 long_vp_offset : 12; 183 u32 reserved2 : 4; 184 185 u32 reserved3; 186 u32 reserved4; 187 }; /* HW DATA */ 188 189 struct mana_tx_oob { 190 struct mana_tx_short_oob s_oob; 191 struct mana_tx_long_oob l_oob; 192 }; /* HW DATA */ 193 194 enum mana_cq_type { 195 MANA_CQ_TYPE_RX, 196 MANA_CQ_TYPE_TX, 197 }; 198 199 enum mana_cqe_type { 200 CQE_INVALID = 0, 201 CQE_RX_OKAY = 1, 202 CQE_RX_COALESCED_4 = 2, 203 CQE_RX_OBJECT_FENCE = 3, 204 CQE_RX_TRUNCATED = 4, 205 206 CQE_TX_OKAY = 32, 207 CQE_TX_SA_DROP = 33, 208 CQE_TX_MTU_DROP = 34, 209 CQE_TX_INVALID_OOB = 35, 210 CQE_TX_INVALID_ETH_TYPE = 36, 211 CQE_TX_HDR_PROCESSING_ERROR = 37, 212 CQE_TX_VF_DISABLED = 38, 213 CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39, 214 CQE_TX_VPORT_DISABLED = 40, 215 CQE_TX_VLAN_TAGGING_VIOLATION = 41, 216 }; 217 218 #define MANA_CQE_COMPLETION 1 219 220 struct mana_cqe_header { 221 u32 cqe_type : 6; 222 u32 client_type : 2; 223 u32 vendor_err : 24; 224 }; /* HW DATA */ 225 226 /* NDIS HASH Types */ 227 #define NDIS_HASH_IPV4 BIT(0) 228 #define NDIS_HASH_TCP_IPV4 BIT(1) 229 #define NDIS_HASH_UDP_IPV4 BIT(2) 230 #define NDIS_HASH_IPV6 BIT(3) 231 #define NDIS_HASH_TCP_IPV6 BIT(4) 232 #define NDIS_HASH_UDP_IPV6 BIT(5) 233 #define NDIS_HASH_IPV6_EX BIT(6) 234 #define NDIS_HASH_TCP_IPV6_EX BIT(7) 235 #define NDIS_HASH_UDP_IPV6_EX BIT(8) 236 237 #define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX) 238 #define MANA_HASH_L4 \ 239 (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \ 240 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX) 241 242 struct mana_rxcomp_perpkt_info { 243 u32 pkt_len : 16; 244 u32 reserved1 : 16; 245 u32 reserved2; 246 u32 pkt_hash; 247 }; /* HW DATA */ 248 249 /* Receive completion OOB */ 250 struct mana_rxcomp_oob { 251 struct mana_cqe_header cqe_hdr; 252 253 u32 rx_vlan_id : 12; 254 u32 rx_vlantag_present : 1; 255 u32 rx_outer_iphdr_csum_succeed : 1; 256 u32 rx_outer_iphdr_csum_fail : 1; 257 u32 reserved1 : 1; 258 u32 rx_hashtype : 9; 259 u32 rx_iphdr_csum_succeed : 1; 260 u32 rx_iphdr_csum_fail : 1; 261 u32 rx_tcp_csum_succeed : 1; 262 u32 rx_tcp_csum_fail : 1; 263 u32 rx_udp_csum_succeed : 1; 264 u32 rx_udp_csum_fail : 1; 265 u32 reserved2 : 1; 266 267 struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI]; 268 269 u32 rx_wqe_offset; 270 }; /* HW DATA */ 271 272 struct mana_tx_comp_oob { 273 struct mana_cqe_header cqe_hdr; 274 275 u32 tx_data_offset; 276 277 u32 tx_sgl_offset : 5; 278 u32 tx_wqe_offset : 27; 279 280 u32 reserved[12]; 281 }; /* HW DATA */ 282 283 struct mana_rxq; 284 285 #define CQE_POLLING_BUFFER 512 286 287 struct mana_cq { 288 struct gdma_queue *gdma_cq; 289 290 /* Cache the CQ id (used to verify if each CQE comes to the right CQ. */ 291 u32 gdma_id; 292 293 /* Type of the CQ: TX or RX */ 294 enum mana_cq_type type; 295 296 /* Pointer to the mana_rxq that is pushing RX CQEs to the queue. 297 * Only and must be non-NULL if type is MANA_CQ_TYPE_RX. 298 */ 299 struct mana_rxq *rxq; 300 301 /* Pointer to the mana_txq that is pushing TX CQEs to the queue. 302 * Only and must be non-NULL if type is MANA_CQ_TYPE_TX. 303 */ 304 struct mana_txq *txq; 305 306 /* Buffer which the CQ handler can copy the CQE's into. */ 307 struct gdma_comp gdma_comp_buf[CQE_POLLING_BUFFER]; 308 309 /* NAPI data */ 310 struct napi_struct napi; 311 int work_done; 312 int work_done_since_doorbell; 313 int budget; 314 315 /* DIM - Dynamic Interrupt Moderation */ 316 struct dim dim; 317 u16 dim_event_ctr; 318 319 /* Cumulative TX completions fed to DIM. Updated and read only in 320 * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they 321 * measure the hardware completion rate and need no u64_stats_sync. 322 */ 323 u64 tx_dim_pkts; 324 u64 tx_dim_bytes; 325 }; 326 327 struct mana_recv_buf_oob { 328 /* A valid GDMA work request representing the data buffer. */ 329 struct gdma_wqe_request wqe_req; 330 331 void *buf_va; 332 bool from_pool; /* allocated from a page pool */ 333 /* head page of the page_pool fragment; valid only when 334 * from_pool && frag_count > 1. 335 */ 336 struct page *pp_page; 337 /* Fragment offset plus rxq->headroom, passed to 338 * page_pool_dma_sync_for_cpu(). 339 */ 340 u32 dma_sync_offset; 341 342 /* SGL of the buffer going to be sent as part of the work request. */ 343 u32 num_sge; 344 struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES]; 345 346 /* Required to store the result of mana_gd_post_work_request. 347 * gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the 348 * work queue when the WQE is consumed. 349 */ 350 struct gdma_posted_wqe_info wqe_inf; 351 }; 352 353 #define MANA_RXBUF_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) \ 354 + ETH_HLEN) 355 356 #define MANA_XDP_MTU_MAX (PAGE_SIZE - MANA_RXBUF_PAD - XDP_PACKET_HEADROOM) 357 358 struct mana_rxq { 359 struct gdma_queue *gdma_rq; 360 /* Cache the gdma receive queue id */ 361 u32 gdma_id; 362 363 /* Index of RQ in the vPort, not gdma receive queue id */ 364 u32 rxq_idx; 365 366 u32 datasize; 367 u32 alloc_size; 368 u32 headroom; 369 u32 frag_count; 370 371 mana_handle_t rxobj; 372 373 struct mana_cq rx_cq; 374 375 struct completion fence_event; 376 377 struct net_device *ndev; 378 379 /* Total number of receive buffers to be allocated */ 380 u32 num_rx_buf; 381 382 u32 buf_index; 383 384 struct mana_stats_rx stats; 385 386 struct bpf_prog __rcu *bpf_prog; 387 struct xdp_rxq_info xdp_rxq; 388 void *xdp_save_va; /* for reusing */ 389 bool xdp_flush; 390 int xdp_rc; /* XDP redirect return code */ 391 392 struct page_pool *page_pool; 393 struct dentry *mana_rx_debugfs; 394 395 /* MUST BE THE LAST MEMBER: 396 * Each receive buffer has an associated mana_recv_buf_oob. 397 */ 398 struct mana_recv_buf_oob rx_oobs[] __counted_by(num_rx_buf); 399 }; 400 401 struct mana_tx_qp { 402 struct mana_txq txq; 403 404 struct mana_cq tx_cq; 405 406 mana_handle_t tx_object; 407 408 struct dentry *mana_tx_debugfs; 409 }; 410 411 struct mana_ethtool_stats { 412 u64 stop_queue; 413 u64 wake_queue; 414 u64 tx_cqe_err; 415 u64 tx_cqe_unknown_type; 416 u64 tx_linear_pkt_cnt; 417 u64 rx_cqe_unknown_type; 418 }; 419 420 struct mana_ethtool_hc_stats { 421 u64 hc_rx_discards_no_wqe; 422 u64 hc_rx_err_vport_disabled; 423 u64 hc_rx_bytes; 424 u64 hc_rx_ucast_pkts; 425 u64 hc_rx_ucast_bytes; 426 u64 hc_rx_bcast_pkts; 427 u64 hc_rx_bcast_bytes; 428 u64 hc_rx_mcast_pkts; 429 u64 hc_rx_mcast_bytes; 430 u64 hc_tx_err_gf_disabled; 431 u64 hc_tx_err_vport_disabled; 432 u64 hc_tx_err_inval_vportoffset_pkt; 433 u64 hc_tx_err_vlan_enforcement; 434 u64 hc_tx_err_eth_type_enforcement; 435 u64 hc_tx_err_sa_enforcement; 436 u64 hc_tx_err_sqpdid_enforcement; 437 u64 hc_tx_err_cqpdid_enforcement; 438 u64 hc_tx_err_mtu_violation; 439 u64 hc_tx_err_inval_oob; 440 u64 hc_tx_bytes; 441 u64 hc_tx_ucast_pkts; 442 u64 hc_tx_ucast_bytes; 443 u64 hc_tx_bcast_pkts; 444 u64 hc_tx_bcast_bytes; 445 u64 hc_tx_mcast_pkts; 446 u64 hc_tx_mcast_bytes; 447 u64 hc_tx_err_gdma; 448 }; 449 450 struct mana_ethtool_phy_stats { 451 /* Drop Counters */ 452 u64 rx_pkt_drop_phy; 453 u64 tx_pkt_drop_phy; 454 455 /* Per TC traffic Counters */ 456 u64 rx_pkt_tc0_phy; 457 u64 tx_pkt_tc0_phy; 458 u64 rx_pkt_tc1_phy; 459 u64 tx_pkt_tc1_phy; 460 u64 rx_pkt_tc2_phy; 461 u64 tx_pkt_tc2_phy; 462 u64 rx_pkt_tc3_phy; 463 u64 tx_pkt_tc3_phy; 464 u64 rx_pkt_tc4_phy; 465 u64 tx_pkt_tc4_phy; 466 u64 rx_pkt_tc5_phy; 467 u64 tx_pkt_tc5_phy; 468 u64 rx_pkt_tc6_phy; 469 u64 tx_pkt_tc6_phy; 470 u64 rx_pkt_tc7_phy; 471 u64 tx_pkt_tc7_phy; 472 473 u64 rx_byte_tc0_phy; 474 u64 tx_byte_tc0_phy; 475 u64 rx_byte_tc1_phy; 476 u64 tx_byte_tc1_phy; 477 u64 rx_byte_tc2_phy; 478 u64 tx_byte_tc2_phy; 479 u64 rx_byte_tc3_phy; 480 u64 tx_byte_tc3_phy; 481 u64 rx_byte_tc4_phy; 482 u64 tx_byte_tc4_phy; 483 u64 rx_byte_tc5_phy; 484 u64 tx_byte_tc5_phy; 485 u64 rx_byte_tc6_phy; 486 u64 tx_byte_tc6_phy; 487 u64 rx_byte_tc7_phy; 488 u64 tx_byte_tc7_phy; 489 490 /* Per TC pause Counters */ 491 u64 rx_pause_tc0_phy; 492 u64 tx_pause_tc0_phy; 493 u64 rx_pause_tc1_phy; 494 u64 tx_pause_tc1_phy; 495 u64 rx_pause_tc2_phy; 496 u64 tx_pause_tc2_phy; 497 u64 rx_pause_tc3_phy; 498 u64 tx_pause_tc3_phy; 499 u64 rx_pause_tc4_phy; 500 u64 tx_pause_tc4_phy; 501 u64 rx_pause_tc5_phy; 502 u64 tx_pause_tc5_phy; 503 u64 rx_pause_tc6_phy; 504 u64 tx_pause_tc6_phy; 505 u64 rx_pause_tc7_phy; 506 u64 tx_pause_tc7_phy; 507 }; 508 509 struct mana_context { 510 struct gdma_dev *gdma_dev; 511 512 u16 num_ports; 513 u8 bm_hostmode; 514 515 struct mana_ethtool_hc_stats hc_stats; 516 struct workqueue_struct *per_port_queue_reset_wq; 517 /* Workqueue for querying hardware stats */ 518 struct delayed_work gf_stats_work; 519 bool hwc_timeout_occurred; 520 521 struct net_device *ports[MAX_PORTS_IN_MANA_DEV]; 522 523 /* Link state change work */ 524 struct work_struct link_change_work; 525 u32 link_event; 526 }; 527 528 struct mana_port_context { 529 struct mana_context *ac; 530 struct net_device *ndev; 531 struct work_struct queue_reset_work; 532 533 /* Debug knob to log TX timeout but skip recovery reset */ 534 bool tx_timeout_skip_reset; 535 536 u8 mac_addr[ETH_ALEN]; 537 538 struct mana_eq *eqs; 539 struct dentry *mana_eqs_debugfs; 540 541 enum TRI_STATE rss_state; 542 543 mana_handle_t default_rxobj; 544 bool tx_shortform_allowed; 545 u16 tx_vp_offset; 546 547 struct mana_tx_qp **tx_qp; 548 549 /* Indirection Table for RX & TX. The values are queue indexes */ 550 u32 *indir_table; 551 u32 indir_table_sz; 552 553 /* Indirection table containing RxObject Handles */ 554 mana_handle_t *rxobj_table; 555 556 /* Hash key used by the NIC */ 557 u8 hashkey[MANA_HASH_KEY_SIZE]; 558 559 /* This points to an array of num_queues of RQ pointers. */ 560 struct mana_rxq **rxqs; 561 562 /* pre-allocated rx buffer array */ 563 void **rxbufs_pre; 564 dma_addr_t *das_pre; 565 int rxbpre_total; 566 u32 rxbpre_datasize; 567 u32 rxbpre_alloc_size; 568 u32 rxbpre_headroom; 569 u32 rxbpre_frag_count; 570 571 struct bpf_prog *bpf_prog; 572 573 /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */ 574 unsigned int max_queues; 575 unsigned int num_queues; 576 577 unsigned int rx_queue_size; 578 unsigned int tx_queue_size; 579 580 mana_handle_t port_handle; 581 mana_handle_t pf_filter_handle; 582 583 /* Mutex for sharing access to vport_use_count */ 584 struct mutex vport_mutex; 585 int vport_use_count; 586 587 /* Set by mana_set_channels() under vport_mutex to block RDMA 588 * from grabbing the vport during the detach/attach window. 589 * Checked by mana_cfg_vport() when called from the RDMA path. 590 */ 591 bool channel_changing; 592 593 /* Net shaper handle*/ 594 struct net_shaper_handle handle; 595 596 u16 port_idx; 597 /* Currently configured speed (mbps) */ 598 u32 speed; 599 /* Maximum speed supported by the SKU (mbps) */ 600 u32 max_speed; 601 /* 1 = not queried, 0 = cached success, negative = permanent error. 602 * Protected by the netdev instance lock. 603 */ 604 int link_cfg_error; 605 606 bool port_is_up; 607 bool port_st_save; /* Saved port state */ 608 609 u8 cqe_coalescing_enable; 610 u32 cqe_coalescing_timeout_ns; 611 612 /* Interrupt moderation settings */ 613 u16 intr_modr_rx_usec; 614 u16 intr_modr_rx_comp; 615 u16 intr_modr_tx_usec; 616 u16 intr_modr_tx_comp; 617 618 bool rx_dim_enabled; 619 bool tx_dim_enabled; 620 621 struct mana_ethtool_stats eth_stats; 622 623 struct mana_ethtool_phy_stats phy_stats; 624 625 /* Debugfs */ 626 struct dentry *mana_port_debugfs; 627 628 /* Cached vport/steering config for debugfs */ 629 u32 vport_max_sq; 630 u32 vport_max_rq; 631 u32 steer_rx; 632 u32 steer_rss; 633 bool steer_update_tab; 634 u32 steer_cqe_coalescing; 635 }; 636 637 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); 638 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, 639 bool update_hash, bool update_tab); 640 int mana_disable_vport_rx(struct mana_port_context *apc); 641 642 int mana_alloc_queues(struct net_device *ndev); 643 int mana_attach(struct net_device *ndev); 644 int mana_detach(struct net_device *ndev, bool from_close); 645 646 void mana_dim_change(struct mana_cq *cq, bool enable); 647 648 int mana_probe(struct gdma_dev *gd, bool resuming); 649 void mana_remove(struct gdma_dev *gd, bool suspending); 650 651 int mana_rdma_probe(struct gdma_dev *gd); 652 void mana_rdma_remove(struct gdma_dev *gd); 653 654 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); 655 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, 656 u32 flags); 657 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, 658 struct xdp_buff *xdp, void *buf_va, uint pkt_len); 659 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc); 660 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog); 661 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf); 662 int mana_query_gf_stats(struct mana_context *ac); 663 int mana_query_link_cfg(struct mana_port_context *apc); 664 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 665 int enable_clamping); 666 void mana_query_phy_stats(struct mana_port_context *apc); 667 int mana_pre_alloc_rxbufs(struct mana_port_context *apc, int mtu, int num_queues); 668 void mana_pre_dealloc_rxbufs(struct mana_port_context *apc); 669 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc); 670 671 extern const struct ethtool_ops mana_ethtool_ops; 672 extern struct dentry *mana_debugfs_root; 673 674 /* A CQ can be created not associated with any EQ */ 675 #define GDMA_CQ_NO_EQ 0xffff 676 677 struct mana_obj_spec { 678 u32 queue_index; 679 u64 gdma_region; 680 u32 queue_size; 681 u32 attached_eq; 682 u32 modr_ctx_id; 683 u8 req_cq_moderation; 684 u16 cq_moderation_comp; 685 u16 cq_moderation_usec; 686 }; 687 688 enum mana_command_code { 689 MANA_QUERY_DEV_CONFIG = 0x20001, 690 MANA_QUERY_GF_STAT = 0x20002, 691 MANA_CONFIG_VPORT_TX = 0x20003, 692 MANA_CREATE_WQ_OBJ = 0x20004, 693 MANA_DESTROY_WQ_OBJ = 0x20005, 694 MANA_FENCE_RQ = 0x20006, 695 MANA_CONFIG_VPORT_RX = 0x20007, 696 MANA_QUERY_VPORT_CONFIG = 0x20008, 697 MANA_QUERY_LINK_CONFIG = 0x2000A, 698 MANA_SET_BW_CLAMP = 0x2000B, 699 MANA_QUERY_PHY_STAT = 0x2000c, 700 701 /* Privileged commands for the PF mode */ 702 MANA_REGISTER_FILTER = 0x28000, 703 MANA_DEREGISTER_FILTER = 0x28001, 704 MANA_REGISTER_HW_PORT = 0x28003, 705 MANA_DEREGISTER_HW_PORT = 0x28004, 706 }; 707 708 /* Query Link Configuration*/ 709 struct mana_query_link_config_req { 710 struct gdma_req_hdr hdr; 711 mana_handle_t vport; 712 }; /* HW DATA */ 713 714 struct mana_query_link_config_resp { 715 struct gdma_resp_hdr hdr; 716 u32 qos_speed_mbps; 717 u8 qos_unconfigured; 718 u8 reserved1[3]; 719 u32 link_speed_mbps; 720 u8 reserved2[4]; 721 }; /* HW DATA */ 722 723 /* Set Bandwidth Clamp*/ 724 struct mana_set_bw_clamp_req { 725 struct gdma_req_hdr hdr; 726 mana_handle_t vport; 727 enum TRI_STATE enable_clamping; 728 u32 link_speed_mbps; 729 }; /* HW DATA */ 730 731 struct mana_set_bw_clamp_resp { 732 struct gdma_resp_hdr hdr; 733 u8 qos_unconfigured; 734 u8 reserved[7]; 735 }; /* HW DATA */ 736 737 /* Query Device Configuration */ 738 struct mana_query_device_cfg_req { 739 struct gdma_req_hdr hdr; 740 741 /* MANA Nic Driver Capability flags */ 742 u64 mn_drv_cap_flags1; 743 u64 mn_drv_cap_flags2; 744 u64 mn_drv_cap_flags3; 745 u64 mn_drv_cap_flags4; 746 747 u32 proto_major_ver; 748 u32 proto_minor_ver; 749 u32 proto_micro_ver; 750 751 u32 reserved; 752 }; /* HW DATA */ 753 754 struct mana_query_device_cfg_resp { 755 struct gdma_resp_hdr hdr; 756 757 u64 pf_cap_flags1; 758 u64 pf_cap_flags2; 759 u64 pf_cap_flags3; 760 u64 pf_cap_flags4; 761 762 u16 max_num_vports; 763 u8 bm_hostmode; /* response v3: Bare Metal Host Mode */ 764 u8 reserved; 765 u32 max_num_eqs; 766 767 /* response v2: */ 768 u16 adapter_mtu; 769 u16 reserved2; 770 u32 reserved3; 771 }; /* HW DATA */ 772 773 /* Query vPort Configuration */ 774 struct mana_query_vport_cfg_req { 775 struct gdma_req_hdr hdr; 776 u32 vport_index; 777 }; /* HW DATA */ 778 779 struct mana_query_vport_cfg_resp { 780 struct gdma_resp_hdr hdr; 781 u32 max_num_sq; 782 u32 max_num_rq; 783 u32 num_indirection_ent; 784 u32 reserved1; 785 u8 mac_addr[6]; 786 u8 reserved2[2]; 787 mana_handle_t vport; 788 }; /* HW DATA */ 789 790 /* Configure vPort */ 791 struct mana_config_vport_req { 792 struct gdma_req_hdr hdr; 793 mana_handle_t vport; 794 u32 pdid; 795 u32 doorbell_pageid; 796 }; /* HW DATA */ 797 798 struct mana_config_vport_resp { 799 struct gdma_resp_hdr hdr; 800 u16 tx_vport_offset; 801 u8 short_form_allowed; 802 u8 reserved; 803 }; /* HW DATA */ 804 805 /* Create WQ Object */ 806 struct mana_create_wqobj_req { 807 struct gdma_req_hdr hdr; 808 mana_handle_t vport; 809 u32 wq_type; 810 u32 reserved; 811 u64 wq_gdma_region; 812 u64 cq_gdma_region; 813 u32 wq_size; 814 u32 cq_size; 815 u32 cq_moderation_ctx_id; 816 u32 cq_parent_qid; 817 818 /* V2 */ 819 u8 allow_rqwqe_chain; 820 821 /* V3 */ 822 u8 req_cq_moderation; 823 u16 cq_moderation_comp; 824 u16 cq_moderation_usec; 825 u8 reserved2[2]; 826 }; /* HW DATA */ 827 828 struct mana_create_wqobj_resp { 829 struct gdma_resp_hdr hdr; 830 u32 wq_id; 831 u32 cq_id; 832 mana_handle_t wq_obj; 833 834 /* V2 */ 835 u16 cq_moderation_comp; 836 u16 cq_moderation_usec; 837 u8 cq_moderation_enabled; 838 u8 reserved1[3]; 839 }; /* HW DATA */ 840 841 /* Destroy WQ Object */ 842 struct mana_destroy_wqobj_req { 843 struct gdma_req_hdr hdr; 844 u32 wq_type; 845 u32 reserved; 846 mana_handle_t wq_obj_handle; 847 }; /* HW DATA */ 848 849 struct mana_destroy_wqobj_resp { 850 struct gdma_resp_hdr hdr; 851 }; /* HW DATA */ 852 853 /* Fence RQ */ 854 struct mana_fence_rq_req { 855 struct gdma_req_hdr hdr; 856 mana_handle_t wq_obj_handle; 857 }; /* HW DATA */ 858 859 struct mana_fence_rq_resp { 860 struct gdma_resp_hdr hdr; 861 }; /* HW DATA */ 862 863 /* Query stats RQ */ 864 struct mana_query_gf_stat_req { 865 struct gdma_req_hdr hdr; 866 u64 req_stats; 867 }; /* HW DATA */ 868 869 struct mana_query_gf_stat_resp { 870 struct gdma_resp_hdr hdr; 871 u64 reported_stats; 872 /* rx errors/discards */ 873 u64 rx_discards_nowqe; 874 u64 rx_err_vport_disabled; 875 /* rx bytes/packets */ 876 u64 hc_rx_bytes; 877 u64 hc_rx_ucast_pkts; 878 u64 hc_rx_ucast_bytes; 879 u64 hc_rx_bcast_pkts; 880 u64 hc_rx_bcast_bytes; 881 u64 hc_rx_mcast_pkts; 882 u64 hc_rx_mcast_bytes; 883 /* tx errors */ 884 u64 tx_err_gf_disabled; 885 u64 tx_err_vport_disabled; 886 u64 tx_err_inval_vport_offset_pkt; 887 u64 tx_err_vlan_enforcement; 888 u64 tx_err_ethtype_enforcement; 889 u64 tx_err_SA_enforcement; 890 u64 tx_err_SQPDID_enforcement; 891 u64 tx_err_CQPDID_enforcement; 892 u64 tx_err_mtu_violation; 893 u64 tx_err_inval_oob; 894 /* tx bytes/packets */ 895 u64 hc_tx_bytes; 896 u64 hc_tx_ucast_pkts; 897 u64 hc_tx_ucast_bytes; 898 u64 hc_tx_bcast_pkts; 899 u64 hc_tx_bcast_bytes; 900 u64 hc_tx_mcast_pkts; 901 u64 hc_tx_mcast_bytes; 902 /* tx error */ 903 u64 tx_err_gdma; 904 }; /* HW DATA */ 905 906 /* Query phy stats */ 907 struct mana_query_phy_stat_req { 908 struct gdma_req_hdr hdr; 909 u64 req_stats; 910 }; /* HW DATA */ 911 912 struct mana_query_phy_stat_resp { 913 struct gdma_resp_hdr hdr; 914 u64 reported_stats; 915 916 /* Aggregate Drop Counters */ 917 u64 rx_pkt_drop_phy; 918 u64 tx_pkt_drop_phy; 919 920 /* Per TC(Traffic class) traffic Counters */ 921 u64 rx_pkt_tc0_phy; 922 u64 tx_pkt_tc0_phy; 923 u64 rx_pkt_tc1_phy; 924 u64 tx_pkt_tc1_phy; 925 u64 rx_pkt_tc2_phy; 926 u64 tx_pkt_tc2_phy; 927 u64 rx_pkt_tc3_phy; 928 u64 tx_pkt_tc3_phy; 929 u64 rx_pkt_tc4_phy; 930 u64 tx_pkt_tc4_phy; 931 u64 rx_pkt_tc5_phy; 932 u64 tx_pkt_tc5_phy; 933 u64 rx_pkt_tc6_phy; 934 u64 tx_pkt_tc6_phy; 935 u64 rx_pkt_tc7_phy; 936 u64 tx_pkt_tc7_phy; 937 938 u64 rx_byte_tc0_phy; 939 u64 tx_byte_tc0_phy; 940 u64 rx_byte_tc1_phy; 941 u64 tx_byte_tc1_phy; 942 u64 rx_byte_tc2_phy; 943 u64 tx_byte_tc2_phy; 944 u64 rx_byte_tc3_phy; 945 u64 tx_byte_tc3_phy; 946 u64 rx_byte_tc4_phy; 947 u64 tx_byte_tc4_phy; 948 u64 rx_byte_tc5_phy; 949 u64 tx_byte_tc5_phy; 950 u64 rx_byte_tc6_phy; 951 u64 tx_byte_tc6_phy; 952 u64 rx_byte_tc7_phy; 953 u64 tx_byte_tc7_phy; 954 955 /* Per TC(Traffic Class) pause Counters */ 956 u64 rx_pause_tc0_phy; 957 u64 tx_pause_tc0_phy; 958 u64 rx_pause_tc1_phy; 959 u64 tx_pause_tc1_phy; 960 u64 rx_pause_tc2_phy; 961 u64 tx_pause_tc2_phy; 962 u64 rx_pause_tc3_phy; 963 u64 tx_pause_tc3_phy; 964 u64 rx_pause_tc4_phy; 965 u64 tx_pause_tc4_phy; 966 u64 rx_pause_tc5_phy; 967 u64 tx_pause_tc5_phy; 968 u64 rx_pause_tc6_phy; 969 u64 tx_pause_tc6_phy; 970 u64 rx_pause_tc7_phy; 971 u64 tx_pause_tc7_phy; 972 }; /* HW DATA */ 973 974 /* Configure vPort Rx Steering */ 975 struct mana_cfg_rx_steer_req_v2 { 976 struct gdma_req_hdr hdr; 977 mana_handle_t vport; 978 u16 num_indir_entries; 979 u16 indir_tab_offset; 980 u32 rx_enable; 981 u32 rss_enable; 982 u8 update_default_rxobj; 983 u8 update_hashkey; 984 u8 update_indir_tab; 985 u8 reserved; 986 mana_handle_t default_rxobj; 987 u8 hashkey[MANA_HASH_KEY_SIZE]; 988 u8 cqe_coalescing_enable; 989 u8 reserved2[7]; 990 mana_handle_t indir_tab[] __counted_by(num_indir_entries); 991 }; /* HW DATA */ 992 993 struct mana_cfg_rx_steer_resp { 994 struct gdma_resp_hdr hdr; 995 996 /* V2 */ 997 u32 cqe_coalescing_timeout_ns; 998 u32 reserved1; 999 }; /* HW DATA */ 1000 1001 /* Register HW vPort */ 1002 struct mana_register_hw_vport_req { 1003 struct gdma_req_hdr hdr; 1004 u16 attached_gfid; 1005 u8 is_pf_default_vport; 1006 u8 reserved1; 1007 u8 allow_all_ether_types; 1008 u8 reserved2; 1009 u8 reserved3; 1010 u8 reserved4; 1011 }; /* HW DATA */ 1012 1013 struct mana_register_hw_vport_resp { 1014 struct gdma_resp_hdr hdr; 1015 mana_handle_t hw_vport_handle; 1016 }; /* HW DATA */ 1017 1018 /* Deregister HW vPort */ 1019 struct mana_deregister_hw_vport_req { 1020 struct gdma_req_hdr hdr; 1021 mana_handle_t hw_vport_handle; 1022 }; /* HW DATA */ 1023 1024 struct mana_deregister_hw_vport_resp { 1025 struct gdma_resp_hdr hdr; 1026 }; /* HW DATA */ 1027 1028 /* Register filter */ 1029 struct mana_register_filter_req { 1030 struct gdma_req_hdr hdr; 1031 mana_handle_t vport; 1032 u8 mac_addr[6]; 1033 u8 reserved1; 1034 u8 reserved2; 1035 u8 reserved3; 1036 u8 reserved4; 1037 u16 reserved5; 1038 u32 reserved6; 1039 u32 reserved7; 1040 u32 reserved8; 1041 }; /* HW DATA */ 1042 1043 struct mana_register_filter_resp { 1044 struct gdma_resp_hdr hdr; 1045 mana_handle_t filter_handle; 1046 }; /* HW DATA */ 1047 1048 /* Deregister filter */ 1049 struct mana_deregister_filter_req { 1050 struct gdma_req_hdr hdr; 1051 mana_handle_t filter_handle; 1052 }; /* HW DATA */ 1053 1054 struct mana_deregister_filter_resp { 1055 struct gdma_resp_hdr hdr; 1056 }; /* HW DATA */ 1057 1058 /* Requested GF stats Flags */ 1059 /* Rx discards/Errors */ 1060 #define STATISTICS_FLAGS_RX_DISCARDS_NO_WQE 0x0000000000000001 1061 #define STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED 0x0000000000000002 1062 /* Rx bytes/pkts */ 1063 #define STATISTICS_FLAGS_HC_RX_BYTES 0x0000000000000004 1064 #define STATISTICS_FLAGS_HC_RX_UCAST_PACKETS 0x0000000000000008 1065 #define STATISTICS_FLAGS_HC_RX_UCAST_BYTES 0x0000000000000010 1066 #define STATISTICS_FLAGS_HC_RX_MCAST_PACKETS 0x0000000000000020 1067 #define STATISTICS_FLAGS_HC_RX_MCAST_BYTES 0x0000000000000040 1068 #define STATISTICS_FLAGS_HC_RX_BCAST_PACKETS 0x0000000000000080 1069 #define STATISTICS_FLAGS_HC_RX_BCAST_BYTES 0x0000000000000100 1070 /* Tx errors */ 1071 #define STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED 0x0000000000000200 1072 #define STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED 0x0000000000000400 1073 #define STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS \ 1074 0x0000000000000800 1075 #define STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT 0x0000000000001000 1076 #define STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT \ 1077 0x0000000000002000 1078 #define STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT 0x0000000000004000 1079 #define STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT 0x0000000000008000 1080 #define STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT 0x0000000000010000 1081 #define STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION 0x0000000000020000 1082 #define STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB 0x0000000000040000 1083 /* Tx bytes/pkts */ 1084 #define STATISTICS_FLAGS_HC_TX_BYTES 0x0000000000080000 1085 #define STATISTICS_FLAGS_HC_TX_UCAST_PACKETS 0x0000000000100000 1086 #define STATISTICS_FLAGS_HC_TX_UCAST_BYTES 0x0000000000200000 1087 #define STATISTICS_FLAGS_HC_TX_MCAST_PACKETS 0x0000000000400000 1088 #define STATISTICS_FLAGS_HC_TX_MCAST_BYTES 0x0000000000800000 1089 #define STATISTICS_FLAGS_HC_TX_BCAST_PACKETS 0x0000000001000000 1090 #define STATISTICS_FLAGS_HC_TX_BCAST_BYTES 0x0000000002000000 1091 /* Tx error */ 1092 #define STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR 0x0000000004000000 1093 1094 #define MANA_MAX_NUM_QUEUES 64 1095 #define MANA_DEF_NUM_QUEUES 16 1096 1097 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1) 1098 1099 struct mana_tx_package { 1100 struct gdma_wqe_request wqe_req; 1101 struct gdma_sge sgl_array[5]; 1102 struct gdma_sge *sgl_ptr; 1103 1104 struct mana_tx_oob tx_oob; 1105 1106 struct gdma_posted_wqe_info wqe_info; 1107 }; 1108 1109 int mana_create_wq_obj(struct mana_port_context *apc, 1110 mana_handle_t vport, 1111 u32 wq_type, struct mana_obj_spec *wq_spec, 1112 struct mana_obj_spec *cq_spec, 1113 mana_handle_t *wq_obj); 1114 1115 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1116 mana_handle_t wq_obj); 1117 1118 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1119 u32 doorbell_pg_id, bool check_channel_changing); 1120 void mana_uncfg_vport(struct mana_port_context *apc); 1121 int mana_create_eq(struct mana_port_context *apc); 1122 void mana_destroy_eq(struct mana_port_context *apc); 1123 1124 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 1125 u32 port_index, 1126 netdevice_tracker *tracker); 1127 #endif /* _MANA_H */ 1128