1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #ifndef _MANA_H 5 #define _MANA_H 6 7 #include <linux/dim.h> 8 #include <net/xdp.h> 9 #include <net/net_shaper.h> 10 11 #include "gdma.h" 12 #include "hw_channel.h" 13 14 /* Microsoft Azure Network Adapter (MANA)'s definitions 15 * 16 * Structures labeled with "HW DATA" are exchanged with the hardware. All of 17 * them are naturally aligned and hence don't need __packed. 18 */ 19 20 /* MANA protocol version */ 21 #define MANA_MAJOR_VERSION 0 22 #define MANA_MINOR_VERSION 1 23 #define MANA_MICRO_VERSION 1 24 25 typedef u64 mana_handle_t; 26 #define INVALID_MANA_HANDLE ((mana_handle_t)-1) 27 28 enum TRI_STATE { 29 TRI_STATE_UNKNOWN = -1, 30 TRI_STATE_FALSE = 0, 31 TRI_STATE_TRUE = 1 32 }; 33 34 /* MANA ethtool private flag bit positions */ 35 enum mana_priv_flag_bits { 36 MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF = 0, 37 MANA_PRIV_FLAG_MAX, 38 }; 39 40 /* Number of entries for hardware indirection table must be in power of 2 */ 41 #define MANA_INDIRECT_TABLE_MAX_SIZE 512 42 #define MANA_INDIRECT_TABLE_DEF_SIZE 64 43 44 /* The Toeplitz hash key's length in bytes: should be multiple of 8 */ 45 #define MANA_HASH_KEY_SIZE 40 46 47 #define COMP_ENTRY_SIZE 64 48 49 /* This Max value for RX buffers is derived from __alloc_page()'s max page 50 * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer 51 * size beyond this value gets rejected by __alloc_page() call. 52 */ 53 #define MAX_RX_BUFFERS_PER_QUEUE 8192 54 #define DEF_RX_BUFFERS_PER_QUEUE 1024 55 #define MIN_RX_BUFFERS_PER_QUEUE 128 56 57 /* This max value for TX buffers is derived as the maximum allocatable 58 * pages supported on host per guest through testing. TX buffer size beyond 59 * this value is rejected by the hardware. 60 */ 61 #define MAX_TX_BUFFERS_PER_QUEUE 16384 62 #define DEF_TX_BUFFERS_PER_QUEUE 256 63 #define MIN_TX_BUFFERS_PER_QUEUE 128 64 65 #define EQ_SIZE (8 * MANA_PAGE_SIZE) 66 67 #define LOG2_EQ_THROTTLE 3 68 69 #define MAX_PORTS_IN_MANA_DEV 256 70 71 /* Maximum number of packets per coalesced CQE */ 72 #define MANA_RXCOMP_OOB_NUM_PPI 4 73 74 /* Default/max interrupt moderation settings */ 75 #define MANA_INTR_MODR_USEC_DEF 0 76 #define MANA_INTR_MODR_COMP_DEF 0 77 78 #define MANA_ADAPTIVE_RX_DEF true 79 #define MANA_ADAPTIVE_TX_DEF true 80 81 /* DIM doorbell value field layout */ 82 #define MANA_INTR_MODR_USEC_MAX GENMASK(9, 0) 83 #define MANA_INTR_MODR_USEC_VLD BIT(15) 84 #define MANA_INTR_MODR_COMP_MAX GENMASK(7, 0) 85 #define MANA_INTR_MODR_COMP_MASK GENMASK(23, 16) 86 87 /* Update this count whenever the respective structures are changed */ 88 #define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1) 89 #define MANA_STATS_TX_COUNT 11 90 91 #define MANA_RX_FRAG_ALIGNMENT 64 92 93 struct mana_stats_rx { 94 u64 packets; 95 u64 bytes; 96 u64 xdp_drop; 97 u64 xdp_tx; 98 u64 xdp_redirect; 99 u64 pkt_len0_err; 100 u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1]; 101 struct u64_stats_sync syncp; 102 }; 103 104 struct mana_stats_tx { 105 u64 packets; 106 u64 bytes; 107 u64 xdp_xmit; 108 u64 tso_packets; 109 u64 tso_bytes; 110 u64 tso_inner_packets; 111 u64 tso_inner_bytes; 112 u64 short_pkt_fmt; 113 u64 long_pkt_fmt; 114 u64 csum_partial; 115 u64 mana_map_err; 116 struct u64_stats_sync syncp; 117 }; 118 119 struct mana_txq { 120 struct gdma_queue *gdma_sq; 121 122 union { 123 u32 gdma_txq_id; 124 struct { 125 u32 reserved1 : 10; 126 u32 vsq_frame : 14; 127 u32 reserved2 : 8; 128 }; 129 }; 130 131 u16 vp_offset; 132 133 struct net_device *ndev; 134 135 /* The SKBs are sent to the HW and we are waiting for the CQEs. */ 136 struct sk_buff_head pending_skbs; 137 struct netdev_queue *net_txq; 138 139 atomic_t pending_sends; 140 141 bool napi_initialized; 142 143 struct mana_stats_tx stats; 144 }; 145 146 /* skb data and frags dma mappings */ 147 struct mana_skb_head { 148 /* GSO pkts may have 2 SGEs for the linear part*/ 149 dma_addr_t dma_handle[MAX_SKB_FRAGS + 2]; 150 151 u32 size[MAX_SKB_FRAGS + 2]; 152 }; 153 154 #define MANA_HEADROOM sizeof(struct mana_skb_head) 155 156 enum mana_tx_pkt_format { 157 MANA_SHORT_PKT_FMT = 0, 158 MANA_LONG_PKT_FMT = 1, 159 }; 160 161 struct mana_tx_short_oob { 162 u32 pkt_fmt : 2; 163 u32 is_outer_ipv4 : 1; 164 u32 is_outer_ipv6 : 1; 165 u32 comp_iphdr_csum : 1; 166 u32 comp_tcp_csum : 1; 167 u32 comp_udp_csum : 1; 168 u32 supress_txcqe_gen : 1; 169 u32 vcq_num : 24; 170 171 u32 trans_off : 10; /* Transport header offset */ 172 u32 vsq_frame : 14; 173 u32 short_vp_offset : 8; 174 }; /* HW DATA */ 175 176 struct mana_tx_long_oob { 177 u32 is_encap : 1; 178 u32 inner_is_ipv6 : 1; 179 u32 inner_tcp_opt : 1; 180 u32 inject_vlan_pri_tag : 1; 181 u32 reserved1 : 12; 182 u32 pcp : 3; /* 802.1Q */ 183 u32 dei : 1; /* 802.1Q */ 184 u32 vlan_id : 12; /* 802.1Q */ 185 186 u32 inner_frame_offset : 10; 187 u32 inner_ip_rel_offset : 6; 188 u32 long_vp_offset : 12; 189 u32 reserved2 : 4; 190 191 u32 reserved3; 192 u32 reserved4; 193 }; /* HW DATA */ 194 195 struct mana_tx_oob { 196 struct mana_tx_short_oob s_oob; 197 struct mana_tx_long_oob l_oob; 198 }; /* HW DATA */ 199 200 enum mana_cq_type { 201 MANA_CQ_TYPE_RX, 202 MANA_CQ_TYPE_TX, 203 }; 204 205 enum mana_cqe_type { 206 CQE_INVALID = 0, 207 CQE_RX_OKAY = 1, 208 CQE_RX_COALESCED_4 = 2, 209 CQE_RX_OBJECT_FENCE = 3, 210 CQE_RX_TRUNCATED = 4, 211 212 CQE_TX_OKAY = 32, 213 CQE_TX_SA_DROP = 33, 214 CQE_TX_MTU_DROP = 34, 215 CQE_TX_INVALID_OOB = 35, 216 CQE_TX_INVALID_ETH_TYPE = 36, 217 CQE_TX_HDR_PROCESSING_ERROR = 37, 218 CQE_TX_VF_DISABLED = 38, 219 CQE_TX_VPORT_IDX_OUT_OF_RANGE = 39, 220 CQE_TX_VPORT_DISABLED = 40, 221 CQE_TX_VLAN_TAGGING_VIOLATION = 41, 222 }; 223 224 #define MANA_CQE_COMPLETION 1 225 226 struct mana_cqe_header { 227 u32 cqe_type : 6; 228 u32 client_type : 2; 229 u32 vendor_err : 24; 230 }; /* HW DATA */ 231 232 /* NDIS HASH Types */ 233 #define NDIS_HASH_IPV4 BIT(0) 234 #define NDIS_HASH_TCP_IPV4 BIT(1) 235 #define NDIS_HASH_UDP_IPV4 BIT(2) 236 #define NDIS_HASH_IPV6 BIT(3) 237 #define NDIS_HASH_TCP_IPV6 BIT(4) 238 #define NDIS_HASH_UDP_IPV6 BIT(5) 239 #define NDIS_HASH_IPV6_EX BIT(6) 240 #define NDIS_HASH_TCP_IPV6_EX BIT(7) 241 #define NDIS_HASH_UDP_IPV6_EX BIT(8) 242 243 #define MANA_HASH_L3 (NDIS_HASH_IPV4 | NDIS_HASH_IPV6 | NDIS_HASH_IPV6_EX) 244 #define MANA_HASH_L4 \ 245 (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \ 246 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX) 247 248 struct mana_rxcomp_perpkt_info { 249 u32 pkt_len : 16; 250 u32 reserved1 : 16; 251 u32 reserved2; 252 u32 pkt_hash; 253 }; /* HW DATA */ 254 255 /* Receive completion OOB */ 256 struct mana_rxcomp_oob { 257 struct mana_cqe_header cqe_hdr; 258 259 u32 rx_vlan_id : 12; 260 u32 rx_vlantag_present : 1; 261 u32 rx_outer_iphdr_csum_succeed : 1; 262 u32 rx_outer_iphdr_csum_fail : 1; 263 u32 reserved1 : 1; 264 u32 rx_hashtype : 9; 265 u32 rx_iphdr_csum_succeed : 1; 266 u32 rx_iphdr_csum_fail : 1; 267 u32 rx_tcp_csum_succeed : 1; 268 u32 rx_tcp_csum_fail : 1; 269 u32 rx_udp_csum_succeed : 1; 270 u32 rx_udp_csum_fail : 1; 271 u32 reserved2 : 1; 272 273 struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI]; 274 275 u32 rx_wqe_offset; 276 }; /* HW DATA */ 277 278 struct mana_tx_comp_oob { 279 struct mana_cqe_header cqe_hdr; 280 281 u32 tx_data_offset; 282 283 u32 tx_sgl_offset : 5; 284 u32 tx_wqe_offset : 27; 285 286 u32 reserved[12]; 287 }; /* HW DATA */ 288 289 struct mana_rxq; 290 291 #define CQE_POLLING_BUFFER 512 292 293 struct mana_cq { 294 struct gdma_queue *gdma_cq; 295 296 /* Cache the CQ id (used to verify if each CQE comes to the right CQ. */ 297 u32 gdma_id; 298 299 /* Type of the CQ: TX or RX */ 300 enum mana_cq_type type; 301 302 /* Pointer to the mana_rxq that is pushing RX CQEs to the queue. 303 * Only and must be non-NULL if type is MANA_CQ_TYPE_RX. 304 */ 305 struct mana_rxq *rxq; 306 307 /* Pointer to the mana_txq that is pushing TX CQEs to the queue. 308 * Only and must be non-NULL if type is MANA_CQ_TYPE_TX. 309 */ 310 struct mana_txq *txq; 311 312 /* Buffer which the CQ handler can copy the CQE's into. */ 313 struct gdma_comp gdma_comp_buf[CQE_POLLING_BUFFER]; 314 315 /* NAPI data */ 316 struct napi_struct napi; 317 int work_done; 318 int work_done_since_doorbell; 319 int budget; 320 321 /* DIM - Dynamic Interrupt Moderation */ 322 struct dim dim; 323 u16 dim_event_ctr; 324 325 /* Cumulative TX completions fed to DIM. Updated and read only in 326 * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they 327 * measure the hardware completion rate and need no u64_stats_sync. 328 */ 329 u64 tx_dim_pkts; 330 u64 tx_dim_bytes; 331 }; 332 333 struct mana_recv_buf_oob { 334 /* A valid GDMA work request representing the data buffer. */ 335 struct gdma_wqe_request wqe_req; 336 337 void *buf_va; 338 bool from_pool; /* allocated from a page pool */ 339 /* head page of the page_pool fragment; valid only when 340 * from_pool && frag_count > 1. 341 */ 342 struct page *pp_page; 343 /* Fragment offset plus rxq->headroom, passed to 344 * page_pool_dma_sync_for_cpu(). 345 */ 346 u32 dma_sync_offset; 347 348 /* SGL of the buffer going to be sent as part of the work request. */ 349 u32 num_sge; 350 struct gdma_sge sgl[MAX_RX_WQE_SGL_ENTRIES]; 351 352 /* Required to store the result of mana_gd_post_work_request. 353 * gdma_posted_wqe_info.wqe_size_in_bu is required for progressing the 354 * work queue when the WQE is consumed. 355 */ 356 struct gdma_posted_wqe_info wqe_inf; 357 }; 358 359 #define MANA_RXBUF_PAD (SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) \ 360 + ETH_HLEN) 361 362 #define MANA_XDP_MTU_MAX (PAGE_SIZE - MANA_RXBUF_PAD - XDP_PACKET_HEADROOM) 363 364 struct mana_rxq { 365 struct gdma_queue *gdma_rq; 366 /* Cache the gdma receive queue id */ 367 u32 gdma_id; 368 369 /* Index of RQ in the vPort, not gdma receive queue id */ 370 u32 rxq_idx; 371 372 u32 datasize; 373 u32 alloc_size; 374 u32 headroom; 375 u32 frag_count; 376 377 mana_handle_t rxobj; 378 379 struct mana_cq rx_cq; 380 381 struct completion fence_event; 382 383 struct net_device *ndev; 384 385 /* Total number of receive buffers to be allocated */ 386 u32 num_rx_buf; 387 388 u32 buf_index; 389 390 struct mana_stats_rx stats; 391 392 struct bpf_prog __rcu *bpf_prog; 393 struct xdp_rxq_info xdp_rxq; 394 void *xdp_save_va; /* for reusing */ 395 bool xdp_flush; 396 int xdp_rc; /* XDP redirect return code */ 397 398 struct page_pool *page_pool; 399 struct dentry *mana_rx_debugfs; 400 401 /* MUST BE THE LAST MEMBER: 402 * Each receive buffer has an associated mana_recv_buf_oob. 403 */ 404 struct mana_recv_buf_oob rx_oobs[] __counted_by(num_rx_buf); 405 }; 406 407 struct mana_tx_qp { 408 struct mana_txq txq; 409 410 struct mana_cq tx_cq; 411 412 mana_handle_t tx_object; 413 414 struct dentry *mana_tx_debugfs; 415 }; 416 417 struct mana_ethtool_stats { 418 u64 stop_queue; 419 u64 wake_queue; 420 u64 tx_cqe_err; 421 u64 tx_cqe_unknown_type; 422 u64 tx_linear_pkt_cnt; 423 u64 rx_cqe_unknown_type; 424 }; 425 426 struct mana_ethtool_hc_stats { 427 u64 hc_rx_discards_no_wqe; 428 u64 hc_rx_err_vport_disabled; 429 u64 hc_rx_bytes; 430 u64 hc_rx_ucast_pkts; 431 u64 hc_rx_ucast_bytes; 432 u64 hc_rx_bcast_pkts; 433 u64 hc_rx_bcast_bytes; 434 u64 hc_rx_mcast_pkts; 435 u64 hc_rx_mcast_bytes; 436 u64 hc_tx_err_gf_disabled; 437 u64 hc_tx_err_vport_disabled; 438 u64 hc_tx_err_inval_vportoffset_pkt; 439 u64 hc_tx_err_vlan_enforcement; 440 u64 hc_tx_err_eth_type_enforcement; 441 u64 hc_tx_err_sa_enforcement; 442 u64 hc_tx_err_sqpdid_enforcement; 443 u64 hc_tx_err_cqpdid_enforcement; 444 u64 hc_tx_err_mtu_violation; 445 u64 hc_tx_err_inval_oob; 446 u64 hc_tx_bytes; 447 u64 hc_tx_ucast_pkts; 448 u64 hc_tx_ucast_bytes; 449 u64 hc_tx_bcast_pkts; 450 u64 hc_tx_bcast_bytes; 451 u64 hc_tx_mcast_pkts; 452 u64 hc_tx_mcast_bytes; 453 u64 hc_tx_err_gdma; 454 }; 455 456 struct mana_ethtool_phy_stats { 457 /* Drop Counters */ 458 u64 rx_pkt_drop_phy; 459 u64 tx_pkt_drop_phy; 460 461 /* Per TC traffic Counters */ 462 u64 rx_pkt_tc0_phy; 463 u64 tx_pkt_tc0_phy; 464 u64 rx_pkt_tc1_phy; 465 u64 tx_pkt_tc1_phy; 466 u64 rx_pkt_tc2_phy; 467 u64 tx_pkt_tc2_phy; 468 u64 rx_pkt_tc3_phy; 469 u64 tx_pkt_tc3_phy; 470 u64 rx_pkt_tc4_phy; 471 u64 tx_pkt_tc4_phy; 472 u64 rx_pkt_tc5_phy; 473 u64 tx_pkt_tc5_phy; 474 u64 rx_pkt_tc6_phy; 475 u64 tx_pkt_tc6_phy; 476 u64 rx_pkt_tc7_phy; 477 u64 tx_pkt_tc7_phy; 478 479 u64 rx_byte_tc0_phy; 480 u64 tx_byte_tc0_phy; 481 u64 rx_byte_tc1_phy; 482 u64 tx_byte_tc1_phy; 483 u64 rx_byte_tc2_phy; 484 u64 tx_byte_tc2_phy; 485 u64 rx_byte_tc3_phy; 486 u64 tx_byte_tc3_phy; 487 u64 rx_byte_tc4_phy; 488 u64 tx_byte_tc4_phy; 489 u64 rx_byte_tc5_phy; 490 u64 tx_byte_tc5_phy; 491 u64 rx_byte_tc6_phy; 492 u64 tx_byte_tc6_phy; 493 u64 rx_byte_tc7_phy; 494 u64 tx_byte_tc7_phy; 495 496 /* Per TC pause Counters */ 497 u64 rx_pause_tc0_phy; 498 u64 tx_pause_tc0_phy; 499 u64 rx_pause_tc1_phy; 500 u64 tx_pause_tc1_phy; 501 u64 rx_pause_tc2_phy; 502 u64 tx_pause_tc2_phy; 503 u64 rx_pause_tc3_phy; 504 u64 tx_pause_tc3_phy; 505 u64 rx_pause_tc4_phy; 506 u64 tx_pause_tc4_phy; 507 u64 rx_pause_tc5_phy; 508 u64 tx_pause_tc5_phy; 509 u64 rx_pause_tc6_phy; 510 u64 tx_pause_tc6_phy; 511 u64 rx_pause_tc7_phy; 512 u64 tx_pause_tc7_phy; 513 }; 514 515 struct mana_context { 516 struct gdma_dev *gdma_dev; 517 518 u16 num_ports; 519 u8 bm_hostmode; 520 521 struct mana_ethtool_hc_stats hc_stats; 522 struct workqueue_struct *per_port_queue_reset_wq; 523 /* Workqueue for querying hardware stats */ 524 struct delayed_work gf_stats_work; 525 bool hwc_timeout_occurred; 526 527 struct net_device *ports[MAX_PORTS_IN_MANA_DEV]; 528 529 /* Link state change work */ 530 struct work_struct link_change_work; 531 u32 link_event; 532 }; 533 534 struct mana_port_context { 535 struct mana_context *ac; 536 struct net_device *ndev; 537 struct work_struct queue_reset_work; 538 539 /* Debug knob to log TX timeout but skip recovery reset */ 540 bool tx_timeout_skip_reset; 541 542 u8 mac_addr[ETH_ALEN]; 543 544 struct mana_eq *eqs; 545 struct dentry *mana_eqs_debugfs; 546 547 enum TRI_STATE rss_state; 548 549 mana_handle_t default_rxobj; 550 bool tx_shortform_allowed; 551 u16 tx_vp_offset; 552 553 struct mana_tx_qp **tx_qp; 554 555 /* Indirection Table for RX & TX. The values are queue indexes */ 556 u32 *indir_table; 557 u32 indir_table_sz; 558 559 /* Indirection table containing RxObject Handles */ 560 mana_handle_t *rxobj_table; 561 562 /* Hash key used by the NIC */ 563 u8 hashkey[MANA_HASH_KEY_SIZE]; 564 565 /* This points to an array of num_queues of RQ pointers. */ 566 struct mana_rxq **rxqs; 567 568 /* pre-allocated rx buffer array */ 569 void **rxbufs_pre; 570 dma_addr_t *das_pre; 571 int rxbpre_total; 572 u32 rxbpre_datasize; 573 u32 rxbpre_alloc_size; 574 u32 rxbpre_headroom; 575 u32 rxbpre_frag_count; 576 577 u32 priv_flags; 578 579 struct bpf_prog *bpf_prog; 580 581 /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */ 582 unsigned int max_queues; 583 unsigned int num_queues; 584 585 unsigned int rx_queue_size; 586 unsigned int tx_queue_size; 587 588 mana_handle_t port_handle; 589 mana_handle_t pf_filter_handle; 590 591 /* Mutex for sharing access to vport_use_count */ 592 struct mutex vport_mutex; 593 int vport_use_count; 594 595 /* Set by mana_set_channels() under vport_mutex to block RDMA 596 * from grabbing the vport during the detach/attach window. 597 * Checked by mana_cfg_vport() when called from the RDMA path. 598 */ 599 bool channel_changing; 600 601 /* Net shaper handle*/ 602 struct net_shaper_handle handle; 603 604 u16 port_idx; 605 /* Currently configured speed (mbps) */ 606 u32 speed; 607 /* Maximum speed supported by the SKU (mbps) */ 608 u32 max_speed; 609 /* 1 = not queried, 0 = cached success, negative = permanent error. 610 * Protected by the netdev instance lock. 611 */ 612 int link_cfg_error; 613 614 bool port_is_up; 615 bool port_st_save; /* Saved port state */ 616 617 u8 cqe_coalescing_enable; 618 u32 cqe_coalescing_timeout_ns; 619 620 /* Interrupt moderation settings */ 621 u16 intr_modr_rx_usec; 622 u16 intr_modr_rx_comp; 623 u16 intr_modr_tx_usec; 624 u16 intr_modr_tx_comp; 625 626 bool rx_dim_enabled; 627 bool tx_dim_enabled; 628 629 struct mana_ethtool_stats eth_stats; 630 631 struct mana_ethtool_phy_stats phy_stats; 632 633 /* Debugfs */ 634 struct dentry *mana_port_debugfs; 635 636 /* Cached vport/steering config for debugfs */ 637 u32 vport_max_sq; 638 u32 vport_max_rq; 639 u32 steer_rx; 640 u32 steer_rss; 641 bool steer_update_tab; 642 u32 steer_cqe_coalescing; 643 }; 644 645 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); 646 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, 647 bool update_hash, bool update_tab); 648 int mana_disable_vport_rx(struct mana_port_context *apc); 649 650 int mana_alloc_queues(struct net_device *ndev); 651 int mana_attach(struct net_device *ndev); 652 int mana_detach(struct net_device *ndev, bool from_close); 653 654 void mana_dim_change(struct mana_cq *cq, bool enable); 655 656 int mana_probe(struct gdma_dev *gd, bool resuming); 657 void mana_remove(struct gdma_dev *gd, bool suspending); 658 659 int mana_rdma_probe(struct gdma_dev *gd); 660 void mana_rdma_remove(struct gdma_dev *gd); 661 662 void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); 663 int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, 664 u32 flags); 665 u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, 666 struct xdp_buff *xdp, void *buf_va, uint pkt_len); 667 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc); 668 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog); 669 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf); 670 int mana_query_gf_stats(struct mana_context *ac); 671 int mana_query_link_cfg(struct mana_port_context *apc); 672 int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, 673 int enable_clamping); 674 void mana_query_phy_stats(struct mana_port_context *apc); 675 int mana_pre_alloc_rxbufs(struct mana_port_context *apc, int mtu, int num_queues); 676 void mana_pre_dealloc_rxbufs(struct mana_port_context *apc); 677 void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc); 678 679 extern const struct ethtool_ops mana_ethtool_ops; 680 extern struct dentry *mana_debugfs_root; 681 682 /* A CQ can be created not associated with any EQ */ 683 #define GDMA_CQ_NO_EQ 0xffff 684 685 struct mana_obj_spec { 686 u32 queue_index; 687 u64 gdma_region; 688 u32 queue_size; 689 u32 attached_eq; 690 u32 modr_ctx_id; 691 u8 req_cq_moderation; 692 u16 cq_moderation_comp; 693 u16 cq_moderation_usec; 694 }; 695 696 enum mana_command_code { 697 MANA_QUERY_DEV_CONFIG = 0x20001, 698 MANA_QUERY_GF_STAT = 0x20002, 699 MANA_CONFIG_VPORT_TX = 0x20003, 700 MANA_CREATE_WQ_OBJ = 0x20004, 701 MANA_DESTROY_WQ_OBJ = 0x20005, 702 MANA_FENCE_RQ = 0x20006, 703 MANA_CONFIG_VPORT_RX = 0x20007, 704 MANA_QUERY_VPORT_CONFIG = 0x20008, 705 MANA_QUERY_LINK_CONFIG = 0x2000A, 706 MANA_SET_BW_CLAMP = 0x2000B, 707 MANA_QUERY_PHY_STAT = 0x2000c, 708 709 /* Privileged commands for the PF mode */ 710 MANA_REGISTER_FILTER = 0x28000, 711 MANA_DEREGISTER_FILTER = 0x28001, 712 MANA_REGISTER_HW_PORT = 0x28003, 713 MANA_DEREGISTER_HW_PORT = 0x28004, 714 }; 715 716 /* Query Link Configuration*/ 717 struct mana_query_link_config_req { 718 struct gdma_req_hdr hdr; 719 mana_handle_t vport; 720 }; /* HW DATA */ 721 722 struct mana_query_link_config_resp { 723 struct gdma_resp_hdr hdr; 724 u32 qos_speed_mbps; 725 u8 qos_unconfigured; 726 u8 reserved1[3]; 727 u32 link_speed_mbps; 728 u8 reserved2[4]; 729 }; /* HW DATA */ 730 731 /* Set Bandwidth Clamp*/ 732 struct mana_set_bw_clamp_req { 733 struct gdma_req_hdr hdr; 734 mana_handle_t vport; 735 enum TRI_STATE enable_clamping; 736 u32 link_speed_mbps; 737 }; /* HW DATA */ 738 739 struct mana_set_bw_clamp_resp { 740 struct gdma_resp_hdr hdr; 741 u8 qos_unconfigured; 742 u8 reserved[7]; 743 }; /* HW DATA */ 744 745 /* Query Device Configuration */ 746 struct mana_query_device_cfg_req { 747 struct gdma_req_hdr hdr; 748 749 /* MANA Nic Driver Capability flags */ 750 u64 mn_drv_cap_flags1; 751 u64 mn_drv_cap_flags2; 752 u64 mn_drv_cap_flags3; 753 u64 mn_drv_cap_flags4; 754 755 u32 proto_major_ver; 756 u32 proto_minor_ver; 757 u32 proto_micro_ver; 758 759 u32 reserved; 760 }; /* HW DATA */ 761 762 struct mana_query_device_cfg_resp { 763 struct gdma_resp_hdr hdr; 764 765 u64 pf_cap_flags1; 766 u64 pf_cap_flags2; 767 u64 pf_cap_flags3; 768 u64 pf_cap_flags4; 769 770 u16 max_num_vports; 771 u8 bm_hostmode; /* response v3: Bare Metal Host Mode */ 772 u8 reserved; 773 u32 max_num_eqs; 774 775 /* response v2: */ 776 u16 adapter_mtu; 777 u16 reserved2; 778 u32 reserved3; 779 }; /* HW DATA */ 780 781 /* Query vPort Configuration */ 782 struct mana_query_vport_cfg_req { 783 struct gdma_req_hdr hdr; 784 u32 vport_index; 785 }; /* HW DATA */ 786 787 struct mana_query_vport_cfg_resp { 788 struct gdma_resp_hdr hdr; 789 u32 max_num_sq; 790 u32 max_num_rq; 791 u32 num_indirection_ent; 792 u32 reserved1; 793 u8 mac_addr[6]; 794 u8 reserved2[2]; 795 mana_handle_t vport; 796 }; /* HW DATA */ 797 798 /* Configure vPort */ 799 struct mana_config_vport_req { 800 struct gdma_req_hdr hdr; 801 mana_handle_t vport; 802 u32 pdid; 803 u32 doorbell_pageid; 804 }; /* HW DATA */ 805 806 struct mana_config_vport_resp { 807 struct gdma_resp_hdr hdr; 808 u16 tx_vport_offset; 809 u8 short_form_allowed; 810 u8 reserved; 811 }; /* HW DATA */ 812 813 /* Create WQ Object */ 814 struct mana_create_wqobj_req { 815 struct gdma_req_hdr hdr; 816 mana_handle_t vport; 817 u32 wq_type; 818 u32 reserved; 819 u64 wq_gdma_region; 820 u64 cq_gdma_region; 821 u32 wq_size; 822 u32 cq_size; 823 u32 cq_moderation_ctx_id; 824 u32 cq_parent_qid; 825 826 /* V2 */ 827 u8 allow_rqwqe_chain; 828 829 /* V3 */ 830 u8 req_cq_moderation; 831 u16 cq_moderation_comp; 832 u16 cq_moderation_usec; 833 u8 reserved2[2]; 834 }; /* HW DATA */ 835 836 struct mana_create_wqobj_resp { 837 struct gdma_resp_hdr hdr; 838 u32 wq_id; 839 u32 cq_id; 840 mana_handle_t wq_obj; 841 842 /* V2 */ 843 u16 cq_moderation_comp; 844 u16 cq_moderation_usec; 845 u8 cq_moderation_enabled; 846 u8 reserved1[3]; 847 }; /* HW DATA */ 848 849 /* Destroy WQ Object */ 850 struct mana_destroy_wqobj_req { 851 struct gdma_req_hdr hdr; 852 u32 wq_type; 853 u32 reserved; 854 mana_handle_t wq_obj_handle; 855 }; /* HW DATA */ 856 857 struct mana_destroy_wqobj_resp { 858 struct gdma_resp_hdr hdr; 859 }; /* HW DATA */ 860 861 /* Fence RQ */ 862 struct mana_fence_rq_req { 863 struct gdma_req_hdr hdr; 864 mana_handle_t wq_obj_handle; 865 }; /* HW DATA */ 866 867 struct mana_fence_rq_resp { 868 struct gdma_resp_hdr hdr; 869 }; /* HW DATA */ 870 871 /* Query stats RQ */ 872 struct mana_query_gf_stat_req { 873 struct gdma_req_hdr hdr; 874 u64 req_stats; 875 }; /* HW DATA */ 876 877 struct mana_query_gf_stat_resp { 878 struct gdma_resp_hdr hdr; 879 u64 reported_stats; 880 /* rx errors/discards */ 881 u64 rx_discards_nowqe; 882 u64 rx_err_vport_disabled; 883 /* rx bytes/packets */ 884 u64 hc_rx_bytes; 885 u64 hc_rx_ucast_pkts; 886 u64 hc_rx_ucast_bytes; 887 u64 hc_rx_bcast_pkts; 888 u64 hc_rx_bcast_bytes; 889 u64 hc_rx_mcast_pkts; 890 u64 hc_rx_mcast_bytes; 891 /* tx errors */ 892 u64 tx_err_gf_disabled; 893 u64 tx_err_vport_disabled; 894 u64 tx_err_inval_vport_offset_pkt; 895 u64 tx_err_vlan_enforcement; 896 u64 tx_err_ethtype_enforcement; 897 u64 tx_err_SA_enforcement; 898 u64 tx_err_SQPDID_enforcement; 899 u64 tx_err_CQPDID_enforcement; 900 u64 tx_err_mtu_violation; 901 u64 tx_err_inval_oob; 902 /* tx bytes/packets */ 903 u64 hc_tx_bytes; 904 u64 hc_tx_ucast_pkts; 905 u64 hc_tx_ucast_bytes; 906 u64 hc_tx_bcast_pkts; 907 u64 hc_tx_bcast_bytes; 908 u64 hc_tx_mcast_pkts; 909 u64 hc_tx_mcast_bytes; 910 /* tx error */ 911 u64 tx_err_gdma; 912 }; /* HW DATA */ 913 914 /* Query phy stats */ 915 struct mana_query_phy_stat_req { 916 struct gdma_req_hdr hdr; 917 u64 req_stats; 918 }; /* HW DATA */ 919 920 struct mana_query_phy_stat_resp { 921 struct gdma_resp_hdr hdr; 922 u64 reported_stats; 923 924 /* Aggregate Drop Counters */ 925 u64 rx_pkt_drop_phy; 926 u64 tx_pkt_drop_phy; 927 928 /* Per TC(Traffic class) traffic Counters */ 929 u64 rx_pkt_tc0_phy; 930 u64 tx_pkt_tc0_phy; 931 u64 rx_pkt_tc1_phy; 932 u64 tx_pkt_tc1_phy; 933 u64 rx_pkt_tc2_phy; 934 u64 tx_pkt_tc2_phy; 935 u64 rx_pkt_tc3_phy; 936 u64 tx_pkt_tc3_phy; 937 u64 rx_pkt_tc4_phy; 938 u64 tx_pkt_tc4_phy; 939 u64 rx_pkt_tc5_phy; 940 u64 tx_pkt_tc5_phy; 941 u64 rx_pkt_tc6_phy; 942 u64 tx_pkt_tc6_phy; 943 u64 rx_pkt_tc7_phy; 944 u64 tx_pkt_tc7_phy; 945 946 u64 rx_byte_tc0_phy; 947 u64 tx_byte_tc0_phy; 948 u64 rx_byte_tc1_phy; 949 u64 tx_byte_tc1_phy; 950 u64 rx_byte_tc2_phy; 951 u64 tx_byte_tc2_phy; 952 u64 rx_byte_tc3_phy; 953 u64 tx_byte_tc3_phy; 954 u64 rx_byte_tc4_phy; 955 u64 tx_byte_tc4_phy; 956 u64 rx_byte_tc5_phy; 957 u64 tx_byte_tc5_phy; 958 u64 rx_byte_tc6_phy; 959 u64 tx_byte_tc6_phy; 960 u64 rx_byte_tc7_phy; 961 u64 tx_byte_tc7_phy; 962 963 /* Per TC(Traffic Class) pause Counters */ 964 u64 rx_pause_tc0_phy; 965 u64 tx_pause_tc0_phy; 966 u64 rx_pause_tc1_phy; 967 u64 tx_pause_tc1_phy; 968 u64 rx_pause_tc2_phy; 969 u64 tx_pause_tc2_phy; 970 u64 rx_pause_tc3_phy; 971 u64 tx_pause_tc3_phy; 972 u64 rx_pause_tc4_phy; 973 u64 tx_pause_tc4_phy; 974 u64 rx_pause_tc5_phy; 975 u64 tx_pause_tc5_phy; 976 u64 rx_pause_tc6_phy; 977 u64 tx_pause_tc6_phy; 978 u64 rx_pause_tc7_phy; 979 u64 tx_pause_tc7_phy; 980 }; /* HW DATA */ 981 982 /* Configure vPort Rx Steering */ 983 struct mana_cfg_rx_steer_req_v2 { 984 struct gdma_req_hdr hdr; 985 mana_handle_t vport; 986 u16 num_indir_entries; 987 u16 indir_tab_offset; 988 u32 rx_enable; 989 u32 rss_enable; 990 u8 update_default_rxobj; 991 u8 update_hashkey; 992 u8 update_indir_tab; 993 u8 reserved; 994 mana_handle_t default_rxobj; 995 u8 hashkey[MANA_HASH_KEY_SIZE]; 996 u8 cqe_coalescing_enable; 997 u8 reserved2[7]; 998 mana_handle_t indir_tab[] __counted_by(num_indir_entries); 999 }; /* HW DATA */ 1000 1001 struct mana_cfg_rx_steer_resp { 1002 struct gdma_resp_hdr hdr; 1003 1004 /* V2 */ 1005 u32 cqe_coalescing_timeout_ns; 1006 u32 reserved1; 1007 }; /* HW DATA */ 1008 1009 /* Register HW vPort */ 1010 struct mana_register_hw_vport_req { 1011 struct gdma_req_hdr hdr; 1012 u16 attached_gfid; 1013 u8 is_pf_default_vport; 1014 u8 reserved1; 1015 u8 allow_all_ether_types; 1016 u8 reserved2; 1017 u8 reserved3; 1018 u8 reserved4; 1019 }; /* HW DATA */ 1020 1021 struct mana_register_hw_vport_resp { 1022 struct gdma_resp_hdr hdr; 1023 mana_handle_t hw_vport_handle; 1024 }; /* HW DATA */ 1025 1026 /* Deregister HW vPort */ 1027 struct mana_deregister_hw_vport_req { 1028 struct gdma_req_hdr hdr; 1029 mana_handle_t hw_vport_handle; 1030 }; /* HW DATA */ 1031 1032 struct mana_deregister_hw_vport_resp { 1033 struct gdma_resp_hdr hdr; 1034 }; /* HW DATA */ 1035 1036 /* Register filter */ 1037 struct mana_register_filter_req { 1038 struct gdma_req_hdr hdr; 1039 mana_handle_t vport; 1040 u8 mac_addr[6]; 1041 u8 reserved1; 1042 u8 reserved2; 1043 u8 reserved3; 1044 u8 reserved4; 1045 u16 reserved5; 1046 u32 reserved6; 1047 u32 reserved7; 1048 u32 reserved8; 1049 }; /* HW DATA */ 1050 1051 struct mana_register_filter_resp { 1052 struct gdma_resp_hdr hdr; 1053 mana_handle_t filter_handle; 1054 }; /* HW DATA */ 1055 1056 /* Deregister filter */ 1057 struct mana_deregister_filter_req { 1058 struct gdma_req_hdr hdr; 1059 mana_handle_t filter_handle; 1060 }; /* HW DATA */ 1061 1062 struct mana_deregister_filter_resp { 1063 struct gdma_resp_hdr hdr; 1064 }; /* HW DATA */ 1065 1066 /* Requested GF stats Flags */ 1067 /* Rx discards/Errors */ 1068 #define STATISTICS_FLAGS_RX_DISCARDS_NO_WQE 0x0000000000000001 1069 #define STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED 0x0000000000000002 1070 /* Rx bytes/pkts */ 1071 #define STATISTICS_FLAGS_HC_RX_BYTES 0x0000000000000004 1072 #define STATISTICS_FLAGS_HC_RX_UCAST_PACKETS 0x0000000000000008 1073 #define STATISTICS_FLAGS_HC_RX_UCAST_BYTES 0x0000000000000010 1074 #define STATISTICS_FLAGS_HC_RX_MCAST_PACKETS 0x0000000000000020 1075 #define STATISTICS_FLAGS_HC_RX_MCAST_BYTES 0x0000000000000040 1076 #define STATISTICS_FLAGS_HC_RX_BCAST_PACKETS 0x0000000000000080 1077 #define STATISTICS_FLAGS_HC_RX_BCAST_BYTES 0x0000000000000100 1078 /* Tx errors */ 1079 #define STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED 0x0000000000000200 1080 #define STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED 0x0000000000000400 1081 #define STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS \ 1082 0x0000000000000800 1083 #define STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT 0x0000000000001000 1084 #define STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT \ 1085 0x0000000000002000 1086 #define STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT 0x0000000000004000 1087 #define STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT 0x0000000000008000 1088 #define STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT 0x0000000000010000 1089 #define STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION 0x0000000000020000 1090 #define STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB 0x0000000000040000 1091 /* Tx bytes/pkts */ 1092 #define STATISTICS_FLAGS_HC_TX_BYTES 0x0000000000080000 1093 #define STATISTICS_FLAGS_HC_TX_UCAST_PACKETS 0x0000000000100000 1094 #define STATISTICS_FLAGS_HC_TX_UCAST_BYTES 0x0000000000200000 1095 #define STATISTICS_FLAGS_HC_TX_MCAST_PACKETS 0x0000000000400000 1096 #define STATISTICS_FLAGS_HC_TX_MCAST_BYTES 0x0000000000800000 1097 #define STATISTICS_FLAGS_HC_TX_BCAST_PACKETS 0x0000000001000000 1098 #define STATISTICS_FLAGS_HC_TX_BCAST_BYTES 0x0000000002000000 1099 /* Tx error */ 1100 #define STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR 0x0000000004000000 1101 1102 #define MANA_MAX_NUM_QUEUES 64 1103 #define MANA_DEF_NUM_QUEUES 16 1104 1105 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1) 1106 1107 struct mana_tx_package { 1108 struct gdma_wqe_request wqe_req; 1109 struct gdma_sge sgl_array[5]; 1110 struct gdma_sge *sgl_ptr; 1111 1112 struct mana_tx_oob tx_oob; 1113 1114 struct gdma_posted_wqe_info wqe_info; 1115 }; 1116 1117 int mana_create_wq_obj(struct mana_port_context *apc, 1118 mana_handle_t vport, 1119 u32 wq_type, struct mana_obj_spec *wq_spec, 1120 struct mana_obj_spec *cq_spec, 1121 mana_handle_t *wq_obj); 1122 1123 void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 1124 mana_handle_t wq_obj); 1125 1126 int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 1127 u32 doorbell_pg_id, bool check_channel_changing); 1128 void mana_uncfg_vport(struct mana_port_context *apc); 1129 int mana_create_eq(struct mana_port_context *apc); 1130 void mana_destroy_eq(struct mana_port_context *apc); 1131 1132 struct net_device *mana_get_primary_netdev(struct mana_context *ac, 1133 u32 port_index, 1134 netdevice_tracker *tracker); 1135 #endif /* _MANA_H */ 1136