1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #ifndef _GDMA_H 5 #define _GDMA_H 6 7 #include <linux/dma-mapping.h> 8 #include <linux/netdevice.h> 9 10 #include "shm_channel.h" 11 12 #define GDMA_STATUS_MORE_ENTRIES 0x00000105 13 #define GDMA_STATUS_CMD_UNSUPPORTED 0xffffffff 14 15 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of 16 * them are naturally aligned and hence don't need __packed. 17 */ 18 19 enum gdma_request_type { 20 GDMA_VERIFY_VF_DRIVER_VERSION = 1, 21 GDMA_QUERY_MAX_RESOURCES = 2, 22 GDMA_LIST_DEVICES = 3, 23 GDMA_REGISTER_DEVICE = 4, 24 GDMA_DEREGISTER_DEVICE = 5, 25 GDMA_GENERATE_TEST_EQE = 10, 26 GDMA_CREATE_QUEUE = 12, 27 GDMA_DISABLE_QUEUE = 13, 28 GDMA_ALLOCATE_RESOURCE_RANGE = 22, 29 GDMA_DESTROY_RESOURCE_RANGE = 24, 30 GDMA_CREATE_DMA_REGION = 25, 31 GDMA_DMA_REGION_ADD_PAGES = 26, 32 GDMA_DESTROY_DMA_REGION = 27, 33 GDMA_CREATE_PD = 29, 34 GDMA_DESTROY_PD = 30, 35 GDMA_CREATE_MR = 31, 36 GDMA_DESTROY_MR = 32, 37 GDMA_QUERY_HWC_TIMEOUT = 84, /* 0x54 */ 38 GDMA_ALLOC_DM = 96, /* 0x60 */ 39 GDMA_DESTROY_DM = 97, /* 0x61 */ 40 }; 41 42 #define GDMA_RESOURCE_DOORBELL_PAGE 27 43 44 enum gdma_queue_type { 45 GDMA_INVALID_QUEUE, 46 GDMA_SQ, 47 GDMA_RQ, 48 GDMA_CQ, 49 GDMA_EQ, 50 GDMA_DIM, 51 }; 52 53 enum gdma_work_request_flags { 54 GDMA_WR_NONE = 0, 55 GDMA_WR_OOB_IN_SGL = BIT(0), 56 GDMA_WR_PAD_BY_SGE0 = BIT(1), 57 }; 58 59 enum gdma_eqe_type { 60 GDMA_EQE_COMPLETION = 3, 61 GDMA_EQE_TEST_EVENT = 64, 62 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129, 63 GDMA_EQE_HWC_INIT_DATA = 130, 64 GDMA_EQE_HWC_INIT_DONE = 131, 65 GDMA_EQE_HWC_FPGA_RECONFIG = 132, 66 GDMA_EQE_HWC_SOC_RECONFIG_DATA = 133, 67 GDMA_EQE_HWC_SOC_SERVICE = 134, 68 GDMA_EQE_HWC_RESET_REQUEST = 135, 69 GDMA_EQE_RNIC_QP_FATAL = 176, 70 }; 71 72 enum { 73 GDMA_DEVICE_NONE = 0, 74 GDMA_DEVICE_HWC = 1, 75 GDMA_DEVICE_MANA = 2, 76 GDMA_DEVICE_MANA_IB = 3, 77 }; 78 79 enum gdma_service_type { 80 GDMA_SERVICE_TYPE_NONE = 0, 81 GDMA_SERVICE_TYPE_RDMA_SUSPEND = 1, 82 GDMA_SERVICE_TYPE_RDMA_RESUME = 2, 83 }; 84 85 struct mana_service_work { 86 struct work_struct work; 87 struct gdma_dev *gdma_dev; 88 enum gdma_service_type event; 89 }; 90 91 struct gdma_resource { 92 /* Protect the bitmap */ 93 spinlock_t lock; 94 95 /* The bitmap size in bits. */ 96 u32 size; 97 98 /* The bitmap tracks the resources. */ 99 unsigned long *map; 100 }; 101 102 union gdma_doorbell_entry { 103 u64 as_uint64; 104 105 struct { 106 u64 id : 24; 107 u64 reserved : 8; 108 u64 tail_ptr : 31; 109 u64 arm : 1; 110 } cq; 111 112 struct { 113 u64 id : 24; 114 u64 wqe_cnt : 8; 115 u64 tail_ptr : 32; 116 } rq; 117 118 struct { 119 u64 id : 24; 120 u64 reserved : 8; 121 u64 tail_ptr : 32; 122 } sq; 123 124 struct { 125 u64 id : 16; 126 u64 reserved : 16; 127 u64 tail_ptr : 31; 128 u64 arm : 1; 129 } eq; 130 131 struct { 132 u64 id : 24; 133 u64 reserved : 8; 134 u64 mod_usec : 10; 135 u64 reserve1 : 5; 136 u64 mod_usec_vld : 1; 137 u64 mod_comps : 8; 138 u64 reserve2 : 7; 139 u64 mod_comps_vld: 1; 140 } dim; 141 }; /* HW DATA */ 142 143 struct gdma_msg_hdr { 144 u32 hdr_type; 145 u32 msg_type; 146 u16 msg_version; 147 u16 hwc_msg_id; 148 u32 msg_size; 149 }; /* HW DATA */ 150 151 struct gdma_dev_id { 152 union { 153 struct { 154 u16 type; 155 u16 instance; 156 }; 157 158 u32 as_uint32; 159 }; 160 }; /* HW DATA */ 161 162 struct gdma_req_hdr { 163 struct gdma_msg_hdr req; 164 struct gdma_msg_hdr resp; /* The expected response */ 165 struct gdma_dev_id dev_id; 166 u32 activity_id; 167 }; /* HW DATA */ 168 169 struct gdma_resp_hdr { 170 struct gdma_msg_hdr response; 171 struct gdma_dev_id dev_id; 172 u32 activity_id; 173 u32 status; 174 u32 reserved; 175 }; /* HW DATA */ 176 177 struct gdma_general_req { 178 struct gdma_req_hdr hdr; 179 }; /* HW DATA */ 180 181 #define GDMA_MESSAGE_V1 1 182 #define GDMA_MESSAGE_V2 2 183 #define GDMA_MESSAGE_V3 3 184 #define GDMA_MESSAGE_V4 4 185 #define GDMA_MESSAGE_V5 5 186 187 struct gdma_general_resp { 188 struct gdma_resp_hdr hdr; 189 }; /* HW DATA */ 190 191 #define GDMA_STANDARD_HEADER_TYPE 0 192 193 static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code, 194 u32 req_size, u32 resp_size) 195 { 196 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE; 197 hdr->req.msg_type = code; 198 hdr->req.msg_version = GDMA_MESSAGE_V1; 199 hdr->req.msg_size = req_size; 200 201 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE; 202 hdr->resp.msg_type = code; 203 hdr->resp.msg_version = GDMA_MESSAGE_V1; 204 hdr->resp.msg_size = resp_size; 205 } 206 207 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */ 208 struct gdma_sge { 209 u64 address; 210 u32 mem_key; 211 u32 size; 212 }; /* HW DATA */ 213 214 struct gdma_wqe_request { 215 struct gdma_sge *sgl; 216 u32 num_sge; 217 218 u32 inline_oob_size; 219 const void *inline_oob_data; 220 221 u32 flags; 222 u32 client_data_unit; 223 }; 224 225 enum gdma_page_type { 226 GDMA_PAGE_TYPE_4K, 227 }; 228 229 #define GDMA_INVALID_DMA_REGION 0 230 231 struct mana_serv_work { 232 struct work_struct serv_work; 233 struct pci_dev *pdev; 234 enum gdma_eqe_type type; 235 }; 236 237 struct gdma_mem_info { 238 struct device *dev; 239 240 dma_addr_t dma_handle; 241 void *virt_addr; 242 u64 length; 243 244 /* Scattered fallback: when @nr_pages > 0 the ring is that many 245 * PAGE_SIZE coherent allocations in @pages_va/@pages_dma, not 246 * @virt_addr/@dma_handle. 247 */ 248 void **pages_va; 249 dma_addr_t *pages_dma; 250 unsigned int nr_pages; 251 252 /* Allocated by the PF driver */ 253 u64 dma_region_handle; 254 }; 255 256 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8 257 258 struct gdma_dev { 259 struct gdma_context *gdma_context; 260 261 struct gdma_dev_id dev_id; 262 263 u32 pdid; 264 u32 doorbell; 265 u32 gpa_mkey; 266 267 /* GDMA driver specific pointer */ 268 void *driver_data; 269 270 struct auxiliary_device *adev; 271 bool is_suspended; 272 bool rdma_teardown; 273 }; 274 275 /* MANA_PAGE_SIZE is the DMA unit */ 276 #define MANA_PAGE_SHIFT 12 277 #define MANA_PAGE_SIZE BIT(MANA_PAGE_SHIFT) 278 #define MANA_PAGE_ALIGN(x) ALIGN((x), MANA_PAGE_SIZE) 279 #define MANA_PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), MANA_PAGE_SIZE) 280 #define MANA_PFN(a) ((a) >> MANA_PAGE_SHIFT) 281 282 /* Required by HW */ 283 #define MANA_MIN_QSIZE MANA_PAGE_SIZE 284 285 #define GDMA_CQE_SIZE 64 286 #define GDMA_EQE_SIZE 16 287 #define GDMA_MAX_SQE_SIZE 512 288 #define GDMA_MAX_RQE_SIZE 256 289 290 #define GDMA_COMP_DATA_SIZE 0x3C 291 292 #define GDMA_EVENT_DATA_SIZE 0xC 293 294 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */ 295 #define GDMA_WQE_BU_SIZE 32 296 297 #define INVALID_PDID UINT_MAX 298 #define INVALID_DOORBELL UINT_MAX 299 #define INVALID_MEM_KEY UINT_MAX 300 #define INVALID_QUEUE_ID UINT_MAX 301 #define INVALID_PCI_MSIX_INDEX UINT_MAX 302 303 struct gdma_comp { 304 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4]; 305 u32 wq_num; 306 bool is_sq; 307 }; 308 309 struct gdma_event { 310 u32 details[GDMA_EVENT_DATA_SIZE / 4]; 311 u8 type; 312 }; 313 314 struct gdma_queue; 315 316 struct mana_eq { 317 struct gdma_queue *eq; 318 struct dentry *mana_eq_debugfs; 319 }; 320 321 typedef void gdma_eq_callback(void *context, struct gdma_queue *q, 322 struct gdma_event *e); 323 324 typedef void gdma_cq_callback(void *context, struct gdma_queue *q); 325 326 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE 327 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the 328 * driver increases the 'head' in BUs rather than in bytes, and notifies 329 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track 330 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE. 331 * 332 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is 333 * processed, the driver increases the 'tail' to indicate that WQEs have 334 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ. 335 * 336 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures 337 * that the EQ/CQ is big enough so they can't overflow, and the driver uses 338 * the owner bits mechanism to detect if the queue has become empty. 339 */ 340 struct gdma_queue { 341 struct gdma_dev *gdma_dev; 342 343 enum gdma_queue_type type; 344 u32 id; 345 346 struct gdma_mem_info mem_info; 347 348 void *queue_mem_ptr; 349 u32 queue_size; 350 351 bool monitor_avl_buf; 352 353 u32 head; 354 u32 tail; 355 struct list_head entry; 356 357 /* Extra fields specific to EQ/CQ. */ 358 union { 359 struct { 360 bool disable_needed; 361 362 gdma_eq_callback *callback; 363 void *context; 364 365 unsigned int msix_index; 366 unsigned int irq; 367 368 u32 log2_throttle_limit; 369 } eq; 370 371 struct { 372 gdma_cq_callback *callback; 373 void *context; 374 375 struct gdma_queue *parent; /* For CQ/EQ relationship */ 376 } cq; 377 }; 378 }; 379 380 struct gdma_queue_spec { 381 enum gdma_queue_type type; 382 bool monitor_avl_buf; 383 unsigned int queue_size; 384 385 /* Extra fields specific to EQ/CQ. */ 386 union { 387 struct { 388 gdma_eq_callback *callback; 389 void *context; 390 391 unsigned long log2_throttle_limit; 392 unsigned int msix_index; 393 } eq; 394 395 struct { 396 gdma_cq_callback *callback; 397 void *context; 398 399 struct gdma_queue *parent_eq; 400 401 } cq; 402 }; 403 }; 404 405 #define MANA_IRQ_NAME_SZ 32 406 407 struct gdma_irq_context { 408 void (*handler)(void *arg); 409 /* Protect the eq_list */ 410 spinlock_t lock; 411 struct list_head eq_list; 412 char name[MANA_IRQ_NAME_SZ]; 413 unsigned int msi; 414 unsigned int irq; 415 refcount_t refcount; 416 unsigned int bitmap_refs; 417 bool dyn_msix; 418 }; 419 420 enum gdma_context_flags { 421 GC_PROBE_SUCCEEDED = 0, 422 GC_IN_SERVICE = 1, 423 }; 424 425 struct gdma_context { 426 struct device *dev; 427 struct dentry *mana_pci_debugfs; 428 429 /* Hardware max number of queues */ 430 unsigned int max_num_queues; 431 /* Per-vPort max number of queues */ 432 unsigned int max_num_queues_vport; 433 unsigned int max_num_msix; 434 unsigned int num_msix_usable; 435 struct xarray irq_contexts; 436 437 /* L2 MTU */ 438 u16 adapter_mtu; 439 440 /* NIC supports CQE x8 coalescing */ 441 bool cqe8_coalescing_sup; 442 443 /* This maps a CQ index to the queue structure. */ 444 unsigned int max_num_cqs; 445 struct gdma_queue **cq_table; 446 447 /* Protect eq_test_event and test_event_eq_id */ 448 struct mutex eq_test_event_mutex; 449 struct completion eq_test_event; 450 u32 test_event_eq_id; 451 452 bool is_pf; 453 bool is_pf2; 454 455 phys_addr_t bar0_pa; 456 void __iomem *bar0_va; 457 resource_size_t bar0_size; 458 void __iomem *shm_base; 459 void __iomem *db_page_base; 460 phys_addr_t phys_db_page_base; 461 u64 db_page_off; 462 u64 db_page_size; 463 int numa_node; 464 465 /* Shared memory chanenl (used to bootstrap HWC) */ 466 struct shm_channel shm_channel; 467 468 /* Hardware communication channel (HWC) */ 469 struct gdma_dev hwc; 470 471 /* Azure network adapter */ 472 struct gdma_dev mana; 473 474 /* Azure RDMA adapter */ 475 struct gdma_dev mana_ib; 476 477 u64 pf_cap_flags1; 478 u64 gdma_protocol_ver; 479 480 struct workqueue_struct *service_wq; 481 482 unsigned long flags; 483 484 /* Protect access to GIC context */ 485 struct mutex gic_mutex; 486 487 /* Indicate if this device is sharing MSI for EQs on MANA */ 488 bool msi_sharing; 489 490 /* Bitmap tracks where MSI is allocated when it is not shared for EQs */ 491 unsigned long *msi_bitmap; 492 }; 493 494 static inline bool mana_gd_is_mana(struct gdma_dev *gd) 495 { 496 return gd->dev_id.type == GDMA_DEVICE_MANA; 497 } 498 499 static inline bool mana_gd_is_hwc(struct gdma_dev *gd) 500 { 501 return gd->dev_id.type == GDMA_DEVICE_HWC; 502 } 503 504 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset); 505 u32 mana_gd_wq_avail_space(struct gdma_queue *wq); 506 507 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq); 508 509 int mana_gd_create_hwc_queue(struct gdma_dev *gd, 510 const struct gdma_queue_spec *spec, 511 struct gdma_queue **queue_ptr); 512 513 int mana_gd_create_mana_eq(struct gdma_dev *gd, 514 const struct gdma_queue_spec *spec, 515 struct gdma_queue **queue_ptr); 516 517 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, 518 const struct gdma_queue_spec *spec, 519 struct gdma_queue **queue_ptr); 520 521 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue); 522 523 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe); 524 525 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit); 526 527 ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf, 528 size_t count, loff_t *pos); 529 530 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type); 531 532 void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, 533 u32 mod_comps, bool mod_comps_vld); 534 535 struct gdma_wqe { 536 u32 reserved :24; 537 u32 last_vbytes :8; 538 539 union { 540 u32 flags; 541 542 struct { 543 u32 num_sge :8; 544 u32 inline_oob_size_div4:3; 545 u32 client_oob_in_sgl :1; 546 u32 reserved1 :4; 547 u32 client_data_unit :14; 548 u32 reserved2 :2; 549 }; 550 }; 551 }; /* HW DATA */ 552 553 #define INLINE_OOB_SMALL_SIZE 8 554 #define INLINE_OOB_LARGE_SIZE 24 555 556 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30 557 558 #define MAX_TX_WQE_SIZE 512 559 #define MAX_RX_WQE_SIZE 256 560 561 #define MAX_TX_WQE_SGL_ENTRIES ((GDMA_MAX_SQE_SIZE - \ 562 sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \ 563 sizeof(struct gdma_sge)) 564 565 #define MAX_RX_WQE_SGL_ENTRIES ((GDMA_MAX_RQE_SIZE - \ 566 sizeof(struct gdma_sge)) / sizeof(struct gdma_sge)) 567 568 struct gdma_cqe { 569 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4]; 570 571 union { 572 u32 as_uint32; 573 574 struct { 575 u32 wq_num : 24; 576 u32 is_sq : 1; 577 u32 reserved : 4; 578 u32 owner_bits : 3; 579 }; 580 } cqe_info; 581 }; /* HW DATA */ 582 583 #define GDMA_CQE_OWNER_BITS 3 584 585 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1) 586 587 #define SET_ARM_BIT 1 588 589 #define GDMA_EQE_OWNER_BITS 3 590 591 union gdma_eqe_info { 592 u32 as_uint32; 593 594 struct { 595 u32 type : 8; 596 u32 reserved1 : 8; 597 u32 client_id : 2; 598 u32 reserved2 : 11; 599 u32 owner_bits : 3; 600 }; 601 }; /* HW DATA */ 602 603 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1) 604 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries)) 605 606 struct gdma_eqe { 607 u32 details[GDMA_EVENT_DATA_SIZE / 4]; 608 u32 eqe_info; 609 }; /* HW DATA */ 610 611 #define GDMA_REG_DB_PAGE_OFFSET 8 612 #define GDMA_REG_DB_PAGE_SIZE 0x10 613 #define GDMA_REG_SHM_OFFSET 0x18 614 615 #define GDMA_PF_REG_DB_PAGE_SIZE 0xD0 616 #define GDMA_PF_REG_DB_PAGE_OFF 0xC8 617 #define GDMA_PF_REG_SHM_OFF 0x70 618 619 #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108 620 621 #define MANA_PF_DEVICE_ID 0x00B9 622 #define MANA_PF2_DEVICE_ID 0x00C1 623 #define MANA_VF_DEVICE_ID 0x00BA 624 625 struct gdma_posted_wqe_info { 626 u32 wqe_size_in_bu; 627 }; 628 629 /* GDMA_GENERATE_TEST_EQE */ 630 struct gdma_generate_test_event_req { 631 struct gdma_req_hdr hdr; 632 u32 queue_index; 633 }; /* HW DATA */ 634 635 /* GDMA_VERIFY_VF_DRIVER_VERSION */ 636 enum { 637 GDMA_PROTOCOL_V1 = 1, 638 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1, 639 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1, 640 }; 641 642 #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0) 643 644 /* Advertise to the NIC firmware: the NAPI work_done variable race is fixed, 645 * so the driver is able to reliably support features like busy_poll. 646 */ 647 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2) 648 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3) 649 #define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4) 650 #define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5) 651 #define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6) 652 653 /* Driver can handle holes (zeros) in the device list */ 654 #define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11) 655 656 /* Driver supports dynamic MSI-X vector allocation */ 657 #define GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT BIT(13) 658 659 /* Driver can self reset on EQE notification */ 660 #define GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE BIT(14) 661 662 /* Driver can self reset on FPGA Reconfig EQE notification */ 663 #define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17) 664 665 /* Driver detects stalled send queues and recovers them */ 666 #define GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY BIT(18) 667 668 /* Driver supports separate EQ/MSIs for each vPort */ 669 #define GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT BIT(19) 670 671 /* Driver supports linearizing the skb when num_sge exceeds hardware limit */ 672 #define GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE BIT(20) 673 674 /* Driver can send HWC periodically to query stats */ 675 #define GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY BIT(21) 676 677 /* Driver can handle hardware recovery events during probe */ 678 #define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22) 679 680 /* Driver supports self recovery on Hardware Channel timeouts */ 681 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25) 682 683 /* Driver supports dynamic interrupt moderation - DIM */ 684 #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28) 685 686 /* Driver supports non-contiguous queue buffers */ 687 #define GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS BIT(30) 688 689 #define GDMA_DRV_CAP_FLAGS1 \ 690 (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \ 691 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \ 692 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \ 693 GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT | \ 694 GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \ 695 GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT | \ 696 GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE | \ 697 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE | \ 698 GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE | \ 699 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \ 700 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \ 701 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \ 702 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \ 703 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \ 704 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \ 705 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION | \ 706 GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS) 707 708 #define GDMA_DRV_CAP_FLAGS2 0 709 710 #define GDMA_DRV_CAP_FLAGS3 0 711 712 #define GDMA_DRV_CAP_FLAGS4 0 713 714 struct gdma_verify_ver_req { 715 struct gdma_req_hdr hdr; 716 717 /* Mandatory fields required for protocol establishment */ 718 u64 protocol_ver_min; 719 u64 protocol_ver_max; 720 721 /* Gdma Driver Capability Flags */ 722 u64 gd_drv_cap_flags1; 723 u64 gd_drv_cap_flags2; 724 u64 gd_drv_cap_flags3; 725 u64 gd_drv_cap_flags4; 726 727 /* Advisory fields */ 728 u64 drv_ver; 729 u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */ 730 u32 reserved; 731 u32 os_ver_major; 732 u32 os_ver_minor; 733 u32 os_ver_build; 734 u32 os_ver_platform; 735 u64 reserved_2; 736 u8 os_ver_str1[128]; 737 u8 os_ver_str2[128]; 738 u8 os_ver_str3[128]; 739 u8 os_ver_str4[128]; 740 }; /* HW DATA */ 741 742 /* HW supports dynamic interrupt moderation - DIM */ 743 #define GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(15) 744 745 struct gdma_verify_ver_resp { 746 struct gdma_resp_hdr hdr; 747 u64 gdma_protocol_ver; 748 u64 pf_cap_flags1; 749 u64 pf_cap_flags2; 750 u64 pf_cap_flags3; 751 u64 pf_cap_flags4; 752 }; /* HW DATA */ 753 754 /* GDMA_QUERY_MAX_RESOURCES */ 755 struct gdma_query_max_resources_resp { 756 struct gdma_resp_hdr hdr; 757 u32 status; 758 u32 max_sq; 759 u32 max_rq; 760 u32 max_cq; 761 u32 max_eq; 762 u32 max_db; 763 u32 max_mst; 764 u32 max_cq_mod_ctx; 765 u32 max_mod_cq; 766 u32 max_msix; 767 }; /* HW DATA */ 768 769 /* GDMA_LIST_DEVICES */ 770 #define GDMA_DEV_LIST_SIZE 64 771 struct gdma_list_devices_resp { 772 struct gdma_resp_hdr hdr; 773 u32 num_of_devs; 774 u32 reserved; 775 struct gdma_dev_id devs[GDMA_DEV_LIST_SIZE]; 776 }; /* HW DATA */ 777 778 /* GDMA_REGISTER_DEVICE */ 779 struct gdma_register_device_resp { 780 struct gdma_resp_hdr hdr; 781 u32 pdid; 782 u32 gpa_mkey; 783 u32 db_id; 784 }; /* HW DATA */ 785 786 struct gdma_allocate_resource_range_req { 787 struct gdma_req_hdr hdr; 788 u32 resource_type; 789 u32 num_resources; 790 u32 alignment; 791 u32 allocated_resources; 792 }; 793 794 struct gdma_allocate_resource_range_resp { 795 struct gdma_resp_hdr hdr; 796 u32 allocated_resources; 797 }; 798 799 struct gdma_destroy_resource_range_req { 800 struct gdma_req_hdr hdr; 801 u32 resource_type; 802 u32 num_resources; 803 u32 allocated_resources; 804 }; 805 806 /* GDMA_CREATE_QUEUE */ 807 struct gdma_create_queue_req { 808 struct gdma_req_hdr hdr; 809 u32 type; 810 u32 reserved1; 811 u32 pdid; 812 u32 doolbell_id; 813 u64 gdma_region; 814 u32 reserved2; 815 u32 queue_size; 816 u32 log2_throttle_limit; 817 u32 eq_pci_msix_index; 818 u32 cq_mod_ctx_id; 819 u32 cq_parent_eq_id; 820 u8 rq_drop_on_overrun; 821 u8 rq_err_on_wqe_overflow; 822 u8 rq_chain_rec_wqes; 823 u8 sq_hw_db; 824 u32 reserved3; 825 }; /* HW DATA */ 826 827 struct gdma_create_queue_resp { 828 struct gdma_resp_hdr hdr; 829 u32 queue_index; 830 }; /* HW DATA */ 831 832 /* GDMA_DISABLE_QUEUE */ 833 struct gdma_disable_queue_req { 834 struct gdma_req_hdr hdr; 835 u32 type; 836 u32 queue_index; 837 u32 alloc_res_id_on_creation; 838 }; /* HW DATA */ 839 840 /* GDMA_QUERY_HWC_TIMEOUT */ 841 struct gdma_query_hwc_timeout_req { 842 struct gdma_req_hdr hdr; 843 u32 timeout_ms; 844 u32 reserved; 845 }; 846 847 struct gdma_query_hwc_timeout_resp { 848 struct gdma_resp_hdr hdr; 849 u32 timeout_ms; 850 u32 reserved; 851 }; 852 853 enum gdma_mr_access_flags { 854 GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0), 855 GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1), 856 GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2), 857 GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3), 858 GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4), 859 GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), 860 }; 861 862 /* GDMA_CREATE_DMA_REGION */ 863 struct gdma_create_dma_region_req { 864 struct gdma_req_hdr hdr; 865 866 /* The total size of the DMA region */ 867 u64 length; 868 869 /* The offset in the first page */ 870 u32 offset_in_page; 871 872 /* enum gdma_page_type */ 873 u32 gdma_page_type; 874 875 /* The total number of pages */ 876 u32 page_count; 877 878 /* If page_addr_list_len is smaller than page_count, 879 * the remaining page addresses will be added via the 880 * message GDMA_DMA_REGION_ADD_PAGES. 881 */ 882 u32 page_addr_list_len; 883 u64 page_addr_list[]; 884 }; /* HW DATA */ 885 886 struct gdma_create_dma_region_resp { 887 struct gdma_resp_hdr hdr; 888 u64 dma_region_handle; 889 }; /* HW DATA */ 890 891 /* GDMA_DMA_REGION_ADD_PAGES */ 892 struct gdma_dma_region_add_pages_req { 893 struct gdma_req_hdr hdr; 894 895 u64 dma_region_handle; 896 897 u32 page_addr_list_len; 898 u32 reserved3; 899 900 u64 page_addr_list[]; 901 }; /* HW DATA */ 902 903 /* GDMA_DESTROY_DMA_REGION */ 904 struct gdma_destroy_dma_region_req { 905 struct gdma_req_hdr hdr; 906 907 u64 dma_region_handle; 908 }; /* HW DATA */ 909 910 enum gdma_pd_flags { 911 GDMA_PD_FLAG_INVALID = 0, 912 GDMA_PD_FLAG_ALLOW_GPA_MR = 1, 913 }; 914 915 struct gdma_create_pd_req { 916 struct gdma_req_hdr hdr; 917 enum gdma_pd_flags flags; 918 u32 reserved; 919 };/* HW DATA */ 920 921 struct gdma_create_pd_resp { 922 struct gdma_resp_hdr hdr; 923 u64 pd_handle; 924 u32 pd_id; 925 u32 reserved; 926 };/* HW DATA */ 927 928 struct gdma_destroy_pd_req { 929 struct gdma_req_hdr hdr; 930 u64 pd_handle; 931 };/* HW DATA */ 932 933 struct gdma_destory_pd_resp { 934 struct gdma_resp_hdr hdr; 935 };/* HW DATA */ 936 937 enum gdma_mr_type { 938 /* 939 * Guest Physical Address - MRs of this type allow access 940 * to any DMA-mapped memory using bus-logical address 941 */ 942 GDMA_MR_TYPE_GPA = 1, 943 /* Guest Virtual Address - MRs of this type allow access 944 * to memory mapped by PTEs associated with this MR using a virtual 945 * address that is set up in the MST 946 */ 947 GDMA_MR_TYPE_GVA = 2, 948 /* Guest zero-based address MRs */ 949 GDMA_MR_TYPE_ZBVA = 4, 950 /* Device address MRs */ 951 GDMA_MR_TYPE_DM = 5, 952 /* Memory Window type 1 */ 953 GDMA_MR_TYPE_MW1 = 6, 954 /* Memory Window type 2 */ 955 GDMA_MR_TYPE_MW2 = 7, 956 }; 957 958 struct gdma_create_mr_params { 959 u64 pd_handle; 960 enum gdma_mr_type mr_type; 961 union { 962 struct { 963 u64 dma_region_handle; 964 u64 virtual_address; 965 enum gdma_mr_access_flags access_flags; 966 } gva; 967 struct { 968 u64 dma_region_handle; 969 enum gdma_mr_access_flags access_flags; 970 } zbva; 971 struct { 972 u64 dm_handle; 973 u64 offset; 974 u64 length; 975 enum gdma_mr_access_flags access_flags; 976 } da; 977 }; 978 }; 979 980 struct gdma_create_mr_request { 981 struct gdma_req_hdr hdr; 982 u64 pd_handle; 983 enum gdma_mr_type mr_type; 984 u32 reserved_1; 985 986 union { 987 struct { 988 u64 dma_region_handle; 989 u64 virtual_address; 990 enum gdma_mr_access_flags access_flags; 991 } __packed gva; 992 struct { 993 u64 dma_region_handle; 994 enum gdma_mr_access_flags access_flags; 995 } __packed zbva; 996 struct { 997 u64 dm_handle; 998 u64 offset; 999 enum gdma_mr_access_flags access_flags; 1000 } __packed da; 1001 } __packed; 1002 u32 reserved_2; 1003 union { 1004 struct { 1005 u64 length; 1006 } da_ext; 1007 }; 1008 };/* HW DATA */ 1009 1010 struct gdma_create_mr_response { 1011 struct gdma_resp_hdr hdr; 1012 u64 mr_handle; 1013 u32 lkey; 1014 u32 rkey; 1015 };/* HW DATA */ 1016 1017 struct gdma_destroy_mr_request { 1018 struct gdma_req_hdr hdr; 1019 u64 mr_handle; 1020 };/* HW DATA */ 1021 1022 struct gdma_destroy_mr_response { 1023 struct gdma_resp_hdr hdr; 1024 };/* HW DATA */ 1025 1026 struct gdma_alloc_dm_req { 1027 struct gdma_req_hdr hdr; 1028 u64 length; 1029 u32 alignment; 1030 u32 flags; 1031 }; /* HW Data */ 1032 1033 struct gdma_alloc_dm_resp { 1034 struct gdma_resp_hdr hdr; 1035 u64 dm_handle; 1036 }; /* HW Data */ 1037 1038 struct gdma_destroy_dm_req { 1039 struct gdma_req_hdr hdr; 1040 u64 dm_handle; 1041 }; /* HW Data */ 1042 1043 struct gdma_destroy_dm_resp { 1044 struct gdma_resp_hdr hdr; 1045 }; /* HW Data */ 1046 1047 int mana_gd_verify_vf_version(struct pci_dev *pdev); 1048 1049 int mana_gd_register_device(struct gdma_dev *gd); 1050 int mana_gd_deregister_device(struct gdma_dev *gd); 1051 1052 int mana_gd_post_work_request(struct gdma_queue *wq, 1053 const struct gdma_wqe_request *wqe_req, 1054 struct gdma_posted_wqe_info *wqe_info); 1055 1056 int mana_gd_post_and_ring(struct gdma_queue *queue, 1057 const struct gdma_wqe_request *wqe, 1058 struct gdma_posted_wqe_info *wqe_info); 1059 1060 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r); 1061 void mana_gd_free_res_map(struct gdma_resource *r); 1062 1063 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, 1064 struct gdma_queue *queue); 1065 1066 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, 1067 struct gdma_mem_info *gmi, bool allow_scatter); 1068 1069 void mana_gd_free_memory(struct gdma_mem_info *gmi); 1070 1071 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, 1072 u32 resp_len, void *resp); 1073 1074 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle); 1075 void mana_register_debugfs(void); 1076 void mana_unregister_debugfs(void); 1077 1078 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event); 1079 1080 int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state); 1081 int mana_gd_resume(struct pci_dev *pdev); 1082 1083 bool mana_need_log(struct gdma_context *gc, int err); 1084 1085 struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, 1086 bool use_msi_bitmap, 1087 int *msi_requested); 1088 void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi); 1089 int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver, 1090 u32 proto_minor_ver, u32 proto_micro_ver, 1091 u16 *max_num_vports, u8 *bm_hostmode); 1092 #endif /* _GDMA_H */ 1093