1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2021 Microsoft Corp.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #ifndef _GDMA_H
33 #define _GDMA_H
34
35 #include <sys/bus.h>
36 #include <sys/bus_dma.h>
37 #include <sys/types.h>
38 #include <sys/limits.h>
39 #include <sys/sx.h>
40
41 #include "gdma_util.h"
42 #include "shm_channel.h"
43
44 #define GDMA_STATUS_MORE_ENTRIES 0x00000105
45
46 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
47 * them are naturally aligned and hence don't need __packed.
48 */
49
50 #define GDMA_BAR0 0
51
52 #define GDMA_IRQNAME_SZ 40
53
54 struct gdma_bus {
55 bus_space_handle_t bar0_h;
56 bus_space_tag_t bar0_t;
57 };
58
59 struct gdma_msix_entry {
60 int entry;
61 int vector;
62 };
63
64 enum gdma_request_type {
65 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
66 GDMA_QUERY_MAX_RESOURCES = 2,
67 GDMA_LIST_DEVICES = 3,
68 GDMA_REGISTER_DEVICE = 4,
69 GDMA_DEREGISTER_DEVICE = 5,
70 GDMA_GENERATE_TEST_EQE = 10,
71 GDMA_CREATE_QUEUE = 12,
72 GDMA_DISABLE_QUEUE = 13,
73 GDMA_ALLOCATE_RESOURCE_RANGE = 22,
74 GDMA_DESTROY_RESOURCE_RANGE = 24,
75 GDMA_CREATE_DMA_REGION = 25,
76 GDMA_DMA_REGION_ADD_PAGES = 26,
77 GDMA_DESTROY_DMA_REGION = 27,
78 GDMA_CREATE_PD = 29,
79 GDMA_DESTROY_PD = 30,
80 GDMA_CREATE_MR = 31,
81 GDMA_DESTROY_MR = 32,
82 };
83
84 #define GDMA_RESOURCE_DOORBELL_PAGE 27
85
86 enum gdma_queue_type {
87 GDMA_INVALID_QUEUE,
88 GDMA_SQ,
89 GDMA_RQ,
90 GDMA_CQ,
91 GDMA_EQ,
92 };
93
94 enum gdma_work_request_flags {
95 GDMA_WR_NONE = 0,
96 GDMA_WR_OOB_IN_SGL = BIT(0),
97 GDMA_WR_PAD_BY_SGE0 = BIT(1),
98 };
99
100 enum gdma_eqe_type {
101 GDMA_EQE_COMPLETION = 3,
102 GDMA_EQE_TEST_EVENT = 64,
103 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
104 GDMA_EQE_HWC_INIT_DATA = 130,
105 GDMA_EQE_HWC_INIT_DONE = 131,
106 };
107
108 enum {
109 GDMA_DEVICE_NONE = 0,
110 GDMA_DEVICE_HWC = 1,
111 GDMA_DEVICE_MANA = 2,
112 };
113
114 typedef uint64_t gdma_obj_handle_t;
115
116 struct gdma_resource {
117 /* Protect the bitmap */
118 struct mtx lock_spin;
119
120 /* The bitmap size in bits. */
121 uint32_t size;
122
123 /* The bitmap tracks the resources. */
124 unsigned long *map;
125 };
126
127 union gdma_doorbell_entry {
128 uint64_t as_uint64;
129
130 struct {
131 uint64_t id : 24;
132 uint64_t reserved : 8;
133 uint64_t tail_ptr : 31;
134 uint64_t arm : 1;
135 } cq;
136
137 struct {
138 uint64_t id : 24;
139 uint64_t wqe_cnt : 8;
140 uint64_t tail_ptr : 32;
141 } rq;
142
143 struct {
144 uint64_t id : 24;
145 uint64_t reserved : 8;
146 uint64_t tail_ptr : 32;
147 } sq;
148
149 struct {
150 uint64_t id : 16;
151 uint64_t reserved : 16;
152 uint64_t tail_ptr : 31;
153 uint64_t arm : 1;
154 } eq;
155 }; /* HW DATA */
156
157 struct gdma_msg_hdr {
158 uint32_t hdr_type;
159 uint32_t msg_type;
160 uint16_t msg_version;
161 uint16_t hwc_msg_id;
162 uint32_t msg_size;
163 }; /* HW DATA */
164
165 struct gdma_dev_id {
166 union {
167 struct {
168 uint16_t type;
169 uint16_t instance;
170 };
171
172 uint32_t as_uint32;
173 };
174 }; /* HW DATA */
175
176 struct gdma_req_hdr {
177 struct gdma_msg_hdr req;
178 struct gdma_msg_hdr resp; /* The expected response */
179 struct gdma_dev_id dev_id;
180 uint32_t activity_id;
181 }; /* HW DATA */
182
183 struct gdma_resp_hdr {
184 struct gdma_msg_hdr response;
185 struct gdma_dev_id dev_id;
186 uint32_t activity_id;
187 uint32_t status;
188 uint32_t reserved;
189 }; /* HW DATA */
190
191 struct gdma_general_req {
192 struct gdma_req_hdr hdr;
193 }; /* HW DATA */
194
195 #define GDMA_MESSAGE_V1 1
196
197 struct gdma_general_resp {
198 struct gdma_resp_hdr hdr;
199 }; /* HW DATA */
200
201 #define GDMA_STANDARD_HEADER_TYPE 0
202
203 static inline void
mana_gd_init_req_hdr(struct gdma_req_hdr * hdr,uint32_t code,uint32_t req_size,uint32_t resp_size)204 mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, uint32_t code,
205 uint32_t req_size, uint32_t resp_size)
206 {
207 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
208 hdr->req.msg_type = code;
209 hdr->req.msg_version = GDMA_MESSAGE_V1;
210 hdr->req.msg_size = req_size;
211
212 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
213 hdr->resp.msg_type = code;
214 hdr->resp.msg_version = GDMA_MESSAGE_V1;
215 hdr->resp.msg_size = resp_size;
216 }
217
218 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
219 struct gdma_sge {
220 uint64_t address;
221 uint32_t mem_key;
222 uint32_t size;
223 }; /* HW DATA */
224
225 struct gdma_wqe_request {
226 struct gdma_sge *sgl;
227 uint32_t num_sge;
228
229 uint32_t inline_oob_size;
230 const void *inline_oob_data;
231
232 uint32_t flags;
233 uint32_t client_data_unit;
234 };
235
236 enum gdma_page_type {
237 GDMA_PAGE_TYPE_4K,
238 };
239
240 #define GDMA_INVALID_DMA_REGION 0
241
242 struct gdma_mem_info {
243 device_t dev;
244
245 bus_dma_tag_t dma_tag;
246 bus_dmamap_t dma_map;
247 bus_addr_t dma_handle; /* Physical address */
248 void *virt_addr; /* Virtual address */
249 uint64_t length;
250
251 /* Allocated by the PF driver */
252 gdma_obj_handle_t dma_region_handle;
253 };
254
255 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
256
257 struct gdma_dev {
258 struct gdma_context *gdma_context;
259
260 struct gdma_dev_id dev_id;
261
262 uint32_t pdid;
263 uint32_t doorbell;
264 uint32_t gpa_mkey;
265
266 /* GDMA driver specific pointer */
267 void *driver_data;
268 };
269
270 #define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
271
272 #define GDMA_CQE_SIZE 64
273 #define GDMA_EQE_SIZE 16
274 #define GDMA_MAX_SQE_SIZE 512
275 #define GDMA_MAX_RQE_SIZE 256
276
277 #define GDMA_COMP_DATA_SIZE 0x3C
278
279 #define GDMA_EVENT_DATA_SIZE 0xC
280
281 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
282 #define GDMA_WQE_BU_SIZE 32
283
284 #define INVALID_PDID UINT_MAX
285 #define INVALID_DOORBELL UINT_MAX
286 #define INVALID_MEM_KEY UINT_MAX
287 #define INVALID_QUEUE_ID UINT_MAX
288 #define INVALID_PCI_MSIX_INDEX UINT_MAX
289
290 struct gdma_comp {
291 uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
292 uint32_t wq_num;
293 bool is_sq;
294 };
295
296 struct gdma_event {
297 uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
298 uint8_t type;
299 };
300
301 struct gdma_queue;
302
303 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
304 struct gdma_event *e);
305
306 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
307
308 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
309 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
310 * driver increases the 'head' in BUs rather than in bytes, and notifies
311 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
312 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
313 *
314 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
315 * processed, the driver increases the 'tail' to indicate that WQEs have
316 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
317 *
318 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
319 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
320 * the owner bits mechanism to detect if the queue has become empty.
321 */
322 struct gdma_queue {
323 struct gdma_dev *gdma_dev;
324
325 enum gdma_queue_type type;
326 uint32_t id;
327
328 struct gdma_mem_info mem_info;
329
330 void *queue_mem_ptr;
331 uint32_t queue_size;
332
333 bool monitor_avl_buf;
334
335 uint32_t head;
336 uint32_t tail;
337
338 /* Extra fields specific to EQ/CQ. */
339 union {
340 struct {
341 bool disable_needed;
342
343 gdma_eq_callback *callback;
344 void *context;
345
346 unsigned int msix_index;
347
348 uint32_t log2_throttle_limit;
349 } eq;
350
351 struct {
352 gdma_cq_callback *callback;
353 void *context;
354
355 /* For CQ/EQ relationship */
356 struct gdma_queue *parent;
357 } cq;
358 };
359 };
360
361 struct gdma_queue_spec {
362 enum gdma_queue_type type;
363 bool monitor_avl_buf;
364 unsigned int queue_size;
365
366 /* Extra fields specific to EQ/CQ. */
367 union {
368 struct {
369 gdma_eq_callback *callback;
370 void *context;
371
372 unsigned long log2_throttle_limit;
373 } eq;
374
375 struct {
376 gdma_cq_callback *callback;
377 void *context;
378
379 struct gdma_queue *parent_eq;
380
381 } cq;
382 };
383 };
384
385 struct mana_eq {
386 struct gdma_queue *eq;
387 };
388
389 struct gdma_irq_context {
390 struct gdma_msix_entry msix_e;
391 struct resource *res;
392 driver_intr_t *handler;
393 void *arg;
394 void *cookie;
395 bool requested;
396 int cpu;
397 char name[GDMA_IRQNAME_SZ];
398 };
399
400 struct gdma_context {
401 device_t dev;
402
403 struct gdma_bus gd_bus;
404
405 /* Per-vPort max number of queues */
406 unsigned int max_num_queues;
407 unsigned int max_num_msix;
408 unsigned int num_msix_usable;
409 struct gdma_resource msix_resource;
410 struct gdma_irq_context *irq_contexts;
411
412 /* This maps a CQ index to the queue structure. */
413 unsigned int max_num_cqs;
414 struct gdma_queue **cq_table;
415
416 /* Protect eq_test_event and test_event_eq_id */
417 struct sx eq_test_event_sx;
418 struct completion eq_test_event;
419 uint32_t test_event_eq_id;
420
421 struct resource *bar0;
422 struct resource *msix;
423 int msix_rid;
424 void __iomem *shm_base;
425 void __iomem *db_page_base;
426 vm_paddr_t phys_db_page_base;
427 uint32_t db_page_size;
428
429 /* Shared memory chanenl (used to bootstrap HWC) */
430 struct shm_channel shm_channel;
431
432 /* Hardware communication channel (HWC) */
433 struct gdma_dev hwc;
434
435 /* Azure network adapter */
436 struct gdma_dev mana;
437 };
438
439 #define MAX_NUM_GDMA_DEVICES 4
440
mana_gd_is_mana(struct gdma_dev * gd)441 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
442 {
443 return gd->dev_id.type == GDMA_DEVICE_MANA;
444 }
445
mana_gd_is_hwc(struct gdma_dev * gd)446 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
447 {
448 return gd->dev_id.type == GDMA_DEVICE_HWC;
449 }
450
451 uint8_t *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, uint32_t wqe_offset);
452 uint32_t mana_gd_wq_avail_space(struct gdma_queue *wq);
453
454 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
455
456 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
457 const struct gdma_queue_spec *spec,
458 struct gdma_queue **queue_ptr);
459
460 int mana_gd_create_mana_eq(struct gdma_dev *gd,
461 const struct gdma_queue_spec *spec,
462 struct gdma_queue **queue_ptr);
463
464 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
465 const struct gdma_queue_spec *spec,
466 struct gdma_queue **queue_ptr);
467
468 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
469
470 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
471
472 void mana_gd_ring_cq(struct gdma_queue *cq, uint8_t arm_bit);
473
474 struct gdma_wqe {
475 uint32_t reserved :24;
476 uint32_t last_vbytes :8;
477
478 union {
479 uint32_t flags;
480
481 struct {
482 uint32_t num_sge :8;
483 uint32_t inline_oob_size_div4 :3;
484 uint32_t client_oob_in_sgl :1;
485 uint32_t reserved1 :4;
486 uint32_t client_data_unit :14;
487 uint32_t reserved2 :2;
488 };
489 };
490 }; /* HW DATA */
491
492 #define INLINE_OOB_SMALL_SIZE 8
493 #define INLINE_OOB_LARGE_SIZE 24
494
495 #define MAX_TX_WQE_SIZE 512
496 #define MAX_RX_WQE_SIZE 256
497
498 #define MAX_TX_WQE_SGL_ENTRIES ((GDMA_MAX_SQE_SIZE - \
499 sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \
500 sizeof(struct gdma_sge))
501
502 #define MAX_RX_WQE_SGL_ENTRIES ((GDMA_MAX_RQE_SIZE - \
503 sizeof(struct gdma_sge)) / sizeof(struct gdma_sge))
504
505 struct gdma_cqe {
506 uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
507
508 union {
509 uint32_t as_uint32;
510
511 struct {
512 uint32_t wq_num :24;
513 uint32_t is_sq :1;
514 uint32_t reserved :4;
515 uint32_t owner_bits :3;
516 };
517 } cqe_info;
518 }; /* HW DATA */
519
520 #define GDMA_CQE_OWNER_BITS 3
521
522 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
523
524 #define SET_ARM_BIT 1
525
526 #define GDMA_EQE_OWNER_BITS 3
527
528 union gdma_eqe_info {
529 uint32_t as_uint32;
530
531 struct {
532 uint32_t type : 8;
533 uint32_t reserved1 : 8;
534 uint32_t client_id : 2;
535 uint32_t reserved2 : 11;
536 uint32_t owner_bits : 3;
537 };
538 }; /* HW DATA */
539
540 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
541 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
542
543 struct gdma_eqe {
544 uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
545 uint32_t eqe_info;
546 }; /* HW DATA */
547
548 #define GDMA_REG_DB_PAGE_OFFSET 8
549 #define GDMA_REG_DB_PAGE_SIZE 0x10
550 #define GDMA_REG_SHM_OFFSET 0x18
551
552 struct gdma_posted_wqe_info {
553 uint32_t wqe_size_in_bu;
554 };
555
556 /* GDMA_GENERATE_TEST_EQE */
557 struct gdma_generate_test_event_req {
558 struct gdma_req_hdr hdr;
559 uint32_t queue_index;
560 }; /* HW DATA */
561
562 /* GDMA_VERIFY_VF_DRIVER_VERSION */
563 enum {
564 GDMA_PROTOCOL_V1 = 1,
565 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
566 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
567 };
568
569 struct gdma_verify_ver_req {
570 struct gdma_req_hdr hdr;
571
572 /* Mandatory fields required for protocol establishment */
573 uint64_t protocol_ver_min;
574 uint64_t protocol_ver_max;
575 uint64_t drv_cap_flags1;
576 uint64_t drv_cap_flags2;
577 uint64_t drv_cap_flags3;
578 uint64_t drv_cap_flags4;
579
580 /* Advisory fields */
581 uint64_t drv_ver;
582 uint32_t os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
583 uint32_t reserved;
584 uint32_t os_ver_major;
585 uint32_t os_ver_minor;
586 uint32_t os_ver_build;
587 uint32_t os_ver_platform;
588 uint64_t reserved_2;
589 uint8_t os_ver_str1[128];
590 uint8_t os_ver_str2[128];
591 uint8_t os_ver_str3[128];
592 uint8_t os_ver_str4[128];
593 }; /* HW DATA */
594
595 struct gdma_verify_ver_resp {
596 struct gdma_resp_hdr hdr;
597 uint64_t gdma_protocol_ver;
598 uint64_t pf_cap_flags1;
599 uint64_t pf_cap_flags2;
600 uint64_t pf_cap_flags3;
601 uint64_t pf_cap_flags4;
602 }; /* HW DATA */
603
604 /* GDMA_QUERY_MAX_RESOURCES */
605 struct gdma_query_max_resources_resp {
606 struct gdma_resp_hdr hdr;
607 uint32_t status;
608 uint32_t max_sq;
609 uint32_t max_rq;
610 uint32_t max_cq;
611 uint32_t max_eq;
612 uint32_t max_db;
613 uint32_t max_mst;
614 uint32_t max_cq_mod_ctx;
615 uint32_t max_mod_cq;
616 uint32_t max_msix;
617 }; /* HW DATA */
618
619 /* GDMA_LIST_DEVICES */
620 struct gdma_list_devices_resp {
621 struct gdma_resp_hdr hdr;
622 uint32_t num_of_devs;
623 uint32_t reserved;
624 struct gdma_dev_id devs[64];
625 }; /* HW DATA */
626
627 /* GDMA_REGISTER_DEVICE */
628 struct gdma_register_device_resp {
629 struct gdma_resp_hdr hdr;
630 uint32_t pdid;
631 uint32_t gpa_mkey;
632 uint32_t db_id;
633 }; /* HW DATA */
634
635 struct gdma_allocate_resource_range_req {
636 struct gdma_req_hdr hdr;
637 uint32_t resource_type;
638 uint32_t num_resources;
639 uint32_t alignment;
640 uint32_t allocated_resources;
641 };
642
643 struct gdma_allocate_resource_range_resp {
644 struct gdma_resp_hdr hdr;
645 uint32_t allocated_resources;
646 };
647
648 struct gdma_destroy_resource_range_req {
649 struct gdma_req_hdr hdr;
650 uint32_t resource_type;
651 uint32_t num_resources;
652 uint32_t allocated_resources;
653 };
654
655 /* GDMA_CREATE_QUEUE */
656 struct gdma_create_queue_req {
657 struct gdma_req_hdr hdr;
658 uint32_t type;
659 uint32_t reserved1;
660 uint32_t pdid;
661 uint32_t doolbell_id;
662 gdma_obj_handle_t gdma_region;
663 uint32_t reserved2;
664 uint32_t queue_size;
665 uint32_t log2_throttle_limit;
666 uint32_t eq_pci_msix_index;
667 uint32_t cq_mod_ctx_id;
668 uint32_t cq_parent_eq_id;
669 uint8_t rq_drop_on_overrun;
670 uint8_t rq_err_on_wqe_overflow;
671 uint8_t rq_chain_rec_wqes;
672 uint8_t sq_hw_db;
673 uint32_t reserved3;
674 }; /* HW DATA */
675
676 struct gdma_create_queue_resp {
677 struct gdma_resp_hdr hdr;
678 uint32_t queue_index;
679 }; /* HW DATA */
680
681 /* GDMA_DISABLE_QUEUE */
682 struct gdma_disable_queue_req {
683 struct gdma_req_hdr hdr;
684 uint32_t type;
685 uint32_t queue_index;
686 uint32_t alloc_res_id_on_creation;
687 }; /* HW DATA */
688
689 enum atb_page_size {
690 ATB_PAGE_SIZE_4K,
691 ATB_PAGE_SIZE_8K,
692 ATB_PAGE_SIZE_16K,
693 ATB_PAGE_SIZE_32K,
694 ATB_PAGE_SIZE_64K,
695 ATB_PAGE_SIZE_128K,
696 ATB_PAGE_SIZE_256K,
697 ATB_PAGE_SIZE_512K,
698 ATB_PAGE_SIZE_1M,
699 ATB_PAGE_SIZE_2M,
700 ATB_PAGE_SIZE_MAX,
701 };
702
703 enum gdma_mr_access_flags {
704 GDMA_ACCESS_FLAG_LOCAL_READ = BIT(0),
705 GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT(1),
706 GDMA_ACCESS_FLAG_REMOTE_READ = BIT(2),
707 GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT(3),
708 GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT(4),
709 };
710
711 /* GDMA_CREATE_DMA_REGION */
712 struct gdma_create_dma_region_req {
713 struct gdma_req_hdr hdr;
714
715 /* The total size of the DMA region */
716 uint64_t length;
717
718 /* The offset in the first page */
719 uint32_t offset_in_page;
720
721 /* enum gdma_page_type */
722 uint32_t gdma_page_type;
723
724 /* The total number of pages */
725 uint32_t page_count;
726
727 /* If page_addr_list_len is smaller than page_count,
728 * the remaining page addresses will be added via the
729 * message GDMA_DMA_REGION_ADD_PAGES.
730 */
731 uint32_t page_addr_list_len;
732 uint64_t page_addr_list[];
733 }; /* HW DATA */
734
735 struct gdma_create_dma_region_resp {
736 struct gdma_resp_hdr hdr;
737 gdma_obj_handle_t dma_region_handle;
738 }; /* HW DATA */
739
740 /* GDMA_DMA_REGION_ADD_PAGES */
741 struct gdma_dma_region_add_pages_req {
742 struct gdma_req_hdr hdr;
743
744 gdma_obj_handle_t dma_region_handle;
745
746 uint32_t page_addr_list_len;
747 uint32_t reserved3;
748
749 uint64_t page_addr_list[];
750 }; /* HW DATA */
751
752 /* GDMA_DESTROY_DMA_REGION */
753 struct gdma_destroy_dma_region_req {
754 struct gdma_req_hdr hdr;
755
756 gdma_obj_handle_t dma_region_handle;
757 }; /* HW DATA */
758
759 enum gdma_pd_flags {
760 GDMA_PD_FLAG_INVALID = 0,
761 };
762
763 struct gdma_create_pd_req {
764 struct gdma_req_hdr hdr;
765 enum gdma_pd_flags flags;
766 uint32_t reserved;
767 };/* HW DATA */
768
769 struct gdma_create_pd_resp {
770 struct gdma_resp_hdr hdr;
771 gdma_obj_handle_t pd_handle;
772 uint32_t pd_id;
773 uint32_t reserved;
774 };/* HW DATA */
775
776 struct gdma_destroy_pd_req {
777 struct gdma_req_hdr hdr;
778 gdma_obj_handle_t pd_handle;
779 };/* HW DATA */
780
781 struct gdma_destory_pd_resp {
782 struct gdma_resp_hdr hdr;
783 };/* HW DATA */
784
785 enum gdma_mr_type {
786 /* Guest Virtual Address - MRs of this type allow access
787 * to memory mapped by PTEs associated with this MR using a virtual
788 * address that is set up in the MST
789 */
790 GDMA_MR_TYPE_GVA = 2,
791 };
792
793 struct gdma_create_mr_params {
794 gdma_obj_handle_t pd_handle;
795 enum gdma_mr_type mr_type;
796 union {
797 struct {
798 gdma_obj_handle_t dma_region_handle;
799 uint64_t virtual_address;
800 enum gdma_mr_access_flags access_flags;
801 } gva;
802 };
803 };
804
805 struct gdma_create_mr_request {
806 struct gdma_req_hdr hdr;
807 gdma_obj_handle_t pd_handle;
808 enum gdma_mr_type mr_type;
809 uint32_t reserved_1;
810
811 union {
812 struct {
813 gdma_obj_handle_t dma_region_handle;
814 uint64_t virtual_address;
815 enum gdma_mr_access_flags access_flags;
816 } gva;
817
818 };
819 uint32_t reserved_2;
820 };/* HW DATA */
821
822 struct gdma_create_mr_response {
823 struct gdma_resp_hdr hdr;
824 gdma_obj_handle_t mr_handle;
825 uint32_t lkey;
826 uint32_t rkey;
827 };/* HW DATA */
828
829 struct gdma_destroy_mr_request {
830 struct gdma_req_hdr hdr;
831 gdma_obj_handle_t mr_handle;
832 };/* HW DATA */
833
834 struct gdma_destroy_mr_response {
835 struct gdma_resp_hdr hdr;
836 };/* HW DATA */
837
838 int mana_gd_verify_vf_version(device_t dev);
839
840 int mana_gd_register_device(struct gdma_dev *gd);
841 int mana_gd_deregister_device(struct gdma_dev *gd);
842
843 int mana_gd_post_work_request(struct gdma_queue *wq,
844 const struct gdma_wqe_request *wqe_req,
845 struct gdma_posted_wqe_info *wqe_info);
846
847 int mana_gd_post_and_ring(struct gdma_queue *queue,
848 const struct gdma_wqe_request *wqe,
849 struct gdma_posted_wqe_info *wqe_info);
850
851 int mana_gd_alloc_res_map(uint32_t res_avil, struct gdma_resource *r,
852 const char *lock_name);
853 void mana_gd_free_res_map(struct gdma_resource *r);
854
855 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
856 struct gdma_queue *queue);
857
858 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
859 struct gdma_mem_info *gmi);
860
861 void mana_gd_free_memory(struct gdma_mem_info *gmi);
862
863 void mana_gd_dma_map_paddr(void *arg, bus_dma_segment_t *segs,
864 int nseg, int error);
865
866 int mana_gd_send_request(struct gdma_context *gc, uint32_t req_len,
867 const void *req, uint32_t resp_len, void *resp);
868
869 int mana_gd_allocate_doorbell_page(struct gdma_context *gc,
870 int *doorbell_page);
871
872 int mana_gd_destroy_doorbell_page(struct gdma_context *gc,
873 int doorbell_page);
874
875 int mana_gd_destroy_dma_region(struct gdma_context *gc,
876 gdma_obj_handle_t dma_region_handle);
877 #endif /* _GDMA_H */
878