1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #ifndef _GDMA_H
5 #define _GDMA_H
6
7 #include <linux/dma-mapping.h>
8 #include <linux/netdevice.h>
9
10 #include "shm_channel.h"
11
12 #define GDMA_STATUS_MORE_ENTRIES 0x00000105
13 #define GDMA_STATUS_CMD_UNSUPPORTED 0xffffffff
14
15 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
16 * them are naturally aligned and hence don't need __packed.
17 */
18
19 enum gdma_request_type {
20 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
21 GDMA_QUERY_MAX_RESOURCES = 2,
22 GDMA_LIST_DEVICES = 3,
23 GDMA_REGISTER_DEVICE = 4,
24 GDMA_DEREGISTER_DEVICE = 5,
25 GDMA_GENERATE_TEST_EQE = 10,
26 GDMA_CREATE_QUEUE = 12,
27 GDMA_DISABLE_QUEUE = 13,
28 GDMA_ALLOCATE_RESOURCE_RANGE = 22,
29 GDMA_DESTROY_RESOURCE_RANGE = 24,
30 GDMA_CREATE_DMA_REGION = 25,
31 GDMA_DMA_REGION_ADD_PAGES = 26,
32 GDMA_DESTROY_DMA_REGION = 27,
33 GDMA_CREATE_PD = 29,
34 GDMA_DESTROY_PD = 30,
35 GDMA_CREATE_MR = 31,
36 GDMA_DESTROY_MR = 32,
37 GDMA_QUERY_HWC_TIMEOUT = 84, /* 0x54 */
38 GDMA_ALLOC_DM = 96, /* 0x60 */
39 GDMA_DESTROY_DM = 97, /* 0x61 */
40 };
41
42 #define GDMA_RESOURCE_DOORBELL_PAGE 27
43
44 enum gdma_queue_type {
45 GDMA_INVALID_QUEUE,
46 GDMA_SQ,
47 GDMA_RQ,
48 GDMA_CQ,
49 GDMA_EQ,
50 GDMA_DIM,
51 };
52
53 enum gdma_work_request_flags {
54 GDMA_WR_NONE = 0,
55 GDMA_WR_OOB_IN_SGL = BIT(0),
56 GDMA_WR_PAD_BY_SGE0 = BIT(1),
57 };
58
59 enum gdma_eqe_type {
60 GDMA_EQE_COMPLETION = 3,
61 GDMA_EQE_TEST_EVENT = 64,
62 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
63 GDMA_EQE_HWC_INIT_DATA = 130,
64 GDMA_EQE_HWC_INIT_DONE = 131,
65 GDMA_EQE_HWC_FPGA_RECONFIG = 132,
66 GDMA_EQE_HWC_SOC_RECONFIG_DATA = 133,
67 GDMA_EQE_HWC_SOC_SERVICE = 134,
68 GDMA_EQE_HWC_RESET_REQUEST = 135,
69 GDMA_EQE_RNIC_QP_FATAL = 176,
70 };
71
72 enum {
73 GDMA_DEVICE_NONE = 0,
74 GDMA_DEVICE_HWC = 1,
75 GDMA_DEVICE_MANA = 2,
76 GDMA_DEVICE_MANA_IB = 3,
77 };
78
79 enum gdma_service_type {
80 GDMA_SERVICE_TYPE_NONE = 0,
81 GDMA_SERVICE_TYPE_RDMA_SUSPEND = 1,
82 GDMA_SERVICE_TYPE_RDMA_RESUME = 2,
83 };
84
85 struct mana_service_work {
86 struct work_struct work;
87 struct gdma_dev *gdma_dev;
88 enum gdma_service_type event;
89 };
90
91 struct gdma_resource {
92 /* Protect the bitmap */
93 spinlock_t lock;
94
95 /* The bitmap size in bits. */
96 u32 size;
97
98 /* The bitmap tracks the resources. */
99 unsigned long *map;
100 };
101
102 union gdma_doorbell_entry {
103 u64 as_uint64;
104
105 struct {
106 u64 id : 24;
107 u64 reserved : 8;
108 u64 tail_ptr : 31;
109 u64 arm : 1;
110 } cq;
111
112 struct {
113 u64 id : 24;
114 u64 wqe_cnt : 8;
115 u64 tail_ptr : 32;
116 } rq;
117
118 struct {
119 u64 id : 24;
120 u64 reserved : 8;
121 u64 tail_ptr : 32;
122 } sq;
123
124 struct {
125 u64 id : 16;
126 u64 reserved : 16;
127 u64 tail_ptr : 31;
128 u64 arm : 1;
129 } eq;
130
131 struct {
132 u64 id : 24;
133 u64 reserved : 8;
134 u64 mod_usec : 10;
135 u64 reserve1 : 5;
136 u64 mod_usec_vld : 1;
137 u64 mod_comps : 8;
138 u64 reserve2 : 7;
139 u64 mod_comps_vld: 1;
140 } dim;
141 }; /* HW DATA */
142
143 struct gdma_msg_hdr {
144 u32 hdr_type;
145 u32 msg_type;
146 u16 msg_version;
147 u16 hwc_msg_id;
148 u32 msg_size;
149 }; /* HW DATA */
150
151 struct gdma_dev_id {
152 union {
153 struct {
154 u16 type;
155 u16 instance;
156 };
157
158 u32 as_uint32;
159 };
160 }; /* HW DATA */
161
162 struct gdma_req_hdr {
163 struct gdma_msg_hdr req;
164 struct gdma_msg_hdr resp; /* The expected response */
165 struct gdma_dev_id dev_id;
166 u32 activity_id;
167 }; /* HW DATA */
168
169 struct gdma_resp_hdr {
170 struct gdma_msg_hdr response;
171 struct gdma_dev_id dev_id;
172 u32 activity_id;
173 u32 status;
174 u32 reserved;
175 }; /* HW DATA */
176
177 struct gdma_general_req {
178 struct gdma_req_hdr hdr;
179 }; /* HW DATA */
180
181 #define GDMA_MESSAGE_V1 1
182 #define GDMA_MESSAGE_V2 2
183 #define GDMA_MESSAGE_V3 3
184 #define GDMA_MESSAGE_V4 4
185 #define GDMA_MESSAGE_V5 5
186
187 struct gdma_general_resp {
188 struct gdma_resp_hdr hdr;
189 }; /* HW DATA */
190
191 #define GDMA_STANDARD_HEADER_TYPE 0
192
mana_gd_init_req_hdr(struct gdma_req_hdr * hdr,u32 code,u32 req_size,u32 resp_size)193 static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
194 u32 req_size, u32 resp_size)
195 {
196 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
197 hdr->req.msg_type = code;
198 hdr->req.msg_version = GDMA_MESSAGE_V1;
199 hdr->req.msg_size = req_size;
200
201 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
202 hdr->resp.msg_type = code;
203 hdr->resp.msg_version = GDMA_MESSAGE_V1;
204 hdr->resp.msg_size = resp_size;
205 }
206
207 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
208 struct gdma_sge {
209 u64 address;
210 u32 mem_key;
211 u32 size;
212 }; /* HW DATA */
213
214 struct gdma_wqe_request {
215 struct gdma_sge *sgl;
216 u32 num_sge;
217
218 u32 inline_oob_size;
219 const void *inline_oob_data;
220
221 u32 flags;
222 u32 client_data_unit;
223 };
224
225 enum gdma_page_type {
226 GDMA_PAGE_TYPE_4K,
227 };
228
229 #define GDMA_INVALID_DMA_REGION 0
230
231 struct mana_serv_work {
232 struct work_struct serv_work;
233 struct pci_dev *pdev;
234 enum gdma_eqe_type type;
235 };
236
237 struct gdma_mem_info {
238 struct device *dev;
239
240 dma_addr_t dma_handle;
241 void *virt_addr;
242 u64 length;
243
244 /* Scattered fallback: when @nr_pages > 0 the ring is that many
245 * PAGE_SIZE coherent allocations in @pages_va/@pages_dma, not
246 * @virt_addr/@dma_handle.
247 */
248 void **pages_va;
249 dma_addr_t *pages_dma;
250 unsigned int nr_pages;
251
252 /* Allocated by the PF driver */
253 u64 dma_region_handle;
254 };
255
256 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
257
258 struct gdma_dev {
259 struct gdma_context *gdma_context;
260
261 struct gdma_dev_id dev_id;
262
263 u32 pdid;
264 u32 doorbell;
265 u32 gpa_mkey;
266
267 /* GDMA driver specific pointer */
268 void *driver_data;
269
270 struct auxiliary_device *adev;
271 bool is_suspended;
272 bool rdma_teardown;
273 };
274
275 /* MANA_PAGE_SIZE is the DMA unit */
276 #define MANA_PAGE_SHIFT 12
277 #define MANA_PAGE_SIZE BIT(MANA_PAGE_SHIFT)
278 #define MANA_PAGE_ALIGN(x) ALIGN((x), MANA_PAGE_SIZE)
279 #define MANA_PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), MANA_PAGE_SIZE)
280 #define MANA_PFN(a) ((a) >> MANA_PAGE_SHIFT)
281
282 /* Required by HW */
283 #define MANA_MIN_QSIZE MANA_PAGE_SIZE
284
285 #define GDMA_CQE_SIZE 64
286 #define GDMA_EQE_SIZE 16
287 #define GDMA_MAX_SQE_SIZE 512
288 #define GDMA_MAX_RQE_SIZE 256
289
290 #define GDMA_COMP_DATA_SIZE 0x3C
291
292 #define GDMA_EVENT_DATA_SIZE 0xC
293
294 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
295 #define GDMA_WQE_BU_SIZE 32
296
297 #define INVALID_PDID UINT_MAX
298 #define INVALID_DOORBELL UINT_MAX
299 #define INVALID_MEM_KEY UINT_MAX
300 #define INVALID_QUEUE_ID UINT_MAX
301 #define INVALID_PCI_MSIX_INDEX UINT_MAX
302
303 struct gdma_comp {
304 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
305 u32 wq_num;
306 bool is_sq;
307 };
308
309 struct gdma_event {
310 u32 details[GDMA_EVENT_DATA_SIZE / 4];
311 u8 type;
312 };
313
314 struct gdma_queue;
315
316 struct mana_eq {
317 struct gdma_queue *eq;
318 struct dentry *mana_eq_debugfs;
319 };
320
321 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
322 struct gdma_event *e);
323
324 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
325
326 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
327 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
328 * driver increases the 'head' in BUs rather than in bytes, and notifies
329 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
330 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
331 *
332 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
333 * processed, the driver increases the 'tail' to indicate that WQEs have
334 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
335 *
336 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
337 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
338 * the owner bits mechanism to detect if the queue has become empty.
339 */
340 struct gdma_queue {
341 struct gdma_dev *gdma_dev;
342
343 enum gdma_queue_type type;
344 u32 id;
345
346 struct gdma_mem_info mem_info;
347
348 void *queue_mem_ptr;
349 u32 queue_size;
350
351 bool monitor_avl_buf;
352
353 u32 head;
354 u32 tail;
355 struct list_head entry;
356
357 /* Extra fields specific to EQ/CQ. */
358 union {
359 struct {
360 bool disable_needed;
361
362 gdma_eq_callback *callback;
363 void *context;
364
365 unsigned int msix_index;
366 unsigned int irq;
367
368 u32 log2_throttle_limit;
369 } eq;
370
371 struct {
372 gdma_cq_callback *callback;
373 void *context;
374
375 struct gdma_queue *parent; /* For CQ/EQ relationship */
376 } cq;
377 };
378 };
379
380 struct gdma_queue_spec {
381 enum gdma_queue_type type;
382 bool monitor_avl_buf;
383 unsigned int queue_size;
384
385 /* Extra fields specific to EQ/CQ. */
386 union {
387 struct {
388 gdma_eq_callback *callback;
389 void *context;
390
391 unsigned long log2_throttle_limit;
392 unsigned int msix_index;
393 } eq;
394
395 struct {
396 gdma_cq_callback *callback;
397 void *context;
398
399 struct gdma_queue *parent_eq;
400
401 } cq;
402 };
403 };
404
405 #define MANA_IRQ_NAME_SZ 32
406
407 struct gdma_irq_context {
408 void (*handler)(void *arg);
409 /* Protect the eq_list */
410 spinlock_t lock;
411 struct list_head eq_list;
412 char name[MANA_IRQ_NAME_SZ];
413 unsigned int msi;
414 unsigned int irq;
415 refcount_t refcount;
416 unsigned int bitmap_refs;
417 bool dyn_msix;
418 };
419
420 enum gdma_context_flags {
421 GC_PROBE_SUCCEEDED = 0,
422 GC_IN_SERVICE = 1,
423 };
424
425 struct gdma_context {
426 struct device *dev;
427 struct dentry *mana_pci_debugfs;
428
429 /* Hardware max number of queues */
430 unsigned int max_num_queues;
431 /* Per-vPort max number of queues */
432 unsigned int max_num_queues_vport;
433 unsigned int max_num_msix;
434 unsigned int num_msix_usable;
435 struct xarray irq_contexts;
436
437 /* L2 MTU */
438 u16 adapter_mtu;
439
440 /* NIC supports CQE x8 coalescing */
441 bool cqe8_coalescing_sup;
442
443 /* This maps a CQ index to the queue structure. */
444 unsigned int max_num_cqs;
445 struct gdma_queue **cq_table;
446
447 /* Protect eq_test_event and test_event_eq_id */
448 struct mutex eq_test_event_mutex;
449 struct completion eq_test_event;
450 u32 test_event_eq_id;
451
452 bool is_pf;
453 bool is_pf2;
454
455 phys_addr_t bar0_pa;
456 void __iomem *bar0_va;
457 resource_size_t bar0_size;
458 void __iomem *shm_base;
459 void __iomem *db_page_base;
460 phys_addr_t phys_db_page_base;
461 u64 db_page_off;
462 u64 db_page_size;
463 int numa_node;
464
465 /* Shared memory chanenl (used to bootstrap HWC) */
466 struct shm_channel shm_channel;
467
468 /* Hardware communication channel (HWC) */
469 struct gdma_dev hwc;
470
471 /* Azure network adapter */
472 struct gdma_dev mana;
473
474 /* Azure RDMA adapter */
475 struct gdma_dev mana_ib;
476
477 u64 pf_cap_flags1;
478 u64 gdma_protocol_ver;
479
480 struct workqueue_struct *service_wq;
481
482 unsigned long flags;
483
484 /* Protect access to GIC context */
485 struct mutex gic_mutex;
486
487 /* Indicate if this device is sharing MSI for EQs on MANA */
488 bool msi_sharing;
489
490 /* Bitmap tracks where MSI is allocated when it is not shared for EQs */
491 unsigned long *msi_bitmap;
492 };
493
mana_gd_is_mana(struct gdma_dev * gd)494 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
495 {
496 return gd->dev_id.type == GDMA_DEVICE_MANA;
497 }
498
mana_gd_is_hwc(struct gdma_dev * gd)499 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
500 {
501 return gd->dev_id.type == GDMA_DEVICE_HWC;
502 }
503
504 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
505 u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
506
507 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
508
509 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
510 const struct gdma_queue_spec *spec,
511 struct gdma_queue **queue_ptr);
512
513 int mana_gd_create_mana_eq(struct gdma_dev *gd,
514 const struct gdma_queue_spec *spec,
515 struct gdma_queue **queue_ptr);
516
517 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
518 const struct gdma_queue_spec *spec,
519 struct gdma_queue **queue_ptr);
520
521 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
522
523 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
524
525 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
526
527 ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf,
528 size_t count, loff_t *pos);
529
530 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type);
531
532 void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
533 u32 mod_comps, bool mod_comps_vld);
534
535 struct gdma_wqe {
536 u32 reserved :24;
537 u32 last_vbytes :8;
538
539 union {
540 u32 flags;
541
542 struct {
543 u32 num_sge :8;
544 u32 inline_oob_size_div4:3;
545 u32 client_oob_in_sgl :1;
546 u32 reserved1 :4;
547 u32 client_data_unit :14;
548 u32 reserved2 :2;
549 };
550 };
551 }; /* HW DATA */
552
553 #define INLINE_OOB_SMALL_SIZE 8
554 #define INLINE_OOB_LARGE_SIZE 24
555
556 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30
557
558 #define MAX_TX_WQE_SIZE 512
559 #define MAX_RX_WQE_SIZE 256
560
561 #define MAX_TX_WQE_SGL_ENTRIES ((GDMA_MAX_SQE_SIZE - \
562 sizeof(struct gdma_sge) - INLINE_OOB_SMALL_SIZE) / \
563 sizeof(struct gdma_sge))
564
565 #define MAX_RX_WQE_SGL_ENTRIES ((GDMA_MAX_RQE_SIZE - \
566 sizeof(struct gdma_sge)) / sizeof(struct gdma_sge))
567
568 struct gdma_cqe {
569 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
570
571 union {
572 u32 as_uint32;
573
574 struct {
575 u32 wq_num : 24;
576 u32 is_sq : 1;
577 u32 reserved : 4;
578 u32 owner_bits : 3;
579 };
580 } cqe_info;
581 }; /* HW DATA */
582
583 #define GDMA_CQE_OWNER_BITS 3
584
585 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
586
587 #define SET_ARM_BIT 1
588
589 #define GDMA_EQE_OWNER_BITS 3
590
591 union gdma_eqe_info {
592 u32 as_uint32;
593
594 struct {
595 u32 type : 8;
596 u32 reserved1 : 8;
597 u32 client_id : 2;
598 u32 reserved2 : 11;
599 u32 owner_bits : 3;
600 };
601 }; /* HW DATA */
602
603 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
604 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
605
606 struct gdma_eqe {
607 u32 details[GDMA_EVENT_DATA_SIZE / 4];
608 u32 eqe_info;
609 }; /* HW DATA */
610
611 #define GDMA_REG_DB_PAGE_OFFSET 8
612 #define GDMA_REG_DB_PAGE_SIZE 0x10
613 #define GDMA_REG_SHM_OFFSET 0x18
614
615 #define GDMA_PF_REG_DB_PAGE_SIZE 0xD0
616 #define GDMA_PF_REG_DB_PAGE_OFF 0xC8
617 #define GDMA_PF_REG_SHM_OFF 0x70
618
619 #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108
620
621 #define MANA_PF_DEVICE_ID 0x00B9
622 #define MANA_PF2_DEVICE_ID 0x00C1
623 #define MANA_VF_DEVICE_ID 0x00BA
624
625 struct gdma_posted_wqe_info {
626 u32 wqe_size_in_bu;
627 };
628
629 /* GDMA_GENERATE_TEST_EQE */
630 struct gdma_generate_test_event_req {
631 struct gdma_req_hdr hdr;
632 u32 queue_index;
633 }; /* HW DATA */
634
635 /* GDMA_VERIFY_VF_DRIVER_VERSION */
636 enum {
637 GDMA_PROTOCOL_V1 = 1,
638 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
639 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
640 };
641
642 #define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
643
644 /* Advertise to the NIC firmware: the NAPI work_done variable race is fixed,
645 * so the driver is able to reliably support features like busy_poll.
646 */
647 #define GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX BIT(2)
648 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3)
649 #define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4)
650 #define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5)
651 #define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6)
652
653 /* Driver can handle holes (zeros) in the device list */
654 #define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11)
655
656 /* Driver supports dynamic MSI-X vector allocation */
657 #define GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT BIT(13)
658
659 /* Driver can self reset on EQE notification */
660 #define GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE BIT(14)
661
662 /* Driver can self reset on FPGA Reconfig EQE notification */
663 #define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17)
664
665 /* Driver detects stalled send queues and recovers them */
666 #define GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY BIT(18)
667
668 /* Driver supports separate EQ/MSIs for each vPort */
669 #define GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT BIT(19)
670
671 /* Driver supports linearizing the skb when num_sge exceeds hardware limit */
672 #define GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE BIT(20)
673
674 /* Driver can send HWC periodically to query stats */
675 #define GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY BIT(21)
676
677 /* Driver can handle hardware recovery events during probe */
678 #define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22)
679
680 /* Driver supports self recovery on Hardware Channel timeouts */
681 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25)
682
683 /* Driver supports dynamic interrupt moderation - DIM */
684 #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
685
686 /* Driver supports non-contiguous queue buffers */
687 #define GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS BIT(30)
688
689 #define GDMA_DRV_CAP_FLAGS1 \
690 (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
691 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
692 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | \
693 GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT | \
694 GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP | \
695 GDMA_DRV_CAP_FLAG_1_DYNAMIC_IRQ_ALLOC_SUPPORT | \
696 GDMA_DRV_CAP_FLAG_1_SELF_RESET_ON_EQE | \
697 GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE | \
698 GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE | \
699 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \
700 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \
701 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \
702 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
703 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
704 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \
705 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION | \
706 GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS)
707
708 #define GDMA_DRV_CAP_FLAGS2 0
709
710 #define GDMA_DRV_CAP_FLAGS3 0
711
712 #define GDMA_DRV_CAP_FLAGS4 0
713
714 struct gdma_verify_ver_req {
715 struct gdma_req_hdr hdr;
716
717 /* Mandatory fields required for protocol establishment */
718 u64 protocol_ver_min;
719 u64 protocol_ver_max;
720
721 /* Gdma Driver Capability Flags */
722 u64 gd_drv_cap_flags1;
723 u64 gd_drv_cap_flags2;
724 u64 gd_drv_cap_flags3;
725 u64 gd_drv_cap_flags4;
726
727 /* Advisory fields */
728 u64 drv_ver;
729 u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
730 u32 reserved;
731 u32 os_ver_major;
732 u32 os_ver_minor;
733 u32 os_ver_build;
734 u32 os_ver_platform;
735 u64 reserved_2;
736 u8 os_ver_str1[128];
737 u8 os_ver_str2[128];
738 u8 os_ver_str3[128];
739 u8 os_ver_str4[128];
740 }; /* HW DATA */
741
742 /* HW supports dynamic interrupt moderation - DIM */
743 #define GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(15)
744
745 struct gdma_verify_ver_resp {
746 struct gdma_resp_hdr hdr;
747 u64 gdma_protocol_ver;
748 u64 pf_cap_flags1;
749 u64 pf_cap_flags2;
750 u64 pf_cap_flags3;
751 u64 pf_cap_flags4;
752 }; /* HW DATA */
753
754 /* GDMA_QUERY_MAX_RESOURCES */
755 struct gdma_query_max_resources_resp {
756 struct gdma_resp_hdr hdr;
757 u32 status;
758 u32 max_sq;
759 u32 max_rq;
760 u32 max_cq;
761 u32 max_eq;
762 u32 max_db;
763 u32 max_mst;
764 u32 max_cq_mod_ctx;
765 u32 max_mod_cq;
766 u32 max_msix;
767 }; /* HW DATA */
768
769 /* GDMA_LIST_DEVICES */
770 #define GDMA_DEV_LIST_SIZE 64
771 struct gdma_list_devices_resp {
772 struct gdma_resp_hdr hdr;
773 u32 num_of_devs;
774 u32 reserved;
775 struct gdma_dev_id devs[GDMA_DEV_LIST_SIZE];
776 }; /* HW DATA */
777
778 /* GDMA_REGISTER_DEVICE */
779 struct gdma_register_device_resp {
780 struct gdma_resp_hdr hdr;
781 u32 pdid;
782 u32 gpa_mkey;
783 u32 db_id;
784 }; /* HW DATA */
785
786 struct gdma_allocate_resource_range_req {
787 struct gdma_req_hdr hdr;
788 u32 resource_type;
789 u32 num_resources;
790 u32 alignment;
791 u32 allocated_resources;
792 };
793
794 struct gdma_allocate_resource_range_resp {
795 struct gdma_resp_hdr hdr;
796 u32 allocated_resources;
797 };
798
799 struct gdma_destroy_resource_range_req {
800 struct gdma_req_hdr hdr;
801 u32 resource_type;
802 u32 num_resources;
803 u32 allocated_resources;
804 };
805
806 /* GDMA_CREATE_QUEUE */
807 struct gdma_create_queue_req {
808 struct gdma_req_hdr hdr;
809 u32 type;
810 u32 reserved1;
811 u32 pdid;
812 u32 doolbell_id;
813 u64 gdma_region;
814 u32 reserved2;
815 u32 queue_size;
816 u32 log2_throttle_limit;
817 u32 eq_pci_msix_index;
818 u32 cq_mod_ctx_id;
819 u32 cq_parent_eq_id;
820 u8 rq_drop_on_overrun;
821 u8 rq_err_on_wqe_overflow;
822 u8 rq_chain_rec_wqes;
823 u8 sq_hw_db;
824 u32 reserved3;
825 }; /* HW DATA */
826
827 struct gdma_create_queue_resp {
828 struct gdma_resp_hdr hdr;
829 u32 queue_index;
830 }; /* HW DATA */
831
832 /* GDMA_DISABLE_QUEUE */
833 struct gdma_disable_queue_req {
834 struct gdma_req_hdr hdr;
835 u32 type;
836 u32 queue_index;
837 u32 alloc_res_id_on_creation;
838 }; /* HW DATA */
839
840 /* GDMA_QUERY_HWC_TIMEOUT */
841 struct gdma_query_hwc_timeout_req {
842 struct gdma_req_hdr hdr;
843 u32 timeout_ms;
844 u32 reserved;
845 };
846
847 struct gdma_query_hwc_timeout_resp {
848 struct gdma_resp_hdr hdr;
849 u32 timeout_ms;
850 u32 reserved;
851 };
852
853 enum gdma_mr_access_flags {
854 GDMA_ACCESS_FLAG_LOCAL_READ = BIT_ULL(0),
855 GDMA_ACCESS_FLAG_LOCAL_WRITE = BIT_ULL(1),
856 GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
857 GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
858 GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
859 GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
860 };
861
862 /* GDMA_CREATE_DMA_REGION */
863 struct gdma_create_dma_region_req {
864 struct gdma_req_hdr hdr;
865
866 /* The total size of the DMA region */
867 u64 length;
868
869 /* The offset in the first page */
870 u32 offset_in_page;
871
872 /* enum gdma_page_type */
873 u32 gdma_page_type;
874
875 /* The total number of pages */
876 u32 page_count;
877
878 /* If page_addr_list_len is smaller than page_count,
879 * the remaining page addresses will be added via the
880 * message GDMA_DMA_REGION_ADD_PAGES.
881 */
882 u32 page_addr_list_len;
883 u64 page_addr_list[];
884 }; /* HW DATA */
885
886 struct gdma_create_dma_region_resp {
887 struct gdma_resp_hdr hdr;
888 u64 dma_region_handle;
889 }; /* HW DATA */
890
891 /* GDMA_DMA_REGION_ADD_PAGES */
892 struct gdma_dma_region_add_pages_req {
893 struct gdma_req_hdr hdr;
894
895 u64 dma_region_handle;
896
897 u32 page_addr_list_len;
898 u32 reserved3;
899
900 u64 page_addr_list[];
901 }; /* HW DATA */
902
903 /* GDMA_DESTROY_DMA_REGION */
904 struct gdma_destroy_dma_region_req {
905 struct gdma_req_hdr hdr;
906
907 u64 dma_region_handle;
908 }; /* HW DATA */
909
910 enum gdma_pd_flags {
911 GDMA_PD_FLAG_INVALID = 0,
912 GDMA_PD_FLAG_ALLOW_GPA_MR = 1,
913 };
914
915 struct gdma_create_pd_req {
916 struct gdma_req_hdr hdr;
917 enum gdma_pd_flags flags;
918 u32 reserved;
919 };/* HW DATA */
920
921 struct gdma_create_pd_resp {
922 struct gdma_resp_hdr hdr;
923 u64 pd_handle;
924 u32 pd_id;
925 u32 reserved;
926 };/* HW DATA */
927
928 struct gdma_destroy_pd_req {
929 struct gdma_req_hdr hdr;
930 u64 pd_handle;
931 };/* HW DATA */
932
933 struct gdma_destory_pd_resp {
934 struct gdma_resp_hdr hdr;
935 };/* HW DATA */
936
937 enum gdma_mr_type {
938 /*
939 * Guest Physical Address - MRs of this type allow access
940 * to any DMA-mapped memory using bus-logical address
941 */
942 GDMA_MR_TYPE_GPA = 1,
943 /* Guest Virtual Address - MRs of this type allow access
944 * to memory mapped by PTEs associated with this MR using a virtual
945 * address that is set up in the MST
946 */
947 GDMA_MR_TYPE_GVA = 2,
948 /* Guest zero-based address MRs */
949 GDMA_MR_TYPE_ZBVA = 4,
950 /* Device address MRs */
951 GDMA_MR_TYPE_DM = 5,
952 /* Memory Window type 1 */
953 GDMA_MR_TYPE_MW1 = 6,
954 /* Memory Window type 2 */
955 GDMA_MR_TYPE_MW2 = 7,
956 };
957
958 struct gdma_create_mr_params {
959 u64 pd_handle;
960 enum gdma_mr_type mr_type;
961 union {
962 struct {
963 u64 dma_region_handle;
964 u64 virtual_address;
965 enum gdma_mr_access_flags access_flags;
966 } gva;
967 struct {
968 u64 dma_region_handle;
969 enum gdma_mr_access_flags access_flags;
970 } zbva;
971 struct {
972 u64 dm_handle;
973 u64 offset;
974 u64 length;
975 enum gdma_mr_access_flags access_flags;
976 } da;
977 };
978 };
979
980 struct gdma_create_mr_request {
981 struct gdma_req_hdr hdr;
982 u64 pd_handle;
983 enum gdma_mr_type mr_type;
984 u32 reserved_1;
985
986 union {
987 struct {
988 u64 dma_region_handle;
989 u64 virtual_address;
990 enum gdma_mr_access_flags access_flags;
991 } __packed gva;
992 struct {
993 u64 dma_region_handle;
994 enum gdma_mr_access_flags access_flags;
995 } __packed zbva;
996 struct {
997 u64 dm_handle;
998 u64 offset;
999 enum gdma_mr_access_flags access_flags;
1000 } __packed da;
1001 } __packed;
1002 u32 reserved_2;
1003 union {
1004 struct {
1005 u64 length;
1006 } da_ext;
1007 };
1008 };/* HW DATA */
1009
1010 struct gdma_create_mr_response {
1011 struct gdma_resp_hdr hdr;
1012 u64 mr_handle;
1013 u32 lkey;
1014 u32 rkey;
1015 };/* HW DATA */
1016
1017 struct gdma_destroy_mr_request {
1018 struct gdma_req_hdr hdr;
1019 u64 mr_handle;
1020 };/* HW DATA */
1021
1022 struct gdma_destroy_mr_response {
1023 struct gdma_resp_hdr hdr;
1024 };/* HW DATA */
1025
1026 struct gdma_alloc_dm_req {
1027 struct gdma_req_hdr hdr;
1028 u64 length;
1029 u32 alignment;
1030 u32 flags;
1031 }; /* HW Data */
1032
1033 struct gdma_alloc_dm_resp {
1034 struct gdma_resp_hdr hdr;
1035 u64 dm_handle;
1036 }; /* HW Data */
1037
1038 struct gdma_destroy_dm_req {
1039 struct gdma_req_hdr hdr;
1040 u64 dm_handle;
1041 }; /* HW Data */
1042
1043 struct gdma_destroy_dm_resp {
1044 struct gdma_resp_hdr hdr;
1045 }; /* HW Data */
1046
1047 int mana_gd_verify_vf_version(struct pci_dev *pdev);
1048
1049 int mana_gd_register_device(struct gdma_dev *gd);
1050 int mana_gd_deregister_device(struct gdma_dev *gd);
1051
1052 int mana_gd_post_work_request(struct gdma_queue *wq,
1053 const struct gdma_wqe_request *wqe_req,
1054 struct gdma_posted_wqe_info *wqe_info);
1055
1056 int mana_gd_post_and_ring(struct gdma_queue *queue,
1057 const struct gdma_wqe_request *wqe,
1058 struct gdma_posted_wqe_info *wqe_info);
1059
1060 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
1061 void mana_gd_free_res_map(struct gdma_resource *r);
1062
1063 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
1064 struct gdma_queue *queue);
1065
1066 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
1067 struct gdma_mem_info *gmi, bool allow_scatter);
1068
1069 void mana_gd_free_memory(struct gdma_mem_info *gmi);
1070
1071 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
1072 u32 resp_len, void *resp);
1073
1074 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle);
1075 void mana_register_debugfs(void);
1076 void mana_unregister_debugfs(void);
1077
1078 int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event);
1079
1080 int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state);
1081 int mana_gd_resume(struct pci_dev *pdev);
1082
1083 bool mana_need_log(struct gdma_context *gc, int err);
1084
1085 struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc,
1086 bool use_msi_bitmap,
1087 int *msi_requested);
1088 void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi);
1089 int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver,
1090 u32 proto_minor_ver, u32 proto_micro_ver,
1091 u16 *max_num_vports, u8 *bm_hostmode);
1092 #endif /* _GDMA_H */
1093