1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #ifndef IRDMA_TYPE_H
4 #define IRDMA_TYPE_H
5 #include "osdep.h"
6 #include "irdma.h"
7 #include "user.h"
8 #include "hmc.h"
9 #include "uda.h"
10 #include "ws.h"
11 #include "virtchnl.h"
12
13 #define IRDMA_DEBUG_ERR "ERR"
14 #define IRDMA_DEBUG_INIT "INIT"
15 #define IRDMA_DEBUG_DEV "DEV"
16 #define IRDMA_DEBUG_CM "CM"
17 #define IRDMA_DEBUG_VERBS "VERBS"
18 #define IRDMA_DEBUG_PUDA "PUDA"
19 #define IRDMA_DEBUG_ILQ "ILQ"
20 #define IRDMA_DEBUG_IEQ "IEQ"
21 #define IRDMA_DEBUG_QP "QP"
22 #define IRDMA_DEBUG_CQ "CQ"
23 #define IRDMA_DEBUG_MR "MR"
24 #define IRDMA_DEBUG_PBLE "PBLE"
25 #define IRDMA_DEBUG_WQE "WQE"
26 #define IRDMA_DEBUG_AEQ "AEQ"
27 #define IRDMA_DEBUG_CQP "CQP"
28 #define IRDMA_DEBUG_HMC "HMC"
29 #define IRDMA_DEBUG_USER "USER"
30 #define IRDMA_DEBUG_VIRT "VIRT"
31 #define IRDMA_DEBUG_DCB "DCB"
32 #define IRDMA_DEBUG_CQE "CQE"
33 #define IRDMA_DEBUG_CLNT "CLNT"
34 #define IRDMA_DEBUG_WS "WS"
35 #define IRDMA_DEBUG_STATS "STATS"
36
37 enum irdma_page_size {
38 IRDMA_PAGE_SIZE_4K = 0,
39 IRDMA_PAGE_SIZE_2M,
40 IRDMA_PAGE_SIZE_1G,
41 };
42
43 enum irdma_hdrct_flags {
44 DDP_LEN_FLAG = 0x80,
45 DDP_HDR_FLAG = 0x40,
46 RDMA_HDR_FLAG = 0x20,
47 };
48
49 enum irdma_term_layers {
50 LAYER_RDMA = 0,
51 LAYER_DDP = 1,
52 LAYER_MPA = 2,
53 };
54
55 enum irdma_term_error_types {
56 RDMAP_REMOTE_PROT = 1,
57 RDMAP_REMOTE_OP = 2,
58 DDP_CATASTROPHIC = 0,
59 DDP_TAGGED_BUF = 1,
60 DDP_UNTAGGED_BUF = 2,
61 DDP_LLP = 3,
62 };
63
64 enum irdma_term_rdma_errors {
65 RDMAP_INV_STAG = 0x00,
66 RDMAP_INV_BOUNDS = 0x01,
67 RDMAP_ACCESS = 0x02,
68 RDMAP_UNASSOC_STAG = 0x03,
69 RDMAP_TO_WRAP = 0x04,
70 RDMAP_INV_RDMAP_VER = 0x05,
71 RDMAP_UNEXPECTED_OP = 0x06,
72 RDMAP_CATASTROPHIC_LOCAL = 0x07,
73 RDMAP_CATASTROPHIC_GLOBAL = 0x08,
74 RDMAP_CANT_INV_STAG = 0x09,
75 RDMAP_UNSPECIFIED = 0xff,
76 };
77
78 enum irdma_term_ddp_errors {
79 DDP_CATASTROPHIC_LOCAL = 0x00,
80 DDP_TAGGED_INV_STAG = 0x00,
81 DDP_TAGGED_BOUNDS = 0x01,
82 DDP_TAGGED_UNASSOC_STAG = 0x02,
83 DDP_TAGGED_TO_WRAP = 0x03,
84 DDP_TAGGED_INV_DDP_VER = 0x04,
85 DDP_UNTAGGED_INV_QN = 0x01,
86 DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
87 DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
88 DDP_UNTAGGED_INV_MO = 0x04,
89 DDP_UNTAGGED_INV_TOO_LONG = 0x05,
90 DDP_UNTAGGED_INV_DDP_VER = 0x06,
91 };
92
93 enum irdma_term_mpa_errors {
94 MPA_CLOSED = 0x01,
95 MPA_CRC = 0x02,
96 MPA_MARKER = 0x03,
97 MPA_REQ_RSP = 0x04,
98 };
99
100 enum irdma_hw_stats_index {
101 /* gen1 - 32-bit */
102 IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0,
103 IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1,
104 IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2,
105 IRDMA_HW_STAT_INDEX_IP6RXDISCARD = 3,
106 IRDMA_HW_STAT_INDEX_IP6RXTRUNC = 4,
107 IRDMA_HW_STAT_INDEX_IP6TXNOROUTE = 5,
108 IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6,
109 IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7,
110 IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8,
111 IRDMA_HW_STAT_INDEX_RXVLANERR = 9,
112 /* gen1 - 64-bit */
113 IRDMA_HW_STAT_INDEX_IP4RXOCTS = 10,
114 IRDMA_HW_STAT_INDEX_IP4RXPKTS = 11,
115 IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 12,
116 IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 13,
117 IRDMA_HW_STAT_INDEX_IP4TXOCTS = 14,
118 IRDMA_HW_STAT_INDEX_IP4TXPKTS = 15,
119 IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 16,
120 IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 17,
121 IRDMA_HW_STAT_INDEX_IP6RXOCTS = 18,
122 IRDMA_HW_STAT_INDEX_IP6RXPKTS = 19,
123 IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 20,
124 IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 21,
125 IRDMA_HW_STAT_INDEX_IP6TXOCTS = 22,
126 IRDMA_HW_STAT_INDEX_IP6TXPKTS = 23,
127 IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 24,
128 IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 25,
129 IRDMA_HW_STAT_INDEX_TCPRXSEGS = 26,
130 IRDMA_HW_STAT_INDEX_TCPTXSEG = 27,
131 IRDMA_HW_STAT_INDEX_RDMARXRDS = 28,
132 IRDMA_HW_STAT_INDEX_RDMARXSNDS = 29,
133 IRDMA_HW_STAT_INDEX_RDMARXWRS = 30,
134 IRDMA_HW_STAT_INDEX_RDMATXRDS = 31,
135 IRDMA_HW_STAT_INDEX_RDMATXSNDS = 32,
136 IRDMA_HW_STAT_INDEX_RDMATXWRS = 33,
137 IRDMA_HW_STAT_INDEX_RDMAVBND = 34,
138 IRDMA_HW_STAT_INDEX_RDMAVINV = 35,
139 IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36,
140 IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37,
141 IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38,
142 IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39,
143 IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40,
144 IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41,
145 IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */
146 /* gen2 - 64-bit */
147 IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42,
148 /* gen2 - 32-bit */
149 IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43,
150 IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44,
151 IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45,
152 IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46,
153
154 /* gen3 */
155 IRDMA_HW_STAT_INDEX_RNR_SENT = 46,
156 IRDMA_HW_STAT_INDEX_RNR_RCVD = 47,
157 IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT = 48,
158 IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT = 49,
159 IRDMA_HW_STAT_INDEX_RDMARXATS = 50,
160 IRDMA_HW_STAT_INDEX_RDMATXATS = 51,
161 IRDMA_HW_STAT_INDEX_NAKSEQERR = 52,
162 IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED = 53,
163 IRDMA_HW_STAT_INDEX_RTO = 54,
164 IRDMA_HW_STAT_INDEX_RXOOOPKTS = 55,
165 IRDMA_HW_STAT_INDEX_ICRCERR = 56,
166
167 IRDMA_HW_STAT_INDEX_MAX_GEN_3 = 57,
168 };
169
170 enum irdma_feature_type {
171 IRDMA_FEATURE_FW_INFO = 0,
172 IRDMA_HW_VERSION_INFO = 1,
173 IRDMA_QP_MAX_INCR = 2,
174 IRDMA_CQ_MAX_INCR = 3,
175 IRDMA_CEQ_MAX_INCR = 4,
176 IRDMA_SD_MAX_INCR = 5,
177 IRDMA_MR_MAX_INCR = 6,
178 IRDMA_Q1_MAX_INCR = 7,
179 IRDMA_AH_MAX_INCR = 8,
180 IRDMA_SRQ_MAX_INCR = 9,
181 IRDMA_TIMER_MAX_INCR = 10,
182 IRDMA_XF_MAX_INCR = 11,
183 IRDMA_RRF_MAX_INCR = 12,
184 IRDMA_PBLE_MAX_INCR = 13,
185 IRDMA_OBJ_1 = 22,
186 IRDMA_OBJ_2 = 23,
187 IRDMA_ENDPT_TRK = 24,
188 IRDMA_FTN_INLINE_MAX = 25,
189 IRDMA_QSETS_MAX = 26,
190 IRDMA_ASO = 27,
191 IRDMA_FTN_FLAGS = 32,
192 IRDMA_FTN_NOP = 33,
193 IRDMA_MAX_FEATURES, /* Must be last entry */
194 };
195
196 enum irdma_sched_prio_type {
197 IRDMA_PRIO_WEIGHTED_RR = 1,
198 IRDMA_PRIO_STRICT = 2,
199 IRDMA_PRIO_WEIGHTED_STRICT = 3,
200 };
201
202 enum irdma_vm_vf_type {
203 IRDMA_VF_TYPE = 0,
204 IRDMA_VM_TYPE,
205 IRDMA_PF_TYPE,
206 };
207
208 enum irdma_cqp_hmc_profile {
209 IRDMA_HMC_PROFILE_DEFAULT = 1,
210 IRDMA_HMC_PROFILE_FAVOR_VF = 2,
211 IRDMA_HMC_PROFILE_EQUAL = 3,
212 };
213
214 enum irdma_quad_entry_type {
215 IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
216 IRDMA_QHASH_TYPE_TCP_SYN,
217 IRDMA_QHASH_TYPE_UDP_UNICAST,
218 IRDMA_QHASH_TYPE_UDP_MCAST,
219 IRDMA_QHASH_TYPE_ROCE_MCAST,
220 IRDMA_QHASH_TYPE_ROCEV2_HW,
221 };
222
223 enum irdma_quad_hash_manage_type {
224 IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
225 IRDMA_QHASH_MANAGE_TYPE_ADD,
226 IRDMA_QHASH_MANAGE_TYPE_MODIFY,
227 };
228
229 enum irdma_syn_rst_handling {
230 IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
231 IRDMA_SYN_RST_HANDLING_HW_TCP,
232 IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
233 IRDMA_SYN_RST_HANDLING_FW_TCP,
234 };
235
236 enum irdma_queue_type {
237 IRDMA_QUEUE_TYPE_SQ_RQ = 0,
238 IRDMA_QUEUE_TYPE_CQP,
239 IRDMA_QUEUE_TYPE_SRQ,
240 };
241
242 struct irdma_sc_dev;
243 struct irdma_vsi_pestat;
244
245 struct irdma_dcqcn_cc_params {
246 u8 cc_cfg_valid;
247 u8 min_dec_factor;
248 u8 min_rate;
249 u8 dcqcn_f;
250 u16 rai_factor;
251 u16 hai_factor;
252 u16 dcqcn_t;
253 u32 dcqcn_b;
254 u32 rreduce_mperiod;
255 };
256
257 struct irdma_cqp_init_info {
258 u64 cqp_compl_ctx;
259 u64 host_ctx_pa;
260 u64 sq_pa;
261 struct irdma_sc_dev *dev;
262 struct irdma_cqp_quanta *sq;
263 struct irdma_dcqcn_cc_params dcqcn_params;
264 __le64 *host_ctx;
265 u64 *scratch_array;
266 u32 sq_size;
267 struct irdma_ooo_cqp_op *ooo_op_array;
268 u32 pe_en_vf_cnt;
269 u16 hw_maj_ver;
270 u16 hw_min_ver;
271 u8 struct_ver;
272 u8 hmc_profile;
273 u8 ena_vf_count;
274 u8 ceqs_per_vf;
275 u8 ooisc_blksize;
276 u8 rrsp_blksize;
277 u8 q1_blksize;
278 u8 xmit_blksize;
279 u8 ts_override;
280 u8 ts_shift;
281 u8 en_fine_grained_timers;
282 u8 blksizes_valid;
283 bool en_datacenter_tcp:1;
284 bool disable_packed:1;
285 bool rocev2_rto_policy:1;
286 enum irdma_protocol_used protocol_used;
287 };
288
289 struct irdma_terminate_hdr {
290 u8 layer_etype;
291 u8 error_code;
292 u8 hdrct;
293 u8 rsvd;
294 };
295
296 struct irdma_cqp_sq_wqe {
297 __le64 buf[IRDMA_CQP_WQE_SIZE];
298 };
299
300 struct irdma_sc_aeqe {
301 __le64 buf[IRDMA_AEQE_SIZE];
302 };
303
304 struct irdma_ceqe {
305 __le64 buf[IRDMA_CEQE_SIZE];
306 };
307
308 struct irdma_cqp_ctx {
309 __le64 buf[IRDMA_CQP_CTX_SIZE];
310 };
311
312 struct irdma_cq_shadow_area {
313 __le64 buf[IRDMA_SHADOW_AREA_SIZE];
314 };
315
316 struct irdma_dev_hw_stats_offsets {
317 u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
318 };
319
320 struct irdma_dev_hw_stats {
321 u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
322 };
323
324 struct irdma_gather_stats {
325 u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
326 };
327
328 struct irdma_hw_stat_map {
329 u16 byteoff;
330 u8 bitoff;
331 u64 bitmask;
332 };
333
334 struct irdma_stats_gather_info {
335 bool use_hmc_fcn_index:1;
336 bool use_stats_inst:1;
337 u8 hmc_fcn_index;
338 u8 stats_inst_index;
339 struct irdma_dma_mem stats_buff_mem;
340 void *gather_stats_va;
341 void *last_gather_stats_va;
342 };
343
344 struct irdma_vsi_pestat {
345 struct irdma_hw *hw;
346 struct irdma_dev_hw_stats hw_stats;
347 struct irdma_stats_gather_info gather_info;
348 struct timer_list stats_timer;
349 struct irdma_sc_vsi *vsi;
350 struct irdma_dev_hw_stats last_hw_stats;
351 spinlock_t lock; /* rdma stats lock */
352 };
353
354 struct irdma_mmio_region {
355 u8 __iomem *addr;
356 resource_size_t len;
357 resource_size_t offset;
358 };
359
360 struct irdma_hw {
361 union {
362 u8 __iomem *hw_addr;
363 struct {
364 struct irdma_mmio_region rdma_reg; /* RDMA region */
365 struct irdma_mmio_region *io_regs; /* Non-RDMA MMIO regions */
366 u16 num_io_regions; /* Number of Non-RDMA MMIO regions */
367 };
368 };
369 struct device *device;
370 struct irdma_hmc_info hmc;
371 };
372
373 struct irdma_pfpdu {
374 struct list_head rxlist;
375 u32 rcv_nxt;
376 u32 fps;
377 u32 max_fpdu_data;
378 u32 nextseqnum;
379 u32 rcv_start_seq;
380 bool mode:1;
381 bool mpa_crc_err:1;
382 u8 marker_len;
383 u64 total_ieq_bufs;
384 u64 fpdu_processed;
385 u64 bad_seq_num;
386 u64 crc_err;
387 u64 no_tx_bufs;
388 u64 tx_err;
389 u64 out_of_order;
390 u64 pmode_count;
391 struct irdma_sc_ah *ah;
392 struct irdma_puda_buf *ah_buf;
393 spinlock_t lock; /* fpdu processing lock */
394 struct irdma_puda_buf *lastrcv_buf;
395 };
396
397 struct irdma_sc_pd {
398 struct irdma_sc_dev *dev;
399 u32 pd_id;
400 int abi_ver;
401 };
402
403 struct irdma_cqp_quanta {
404 __le64 elem[IRDMA_CQP_WQE_SIZE];
405 };
406
407 struct irdma_ooo_cqp_op {
408 struct list_head list_entry;
409 u64 scratch;
410 u32 def_info;
411 u32 sw_def_info;
412 u32 wqe_idx;
413 bool deferred:1;
414 };
415
416 struct irdma_sc_cqp {
417 spinlock_t ooo_list_lock; /* protects list of pending completions */
418 struct list_head ooo_avail;
419 struct list_head ooo_pnd;
420 u32 last_def_cmpl_ticket;
421 u32 sw_def_cmpl_ticket;
422 u32 size;
423 u64 sq_pa;
424 u64 host_ctx_pa;
425 void *back_cqp;
426 struct irdma_sc_dev *dev;
427 int (*process_cqp_sds)(struct irdma_sc_dev *dev,
428 struct irdma_update_sds_info *info);
429 struct irdma_dma_mem sdbuf;
430 struct irdma_ring sq_ring;
431 struct irdma_cqp_quanta *sq_base;
432 struct irdma_dcqcn_cc_params dcqcn_params;
433 __le64 *host_ctx;
434 u64 *scratch_array;
435 u64 requested_ops;
436 atomic64_t completed_ops;
437 struct irdma_ooo_cqp_op *ooo_op_array;
438 u32 cqp_id;
439 u32 sq_size;
440 u32 pe_en_vf_cnt;
441 u32 hw_sq_size;
442 u16 hw_maj_ver;
443 u16 hw_min_ver;
444 u8 struct_ver;
445 u8 polarity;
446 u8 hmc_profile;
447 u8 ena_vf_count;
448 u8 timeout_count;
449 u8 ceqs_per_vf;
450 u8 ooisc_blksize;
451 u8 rrsp_blksize;
452 u8 q1_blksize;
453 u8 xmit_blksize;
454 u8 ts_override;
455 u8 ts_shift;
456 u8 en_fine_grained_timers;
457 u8 blksizes_valid;
458 bool en_datacenter_tcp:1;
459 bool disable_packed:1;
460 bool rocev2_rto_policy:1;
461 enum irdma_protocol_used protocol_used;
462 };
463
464 struct irdma_sc_aeq {
465 u32 size;
466 u64 aeq_elem_pa;
467 struct irdma_sc_dev *dev;
468 struct irdma_sc_aeqe *aeqe_base;
469 void *pbl_list;
470 u32 elem_cnt;
471 struct irdma_ring aeq_ring;
472 u8 pbl_chunk_size;
473 u32 first_pm_pbl_idx;
474 u32 msix_idx;
475 u8 polarity;
476 bool virtual_map:1;
477 bool pasid_valid:1;
478 u32 pasid;
479 };
480
481 struct irdma_sc_ceq {
482 u32 size;
483 u64 ceq_elem_pa;
484 struct irdma_sc_dev *dev;
485 struct irdma_ceqe *ceqe_base;
486 void *pbl_list;
487 u32 ceq_id;
488 u32 elem_cnt;
489 struct irdma_ring ceq_ring;
490 u8 pbl_chunk_size;
491 u8 tph_val;
492 u32 first_pm_pbl_idx;
493 u8 polarity;
494 u16 vsi_idx;
495 bool virtual_map:1;
496 bool tph_en:1;
497 bool itr_no_expire:1;
498 bool pasid_valid:1;
499 u32 pasid;
500 };
501
502 struct irdma_sc_cq {
503 struct irdma_cq_uk cq_uk;
504 u64 cq_pa;
505 u64 shadow_area_pa;
506 struct irdma_sc_dev *dev;
507 u16 vsi_idx;
508 struct irdma_sc_vsi *vsi;
509 void *pbl_list;
510 void *back_cq;
511 u32 ceq_id;
512 u32 shadow_read_threshold;
513 u8 pbl_chunk_size;
514 u8 cq_type;
515 u8 tph_val;
516 u32 first_pm_pbl_idx;
517 bool ceqe_mask:1;
518 bool virtual_map:1;
519 bool check_overflow:1;
520 bool ceq_id_valid:1;
521 bool tph_en;
522 };
523
524 struct irdma_sc_qp {
525 struct irdma_qp_uk qp_uk;
526 u64 sq_pa;
527 u64 rq_pa;
528 u64 hw_host_ctx_pa;
529 u64 shadow_area_pa;
530 u64 q2_pa;
531 struct irdma_sc_dev *dev;
532 struct irdma_sc_vsi *vsi;
533 struct irdma_sc_pd *pd;
534 __le64 *hw_host_ctx;
535 void *llp_stream_handle;
536 struct irdma_pfpdu pfpdu;
537 u32 ieq_qp;
538 u8 *q2_buf;
539 u64 qp_compl_ctx;
540 u32 push_idx;
541 u16 qs_handle;
542 u16 push_offset;
543 u8 flush_wqes_count;
544 u8 sq_tph_val;
545 u8 rq_tph_val;
546 u8 qp_state;
547 u8 hw_sq_size;
548 u8 hw_rq_size;
549 u8 src_mac_addr_idx;
550 bool on_qoslist:1;
551 bool ieq_pass_thru:1;
552 bool sq_tph_en:1;
553 bool rq_tph_en:1;
554 bool rcv_tph_en:1;
555 bool xmit_tph_en:1;
556 bool virtual_map:1;
557 bool flush_sq:1;
558 bool flush_rq:1;
559 bool err_sq_idx_valid:1;
560 bool err_rq_idx_valid:1;
561 u32 err_sq_idx;
562 u32 err_rq_idx;
563 bool sq_flush_code:1;
564 bool rq_flush_code:1;
565 u32 pkt_limit;
566 enum irdma_flush_opcode flush_code;
567 enum irdma_qp_event_type event_type;
568 u8 term_flags;
569 u8 user_pri;
570 struct list_head list;
571 };
572
573 struct irdma_stats_inst_info {
574 bool use_hmc_fcn_index;
575 u8 hmc_fn_id;
576 u16 stats_idx;
577 };
578
579 struct irdma_up_info {
580 u8 map[8];
581 u8 cnp_up_override;
582 u16 hmc_fcn_idx;
583 bool use_vlan:1;
584 bool use_cnp_up_override:1;
585 };
586
587 #define IRDMA_MAX_WS_NODES 0x3FF
588 #define IRDMA_WS_NODE_INVALID 0xFFFF
589
590 struct irdma_ws_node_info {
591 u16 id;
592 u16 vsi;
593 u16 parent_id;
594 u16 qs_handle;
595 bool type_leaf:1;
596 bool enable:1;
597 u8 prio_type;
598 u8 tc;
599 u8 weight;
600 };
601
602 struct irdma_hmc_fpm_misc {
603 u32 max_ceqs;
604 u32 max_sds;
605 u32 loc_mem_pages;
606 u8 ird;
607 u32 xf_block_size;
608 u32 q1_block_size;
609 u32 ht_multiplier;
610 u32 timer_bucket;
611 u32 rrf_block_size;
612 u32 ooiscf_block_size;
613 };
614
615 #define IRDMA_VCHNL_MAX_MSG_SIZE 512
616 #define IRDMA_LEAF_DEFAULT_REL_BW 64
617 #define IRDMA_PARENT_DEFAULT_REL_BW 1
618
619 struct irdma_qos {
620 struct list_head qplist;
621 struct mutex qos_mutex; /* protect QoS attributes per QoS level */
622 u64 lan_qos_handle;
623 u32 l2_sched_node_id;
624 u16 qs_handle;
625 u8 traffic_class;
626 u8 rel_bw;
627 u8 prio_type;
628 bool valid;
629 };
630
631 #define IRDMA_INVALID_STATS_IDX 0xff
632 struct irdma_sc_vsi {
633 u16 vsi_idx;
634 struct irdma_sc_dev *dev;
635 void *back_vsi;
636 u32 ilq_count;
637 struct irdma_virt_mem ilq_mem;
638 struct irdma_puda_rsrc *ilq;
639 u32 ieq_count;
640 struct irdma_virt_mem ieq_mem;
641 struct irdma_puda_rsrc *ieq;
642 u32 exception_lan_q;
643 u16 mtu;
644 u16 vm_id;
645 enum irdma_vm_vf_type vm_vf_type;
646 bool stats_inst_alloc:1;
647 bool tc_change_pending:1;
648 struct irdma_vsi_pestat *pestat;
649 atomic_t qp_suspend_reqs;
650 int (*register_qset)(struct irdma_sc_vsi *vsi,
651 struct irdma_ws_node *tc_node);
652 void (*unregister_qset)(struct irdma_sc_vsi *vsi,
653 struct irdma_ws_node *tc_node);
654 u8 qos_rel_bw;
655 u8 qos_prio_type;
656 u8 stats_idx;
657 u8 dscp_map[DSCP_MAX];
658 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
659 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
660 bool dscp_mode:1;
661 };
662
663 struct irdma_sc_dev {
664 struct list_head cqp_cmd_head; /* head of the CQP command list */
665 spinlock_t cqp_lock; /* protect CQP list access */
666 bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
667 struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
668 u64 fpm_query_buf_pa;
669 u64 fpm_commit_buf_pa;
670 __le64 *fpm_query_buf;
671 __le64 *fpm_commit_buf;
672 struct irdma_hw *hw;
673 u8 __iomem *db_addr;
674 u32 __iomem *wqe_alloc_db;
675 u32 __iomem *cq_arm_db;
676 u32 __iomem *aeq_alloc_db;
677 u32 __iomem *cqp_db;
678 u32 __iomem *cq_ack_db;
679 u32 __iomem *ceq_itr_mask_db;
680 u32 __iomem *aeq_itr_mask_db;
681 u32 __iomem *hw_regs[IRDMA_MAX_REGS];
682 u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
683 u64 hw_masks[IRDMA_MAX_MASKS];
684 u64 hw_shifts[IRDMA_MAX_SHIFTS];
685 const struct irdma_hw_stat_map *hw_stats_map;
686 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
687 u64 feature_info[IRDMA_MAX_FEATURES];
688 u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
689 struct irdma_hw_attrs hw_attrs;
690 struct irdma_hmc_info *hmc_info;
691 struct irdma_vchnl_rdma_caps vc_caps;
692 u8 vc_recv_buf[IRDMA_VCHNL_MAX_MSG_SIZE];
693 u16 vc_recv_len;
694 struct irdma_sc_cqp *cqp;
695 struct irdma_sc_aeq *aeq;
696 struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
697 struct irdma_sc_cq *ccq;
698 const struct irdma_irq_ops *irq_ops;
699 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
700 struct irdma_hmc_fpm_misc hmc_fpm_misc;
701 struct irdma_ws_node *ws_tree_root;
702 struct mutex ws_mutex; /* ws tree mutex */
703 u32 vchnl_ver;
704 u16 num_vfs;
705 u16 hmc_fn_id;
706 u16 vf_id;
707 bool privileged:1;
708 bool vchnl_up:1;
709 bool ceq_valid:1;
710 bool is_pf:1;
711 u8 protocol_used;
712 struct mutex vchnl_mutex; /* mutex to synchronize RDMA virtual channel messages */
713 u8 pci_rev;
714 int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
715 void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
716 void (*ws_reset)(struct irdma_sc_vsi *vsi);
717 };
718
719 struct irdma_modify_cq_info {
720 u64 cq_pa;
721 struct irdma_cqe *cq_base;
722 u32 cq_size;
723 u32 shadow_read_threshold;
724 u8 pbl_chunk_size;
725 u32 first_pm_pbl_idx;
726 bool virtual_map:1;
727 bool check_overflow;
728 bool cq_resize:1;
729 };
730
731 struct irdma_srq_init_info {
732 struct irdma_sc_pd *pd;
733 struct irdma_sc_vsi *vsi;
734 u64 srq_pa;
735 u64 shadow_area_pa;
736 u32 first_pm_pbl_idx;
737 u32 pasid;
738 u32 srq_size;
739 u16 srq_limit;
740 u8 pasid_valid;
741 u8 wqe_size;
742 u8 leaf_pbl_size;
743 u8 virtual_map;
744 u8 tph_en;
745 u8 arm_limit_event;
746 u8 tph_value;
747 u8 pbl_chunk_size;
748 struct irdma_srq_uk_init_info srq_uk_init_info;
749 };
750
751 struct irdma_sc_srq {
752 struct irdma_sc_dev *dev;
753 struct irdma_sc_vsi *vsi;
754 struct irdma_sc_pd *pd;
755 struct irdma_srq_uk srq_uk;
756 void *back_srq;
757 u64 srq_pa;
758 u64 shadow_area_pa;
759 u32 first_pm_pbl_idx;
760 u32 pasid;
761 u32 hw_srq_size;
762 u16 srq_limit;
763 u8 pasid_valid;
764 u8 leaf_pbl_size;
765 u8 virtual_map;
766 u8 tph_en;
767 u8 arm_limit_event;
768 u8 tph_val;
769 };
770
771 struct irdma_modify_srq_info {
772 u16 srq_limit;
773 u8 arm_limit_event;
774 };
775
776 struct irdma_create_qp_info {
777 bool ord_valid:1;
778 bool tcp_ctx_valid:1;
779 bool cq_num_valid:1;
780 bool arp_cache_idx_valid:1;
781 bool mac_valid:1;
782 bool force_lpb;
783 u8 next_iwarp_state;
784 };
785
786 struct irdma_modify_qp_info {
787 u64 rx_win0;
788 u64 rx_win1;
789 u16 new_mss;
790 u8 next_iwarp_state;
791 u8 curr_iwarp_state;
792 u8 termlen;
793 bool ord_valid:1;
794 bool tcp_ctx_valid:1;
795 bool udp_ctx_valid:1;
796 bool cq_num_valid:1;
797 bool arp_cache_idx_valid:1;
798 bool reset_tcp_conn:1;
799 bool remove_hash_idx:1;
800 bool dont_send_term:1;
801 bool dont_send_fin:1;
802 bool cached_var_valid:1;
803 bool mss_change:1;
804 bool force_lpb:1;
805 bool mac_valid:1;
806 };
807
808 struct irdma_ccq_cqe_info {
809 struct irdma_sc_cqp *cqp;
810 u64 scratch;
811 u32 op_ret_val;
812 u16 maj_err_code;
813 u16 min_err_code;
814 u8 op_code;
815 bool error:1;
816 bool pending:1;
817 };
818
819 struct irdma_dcb_app_info {
820 u8 priority;
821 u8 selector;
822 u16 prot_id;
823 };
824
825 struct irdma_qos_tc_info {
826 u64 tc_ctx;
827 u8 rel_bw;
828 u8 prio_type;
829 u8 egress_virt_up;
830 u8 ingress_virt_up;
831 };
832
833 struct irdma_l2params {
834 struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
835 struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
836 u32 num_apps;
837 u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
838 u16 mtu;
839 u8 up2tc[IRDMA_MAX_USER_PRIORITY];
840 u8 dscp_map[DSCP_MAX];
841 u8 num_tc;
842 u8 vsi_rel_bw;
843 u8 vsi_prio_type;
844 bool mtu_changed:1;
845 bool tc_changed:1;
846 bool dscp_mode:1;
847 };
848
849 struct irdma_vsi_init_info {
850 struct irdma_sc_dev *dev;
851 void *back_vsi;
852 struct irdma_l2params *params;
853 u16 exception_lan_q;
854 u16 pf_data_vsi_num;
855 enum irdma_vm_vf_type vm_vf_type;
856 u16 vm_id;
857 int (*register_qset)(struct irdma_sc_vsi *vsi,
858 struct irdma_ws_node *tc_node);
859 void (*unregister_qset)(struct irdma_sc_vsi *vsi,
860 struct irdma_ws_node *tc_node);
861 };
862
863 struct irdma_vsi_stats_info {
864 struct irdma_vsi_pestat *pestat;
865 u16 fcn_id;
866 bool alloc_stats_inst;
867 };
868
869 struct irdma_device_init_info {
870 u64 fpm_query_buf_pa;
871 u64 fpm_commit_buf_pa;
872 __le64 *fpm_query_buf;
873 __le64 *fpm_commit_buf;
874 struct irdma_hw *hw;
875 void __iomem *bar0;
876 enum irdma_protocol_used protocol_used;
877 u16 hmc_fn_id;
878 };
879
880 struct irdma_ceq_init_info {
881 u64 ceqe_pa;
882 struct irdma_sc_dev *dev;
883 u64 *ceqe_base;
884 void *pbl_list;
885 u32 elem_cnt;
886 u32 ceq_id;
887 bool virtual_map:1;
888 bool tph_en:1;
889 bool itr_no_expire:1;
890 u8 pbl_chunk_size;
891 u8 tph_val;
892 u16 vsi_idx;
893 u32 first_pm_pbl_idx;
894 };
895
896 struct irdma_aeq_init_info {
897 u64 aeq_elem_pa;
898 struct irdma_sc_dev *dev;
899 u32 *aeqe_base;
900 void *pbl_list;
901 u32 elem_cnt;
902 bool virtual_map;
903 u8 pbl_chunk_size;
904 u32 first_pm_pbl_idx;
905 u32 msix_idx;
906 };
907
908 struct irdma_ccq_init_info {
909 u64 cq_pa;
910 u64 shadow_area_pa;
911 struct irdma_sc_dev *dev;
912 struct irdma_cqe *cq_base;
913 __le64 *shadow_area;
914 void *pbl_list;
915 u32 num_elem;
916 u32 ceq_id;
917 u32 shadow_read_threshold;
918 bool ceqe_mask:1;
919 bool ceq_id_valid:1;
920 bool avoid_mem_cflct:1;
921 bool virtual_map:1;
922 bool tph_en:1;
923 u8 tph_val;
924 u8 pbl_chunk_size;
925 u32 first_pm_pbl_idx;
926 struct irdma_sc_vsi *vsi;
927 };
928
929 struct irdma_udp_offload_info {
930 bool ipv4:1;
931 bool insert_vlan_tag:1;
932 u8 ttl;
933 u8 tos;
934 u16 src_port;
935 u16 dst_port;
936 u32 dest_ip_addr[4];
937 u32 snd_mss;
938 u16 vlan_tag;
939 u16 arp_idx;
940 u32 flow_label;
941 u8 udp_state;
942 u32 psn_nxt;
943 u32 lsn;
944 u32 epsn;
945 u32 psn_max;
946 u32 psn_una;
947 u32 local_ipaddr[4];
948 u32 cwnd;
949 u8 rexmit_thresh;
950 u8 rnr_nak_thresh;
951 u8 rnr_nak_tmr;
952 u8 min_rnr_timer;
953 };
954
955 struct irdma_roce_offload_info {
956 u16 p_key;
957 u16 err_rq_idx;
958 u32 qkey;
959 u32 dest_qp;
960 u8 roce_tver;
961 u8 ack_credits;
962 u8 err_rq_idx_valid;
963 u32 pd_id;
964 u16 ord_size;
965 u16 ird_size;
966 bool is_qp1:1;
967 bool udprivcq_en:1;
968 bool dcqcn_en:1;
969 bool rcv_no_icrc:1;
970 bool wr_rdresp_en:1;
971 bool bind_en:1;
972 bool fast_reg_en:1;
973 bool priv_mode_en:1;
974 bool rd_en:1;
975 bool timely_en:1;
976 bool dctcp_en:1;
977 bool fw_cc_enable:1;
978 bool use_stats_inst:1;
979 u8 local_ack_timeout;
980 u16 t_high;
981 u16 t_low;
982 u8 last_byte_sent;
983 u8 mac_addr[ETH_ALEN];
984 u8 rtomin;
985 };
986
987 struct irdma_iwarp_offload_info {
988 u16 rcv_mark_offset;
989 u16 snd_mark_offset;
990 u8 ddp_ver;
991 u8 rdmap_ver;
992 u8 iwarp_mode;
993 u16 err_rq_idx;
994 u32 pd_id;
995 u16 ord_size;
996 u16 ird_size;
997 bool ib_rd_en:1;
998 bool align_hdrs:1;
999 bool rcv_no_mpa_crc:1;
1000 bool err_rq_idx_valid:1;
1001 bool snd_mark_en:1;
1002 bool rcv_mark_en:1;
1003 bool wr_rdresp_en:1;
1004 bool bind_en:1;
1005 bool fast_reg_en:1;
1006 bool priv_mode_en:1;
1007 bool rd_en:1;
1008 bool timely_en:1;
1009 bool use_stats_inst:1;
1010 bool ecn_en:1;
1011 bool dctcp_en:1;
1012 u16 t_high;
1013 u16 t_low;
1014 u8 last_byte_sent;
1015 u8 mac_addr[ETH_ALEN];
1016 u8 rtomin;
1017 };
1018
1019 struct irdma_tcp_offload_info {
1020 bool ipv4:1;
1021 bool no_nagle:1;
1022 bool insert_vlan_tag:1;
1023 bool time_stamp:1;
1024 bool drop_ooo_seg:1;
1025 bool avoid_stretch_ack:1;
1026 bool wscale:1;
1027 bool ignore_tcp_opt:1;
1028 bool ignore_tcp_uns_opt:1;
1029 u8 cwnd_inc_limit;
1030 u8 dup_ack_thresh;
1031 u8 ttl;
1032 u8 src_mac_addr_idx;
1033 u8 tos;
1034 u16 src_port;
1035 u16 dst_port;
1036 u32 dest_ip_addr[4];
1037 //u32 dest_ip_addr0;
1038 //u32 dest_ip_addr1;
1039 //u32 dest_ip_addr2;
1040 //u32 dest_ip_addr3;
1041 u32 snd_mss;
1042 u16 syn_rst_handling;
1043 u16 vlan_tag;
1044 u16 arp_idx;
1045 u32 flow_label;
1046 u8 tcp_state;
1047 u8 snd_wscale;
1048 u8 rcv_wscale;
1049 u32 time_stamp_recent;
1050 u32 time_stamp_age;
1051 u32 snd_nxt;
1052 u32 snd_wnd;
1053 u32 rcv_nxt;
1054 u32 rcv_wnd;
1055 u32 snd_max;
1056 u32 snd_una;
1057 u32 srtt;
1058 u32 rtt_var;
1059 u32 ss_thresh;
1060 u32 cwnd;
1061 u32 snd_wl1;
1062 u32 snd_wl2;
1063 u32 max_snd_window;
1064 u8 rexmit_thresh;
1065 u32 local_ipaddr[4];
1066 };
1067
1068 struct irdma_qp_host_ctx_info {
1069 u64 qp_compl_ctx;
1070 union {
1071 struct irdma_tcp_offload_info *tcp_info;
1072 struct irdma_udp_offload_info *udp_info;
1073 };
1074 union {
1075 struct irdma_iwarp_offload_info *iwarp_info;
1076 struct irdma_roce_offload_info *roce_info;
1077 };
1078 u32 send_cq_num;
1079 u32 rcv_cq_num;
1080 u32 srq_id;
1081 u32 rem_endpoint_idx;
1082 u16 stats_idx;
1083 bool remote_atomics_en:1;
1084 bool srq_valid:1;
1085 bool tcp_info_valid:1;
1086 bool iwarp_info_valid:1;
1087 bool stats_idx_valid:1;
1088 u8 user_pri;
1089 };
1090
1091 struct irdma_aeqe_info {
1092 u64 compl_ctx;
1093 u32 qp_cq_id;
1094 u32 def_info; /* only valid for DEF_CMPL */
1095 u16 ae_id;
1096 u16 wqe_idx;
1097 u8 tcp_state;
1098 u8 iwarp_state;
1099 bool qp:1;
1100 bool cq:1;
1101 bool sq:1;
1102 bool rq:1;
1103 bool srq:1;
1104 bool in_rdrsp_wr:1;
1105 bool out_rdrsp:1;
1106 bool aeqe_overflow:1;
1107 bool err_rq_idx_valid:1;
1108 u8 q2_data_written;
1109 u8 ae_src;
1110 };
1111
1112 struct irdma_allocate_stag_info {
1113 u64 total_len;
1114 u64 first_pm_pbl_idx;
1115 u32 chunk_size;
1116 u32 stag_idx;
1117 u32 page_size;
1118 u32 pd_id;
1119 u16 access_rights;
1120 bool remote_access:1;
1121 bool use_hmc_fcn_index:1;
1122 bool use_pf_rid:1;
1123 bool all_memory:1;
1124 bool remote_atomics_en:1;
1125 u16 hmc_fcn_index;
1126 };
1127
1128 struct irdma_mw_alloc_info {
1129 u32 mw_stag_index;
1130 u32 page_size;
1131 u32 pd_id;
1132 bool remote_access:1;
1133 bool mw_wide:1;
1134 bool mw1_bind_dont_vldt_key:1;
1135 };
1136
1137 struct irdma_reg_ns_stag_info {
1138 u64 reg_addr_pa;
1139 u64 va;
1140 u64 total_len;
1141 u32 page_size;
1142 u32 chunk_size;
1143 u32 first_pm_pbl_index;
1144 enum irdma_addressing_type addr_type;
1145 irdma_stag_index stag_idx;
1146 u16 access_rights;
1147 u32 pd_id;
1148 irdma_stag_key stag_key;
1149 bool use_hmc_fcn_index:1;
1150 u8 hmc_fcn_index;
1151 bool use_pf_rid:1;
1152 bool all_memory:1;
1153 bool remote_atomics_en:1;
1154 };
1155
1156 struct irdma_fast_reg_stag_info {
1157 u64 wr_id;
1158 u64 reg_addr_pa;
1159 u64 fbo;
1160 void *va;
1161 u64 total_len;
1162 u32 page_size;
1163 u32 chunk_size;
1164 u32 first_pm_pbl_index;
1165 enum irdma_addressing_type addr_type;
1166 irdma_stag_index stag_idx;
1167 u16 access_rights;
1168 u32 pd_id;
1169 irdma_stag_key stag_key;
1170 bool local_fence:1;
1171 bool read_fence:1;
1172 bool signaled:1;
1173 bool use_hmc_fcn_index:1;
1174 u8 hmc_fcn_index;
1175 bool use_pf_rid:1;
1176 bool defer_flag:1;
1177 bool remote_atomics_en:1;
1178 };
1179
1180 struct irdma_dealloc_stag_info {
1181 u32 stag_idx;
1182 u32 pd_id;
1183 bool mr:1;
1184 bool dealloc_pbl:1;
1185 };
1186
1187 struct irdma_register_shared_stag {
1188 u64 va;
1189 enum irdma_addressing_type addr_type;
1190 irdma_stag_index new_stag_idx;
1191 irdma_stag_index parent_stag_idx;
1192 u32 access_rights;
1193 u32 pd_id;
1194 u32 page_size;
1195 irdma_stag_key new_stag_key;
1196 };
1197
1198 struct irdma_qp_init_info {
1199 struct irdma_qp_uk_init_info qp_uk_init_info;
1200 struct irdma_sc_pd *pd;
1201 struct irdma_sc_vsi *vsi;
1202 __le64 *host_ctx;
1203 u8 *q2;
1204 u64 sq_pa;
1205 u64 rq_pa;
1206 u64 host_ctx_pa;
1207 u64 q2_pa;
1208 u64 shadow_area_pa;
1209 u8 sq_tph_val;
1210 u8 rq_tph_val;
1211 bool sq_tph_en:1;
1212 bool rq_tph_en:1;
1213 bool rcv_tph_en:1;
1214 bool xmit_tph_en:1;
1215 bool virtual_map:1;
1216 };
1217
1218 struct irdma_cq_init_info {
1219 struct irdma_sc_dev *dev;
1220 u64 cq_base_pa;
1221 u64 shadow_area_pa;
1222 u32 ceq_id;
1223 u32 shadow_read_threshold;
1224 u8 pbl_chunk_size;
1225 u32 first_pm_pbl_idx;
1226 bool virtual_map:1;
1227 bool ceqe_mask:1;
1228 bool ceq_id_valid:1;
1229 bool tph_en:1;
1230 u8 tph_val;
1231 u8 type;
1232 struct irdma_cq_uk_init_info cq_uk_init_info;
1233 struct irdma_sc_vsi *vsi;
1234 };
1235
1236 struct irdma_upload_context_info {
1237 u64 buf_pa;
1238 u32 qp_id;
1239 u8 qp_type;
1240 bool freeze_qp:1;
1241 bool raw_format:1;
1242 };
1243
1244 struct irdma_local_mac_entry_info {
1245 u8 mac_addr[6];
1246 u16 entry_idx;
1247 };
1248
1249 struct irdma_add_arp_cache_entry_info {
1250 u8 mac_addr[ETH_ALEN];
1251 u32 reach_max;
1252 u16 arp_index;
1253 bool permanent;
1254 };
1255
1256 struct irdma_apbvt_info {
1257 u16 port;
1258 bool add;
1259 };
1260
1261 struct irdma_qhash_table_info {
1262 struct irdma_sc_vsi *vsi;
1263 enum irdma_quad_hash_manage_type manage;
1264 enum irdma_quad_entry_type entry_type;
1265 bool vlan_valid:1;
1266 bool ipv4_valid:1;
1267 u8 mac_addr[ETH_ALEN];
1268 u16 vlan_id;
1269 u8 user_pri;
1270 u32 qp_num;
1271 u32 dest_ip[4];
1272 u32 src_ip[4];
1273 u16 dest_port;
1274 u16 src_port;
1275 };
1276
1277 struct irdma_cqp_manage_push_page_info {
1278 u32 push_idx;
1279 u16 qs_handle;
1280 u8 free_page;
1281 u8 push_page_type;
1282 };
1283
1284 struct irdma_qp_flush_info {
1285 u32 err_sq_idx;
1286 u32 err_rq_idx;
1287 u16 sq_minor_code;
1288 u16 sq_major_code;
1289 u16 rq_minor_code;
1290 u16 rq_major_code;
1291 u16 ae_code;
1292 u8 ae_src;
1293 bool sq:1;
1294 bool rq:1;
1295 bool userflushcode:1;
1296 bool generate_ae:1;
1297 bool err_sq_idx_valid:1;
1298 bool err_rq_idx_valid:1;
1299 };
1300
1301 struct irdma_gen_ae_info {
1302 u16 ae_code;
1303 u8 ae_src;
1304 };
1305
1306 struct irdma_cqp_timeout {
1307 u64 compl_cqp_cmds;
1308 u32 count;
1309 };
1310
1311 struct irdma_irq_ops {
1312 void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1313 void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1314 bool enable);
1315 void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1316 void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1317 };
1318
1319 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1320 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1321 bool check_overflow, bool post_sq);
1322 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1323 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1324 struct irdma_ccq_cqe_info *info);
1325 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1326 struct irdma_ccq_init_info *info);
1327
1328 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1329 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1330
1331 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1332 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1333 struct irdma_ceq_init_info *info);
1334 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1335 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1336
1337 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1338 struct irdma_aeq_init_info *info);
1339 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1340 struct irdma_aeqe_info *info);
1341 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1342
1343 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1344 int abi_ver);
1345 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1346 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1347 struct irdma_sc_dev *dev);
1348 void irdma_sc_cqp_def_cmpl_ae_handler(struct irdma_sc_dev *dev,
1349 struct irdma_aeqe_info *info,
1350 bool first, u64 *scratch,
1351 u32 *sw_def_info);
1352 u64 irdma_sc_cqp_cleanup_handler(struct irdma_sc_dev *dev);
1353 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1354 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1355 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1356 struct irdma_cqp_init_info *info);
1357 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1358 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1359 struct irdma_ccq_cqe_info *cmpl_info);
1360 int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1361 struct irdma_fast_reg_stag_info *info, bool post_sq);
1362 int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1363 struct irdma_create_qp_info *info, u64 scratch,
1364 bool post_sq);
1365 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1366 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1367 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1368 struct irdma_qp_flush_info *info, u64 scratch,
1369 bool post_sq);
1370 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1371 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1372 struct irdma_modify_qp_info *info, u64 scratch,
1373 bool post_sq);
1374 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1375 irdma_stag stag);
1376
1377 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1378 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1379 struct irdma_qp_host_ctx_info *info);
1380 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1381 struct irdma_qp_host_ctx_info *info);
1382 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1383 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1384 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1385 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1386 u8 hmc_fn_id, bool post_sq,
1387 bool poll_registers);
1388 int irdma_sc_srq_init(struct irdma_sc_srq *srq,
1389 struct irdma_srq_init_info *info);
1390
1391 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1392 struct cqp_info {
1393 union {
1394 struct {
1395 struct irdma_sc_qp *qp;
1396 struct irdma_create_qp_info info;
1397 u64 scratch;
1398 } qp_create;
1399
1400 struct {
1401 struct irdma_sc_qp *qp;
1402 struct irdma_modify_qp_info info;
1403 u64 scratch;
1404 } qp_modify;
1405
1406 struct {
1407 struct irdma_sc_qp *qp;
1408 u64 scratch;
1409 bool remove_hash_idx;
1410 bool ignore_mw_bnd;
1411 } qp_destroy;
1412
1413 struct {
1414 struct irdma_sc_cq *cq;
1415 u64 scratch;
1416 bool check_overflow;
1417 } cq_create;
1418
1419 struct {
1420 struct irdma_sc_cq *cq;
1421 struct irdma_modify_cq_info info;
1422 u64 scratch;
1423 } cq_modify;
1424
1425 struct {
1426 struct irdma_sc_cq *cq;
1427 u64 scratch;
1428 } cq_destroy;
1429
1430 struct {
1431 struct irdma_sc_dev *dev;
1432 struct irdma_allocate_stag_info info;
1433 u64 scratch;
1434 } alloc_stag;
1435
1436 struct {
1437 struct irdma_sc_dev *dev;
1438 struct irdma_mw_alloc_info info;
1439 u64 scratch;
1440 } mw_alloc;
1441
1442 struct {
1443 struct irdma_sc_dev *dev;
1444 struct irdma_reg_ns_stag_info info;
1445 u64 scratch;
1446 } mr_reg_non_shared;
1447
1448 struct {
1449 struct irdma_sc_dev *dev;
1450 struct irdma_dealloc_stag_info info;
1451 u64 scratch;
1452 } dealloc_stag;
1453
1454 struct {
1455 struct irdma_sc_cqp *cqp;
1456 struct irdma_add_arp_cache_entry_info info;
1457 u64 scratch;
1458 } add_arp_cache_entry;
1459
1460 struct {
1461 struct irdma_sc_cqp *cqp;
1462 u64 scratch;
1463 u16 arp_index;
1464 } del_arp_cache_entry;
1465
1466 struct {
1467 struct irdma_sc_cqp *cqp;
1468 struct irdma_local_mac_entry_info info;
1469 u64 scratch;
1470 } add_local_mac_entry;
1471
1472 struct {
1473 struct irdma_sc_cqp *cqp;
1474 u64 scratch;
1475 u8 entry_idx;
1476 u8 ignore_ref_count;
1477 } del_local_mac_entry;
1478
1479 struct {
1480 struct irdma_sc_cqp *cqp;
1481 u64 scratch;
1482 } alloc_local_mac_entry;
1483
1484 struct {
1485 struct irdma_sc_cqp *cqp;
1486 struct irdma_cqp_manage_push_page_info info;
1487 u64 scratch;
1488 } manage_push_page;
1489
1490 struct {
1491 struct irdma_sc_dev *dev;
1492 struct irdma_upload_context_info info;
1493 u64 scratch;
1494 } qp_upload_context;
1495
1496 struct {
1497 struct irdma_sc_dev *dev;
1498 struct irdma_hmc_fcn_info info;
1499 u64 scratch;
1500 } manage_hmc_pm;
1501
1502 struct {
1503 struct irdma_sc_ceq *ceq;
1504 u64 scratch;
1505 } ceq_create;
1506
1507 struct {
1508 struct irdma_sc_ceq *ceq;
1509 u64 scratch;
1510 } ceq_destroy;
1511
1512 struct {
1513 struct irdma_sc_aeq *aeq;
1514 u64 scratch;
1515 } aeq_create;
1516
1517 struct {
1518 struct irdma_sc_aeq *aeq;
1519 u64 scratch;
1520 } aeq_destroy;
1521
1522 struct {
1523 struct irdma_sc_qp *qp;
1524 struct irdma_qp_flush_info info;
1525 u64 scratch;
1526 } qp_flush_wqes;
1527
1528 struct {
1529 struct irdma_sc_qp *qp;
1530 struct irdma_gen_ae_info info;
1531 u64 scratch;
1532 } gen_ae;
1533
1534 struct {
1535 struct irdma_sc_cqp *cqp;
1536 void *fpm_val_va;
1537 u64 fpm_val_pa;
1538 u8 hmc_fn_id;
1539 u64 scratch;
1540 } query_fpm_val;
1541
1542 struct {
1543 struct irdma_sc_cqp *cqp;
1544 void *fpm_val_va;
1545 u64 fpm_val_pa;
1546 u8 hmc_fn_id;
1547 u64 scratch;
1548 } commit_fpm_val;
1549
1550 struct {
1551 struct irdma_sc_cqp *cqp;
1552 struct irdma_apbvt_info info;
1553 u64 scratch;
1554 } manage_apbvt_entry;
1555
1556 struct {
1557 struct irdma_sc_cqp *cqp;
1558 struct irdma_qhash_table_info info;
1559 u64 scratch;
1560 } manage_qhash_table_entry;
1561
1562 struct {
1563 struct irdma_sc_dev *dev;
1564 struct irdma_update_sds_info info;
1565 u64 scratch;
1566 } update_pe_sds;
1567
1568 struct {
1569 struct irdma_sc_cqp *cqp;
1570 struct irdma_sc_qp *qp;
1571 u64 scratch;
1572 } suspend_resume;
1573
1574 struct {
1575 struct irdma_sc_cqp *cqp;
1576 struct irdma_ah_info info;
1577 u64 scratch;
1578 } ah_create;
1579
1580 struct {
1581 struct irdma_sc_cqp *cqp;
1582 struct irdma_ah_info info;
1583 u64 scratch;
1584 } ah_destroy;
1585
1586 struct {
1587 struct irdma_sc_cqp *cqp;
1588 struct irdma_mcast_grp_info info;
1589 u64 scratch;
1590 } mc_create;
1591
1592 struct {
1593 struct irdma_sc_cqp *cqp;
1594 struct irdma_mcast_grp_info info;
1595 u64 scratch;
1596 } mc_destroy;
1597
1598 struct {
1599 struct irdma_sc_cqp *cqp;
1600 struct irdma_mcast_grp_info info;
1601 u64 scratch;
1602 } mc_modify;
1603
1604 struct {
1605 struct irdma_sc_cqp *cqp;
1606 struct irdma_stats_inst_info info;
1607 u64 scratch;
1608 } stats_manage;
1609
1610 struct {
1611 struct irdma_sc_cqp *cqp;
1612 struct irdma_stats_gather_info info;
1613 u64 scratch;
1614 } stats_gather;
1615
1616 struct {
1617 struct irdma_sc_cqp *cqp;
1618 struct irdma_ws_node_info info;
1619 u64 scratch;
1620 } ws_node;
1621
1622 struct {
1623 struct irdma_sc_cqp *cqp;
1624 struct irdma_up_info info;
1625 u64 scratch;
1626 } up_map;
1627
1628 struct {
1629 struct irdma_sc_cqp *cqp;
1630 struct irdma_dma_mem query_buff_mem;
1631 u64 scratch;
1632 } query_rdma;
1633
1634 struct {
1635 struct irdma_sc_srq *srq;
1636 u64 scratch;
1637 } srq_create;
1638
1639 struct {
1640 struct irdma_sc_srq *srq;
1641 struct irdma_modify_srq_info info;
1642 u64 scratch;
1643 } srq_modify;
1644
1645 struct {
1646 struct irdma_sc_srq *srq;
1647 u64 scratch;
1648 } srq_destroy;
1649
1650 } u;
1651 };
1652
1653 struct cqp_cmds_info {
1654 struct list_head cqp_cmd_entry;
1655 u8 cqp_cmd;
1656 u8 post_sq;
1657 struct cqp_info in;
1658 };
1659
1660 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1661 u32 *wqe_idx);
1662
1663 /**
1664 * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1665 * @cqp: struct for cqp hw
1666 * @scratch: private data for CQP WQE
1667 */
irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp * cqp,u64 scratch)1668 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1669 {
1670 u32 wqe_idx;
1671
1672 return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1673 }
1674 #endif /* IRDMA_TYPE_H */
1675