1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #ifndef IRDMA_TYPE_H
4 #define IRDMA_TYPE_H
5 #include "osdep.h"
6 #include "irdma.h"
7 #include "user.h"
8 #include "hmc.h"
9 #include "uda.h"
10 #include "ws.h"
11 #include "virtchnl.h"
12
13 #define IRDMA_DEBUG_ERR "ERR"
14 #define IRDMA_DEBUG_INIT "INIT"
15 #define IRDMA_DEBUG_DEV "DEV"
16 #define IRDMA_DEBUG_CM "CM"
17 #define IRDMA_DEBUG_VERBS "VERBS"
18 #define IRDMA_DEBUG_PUDA "PUDA"
19 #define IRDMA_DEBUG_ILQ "ILQ"
20 #define IRDMA_DEBUG_IEQ "IEQ"
21 #define IRDMA_DEBUG_QP "QP"
22 #define IRDMA_DEBUG_CQ "CQ"
23 #define IRDMA_DEBUG_MR "MR"
24 #define IRDMA_DEBUG_PBLE "PBLE"
25 #define IRDMA_DEBUG_WQE "WQE"
26 #define IRDMA_DEBUG_AEQ "AEQ"
27 #define IRDMA_DEBUG_CQP "CQP"
28 #define IRDMA_DEBUG_HMC "HMC"
29 #define IRDMA_DEBUG_USER "USER"
30 #define IRDMA_DEBUG_VIRT "VIRT"
31 #define IRDMA_DEBUG_DCB "DCB"
32 #define IRDMA_DEBUG_CQE "CQE"
33 #define IRDMA_DEBUG_CLNT "CLNT"
34 #define IRDMA_DEBUG_WS "WS"
35 #define IRDMA_DEBUG_STATS "STATS"
36
37 enum irdma_page_size {
38 IRDMA_PAGE_SIZE_4K = 0,
39 IRDMA_PAGE_SIZE_2M,
40 IRDMA_PAGE_SIZE_1G,
41 };
42
43 enum irdma_hdrct_flags {
44 DDP_LEN_FLAG = 0x80,
45 DDP_HDR_FLAG = 0x40,
46 RDMA_HDR_FLAG = 0x20,
47 };
48
49 enum irdma_term_layers {
50 LAYER_RDMA = 0,
51 LAYER_DDP = 1,
52 LAYER_MPA = 2,
53 };
54
55 enum irdma_term_error_types {
56 RDMAP_REMOTE_PROT = 1,
57 RDMAP_REMOTE_OP = 2,
58 DDP_CATASTROPHIC = 0,
59 DDP_TAGGED_BUF = 1,
60 DDP_UNTAGGED_BUF = 2,
61 DDP_LLP = 3,
62 };
63
64 enum irdma_term_rdma_errors {
65 RDMAP_INV_STAG = 0x00,
66 RDMAP_INV_BOUNDS = 0x01,
67 RDMAP_ACCESS = 0x02,
68 RDMAP_UNASSOC_STAG = 0x03,
69 RDMAP_TO_WRAP = 0x04,
70 RDMAP_INV_RDMAP_VER = 0x05,
71 RDMAP_UNEXPECTED_OP = 0x06,
72 RDMAP_CATASTROPHIC_LOCAL = 0x07,
73 RDMAP_CATASTROPHIC_GLOBAL = 0x08,
74 RDMAP_CANT_INV_STAG = 0x09,
75 RDMAP_UNSPECIFIED = 0xff,
76 };
77
78 enum irdma_term_ddp_errors {
79 DDP_CATASTROPHIC_LOCAL = 0x00,
80 DDP_TAGGED_INV_STAG = 0x00,
81 DDP_TAGGED_BOUNDS = 0x01,
82 DDP_TAGGED_UNASSOC_STAG = 0x02,
83 DDP_TAGGED_TO_WRAP = 0x03,
84 DDP_TAGGED_INV_DDP_VER = 0x04,
85 DDP_UNTAGGED_INV_QN = 0x01,
86 DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
87 DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
88 DDP_UNTAGGED_INV_MO = 0x04,
89 DDP_UNTAGGED_INV_TOO_LONG = 0x05,
90 DDP_UNTAGGED_INV_DDP_VER = 0x06,
91 };
92
93 enum irdma_term_mpa_errors {
94 MPA_CLOSED = 0x01,
95 MPA_CRC = 0x02,
96 MPA_MARKER = 0x03,
97 MPA_REQ_RSP = 0x04,
98 };
99
100 enum irdma_hw_stats_index {
101 /* gen1 - 32-bit */
102 IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0,
103 IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1,
104 IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2,
105 IRDMA_HW_STAT_INDEX_IP6RXDISCARD = 3,
106 IRDMA_HW_STAT_INDEX_IP6RXTRUNC = 4,
107 IRDMA_HW_STAT_INDEX_IP6TXNOROUTE = 5,
108 IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6,
109 IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7,
110 IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8,
111 IRDMA_HW_STAT_INDEX_RXVLANERR = 9,
112 /* gen1 - 64-bit */
113 IRDMA_HW_STAT_INDEX_IP4RXOCTS = 10,
114 IRDMA_HW_STAT_INDEX_IP4RXPKTS = 11,
115 IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 12,
116 IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 13,
117 IRDMA_HW_STAT_INDEX_IP4TXOCTS = 14,
118 IRDMA_HW_STAT_INDEX_IP4TXPKTS = 15,
119 IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 16,
120 IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 17,
121 IRDMA_HW_STAT_INDEX_IP6RXOCTS = 18,
122 IRDMA_HW_STAT_INDEX_IP6RXPKTS = 19,
123 IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 20,
124 IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 21,
125 IRDMA_HW_STAT_INDEX_IP6TXOCTS = 22,
126 IRDMA_HW_STAT_INDEX_IP6TXPKTS = 23,
127 IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 24,
128 IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 25,
129 IRDMA_HW_STAT_INDEX_TCPRXSEGS = 26,
130 IRDMA_HW_STAT_INDEX_TCPTXSEG = 27,
131 IRDMA_HW_STAT_INDEX_RDMARXRDS = 28,
132 IRDMA_HW_STAT_INDEX_RDMARXSNDS = 29,
133 IRDMA_HW_STAT_INDEX_RDMARXWRS = 30,
134 IRDMA_HW_STAT_INDEX_RDMATXRDS = 31,
135 IRDMA_HW_STAT_INDEX_RDMATXSNDS = 32,
136 IRDMA_HW_STAT_INDEX_RDMATXWRS = 33,
137 IRDMA_HW_STAT_INDEX_RDMAVBND = 34,
138 IRDMA_HW_STAT_INDEX_RDMAVINV = 35,
139 IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36,
140 IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37,
141 IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38,
142 IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39,
143 IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40,
144 IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41,
145 IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */
146 /* gen2 - 64-bit */
147 IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42,
148 /* gen2 - 32-bit */
149 IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43,
150 IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44,
151 IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45,
152 IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46,
153
154 /* gen3 */
155 IRDMA_HW_STAT_INDEX_RNR_SENT = 46,
156 IRDMA_HW_STAT_INDEX_RNR_RCVD = 47,
157 IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT = 48,
158 IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT = 49,
159 IRDMA_HW_STAT_INDEX_RDMARXATS = 50,
160 IRDMA_HW_STAT_INDEX_RDMATXATS = 51,
161 IRDMA_HW_STAT_INDEX_NAKSEQERR = 52,
162 IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED = 53,
163 IRDMA_HW_STAT_INDEX_RTO = 54,
164 IRDMA_HW_STAT_INDEX_RXOOOPKTS = 55,
165 IRDMA_HW_STAT_INDEX_ICRCERR = 56,
166
167 IRDMA_HW_STAT_INDEX_MAX_GEN_3 = 57,
168 };
169
170 enum irdma_feature_type {
171 IRDMA_FEATURE_FW_INFO = 0,
172 IRDMA_HW_VERSION_INFO = 1,
173 IRDMA_QP_MAX_INCR = 2,
174 IRDMA_CQ_MAX_INCR = 3,
175 IRDMA_CEQ_MAX_INCR = 4,
176 IRDMA_SD_MAX_INCR = 5,
177 IRDMA_MR_MAX_INCR = 6,
178 IRDMA_Q1_MAX_INCR = 7,
179 IRDMA_AH_MAX_INCR = 8,
180 IRDMA_SRQ_MAX_INCR = 9,
181 IRDMA_TIMER_MAX_INCR = 10,
182 IRDMA_XF_MAX_INCR = 11,
183 IRDMA_RRF_MAX_INCR = 12,
184 IRDMA_PBLE_MAX_INCR = 13,
185 IRDMA_OBJ_1 = 22,
186 IRDMA_OBJ_2 = 23,
187 IRDMA_ENDPT_TRK = 24,
188 IRDMA_FTN_INLINE_MAX = 25,
189 IRDMA_QSETS_MAX = 26,
190 IRDMA_ASO = 27,
191 IRDMA_FTN_FLAGS = 32,
192 IRDMA_FTN_NOP = 33,
193 IRDMA_MAX_FEATURES, /* Must be last entry */
194 };
195
196 enum irdma_sched_prio_type {
197 IRDMA_PRIO_WEIGHTED_RR = 1,
198 IRDMA_PRIO_STRICT = 2,
199 IRDMA_PRIO_WEIGHTED_STRICT = 3,
200 };
201
202 enum irdma_vm_vf_type {
203 IRDMA_VF_TYPE = 0,
204 IRDMA_VM_TYPE,
205 IRDMA_PF_TYPE,
206 };
207
208 enum irdma_cqp_hmc_profile {
209 IRDMA_HMC_PROFILE_DEFAULT = 1,
210 IRDMA_HMC_PROFILE_FAVOR_VF = 2,
211 IRDMA_HMC_PROFILE_EQUAL = 3,
212 };
213
214 enum irdma_quad_entry_type {
215 IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1,
216 IRDMA_QHASH_TYPE_TCP_SYN,
217 IRDMA_QHASH_TYPE_UDP_UNICAST,
218 IRDMA_QHASH_TYPE_UDP_MCAST,
219 IRDMA_QHASH_TYPE_ROCE_MCAST,
220 IRDMA_QHASH_TYPE_ROCEV2_HW,
221 };
222
223 enum irdma_quad_hash_manage_type {
224 IRDMA_QHASH_MANAGE_TYPE_DELETE = 0,
225 IRDMA_QHASH_MANAGE_TYPE_ADD,
226 IRDMA_QHASH_MANAGE_TYPE_MODIFY,
227 };
228
229 enum irdma_syn_rst_handling {
230 IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0,
231 IRDMA_SYN_RST_HANDLING_HW_TCP,
232 IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE,
233 IRDMA_SYN_RST_HANDLING_FW_TCP,
234 };
235
236 enum irdma_queue_type {
237 IRDMA_QUEUE_TYPE_SQ_RQ = 0,
238 IRDMA_QUEUE_TYPE_CQP,
239 IRDMA_QUEUE_TYPE_SRQ,
240 };
241
242 struct irdma_sc_dev;
243 struct irdma_vsi_pestat;
244
245 struct irdma_dcqcn_cc_params {
246 u8 cc_cfg_valid;
247 u8 min_dec_factor;
248 u8 min_rate;
249 u8 dcqcn_f;
250 u16 rai_factor;
251 u16 hai_factor;
252 u16 dcqcn_t;
253 u32 dcqcn_b;
254 u32 rreduce_mperiod;
255 };
256
257 struct irdma_cqp_init_info {
258 u64 cqp_compl_ctx;
259 u64 host_ctx_pa;
260 u64 sq_pa;
261 struct irdma_sc_dev *dev;
262 struct irdma_cqp_quanta *sq;
263 struct irdma_dcqcn_cc_params dcqcn_params;
264 __le64 *host_ctx;
265 u64 *scratch_array;
266 u32 sq_size;
267 struct irdma_ooo_cqp_op *ooo_op_array;
268 u32 pe_en_vf_cnt;
269 u16 hw_maj_ver;
270 u16 hw_min_ver;
271 u8 struct_ver;
272 u8 hmc_profile;
273 u8 ena_vf_count;
274 u8 ceqs_per_vf;
275 u8 ooisc_blksize;
276 u8 rrsp_blksize;
277 u8 q1_blksize;
278 u8 xmit_blksize;
279 u8 ts_override;
280 u8 ts_shift;
281 u8 en_fine_grained_timers;
282 u8 blksizes_valid;
283 bool en_datacenter_tcp:1;
284 bool disable_packed:1;
285 bool rocev2_rto_policy:1;
286 enum irdma_protocol_used protocol_used;
287 };
288
289 struct irdma_terminate_hdr {
290 u8 layer_etype;
291 u8 error_code;
292 u8 hdrct;
293 u8 rsvd;
294 };
295
296 struct irdma_cqp_sq_wqe {
297 __le64 buf[IRDMA_CQP_WQE_SIZE];
298 };
299
300 struct irdma_sc_aeqe {
301 __le64 buf[IRDMA_AEQE_SIZE];
302 };
303
304 struct irdma_ceqe {
305 __le64 buf[IRDMA_CEQE_SIZE];
306 };
307
308 struct irdma_cqp_ctx {
309 __le64 buf[IRDMA_CQP_CTX_SIZE];
310 };
311
312 struct irdma_cq_shadow_area {
313 __le64 buf[IRDMA_SHADOW_AREA_SIZE];
314 };
315
316 struct irdma_dev_hw_stats_offsets {
317 u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
318 };
319
320 struct irdma_dev_hw_stats {
321 u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
322 };
323
324 struct irdma_gather_stats {
325 u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)];
326 };
327
328 struct irdma_hw_stat_map {
329 u16 byteoff;
330 u8 bitoff;
331 u64 bitmask;
332 };
333
334 struct irdma_stats_gather_info {
335 bool use_hmc_fcn_index:1;
336 bool use_stats_inst:1;
337 u8 hmc_fcn_index;
338 u8 stats_inst_index;
339 struct irdma_dma_mem stats_buff_mem;
340 void *gather_stats_va;
341 void *last_gather_stats_va;
342 };
343
344 struct irdma_vsi_pestat {
345 struct irdma_hw *hw;
346 struct irdma_dev_hw_stats hw_stats;
347 struct irdma_stats_gather_info gather_info;
348 struct timer_list stats_timer;
349 struct irdma_sc_vsi *vsi;
350 struct irdma_dev_hw_stats last_hw_stats;
351 spinlock_t lock; /* rdma stats lock */
352 };
353
354 struct irdma_mmio_region {
355 u8 __iomem *addr;
356 resource_size_t len;
357 resource_size_t offset;
358 };
359
360 struct irdma_hw {
361 union {
362 u8 __iomem *hw_addr;
363 struct {
364 struct irdma_mmio_region rdma_reg; /* RDMA region */
365 struct irdma_mmio_region *io_regs; /* Non-RDMA MMIO regions */
366 u16 num_io_regions; /* Number of Non-RDMA MMIO regions */
367 };
368 };
369 struct device *device;
370 struct irdma_hmc_info hmc;
371 };
372
373 struct irdma_pfpdu {
374 struct list_head rxlist;
375 u32 rcv_nxt;
376 u32 fps;
377 u32 max_fpdu_data;
378 u32 nextseqnum;
379 u32 rcv_start_seq;
380 bool mode:1;
381 bool mpa_crc_err:1;
382 u8 marker_len;
383 u64 total_ieq_bufs;
384 u64 fpdu_processed;
385 u64 bad_seq_num;
386 u64 crc_err;
387 u64 no_tx_bufs;
388 u64 tx_err;
389 u64 out_of_order;
390 u64 pmode_count;
391 struct irdma_sc_ah *ah;
392 struct irdma_puda_buf *ah_buf;
393 spinlock_t lock; /* fpdu processing lock */
394 struct irdma_puda_buf *lastrcv_buf;
395 };
396
397 struct irdma_sc_pd {
398 struct irdma_sc_dev *dev;
399 u32 pd_id;
400 int abi_ver;
401 };
402
403 struct irdma_cqp_quanta {
404 __le64 elem[IRDMA_CQP_WQE_SIZE];
405 };
406
407 struct irdma_ooo_cqp_op {
408 struct list_head list_entry;
409 u64 scratch;
410 u32 def_info;
411 u32 sw_def_info;
412 u32 wqe_idx;
413 bool deferred:1;
414 };
415
416 struct irdma_sc_cqp {
417 spinlock_t ooo_list_lock; /* protects list of pending completions */
418 struct list_head ooo_avail;
419 struct list_head ooo_pnd;
420 u32 last_def_cmpl_ticket;
421 u32 sw_def_cmpl_ticket;
422 u32 size;
423 u64 sq_pa;
424 u64 host_ctx_pa;
425 void *back_cqp;
426 struct irdma_sc_dev *dev;
427 int (*process_cqp_sds)(struct irdma_sc_dev *dev,
428 struct irdma_update_sds_info *info);
429 struct irdma_dma_mem sdbuf;
430 struct irdma_ring sq_ring;
431 struct irdma_cqp_quanta *sq_base;
432 struct irdma_dcqcn_cc_params dcqcn_params;
433 __le64 *host_ctx;
434 u64 *scratch_array;
435 u64 requested_ops;
436 atomic64_t completed_ops;
437 struct irdma_ooo_cqp_op *ooo_op_array;
438 u32 cqp_id;
439 u32 sq_size;
440 u32 pe_en_vf_cnt;
441 u32 hw_sq_size;
442 u16 hw_maj_ver;
443 u16 hw_min_ver;
444 u8 struct_ver;
445 u8 polarity;
446 u8 hmc_profile;
447 u8 ena_vf_count;
448 u8 timeout_count;
449 u8 ceqs_per_vf;
450 u8 ooisc_blksize;
451 u8 rrsp_blksize;
452 u8 q1_blksize;
453 u8 xmit_blksize;
454 u8 ts_override;
455 u8 ts_shift;
456 u8 en_fine_grained_timers;
457 u8 blksizes_valid;
458 bool en_datacenter_tcp:1;
459 bool disable_packed:1;
460 bool rocev2_rto_policy:1;
461 enum irdma_protocol_used protocol_used;
462 };
463
464 struct irdma_sc_aeq {
465 u32 size;
466 u64 aeq_elem_pa;
467 struct irdma_sc_dev *dev;
468 struct irdma_sc_aeqe *aeqe_base;
469 void *pbl_list;
470 u32 elem_cnt;
471 struct irdma_ring aeq_ring;
472 u8 pbl_chunk_size;
473 u32 first_pm_pbl_idx;
474 u32 msix_idx;
475 u8 polarity;
476 bool virtual_map:1;
477 bool pasid_valid:1;
478 u32 pasid;
479 };
480
481 struct irdma_sc_ceq {
482 u32 size;
483 u64 ceq_elem_pa;
484 struct irdma_sc_dev *dev;
485 struct irdma_ceqe *ceqe_base;
486 void *pbl_list;
487 u32 ceq_id;
488 u32 elem_cnt;
489 struct irdma_ring ceq_ring;
490 u8 pbl_chunk_size;
491 u8 tph_val;
492 u32 first_pm_pbl_idx;
493 u8 polarity;
494 u16 vsi_idx;
495 struct irdma_sc_cq **reg_cq;
496 u32 reg_cq_size;
497 spinlock_t req_cq_lock; /* protect access to reg_cq array */
498 bool virtual_map:1;
499 bool tph_en:1;
500 bool itr_no_expire:1;
501 bool pasid_valid:1;
502 u32 pasid;
503 };
504
505 struct irdma_sc_cq {
506 struct irdma_cq_uk cq_uk;
507 u64 cq_pa;
508 u64 shadow_area_pa;
509 struct irdma_sc_dev *dev;
510 u16 vsi_idx;
511 struct irdma_sc_vsi *vsi;
512 void *pbl_list;
513 void *back_cq;
514 u32 ceq_id;
515 u32 shadow_read_threshold;
516 u8 pbl_chunk_size;
517 u8 cq_type;
518 u8 tph_val;
519 u32 first_pm_pbl_idx;
520 bool ceqe_mask:1;
521 bool virtual_map:1;
522 bool check_overflow:1;
523 bool ceq_id_valid:1;
524 bool tph_en;
525 };
526
527 struct irdma_sc_qp {
528 struct irdma_qp_uk qp_uk;
529 u64 sq_pa;
530 u64 rq_pa;
531 u64 hw_host_ctx_pa;
532 u64 shadow_area_pa;
533 u64 q2_pa;
534 struct irdma_sc_dev *dev;
535 struct irdma_sc_vsi *vsi;
536 struct irdma_sc_pd *pd;
537 __le64 *hw_host_ctx;
538 void *llp_stream_handle;
539 struct irdma_pfpdu pfpdu;
540 u32 ieq_qp;
541 u8 *q2_buf;
542 u64 qp_compl_ctx;
543 u32 push_idx;
544 u16 qs_handle;
545 u16 push_offset;
546 u8 flush_wqes_count;
547 u8 sq_tph_val;
548 u8 rq_tph_val;
549 u8 qp_state;
550 u8 hw_sq_size;
551 u8 hw_rq_size;
552 u8 src_mac_addr_idx;
553 bool on_qoslist:1;
554 bool ieq_pass_thru:1;
555 bool sq_tph_en:1;
556 bool rq_tph_en:1;
557 bool rcv_tph_en:1;
558 bool xmit_tph_en:1;
559 bool virtual_map:1;
560 bool flush_sq:1;
561 bool flush_rq:1;
562 bool err_sq_idx_valid:1;
563 bool err_rq_idx_valid:1;
564 u32 err_sq_idx;
565 u32 err_rq_idx;
566 bool sq_flush_code:1;
567 bool rq_flush_code:1;
568 u32 pkt_limit;
569 enum irdma_flush_opcode flush_code;
570 enum irdma_qp_event_type event_type;
571 u8 term_flags;
572 u8 user_pri;
573 struct list_head list;
574 };
575
576 struct irdma_stats_inst_info {
577 bool use_hmc_fcn_index;
578 u8 hmc_fn_id;
579 u16 stats_idx;
580 };
581
582 struct irdma_up_info {
583 u8 map[8];
584 u8 cnp_up_override;
585 u16 hmc_fcn_idx;
586 bool use_vlan:1;
587 bool use_cnp_up_override:1;
588 };
589
590 #define IRDMA_MAX_WS_NODES 0x3FF
591 #define IRDMA_WS_NODE_INVALID 0xFFFF
592
593 struct irdma_ws_node_info {
594 u16 id;
595 u16 vsi;
596 u16 parent_id;
597 u16 qs_handle;
598 bool type_leaf:1;
599 bool enable:1;
600 u8 prio_type;
601 u8 tc;
602 u8 weight;
603 };
604
605 struct irdma_hmc_fpm_misc {
606 u32 max_ceqs;
607 u32 max_sds;
608 u32 loc_mem_pages;
609 u8 ird;
610 u32 xf_block_size;
611 u32 q1_block_size;
612 u32 ht_multiplier;
613 u32 timer_bucket;
614 u32 rrf_block_size;
615 u32 ooiscf_block_size;
616 };
617
618 #define IRDMA_VCHNL_MAX_MSG_SIZE 512
619 #define IRDMA_LEAF_DEFAULT_REL_BW 64
620 #define IRDMA_PARENT_DEFAULT_REL_BW 1
621
622 struct irdma_qos {
623 struct list_head qplist;
624 struct mutex qos_mutex; /* protect QoS attributes per QoS level */
625 u64 lan_qos_handle;
626 u32 l2_sched_node_id;
627 u16 qs_handle;
628 u8 traffic_class;
629 u8 rel_bw;
630 u8 prio_type;
631 bool valid;
632 };
633
634 #define IRDMA_INVALID_STATS_IDX 0xff
635 struct irdma_sc_vsi {
636 u16 vsi_idx;
637 struct irdma_sc_dev *dev;
638 void *back_vsi;
639 u32 ilq_count;
640 struct irdma_virt_mem ilq_mem;
641 struct irdma_puda_rsrc *ilq;
642 u32 ieq_count;
643 struct irdma_virt_mem ieq_mem;
644 struct irdma_puda_rsrc *ieq;
645 u32 exception_lan_q;
646 u16 mtu;
647 u16 vm_id;
648 enum irdma_vm_vf_type vm_vf_type;
649 bool stats_inst_alloc:1;
650 bool tc_change_pending:1;
651 struct irdma_vsi_pestat *pestat;
652 atomic_t qp_suspend_reqs;
653 int (*register_qset)(struct irdma_sc_vsi *vsi,
654 struct irdma_ws_node *tc_node);
655 void (*unregister_qset)(struct irdma_sc_vsi *vsi,
656 struct irdma_ws_node *tc_node);
657 u8 qos_rel_bw;
658 u8 qos_prio_type;
659 u8 stats_idx;
660 u8 dscp_map[DSCP_MAX];
661 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
662 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
663 bool dscp_mode:1;
664 };
665
666 struct irdma_sc_dev {
667 struct list_head cqp_cmd_head; /* head of the CQP command list */
668 spinlock_t cqp_lock; /* protect CQP list access */
669 bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1];
670 struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT];
671 u64 fpm_query_buf_pa;
672 u64 fpm_commit_buf_pa;
673 __le64 *fpm_query_buf;
674 __le64 *fpm_commit_buf;
675 struct irdma_hw *hw;
676 u8 __iomem *db_addr;
677 u32 __iomem *wqe_alloc_db;
678 u32 __iomem *cq_arm_db;
679 u32 __iomem *aeq_alloc_db;
680 u32 __iomem *cqp_db;
681 u32 __iomem *cq_ack_db;
682 u32 __iomem *ceq_itr_mask_db;
683 u32 __iomem *aeq_itr_mask_db;
684 u32 __iomem *hw_regs[IRDMA_MAX_REGS];
685 u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
686 u64 hw_masks[IRDMA_MAX_MASKS];
687 u64 hw_shifts[IRDMA_MAX_SHIFTS];
688 const struct irdma_hw_stat_map *hw_stats_map;
689 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1];
690 u64 feature_info[IRDMA_MAX_FEATURES];
691 u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS];
692 struct irdma_hw_attrs hw_attrs;
693 struct irdma_hmc_info *hmc_info;
694 struct irdma_vchnl_rdma_caps vc_caps;
695 u8 vc_recv_buf[IRDMA_VCHNL_MAX_MSG_SIZE];
696 u16 vc_recv_len;
697 struct irdma_sc_cqp *cqp;
698 struct irdma_sc_aeq *aeq;
699 struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT];
700 struct irdma_sc_cq *ccq;
701 const struct irdma_irq_ops *irq_ops;
702 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY];
703 struct irdma_hmc_fpm_misc hmc_fpm_misc;
704 struct irdma_ws_node *ws_tree_root;
705 struct mutex ws_mutex; /* ws tree mutex */
706 u32 vchnl_ver;
707 u16 num_vfs;
708 u16 hmc_fn_id;
709 u16 vf_id;
710 bool privileged:1;
711 bool vchnl_up:1;
712 bool ceq_valid:1;
713 bool is_pf:1;
714 u8 protocol_used;
715 struct mutex vchnl_mutex; /* mutex to synchronize RDMA virtual channel messages */
716 u8 pci_rev;
717 int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
718 void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri);
719 void (*ws_reset)(struct irdma_sc_vsi *vsi);
720 };
721
722 struct irdma_modify_cq_info {
723 u64 cq_pa;
724 struct irdma_cqe *cq_base;
725 u32 cq_size;
726 u32 shadow_read_threshold;
727 u8 pbl_chunk_size;
728 u32 first_pm_pbl_idx;
729 bool virtual_map:1;
730 bool check_overflow;
731 bool cq_resize:1;
732 };
733
734 struct irdma_srq_init_info {
735 struct irdma_sc_pd *pd;
736 struct irdma_sc_vsi *vsi;
737 u64 srq_pa;
738 u64 shadow_area_pa;
739 u32 first_pm_pbl_idx;
740 u32 pasid;
741 u32 srq_size;
742 u16 srq_limit;
743 u8 pasid_valid;
744 u8 wqe_size;
745 u8 leaf_pbl_size;
746 u8 virtual_map;
747 u8 tph_en;
748 u8 arm_limit_event;
749 u8 tph_value;
750 u8 pbl_chunk_size;
751 struct irdma_srq_uk_init_info srq_uk_init_info;
752 };
753
754 struct irdma_sc_srq {
755 struct irdma_sc_dev *dev;
756 struct irdma_sc_vsi *vsi;
757 struct irdma_sc_pd *pd;
758 struct irdma_srq_uk srq_uk;
759 void *back_srq;
760 u64 srq_pa;
761 u64 shadow_area_pa;
762 u32 first_pm_pbl_idx;
763 u32 pasid;
764 u32 hw_srq_size;
765 u16 srq_limit;
766 u8 pasid_valid;
767 u8 leaf_pbl_size;
768 u8 virtual_map;
769 u8 tph_en;
770 u8 arm_limit_event;
771 u8 tph_val;
772 };
773
774 struct irdma_modify_srq_info {
775 u16 srq_limit;
776 u8 arm_limit_event;
777 };
778
779 struct irdma_create_qp_info {
780 bool ord_valid:1;
781 bool tcp_ctx_valid:1;
782 bool cq_num_valid:1;
783 bool arp_cache_idx_valid:1;
784 bool mac_valid:1;
785 bool force_lpb;
786 u8 next_iwarp_state;
787 };
788
789 struct irdma_modify_qp_info {
790 u64 rx_win0;
791 u64 rx_win1;
792 u16 new_mss;
793 u8 next_iwarp_state;
794 u8 curr_iwarp_state;
795 u8 termlen;
796 bool ord_valid:1;
797 bool tcp_ctx_valid:1;
798 bool udp_ctx_valid:1;
799 bool cq_num_valid:1;
800 bool arp_cache_idx_valid:1;
801 bool reset_tcp_conn:1;
802 bool remove_hash_idx:1;
803 bool dont_send_term:1;
804 bool dont_send_fin:1;
805 bool cached_var_valid:1;
806 bool mss_change:1;
807 bool force_lpb:1;
808 bool mac_valid:1;
809 };
810
811 struct irdma_ccq_cqe_info {
812 struct irdma_sc_cqp *cqp;
813 u64 scratch;
814 u32 op_ret_val;
815 u16 maj_err_code;
816 u16 min_err_code;
817 u8 op_code;
818 bool error:1;
819 bool pending:1;
820 };
821
822 struct irdma_dcb_app_info {
823 u8 priority;
824 u8 selector;
825 u16 prot_id;
826 };
827
828 struct irdma_qos_tc_info {
829 u64 tc_ctx;
830 u8 rel_bw;
831 u8 prio_type;
832 u8 egress_virt_up;
833 u8 ingress_virt_up;
834 };
835
836 struct irdma_l2params {
837 struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY];
838 struct irdma_dcb_app_info apps[IRDMA_MAX_APPS];
839 u32 num_apps;
840 u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY];
841 u16 mtu;
842 u8 up2tc[IRDMA_MAX_USER_PRIORITY];
843 u8 dscp_map[DSCP_MAX];
844 u8 num_tc;
845 u8 vsi_rel_bw;
846 u8 vsi_prio_type;
847 bool mtu_changed:1;
848 bool tc_changed:1;
849 bool dscp_mode:1;
850 };
851
852 struct irdma_vsi_init_info {
853 struct irdma_sc_dev *dev;
854 void *back_vsi;
855 struct irdma_l2params *params;
856 u16 exception_lan_q;
857 u16 pf_data_vsi_num;
858 enum irdma_vm_vf_type vm_vf_type;
859 u16 vm_id;
860 int (*register_qset)(struct irdma_sc_vsi *vsi,
861 struct irdma_ws_node *tc_node);
862 void (*unregister_qset)(struct irdma_sc_vsi *vsi,
863 struct irdma_ws_node *tc_node);
864 };
865
866 struct irdma_vsi_stats_info {
867 struct irdma_vsi_pestat *pestat;
868 u16 fcn_id;
869 bool alloc_stats_inst;
870 };
871
872 struct irdma_device_init_info {
873 u64 fpm_query_buf_pa;
874 u64 fpm_commit_buf_pa;
875 __le64 *fpm_query_buf;
876 __le64 *fpm_commit_buf;
877 struct irdma_hw *hw;
878 void __iomem *bar0;
879 enum irdma_protocol_used protocol_used;
880 u16 hmc_fn_id;
881 };
882
883 struct irdma_ceq_init_info {
884 u64 ceqe_pa;
885 struct irdma_sc_dev *dev;
886 u64 *ceqe_base;
887 void *pbl_list;
888 u32 elem_cnt;
889 u32 ceq_id;
890 bool virtual_map:1;
891 bool tph_en:1;
892 bool itr_no_expire:1;
893 u8 pbl_chunk_size;
894 u8 tph_val;
895 u16 vsi_idx;
896 u32 first_pm_pbl_idx;
897 struct irdma_sc_cq **reg_cq;
898 u32 reg_cq_idx;
899 };
900
901 struct irdma_aeq_init_info {
902 u64 aeq_elem_pa;
903 struct irdma_sc_dev *dev;
904 u32 *aeqe_base;
905 void *pbl_list;
906 u32 elem_cnt;
907 bool virtual_map;
908 u8 pbl_chunk_size;
909 u32 first_pm_pbl_idx;
910 u32 msix_idx;
911 };
912
913 struct irdma_ccq_init_info {
914 u64 cq_pa;
915 u64 shadow_area_pa;
916 struct irdma_sc_dev *dev;
917 struct irdma_cqe *cq_base;
918 __le64 *shadow_area;
919 void *pbl_list;
920 u32 num_elem;
921 u32 ceq_id;
922 u32 shadow_read_threshold;
923 bool ceqe_mask:1;
924 bool ceq_id_valid:1;
925 bool avoid_mem_cflct:1;
926 bool virtual_map:1;
927 bool tph_en:1;
928 u8 tph_val;
929 u8 pbl_chunk_size;
930 u32 first_pm_pbl_idx;
931 struct irdma_sc_vsi *vsi;
932 };
933
934 struct irdma_udp_offload_info {
935 bool ipv4:1;
936 bool insert_vlan_tag:1;
937 u8 ttl;
938 u8 tos;
939 u16 src_port;
940 u16 dst_port;
941 u32 dest_ip_addr[4];
942 u32 snd_mss;
943 u16 vlan_tag;
944 u16 arp_idx;
945 u32 flow_label;
946 u8 udp_state;
947 u32 psn_nxt;
948 u32 lsn;
949 u32 epsn;
950 u32 psn_max;
951 u32 psn_una;
952 u32 local_ipaddr[4];
953 u32 cwnd;
954 u8 rexmit_thresh;
955 u8 rnr_nak_thresh;
956 u8 rnr_nak_tmr;
957 u8 min_rnr_timer;
958 };
959
960 struct irdma_roce_offload_info {
961 u16 p_key;
962 u16 err_rq_idx;
963 u32 qkey;
964 u32 dest_qp;
965 u8 roce_tver;
966 u8 ack_credits;
967 u8 err_rq_idx_valid;
968 u32 pd_id;
969 u16 ord_size;
970 u16 ird_size;
971 bool is_qp1:1;
972 bool udprivcq_en:1;
973 bool dcqcn_en:1;
974 bool rcv_no_icrc:1;
975 bool wr_rdresp_en:1;
976 bool bind_en:1;
977 bool fast_reg_en:1;
978 bool priv_mode_en:1;
979 bool rd_en:1;
980 bool timely_en:1;
981 bool dctcp_en:1;
982 bool fw_cc_enable:1;
983 bool use_stats_inst:1;
984 u8 local_ack_timeout;
985 u16 t_high;
986 u16 t_low;
987 u8 last_byte_sent;
988 u8 mac_addr[ETH_ALEN];
989 u8 rtomin;
990 };
991
992 struct irdma_iwarp_offload_info {
993 u16 rcv_mark_offset;
994 u16 snd_mark_offset;
995 u8 ddp_ver;
996 u8 rdmap_ver;
997 u8 iwarp_mode;
998 u16 err_rq_idx;
999 u32 pd_id;
1000 u16 ord_size;
1001 u16 ird_size;
1002 bool ib_rd_en:1;
1003 bool align_hdrs:1;
1004 bool rcv_no_mpa_crc:1;
1005 bool err_rq_idx_valid:1;
1006 bool snd_mark_en:1;
1007 bool rcv_mark_en:1;
1008 bool wr_rdresp_en:1;
1009 bool bind_en:1;
1010 bool fast_reg_en:1;
1011 bool priv_mode_en:1;
1012 bool rd_en:1;
1013 bool timely_en:1;
1014 bool use_stats_inst:1;
1015 bool ecn_en:1;
1016 bool dctcp_en:1;
1017 u16 t_high;
1018 u16 t_low;
1019 u8 last_byte_sent;
1020 u8 mac_addr[ETH_ALEN];
1021 u8 rtomin;
1022 };
1023
1024 struct irdma_tcp_offload_info {
1025 bool ipv4:1;
1026 bool no_nagle:1;
1027 bool insert_vlan_tag:1;
1028 bool time_stamp:1;
1029 bool drop_ooo_seg:1;
1030 bool avoid_stretch_ack:1;
1031 bool wscale:1;
1032 bool ignore_tcp_opt:1;
1033 bool ignore_tcp_uns_opt:1;
1034 u8 cwnd_inc_limit;
1035 u8 dup_ack_thresh;
1036 u8 ttl;
1037 u8 src_mac_addr_idx;
1038 u8 tos;
1039 u16 src_port;
1040 u16 dst_port;
1041 u32 dest_ip_addr[4];
1042 //u32 dest_ip_addr0;
1043 //u32 dest_ip_addr1;
1044 //u32 dest_ip_addr2;
1045 //u32 dest_ip_addr3;
1046 u32 snd_mss;
1047 u16 syn_rst_handling;
1048 u16 vlan_tag;
1049 u16 arp_idx;
1050 u32 flow_label;
1051 u8 tcp_state;
1052 u8 snd_wscale;
1053 u8 rcv_wscale;
1054 u32 time_stamp_recent;
1055 u32 time_stamp_age;
1056 u32 snd_nxt;
1057 u32 snd_wnd;
1058 u32 rcv_nxt;
1059 u32 rcv_wnd;
1060 u32 snd_max;
1061 u32 snd_una;
1062 u32 srtt;
1063 u32 rtt_var;
1064 u32 ss_thresh;
1065 u32 cwnd;
1066 u32 snd_wl1;
1067 u32 snd_wl2;
1068 u32 max_snd_window;
1069 u8 rexmit_thresh;
1070 u32 local_ipaddr[4];
1071 };
1072
1073 struct irdma_qp_host_ctx_info {
1074 u64 qp_compl_ctx;
1075 union {
1076 struct irdma_tcp_offload_info *tcp_info;
1077 struct irdma_udp_offload_info *udp_info;
1078 };
1079 union {
1080 struct irdma_iwarp_offload_info *iwarp_info;
1081 struct irdma_roce_offload_info *roce_info;
1082 };
1083 u32 send_cq_num;
1084 u32 rcv_cq_num;
1085 u32 srq_id;
1086 u32 rem_endpoint_idx;
1087 u16 stats_idx;
1088 bool remote_atomics_en:1;
1089 bool srq_valid:1;
1090 bool tcp_info_valid:1;
1091 bool iwarp_info_valid:1;
1092 bool stats_idx_valid:1;
1093 u8 user_pri;
1094 };
1095
1096 struct irdma_aeqe_info {
1097 u64 compl_ctx;
1098 u32 qp_cq_id;
1099 u32 def_info; /* only valid for DEF_CMPL */
1100 u16 ae_id;
1101 u16 wqe_idx;
1102 u8 tcp_state;
1103 u8 iwarp_state;
1104 bool qp:1;
1105 bool cq:1;
1106 bool sq:1;
1107 bool rq:1;
1108 bool srq:1;
1109 bool in_rdrsp_wr:1;
1110 bool out_rdrsp:1;
1111 bool aeqe_overflow:1;
1112 bool err_rq_idx_valid:1;
1113 u8 q2_data_written;
1114 u8 ae_src;
1115 };
1116
1117 struct irdma_allocate_stag_info {
1118 u64 total_len;
1119 u64 first_pm_pbl_idx;
1120 u32 chunk_size;
1121 u32 stag_idx;
1122 u32 page_size;
1123 u32 pd_id;
1124 u16 access_rights;
1125 bool remote_access:1;
1126 bool use_hmc_fcn_index:1;
1127 bool use_pf_rid:1;
1128 bool all_memory:1;
1129 bool remote_atomics_en:1;
1130 u16 hmc_fcn_index;
1131 };
1132
1133 struct irdma_mw_alloc_info {
1134 u32 mw_stag_index;
1135 u32 page_size;
1136 u32 pd_id;
1137 bool remote_access:1;
1138 bool mw_wide:1;
1139 bool mw1_bind_dont_vldt_key:1;
1140 };
1141
1142 struct irdma_reg_ns_stag_info {
1143 u64 reg_addr_pa;
1144 u64 va;
1145 u64 total_len;
1146 u32 page_size;
1147 u32 chunk_size;
1148 u32 first_pm_pbl_index;
1149 enum irdma_addressing_type addr_type;
1150 irdma_stag_index stag_idx;
1151 u16 access_rights;
1152 u32 pd_id;
1153 irdma_stag_key stag_key;
1154 bool use_hmc_fcn_index:1;
1155 u8 hmc_fcn_index;
1156 bool use_pf_rid:1;
1157 bool all_memory:1;
1158 bool remote_atomics_en:1;
1159 };
1160
1161 struct irdma_fast_reg_stag_info {
1162 u64 wr_id;
1163 u64 reg_addr_pa;
1164 u64 fbo;
1165 void *va;
1166 u64 total_len;
1167 u32 page_size;
1168 u32 chunk_size;
1169 u32 first_pm_pbl_index;
1170 enum irdma_addressing_type addr_type;
1171 irdma_stag_index stag_idx;
1172 u16 access_rights;
1173 u32 pd_id;
1174 irdma_stag_key stag_key;
1175 bool local_fence:1;
1176 bool read_fence:1;
1177 bool signaled:1;
1178 bool use_hmc_fcn_index:1;
1179 u8 hmc_fcn_index;
1180 bool use_pf_rid:1;
1181 bool defer_flag:1;
1182 bool remote_atomics_en:1;
1183 };
1184
1185 struct irdma_dealloc_stag_info {
1186 u32 stag_idx;
1187 u32 pd_id;
1188 bool mr:1;
1189 bool dealloc_pbl:1;
1190 };
1191
1192 struct irdma_register_shared_stag {
1193 u64 va;
1194 enum irdma_addressing_type addr_type;
1195 irdma_stag_index new_stag_idx;
1196 irdma_stag_index parent_stag_idx;
1197 u32 access_rights;
1198 u32 pd_id;
1199 u32 page_size;
1200 irdma_stag_key new_stag_key;
1201 };
1202
1203 struct irdma_qp_init_info {
1204 struct irdma_qp_uk_init_info qp_uk_init_info;
1205 struct irdma_sc_pd *pd;
1206 struct irdma_sc_vsi *vsi;
1207 __le64 *host_ctx;
1208 u8 *q2;
1209 u64 sq_pa;
1210 u64 rq_pa;
1211 u64 host_ctx_pa;
1212 u64 q2_pa;
1213 u64 shadow_area_pa;
1214 u8 sq_tph_val;
1215 u8 rq_tph_val;
1216 bool sq_tph_en:1;
1217 bool rq_tph_en:1;
1218 bool rcv_tph_en:1;
1219 bool xmit_tph_en:1;
1220 bool virtual_map:1;
1221 };
1222
1223 struct irdma_cq_init_info {
1224 struct irdma_sc_dev *dev;
1225 u64 cq_base_pa;
1226 u64 shadow_area_pa;
1227 u32 ceq_id;
1228 u32 shadow_read_threshold;
1229 u8 pbl_chunk_size;
1230 u32 first_pm_pbl_idx;
1231 bool virtual_map:1;
1232 bool ceqe_mask:1;
1233 bool ceq_id_valid:1;
1234 bool tph_en:1;
1235 u8 tph_val;
1236 u8 type;
1237 struct irdma_cq_uk_init_info cq_uk_init_info;
1238 struct irdma_sc_vsi *vsi;
1239 };
1240
1241 struct irdma_upload_context_info {
1242 u64 buf_pa;
1243 u32 qp_id;
1244 u8 qp_type;
1245 bool freeze_qp:1;
1246 bool raw_format:1;
1247 };
1248
1249 struct irdma_local_mac_entry_info {
1250 u8 mac_addr[6];
1251 u16 entry_idx;
1252 };
1253
1254 struct irdma_add_arp_cache_entry_info {
1255 u8 mac_addr[ETH_ALEN];
1256 u32 reach_max;
1257 u16 arp_index;
1258 bool permanent;
1259 };
1260
1261 struct irdma_apbvt_info {
1262 u16 port;
1263 bool add;
1264 };
1265
1266 struct irdma_qhash_table_info {
1267 struct irdma_sc_vsi *vsi;
1268 enum irdma_quad_hash_manage_type manage;
1269 enum irdma_quad_entry_type entry_type;
1270 bool vlan_valid:1;
1271 bool ipv4_valid:1;
1272 u8 mac_addr[ETH_ALEN];
1273 u16 vlan_id;
1274 u8 user_pri;
1275 u32 qp_num;
1276 u32 dest_ip[4];
1277 u32 src_ip[4];
1278 u16 dest_port;
1279 u16 src_port;
1280 };
1281
1282 struct irdma_cqp_manage_push_page_info {
1283 u32 push_idx;
1284 u16 qs_handle;
1285 u8 free_page;
1286 u8 push_page_type;
1287 };
1288
1289 struct irdma_qp_flush_info {
1290 u32 err_sq_idx;
1291 u32 err_rq_idx;
1292 u16 sq_minor_code;
1293 u16 sq_major_code;
1294 u16 rq_minor_code;
1295 u16 rq_major_code;
1296 u16 ae_code;
1297 u8 ae_src;
1298 bool sq:1;
1299 bool rq:1;
1300 bool userflushcode:1;
1301 bool generate_ae:1;
1302 bool err_sq_idx_valid:1;
1303 bool err_rq_idx_valid:1;
1304 };
1305
1306 struct irdma_gen_ae_info {
1307 u16 ae_code;
1308 u8 ae_src;
1309 };
1310
1311 struct irdma_cqp_timeout {
1312 u64 compl_cqp_cmds;
1313 u32 count;
1314 };
1315
1316 struct irdma_irq_ops {
1317 void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable);
1318 void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
1319 bool enable);
1320 void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx);
1321 void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx);
1322 };
1323
1324 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq);
1325 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
1326 bool check_overflow, bool post_sq);
1327 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq);
1328 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
1329 struct irdma_ccq_cqe_info *info);
1330 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq,
1331 struct irdma_ccq_init_info *info);
1332
1333 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch);
1334 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq);
1335
1336 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq);
1337 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
1338 struct irdma_ceq_init_info *info);
1339 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq);
1340 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq);
1341
1342 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
1343 struct irdma_aeq_init_info *info);
1344 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
1345 struct irdma_aeqe_info *info);
1346 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count);
1347
1348 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
1349 int abi_ver);
1350 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable);
1351 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout,
1352 struct irdma_sc_dev *dev);
1353 void irdma_sc_cqp_def_cmpl_ae_handler(struct irdma_sc_dev *dev,
1354 struct irdma_aeqe_info *info,
1355 bool first, u64 *scratch,
1356 u32 *sw_def_info);
1357 u64 irdma_sc_cqp_cleanup_handler(struct irdma_sc_dev *dev);
1358 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err);
1359 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp);
1360 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
1361 struct irdma_cqp_init_info *info);
1362 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
1363 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode,
1364 struct irdma_ccq_cqe_info *cmpl_info);
1365 int irdma_sc_fast_register(struct irdma_sc_qp *qp,
1366 struct irdma_fast_reg_stag_info *info, bool post_sq);
1367 int irdma_sc_qp_create(struct irdma_sc_qp *qp,
1368 struct irdma_create_qp_info *info, u64 scratch,
1369 bool post_sq);
1370 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
1371 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
1372 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
1373 struct irdma_qp_flush_info *info, u64 scratch,
1374 bool post_sq);
1375 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
1376 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
1377 struct irdma_modify_qp_info *info, u64 scratch,
1378 bool post_sq);
1379 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1380 irdma_stag stag);
1381
1382 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
1383 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1384 struct irdma_qp_host_ctx_info *info);
1385 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
1386 struct irdma_qp_host_ctx_info *info);
1387 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq);
1388 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info);
1389 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info);
1390 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1391 u8 hmc_fn_id, bool post_sq,
1392 bool poll_registers);
1393 int irdma_sc_srq_init(struct irdma_sc_srq *srq,
1394 struct irdma_srq_init_info *info);
1395
1396 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1397 struct cqp_info {
1398 union {
1399 struct {
1400 struct irdma_sc_qp *qp;
1401 struct irdma_create_qp_info info;
1402 u64 scratch;
1403 } qp_create;
1404
1405 struct {
1406 struct irdma_sc_qp *qp;
1407 struct irdma_modify_qp_info info;
1408 u64 scratch;
1409 } qp_modify;
1410
1411 struct {
1412 struct irdma_sc_qp *qp;
1413 u64 scratch;
1414 bool remove_hash_idx;
1415 bool ignore_mw_bnd;
1416 } qp_destroy;
1417
1418 struct {
1419 struct irdma_sc_cq *cq;
1420 u64 scratch;
1421 bool check_overflow;
1422 } cq_create;
1423
1424 struct {
1425 struct irdma_sc_cq *cq;
1426 struct irdma_modify_cq_info info;
1427 u64 scratch;
1428 } cq_modify;
1429
1430 struct {
1431 struct irdma_sc_cq *cq;
1432 u64 scratch;
1433 } cq_destroy;
1434
1435 struct {
1436 struct irdma_sc_dev *dev;
1437 struct irdma_allocate_stag_info info;
1438 u64 scratch;
1439 } alloc_stag;
1440
1441 struct {
1442 struct irdma_sc_dev *dev;
1443 struct irdma_mw_alloc_info info;
1444 u64 scratch;
1445 } mw_alloc;
1446
1447 struct {
1448 struct irdma_sc_dev *dev;
1449 struct irdma_reg_ns_stag_info info;
1450 u64 scratch;
1451 } mr_reg_non_shared;
1452
1453 struct {
1454 struct irdma_sc_dev *dev;
1455 struct irdma_dealloc_stag_info info;
1456 u64 scratch;
1457 } dealloc_stag;
1458
1459 struct {
1460 struct irdma_sc_cqp *cqp;
1461 struct irdma_add_arp_cache_entry_info info;
1462 u64 scratch;
1463 } add_arp_cache_entry;
1464
1465 struct {
1466 struct irdma_sc_cqp *cqp;
1467 u64 scratch;
1468 u16 arp_index;
1469 } del_arp_cache_entry;
1470
1471 struct {
1472 struct irdma_sc_cqp *cqp;
1473 struct irdma_local_mac_entry_info info;
1474 u64 scratch;
1475 } add_local_mac_entry;
1476
1477 struct {
1478 struct irdma_sc_cqp *cqp;
1479 u64 scratch;
1480 u8 entry_idx;
1481 u8 ignore_ref_count;
1482 } del_local_mac_entry;
1483
1484 struct {
1485 struct irdma_sc_cqp *cqp;
1486 u64 scratch;
1487 } alloc_local_mac_entry;
1488
1489 struct {
1490 struct irdma_sc_cqp *cqp;
1491 struct irdma_cqp_manage_push_page_info info;
1492 u64 scratch;
1493 } manage_push_page;
1494
1495 struct {
1496 struct irdma_sc_dev *dev;
1497 struct irdma_upload_context_info info;
1498 u64 scratch;
1499 } qp_upload_context;
1500
1501 struct {
1502 struct irdma_sc_dev *dev;
1503 struct irdma_hmc_fcn_info info;
1504 u64 scratch;
1505 } manage_hmc_pm;
1506
1507 struct {
1508 struct irdma_sc_ceq *ceq;
1509 u64 scratch;
1510 } ceq_create;
1511
1512 struct {
1513 struct irdma_sc_ceq *ceq;
1514 u64 scratch;
1515 } ceq_destroy;
1516
1517 struct {
1518 struct irdma_sc_aeq *aeq;
1519 u64 scratch;
1520 } aeq_create;
1521
1522 struct {
1523 struct irdma_sc_aeq *aeq;
1524 u64 scratch;
1525 } aeq_destroy;
1526
1527 struct {
1528 struct irdma_sc_qp *qp;
1529 struct irdma_qp_flush_info info;
1530 u64 scratch;
1531 } qp_flush_wqes;
1532
1533 struct {
1534 struct irdma_sc_qp *qp;
1535 struct irdma_gen_ae_info info;
1536 u64 scratch;
1537 } gen_ae;
1538
1539 struct {
1540 struct irdma_sc_cqp *cqp;
1541 void *fpm_val_va;
1542 u64 fpm_val_pa;
1543 u8 hmc_fn_id;
1544 u64 scratch;
1545 } query_fpm_val;
1546
1547 struct {
1548 struct irdma_sc_cqp *cqp;
1549 void *fpm_val_va;
1550 u64 fpm_val_pa;
1551 u8 hmc_fn_id;
1552 u64 scratch;
1553 } commit_fpm_val;
1554
1555 struct {
1556 struct irdma_sc_cqp *cqp;
1557 struct irdma_apbvt_info info;
1558 u64 scratch;
1559 } manage_apbvt_entry;
1560
1561 struct {
1562 struct irdma_sc_cqp *cqp;
1563 struct irdma_qhash_table_info info;
1564 u64 scratch;
1565 } manage_qhash_table_entry;
1566
1567 struct {
1568 struct irdma_sc_dev *dev;
1569 struct irdma_update_sds_info info;
1570 u64 scratch;
1571 } update_pe_sds;
1572
1573 struct {
1574 struct irdma_sc_cqp *cqp;
1575 struct irdma_sc_qp *qp;
1576 u64 scratch;
1577 } suspend_resume;
1578
1579 struct {
1580 struct irdma_sc_cqp *cqp;
1581 struct irdma_ah_info info;
1582 u64 scratch;
1583 } ah_create;
1584
1585 struct {
1586 struct irdma_sc_cqp *cqp;
1587 struct irdma_ah_info info;
1588 u64 scratch;
1589 } ah_destroy;
1590
1591 struct {
1592 struct irdma_sc_cqp *cqp;
1593 struct irdma_mcast_grp_info info;
1594 u64 scratch;
1595 } mc_create;
1596
1597 struct {
1598 struct irdma_sc_cqp *cqp;
1599 struct irdma_mcast_grp_info info;
1600 u64 scratch;
1601 } mc_destroy;
1602
1603 struct {
1604 struct irdma_sc_cqp *cqp;
1605 struct irdma_mcast_grp_info info;
1606 u64 scratch;
1607 } mc_modify;
1608
1609 struct {
1610 struct irdma_sc_cqp *cqp;
1611 struct irdma_stats_inst_info info;
1612 u64 scratch;
1613 } stats_manage;
1614
1615 struct {
1616 struct irdma_sc_cqp *cqp;
1617 struct irdma_stats_gather_info info;
1618 u64 scratch;
1619 } stats_gather;
1620
1621 struct {
1622 struct irdma_sc_cqp *cqp;
1623 struct irdma_ws_node_info info;
1624 u64 scratch;
1625 } ws_node;
1626
1627 struct {
1628 struct irdma_sc_cqp *cqp;
1629 struct irdma_up_info info;
1630 u64 scratch;
1631 } up_map;
1632
1633 struct {
1634 struct irdma_sc_cqp *cqp;
1635 struct irdma_dma_mem query_buff_mem;
1636 u64 scratch;
1637 } query_rdma;
1638
1639 struct {
1640 struct irdma_sc_srq *srq;
1641 u64 scratch;
1642 } srq_create;
1643
1644 struct {
1645 struct irdma_sc_srq *srq;
1646 struct irdma_modify_srq_info info;
1647 u64 scratch;
1648 } srq_modify;
1649
1650 struct {
1651 struct irdma_sc_srq *srq;
1652 u64 scratch;
1653 } srq_destroy;
1654
1655 } u;
1656 };
1657
1658 struct cqp_cmds_info {
1659 struct list_head cqp_cmd_entry;
1660 u8 cqp_cmd;
1661 u8 post_sq;
1662 struct cqp_info in;
1663 };
1664
1665 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
1666 u32 *wqe_idx);
1667
1668 /**
1669 * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
1670 * @cqp: struct for cqp hw
1671 * @scratch: private data for CQP WQE
1672 */
irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp * cqp,u64 scratch)1673 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch)
1674 {
1675 u32 wqe_idx;
1676
1677 return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
1678 }
1679 #endif /* IRDMA_TYPE_H */
1680