xref: /freebsd/sys/dev/qlnx/qlnxe/rdma_common.h (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*
2  * Copyright (c) 2017-2018 Cavium, Inc.
3  * All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions
7  *  are met:
8  *
9  *  1. Redistributions of source code must retain the above copyright
10  *     notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *     notice, this list of conditions and the following disclaimer in the
13  *     documentation and/or other materials provided with the distribution.
14  *
15  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25  *  POSSIBILITY OF SUCH DAMAGE.
26  *
27  * $FreeBSD$
28  *
29  */
30 
31 #ifndef __RDMA_COMMON__
32 #define __RDMA_COMMON__
33 /************************************************************************/
34 /* Add include to common rdma target for both eCore and protocol rdma driver */
35 /************************************************************************/
36 
37 #define RDMA_RESERVED_LKEY                      (0)                     //Reserved lkey
38 #define RDMA_RING_PAGE_SIZE                     (0x1000)        //4KB pages
39 
40 #define RDMA_MAX_SGE_PER_SQ_WQE         (4)             //max number of SGEs in a single request
41 #define RDMA_MAX_SGE_PER_RQ_WQE         (4)             //max number of SGEs in a single request
42 
43 #define RDMA_MAX_DATA_SIZE_IN_WQE       (0x80000000)    //max size of data in single request
44 
45 #define RDMA_REQ_RD_ATOMIC_ELM_SIZE             (0x50)
46 #define RDMA_RESP_RD_ATOMIC_ELM_SIZE    (0x20)
47 
48 #define RDMA_MAX_CQS                            (64*1024)
49 #define RDMA_MAX_TIDS                           (128*1024-1)
50 #define RDMA_MAX_PDS                            (64*1024)
51 #define RDMA_MAX_XRC_SRQS                       (1024)
52 #define RDMA_MAX_SRQS                           (32*1024)
53 
54 #define RDMA_NUM_STATISTIC_COUNTERS                     MAX_NUM_VPORTS
55 #define RDMA_NUM_STATISTIC_COUNTERS_K2                  MAX_NUM_VPORTS_K2
56 #define RDMA_NUM_STATISTIC_COUNTERS_BB                  MAX_NUM_VPORTS_BB
57 
58 #define RDMA_TASK_TYPE (PROTOCOLID_ROCE)
59 
60 
61 struct rdma_srq_id
62 {
63         __le16 srq_idx /* SRQ index */;
64         __le16 opaque_fid;
65 };
66 
67 
68 struct rdma_srq_producers
69 {
70         __le32 sge_prod /* Current produced sge in SRQ */;
71         __le32 wqe_prod /* Current produced WQE to SRQ */;
72 };
73 
74 /*
75  * rdma completion notification queue element
76  */
77 struct rdma_cnqe
78 {
79 	struct regpair cq_handle;
80 };
81 
82 
83 struct rdma_cqe_responder
84 {
85 	struct regpair srq_wr_id;
86 	struct regpair qp_handle;
87 	__le32 imm_data_or_inv_r_Key /* immediate data in case imm_flg is set, or invalidated r_key in case inv_flg is set */;
88 	__le32 length;
89 	__le32 imm_data_hi /* High bytes of immediate data in case imm_flg is set in iWARP only */;
90 	__le16 rq_cons /* Valid only when status is WORK_REQUEST_FLUSHED_ERR. Indicates an aggregative flush on all posted RQ WQEs until the reported rq_cons. */;
91 	u8 flags;
92 #define RDMA_CQE_RESPONDER_TOGGLE_BIT_MASK  0x1 /* indicates a valid completion written by FW. FW toggle this bit each time it finishes producing all PBL entries */
93 #define RDMA_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0
94 #define RDMA_CQE_RESPONDER_TYPE_MASK        0x3 /*  (use enum rdma_cqe_type) */
95 #define RDMA_CQE_RESPONDER_TYPE_SHIFT       1
96 #define RDMA_CQE_RESPONDER_INV_FLG_MASK     0x1 /* r_key invalidated indicator */
97 #define RDMA_CQE_RESPONDER_INV_FLG_SHIFT    3
98 #define RDMA_CQE_RESPONDER_IMM_FLG_MASK     0x1 /* immediate data indicator */
99 #define RDMA_CQE_RESPONDER_IMM_FLG_SHIFT    4
100 #define RDMA_CQE_RESPONDER_RDMA_FLG_MASK    0x1 /* 1=this CQE relates to an RDMA Write. 0=Send. */
101 #define RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT   5
102 #define RDMA_CQE_RESPONDER_RESERVED2_MASK   0x3
103 #define RDMA_CQE_RESPONDER_RESERVED2_SHIFT  6
104 	u8 status;
105 };
106 
107 struct rdma_cqe_requester
108 {
109 	__le16 sq_cons;
110 	__le16 reserved0;
111 	__le32 reserved1;
112 	struct regpair qp_handle;
113 	struct regpair reserved2;
114 	__le32 reserved3;
115 	__le16 reserved4;
116 	u8 flags;
117 #define RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK  0x1 /* indicates a valid completion written by FW. FW toggle this bit each time it finishes producing all PBL entries */
118 #define RDMA_CQE_REQUESTER_TOGGLE_BIT_SHIFT 0
119 #define RDMA_CQE_REQUESTER_TYPE_MASK        0x3 /*  (use enum rdma_cqe_type) */
120 #define RDMA_CQE_REQUESTER_TYPE_SHIFT       1
121 #define RDMA_CQE_REQUESTER_RESERVED5_MASK   0x1F
122 #define RDMA_CQE_REQUESTER_RESERVED5_SHIFT  3
123 	u8 status;
124 };
125 
126 struct rdma_cqe_common
127 {
128 	struct regpair reserved0;
129 	struct regpair qp_handle;
130 	__le16 reserved1[7];
131 	u8 flags;
132 #define RDMA_CQE_COMMON_TOGGLE_BIT_MASK  0x1 /* indicates a valid completion written by FW. FW toggle this bit each time it finishes producing all PBL entries */
133 #define RDMA_CQE_COMMON_TOGGLE_BIT_SHIFT 0
134 #define RDMA_CQE_COMMON_TYPE_MASK        0x3 /*  (use enum rdma_cqe_type) */
135 #define RDMA_CQE_COMMON_TYPE_SHIFT       1
136 #define RDMA_CQE_COMMON_RESERVED2_MASK   0x1F
137 #define RDMA_CQE_COMMON_RESERVED2_SHIFT  3
138 	u8 status;
139 };
140 
141 /*
142  * rdma completion queue element
143  */
144 union rdma_cqe
145 {
146 	struct rdma_cqe_responder resp;
147 	struct rdma_cqe_requester req;
148 	struct rdma_cqe_common cmn;
149 };
150 
151 
152 
153 
154 /*
155  * CQE requester status enumeration
156  */
157 enum rdma_cqe_requester_status_enum
158 {
159 	RDMA_CQE_REQ_STS_OK,
160 	RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR,
161 	RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR,
162 	RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR,
163 	RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR,
164 	RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR,
165 	RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR,
166 	RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR,
167 	RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR,
168 	RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR,
169 	RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR,
170 	RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR,
171 	MAX_RDMA_CQE_REQUESTER_STATUS_ENUM
172 };
173 
174 
175 
176 /*
177  * CQE responder status enumeration
178  */
179 enum rdma_cqe_responder_status_enum
180 {
181 	RDMA_CQE_RESP_STS_OK,
182 	RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR,
183 	RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR,
184 	RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR,
185 	RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR,
186 	RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR,
187 	RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR,
188 	RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR,
189 	MAX_RDMA_CQE_RESPONDER_STATUS_ENUM
190 };
191 
192 
193 /*
194  * CQE type enumeration
195  */
196 enum rdma_cqe_type
197 {
198 	RDMA_CQE_TYPE_REQUESTER,
199 	RDMA_CQE_TYPE_RESPONDER_RQ,
200 	RDMA_CQE_TYPE_RESPONDER_SRQ,
201 	RDMA_CQE_TYPE_INVALID,
202 	MAX_RDMA_CQE_TYPE
203 };
204 
205 
206 /*
207  * DIF Block size options
208  */
209 enum rdma_dif_block_size
210 {
211 	RDMA_DIF_BLOCK_512=0,
212 	RDMA_DIF_BLOCK_4096=1,
213 	MAX_RDMA_DIF_BLOCK_SIZE
214 };
215 
216 
217 /*
218  * DIF CRC initial value
219  */
220 enum rdma_dif_crc_seed
221 {
222 	RDMA_DIF_CRC_SEED_0000=0,
223 	RDMA_DIF_CRC_SEED_FFFF=1,
224 	MAX_RDMA_DIF_CRC_SEED
225 };
226 
227 
228 /*
229  * RDMA DIF Error Result Structure
230  */
231 struct rdma_dif_error_result
232 {
233 	__le32 error_intervals /* Total number of error intervals in the IO. */;
234 	__le32 dif_error_1st_interval /* Number of the first interval that contained error. Set to 0xFFFFFFFF if error occurred in the Runt Block. */;
235 	u8 flags;
236 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_CRC_MASK      0x1 /* CRC error occurred. */
237 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_CRC_SHIFT     0
238 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_APP_TAG_MASK  0x1 /* App Tag error occurred. */
239 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_APP_TAG_SHIFT 1
240 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_REF_TAG_MASK  0x1 /* Ref Tag error occurred. */
241 #define RDMA_DIF_ERROR_RESULT_DIF_ERROR_TYPE_REF_TAG_SHIFT 2
242 #define RDMA_DIF_ERROR_RESULT_RESERVED0_MASK               0xF
243 #define RDMA_DIF_ERROR_RESULT_RESERVED0_SHIFT              3
244 #define RDMA_DIF_ERROR_RESULT_TOGGLE_BIT_MASK              0x1 /* Used to indicate the structure is valid. Toggles each time an invalidate region is performed. */
245 #define RDMA_DIF_ERROR_RESULT_TOGGLE_BIT_SHIFT             7
246 	u8 reserved1[55] /* Pad to 64 bytes to ensure efficient word line writing. */;
247 };
248 
249 
250 /*
251  * DIF IO direction
252  */
253 enum rdma_dif_io_direction_flg
254 {
255 	RDMA_DIF_DIR_RX=0,
256 	RDMA_DIF_DIR_TX=1,
257 	MAX_RDMA_DIF_IO_DIRECTION_FLG
258 };
259 
260 
261 /*
262  * RDMA DIF Runt Result Structure
263  */
264 struct rdma_dif_runt_result
265 {
266 	__le16 guard_tag /* CRC result of received IO. */;
267 	__le16 reserved[3];
268 };
269 
270 
271 /*
272  * memory window type enumeration
273  */
274 enum rdma_mw_type
275 {
276 	RDMA_MW_TYPE_1,
277 	RDMA_MW_TYPE_2A,
278 	MAX_RDMA_MW_TYPE
279 };
280 
281 
282 struct rdma_rq_sge
283 {
284 	struct regpair addr;
285 	__le32 length;
286 	__le32 flags;
287 #define RDMA_RQ_SGE_L_KEY_MASK      0x3FFFFFF /* key of memory relating to this RQ */
288 #define RDMA_RQ_SGE_L_KEY_SHIFT     0
289 #define RDMA_RQ_SGE_NUM_SGES_MASK   0x7 /* first SGE - number of SGEs in this RQ WQE. Other SGEs - should be set to 0 */
290 #define RDMA_RQ_SGE_NUM_SGES_SHIFT  26
291 #define RDMA_RQ_SGE_RESERVED0_MASK  0x7
292 #define RDMA_RQ_SGE_RESERVED0_SHIFT 29
293 };
294 
295 
296 struct rdma_sq_atomic_wqe
297 {
298 	__le32 reserved1;
299 	__le32 length /* Total data length (8 bytes for Atomic) */;
300 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
301 	u8 req_type /* Type of WQE */;
302 	u8 flags;
303 #define RDMA_SQ_ATOMIC_WQE_COMP_FLG_MASK         0x1 /* If set, completion will be generated when the WQE is completed */
304 #define RDMA_SQ_ATOMIC_WQE_COMP_FLG_SHIFT        0
305 #define RDMA_SQ_ATOMIC_WQE_RD_FENCE_FLG_MASK     0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
306 #define RDMA_SQ_ATOMIC_WQE_RD_FENCE_FLG_SHIFT    1
307 #define RDMA_SQ_ATOMIC_WQE_INV_FENCE_FLG_MASK    0x1 /* If set, all pending operations will be completed before start processing this WQE */
308 #define RDMA_SQ_ATOMIC_WQE_INV_FENCE_FLG_SHIFT   2
309 #define RDMA_SQ_ATOMIC_WQE_SE_FLG_MASK           0x1 /* Dont care for atomic wqe */
310 #define RDMA_SQ_ATOMIC_WQE_SE_FLG_SHIFT          3
311 #define RDMA_SQ_ATOMIC_WQE_INLINE_FLG_MASK       0x1 /* Should be 0 for atomic wqe */
312 #define RDMA_SQ_ATOMIC_WQE_INLINE_FLG_SHIFT      4
313 #define RDMA_SQ_ATOMIC_WQE_DIF_ON_HOST_FLG_MASK  0x1 /* Should be 0 for atomic wqe */
314 #define RDMA_SQ_ATOMIC_WQE_DIF_ON_HOST_FLG_SHIFT 5
315 #define RDMA_SQ_ATOMIC_WQE_RESERVED0_MASK        0x3
316 #define RDMA_SQ_ATOMIC_WQE_RESERVED0_SHIFT       6
317 	u8 wqe_size /* Size of WQE in 16B chunks including SGE */;
318 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
319 	struct regpair remote_va /* remote virtual address */;
320 	__le32 r_key /* Remote key */;
321 	__le32 reserved2;
322 	struct regpair cmp_data /* Data to compare in case of ATOMIC_CMP_AND_SWAP */;
323 	struct regpair swap_data /* Swap or add data */;
324 };
325 
326 
327 /*
328  * First element (16 bytes) of atomic wqe
329  */
330 struct rdma_sq_atomic_wqe_1st
331 {
332 	__le32 reserved1;
333 	__le32 length /* Total data length (8 bytes for Atomic) */;
334 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
335 	u8 req_type /* Type of WQE */;
336 	u8 flags;
337 #define RDMA_SQ_ATOMIC_WQE_1ST_COMP_FLG_MASK       0x1 /* If set, completion will be generated when the WQE is completed */
338 #define RDMA_SQ_ATOMIC_WQE_1ST_COMP_FLG_SHIFT      0
339 #define RDMA_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_MASK   0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
340 #define RDMA_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_SHIFT  1
341 #define RDMA_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_MASK  0x1 /* If set, all pending operations will be completed before start processing this WQE */
342 #define RDMA_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_SHIFT 2
343 #define RDMA_SQ_ATOMIC_WQE_1ST_SE_FLG_MASK         0x1 /* Dont care for atomic wqe */
344 #define RDMA_SQ_ATOMIC_WQE_1ST_SE_FLG_SHIFT        3
345 #define RDMA_SQ_ATOMIC_WQE_1ST_INLINE_FLG_MASK     0x1 /* Should be 0 for atomic wqe */
346 #define RDMA_SQ_ATOMIC_WQE_1ST_INLINE_FLG_SHIFT    4
347 #define RDMA_SQ_ATOMIC_WQE_1ST_RESERVED0_MASK      0x7
348 #define RDMA_SQ_ATOMIC_WQE_1ST_RESERVED0_SHIFT     5
349 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs. Set to number of SGEs + 1. */;
350 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
351 };
352 
353 
354 /*
355  * Second element (16 bytes) of atomic wqe
356  */
357 struct rdma_sq_atomic_wqe_2nd
358 {
359 	struct regpair remote_va /* remote virtual address */;
360 	__le32 r_key /* Remote key */;
361 	__le32 reserved2;
362 };
363 
364 
365 /*
366  * Third element (16 bytes) of atomic wqe
367  */
368 struct rdma_sq_atomic_wqe_3rd
369 {
370 	struct regpair cmp_data /* Data to compare in case of ATOMIC_CMP_AND_SWAP */;
371 	struct regpair swap_data /* Swap or add data */;
372 };
373 
374 
375 struct rdma_sq_bind_wqe
376 {
377 	struct regpair addr;
378 	__le32 l_key;
379 	u8 req_type /* Type of WQE */;
380 	u8 flags;
381 #define RDMA_SQ_BIND_WQE_COMP_FLG_MASK       0x1 /* If set, completion will be generated when the WQE is completed */
382 #define RDMA_SQ_BIND_WQE_COMP_FLG_SHIFT      0
383 #define RDMA_SQ_BIND_WQE_RD_FENCE_FLG_MASK   0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
384 #define RDMA_SQ_BIND_WQE_RD_FENCE_FLG_SHIFT  1
385 #define RDMA_SQ_BIND_WQE_INV_FENCE_FLG_MASK  0x1 /* If set, all pending operations will be completed before start processing this WQE */
386 #define RDMA_SQ_BIND_WQE_INV_FENCE_FLG_SHIFT 2
387 #define RDMA_SQ_BIND_WQE_SE_FLG_MASK         0x1 /* Dont care for bind wqe */
388 #define RDMA_SQ_BIND_WQE_SE_FLG_SHIFT        3
389 #define RDMA_SQ_BIND_WQE_INLINE_FLG_MASK     0x1 /* Should be 0 for bind wqe */
390 #define RDMA_SQ_BIND_WQE_INLINE_FLG_SHIFT    4
391 #define RDMA_SQ_BIND_WQE_RESERVED0_MASK      0x7
392 #define RDMA_SQ_BIND_WQE_RESERVED0_SHIFT     5
393 	u8 wqe_size /* Size of WQE in 16B chunks */;
394 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
395 	u8 bind_ctrl;
396 #define RDMA_SQ_BIND_WQE_ZERO_BASED_MASK     0x1 /* zero based indication */
397 #define RDMA_SQ_BIND_WQE_ZERO_BASED_SHIFT    0
398 #define RDMA_SQ_BIND_WQE_MW_TYPE_MASK        0x1 /*  (use enum rdma_mw_type) */
399 #define RDMA_SQ_BIND_WQE_MW_TYPE_SHIFT       1
400 #define RDMA_SQ_BIND_WQE_RESERVED1_MASK      0x3F
401 #define RDMA_SQ_BIND_WQE_RESERVED1_SHIFT     2
402 	u8 access_ctrl;
403 #define RDMA_SQ_BIND_WQE_REMOTE_READ_MASK    0x1
404 #define RDMA_SQ_BIND_WQE_REMOTE_READ_SHIFT   0
405 #define RDMA_SQ_BIND_WQE_REMOTE_WRITE_MASK   0x1
406 #define RDMA_SQ_BIND_WQE_REMOTE_WRITE_SHIFT  1
407 #define RDMA_SQ_BIND_WQE_ENABLE_ATOMIC_MASK  0x1
408 #define RDMA_SQ_BIND_WQE_ENABLE_ATOMIC_SHIFT 2
409 #define RDMA_SQ_BIND_WQE_LOCAL_READ_MASK     0x1
410 #define RDMA_SQ_BIND_WQE_LOCAL_READ_SHIFT    3
411 #define RDMA_SQ_BIND_WQE_LOCAL_WRITE_MASK    0x1
412 #define RDMA_SQ_BIND_WQE_LOCAL_WRITE_SHIFT   4
413 #define RDMA_SQ_BIND_WQE_RESERVED2_MASK      0x7
414 #define RDMA_SQ_BIND_WQE_RESERVED2_SHIFT     5
415 	u8 reserved3;
416 	u8 length_hi /* upper 8 bits of the registered MW length */;
417 	__le32 length_lo /* lower 32 bits of the registered MW length */;
418 	__le32 parent_l_key /* l_key of the parent MR */;
419 	__le32 reserved4;
420 };
421 
422 
423 /*
424  * First element (16 bytes) of bind wqe
425  */
426 struct rdma_sq_bind_wqe_1st
427 {
428 	struct regpair addr;
429 	__le32 l_key;
430 	u8 req_type /* Type of WQE */;
431 	u8 flags;
432 #define RDMA_SQ_BIND_WQE_1ST_COMP_FLG_MASK       0x1 /* If set, completion will be generated when the WQE is completed */
433 #define RDMA_SQ_BIND_WQE_1ST_COMP_FLG_SHIFT      0
434 #define RDMA_SQ_BIND_WQE_1ST_RD_FENCE_FLG_MASK   0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
435 #define RDMA_SQ_BIND_WQE_1ST_RD_FENCE_FLG_SHIFT  1
436 #define RDMA_SQ_BIND_WQE_1ST_INV_FENCE_FLG_MASK  0x1 /* If set, all pending operations will be completed before start processing this WQE */
437 #define RDMA_SQ_BIND_WQE_1ST_INV_FENCE_FLG_SHIFT 2
438 #define RDMA_SQ_BIND_WQE_1ST_SE_FLG_MASK         0x1 /* Dont care for bind wqe */
439 #define RDMA_SQ_BIND_WQE_1ST_SE_FLG_SHIFT        3
440 #define RDMA_SQ_BIND_WQE_1ST_INLINE_FLG_MASK     0x1 /* Should be 0 for bind wqe */
441 #define RDMA_SQ_BIND_WQE_1ST_INLINE_FLG_SHIFT    4
442 #define RDMA_SQ_BIND_WQE_1ST_RESERVED0_MASK      0x7
443 #define RDMA_SQ_BIND_WQE_1ST_RESERVED0_SHIFT     5
444 	u8 wqe_size /* Size of WQE in 16B chunks */;
445 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
446 };
447 
448 
449 /*
450  * Second element (16 bytes) of bind wqe
451  */
452 struct rdma_sq_bind_wqe_2nd
453 {
454 	u8 bind_ctrl;
455 #define RDMA_SQ_BIND_WQE_2ND_ZERO_BASED_MASK     0x1 /* zero based indication */
456 #define RDMA_SQ_BIND_WQE_2ND_ZERO_BASED_SHIFT    0
457 #define RDMA_SQ_BIND_WQE_2ND_MW_TYPE_MASK        0x1 /*  (use enum rdma_mw_type) */
458 #define RDMA_SQ_BIND_WQE_2ND_MW_TYPE_SHIFT       1
459 #define RDMA_SQ_BIND_WQE_2ND_RESERVED1_MASK      0x3F
460 #define RDMA_SQ_BIND_WQE_2ND_RESERVED1_SHIFT     2
461 	u8 access_ctrl;
462 #define RDMA_SQ_BIND_WQE_2ND_REMOTE_READ_MASK    0x1
463 #define RDMA_SQ_BIND_WQE_2ND_REMOTE_READ_SHIFT   0
464 #define RDMA_SQ_BIND_WQE_2ND_REMOTE_WRITE_MASK   0x1
465 #define RDMA_SQ_BIND_WQE_2ND_REMOTE_WRITE_SHIFT  1
466 #define RDMA_SQ_BIND_WQE_2ND_ENABLE_ATOMIC_MASK  0x1
467 #define RDMA_SQ_BIND_WQE_2ND_ENABLE_ATOMIC_SHIFT 2
468 #define RDMA_SQ_BIND_WQE_2ND_LOCAL_READ_MASK     0x1
469 #define RDMA_SQ_BIND_WQE_2ND_LOCAL_READ_SHIFT    3
470 #define RDMA_SQ_BIND_WQE_2ND_LOCAL_WRITE_MASK    0x1
471 #define RDMA_SQ_BIND_WQE_2ND_LOCAL_WRITE_SHIFT   4
472 #define RDMA_SQ_BIND_WQE_2ND_RESERVED2_MASK      0x7
473 #define RDMA_SQ_BIND_WQE_2ND_RESERVED2_SHIFT     5
474 	u8 reserved3;
475 	u8 length_hi /* upper 8 bits of the registered MW length */;
476 	__le32 length_lo /* lower 32 bits of the registered MW length */;
477 	__le32 parent_l_key /* l_key of the parent MR */;
478 	__le32 reserved4;
479 };
480 
481 
482 /*
483  * Structure with only the SQ WQE common fields. Size is of one SQ element (16B)
484  */
485 struct rdma_sq_common_wqe
486 {
487 	__le32 reserved1[3];
488 	u8 req_type /* Type of WQE */;
489 	u8 flags;
490 #define RDMA_SQ_COMMON_WQE_COMP_FLG_MASK       0x1 /* If set, completion will be generated when the WQE is completed */
491 #define RDMA_SQ_COMMON_WQE_COMP_FLG_SHIFT      0
492 #define RDMA_SQ_COMMON_WQE_RD_FENCE_FLG_MASK   0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
493 #define RDMA_SQ_COMMON_WQE_RD_FENCE_FLG_SHIFT  1
494 #define RDMA_SQ_COMMON_WQE_INV_FENCE_FLG_MASK  0x1 /* If set, all pending operations will be completed before start processing this WQE */
495 #define RDMA_SQ_COMMON_WQE_INV_FENCE_FLG_SHIFT 2
496 #define RDMA_SQ_COMMON_WQE_SE_FLG_MASK         0x1 /* If set, signal the responder to generate a solicited event on this WQE (only relevant in SENDs and RDMA write with Imm) */
497 #define RDMA_SQ_COMMON_WQE_SE_FLG_SHIFT        3
498 #define RDMA_SQ_COMMON_WQE_INLINE_FLG_MASK     0x1 /* if set, indicates inline data is following this WQE instead of SGEs (only relevant in SENDs and RDMA writes) */
499 #define RDMA_SQ_COMMON_WQE_INLINE_FLG_SHIFT    4
500 #define RDMA_SQ_COMMON_WQE_RESERVED0_MASK      0x7
501 #define RDMA_SQ_COMMON_WQE_RESERVED0_SHIFT     5
502 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs or inline data. In case there are SGEs: set to number of SGEs + 1. In case of inline data: set to the whole number of 16B which contain the inline data + 1. */;
503 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
504 };
505 
506 
507 struct rdma_sq_fmr_wqe
508 {
509 	struct regpair addr;
510 	__le32 l_key;
511 	u8 req_type /* Type of WQE */;
512 	u8 flags;
513 #define RDMA_SQ_FMR_WQE_COMP_FLG_MASK                0x1 /* If set, completion will be generated when the WQE is completed */
514 #define RDMA_SQ_FMR_WQE_COMP_FLG_SHIFT               0
515 #define RDMA_SQ_FMR_WQE_RD_FENCE_FLG_MASK            0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
516 #define RDMA_SQ_FMR_WQE_RD_FENCE_FLG_SHIFT           1
517 #define RDMA_SQ_FMR_WQE_INV_FENCE_FLG_MASK           0x1 /* If set, all pending operations will be completed before start processing this WQE */
518 #define RDMA_SQ_FMR_WQE_INV_FENCE_FLG_SHIFT          2
519 #define RDMA_SQ_FMR_WQE_SE_FLG_MASK                  0x1 /* Dont care for FMR wqe */
520 #define RDMA_SQ_FMR_WQE_SE_FLG_SHIFT                 3
521 #define RDMA_SQ_FMR_WQE_INLINE_FLG_MASK              0x1 /* Should be 0 for FMR wqe */
522 #define RDMA_SQ_FMR_WQE_INLINE_FLG_SHIFT             4
523 #define RDMA_SQ_FMR_WQE_DIF_ON_HOST_FLG_MASK         0x1 /* If set, indicated host memory of this WQE is DIF protected. */
524 #define RDMA_SQ_FMR_WQE_DIF_ON_HOST_FLG_SHIFT        5
525 #define RDMA_SQ_FMR_WQE_RESERVED0_MASK               0x3
526 #define RDMA_SQ_FMR_WQE_RESERVED0_SHIFT              6
527 	u8 wqe_size /* Size of WQE in 16B chunks */;
528 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
529 	u8 fmr_ctrl;
530 #define RDMA_SQ_FMR_WQE_PAGE_SIZE_LOG_MASK           0x1F /* 0 is 4k, 1 is 8k... */
531 #define RDMA_SQ_FMR_WQE_PAGE_SIZE_LOG_SHIFT          0
532 #define RDMA_SQ_FMR_WQE_ZERO_BASED_MASK              0x1 /* zero based indication */
533 #define RDMA_SQ_FMR_WQE_ZERO_BASED_SHIFT             5
534 #define RDMA_SQ_FMR_WQE_BIND_EN_MASK                 0x1 /* indication whether bind is enabled for this MR */
535 #define RDMA_SQ_FMR_WQE_BIND_EN_SHIFT                6
536 #define RDMA_SQ_FMR_WQE_RESERVED1_MASK               0x1
537 #define RDMA_SQ_FMR_WQE_RESERVED1_SHIFT              7
538 	u8 access_ctrl;
539 #define RDMA_SQ_FMR_WQE_REMOTE_READ_MASK             0x1
540 #define RDMA_SQ_FMR_WQE_REMOTE_READ_SHIFT            0
541 #define RDMA_SQ_FMR_WQE_REMOTE_WRITE_MASK            0x1
542 #define RDMA_SQ_FMR_WQE_REMOTE_WRITE_SHIFT           1
543 #define RDMA_SQ_FMR_WQE_ENABLE_ATOMIC_MASK           0x1
544 #define RDMA_SQ_FMR_WQE_ENABLE_ATOMIC_SHIFT          2
545 #define RDMA_SQ_FMR_WQE_LOCAL_READ_MASK              0x1
546 #define RDMA_SQ_FMR_WQE_LOCAL_READ_SHIFT             3
547 #define RDMA_SQ_FMR_WQE_LOCAL_WRITE_MASK             0x1
548 #define RDMA_SQ_FMR_WQE_LOCAL_WRITE_SHIFT            4
549 #define RDMA_SQ_FMR_WQE_RESERVED2_MASK               0x7
550 #define RDMA_SQ_FMR_WQE_RESERVED2_SHIFT              5
551 	u8 reserved3;
552 	u8 length_hi /* upper 8 bits of the registered MR length */;
553 	__le32 length_lo /* lower 32 bits of the registered MR length. In case of DIF the length is specified including the DIF guards. */;
554 	struct regpair pbl_addr /* Address of PBL */;
555 	__le32 dif_base_ref_tag /* Ref tag of the first DIF Block. */;
556 	__le16 dif_app_tag /* App tag of all DIF Blocks. */;
557 	__le16 dif_app_tag_mask /* Bitmask for verifying dif_app_tag. */;
558 	__le16 dif_runt_crc_value /* In TX IO, in case the runt_valid_flg is set, this value is used to validate the last Block in the IO. */;
559 	__le16 dif_flags;
560 #define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_MASK    0x1 /* 0=RX, 1=TX (use enum rdma_dif_io_direction_flg) */
561 #define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_SHIFT   0
562 #define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_MASK          0x1 /* DIF block size. 0=512B 1=4096B (use enum rdma_dif_block_size) */
563 #define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_SHIFT         1
564 #define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_MASK      0x1 /* In TX IO, indicates the runt_value field is valid. In RX IO, indicates the calculated runt value is to be placed on host buffer. */
565 #define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_SHIFT     2
566 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_MASK  0x1 /* In TX IO, indicates CRC of each DIF guard tag is checked. */
567 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_SHIFT 3
568 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_MASK    0x1 /* In TX IO, indicates Ref tag of each DIF guard tag is checked. */
569 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_SHIFT   4
570 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_MASK    0x1 /* In TX IO, indicates App tag of each DIF guard tag is checked. */
571 #define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_SHIFT   5
572 #define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_MASK            0x1 /* DIF CRC Seed to use. 0=0x000 1=0xFFFF (use enum rdma_dif_crc_seed) */
573 #define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_SHIFT           6
574 #define RDMA_SQ_FMR_WQE_DIF_RX_REF_TAG_CONST_MASK    0x1 /* In RX IO, Ref Tag will remain at constant value of dif_base_ref_tag */
575 #define RDMA_SQ_FMR_WQE_DIF_RX_REF_TAG_CONST_SHIFT   7
576 #define RDMA_SQ_FMR_WQE_RESERVED4_MASK               0xFF
577 #define RDMA_SQ_FMR_WQE_RESERVED4_SHIFT              8
578 	__le32 Reserved5;
579 };
580 
581 
582 /*
583  * First element (16 bytes) of fmr wqe
584  */
585 struct rdma_sq_fmr_wqe_1st
586 {
587 	struct regpair addr;
588 	__le32 l_key;
589 	u8 req_type /* Type of WQE */;
590 	u8 flags;
591 #define RDMA_SQ_FMR_WQE_1ST_COMP_FLG_MASK         0x1 /* If set, completion will be generated when the WQE is completed */
592 #define RDMA_SQ_FMR_WQE_1ST_COMP_FLG_SHIFT        0
593 #define RDMA_SQ_FMR_WQE_1ST_RD_FENCE_FLG_MASK     0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
594 #define RDMA_SQ_FMR_WQE_1ST_RD_FENCE_FLG_SHIFT    1
595 #define RDMA_SQ_FMR_WQE_1ST_INV_FENCE_FLG_MASK    0x1 /* If set, all pending operations will be completed before start processing this WQE */
596 #define RDMA_SQ_FMR_WQE_1ST_INV_FENCE_FLG_SHIFT   2
597 #define RDMA_SQ_FMR_WQE_1ST_SE_FLG_MASK           0x1 /* Dont care for FMR wqe */
598 #define RDMA_SQ_FMR_WQE_1ST_SE_FLG_SHIFT          3
599 #define RDMA_SQ_FMR_WQE_1ST_INLINE_FLG_MASK       0x1 /* Should be 0 for FMR wqe */
600 #define RDMA_SQ_FMR_WQE_1ST_INLINE_FLG_SHIFT      4
601 #define RDMA_SQ_FMR_WQE_1ST_DIF_ON_HOST_FLG_MASK  0x1 /* If set, indicated host memory of this WQE is DIF protected. */
602 #define RDMA_SQ_FMR_WQE_1ST_DIF_ON_HOST_FLG_SHIFT 5
603 #define RDMA_SQ_FMR_WQE_1ST_RESERVED0_MASK        0x3
604 #define RDMA_SQ_FMR_WQE_1ST_RESERVED0_SHIFT       6
605 	u8 wqe_size /* Size of WQE in 16B chunks */;
606 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
607 };
608 
609 
610 /*
611  * Second element (16 bytes) of fmr wqe
612  */
613 struct rdma_sq_fmr_wqe_2nd
614 {
615 	u8 fmr_ctrl;
616 #define RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_MASK  0x1F /* 0 is 4k, 1 is 8k... */
617 #define RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_SHIFT 0
618 #define RDMA_SQ_FMR_WQE_2ND_ZERO_BASED_MASK     0x1 /* zero based indication */
619 #define RDMA_SQ_FMR_WQE_2ND_ZERO_BASED_SHIFT    5
620 #define RDMA_SQ_FMR_WQE_2ND_BIND_EN_MASK        0x1 /* indication whether bind is enabled for this MR */
621 #define RDMA_SQ_FMR_WQE_2ND_BIND_EN_SHIFT       6
622 #define RDMA_SQ_FMR_WQE_2ND_RESERVED1_MASK      0x1
623 #define RDMA_SQ_FMR_WQE_2ND_RESERVED1_SHIFT     7
624 	u8 access_ctrl;
625 #define RDMA_SQ_FMR_WQE_2ND_REMOTE_READ_MASK    0x1
626 #define RDMA_SQ_FMR_WQE_2ND_REMOTE_READ_SHIFT   0
627 #define RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE_MASK   0x1
628 #define RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE_SHIFT  1
629 #define RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC_MASK  0x1
630 #define RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC_SHIFT 2
631 #define RDMA_SQ_FMR_WQE_2ND_LOCAL_READ_MASK     0x1
632 #define RDMA_SQ_FMR_WQE_2ND_LOCAL_READ_SHIFT    3
633 #define RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE_MASK    0x1
634 #define RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE_SHIFT   4
635 #define RDMA_SQ_FMR_WQE_2ND_RESERVED2_MASK      0x7
636 #define RDMA_SQ_FMR_WQE_2ND_RESERVED2_SHIFT     5
637 	u8 reserved3;
638 	u8 length_hi /* upper 8 bits of the registered MR length */;
639 	__le32 length_lo /* lower 32 bits of the registered MR length. */;
640 	struct regpair pbl_addr /* Address of PBL */;
641 };
642 
643 
644 /*
645  * Third element (16 bytes) of fmr wqe
646  */
647 struct rdma_sq_fmr_wqe_3rd
648 {
649 	__le32 dif_base_ref_tag /* Ref tag of the first DIF Block. */;
650 	__le16 dif_app_tag /* App tag of all DIF Blocks. */;
651 	__le16 dif_app_tag_mask /* Bitmask for verifying dif_app_tag. */;
652 	__le16 dif_runt_crc_value /* In TX IO, in case the runt_valid_flg is set, this value is used to validate the last Block in the IO. */;
653 	__le16 dif_flags;
654 #define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_MASK    0x1 /* 0=RX, 1=TX (use enum rdma_dif_io_direction_flg) */
655 #define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_SHIFT   0
656 #define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_MASK          0x1 /* DIF block size. 0=512B 1=4096B (use enum rdma_dif_block_size) */
657 #define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_SHIFT         1
658 #define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_MASK      0x1 /* In TX IO, indicates the runt_value field is valid. In RX IO, indicates the calculated runt value is to be placed on host buffer. */
659 #define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_SHIFT     2
660 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_MASK  0x1 /* In TX IO, indicates CRC of each DIF guard tag is checked. */
661 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_SHIFT 3
662 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_MASK    0x1 /* In TX IO, indicates Ref tag of each DIF guard tag is checked. */
663 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_SHIFT   4
664 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_MASK    0x1 /* In TX IO, indicates App tag of each DIF guard tag is checked. */
665 #define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_SHIFT   5
666 #define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_MASK            0x1 /* DIF CRC Seed to use. 0=0x000 1=0xFFFF (use enum rdma_dif_crc_seed) */
667 #define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_SHIFT           6
668 #define RDMA_SQ_FMR_WQE_3RD_RESERVED4_MASK               0x1FF
669 #define RDMA_SQ_FMR_WQE_3RD_RESERVED4_SHIFT              7
670 	__le32 Reserved5;
671 };
672 
673 
674 struct rdma_sq_local_inv_wqe
675 {
676 	struct regpair reserved;
677 	__le32 inv_l_key /* The invalidate local key */;
678 	u8 req_type /* Type of WQE */;
679 	u8 flags;
680 #define RDMA_SQ_LOCAL_INV_WQE_COMP_FLG_MASK         0x1 /* If set, completion will be generated when the WQE is completed */
681 #define RDMA_SQ_LOCAL_INV_WQE_COMP_FLG_SHIFT        0
682 #define RDMA_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_MASK     0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
683 #define RDMA_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_SHIFT    1
684 #define RDMA_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_MASK    0x1 /* If set, all pending operations will be completed before start processing this WQE */
685 #define RDMA_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_SHIFT   2
686 #define RDMA_SQ_LOCAL_INV_WQE_SE_FLG_MASK           0x1 /* Dont care for local invalidate wqe */
687 #define RDMA_SQ_LOCAL_INV_WQE_SE_FLG_SHIFT          3
688 #define RDMA_SQ_LOCAL_INV_WQE_INLINE_FLG_MASK       0x1 /* Should be 0 for local invalidate wqe */
689 #define RDMA_SQ_LOCAL_INV_WQE_INLINE_FLG_SHIFT      4
690 #define RDMA_SQ_LOCAL_INV_WQE_DIF_ON_HOST_FLG_MASK  0x1 /* If set, indicated host memory of this WQE is DIF protected. */
691 #define RDMA_SQ_LOCAL_INV_WQE_DIF_ON_HOST_FLG_SHIFT 5
692 #define RDMA_SQ_LOCAL_INV_WQE_RESERVED0_MASK        0x3
693 #define RDMA_SQ_LOCAL_INV_WQE_RESERVED0_SHIFT       6
694 	u8 wqe_size /* Size of WQE in 16B chunks */;
695 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
696 };
697 
698 
699 struct rdma_sq_rdma_wqe
700 {
701 	__le32 imm_data /* The immediate data in case of RDMA_WITH_IMM */;
702 	__le32 length /* Total data length. If DIF on host is enabled, length does NOT include DIF guards. */;
703 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
704 	u8 req_type /* Type of WQE */;
705 	u8 flags;
706 #define RDMA_SQ_RDMA_WQE_COMP_FLG_MASK                  0x1 /* If set, completion will be generated when the WQE is completed */
707 #define RDMA_SQ_RDMA_WQE_COMP_FLG_SHIFT                 0
708 #define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_MASK              0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
709 #define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_SHIFT             1
710 #define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_MASK             0x1 /* If set, all pending operations will be completed before start processing this WQE */
711 #define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_SHIFT            2
712 #define RDMA_SQ_RDMA_WQE_SE_FLG_MASK                    0x1 /* If set, signal the responder to generate a solicited event on this WQE */
713 #define RDMA_SQ_RDMA_WQE_SE_FLG_SHIFT                   3
714 #define RDMA_SQ_RDMA_WQE_INLINE_FLG_MASK                0x1 /* if set, indicates inline data is following this WQE instead of SGEs. Applicable for RDMA_WR or RDMA_WR_WITH_IMM. Should be 0 for RDMA_RD */
715 #define RDMA_SQ_RDMA_WQE_INLINE_FLG_SHIFT               4
716 #define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_MASK           0x1 /* If set, indicated host memory of this WQE is DIF protected. */
717 #define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_SHIFT          5
718 #define RDMA_SQ_RDMA_WQE_READ_INV_FLG_MASK              0x1 /* If set, indicated read with invalidate WQE. iWARP only */
719 #define RDMA_SQ_RDMA_WQE_READ_INV_FLG_SHIFT             6
720 #define RDMA_SQ_RDMA_WQE_RESERVED0_MASK                 0x1
721 #define RDMA_SQ_RDMA_WQE_RESERVED0_SHIFT                7
722 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs or inline data. In case there are SGEs: set to number of SGEs + 1. In case of inline data: set to the whole number of 16B which contain the inline data + 1. */;
723 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
724 	struct regpair remote_va /* Remote virtual address */;
725 	__le32 r_key /* Remote key */;
726 	u8 dif_flags;
727 #define RDMA_SQ_RDMA_WQE_DIF_BLOCK_SIZE_MASK            0x1 /* if dif_on_host_flg set: DIF block size. 0=512B 1=4096B (use enum rdma_dif_block_size) */
728 #define RDMA_SQ_RDMA_WQE_DIF_BLOCK_SIZE_SHIFT           0
729 #define RDMA_SQ_RDMA_WQE_DIF_FIRST_RDMA_IN_IO_FLG_MASK  0x1 /* if dif_on_host_flg set: WQE executes first RDMA on related IO. */
730 #define RDMA_SQ_RDMA_WQE_DIF_FIRST_RDMA_IN_IO_FLG_SHIFT 1
731 #define RDMA_SQ_RDMA_WQE_DIF_LAST_RDMA_IN_IO_FLG_MASK   0x1 /* if dif_on_host_flg set: WQE executes last RDMA on related IO. */
732 #define RDMA_SQ_RDMA_WQE_DIF_LAST_RDMA_IN_IO_FLG_SHIFT  2
733 #define RDMA_SQ_RDMA_WQE_RESERVED1_MASK                 0x1F
734 #define RDMA_SQ_RDMA_WQE_RESERVED1_SHIFT                3
735 	u8 reserved2[3];
736 };
737 
738 
739 /*
740  * First element (16 bytes) of rdma wqe
741  */
742 struct rdma_sq_rdma_wqe_1st
743 {
744 	__le32 imm_data /* The immediate data in case of RDMA_WITH_IMM */;
745 	__le32 length /* Total data length */;
746 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
747 	u8 req_type /* Type of WQE */;
748 	u8 flags;
749 #define RDMA_SQ_RDMA_WQE_1ST_COMP_FLG_MASK         0x1 /* If set, completion will be generated when the WQE is completed */
750 #define RDMA_SQ_RDMA_WQE_1ST_COMP_FLG_SHIFT        0
751 #define RDMA_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_MASK     0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
752 #define RDMA_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_SHIFT    1
753 #define RDMA_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_MASK    0x1 /* If set, all pending operations will be completed before start processing this WQE */
754 #define RDMA_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_SHIFT   2
755 #define RDMA_SQ_RDMA_WQE_1ST_SE_FLG_MASK           0x1 /* If set, signal the responder to generate a solicited event on this WQE */
756 #define RDMA_SQ_RDMA_WQE_1ST_SE_FLG_SHIFT          3
757 #define RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG_MASK       0x1 /* if set, indicates inline data is following this WQE instead of SGEs. Applicable for RDMA_WR or RDMA_WR_WITH_IMM. Should be 0 for RDMA_RD */
758 #define RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG_SHIFT      4
759 #define RDMA_SQ_RDMA_WQE_1ST_DIF_ON_HOST_FLG_MASK  0x1 /* If set, indicated host memory of this WQE is DIF protected. */
760 #define RDMA_SQ_RDMA_WQE_1ST_DIF_ON_HOST_FLG_SHIFT 5
761 #define RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG_MASK     0x1 /* If set, indicated read with invalidate WQE. iWARP only */
762 #define RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG_SHIFT    6
763 #define RDMA_SQ_RDMA_WQE_1ST_RESERVED0_MASK        0x1
764 #define RDMA_SQ_RDMA_WQE_1ST_RESERVED0_SHIFT       7
765 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs or inline data. In case there are SGEs: set to number of SGEs + 1. In case of inline data: set to the whole number of 16B which contain the inline data + 1. */;
766 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
767 };
768 
769 
770 /*
771  * Second element (16 bytes) of rdma wqe
772  */
773 struct rdma_sq_rdma_wqe_2nd
774 {
775 	struct regpair remote_va /* Remote virtual address */;
776 	__le32 r_key /* Remote key */;
777 	u8 dif_flags;
778 #define RDMA_SQ_RDMA_WQE_2ND_DIF_BLOCK_SIZE_MASK         0x1 /* if dif_on_host_flg set: DIF block size. 0=512B 1=4096B (use enum rdma_dif_block_size) */
779 #define RDMA_SQ_RDMA_WQE_2ND_DIF_BLOCK_SIZE_SHIFT        0
780 #define RDMA_SQ_RDMA_WQE_2ND_DIF_FIRST_SEGMENT_FLG_MASK  0x1 /* if dif_on_host_flg set: WQE executes first DIF on related MR. */
781 #define RDMA_SQ_RDMA_WQE_2ND_DIF_FIRST_SEGMENT_FLG_SHIFT 1
782 #define RDMA_SQ_RDMA_WQE_2ND_DIF_LAST_SEGMENT_FLG_MASK   0x1 /* if dif_on_host_flg set: WQE executes last DIF on related MR. */
783 #define RDMA_SQ_RDMA_WQE_2ND_DIF_LAST_SEGMENT_FLG_SHIFT  2
784 #define RDMA_SQ_RDMA_WQE_2ND_RESERVED1_MASK              0x1F
785 #define RDMA_SQ_RDMA_WQE_2ND_RESERVED1_SHIFT             3
786 	u8 reserved2[3];
787 };
788 
789 
790 /*
791  * SQ WQE req type enumeration
792  */
793 enum rdma_sq_req_type
794 {
795 	RDMA_SQ_REQ_TYPE_SEND,
796 	RDMA_SQ_REQ_TYPE_SEND_WITH_IMM,
797 	RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE,
798 	RDMA_SQ_REQ_TYPE_RDMA_WR,
799 	RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM,
800 	RDMA_SQ_REQ_TYPE_RDMA_RD,
801 	RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP,
802 	RDMA_SQ_REQ_TYPE_ATOMIC_ADD,
803 	RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE,
804 	RDMA_SQ_REQ_TYPE_FAST_MR,
805 	RDMA_SQ_REQ_TYPE_BIND,
806 	RDMA_SQ_REQ_TYPE_INVALID,
807 	MAX_RDMA_SQ_REQ_TYPE
808 };
809 
810 
811 struct rdma_sq_send_wqe
812 {
813 	__le32 inv_key_or_imm_data /* the r_key to invalidate in case of SEND_WITH_INVALIDATE, or the immediate data in case of SEND_WITH_IMM */;
814 	__le32 length /* Total data length */;
815 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
816 	u8 req_type /* Type of WQE */;
817 	u8 flags;
818 #define RDMA_SQ_SEND_WQE_COMP_FLG_MASK         0x1 /* If set, completion will be generated when the WQE is completed */
819 #define RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT        0
820 #define RDMA_SQ_SEND_WQE_RD_FENCE_FLG_MASK     0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
821 #define RDMA_SQ_SEND_WQE_RD_FENCE_FLG_SHIFT    1
822 #define RDMA_SQ_SEND_WQE_INV_FENCE_FLG_MASK    0x1 /* If set, all pending operations will be completed before start processing this WQE */
823 #define RDMA_SQ_SEND_WQE_INV_FENCE_FLG_SHIFT   2
824 #define RDMA_SQ_SEND_WQE_SE_FLG_MASK           0x1 /* If set, signal the responder to generate a solicited event on this WQE */
825 #define RDMA_SQ_SEND_WQE_SE_FLG_SHIFT          3
826 #define RDMA_SQ_SEND_WQE_INLINE_FLG_MASK       0x1 /* if set, indicates inline data is following this WQE instead of SGEs */
827 #define RDMA_SQ_SEND_WQE_INLINE_FLG_SHIFT      4
828 #define RDMA_SQ_SEND_WQE_DIF_ON_HOST_FLG_MASK  0x1 /* Should be 0 for send wqe */
829 #define RDMA_SQ_SEND_WQE_DIF_ON_HOST_FLG_SHIFT 5
830 #define RDMA_SQ_SEND_WQE_RESERVED0_MASK        0x3
831 #define RDMA_SQ_SEND_WQE_RESERVED0_SHIFT       6
832 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs or inline data. In case there are SGEs: set to number of SGEs + 1. In case of inline data: set to the whole number of 16B which contain the inline data + 1. */;
833 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
834 	__le32 reserved1[4];
835 };
836 
837 
838 struct rdma_sq_send_wqe_1st
839 {
840 	__le32 inv_key_or_imm_data /* the r_key to invalidate in case of SEND_WITH_INVALIDATE, or the immediate data in case of SEND_WITH_IMM */;
841 	__le32 length /* Total data length */;
842 	__le32 xrc_srq /* Valid only when XRC is set for the QP */;
843 	u8 req_type /* Type of WQE */;
844 	u8 flags;
845 #define RDMA_SQ_SEND_WQE_1ST_COMP_FLG_MASK       0x1 /* If set, completion will be generated when the WQE is completed */
846 #define RDMA_SQ_SEND_WQE_1ST_COMP_FLG_SHIFT      0
847 #define RDMA_SQ_SEND_WQE_1ST_RD_FENCE_FLG_MASK   0x1 /* If set, all pending RDMA read or Atomic operations will be completed before start processing this WQE */
848 #define RDMA_SQ_SEND_WQE_1ST_RD_FENCE_FLG_SHIFT  1
849 #define RDMA_SQ_SEND_WQE_1ST_INV_FENCE_FLG_MASK  0x1 /* If set, all pending operations will be completed before start processing this WQE */
850 #define RDMA_SQ_SEND_WQE_1ST_INV_FENCE_FLG_SHIFT 2
851 #define RDMA_SQ_SEND_WQE_1ST_SE_FLG_MASK         0x1 /* If set, signal the responder to generate a solicited event on this WQE */
852 #define RDMA_SQ_SEND_WQE_1ST_SE_FLG_SHIFT        3
853 #define RDMA_SQ_SEND_WQE_1ST_INLINE_FLG_MASK     0x1 /* if set, indicates inline data is following this WQE instead of SGEs */
854 #define RDMA_SQ_SEND_WQE_1ST_INLINE_FLG_SHIFT    4
855 #define RDMA_SQ_SEND_WQE_1ST_RESERVED0_MASK      0x7
856 #define RDMA_SQ_SEND_WQE_1ST_RESERVED0_SHIFT     5
857 	u8 wqe_size /* Size of WQE in 16B chunks including all SGEs or inline data. In case there are SGEs: set to number of SGEs + 1. In case of inline data: set to the whole number of 16B which contain the inline data + 1. */;
858 	u8 prev_wqe_size /* Previous WQE size in 16B chunks */;
859 };
860 
861 
862 struct rdma_sq_send_wqe_2st
863 {
864 	__le32 reserved1[4];
865 };
866 
867 
868 struct rdma_sq_sge
869 {
870 	__le32 length /* Total length of the send. If DIF on host is enabled, SGE length includes the DIF guards. */;
871 	struct regpair addr;
872 	__le32 l_key;
873 };
874 
875 
876 struct rdma_srq_wqe_header
877 {
878 	struct regpair wr_id;
879 	u8 num_sges /* number of SGEs in WQE */;
880 	u8 reserved2[7];
881 };
882 
883 struct rdma_srq_sge
884 {
885 	struct regpair addr;
886 	__le32 length;
887 	__le32 l_key;
888 };
889 
890 /*
891  * rdma srq sge
892  */
893 union rdma_srq_elm
894 {
895 	struct rdma_srq_wqe_header header;
896 	struct rdma_srq_sge sge;
897 };
898 
899 
900 
901 
902 /*
903  * Rdma doorbell data for flags update
904  */
905 struct rdma_pwm_flags_data
906 {
907 	__le16 icid /* internal CID */;
908 	u8 agg_flags /* aggregative flags */;
909 	u8 reserved;
910 };
911 
912 
913 /*
914  * Rdma doorbell data for SQ and RQ
915  */
916 struct rdma_pwm_val16_data
917 {
918 	__le16 icid /* internal CID */;
919 	__le16 value /* aggregated value to update */;
920 };
921 
922 
923 union rdma_pwm_val16_data_union
924 {
925 	struct rdma_pwm_val16_data as_struct /* Parameters field */;
926 	__le32 as_dword;
927 };
928 
929 
930 /*
931  * Rdma doorbell data for CQ
932  */
933 struct rdma_pwm_val32_data
934 {
935 	__le16 icid /* internal CID */;
936 	u8 agg_flags /* bit for every DQ counter flags in CM context that DQ can increment */;
937 	u8 params;
938 #define RDMA_PWM_VAL32_DATA_AGG_CMD_MASK             0x3 /* aggregative command to CM (use enum db_agg_cmd_sel) */
939 #define RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT            0
940 #define RDMA_PWM_VAL32_DATA_BYPASS_EN_MASK           0x1 /* enable QM bypass */
941 #define RDMA_PWM_VAL32_DATA_BYPASS_EN_SHIFT          2
942 #define RDMA_PWM_VAL32_DATA_CONN_TYPE_IS_IWARP_MASK  0x1 /* Connection type is iWARP */
943 #define RDMA_PWM_VAL32_DATA_CONN_TYPE_IS_IWARP_SHIFT 3
944 #define RDMA_PWM_VAL32_DATA_SET_16B_VAL_MASK         0x1 /* Flag indicating 16b variable should be updated. Should be used when conn_type_is_iwarp is used */
945 #define RDMA_PWM_VAL32_DATA_SET_16B_VAL_SHIFT        4
946 #define RDMA_PWM_VAL32_DATA_RESERVED_MASK            0x7
947 #define RDMA_PWM_VAL32_DATA_RESERVED_SHIFT           5
948 	__le32 value /* aggregated value to update */;
949 };
950 
951 
952 union rdma_pwm_val32_data_union
953 {
954 	struct rdma_pwm_val32_data as_struct /* Parameters field */;
955 	struct regpair as_repair;
956 };
957 
958 #endif /* __RDMA_COMMON__ */
959