1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2015 - 2023 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #ifndef IRDMA_USER_H
36 #define IRDMA_USER_H
37
38 #include <rdma/ib_verbs.h>
39
40 #define irdma_handle void *
41 #define irdma_adapter_handle irdma_handle
42 #define irdma_qp_handle irdma_handle
43 #define irdma_cq_handle irdma_handle
44 #define irdma_pd_id irdma_handle
45 #define irdma_stag_handle irdma_handle
46 #define irdma_stag_index u32
47 #define irdma_stag u32
48 #define irdma_stag_key u8
49 #define irdma_tagged_offset u64
50 #define irdma_access_privileges u32
51 #define irdma_physical_fragment u64
52 #define irdma_address_list u64 *
53 #define irdma_sgl struct ib_sge *
54
55 #define IRDMA_MAX_MR_SIZE 0x200000000000ULL
56
57 #define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
58 #define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
59 #define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
60 #define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05
61 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
62 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a
63 #define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10
64 #define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20
65 #define IRDMA_ACCESS_FLAGS_ALL 0x3f
66
67 #define IRDMA_OP_TYPE_RDMA_WRITE 0x00
68 #define IRDMA_OP_TYPE_RDMA_READ 0x01
69 #define IRDMA_OP_TYPE_SEND 0x03
70 #define IRDMA_OP_TYPE_SEND_INV 0x04
71 #define IRDMA_OP_TYPE_SEND_SOL 0x05
72 #define IRDMA_OP_TYPE_SEND_SOL_INV 0x06
73 #define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d
74 #define IRDMA_OP_TYPE_BIND_MW 0x08
75 #define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09
76 #define IRDMA_OP_TYPE_INV_STAG 0x0a
77 #define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b
78 #define IRDMA_OP_TYPE_NOP 0x0c
79 #define IRDMA_OP_TYPE_REC 0x3e
80 #define IRDMA_OP_TYPE_REC_IMM 0x3f
81
82 #define IRDMA_FLUSH_MAJOR_ERR 1
83 /* Async Events codes */
84 #define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
85 #define IRDMA_AE_AMP_INVALID_STAG 0x0103
86 #define IRDMA_AE_AMP_BAD_QP 0x0104
87 #define IRDMA_AE_AMP_BAD_PD 0x0105
88 #define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
89 #define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
90 #define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
91 #define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
92 #define IRDMA_AE_AMP_TO_WRAP 0x010a
93 #define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
94 #define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
95 #define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
96 #define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
97 #define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
98 #define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
99 #define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
100 #define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
101 #define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
102 #define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
103 #define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
104 #define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
105 #define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
106 #define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
107 #define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
108 #define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
109 #define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
110 #define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
111 #define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
112 #define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
113 #define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
114 #define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
115 #define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
116 #define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
117 #define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
118 #define IRDMA_AE_BAD_CLOSE 0x0201
119 #define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
120 #define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
121 #define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
122 #define IRDMA_AE_STAG_ZERO_INVALID 0x0206
123 #define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
124 #define IRDMA_AE_IB_INVALID_REQUEST 0x0208
125 #define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
126 #define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
127 #define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
128 #define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
129 #define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
130 #define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
131 #define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
132 #define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
133 #define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
134 #define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
135 #define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
136 #define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
137 #define IRDMA_AE_DDP_NO_L_BIT 0x0308
138 #define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
139 #define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
140 #define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
141 #define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
142 #define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
143 #define IRDMA_AE_ROCE_REQ_LENGTH_ERROR 0x0318
144 #define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
145 #define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
146 #define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
147 #define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
148 #define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
149 #define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
150 #define IRDMA_AE_STALE_ARP_ENTRY 0x0403
151 #define IRDMA_AE_INVALID_AH_ENTRY 0x0406
152 #define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
153 #define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
154 #define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
155 #define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
156 #define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
157 #define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
158 #define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
159 #define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
160 #define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
161 #define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
162 #define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
163 #define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
164 #define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f
165 #define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
166 #define IRDMA_AE_RESET_SENT 0x0601
167 #define IRDMA_AE_TERMINATE_SENT 0x0602
168 #define IRDMA_AE_RESET_NOT_SENT 0x0603
169 #define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
170 #define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
171 #define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
172 #define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
173
174 enum irdma_device_caps_const {
175 IRDMA_WQE_SIZE = 4,
176 IRDMA_CQP_WQE_SIZE = 8,
177 IRDMA_CQE_SIZE = 4,
178 IRDMA_EXTENDED_CQE_SIZE = 8,
179 IRDMA_AEQE_SIZE = 2,
180 IRDMA_CEQE_SIZE = 1,
181 IRDMA_CQP_CTX_SIZE = 8,
182 IRDMA_SHADOW_AREA_SIZE = 8,
183 IRDMA_GATHER_STATS_BUF_SIZE = 1024,
184 IRDMA_MIN_IW_QP_ID = 0,
185 IRDMA_QUERY_FPM_BUF_SIZE = 176,
186 IRDMA_COMMIT_FPM_BUF_SIZE = 176,
187 IRDMA_MAX_IW_QP_ID = 262143,
188 IRDMA_MIN_CEQID = 0,
189 IRDMA_MAX_CEQID = 1023,
190 IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
191 IRDMA_MIN_CQID = 0,
192 IRDMA_MAX_CQID = 524287,
193 IRDMA_MIN_AEQ_ENTRIES = 1,
194 IRDMA_MAX_AEQ_ENTRIES = 524287,
195 IRDMA_MIN_CEQ_ENTRIES = 1,
196 IRDMA_MAX_CEQ_ENTRIES = 262143,
197 IRDMA_MIN_CQ_SIZE = 1,
198 IRDMA_MAX_CQ_SIZE = 1048575,
199 IRDMA_DB_ID_ZERO = 0,
200 /* 64K + 1 */
201 IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
202 /* 64K +1 */
203 IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
204 IRDMA_MAX_PE_ENA_VF_COUNT = 32,
205 IRDMA_MAX_VF_FPM_ID = 47,
206 IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
207 IRDMA_MAX_INLINE_DATA_SIZE = 101,
208 IRDMA_MAX_WQ_ENTRIES = 32768,
209 IRDMA_Q2_BUF_SIZE = 256,
210 IRDMA_QP_CTX_SIZE = 256,
211 IRDMA_MAX_PDS = 262144,
212 };
213
214 enum irdma_addressing_type {
215 IRDMA_ADDR_TYPE_ZERO_BASED = 0,
216 IRDMA_ADDR_TYPE_VA_BASED = 1,
217 };
218
219 enum irdma_flush_opcode {
220 FLUSH_INVALID = 0,
221 FLUSH_GENERAL_ERR,
222 FLUSH_PROT_ERR,
223 FLUSH_REM_ACCESS_ERR,
224 FLUSH_LOC_QP_OP_ERR,
225 FLUSH_REM_OP_ERR,
226 FLUSH_LOC_LEN_ERR,
227 FLUSH_FATAL_ERR,
228 FLUSH_RETRY_EXC_ERR,
229 FLUSH_MW_BIND_ERR,
230 FLUSH_REM_INV_REQ_ERR,
231 FLUSH_RNR_RETRY_EXC_ERR,
232 };
233
234 enum irdma_qp_event_type {
235 IRDMA_QP_EVENT_CATASTROPHIC,
236 IRDMA_QP_EVENT_ACCESS_ERR,
237 IRDMA_QP_EVENT_REQ_ERR,
238 };
239
240 enum irdma_cmpl_status {
241 IRDMA_COMPL_STATUS_SUCCESS = 0,
242 IRDMA_COMPL_STATUS_FLUSHED,
243 IRDMA_COMPL_STATUS_INVALID_WQE,
244 IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
245 IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
246 IRDMA_COMPL_STATUS_INVALID_STAG,
247 IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
248 IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
249 IRDMA_COMPL_STATUS_INVALID_PD_ID,
250 IRDMA_COMPL_STATUS_WRAP_ERROR,
251 IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
252 IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
253 IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
254 IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
255 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
256 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
257 IRDMA_COMPL_STATUS_INVALID_FBO,
258 IRDMA_COMPL_STATUS_INVALID_LEN,
259 IRDMA_COMPL_STATUS_INVALID_ACCESS,
260 IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
261 IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
262 IRDMA_COMPL_STATUS_INVALID_REGION,
263 IRDMA_COMPL_STATUS_INVALID_WINDOW,
264 IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
265 IRDMA_COMPL_STATUS_UNKNOWN,
266 };
267
268 enum irdma_cmpl_notify {
269 IRDMA_CQ_COMPL_EVENT = 0,
270 IRDMA_CQ_COMPL_SOLICITED = 1,
271 };
272
273 enum irdma_qp_caps {
274 IRDMA_WRITE_WITH_IMM = 1,
275 IRDMA_SEND_WITH_IMM = 2,
276 IRDMA_ROCE = 4,
277 IRDMA_PUSH_MODE = 8,
278 };
279
280 struct irdma_qp_uk;
281 struct irdma_cq_uk;
282 struct irdma_qp_uk_init_info;
283 struct irdma_cq_uk_init_info;
284
285 struct irdma_ring {
286 volatile u32 head;
287 volatile u32 tail; /* effective tail */
288 u32 size;
289 };
290
291 struct irdma_cqe {
292 __le64 buf[IRDMA_CQE_SIZE];
293 };
294
295 struct irdma_extended_cqe {
296 __le64 buf[IRDMA_EXTENDED_CQE_SIZE];
297 };
298
299 struct irdma_post_send {
300 irdma_sgl sg_list;
301 u32 num_sges;
302 u32 qkey;
303 u32 dest_qp;
304 u32 ah_id;
305 };
306
307 struct irdma_post_rq_info {
308 u64 wr_id;
309 irdma_sgl sg_list;
310 u32 num_sges;
311 };
312
313 struct irdma_rdma_write {
314 irdma_sgl lo_sg_list;
315 u32 num_lo_sges;
316 struct ib_sge rem_addr;
317 };
318
319 struct irdma_rdma_read {
320 irdma_sgl lo_sg_list;
321 u32 num_lo_sges;
322 struct ib_sge rem_addr;
323 };
324
325 struct irdma_bind_window {
326 irdma_stag mr_stag;
327 u64 bind_len;
328 void *va;
329 enum irdma_addressing_type addressing_type;
330 bool ena_reads:1;
331 bool ena_writes:1;
332 irdma_stag mw_stag;
333 bool mem_window_type_1:1;
334 };
335
336 struct irdma_inv_local_stag {
337 irdma_stag target_stag;
338 };
339
340 struct irdma_post_sq_info {
341 u64 wr_id;
342 u8 op_type;
343 u8 l4len;
344 bool signaled:1;
345 bool read_fence:1;
346 bool local_fence:1;
347 bool inline_data:1;
348 bool imm_data_valid:1;
349 bool push_wqe:1;
350 bool report_rtt:1;
351 bool udp_hdr:1;
352 bool defer_flag:1;
353 u32 imm_data;
354 u32 stag_to_inv;
355 union {
356 struct irdma_post_send send;
357 struct irdma_rdma_write rdma_write;
358 struct irdma_rdma_read rdma_read;
359 struct irdma_bind_window bind_window;
360 struct irdma_inv_local_stag inv_local_stag;
361 } op;
362 };
363
364 struct irdma_cq_poll_info {
365 u64 wr_id;
366 irdma_qp_handle qp_handle;
367 u32 bytes_xfered;
368 u32 qp_id;
369 u32 ud_src_qpn;
370 u32 imm_data;
371 irdma_stag inv_stag; /* or L_R_Key */
372 enum irdma_cmpl_status comp_status;
373 u16 major_err;
374 u16 minor_err;
375 u16 ud_vlan;
376 u8 ud_smac[6];
377 u8 op_type;
378 u8 q_type;
379 bool stag_invalid_set:1; /* or L_R_Key set */
380 bool push_dropped:1;
381 bool error:1;
382 bool solicited_event:1;
383 bool ipv4:1;
384 bool ud_vlan_valid:1;
385 bool ud_smac_valid:1;
386 bool imm_valid:1;
387 bool signaled:1;
388 union {
389 u32 tcp_sqn;
390 u32 roce_psn;
391 u32 rtt;
392 u32 raw;
393 } stat;
394 };
395
396 struct qp_err_code {
397 enum irdma_flush_opcode flush_code;
398 enum irdma_qp_event_type event_type;
399 };
400
401 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
402 struct irdma_post_sq_info *info, bool post_sq);
403 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
404 struct irdma_post_sq_info *info, bool post_sq);
405 int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
406 bool post_sq);
407 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
408 struct irdma_post_rq_info *info);
409 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
410 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
411 bool inv_stag, bool post_sq);
412 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
413 bool post_sq);
414 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
415 bool post_sq);
416 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
417 struct irdma_post_sq_info *info,
418 bool post_sq);
419
420 struct irdma_wqe_uk_ops {
421 void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list, u32 num_sges, u8 polarity);
422 u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
423 void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
424 u8 valid);
425 void (*iw_set_mw_bind_wqe)(__le64 *wqe,
426 struct irdma_bind_window *op_info);
427 };
428
429 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
430 struct irdma_cq_poll_info *info);
431 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
432 enum irdma_cmpl_notify cq_notify);
433 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
434 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
435 int irdma_uk_cq_init(struct irdma_cq_uk *cq,
436 struct irdma_cq_uk_init_info *info);
437 int irdma_uk_qp_init(struct irdma_qp_uk *qp,
438 struct irdma_qp_uk_init_info *info);
439 void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
440 u8 *rq_shift);
441 int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
442 u32 *sq_depth, u8 *sq_shift);
443 int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
444 u32 *rq_depth, u8 *rq_shift);
445 struct irdma_sq_uk_wr_trk_info {
446 u64 wrid;
447 u32 wr_len;
448 u16 quanta;
449 u8 signaled;
450 u8 reserved[1];
451 };
452
453 struct irdma_qp_quanta {
454 __le64 elem[IRDMA_WQE_SIZE];
455 };
456
457 struct irdma_qp_uk {
458 struct irdma_qp_quanta *sq_base;
459 struct irdma_qp_quanta *rq_base;
460 struct irdma_uk_attrs *uk_attrs;
461 u32 IOMEM *wqe_alloc_db;
462 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
463 struct irdma_sig_wr_trk_info *sq_sigwrtrk_array;
464 u64 *rq_wrid_array;
465 __le64 *shadow_area;
466 __le32 *push_db;
467 __le64 *push_wqe;
468 struct irdma_ring sq_ring;
469 struct irdma_ring sq_sig_ring;
470 struct irdma_ring rq_ring;
471 struct irdma_ring initial_ring;
472 u32 qp_id;
473 u32 qp_caps;
474 u32 sq_size;
475 u32 rq_size;
476 u32 max_sq_frag_cnt;
477 u32 max_rq_frag_cnt;
478 u32 max_inline_data;
479 u32 last_rx_cmpl_idx;
480 u32 last_tx_cmpl_idx;
481 struct irdma_wqe_uk_ops wqe_ops;
482 u16 conn_wqes;
483 u8 qp_type;
484 u8 swqe_polarity;
485 u8 swqe_polarity_deferred;
486 u8 rwqe_polarity;
487 u8 rq_wqe_size;
488 u8 rq_wqe_size_multiplier;
489 u8 start_wqe_idx;
490 bool deferred_flag:1;
491 bool push_mode:1; /* whether the last post wqe was pushed */
492 bool push_dropped:1;
493 bool first_sq_wq:1;
494 bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
495 bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
496 bool destroy_pending:1; /* Indicates the QP is being destroyed */
497 void *back_qp;
498 spinlock_t *lock;
499 u8 dbg_rq_flushed;
500 u16 ord_cnt;
501 u8 sq_flush_seen;
502 u8 rq_flush_seen;
503 u8 rd_fence_rate;
504 };
505
506 struct irdma_cq_uk {
507 struct irdma_cqe *cq_base;
508 u32 IOMEM *cqe_alloc_db;
509 u32 IOMEM *cq_ack_db;
510 __le64 *shadow_area;
511 u32 cq_id;
512 u32 cq_size;
513 struct irdma_ring cq_ring;
514 u8 polarity;
515 bool avoid_mem_cflct:1;
516 };
517
518 struct irdma_qp_uk_init_info {
519 struct irdma_qp_quanta *sq;
520 struct irdma_qp_quanta *rq;
521 struct irdma_uk_attrs *uk_attrs;
522 u32 IOMEM *wqe_alloc_db;
523 __le64 *shadow_area;
524 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
525 struct irdma_sig_wr_trk_info *sq_sigwrtrk_array;
526 u64 *rq_wrid_array;
527 u32 qp_id;
528 u32 qp_caps;
529 u32 sq_size;
530 u32 rq_size;
531 u32 max_sq_frag_cnt;
532 u32 max_rq_frag_cnt;
533 u32 max_inline_data;
534 u32 sq_depth;
535 u32 rq_depth;
536 u8 first_sq_wq;
537 u8 start_wqe_idx;
538 u8 type;
539 u8 sq_shift;
540 u8 rq_shift;
541 u8 rd_fence_rate;
542 int abi_ver;
543 bool legacy_mode;
544 };
545
546 struct irdma_cq_uk_init_info {
547 u32 IOMEM *cqe_alloc_db;
548 u32 IOMEM *cq_ack_db;
549 struct irdma_cqe *cq_base;
550 __le64 *shadow_area;
551 u32 cq_size;
552 u32 cq_id;
553 bool avoid_mem_cflct;
554 };
555
556 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
557 u16 *quanta, u32 total_size,
558 struct irdma_post_sq_info *info);
559 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
560 int irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
561 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
562 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
563 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
564 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
565 u32 inline_data, u8 *shift);
566 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
567 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
568 void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
569 u32 wqe_idx, bool post_sq);
570 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
571
irdma_ae_to_qp_err_code(u16 ae_id)572 static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
573 {
574 struct qp_err_code qp_err = { 0 };
575
576 switch (ae_id) {
577 case IRDMA_AE_AMP_BOUNDS_VIOLATION:
578 case IRDMA_AE_AMP_INVALID_STAG:
579 case IRDMA_AE_AMP_RIGHTS_VIOLATION:
580 case IRDMA_AE_AMP_UNALLOCATED_STAG:
581 case IRDMA_AE_AMP_BAD_PD:
582 case IRDMA_AE_AMP_BAD_QP:
583 case IRDMA_AE_AMP_BAD_STAG_KEY:
584 case IRDMA_AE_AMP_BAD_STAG_INDEX:
585 case IRDMA_AE_AMP_TO_WRAP:
586 case IRDMA_AE_PRIV_OPERATION_DENIED:
587 qp_err.flush_code = FLUSH_PROT_ERR;
588 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
589 break;
590 case IRDMA_AE_UDA_XMIT_BAD_PD:
591 case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
592 qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
593 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
594 break;
595 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
596 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
597 case IRDMA_AE_UDA_L4LEN_INVALID:
598 case IRDMA_AE_DDP_UBE_INVALID_MO:
599 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
600 qp_err.flush_code = FLUSH_LOC_LEN_ERR;
601 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
602 break;
603 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
604 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
605 qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
606 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
607 break;
608 case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
609 case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
610 case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
611 case IRDMA_AE_AMP_MWBIND_VALID_STAG:
612 qp_err.flush_code = FLUSH_MW_BIND_ERR;
613 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
614 break;
615 case IRDMA_AE_LLP_TOO_MANY_RETRIES:
616 qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
617 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
618 break;
619 case IRDMA_AE_IB_INVALID_REQUEST:
620 qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
621 qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
622 break;
623 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
624 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
625 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
626 case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
627 case IRDMA_AE_IB_REMOTE_OP_ERROR:
628 qp_err.flush_code = FLUSH_REM_OP_ERR;
629 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
630 break;
631 case IRDMA_AE_LLP_TOO_MANY_RNRS:
632 qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
633 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
634 break;
635 case IRDMA_AE_LCE_QP_CATASTROPHIC:
636 qp_err.flush_code = FLUSH_FATAL_ERR;
637 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
638 break;
639 default:
640 qp_err.flush_code = FLUSH_GENERAL_ERR;
641 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
642 break;
643 }
644
645 return qp_err;
646 }
647 #endif /* IRDMA_USER_H */
648