1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2022 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 /*$FreeBSD$*/ 35 36 #ifndef IRDMA_USER_H 37 #define IRDMA_USER_H 38 39 #include "osdep.h" 40 41 #define irdma_handle void * 42 #define irdma_adapter_handle irdma_handle 43 #define irdma_qp_handle irdma_handle 44 #define irdma_cq_handle irdma_handle 45 #define irdma_pd_id irdma_handle 46 #define irdma_stag_handle irdma_handle 47 #define irdma_stag_index u32 48 #define irdma_stag u32 49 #define irdma_stag_key u8 50 #define irdma_tagged_offset u64 51 #define irdma_access_privileges u32 52 #define irdma_physical_fragment u64 53 #define irdma_address_list u64 * 54 #define irdma_sgl struct irdma_sge * 55 56 #define IRDMA_MAX_MR_SIZE 0x200000000000ULL 57 58 #define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01 59 #define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02 60 #define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04 61 #define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05 62 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08 63 #define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a 64 #define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10 65 #define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20 66 #define IRDMA_ACCESS_FLAGS_ALL 0x3f 67 68 #define IRDMA_OP_TYPE_RDMA_WRITE 0x00 69 #define IRDMA_OP_TYPE_RDMA_READ 0x01 70 #define IRDMA_OP_TYPE_SEND 0x03 71 #define IRDMA_OP_TYPE_SEND_INV 0x04 72 #define IRDMA_OP_TYPE_SEND_SOL 0x05 73 #define IRDMA_OP_TYPE_SEND_SOL_INV 0x06 74 #define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d 75 #define IRDMA_OP_TYPE_BIND_MW 0x08 76 #define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09 77 #define IRDMA_OP_TYPE_INV_STAG 0x0a 78 #define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b 79 #define IRDMA_OP_TYPE_NOP 0x0c 80 #define IRDMA_OP_TYPE_REC 0x3e 81 #define IRDMA_OP_TYPE_REC_IMM 0x3f 82 83 #define IRDMA_FLUSH_MAJOR_ERR 1 84 #define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe 85 86 /* Async Events codes */ 87 #define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102 88 #define IRDMA_AE_AMP_INVALID_STAG 0x0103 89 #define IRDMA_AE_AMP_BAD_QP 0x0104 90 #define IRDMA_AE_AMP_BAD_PD 0x0105 91 #define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106 92 #define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107 93 #define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108 94 #define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109 95 #define IRDMA_AE_AMP_TO_WRAP 0x010a 96 #define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c 97 #define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d 98 #define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e 99 #define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110 100 #define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111 101 #define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112 102 #define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113 103 #define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114 104 #define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115 105 #define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116 106 #define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117 107 #define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118 108 #define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119 109 #define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a 110 #define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b 111 #define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c 112 #define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d 113 #define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e 114 #define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f 115 #define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120 116 #define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121 117 #define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132 118 #define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133 119 #define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134 120 #define IRDMA_AE_UDA_L4LEN_INVALID 0x0135 121 #define IRDMA_AE_BAD_CLOSE 0x0201 122 #define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202 123 #define IRDMA_AE_CQ_OPERATION_ERROR 0x0203 124 #define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205 125 #define IRDMA_AE_STAG_ZERO_INVALID 0x0206 126 #define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207 127 #define IRDMA_AE_IB_INVALID_REQUEST 0x0208 128 #define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a 129 #define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b 130 #define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c 131 #define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d 132 #define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e 133 #define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220 134 #define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301 135 #define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303 136 #define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304 137 #define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305 138 #define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306 139 #define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307 140 #define IRDMA_AE_DDP_NO_L_BIT 0x0308 141 #define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311 142 #define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312 143 #define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313 144 #define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314 145 #define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316 146 #define IRDMA_AE_ROCE_EMPTY_MCG 0x0380 147 #define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381 148 #define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382 149 #define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383 150 #define IRDMA_AE_INVALID_ARP_ENTRY 0x0401 151 #define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402 152 #define IRDMA_AE_STALE_ARP_ENTRY 0x0403 153 #define IRDMA_AE_INVALID_AH_ENTRY 0x0406 154 #define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501 155 #define IRDMA_AE_LLP_CONNECTION_RESET 0x0502 156 #define IRDMA_AE_LLP_FIN_RECEIVED 0x0503 157 #define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504 158 #define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505 159 #define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507 160 #define IRDMA_AE_LLP_SYN_RECEIVED 0x0508 161 #define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509 162 #define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a 163 #define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b 164 #define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c 165 #define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e 166 #define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520 167 #define IRDMA_AE_RESET_SENT 0x0601 168 #define IRDMA_AE_TERMINATE_SENT 0x0602 169 #define IRDMA_AE_RESET_NOT_SENT 0x0603 170 #define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700 171 #define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701 172 #define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702 173 #define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900 174 175 enum irdma_device_caps_const { 176 IRDMA_WQE_SIZE = 4, 177 IRDMA_CQP_WQE_SIZE = 8, 178 IRDMA_CQE_SIZE = 4, 179 IRDMA_EXTENDED_CQE_SIZE = 8, 180 IRDMA_AEQE_SIZE = 2, 181 IRDMA_CEQE_SIZE = 1, 182 IRDMA_CQP_CTX_SIZE = 8, 183 IRDMA_SHADOW_AREA_SIZE = 8, 184 IRDMA_GATHER_STATS_BUF_SIZE = 1024, 185 IRDMA_MIN_IW_QP_ID = 0, 186 IRDMA_QUERY_FPM_BUF_SIZE = 176, 187 IRDMA_COMMIT_FPM_BUF_SIZE = 176, 188 IRDMA_MAX_IW_QP_ID = 262143, 189 IRDMA_MIN_CEQID = 0, 190 IRDMA_MAX_CEQID = 1023, 191 IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1, 192 IRDMA_MIN_CQID = 0, 193 IRDMA_MAX_CQID = 524287, 194 IRDMA_MIN_AEQ_ENTRIES = 1, 195 IRDMA_MAX_AEQ_ENTRIES = 524287, 196 IRDMA_MIN_CEQ_ENTRIES = 1, 197 IRDMA_MAX_CEQ_ENTRIES = 262143, 198 IRDMA_MIN_CQ_SIZE = 1, 199 IRDMA_MAX_CQ_SIZE = 1048575, 200 IRDMA_DB_ID_ZERO = 0, 201 /* 64K + 1 */ 202 IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537, 203 /* 64K +1 */ 204 IRDMA_MAX_INBOUND_MSG_SIZE = 65537, 205 IRDMA_MAX_PUSH_PAGE_COUNT = 1024, 206 IRDMA_MAX_PE_ENA_VF_COUNT = 32, 207 IRDMA_MAX_VF_FPM_ID = 47, 208 IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496, 209 IRDMA_MAX_INLINE_DATA_SIZE = 101, 210 IRDMA_MAX_WQ_ENTRIES = 32768, 211 IRDMA_Q2_BUF_SIZE = 256, 212 IRDMA_QP_CTX_SIZE = 256, 213 IRDMA_MAX_PDS = 262144, 214 IRDMA_MIN_WQ_SIZE_GEN2 = 8, 215 }; 216 217 enum irdma_addressing_type { 218 IRDMA_ADDR_TYPE_ZERO_BASED = 0, 219 IRDMA_ADDR_TYPE_VA_BASED = 1, 220 }; 221 222 enum irdma_flush_opcode { 223 FLUSH_INVALID = 0, 224 FLUSH_GENERAL_ERR, 225 FLUSH_PROT_ERR, 226 FLUSH_REM_ACCESS_ERR, 227 FLUSH_LOC_QP_OP_ERR, 228 FLUSH_REM_OP_ERR, 229 FLUSH_LOC_LEN_ERR, 230 FLUSH_FATAL_ERR, 231 FLUSH_RETRY_EXC_ERR, 232 FLUSH_MW_BIND_ERR, 233 FLUSH_REM_INV_REQ_ERR, 234 }; 235 236 enum irdma_qp_event_type { 237 IRDMA_QP_EVENT_CATASTROPHIC, 238 IRDMA_QP_EVENT_ACCESS_ERR, 239 IRDMA_QP_EVENT_REQ_ERR, 240 }; 241 242 enum irdma_cmpl_status { 243 IRDMA_COMPL_STATUS_SUCCESS = 0, 244 IRDMA_COMPL_STATUS_FLUSHED, 245 IRDMA_COMPL_STATUS_INVALID_WQE, 246 IRDMA_COMPL_STATUS_QP_CATASTROPHIC, 247 IRDMA_COMPL_STATUS_REMOTE_TERMINATION, 248 IRDMA_COMPL_STATUS_INVALID_STAG, 249 IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION, 250 IRDMA_COMPL_STATUS_ACCESS_VIOLATION, 251 IRDMA_COMPL_STATUS_INVALID_PD_ID, 252 IRDMA_COMPL_STATUS_WRAP_ERROR, 253 IRDMA_COMPL_STATUS_STAG_INVALID_PDID, 254 IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD, 255 IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED, 256 IRDMA_COMPL_STATUS_STAG_NOT_INVALID, 257 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE, 258 IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY, 259 IRDMA_COMPL_STATUS_INVALID_FBO, 260 IRDMA_COMPL_STATUS_INVALID_LEN, 261 IRDMA_COMPL_STATUS_INVALID_ACCESS, 262 IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG, 263 IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS, 264 IRDMA_COMPL_STATUS_INVALID_REGION, 265 IRDMA_COMPL_STATUS_INVALID_WINDOW, 266 IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN, 267 IRDMA_COMPL_STATUS_UNKNOWN, 268 }; 269 270 enum irdma_cmpl_notify { 271 IRDMA_CQ_COMPL_EVENT = 0, 272 IRDMA_CQ_COMPL_SOLICITED = 1, 273 }; 274 275 enum irdma_qp_caps { 276 IRDMA_WRITE_WITH_IMM = 1, 277 IRDMA_SEND_WITH_IMM = 2, 278 IRDMA_ROCE = 4, 279 IRDMA_PUSH_MODE = 8, 280 }; 281 282 struct irdma_qp_uk; 283 struct irdma_cq_uk; 284 struct irdma_qp_uk_init_info; 285 struct irdma_cq_uk_init_info; 286 287 struct irdma_sge { 288 irdma_tagged_offset tag_off; 289 u32 len; 290 irdma_stag stag; 291 }; 292 293 struct irdma_ring { 294 volatile u32 head; 295 volatile u32 tail; /* effective tail */ 296 u32 size; 297 }; 298 299 struct irdma_cqe { 300 __le64 buf[IRDMA_CQE_SIZE]; 301 }; 302 303 struct irdma_extended_cqe { 304 __le64 buf[IRDMA_EXTENDED_CQE_SIZE]; 305 }; 306 307 struct irdma_post_send { 308 irdma_sgl sg_list; 309 u32 num_sges; 310 u32 qkey; 311 u32 dest_qp; 312 u32 ah_id; 313 }; 314 315 struct irdma_post_rq_info { 316 u64 wr_id; 317 irdma_sgl sg_list; 318 u32 num_sges; 319 }; 320 321 struct irdma_rdma_write { 322 irdma_sgl lo_sg_list; 323 u32 num_lo_sges; 324 struct irdma_sge rem_addr; 325 }; 326 327 struct irdma_rdma_read { 328 irdma_sgl lo_sg_list; 329 u32 num_lo_sges; 330 struct irdma_sge rem_addr; 331 }; 332 333 struct irdma_bind_window { 334 irdma_stag mr_stag; 335 u64 bind_len; 336 void *va; 337 enum irdma_addressing_type addressing_type; 338 bool ena_reads:1; 339 bool ena_writes:1; 340 irdma_stag mw_stag; 341 bool mem_window_type_1:1; 342 }; 343 344 struct irdma_inv_local_stag { 345 irdma_stag target_stag; 346 }; 347 348 struct irdma_post_sq_info { 349 u64 wr_id; 350 u8 op_type; 351 u8 l4len; 352 bool signaled:1; 353 bool read_fence:1; 354 bool local_fence:1; 355 bool inline_data:1; 356 bool imm_data_valid:1; 357 bool push_wqe:1; 358 bool report_rtt:1; 359 bool udp_hdr:1; 360 bool defer_flag:1; 361 u32 imm_data; 362 u32 stag_to_inv; 363 union { 364 struct irdma_post_send send; 365 struct irdma_rdma_write rdma_write; 366 struct irdma_rdma_read rdma_read; 367 struct irdma_bind_window bind_window; 368 struct irdma_inv_local_stag inv_local_stag; 369 } op; 370 }; 371 372 struct irdma_cq_poll_info { 373 u64 wr_id; 374 irdma_qp_handle qp_handle; 375 u32 bytes_xfered; 376 u32 qp_id; 377 u32 ud_src_qpn; 378 u32 imm_data; 379 irdma_stag inv_stag; /* or L_R_Key */ 380 enum irdma_cmpl_status comp_status; 381 u16 major_err; 382 u16 minor_err; 383 u16 ud_vlan; 384 u8 ud_smac[6]; 385 u8 op_type; 386 u8 q_type; 387 bool stag_invalid_set:1; /* or L_R_Key set */ 388 bool push_dropped:1; 389 bool error:1; 390 bool solicited_event:1; 391 bool ipv4:1; 392 bool ud_vlan_valid:1; 393 bool ud_smac_valid:1; 394 bool imm_valid:1; 395 bool signaled:1; 396 union { 397 u32 tcp_sqn; 398 u32 roce_psn; 399 u32 rtt; 400 u32 raw; 401 } stat; 402 }; 403 404 struct qp_err_code { 405 enum irdma_flush_opcode flush_code; 406 enum irdma_qp_event_type event_type; 407 }; 408 409 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, 410 struct irdma_post_sq_info *info, bool post_sq); 411 int irdma_uk_inline_send(struct irdma_qp_uk *qp, 412 struct irdma_post_sq_info *info, bool post_sq); 413 int irdma_uk_mw_bind(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, 414 bool post_sq); 415 int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, 416 bool post_sq); 417 int irdma_uk_post_receive(struct irdma_qp_uk *qp, 418 struct irdma_post_rq_info *info); 419 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp); 420 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, 421 bool inv_stag, bool post_sq); 422 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, 423 bool post_sq); 424 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, 425 bool post_sq); 426 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, 427 struct irdma_post_sq_info *info, 428 bool post_sq); 429 430 struct irdma_wqe_uk_ops { 431 void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity); 432 u16 (*iw_inline_data_size_to_quanta)(u32 data_size); 433 void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge, 434 u8 valid); 435 void (*iw_set_mw_bind_wqe)(__le64 *wqe, 436 struct irdma_bind_window *op_info); 437 }; 438 439 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, 440 struct irdma_cq_poll_info *info); 441 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq, 442 enum irdma_cmpl_notify cq_notify); 443 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size); 444 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt); 445 int irdma_uk_cq_init(struct irdma_cq_uk *cq, 446 struct irdma_cq_uk_init_info *info); 447 int irdma_uk_qp_init(struct irdma_qp_uk *qp, 448 struct irdma_qp_uk_init_info *info); 449 void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, 450 u8 *rq_shift); 451 int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, 452 u32 *sq_depth, u8 *sq_shift); 453 int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo, 454 u32 *rq_depth, u8 *rq_shift); 455 struct irdma_sq_uk_wr_trk_info { 456 u64 wrid; 457 u32 wr_len; 458 u16 quanta; 459 u8 signaled; 460 u8 reserved[1]; 461 }; 462 463 struct irdma_qp_quanta { 464 __le64 elem[IRDMA_WQE_SIZE]; 465 }; 466 467 struct irdma_qp_uk { 468 struct irdma_qp_quanta *sq_base; 469 struct irdma_qp_quanta *rq_base; 470 struct irdma_uk_attrs *uk_attrs; 471 u32 IOMEM *wqe_alloc_db; 472 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array; 473 struct irdma_sig_wr_trk_info *sq_sigwrtrk_array; 474 u64 *rq_wrid_array; 475 __le64 *shadow_area; 476 __le32 *push_db; 477 __le64 *push_wqe; 478 struct irdma_ring sq_ring; 479 struct irdma_ring sq_sig_ring; 480 struct irdma_ring rq_ring; 481 struct irdma_ring initial_ring; 482 u32 qp_id; 483 u32 qp_caps; 484 u32 sq_size; 485 u32 rq_size; 486 u32 max_sq_frag_cnt; 487 u32 max_rq_frag_cnt; 488 u32 max_inline_data; 489 u32 last_rx_cmpl_idx; 490 u32 last_tx_cmpl_idx; 491 struct irdma_wqe_uk_ops wqe_ops; 492 u16 conn_wqes; 493 u8 qp_type; 494 u8 swqe_polarity; 495 u8 swqe_polarity_deferred; 496 u8 rwqe_polarity; 497 u8 rq_wqe_size; 498 u8 rq_wqe_size_multiplier; 499 bool deferred_flag:1; 500 bool push_mode:1; /* whether the last post wqe was pushed */ 501 bool push_dropped:1; 502 bool first_sq_wq:1; 503 bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */ 504 bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */ 505 bool destroy_pending:1; /* Indicates the QP is being destroyed */ 506 void *back_qp; 507 pthread_spinlock_t *lock; 508 u8 dbg_rq_flushed; 509 u16 ord_cnt; 510 u8 sq_flush_seen; 511 u8 rq_flush_seen; 512 u8 rd_fence_rate; 513 }; 514 515 struct irdma_cq_uk { 516 struct irdma_cqe *cq_base; 517 u32 IOMEM *cqe_alloc_db; 518 u32 IOMEM *cq_ack_db; 519 __le64 *shadow_area; 520 u32 cq_id; 521 u32 cq_size; 522 struct irdma_ring cq_ring; 523 u8 polarity; 524 bool armed:1; 525 bool avoid_mem_cflct:1; 526 }; 527 528 struct irdma_qp_uk_init_info { 529 struct irdma_qp_quanta *sq; 530 struct irdma_qp_quanta *rq; 531 struct irdma_uk_attrs *uk_attrs; 532 u32 IOMEM *wqe_alloc_db; 533 __le64 *shadow_area; 534 struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array; 535 struct irdma_sig_wr_trk_info *sq_sigwrtrk_array; 536 u64 *rq_wrid_array; 537 u32 qp_id; 538 u32 qp_caps; 539 u32 sq_size; 540 u32 rq_size; 541 u32 max_sq_frag_cnt; 542 u32 max_rq_frag_cnt; 543 u32 max_inline_data; 544 u32 sq_depth; 545 u32 rq_depth; 546 u8 first_sq_wq; 547 u8 type; 548 u8 sq_shift; 549 u8 rq_shift; 550 u8 rd_fence_rate; 551 int abi_ver; 552 bool legacy_mode; 553 }; 554 555 struct irdma_cq_uk_init_info { 556 u32 IOMEM *cqe_alloc_db; 557 u32 IOMEM *cq_ack_db; 558 struct irdma_cqe *cq_base; 559 __le64 *shadow_area; 560 u32 cq_size; 561 u32 cq_id; 562 bool avoid_mem_cflct; 563 }; 564 565 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, 566 u16 *quanta, u32 total_size, 567 struct irdma_post_sq_info *info); 568 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx); 569 int irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq); 570 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq); 571 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta); 572 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size); 573 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, 574 u32 inline_data, u8 *shift); 575 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth); 576 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth); 577 int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth); 578 void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, 579 u32 wqe_idx, bool post_sq); 580 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); 581 582 static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id) 583 { 584 struct qp_err_code qp_err = { 0 }; 585 586 switch (ae_id) { 587 case IRDMA_AE_AMP_BOUNDS_VIOLATION: 588 case IRDMA_AE_AMP_INVALID_STAG: 589 case IRDMA_AE_AMP_RIGHTS_VIOLATION: 590 case IRDMA_AE_AMP_UNALLOCATED_STAG: 591 case IRDMA_AE_AMP_BAD_PD: 592 case IRDMA_AE_AMP_BAD_QP: 593 case IRDMA_AE_AMP_BAD_STAG_KEY: 594 case IRDMA_AE_AMP_BAD_STAG_INDEX: 595 case IRDMA_AE_AMP_TO_WRAP: 596 case IRDMA_AE_PRIV_OPERATION_DENIED: 597 qp_err.flush_code = FLUSH_PROT_ERR; 598 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; 599 break; 600 case IRDMA_AE_UDA_XMIT_BAD_PD: 601 case IRDMA_AE_WQE_UNEXPECTED_OPCODE: 602 qp_err.flush_code = FLUSH_LOC_QP_OP_ERR; 603 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 604 break; 605 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: 606 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: 607 case IRDMA_AE_UDA_L4LEN_INVALID: 608 case IRDMA_AE_DDP_UBE_INVALID_MO: 609 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: 610 qp_err.flush_code = FLUSH_LOC_LEN_ERR; 611 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 612 break; 613 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: 614 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: 615 qp_err.flush_code = FLUSH_REM_ACCESS_ERR; 616 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; 617 break; 618 case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS: 619 case IRDMA_AE_AMP_MWBIND_BIND_DISABLED: 620 case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS: 621 case IRDMA_AE_AMP_MWBIND_VALID_STAG: 622 qp_err.flush_code = FLUSH_MW_BIND_ERR; 623 qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; 624 break; 625 case IRDMA_AE_LLP_TOO_MANY_RETRIES: 626 qp_err.flush_code = FLUSH_RETRY_EXC_ERR; 627 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 628 break; 629 case IRDMA_AE_IB_INVALID_REQUEST: 630 qp_err.flush_code = FLUSH_REM_INV_REQ_ERR; 631 qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR; 632 break; 633 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 634 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 635 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: 636 case IRDMA_AE_IB_REMOTE_OP_ERROR: 637 qp_err.flush_code = FLUSH_REM_OP_ERR; 638 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 639 break; 640 case IRDMA_AE_LCE_QP_CATASTROPHIC: 641 qp_err.flush_code = FLUSH_FATAL_ERR; 642 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 643 break; 644 default: 645 qp_err.flush_code = FLUSH_GENERAL_ERR; 646 qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; 647 break; 648 } 649 650 return qp_err; 651 } 652 #endif /* IRDMA_USER_H */ 653