1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ 2 /* 3 * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #ifndef _EFA_IO_H_ 7 #define _EFA_IO_H_ 8 9 #define EFA_IO_TX_DESC_NUM_BUFS 2 10 #define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1 11 #define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32 12 #define EFA_IO_TX_DESC_IMM_DATA_SIZE 4 13 #define EFA_IO_TX_DESC_INLINE_PBL_SIZE 1 14 15 enum efa_io_queue_type { 16 /* send queue (of a QP) */ 17 EFA_IO_SEND_QUEUE = 1, 18 /* recv queue (of a QP) */ 19 EFA_IO_RECV_QUEUE = 2, 20 }; 21 22 enum efa_io_send_op_type { 23 /* send message */ 24 EFA_IO_SEND = 0, 25 /* RDMA read */ 26 EFA_IO_RDMA_READ = 1, 27 /* RDMA write */ 28 EFA_IO_RDMA_WRITE = 2, 29 /* Fast MR registration */ 30 EFA_IO_FAST_REG = 3, 31 /* Fast MR invalidation */ 32 EFA_IO_FAST_INV = 4, 33 }; 34 35 enum efa_io_comp_status { 36 /* Successful completion */ 37 EFA_IO_COMP_STATUS_OK = 0, 38 /* Flushed during QP destroy */ 39 EFA_IO_COMP_STATUS_FLUSHED = 1, 40 /* Internal QP error */ 41 EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2, 42 /* Unsupported operation */ 43 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNSUPPORTED_OP = 3, 44 /* Bad AH */ 45 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4, 46 /* LKEY not registered or does not match IOVA */ 47 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5, 48 /* Message too long */ 49 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6, 50 /* RKEY not registered or does not match remote IOVA */ 51 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7, 52 /* Connection was reset by remote side */ 53 EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8, 54 /* Bad dest QP number (QP does not exist or is in error state) */ 55 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9, 56 /* Destination resource not ready (no WQEs posted on RQ) */ 57 EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10, 58 /* Receiver SGL too short */ 59 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11, 60 /* Unexpected status returned by responder */ 61 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12, 62 /* Unresponsive remote - was previously responsive */ 63 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13, 64 /* No valid AH at remote side (required for RDMA operations) */ 65 EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14, 66 /* Unreachable remote - never received a response */ 67 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15, 68 }; 69 70 enum efa_io_frwr_pbl_mode { 71 EFA_IO_FRWR_INLINE_PBL = 0, 72 EFA_IO_FRWR_DIRECT_PBL = 1, 73 }; 74 75 struct efa_io_tx_meta_desc { 76 /* Verbs-generated Request ID */ 77 u16 req_id; 78 79 /* 80 * control flags 81 * 3:0 : op_type - enum efa_io_send_op_type 82 * 4 : has_imm - immediate_data field carries valid 83 * data. 84 * 5 : inline_msg - inline mode - inline message data 85 * follows this descriptor (no buffer descriptors). 86 * Note that it is different from immediate data 87 * 6 : meta_extension - Extended metadata. MBZ 88 * 7 : meta_desc - Indicates metadata descriptor. 89 * Must be set. 90 */ 91 u8 ctrl1; 92 93 /* 94 * control flags 95 * 0 : phase 96 * 1 : reserved25 - MBZ 97 * 2 : first - Indicates first descriptor in 98 * transaction. Must be set. 99 * 3 : last - Indicates last descriptor in 100 * transaction. Must be set. 101 * 4 : comp_req - Indicates whether completion should 102 * be posted, after packet is transmitted. Valid only 103 * for the first descriptor 104 * 7:5 : reserved29 - MBZ 105 */ 106 u8 ctrl2; 107 108 u16 dest_qp_num; 109 110 /* 111 * If inline_msg bit is set, length of inline message in bytes, 112 * otherwise length of SGL (number of buffers). 113 */ 114 u16 length; 115 116 /* 117 * immediate data: if has_imm is set, then this field is included within 118 * Tx message and reported in remote Rx completion. 119 */ 120 u32 immediate_data; 121 122 u16 ah; 123 124 u16 reserved; 125 126 /* Queue key */ 127 u32 qkey; 128 129 u8 reserved2[12]; 130 }; 131 132 /* 133 * Tx queue buffer descriptor, for any transport type. Preceded by metadata 134 * descriptor. 135 */ 136 struct efa_io_tx_buf_desc { 137 /* length in bytes */ 138 u32 length; 139 140 /* 141 * 23:0 : lkey - local memory translation key 142 * 31:24 : reserved - MBZ 143 */ 144 u32 lkey; 145 146 /* Buffer address bits[31:0] */ 147 u32 buf_addr_lo; 148 149 /* Buffer address bits[63:32] */ 150 u32 buf_addr_hi; 151 }; 152 153 struct efa_io_remote_mem_addr { 154 /* length in bytes */ 155 u32 length; 156 157 /* remote memory translation key */ 158 u32 rkey; 159 160 /* Buffer address bits[31:0] */ 161 u32 buf_addr_lo; 162 163 /* Buffer address bits[63:32] */ 164 u32 buf_addr_hi; 165 }; 166 167 struct efa_io_rdma_req { 168 /* Remote memory address */ 169 struct efa_io_remote_mem_addr remote_mem; 170 171 /* Local memory address */ 172 struct efa_io_tx_buf_desc local_mem[1]; 173 }; 174 175 struct efa_io_fast_mr_reg_req { 176 /* Updated local key of the MR after lkey/rkey increment */ 177 u32 lkey; 178 179 /* 180 * permissions 181 * 0 : local_write_enable - Local write permissions: 182 * must be set for RQ buffers and buffers posted for 183 * RDMA Read requests 184 * 1 : remote_write_enable - Remote write 185 * permissions: must be set to enable RDMA write to 186 * the region 187 * 2 : remote_read_enable - Remote read permissions: 188 * must be set to enable RDMA read from the region 189 * 7:3 : reserved2 - MBZ 190 */ 191 u8 permissions; 192 193 /* 194 * control flags 195 * 4:0 : phys_page_size_shift - page size is (1 << 196 * phys_page_size_shift) 197 * 6:5 : pbl_mode - enum efa_io_frwr_pbl_mode 198 * 7 : reserved - MBZ 199 */ 200 u8 flags; 201 202 /* MBZ */ 203 u8 reserved[2]; 204 205 /* IO Virtual Address associated with this MR */ 206 u64 iova; 207 208 /* Memory region length, in bytes */ 209 u64 mr_length; 210 211 /* Physical Buffer List, each element is page-aligned. */ 212 union { 213 /* 214 * Inline array of physical page addresses (optimization 215 * for short region activation). 216 */ 217 u64 inline_array[1]; 218 219 /* points to PBL (Currently only direct) */ 220 u64 dma_addr; 221 } pbl; 222 }; 223 224 struct efa_io_fast_mr_inv_req { 225 /* Local key of the MR to invalidate */ 226 u32 lkey; 227 228 /* MBZ */ 229 u8 reserved[28]; 230 }; 231 232 /* 233 * Tx WQE, composed of tx meta descriptors followed by either tx buffer 234 * descriptors or inline data 235 */ 236 struct efa_io_tx_wqe { 237 /* TX meta */ 238 struct efa_io_tx_meta_desc meta; 239 240 union { 241 /* Send buffer descriptors */ 242 struct efa_io_tx_buf_desc sgl[2]; 243 244 u8 inline_data[32]; 245 246 /* RDMA local and remote memory addresses */ 247 struct efa_io_rdma_req rdma_req; 248 249 /* Fast registration */ 250 struct efa_io_fast_mr_reg_req reg_mr_req; 251 252 /* Fast invalidation */ 253 struct efa_io_fast_mr_inv_req inv_mr_req; 254 } data; 255 }; 256 257 /* 258 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer 259 * descriptors. 260 */ 261 struct efa_io_rx_desc { 262 /* Buffer address bits[31:0] */ 263 u32 buf_addr_lo; 264 265 /* Buffer Pointer[63:32] */ 266 u32 buf_addr_hi; 267 268 /* Verbs-generated request id. */ 269 u16 req_id; 270 271 /* Length in bytes. */ 272 u16 length; 273 274 /* 275 * LKey and control flags 276 * 23:0 : lkey 277 * 29:24 : reserved - MBZ 278 * 30 : first - Indicates first descriptor in WQE 279 * 31 : last - Indicates last descriptor in WQE 280 */ 281 u32 lkey_ctrl; 282 }; 283 284 /* Common IO completion descriptor */ 285 struct efa_io_cdesc_common { 286 /* 287 * verbs-generated request ID, as provided in the completed tx or rx 288 * descriptor. 289 */ 290 u16 req_id; 291 292 u8 status; 293 294 /* 295 * flags 296 * 0 : phase - Phase bit 297 * 2:1 : q_type - enum efa_io_queue_type: send/recv 298 * 3 : has_imm - indicates that immediate data is 299 * present - for RX completions only 300 * 6:4 : op_type - enum efa_io_send_op_type 301 * 7 : unsolicited - indicates that there is no 302 * matching request - for RDMA with imm. RX only 303 */ 304 u8 flags; 305 306 /* local QP number */ 307 u16 qp_num; 308 }; 309 310 /* Tx completion descriptor */ 311 struct efa_io_tx_cdesc { 312 /* Common completion info */ 313 struct efa_io_cdesc_common common; 314 315 /* MBZ */ 316 u16 reserved16; 317 }; 318 319 /* Rx Completion Descriptor */ 320 struct efa_io_rx_cdesc { 321 /* Common completion info */ 322 struct efa_io_cdesc_common common; 323 324 /* Transferred length bits[15:0] */ 325 u16 length; 326 327 /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */ 328 u16 ah; 329 330 u16 src_qp_num; 331 332 /* Immediate data */ 333 u32 imm; 334 }; 335 336 /* Rx Completion Descriptor RDMA write info */ 337 struct efa_io_rx_cdesc_rdma_write { 338 /* Transferred length bits[31:16] */ 339 u16 length_hi; 340 }; 341 342 /* Extended Rx Completion Descriptor */ 343 struct efa_io_rx_cdesc_ex { 344 /* Base RX completion info */ 345 struct efa_io_rx_cdesc base; 346 347 union { 348 struct efa_io_rx_cdesc_rdma_write rdma_write; 349 350 /* 351 * Valid only in case of unknown AH (0xFFFF) and CQ 352 * set_src_addr is enabled. 353 */ 354 u8 src_addr[16]; 355 } u; 356 }; 357 358 /* tx_meta_desc */ 359 #define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0) 360 #define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4) 361 #define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5) 362 #define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6) 363 #define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7) 364 #define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0) 365 #define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2) 366 #define EFA_IO_TX_META_DESC_LAST_MASK BIT(3) 367 #define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4) 368 369 /* tx_buf_desc */ 370 #define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0) 371 372 /* fast_mr_reg_req */ 373 #define EFA_IO_FAST_MR_REG_REQ_LOCAL_WRITE_ENABLE_MASK BIT(0) 374 #define EFA_IO_FAST_MR_REG_REQ_REMOTE_WRITE_ENABLE_MASK BIT(1) 375 #define EFA_IO_FAST_MR_REG_REQ_REMOTE_READ_ENABLE_MASK BIT(2) 376 #define EFA_IO_FAST_MR_REG_REQ_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0) 377 #define EFA_IO_FAST_MR_REG_REQ_PBL_MODE_MASK GENMASK(6, 5) 378 379 /* rx_desc */ 380 #define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0) 381 #define EFA_IO_RX_DESC_FIRST_MASK BIT(30) 382 #define EFA_IO_RX_DESC_LAST_MASK BIT(31) 383 384 /* cdesc_common */ 385 #define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0) 386 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1) 387 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3) 388 #define EFA_IO_CDESC_COMMON_OP_TYPE_MASK GENMASK(6, 4) 389 #define EFA_IO_CDESC_COMMON_UNSOLICITED_MASK BIT(7) 390 391 #endif /* _EFA_IO_H_ */ 392