1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #ifndef _SYS_IB_IBTL_IBTI_H 26 #define _SYS_IB_IBTL_IBTI_H 27 28 /* 29 * ibti.h 30 * 31 * This file contains the IBTI prototypes and associated data structures. 32 * It is the only header file that should be included by IBTI clients. 33 */ 34 #include <sys/ib/ibtl/ibti_common.h> 35 36 #ifdef __cplusplus 37 extern "C" { 38 #endif 39 40 /* 41 * Allocate channel flags. 42 */ 43 typedef enum ibt_chan_alloc_flags_e { 44 IBT_ACHAN_NO_FLAGS = 0, 45 IBT_ACHAN_CLONE = (1 << 0), 46 IBT_ACHAN_USER_MAP = (1 << 1), 47 IBT_ACHAN_DEFER_ALLOC = (1 << 2), 48 IBT_ACHAN_USES_SRQ = (1 << 3), 49 IBT_ACHAN_USES_RSS = (1 << 4), 50 51 /* UD variants for FC support */ 52 IBT_ACHAN_USES_RFCI = (1 << 5), /* from RFCI pool */ 53 IBT_ACHAN_USES_FCMD = (1 << 6), 54 IBT_ACHAN_USES_FEXCH = (1 << 7) /* from FEXCH pool */ 55 } ibt_chan_alloc_flags_t; 56 57 58 /* 59 * Allocate RC channel ibt_alloc_rc_channel() argument. 60 */ 61 typedef struct ibt_rc_chan_alloc_args_s { 62 ibt_attr_flags_t rc_flags; /* Signal type etc */ 63 ibt_cep_flags_t rc_control; 64 uint8_t rc_hca_port_num; 65 ibt_chan_sizes_t rc_sizes; 66 ibt_cq_hdl_t rc_scq; /* Send CQ */ 67 ibt_cq_hdl_t rc_rcq; /* Recv CQ */ 68 ibt_pd_hdl_t rc_pd; /* PD */ 69 ibt_channel_hdl_t rc_clone_chan; /* Optional Clone handle */ 70 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 71 } ibt_rc_chan_alloc_args_t; 72 73 /* 74 * RC channel query attributes structure. 75 */ 76 typedef struct ibt_rc_chan_query_attr_s { 77 ib_guid_t rc_hca_guid; /* Local HCA GUID */ 78 ibt_cq_hdl_t rc_scq; /* SendCQ handle */ 79 ibt_cq_hdl_t rc_rcq; /* RecvCQ handle */ 80 ibt_pd_hdl_t rc_pd; /* PD Handle. */ 81 ibt_cep_state_t rc_state; /* Channel state */ 82 ib_mtu_t rc_path_mtu; 83 uint8_t rc_path_retry_cnt:3; 84 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 85 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 86 ibt_cep_path_t rc_prim_path; 87 ibt_cep_path_t rc_alt_path; 88 ibt_chan_sizes_t rc_chan_sizes; /* Queue/SGL sizes */ 89 uint8_t rc_rdma_ra_out; /* max RDMA-Reads/Atomics out */ 90 uint8_t rc_rdma_ra_in; /* max RDMA-Reads/Atomics in */ 91 ibt_attr_flags_t rc_flags; /* SQ Signaling Type etc */ 92 ibt_cep_flags_t rc_control; /* Control Flags */ 93 ibt_cep_cmstate_t rc_mig_state; 94 ib_qpn_t rc_qpn; /* Local QPN */ 95 ib_qpn_t rc_dst_qpn; /* Destination QPN */ 96 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 97 } ibt_rc_chan_query_attr_t; 98 99 /* 100 * RC Channel Modify Attributes definition. 101 * 102 * It is only possible to modify a channel that has previously been 103 * opened. The channel must either be in operational state (IBT_STATE_RTS) 104 * or paused (IBT_STATE_SQD). If channel is in paused state, then a modify 105 * operation will unpause the channel. 106 * 107 * See the ibt_modify_rc_channel() for details of the required corresponding 108 * modify flags. 109 * Not specified attributes should be set to "NULL" or "0". 110 */ 111 typedef struct ibt_rc_chan_modify_attr_s { 112 ibt_cep_flags_t rc_control; /* Channel Control Flags */ 113 uint_t rc_sq_sz; /* Set SQ Max outstanding WRs */ 114 uint_t rc_rq_sz; /* Set RQ Max outstanding WRs */ 115 116 ibt_adds_vect_t rc_prim_adds_vect; /* Primary Path Address */ 117 ibt_adds_vect_t rc_alt_adds_vect; /* Alternate Path Address */ 118 uint8_t rc_path_retry_cnt:3; 119 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 120 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 121 uint8_t rc_prim_port_num; /* Port of Primary Path */ 122 uint8_t rc_alt_port_num; /* Port of Alternate Path */ 123 uint8_t rc_rdma_ra_out; /* Initiator Depth, Number of */ 124 /* RDMA RD's & Atomics */ 125 /* outstanding. */ 126 uint8_t rc_rdma_ra_in; /* Responder resources for */ 127 /* handling incoming RDMA rds */ 128 /* and Atomics. */ 129 } ibt_rc_chan_modify_attr_t; 130 131 /* 132 * UD remote destination query attributes 133 */ 134 typedef struct ibt_ud_dest_query_attr_s { 135 ibt_hca_hdl_t ud_hca_hdl; /* Local HCA Handle */ 136 ib_qpn_t ud_dst_qpn; /* Destination QPN */ 137 ib_qkey_t ud_qkey; /* Q_Key */ 138 ibt_adds_vect_t ud_addr_vect; /* Address Information */ 139 ibt_pd_hdl_t ud_pd; 140 } ibt_ud_dest_query_attr_t; 141 142 /* 143 * Allocate UD channel ibt_alloc_ud_channel() arguments; see below at 144 * ibt_alloc_ud_channel() for a description of what's required and optional. 145 */ 146 typedef struct ibt_ud_chan_alloc_args_s { 147 ibt_attr_flags_t ud_flags; /* Sig type etc */ 148 uint8_t ud_hca_port_num; 149 uint16_t ud_pkey_ix; /* P_Key Index */ 150 ibt_chan_sizes_t ud_sizes; 151 ib_qkey_t ud_qkey; /* Q_Key */ 152 ibt_cq_hdl_t ud_scq; /* Send CQ */ 153 ibt_cq_hdl_t ud_rcq; /* Recv CQ */ 154 ibt_pd_hdl_t ud_pd; /* PD */ 155 ibt_channel_hdl_t ud_clone_chan; /* Optional clone handle */ 156 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 157 ibt_rss_attr_t ud_rss; 158 ibt_fc_attr_t ud_fc; 159 } ibt_ud_chan_alloc_args_t; 160 161 /* 162 * UD channel query attributes. 163 */ 164 typedef struct ibt_ud_chan_query_attr_s { 165 ib_qpn_t ud_qpn; /* QPN */ 166 ib_guid_t ud_hca_guid; /* Local HCA GUID */ 167 ibt_cq_hdl_t ud_scq; /* SendCQ handle. */ 168 ibt_cq_hdl_t ud_rcq; /* RecvCQ handle. */ 169 ibt_pd_hdl_t ud_pd; /* PD Handle. */ 170 uint8_t ud_hca_port_num; /* Local HCA port */ 171 ibt_cep_state_t ud_state; /* Channel state */ 172 uint16_t ud_pkey_ix; /* P_Key Index */ 173 ib_qkey_t ud_qkey; /* Q_Key */ 174 ibt_chan_sizes_t ud_chan_sizes; /* Queue/SGL sizes */ 175 ibt_attr_flags_t ud_flags; /* Signaling Type etc */ 176 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 177 ibt_rss_attr_t ud_rss; 178 ibt_fc_attr_t ud_fc; 179 ibt_fexch_query_attr_t ud_query_fc; /* query only set */ 180 } ibt_ud_chan_query_attr_t; 181 182 /* 183 * UD Channel Modify Attributes definition. 184 * 185 * It is only possible to modify a channel that is either in the operational 186 * state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If channel is in paused 187 * state, then a modify operation will unpause the channel. 188 * 189 * See the ibt_modify_ud_channel() for details of the required corresponding 190 * modify flags. 191 */ 192 typedef struct ibt_ud_chan_modify_attr_s { 193 uint_t ud_sq_sz; /* Set SQ Max outstanding WRs */ 194 uint_t ud_rq_sz; /* Set RQ Max outstanding WRs */ 195 ib_qkey_t ud_qkey; /* Set Q_Key */ 196 ibt_rss_attr_t ud_rss; /* Set RSS stuff */ 197 ibt_fc_attr_t ud_fc; 198 } ibt_ud_chan_modify_attr_t; 199 200 201 /* 202 * FUNCTION PROTOTYPES 203 */ 204 205 /* 206 * CONNECTION ESTABLISHMENT/TEAR DOWN FUNCTIONS. 207 */ 208 209 /* 210 * ibt_alloc_rc_channel 211 * Allocates a RC communication channels that satisfy the specified 212 * channel attributes. 213 */ 214 ibt_status_t ibt_alloc_rc_channel(ibt_hca_hdl_t hca_hdl, 215 ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args, 216 ibt_channel_hdl_t *rc_chan_p, ibt_chan_sizes_t *sizes); 217 218 /* 219 * ibt_flush_channel 220 * Flush the specified channel. Outstanding work requests are flushed 221 * so that the client can do the associated clean up. After that, the 222 * client will usually deregister the previously registered memory, 223 * then free the channel by calling ibt_free_channel(). RC channels 224 * that have been successfully opened will fail this call, as they 225 * need to instead be handled by ibt_close_rc_channel(). 226 */ 227 ibt_status_t ibt_flush_channel(ibt_channel_hdl_t chan); 228 229 /* 230 * ibt_free_channel 231 * Releases the resources associated with the specified channel 232 */ 233 ibt_status_t ibt_free_channel(ibt_channel_hdl_t chan); 234 235 /* 236 * ibt_query_rc_channel 237 * Query an RC channel's attributes. Should only be called on an 238 * opened RC channel. If called on a channel before it is opened, 239 * some channel attributes may change when the channel is opened. 240 */ 241 ibt_status_t ibt_query_rc_channel(ibt_channel_hdl_t rc_chan, 242 ibt_rc_chan_query_attr_t *chan_attrs); 243 244 /* 245 * ibt_modify_rc_channel() 246 * Modifies a previous opened operational or paused RC channel's 247 * attributes. 248 * 249 * NOTE: 250 * It is only possible to modify a channel that has previously been opened. 251 * The channel must either be in operational state (IBT_STATE_RTS) or 252 * paused (IBT_STATE_SQD). If channel is in paused state, then a modify 253 * operation will will unpause the channel. 254 */ 255 ibt_status_t ibt_modify_rc_channel(ibt_channel_hdl_t rc_chan, 256 ibt_cep_modify_flags_t flags, ibt_rc_chan_modify_attr_t *attrs, 257 ibt_queue_sizes_t *actual_sz); 258 259 /* 260 * ibt_alloc_ud_channel 261 * Allocate UD channels that satisfy the specified channel attributes. 262 */ 263 ibt_status_t ibt_alloc_ud_channel(ibt_hca_hdl_t hca_hdl, 264 ibt_chan_alloc_flags_t flags, ibt_ud_chan_alloc_args_t *args, 265 ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes); 266 267 /* 268 * ibt_alloc_ud_channel_range 269 * Allocate a range of UD channels that have consecutive QPNs for RSS. 270 */ 271 ibt_status_t ibt_alloc_ud_channel_range(ibt_hca_hdl_t hca_hdl, 272 uint_t log2, ibt_chan_alloc_flags_t flags, 273 ibt_ud_chan_alloc_args_t *args, ibt_cq_hdl_t *send_cq_p, 274 ibt_cq_hdl_t *recv_cq_p, ib_qpn_t *base_qpn_p, 275 ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes); 276 277 /* 278 * ibt_query_ud_channel 279 * Query a UD channel's attributes. 280 */ 281 ibt_status_t ibt_query_ud_channel(ibt_channel_hdl_t ud_chan, 282 ibt_ud_chan_query_attr_t *ud_chan_attrs); 283 284 /* 285 * ibt_modify_ud_channel() 286 * Modifies an UD channel's attributes, as specified by a 287 * ibt_cep_modify_flags_t parameter to those specified in the 288 * ibt_ud_chan_modify_attr_t structure. 289 * 290 * NOTE: 291 * It is only possible to modify a channel that is either in the 292 * operational state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If 293 * channel is in paused state, then a modify operation will unpause the 294 * channel. 295 */ 296 ibt_status_t ibt_modify_ud_channel(ibt_channel_hdl_t ud_chan, 297 ibt_cep_modify_flags_t flags, ibt_ud_chan_modify_attr_t *attrs, 298 ibt_queue_sizes_t *actual_sz); 299 300 /* 301 * ibt_recover_ud_channel() 302 * Recover an UD Channel which has transitioned to SQ Error state. The 303 * ibt_recover_ud_channel() transitions the channel from SQ Error state 304 * to Ready-To-Send channel state. 305 * 306 * If a work request posted to a UD channel's send queue completes with 307 * an error (see ibt_wc_status_t), the channel gets transitioned to SQ 308 * Error state. In order to reuse this channel, ibt_recover_ud_channel() 309 * can be used to recover the channel to a usable (Ready-to-Send) state. 310 */ 311 ibt_status_t ibt_recover_ud_channel(ibt_channel_hdl_t ud_chan); 312 313 314 /* 315 * ibt_alloc_ud_dest 316 * Allocate a UD destination handle. This allocates local resources 317 * that will need subsequent modification/initialization before use 318 * (in send work requests). Functions that can be used to do this are 319 * ibt_modify_ud_dest (data supplied by caller), ibt_modify_reply_ud_dest 320 * (data supplied from a successfully completed receive work request), 321 * and ibt_request_ud_dest (data retrieved using SIDR protocol). 322 */ 323 ibt_status_t ibt_alloc_ud_dest(ibt_hca_hdl_t hca_hdl, 324 ibt_ud_dest_flags_t flags, ibt_pd_hdl_t pd, ibt_ud_dest_hdl_t *ud_dest_p); 325 326 /* 327 * ibt_modify_ud_dest 328 * Modify a previously allocated UD destination handle from the 329 * argument data. After the ud_dest has already been made usable, 330 * the adds_vect argument to this function is optional (NULL). 331 */ 332 ibt_status_t ibt_modify_ud_dest(ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, 333 ib_qpn_t dest_qpn, ibt_adds_vect_t *adds_vect); 334 335 /* 336 * ibt_modify_reply_ud_dest 337 * Modify a previously allocated UD destination handle, so that it 338 * can be used to reply to the sender of the datagram contained in the 339 * specified work request completion. 340 */ 341 ibt_status_t ibt_modify_reply_ud_dest(ibt_channel_hdl_t ud_chan, 342 ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, ibt_wc_t *wc, 343 ib_vaddr_t recv_buf); 344 345 /* 346 * ibt_request_ud_dest 347 * Modify a previously allocated UD destination handle based on the 348 * data retrieved by making an SIDR request. 349 */ 350 ibt_status_t ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, 351 ibt_execution_mode_t mode, ibt_ud_dest_attr_t *dest_attrs, 352 ibt_ud_returns_t *ud_ret_args); 353 354 /* 355 * ibt_free_ud_dest 356 * Releases the resources associated with the specified UD destination 357 * handle. 358 */ 359 ibt_status_t ibt_free_ud_dest(ibt_ud_dest_hdl_t ud_dest); 360 361 /* 362 * ibt_query_ud_dest 363 * Query a UD destination's attributes. 364 */ 365 ibt_status_t ibt_query_ud_dest(ibt_ud_dest_hdl_t ud_dest, 366 ibt_ud_dest_query_attr_t *dest_attrs); 367 368 369 /* 370 * ibt_is_privileged_ud_dest 371 * Determine if a UD destination Handle is a privileged handle. 372 */ 373 boolean_t ibt_is_privileged_ud_dest(ibt_ud_dest_hdl_t ud_dest); 374 375 376 /* 377 * ibt_update_channel_qkey 378 * 379 * ibt_update_channel_qkey() set's the Qkey in the specified channel context 380 * to the Qkey in the specified destination handle. This function can be used 381 * to enable sends to a privileged destination. All posted Send Work Requests 382 * that contain a privileged destination handle now use the Qkey in the 383 * channel context. 384 * 385 * ibt_update_channel_qkey() can also be used to enable the caller to receive 386 * from the specified remote destination on the specified channel. 387 * 388 */ 389 ibt_status_t ibt_update_channel_qkey(ibt_channel_hdl_t ud_chan, 390 ibt_ud_dest_hdl_t ud_dest); 391 392 393 /* 394 * ibt_set_chan_private() 395 * ibt_get_chan_private() 396 * Set/get a pointer to client private data. 397 * Applicable for both RC and UD channels. 398 */ 399 void ibt_set_chan_private(ibt_channel_hdl_t chan, void *clnt_private); 400 401 void *ibt_get_chan_private(ibt_channel_hdl_t chan); 402 403 /* 404 * ibt_channel_to_hca_guid() 405 * 406 * A helper function to retrieve HCA GUID for the specified Channel. 407 */ 408 ib_guid_t ibt_channel_to_hca_guid(ibt_channel_hdl_t chan); 409 410 #ifdef __cplusplus 411 } 412 #endif 413 414 #endif /* _SYS_IB_IBTL_IBTI_H */ 415