1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IB_IBTL_IBTI_H 28 #define _SYS_IB_IBTL_IBTI_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 /* 33 * ibti.h 34 * 35 * This file contains the IBTI prototypes and associated data structures. 36 * It is the only header file that should be included by IBTI clients. 37 */ 38 #include <sys/ib/ibtl/ibti_common.h> 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 /* 45 * Allocate channel flags. 46 */ 47 typedef enum ibt_chan_alloc_flags_e { 48 IBT_ACHAN_NO_FLAGS = 0, 49 IBT_ACHAN_CLONE = (1 << 0), 50 IBT_ACHAN_USER_MAP = (1 << 1), 51 IBT_ACHAN_DEFER_ALLOC = (1 << 2), 52 IBT_ACHAN_USES_SRQ = (1 << 3) 53 } ibt_chan_alloc_flags_t; 54 55 56 /* 57 * Allocate RC channel ibt_alloc_rc_channel() argument. 58 */ 59 typedef struct ibt_rc_chan_alloc_args_s { 60 ibt_attr_flags_t rc_flags; /* Signal type etc */ 61 ibt_cep_flags_t rc_control; 62 uint8_t rc_hca_port_num; 63 ibt_chan_sizes_t rc_sizes; 64 ibt_cq_hdl_t rc_scq; /* Send CQ */ 65 ibt_cq_hdl_t rc_rcq; /* Recv CQ */ 66 ibt_pd_hdl_t rc_pd; /* PD */ 67 ibt_channel_hdl_t rc_clone_chan; /* Optional Clone handle */ 68 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 69 } ibt_rc_chan_alloc_args_t; 70 71 /* 72 * RC channel query attributes structure. 73 */ 74 typedef struct ibt_rc_chan_query_attr_s { 75 ib_guid_t rc_hca_guid; /* Local HCA GUID */ 76 ibt_cq_hdl_t rc_scq; /* SendCQ handle */ 77 ibt_cq_hdl_t rc_rcq; /* RecvCQ handle */ 78 ibt_pd_hdl_t rc_pd; /* PD Handle. */ 79 ibt_cep_state_t rc_state; /* Channel state */ 80 ib_mtu_t rc_path_mtu; 81 uint8_t rc_path_retry_cnt:3; 82 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 83 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 84 ibt_cep_path_t rc_prim_path; 85 ibt_cep_path_t rc_alt_path; 86 ibt_chan_sizes_t rc_chan_sizes; /* Queue/SGL sizes */ 87 uint8_t rc_rdma_ra_out; /* max RDMA-Reads/Atomics out */ 88 uint8_t rc_rdma_ra_in; /* max RDMA-Reads/Atomics in */ 89 ibt_attr_flags_t rc_flags; /* SQ Signaling Type etc */ 90 ibt_cep_flags_t rc_control; /* Control Flags */ 91 ibt_cep_cmstate_t rc_mig_state; 92 ib_qpn_t rc_qpn; /* Local QPN */ 93 ib_qpn_t rc_dst_qpn; /* Destination QPN */ 94 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 95 } ibt_rc_chan_query_attr_t; 96 97 /* 98 * RC Channel Modify Attributes definition. 99 * 100 * It is only possible to modify a channel that has previously been 101 * opened. The channel must either be in operational state (IBT_STATE_RTS) 102 * or paused (IBT_STATE_SQD). If channel is in paused state, then a modify 103 * operation will unpause the channel. 104 * 105 * See the ibt_modify_rc_channel() for details of the required corresponding 106 * modify flags. 107 * Not specified attributes should be set to "NULL" or "0". 108 */ 109 typedef struct ibt_rc_chan_modify_attr_s { 110 ibt_cep_flags_t rc_control; /* Channel Control Flags */ 111 uint_t rc_sq_sz; /* Set SQ Max outstanding WRs */ 112 uint_t rc_rq_sz; /* Set RQ Max outstanding WRs */ 113 114 ibt_adds_vect_t rc_prim_adds_vect; /* Primary Path Address */ 115 ibt_adds_vect_t rc_alt_adds_vect; /* Alternate Path Address */ 116 uint8_t rc_path_retry_cnt:3; 117 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 118 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 119 uint8_t rc_prim_port_num; /* Port of Primary Path */ 120 uint8_t rc_alt_port_num; /* Port of Alternate Path */ 121 uint8_t rc_rdma_ra_out; /* Initiator Depth, Number of */ 122 /* RDMA RD's & Atomics */ 123 /* outstanding. */ 124 uint8_t rc_rdma_ra_in; /* Responder resources for */ 125 /* handling incoming RDMA rds */ 126 /* and Atomics. */ 127 } ibt_rc_chan_modify_attr_t; 128 129 /* 130 * UD remote destination query attributes 131 */ 132 typedef struct ibt_ud_dest_query_attr_s { 133 ibt_hca_hdl_t ud_hca_hdl; /* Local HCA Handle */ 134 ib_qpn_t ud_dst_qpn; /* Destination QPN */ 135 ib_qkey_t ud_qkey; /* Q_Key */ 136 ibt_adds_vect_t ud_addr_vect; /* Address Information */ 137 ibt_pd_hdl_t ud_pd; 138 } ibt_ud_dest_query_attr_t; 139 140 141 /* 142 * Allocate UD channel ibt_alloc_ud_channel() arguments; see below at 143 * ibt_alloc_ud_channel() for a description of what's required and optional. 144 */ 145 typedef struct ibt_ud_chan_alloc_args_s { 146 ibt_attr_flags_t ud_flags; /* Sig type etc */ 147 uint8_t ud_hca_port_num; 148 uint16_t ud_pkey_ix; /* P_Key Index */ 149 ibt_chan_sizes_t ud_sizes; 150 ib_qkey_t ud_qkey; /* Q_Key */ 151 ibt_cq_hdl_t ud_scq; /* Send CQ */ 152 ibt_cq_hdl_t ud_rcq; /* Recv CQ */ 153 ibt_pd_hdl_t ud_pd; /* PD */ 154 ibt_channel_hdl_t ud_clone_chan; /* Optional clone handle */ 155 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 156 } ibt_ud_chan_alloc_args_t; 157 158 /* 159 * UD channel query attributes. 160 */ 161 typedef struct ibt_ud_chan_query_attr_s { 162 ib_qpn_t ud_qpn; /* QPN */ 163 ib_guid_t ud_hca_guid; /* Local HCA GUID */ 164 ibt_cq_hdl_t ud_scq; /* SendCQ handle. */ 165 ibt_cq_hdl_t ud_rcq; /* RecvCQ handle. */ 166 ibt_pd_hdl_t ud_pd; /* PD Handle. */ 167 uint8_t ud_hca_port_num; /* Local HCA port */ 168 ibt_cep_state_t ud_state; /* Channel state */ 169 uint16_t ud_pkey_ix; /* P_Key Index */ 170 ib_qkey_t ud_qkey; /* Q_Key */ 171 ibt_chan_sizes_t ud_chan_sizes; /* Queue/SGL sizes */ 172 ibt_attr_flags_t ud_flags; /* Signaling Type etc */ 173 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 174 } ibt_ud_chan_query_attr_t; 175 176 /* 177 * UD Channel Modify Attributes definition. 178 * 179 * It is only possible to modify a channel that is either in the operational 180 * state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If channel is in paused 181 * state, then a modify operation will unpause the channel. 182 * 183 * See the ibt_modify_ud_channel() for details of the required corresponding 184 * modify flags. 185 */ 186 typedef struct ibt_ud_chan_modify_attr_s { 187 uint_t ud_sq_sz; /* Set SQ Max outstanding WRs */ 188 uint_t ud_rq_sz; /* Set RQ Max outstanding WRs */ 189 ib_qkey_t ud_qkey; /* Set Q_Key */ 190 } ibt_ud_chan_modify_attr_t; 191 192 193 /* 194 * FUNCTION PROTOTYPES 195 */ 196 197 /* 198 * CONNECTION ESTABLISHMENT/TEAR DOWN FUNCTIONS. 199 */ 200 201 /* 202 * ibt_alloc_rc_channel 203 * Allocates a RC communication channels that satisfy the specified 204 * channel attributes. 205 */ 206 ibt_status_t ibt_alloc_rc_channel(ibt_hca_hdl_t hca_hdl, 207 ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args, 208 ibt_channel_hdl_t *rc_chan_p, ibt_chan_sizes_t *sizes); 209 210 /* 211 * ibt_flush_channel 212 * Flush the specified channel. Outstanding work requests are flushed 213 * so that the client can do the associated clean up. After that, the 214 * client will usually deregister the previously registered memory, 215 * then free the channel by calling ibt_free_channel(). RC channels 216 * that have been successfully opened will fail this call, as they 217 * need to instead be handled by ibt_close_rc_channel(). 218 */ 219 ibt_status_t ibt_flush_channel(ibt_channel_hdl_t chan); 220 221 /* 222 * ibt_free_channel 223 * Releases the resources associated with the specified channel 224 */ 225 ibt_status_t ibt_free_channel(ibt_channel_hdl_t chan); 226 227 /* 228 * ibt_query_rc_channel 229 * Query an RC channel's attributes. Should only be called on an 230 * opened RC channel. If called on a channel before it is opened, 231 * some channel attributes may change when the channel is opened. 232 */ 233 ibt_status_t ibt_query_rc_channel(ibt_channel_hdl_t rc_chan, 234 ibt_rc_chan_query_attr_t *chan_attrs); 235 236 /* 237 * ibt_modify_rc_channel() 238 * Modifies a previous opened operational or paused RC channel's 239 * attributes. 240 * 241 * NOTE: 242 * It is only possible to modify a channel that has previously been opened. 243 * The channel must either be in operational state (IBT_STATE_RTS) or 244 * paused (IBT_STATE_SQD). If channel is in paused state, then a modify 245 * operation will will unpause the channel. 246 */ 247 ibt_status_t ibt_modify_rc_channel(ibt_channel_hdl_t rc_chan, 248 ibt_cep_modify_flags_t flags, ibt_rc_chan_modify_attr_t *attrs, 249 ibt_queue_sizes_t *actual_sz); 250 251 /* 252 * ibt_alloc_ud_channel 253 * Allocate UD channels that satisfy the specified channel attributes. 254 */ 255 ibt_status_t ibt_alloc_ud_channel(ibt_hca_hdl_t hca_hdl, 256 ibt_chan_alloc_flags_t flags, ibt_ud_chan_alloc_args_t *args, 257 ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes); 258 259 /* 260 * ibt_query_ud_channel 261 * Query a UD channel's attributes. 262 */ 263 ibt_status_t ibt_query_ud_channel(ibt_channel_hdl_t ud_chan, 264 ibt_ud_chan_query_attr_t *ud_chan_attrs); 265 266 /* 267 * ibt_modify_ud_channel() 268 * Modifies an UD channel's attributes, as specified by a 269 * ibt_cep_modify_flags_t parameter to those specified in the 270 * ibt_ud_chan_modify_attr_t structure. 271 * 272 * NOTE: 273 * It is only possible to modify a channel that is either in the 274 * operational state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If 275 * channel is in paused state, then a modify operation will unpause the 276 * channel. 277 */ 278 ibt_status_t ibt_modify_ud_channel(ibt_channel_hdl_t ud_chan, 279 ibt_cep_modify_flags_t flags, ibt_ud_chan_modify_attr_t *attrs, 280 ibt_queue_sizes_t *actual_sz); 281 282 /* 283 * ibt_recover_ud_channel() 284 * Recover an UD Channel which has transitioned to SQ Error state. The 285 * ibt_recover_ud_channel() transitions the channel from SQ Error state 286 * to Ready-To-Send channel state. 287 * 288 * If a work request posted to a UD channel's send queue completes with 289 * an error (see ibt_wc_status_t), the channel gets transitioned to SQ 290 * Error state. In order to reuse this channel, ibt_recover_ud_channel() 291 * can be used to recover the channel to a usable (Ready-to-Send) state. 292 */ 293 ibt_status_t ibt_recover_ud_channel(ibt_channel_hdl_t ud_chan); 294 295 296 /* 297 * ibt_alloc_ud_dest 298 * Allocate a UD destination handle. This allocates local resources 299 * that will need subsequent modification/initialization before use 300 * (in send work requests). Functions that can be used to do this are 301 * ibt_modify_ud_dest (data supplied by caller), ibt_modify_reply_ud_dest 302 * (data supplied from a successfully completed receive work request), 303 * and ibt_request_ud_dest (data retrieved using SIDR protocol). 304 */ 305 ibt_status_t ibt_alloc_ud_dest(ibt_hca_hdl_t hca_hdl, 306 ibt_ud_dest_flags_t flags, ibt_pd_hdl_t pd, ibt_ud_dest_hdl_t *ud_dest_p); 307 308 /* 309 * ibt_modify_ud_dest 310 * Modify a previously allocated UD destination handle from the 311 * argument data. After the ud_dest has already been made usable, 312 * the adds_vect argument to this function is optional (NULL). 313 */ 314 ibt_status_t ibt_modify_ud_dest(ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, 315 ib_qpn_t dest_qpn, ibt_adds_vect_t *adds_vect); 316 317 /* 318 * ibt_modify_reply_ud_dest 319 * Modify a previously allocated UD destination handle, so that it 320 * can be used to reply to the sender of the datagram contained in the 321 * specified work request completion. 322 */ 323 ibt_status_t ibt_modify_reply_ud_dest(ibt_channel_hdl_t ud_chan, 324 ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, ibt_wc_t *wc, 325 ib_vaddr_t recv_buf); 326 327 /* 328 * ibt_request_ud_dest 329 * Modify a previously allocated UD destination handle based on the 330 * data retrieved by making an SIDR request. 331 */ 332 ibt_status_t ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, 333 ibt_execution_mode_t mode, ibt_ud_dest_attr_t *dest_attrs, 334 ibt_ud_returns_t *ud_ret_args); 335 336 /* 337 * ibt_free_ud_dest 338 * Releases the resources associated with the specified UD destination 339 * handle. 340 */ 341 ibt_status_t ibt_free_ud_dest(ibt_ud_dest_hdl_t ud_dest); 342 343 /* 344 * ibt_query_ud_dest 345 * Query a UD destination's attributes. 346 */ 347 ibt_status_t ibt_query_ud_dest(ibt_ud_dest_hdl_t ud_dest, 348 ibt_ud_dest_query_attr_t *dest_attrs); 349 350 351 /* 352 * ibt_is_privileged_ud_dest 353 * Determine if a UD destination Handle is a privileged handle. 354 */ 355 boolean_t ibt_is_privileged_ud_dest(ibt_ud_dest_hdl_t ud_dest); 356 357 358 /* 359 * ibt_update_channel_qkey 360 * 361 * ibt_update_channel_qkey() set's the Qkey in the specified channel context 362 * to the Qkey in the specified destination handle. This function can be used 363 * to enable sends to a privileged destination. All posted Send Work Requests 364 * that contain a privileged destination handle now use the Qkey in the 365 * channel context. 366 * 367 * ibt_update_channel_qkey() can also be used to enable the caller to receive 368 * from the specified remote destination on the specified channel. 369 * 370 */ 371 ibt_status_t ibt_update_channel_qkey(ibt_channel_hdl_t ud_chan, 372 ibt_ud_dest_hdl_t ud_dest); 373 374 375 /* 376 * ibt_set_chan_private() 377 * ibt_get_chan_private() 378 * Set/get a pointer to client private data. 379 * Applicable for both RC and UD channels. 380 */ 381 void ibt_set_chan_private(ibt_channel_hdl_t chan, void *clnt_private); 382 383 void *ibt_get_chan_private(ibt_channel_hdl_t chan); 384 385 /* 386 * ibt_channel_to_hca_guid() 387 * 388 * A helper function to retrieve HCA GUID for the specified Channel. 389 */ 390 ib_guid_t ibt_channel_to_hca_guid(ibt_channel_hdl_t chan); 391 392 #ifdef __cplusplus 393 } 394 #endif 395 396 #endif /* _SYS_IB_IBTL_IBTI_H */ 397