1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_IB_IBTL_IBTI_H 27 #define _SYS_IB_IBTL_IBTI_H 28 29 /* 30 * ibti.h 31 * 32 * This file contains the IBTI prototypes and associated data structures. 33 * It is the only header file that should be included by IBTI clients. 34 */ 35 #include <sys/ib/ibtl/ibti_common.h> 36 37 #ifdef __cplusplus 38 extern "C" { 39 #endif 40 41 /* 42 * Allocate channel flags. 43 */ 44 typedef enum ibt_chan_alloc_flags_e { 45 IBT_ACHAN_NO_FLAGS = 0, 46 IBT_ACHAN_CLONE = (1 << 0), 47 IBT_ACHAN_USER_MAP = (1 << 1), 48 IBT_ACHAN_DEFER_ALLOC = (1 << 2), 49 IBT_ACHAN_USES_SRQ = (1 << 3), 50 IBT_ACHAN_USES_RSS = (1 << 4) 51 } ibt_chan_alloc_flags_t; 52 53 54 /* 55 * Allocate RC channel ibt_alloc_rc_channel() argument. 56 */ 57 typedef struct ibt_rc_chan_alloc_args_s { 58 ibt_attr_flags_t rc_flags; /* Signal type etc */ 59 ibt_cep_flags_t rc_control; 60 uint8_t rc_hca_port_num; 61 ibt_chan_sizes_t rc_sizes; 62 ibt_cq_hdl_t rc_scq; /* Send CQ */ 63 ibt_cq_hdl_t rc_rcq; /* Recv CQ */ 64 ibt_pd_hdl_t rc_pd; /* PD */ 65 ibt_channel_hdl_t rc_clone_chan; /* Optional Clone handle */ 66 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 67 } ibt_rc_chan_alloc_args_t; 68 69 /* 70 * RC channel query attributes structure. 71 */ 72 typedef struct ibt_rc_chan_query_attr_s { 73 ib_guid_t rc_hca_guid; /* Local HCA GUID */ 74 ibt_cq_hdl_t rc_scq; /* SendCQ handle */ 75 ibt_cq_hdl_t rc_rcq; /* RecvCQ handle */ 76 ibt_pd_hdl_t rc_pd; /* PD Handle. */ 77 ibt_cep_state_t rc_state; /* Channel state */ 78 ib_mtu_t rc_path_mtu; 79 uint8_t rc_path_retry_cnt:3; 80 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 81 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 82 ibt_cep_path_t rc_prim_path; 83 ibt_cep_path_t rc_alt_path; 84 ibt_chan_sizes_t rc_chan_sizes; /* Queue/SGL sizes */ 85 uint8_t rc_rdma_ra_out; /* max RDMA-Reads/Atomics out */ 86 uint8_t rc_rdma_ra_in; /* max RDMA-Reads/Atomics in */ 87 ibt_attr_flags_t rc_flags; /* SQ Signaling Type etc */ 88 ibt_cep_flags_t rc_control; /* Control Flags */ 89 ibt_cep_cmstate_t rc_mig_state; 90 ib_qpn_t rc_qpn; /* Local QPN */ 91 ib_qpn_t rc_dst_qpn; /* Destination QPN */ 92 ibt_srq_hdl_t rc_srq; /* Optional Shared Rcv Queue */ 93 } ibt_rc_chan_query_attr_t; 94 95 /* 96 * RC Channel Modify Attributes definition. 97 * 98 * It is only possible to modify a channel that has previously been 99 * opened. The channel must either be in operational state (IBT_STATE_RTS) 100 * or paused (IBT_STATE_SQD). If channel is in paused state, then a modify 101 * operation will unpause the channel. 102 * 103 * See the ibt_modify_rc_channel() for details of the required corresponding 104 * modify flags. 105 * Not specified attributes should be set to "NULL" or "0". 106 */ 107 typedef struct ibt_rc_chan_modify_attr_s { 108 ibt_cep_flags_t rc_control; /* Channel Control Flags */ 109 uint_t rc_sq_sz; /* Set SQ Max outstanding WRs */ 110 uint_t rc_rq_sz; /* Set RQ Max outstanding WRs */ 111 112 ibt_adds_vect_t rc_prim_adds_vect; /* Primary Path Address */ 113 ibt_adds_vect_t rc_alt_adds_vect; /* Alternate Path Address */ 114 uint8_t rc_path_retry_cnt:3; 115 ibt_rnr_retry_cnt_t rc_path_rnr_retry_cnt; 116 ibt_rnr_nak_time_t rc_min_rnr_nak; /* min RNR-NAK timer */ 117 uint8_t rc_prim_port_num; /* Port of Primary Path */ 118 uint8_t rc_alt_port_num; /* Port of Alternate Path */ 119 uint8_t rc_rdma_ra_out; /* Initiator Depth, Number of */ 120 /* RDMA RD's & Atomics */ 121 /* outstanding. */ 122 uint8_t rc_rdma_ra_in; /* Responder resources for */ 123 /* handling incoming RDMA rds */ 124 /* and Atomics. */ 125 } ibt_rc_chan_modify_attr_t; 126 127 /* 128 * UD remote destination query attributes 129 */ 130 typedef struct ibt_ud_dest_query_attr_s { 131 ibt_hca_hdl_t ud_hca_hdl; /* Local HCA Handle */ 132 ib_qpn_t ud_dst_qpn; /* Destination QPN */ 133 ib_qkey_t ud_qkey; /* Q_Key */ 134 ibt_adds_vect_t ud_addr_vect; /* Address Information */ 135 ibt_pd_hdl_t ud_pd; 136 } ibt_ud_dest_query_attr_t; 137 138 /* 139 * Allocate UD channel ibt_alloc_ud_channel() arguments; see below at 140 * ibt_alloc_ud_channel() for a description of what's required and optional. 141 */ 142 typedef struct ibt_ud_chan_alloc_args_s { 143 ibt_attr_flags_t ud_flags; /* Sig type etc */ 144 uint8_t ud_hca_port_num; 145 uint16_t ud_pkey_ix; /* P_Key Index */ 146 ibt_chan_sizes_t ud_sizes; 147 ib_qkey_t ud_qkey; /* Q_Key */ 148 ibt_cq_hdl_t ud_scq; /* Send CQ */ 149 ibt_cq_hdl_t ud_rcq; /* Recv CQ */ 150 ibt_pd_hdl_t ud_pd; /* PD */ 151 ibt_channel_hdl_t ud_clone_chan; /* Optional clone handle */ 152 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 153 ibt_rss_attr_t ud_rss; 154 } ibt_ud_chan_alloc_args_t; 155 156 /* 157 * UD channel query attributes. 158 */ 159 typedef struct ibt_ud_chan_query_attr_s { 160 ib_qpn_t ud_qpn; /* QPN */ 161 ib_guid_t ud_hca_guid; /* Local HCA GUID */ 162 ibt_cq_hdl_t ud_scq; /* SendCQ handle. */ 163 ibt_cq_hdl_t ud_rcq; /* RecvCQ handle. */ 164 ibt_pd_hdl_t ud_pd; /* PD Handle. */ 165 uint8_t ud_hca_port_num; /* Local HCA port */ 166 ibt_cep_state_t ud_state; /* Channel state */ 167 uint16_t ud_pkey_ix; /* P_Key Index */ 168 ib_qkey_t ud_qkey; /* Q_Key */ 169 ibt_chan_sizes_t ud_chan_sizes; /* Queue/SGL sizes */ 170 ibt_attr_flags_t ud_flags; /* Signaling Type etc */ 171 ibt_srq_hdl_t ud_srq; /* Optional Shared Rcv Queue */ 172 ibt_rss_attr_t ud_rss; 173 } ibt_ud_chan_query_attr_t; 174 175 /* 176 * UD Channel Modify Attributes definition. 177 * 178 * It is only possible to modify a channel that is either in the operational 179 * state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If channel is in paused 180 * state, then a modify operation will unpause the channel. 181 * 182 * See the ibt_modify_ud_channel() for details of the required corresponding 183 * modify flags. 184 */ 185 typedef struct ibt_ud_chan_modify_attr_s { 186 uint_t ud_sq_sz; /* Set SQ Max outstanding WRs */ 187 uint_t ud_rq_sz; /* Set RQ Max outstanding WRs */ 188 ib_qkey_t ud_qkey; /* Set Q_Key */ 189 ibt_rss_attr_t ud_rss; /* Set RSS stuff */ 190 } ibt_ud_chan_modify_attr_t; 191 192 193 /* 194 * FUNCTION PROTOTYPES 195 */ 196 197 /* 198 * CONNECTION ESTABLISHMENT/TEAR DOWN FUNCTIONS. 199 */ 200 201 /* 202 * ibt_alloc_rc_channel 203 * Allocates a RC communication channels that satisfy the specified 204 * channel attributes. 205 */ 206 ibt_status_t ibt_alloc_rc_channel(ibt_hca_hdl_t hca_hdl, 207 ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args, 208 ibt_channel_hdl_t *rc_chan_p, ibt_chan_sizes_t *sizes); 209 210 /* 211 * ibt_flush_channel 212 * Flush the specified channel. Outstanding work requests are flushed 213 * so that the client can do the associated clean up. After that, the 214 * client will usually deregister the previously registered memory, 215 * then free the channel by calling ibt_free_channel(). RC channels 216 * that have been successfully opened will fail this call, as they 217 * need to instead be handled by ibt_close_rc_channel(). 218 */ 219 ibt_status_t ibt_flush_channel(ibt_channel_hdl_t chan); 220 221 /* 222 * ibt_free_channel 223 * Releases the resources associated with the specified channel 224 */ 225 ibt_status_t ibt_free_channel(ibt_channel_hdl_t chan); 226 227 /* 228 * ibt_query_rc_channel 229 * Query an RC channel's attributes. Should only be called on an 230 * opened RC channel. If called on a channel before it is opened, 231 * some channel attributes may change when the channel is opened. 232 */ 233 ibt_status_t ibt_query_rc_channel(ibt_channel_hdl_t rc_chan, 234 ibt_rc_chan_query_attr_t *chan_attrs); 235 236 /* 237 * ibt_modify_rc_channel() 238 * Modifies a previous opened operational or paused RC channel's 239 * attributes. 240 * 241 * NOTE: 242 * It is only possible to modify a channel that has previously been opened. 243 * The channel must either be in operational state (IBT_STATE_RTS) or 244 * paused (IBT_STATE_SQD). If channel is in paused state, then a modify 245 * operation will will unpause the channel. 246 */ 247 ibt_status_t ibt_modify_rc_channel(ibt_channel_hdl_t rc_chan, 248 ibt_cep_modify_flags_t flags, ibt_rc_chan_modify_attr_t *attrs, 249 ibt_queue_sizes_t *actual_sz); 250 251 /* 252 * ibt_alloc_ud_channel 253 * Allocate UD channels that satisfy the specified channel attributes. 254 */ 255 ibt_status_t ibt_alloc_ud_channel(ibt_hca_hdl_t hca_hdl, 256 ibt_chan_alloc_flags_t flags, ibt_ud_chan_alloc_args_t *args, 257 ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes); 258 259 /* 260 * ibt_alloc_ud_channel_range 261 * Allocate a range of UD channels that have consecutive QPNs for RSS. 262 */ 263 ibt_status_t ibt_alloc_ud_channel_range(ibt_hca_hdl_t hca_hdl, 264 uint_t log2, ibt_chan_alloc_flags_t flags, 265 ibt_ud_chan_alloc_args_t *args, ibt_cq_hdl_t *send_cq_p, 266 ibt_cq_hdl_t *recv_cq_p, ib_qpn_t *base_qpn_p, 267 ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes); 268 269 /* 270 * ibt_query_ud_channel 271 * Query a UD channel's attributes. 272 */ 273 ibt_status_t ibt_query_ud_channel(ibt_channel_hdl_t ud_chan, 274 ibt_ud_chan_query_attr_t *ud_chan_attrs); 275 276 /* 277 * ibt_modify_ud_channel() 278 * Modifies an UD channel's attributes, as specified by a 279 * ibt_cep_modify_flags_t parameter to those specified in the 280 * ibt_ud_chan_modify_attr_t structure. 281 * 282 * NOTE: 283 * It is only possible to modify a channel that is either in the 284 * operational state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If 285 * channel is in paused state, then a modify operation will unpause the 286 * channel. 287 */ 288 ibt_status_t ibt_modify_ud_channel(ibt_channel_hdl_t ud_chan, 289 ibt_cep_modify_flags_t flags, ibt_ud_chan_modify_attr_t *attrs, 290 ibt_queue_sizes_t *actual_sz); 291 292 /* 293 * ibt_recover_ud_channel() 294 * Recover an UD Channel which has transitioned to SQ Error state. The 295 * ibt_recover_ud_channel() transitions the channel from SQ Error state 296 * to Ready-To-Send channel state. 297 * 298 * If a work request posted to a UD channel's send queue completes with 299 * an error (see ibt_wc_status_t), the channel gets transitioned to SQ 300 * Error state. In order to reuse this channel, ibt_recover_ud_channel() 301 * can be used to recover the channel to a usable (Ready-to-Send) state. 302 */ 303 ibt_status_t ibt_recover_ud_channel(ibt_channel_hdl_t ud_chan); 304 305 306 /* 307 * ibt_alloc_ud_dest 308 * Allocate a UD destination handle. This allocates local resources 309 * that will need subsequent modification/initialization before use 310 * (in send work requests). Functions that can be used to do this are 311 * ibt_modify_ud_dest (data supplied by caller), ibt_modify_reply_ud_dest 312 * (data supplied from a successfully completed receive work request), 313 * and ibt_request_ud_dest (data retrieved using SIDR protocol). 314 */ 315 ibt_status_t ibt_alloc_ud_dest(ibt_hca_hdl_t hca_hdl, 316 ibt_ud_dest_flags_t flags, ibt_pd_hdl_t pd, ibt_ud_dest_hdl_t *ud_dest_p); 317 318 /* 319 * ibt_modify_ud_dest 320 * Modify a previously allocated UD destination handle from the 321 * argument data. After the ud_dest has already been made usable, 322 * the adds_vect argument to this function is optional (NULL). 323 */ 324 ibt_status_t ibt_modify_ud_dest(ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, 325 ib_qpn_t dest_qpn, ibt_adds_vect_t *adds_vect); 326 327 /* 328 * ibt_modify_reply_ud_dest 329 * Modify a previously allocated UD destination handle, so that it 330 * can be used to reply to the sender of the datagram contained in the 331 * specified work request completion. 332 */ 333 ibt_status_t ibt_modify_reply_ud_dest(ibt_channel_hdl_t ud_chan, 334 ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, ibt_wc_t *wc, 335 ib_vaddr_t recv_buf); 336 337 /* 338 * ibt_request_ud_dest 339 * Modify a previously allocated UD destination handle based on the 340 * data retrieved by making an SIDR request. 341 */ 342 ibt_status_t ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, 343 ibt_execution_mode_t mode, ibt_ud_dest_attr_t *dest_attrs, 344 ibt_ud_returns_t *ud_ret_args); 345 346 /* 347 * ibt_free_ud_dest 348 * Releases the resources associated with the specified UD destination 349 * handle. 350 */ 351 ibt_status_t ibt_free_ud_dest(ibt_ud_dest_hdl_t ud_dest); 352 353 /* 354 * ibt_query_ud_dest 355 * Query a UD destination's attributes. 356 */ 357 ibt_status_t ibt_query_ud_dest(ibt_ud_dest_hdl_t ud_dest, 358 ibt_ud_dest_query_attr_t *dest_attrs); 359 360 361 /* 362 * ibt_is_privileged_ud_dest 363 * Determine if a UD destination Handle is a privileged handle. 364 */ 365 boolean_t ibt_is_privileged_ud_dest(ibt_ud_dest_hdl_t ud_dest); 366 367 368 /* 369 * ibt_update_channel_qkey 370 * 371 * ibt_update_channel_qkey() set's the Qkey in the specified channel context 372 * to the Qkey in the specified destination handle. This function can be used 373 * to enable sends to a privileged destination. All posted Send Work Requests 374 * that contain a privileged destination handle now use the Qkey in the 375 * channel context. 376 * 377 * ibt_update_channel_qkey() can also be used to enable the caller to receive 378 * from the specified remote destination on the specified channel. 379 * 380 */ 381 ibt_status_t ibt_update_channel_qkey(ibt_channel_hdl_t ud_chan, 382 ibt_ud_dest_hdl_t ud_dest); 383 384 385 /* 386 * ibt_set_chan_private() 387 * ibt_get_chan_private() 388 * Set/get a pointer to client private data. 389 * Applicable for both RC and UD channels. 390 */ 391 void ibt_set_chan_private(ibt_channel_hdl_t chan, void *clnt_private); 392 393 void *ibt_get_chan_private(ibt_channel_hdl_t chan); 394 395 /* 396 * ibt_channel_to_hca_guid() 397 * 398 * A helper function to retrieve HCA GUID for the specified Channel. 399 */ 400 ib_guid_t ibt_channel_to_hca_guid(ibt_channel_hdl_t chan); 401 402 #ifdef __cplusplus 403 } 404 #endif 405 406 #endif /* _SYS_IB_IBTL_IBTI_H */ 407