1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 30 #include <sys/ib/ibtl/ibti.h> 31 32 /* 33 * ibcm_ti.c 34 * These routines implement the Communication Manager's interfaces to IBTL. 35 */ 36 37 /* CM rc recycle task args structure definition */ 38 typedef struct ibcm_taskq_recycle_arg_s { 39 ibt_channel_hdl_t rc_chan; 40 ibt_cep_flags_t control; 41 uint8_t hca_port_num; 42 ibt_recycle_handler_t func; 43 void *arg; 44 } ibcm_taskq_recycle_arg_t; 45 46 _NOTE(READ_ONLY_DATA(ibcm_taskq_recycle_arg_s)) 47 48 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap, 49 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args, 50 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid); 51 static void ibcm_process_abort_via_taskq(void *args); 52 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg); 53 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg); 54 static void ibcm_process_async_join_mcg(void *tq_arg); 55 56 static ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *, 57 uint64_t c_mask, void *, size_t *); 58 59 /* Address Record management definitions */ 60 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service" 61 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL 62 #define IBCM_DAPL_ATS_NBYTES 16 63 ibcm_svc_info_t *ibcm_ar_svcinfop; 64 ibcm_ar_t *ibcm_ar_list; 65 66 #ifdef DEBUG 67 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel, 68 ibcm_mad_addr_t *cm_reply_addr); 69 #endif 70 71 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_hdl})) 72 73 /* access is controlled between ibcm_sm.c and ibcm_ti.c by CVs */ 74 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by CV", {ibt_rc_returns_t 75 ibt_ud_returns_t ibt_ap_returns_t ibt_ar_t})) 76 77 /* 78 * Typically, clients initialize these args in one api call, and use in 79 * another api 80 */ 81 _NOTE(SCHEME_PROTECTS_DATA("Expected usage of ibtl api by client", 82 {ibt_path_info_s ibt_cep_path_s ibt_adds_vect_s ibt_mcg_info_s ib_gid_s 83 ibt_ud_dest_attr_s ibt_ud_dest_s ibt_srv_data_s ibt_redirect_info_s})) 84 85 /* 86 * ibt_open_rc_channel() 87 * ibt_open_rc_channel opens a communication channel on the specified 88 * channel to the specified service. For connection service type qp's 89 * the CM initiates the CEP to establish the connection and transitions 90 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's 91 * attributes as necessary. 92 * The implementation of this function assumes that alt path is different 93 * from primary path. It is assumed that the Path functions ensure that. 94 * 95 * RETURN VALUES: 96 * IBT_SUCCESS on success (or respective failure on error) 97 */ 98 ibt_status_t 99 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags, 100 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args, 101 ibt_rc_returns_t *ret_args) 102 { 103 /* all fields that are related to REQ MAD formation */ 104 105 ib_pkey_t prim_pkey; 106 ib_lid_t primary_slid, alternate_slid; 107 ib_qpn_t local_qpn = 0; 108 ib_guid_t hca_guid; 109 ib_qkey_t local_qkey = 0; 110 ib_eecn_t local_eecn = 0; 111 ib_eecn_t remote_eecn = 0; 112 boolean_t primary_grh; 113 boolean_t alternate_grh = B_FALSE; 114 ib_lid_t base_lid; 115 ib_com_id_t local_comid; 116 ibmf_msg_t *ibmf_msg; 117 ibcm_req_msg_t *req_msgp; 118 119 uint8_t rdma_in, rdma_out; 120 uint8_t cm_retries; 121 uint64_t local_cm_proc_time; /* In usec */ 122 uint8_t local_cm_resp_time; /* IB time */ 123 uint64_t remote_cm_resp_time; /* In usec */ 124 uint32_t starting_psn = 0; 125 126 /* CM path related fields */ 127 ibmf_handle_t ibmf_hdl; 128 ibcm_qp_list_t *cm_qp_entry; 129 ibcm_mad_addr_t cm_reply_addr; 130 131 uint8_t cm_pkt_lt; 132 133 /* Local args for ibtl/internal CM functions called within */ 134 ibt_status_t status; 135 ibcm_status_t lkup_status; 136 ibt_qp_query_attr_t qp_query_attr; 137 138 /* Other misc local args */ 139 ibt_priv_data_len_t len; 140 ibcm_hca_info_t *hcap; 141 ibcm_state_data_t *statep; 142 uint8_t port_no; 143 144 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)", 145 channel, flags, mode, chan_args, ret_args); 146 147 if (IBCM_INVALID_CHANNEL(channel)) { 148 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel"); 149 return (IBT_CHAN_HDL_INVALID); 150 } 151 152 /* cm handler should always be specified */ 153 if (chan_args->oc_cm_handler == NULL) { 154 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 155 "CM handler is not be specified", channel); 156 return (IBT_INVALID_PARAM); 157 } 158 159 if (mode == IBT_NONBLOCKING) { 160 if (ret_args != NULL) { 161 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 162 " ret_args should be NULL when called in " 163 "non-blocking mode", channel); 164 return (IBT_INVALID_PARAM); 165 } 166 } else if (mode == IBT_BLOCKING) { 167 if (ret_args == NULL) { 168 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 169 " ret_args should be Non-NULL when called in " 170 "blocking mode", channel); 171 return (IBT_INVALID_PARAM); 172 } 173 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) { 174 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 175 " private data length is too large", channel); 176 return (IBT_INVALID_PARAM); 177 } 178 if ((ret_args->rc_priv_data_len > 0) && 179 (ret_args->rc_priv_data == NULL)) { 180 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 181 " rc_priv_data_len > 0, but rc_priv_data NULL", 182 channel); 183 return (IBT_INVALID_PARAM); 184 } 185 } else { /* any other mode is not valid for ibt_open_rc_channel */ 186 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 187 "invalid mode %x specified", channel, mode); 188 return (IBT_INVALID_PARAM); 189 } 190 191 /* 192 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP 193 */ 194 if (flags & IBT_OCHAN_DUP) { 195 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 196 "Unsupported Flags specified: 0x%X", channel, flags); 197 return (IBT_INVALID_PARAM); 198 } 199 200 if ((flags & IBT_OCHAN_REDIRECTED) && 201 (flags & IBT_OCHAN_PORT_REDIRECTED)) { 202 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 203 "Illegal to specify IBT_OCHAN_REDIRECTED and " 204 "IBT_OCHAN_PORT_REDIRECTED flags together", channel); 205 return (IBT_INVALID_PARAM); 206 } 207 208 if (((flags & IBT_OCHAN_REDIRECTED) && 209 (chan_args->oc_cm_redirect_info == NULL)) || 210 ((flags & IBT_OCHAN_PORT_REDIRECTED) && 211 (chan_args->oc_cm_cep_path == NULL))) { 212 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 213 "Redirect flag specified, but respective arg is NULL", 214 channel); 215 return (IBT_INVALID_PARAM); 216 } 217 218 if ((flags & IBT_OCHAN_REDIRECTED) && 219 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) && 220 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) { 221 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 222 "Either rdi_dlid or rdi_gid must be specified for" 223 " IBT_OCHAN_REDIRECTED", channel); 224 return (IBT_INVALID_PARAM); 225 } 226 227 /* primary dlid and hca_port_num should never be zero */ 228 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 229 230 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) { 231 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 232 "Primary Path's information is not valid", channel); 233 return (IBT_INVALID_PARAM); 234 } 235 236 /* validate SID */ 237 if (chan_args->oc_path->pi_sid == 0) { 238 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 239 "ERROR: Service ID in path information is 0", channel); 240 return (IBT_INVALID_PARAM); 241 } 242 243 /* validate rnr_retry_cnt (enum has more than 3 bits) */ 244 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) { 245 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 246 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range", 247 channel, chan_args->oc_path_rnr_retry_cnt); 248 return (IBT_INVALID_PARAM); 249 } 250 251 /* 252 * Ensure that client is not re-using a QP that is still associated 253 * with a statep 254 */ 255 IBCM_GET_CHAN_PRIVATE(channel, statep); 256 if (statep != NULL) { 257 IBCM_RELEASE_CHAN_PRIVATE(channel); 258 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 259 "Channel being re-used on active side", channel); 260 return (IBT_CHAN_IN_USE); 261 } 262 263 /* Get GUID from Channel */ 264 hca_guid = ibt_channel_to_hca_guid(channel); 265 266 /* validate QP's hca guid with that from primary path */ 267 if (hca_guid != chan_args->oc_path->pi_hca_guid) { 268 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 269 "GUID from Channel and primary path don't match", channel); 270 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 271 "Channel GUID %llX primary path GUID %llX", channel, 272 hca_guid, chan_args->oc_path->pi_hca_guid); 273 return (IBT_CHAN_HDL_INVALID); 274 } 275 276 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 277 "Local HCA GUID %llX", channel, hca_guid); 278 279 status = ibt_query_qp(channel, &qp_query_attr); 280 if (status != IBT_SUCCESS) { 281 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 282 "ibt_query_qp failed %d", channel, status); 283 return (status); 284 } 285 286 /* If client specified "no port change on QP" */ 287 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 288 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) { 289 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 290 "chan port %d and path port %d does not match", channel, 291 qp_query_attr.qp_info.qp_transport.rc.rc_path. \ 292 cep_hca_port_num, port_no); 293 return (IBT_INVALID_PARAM); 294 } 295 296 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) { 297 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 298 "Invalid Channel type: Applicable only to RC Channel", 299 channel); 300 return (IBT_CHAN_SRV_TYPE_INVALID); 301 } 302 303 /* Check if QP is in INIT state or not */ 304 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) { 305 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 306 "QP is not in INIT state %x", channel, 307 qp_query_attr.qp_info.qp_state); 308 return (IBT_CHAN_STATE_INVALID); 309 } 310 311 local_qpn = qp_query_attr.qp_qpn; 312 313 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x", 314 channel, local_qpn); 315 316 #ifdef NO_EEC_SUPPORT_YET 317 318 if (flags & IBT_OCHAN_RDC_EXISTS) { 319 ibt_eec_query_attr_t eec_query_attr; 320 321 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey; 322 323 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD"); 324 325 status = ibt_query_eec(channel, &eec_query_attr); 326 if (status != IBT_SUCCESS) { 327 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 328 " ibt_query_eec failed %d", channel, status); 329 return (status); 330 } 331 local_eecn = eec_query_attr.eec_eecn; 332 } 333 334 #endif 335 336 /* If no HCA found return failure */ 337 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) { 338 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 339 "hcap is NULL. Probably hca is not in active state", 340 channel); 341 return (IBT_CHAN_HDL_INVALID); 342 } 343 344 rdma_out = chan_args->oc_rdma_ra_out; 345 rdma_in = chan_args->oc_rdma_ra_in; 346 347 if ((rdma_in > hcap->hca_max_rdma_in_qp) || 348 (rdma_out > hcap->hca_max_rdma_out_qp)) { 349 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 350 "rdma in %d/out %d values exceed hca limits", channel, 351 rdma_in, rdma_out); 352 ibcm_dec_hca_acc_cnt(hcap); 353 return (IBT_INVALID_PARAM); 354 } 355 356 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 357 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out); 358 359 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) { 360 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 361 "Huge Primary Pkt lt %d", channel, 362 chan_args->oc_path->pi_prim_pkt_lt); 363 ibcm_dec_hca_acc_cnt(hcap); 364 return (IBT_PATH_PKT_LT_TOO_HIGH); 365 } 366 367 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, 368 NULL, &base_lid); 369 if (status != IBT_SUCCESS) { 370 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 371 "primary port_num %d not active", channel, port_no); 372 ibcm_dec_hca_acc_cnt(hcap); 373 return (status); 374 } 375 376 /* Validate P_KEY Index */ 377 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no, 378 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey); 379 if (status != IBT_SUCCESS) { 380 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 381 "Invalid Primary PKeyIx %x", channel, 382 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix); 383 ibcm_dec_hca_acc_cnt(hcap); 384 return (status); 385 } 386 387 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 388 "primary_port_num %d primary_pkey 0x%x", channel, port_no, 389 prim_pkey); 390 391 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 392 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 393 != IBT_SUCCESS)) { 394 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 395 "ibmf reg or callback setup failed during re-initialize", 396 channel); 397 ibcm_dec_hca_acc_cnt(hcap); 398 return (status); 399 } 400 401 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl; 402 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 403 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl); 404 405 406 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 407 408 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p " 409 "primary SLID = %x", channel, primary_slid); 410 411 /* check first if alternate path exists or not as it is OPTIONAL */ 412 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) { 413 uint8_t alt_port_no; 414 415 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 416 417 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) { 418 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 419 "Huge Alt Pkt lt %d", channel, 420 chan_args->oc_path->pi_alt_pkt_lt); 421 ibcm_dec_hca_acc_cnt(hcap); 422 return (IBT_PATH_PKT_LT_TOO_HIGH); 423 } 424 425 if (port_no != alt_port_no) { 426 427 status = ibt_get_port_state_byguid(hcap->hca_guid, 428 alt_port_no, NULL, &base_lid); 429 if (status != IBT_SUCCESS) { 430 431 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 432 "chan 0x%p alt_port_num %d inactive %d", 433 channel, alt_port_no, status); 434 ibcm_dec_hca_acc_cnt(hcap); 435 return (status); 436 } 437 438 } 439 alternate_slid = 440 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 441 442 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan %0xp " 443 "alternate SLID = %x", channel, alternate_slid); 444 } 445 446 /* 447 * only pkey needs to be zero'ed, because all other fields are set in 448 * in ibcm_init_reply_addr. But, let's bzero the complete struct for 449 * any future modifications. 450 */ 451 bzero(&cm_reply_addr, sizeof (cm_reply_addr)); 452 453 /* Initialize the MAD destination address in stored_reply_addr */ 454 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args, 455 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) { 456 457 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 458 "ibcm_init_reply_addr failed status %d ", channel, status); 459 ibcm_dec_hca_acc_cnt(hcap); 460 return (status); 461 } 462 463 464 /* Initialize the pkey for CM MAD communication */ 465 if (cm_reply_addr.rcvd_addr.ia_p_key == 0) 466 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey; 467 468 #ifdef DEBUG 469 ibcm_print_reply_addr(channel, &cm_reply_addr); 470 #endif 471 472 /* Retrieve an ibmf qp for sending CM MADs */ 473 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no, 474 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) { 475 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 476 "unable to allocate ibmf qp for CM MADs", channel); 477 ibcm_dec_hca_acc_cnt(hcap); 478 return (IBT_INSUFF_RESOURCE); 479 } 480 481 482 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) { 483 ibcm_release_qp(cm_qp_entry); 484 ibcm_dec_hca_acc_cnt(hcap); 485 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 486 " Unable to allocate comid", channel); 487 return (IBT_INSUFF_KERNEL_RESOURCE); 488 } 489 490 /* allocate an IBMF mad buffer */ 491 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, 492 MAD_METHOD_SEND)) != IBT_SUCCESS) { 493 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 494 "chan 0x%p ibcm_alloc_out_msg failed", channel); 495 ibcm_release_qp(cm_qp_entry); 496 ibcm_free_comid(hcap, local_comid); 497 ibcm_dec_hca_acc_cnt(hcap); 498 return (status); 499 } 500 501 /* Init to Init, if QP's port does not match with path information */ 502 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 503 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) { 504 505 ibt_qp_info_t qp_info; 506 ibt_cep_modify_flags_t cep_flags; 507 508 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 509 "chan 0x%p chan port %d", channel, 510 qp_query_attr.qp_info.qp_transport.rc.rc_path.\ 511 cep_hca_port_num); 512 513 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 514 "chan 0x%p path port %d", channel, port_no); 515 516 bzero(&qp_info, sizeof (qp_info)); 517 /* For now, set it to RC type */ 518 519 qp_info.qp_trans = IBT_RC_SRV; 520 qp_info.qp_state = IBT_STATE_INIT; 521 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no; 522 523 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 524 525 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL); 526 527 if (status != IBT_SUCCESS) { 528 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 529 "chan 0x%p ibt_modify_qp() = %d", channel, status); 530 ibcm_release_qp(cm_qp_entry); 531 ibcm_free_comid(hcap, local_comid); 532 ibcm_dec_hca_acc_cnt(hcap); 533 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 534 return (status); 535 } else 536 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 537 "chan 0x%p ibt_modify_qp() = %d", channel, status); 538 } 539 540 /* allocate ibcm_state_data_t before grabbing the WRITER lock */ 541 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP); 542 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 543 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0, 544 hcap, &statep); 545 rw_exit(&hcap->hca_state_rwlock); 546 547 /* CM should be seeing this for the first time */ 548 ASSERT(lkup_status == IBCM_LOOKUP_NEW); 549 550 /* Increment the hca's resource count */ 551 ibcm_inc_hca_res_cnt(hcap); 552 553 /* Once a resource created on hca, no need to hold the acc cnt */ 554 ibcm_dec_hca_acc_cnt(hcap); 555 556 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 557 558 statep->timerid = 0; 559 statep->local_hca_guid = hca_guid; 560 statep->local_qpn = local_qpn; 561 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry; 562 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 563 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 564 565 566 /* Save "statep" as channel's CM private data. */ 567 statep->channel = channel; 568 IBCM_SET_CHAN_PRIVATE(statep->channel, statep); 569 570 statep->stored_msg = ibmf_msg; 571 572 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*req_msgp)) 573 574 /* Start filling in the REQ MAD */ 575 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg); 576 req_msgp->req_local_comm_id = h2b32(local_comid); 577 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid); 578 req_msgp->req_local_ca_guid = h2b64(hca_guid); 579 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */ 580 581 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */ 582 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in); 583 584 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */ 585 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out); 586 587 if (flags & IBT_OCHAN_REMOTE_CM_TM) 588 remote_cm_resp_time = chan_args->oc_remote_cm_time; 589 else 590 remote_cm_resp_time = ibcm_remote_response_time; 591 592 /* 593 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type, 594 * IBT_CM_FLOW_CONTROL is always set by default. 595 */ 596 req_msgp->req_remote_eecn_plus = h2b32( 597 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 | 598 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL); 599 600 if (flags & IBT_OCHAN_LOCAL_CM_TM) 601 local_cm_proc_time = chan_args->oc_local_cm_time; 602 else 603 local_cm_proc_time = ibcm_local_processing_time; 604 605 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time + 606 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) + 607 ibcm_sw_delay); 608 609 /* save retry count */ 610 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt; 611 612 if (flags & IBT_OCHAN_STARTING_PSN) 613 starting_psn = chan_args->oc_starting_psn; 614 615 if (local_cm_resp_time > 0x1f) 616 local_cm_resp_time = 0x1f; 617 618 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */ 619 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 | 620 local_cm_resp_time << 3 | statep->cep_retry_cnt); 621 622 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 623 "Prim Pkt lt (IB time) 0x%x", channel, 624 chan_args->oc_path->pi_prim_pkt_lt); 625 626 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 627 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time); 628 629 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 630 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time); 631 632 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 633 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time); 634 635 statep->starting_psn = starting_psn; 636 637 /* Pkey - bytes 48-49 */ 638 req_msgp->req_part_key = h2b16(prim_pkey); 639 640 if (flags & IBT_OCHAN_CM_RETRY) 641 cm_retries = chan_args->oc_cm_retry_cnt; 642 else 643 cm_retries = ibcm_max_retries; 644 645 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries; 646 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4; 647 648 /* 649 * Check whether SRQ is associated with this Channel, if yes, then 650 * set the SRQ Exists bit in the REQ. 651 */ 652 if (qp_query_attr.qp_srq != NULL) { 653 req_msgp->req_max_cm_retries_plus |= (1 << 3); 654 } 655 656 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 | 657 chan_args->oc_path_rnr_retry_cnt; 658 659 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d" 660 " staring PSN %x", channel, cm_retries, starting_psn); 661 662 663 #ifdef NO_EEC_SUPPORT_YET 664 if (flags & IBT_OCHAN_RDC_EXISTS) 665 req_msgp->req_mtu_plus |= 8; 666 #endif 667 668 /* Initialize the "primary" port stuff next - bytes 52-95 */ 669 req_msgp->req_primary_l_port_lid = h2b16(primary_slid); 670 req_msgp->req_primary_r_port_lid = 671 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 672 req_msgp->req_primary_l_port_gid.gid_prefix = 673 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix); 674 req_msgp->req_primary_l_port_gid.gid_guid = 675 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid); 676 req_msgp->req_primary_r_port_gid.gid_prefix = 677 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 678 req_msgp->req_primary_r_port_gid.gid_guid = 679 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 680 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh; 681 682 statep->remote_hca_guid = /* not correct, but helpful for debugging */ 683 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid; 684 685 /* Bytes 88-91 - primary_flowlbl, and primary_srate */ 686 req_msgp->req_primary_flow_label_plus = 687 h2b32(((primary_grh == B_TRUE) ? 688 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) | 689 IBCM_PRIM_ADDS_VECT(chan_args).av_srate); 690 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ? 691 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0; 692 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ? 693 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 0xff; 694 req_msgp->req_primary_sl_plus = 695 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 | 696 ((primary_grh == B_TRUE) ? 0 : 8); 697 698 req_msgp->req_primary_localtime_plus = 699 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) + 700 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 701 702 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p", 703 channel, statep); 704 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 705 "active hca_ack_delay (usec) %d", channel, 706 req_msgp->req_primary_localtime_plus); 707 708 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 709 "Sent primary cep timeout (IB Time) %d", channel, 710 hcap->hca_ack_delay); 711 712 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ", 713 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 714 715 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 716 "prim GID %llX:%llX", channel, 717 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix, 718 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 719 720 /* Initialize the "alternate" port stuff - optional */ 721 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) { 722 ib_gid_t tmp_gid; 723 724 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid); 725 req_msgp->req_alt_r_port_lid = 726 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 727 /* 728 * doing all this as req_alt_r/l_port_gid is at offset 729 * 100, 116 which is not divisible by 8 730 */ 731 732 tmp_gid.gid_prefix = 733 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 734 tmp_gid.gid_guid = 735 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 736 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0], 737 sizeof (ib_gid_t)); 738 tmp_gid.gid_prefix = 739 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix); 740 tmp_gid.gid_guid = 741 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid); 742 743 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0], 744 sizeof (ib_gid_t)); 745 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh; 746 747 /* Bytes 132-135 - alternate_flow_label, and alternate srate */ 748 req_msgp->req_alt_flow_label_plus = h2b32( 749 (((alternate_grh == B_TRUE) ? 750 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) | 751 IBCM_ALT_ADDS_VECT(chan_args).av_srate)); 752 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ? 753 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0; 754 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ? 755 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 0xff; 756 req_msgp->req_alt_sl_plus = 757 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 | 758 ((alternate_grh == B_TRUE) ? 0 : 8); 759 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 * 760 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) + 761 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 762 763 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 764 "alt_dlid %x ", channel, 765 IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 766 767 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 768 "alt GID %llX:%llX", channel, 769 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix, 770 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 771 } 772 773 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ); 774 if ((len > 0) && chan_args->oc_priv_data) 775 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len); 776 777 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*req_msgp)) 778 779 /* return_data is filled up in the state machine code */ 780 if (ret_args != NULL) { 781 statep->open_return_data = ret_args; 782 } 783 784 /* initialize some statep fields here */ 785 statep->mode = IBCM_ACTIVE_MODE; 786 statep->hcap = hcap; 787 788 statep->cm_handler = chan_args->oc_cm_handler; 789 statep->state_cm_private = chan_args->oc_cm_clnt_private; 790 791 statep->pkt_life_time = 792 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt); 793 794 statep->timer_value = ibt_ib2usec(ibt_usec2ib( 795 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time)); 796 797 /* Initialize statep->stored_reply_addr */ 798 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl; 799 800 /* Initialize stored reply addr fields */ 801 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr; 802 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr; 803 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists; 804 statep->stored_reply_addr.port_num = cm_reply_addr.port_num; 805 806 /* 807 * The IPD on local/active side is calculated by path functions, 808 * hence available in the args of ibt_open_rc_channel 809 */ 810 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate; 811 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate; 812 813 /* Store the source path bits for primary and alt paths */ 814 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 815 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 816 817 statep->open_flow = 1; 818 statep->open_done = B_FALSE; 819 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT; 820 IBCM_REF_CNT_INCR(statep); /* Decremented before return */ 821 IBCM_REF_CNT_INCR(statep); /* Decremented after REQ is posted */ 822 statep->send_mad_flags |= IBCM_REQ_POST_BUSY; 823 824 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID = 825 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID); 826 827 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID = 828 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid, 829 0)); 830 831 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 832 833 ibcm_open_enqueue(statep); 834 835 mutex_enter(&statep->state_mutex); 836 837 if (mode == IBT_BLOCKING) { 838 839 /* wait for REQ/REP/RTU */ 840 while (statep->open_done != B_TRUE) { 841 cv_wait(&statep->block_client_cv, &statep->state_mutex); 842 } 843 844 /* 845 * In the case that open_channel() fails because of a 846 * REJ or timeout, change retval to IBT_CM_FAILURE 847 */ 848 if (statep->open_return_data->rc_status != IBT_CM_ACCEPT) 849 status = IBT_CM_FAILURE; 850 851 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p " 852 "ret status %d cm status %d", channel, status, 853 statep->open_return_data->rc_status); 854 } 855 856 /* decrement the ref-count before leaving here */ 857 IBCM_REF_CNT_DECR(statep); 858 859 mutex_exit(&statep->state_mutex); 860 861 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel); 862 return (status); 863 } 864 865 /* 866 * ibcm_init_reply_addr: 867 * 868 * The brief description of functionality below. 869 * 870 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case): 871 * Build CM path from chan_args->oc_cm_cep_path 872 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt 873 * 874 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case): 875 * If Redirect LID is specified, 876 * If Redirect GID is not specified or specified to be on the same 877 * subnet, then 878 * Build CM path from chan_args->oc_cm_redirect_info 879 * Set CM pkt lt to subnet timeout 880 * Else (ie., GID specified, but on a different subnet) 881 * Do a path lookup to build CM Path and set CM pkt lt 882 * 883 */ 884 static ibt_status_t 885 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr, 886 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags, 887 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid) 888 { 889 ibt_adds_vect_t *cm_adds; 890 ibt_path_info_t path; 891 boolean_t cm_grh; 892 ibt_status_t status; 893 894 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:"); 895 896 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*reply_addr)) 897 898 /* 899 * sending side CM lid/gid/port num are not based on any redirect 900 * params. These values are set to primary RC path lid/gid/port num. 901 * In the future, these values can be set based on framework policy 902 * decisions ensuring reachability. 903 */ 904 reply_addr->grh_hdr.ig_sender_gid = 905 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid; 906 reply_addr->rcvd_addr.ia_local_lid = prim_slid; 907 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 908 909 if (flags & IBT_OCHAN_PORT_REDIRECTED) { 910 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 911 "IBT_OCHAN_PORT_REDIRECTED specified"); 912 913 status = ibt_index2pkey_byguid(hcap->hca_guid, 914 chan_args->oc_cm_cep_path->cep_hca_port_num, 915 chan_args->oc_cm_cep_path->cep_pkey_ix, 916 &reply_addr->rcvd_addr.ia_p_key); 917 918 if (status != IBT_SUCCESS) { 919 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid " 920 "CM PKeyIx %x port_num %x", 921 chan_args->oc_cm_cep_path->cep_pkey_ix, 922 chan_args->oc_cm_cep_path->cep_hca_port_num); 923 return (status); 924 } 925 926 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect); 927 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x", 928 cm_adds->av_dlid); 929 930 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 931 reply_addr->rcvd_addr.ia_remote_qno = 1; 932 *cm_pkt_lt = chan_args->oc_cm_pkt_lt; 933 934 } else if (flags & IBT_OCHAN_REDIRECTED) { 935 ibt_redirect_info_t *redirect_info; 936 ibt_hca_portinfo_t *port_infop; 937 uint_t psize, nports; 938 939 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 940 "IBT_OCHAN_REDIRECTED specified"); 941 942 redirect_info = chan_args->oc_cm_redirect_info; 943 944 if ((redirect_info->rdi_gid.gid_prefix == 0) || 945 (redirect_info->rdi_gid.gid_guid == 0)) { 946 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 947 "ERROR: Re-direct GID value NOT Provided."); 948 return (IBT_INVALID_PARAM); 949 } 950 951 /* As per spec definition 1.1, it's always IB_GSI_QKEY */ 952 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey; 953 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn; 954 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey; 955 956 /* 957 * if LID is non-zero in classportinfo then use classportinfo 958 * fields to form CM MAD destination address. 959 */ 960 if (redirect_info->rdi_dlid != 0) { 961 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid, 962 reply_addr->port_num, &port_infop, &nports, &psize); 963 if ((status != IBT_SUCCESS) || (nports == 0)) { 964 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 965 "Query Ports Failed: %d", status); 966 return (status); 967 } else if (port_infop->p_subnet_timeout > 968 IBCM_MAX_IB_PKT_LT) { 969 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 970 "large subnet timeout %x port_no %x", 971 port_infop->p_subnet_timeout, 972 reply_addr->port_num); 973 ibt_free_portinfo(port_infop, psize); 974 return (IBT_PATH_PKT_LT_TOO_HIGH); 975 } else { 976 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: " 977 "subnet timeout %x port_no %x", 978 port_infop->p_subnet_timeout, 979 reply_addr->port_num); 980 981 *cm_pkt_lt = 982 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt, 983 port_infop->p_subnet_timeout)); 984 985 ibt_free_portinfo(port_infop, psize); 986 } 987 988 reply_addr->rcvd_addr.ia_remote_lid = 989 redirect_info->rdi_dlid; 990 reply_addr->rcvd_addr.ia_service_level = 991 redirect_info->rdi_sl; 992 reply_addr->grh_exists = B_TRUE; 993 reply_addr->grh_hdr.ig_recver_gid = 994 redirect_info->rdi_gid; 995 reply_addr->grh_hdr.ig_tclass = 996 redirect_info->rdi_tclass; 997 reply_addr->grh_hdr.ig_flow_label = 998 redirect_info->rdi_flow; 999 1000 /* Classportinfo doesn't have hoplimit field */ 1001 reply_addr->grh_hdr.ig_hop_limit = 0xff; 1002 return (IBT_SUCCESS); 1003 1004 } else { 1005 ibt_path_attr_t path_attr; 1006 ib_gid_t path_dgid[1]; 1007 1008 /* 1009 * If GID is specified, and LID is zero in classportinfo 1010 * do a path lookup using specified GID, Pkey, 1011 * in classportinfo 1012 */ 1013 1014 bzero(&path_attr, sizeof (path_attr)); 1015 1016 path_attr.pa_dgids = &path_dgid[0]; 1017 path_attr.pa_dgids[0] = redirect_info->rdi_gid; 1018 1019 /* 1020 * use reply_addr below, as sender_gid in reply_addr 1021 * may have been set above based on some policy decision 1022 * for originating end point for CM MADs above 1023 */ 1024 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid; 1025 path_attr.pa_num_dgids = 1; 1026 path_attr.pa_pkey = redirect_info->rdi_pkey; 1027 1028 if ((status = ibt_get_paths(ibcm_ibt_handle, 1029 IBT_PATH_PKEY, &path_attr, 1, &path, NULL)) != 1030 IBT_SUCCESS) 1031 return (status); 1032 1033 /* Initialize cm_adds */ 1034 cm_adds = &path.pi_prim_cep_path.cep_adds_vect; 1035 *cm_pkt_lt = path.pi_prim_pkt_lt; 1036 } 1037 1038 } else { /* cm_pkey initialized in ibt_open_rc_channel */ 1039 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 1040 reply_addr->rcvd_addr.ia_remote_qno = 1; 1041 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt; 1042 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args)); 1043 } 1044 1045 1046 cm_grh = cm_adds->av_send_grh; 1047 reply_addr->grh_exists = cm_grh; 1048 1049 reply_addr->rcvd_addr.ia_remote_lid = 1050 cm_adds->av_dlid; 1051 reply_addr->grh_hdr.ig_recver_gid = 1052 cm_adds->av_dgid; 1053 reply_addr->grh_hdr.ig_flow_label = 1054 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK; 1055 reply_addr->grh_hdr.ig_tclass = 1056 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0; 1057 reply_addr->grh_hdr.ig_hop_limit = 1058 (cm_grh == B_TRUE) ? cm_adds->av_hop : 0xff; 1059 reply_addr->rcvd_addr.ia_service_level = 1060 cm_adds->av_srvl; 1061 1062 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*reply_addr)) 1063 1064 return (IBT_SUCCESS); 1065 } 1066 1067 1068 /* 1069 * ibt_prime_close_rc_channel() 1070 * It allocates resources required for close channel operation, so 1071 * ibt_close_rc_channel can be called from interrupt routine. 1072 * 1073 * INPUTS: 1074 * channel The address of an ibt_channel_t struct that 1075 * specifies the channel to open. 1076 * 1077 * RETURN VALUES: 1078 * IBT_SUCCESS on success(or respective failure on error) 1079 * 1080 * Clients are typically expected to call this function in established state 1081 */ 1082 ibt_status_t 1083 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel) 1084 { 1085 ibcm_state_data_t *statep; 1086 ibt_status_t status = IBT_SUCCESS; 1087 1088 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel); 1089 1090 /* validate channel, first */ 1091 if (IBCM_INVALID_CHANNEL(channel)) { 1092 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1093 "invalid channel", channel); 1094 return (IBT_CHAN_HDL_INVALID); 1095 } 1096 1097 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1098 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1099 "Invalid Channel type: Applicable only to RC Channel", 1100 channel); 1101 return (IBT_CHAN_SRV_TYPE_INVALID); 1102 } 1103 1104 /* get the statep */ 1105 IBCM_GET_CHAN_PRIVATE(channel, statep); 1106 1107 /* 1108 * This can happen, if the statep is already gone by a DREQ from 1109 * the remote side 1110 */ 1111 1112 if (statep == NULL) { 1113 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1114 "statep NULL", channel); 1115 return (IBT_SUCCESS); 1116 } 1117 1118 mutex_enter(&statep->state_mutex); 1119 IBCM_RELEASE_CHAN_PRIVATE(channel); 1120 if (statep->state != IBCM_STATE_ESTABLISHED) { 1121 mutex_exit(&statep->state_mutex); 1122 return (IBT_CHAN_STATE_INVALID); 1123 } 1124 IBCM_REF_CNT_INCR(statep); 1125 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p" 1126 " state %x", channel, statep, statep->state); 1127 mutex_exit(&statep->state_mutex); 1128 1129 /* clients could pre-allocate dreq mad, even before connection est */ 1130 if (statep->dreq_msg == NULL) 1131 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, 1132 &statep->dreq_msg, MAD_METHOD_SEND); 1133 1134 mutex_enter(&statep->state_mutex); 1135 IBCM_REF_CNT_DECR(statep); 1136 mutex_exit(&statep->state_mutex); 1137 1138 if (status != IBT_SUCCESS) { 1139 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1140 "ibcm_alloc_out_msg failed ", channel); 1141 return (status); 1142 } 1143 1144 /* If this message isn't seen then ibt_prime_close_rc_channel failed */ 1145 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done", 1146 channel); 1147 1148 return (IBT_SUCCESS); 1149 } 1150 1151 /* 1152 * ibt_close_rc_channel() 1153 * It closes an established channel. 1154 * 1155 * RETURN VALUES: 1156 * IBT_SUCCESS on success(or respective failure on error) 1157 */ 1158 ibt_status_t 1159 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 1160 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status, 1161 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p) 1162 { 1163 ibcm_hca_info_t *hcap; 1164 ibcm_state_data_t *statep; 1165 ibt_status_t status; 1166 1167 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)", 1168 channel, mode, priv_data, priv_data_len, 1169 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p); 1170 1171 /* validate channel, first */ 1172 if (IBCM_INVALID_CHANNEL(channel)) { 1173 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1174 "invalid channel", channel); 1175 return (IBT_CHAN_HDL_INVALID); 1176 } 1177 1178 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1179 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1180 "Invalid Channel type: Applicable only to RC Channel", 1181 channel); 1182 return (IBT_CHAN_SRV_TYPE_INVALID); 1183 } 1184 1185 if (mode == IBT_BLOCKING) { 1186 /* valid only for BLOCKING MODE */ 1187 if ((ret_priv_data_len_p != NULL) && 1188 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) { 1189 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1190 " private data len %d is too large", channel, 1191 *ret_priv_data_len_p); 1192 return (IBT_INVALID_PARAM); 1193 } 1194 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) { 1195 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1196 "invalid mode %x specified", channel, mode); 1197 return (IBT_INVALID_PARAM); 1198 } 1199 1200 /* get the statep */ 1201 IBCM_GET_CHAN_PRIVATE(channel, statep); 1202 1203 if (statep == NULL) { 1204 1205 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1206 "statep NULL", channel); 1207 if (ibtl_cm_is_chan_closing(channel) || 1208 ibtl_cm_is_chan_closed(channel)) { 1209 if (ret_status) 1210 *ret_status = IBT_CM_CLOSED_ALREADY; 1211 1212 /* No private data to return to the client */ 1213 if (ret_priv_data_len_p != NULL) 1214 *ret_priv_data_len_p = 0; 1215 1216 return (IBT_SUCCESS); 1217 } 1218 return (IBT_CHAN_STATE_INVALID); 1219 } 1220 1221 mutex_enter(&statep->state_mutex); 1222 IBCM_RELEASE_CHAN_PRIVATE(channel); 1223 IBCM_REF_CNT_INCR(statep); 1224 mutex_exit(&statep->state_mutex); 1225 1226 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p statep %p", 1227 channel, statep); 1228 1229 mutex_enter(&statep->state_mutex); 1230 hcap = statep->hcap; 1231 1232 /* HCA must have been in active state. If not, it's a client bug */ 1233 if (!IBCM_ACCESS_HCA_OK(hcap)) { 1234 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1235 "hcap 0x%p not active", channel, hcap); 1236 IBCM_REF_CNT_DECR(statep); 1237 mutex_exit(&statep->state_mutex); 1238 return (IBT_CHAN_HDL_INVALID); 1239 } 1240 1241 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) { 1242 while (statep->cep_in_rts == IBCM_BLOCK) 1243 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1244 } 1245 1246 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */ 1247 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT) 1248 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1249 1250 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1251 "connection state is %x", channel, statep->state); 1252 1253 statep->close_ret_status = ret_status; 1254 1255 /* If state is in pre-established states, abort the connection est */ 1256 if (statep->state != IBCM_STATE_ESTABLISHED) { 1257 statep->cm_retries++; /* ensure connection trace is dumped */ 1258 1259 /* No DREP private data possible */ 1260 if (ret_priv_data_len_p != NULL) 1261 *ret_priv_data_len_p = 0; 1262 1263 /* 1264 * If waiting for a response mad, then cancel the timer, 1265 * and delete the connection 1266 */ 1267 if (statep->state == IBCM_STATE_REQ_SENT || 1268 statep->state == IBCM_STATE_REP_SENT || 1269 statep->state == IBCM_STATE_REP_WAIT || 1270 statep->state == IBCM_STATE_MRA_REP_RCVD) { 1271 timeout_id_t timer_val = statep->timerid; 1272 ibcm_conn_state_t old_state; 1273 1274 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1275 "chan 0x%p connection aborted in state %x", channel, 1276 statep->state); 1277 1278 old_state = statep->state; 1279 statep->state = IBCM_STATE_DELETE; 1280 1281 if (mode == IBT_NONBLOCKING) { 1282 if (taskq_dispatch(ibcm_taskq, 1283 ibcm_process_abort_via_taskq, statep, 1284 TQ_NOSLEEP) == 0) { 1285 1286 IBCM_REF_CNT_DECR(statep); 1287 statep->state = old_state; 1288 mutex_exit(&statep->state_mutex); 1289 return (IBT_INSUFF_KERNEL_RESOURCE); 1290 } /* if taskq_dispatch succeeds */ 1291 /* Cancel the timer */ 1292 statep->timerid = 0; 1293 mutex_exit(&statep->state_mutex); 1294 } else { 1295 /* Cancel the timer */ 1296 statep->timerid = 0; 1297 mutex_exit(&statep->state_mutex); 1298 (void) taskq_dispatch(ibcm_taskq, 1299 ibcm_process_abort_via_taskq, statep, 1300 TQ_SLEEP); 1301 } 1302 1303 /* cancel the currently running timer */ 1304 if (timer_val != 0) 1305 (void) untimeout(timer_val); 1306 1307 /* wait until cm handler returns for BLOCKING cases */ 1308 if ((mode == IBT_BLOCKING) || 1309 (mode == IBT_NOCALLBACKS)) { 1310 mutex_enter(&statep->state_mutex); 1311 while (statep->close_done != B_TRUE) 1312 cv_wait(&statep->block_client_cv, 1313 &statep->state_mutex); 1314 mutex_exit(&statep->state_mutex); 1315 } 1316 1317 if (ret_status) 1318 *ret_status = IBT_CM_CLOSED_ABORT; 1319 1320 /* 1321 * It would ideal to post a REJ MAD, but that would 1322 * be non-conformance to spec. Hence, delete the state 1323 * data. Assuming that happens quickly, any retransmits 1324 * from the remote are replied by CM with reject 1325 * reason " no valid com id". That would stop remote 1326 * sending any more MADs. 1327 */ 1328 ibcm_delete_state_data(statep); 1329 return (IBT_SUCCESS); 1330 1331 /* if CM busy in cm handler, wait until cm handler returns */ 1332 } else if (statep->state == IBCM_STATE_REQ_RCVD || 1333 statep->state == IBCM_STATE_REP_RCVD || 1334 statep->state == IBCM_STATE_MRA_SENT || 1335 statep->state == IBCM_STATE_MRA_REP_SENT) { 1336 1337 /* take control of statep */ 1338 statep->abort_flag |= IBCM_ABORT_CLIENT; 1339 1340 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1341 "chan 0x%p connection aborted in state = %x", 1342 channel, statep->state); 1343 1344 /* 1345 * wait until state machine modifies qp state to error, 1346 * including disassociating statep and QP 1347 */ 1348 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS)) 1349 while (statep->close_done != B_TRUE) 1350 cv_wait(&statep->block_client_cv, 1351 &statep->state_mutex); 1352 1353 /* a sanity setting */ 1354 if (mode == IBT_NOCALLBACKS) 1355 statep->cm_handler = NULL; 1356 IBCM_REF_CNT_DECR(statep); 1357 mutex_exit(&statep->state_mutex); 1358 1359 /* 1360 * In rare situations, connection attempt could be 1361 * terminated for some other reason, before abort is 1362 * processed, but CM still returns ret_status as abort 1363 */ 1364 if (ret_status) 1365 *ret_status = IBT_CM_CLOSED_ABORT; 1366 1367 /* 1368 * REJ MAD is posted by the CM state machine for this 1369 * case, hence state structure is deleted in the 1370 * state machine processing. 1371 */ 1372 return (IBT_SUCCESS); 1373 1374 } else if ((statep->state == IBCM_STATE_TIMEWAIT) || 1375 (statep->state == IBCM_STATE_DELETE)) { 1376 1377 /* State already in timewait, so no return priv data */ 1378 IBCM_REF_CNT_DECR(statep); 1379 mutex_exit(&statep->state_mutex); 1380 1381 /* The teardown has already been done */ 1382 if (ret_status) 1383 *ret_status = IBT_CM_CLOSED_ALREADY; 1384 1385 return (IBT_SUCCESS); 1386 1387 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) || 1388 (statep->state == IBCM_STATE_DREQ_SENT) || 1389 (statep->state == IBCM_STATE_DREP_RCVD) || 1390 ((statep->state == IBCM_STATE_TIMED_OUT) && 1391 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) { 1392 1393 /* 1394 * Either the remote or local client has already 1395 * initiated the teardown. IBCM_STATE_DREP_RCVD is 1396 * possible, if CM initiated teardown without client's 1397 * knowledge, for stale handling, etc., 1398 */ 1399 if (mode == IBT_NOCALLBACKS) { 1400 if (statep->close_nocb_state == IBCM_UNBLOCK) { 1401 statep->close_nocb_state = IBCM_FAIL; 1402 /* enable free qp after return */ 1403 ibtl_cm_chan_is_closing( 1404 statep->channel); 1405 } else while (statep->close_nocb_state == 1406 IBCM_BLOCK) 1407 cv_wait(&statep->block_client_cv, 1408 &statep->state_mutex); 1409 statep->cm_handler = NULL; /* sanity setting */ 1410 if (ret_status) 1411 *ret_status = IBT_CM_CLOSED_ALREADY; 1412 } else if (mode == IBT_BLOCKING) { 1413 /* wait until state is moved to timewait */ 1414 while (statep->close_done != B_TRUE) 1415 cv_wait(&statep->block_client_cv, 1416 &statep->state_mutex); 1417 } 1418 1419 IBCM_REF_CNT_DECR(statep); 1420 mutex_exit(&statep->state_mutex); 1421 1422 /* ret_status is set in state machine code */ 1423 return (IBT_SUCCESS); 1424 1425 } else if (statep->state == IBCM_STATE_TIMED_OUT) { 1426 1427 if ((mode == IBT_BLOCKING) || 1428 (mode == IBT_NOCALLBACKS)) { 1429 1430 /* 1431 * wait until cm handler invocation and 1432 * disassociation between statep and channel 1433 * is complete 1434 */ 1435 while (statep->close_done != B_TRUE) 1436 cv_wait(&statep->block_client_cv, 1437 &statep->state_mutex); 1438 } 1439 1440 if (ret_status) 1441 *ret_status = IBT_CM_CLOSED_ABORT; 1442 IBCM_REF_CNT_DECR(statep); 1443 mutex_exit(&statep->state_mutex); 1444 1445 return (IBT_SUCCESS); 1446 } else { 1447 IBCM_REF_CNT_DECR(statep); 1448 mutex_exit(&statep->state_mutex); 1449 1450 return (IBT_CM_FAILURE); 1451 } 1452 } 1453 1454 ASSERT(statep->close_nocb_state != IBCM_BLOCK); 1455 1456 if (mode == IBT_NOCALLBACKS) { 1457 statep->close_nocb_state = IBCM_FAIL; 1458 statep->cm_handler = NULL; 1459 ibtl_cm_chan_is_closing(statep->channel); 1460 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1461 "NOCALLBACKS on in statep = %p", statep); 1462 } 1463 mutex_exit(&statep->state_mutex); 1464 1465 mutex_enter(&statep->state_mutex); 1466 if (statep->state != IBCM_STATE_ESTABLISHED) { 1467 goto lost_race; 1468 } 1469 1470 /* 1471 * Cancel/wait for any pending ibt_set_alt_path, and 1472 * release state mutex 1473 */ 1474 ibcm_sync_lapr_idle(statep); 1475 1476 ibcm_close_enter(); 1477 1478 mutex_enter(&statep->state_mutex); 1479 if (statep->state != IBCM_STATE_ESTABLISHED) { 1480 ibcm_close_exit(); 1481 goto lost_race; 1482 } 1483 1484 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT; 1485 statep->timerid = 0; 1486 statep->close_done = B_FALSE; 1487 statep->close_flow = 1; 1488 mutex_exit(&statep->state_mutex); 1489 1490 if (statep->dreq_msg == NULL) { 1491 if ((status = ibcm_alloc_out_msg( 1492 statep->stored_reply_addr.ibmf_hdl, &statep->dreq_msg, 1493 MAD_METHOD_SEND)) != IBT_SUCCESS) { 1494 1495 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: " 1496 "chan 0x%p ibcm_alloc_out_msg failed ", channel); 1497 mutex_enter(&statep->state_mutex); 1498 ibcm_close_exit(); 1499 statep->state = IBCM_STATE_ESTABLISHED; 1500 IBCM_REF_CNT_DECR(statep); 1501 cv_broadcast(&statep->block_mad_cv); 1502 statep->close_flow = 0; 1503 mutex_exit(&statep->state_mutex); 1504 return (status); 1505 } 1506 } else 1507 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: " 1508 "DREQ MAD already allocated in statep %p", statep); 1509 1510 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) { 1511 statep->close_priv_data = NULL; 1512 statep->close_priv_data_len = NULL; 1513 } else { 1514 statep->close_priv_data = ret_priv_data; 1515 statep->close_priv_data_len = ret_priv_data_len_p; 1516 } 1517 1518 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ); 1519 if ((priv_data != NULL) && (priv_data_len > 0)) 1520 bcopy(priv_data, ((ibcm_dreq_msg_t *) 1521 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data, 1522 priv_data_len); 1523 1524 ibcm_post_dreq_mad(statep); 1525 1526 mutex_enter(&statep->state_mutex); 1527 1528 lost_race: 1529 if (mode == IBT_BLOCKING) { 1530 1531 /* wait for DREP */ 1532 while (statep->close_done != B_TRUE) 1533 cv_wait(&statep->block_client_cv, 1534 &statep->state_mutex); 1535 1536 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1537 "done blocking", channel); 1538 } 1539 1540 IBCM_REF_CNT_DECR(statep); 1541 mutex_exit(&statep->state_mutex); 1542 1543 /* If this message isn't seen then ibt_close_rc_channel failed */ 1544 IBTF_DPRINTF_L5(cmlog, "ibt_close_rc_channel: chan 0x%p done", 1545 channel); 1546 1547 return (IBT_SUCCESS); 1548 } 1549 1550 ibt_status_t 1551 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control, 1552 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg) 1553 { 1554 ibcm_state_data_t *statep; 1555 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg; 1556 ibt_qp_query_attr_t qp_attr; 1557 ibt_status_t retval; 1558 1559 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan, 1560 control, hca_port_num, func, arg); 1561 1562 if (IBCM_INVALID_CHANNEL(rc_chan)) { 1563 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel"); 1564 return (IBT_CHAN_HDL_INVALID); 1565 } 1566 1567 /* check qp state */ 1568 retval = ibt_query_qp(rc_chan, &qp_attr); 1569 1570 if (retval != IBT_SUCCESS) 1571 return (retval); 1572 1573 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV) 1574 return (IBT_CHAN_SRV_TYPE_INVALID); 1575 1576 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR) 1577 return (IBT_CHAN_STATE_INVALID); 1578 1579 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1580 1581 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t), 1582 KM_SLEEP); 1583 1584 ibcm_tq_recycle_arg->rc_chan = rc_chan; 1585 ibcm_tq_recycle_arg->control = control; 1586 ibcm_tq_recycle_arg->hca_port_num = hca_port_num; 1587 ibcm_tq_recycle_arg->func = func; 1588 ibcm_tq_recycle_arg->arg = arg; 1589 1590 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1591 1592 IBCM_GET_CHAN_PRIVATE(rc_chan, statep); 1593 1594 /* 1595 * If non-blocking ie., func specified and channel has not yet completed 1596 * the timewait, then schedule the work for later 1597 */ 1598 if ((func != NULL) && (statep != NULL)) { 1599 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1600 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1601 statep->recycle_arg = ibcm_tq_recycle_arg; 1602 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1603 return (IBT_SUCCESS); 1604 } 1605 1606 /* 1607 * if blocking ie., func specified, and channel has not yet completed 1608 * the timewait, then block until the channel completes the timewait 1609 */ 1610 if (statep != NULL) 1611 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1612 IBCM_WAIT_CHAN_PRIVATE(rc_chan); 1613 1614 if (func) { /* NON BLOCKING case. Taskq for QP state change */ 1615 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle, 1616 ibcm_tq_recycle_arg, TQ_SLEEP); 1617 return (IBT_SUCCESS); 1618 } else /* BLOCKING case */ 1619 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg)); 1620 } 1621 1622 void 1623 ibcm_process_rc_recycle(void *recycle_arg) 1624 { 1625 (void) ibcm_process_rc_recycle_ret(recycle_arg); 1626 } 1627 1628 static ibt_status_t 1629 ibcm_process_rc_recycle_ret(void *recycle_arg) 1630 { 1631 ibt_qp_info_t qp_info; 1632 ibt_status_t ibt_status = IBT_SUCCESS; 1633 ibt_cep_modify_flags_t cep_flags; 1634 ibt_qp_query_attr_t qp_attr; 1635 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg = 1636 (ibcm_taskq_recycle_arg_t *)recycle_arg; 1637 1638 /* QP must have been in error state */ 1639 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr); 1640 if (ibt_status != IBT_SUCCESS) 1641 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1642 "chanp %p ibt_query_qp() = %d", 1643 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1644 else { 1645 /* perform the QP state change from ERROR to RESET */ 1646 bzero(&qp_info, sizeof (qp_info)); 1647 1648 qp_info.qp_trans = IBT_RC_SRV; 1649 qp_info.qp_state = IBT_STATE_RESET; 1650 1651 /* Call modify_qp to move to RESET state */ 1652 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1653 IBT_CEP_SET_STATE, &qp_info, NULL); 1654 1655 if (ibt_status != IBT_SUCCESS) 1656 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1657 "chanp %p ibt_modify_qp() = %d for ERROR to RESET", 1658 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1659 } 1660 1661 if (ibt_status == IBT_SUCCESS) { 1662 1663 qp_info.qp_state = IBT_STATE_INIT; 1664 1665 /* set flags for all mandatory args from RESET to INIT */ 1666 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 1667 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W; 1668 cep_flags |= IBT_CEP_SET_ATOMIC; 1669 1670 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 1671 ibcm_tq_recycle_arg->hca_port_num; 1672 qp_info.qp_flags |= 1673 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD; 1674 qp_info.qp_flags |= 1675 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR; 1676 qp_info.qp_flags |= 1677 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC; 1678 1679 /* Always use the existing pkey */ 1680 qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 1681 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix; 1682 1683 /* Call modify_qp to move to INIT state */ 1684 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1685 cep_flags, &qp_info, NULL); 1686 1687 if (ibt_status != IBT_SUCCESS) 1688 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1689 "chanp %p ibt_modify_qp() = %d for RESET to INIT", 1690 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1691 } 1692 1693 /* Change the QP CM state to indicate QP being re-used */ 1694 if (ibt_status == IBT_SUCCESS) 1695 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan); 1696 1697 /* Call func, if defined */ 1698 if (ibcm_tq_recycle_arg->func) 1699 (*(ibcm_tq_recycle_arg->func))(ibt_status, 1700 ibcm_tq_recycle_arg->arg); 1701 1702 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t)); 1703 1704 return (ibt_status); 1705 } 1706 1707 static void 1708 ibcm_process_abort_via_taskq(void *args) 1709 { 1710 ibcm_state_data_t *statep = (ibcm_state_data_t *)args; 1711 1712 ibcm_process_abort(statep); 1713 mutex_enter(&statep->state_mutex); 1714 IBCM_REF_CNT_DECR(statep); 1715 mutex_exit(&statep->state_mutex); 1716 } 1717 1718 /* 1719 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in 1720 * Non-Blocking mode operations. 1721 */ 1722 typedef struct ibcm_local_handler_s { 1723 ibt_cm_ud_handler_t actual_cm_handler; 1724 void *actual_cm_private; 1725 ibt_ud_dest_t *dest_hdl; 1726 } ibcm_local_handler_t; 1727 1728 _NOTE(READ_ONLY_DATA(ibcm_local_handler_s)) 1729 1730 /* 1731 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in 1732 * NON-Blocking mode. 1733 * 1734 * Out here, we update the UD Destination handle with 1735 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client 1736 * handler that was specified by the client. 1737 */ 1738 static ibt_cm_status_t 1739 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event, 1740 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 1741 { 1742 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv; 1743 1744 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d", 1745 event->cm_type); 1746 1747 ASSERT(handler_priv != NULL); 1748 1749 switch (event->cm_type) { 1750 case IBT_CM_UD_EVENT_SIDR_REP: 1751 /* Update QPN & QKey from event into destination handle. */ 1752 if (handler_priv->dest_hdl != NULL) { 1753 handler_priv->dest_hdl->ud_dst_qpn = 1754 event->cm_event.sidr_rep.srep_remote_qpn; 1755 handler_priv->dest_hdl->ud_qkey = 1756 event->cm_event.sidr_rep.srep_remote_qkey; 1757 } 1758 1759 /* Invoke the client handler - inform only, so ignore retval */ 1760 (void) handler_priv->actual_cm_handler( 1761 handler_priv->actual_cm_private, event, ret_args, priv_data, 1762 len); 1763 1764 /* Free memory allocated for local handler's private data. */ 1765 if (handler_priv != NULL) 1766 kmem_free(handler_priv, sizeof (*handler_priv)); 1767 1768 break; 1769 default: 1770 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR"); 1771 break; 1772 } 1773 1774 return (IBT_CM_ACCEPT); 1775 } 1776 1777 1778 /* Validate the input UD destination attributes. */ 1779 static ibt_status_t 1780 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1781 ibt_ud_returns_t *ret_args) 1782 { 1783 /* cm handler must always be specified */ 1784 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) { 1785 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1786 "CM handler is not specified "); 1787 return (IBT_INVALID_PARAM); 1788 } 1789 1790 if (mode == IBT_NONBLOCKING) { 1791 if (ret_args != NULL) { 1792 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1793 "ret_args should be NULL when called in " 1794 "non-blocking mode"); 1795 return (IBT_INVALID_PARAM); 1796 } 1797 } else if (mode == IBT_BLOCKING) { 1798 if (ret_args == NULL) { 1799 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1800 "ret_args should be Non-NULL when called in " 1801 "blocking mode"); 1802 return (IBT_INVALID_PARAM); 1803 } 1804 } else { 1805 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1806 "invalid mode %x specified ", mode); 1807 return (IBT_INVALID_PARAM); 1808 } 1809 1810 if (attr->ud_sid == 0) { 1811 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1812 "ServiceID must be specified. "); 1813 return (IBT_INVALID_PARAM); 1814 } 1815 1816 if (attr->ud_addr == NULL) { 1817 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1818 "Address Info NULL"); 1819 return (IBT_INVALID_PARAM); 1820 } 1821 1822 /* Validate SGID */ 1823 if ((attr->ud_addr->av_sgid.gid_prefix == 0) || 1824 (attr->ud_addr->av_sgid.gid_guid == 0)) { 1825 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID"); 1826 return (IBT_INVALID_PARAM); 1827 } 1828 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>", 1829 attr->ud_addr->av_sgid.gid_prefix, 1830 attr->ud_addr->av_sgid.gid_guid); 1831 1832 /* Validate DGID */ 1833 if ((attr->ud_addr->av_dgid.gid_prefix == 0) || 1834 (attr->ud_addr->av_dgid.gid_guid == 0)) { 1835 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID"); 1836 return (IBT_INVALID_PARAM); 1837 } 1838 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>", 1839 attr->ud_addr->av_dgid.gid_prefix, 1840 attr->ud_addr->av_dgid.gid_guid); 1841 1842 return (IBT_SUCCESS); 1843 } 1844 1845 1846 /* Perform SIDR to retrieve DQPN and QKey. */ 1847 static ibt_status_t 1848 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1849 ibt_ud_returns_t *ret_args) 1850 { 1851 ibt_status_t retval; 1852 ib_pkey_t ud_pkey; 1853 ibmf_handle_t ibmf_hdl; 1854 ibmf_msg_t *ibmf_msg; 1855 ibcm_hca_info_t *hcap; 1856 ibcm_sidr_req_msg_t *sidr_req_msgp; 1857 ibcm_ud_state_data_t *ud_statep; 1858 ibtl_cm_hca_port_t port; 1859 ibcm_sidr_srch_t sidr_entry; 1860 ibcm_qp_list_t *cm_qp_entry; 1861 1862 /* Retrieve HCA GUID value from the available SGID info. */ 1863 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port); 1864 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) { 1865 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1866 "ibtl_cm_get_hca_port failed: %d", retval); 1867 return (retval); 1868 } 1869 1870 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: " 1871 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port); 1872 1873 /* Lookup the HCA info for this GUID */ 1874 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) { 1875 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL"); 1876 return (IBT_HCA_INVALID); 1877 } 1878 1879 /* Return failure if the HCA device or Port is not operational */ 1880 1881 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port, 1882 NULL, NULL)) != IBT_SUCCESS) { 1883 /* Device Port is not in good state, don't use it. */ 1884 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid " 1885 "port specified or port not active"); 1886 ibcm_dec_hca_acc_cnt(hcap); 1887 return (retval); 1888 } 1889 1890 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port, 1891 attr->ud_pkey_ix, &ud_pkey); 1892 if (retval != IBT_SUCCESS) { 1893 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1894 "Failed to convert index2pkey: %d", retval); 1895 ibcm_dec_hca_acc_cnt(hcap); 1896 return (retval); 1897 } 1898 1899 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(sidr_entry)) 1900 1901 /* Allocate a new request id */ 1902 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) { 1903 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1904 "no req id available"); 1905 ibcm_dec_hca_acc_cnt(hcap); 1906 return (IBT_INSUFF_KERNEL_RESOURCE); 1907 } 1908 1909 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) && 1910 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1)) 1911 != IBT_SUCCESS)) { 1912 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1913 "ibmf reg or callback setup failed during re-initialize"); 1914 return (retval); 1915 } 1916 1917 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl; 1918 1919 /* find the ibmf QP to post the SIDR REQ */ 1920 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) == 1921 NULL) { 1922 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation" 1923 " failed"); 1924 ibcm_dec_hca_acc_cnt(hcap); 1925 return (IBT_INSUFF_RESOURCE); 1926 } 1927 1928 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND)) 1929 != IBT_SUCCESS) { 1930 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation" 1931 " failed"); 1932 ibcm_release_qp(cm_qp_entry); 1933 ibcm_dec_hca_acc_cnt(hcap); 1934 return (retval); 1935 } 1936 1937 sidr_entry.srch_lid = port.hp_base_lid; 1938 sidr_entry.srch_gid = attr->ud_addr->av_sgid; 1939 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh; 1940 sidr_entry.srch_mode = IBCM_ACTIVE_MODE; 1941 1942 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(sidr_entry)) 1943 1944 /* do various allocations needed here */ 1945 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 1946 1947 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep, 1948 IBCM_FLAG_ADD); 1949 rw_exit(&hcap->hca_sidr_list_lock); 1950 1951 /* Increment hca's resource count */ 1952 ibcm_inc_hca_res_cnt(hcap); 1953 1954 /* After a resource created on hca, no need to hold the acc cnt */ 1955 ibcm_dec_hca_acc_cnt(hcap); 1956 1957 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 1958 1959 /* Initialize some ud_statep fields */ 1960 ud_statep->ud_stored_msg = ibmf_msg; 1961 ud_statep->ud_svc_id = attr->ud_sid; 1962 ud_statep->ud_pkt_life_time = 1963 ibt_ib2usec(attr->ud_pkt_lt); 1964 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry; 1965 1966 /* set remaining retry cnt */ 1967 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries; 1968 1969 /* 1970 * Get UD handler and corresponding args which is pass it back 1971 * as first argument for the handler. 1972 */ 1973 ud_statep->ud_state_cm_private = attr->ud_cm_private; 1974 1975 if (mode == IBT_BLOCKING) 1976 ud_statep->ud_return_data = ret_args; 1977 else 1978 ud_statep->ud_cm_handler = attr->ud_cm_handler; 1979 1980 /* Initialize the fields of ud_statep->ud_stored_reply_addr */ 1981 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh; 1982 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl; 1983 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit = 1984 attr->ud_addr->av_hop; 1985 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid = 1986 attr->ud_addr->av_sgid; 1987 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid = 1988 attr->ud_addr->av_dgid; 1989 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass = 1990 attr->ud_addr->av_tclass; 1991 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label = 1992 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK; 1993 1994 /* needs to be derived based on the base LID and path bits */ 1995 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid = 1996 port.hp_base_lid; 1997 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid = 1998 attr->ud_addr->av_dlid; 1999 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey; 2000 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY; 2001 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level = 2002 attr->ud_addr->av_srvl; 2003 2004 /* 2005 * This may be enchanced later, to use a remote qno based on past 2006 * redirect rej mad responses. This would be the place to specify 2007 * appropriate remote qno 2008 */ 2009 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1; 2010 2011 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2012 2013 /* Initialize the SIDR REQ message fields */ 2014 sidr_req_msgp = 2015 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg); 2016 2017 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id); 2018 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid); 2019 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey); 2020 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID = 2021 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID); 2022 2023 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) { 2024 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data, 2025 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ)); 2026 } 2027 2028 /* Send out the SIDR REQ message */ 2029 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT; 2030 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT; 2031 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */ 2032 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) + 2033 (ud_statep->ud_pkt_life_time * 2); 2034 2035 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID = 2036 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ, 2037 ud_statep->ud_req_id, 0)); 2038 2039 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x", 2040 ud_statep->ud_timer_value); 2041 2042 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 2043 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2044 2045 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg, 2046 ibcm_post_sidr_req_complete, ud_statep); 2047 2048 mutex_enter(&ud_statep->ud_state_mutex); 2049 2050 /* Wait for SIDR_REP */ 2051 if (mode == IBT_BLOCKING) { 2052 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking"); 2053 2054 while (ud_statep->ud_blocking_done != B_TRUE) { 2055 cv_wait(&ud_statep->ud_block_client_cv, 2056 &ud_statep->ud_state_mutex); 2057 } 2058 2059 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking"); 2060 2061 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) { 2062 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, " 2063 "status = %x, QKey = %x", ret_args->ud_dqpn, 2064 ret_args->ud_status, ret_args->ud_qkey); 2065 2066 } else { 2067 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>", 2068 ret_args->ud_status); 2069 retval = IBT_CM_FAILURE; 2070 } 2071 } 2072 2073 IBCM_UD_REF_CNT_DECR(ud_statep); 2074 mutex_exit(&ud_statep->ud_state_mutex); 2075 2076 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done"); 2077 2078 return (retval); 2079 } 2080 2081 2082 /* 2083 * Function: 2084 * ibt_request_ud_dest 2085 * Input: 2086 * ud_dest A previously allocated UD destination handle. 2087 * mode This function can execute in blocking or non blocking 2088 * modes. 2089 * attr UD destination attributes to be modified. 2090 * Output: 2091 * ud_ret_args If the function is called in blocking mode, ud_ret_args 2092 * should be a pointer to an ibt_ud_returns_t struct. 2093 * Returns: 2094 * IBT_SUCCESS 2095 * Description: 2096 * Modify a previously allocated UD destination handle based on the 2097 * results of doing the SIDR protocol. 2098 */ 2099 ibt_status_t 2100 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode, 2101 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args) 2102 { 2103 ibt_status_t retval; 2104 ibt_ud_dest_t *ud_destp; 2105 ibcm_local_handler_t *local_handler_priv = NULL; 2106 2107 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)", 2108 ud_dest, mode, attr, ud_ret_args); 2109 2110 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args); 2111 if (retval != IBT_SUCCESS) { 2112 return (retval); 2113 } 2114 2115 ud_destp = ud_dest; 2116 2117 /* Allocate an Address handle. */ 2118 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah, 2119 attr->ud_addr); 2120 if (retval != IBT_SUCCESS) { 2121 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2122 "Address Handle Modification failed: %d", retval); 2123 return (retval); 2124 } 2125 2126 if (mode == IBT_NONBLOCKING) { 2127 /* 2128 * In NON-BLOCKING mode, and we need to update the destination 2129 * handle with the DQPN and QKey that are obtained from 2130 * SIDR REP, hook-up our own handler, so that we can catch 2131 * the event, and we ourselves call the actual client's 2132 * ud_cm_handler, in our handler. 2133 */ 2134 2135 /* Allocate memory for local handler's private data. */ 2136 local_handler_priv = 2137 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP); 2138 2139 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2140 2141 local_handler_priv->actual_cm_handler = attr->ud_cm_handler; 2142 local_handler_priv->actual_cm_private = attr->ud_cm_private; 2143 local_handler_priv->dest_hdl = ud_destp; 2144 2145 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2146 2147 attr->ud_cm_handler = ibcm_local_cm_handler; 2148 attr->ud_cm_private = local_handler_priv; 2149 } 2150 2151 /* In order to get DQPN and Destination QKey, perform SIDR */ 2152 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args); 2153 if (retval != IBT_SUCCESS) { 2154 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2155 "Failed to get DQPN: %d", retval); 2156 2157 /* Free memory allocated for local handler's private data. */ 2158 if (local_handler_priv != NULL) 2159 kmem_free(local_handler_priv, 2160 sizeof (*local_handler_priv)); 2161 return (retval); 2162 } 2163 2164 /* 2165 * Fill in the dqpn and dqkey as obtained from ud_ret_args, 2166 * values will be valid only on BLOCKING mode. 2167 */ 2168 if (mode == IBT_BLOCKING) { 2169 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn; 2170 ud_destp->ud_qkey = ud_ret_args->ud_qkey; 2171 } 2172 2173 return (retval); 2174 } 2175 2176 /* 2177 * Function: 2178 * ibt_ud_get_dqpn 2179 * Input: 2180 * attr A pointer to an ibt_ud_dest_attr_t struct that are 2181 * required for SIDR REQ message. Not specified attributes 2182 * should be set to "NULL" or "0". 2183 * ud_sid, ud_addr and ud_pkt_lt must be specified. 2184 * mode This function can execute in blocking or non blocking 2185 * modes. 2186 * Output: 2187 * returns If the function is called in blocking mode, returns 2188 * should be a pointer to an ibt_ud_returns_t struct. 2189 * Return: 2190 * IBT_SUCCESS on success or respective failure on error. 2191 * Description: 2192 * Finds the destination QPN at the specified destination that the 2193 * specified service can be reached on. The IBTF CM initiates the 2194 * service ID resolution protocol (SIDR) to determine a destination QPN. 2195 * 2196 * NOTE: SIDR_REQ is initiated from active side. 2197 */ 2198 ibt_status_t 2199 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 2200 ibt_ud_returns_t *returns) 2201 { 2202 ibt_status_t retval; 2203 2204 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)", 2205 attr, mode, returns); 2206 2207 retval = ibcm_validate_dqpn_data(attr, mode, returns); 2208 if (retval != IBT_SUCCESS) { 2209 return (retval); 2210 } 2211 2212 return (ibcm_ud_get_dqpn(attr, mode, returns)); 2213 } 2214 2215 2216 /* 2217 * ibt_cm_delay: 2218 * A client CM handler function can call this function 2219 * to extend its response time to a CM event. 2220 * INPUTS: 2221 * flags Indicates what CM message processing is being delayed 2222 * by the CM handler, valid values are: 2223 * IBT_CM_DELAY_REQ 2224 * IBT_CM_DELAY_REP 2225 * IBT_CM_DELAY_LAP 2226 * cm_session_id The session ID that was passed to client srv_handler 2227 * by the CM 2228 * service_time The extended service time 2229 * priv_data Vendor specific data to be sent in the CM generated 2230 * MRA message. Should be NULL if not specified. 2231 * len The number of bytes of data specified by priv_data. 2232 * 2233 * RETURN VALUES: 2234 * IBT_SUCCESS on success (or respective failure on error) 2235 */ 2236 ibt_status_t 2237 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id, 2238 clock_t service_time, void *priv_data, ibt_priv_data_len_t len) 2239 { 2240 uint8_t msg_typ = 0; 2241 ibcm_mra_msg_t *mra_msgp; 2242 ibcm_state_data_t *statep; 2243 ibt_status_t status; 2244 2245 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)", 2246 flags, cm_session_id, service_time); 2247 2248 /* 2249 * Make sure channel is associated with a statep 2250 */ 2251 statep = (ibcm_state_data_t *)cm_session_id; 2252 2253 if (statep == NULL) { 2254 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL"); 2255 return (IBT_INVALID_PARAM); 2256 } 2257 2258 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep); 2259 2260 /* Allocate an ibmf msg for mra, if not allocated yet */ 2261 if (statep->mra_msg == NULL) { 2262 if ((status = ibcm_alloc_out_msg( 2263 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg, 2264 MAD_METHOD_SEND)) != IBT_SUCCESS) { 2265 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p" 2266 "IBMF MSG allocation failed", statep->channel); 2267 return (status); 2268 } 2269 } 2270 2271 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2272 2273 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg); 2274 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid); 2275 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid); 2276 2277 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */ 2278 if (flags == IBT_CM_DELAY_REQ) { 2279 msg_typ = IBT_CM_MRA_TYPE_REQ; 2280 } else if (flags == IBT_CM_DELAY_REP) { 2281 msg_typ = IBT_CM_MRA_TYPE_REP; 2282 } else if (flags == IBT_CM_DELAY_LAP) { 2283 msg_typ = IBT_CM_MRA_TYPE_LAP; 2284 } 2285 2286 mra_msgp->mra_message_type_plus = msg_typ << 6; 2287 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3; 2288 2289 len = min(len, IBT_MRA_PRIV_DATA_SZ); 2290 if (priv_data && (len > 0)) 2291 bcopy(priv_data, mra_msgp->mra_private_data, len); 2292 2293 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID = 2294 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID); 2295 2296 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2297 2298 mutex_enter(&statep->state_mutex); 2299 2300 if ((statep->mode == IBCM_ACTIVE_MODE) && 2301 (statep->state == IBCM_STATE_REP_RCVD)) { 2302 statep->state = IBCM_STATE_MRA_REP_SENT; 2303 } else if (statep->mode == IBCM_PASSIVE_MODE) { 2304 if (statep->state == IBCM_STATE_REQ_RCVD) { 2305 statep->state = IBCM_STATE_MRA_SENT; 2306 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) { 2307 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD; 2308 } else { 2309 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2310 "/ap_state/mode %x, %x, %x", statep->state, 2311 statep->ap_state, statep->mode); 2312 mutex_exit(&statep->state_mutex); 2313 return (IBT_CHAN_STATE_INVALID); 2314 } 2315 } else { 2316 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2317 "/ap_state/mode %x, %x, %x", statep->state, 2318 statep->ap_state, statep->mode); 2319 mutex_exit(&statep->state_mutex); 2320 2321 return (IBT_CHAN_STATE_INVALID); 2322 } 2323 /* service time is usecs, stale_clock is nsecs */ 2324 statep->stale_clock = gethrtime() + 2325 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 * 2326 statep->max_cm_retries); 2327 2328 statep->send_mad_flags |= IBCM_MRA_POST_BUSY; 2329 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */ 2330 mutex_exit(&statep->state_mutex); 2331 2332 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID = 2333 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID; 2334 2335 /* post the MRA mad in blocking mode, as no timers involved */ 2336 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete, 2337 statep); 2338 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_MRA); 2339 /* If this message isn't seen then ibt_cm_delay failed */ 2340 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay: done !!"); 2341 2342 return (IBT_SUCCESS); 2343 } 2344 2345 2346 /* 2347 * ibt_register_service() 2348 * Register a service with the IBCM 2349 * 2350 * INPUTS: 2351 * ibt_hdl The IBT client handle returned to the client 2352 * on an ibt_attach() call. 2353 * 2354 * srv The address of a ibt_srv_desc_t that describes 2355 * the service, containing the following: 2356 * 2357 * sd_ud_handler The Service CM UD event Handler. 2358 * sd_handler The Service CM RC/UC/RD event Handler. 2359 * sd_flags Service flags (peer-to-peer, or not). 2360 * 2361 * sid This tells CM if the service is local (sid is 0) or 2362 * wellknown (sid is the starting service id of the range). 2363 * 2364 * num_sids The number of contiguous service-ids to reserve. 2365 * 2366 * srv_hdl The address of a service identification handle, used 2367 * to deregister a service, and to bind GIDs to. 2368 * 2369 * ret_sid The address to store the Service ID return value. 2370 * If num_sids > 1, ret_sid is the first Service ID 2371 * in the range. 2372 * 2373 * ibt_register_service() returns: 2374 * IBT_SUCCESS - added a service successfully. 2375 * IBT_INVALID_PARAM - invalid input parameter. 2376 * IBT_CM_FAILURE - failed to add the service. 2377 * IBT_CM_SERVICE_EXISTS - service already exists. 2378 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should 2379 * never happen). 2380 */ 2381 ibt_status_t 2382 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv, 2383 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid) 2384 { 2385 ibcm_svc_info_t *svcinfop; 2386 2387 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p, %p, %llx, %d)", 2388 ibt_hdl, srv, (longlong_t)sid, num_sids); 2389 2390 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 2391 2392 *srv_hdl = NULL; 2393 2394 if (num_sids <= 0) { 2395 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2396 "Invalid number of service-ids specified (%d)", num_sids); 2397 return (IBT_INVALID_PARAM); 2398 } 2399 2400 if (sid == 0) { 2401 if (ret_sid == NULL) 2402 return (IBT_INVALID_PARAM); 2403 sid = ibcm_alloc_local_sids(num_sids); 2404 if (sid == 0) 2405 return (IBT_INSUFF_KERNEL_RESOURCE); 2406 2407 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */ 2408 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) { 2409 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2410 "Invalid non-LOCAL SID specified: 0x%llX", 2411 (longlong_t)sid); 2412 return (IBT_INVALID_PARAM); 2413 } 2414 2415 svcinfop = ibcm_create_svc_entry(sid, num_sids); 2416 2417 if (svcinfop == NULL) { 2418 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2419 "Service-ID 0x%llx already registered", (longlong_t)sid); 2420 return (IBT_CM_SERVICE_EXISTS); 2421 } 2422 2423 /* 2424 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry() 2425 */ 2426 svcinfop->svc_flags = srv->sd_flags; 2427 svcinfop->svc_rc_handler = srv->sd_handler; 2428 svcinfop->svc_ud_handler = srv->sd_ud_handler; 2429 2430 if (ret_sid != NULL) 2431 *ret_sid = sid; 2432 2433 *srv_hdl = svcinfop; 2434 2435 ibtl_cm_change_service_cnt(ibt_hdl, num_sids); 2436 2437 /* If this message isn't seen, then ibt_register_service failed. */ 2438 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)", 2439 svcinfop, sid); 2440 2441 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 2442 2443 return (IBT_SUCCESS); 2444 } 2445 2446 2447 static ibt_status_t 2448 ibcm_write_service_record(ibmf_saa_handle_t saa_handle, 2449 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type) 2450 { 2451 int rval; 2452 int retry; 2453 2454 ibcm_sa_access_enter(); 2455 for (retry = 0; retry < ibcm_max_sa_retries; retry++) { 2456 rval = ibmf_saa_update_service_record( 2457 saa_handle, srv_recp, saa_type, 0); 2458 if (rval != IBMF_TRANS_TIMEOUT) { 2459 break; 2460 } 2461 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2462 "ibmf_saa_update_service_record timed out" 2463 " SID = %llX, rval = %d, saa_type = %d", 2464 (longlong_t)srv_recp->ServiceID, rval, saa_type); 2465 delay(ibcm_sa_timeout_delay); 2466 } 2467 ibcm_sa_access_exit(); 2468 2469 if (rval != IBMF_SUCCESS) { 2470 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2471 "ibmf_saa_update_service_record() : Failed - %d", rval); 2472 return (ibcm_ibmf_analyze_error(rval)); 2473 } else 2474 return (IBT_SUCCESS); 2475 } 2476 2477 2478 static void 2479 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec) 2480 { 2481 ibt_status_t retval; 2482 uint_t num_found; 2483 size_t length; 2484 sa_service_record_t *srv_resp; 2485 void *results_p; 2486 uint_t i; 2487 uint64_t component_mask; 2488 ibmf_saa_access_args_t access_args; 2489 2490 component_mask = 2491 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID; 2492 2493 /* Call in SA Access retrieve routine to get Service Records. */ 2494 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 2495 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 2496 access_args.sq_component_mask = component_mask; 2497 access_args.sq_template = srec; 2498 access_args.sq_template_length = sizeof (sa_service_record_t); 2499 access_args.sq_callback = NULL; 2500 access_args.sq_callback_arg = NULL; 2501 2502 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 2503 &results_p); 2504 if (retval != IBT_SUCCESS) { 2505 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2506 "SA Access Failure"); 2507 return; 2508 } 2509 2510 num_found = length / sizeof (sa_service_record_t); 2511 2512 if (num_found) 2513 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2514 "Found %d matching Service Records.", num_found); 2515 2516 /* Validate the returned number of records. */ 2517 if ((results_p != NULL) && (num_found > 0)) { 2518 2519 /* Remove all the records. */ 2520 for (i = 0; i < num_found; i++) { 2521 2522 srv_resp = (sa_service_record_t *) 2523 ((uchar_t *)results_p + 2524 i * sizeof (sa_service_record_t)); 2525 2526 /* 2527 * Found some matching records, but check out whether 2528 * this Record is really stale or just happens to match 2529 * the current session records. If yes, don't remove it. 2530 */ 2531 mutex_enter(&ibcm_svc_info_lock); 2532 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) { 2533 /* This record is NOT STALE. */ 2534 mutex_exit(&ibcm_svc_info_lock); 2535 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2536 "This is not Stale, it's an active record"); 2537 continue; 2538 } 2539 mutex_exit(&ibcm_svc_info_lock); 2540 2541 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2542 "Removing Stale Rec: %s, %llX", 2543 srv_resp->ServiceName, srv_resp->ServiceID); 2544 2545 IBCM_DUMP_SERVICE_REC(srv_resp); 2546 2547 /* 2548 * Remove the Service Record Entry from SA. 2549 * 2550 * Get ServiceID info from Response Buf, other 2551 * attributes are already filled-in. 2552 */ 2553 2554 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2555 2556 srec->ServiceID = srv_resp->ServiceID; 2557 2558 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2559 2560 (void) ibcm_write_service_record(saa_handle, srec, 2561 IBMF_SAA_DELETE); 2562 } 2563 2564 /* Deallocate the memory for results_p. */ 2565 kmem_free(results_p, length); 2566 } 2567 } 2568 2569 2570 2571 /* 2572 * ibt_bind_service() 2573 * Register a service with the IBCM 2574 * 2575 * INPUTS: 2576 * srv_hdl The service id handle returned to the client 2577 * on an ibt_service_register() call. 2578 * 2579 * gid The GID to which to bind the service. 2580 * 2581 * srv_bind The address of a ibt_srv_bind_t that describes 2582 * the service record. This should be NULL if there 2583 * is to be no service record. This contains: 2584 * 2585 * sb_lease Lease period 2586 * sb_pkey Partition 2587 * sb_name pointer to ASCII string Service Name, 2588 * NULL terminated. 2589 * sb_key[] Key to secure the service record. 2590 * sb_data Service Data structure (64-byte) 2591 * 2592 * cm_private First argument of Service handler. 2593 * 2594 * sb_hdl_p The address of a service bind handle, used 2595 * to undo the service binding. 2596 * 2597 * ibt_bind_service() returns: 2598 * IBT_SUCCESS - added a service successfully. 2599 * IBT_INVALID_PARAM - invalid input parameter. 2600 * IBT_CM_FAILURE - failed to add the service. 2601 * IBT_CM_SERVICE_EXISTS - service already exists. 2602 */ 2603 ibt_status_t 2604 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind, 2605 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p) 2606 { 2607 ibt_status_t status; 2608 ibtl_cm_hca_port_t port; 2609 ibcm_svc_bind_t *sbindp, *sbp; 2610 ibcm_hca_info_t *hcap; 2611 ib_svc_id_t sid, start_sid, end_sid; 2612 ibmf_saa_handle_t saa_handle; 2613 sa_service_record_t srv_rec; 2614 uint16_t pkey_ix; 2615 2616 if (sb_hdl_p != NULL) 2617 *sb_hdl_p = NULL; /* return value for error cases */ 2618 2619 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)", 2620 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid); 2621 2622 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sbindp)) 2623 2624 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */ 2625 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) { 2626 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2627 "ibtl_cm_get_hca_port failed: %d", status); 2628 return (status); 2629 } 2630 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX", 2631 port.hp_port, port.hp_hca_guid); 2632 2633 hcap = ibcm_find_hca_entry(port.hp_hca_guid); 2634 if (hcap == NULL) { 2635 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found"); 2636 return (IBT_HCA_BUSY_DETACHING); 2637 } 2638 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap); 2639 2640 if (srv_bind != NULL) { 2641 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2642 if (saa_handle == NULL) { 2643 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2644 "saa_handle is NULL"); 2645 ibcm_dec_hca_acc_cnt(hcap); 2646 return (IBT_HCA_PORT_NOT_ACTIVE); 2647 } 2648 if (srv_bind->sb_pkey == 0) { 2649 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2650 "P_Key must not be 0"); 2651 ibcm_dec_hca_acc_cnt(hcap); 2652 return (IBT_INVALID_PARAM); 2653 } 2654 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) { 2655 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2656 "Service Name is too long"); 2657 ibcm_dec_hca_acc_cnt(hcap); 2658 return (IBT_INVALID_PARAM); 2659 } else 2660 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: " 2661 "Service Name='%s'", srv_bind->sb_name); 2662 status = ibt_pkey2index_byguid(port.hp_hca_guid, 2663 port.hp_port, srv_bind->sb_pkey, &pkey_ix); 2664 if (status != IBT_SUCCESS) { 2665 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2666 "P_Key 0x%x not found in P_Key_Table", 2667 srv_bind->sb_pkey); 2668 ibcm_dec_hca_acc_cnt(hcap); 2669 return (status); 2670 } 2671 } 2672 2673 /* assume success - allocate before locking */ 2674 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP); 2675 sbindp->sbind_cm_private = cm_private; 2676 sbindp->sbind_gid = gid; 2677 sbindp->sbind_hcaguid = port.hp_hca_guid; 2678 sbindp->sbind_port = port.hp_port; 2679 2680 mutex_enter(&ibcm_svc_info_lock); 2681 2682 sbp = srv_hdl->svc_bind_list; 2683 while (sbp != NULL) { 2684 if (sbp->sbind_gid.gid_guid == gid.gid_guid && 2685 sbp->sbind_gid.gid_prefix == gid.gid_prefix) { 2686 if (srv_bind == NULL || 2687 srv_bind->sb_pkey == sbp->sbind_pkey) { 2688 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2689 "failed: GID %llx:%llx and PKEY %x is " 2690 "already bound", gid.gid_guid, 2691 gid.gid_prefix, srv_bind->sb_pkey); 2692 mutex_exit(&ibcm_svc_info_lock); 2693 ibcm_dec_hca_acc_cnt(hcap); 2694 kmem_free(sbindp, sizeof (*sbindp)); 2695 return (IBT_CM_SERVICE_EXISTS); 2696 } 2697 } 2698 sbp = sbp->sbind_link; 2699 } 2700 /* no entry found */ 2701 2702 sbindp->sbind_link = srv_hdl->svc_bind_list; 2703 srv_hdl->svc_bind_list = sbindp; 2704 2705 mutex_exit(&ibcm_svc_info_lock); 2706 2707 if (srv_bind != NULL) { 2708 bzero(&srv_rec, sizeof (srv_rec)); 2709 2710 srv_rec.ServiceLease = 2711 sbindp->sbind_lease = srv_bind->sb_lease; 2712 srv_rec.ServiceP_Key = 2713 sbindp->sbind_pkey = srv_bind->sb_pkey; 2714 srv_rec.ServiceKey_hi = 2715 sbindp->sbind_key[0] = srv_bind->sb_key[0]; 2716 srv_rec.ServiceKey_lo = 2717 sbindp->sbind_key[1] = srv_bind->sb_key[1]; 2718 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name); 2719 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name); 2720 srv_rec.ServiceGID = gid; 2721 2722 /* 2723 * Find out whether we have any stale Local Service records 2724 * matching the current attributes. If yes, we shall try to 2725 * remove them from SA using the current request's ServiceKey. 2726 * 2727 * We will perform this operation only for Local Services, as 2728 * it is handled by SA automatically for WellKnown Services. 2729 * 2730 * Ofcourse, clients can specify NOT to do this clean-up by 2731 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag). 2732 */ 2733 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) && 2734 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) { 2735 ibcm_rem_stale_srec(saa_handle, &srv_rec); 2736 } 2737 2738 /* Handle endianess for service data. */ 2739 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data); 2740 2741 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 2742 2743 /* insert srv record into the SA */ 2744 start_sid = srv_hdl->svc_id; 2745 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 2746 for (sid = start_sid; sid <= end_sid; sid++) { 2747 2748 srv_rec.ServiceID = sid; 2749 2750 IBCM_DUMP_SERVICE_REC(&srv_rec); 2751 2752 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: " 2753 "ibmf_saa_write_service_record, SvcId = %llX", 2754 (longlong_t)sid); 2755 2756 status = ibcm_write_service_record(saa_handle, &srv_rec, 2757 IBMF_SAA_UPDATE); 2758 if (status != IBT_SUCCESS) { 2759 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:" 2760 " ibcm_write_service_record fails %d, " 2761 "sid %llX", status, (longlong_t)sid); 2762 2763 if (sid != start_sid) { 2764 /* 2765 * Bind failed while bind SID other than 2766 * first in the sid_range. So we need 2767 * to unbind those, which are passed. 2768 * 2769 * Need to increment svc count to 2770 * compensate for ibt_unbind_service(). 2771 */ 2772 ibcm_inc_hca_svc_cnt(hcap); 2773 ibcm_dec_hca_acc_cnt(hcap); 2774 2775 (void) ibt_unbind_service(srv_hdl, 2776 sbindp); 2777 } else { 2778 ibcm_svc_bind_t **sbpp; 2779 2780 /* 2781 * Bind failed for the first SID or the 2782 * only SID in question, then no need 2783 * to unbind, just free memory and 2784 * return error. 2785 */ 2786 mutex_enter(&ibcm_svc_info_lock); 2787 2788 sbpp = &srv_hdl->svc_bind_list; 2789 sbp = *sbpp; 2790 while (sbp != NULL) { 2791 if (sbp == sbindp) { 2792 *sbpp = sbp->sbind_link; 2793 break; 2794 } 2795 sbpp = &sbp->sbind_link; 2796 sbp = *sbpp; 2797 } 2798 mutex_exit(&ibcm_svc_info_lock); 2799 ibcm_dec_hca_acc_cnt(hcap); 2800 2801 kmem_free(sbindp, sizeof (*sbindp)); 2802 } 2803 return (status); 2804 } 2805 } 2806 } 2807 ibcm_inc_hca_svc_cnt(hcap); 2808 ibcm_dec_hca_acc_cnt(hcap); 2809 2810 /* If this message isn't seen then ibt_bind_service failed */ 2811 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llX)", 2812 srv_hdl, gid.gid_prefix, gid.gid_guid); 2813 2814 if (sb_hdl_p != NULL) 2815 *sb_hdl_p = sbindp; 2816 2817 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sbindp)) 2818 2819 return (IBT_SUCCESS); 2820 } 2821 2822 ibt_status_t 2823 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp) 2824 { 2825 ib_svc_id_t sid, end_sid; 2826 ibt_status_t rval; 2827 ibcm_hca_info_t *hcap; 2828 ibcm_svc_bind_t *sbp, **sbpp; 2829 2830 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)", 2831 srv_hdl, sbindp); 2832 2833 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 2834 2835 /* If there is a service on hca, respective hcap cannot go away */ 2836 ASSERT(hcap != NULL); 2837 2838 mutex_enter(&ibcm_svc_info_lock); 2839 2840 sbpp = &srv_hdl->svc_bind_list; 2841 sbp = *sbpp; 2842 while (sbp != NULL) { 2843 if (sbp == sbindp) { 2844 *sbpp = sbp->sbind_link; 2845 break; 2846 } 2847 sbpp = &sbp->sbind_link; 2848 sbp = *sbpp; 2849 } 2850 sid = srv_hdl->svc_id; 2851 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1; 2852 if (sbp != NULL) 2853 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY) 2854 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 2855 mutex_exit(&ibcm_svc_info_lock); 2856 2857 if (sbp == NULL) { 2858 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2859 "service binding not found: srv_hdl %p, srv_bind %p", 2860 srv_hdl, sbindp); 2861 ibcm_dec_hca_acc_cnt(hcap); 2862 return (IBT_INVALID_PARAM); 2863 } 2864 2865 if (sbindp->sbind_pkey != 0) { /* Are there service records? */ 2866 ibtl_cm_hca_port_t port; 2867 sa_service_record_t srv_rec; 2868 ibmf_saa_handle_t saa_handle; 2869 ibt_status_t status; 2870 2871 /* get the default SGID of the port */ 2872 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port)) 2873 != IBT_SUCCESS) { 2874 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2875 "ibtl_cm_get_hca_port failed: %d", status); 2876 /* we're done, but there may be stale service records */ 2877 goto done; 2878 } 2879 2880 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2881 if (saa_handle == NULL) { 2882 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2883 "saa_handle is NULL"); 2884 /* we're done, but there may be stale service records */ 2885 goto done; 2886 } 2887 2888 /* Fill in fields of srv_rec */ 2889 bzero(&srv_rec, sizeof (srv_rec)); 2890 2891 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 2892 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 2893 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 2894 srv_rec.ServiceGID = sbindp->sbind_gid; 2895 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 2896 2897 while (sid <= end_sid) { 2898 2899 srv_rec.ServiceID = sid; 2900 IBCM_DUMP_SERVICE_REC(&srv_rec); 2901 2902 rval = ibcm_write_service_record(saa_handle, &srv_rec, 2903 IBMF_SAA_DELETE); 2904 2905 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: " 2906 "ibcm_write_service_record rval = %d, SID %llx", 2907 rval, sid); 2908 if (rval != IBT_SUCCESS) { 2909 /* this is not considered a reason to fail */ 2910 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2911 "ibcm_write_service_record fails %d, " 2912 "sid %llx", rval, sid); 2913 } 2914 sid++; 2915 } 2916 } 2917 done: 2918 ibcm_dec_hca_svc_cnt(hcap); 2919 ibcm_dec_hca_acc_cnt(hcap); 2920 kmem_free(sbindp, sizeof (*sbindp)); 2921 2922 /* If this message isn't seen then ibt_unbind_service failed */ 2923 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!"); 2924 2925 return (IBT_SUCCESS); 2926 } 2927 2928 /* 2929 * Simply pull off each binding from the list and unbind it. 2930 * If any of the unbind calls fail, we fail. 2931 */ 2932 ibt_status_t 2933 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl) 2934 { 2935 ibt_status_t status; 2936 ibcm_svc_bind_t *sbp; 2937 2938 mutex_enter(&ibcm_svc_info_lock); 2939 sbp = NULL; 2940 2941 /* this compare keeps the loop from being infinite */ 2942 while (sbp != srv_hdl->svc_bind_list) { 2943 sbp = srv_hdl->svc_bind_list; 2944 mutex_exit(&ibcm_svc_info_lock); 2945 status = ibt_unbind_service(srv_hdl, sbp); 2946 if (status != IBT_SUCCESS) 2947 return (status); 2948 mutex_enter(&ibcm_svc_info_lock); 2949 if (srv_hdl->svc_bind_list == NULL) 2950 break; 2951 } 2952 mutex_exit(&ibcm_svc_info_lock); 2953 return (IBT_SUCCESS); 2954 } 2955 2956 /* 2957 * ibt_deregister_service() 2958 * Deregister a service with the IBCM 2959 * 2960 * INPUTS: 2961 * ibt_hdl The IBT client handle returned to the client 2962 * on an ibt_attach() call. 2963 * 2964 * srv_hdl The address of a service identification handle, used 2965 * to de-register a service. 2966 * RETURN VALUES: 2967 * IBT_SUCCESS on success (or respective failure on error) 2968 */ 2969 ibt_status_t 2970 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl) 2971 { 2972 ibcm_svc_info_t *svcp; 2973 ibcm_svc_lookup_t svc; 2974 2975 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p, %p)", 2976 ibt_hdl, srv_hdl); 2977 2978 mutex_enter(&ibcm_svc_info_lock); 2979 2980 if (srv_hdl->svc_bind_list != NULL) { 2981 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:" 2982 " srv_hdl %p still has bindings", srv_hdl); 2983 mutex_exit(&ibcm_svc_info_lock); 2984 return (IBT_CM_SERVICE_BUSY); 2985 } 2986 svc.sid = srv_hdl->svc_id; 2987 svc.num_sids = 1; 2988 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d", 2989 srv_hdl->svc_id, srv_hdl->svc_num_sids); 2990 2991 #ifdef __lock_lint 2992 ibcm_svc_compare(NULL, NULL); 2993 #endif 2994 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 2995 if (svcp != srv_hdl) { 2996 mutex_exit(&ibcm_svc_info_lock); 2997 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): " 2998 "srv_hdl %p not found", srv_hdl); 2999 return (IBT_INVALID_PARAM); 3000 } 3001 avl_remove(&ibcm_svc_avl_tree, svcp); 3002 3003 /* wait for active REQ/SREQ handling to be done */ 3004 svcp->svc_to_delete = 1; 3005 while (svcp->svc_ref_cnt != 0) 3006 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3007 3008 mutex_exit(&ibcm_svc_info_lock); 3009 3010 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) 3011 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids); 3012 3013 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids); 3014 kmem_free(srv_hdl, sizeof (*srv_hdl)); 3015 3016 /* If this message isn't seen then ibt_deregister_service failed */ 3017 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!"); 3018 3019 return (IBT_SUCCESS); 3020 } 3021 3022 ibcm_status_t 3023 ibcm_ar_init(void) 3024 { 3025 ib_svc_id_t sid = IBCM_DAPL_ATS_SID; 3026 ibcm_svc_info_t *tmp_svcp; 3027 3028 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()"); 3029 3030 /* remove this special SID from the pool of available SIDs */ 3031 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) { 3032 IBTF_DPRINTF_L2(cmlog, "ibcm_ar_init: " 3033 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid); 3034 return (IBCM_FAILURE); 3035 } 3036 mutex_enter(&ibcm_svc_info_lock); 3037 ibcm_ar_svcinfop = tmp_svcp; 3038 ibcm_ar_list = NULL; /* no address records registered yet */ 3039 mutex_exit(&ibcm_svc_info_lock); 3040 return (IBCM_SUCCESS); 3041 } 3042 3043 ibcm_status_t 3044 ibcm_ar_fini(void) 3045 { 3046 ibcm_ar_t *ar_list; 3047 ibcm_svc_info_t *tmp_svcp; 3048 3049 mutex_enter(&ibcm_svc_info_lock); 3050 ar_list = ibcm_ar_list; 3051 3052 if (ar_list == NULL && 3053 avl_numnodes(&ibcm_svc_avl_tree) == 1 && 3054 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) { 3055 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop); 3056 tmp_svcp = ibcm_ar_svcinfop; 3057 mutex_exit(&ibcm_svc_info_lock); 3058 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop)); 3059 return (IBCM_SUCCESS); 3060 } 3061 mutex_exit(&ibcm_svc_info_lock); 3062 return (IBCM_FAILURE); 3063 } 3064 3065 3066 /* 3067 * Return to the caller: 3068 * IBT_SUCCESS Found a perfect match. 3069 * *arpp is set to the record. 3070 * IBT_INCONSISTENT_AR Found a record that's inconsistent. 3071 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and 3072 * found no record with same data. 3073 */ 3074 static ibt_status_t 3075 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp) 3076 { 3077 ibcm_ar_t *tmp; 3078 int i; 3079 3080 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3081 tmp = ibcm_ar_list; 3082 while (tmp != NULL) { 3083 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix && 3084 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid && 3085 tmp->ar.ar_pkey == arp->ar_pkey) { 3086 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3087 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3088 return (IBT_INCONSISTENT_AR); 3089 *arpp = tmp; 3090 return (IBT_SUCCESS); 3091 } else { 3092 /* if all the data bytes match, we have inconsistency */ 3093 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3094 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3095 break; 3096 if (i == IBCM_DAPL_ATS_NBYTES) 3097 return (IBT_INCONSISTENT_AR); 3098 /* try next address record */ 3099 } 3100 tmp = tmp->ar_link; 3101 } 3102 return (IBT_AR_NOT_REGISTERED); 3103 } 3104 3105 ibt_status_t 3106 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3107 { 3108 ibcm_ar_t *found; 3109 ibcm_ar_t *tmp; 3110 ibt_status_t status; 3111 ibt_status_t s1, s2; 3112 char *s; 3113 ibcm_ar_ref_t *hdlp; 3114 ibcm_ar_t *new; 3115 ibcm_ar_t **linkp; 3116 ibtl_cm_hca_port_t cm_port; 3117 uint16_t pkey_ix; 3118 ibcm_hca_info_t *hcap; 3119 ibmf_saa_handle_t saa_handle; 3120 sa_service_record_t *srv_recp; 3121 uint64_t gid_ored; 3122 3123 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX", 3124 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix, 3125 (longlong_t)arp->ar_gid.gid_guid); 3126 3127 /* 3128 * If P_Key is 0, but GID is not, this query is invalid. 3129 * If GID is 0, but P_Key is not, this query is invalid. 3130 */ 3131 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3132 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3133 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3134 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3135 "GID/P_Key is not valid"); 3136 return (IBT_INVALID_PARAM); 3137 } 3138 3139 /* assume success, so these might be needed */ 3140 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP); 3141 new = kmem_alloc(sizeof (*new), KM_SLEEP); 3142 3143 mutex_enter(&ibcm_svc_info_lock); 3144 /* search for existing GID/pkey (there can be at most 1) */ 3145 status = ibcm_search_ar(arp, &found); 3146 if (status == IBT_INCONSISTENT_AR) { 3147 mutex_exit(&ibcm_svc_info_lock); 3148 kmem_free(new, sizeof (*new)); 3149 kmem_free(hdlp, sizeof (*hdlp)); 3150 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3151 "address record is inconsistent with a known one"); 3152 return (IBT_INCONSISTENT_AR); 3153 } else if (status == IBT_SUCCESS) { 3154 if (found->ar_flags == IBCM_AR_INITING) { 3155 found->ar_waiters++; 3156 cv_wait(&found->ar_cv, &ibcm_svc_info_lock); 3157 found->ar_waiters--; 3158 } 3159 if (found->ar_flags == IBCM_AR_FAILED) { 3160 if (found->ar_waiters == 0) { 3161 cv_destroy(&found->ar_cv); 3162 kmem_free(found, sizeof (*found)); 3163 } 3164 mutex_exit(&ibcm_svc_info_lock); 3165 kmem_free(new, sizeof (*new)); 3166 kmem_free(hdlp, sizeof (*hdlp)); 3167 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0)); 3168 } 3169 hdlp->ar_ibt_hdl = ibt_hdl; 3170 hdlp->ar_ref_link = found->ar_ibt_hdl_list; 3171 found->ar_ibt_hdl_list = hdlp; 3172 mutex_exit(&ibcm_svc_info_lock); 3173 kmem_free(new, sizeof (*new)); 3174 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3175 return (IBT_SUCCESS); 3176 } else { 3177 ASSERT(status == IBT_AR_NOT_REGISTERED); 3178 } 3179 hdlp->ar_ref_link = NULL; 3180 hdlp->ar_ibt_hdl = ibt_hdl; 3181 new->ar_ibt_hdl_list = hdlp; 3182 new->ar = *arp; 3183 new->ar_flags = IBCM_AR_INITING; 3184 new->ar_waiters = 0; 3185 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL); 3186 new->ar_link = ibcm_ar_list; 3187 ibcm_ar_list = new; 3188 3189 /* verify GID/pkey is valid for a local port, etc. */ 3190 hcap = NULL; 3191 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port)) 3192 != IBT_SUCCESS || 3193 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port, 3194 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS || 3195 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) { 3196 cv_destroy(&new->ar_cv); 3197 ibcm_ar_list = new->ar_link; 3198 mutex_exit(&ibcm_svc_info_lock); 3199 kmem_free(new, sizeof (*new)); 3200 kmem_free(hdlp, sizeof (*hdlp)); 3201 status = IBT_INVALID_PARAM; 3202 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) { 3203 s = "PORT DOWN"; 3204 status = IBT_HCA_PORT_NOT_ACTIVE; 3205 } else if (s1 != IBT_SUCCESS) 3206 s = "GID not found"; 3207 else if (s2 != IBT_SUCCESS) 3208 s = "PKEY not found"; 3209 else 3210 s = "CM could not find its HCA entry"; 3211 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d", 3212 s, status); 3213 return (status); 3214 } 3215 mutex_exit(&ibcm_svc_info_lock); 3216 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port); 3217 3218 /* create service record */ 3219 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP); 3220 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */ 3221 srv_recp->ServiceP_Key = arp->ar_pkey; 3222 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */ 3223 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */ 3224 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME); 3225 srv_recp->ServiceGID = arp->ar_gid; 3226 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES); 3227 srv_recp->ServiceID = IBCM_DAPL_ATS_SID; 3228 3229 /* insert service record into the SA */ 3230 3231 IBCM_DUMP_SERVICE_REC(srv_recp); 3232 3233 if (saa_handle != NULL) 3234 status = ibcm_write_service_record(saa_handle, srv_recp, 3235 IBMF_SAA_UPDATE); 3236 else 3237 status = IBT_HCA_PORT_NOT_ACTIVE; 3238 3239 if (status != IBT_SUCCESS) { 3240 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, " 3241 "sid %llX", status, (longlong_t)srv_recp->ServiceID); 3242 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid " 3243 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix, 3244 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3245 3246 kmem_free(srv_recp, sizeof (*srv_recp)); 3247 kmem_free(hdlp, sizeof (*hdlp)); 3248 3249 mutex_enter(&ibcm_svc_info_lock); 3250 linkp = &ibcm_ar_list; 3251 tmp = *linkp; 3252 while (tmp != NULL) { 3253 if (tmp == new) { 3254 *linkp = new->ar_link; 3255 break; 3256 } 3257 linkp = &tmp->ar_link; 3258 tmp = *linkp; 3259 } 3260 if (new->ar_waiters > 0) { 3261 new->ar_flags = IBCM_AR_FAILED; 3262 cv_broadcast(&new->ar_cv); 3263 mutex_exit(&ibcm_svc_info_lock); 3264 } else { 3265 cv_destroy(&new->ar_cv); 3266 mutex_exit(&ibcm_svc_info_lock); 3267 kmem_free(new, sizeof (*new)); 3268 } 3269 ibcm_dec_hca_acc_cnt(hcap); 3270 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3271 "IBMF_SAA failed to write address record"); 3272 } else { /* SUCCESS */ 3273 uint8_t *b; 3274 3275 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: SUCCESS for gid " 3276 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix, 3277 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3278 b = arp->ar_data; 3279 3280 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar:" 3281 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3282 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3283 b[10], b[11], b[12], b[13], b[14], b[15]); 3284 mutex_enter(&ibcm_svc_info_lock); 3285 new->ar_srv_recp = srv_recp; 3286 new->ar_saa_handle = saa_handle; 3287 new->ar_port = cm_port.hp_port; 3288 new->ar_hcap = hcap; 3289 new->ar_flags = IBCM_AR_SUCCESS; 3290 if (new->ar_waiters > 0) 3291 cv_broadcast(&new->ar_cv); 3292 mutex_exit(&ibcm_svc_info_lock); 3293 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3294 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */ 3295 } 3296 return (status); 3297 } 3298 3299 ibt_status_t 3300 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3301 { 3302 ibcm_ar_t *found; 3303 ibcm_ar_t *tmp; 3304 ibcm_ar_t **linkp; 3305 ibcm_ar_ref_t *hdlp; 3306 ibcm_ar_ref_t **hdlpp; 3307 ibt_status_t status; 3308 ibmf_saa_handle_t saa_handle; 3309 sa_service_record_t *srv_recp; 3310 uint64_t gid_ored; 3311 3312 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey); 3313 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: gid %llx:%llx", 3314 (longlong_t)arp->ar_gid.gid_prefix, 3315 (longlong_t)arp->ar_gid.gid_guid); 3316 3317 /* 3318 * If P_Key is 0, but GID is not, this query is invalid. 3319 * If GID is 0, but P_Key is not, this query is invalid. 3320 */ 3321 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3322 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3323 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3324 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3325 "GID/P_Key is not valid"); 3326 return (IBT_INVALID_PARAM); 3327 } 3328 3329 mutex_enter(&ibcm_svc_info_lock); 3330 /* search for existing GID/pkey (there can be at most 1) */ 3331 status = ibcm_search_ar(arp, &found); 3332 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) { 3333 mutex_exit(&ibcm_svc_info_lock); 3334 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3335 "address record not found"); 3336 return (IBT_AR_NOT_REGISTERED); 3337 } 3338 ASSERT(status == IBT_SUCCESS); 3339 3340 hdlpp = &found->ar_ibt_hdl_list; 3341 hdlp = *hdlpp; 3342 while (hdlp != NULL) { 3343 if (hdlp->ar_ibt_hdl == ibt_hdl) 3344 break; 3345 hdlpp = &hdlp->ar_ref_link; 3346 hdlp = *hdlpp; 3347 } 3348 if (hdlp == NULL) { /* could not find ibt_hdl on list */ 3349 mutex_exit(&ibcm_svc_info_lock); 3350 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3351 "address record found, but not for this client"); 3352 return (IBT_AR_NOT_REGISTERED); 3353 } 3354 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */ 3355 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) { 3356 /* last entry was removed */ 3357 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */ 3358 saa_handle = found->ar_saa_handle; 3359 srv_recp = found->ar_srv_recp; 3360 3361 /* wait if this service record is being rewritten */ 3362 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY) 3363 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3364 mutex_exit(&ibcm_svc_info_lock); 3365 3366 /* remove service record */ 3367 status = ibcm_write_service_record(saa_handle, srv_recp, 3368 IBMF_SAA_DELETE); 3369 if (status != IBT_SUCCESS) 3370 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3371 "IBMF_SAA failed to delete address record"); 3372 mutex_enter(&ibcm_svc_info_lock); 3373 if (found->ar_waiters == 0) { /* still no waiters */ 3374 linkp = &ibcm_ar_list; 3375 tmp = *linkp; 3376 while (tmp != found) { 3377 linkp = &tmp->ar_link; 3378 tmp = *linkp; 3379 } 3380 *linkp = tmp->ar_link; 3381 ibcm_dec_hca_acc_cnt(found->ar_hcap); 3382 kmem_free(srv_recp, sizeof (*srv_recp)); 3383 cv_destroy(&found->ar_cv); 3384 kmem_free(found, sizeof (*found)); 3385 } else { 3386 /* add service record back in for the waiters */ 3387 mutex_exit(&ibcm_svc_info_lock); 3388 status = ibcm_write_service_record(saa_handle, srv_recp, 3389 IBMF_SAA_UPDATE); 3390 mutex_enter(&ibcm_svc_info_lock); 3391 if (status == IBT_SUCCESS) 3392 found->ar_flags = IBCM_AR_SUCCESS; 3393 else { 3394 found->ar_flags = IBCM_AR_FAILED; 3395 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3396 "IBMF_SAA failed to write address record"); 3397 } 3398 cv_broadcast(&found->ar_cv); 3399 } 3400 } 3401 mutex_exit(&ibcm_svc_info_lock); 3402 kmem_free(hdlp, sizeof (*hdlp)); 3403 ibtl_cm_change_service_cnt(ibt_hdl, -1); 3404 return (status); 3405 } 3406 3407 ibt_status_t 3408 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp) 3409 { 3410 sa_service_record_t svcrec_req; 3411 sa_service_record_t *svcrec_resp; 3412 void *results_p; 3413 uint64_t component_mask = 0; 3414 uint64_t gid_ored; 3415 size_t length; 3416 int num_rec; 3417 int i; 3418 ibmf_saa_access_args_t access_args; 3419 ibt_status_t retval; 3420 ibtl_cm_hca_port_t cm_port; 3421 ibcm_hca_info_t *hcap; 3422 ibmf_saa_handle_t saa_handle; 3423 3424 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp); 3425 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx", 3426 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid); 3427 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey); 3428 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx", 3429 (longlong_t)queryp->ar_gid.gid_prefix, 3430 (longlong_t)queryp->ar_gid.gid_guid); 3431 3432 /* 3433 * If P_Key is 0, but GID is not, this query is invalid. 3434 * If GID is 0, but P_Key is not, this query is invalid. 3435 */ 3436 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix; 3437 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) || 3438 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) { 3439 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid"); 3440 return (IBT_INVALID_PARAM); 3441 } 3442 3443 hcap = NULL; 3444 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS || 3445 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL || 3446 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) { 3447 if (hcap != NULL) 3448 ibcm_dec_hca_acc_cnt(hcap); 3449 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid"); 3450 return (IBT_INVALID_PARAM); 3451 } 3452 3453 bzero(&svcrec_req, sizeof (svcrec_req)); 3454 3455 /* Is GID/P_Key Specified. */ 3456 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */ 3457 svcrec_req.ServiceP_Key = queryp->ar_pkey; 3458 component_mask |= SA_SR_COMPMASK_PKEY; 3459 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X", 3460 queryp->ar_pkey); 3461 svcrec_req.ServiceGID = queryp->ar_gid; 3462 component_mask |= SA_SR_COMPMASK_GID; 3463 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX", 3464 (longlong_t)queryp->ar_gid.gid_prefix, 3465 (longlong_t)queryp->ar_gid.gid_guid); 3466 } 3467 3468 /* Is ServiceData Specified. */ 3469 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) { 3470 if (queryp->ar_data[i] != 0) { 3471 bcopy(queryp->ar_data, svcrec_req.ServiceData, 3472 IBCM_DAPL_ATS_NBYTES); 3473 component_mask |= 0xFFFF << 7; /* all 16 Data8 */ 3474 /* components */ 3475 break; 3476 } 3477 } 3478 3479 /* Service Name */ 3480 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME); 3481 component_mask |= SA_SR_COMPMASK_NAME; 3482 3483 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID; 3484 component_mask |= SA_SR_COMPMASK_ID; 3485 3486 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3487 "Perform SA Access: Mask: 0x%X", component_mask); 3488 3489 /* 3490 * Call in SA Access retrieve routine to get Service Records. 3491 * 3492 * SA Access framework allocated memory for the "results_p". 3493 * Make sure to deallocate once we are done with the results_p. 3494 * The size of the buffer allocated will be as returned in 3495 * "length" field. 3496 */ 3497 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 3498 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 3499 access_args.sq_component_mask = component_mask; 3500 access_args.sq_template = &svcrec_req; 3501 access_args.sq_template_length = sizeof (sa_service_record_t); 3502 access_args.sq_callback = NULL; 3503 access_args.sq_callback_arg = NULL; 3504 3505 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 3506 &results_p); 3507 3508 ibcm_dec_hca_acc_cnt(hcap); 3509 if (retval != IBT_SUCCESS) { 3510 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed"); 3511 return (retval); 3512 } 3513 3514 num_rec = length / sizeof (sa_service_record_t); 3515 3516 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3517 "Found %d Service Records.", num_rec); 3518 3519 /* Validate the returned number of records. */ 3520 if ((results_p != NULL) && (num_rec > 0)) { 3521 uint8_t *b; 3522 3523 /* Just return info from the first service record. */ 3524 svcrec_resp = (sa_service_record_t *)results_p; 3525 3526 /* The Service GID and Service ID */ 3527 resultp->ar_gid = svcrec_resp->ServiceGID; 3528 resultp->ar_pkey = svcrec_resp->ServiceP_Key; 3529 bcopy(svcrec_resp->ServiceData, 3530 resultp->ar_data, IBCM_DAPL_ATS_NBYTES); 3531 3532 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3533 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey, 3534 (longlong_t)resultp->ar_gid.gid_prefix, 3535 (longlong_t)resultp->ar_gid.gid_guid); 3536 b = resultp->ar_data; 3537 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar:" 3538 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3539 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3540 b[10], b[11], b[12], b[13], b[14], b[15]); 3541 3542 /* Deallocate the memory for results_p. */ 3543 kmem_free(results_p, length); 3544 if (num_rec > 1) 3545 retval = IBT_MULTIPLE_AR; 3546 else 3547 retval = IBT_SUCCESS; 3548 } else { 3549 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3550 "ibmf_sa_access found 0 matching records"); 3551 retval = IBT_AR_NOT_REGISTERED; 3552 } 3553 return (retval); 3554 } 3555 3556 /* mark all ATS service records associated with the port */ 3557 static void 3558 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port) 3559 { 3560 ibcm_ar_t *tmp; 3561 3562 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3563 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3564 if (tmp->ar_hcap->hca_guid == hca_guid && 3565 tmp->ar_port == port) { 3566 /* even if it's busy, we mark it for rewrite */ 3567 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED; 3568 } 3569 } 3570 } 3571 3572 /* rewrite all ATS service records */ 3573 static int 3574 ibcm_rewrite_ar(void) 3575 { 3576 ibcm_ar_t *tmp; 3577 ibmf_saa_handle_t saa_handle; 3578 sa_service_record_t *srv_recp; 3579 ibt_status_t rval; 3580 int did_something = 0; 3581 3582 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3583 check_for_work: 3584 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3585 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) { 3586 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY; 3587 saa_handle = tmp->ar_saa_handle; 3588 srv_recp = tmp->ar_srv_recp; 3589 mutex_exit(&ibcm_svc_info_lock); 3590 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: " 3591 "rewriting ar @ %p", tmp); 3592 did_something = 1; 3593 rval = ibcm_write_service_record(saa_handle, srv_recp, 3594 IBMF_SAA_UPDATE); 3595 if (rval != IBT_SUCCESS) 3596 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: " 3597 "ibcm_write_service_record failed: " 3598 "status = %d", rval); 3599 mutex_enter(&ibcm_svc_info_lock); 3600 /* if it got marked again, then we want to rewrite */ 3601 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY) 3602 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE; 3603 /* in case there was a waiter... */ 3604 cv_broadcast(&ibcm_svc_info_cv); 3605 goto check_for_work; 3606 } 3607 } 3608 return (did_something); 3609 } 3610 3611 static void 3612 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp) 3613 { 3614 ibcm_hca_info_t *hcap; 3615 ib_svc_id_t sid, start_sid, end_sid; 3616 ibmf_saa_handle_t saa_handle; 3617 sa_service_record_t srv_rec; 3618 ibt_status_t rval; 3619 3620 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 3621 if (hcap == NULL) { 3622 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3623 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid); 3624 return; 3625 } 3626 3627 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port); 3628 if (saa_handle == NULL) { 3629 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3630 "saa_handle is NULL"); 3631 ibcm_dec_hca_acc_cnt(hcap); 3632 return; 3633 } 3634 3635 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: " 3636 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name, 3637 sbindp->sbind_gid.gid_guid); 3638 3639 bzero(&srv_rec, sizeof (srv_rec)); 3640 3641 srv_rec.ServiceLease = sbindp->sbind_lease; 3642 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 3643 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 3644 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 3645 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 3646 srv_rec.ServiceGID = sbindp->sbind_gid; 3647 3648 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 3649 3650 /* insert srv record into the SA */ 3651 start_sid = srv_hdl->svc_id; 3652 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 3653 for (sid = start_sid; sid <= end_sid; sid++) { 3654 srv_rec.ServiceID = sid; 3655 3656 rval = ibcm_write_service_record(saa_handle, &srv_rec, 3657 IBMF_SAA_UPDATE); 3658 3659 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: " 3660 "ibcm_write_service_record, SvcId = %llX, " 3661 "rval = %d", (longlong_t)sid, rval); 3662 if (rval != IBT_SUCCESS) { 3663 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:" 3664 " ibcm_write_service_record fails %d sid %llX", 3665 rval, (longlong_t)sid); 3666 } 3667 } 3668 ibcm_dec_hca_acc_cnt(hcap); 3669 } 3670 3671 /* 3672 * Task to mark all service records as needing to be rewritten to the SM/SA. 3673 * This task does not return until all of them have been rewritten. 3674 */ 3675 void 3676 ibcm_service_record_rewrite_task(void *arg) 3677 { 3678 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg; 3679 ib_guid_t hca_guid = pup->pup_hca_guid; 3680 uint8_t port = pup->pup_port; 3681 ibcm_svc_info_t *svcp; 3682 ibcm_svc_bind_t *sbp; 3683 avl_tree_t *avl_tree = &ibcm_svc_avl_tree; 3684 static int task_is_running = 0; 3685 3686 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task STARTED " 3687 "for hca_guid %llX, port %d", hca_guid, port); 3688 3689 mutex_enter(&ibcm_svc_info_lock); 3690 ibcm_mark_ar(hca_guid, port); 3691 for (svcp = avl_first(avl_tree); svcp != NULL; 3692 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3693 sbp = svcp->svc_bind_list; 3694 while (sbp != NULL) { 3695 if (sbp->sbind_pkey != 0 && 3696 sbp->sbind_port == port && 3697 sbp->sbind_hcaguid == hca_guid) { 3698 /* even if it's busy, we mark it for rewrite */ 3699 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED; 3700 } 3701 sbp = sbp->sbind_link; 3702 } 3703 } 3704 if (task_is_running) { 3705 /* let the other task thread finish the work */ 3706 mutex_exit(&ibcm_svc_info_lock); 3707 return; 3708 } 3709 task_is_running = 1; 3710 3711 (void) ibcm_rewrite_ar(); 3712 3713 check_for_work: 3714 for (svcp = avl_first(avl_tree); svcp != NULL; 3715 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3716 sbp = svcp->svc_bind_list; 3717 while (sbp != NULL) { 3718 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) { 3719 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY; 3720 mutex_exit(&ibcm_svc_info_lock); 3721 ibcm_rewrite_svc_record(svcp, sbp); 3722 mutex_enter(&ibcm_svc_info_lock); 3723 /* if it got marked again, we want to rewrite */ 3724 if (sbp->sbind_rewrite_state == 3725 IBCM_REWRITE_BUSY) 3726 sbp->sbind_rewrite_state = 3727 IBCM_REWRITE_IDLE; 3728 /* in case there was a waiter... */ 3729 cv_broadcast(&ibcm_svc_info_cv); 3730 goto check_for_work; 3731 } 3732 sbp = sbp->sbind_link; 3733 } 3734 } 3735 /* 3736 * If there were no service records to write, and we failed to 3737 * have to rewrite any more ATS service records, then we're done. 3738 */ 3739 if (ibcm_rewrite_ar() != 0) 3740 goto check_for_work; 3741 task_is_running = 0; 3742 mutex_exit(&ibcm_svc_info_lock); 3743 3744 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task DONE"); 3745 kmem_free(pup, sizeof (ibcm_port_up_t)); 3746 } 3747 3748 3749 /* 3750 * Function: 3751 * ibt_cm_proceed 3752 * 3753 * Verifies the arguments and dispatches the cm state machine processing 3754 * via taskq 3755 */ 3756 3757 ibt_status_t 3758 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id, 3759 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3760 void *priv_data, ibt_priv_data_len_t priv_data_len) 3761 { 3762 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3763 ibcm_proceed_targs_t *proceed_targs; 3764 ibcm_proceed_error_t proceed_error; 3765 3766 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x " 3767 "session_id %p", statep->channel, event, status, session_id); 3768 3769 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, " 3770 "priv_data %p priv_data_len %x", statep->channel, cm_event_data, 3771 priv_data, priv_data_len); 3772 3773 /* validate session_id and status */ 3774 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3775 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args"); 3776 return (IBT_INVALID_PARAM); 3777 } 3778 3779 /* If priv data len specified, then priv_data cannot be NULL */ 3780 if ((priv_data_len > 0) && (priv_data == NULL)) 3781 return (IBT_INVALID_PARAM); 3782 3783 proceed_error = IBCM_PROCEED_INVALID_NONE; 3784 3785 mutex_enter(&statep->state_mutex); 3786 if (event == IBT_CM_EVENT_REQ_RCV) { 3787 3788 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3789 (statep->state != IBCM_STATE_MRA_SENT)) 3790 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3791 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ) 3792 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3793 3794 } else if (event == IBT_CM_EVENT_REP_RCV) { 3795 if ((statep->state != IBCM_STATE_REP_RCVD) && 3796 (statep->state != IBCM_STATE_MRA_REP_SENT)) 3797 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3798 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ) 3799 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3800 } else if (event == IBT_CM_EVENT_LAP_RCV) { 3801 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) && 3802 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT)) 3803 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3804 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ) 3805 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3806 } else if (event == IBT_CM_EVENT_CONN_CLOSED) { 3807 if (statep->state != IBCM_STATE_DREQ_RCVD) 3808 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3809 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ) 3810 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3811 } else { 3812 proceed_error = IBCM_PROCEED_INVALID_EVENT; 3813 } 3814 3815 /* if there is an error, print an error message and return */ 3816 if (proceed_error != IBCM_PROCEED_INVALID_NONE) { 3817 mutex_exit(&statep->state_mutex); 3818 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) { 3819 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3820 "Invalid Event/State combination specified", 3821 statep->channel); 3822 return (IBT_INVALID_PARAM); 3823 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) { 3824 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3825 "Invalid Event/priv len combination specified", 3826 statep->channel); 3827 return (IBT_INVALID_PARAM); 3828 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) { 3829 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3830 "Invalid Event specified", statep->channel); 3831 return (IBT_INVALID_PARAM); 3832 } else { 3833 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP); 3834 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3835 "IBT_CM_EVENT_LAP_RCV not supported", 3836 statep->channel); 3837 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */ 3838 return (IBT_APM_NOT_SUPPORTED); 3839 } 3840 } 3841 3842 3843 /* wait until client's CM handler returns DEFER status back to CM */ 3844 3845 while (statep->clnt_proceed == IBCM_BLOCK) { 3846 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for " 3847 "return of client's cm handler", statep->channel); 3848 cv_wait(&statep->block_client_cv, &statep->state_mutex); 3849 } 3850 3851 if (statep->clnt_proceed == IBCM_FAIL) { 3852 mutex_exit(&statep->state_mutex); 3853 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as " 3854 "client returned non-DEFER status from cm handler", 3855 statep->channel); 3856 return (IBT_CHAN_STATE_INVALID); 3857 } 3858 3859 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK); 3860 statep->clnt_proceed = IBCM_FAIL; 3861 mutex_exit(&statep->state_mutex); 3862 3863 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3864 3865 /* the state machine processing is done in a separate thread */ 3866 3867 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 3868 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 3869 KM_SLEEP); 3870 3871 proceed_targs->event = event; 3872 proceed_targs->status = status; 3873 proceed_targs->priv_data_len = priv_data_len; 3874 3875 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 3876 3877 proceed_targs->tst.rc.statep = statep; 3878 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data, 3879 sizeof (ibt_cm_proceed_reply_t)); 3880 3881 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3882 3883 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq, 3884 proceed_targs, TQ_SLEEP); 3885 3886 return (IBT_SUCCESS); 3887 } 3888 3889 /* 3890 * Function: 3891 * ibcm_proceed_via_taskq 3892 * 3893 * Called from taskq, dispatched by ibt_cm_proceed 3894 * Completes the cm state processing for ibt_cm_proceed 3895 */ 3896 void 3897 ibcm_proceed_via_taskq(void *targs) 3898 { 3899 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 3900 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep; 3901 ibt_cm_reason_t reject_reason; 3902 uint8_t arej_len; 3903 ibcm_status_t response; 3904 ibcm_clnt_reply_info_t clnt_info; 3905 3906 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data; 3907 clnt_info.priv_data = proceed_targs->priv_data; 3908 clnt_info.priv_data_len = proceed_targs->priv_data_len; 3909 3910 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x", 3911 statep->channel, targs); 3912 3913 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) { 3914 response = 3915 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status, 3916 &clnt_info, &reject_reason, &arej_len, 3917 (ibcm_req_msg_t *)statep->defer_cm_msg); 3918 3919 ibcm_handle_cep_req_response(statep, response, reject_reason, 3920 arej_len); 3921 3922 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) { 3923 response = 3924 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status, 3925 &clnt_info, &reject_reason, &arej_len, 3926 (ibcm_rep_msg_t *)statep->defer_cm_msg); 3927 3928 ibcm_handle_cep_rep_response(statep, response, reject_reason, 3929 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg); 3930 3931 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) { 3932 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status, 3933 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg, 3934 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg)); 3935 3936 ibcm_post_apr_mad(statep); 3937 3938 } else { 3939 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED); 3940 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data, 3941 proceed_targs->priv_data_len); 3942 } 3943 3944 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 3945 } 3946 3947 /* 3948 * Function: 3949 * ibt_cm_ud_proceed 3950 * 3951 * Verifies the arguments and dispatches the cm state machine processing 3952 * via taskq 3953 */ 3954 ibt_status_t 3955 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel, 3956 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop, 3957 void *priv_data, ibt_priv_data_len_t priv_data_len) 3958 { 3959 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id; 3960 ibcm_proceed_targs_t *proceed_targs; 3961 ibt_qp_query_attr_t qp_attr; 3962 ibt_status_t retval; 3963 3964 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p " 3965 "ud_channel %p ", session_id, ud_channel); 3966 3967 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p " 3968 "priv_data_len %x", status, priv_data, priv_data_len); 3969 3970 /* validate session_id and status */ 3971 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) { 3972 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args"); 3973 return (IBT_INVALID_PARAM); 3974 } 3975 3976 /* If priv data len specified, then priv_data cannot be NULL */ 3977 if ((priv_data_len > 0) && (priv_data == NULL)) 3978 return (IBT_INVALID_PARAM); 3979 3980 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ) 3981 return (IBT_INVALID_PARAM); 3982 3983 /* retrieve qpn and qkey from ud channel */ 3984 3985 /* validate event and statep's state */ 3986 3987 if (status == IBT_CM_ACCEPT) { 3988 retval = ibt_query_qp(ud_channel, &qp_attr); 3989 if ((retval != IBT_SUCCESS) || 3990 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) { 3991 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: " 3992 "Failed to retrieve QPN from the channel: %d", 3993 retval); 3994 return (IBT_INVALID_PARAM); 3995 } 3996 } 3997 3998 3999 mutex_enter(&ud_statep->ud_state_mutex); 4000 4001 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) { 4002 mutex_exit(&ud_statep->ud_state_mutex); 4003 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State " 4004 "specified"); 4005 return (IBT_INVALID_PARAM); 4006 } 4007 4008 /* wait until client's CM handler returns DEFER status back to CM */ 4009 4010 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) { 4011 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return" 4012 " of client's ud cm handler"); 4013 cv_wait(&ud_statep->ud_block_client_cv, 4014 &ud_statep->ud_state_mutex); 4015 } 4016 4017 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) { 4018 mutex_exit(&ud_statep->ud_state_mutex); 4019 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client " 4020 "returned non-DEFER status from cm handler"); 4021 return (IBT_INVALID_PARAM); 4022 } 4023 4024 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK); 4025 ud_statep->ud_clnt_proceed = IBCM_FAIL; 4026 mutex_exit(&ud_statep->ud_state_mutex); 4027 4028 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4029 4030 /* the state machine processing is done in a separate thread */ 4031 4032 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4033 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 4034 KM_SLEEP); 4035 4036 proceed_targs->status = status; 4037 proceed_targs->priv_data_len = priv_data_len; 4038 4039 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4040 4041 if (status == IBT_CM_ACCEPT) { 4042 proceed_targs->tst.ud.ud_qkey = 4043 qp_attr.qp_info.qp_transport.ud.ud_qkey; 4044 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn; 4045 } 4046 4047 proceed_targs->tst.ud.ud_statep = ud_statep; 4048 4049 /* copy redirect info based on status */ 4050 if (status == IBT_CM_REDIRECT) 4051 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info, 4052 sizeof (ibt_redirect_info_t)); 4053 4054 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4055 4056 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq, 4057 proceed_targs, TQ_SLEEP); 4058 4059 return (IBT_SUCCESS); 4060 } 4061 4062 /* 4063 * Function: 4064 * ibcm_ud_proceed_via_taskq 4065 * 4066 * Called from taskq, dispatched by ibt_cm_ud_proceed 4067 * Completes the cm state processing for ibt_cm_ud_proceed 4068 */ 4069 void 4070 ibcm_ud_proceed_via_taskq(void *targs) 4071 { 4072 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4073 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep; 4074 ibcm_ud_clnt_reply_info_t ud_clnt_info; 4075 ibt_sidr_status_t sidr_status; 4076 4077 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs); 4078 4079 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn; 4080 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey; 4081 ud_clnt_info.priv_data = proceed_targs->priv_data; 4082 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len; 4083 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info; 4084 4085 /* validate event and statep's state */ 4086 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status, 4087 &ud_clnt_info, &sidr_status, 4088 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg)); 4089 4090 ibcm_post_sidr_rep_mad(ud_statep, sidr_status); 4091 4092 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */ 4093 mutex_enter(&ud_statep->ud_state_mutex); 4094 IBCM_UD_REF_CNT_DECR(ud_statep); 4095 mutex_exit(&ud_statep->ud_state_mutex); 4096 4097 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4098 } 4099 4100 /* 4101 * Function: 4102 * ibt_set_alt_path 4103 * Input: 4104 * channel Channel handle returned from ibt_alloc_rc_channel(9F). 4105 * 4106 * mode Execute in blocking or non blocking mode. 4107 * 4108 * alt_path A pointer to an ibt_alt_path_info_t as returned from an 4109 * ibt_get_alt_path(9F) call that specifies the new 4110 * alternate path. 4111 * 4112 * priv_data A pointer to a buffer specified by caller for the 4113 * private data in the outgoing CM Load Alternate Path 4114 * (LAP) message sent to the remote host. This can be NULL 4115 * if no private data is available to communicate to the 4116 * remote node. 4117 * 4118 * priv_data_len Length of valid data in priv_data, this should be less 4119 * than or equal to IBT_LAP_PRIV_DATA_SZ. 4120 * 4121 * Output: 4122 * ret_args If called in blocking mode, points to a return argument 4123 * structure of type ibt_ap_returns_t. 4124 * 4125 * Returns: 4126 * IBT_SUCCESS on Success else appropriate error. 4127 * Description: 4128 * Load the specified alternate path. Causes the CM to send an LAP message 4129 * to the remote node. 4130 * Can only be called on a previously opened RC channel. 4131 */ 4132 ibt_status_t 4133 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 4134 ibt_alt_path_info_t *alt_path, void *priv_data, 4135 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args) 4136 { 4137 ibmf_handle_t ibmf_hdl; 4138 ibt_status_t status = IBT_SUCCESS; 4139 ibcm_lap_msg_t *lap_msgp; 4140 ibcm_hca_info_t *hcap; 4141 ibcm_state_data_t *statep; 4142 uint8_t port_no; 4143 ib_lid_t alternate_slid; 4144 ibt_priv_data_len_t len; 4145 ib_lid_t base_lid; 4146 boolean_t alt_grh; 4147 4148 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)", 4149 channel, mode, alt_path, priv_data, priv_data_len, ret_args); 4150 4151 /* validate channel */ 4152 if (IBCM_INVALID_CHANNEL(channel)) { 4153 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel"); 4154 return (IBT_CHAN_HDL_INVALID); 4155 } 4156 4157 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 4158 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4159 "Invalid Channel type: Applicable only to RC Channel"); 4160 return (IBT_CHAN_SRV_TYPE_INVALID); 4161 } 4162 4163 if (mode == IBT_NONBLOCKING) { 4164 if (ret_args != NULL) { 4165 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4166 "ret_args should be NULL when called in " 4167 "non-blocking mode"); 4168 return (IBT_INVALID_PARAM); 4169 } 4170 } else if (mode == IBT_BLOCKING) { 4171 if (ret_args == NULL) { 4172 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4173 "ret_args should be Non-NULL when called in " 4174 "blocking mode"); 4175 return (IBT_INVALID_PARAM); 4176 } 4177 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) { 4178 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4179 "expected private data length is too large"); 4180 return (IBT_INVALID_PARAM); 4181 } 4182 if ((ret_args->ap_priv_data_len > 0) && 4183 (ret_args->ap_priv_data == NULL)) { 4184 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4185 "apr_priv_data_len > 0, but apr_priv_data NULL"); 4186 return (IBT_INVALID_PARAM); 4187 } 4188 } else { /* any other mode is not valid for ibt_set_alt_path */ 4189 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4190 "invalid mode %x specified", mode); 4191 return (IBT_INVALID_PARAM); 4192 } 4193 4194 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0) 4195 return (IBT_INVALID_PARAM); 4196 4197 /* get the statep */ 4198 IBCM_GET_CHAN_PRIVATE(channel, statep); 4199 if (statep == NULL) { 4200 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL"); 4201 return (IBT_CM_FAILURE); 4202 } 4203 4204 mutex_enter(&statep->state_mutex); 4205 IBCM_RELEASE_CHAN_PRIVATE(channel); 4206 IBCM_REF_CNT_INCR(statep); 4207 mutex_exit(&statep->state_mutex); 4208 4209 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep); 4210 4211 hcap = statep->hcap; 4212 4213 /* HCA must have been in active state. If not, it's a client bug */ 4214 if (!IBCM_ACCESS_HCA_OK(hcap)) 4215 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state"); 4216 4217 ASSERT(statep->cm_handler != NULL); 4218 4219 /* Check Alternate port */ 4220 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL, 4221 &base_lid); 4222 if (status != IBT_SUCCESS) { 4223 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4224 "ibt_get_port_state_byguid status %d ", status); 4225 mutex_enter(&statep->state_mutex); 4226 IBCM_REF_CNT_DECR(statep); 4227 mutex_exit(&statep->state_mutex); 4228 return (status); 4229 } 4230 4231 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 4232 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 4233 != IBT_SUCCESS)) { 4234 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4235 "ibmf reg or callback setup failed during re-initialize"); 4236 mutex_enter(&statep->state_mutex); 4237 IBCM_REF_CNT_DECR(statep); 4238 mutex_exit(&statep->state_mutex); 4239 return (status); 4240 } 4241 4242 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl; 4243 4244 alternate_slid = base_lid + 4245 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path; 4246 4247 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x", 4248 h2b16(alternate_slid)); 4249 4250 ibcm_lapr_enter(); /* limit how many run simultaneously */ 4251 4252 /* Allocate MAD for LAP */ 4253 if (statep->lapr_msg == NULL) 4254 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg, 4255 MAD_METHOD_SEND)) != IBT_SUCCESS) { 4256 ibcm_lapr_exit(); 4257 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4258 "chan 0x%p ibcm_alloc_out_msg failed", channel); 4259 mutex_enter(&statep->state_mutex); 4260 IBCM_REF_CNT_DECR(statep); 4261 mutex_exit(&statep->state_mutex); 4262 return (status); 4263 } 4264 4265 mutex_enter(&statep->state_mutex); 4266 4267 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is" 4268 " %x", statep->state); 4269 4270 /* Check state */ 4271 if ((statep->state != IBCM_STATE_ESTABLISHED) || 4272 (statep->ap_state != IBCM_AP_STATE_IDLE)) { 4273 IBCM_REF_CNT_DECR(statep); 4274 mutex_exit(&statep->state_mutex); 4275 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg); 4276 ibcm_lapr_exit(); 4277 return (IBT_CHAN_STATE_INVALID); 4278 } else { 4279 /* Set to LAP Sent state */ 4280 statep->ap_state = IBCM_AP_STATE_LAP_SENT; 4281 statep->ap_done = B_FALSE; 4282 statep->remaining_retry_cnt = statep->max_cm_retries; 4283 statep->timer_stored_state = statep->state; 4284 statep->timer_stored_ap_state = statep->ap_state; 4285 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */ 4286 } 4287 4288 mutex_exit(&statep->state_mutex); 4289 4290 /* No more failure returns below */ 4291 4292 /* Allocate MAD for LAP */ 4293 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:" 4294 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg)); 4295 4296 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4297 4298 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg); 4299 4300 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid); 4301 lap_msgp->lap_alt_r_port_lid = 4302 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid); 4303 4304 /* Fill in remote port gid */ 4305 lap_msgp->lap_alt_r_port_gid.gid_prefix = 4306 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix); 4307 lap_msgp->lap_alt_r_port_gid.gid_guid = 4308 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid); 4309 4310 /* Fill in local port gid */ 4311 lap_msgp->lap_alt_l_port_gid.gid_prefix = 4312 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix); 4313 lap_msgp->lap_alt_l_port_gid.gid_guid = 4314 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid); 4315 4316 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh; 4317 4318 /* alternate_flow_label, and alternate srate, alternate traffic class */ 4319 lap_msgp->lap_alt_srate_plus = 4320 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f; 4321 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ? 4322 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) | 4323 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass); 4324 4325 /* Alternate hop limit, service level */ 4326 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ? 4327 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 0xff; 4328 lap_msgp->lap_alt_sl_plus = 4329 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 | 4330 ((alt_grh == B_FALSE) ? 0x8 : 0); 4331 4332 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib( 4333 (2 * statep->rc_alt_pkt_lt) + 4334 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 4335 4336 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid); 4337 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid); 4338 4339 lap_msgp->lap_remote_qpn_eecn_plus = 4340 h2b32((statep->remote_qpn << 8) | 4341 ibt_usec2ib(ibcm_remote_response_time) << 3); 4342 4343 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ); 4344 if ((len > 0) && priv_data) { 4345 bcopy(priv_data, lap_msgp->lap_private_data, len); 4346 } 4347 4348 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */ 4349 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 4350 4351 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt); 4352 4353 /* return_data is filled up in the state machine code */ 4354 statep->ap_return_data = ret_args; 4355 4356 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 4357 4358 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID = 4359 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID); 4360 4361 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID = 4362 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid, 4363 0)); 4364 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx", 4365 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID); 4366 4367 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4368 4369 /* Send LAP */ 4370 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete, 4371 statep); 4372 4373 mutex_enter(&statep->state_mutex); 4374 4375 if (mode == IBT_BLOCKING) { 4376 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking"); 4377 4378 /* wait for APR */ 4379 while (statep->ap_done != B_TRUE) { 4380 cv_wait(&statep->block_client_cv, 4381 &statep->state_mutex); 4382 } 4383 4384 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking"); 4385 4386 /* 4387 * In the case that ibt_set_alt_path fails, 4388 * change retval to IBT_CM_FAILURE 4389 */ 4390 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED) 4391 status = IBT_CM_FAILURE; 4392 4393 } 4394 4395 /* decrement the ref-count before leaving here */ 4396 IBCM_REF_CNT_DECR(statep); 4397 4398 mutex_exit(&statep->state_mutex); 4399 4400 ibcm_lapr_exit(); 4401 4402 /* If this message isn't seen then ibt_set_alt_path failed */ 4403 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done"); 4404 4405 return (status); 4406 } 4407 4408 4409 #ifdef DEBUG 4410 4411 /* 4412 * ibcm_query_classport_info: 4413 * Query classportinfo 4414 * 4415 * INPUTS: 4416 * channel - Channel that is associated with a statep 4417 * 4418 * RETURN VALUE: NONE 4419 * This function is currently used to generate a valid get method classport 4420 * info, and test CM functionality. There is no ibtl client interface to 4421 * generate a classportinfo. It is possible that CM may use classportinfo 4422 * from other nodes in the future, and most of the code below could be re-used. 4423 */ 4424 void 4425 ibcm_query_classport_info(ibt_channel_hdl_t channel) 4426 { 4427 ibcm_state_data_t *statep; 4428 ibmf_msg_t *msgp; 4429 4430 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p)", channel); 4431 4432 /* validate channel, first */ 4433 if (IBCM_INVALID_CHANNEL(channel)) { 4434 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4435 "invalid channel (%p)", channel); 4436 return; 4437 } 4438 4439 /* get the statep */ 4440 IBCM_GET_CHAN_PRIVATE(channel, statep); 4441 4442 /* 4443 * This can happen, if the statep is already gone by a DREQ from 4444 * the remote side 4445 */ 4446 if (statep == NULL) { 4447 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4448 "statep NULL"); 4449 return; 4450 } 4451 4452 mutex_enter(&statep->state_mutex); 4453 IBCM_RELEASE_CHAN_PRIVATE(channel); 4454 IBCM_REF_CNT_INCR(statep); 4455 mutex_exit(&statep->state_mutex); 4456 4457 /* Debug/test code, so don't care about return status */ 4458 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp, 4459 MAD_METHOD_GET); 4460 4461 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid( 4462 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0)); 4463 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO); 4464 4465 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL); 4466 4467 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p) " 4468 "Get method MAD posted ", channel); 4469 4470 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp); 4471 4472 mutex_enter(&statep->state_mutex); 4473 IBCM_REF_CNT_DECR(statep); 4474 mutex_exit(&statep->state_mutex); 4475 } 4476 4477 static void 4478 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr) 4479 { 4480 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, " 4481 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid, 4482 cm_reply_addr->rcvd_addr.ia_remote_lid); 4483 4484 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, " 4485 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key, 4486 cm_reply_addr->rcvd_addr.ia_p_key, 4487 cm_reply_addr->rcvd_addr.ia_remote_qno, 4488 cm_reply_addr->rcvd_addr.ia_service_level); 4489 4490 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ", 4491 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix, 4492 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid); 4493 4494 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX", 4495 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix, 4496 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid); 4497 4498 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x", 4499 cm_reply_addr->grh_hdr.ig_flow_label, 4500 cm_reply_addr->grh_hdr.ig_tclass, 4501 cm_reply_addr->grh_hdr.ig_hop_limit); 4502 } 4503 4504 #endif 4505 4506 typedef struct ibcm_join_mcg_tqarg_s { 4507 ib_gid_t rgid; 4508 ibt_mcg_attr_t mcg_attr; 4509 ibt_mcg_info_t *mcg_infop; 4510 ibt_mcg_handler_t func; 4511 void *arg; 4512 } ibcm_join_mcg_tqarg_t; 4513 4514 _NOTE(READ_ONLY_DATA(ibcm_join_mcg_tqarg_s)) 4515 4516 /* 4517 * Function: 4518 * ibt_join_mcg 4519 * Input: 4520 * rgid The request GID that defines the HCA port from which a 4521 * contact to SA Access is performed to add the specified 4522 * endport GID ((mcg_attr->mc_pgid) to a multicast group. 4523 * If mcg_attr->mc_pgid is null, then this (rgid) will be 4524 * treated as endport GID that is to be added to the 4525 * multicast group. 4526 * 4527 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines 4528 * the attributes of the desired multicast group to be 4529 * created or joined. 4530 * 4531 * func NULL or a pointer to a function to call when 4532 * ibt_join_mcg() completes. If 'func' is not NULL then 4533 * ibt_join_mcg() will return as soon as possible after 4534 * initiating the multicast group join/create process. 4535 * 'func' is then called when the process completes. 4536 * 4537 * arg Argument to the 'func'. 4538 * 4539 * Output: 4540 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated 4541 * by the caller, where the attributes of the created or 4542 * joined multicast group are copied. 4543 * Returns: 4544 * IBT_SUCCESS 4545 * IBT_INVALID_PARAM 4546 * IBT_MCG_RECORDS_NOT_FOUND 4547 * IBT_INSUFF_RESOURCE 4548 * Description: 4549 * Join a multicast group. The first full member "join" causes the MCG 4550 * to be created. 4551 */ 4552 ibt_status_t 4553 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, 4554 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg) 4555 { 4556 ibcm_join_mcg_tqarg_t *mcg_tq; 4557 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP); 4558 4559 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix, 4560 rgid.gid_guid, mcg_attr); 4561 4562 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4563 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required"); 4564 return (IBT_INVALID_PARAM); 4565 } 4566 4567 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) || 4568 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) { 4569 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified"); 4570 return (IBT_INVALID_PARAM); 4571 } 4572 4573 if (mcg_attr->mc_join_state == 0) { 4574 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified"); 4575 return (IBT_INVALID_PARAM); 4576 } 4577 4578 if (mcg_info_p == NULL) { 4579 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL"); 4580 return (IBT_INVALID_PARAM); 4581 } 4582 4583 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4584 4585 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag); 4586 if (mcg_tq == NULL) { 4587 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: " 4588 "Unable to allocate memory for local usage."); 4589 return (IBT_INSUFF_KERNEL_RESOURCE); 4590 } 4591 4592 mcg_tq->rgid = rgid; 4593 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t)); 4594 mcg_tq->mcg_infop = mcg_info_p; 4595 mcg_tq->func = func; 4596 mcg_tq->arg = arg; 4597 4598 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4599 4600 if (func != NULL) { /* Non-Blocking */ 4601 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call"); 4602 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg, 4603 mcg_tq, TQ_NOSLEEP) == 0) { 4604 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to " 4605 "Dispatch the TaskQ"); 4606 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t)); 4607 return (IBT_INSUFF_KERNEL_RESOURCE); 4608 } else 4609 return (IBT_SUCCESS); 4610 } else { /* Blocking */ 4611 return (ibcm_process_join_mcg(mcg_tq)); 4612 } 4613 } 4614 4615 static void 4616 ibcm_process_async_join_mcg(void *tq_arg) 4617 { 4618 (void) ibcm_process_join_mcg(tq_arg); 4619 } 4620 4621 static ibt_status_t 4622 ibcm_process_join_mcg(void *taskq_arg) 4623 { 4624 sa_mcmember_record_t mcg_req; 4625 sa_mcmember_record_t *mcg_resp; 4626 ibmf_saa_access_args_t access_args; 4627 ibmf_saa_handle_t saa_handle; 4628 uint64_t component_mask = 0; 4629 ibt_status_t retval; 4630 ibtl_cm_hca_port_t hca_port; 4631 uint_t num_records; 4632 size_t length; 4633 ibcm_hca_info_t *hcap; 4634 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg; 4635 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop; 4636 4637 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg); 4638 4639 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port); 4640 if (retval != IBT_SUCCESS) { 4641 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get " 4642 "port info from specified RGID: status = %d", retval); 4643 goto ibcm_join_mcg_exit1; 4644 } 4645 4646 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4647 4648 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) || 4649 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) { 4650 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4651 "Request GID is Port GID"); 4652 mcg_req.PortGID = mcg_arg->rgid; 4653 } else { 4654 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid; 4655 } 4656 component_mask |= SA_MC_COMPMASK_PORTGID; 4657 4658 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey; 4659 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey; 4660 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state; 4661 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass; 4662 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow; 4663 mcg_req.SL = mcg_arg->mcg_attr.mc_sl; 4664 4665 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY | 4666 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS | 4667 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL; 4668 4669 /* If client has specified MGID, use it else SA will assign one. */ 4670 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4671 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid; 4672 component_mask |= SA_MC_COMPMASK_MGID; 4673 } 4674 4675 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: PGID=%llX:%llX, " 4676 "MGID=%llX:%llX", mcg_req.PortGID.gid_prefix, 4677 mcg_req.PortGID.gid_guid, mcg_req.MGID.gid_prefix, 4678 mcg_req.MGID.gid_guid); 4679 4680 /* Is MTU specified. */ 4681 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) { 4682 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu; 4683 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector; 4684 4685 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 4686 SA_MC_COMPMASK_MTU; 4687 } 4688 4689 /* Is RATE specified. */ 4690 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) { 4691 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate; 4692 mcg_req.RateSelector = 4693 mcg_arg->mcg_attr.mc_rate_req.r_selector; 4694 4695 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 4696 SA_MC_COMPMASK_RATE; 4697 } 4698 4699 /* Is Packet Life Time specified. */ 4700 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) { 4701 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt; 4702 mcg_req.RateSelector = 4703 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector; 4704 4705 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 4706 SA_MC_COMPMASK_PKTLT; 4707 } 4708 4709 if (mcg_arg->mcg_attr.mc_hop) { 4710 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop; 4711 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 4712 } 4713 4714 if (mcg_arg->mcg_attr.mc_scope) { 4715 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope; 4716 component_mask |= SA_MC_COMPMASK_SCOPE; 4717 } 4718 4719 if (mcg_arg->mcg_attr.mc_mlid) { 4720 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid; 4721 component_mask |= SA_MC_COMPMASK_MLID; 4722 } 4723 4724 /* Get SA Access Handle. */ 4725 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4726 if (hcap == NULL) { 4727 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found"); 4728 4729 retval = IBT_HCA_BUSY_DETACHING; 4730 goto ibcm_join_mcg_exit1; 4731 } 4732 4733 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4734 if (saa_handle == NULL) { 4735 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL"); 4736 4737 retval = IBT_HCA_PORT_NOT_ACTIVE; 4738 goto ibcm_join_mcg_exit; 4739 } 4740 4741 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) && 4742 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) { 4743 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0, 4744 &hca_port); 4745 if (retval != IBT_SUCCESS) { 4746 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed " 4747 "to get PortInfo of specified PGID: status = %d", 4748 retval); 4749 goto ibcm_join_mcg_exit1; 4750 } 4751 } 4752 4753 /* Contact SA Access */ 4754 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4755 access_args.sq_access_type = IBMF_SAA_UPDATE; 4756 access_args.sq_component_mask = component_mask; 4757 access_args.sq_template = &mcg_req; 4758 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4759 access_args.sq_callback = NULL; 4760 access_args.sq_callback_arg = NULL; 4761 4762 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 4763 (void **)&mcg_resp); 4764 if (retval != IBT_SUCCESS) { 4765 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: " 4766 "SA Access Failed"); 4767 goto ibcm_join_mcg_exit; 4768 } 4769 4770 num_records = length/sizeof (sa_mcmember_record_t); 4771 4772 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: " 4773 "Found %d MCMember Records", num_records); 4774 4775 /* Validate the returned number of records. */ 4776 if ((mcg_resp != NULL) && (num_records > 0)) { 4777 /* Update the return values. */ 4778 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID; 4779 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID; 4780 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate; 4781 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL; 4782 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel; 4783 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass; 4784 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit; 4785 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE; 4786 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID; 4787 mcg_info_p->mc_mtu = mcg_resp->MTU; 4788 mcg_info_p->mc_qkey = mcg_resp->Q_Key; 4789 4790 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid, 4791 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix); 4792 if (retval != IBT_SUCCESS) { 4793 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4794 "Pkey2Index Conversion failed<%d>", retval); 4795 mcg_info_p->mc_pkey_ix = 0; 4796 } 4797 4798 mcg_info_p->mc_scope = mcg_resp->Scope; 4799 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime; 4800 4801 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port; 4802 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix; 4803 mcg_info_p->mc_adds_vect.av_src_path = 0; 4804 4805 /* Deallocate the memory allocated by SA for mcg_resp. */ 4806 kmem_free(mcg_resp, length); 4807 retval = IBT_SUCCESS; 4808 } else { 4809 retval = IBT_MCG_RECORDS_NOT_FOUND; 4810 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4811 "MCG RECORDS NOT FOUND"); 4812 } 4813 4814 ibcm_join_mcg_exit: 4815 ibcm_dec_hca_acc_cnt(hcap); 4816 4817 ibcm_join_mcg_exit1: 4818 if (mcg_arg->func) 4819 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p); 4820 4821 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t)); 4822 4823 return (retval); 4824 } 4825 4826 4827 /* 4828 * Function: 4829 * ibt_leave_mcg 4830 * Input: 4831 * rgid The request GID that defines the HCA port upon which 4832 * to send the request to the Subnet Administrator, to 4833 * remove the specified port (port_gid) from the multicast 4834 * group. If 'port_gid' is the Reserved GID (i.e. 4835 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0), 4836 * then the end-port associated with 'rgid' is removed 4837 * from the multicast group. 4838 * 4839 * mc_gid A multicast group GID as returned from ibt_join_mcg() 4840 * call. This is optional, if not specified (i.e. 4841 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to 4842 * identify this as being a multicast GID), then the 4843 * port is removed from all the multicast groups of 4844 * which it is a member. 4845 * 4846 * port_gid This is optional, if not the Reserved GID (gid_prefix 4847 * and gid_guid not equal to 0), then this specifies the 4848 * endport GID of the multicast group member being deleted 4849 * from the group. If it is the Reserved GID (gid_prefix 4850 * and gid_guid equal to 0) then the member endport GID is 4851 * determined from 'rgid'. 4852 * 4853 * mc_join_state The Join State attribute used when the group was joined 4854 * using ibt_join_mcg(). This Join State component must 4855 * contains at least one bit set to 1 in the same position 4856 * as that used during ibt_join_mcg(). i.e. the logical 4857 * AND of the two JoinState components is not all zeros. 4858 * This Join State component must not have some bits set 4859 * which are not set using ibt_join_mcg(). 4860 * Output: 4861 * None. 4862 * Returns: 4863 * IBT_SUCCESS 4864 * IBT_INVALID_PARAM 4865 * IBT_MC_GROUP_INVALID 4866 * IBT_INSUFF_RESOURCE 4867 * Description: 4868 * The port associated with the port GID shall be removed from the 4869 * multicast group specified by MGID (mc_gid) or from all the multicast 4870 * groups of which it is a member if the MGID (mc_gid) is not specified. 4871 * 4872 * The last full member to leave causes the destruction of the Multicast 4873 * Group. 4874 */ 4875 ibt_status_t 4876 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid, 4877 uint8_t mc_join_state) 4878 { 4879 sa_mcmember_record_t mcg_req; 4880 ibmf_saa_access_args_t access_args; 4881 ibmf_saa_handle_t saa_handle; 4882 uint64_t component_mask = 0; 4883 int sa_retval; 4884 ibt_status_t retval; 4885 ibtl_cm_hca_port_t hca_port; 4886 size_t length; 4887 void *results_p; 4888 ibcm_hca_info_t *hcap; 4889 4890 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)", 4891 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid); 4892 4893 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)", 4894 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state); 4895 4896 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4897 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required"); 4898 return (IBT_INVALID_PARAM); 4899 } 4900 4901 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4902 4903 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX", 4904 mc_gid.gid_prefix, mc_gid.gid_guid); 4905 4906 /* Validate MGID */ 4907 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4908 mcg_req.MGID = mc_gid; 4909 component_mask |= SA_MC_COMPMASK_MGID; 4910 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) { 4911 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified"); 4912 return (IBT_MC_MGID_INVALID); 4913 } 4914 4915 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) { 4916 mcg_req.PortGID = rgid; 4917 } else { 4918 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave"); 4919 mcg_req.PortGID = port_gid; 4920 } 4921 component_mask |= SA_MC_COMPMASK_PORTGID; 4922 4923 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>", 4924 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 4925 4926 /* Join State */ 4927 mcg_req.JoinState = mc_join_state; 4928 component_mask |= SA_MC_COMPMASK_JOINSTATE; 4929 4930 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port); 4931 if (retval != IBT_SUCCESS) { 4932 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info " 4933 "from specified RGID : status = %d", retval); 4934 return (retval); 4935 } 4936 4937 /* Get SA Access Handle. */ 4938 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4939 if (hcap == NULL) { 4940 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: " 4941 "NO HCA found"); 4942 return (IBT_HCA_BUSY_DETACHING); 4943 } 4944 4945 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4946 if (saa_handle == NULL) { 4947 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL"); 4948 ibcm_dec_hca_acc_cnt(hcap); 4949 return (IBT_HCA_PORT_NOT_ACTIVE); 4950 } 4951 4952 /* Contact SA Access */ 4953 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4954 access_args.sq_access_type = IBMF_SAA_DELETE; 4955 access_args.sq_component_mask = component_mask; 4956 access_args.sq_template = &mcg_req; 4957 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4958 access_args.sq_callback = NULL; 4959 access_args.sq_callback_arg = NULL; 4960 4961 ibcm_sa_access_enter(); 4962 4963 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length, 4964 &results_p); 4965 if (sa_retval != IBMF_SUCCESS) { 4966 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d", 4967 sa_retval); 4968 (void) ibcm_ibmf_analyze_error(sa_retval); 4969 retval = IBT_MC_GROUP_INVALID; 4970 } 4971 4972 ibcm_sa_access_exit(); 4973 4974 ibcm_dec_hca_acc_cnt(hcap); 4975 4976 return (retval); 4977 } 4978 4979 4980 /* 4981 * Function: 4982 * ibt_query_mcg 4983 * Input: 4984 * rgid The request GID that defines the HCA port upon which 4985 * to send the request to the Subnet Administrator, to 4986 * retrieve Multicast Records matching attributes as 4987 * specified through 'mcg_attr' argument. 4988 * 4989 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that 4990 * specifies MCG attributes that are to be matched. 4991 * Attributes that are not required can be wild carded 4992 * by specifying as '0'. 4993 * 4994 * mcgs_max_num The maximum number of matching multicast groups to 4995 * return. If zero, then all available matching multicast 4996 * groups are returned. 4997 * Output: 4998 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where 4999 * multicast group information is returned. The actual 5000 * number of entries filled in the array is returned in 5001 * entries_p. 5002 * 5003 * entries_p The number of ibt_mcg_attr_t entries returned. 5004 * Returns: 5005 * IBT_SUCCESS 5006 * IBT_INVALID_PARAM 5007 * IBT_MCG_RECORDS_NOT_FOUND 5008 * Description: 5009 * Request information on multicast groups that match the parameters 5010 * specified in mcg_attr. Information on each multicast group is returned 5011 * to the caller in the form of an array of ibt_mcg_info_t. 5012 * ibt_query_mcg() allocates the memory for this array and returns a 5013 * pointer to the array (mcgs_p) and the number of entries in the array 5014 * (entries_p). This memory should be freed by the client using 5015 * ibt_free_mcg_info(). 5016 */ 5017 ibt_status_t 5018 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num, 5019 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p) 5020 { 5021 sa_mcmember_record_t mcg_req; 5022 sa_mcmember_record_t *mcg_resp; 5023 ibt_mcg_info_t *mcg_infop; 5024 ibmf_saa_access_args_t access_args; 5025 ibmf_saa_handle_t saa_handle; 5026 uint64_t component_mask = 0; 5027 ibt_status_t retval; 5028 ibtl_cm_hca_port_t hport; 5029 uint_t num_records; 5030 size_t length; 5031 void *results_p; 5032 ib_gid_t port_gid; 5033 ibcm_hca_info_t *hcap; 5034 5035 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num); 5036 5037 if ((entries_p == NULL) || (mcgs_info_p == NULL)) { 5038 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5039 "entries_p or mcgs_info_p is NULL"); 5040 return (IBT_INVALID_PARAM); 5041 } 5042 5043 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5044 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required"); 5045 return (IBT_INVALID_PARAM); 5046 } 5047 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>", 5048 rgid.gid_prefix, rgid.gid_guid); 5049 5050 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5051 port_gid.gid_prefix = port_gid.gid_guid = 0; 5052 5053 if (mcg_attr != NULL) { 5054 port_gid = mcg_attr->mc_pgid; 5055 5056 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5057 mcg_req.PortGID = mcg_attr->mc_pgid; 5058 component_mask |= SA_MC_COMPMASK_PORTGID; 5059 5060 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX", 5061 port_gid.gid_prefix, port_gid.gid_guid); 5062 } 5063 5064 /* Is Q_Key specified. */ 5065 if (mcg_attr->mc_qkey != 0) { 5066 mcg_req.Q_Key = mcg_attr->mc_qkey; 5067 component_mask |= SA_MC_COMPMASK_QKEY; 5068 } 5069 5070 /* Is P_Key specified. */ 5071 if (mcg_attr->mc_pkey != 0) { 5072 mcg_req.P_Key = mcg_attr->mc_pkey; 5073 component_mask |= SA_MC_COMPMASK_PKEY; 5074 } 5075 5076 /* Is MGID specified. */ 5077 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5078 mcg_req.MGID = mcg_attr->mc_mgid; 5079 component_mask |= SA_MC_COMPMASK_MGID; 5080 } 5081 5082 /* Is MTU specified. */ 5083 if (mcg_attr->mc_mtu_req.r_mtu) { 5084 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu; 5085 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector; 5086 5087 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 5088 SA_MC_COMPMASK_MTU; 5089 } 5090 5091 if (mcg_attr->mc_tclass) { 5092 mcg_req.TClass = mcg_attr->mc_tclass; 5093 component_mask |= SA_MC_COMPMASK_TCLASS; 5094 } 5095 5096 /* Is RATE specified. */ 5097 if (mcg_attr->mc_rate_req.r_srate) { 5098 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate; 5099 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector; 5100 5101 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 5102 SA_MC_COMPMASK_RATE; 5103 } 5104 5105 /* Is Packet Life Time specified. */ 5106 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) { 5107 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt; 5108 mcg_req.RateSelector = 5109 mcg_attr->mc_pkt_lt_req.p_selector; 5110 5111 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5112 SA_MC_COMPMASK_PKTLT; 5113 } 5114 5115 if (mcg_attr->mc_hop) { 5116 mcg_req.HopLimit = mcg_attr->mc_hop; 5117 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5118 } 5119 5120 if (mcg_attr->mc_flow) { 5121 mcg_req.FlowLabel = mcg_attr->mc_flow; 5122 component_mask |= SA_MC_COMPMASK_FLOWLABEL; 5123 } 5124 5125 if (mcg_attr->mc_sl) { 5126 mcg_req.SL = mcg_attr->mc_sl; 5127 component_mask |= SA_MC_COMPMASK_SL; 5128 } 5129 5130 if (mcg_attr->mc_scope) { 5131 mcg_req.Scope = mcg_attr->mc_scope; 5132 component_mask |= SA_MC_COMPMASK_SCOPE; 5133 } 5134 5135 if (mcg_attr->mc_join_state) { 5136 mcg_req.JoinState = mcg_attr->mc_join_state; 5137 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5138 } 5139 5140 if (mcg_attr->mc_mlid) { 5141 mcg_req.MLID = mcg_attr->mc_mlid; 5142 component_mask |= SA_MC_COMPMASK_MLID; 5143 } 5144 } 5145 5146 retval = ibtl_cm_get_hca_port(rgid, 0, &hport); 5147 if (retval != IBT_SUCCESS) { 5148 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info " 5149 "from specified RGID : status = %d", retval); 5150 return (retval); 5151 } 5152 5153 /* Get SA Access Handle. */ 5154 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5155 if (hcap == NULL) { 5156 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found"); 5157 return (IBT_HCA_BUSY_DETACHING); 5158 } 5159 5160 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port); 5161 if (saa_handle == NULL) { 5162 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL"); 5163 ibcm_dec_hca_acc_cnt(hcap); 5164 return (IBT_HCA_PORT_NOT_ACTIVE); 5165 } 5166 5167 /* Contact SA Access */ 5168 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5169 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 5170 access_args.sq_component_mask = component_mask; 5171 access_args.sq_template = &mcg_req; 5172 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5173 access_args.sq_callback = NULL; 5174 access_args.sq_callback_arg = NULL; 5175 5176 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5177 &results_p); 5178 if (retval != IBT_SUCCESS) { 5179 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed"); 5180 ibcm_dec_hca_acc_cnt(hcap); 5181 return (retval); 5182 } 5183 5184 num_records = length/sizeof (sa_mcmember_record_t); 5185 5186 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records", 5187 num_records); 5188 5189 /* Validate the returned number of records. */ 5190 if ((results_p != NULL) && (num_records > 0)) { 5191 uint_t i; 5192 5193 /* 5194 * If mcgs_max_num is zero, then return all records else 5195 * return only requested number of records 5196 */ 5197 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) { 5198 /* we are interested in only mcgs_max_num records */ 5199 num_records = mcgs_max_num; 5200 } 5201 5202 /* 5203 * The SGID returned in "mcg_info_p" buffer should be PortGID, 5204 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified, 5205 * else RequestGID (rgid) should be returned. 5206 */ 5207 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5208 5209 /* Get sgid_ix and port number of 'port_gid' */ 5210 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport); 5211 if (retval != IBT_SUCCESS) { 5212 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5213 "Failed to Get Portinfo for PortGID :" 5214 "status = %d", retval); 5215 return (retval); 5216 } 5217 } else { 5218 /* 5219 * The sgid_ix and port number related to RequestGID 5220 * are already obtained at the beginning. 5221 */ 5222 port_gid = rgid; 5223 } 5224 5225 /* 5226 * Allocate memory for return buffer, to be freed in 5227 * ibt_free_mcg_info(). 5228 */ 5229 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)), 5230 KM_SLEEP); 5231 5232 *mcgs_info_p = mcg_infop; 5233 *entries_p = num_records; 5234 5235 /* Update the return values. */ 5236 for (i = 0; i < num_records; i++) { 5237 5238 mcg_resp = (sa_mcmember_record_t *)((uchar_t *) 5239 results_p + i * sizeof (sa_mcmember_record_t)); 5240 5241 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID; 5242 mcg_infop[i].mc_adds_vect.av_sgid = port_gid; 5243 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate; 5244 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL; 5245 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5246 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass; 5247 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit; 5248 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port; 5249 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE; 5250 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID; 5251 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix; 5252 mcg_infop[i].mc_adds_vect.av_src_path = 0; 5253 mcg_infop[i].mc_mtu = mcg_resp->MTU; 5254 mcg_infop[i].mc_qkey = mcg_resp->Q_Key; 5255 mcg_infop[i].mc_scope = mcg_resp->Scope; 5256 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime; 5257 5258 if (ibt_pkey2index_byguid(hport.hp_hca_guid, 5259 hport.hp_port, mcg_resp->P_Key, 5260 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) { 5261 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: " 5262 "Pkey2Index Conversion failed"); 5263 mcg_infop[i].mc_pkey_ix = 0; 5264 } 5265 } 5266 5267 /* 5268 * Deallocate the memory allocated by SA for results_p. 5269 */ 5270 kmem_free(results_p, length); 5271 retval = IBT_SUCCESS; 5272 5273 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords", 5274 num_records); 5275 5276 } else { 5277 retval = IBT_MCG_RECORDS_NOT_FOUND; 5278 *entries_p = 0; 5279 5280 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND"); 5281 } 5282 5283 ibcm_dec_hca_acc_cnt(hcap); 5284 5285 return (retval); 5286 } 5287 5288 5289 /* 5290 * ibt_free_mcg_info() 5291 * Free the memory allocated by successful ibt_query_mcg() 5292 * 5293 * mcgs_info Pointer returned by ibt_query_mcg(). 5294 * 5295 * entries The number of ibt_mcg_info_t entries to free. 5296 */ 5297 void 5298 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries) 5299 { 5300 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: " 5301 "Free <%d> entries from 0x%p", entries, mcgs_info); 5302 5303 if ((mcgs_info != NULL) && (entries > 0)) 5304 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t)); 5305 else 5306 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: " 5307 "ERROR: NULL buf pointer or length specified."); 5308 } 5309 5310 5311 /* 5312 * Function: 5313 * ibt_gid_to_node_info() 5314 * Input: 5315 * gid Identifies the IB Node and port for which to obtain 5316 * Node information. 5317 * Output: 5318 * node_info_p A pointer to an ibt_node_info_t structure (allocated 5319 * by the caller) in which to return the node information. 5320 * Returns: 5321 * IBT_SUCCESS 5322 * IBT_INVALID_PARAM 5323 * IBT_NODE_RECORDS_NOT_FOUND 5324 * IBT_NO_HCAS_AVAILABLE 5325 * Description: 5326 * Retrieve Node Information for the specified GID. 5327 */ 5328 ibt_status_t 5329 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p) 5330 { 5331 sa_node_record_t nr_req, *nr_resp; 5332 ibmf_saa_handle_t saa_handle; 5333 ibt_status_t retval; 5334 ibcm_hca_info_t *hcap; 5335 ibtl_cm_hca_port_t hport; 5336 int i, j; 5337 uint_t num_rec; 5338 ib_guid_t *guid_array = NULL; 5339 sa_path_record_t *path; 5340 size_t len; 5341 uint8_t npaths; 5342 uint32_t num_hcas = 0; 5343 ib_lid_t node_lid; 5344 boolean_t local_node = B_FALSE; 5345 void *res_p; 5346 uint8_t num_ports = 0; 5347 5348 5349 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)", 5350 gid.gid_prefix, gid.gid_guid, node_info_p); 5351 5352 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) { 5353 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required"); 5354 return (IBT_INVALID_PARAM); 5355 } 5356 5357 if (node_info_p == NULL) { 5358 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5359 "Return Buf (node_info_p) is NULL."); 5360 return (IBT_INVALID_PARAM); 5361 } 5362 5363 /* 5364 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5365 * associated port) info via ibtl_cm_get_hca_port() call. 5366 */ 5367 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5368 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) { 5369 5370 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5371 if (hcap == NULL) { 5372 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5373 "HCA(%llX) info not found", hport.hp_hca_guid); 5374 return (IBT_NO_HCAS_AVAILABLE); 5375 } 5376 num_ports = 1; 5377 num_hcas = 1; 5378 node_lid = hport.hp_base_lid; 5379 local_node = B_TRUE; 5380 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: " 5381 "LID = 0x%X", node_lid); 5382 } else { 5383 /* Get the number of HCAs and their GUIDs */ 5384 num_hcas = ibt_get_hca_list(&guid_array); 5385 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list " 5386 "returned %d hcas", num_hcas); 5387 5388 if (num_hcas == 0) { 5389 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5390 "NO HCA's Found on this system"); 5391 return (IBT_NO_HCAS_AVAILABLE); 5392 } 5393 } 5394 5395 for (i = 0; i < num_hcas; i++) { 5396 if (local_node == B_FALSE) { 5397 hcap = ibcm_find_hca_entry(guid_array[i]); 5398 if (hcap == NULL) { 5399 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5400 "HCA(%llX) info not found", guid_array[i]); 5401 retval = IBT_NO_HCAS_AVAILABLE; 5402 continue; 5403 } 5404 num_ports = hcap->hca_num_ports; 5405 } 5406 5407 for (j = 0; j < num_ports; j++) { 5408 uint8_t port = 0; 5409 5410 if (local_node == B_TRUE) 5411 port = hport.hp_port; 5412 else 5413 port = j + 1; 5414 5415 /* Get SA Access Handle. */ 5416 saa_handle = ibcm_get_saa_handle(hcap, port); 5417 if (saa_handle == NULL) { 5418 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5419 "Port %d of HCA (%llX) is NOT ACTIVE", 5420 port, hport.hp_hca_guid); 5421 retval = IBT_NODE_RECORDS_NOT_FOUND; 5422 continue; 5423 } 5424 5425 if (local_node == B_FALSE) { 5426 ib_gid_t sgid; 5427 int sa_ret; 5428 5429 /* 5430 * Check whether 'gid' and this port has same 5431 * subnet prefix. If not, then there is no use 5432 * in searching from this port. 5433 */ 5434 sgid = hcap->hca_port_info[j].port_sgid0; 5435 if (gid.gid_prefix != sgid.gid_prefix) { 5436 IBTF_DPRINTF_L3(cmlog, 5437 "ibt_gid_to_node_info:Sn_Prefix of " 5438 "GID(%llX) and Port's(%llX) differ", 5439 gid.gid_prefix, sgid.gid_prefix); 5440 retval = IBT_NODE_RECORDS_NOT_FOUND; 5441 continue; 5442 } 5443 5444 /* 5445 * First Get Path Records for the specified DGID 5446 * from this port (SGID). From Path Records, 5447 * note down DLID, then use this DLID as Input 5448 * attribute to get NodeRecords from SA Access. 5449 */ 5450 npaths = 1; 5451 path = NULL; 5452 5453 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5454 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5455 &path); 5456 if (sa_ret != IBMF_SUCCESS) { 5457 IBTF_DPRINTF_L2(cmlog, 5458 "ibt_gid_to_node_info: " 5459 "ibmf_saa_gid_to_pathrecords() " 5460 "returned error: %d ", sa_ret); 5461 retval = 5462 ibcm_ibmf_analyze_error(sa_ret); 5463 continue; 5464 } else if ((npaths == 0) || (path == NULL)) { 5465 IBTF_DPRINTF_L3(cmlog, 5466 "ibt_gid_to_node_info: failed (%d) " 5467 "to get path records for the DGID " 5468 "0x%llX from SGID 0x%llX", sa_ret, 5469 gid.gid_guid, sgid.gid_guid); 5470 retval = IBT_NODE_RECORDS_NOT_FOUND; 5471 continue; 5472 } 5473 node_lid = path->DLID; /* LID */ 5474 5475 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5476 "Remote Node: LID = 0x%X", node_lid); 5477 5478 /* Free SA_Access memory for path record. */ 5479 kmem_free(path, len); 5480 } 5481 5482 /* Retrieve Node Records from SA Access. */ 5483 bzero(&nr_req, sizeof (sa_node_record_t)); 5484 5485 nr_req.LID = node_lid; /* LID */ 5486 5487 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5488 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5489 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5490 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5491 "failed (%d) to get Node records", retval); 5492 continue; 5493 } else if (retval != IBT_SUCCESS) { 5494 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5495 "failed (%d) to get Node records", retval); 5496 ibcm_dec_hca_acc_cnt(hcap); 5497 goto gid_to_ni_exit; 5498 } 5499 5500 num_rec = len/sizeof (sa_node_record_t); 5501 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5502 5503 /* Validate the returned number of records. */ 5504 if ((nr_resp != NULL) && (num_rec > 0)) { 5505 5506 IBCM_DUMP_NODE_REC(nr_resp); 5507 5508 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS( 5509 *node_info_p)) 5510 5511 node_info_p->n_sys_img_guid = 5512 nr_resp->NodeInfo.SystemImageGUID; 5513 node_info_p->n_node_guid = 5514 nr_resp->NodeInfo.NodeGUID; 5515 node_info_p->n_port_guid = 5516 nr_resp->NodeInfo.PortGUID; 5517 node_info_p->n_dev_id = 5518 nr_resp->NodeInfo.DeviceID; 5519 node_info_p->n_revision = 5520 nr_resp->NodeInfo.Revision; 5521 node_info_p->n_vendor_id = 5522 nr_resp->NodeInfo.VendorID; 5523 node_info_p->n_num_ports = 5524 nr_resp->NodeInfo.NumPorts; 5525 node_info_p->n_port_num = 5526 nr_resp->NodeInfo.LocalPortNum; 5527 node_info_p->n_node_type = 5528 nr_resp->NodeInfo.NodeType; 5529 (void) strncpy(node_info_p->n_description, 5530 (char *)&nr_resp->NodeDescription, 64); 5531 5532 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS( 5533 *node_info_p)) 5534 5535 /* 5536 * Deallocate the memory allocated by SA for 5537 * 'nr_resp'. 5538 */ 5539 ibcm_dec_hca_acc_cnt(hcap); 5540 kmem_free(nr_resp, len); 5541 retval = IBT_SUCCESS; 5542 5543 goto gid_to_ni_exit; 5544 } else { 5545 retval = IBT_NODE_RECORDS_NOT_FOUND; 5546 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5547 "Node Records NOT found - PortGUID %016llX", 5548 gid.gid_guid); 5549 } 5550 } 5551 ibcm_dec_hca_acc_cnt(hcap); 5552 5553 if (local_node == B_TRUE) 5554 break; 5555 } 5556 5557 gid_to_ni_exit: 5558 if (guid_array) 5559 ibt_free_hca_list(guid_array, num_hcas); 5560 5561 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval); 5562 5563 return (retval); 5564 } 5565 5566 5567 static ibt_status_t 5568 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req, 5569 uint64_t component_mask, void *result_p, size_t *len) 5570 { 5571 ibmf_saa_access_args_t args; 5572 size_t length; 5573 ibt_status_t retval; 5574 5575 args.sq_attr_id = SA_NODERECORD_ATTRID; 5576 args.sq_template = nr_req; 5577 args.sq_access_type = IBMF_SAA_RETRIEVE; 5578 args.sq_template_length = sizeof (sa_node_record_t); 5579 args.sq_component_mask = component_mask; 5580 args.sq_callback = NULL; 5581 args.sq_callback_arg = NULL; 5582 5583 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p); 5584 if (retval != IBT_SUCCESS) { 5585 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed"); 5586 return (retval); 5587 } 5588 5589 *len = length; 5590 5591 /* Validate the returned number of records. */ 5592 if ((result_p != NULL) && (length > 0)) { 5593 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND"); 5594 5595 /* Got it, done!. */ 5596 return (IBT_SUCCESS); 5597 } else { 5598 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found"); 5599 return (IBT_NODE_RECORDS_NOT_FOUND); 5600 } 5601 } 5602 5603 5604 /* 5605 * Function: 5606 * ibt_get_companion_port_gids() 5607 * Description: 5608 * Get list of GID's available on a companion port(s) of the specified 5609 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID. 5610 */ 5611 ibt_status_t 5612 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid, 5613 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p) 5614 { 5615 sa_node_record_t nr_req, *nr_resp; 5616 void *res_p; 5617 ibmf_saa_handle_t saa_handle; 5618 int sa_ret; 5619 ibt_status_t retval; 5620 ibcm_hca_info_t *hcap; 5621 ibtl_cm_hca_port_t hport; 5622 int i, j; 5623 uint_t num_rec; 5624 ib_guid_t *guid_array = NULL; 5625 sa_path_record_t *path; 5626 size_t len; 5627 uint8_t npaths; 5628 uint32_t num_hcas = 0; 5629 boolean_t local_node = B_FALSE; 5630 boolean_t local_hca = B_FALSE; 5631 ib_guid_t h_guid = hca_guid; 5632 ib_gid_t *gidp = NULL, *t_gidp = NULL; 5633 int multi_hca_loop = 0; 5634 5635 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, " 5636 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid); 5637 5638 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) && 5639 (sysimg_guid == 0)) { 5640 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5641 "Null Input attribute specified."); 5642 return (IBT_INVALID_PARAM); 5643 } 5644 5645 if ((num_gids_p == NULL) || (gids_p == NULL)) { 5646 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5647 "num_gids_p or gids_p is NULL"); 5648 return (IBT_INVALID_PARAM); 5649 } 5650 5651 *num_gids_p = 0; 5652 5653 /* Get the number of HCAs and their GUIDs */ 5654 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) { 5655 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5656 "NO HCA's Found on this system"); 5657 return (IBT_NO_HCAS_AVAILABLE); 5658 } 5659 5660 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5661 "ibt_get_hca_list() returned %d hcas", num_hcas); 5662 5663 /* 5664 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5665 * associated port) info via ibtl_cm_get_hca_port() call. 5666 */ 5667 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5668 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) && 5669 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) { 5670 5671 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) { 5672 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5673 "Invalid GID<->HCAGUID combination specified."); 5674 retval = IBT_INVALID_PARAM; 5675 goto get_comp_pgid_exit; 5676 } 5677 h_guid = hport.hp_hca_guid; 5678 local_node = B_TRUE; 5679 5680 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5681 "Local Node: HCA (0x%llX)", h_guid); 5682 } else if (h_guid) { /* Is specified HCA GUID - local? */ 5683 for (i = 0; i < num_hcas; i++) { 5684 if (h_guid == guid_array[i]) { 5685 local_hca = B_TRUE; 5686 break; 5687 } 5688 } 5689 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */ 5690 for (i = 0; i < num_hcas; i++) { 5691 ibt_status_t ret; 5692 ibt_hca_attr_t hca_attr; 5693 5694 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr); 5695 if (ret != IBT_SUCCESS) { 5696 IBTF_DPRINTF_L2(cmlog, 5697 "ibt_get_companion_port_gids: HCA(%llX) " 5698 "info not found", guid_array[i]); 5699 retval = IBT_NO_HCAS_AVAILABLE; 5700 continue; 5701 } 5702 if (hca_attr.hca_si_guid == sysimg_guid) { 5703 if ((hca_guid != 0) && 5704 (hca_guid != hca_attr.hca_node_guid)) { 5705 IBTF_DPRINTF_L2(cmlog, 5706 "ibt_get_companion_port_gids: " 5707 "Invalid SysImg<->HCA GUID " 5708 "combination specified."); 5709 retval = IBT_INVALID_PARAM; 5710 goto get_comp_pgid_exit; 5711 } 5712 local_hca = B_TRUE; 5713 h_guid = hca_attr.hca_node_guid; 5714 break; 5715 } 5716 } 5717 } 5718 5719 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) { 5720 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p, 5721 num_gids_p); 5722 goto get_comp_pgid_exit; 5723 } 5724 5725 get_comp_for_multihca: 5726 /* We will be here, if request is for remote node */ 5727 for (i = 0; i < num_hcas; i++) { 5728 int multism; 5729 uint8_t count = 0; 5730 int multi_sm_loop = 0; 5731 uint_t k = 0, l; 5732 5733 hcap = ibcm_find_hca_entry(guid_array[i]); 5734 if (hcap == NULL) { 5735 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5736 "HCA(%llX) info not found", guid_array[i]); 5737 retval = IBT_NO_HCAS_AVAILABLE; 5738 continue; 5739 } 5740 5741 /* 1 - MultiSM, 0 - Single SM */ 5742 multism = ibtl_cm_is_multi_sm(guid_array[i]); 5743 5744 for (j = 0; j < hcap->hca_num_ports; j++) { 5745 ib_gid_t sgid; 5746 uint64_t c_mask = 0; 5747 ib_guid_t pg; 5748 uint_t port = j; 5749 5750 get_comp_for_multism: 5751 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5752 "Port %d, HCA %llX, MultiSM= %d, Loop=%d", 5753 port + 1, h_guid, multism, multi_sm_loop); 5754 5755 /* Get SA Access Handle. */ 5756 saa_handle = ibcm_get_saa_handle(hcap, port + 1); 5757 if (saa_handle == NULL) { 5758 IBTF_DPRINTF_L2(cmlog, 5759 "ibt_get_companion_port_gids: " 5760 "Port (%d) - NOT ACTIVE", port); 5761 retval = IBT_HCA_PORT_NOT_ACTIVE; 5762 continue; 5763 } 5764 5765 /* 5766 * Check whether 'gid' and this port has same subnet 5767 * prefix. If not, then there is no use in searching 5768 * from this port. 5769 */ 5770 sgid = hcap->hca_port_info[port].port_sgid0; 5771 if ((h_guid == 0) && (gid.gid_prefix != 0) && 5772 (multi_sm_loop == 0) && 5773 (gid.gid_prefix != sgid.gid_prefix)) { 5774 IBTF_DPRINTF_L2(cmlog, 5775 "ibt_get_companion_port_gids: SnPrefix of " 5776 "GID(%llX) and Port SN_Pfx(%llX) differ", 5777 gid.gid_prefix, sgid.gid_prefix); 5778 retval = IBT_NODE_RECORDS_NOT_FOUND; 5779 continue; 5780 } 5781 5782 /* 5783 * If HCA GUID or System Image GUID is specified, then 5784 * we can achieve our goal sooner!. 5785 */ 5786 if ((h_guid == 0) && (sysimg_guid == 0)) { 5787 /* So only GID info is provided. */ 5788 5789 /* 5790 * First Get Path Records for the specified DGID 5791 * from this port (SGID). From Path Records, 5792 * note down DLID, then use this DLID as Input 5793 * attribute to get NodeRecords. 5794 */ 5795 npaths = 1; 5796 path = NULL; 5797 5798 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5799 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5800 &path); 5801 if (sa_ret != IBMF_SUCCESS) { 5802 IBTF_DPRINTF_L2(cmlog, 5803 "ibt_get_companion_port_gids: " 5804 "ibmf_saa_gid_to_pathrecords() " 5805 "returned error: %d ", sa_ret); 5806 retval = 5807 ibcm_ibmf_analyze_error(sa_ret); 5808 ibcm_dec_hca_acc_cnt(hcap); 5809 goto get_comp_pgid_exit; 5810 } else if ((npaths == 0) || (path == NULL)) { 5811 IBTF_DPRINTF_L2(cmlog, 5812 "ibt_get_companion_port_gids: " 5813 "failed (%d) to get path records " 5814 "for the DGID (0x%llX) from SGID " 5815 "(0x%llX)", sa_ret, gid.gid_guid, 5816 sgid.gid_guid); 5817 retval = IBT_NODE_RECORDS_NOT_FOUND; 5818 continue; 5819 } 5820 5821 bzero(&nr_req, sizeof (sa_node_record_t)); 5822 nr_req.LID = path->DLID; /* LID */ 5823 5824 IBTF_DPRINTF_L3(cmlog, 5825 "ibt_get_companion_port_gids: " 5826 "Remote Node: LID = 0x%X", nr_req.LID); 5827 5828 /* Free SA_Access memory for path record. */ 5829 kmem_free(path, len); 5830 5831 IBTF_DPRINTF_L3(cmlog, 5832 "ibt_get_companion_port_gids: SAA Call: " 5833 "based on LID "); 5834 5835 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5836 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5837 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5838 IBTF_DPRINTF_L2(cmlog, 5839 "ibt_get_companion_port_gids: " 5840 "failed (%d) to get Node records", 5841 retval); 5842 continue; 5843 } else if (retval != IBT_SUCCESS) { 5844 IBTF_DPRINTF_L2(cmlog, 5845 "ibt_get_companion_port_gids: " 5846 "failed (%d) to get Node records", 5847 retval); 5848 ibcm_dec_hca_acc_cnt(hcap); 5849 goto get_comp_pgid_exit; 5850 } 5851 5852 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5853 /* Note down HCA GUID info. */ 5854 h_guid = nr_resp->NodeInfo.NodeGUID; 5855 5856 IBTF_DPRINTF_L3(cmlog, 5857 "ibt_get_companion_port_gids: " 5858 "Remote HCA GUID: 0x%llX", h_guid); 5859 5860 IBCM_DUMP_NODE_REC(nr_resp); 5861 5862 kmem_free(res_p, len); 5863 } 5864 5865 bzero(&nr_req, sizeof (sa_node_record_t)); 5866 if (h_guid != 0) { 5867 nr_req.NodeInfo.NodeGUID = h_guid; 5868 c_mask = SA_NODEINFO_COMPMASK_NODEGUID; 5869 } 5870 5871 if (sysimg_guid != 0) { 5872 nr_req.NodeInfo.SystemImageGUID = sysimg_guid; 5873 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID; 5874 } 5875 5876 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5877 "SAA Call: CMASK= 0x%llX", c_mask); 5878 5879 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask, 5880 &res_p, &len); 5881 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5882 IBTF_DPRINTF_L3(cmlog, 5883 "ibt_get_companion_port_gids: " 5884 "failed (%d) to get Node records", retval); 5885 continue; 5886 } else if (retval != IBT_SUCCESS) { 5887 IBTF_DPRINTF_L2(cmlog, 5888 "ibt_get_companion_port_gids: Error: (%d) " 5889 "while getting Node records", retval); 5890 ibcm_dec_hca_acc_cnt(hcap); 5891 goto get_comp_pgid_exit; 5892 } 5893 5894 num_rec = len/sizeof (sa_node_record_t); 5895 5896 /* We will be here, only if we found some NodeRec */ 5897 if (gid.gid_prefix && gid.gid_guid) { 5898 nr_resp = (sa_node_record_t *)res_p; 5899 for (l = 0; l < num_rec; l++, nr_resp++) { 5900 pg = nr_resp->NodeInfo.PortGUID; 5901 if (gid.gid_guid != pg) 5902 count++; 5903 } 5904 } else { 5905 count = num_rec; 5906 } 5907 5908 if (count != 0) { 5909 if (multi_sm_loop == 1) { 5910 count += k; 5911 t_gidp = kmem_zalloc(count * 5912 sizeof (ib_gid_t), KM_SLEEP); 5913 5914 if ((k != 0) && (gidp != NULL)) { 5915 bcopy(gidp, t_gidp, 5916 k * sizeof (ib_gid_t)); 5917 kmem_free(gidp, 5918 k * sizeof (ib_gid_t)); 5919 } 5920 gidp = t_gidp; 5921 } else { 5922 gidp = kmem_zalloc(count * 5923 sizeof (ib_gid_t), KM_SLEEP); 5924 } 5925 *num_gids_p = count; 5926 *gids_p = gidp; 5927 5928 nr_resp = (sa_node_record_t *)res_p; 5929 for (l = 0; l < num_rec; l++, nr_resp++) { 5930 IBCM_DUMP_NODE_REC(nr_resp); 5931 5932 pg = nr_resp->NodeInfo.PortGUID; 5933 IBTF_DPRINTF_L4(cmlog, 5934 "ibt_get_companion_port_gids: " 5935 "PortGID %llX", pg); 5936 5937 if (pg != gid.gid_guid) { 5938 gidp[k].gid_prefix = 5939 sgid.gid_prefix; 5940 gidp[k].gid_guid = pg; 5941 5942 IBTF_DPRINTF_L3(cmlog, 5943 "ibt_get_companion_pgids: " 5944 "GID[%d] = %llX:%llX", k, 5945 gidp[k].gid_prefix, 5946 gidp[k].gid_guid); 5947 5948 k++; 5949 if (k == count) 5950 break; 5951 } 5952 } 5953 retval = IBT_SUCCESS; /* done!. */ 5954 kmem_free(res_p, len); 5955 ibcm_dec_hca_acc_cnt(hcap); 5956 goto get_comp_pgid_exit; 5957 } else { 5958 IBTF_DPRINTF_L2(cmlog, 5959 "ibt_get_companion_port_gids: " 5960 "Companion PortGIDs not available"); 5961 retval = IBT_GIDS_NOT_FOUND; 5962 } 5963 /* Deallocate the memory for 'res_p'. */ 5964 kmem_free(res_p, len); 5965 5966 /* 5967 * If we are on MultiSM setup, then we need to lookout 5968 * from that subnet port too. 5969 */ 5970 if (multism) { 5971 /* break if already searched both the subnet */ 5972 if (multi_sm_loop == 1) 5973 break; 5974 5975 port = (j == 0) ? 1 : 0; 5976 multi_sm_loop = 1; 5977 goto get_comp_for_multism; 5978 } else { 5979 break; 5980 } 5981 } 5982 ibcm_dec_hca_acc_cnt(hcap); 5983 5984 /* 5985 * We may be on dual HCA with dual SM configured system. And 5986 * the input attr GID was visible from second HCA. So in order 5987 * to get the companion portgid we need to re-look from the 5988 * first HCA ports. 5989 */ 5990 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) && 5991 (multi_hca_loop != 1)) { 5992 multi_hca_loop = 1; 5993 goto get_comp_for_multihca; 5994 } 5995 } 5996 5997 get_comp_pgid_exit: 5998 if (guid_array) 5999 ibt_free_hca_list(guid_array, num_hcas); 6000 6001 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) { 6002 retval = IBT_SUCCESS; 6003 } 6004 6005 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, " 6006 "Found %d GIDs", retval, *num_gids_p); 6007 6008 return (retval); 6009 } 6010 6011 6012 /* Routines for warlock */ 6013 6014 /* ARGSUSED */ 6015 static void 6016 ibcm_dummy_mcg_handler(void *arg, ibt_status_t retval, ibt_mcg_info_t *minfo) 6017 { 6018 ibcm_join_mcg_tqarg_t dummy_mcg; 6019 6020 dummy_mcg.func = ibcm_dummy_mcg_handler; 6021 6022 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_mcg_handler: " 6023 "dummy_mcg.func %p", dummy_mcg.func); 6024 } 6025 6026 6027 /* ARGSUSED */ 6028 static void 6029 ibcm_dummy_recycle_rc_handler(ibt_status_t retval, void *arg) 6030 { 6031 ibcm_taskq_recycle_arg_t dummy_rc_recycle; 6032 6033 dummy_rc_recycle.func = ibcm_dummy_recycle_rc_handler; 6034 6035 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_recycle_rc_handler: " 6036 "dummy_rc_recycle.func %p", dummy_rc_recycle.func); 6037 } 6038 6039 6040 /* ARGSUSED */ 6041 static ibt_cm_status_t 6042 ibcm_dummy_ud_handler(void *priv, ibt_cm_ud_event_t *event, 6043 ibt_cm_ud_return_args_t *ret_args, 6044 void *priv_data, ibt_priv_data_len_t len) 6045 { 6046 /* 6047 * Let warlock see that ibcm_local_handler_s::actual_cm_handler 6048 * points to this routine. 6049 */ 6050 ibcm_local_handler_t p; 6051 ibcm_ud_state_data_t dummy_ud; 6052 6053 p.actual_cm_handler = ibcm_dummy_ud_handler; 6054 dummy_ud.ud_cm_handler = ibcm_dummy_ud_handler; 6055 6056 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_ud_handler: p.actual_cm_handler %p" 6057 "dummy_ud.ud_cm_handler %p", p.actual_cm_handler, 6058 dummy_ud.ud_cm_handler); 6059 /* 6060 * Call all routines that the client's callback routine could call. 6061 */ 6062 6063 return (IBT_CM_ACCEPT); 6064 } 6065 6066 /* ARGSUSED */ 6067 static ibt_cm_status_t 6068 ibcm_dummy_rc_handler(void *priv, ibt_cm_event_t *event, 6069 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 6070 { 6071 ibcm_state_data_t dummy_rc; 6072 6073 dummy_rc.cm_handler = ibcm_dummy_rc_handler; 6074 6075 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_rc_handler: " 6076 "dummy_ud.ud_cm_handler %p", dummy_rc.cm_handler); 6077 /* 6078 * Call all routines that the client's callback routine could call. 6079 */ 6080 6081 return (IBT_CM_ACCEPT); 6082 } 6083