1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 28 29 /* Global sa_access State Pointer */ 30 saa_state_t *saa_statep; 31 _NOTE(READ_ONLY_DATA(saa_statep)) 32 33 extern int ibmf_trace_level; 34 35 extern int ibmf_taskq_max_tasks; 36 37 static int 38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 40 static int 41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags); 43 static int 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 46 hrtime_t trans_send_time, int transport_flags); 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 49 ibmf_retrans_t *ibmf_retransp); 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 55 int status); 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 57 void *clnt_private, ibmf_async_event_t event_type); 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event); 61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num); 62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 65 size_t *length, boolean_t sleep_flag); 66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 70 ib_guid_t *guid_ret); 71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 72 ibt_hca_portinfo_t *portinfop); 73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, 74 ibmf_msg_t *msgp); 75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 76 77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 79 int ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME; 80 int ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 81 int ibmf_saa_retrans_retries = IBMF_SAA_RETRANS_RETRIES; 82 83 /* 84 * ibmf_saa_impl_init: 85 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 86 * Called from the modules init() routine. 87 * 88 * Input Arguments 89 * none 90 * 91 * Output Arguments 92 * none 93 * 94 * Returns 95 * IBMF_NO_RESOURCES if taskq could not be created. 96 * IBMF_SUCCESS on success 97 * 98 */ 99 int 100 ibmf_saa_impl_init() 101 { 102 int res; 103 104 /* CONSTCOND */ 105 ASSERT(NO_COMPETING_THREADS); 106 107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 109 110 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 111 112 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 113 114 /* create taskq for notifying event subscribers */ 115 saa_statep->saa_event_taskq = taskq_create( 116 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 117 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 118 TASKQ_PREPOPULATE); 119 if (saa_statep->saa_event_taskq == NULL) { 120 121 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 122 ibmf_saa_impl_init_end_err, 123 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 124 tnf_string, msg, "event taskq create failed"); 125 126 kmem_free(saa_statep, sizeof (saa_state_t)); 127 128 res = IBMF_NO_RESOURCES; 129 130 goto bail; 131 } 132 133 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 134 NULL); 135 136 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 137 138 res = IBMF_SUCCESS; 139 bail: 140 141 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 142 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 143 tnf_int, res, res); 144 145 return (res); 146 } 147 148 /* 149 * ibmf_saa_impl_fini: 150 * If there are no registered clients, cleans up all memory associated with the 151 * state, including each of the port list entries. 152 * Called from the modules fini() routine. 153 * 154 * Input Arguments 155 * none 156 * 157 * Output Arguments 158 * none 159 * 160 * Returns 161 * EBUSY if there are outstanding transactions or registered clients 162 * 0 if cleanup was sucessfull 163 * 164 */ 165 int 166 ibmf_saa_impl_fini() 167 { 168 int ret = 0; 169 saa_port_t *saa_portp; 170 saa_port_t *next; 171 172 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 173 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 174 175 /* make sure there are no registered clients */ 176 mutex_enter(&saa_statep->saa_port_list_mutex); 177 178 saa_portp = saa_statep->saa_port_list; 179 while (saa_portp != NULL) { 180 181 mutex_enter(&saa_portp->saa_pt_mutex); 182 183 if (saa_portp->saa_pt_reference_count > 0) { 184 185 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 186 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 187 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 188 tnf_string, msg, 189 "cannot unload ibmf_saa. Client on port still" 190 " registered", tnf_opaque, port, 191 saa_portp->saa_pt_port_guid); 192 193 mutex_exit(&saa_portp->saa_pt_mutex); 194 195 mutex_exit(&saa_statep->saa_port_list_mutex); 196 197 ret = EBUSY; 198 goto bail; 199 } 200 201 /* make sure there are no outstanding transactions */ 202 203 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 204 205 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 206 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 207 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 208 ", num transactions = %d\n", 209 tnf_string, msg, "Cannot unload ibmf_saa." 210 " Outstanding transactions on port.", 211 tnf_opaque, port, 212 saa_portp->saa_pt_port_guid, 213 tnf_uint, outstanding_transactions, 214 saa_portp->saa_pt_num_outstanding_trans); 215 216 mutex_exit(&saa_portp->saa_pt_mutex); 217 218 mutex_exit(&saa_statep->saa_port_list_mutex); 219 220 ret = EBUSY; 221 goto bail; 222 } 223 224 mutex_exit(&saa_portp->saa_pt_mutex); 225 226 saa_portp = saa_portp->next; 227 } 228 229 mutex_exit(&saa_statep->saa_port_list_mutex); 230 231 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 232 *saa_portp)) 233 234 /* 235 * no more clients nor pending transaction: 236 * unregister ibmf and destroy port entries 237 */ 238 while (saa_statep->saa_port_list != NULL) { 239 240 saa_portp = saa_statep->saa_port_list; 241 next = saa_portp->next; 242 243 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 244 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 245 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 246 tnf_string, msg, "deinitializing port", 247 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 248 249 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 250 251 mutex_enter(&saa_portp->saa_pt_mutex); 252 253 /* unregister from ibmf */ 254 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 255 256 mutex_exit(&saa_portp->saa_pt_mutex); 257 258 if (ibmf_saa_impl_ibmf_unreg(saa_portp) 259 != IBMF_SUCCESS) { 260 ret = EBUSY; 261 goto bail; 262 } 263 } else 264 mutex_exit(&saa_portp->saa_pt_mutex); 265 266 ibmf_saa_impl_destroy_port(saa_portp); 267 268 saa_statep->saa_port_list = next; 269 } 270 271 taskq_destroy(saa_statep->saa_event_taskq); 272 273 mutex_destroy(&saa_statep->saa_port_list_mutex); 274 275 kmem_free(saa_statep, sizeof (saa_state_t)); 276 277 bail: 278 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 279 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 280 281 return (ret); 282 } 283 284 /* 285 * ibmf_saa_is_valid 286 * Returns true the entry is valid. 287 * 288 * Input Arguments 289 * saa_portp pointer to state structure 290 * add_ref if B_TRUE ref count is incremented on a valid portp 291 * 292 * Output Arguments 293 * none 294 * 295 * Returns 296 * B_TRUE if entry was in a valid state, B_FALSE otherwise 297 */ 298 boolean_t 299 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 300 { 301 boolean_t is_valid = B_TRUE; 302 303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 304 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 305 306 mutex_enter(&saa_portp->saa_pt_mutex); 307 308 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 309 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 310 311 is_valid = B_FALSE; 312 313 } else if (add_ref == B_TRUE) { 314 /* 315 * increment reference count here to ensure that 316 * entry does not get purged behind our backs 317 */ 318 saa_portp->saa_pt_reference_count++; 319 } 320 mutex_exit(&saa_portp->saa_pt_mutex); 321 322 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 323 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 324 325 return (is_valid); 326 } 327 328 /* 329 * ibmf_saa_must_purge 330 * Determines if we can purge a portp (remove it from the list) based on the 331 * state and number of clients 332 * 333 * Input Arguments 334 * saa_portp pointer to state structure 335 * 336 * Output Arguments 337 * none 338 * 339 * Returns 340 * B_TRUE if the entry can be removed, B_FALSE otherwise 341 */ 342 static int 343 ibmf_saa_must_purge(saa_port_t *saa_portp) 344 { 345 int must_purge = B_FALSE; 346 347 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 348 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 349 350 mutex_enter(&saa_portp->saa_pt_mutex); 351 352 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 353 saa_portp->saa_pt_reference_count == 0) { 354 355 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 356 must_purge = B_TRUE; 357 } 358 359 mutex_exit(&saa_portp->saa_pt_mutex); 360 361 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 362 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 363 364 return (must_purge); 365 } 366 367 368 /* 369 * ibmf_saa_impl_purge: 370 * Removes invalid port state entries from the list 371 * 372 * Input Arguments 373 * none 374 * 375 * Output Arguments 376 * none 377 * 378 * Returns 379 * void 380 */ 381 void 382 ibmf_saa_impl_purge() 383 { 384 saa_port_t *cur_portp = NULL; 385 saa_port_t *prev_portp = NULL; 386 saa_port_t *rem_portp = NULL; 387 388 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 389 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 390 391 mutex_enter(&saa_statep->saa_port_list_mutex); 392 393 cur_portp = saa_statep->saa_port_list; 394 prev_portp = cur_portp; 395 396 while (cur_portp != NULL) { 397 398 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 399 400 rem_portp = cur_portp; 401 402 /* unlink entry */ 403 if (cur_portp == saa_statep->saa_port_list) { 404 405 saa_statep->saa_port_list = cur_portp->next; 406 cur_portp = saa_statep->saa_port_list; 407 prev_portp = cur_portp; 408 409 } else { 410 411 prev_portp->next = cur_portp->next; 412 cur_portp = cur_portp->next; 413 } 414 415 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 416 417 /* destroy entry */ 418 ASSERT(rem_portp != NULL); 419 ibmf_saa_impl_destroy_port(rem_portp); 420 421 } else { 422 423 prev_portp = cur_portp; 424 cur_portp = cur_portp->next; 425 } 426 } 427 428 mutex_exit(&saa_statep->saa_port_list_mutex); 429 430 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 431 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 432 } 433 434 /* 435 * saa_impl_add_client: 436 * Adds a client for a particular portp. Reference count has been incremented 437 * before this call. It is decremented by saa_impl_add_client() if the call 438 * fails. 439 * 440 * Input Arguments 441 * none 442 * 443 * Output Arguments 444 * none 445 * 446 * Returns 447 * IBMF_BUSY if there are already too many clients registered, 448 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 449 * client failed during registration for this port) 450 * IBMF_SUCCESS otherwise 451 */ 452 int 453 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 454 { 455 int status = IBMF_SUCCESS; 456 457 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 458 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 459 460 mutex_enter(&saa_portp->saa_pt_mutex); 461 462 /* 463 * check that we don't exceed max clients 464 */ 465 if (saa_portp->saa_pt_reference_count > 466 SAA_MAX_CLIENTS_PER_PORT) { 467 468 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 469 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 470 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 471 tnf_string, msg, "too many clients registered for" 472 " port", tnf_uint, num_reg_clients, 473 saa_portp->saa_pt_reference_count); 474 475 status = IBMF_BUSY; 476 goto bail; 477 } 478 479 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 480 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 481 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 482 tnf_uint, num_registered_clients, 483 saa_portp->saa_pt_reference_count); 484 485 /* 486 * wait until anyone who is currently registering 487 * this port with ibmf is done 488 */ 489 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 490 491 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 492 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 493 "ibmf_saa_impl_add_client: %s\n", 494 tnf_string, msg, "someone is registering. waiting" 495 " for them to finish"); 496 497 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 498 &saa_portp->saa_pt_mutex); 499 500 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 501 ibmf_saa_impl_add_client, 502 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 503 tnf_string, msg, "done waiting"); 504 } 505 506 /* 507 * if port isn't ready here, fail. 508 */ 509 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 510 511 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 512 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 513 "", "ibmf_saa_impl_add_client: %s\n", 514 tnf_string, msg, "port state not ready," 515 " removing client."); 516 517 status = IBMF_BAD_PORT_STATE; 518 goto bail; 519 } 520 521 bail: 522 mutex_exit(&saa_portp->saa_pt_mutex); 523 524 if (status != IBMF_SUCCESS) { 525 526 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 527 528 IBMF_SAA_ADD32_KSTATS(saa_portp, 529 clients_reg_failed, 1); 530 531 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 532 533 /* decrementing refcount is last thing we do on entry */ 534 535 mutex_enter(&saa_portp->saa_pt_mutex); 536 537 ASSERT(saa_portp->saa_pt_reference_count > 0); 538 saa_portp->saa_pt_reference_count--; 539 540 mutex_exit(&saa_portp->saa_pt_mutex); 541 } 542 543 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 544 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 545 "ibmf_saa_impl_add_client() exit\n"); 546 547 return (status); 548 } 549 550 /* 551 * ibmf_saa_impl_create_port() 552 * Create port entry with mimimal inits because 553 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 554 * 555 * Initialize port state to "registering", so that clients accessing 556 * same port concurrently will wait for the end of the ibmf registration. 557 * Note: this thread will access port members without locking mutex. 558 * 559 * Input Arguments 560 * pt_guid guid of port 561 * 562 * Output Arguments 563 * saa_portpp pointer to new saa_portp structure 564 * 565 * Returns 566 * IBMF_NO_MEMORY if memory could not be allocated 567 * IBMF_SUCCESS otherwise 568 */ 569 int 570 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 571 { 572 int status = IBMF_SUCCESS; 573 saa_port_t *saa_portp = NULL; 574 575 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 576 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 577 " guid %016" PRIx64 "\n", 578 tnf_opaque, port_guid, pt_guid); 579 580 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 581 582 /* create & initialize new port */ 583 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 584 585 if (saa_portp == NULL) { 586 587 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 588 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 589 "ibmf_saa_impl_create_port: %s\n", 590 tnf_string, msg, "could not allocate memory for " 591 "new port"); 592 593 status = IBMF_NO_MEMORY; 594 goto bail; 595 } 596 597 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 598 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 599 tnf_string, msg, "first client registering, initializing"); 600 601 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 602 603 /* tell everyone that kstats are not initialized */ 604 saa_portp->saa_pt_kstatp = NULL; 605 606 /* 607 * set up mutexe and state variable to indicate to 608 * other clients that were currently in the process of 609 * setting up the port data. This will prevent a subsequent 610 * client from trying to to register with ibmf before the 611 * port data has been initialized. 612 */ 613 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 614 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 615 616 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 617 618 /* create other mutexes */ 619 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 620 621 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 622 NULL); 623 624 /* 625 * clients assume all arrive; set mask to this so we only notify 626 * if something failed 627 */ 628 saa_portp->saa_pt_event_sub_last_success_mask = 629 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 630 631 /* 632 * set port_guid now so any immediately subsequent clients 633 * registering on this port, guid will know we're already here 634 */ 635 saa_portp->saa_pt_port_guid = pt_guid; 636 saa_portp->saa_pt_reference_count = 1; 637 saa_portp->saa_pt_current_tid = pt_guid << 32; 638 639 saa_portp->saa_pt_redirect_active = B_FALSE; 640 641 /* set sa_uptime now in case we never receive anything from SA */ 642 saa_portp->saa_pt_sa_uptime = gethrtime(); 643 644 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 645 646 /* Set new pointer in caller's */ 647 *saa_portpp = saa_portp; 648 649 bail: 650 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 652 653 return (status); 654 } 655 656 /* 657 * ibmf_saa_impl_invalidate_port: 658 * invalidates port entry (assumes exist) and deletes kstat object 659 * kstat object is destroyed in order to allow creating port entry 660 * even if this entry is not purged 661 */ 662 static void 663 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 664 { 665 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 666 ibmf_saa_impl_invalidate_port_start, 667 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 668 669 ASSERT(saa_portp != NULL); 670 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 671 672 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 673 ibmf_saa_impl_uninit_kstats(saa_portp); 674 675 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 676 ibmf_saa_impl_invalidate_port_end, 677 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 678 } 679 680 /* 681 * ibmf_saa_impl_destroy_port: 682 * Frees the resources associated with an saa_portp structure. Assumes the 683 * saa_portp exists 684 * 685 * Input Arguments 686 * saa_portp pointer to saa_portp structure 687 * 688 * Output Arguments 689 * none 690 * 691 * Returns 692 * void 693 */ 694 static void 695 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 696 { 697 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 698 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 699 700 ASSERT(saa_portp != NULL); 701 702 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 703 704 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 705 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 706 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 707 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 708 709 ibmf_saa_impl_uninit_kstats(saa_portp); 710 711 /* uninit synchronization variables used for registration */ 712 mutex_destroy(&saa_portp->saa_pt_mutex); 713 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 714 715 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 716 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 717 718 kmem_free(saa_portp, sizeof (saa_port_t)); 719 720 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 721 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 722 } 723 724 /* 725 * ibmf_saa_impl_init_kstats: 726 * Create kstats structure. Should be called when memory is alloced for a new 727 * port entry. 728 */ 729 int 730 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 731 { 732 char buf[128]; 733 ibmf_saa_kstat_t *ksp; 734 735 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 736 737 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 738 ibmf_saa_impl_init_kstats_start, 739 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 740 741 /* set up kstats structure */ 742 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 743 saa_portp->saa_pt_port_guid); 744 745 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 746 0, buf, "misc", KSTAT_TYPE_NAMED, 747 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 748 KSTAT_FLAG_WRITABLE); 749 750 if (saa_portp->saa_pt_kstatp == NULL) 751 return (IBMF_NO_RESOURCES); 752 753 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 754 755 kstat_named_init(&ksp->clients_registered, 756 "clients_registered", KSTAT_DATA_UINT32); 757 758 kstat_named_init(&ksp->clients_reg_failed, 759 "clients_reg_failed", KSTAT_DATA_UINT32); 760 761 kstat_named_init(&ksp->outstanding_requests, 762 "outstanding_requests", KSTAT_DATA_UINT32); 763 764 kstat_named_init(&ksp->total_requests, 765 "total_requests", KSTAT_DATA_UINT32); 766 767 kstat_named_init(&ksp->failed_requests, 768 "failed_requests", KSTAT_DATA_UINT32); 769 770 kstat_named_init(&ksp->requests_timedout, 771 "requests_timedout", KSTAT_DATA_UINT32); 772 773 kstat_install(saa_portp->saa_pt_kstatp); 774 775 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 776 ibmf_saa_impl_init_kstats_end, 777 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 778 779 return (IBMF_SUCCESS); 780 } 781 782 /* 783 * ibmf_saa_impl_uninit_kstats: 784 * Free kstats context. Should be called when port is either destroyed 785 * or invalidated. 786 */ 787 static void 788 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 789 { 790 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 791 ibmf_saa_impl_uninit_kstats_start, 792 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 793 794 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 795 796 if (saa_portp->saa_pt_kstatp != NULL) { 797 kstat_delete(saa_portp->saa_pt_kstatp); 798 } 799 saa_portp->saa_pt_kstatp = NULL; 800 801 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 802 803 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 804 ibmf_saa_impl_uninit_kstats_end, 805 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 806 } 807 808 /* 809 * ibmf_saa_impl_register_failed: 810 * invalidate entry and kick waiters 811 */ 812 void 813 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 814 { 815 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 816 ibmf_saa_impl_register_failed_start, 817 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 818 819 mutex_enter(&saa_portp->saa_pt_mutex); 820 821 ibmf_saa_impl_invalidate_port(saa_portp); 822 823 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 824 825 /* decrementing refcount is last thing we do on entry */ 826 827 ASSERT(saa_portp->saa_pt_reference_count > 0); 828 saa_portp->saa_pt_reference_count--; 829 830 mutex_exit(&saa_portp->saa_pt_mutex); 831 832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 833 ibmf_saa_impl_register_failed_end, 834 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 835 } 836 837 static int 838 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only) 839 { 840 int status; 841 int unreg_status; 842 ib_pkey_t p_key; 843 ib_qkey_t q_key; 844 uint8_t portnum; 845 boolean_t qp_alloced = B_FALSE; 846 847 if (setup_async_cb_only == 0) { 848 849 /* allocate a qp through ibmf */ 850 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 851 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, 852 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle); 853 854 if (status != IBMF_SUCCESS) { 855 856 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 857 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 858 "ibmf_saa_impl_setup_qp_async_cb: %s, " 859 "ibmf_status = %d\n", 860 tnf_string, msg, "Cannot alloc qp with ibmf", 861 tnf_int, status, status); 862 863 return (status); 864 } 865 866 qp_alloced = B_TRUE; 867 868 /* 869 * query the queue pair number; we will need it to unsubscribe 870 * from notice reports 871 */ 872 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 873 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, 874 &p_key, &q_key, &portnum, 0); 875 876 if (status != IBMF_SUCCESS) { 877 878 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 879 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 880 "ibmf_saa_impl_setup_qp_async_cb: %s, " 881 "ibmf_status = %d\n", 882 tnf_string, msg, 883 "Cannot query alt qp to get qp num", 884 tnf_int, status, status); 885 886 goto bail; 887 } 888 } 889 890 /* 891 * core ibmf is taking advantage of the fact that saa_portp is our 892 * callback arg. If this changes, the code in ibmf_recv would need to 893 * change as well 894 */ 895 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 896 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 897 if (status != IBMF_SUCCESS) { 898 899 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 900 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 901 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n", 902 tnf_string, msg, "Cannot register async cb with ibmf", 903 tnf_int, status, status); 904 905 goto bail; 906 } 907 908 return (IBMF_SUCCESS); 909 910 bail: 911 if (qp_alloced == B_TRUE) { 912 /* free alternate qp */ 913 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 914 &saa_portp->saa_pt_qp_handle, 0); 915 if (unreg_status != IBMF_SUCCESS) { 916 917 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 918 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 919 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status =" 920 " %d\n", tnf_string, msg, 921 "Cannot free alternate queue pair with ibmf", 922 tnf_int, unreg_status, unreg_status); 923 } 924 } 925 926 return (status); 927 } 928 929 /* 930 * ibmf_saa_impl_register_port: 931 */ 932 int 933 ibmf_saa_impl_register_port( 934 saa_port_t *saa_portp) 935 { 936 uint_t hca_count = 0; 937 ib_guid_t *hca_list = NULL; 938 int status = IBMF_SUCCESS; 939 int unreg_status = IBMF_SUCCESS; 940 int ibt_status = IBT_SUCCESS; 941 ibt_hca_portinfo_t *port_info_list = NULL; 942 uint_t port_count = 0; 943 uint_t port_size = 0; 944 int ihca, iport; 945 ib_guid_t port_guid; 946 boolean_t ibmf_reg = B_FALSE; 947 948 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 949 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 950 "ibmf_saa_impl_register_port() enter\n"); 951 952 ASSERT(saa_portp != NULL); 953 954 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 955 956 /* get the HCA list */ 957 958 hca_count = ibt_get_hca_list(&hca_list); 959 960 if (hca_count == 0) { 961 962 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 963 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 964 "ibmf_saa_impl_register_port: %s\n", 965 tnf_string, msg, "cannot register port (no HCAs).\n"); 966 967 status = IBMF_BAD_PORT; 968 goto bail; 969 } 970 971 /* lookup requested port guid in hca list */ 972 for (ihca = 0; ihca != hca_count; ihca++) { 973 974 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 975 0 /* all ports */, &port_info_list, 976 &port_count, &port_size); 977 978 if (ibt_status != IBT_SUCCESS) { 979 980 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 981 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 982 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 983 tnf_string, msg, "Could not query hca. Exiting.", 984 tnf_opaque, guid, hca_list[ihca]); 985 986 status = IBMF_TRANSPORT_FAILURE; 987 break; 988 } 989 990 for (iport = 0; iport < port_count; iport++) { 991 992 /* get port guid associated with hca guid, port num */ 993 if (ibmf_saa_impl_get_port_guid( 994 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 995 continue; 996 997 if (saa_portp->saa_pt_port_guid != port_guid) 998 continue; 999 1000 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1001 ibmf_saa_impl_register_port, 1002 IBMF_TNF_TRACE, "", 1003 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 1004 PRIx64 ", port_guid = %016" PRIx64 1005 ", number = %d\n", 1006 tnf_string, msg, "found port", 1007 tnf_opaque, hca_guid, hca_list[ihca], 1008 tnf_opaque, port_guid, port_guid, 1009 tnf_uint, port, iport + 1); 1010 1011 /* 1012 * we're here? then we found our port: 1013 * fill in ibmf registration info 1014 * and address parameters from the portinfo 1015 */ 1016 1017 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 1018 = hca_list[ihca]; 1019 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 1020 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 1021 = SUBN_ADM_MANAGER; 1022 1023 saa_portp->saa_pt_node_guid = hca_list[ihca]; 1024 saa_portp->saa_pt_port_num = iport + 1; 1025 1026 ibmf_saa_impl_set_transaction_params( 1027 saa_portp, port_info_list + iport); 1028 break; 1029 } 1030 1031 ibt_free_portinfo(port_info_list, port_size); 1032 1033 if (iport != port_count) 1034 break; /* found our port */ 1035 } 1036 1037 ibt_free_hca_list(hca_list, hca_count); 1038 1039 if (ihca == hca_count) { 1040 1041 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1042 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1043 "ibmf_saa_impl_register_port: %s, port_guid %016" 1044 PRIx64 "\n", 1045 tnf_string, msg, "Could not find port, exiting", 1046 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 1047 1048 status = IBMF_BAD_PORT; 1049 } 1050 1051 if (status != IBMF_SUCCESS) { 1052 1053 goto bail; 1054 } 1055 1056 /* 1057 * Now we found the port we searched for, 1058 * and open an ibmf session on that port. 1059 */ 1060 1061 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1062 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 1063 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 1064 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 1065 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1066 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1067 1068 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 1069 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 1070 ibmf_saa_impl_async_event_cb, saa_portp, 1071 &saa_portp->saa_pt_ibmf_handle, 1072 &saa_portp->saa_pt_ibmf_impl_features); 1073 1074 if (status != IBMF_SUCCESS) { 1075 1076 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1077 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1078 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1079 tnf_string, msg, "Could not register with ibmf", 1080 tnf_int, status, status); 1081 1082 goto bail; 1083 } 1084 1085 ibmf_reg = B_TRUE; 1086 1087 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS) 1088 return (IBMF_SUCCESS); 1089 1090 bail: 1091 if (ibmf_reg == B_TRUE) { 1092 /* unregister from ibmf */ 1093 unreg_status = ibmf_unregister( 1094 &saa_portp->saa_pt_ibmf_handle, 0); 1095 1096 if (unreg_status != IBMF_SUCCESS) { 1097 1098 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1099 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1100 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1101 " %d\n", tnf_string, msg, 1102 "Cannot unregister from ibmf", 1103 tnf_int, unreg_status, unreg_status); 1104 } 1105 } 1106 1107 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1108 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1109 1110 return (status); 1111 } 1112 1113 /* 1114 * ibmf_saa_impl_getclassportinfo: 1115 */ 1116 void 1117 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1118 { 1119 int res; 1120 saa_impl_trans_info_t *trans_info; 1121 1122 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1123 ibmf_saa_impl_get_classportinfo_start, 1124 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1125 1126 /* 1127 * allocate memory for trans_info; send_request's callback will free up 1128 * memory since request is asynchronous 1129 */ 1130 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1131 if (trans_info == NULL) { 1132 1133 mutex_enter(&saa_portp->saa_pt_mutex); 1134 1135 /* cpi transaction is handled as a client, decrement refcount */ 1136 ASSERT(saa_portp->saa_pt_reference_count > 0); 1137 saa_portp->saa_pt_reference_count--; 1138 1139 mutex_exit(&saa_portp->saa_pt_mutex); 1140 1141 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1142 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1143 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1144 "Could not allocate memory for classportinfo trans_info"); 1145 1146 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1147 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1148 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1149 1150 return; 1151 } 1152 1153 /* no specific client associated with this transaction */ 1154 trans_info->si_trans_client_data = NULL; 1155 trans_info->si_trans_port = saa_portp; 1156 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1157 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1158 1159 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1160 trans_info->si_trans_callback_arg = saa_portp; 1161 1162 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1163 1164 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1165 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1166 1167 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1168 1169 res = ibmf_saa_impl_send_request(trans_info); 1170 1171 if (res != IBMF_SUCCESS) { 1172 1173 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1174 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1175 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1176 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1177 tnf_opaque, res, res); 1178 1179 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1180 1181 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1182 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1183 1184 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1185 1186 mutex_enter(&saa_portp->saa_pt_mutex); 1187 1188 /* cpi transaction is handled as a client, decrement refcount */ 1189 ASSERT(saa_portp->saa_pt_reference_count > 0); 1190 saa_portp->saa_pt_reference_count--; 1191 1192 mutex_exit(&saa_portp->saa_pt_mutex); 1193 1194 } 1195 1196 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1197 ibmf_saa_impl_get_classportinfo_end, 1198 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1199 } 1200 1201 /* 1202 * ibmf_saa_impl_get_cpi_cb: 1203 * 1204 * Called when the asynchronous getportinfo request receives its response. 1205 * Checks the status. If success, updates the times in the port's 1206 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1207 * just use default values. 1208 * 1209 * Input Arguments 1210 * arg user-specified pointer (points to the current port data) 1211 * length length of payload returned (should be size of classportinfo_t) 1212 * buffer pointer to classportinfo returned (should not be null) 1213 * status status of sa access request 1214 * 1215 * Output Arguments 1216 * none 1217 * 1218 * Returns void 1219 */ 1220 static void 1221 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1222 { 1223 saa_port_t *saa_portp; 1224 uint64_t base_time, resp_timeout, rttv_timeout; 1225 ib_mad_classportinfo_t *classportinfo; 1226 int resp_time_value; 1227 uint16_t sa_cap_mask; 1228 1229 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1231 1232 /* 1233 * access port entry: note that it may have become invalid 1234 * but we hold a ref count for cpi and the interactions on 1235 * the entry are harmless 1236 */ 1237 saa_portp = (saa_port_t *)arg; 1238 1239 /* process response */ 1240 1241 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1242 1243 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1244 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1245 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1246 " 0x%p, length = %d\n", tnf_string, msg, 1247 "could not get classportinfo. Check node and path to sm" 1248 " lid", tnf_int, status, status, 1249 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1250 1251 /* 1252 * IB spec (C13-13) indicates 20 can be used as default or 1253 * intial value for classportinfo->resptimeout value 1254 */ 1255 resp_time_value = 20; 1256 1257 sa_cap_mask = 0xFFFF; 1258 1259 } else if (buffer != NULL) { 1260 1261 classportinfo = (ib_mad_classportinfo_t *)buffer; 1262 1263 resp_time_value = classportinfo->RespTimeValue & 0x1f; 1264 1265 /* 1266 * Because some subnet managers might not provide sane 1267 * value for "resp_time_value", we limit it here. In 1268 * case this limit is too restrictive (very large fabric), 1269 * we allow the limit to be raised (/etc/system). 1270 */ 1271 if (resp_time_value > ibmf_saa_max_resp_time) { 1272 cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) " 1273 "exceeded.", ibmf_saa_max_resp_time); 1274 cmn_err(CE_CONT, "!Reducing subnet administrator " 1275 "resp_time value from %d to %d.", 1276 resp_time_value, ibmf_saa_max_resp_time); 1277 resp_time_value = ibmf_saa_max_resp_time; 1278 } 1279 1280 sa_cap_mask = classportinfo->CapabilityMask; 1281 1282 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1283 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1284 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1285 " cap_mask = 0x%x\n", 1286 tnf_string, msg, "got classportinfo", 1287 tnf_opaque, timeout, resp_time_value, 1288 tnf_opaque, cap_mask, sa_cap_mask); 1289 1290 kmem_free(buffer, length); 1291 } 1292 1293 /* 1294 * using IB spec calculation from 13.4.6.2 1295 * use bit shifting for 2^x. 1296 */ 1297 base_time = (1 << resp_time_value); 1298 1299 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1300 1301 mutex_enter(&saa_portp->saa_pt_mutex); 1302 1303 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1304 1305 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1306 1307 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1308 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1309 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1310 1311 /* 1312 * cpi transaction is handled as a client, 1313 * decrement refcount; make sure it's the last 1314 * thing we do on this entry 1315 */ 1316 ASSERT(saa_portp->saa_pt_reference_count > 0); 1317 saa_portp->saa_pt_reference_count--; 1318 1319 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1320 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1321 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1322 "resp_time_value = 0x%x\n", 1323 tnf_string, msg, "updated resp timeout", 1324 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1325 tnf_opaque, resp_time_value, resp_time_value); 1326 1327 mutex_exit(&saa_portp->saa_pt_mutex); 1328 1329 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1330 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1331 } 1332 1333 /* 1334 * ibmf_saa_impl_send_request: 1335 * Sends a request to the sa. Can be used for both classportinfo and record 1336 * requests. Will set up all data structures for using the multi-packet 1337 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1338 * worked, meaning succesful send for the async case and a succesful send and 1339 * recv for the sync case. 1340 */ 1341 int 1342 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1343 { 1344 uint16_t attr_id; 1345 saa_client_data_t *client_data; 1346 saa_port_t *saa_portp; 1347 uint32_t transport_flags; 1348 ibmf_msg_cb_t ibmf_callback; 1349 void *ibmf_callback_arg; 1350 ibmf_msg_t *msgp; 1351 ibmf_retrans_t ibmf_retrans; 1352 uint16_t sa_cap_mask; 1353 boolean_t sleep_flag; 1354 int ibmf_status = IBMF_SUCCESS; 1355 int retry_count; 1356 uint16_t mad_status; 1357 boolean_t sa_is_redirected = B_FALSE; 1358 1359 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1360 ibmf_saa_impl_send_request_start, 1361 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1362 1363 attr_id = trans_info->si_trans_attr_id; 1364 client_data = trans_info->si_trans_client_data; 1365 saa_portp = trans_info->si_trans_port; 1366 1367 /* 1368 * don't send on invalid entry 1369 * Note that there is a window where it could become 1370 * invalid after this test is done, but we'd rely on ibmf errors... 1371 */ 1372 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1373 1374 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1375 ibmf_saa_impl_send_request, 1376 IBMF_TNF_ERROR, "", 1377 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1378 PRIx64 ", port_guid = %016" PRIx64 1379 ", number = %d\n", 1380 tnf_string, msg, "sending on invalid port", 1381 tnf_opaque, hca_guid, 1382 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1383 tnf_opaque, port_guid, 1384 saa_portp->saa_pt_port_guid, 1385 tnf_uint, port, 1386 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1387 1388 ibmf_status = IBMF_REQ_INVALID; 1389 goto bail; 1390 } 1391 1392 /* check whether SA supports this attribute */ 1393 mutex_enter(&saa_portp->saa_pt_mutex); 1394 1395 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1396 sa_is_redirected = saa_portp->saa_pt_redirect_active; 1397 1398 mutex_exit(&saa_portp->saa_pt_mutex); 1399 1400 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1401 1402 if (ibmf_status != IBMF_SUCCESS) { 1403 1404 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1405 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1406 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1407 tnf_string, msg, "SA does not support attribute", 1408 tnf_int, ibmf_status, ibmf_status); 1409 1410 goto bail; 1411 } 1412 1413 /* make only non-blocking calls if this is an async request */ 1414 if ((trans_info->si_trans_callback == NULL) && 1415 (trans_info->si_trans_sub_callback == NULL)) { 1416 ibmf_callback = NULL; 1417 ibmf_callback_arg = NULL; 1418 sleep_flag = B_TRUE; 1419 } else { 1420 ibmf_callback = ibmf_saa_async_cb; 1421 ibmf_callback_arg = (void *)trans_info; 1422 sleep_flag = B_FALSE; 1423 } 1424 1425 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1426 &transport_flags, &ibmf_retrans); 1427 if (ibmf_status != IBMF_SUCCESS) { 1428 1429 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1430 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1431 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1432 tnf_string, msg, "init_msg() failed", 1433 tnf_int, ibmf_status, ibmf_status); 1434 1435 goto bail; 1436 } 1437 1438 mutex_enter(&saa_portp->saa_pt_mutex); 1439 1440 saa_portp->saa_pt_num_outstanding_trans++; 1441 1442 mutex_exit(&saa_portp->saa_pt_mutex); 1443 1444 /* 1445 * increment the number of outstanding transaction so 1446 * ibmf_close_sa_session() will wait. classportinfo requests 1447 * don't have associated clients so check for valid clientp 1448 */ 1449 if (client_data != NULL) { 1450 1451 mutex_enter(&client_data->saa_client_mutex); 1452 1453 client_data->saa_client_num_pending_trans++; 1454 1455 mutex_exit(&client_data->saa_client_mutex); 1456 } 1457 1458 /* 1459 * make the call to msg_transport. If synchronous and success, 1460 * check that the response mad isn't status busy. If so, repeat the 1461 * call 1462 */ 1463 retry_count = 0; 1464 1465 /* 1466 * set the send time here. We only set this once at the beginning of 1467 * the transaction. Retrying because of busys or mastersmlid changes 1468 * does not change the original send time. It is meant to be an 1469 * absolute time out value and will only be used if there are other 1470 * problems (i.e. a buggy SA) 1471 */ 1472 trans_info->si_trans_send_time = gethrtime(); 1473 1474 for (;;) { 1475 1476 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1477 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1478 ibmf_callback, ibmf_callback_arg, transport_flags); 1479 1480 if (ibmf_callback != NULL) 1481 break; 1482 1483 /* 1484 * stop here for non-sequenced transactions since they wouldn't 1485 * receive a timeout or busy response 1486 */ 1487 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1488 break; 1489 1490 /* 1491 * if the transaction timed out and this was a synchronous 1492 * request there's a possiblity we were talking to the wrong 1493 * master smlid or that the SA has stopped responding on the 1494 * redirected desination (if redirect is active). 1495 * Check this and retry if necessary. 1496 */ 1497 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1498 (sleep_flag == B_TRUE)) { 1499 if (sa_is_redirected == B_TRUE) { 1500 ibmf_status = ibmf_saa_impl_revert_to_qp1( 1501 saa_portp, msgp, ibmf_callback, 1502 ibmf_callback_arg, transport_flags); 1503 } else { 1504 ibmf_status = ibmf_saa_impl_new_smlid_retry( 1505 saa_portp, msgp, ibmf_callback, 1506 ibmf_callback_arg, transport_flags); 1507 } 1508 } 1509 1510 /* 1511 * if the transaction timed out (and retrying with a new SM LID 1512 * didn't help) check how long it's been since we received an SA 1513 * packet. If it hasn't been max_wait_time then retry the 1514 * request. 1515 */ 1516 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1517 (sleep_flag == B_TRUE)) { 1518 1519 ibmf_status = ibmf_saa_check_sa_and_retry( 1520 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1521 trans_info->si_trans_send_time, transport_flags); 1522 } 1523 1524 if (ibmf_status != IBMF_SUCCESS) 1525 break; 1526 1527 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1528 break; 1529 1530 /* sync transaction with status SUCCESS should have response */ 1531 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1532 1533 mad_status = b2h16(msgp->im_msgbufs_recv. 1534 im_bufs_mad_hdr->Status); 1535 1536 if ((mad_status != MAD_STATUS_BUSY) && 1537 (mad_status != MAD_STATUS_REDIRECT_REQUIRED)) 1538 break; 1539 1540 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) { 1541 1542 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1543 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1544 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1545 tnf_string, msg, 1546 "response returned redirect status", 1547 tnf_int, retry_count, retry_count); 1548 1549 /* update address info and copy it into msgp */ 1550 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 1551 } else { 1552 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1553 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1554 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1555 tnf_string, msg, "response returned busy status", 1556 tnf_int, retry_count, retry_count); 1557 } 1558 1559 retry_count++; 1560 1561 /* 1562 * since this is a blocking call, sleep for some time 1563 * to allow SA to transition from busy state (if busy) 1564 */ 1565 if (mad_status == MAD_STATUS_BUSY) 1566 delay(drv_usectohz( 1567 IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000)); 1568 } 1569 1570 if (ibmf_status != IBMF_SUCCESS) { 1571 1572 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1573 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1574 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1575 tnf_string, msg, "ibmf_msg_transport() failed", 1576 tnf_int, ibmf_status, ibmf_status); 1577 1578 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1579 1580 mutex_enter(&saa_portp->saa_pt_mutex); 1581 1582 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1583 saa_portp->saa_pt_num_outstanding_trans--; 1584 1585 mutex_exit(&saa_portp->saa_pt_mutex); 1586 1587 if (client_data != NULL) { 1588 1589 mutex_enter(&client_data->saa_client_mutex); 1590 1591 ASSERT(client_data->saa_client_num_pending_trans > 0); 1592 client_data->saa_client_num_pending_trans--; 1593 1594 if ((client_data->saa_client_num_pending_trans == 0) && 1595 (client_data->saa_client_state == 1596 SAA_CLIENT_STATE_WAITING)) 1597 cv_signal(&client_data->saa_client_state_cv); 1598 1599 mutex_exit(&client_data->saa_client_mutex); 1600 } 1601 1602 } else if (sleep_flag == B_TRUE) { 1603 1604 mutex_enter(&saa_portp->saa_pt_mutex); 1605 1606 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1607 saa_portp->saa_pt_num_outstanding_trans--; 1608 1609 mutex_exit(&saa_portp->saa_pt_mutex); 1610 1611 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1612 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1613 "ibmf_saa_impl_send_request: %s\n", 1614 tnf_string, msg, "Message sent and received successfully"); 1615 1616 /* fill in response values and free the message */ 1617 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1618 msgp, B_FALSE, &trans_info->si_trans_status, 1619 &trans_info->si_trans_result, 1620 &trans_info->si_trans_length, sleep_flag); 1621 1622 if (client_data != NULL) { 1623 mutex_enter(&client_data->saa_client_mutex); 1624 1625 ASSERT(client_data->saa_client_num_pending_trans > 0); 1626 client_data->saa_client_num_pending_trans--; 1627 1628 if ((client_data->saa_client_num_pending_trans == 0) && 1629 (client_data->saa_client_state == 1630 SAA_CLIENT_STATE_WAITING)) 1631 cv_signal(&client_data->saa_client_state_cv); 1632 1633 mutex_exit(&client_data->saa_client_mutex); 1634 } 1635 } else { 1636 1637 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1638 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1639 "ibmf_saa_impl_send_request: %s\n", 1640 tnf_string, msg, "Message sent successfully"); 1641 } 1642 1643 bail: 1644 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1645 ibmf_saa_impl_send_request_end, 1646 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1647 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1648 1649 return (ibmf_status); 1650 } 1651 1652 /* 1653 * ibmf_saa_impl_init_msg: 1654 * Allocates an ibmf message and fills out the header fields and formatted data 1655 * fields. Also sets up the correct transport_flags and retrans argument for 1656 * the message transport call based on the request information. 1657 * 1658 * Input Arguments 1659 * trans_info saa_trans_info structure passed to send_request 1660 * sleep_flag B_TRUE if init_msg can sleep in function calls 1661 * 1662 * Output Arguments 1663 * msgp ibmf message that should be given to msg_transport 1664 * transport_flagsp transport flags that should be given to msg_transport 1665 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1666 * 1667 * Returns 1668 * ibmf_status 1669 */ 1670 static int 1671 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1672 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1673 ibmf_retrans_t *ibmf_retransp) 1674 { 1675 int ibmf_status; 1676 ibmf_msg_bufs_t *req_mad; 1677 ib_mad_hdr_t *mad_hdr; 1678 int ibmf_sleep_flag, km_sleep_flag; 1679 int free_res; 1680 ib_sa_hdr_t sa_hdr; 1681 ibmf_msg_t *ibmf_msg; 1682 uint16_t attr_id, pack_attr_id; 1683 uint8_t method; 1684 saa_client_data_t *client_data; 1685 saa_port_t *saa_portp; 1686 sa_multipath_record_t *multipath_template; 1687 size_t payload_length; 1688 uint32_t transport_flags; 1689 1690 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1691 ibmf_saa_impl_init_msg_start, 1692 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1693 1694 attr_id = trans_info->si_trans_attr_id; 1695 method = trans_info->si_trans_method; 1696 client_data = trans_info->si_trans_client_data; 1697 saa_portp = trans_info->si_trans_port; 1698 1699 if (sleep_flag == B_TRUE) { 1700 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1701 km_sleep_flag = KM_SLEEP; 1702 } else { 1703 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1704 km_sleep_flag = KM_NOSLEEP; 1705 } 1706 1707 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1708 ibmf_sleep_flag, &ibmf_msg); 1709 if (ibmf_status != IBMF_SUCCESS) { 1710 1711 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1712 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1713 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1714 tnf_string, msg, "Cannot allocate msg_buf.", 1715 tnf_int, ibmf_status, ibmf_status); 1716 1717 goto bail; 1718 } 1719 1720 req_mad = &ibmf_msg->im_msgbufs_send; 1721 1722 /* create a template (SA MAD) */ 1723 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1724 1725 if (mad_hdr == NULL) { 1726 1727 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1728 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1729 "ibmf_saa_impl_init_msg: %s\n", 1730 tnf_string, msg, "Cannot allocate mad header."); 1731 1732 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1733 &ibmf_msg); 1734 ASSERT(free_res == IBMF_SUCCESS); 1735 1736 ibmf_status = IBMF_NO_MEMORY; 1737 goto bail; 1738 } 1739 1740 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1741 *req_mad)) 1742 1743 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1744 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1745 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1746 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1747 mad_hdr->R_Method = method; 1748 mad_hdr->AttributeID = h2b16(attr_id); 1749 1750 /* attribute modifier is all Fs since RIDs are no longer used */ 1751 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1752 1753 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1754 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1755 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1756 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1757 tnf_opaque, class, mad_hdr->MgmtClass, 1758 tnf_opaque, method, mad_hdr->R_Method, 1759 tnf_opaque, attr_id, attr_id); 1760 1761 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1762 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1763 1764 if (client_data != NULL) 1765 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1766 1767 /* 1768 * pack data for IB wire format; req_mad will have different pointers to 1769 * sa header and payload, mad_hdr will be the same 1770 */ 1771 req_mad->im_bufs_mad_hdr = mad_hdr; 1772 1773 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1774 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1775 km_sleep_flag); 1776 1777 if (ibmf_status != IBMF_SUCCESS) { 1778 1779 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1780 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1781 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1782 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1783 tnf_int, ibmf_status, ibmf_status); 1784 1785 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1786 1787 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1788 &ibmf_msg); 1789 ASSERT(free_res == IBMF_SUCCESS); 1790 1791 goto bail; 1792 } 1793 1794 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1795 1796 multipath_template = 1797 (sa_multipath_record_t *)trans_info->si_trans_template; 1798 1799 payload_length = sizeof (sa_multipath_record_t) + 1800 ((multipath_template->SGIDCount + 1801 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1802 1803 pack_attr_id = attr_id; 1804 } else { 1805 1806 /* trace record template is a path record */ 1807 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1808 SA_PATHRECORD_ATTRID : attr_id; 1809 1810 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1811 1812 if (payload_length == 0) { 1813 payload_length = trans_info->si_trans_template_length; 1814 1815 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1816 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1817 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1818 tnf_string, msg, 1819 "Unknown attribute. Using user-defined length.", 1820 tnf_uint, length, payload_length) 1821 } 1822 } 1823 1824 /* transport type depends on method */ 1825 switch (method) { 1826 1827 case SA_SUBN_ADM_GET: 1828 case SA_SUBN_ADM_DELETE: 1829 case SA_SUBN_ADM_GET_TABLE: 1830 case SA_SUBN_ADM_GET_TRACE_TABLE: 1831 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1832 break; 1833 case SA_SUBN_ADM_SET: 1834 /* unsubscribes can be sequenced or unsequenced */ 1835 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1836 transport_flags = 0; 1837 } else { 1838 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1839 } 1840 break; 1841 case SA_SUBN_ADM_GET_MULTI: 1842 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1843 IBMF_MSG_TRANS_FLAG_RMPP; 1844 break; 1845 default : 1846 ibmf_status = IBMF_UNSUPP_METHOD; 1847 goto bail; 1848 } 1849 1850 trans_info->si_trans_transport_flags = transport_flags; 1851 1852 if (trans_info->si_trans_template != NULL) { 1853 1854 ibmf_status = ibmf_saa_utils_pack_payload( 1855 trans_info->si_trans_template, payload_length, pack_attr_id, 1856 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1857 km_sleep_flag); 1858 if (ibmf_status != IBMF_SUCCESS) { 1859 1860 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1861 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1862 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1863 " %d\n", tnf_string, msg, 1864 "ibmf_saa_utils_pack_payload() failed", 1865 tnf_int, ibmf_status, ibmf_status); 1866 1867 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1868 1869 kmem_free(req_mad->im_bufs_cl_hdr, 1870 req_mad->im_bufs_cl_hdr_len); 1871 1872 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1873 &ibmf_msg); 1874 ASSERT(free_res == IBMF_SUCCESS); 1875 1876 goto bail; 1877 } 1878 1879 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1880 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1881 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1882 " %d\n", tnf_string, msg, "Packed payload successfully", 1883 tnf_opaque, attr_id, attr_id, 1884 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1885 1886 /* non-RMPP transactions have template size limit */ 1887 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1888 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1889 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1890 1891 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1892 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1893 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1894 "Template too large to fit in single packet"); 1895 1896 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1897 1898 kmem_free(req_mad->im_bufs_cl_hdr, 1899 req_mad->im_bufs_cl_hdr_len); 1900 1901 kmem_free(req_mad->im_bufs_cl_data, 1902 req_mad->im_bufs_cl_data_len); 1903 1904 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1905 &ibmf_msg); 1906 ASSERT(free_res == IBMF_SUCCESS); 1907 1908 ibmf_status = IBMF_REQ_INVALID; 1909 goto bail; 1910 } 1911 } 1912 1913 mutex_enter(&saa_portp->saa_pt_mutex); 1914 1915 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1916 1917 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1918 sizeof (ibmf_retrans_t)); 1919 1920 /* copy local addressing information to message */ 1921 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1922 sizeof (ibmf_addr_info_t)); 1923 1924 /* copy global addressing information to message if in use */ 1925 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 1926 1927 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 1928 1929 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 1930 &ibmf_msg->im_global_addr, 1931 sizeof (ibmf_global_addr_info_t)); 1932 } else { 1933 ibmf_msg->im_msg_flags = 0; 1934 } 1935 1936 mutex_exit(&saa_portp->saa_pt_mutex); 1937 1938 *msgp = ibmf_msg; 1939 *transport_flagsp = transport_flags; 1940 bail: 1941 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1942 ibmf_saa_impl_init_msg_end, 1943 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1944 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1945 1946 return (ibmf_status); 1947 1948 } 1949 1950 /* 1951 * ibmf_saa_impl_new_smlid_retry: 1952 * 1953 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1954 * MasterSMLID is set when we first register with ibmf_saa. If a request 1955 * timesout, this function should be called to check whether the SM LID changed. 1956 * If so, it will call msg_transport again with the request. 1957 * 1958 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1959 * same values passed to the original ibmf_msg_transport that timedout. The 1960 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1961 * 1962 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1963 * That way, callers can simply return the result of this function. 1964 * 1965 * Input Arguments 1966 * saa_portp pointer to saa_port structure 1967 * msgp ibmf message that timedout 1968 * ibmf_callback callback that should be called by msg_transport 1969 * ibmf_callback_arg args for ibmf_callback 1970 * transport_flags flags for ibmf_msg_transport 1971 * 1972 * Output Arguments 1973 * none 1974 * 1975 * Returns 1976 * IBMF_SUCCESS if lid changed and request was resent successfully, 1977 * IBMF_TRANS_TIMEOUT if lid did not change, 1978 * same values as ibmf_msg_transport() if lid changed but request could not be 1979 * resent. 1980 */ 1981 static int 1982 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1983 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1984 { 1985 ibt_hca_portinfo_t *ibt_portinfop; 1986 ib_lid_t master_sm_lid; 1987 int subnet_timeout; 1988 uint_t nports, size; 1989 ibmf_retrans_t ibmf_retrans; 1990 int ibmf_status; 1991 ibt_status_t ibt_status; 1992 1993 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1994 ibmf_saa_impl_new_smlid_retry_start, 1995 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1996 1997 _NOTE(ASSUMING_PROTECTED(*msgp)) 1998 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1999 2000 /* first query the portinfo to see if the lid changed */ 2001 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2002 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2003 2004 if (ibt_status != IBT_SUCCESS) { 2005 2006 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2007 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 2008 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 2009 " %d\n", tnf_string, msg, 2010 "ibt_query_hca_ports_byguid() failed", 2011 tnf_int, ibt_status, ibt_status); 2012 2013 ibmf_status = IBMF_TRANSPORT_FAILURE; 2014 2015 goto bail; 2016 } 2017 2018 master_sm_lid = ibt_portinfop->p_sm_lid; 2019 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2020 2021 ibt_free_portinfo(ibt_portinfop, size); 2022 2023 /* if master smlid is different than the remote lid we sent to */ 2024 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 2025 2026 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 2027 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2028 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 2029 " old_lid 0x%x\n", tnf_string, msg, 2030 "master smlid has changed. retrying msg_transport", 2031 tnf_opaque, new_lid, master_sm_lid, 2032 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 2033 2034 mutex_enter(&saa_portp->saa_pt_mutex); 2035 2036 /* update the master sm lid value in ibmf_saa */ 2037 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2038 master_sm_lid; 2039 2040 /* new tid needed */ 2041 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2042 h2b64(saa_portp->saa_pt_current_tid++); 2043 2044 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2045 sizeof (ibmf_retrans_t)); 2046 2047 /* update the subnet timeout since this may be a new sm/sa */ 2048 saa_portp->saa_pt_timeout = subnet_timeout; 2049 2050 /* place upper bound on subnet timeout in case of faulty SM */ 2051 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2052 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2053 2054 /* increment the reference count to account for the cpi call */ 2055 saa_portp->saa_pt_reference_count++; 2056 2057 mutex_exit(&saa_portp->saa_pt_mutex); 2058 2059 /* update the remote lid for this particular message */ 2060 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 2061 2062 /* get the classportinfo again since this may be a new sm/sa */ 2063 ibmf_saa_impl_get_classportinfo(saa_portp); 2064 2065 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2066 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2067 ibmf_callback, ibmf_callback_arg, transport_flags); 2068 2069 if (ibmf_status != IBMF_SUCCESS) { 2070 2071 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2072 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2073 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 2074 "%d\n", tnf_string, msg, 2075 "ibmf_msg_transport() failed", 2076 tnf_int, ibmf_status, ibmf_status); 2077 } 2078 2079 goto bail; 2080 } 2081 2082 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2083 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2084 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2085 tnf_string, msg, 2086 "master smlid did not change. returning failure", 2087 tnf_opaque, master_smlid, master_sm_lid); 2088 2089 /* mark status as timeout since that was original failure */ 2090 ibmf_status = IBMF_TRANS_TIMEOUT; 2091 2092 bail: 2093 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2094 ibmf_saa_impl_new_smlid_retry_end, 2095 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2096 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2097 2098 return (ibmf_status); 2099 } 2100 2101 /* 2102 * ibmf_saa_impl_revert_to_qp1() 2103 * 2104 * The SA that we had contact with via redirect may fail to respond. If this 2105 * occurs SA should revert back to qp1 and the SMLID set in the port. 2106 * msg_transport for the message that timed out will be retried with 2107 * these new parameters. 2108 * 2109 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2110 * same values passed to the original ibmf_msg_transport that timedout. The 2111 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2112 * 2113 * Input Arguments 2114 * saa_portp pointer to saa_port structure 2115 * msgp ibmf message that timedout 2116 * ibmf_callback callback that should be called by msg_transport 2117 * ibmf_callback_arg args for ibmf_callback 2118 * transport_flags flags for ibmf_msg_transport 2119 * 2120 * Output Arguments 2121 * none 2122 * 2123 * Returns 2124 * none 2125 */ 2126 static int 2127 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2128 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags) 2129 { 2130 ibt_hca_portinfo_t *ibt_portinfop; 2131 ib_lid_t master_sm_lid, base_lid; 2132 uint8_t sm_sl; 2133 int subnet_timeout; 2134 uint_t nports, size; 2135 ibmf_retrans_t ibmf_retrans; 2136 int ibmf_status; 2137 ibt_status_t ibt_status; 2138 2139 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2140 ibmf_saa_impl_revert_to_qp1_start, 2141 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n"); 2142 2143 _NOTE(ASSUMING_PROTECTED(*msgp)) 2144 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 2145 2146 /* first query the portinfo to see if the lid changed */ 2147 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2148 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2149 2150 if (ibt_status != IBT_SUCCESS) { 2151 2152 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2153 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "", 2154 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status =" 2155 " %d\n", tnf_string, msg, 2156 "ibt_query_hca_ports_byguid() failed", 2157 tnf_int, ibt_status, ibt_status); 2158 2159 ibmf_status = IBMF_TRANSPORT_FAILURE; 2160 2161 goto bail; 2162 } 2163 2164 master_sm_lid = ibt_portinfop->p_sm_lid; 2165 base_lid = ibt_portinfop->p_base_lid; 2166 sm_sl = ibt_portinfop->p_sm_sl; 2167 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2168 2169 ibt_free_portinfo(ibt_portinfop, size); 2170 2171 2172 mutex_enter(&saa_portp->saa_pt_mutex); 2173 2174 saa_portp->saa_pt_redirect_active = B_FALSE; 2175 2176 /* update the address info in ibmf_saa */ 2177 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid; 2178 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid; 2179 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl; 2180 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 2181 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED; 2182 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 2183 saa_portp->saa_pt_ibmf_msg_flags = 0; 2184 2185 /* new tid needed */ 2186 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2187 h2b64(saa_portp->saa_pt_current_tid++); 2188 2189 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2190 sizeof (ibmf_retrans_t)); 2191 2192 /* update the subnet timeout since this may be a new sm/sa */ 2193 saa_portp->saa_pt_timeout = subnet_timeout; 2194 2195 /* place upper bound on subnet timeout in case of faulty SM */ 2196 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2197 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2198 2199 /* increment the reference count to account for the cpi call */ 2200 saa_portp->saa_pt_reference_count++; 2201 2202 mutex_exit(&saa_portp->saa_pt_mutex); 2203 2204 /* update the address info for this particular message */ 2205 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 2206 sizeof (ibmf_addr_info_t)); 2207 msgp->im_msg_flags = 0; /* No GRH */ 2208 2209 /* get the classportinfo again since this may be a new sm/sa */ 2210 ibmf_saa_impl_get_classportinfo(saa_portp); 2211 2212 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2213 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2214 ibmf_callback, ibmf_callback_args, transport_flags); 2215 2216 if (ibmf_status != IBMF_SUCCESS) { 2217 2218 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2219 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "", 2220 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = " 2221 "%d\n", tnf_string, msg, 2222 "ibmf_msg_transport() failed", 2223 tnf_int, ibmf_status, ibmf_status); 2224 } 2225 2226 bail: 2227 2228 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2229 ibmf_saa_impl_revert_to_qp1_end, 2230 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting" 2231 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2232 2233 return (ibmf_status); 2234 } 2235 2236 /* 2237 * ibmf_saa_impl_async_event_cb: 2238 * ibmf event callback, argument to ibmf_register 2239 * ibmf_handle is unused 2240 */ 2241 /* ARGSUSED */ 2242 static void 2243 ibmf_saa_impl_async_event_cb( 2244 ibmf_handle_t ibmf_handle, 2245 void *clnt_private, 2246 ibmf_async_event_t event_type) 2247 { 2248 saa_port_t *saa_portp; 2249 2250 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2251 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2252 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2253 tnf_opaque, event_type, event_type); 2254 2255 saa_portp = (saa_port_t *)clnt_private; 2256 ASSERT(saa_portp != NULL); 2257 2258 switch (event_type) { 2259 2260 case IBMF_CI_OFFLINE: 2261 ibmf_saa_impl_hca_detach(saa_portp); 2262 break; 2263 default: 2264 break; 2265 } 2266 2267 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2268 ibmf_saa_impl_async_event_cb_end, 2269 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2270 } 2271 2272 2273 /* 2274 * ibmf_saa_impl_ibt_async_handler: 2275 * MUST NOT BE STATIC (referred from within IBMF) 2276 */ 2277 void 2278 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2279 { 2280 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2281 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2282 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2283 tnf_opaque, code, code); 2284 2285 switch (code) { 2286 2287 case IBT_EVENT_PORT_UP: 2288 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2289 break; 2290 case IBT_ERROR_PORT_DOWN: 2291 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2292 break; 2293 case IBT_PORT_CHANGE_EVENT: 2294 ibmf_saa_impl_port_chg(event); 2295 break; 2296 case IBT_CLNT_REREG_EVENT: 2297 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port); 2298 break; 2299 default: 2300 break; 2301 } 2302 2303 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2304 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2305 } 2306 2307 /* 2308 * ibmf_saa_impl_port_chg: 2309 */ 2310 static void 2311 ibmf_saa_impl_port_chg(ibt_async_event_t *event) 2312 { 2313 saa_port_t *saa_portp = NULL; 2314 boolean_t is_ready = B_FALSE; 2315 ibt_hca_portinfo_t *ibt_portinfop; 2316 uint_t nports, size; 2317 ibt_status_t ibt_status; 2318 ib_guid_t ci_guid; 2319 int port_num; 2320 2321 ci_guid = event->ev_hca_guid; 2322 port_num = event->ev_port; 2323 2324 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start, 2325 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg" 2326 " guid %016" PRIx64 " port %d\n", 2327 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2328 2329 /* Get classportinfo of corresponding entry */ 2330 mutex_enter(&saa_statep->saa_port_list_mutex); 2331 2332 saa_portp = saa_statep->saa_port_list; 2333 while (saa_portp != NULL) { 2334 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2335 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2336 mutex_enter(&saa_portp->saa_pt_mutex); 2337 2338 is_ready = (saa_portp->saa_pt_state 2339 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2340 2341 /* 2342 * increment reference count to account for cpi and 2343 * informinfos. All 4 informinfo's sent are treated as 2344 * one port client reference 2345 */ 2346 if (is_ready) 2347 saa_portp->saa_pt_reference_count ++; 2348 2349 mutex_exit(&saa_portp->saa_pt_mutex); 2350 2351 if (is_ready) 2352 break; /* normally, only 1 port entry */ 2353 } 2354 saa_portp = saa_portp->next; 2355 } 2356 2357 mutex_exit(&saa_statep->saa_port_list_mutex); 2358 2359 if (saa_portp != NULL) { 2360 /* first query the portinfo to see if the lid changed */ 2361 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2362 &ibt_portinfop, &nports, &size); 2363 2364 if (ibt_status != IBT_SUCCESS) { 2365 2366 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2367 ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "", 2368 "ibmf_saa_impl_port_chg: %s, ibmf_status =" 2369 " %d\n", tnf_string, msg, 2370 "ibt_query_hca_ports_byguid() failed", 2371 tnf_int, ibt_status, ibt_status); 2372 2373 goto bail; 2374 } 2375 2376 mutex_enter(&saa_portp->saa_pt_mutex); 2377 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) { 2378 /* update the Master SM Lid value in ibmf_saa */ 2379 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2380 ibt_portinfop->p_sm_lid; 2381 } 2382 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) { 2383 /* update the Master SM SL value in ibmf_saa */ 2384 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = 2385 ibt_portinfop->p_sm_sl; 2386 } 2387 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) { 2388 /* update the Subnet timeout value in ibmf_saa */ 2389 saa_portp->saa_pt_timeout = 2390 ibt_portinfop->p_subnet_timeout; 2391 } 2392 mutex_exit(&saa_portp->saa_pt_mutex); 2393 2394 ibt_free_portinfo(ibt_portinfop, size); 2395 2396 /* get the classportinfo again */ 2397 ibmf_saa_impl_get_classportinfo(saa_portp); 2398 } 2399 bail: 2400 2401 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end, 2402 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n"); 2403 } 2404 /* 2405 * ibmf_saa_impl_client_rereg: 2406 */ 2407 static void 2408 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num) 2409 { 2410 saa_port_t *saa_portp = NULL; 2411 boolean_t is_ready = B_FALSE; 2412 ibt_hca_portinfo_t *ibt_portinfop; 2413 ib_lid_t master_sm_lid; 2414 uint_t nports, size; 2415 ibt_status_t ibt_status; 2416 boolean_t event_subs = B_FALSE; 2417 2418 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start, 2419 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt " 2420 "rereg guid %016" PRIx64 " port %d\n", 2421 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2422 2423 /* Get classportinfo of corresponding entry */ 2424 mutex_enter(&saa_statep->saa_port_list_mutex); 2425 2426 saa_portp = saa_statep->saa_port_list; 2427 while (saa_portp != NULL) { 2428 2429 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2430 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2431 2432 mutex_enter(&saa_portp->saa_pt_mutex); 2433 2434 is_ready = (saa_portp->saa_pt_state 2435 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2436 2437 /* 2438 * increment reference count to account for cpi and 2439 * informinfos. All 4 informinfo's sent are treated as 2440 * one port client reference 2441 */ 2442 if (is_ready) 2443 saa_portp->saa_pt_reference_count += 2; 2444 2445 mutex_exit(&saa_portp->saa_pt_mutex); 2446 2447 if (is_ready) 2448 break; /* normally, only 1 port entry */ 2449 } 2450 saa_portp = saa_portp->next; 2451 } 2452 2453 mutex_exit(&saa_statep->saa_port_list_mutex); 2454 2455 if (saa_portp != NULL && is_ready == B_TRUE) { 2456 2457 /* verify whether master sm lid changed */ 2458 2459 /* first query the portinfo to see if the lid changed */ 2460 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2461 &ibt_portinfop, &nports, &size); 2462 2463 if (ibt_status != IBT_SUCCESS) { 2464 2465 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2466 ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "", 2467 "ibmf_saa_impl_client_rereg: %s, ibmf_status =" 2468 " %d\n", tnf_string, msg, 2469 "ibt_query_hca_ports_byguid() failed", 2470 tnf_int, ibt_status, ibt_status); 2471 2472 goto bail; 2473 } 2474 2475 master_sm_lid = ibt_portinfop->p_sm_lid; 2476 2477 ibt_free_portinfo(ibt_portinfop, size); 2478 2479 /* check whether we need to subscribe for events */ 2480 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2481 2482 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2483 B_TRUE : B_FALSE; 2484 2485 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2486 2487 /* update the master smlid */ 2488 mutex_enter(&saa_portp->saa_pt_mutex); 2489 2490 /* update the master sm lid value in ibmf_saa */ 2491 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2492 master_sm_lid; 2493 2494 /* if we're not subscribed for events, dec reference count */ 2495 if (event_subs == B_FALSE) 2496 saa_portp->saa_pt_reference_count--; 2497 2498 mutex_exit(&saa_portp->saa_pt_mutex); 2499 2500 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2501 ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "", 2502 "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n", 2503 tnf_string, msg, 2504 "port is up. Sending classportinfo request", 2505 tnf_opaque, master_sm_lid, master_sm_lid); 2506 2507 /* get the classportinfo again */ 2508 ibmf_saa_impl_get_classportinfo(saa_portp); 2509 2510 /* 2511 * resubscribe to events if there are subscribers since SA may 2512 * have removed our subscription records when the port went down 2513 */ 2514 if (event_subs == B_TRUE) 2515 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2516 } 2517 2518 bail: 2519 2520 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end, 2521 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n"); 2522 } 2523 /* 2524 * ibmf_saa_impl_port_up: 2525 */ 2526 static void 2527 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2528 { 2529 saa_port_t *saa_portp = NULL; 2530 int is_ready; 2531 ibt_hca_portinfo_t *ibt_portinfop; 2532 ib_lid_t master_sm_lid; 2533 uint_t nports, size; 2534 ibt_status_t ibt_status; 2535 boolean_t event_subs = B_FALSE; 2536 2537 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2538 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2539 " guid %016" PRIx64 " port %d\n", 2540 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2541 2542 /* Get classportinfo of corresponding entry */ 2543 mutex_enter(&saa_statep->saa_port_list_mutex); 2544 2545 saa_portp = saa_statep->saa_port_list; 2546 while (saa_portp != NULL) { 2547 2548 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2549 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2550 2551 mutex_enter(&saa_portp->saa_pt_mutex); 2552 2553 is_ready = (saa_portp->saa_pt_state 2554 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2555 2556 /* 2557 * increment reference count to account for cpi and 2558 * informinfos. All 4 informinfo's sent are treated as 2559 * one port client reference 2560 */ 2561 if (is_ready == B_TRUE) 2562 saa_portp->saa_pt_reference_count += 2; 2563 2564 mutex_exit(&saa_portp->saa_pt_mutex); 2565 2566 if (is_ready == B_TRUE) 2567 break; /* normally, only 1 port entry */ 2568 } 2569 saa_portp = saa_portp->next; 2570 } 2571 2572 mutex_exit(&saa_statep->saa_port_list_mutex); 2573 2574 if (saa_portp != NULL && is_ready == B_TRUE) { 2575 2576 /* verify whether master sm lid changed */ 2577 2578 /* first query the portinfo to see if the lid changed */ 2579 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2580 &ibt_portinfop, &nports, &size); 2581 2582 if (ibt_status != IBT_SUCCESS) { 2583 2584 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2585 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2586 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2587 " %d\n", tnf_string, msg, 2588 "ibt_query_hca_ports_byguid() failed", 2589 tnf_int, ibt_status, ibt_status); 2590 2591 goto bail; 2592 } 2593 2594 master_sm_lid = ibt_portinfop->p_sm_lid; 2595 2596 ibt_free_portinfo(ibt_portinfop, size); 2597 2598 /* check whether we need to subscribe for events */ 2599 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2600 2601 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2602 B_TRUE : B_FALSE; 2603 2604 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2605 2606 /* update the master smlid */ 2607 mutex_enter(&saa_portp->saa_pt_mutex); 2608 2609 /* update the master sm lid value in ibmf_saa */ 2610 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2611 master_sm_lid; 2612 2613 /* if we're not subscribed for events, dec reference count */ 2614 if (event_subs == B_FALSE) 2615 saa_portp->saa_pt_reference_count--; 2616 2617 mutex_exit(&saa_portp->saa_pt_mutex); 2618 2619 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2620 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2621 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2622 tnf_string, msg, 2623 "port is up. Sending classportinfo request", 2624 tnf_opaque, master_sm_lid, master_sm_lid); 2625 2626 /* get the classportinfo again */ 2627 ibmf_saa_impl_get_classportinfo(saa_portp); 2628 2629 /* 2630 * resubscribe to events if there are subscribers since SA may 2631 * have removed our subscription records when the port went down 2632 */ 2633 if (event_subs == B_TRUE) 2634 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2635 } 2636 2637 bail: 2638 2639 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2640 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2641 } 2642 2643 /* 2644 * ibmf_saa_impl_port_down: 2645 */ 2646 static void 2647 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2648 { 2649 2650 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2651 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2652 " guid %016" PRIx64 " port %d\n", 2653 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2654 2655 2656 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2657 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2658 } 2659 2660 /* 2661 * ibmf_saa_impl_hca_detach: 2662 * find entry, unregister if there are no clients 2663 * have to unregister since ibmf needs to close the hca and will only do this if 2664 * no clients are registered 2665 */ 2666 static void 2667 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2668 { 2669 saa_port_t *saa_portp; 2670 boolean_t must_unreg, must_unsub; 2671 2672 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2673 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2674 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2675 2676 /* find this entry */ 2677 mutex_enter(&saa_statep->saa_port_list_mutex); 2678 2679 saa_portp = saa_statep->saa_port_list; 2680 while (saa_portp != NULL) { 2681 2682 if (saa_portp == saa_removed) 2683 break; 2684 2685 saa_portp = saa_portp->next; 2686 } 2687 mutex_exit(&saa_statep->saa_port_list_mutex); 2688 2689 ASSERT(saa_portp != NULL); 2690 2691 if (saa_portp == NULL) { 2692 2693 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2694 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2695 "ibmf_saa_impl_hca_detach: %s, entry %016" 2696 PRIx64 "\n", 2697 tnf_string, msg, 2698 "Port entry NOT found", 2699 tnf_opaque, entryp, saa_removed); 2700 2701 goto bail; 2702 } 2703 2704 /* if there are clients expecting Reports(), unsusbscribe */ 2705 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2706 2707 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2708 B_TRUE : B_FALSE; 2709 2710 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2711 2712 /* fail if outstanding transactions */ 2713 mutex_enter(&saa_portp->saa_pt_mutex); 2714 2715 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2716 2717 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2718 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2719 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2720 ", num transactions = %d\n", 2721 tnf_string, msg, "Detaching HCA." 2722 " Outstanding transactions on port.", 2723 tnf_opaque, port, 2724 saa_portp->saa_pt_port_guid, 2725 tnf_uint, outstanding_transactions, 2726 saa_portp->saa_pt_num_outstanding_trans); 2727 2728 mutex_exit(&saa_portp->saa_pt_mutex); 2729 2730 goto bail; 2731 } 2732 2733 2734 /* 2735 * increment reference count by one to account for unsubscribe requests 2736 * that are about to be sent. All four informinfo's are treated as one 2737 * port client reference. The count will be decremented by 2738 * subscribe_events() before the call returns. 2739 */ 2740 if (must_unsub == B_TRUE) 2741 saa_portp->saa_pt_reference_count++; 2742 2743 mutex_exit(&saa_portp->saa_pt_mutex); 2744 2745 /* 2746 * try and unsubscribe from SA. Generate synchronous, unsequenced 2747 * unsubscribe requests. 2748 */ 2749 if (must_unsub == B_TRUE) 2750 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2751 2752 /* warning if registered clients */ 2753 mutex_enter(&saa_portp->saa_pt_mutex); 2754 2755 if (saa_portp->saa_pt_reference_count > 0) { 2756 2757 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2758 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2759 "ibmf_saa_impl_hca_detach: %s, port %016" 2760 PRIx64 "\n", 2761 tnf_string, msg, 2762 "Detaching HCA for port with clients still" 2763 " registered", tnf_opaque, port, 2764 saa_portp->saa_pt_port_guid); 2765 } 2766 2767 /* synchronize on end of registration */ 2768 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2769 2770 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2771 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2772 "ibmf_saa_impl_hca_detach: %s\n", 2773 tnf_string, msg, "someone is registering. waiting" 2774 " for them to finish"); 2775 2776 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2777 &saa_portp->saa_pt_mutex); 2778 2779 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2780 ibmf_saa_impl_hca_detach, 2781 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2782 tnf_string, msg, "done waiting"); 2783 } 2784 2785 /* unregister from ibmf */ 2786 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2787 must_unreg = B_TRUE; 2788 } else 2789 must_unreg = B_FALSE; 2790 2791 ibmf_saa_impl_invalidate_port(saa_portp); 2792 2793 mutex_exit(&saa_portp->saa_pt_mutex); 2794 2795 if (must_unreg == B_TRUE) { 2796 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) { 2797 mutex_enter(&saa_portp->saa_pt_mutex); 2798 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2799 (void) ibmf_saa_impl_init_kstats(saa_portp); 2800 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2801 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY; 2802 if (must_unsub == B_TRUE) 2803 saa_portp->saa_pt_reference_count++; 2804 mutex_exit(&saa_portp->saa_pt_mutex); 2805 2806 if (must_unsub == B_TRUE) { 2807 ibmf_saa_subscribe_events(saa_portp, B_TRUE, 2808 B_FALSE); 2809 } 2810 } 2811 } 2812 bail: 2813 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2814 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2815 } 2816 2817 /* ARGSUSED */ 2818 void 2819 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2820 { 2821 saa_impl_trans_info_t *trans_info; 2822 int status; 2823 size_t length; 2824 void *result; 2825 saa_port_t *saa_portp; 2826 saa_client_data_t *client_data; 2827 int ibmf_status; 2828 boolean_t ignore_data; 2829 ibmf_retrans_t ibmf_retrans; 2830 boolean_t sa_is_redirected = B_FALSE; 2831 2832 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2833 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2834 2835 trans_info = (saa_impl_trans_info_t *)args; 2836 2837 client_data = trans_info->si_trans_client_data; 2838 saa_portp = trans_info->si_trans_port; 2839 2840 mutex_enter(&saa_portp->saa_pt_mutex); 2841 sa_is_redirected = saa_portp->saa_pt_redirect_active; 2842 mutex_exit(&saa_portp->saa_pt_mutex); 2843 2844 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) && 2845 (sa_is_redirected == B_TRUE)) { 2846 2847 /* 2848 * We should retry the request using SM_LID and QP1 if we 2849 * have been using redirect up until now 2850 */ 2851 ibmf_status = ibmf_saa_impl_revert_to_qp1( 2852 saa_portp, msgp, ibmf_saa_async_cb, args, 2853 trans_info->si_trans_transport_flags); 2854 2855 /* 2856 * If revert_to_qp1 returns success msg was resent. 2857 * Otherwise msg could not be resent. Continue normally 2858 */ 2859 if (ibmf_status == IBMF_SUCCESS) 2860 goto bail; 2861 2862 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2863 2864 2865 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2866 ibmf_saa_async_cb, args, 2867 trans_info->si_trans_transport_flags); 2868 2869 /* 2870 * if smlid_retry() returns success sm lid changed and msg 2871 * was resent. Otherwise, lid did not change or msg could not 2872 * be resent. Continue normally. 2873 */ 2874 if (ibmf_status == IBMF_SUCCESS) 2875 goto bail; 2876 2877 /* 2878 * check whether we've received anything from the SA in a while. 2879 * If we have, this function will retry and return success. If 2880 * we haven't continue normally so that we return a timeout to 2881 * the client 2882 */ 2883 ibmf_status = ibmf_saa_check_sa_and_retry( 2884 saa_portp, msgp, ibmf_saa_async_cb, args, 2885 trans_info->si_trans_send_time, 2886 trans_info->si_trans_transport_flags); 2887 2888 if (ibmf_status == IBMF_SUCCESS) 2889 goto bail; 2890 } 2891 2892 /* 2893 * If SA returned success but mad status is busy, retry a few times. 2894 * If SA returned success but mad status says redirect is required, 2895 * update the address info and retry the request to the new SA address 2896 */ 2897 if (msgp->im_msg_status == IBMF_SUCCESS) { 2898 2899 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2900 2901 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2902 MAD_STATUS_BUSY) && 2903 (trans_info->si_trans_retry_busy_count < 2904 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2905 2906 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2907 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2908 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2909 tnf_string, msg, 2910 "async response returned busy status", 2911 tnf_int, retry_count, 2912 trans_info->si_trans_retry_busy_count); 2913 2914 trans_info->si_trans_retry_busy_count++; 2915 2916 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2917 sizeof (ibmf_retrans_t)); 2918 2919 ibmf_status = ibmf_msg_transport( 2920 saa_portp->saa_pt_ibmf_handle, 2921 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2922 ibmf_saa_async_cb, args, 2923 trans_info->si_trans_transport_flags); 2924 2925 /* 2926 * if retry is successful, quit here since async_cb will 2927 * get called again; otherwise, let this function call 2928 * handle the cleanup 2929 */ 2930 if (ibmf_status == IBMF_SUCCESS) 2931 goto bail; 2932 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) 2933 == MAD_STATUS_REDIRECT_REQUIRED) { 2934 2935 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2, 2936 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2937 "ibmf_saa_async_cb: " 2938 "async response returned redirect status\n"); 2939 2940 /* update address info and copy it into msgp */ 2941 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 2942 2943 /* retry with new address info */ 2944 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2945 sizeof (ibmf_retrans_t)); 2946 2947 ibmf_status = ibmf_msg_transport( 2948 saa_portp->saa_pt_ibmf_handle, 2949 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2950 ibmf_saa_async_cb, args, 2951 trans_info->si_trans_transport_flags); 2952 2953 /* 2954 * if retry is successful, quit here since async_cb will 2955 * get called again; otherwise, let this function call 2956 * handle the cleanup 2957 */ 2958 if (ibmf_status == IBMF_SUCCESS) 2959 goto bail; 2960 } 2961 } 2962 2963 mutex_enter(&saa_portp->saa_pt_mutex); 2964 2965 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2966 saa_portp->saa_pt_num_outstanding_trans--; 2967 2968 mutex_exit(&saa_portp->saa_pt_mutex); 2969 2970 if ((trans_info->si_trans_callback == NULL) && 2971 (trans_info->si_trans_sub_callback == NULL)) 2972 ignore_data = B_TRUE; 2973 else 2974 ignore_data = B_FALSE; 2975 2976 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2977 &result, &length, B_FALSE); 2978 2979 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2980 2981 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2982 2983 if (status != IBMF_SUCCESS) 2984 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2985 2986 if (status == IBMF_TRANS_TIMEOUT) 2987 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2988 2989 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2990 2991 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2992 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2993 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2994 "Calling ibmf_saa client's callback"); 2995 2996 /* 2997 * there are three classes or trans_info users: ibmf_saa clients and 2998 * classportinfo requests; informinfo subscribe requests, and report 2999 * responses. For the first two, call the correct callback. For report 3000 * responses there's no need to notify anyone. 3001 */ 3002 if (trans_info->si_trans_callback != NULL) { 3003 /* ibmf_saa client or classportinfo request */ 3004 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 3005 length, result, status); 3006 } else if (trans_info->si_trans_sub_callback != NULL) { 3007 /* informinfo subscribe request */ 3008 trans_info->si_trans_sub_callback( 3009 trans_info->si_trans_callback_arg, length, result, status, 3010 trans_info->si_trans_sub_producer_type); 3011 } 3012 3013 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3014 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 3015 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 3016 "Returned from callback"); 3017 3018 if (client_data != NULL) { 3019 mutex_enter(&client_data->saa_client_mutex); 3020 3021 ASSERT(client_data->saa_client_num_pending_trans > 0); 3022 client_data->saa_client_num_pending_trans--; 3023 3024 if ((client_data->saa_client_num_pending_trans == 0) && 3025 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 3026 cv_signal(&client_data->saa_client_state_cv); 3027 3028 mutex_exit(&client_data->saa_client_mutex); 3029 } 3030 3031 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 3032 3033 bail: 3034 3035 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 3036 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 3037 } 3038 3039 /* 3040 * ibmf_saa_check_sa_and_retry: 3041 * 3042 * If a particular transaction times out, we don't want to give up if we know 3043 * the SA is responding. Check the time since we last received a response. If 3044 * it's less than ibmf_saa_max_wait_time retry the request. 3045 * 3046 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 3047 * same values passed to the original ibmf_msg_transport that timed out. The 3048 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 3049 * 3050 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 3051 * That way, callers can simply return the result of this function. 3052 * 3053 * Input Arguments 3054 * saa_portp pointer to saa_port structure 3055 * msgp ibmf message that timedout 3056 * ibmf_callback callback that should be called by msg_transport 3057 * ibmf_callback_arg args for ibmf_callback 3058 * transport_flags flags for ibmf_msg_transport 3059 * 3060 * Output Arguments 3061 * none 3062 * 3063 * Returns 3064 * IBMF_SUCCESS if we've recently received data from the SA and request was 3065 * resent. 3066 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 3067 * same values as ibmf_msg_transport() if data has been received but request 3068 * could not be resent. 3069 */ 3070 static int 3071 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 3072 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 3073 hrtime_t trans_send_time, int transport_flags) 3074 { 3075 hrtime_t curr_time, sa_uptime; 3076 ibmf_retrans_t ibmf_retrans; 3077 int ibmf_status; 3078 3079 do { 3080 3081 mutex_enter(&saa_portp->saa_pt_mutex); 3082 3083 sa_uptime = saa_portp->saa_pt_sa_uptime; 3084 3085 /* if nothing received from SA since we sent */ 3086 curr_time = gethrtime(); 3087 3088 /* 3089 * check if it's been a very long time since this 3090 * particular transaction was sent 3091 */ 3092 if (((curr_time - trans_send_time) / 1000000000) > 3093 ibmf_saa_trans_wait_time) { 3094 3095 mutex_exit(&saa_portp->saa_pt_mutex); 3096 3097 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3098 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3099 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3100 "%p sa_uptime = %" PRIu64 ", trans send time = %" 3101 PRIu64 ", curr_time = %" PRIu64 "\n", 3102 tnf_string, msg, 3103 "Nothing received for this transaction", 3104 tnf_opaque, msgp, msgp, 3105 tnf_long, sa_uptime, sa_uptime, 3106 tnf_long, trans_send_time, trans_send_time, 3107 tnf_long, curr_time, curr_time); 3108 3109 ibmf_status = IBMF_TRANS_TIMEOUT; 3110 3111 break; 3112 } 3113 3114 /* 3115 * check time since we received something, 3116 * and make sure that it hasn't been an extra long 3117 * time for this particular transaction 3118 */ 3119 if (((curr_time - sa_uptime) / 1000000000) < 3120 ibmf_saa_max_wait_time) { 3121 3122 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 3123 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3124 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3125 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 3126 PRIu64 " curr_time = %" PRIu64 "\n", 3127 tnf_string, msg, "Something received. Retrying", 3128 tnf_opaque, msgp, msgp, 3129 tnf_long, sa_uptime, sa_uptime, 3130 tnf_long, trans_send_time, trans_send_time, 3131 tnf_long, curr_time, curr_time); 3132 3133 /* 3134 * something received in WAIT_TIME_IN_SECS; 3135 * resend request 3136 */ 3137 3138 /* new tid needed */ 3139 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 3140 h2b64(saa_portp->saa_pt_current_tid++); 3141 3142 /* 3143 * We are going to retry the access to the SM but 3144 * Master SMLID could have changed due to a port change 3145 * event. So update the remote_lid of the message with 3146 * the SMLID from saa_portp for this port before the 3147 * retry. 3148 */ 3149 msgp->im_local_addr.ia_remote_lid = 3150 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid; 3151 3152 bcopy(&saa_portp->saa_pt_ibmf_retrans, 3153 &ibmf_retrans, sizeof (ibmf_retrans_t)); 3154 3155 mutex_exit(&saa_portp->saa_pt_mutex); 3156 3157 ibmf_status = ibmf_msg_transport( 3158 saa_portp->saa_pt_ibmf_handle, 3159 saa_portp->saa_pt_qp_handle, msgp, 3160 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 3161 transport_flags); 3162 3163 if (ibmf_status == IBMF_SUCCESS) 3164 goto bail; 3165 3166 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3167 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3168 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 3169 "%d\n", tnf_string, msg, 3170 "ibmf_msg_transport() failed", 3171 tnf_int, ibmf_status, ibmf_status); 3172 } else { 3173 3174 mutex_exit(&saa_portp->saa_pt_mutex); 3175 3176 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 3177 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3178 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3179 "%p sa_uptime = %" PRIu64 " curr_time = %" 3180 PRIu64 "\n", tnf_string, msg, 3181 "Nothing received. Timing out", 3182 tnf_opaque, msgp, msgp, 3183 tnf_long, sa_uptime, sa_uptime, 3184 tnf_long, curr_time, curr_time); 3185 3186 ibmf_status = IBMF_TRANS_TIMEOUT; 3187 3188 break; 3189 } 3190 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 3191 3192 bail: 3193 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3194 ibmf_saa_check_sa_and_retry_end, 3195 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 3196 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 3197 3198 return (ibmf_status); 3199 } 3200 3201 3202 /* 3203 * ibmf_saa_impl_prepare_response: 3204 */ 3205 static void 3206 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 3207 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 3208 size_t *length, boolean_t sleep_flag) 3209 { 3210 ibmf_msg_bufs_t *resp_buf; 3211 uint16_t attr_id; 3212 uint8_t method; 3213 boolean_t is_get_resp; 3214 uint16_t mad_status; 3215 uint16_t attr_offset; 3216 ib_sa_hdr_t *sa_hdr; 3217 3218 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3219 ibmf_saa_impl_prepare_response_start, 3220 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 3221 3222 _NOTE(ASSUMING_PROTECTED(*msgp)) 3223 3224 *result = NULL; 3225 *length = 0; 3226 sa_hdr = NULL; 3227 3228 resp_buf = &msgp->im_msgbufs_recv; 3229 3230 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 3231 3232 if (msgp->im_msg_status != IBMF_SUCCESS) { 3233 3234 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3235 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3236 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 3237 tnf_string, msg, "Bad ibmf status", 3238 tnf_int, msg_status, msgp->im_msg_status); 3239 3240 *status = msgp->im_msg_status; 3241 3242 goto exit; 3243 } 3244 3245 if (resp_buf->im_bufs_mad_hdr == NULL) { 3246 3247 /* 3248 * this was an unsequenced transaction (from an unsubscribe for 3249 * following a CI_OFFLINE event) 3250 */ 3251 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3252 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3253 "ibmf_saa_impl_prepare_response: %s\n", 3254 tnf_string, msg, "Unsequenced transaction callback"); 3255 3256 goto exit; 3257 } 3258 3259 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 3260 MAD_STATUS_NO_INVALID_FIELDS) { 3261 3262 /* convert mad packet status to IBMF status */ 3263 switch (mad_status) { 3264 3265 case SA_STATUS_ERR_NO_RESOURCES: 3266 *status = IBMF_NO_RESOURCES; 3267 break; 3268 case SA_STATUS_ERR_REQ_INVALID: 3269 *status = IBMF_REQ_INVALID; 3270 break; 3271 case SA_STATUS_ERR_NO_RECORDS: 3272 *status = IBMF_NO_RECORDS; 3273 break; 3274 case SA_STATUS_ERR_TOO_MANY_RECORDS: 3275 *status = IBMF_TOO_MANY_RECORDS; 3276 break; 3277 case SA_STATUS_ERR_REQ_INVALID_GID: 3278 *status = IBMF_INVALID_GID; 3279 break; 3280 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 3281 *status = IBMF_INSUFF_COMPS; 3282 break; 3283 case MAD_STATUS_UNSUPP_METHOD: 3284 *status = IBMF_UNSUPP_METHOD; 3285 break; 3286 case MAD_STATUS_UNSUPP_METHOD_ATTR: 3287 *status = IBMF_UNSUPP_METHOD_ATTR; 3288 break; 3289 case MAD_STATUS_INVALID_FIELD: 3290 *status = IBMF_INVALID_FIELD; 3291 break; 3292 default: 3293 *status = IBMF_REQ_INVALID; 3294 break; 3295 } 3296 3297 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3298 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3299 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 3300 tnf_string, msg, "Bad MAD status", 3301 tnf_int, mad_status, mad_status); 3302 3303 goto exit; 3304 } 3305 3306 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 3307 method = resp_buf->im_bufs_mad_hdr->R_Method; 3308 3309 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3310 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3311 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 3312 "0x%x\n", 3313 tnf_opaque, attr_id, attr_id, 3314 tnf_opaque, method, method); 3315 3316 /* 3317 * ignore any data from deleteresp since there's no way to know whether 3318 * real data was returned; also ignore data if this was a Report 3319 * response 3320 */ 3321 if (method == SA_SUBN_ADM_DELETE_RESP) { 3322 3323 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3324 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3325 "impf_saa_impl_prepare_response: %s\n", 3326 tnf_string, msg, 3327 "DeleteResp or NoticeResp returned. " 3328 "Ignoring response data"); 3329 3330 *status = IBMF_SUCCESS; 3331 3332 *length = 0; 3333 *result = NULL; 3334 3335 goto exit; 3336 } 3337 3338 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 3339 3340 /* 3341 * getmulti is only for requests; attribute should not 3342 * be returned from SA 3343 */ 3344 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 3345 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 3346 "", "ibmf_saa_impl_prepare_response: %s\n", 3347 tnf_string, msg, "SA returned getmulti record"); 3348 3349 *status = IBMF_REQ_INVALID; 3350 3351 goto exit; 3352 } 3353 3354 /* if we are supposed to ignore data, stop here */ 3355 if (ignore_data == B_TRUE) { 3356 3357 *status = IBMF_SUCCESS; 3358 3359 goto exit; 3360 } 3361 3362 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 3363 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 3364 3365 /* unpack the sa header to get the attribute offset */ 3366 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 3367 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 3368 if (*status != IBMF_SUCCESS) { 3369 3370 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3371 ibmf_saa_impl_prepare_response_err, 3372 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3373 " ibmf_status = %d\n", tnf_string, msg, 3374 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 3375 3376 goto exit; 3377 } 3378 3379 attr_offset = sa_hdr->AttributeOffset; 3380 3381 /* 3382 * unpack data payload; if unpack function doesn't return success 3383 * (because it could not allocate memory) forward this status to waiting 3384 * client 3385 */ 3386 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 3387 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 3388 attr_offset, is_get_resp, sleep_flag); 3389 if (*status == IBMF_SUCCESS) { 3390 3391 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3392 ibmf_saa_impl_prepare_response, 3393 IBMF_TNF_TRACE, "", 3394 "ibmf_saa_impl_prepare_response: attr_id = " 3395 "0x%x, attr_offset = %d, packed_payload_len = %d, " 3396 "unpacked_payload_len = %d\n", 3397 tnf_opaque, attr_id, attr_id, 3398 tnf_opaque, attr_offset, attr_offset, 3399 tnf_opaque, packed_payload_len, 3400 resp_buf->im_bufs_cl_data_len, 3401 tnf_opaque, unpacked_payload_len, *length); 3402 } else { 3403 3404 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3405 ibmf_saa_impl_prepare_response_err, 3406 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3407 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 3408 "status = %d\n", 3409 tnf_string, msg, "Could not unpack payload", 3410 tnf_opaque, attr_id, attr_id, 3411 tnf_int, attr_offset, attr_offset, 3412 tnf_int, packed_payload_len, 3413 resp_buf->im_bufs_cl_data_len, 3414 tnf_int, status, *status); 3415 } 3416 exit: 3417 if (sa_hdr != NULL) 3418 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3419 3420 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 3421 3422 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3423 ibmf_saa_impl_prepare_response_end, 3424 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 3425 " status = 0x%d\n", tnf_int, status, *status); 3426 } 3427 3428 3429 /* 3430 * ibmf_saa_impl_check_sa_support: 3431 * Checks the capability mask (returned from the SA classportinfo response) to 3432 * determine whether the sa supports the specified attribute ID. 3433 * 3434 * Input Arguments 3435 * cap_mask 16-bit capability mask returned in SA's classportinfo 3436 * attr_id attribute ID of current request 3437 * 3438 * Returns 3439 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 3440 * IBMF_SUCCESS otherwise 3441 */ 3442 static int 3443 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 3444 { 3445 boolean_t attr_supported = B_TRUE; 3446 3447 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3448 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 3449 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 3450 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 3451 tnf_opaque, attr_id, attr_id); 3452 3453 switch (attr_id) { 3454 3455 case SA_SWITCHINFORECORD_ATTRID: 3456 case SA_LINEARFDBRECORD_ATTRID: 3457 case SA_RANDOMFDBRECORD_ATTRID: 3458 case SA_MULTICASTFDBRECORD_ATTRID: 3459 case SA_SMINFORECORD_ATTRID: 3460 case SA_INFORMINFORECORD_ATTRID: 3461 case SA_LINKRECORD_ATTRID: 3462 case SA_GUIDINFORECORD_ATTRID: 3463 case SA_TRACERECORD_ATTRID: 3464 case SA_SERVICEASSNRECORD_ATTRID: 3465 3466 if ((cap_mask & 3467 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 3468 3469 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 3470 ibmf_saa_impl_check_sa_support, 3471 IBMF_TNF_ERROR, "", 3472 "ibmf_saa_impl_check_sa_support: %s, " 3473 "cap_mask = 0x%x\n", tnf_string, msg, 3474 "SA does not support optional records", 3475 tnf_opaque, cap_mask, cap_mask, 3476 tnf_opaque, attr_id, attr_id); 3477 3478 attr_supported = B_FALSE; 3479 } 3480 break; 3481 3482 case SA_MULTIPATHRECORD_ATTRID: 3483 3484 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 3485 3486 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3487 ibmf_saa_impl_check_sa_support, 3488 IBMF_TNF_ERROR, "", 3489 "ibmf_saa_impl_check_sa_support: %s, " 3490 "cap_mask = 0x%x\n", tnf_string, msg, 3491 "SA does not support multipath records", 3492 tnf_opaque, cap_mask, cap_mask); 3493 3494 attr_supported = B_FALSE; 3495 } 3496 break; 3497 3498 case SA_MCMEMBERRECORD_ATTRID: 3499 3500 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 3501 3502 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3503 ibmf_saa_impl_check_sa_support, 3504 IBMF_TNF_ERROR, "", 3505 "ibmf_saa_impl_check_sa_support: %s, " 3506 "cap_mask = 0x%x\n", tnf_string, msg, 3507 "SA does not support ud multicast", 3508 tnf_opaque, cap_mask, cap_mask); 3509 3510 attr_supported = B_FALSE; 3511 } 3512 break; 3513 3514 default: 3515 break; 3516 } /* switch */ 3517 3518 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3519 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 3520 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 3521 tnf_opaque, attr_supported, attr_supported); 3522 3523 if (attr_supported == B_FALSE) 3524 return (IBMF_UNSUPP_METHOD_ATTR); 3525 else 3526 return (IBMF_SUCCESS); 3527 } 3528 3529 /* 3530 * ibmf_saa_impl_get_attr_id_length: 3531 * 3532 * Returns the host size of the specified sa record. Returns 0 for unknown 3533 * attributes. multipath record size is a dynamic value given as a parameter 3534 * specified with the ibmf_sa_access() call. 3535 */ 3536 static uint_t 3537 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3538 { 3539 uint_t attr_length; 3540 3541 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3542 ibmf_saa_impl_get_attr_id_length_start, 3543 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3544 3545 /* this function should not be used for multipath record */ 3546 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3547 3548 switch (attr_id) { 3549 case SA_CLASSPORTINFO_ATTRID: 3550 attr_length = sizeof (ib_mad_classportinfo_t); 3551 break; 3552 case SA_NOTICE_ATTRID: 3553 attr_length = sizeof (ib_mad_notice_t); 3554 break; 3555 case SA_INFORMINFO_ATTRID: 3556 attr_length = sizeof (ib_mad_informinfo_t); 3557 break; 3558 case SA_NODERECORD_ATTRID: 3559 attr_length = sizeof (sa_node_record_t); 3560 break; 3561 case SA_PORTINFORECORD_ATTRID: 3562 attr_length = sizeof (sa_portinfo_record_t); 3563 break; 3564 case SA_SLTOVLRECORD_ATTRID: 3565 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3566 break; 3567 case SA_SWITCHINFORECORD_ATTRID: 3568 attr_length = sizeof (sa_switchinfo_record_t); 3569 break; 3570 case SA_LINEARFDBRECORD_ATTRID: 3571 attr_length = sizeof (sa_linearft_record_t); 3572 break; 3573 case SA_RANDOMFDBRECORD_ATTRID: 3574 attr_length = sizeof (sa_randomft_record_t); 3575 break; 3576 case SA_MULTICASTFDBRECORD_ATTRID: 3577 attr_length = sizeof (sa_multicastft_record_t); 3578 break; 3579 case SA_SMINFORECORD_ATTRID: 3580 attr_length = sizeof (sa_sminfo_record_t); 3581 break; 3582 case SA_INFORMINFORECORD_ATTRID: 3583 attr_length = sizeof (sa_informinfo_record_t); 3584 break; 3585 case SA_LINKRECORD_ATTRID: 3586 attr_length = sizeof (sa_link_record_t); 3587 break; 3588 case SA_GUIDINFORECORD_ATTRID: 3589 attr_length = sizeof (sa_guidinfo_record_t); 3590 break; 3591 case SA_SERVICERECORD_ATTRID: 3592 attr_length = sizeof (sa_service_record_t); 3593 break; 3594 case SA_PARTITIONRECORD_ATTRID: 3595 attr_length = sizeof (sa_pkey_table_record_t); 3596 break; 3597 case SA_PATHRECORD_ATTRID: 3598 attr_length = sizeof (sa_path_record_t); 3599 break; 3600 case SA_VLARBRECORD_ATTRID: 3601 attr_length = sizeof (sa_VLarb_table_record_t); 3602 break; 3603 case SA_MCMEMBERRECORD_ATTRID: 3604 attr_length = sizeof (sa_mcmember_record_t); 3605 break; 3606 case SA_TRACERECORD_ATTRID: 3607 attr_length = sizeof (sa_trace_record_t); 3608 break; 3609 case SA_SERVICEASSNRECORD_ATTRID: 3610 attr_length = sizeof (sa_service_assn_record_t); 3611 break; 3612 default: 3613 /* should only get the above type of packets */ 3614 attr_length = 0; 3615 break; 3616 } 3617 3618 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3619 ibmf_saa_impl_get_attr_id_length_end, 3620 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3621 " attr_id: 0x%x size %d\n", 3622 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3623 3624 return (attr_length); 3625 } 3626 3627 /* 3628 * ibmf_saa_impl_free_msg: 3629 * Takes a completed message and free memory associated with the message, 3630 * including the individual fields of the im_msgbufs_send. 3631 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3632 * message pointer so that it can set the message pointer to NULL. This 3633 * function takes just the message pointer so the msgp will not be NULL after 3634 * this function returns. 3635 * 3636 * Input Arguments 3637 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3638 * msgp pointer to ibmf_msg_t to free 3639 * 3640 * Returns 3641 * void 3642 */ 3643 static void 3644 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3645 { 3646 int res; 3647 3648 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3649 ibmf_saa_impl_free_msg_start, 3650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3651 tnf_opaque, msg, msgp); 3652 3653 ASSERT(msgp != NULL); 3654 3655 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3656 sizeof (ib_mad_hdr_t)); 3657 3658 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3659 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3660 3661 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3662 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3663 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3664 3665 res = ibmf_free_msg(ibmf_hdl, &msgp); 3666 ASSERT(res == IBMF_SUCCESS); 3667 3668 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3669 ibmf_saa_impl_free_msg_end, 3670 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3671 } 3672 3673 /* 3674 * ibmf_saa_impl_get_port_guid: 3675 */ 3676 static int 3677 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3678 ib_guid_t *guid_ret) 3679 { 3680 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3681 ibmf_saa_impl_get_port_guid_start, 3682 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3683 3684 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3685 3686 return (IBMF_BAD_PORT_STATE); 3687 } 3688 3689 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3690 3691 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3692 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3693 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3694 "portinfo sgid table size is 0. Exiting.\n"); 3695 3696 return (IBMF_TRANSPORT_FAILURE); 3697 } 3698 3699 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3700 3701 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3702 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3703 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3704 tnf_opaque, port_guid, *guid_ret); 3705 3706 return (IBMF_SUCCESS); 3707 } 3708 3709 /* 3710 * ibmf_saa_impl_set_transaction_params: 3711 */ 3712 static void 3713 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3714 ibt_hca_portinfo_t *portinfop) 3715 { 3716 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3717 ibmf_saa_impl_set_transaction_params_start, 3718 IBMF_TNF_TRACE, "", 3719 "ibmf_saa_impl_set_transaction_params() enter\n"); 3720 3721 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3722 3723 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3724 ibmf_saa_retrans_retries; 3725 /* 3726 * For the first transaction (generally getting the 3727 * classportinfo) have ibmf pick our timeouts. It should be using the 3728 * default IB spec values. 3729 * Once we get the classportinfo we'll update the correct response time 3730 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3731 * trans_to since it depends on the particular transaction's number of 3732 * packets. 3733 */ 3734 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3735 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3736 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3737 3738 /* 3739 * Assume that the SA supports all optional records. If it 3740 * does not, the request will get returned with ERR_NOT_SUPP. When 3741 * the classportinfo response comes back we will update the cap mask 3742 * to prevent unnecessary unsupported requests. 3743 */ 3744 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3745 3746 saa_portp->saa_pt_ibmf_msg_flags = 0; 3747 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3748 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3749 IB_PKEY_DEFAULT_LIMITED; 3750 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3751 3752 /* 3753 * fill out addr information for MADs that will be sent 3754 * to SA on this port 3755 */ 3756 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3757 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3758 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3759 3760 /* place upper bound on subnet timeout in case of faulty SM */ 3761 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3762 3763 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 3764 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 3765 3766 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3767 ibmf_saa_impl_set_transaction_params, 3768 IBMF_TNF_TRACE, "", 3769 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3770 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3771 tnf_opaque, local_lid, portinfop->p_base_lid, 3772 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3773 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3774 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3775 3776 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3777 ibmf_saa_impl_set_transaction_params_end, 3778 IBMF_TNF_TRACE, "", 3779 "ibmf_saa_impl_set_transaction_params() exit\n"); 3780 } 3781 3782 3783 /* 3784 * ibmf_saa_impl_update_sa_address_info 3785 */ 3786 static void 3787 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp) 3788 { 3789 void *result; 3790 ib_sa_hdr_t *sa_hdr; 3791 int rv; 3792 size_t length; 3793 uint16_t attr_id; 3794 ib_mad_classportinfo_t *cpi; 3795 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr; 3796 ibt_hca_portinfo_t *ibt_pinfo; 3797 uint_t nports, size; 3798 ibt_status_t ibt_status; 3799 3800 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3801 ibmf_saa_impl_update_sa_address_info, 3802 IBMF_TNF_TRACE, "", 3803 "ibmf_saa_impl_update_sa_address_info() enter\n"); 3804 3805 /* 3806 * decode the respons of msgp as a classportinfo attribute 3807 */ 3808 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr, 3809 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP); 3810 if (rv != IBMF_SUCCESS) { 3811 3812 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3813 ibmf_saa_impl_update_sa_address_err, 3814 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3815 "%s, ibmf_status = %d\n", tnf_string, msg, 3816 "Could not unpack sa hdr", tnf_int, ibmf_status, rv); 3817 3818 return; 3819 } 3820 3821 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID); 3822 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) { 3823 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3824 ibmf_saa_impl_update_sa_address_info_err, 3825 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3826 "%s, attrID = %x\n", tnf_string, msg, 3827 "Wrong attribute ID", tnf_int, ibmf_status, attr_id); 3828 3829 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3830 return; 3831 } 3832 rv = ibmf_saa_utils_unpack_payload( 3833 msgp->im_msgbufs_recv.im_bufs_cl_data, 3834 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result, 3835 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP); 3836 if (rv != IBMF_SUCCESS) { 3837 3838 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3839 ibmf_saa_impl_update_sa_address_err, 3840 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3841 "%s, ibmf_status = %d\n", tnf_string, msg, 3842 "Could not unpack payload", tnf_int, ibmf_status, rv); 3843 3844 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3845 return; 3846 } 3847 3848 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3849 3850 /* 3851 * Use the classportinfo contents to update the SA address info 3852 */ 3853 cpi = (ib_mad_classportinfo_t *)result; 3854 mutex_enter(&saa_portp->saa_pt_mutex); 3855 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID; 3856 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP; 3857 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key; 3858 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key; 3859 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL; 3860 3861 saa_portp->saa_pt_redirect_active = B_TRUE; 3862 3863 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) { 3864 3865 mutex_exit(&saa_portp->saa_pt_mutex); 3866 ibt_status = ibt_query_hca_ports_byguid( 3867 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num, 3868 &ibt_pinfo, &nports, &size); 3869 if (ibt_status != IBT_SUCCESS) { 3870 3871 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3872 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE, 3873 "", "ibmf_saa_impl_update_sa_address_info: " 3874 "%s, ibt_status = %d\n", tnf_string, msg, 3875 "Could not query hca port", 3876 tnf_int, ibt_status, ibt_status); 3877 3878 kmem_free(result, length); 3879 return; 3880 } 3881 3882 mutex_enter(&saa_portp->saa_pt_mutex); 3883 /* 3884 * Fill in global address info parameters 3885 * 3886 * NOTE: The HopLimit value is not specified through the 3887 * contents of ClassPortInfo. It may be possible to find 3888 * out the proper value to use even for SA beeing redirected 3889 * to another subnet. But we do only support redirect within 3890 * our local subnet 3891 */ 3892 gaddrp->ig_sender_gid.gid_prefix = 3893 ibt_pinfo->p_sgid_tbl[0].gid_prefix; 3894 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid; 3895 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi; 3896 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo; 3897 gaddrp->ig_flow_label = cpi->RedirectFL; 3898 gaddrp->ig_tclass = cpi->RedirectTC; 3899 gaddrp->ig_hop_limit = 0; 3900 3901 saa_portp->saa_pt_ibmf_msg_flags = 3902 IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3903 3904 mutex_exit(&saa_portp->saa_pt_mutex); 3905 ibt_free_portinfo(ibt_pinfo, size); 3906 } else { 3907 saa_portp->saa_pt_ibmf_msg_flags = 0; 3908 mutex_exit(&saa_portp->saa_pt_mutex); 3909 } 3910 kmem_free(result, length); 3911 3912 /* 3913 * Update the address info of msgp with the new address parameters 3914 */ 3915 mutex_enter(&saa_portp->saa_pt_mutex); 3916 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 3917 sizeof (ibmf_addr_info_t)); 3918 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 3919 3920 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3921 3922 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 3923 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t)); 3924 } else { 3925 msgp->im_msg_flags = 0; 3926 } 3927 mutex_exit(&saa_portp->saa_pt_mutex); 3928 } 3929 3930 /* 3931 * ibmf_saa_impl_ibmf_unreg: 3932 */ 3933 static int 3934 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3935 { 3936 int ibmf_status; 3937 3938 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3939 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3940 3941 /* teardown async cb */ 3942 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3943 saa_portp->saa_pt_qp_handle, 0); 3944 if (ibmf_status != IBMF_SUCCESS) { 3945 3946 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3947 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3948 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3949 tnf_string, msg, "Could not tear down async cb", 3950 tnf_int, ibmf_status, ibmf_status); 3951 3952 goto bail; 3953 } 3954 3955 /* free qp */ 3956 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3957 &saa_portp->saa_pt_qp_handle, 0); 3958 3959 if (ibmf_status != IBMF_SUCCESS) { 3960 3961 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3962 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3963 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3964 tnf_string, msg, "Could not free queue pair", 3965 tnf_int, ibmf_status, ibmf_status); 3966 3967 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1); 3968 goto bail; 3969 } 3970 3971 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3972 3973 if (ibmf_status != IBMF_SUCCESS) { 3974 3975 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3976 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3977 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3978 tnf_string, msg, "ibmf_unregister() failed", 3979 tnf_int, ibmf_status, ibmf_status); 3980 3981 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0); 3982 } 3983 3984 bail: 3985 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3986 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3987 3988 return (ibmf_status); 3989 } 3990