1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/ib/mgt/ibmf/ibmf_saa_impl.h> 27 #include <sys/ib/mgt/ibmf/ibmf_saa_utils.h> 28 29 /* Global sa_access State Pointer */ 30 saa_state_t *saa_statep; 31 _NOTE(READ_ONLY_DATA(saa_statep)) 32 33 extern int ibmf_trace_level; 34 35 extern int ibmf_taskq_max_tasks; 36 37 static int 38 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 39 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags); 40 static int 41 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 42 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags); 43 static int 44 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 45 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 46 hrtime_t trans_send_time, int transport_flags); 47 static int ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, 48 boolean_t sleep_flag, ibmf_msg_t **msgp, uint32_t *transport_flagsp, 49 ibmf_retrans_t *ibmf_retransp); 50 static int ibmf_saa_must_purge(saa_port_t *saa_portp); 51 static void ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp); 52 static void ibmf_saa_impl_destroy_port(saa_port_t *saa_portp); 53 static void ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp); 54 static void ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, 55 int status); 56 static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle, 57 void *clnt_private, ibmf_async_event_t event_type); 58 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num); 59 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num); 60 static void ibmf_saa_impl_port_chg(ibt_async_event_t *event); 61 static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num); 62 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed); 63 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 64 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 65 size_t *length, boolean_t sleep_flag); 66 static int ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id); 67 static uint_t ibmf_saa_impl_get_attr_id_length(uint16_t attr_id); 68 static void ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp); 69 static int ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 70 ib_guid_t *guid_ret); 71 static void ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 72 ibt_hca_portinfo_t *portinfop); 73 static void ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, 74 ibmf_msg_t *msgp); 75 static int ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp); 76 77 int ibmf_saa_max_wait_time = IBMF_SAA_MAX_WAIT_TIME_IN_SECS; 78 int ibmf_saa_trans_wait_time = IBMF_SAA_TRANS_WAIT_TIME_IN_SECS; 79 int ibmf_saa_max_resp_time = IBMF_SAA_MAX_RESP_TIME; 80 int ibmf_saa_max_subnet_timeout = IBMF_SAA_MAX_SUBNET_TIMEOUT; 81 82 /* 83 * ibmf_saa_impl_init: 84 * Allocates memory for the ibmf_saa state structure and initializes the taskq. 85 * Called from the modules init() routine. 86 * 87 * Input Arguments 88 * none 89 * 90 * Output Arguments 91 * none 92 * 93 * Returns 94 * IBMF_NO_RESOURCES if taskq could not be created. 95 * IBMF_SUCCESS on success 96 * 97 */ 98 int 99 ibmf_saa_impl_init() 100 { 101 int res; 102 103 /* CONSTCOND */ 104 ASSERT(NO_COMPETING_THREADS); 105 106 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_init_start, 107 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() enter\n"); 108 109 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_statep)) 110 111 saa_statep = kmem_zalloc(sizeof (saa_state_t), KM_SLEEP); 112 113 /* create taskq for notifying event subscribers */ 114 saa_statep->saa_event_taskq = taskq_create( 115 "ibmf_saa_event_taskq", IBMF_TASKQ_NTHREADS, 116 MINCLSYSPRI, 1, ibmf_taskq_max_tasks, TASKQ_DYNAMIC | 117 TASKQ_PREPOPULATE); 118 if (saa_statep->saa_event_taskq == NULL) { 119 120 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L4, 121 ibmf_saa_impl_init_end_err, 122 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init(): %s\n", 123 tnf_string, msg, "event taskq create failed"); 124 125 kmem_free(saa_statep, sizeof (saa_state_t)); 126 127 res = IBMF_NO_RESOURCES; 128 129 goto bail; 130 } 131 132 mutex_init(&saa_statep->saa_port_list_mutex, NULL, MUTEX_DRIVER, 133 NULL); 134 135 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_statep)) 136 137 res = IBMF_SUCCESS; 138 bail: 139 140 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_init_end, 141 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init() exit: status = %d\n", 142 tnf_int, res, res); 143 144 return (res); 145 } 146 147 /* 148 * ibmf_saa_impl_fini: 149 * If there are no registered clients, cleans up all memory associated with the 150 * state, including each of the port list entries. 151 * Called from the modules fini() routine. 152 * 153 * Input Arguments 154 * none 155 * 156 * Output Arguments 157 * none 158 * 159 * Returns 160 * EBUSY if there are outstanding transactions or registered clients 161 * 0 if cleanup was sucessfull 162 * 163 */ 164 int 165 ibmf_saa_impl_fini() 166 { 167 int ret = 0; 168 saa_port_t *saa_portp; 169 saa_port_t *next; 170 171 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_start, 172 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() enter\n"); 173 174 /* make sure there are no registered clients */ 175 mutex_enter(&saa_statep->saa_port_list_mutex); 176 177 saa_portp = saa_statep->saa_port_list; 178 while (saa_portp != NULL) { 179 180 mutex_enter(&saa_portp->saa_pt_mutex); 181 182 if (saa_portp->saa_pt_reference_count > 0) { 183 184 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 185 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 186 "ibmf_saa_impl_fini: %s, port %016" PRIx64 "\n", 187 tnf_string, msg, 188 "cannot unload ibmf_saa. Client on port still" 189 " registered", tnf_opaque, port, 190 saa_portp->saa_pt_port_guid); 191 192 mutex_exit(&saa_portp->saa_pt_mutex); 193 194 mutex_exit(&saa_statep->saa_port_list_mutex); 195 196 ret = EBUSY; 197 goto bail; 198 } 199 200 /* make sure there are no outstanding transactions */ 201 202 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 203 204 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 205 ibmf_saa_impl_fini_err, IBMF_TNF_ERROR, "", 206 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 207 ", num transactions = %d\n", 208 tnf_string, msg, "Cannot unload ibmf_saa." 209 " Outstanding transactions on port.", 210 tnf_opaque, port, 211 saa_portp->saa_pt_port_guid, 212 tnf_uint, outstanding_transactions, 213 saa_portp->saa_pt_num_outstanding_trans); 214 215 mutex_exit(&saa_portp->saa_pt_mutex); 216 217 mutex_exit(&saa_statep->saa_port_list_mutex); 218 219 ret = EBUSY; 220 goto bail; 221 } 222 223 mutex_exit(&saa_portp->saa_pt_mutex); 224 225 saa_portp = saa_portp->next; 226 } 227 228 mutex_exit(&saa_statep->saa_port_list_mutex); 229 230 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(saa_statep->saa_port_list, 231 *saa_portp)) 232 233 /* 234 * no more clients nor pending transaction: 235 * unregister ibmf and destroy port entries 236 */ 237 while (saa_statep->saa_port_list != NULL) { 238 239 saa_portp = saa_statep->saa_port_list; 240 next = saa_portp->next; 241 242 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 243 ibmf_saa_impl_fini, IBMF_TNF_TRACE, "", 244 "ibmf_saa_impl_fini: %s, prefix = %016" PRIx64 "\n", 245 tnf_string, msg, "deinitializing port", 246 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 247 248 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 249 250 mutex_enter(&saa_portp->saa_pt_mutex); 251 252 /* unregister from ibmf */ 253 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 254 255 mutex_exit(&saa_portp->saa_pt_mutex); 256 257 if (ibmf_saa_impl_ibmf_unreg(saa_portp) 258 != IBMF_SUCCESS) { 259 ret = EBUSY; 260 goto bail; 261 } 262 } else 263 mutex_exit(&saa_portp->saa_pt_mutex); 264 265 ibmf_saa_impl_destroy_port(saa_portp); 266 267 saa_statep->saa_port_list = next; 268 } 269 270 taskq_destroy(saa_statep->saa_event_taskq); 271 272 mutex_destroy(&saa_statep->saa_port_list_mutex); 273 274 kmem_free(saa_statep, sizeof (saa_state_t)); 275 276 bail: 277 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_fini_end, 278 IBMF_TNF_TRACE, "", "ibmf_saa_impl_fini() exit\n"); 279 280 return (ret); 281 } 282 283 /* 284 * ibmf_saa_is_valid 285 * Returns true the entry is valid. 286 * 287 * Input Arguments 288 * saa_portp pointer to state structure 289 * add_ref if B_TRUE ref count is incremented on a valid portp 290 * 291 * Output Arguments 292 * none 293 * 294 * Returns 295 * B_TRUE if entry was in a valid state, B_FALSE otherwise 296 */ 297 boolean_t 298 ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref) 299 { 300 boolean_t is_valid = B_TRUE; 301 302 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_start, 303 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() enter\n"); 304 305 mutex_enter(&saa_portp->saa_pt_mutex); 306 307 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID || 308 saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_PURGING) { 309 310 is_valid = B_FALSE; 311 312 } else if (add_ref == B_TRUE) { 313 /* 314 * increment reference count here to ensure that 315 * entry does not get purged behind our backs 316 */ 317 saa_portp->saa_pt_reference_count++; 318 } 319 mutex_exit(&saa_portp->saa_pt_mutex); 320 321 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_is_valid_end, 322 IBMF_TNF_TRACE, "", "ibmf_saa_is_valid() exit\n"); 323 324 return (is_valid); 325 } 326 327 /* 328 * ibmf_saa_must_purge 329 * Determines if we can purge a portp (remove it from the list) based on the 330 * state and number of clients 331 * 332 * Input Arguments 333 * saa_portp pointer to state structure 334 * 335 * Output Arguments 336 * none 337 * 338 * Returns 339 * B_TRUE if the entry can be removed, B_FALSE otherwise 340 */ 341 static int 342 ibmf_saa_must_purge(saa_port_t *saa_portp) 343 { 344 int must_purge = B_FALSE; 345 346 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_start, 347 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() enter\n"); 348 349 mutex_enter(&saa_portp->saa_pt_mutex); 350 351 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_INVALID && 352 saa_portp->saa_pt_reference_count == 0) { 353 354 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_PURGING; 355 must_purge = B_TRUE; 356 } 357 358 mutex_exit(&saa_portp->saa_pt_mutex); 359 360 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_must_purge_end, 361 IBMF_TNF_TRACE, "", "ibmf_saa_must_purge() exit\n"); 362 363 return (must_purge); 364 } 365 366 367 /* 368 * ibmf_saa_impl_purge: 369 * Removes invalid port state entries from the list 370 * 371 * Input Arguments 372 * none 373 * 374 * Output Arguments 375 * none 376 * 377 * Returns 378 * void 379 */ 380 void 381 ibmf_saa_impl_purge() 382 { 383 saa_port_t *cur_portp = NULL; 384 saa_port_t *prev_portp = NULL; 385 saa_port_t *rem_portp = NULL; 386 387 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_start, 388 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() enter\n"); 389 390 mutex_enter(&saa_statep->saa_port_list_mutex); 391 392 cur_portp = saa_statep->saa_port_list; 393 prev_portp = cur_portp; 394 395 while (cur_portp != NULL) { 396 397 if (ibmf_saa_must_purge(cur_portp) == B_TRUE) { 398 399 rem_portp = cur_portp; 400 401 /* unlink entry */ 402 if (cur_portp == saa_statep->saa_port_list) { 403 404 saa_statep->saa_port_list = cur_portp->next; 405 cur_portp = saa_statep->saa_port_list; 406 prev_portp = cur_portp; 407 408 } else { 409 410 prev_portp->next = cur_portp->next; 411 cur_portp = cur_portp->next; 412 } 413 414 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*rem_portp)) 415 416 /* destroy entry */ 417 ASSERT(rem_portp != NULL); 418 ibmf_saa_impl_destroy_port(rem_portp); 419 420 } else { 421 422 prev_portp = cur_portp; 423 cur_portp = cur_portp->next; 424 } 425 } 426 427 mutex_exit(&saa_statep->saa_port_list_mutex); 428 429 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_purge_end, 430 IBMF_TNF_TRACE, "", "ibmf_saa_impl_purge() exit\n"); 431 } 432 433 /* 434 * saa_impl_add_client: 435 * Adds a client for a particular portp. Reference count has been incremented 436 * before this call. It is decremented by saa_impl_add_client() if the call 437 * fails. 438 * 439 * Input Arguments 440 * none 441 * 442 * Output Arguments 443 * none 444 * 445 * Returns 446 * IBMF_BUSY if there are already too many clients registered, 447 * IBMF_BAD_PORT_STATE if the port is invalid (generally because a previous 448 * client failed during registration for this port) 449 * IBMF_SUCCESS otherwise 450 */ 451 int 452 ibmf_saa_impl_add_client(saa_port_t *saa_portp) 453 { 454 int status = IBMF_SUCCESS; 455 456 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_add_client_start, 457 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client() enter\n"); 458 459 mutex_enter(&saa_portp->saa_pt_mutex); 460 461 /* 462 * check that we don't exceed max clients 463 */ 464 if (saa_portp->saa_pt_reference_count > 465 SAA_MAX_CLIENTS_PER_PORT) { 466 467 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 468 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, "", 469 "ibmf_saa_impl_add_client: %s, num_reg_clients %d\n", 470 tnf_string, msg, "too many clients registered for" 471 " port", tnf_uint, num_reg_clients, 472 saa_portp->saa_pt_reference_count); 473 474 status = IBMF_BUSY; 475 goto bail; 476 } 477 478 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 479 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 480 "ibmf_saa_impl_add_client: num_registered_clients %d\n", 481 tnf_uint, num_registered_clients, 482 saa_portp->saa_pt_reference_count); 483 484 /* 485 * wait until anyone who is currently registering 486 * this port with ibmf is done 487 */ 488 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 489 490 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 491 ibmf_saa_impl_add_client, IBMF_TNF_TRACE, "", 492 "ibmf_saa_impl_add_client: %s\n", 493 tnf_string, msg, "someone is registering. waiting" 494 " for them to finish"); 495 496 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 497 &saa_portp->saa_pt_mutex); 498 499 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 500 ibmf_saa_impl_add_client, 501 IBMF_TNF_TRACE, "", "ibmf_saa_impl_add_client: %s\n", 502 tnf_string, msg, "done waiting"); 503 } 504 505 /* 506 * if port isn't ready here, fail. 507 */ 508 if (saa_portp->saa_pt_state != IBMF_SAA_PORT_STATE_READY) { 509 510 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 511 ibmf_saa_impl_add_client_err, IBMF_TNF_ERROR, 512 "", "ibmf_saa_impl_add_client: %s\n", 513 tnf_string, msg, "port state not ready," 514 " removing client."); 515 516 status = IBMF_BAD_PORT_STATE; 517 goto bail; 518 } 519 520 bail: 521 mutex_exit(&saa_portp->saa_pt_mutex); 522 523 if (status != IBMF_SUCCESS) { 524 525 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 526 527 IBMF_SAA_ADD32_KSTATS(saa_portp, 528 clients_reg_failed, 1); 529 530 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 531 532 /* decrementing refcount is last thing we do on entry */ 533 534 mutex_enter(&saa_portp->saa_pt_mutex); 535 536 ASSERT(saa_portp->saa_pt_reference_count > 0); 537 saa_portp->saa_pt_reference_count--; 538 539 mutex_exit(&saa_portp->saa_pt_mutex); 540 } 541 542 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 543 ibmf_saa_impl_add_client_end, IBMF_TNF_TRACE, "", 544 "ibmf_saa_impl_add_client() exit\n"); 545 546 return (status); 547 } 548 549 /* 550 * ibmf_saa_impl_create_port() 551 * Create port entry with mimimal inits because 552 * we're holding the list mutex: NO BLOCKING CALLS HERE, please. 553 * 554 * Initialize port state to "registering", so that clients accessing 555 * same port concurrently will wait for the end of the ibmf registration. 556 * Note: this thread will access port members without locking mutex. 557 * 558 * Input Arguments 559 * pt_guid guid of port 560 * 561 * Output Arguments 562 * saa_portpp pointer to new saa_portp structure 563 * 564 * Returns 565 * IBMF_NO_MEMORY if memory could not be allocated 566 * IBMF_SUCCESS otherwise 567 */ 568 int 569 ibmf_saa_impl_create_port(ib_guid_t pt_guid, saa_port_t **saa_portpp) 570 { 571 int status = IBMF_SUCCESS; 572 saa_port_t *saa_portp = NULL; 573 574 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_create_port_start, 575 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port:" 576 " guid %016" PRIx64 "\n", 577 tnf_opaque, port_guid, pt_guid); 578 579 ASSERT(MUTEX_HELD(&saa_statep->saa_port_list_mutex)); 580 581 /* create & initialize new port */ 582 saa_portp = kmem_zalloc(sizeof (saa_port_t), KM_NOSLEEP); 583 584 if (saa_portp == NULL) { 585 586 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 587 ibmf_sa_session_open_err, IBMF_TNF_ERROR, "", 588 "ibmf_saa_impl_create_port: %s\n", 589 tnf_string, msg, "could not allocate memory for " 590 "new port"); 591 592 status = IBMF_NO_MEMORY; 593 goto bail; 594 } 595 596 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_sa_session_open, 597 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port: %s\n", 598 tnf_string, msg, "first client registering, initializing"); 599 600 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*saa_portp)) 601 602 /* tell everyone that kstats are not initialized */ 603 saa_portp->saa_pt_kstatp = NULL; 604 605 /* 606 * set up mutexe and state variable to indicate to 607 * other clients that were currently in the process of 608 * setting up the port data. This will prevent a subsequent 609 * client from trying to to register with ibmf before the 610 * port data has been initialized. 611 */ 612 mutex_init(&saa_portp->saa_pt_mutex, NULL, MUTEX_DRIVER, NULL); 613 cv_init(&saa_portp->saa_pt_ibmf_reg_cv, NULL, CV_DRIVER, NULL); 614 615 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_REGISTERING; 616 617 /* create other mutexes */ 618 mutex_init(&saa_portp->saa_pt_kstat_mutex, NULL, MUTEX_DRIVER, NULL); 619 620 mutex_init(&saa_portp->saa_pt_event_sub_mutex, NULL, MUTEX_DRIVER, 621 NULL); 622 623 /* 624 * clients assume all arrive; set mask to this so we only notify 625 * if something failed 626 */ 627 saa_portp->saa_pt_event_sub_last_success_mask = 628 IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE; 629 630 /* 631 * set port_guid now so any immediately subsequent clients 632 * registering on this port, guid will know we're already here 633 */ 634 saa_portp->saa_pt_port_guid = pt_guid; 635 saa_portp->saa_pt_reference_count = 1; 636 saa_portp->saa_pt_current_tid = pt_guid << 32; 637 638 saa_portp->saa_pt_redirect_active = B_FALSE; 639 640 /* set sa_uptime now in case we never receive anything from SA */ 641 saa_portp->saa_pt_sa_uptime = gethrtime(); 642 643 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*saa_portp)) 644 645 /* Set new pointer in caller's */ 646 *saa_portpp = saa_portp; 647 648 bail: 649 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_create_port_end, 650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_create_port() exit\n"); 651 652 return (status); 653 } 654 655 /* 656 * ibmf_saa_impl_invalidate_port: 657 * invalidates port entry (assumes exist) and deletes kstat object 658 * kstat object is destroyed in order to allow creating port entry 659 * even if this entry is not purged 660 */ 661 static void 662 ibmf_saa_impl_invalidate_port(saa_port_t *saa_portp) 663 { 664 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 665 ibmf_saa_impl_invalidate_port_start, 666 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() enter\n"); 667 668 ASSERT(saa_portp != NULL); 669 ASSERT(MUTEX_HELD(&saa_portp->saa_pt_mutex)); 670 671 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_INVALID; 672 ibmf_saa_impl_uninit_kstats(saa_portp); 673 674 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 675 ibmf_saa_impl_invalidate_port_end, 676 IBMF_TNF_TRACE, "", "ibmf_saa_impl_invalidate_port() exit\n"); 677 } 678 679 /* 680 * ibmf_saa_impl_destroy_port: 681 * Frees the resources associated with an saa_portp structure. Assumes the 682 * saa_portp exists 683 * 684 * Input Arguments 685 * saa_portp pointer to saa_portp structure 686 * 687 * Output Arguments 688 * none 689 * 690 * Returns 691 * void 692 */ 693 static void 694 ibmf_saa_impl_destroy_port(saa_port_t *saa_portp) 695 { 696 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_start, 697 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() enter\n"); 698 699 ASSERT(saa_portp != NULL); 700 701 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 702 703 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 704 ibmf_saa_impl_destroy, IBMF_TNF_TRACE, "", 705 "ibmf_saa_impl_destroy(): destroying port_guid %016" PRIx64 "\n", 706 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 707 708 ibmf_saa_impl_uninit_kstats(saa_portp); 709 710 /* uninit synchronization variables used for registration */ 711 mutex_destroy(&saa_portp->saa_pt_mutex); 712 cv_destroy(&saa_portp->saa_pt_ibmf_reg_cv); 713 714 mutex_destroy(&saa_portp->saa_pt_event_sub_mutex); 715 mutex_destroy(&saa_portp->saa_pt_kstat_mutex); 716 717 kmem_free(saa_portp, sizeof (saa_port_t)); 718 719 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_destroy_end, 720 IBMF_TNF_TRACE, "", "ibmf_saa_impl_destroy() exit\n"); 721 } 722 723 /* 724 * ibmf_saa_impl_init_kstats: 725 * Create kstats structure. Should be called when memory is alloced for a new 726 * port entry. 727 */ 728 int 729 ibmf_saa_impl_init_kstats(saa_port_t *saa_portp) 730 { 731 char buf[128]; 732 ibmf_saa_kstat_t *ksp; 733 734 _NOTE(ASSUMING_PROTECTED(saa_portp->saa_pt_kstatp)) 735 736 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 737 ibmf_saa_impl_init_kstats_start, 738 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() enter\n"); 739 740 /* set up kstats structure */ 741 (void) sprintf(buf, "ibmf_saa_%016" PRIx64 "_stat", 742 saa_portp->saa_pt_port_guid); 743 744 saa_portp->saa_pt_kstatp = kstat_create("ibmf_saa", 745 0, buf, "misc", KSTAT_TYPE_NAMED, 746 sizeof (ibmf_saa_kstat_t) / sizeof (kstat_named_t), 747 KSTAT_FLAG_WRITABLE); 748 749 if (saa_portp->saa_pt_kstatp == NULL) 750 return (IBMF_NO_RESOURCES); 751 752 ksp = (ibmf_saa_kstat_t *)saa_portp->saa_pt_kstatp->ks_data; 753 754 kstat_named_init(&ksp->clients_registered, 755 "clients_registered", KSTAT_DATA_UINT32); 756 757 kstat_named_init(&ksp->clients_reg_failed, 758 "clients_reg_failed", KSTAT_DATA_UINT32); 759 760 kstat_named_init(&ksp->outstanding_requests, 761 "outstanding_requests", KSTAT_DATA_UINT32); 762 763 kstat_named_init(&ksp->total_requests, 764 "total_requests", KSTAT_DATA_UINT32); 765 766 kstat_named_init(&ksp->failed_requests, 767 "failed_requests", KSTAT_DATA_UINT32); 768 769 kstat_named_init(&ksp->requests_timedout, 770 "requests_timedout", KSTAT_DATA_UINT32); 771 772 kstat_install(saa_portp->saa_pt_kstatp); 773 774 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 775 ibmf_saa_impl_init_kstats_end, 776 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_kstats() exit\n"); 777 778 return (IBMF_SUCCESS); 779 } 780 781 /* 782 * ibmf_saa_impl_uninit_kstats: 783 * Free kstats context. Should be called when port is either destroyed 784 * or invalidated. 785 */ 786 static void 787 ibmf_saa_impl_uninit_kstats(saa_port_t *saa_portp) 788 { 789 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 790 ibmf_saa_impl_uninit_kstats_start, 791 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() enter\n"); 792 793 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 794 795 if (saa_portp->saa_pt_kstatp != NULL) { 796 kstat_delete(saa_portp->saa_pt_kstatp); 797 } 798 saa_portp->saa_pt_kstatp = NULL; 799 800 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 801 802 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 803 ibmf_saa_impl_uninit_kstats_end, 804 IBMF_TNF_TRACE, "", "ibmf_saa_impl_uninit_kstats() exit\n"); 805 } 806 807 /* 808 * ibmf_saa_impl_register_failed: 809 * invalidate entry and kick waiters 810 */ 811 void 812 ibmf_saa_impl_register_failed(saa_port_t *saa_portp) 813 { 814 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 815 ibmf_saa_impl_register_failed_start, 816 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() enter\n"); 817 818 mutex_enter(&saa_portp->saa_pt_mutex); 819 820 ibmf_saa_impl_invalidate_port(saa_portp); 821 822 cv_broadcast(&saa_portp->saa_pt_ibmf_reg_cv); 823 824 /* decrementing refcount is last thing we do on entry */ 825 826 ASSERT(saa_portp->saa_pt_reference_count > 0); 827 saa_portp->saa_pt_reference_count--; 828 829 mutex_exit(&saa_portp->saa_pt_mutex); 830 831 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 832 ibmf_saa_impl_register_failed_end, 833 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_failed() exit\n"); 834 } 835 836 static int 837 ibmf_saa_impl_setup_qp_async_cb(saa_port_t *saa_portp, int setup_async_cb_only) 838 { 839 int status; 840 int unreg_status; 841 ib_pkey_t p_key; 842 ib_qkey_t q_key; 843 uint8_t portnum; 844 boolean_t qp_alloced = B_FALSE; 845 846 if (setup_async_cb_only == 0) { 847 848 /* allocate a qp through ibmf */ 849 status = ibmf_alloc_qp(saa_portp->saa_pt_ibmf_handle, 850 IB_PKEY_DEFAULT_LIMITED, IB_GSI_QKEY, 851 IBMF_ALT_QP_MAD_RMPP, &saa_portp->saa_pt_qp_handle); 852 853 if (status != IBMF_SUCCESS) { 854 855 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 856 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 857 "ibmf_saa_impl_setup_qp_async_cb: %s, " 858 "ibmf_status = %d\n", 859 tnf_string, msg, "Cannot alloc qp with ibmf", 860 tnf_int, status, status); 861 862 return (status); 863 } 864 865 qp_alloced = B_TRUE; 866 867 /* 868 * query the queue pair number; we will need it to unsubscribe 869 * from notice reports 870 */ 871 status = ibmf_query_qp(saa_portp->saa_pt_ibmf_handle, 872 saa_portp->saa_pt_qp_handle, &saa_portp->saa_pt_qpn, 873 &p_key, &q_key, &portnum, 0); 874 875 if (status != IBMF_SUCCESS) { 876 877 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 878 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 879 "ibmf_saa_impl_setup_qp_async_cb: %s, " 880 "ibmf_status = %d\n", 881 tnf_string, msg, 882 "Cannot query alt qp to get qp num", 883 tnf_int, status, status); 884 885 goto bail; 886 } 887 } 888 889 /* 890 * core ibmf is taking advantage of the fact that saa_portp is our 891 * callback arg. If this changes, the code in ibmf_recv would need to 892 * change as well 893 */ 894 status = ibmf_setup_async_cb(saa_portp->saa_pt_ibmf_handle, 895 saa_portp->saa_pt_qp_handle, ibmf_saa_report_cb, saa_portp, 0); 896 if (status != IBMF_SUCCESS) { 897 898 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 899 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 900 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status = %d\n", 901 tnf_string, msg, "Cannot register async cb with ibmf", 902 tnf_int, status, status); 903 904 goto bail; 905 } 906 907 return (IBMF_SUCCESS); 908 909 bail: 910 if (qp_alloced == B_TRUE) { 911 /* free alternate qp */ 912 unreg_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 913 &saa_portp->saa_pt_qp_handle, 0); 914 if (unreg_status != IBMF_SUCCESS) { 915 916 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 917 ibmf_saa_impl_setup_qp_async_cb, IBMF_TNF_ERROR, "", 918 "ibmf_saa_impl_setup_qp_async_cb: %s, ibmf_status =" 919 " %d\n", tnf_string, msg, 920 "Cannot free alternate queue pair with ibmf", 921 tnf_int, unreg_status, unreg_status); 922 } 923 } 924 925 return (status); 926 } 927 928 /* 929 * ibmf_saa_impl_register_port: 930 */ 931 int 932 ibmf_saa_impl_register_port( 933 saa_port_t *saa_portp) 934 { 935 uint_t hca_count = 0; 936 ib_guid_t *hca_list = NULL; 937 int status = IBMF_SUCCESS; 938 int unreg_status = IBMF_SUCCESS; 939 int ibt_status = IBT_SUCCESS; 940 ibt_hca_portinfo_t *port_info_list = NULL; 941 uint_t port_count = 0; 942 uint_t port_size = 0; 943 int ihca, iport; 944 ib_guid_t port_guid; 945 boolean_t ibmf_reg = B_FALSE; 946 947 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 948 ibmf_saa_impl_register_port_start, IBMF_TNF_TRACE, "", 949 "ibmf_saa_impl_register_port() enter\n"); 950 951 ASSERT(saa_portp != NULL); 952 953 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 954 955 /* get the HCA list */ 956 957 hca_count = ibt_get_hca_list(&hca_list); 958 959 if (hca_count == 0) { 960 961 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 962 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 963 "ibmf_saa_impl_register_port: %s\n", 964 tnf_string, msg, "cannot register port (no HCAs).\n"); 965 966 status = IBMF_BAD_PORT; 967 goto bail; 968 } 969 970 /* lookup requested port guid in hca list */ 971 for (ihca = 0; ihca != hca_count; ihca++) { 972 973 ibt_status = ibt_query_hca_ports_byguid(hca_list[ihca], 974 0 /* all ports */, &port_info_list, 975 &port_count, &port_size); 976 977 if (ibt_status != IBT_SUCCESS) { 978 979 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 980 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 981 "ibmf_saa_impl_register_port: %s, %016" PRIx64 "\n", 982 tnf_string, msg, "Could not query hca. Exiting.", 983 tnf_opaque, guid, hca_list[ihca]); 984 985 status = IBMF_TRANSPORT_FAILURE; 986 break; 987 } 988 989 for (iport = 0; iport < port_count; iport++) { 990 991 /* get port guid associated with hca guid, port num */ 992 if (ibmf_saa_impl_get_port_guid( 993 port_info_list + iport, &port_guid) != IBMF_SUCCESS) 994 continue; 995 996 if (saa_portp->saa_pt_port_guid != port_guid) 997 continue; 998 999 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1000 ibmf_saa_impl_register_port, 1001 IBMF_TNF_TRACE, "", 1002 "ibmf_saa_impl_register_port: %s, hca_guid = %016" 1003 PRIx64 ", port_guid = %016" PRIx64 1004 ", number = %d\n", 1005 tnf_string, msg, "found port", 1006 tnf_opaque, hca_guid, hca_list[ihca], 1007 tnf_opaque, port_guid, port_guid, 1008 tnf_uint, port, iport + 1); 1009 1010 /* 1011 * we're here? then we found our port: 1012 * fill in ibmf registration info 1013 * and address parameters from the portinfo 1014 */ 1015 1016 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid 1017 = hca_list[ihca]; 1018 saa_portp->saa_pt_ibmf_reginfo.ir_port_num = iport+1; 1019 saa_portp->saa_pt_ibmf_reginfo.ir_client_class 1020 = SUBN_ADM_MANAGER; 1021 1022 saa_portp->saa_pt_node_guid = hca_list[ihca]; 1023 saa_portp->saa_pt_port_num = iport + 1; 1024 1025 ibmf_saa_impl_set_transaction_params( 1026 saa_portp, port_info_list + iport); 1027 break; 1028 } 1029 1030 ibt_free_portinfo(port_info_list, port_size); 1031 1032 if (iport != port_count) 1033 break; /* found our port */ 1034 } 1035 1036 ibt_free_hca_list(hca_list, hca_count); 1037 1038 if (ihca == hca_count) { 1039 1040 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1041 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1042 "ibmf_saa_impl_register_port: %s, port_guid %016" 1043 PRIx64 "\n", 1044 tnf_string, msg, "Could not find port, exiting", 1045 tnf_opaque, port_guid, saa_portp->saa_pt_port_guid); 1046 1047 status = IBMF_BAD_PORT; 1048 } 1049 1050 if (status != IBMF_SUCCESS) { 1051 1052 goto bail; 1053 } 1054 1055 /* 1056 * Now we found the port we searched for, 1057 * and open an ibmf session on that port. 1058 */ 1059 1060 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1061 ibmf_saa_impl_register_port, IBMF_TNF_TRACE, "", 1062 "ibmf_saa_impl_register_port: %s, port_guid = %016" PRIx64 1063 ", port = %d\n", tnf_string, msg, "Registering with ibmf", 1064 tnf_opaque, port_guid, saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1065 tnf_uint, port, saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1066 1067 status = ibmf_register(&saa_portp->saa_pt_ibmf_reginfo, 1068 IBMF_VERSION, IBMF_REG_FLAG_RMPP, 1069 ibmf_saa_impl_async_event_cb, saa_portp, 1070 &saa_portp->saa_pt_ibmf_handle, 1071 &saa_portp->saa_pt_ibmf_impl_features); 1072 1073 if (status != IBMF_SUCCESS) { 1074 1075 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1076 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1077 "ibmf_saa_impl_register_port: %s, ibmf_status = %d\n", 1078 tnf_string, msg, "Could not register with ibmf", 1079 tnf_int, status, status); 1080 1081 goto bail; 1082 } 1083 1084 ibmf_reg = B_TRUE; 1085 1086 if (ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0) == IBMF_SUCCESS) 1087 return (IBMF_SUCCESS); 1088 1089 bail: 1090 if (ibmf_reg == B_TRUE) { 1091 /* unregister from ibmf */ 1092 unreg_status = ibmf_unregister( 1093 &saa_portp->saa_pt_ibmf_handle, 0); 1094 1095 if (unreg_status != IBMF_SUCCESS) { 1096 1097 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1098 ibmf_saa_impl_register_port, IBMF_TNF_ERROR, "", 1099 "ibmf_saa_impl_register_port: %s, ibmf_status =" 1100 " %d\n", tnf_string, msg, 1101 "Cannot unregister from ibmf", 1102 tnf_int, unreg_status, unreg_status); 1103 } 1104 } 1105 1106 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_register_port_end, 1107 IBMF_TNF_TRACE, "", "ibmf_saa_impl_register_port() exit\n"); 1108 1109 return (status); 1110 } 1111 1112 /* 1113 * ibmf_saa_impl_getclassportinfo: 1114 */ 1115 void 1116 ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp) 1117 { 1118 int res; 1119 saa_impl_trans_info_t *trans_info; 1120 1121 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1122 ibmf_saa_impl_get_classportinfo_start, 1123 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() enter\n"); 1124 1125 /* 1126 * allocate memory for trans_info; send_request's callback will free up 1127 * memory since request is asynchronous 1128 */ 1129 trans_info = kmem_zalloc(sizeof (saa_impl_trans_info_t), KM_NOSLEEP); 1130 if (trans_info == NULL) { 1131 1132 mutex_enter(&saa_portp->saa_pt_mutex); 1133 1134 /* cpi transaction is handled as a client, decrement refcount */ 1135 ASSERT(saa_portp->saa_pt_reference_count > 0); 1136 saa_portp->saa_pt_reference_count--; 1137 1138 mutex_exit(&saa_portp->saa_pt_mutex); 1139 1140 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1141 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_ERROR, "", 1142 "ibmf_saa_impl_get_classportinfo: %s\n", tnf_string, msg, 1143 "Could not allocate memory for classportinfo trans_info"); 1144 1145 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1146 ibmf_saa_impl_get_classportinfo_end, IBMF_TNF_TRACE, "", 1147 "ibmf_saa_impl_get_classportinfo() exiting\n"); 1148 1149 return; 1150 } 1151 1152 /* no specific client associated with this transaction */ 1153 trans_info->si_trans_client_data = NULL; 1154 trans_info->si_trans_port = saa_portp; 1155 trans_info->si_trans_method = SA_SUBN_ADM_GET; 1156 trans_info->si_trans_attr_id = MAD_ATTR_ID_CLASSPORTINFO; 1157 1158 trans_info->si_trans_callback = ibmf_saa_impl_get_cpi_cb; 1159 trans_info->si_trans_callback_arg = saa_portp; 1160 1161 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1162 1163 IBMF_SAA_ADD32_KSTATS(saa_portp, outstanding_requests, 1); 1164 IBMF_SAA_ADD32_KSTATS(saa_portp, total_requests, 1); 1165 1166 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1167 1168 res = ibmf_saa_impl_send_request(trans_info); 1169 1170 if (res != IBMF_SUCCESS) { 1171 1172 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1173 ibmf_saa_impl_get_classportinfo_err, IBMF_TNF_TRACE, "", 1174 "ibmf_saa_impl_get_classportinfo: %s, res = 0x%x\n", 1175 tnf_string, msg, "ibmf_saa_impl_send_request failed", 1176 tnf_opaque, res, res); 1177 1178 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 1179 1180 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 1181 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 1182 1183 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 1184 1185 mutex_enter(&saa_portp->saa_pt_mutex); 1186 1187 /* cpi transaction is handled as a client, decrement refcount */ 1188 ASSERT(saa_portp->saa_pt_reference_count > 0); 1189 saa_portp->saa_pt_reference_count--; 1190 1191 mutex_exit(&saa_portp->saa_pt_mutex); 1192 1193 } 1194 1195 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1196 ibmf_saa_impl_get_classportinfo_end, 1197 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_classportinfo() exit\n"); 1198 } 1199 1200 /* 1201 * ibmf_saa_impl_get_cpi_cb: 1202 * 1203 * Called when the asynchronous getportinfo request receives its response. 1204 * Checks the status. If success, updates the times in the port's 1205 * ibmf_retrans structure that is used in ibmf_msg_transport calls. If failure, 1206 * just use default values. 1207 * 1208 * Input Arguments 1209 * arg user-specified pointer (points to the current port data) 1210 * length length of payload returned (should be size of classportinfo_t) 1211 * buffer pointer to classportinfo returned (should not be null) 1212 * status status of sa access request 1213 * 1214 * Output Arguments 1215 * none 1216 * 1217 * Returns void 1218 */ 1219 static void 1220 ibmf_saa_impl_get_cpi_cb(void *arg, size_t length, char *buffer, int status) 1221 { 1222 saa_port_t *saa_portp; 1223 uint64_t base_time, resp_timeout, rttv_timeout; 1224 ib_mad_classportinfo_t *classportinfo; 1225 int resp_time_value; 1226 uint16_t sa_cap_mask; 1227 1228 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_start, 1229 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() enter\n"); 1230 1231 /* 1232 * access port entry: note that it may have become invalid 1233 * but we hold a ref count for cpi and the interactions on 1234 * the entry are harmless 1235 */ 1236 saa_portp = (saa_port_t *)arg; 1237 1238 /* process response */ 1239 1240 if ((status != IBMF_SUCCESS) || (buffer == NULL)) { 1241 1242 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1243 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_ERROR, "", 1244 "ibmf_saa_impl_get_cpi_cb: %s, status = %d, buffer = " 1245 " 0x%p, length = %d\n", tnf_string, msg, 1246 "could not get classportinfo. Check node and path to sm" 1247 " lid", tnf_int, status, status, 1248 tnf_opaque, buffer, buffer, tnf_uint, length, length); 1249 1250 /* 1251 * IB spec (C13-13) indicates 20 can be used as default or 1252 * intial value for classportinfo->resptimeout value 1253 */ 1254 resp_time_value = 20; 1255 1256 sa_cap_mask = 0xFFFF; 1257 1258 } else if (buffer != NULL) { 1259 1260 classportinfo = (ib_mad_classportinfo_t *)buffer; 1261 1262 resp_time_value = classportinfo->RespTimeValue & 0x1f; 1263 1264 /* 1265 * Because some subnet managers might not provide sane 1266 * value for "resp_time_value", we limit it here. In 1267 * case this limit is too restrictive (very large fabric), 1268 * we allow the limit to be raised (/etc/system). 1269 */ 1270 if (resp_time_value > ibmf_saa_max_resp_time) { 1271 cmn_err(CE_CONT, "!ibmf_saa_max_resp_time (%d) " 1272 "exceeded.", ibmf_saa_max_resp_time); 1273 cmn_err(CE_CONT, "!Reducing subnet administrator " 1274 "resp_time value from %d to %d.", 1275 resp_time_value, ibmf_saa_max_resp_time); 1276 resp_time_value = ibmf_saa_max_resp_time; 1277 } 1278 1279 sa_cap_mask = classportinfo->CapabilityMask; 1280 1281 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1282 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1283 "ibmf_saa_impl_get_cpi_cb: %s, timeout = 0x%x," 1284 " cap_mask = 0x%x\n", 1285 tnf_string, msg, "got classportinfo", 1286 tnf_opaque, timeout, resp_time_value, 1287 tnf_opaque, cap_mask, sa_cap_mask); 1288 1289 kmem_free(buffer, length); 1290 } 1291 1292 /* 1293 * using IB spec calculation from 13.4.6.2 1294 * use bit shifting for 2^x. 1295 */ 1296 base_time = (1 << resp_time_value); 1297 1298 resp_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1299 1300 mutex_enter(&saa_portp->saa_pt_mutex); 1301 1302 base_time = 2 * (1 << saa_portp->saa_pt_timeout); 1303 1304 rttv_timeout = (4 * base_time * 1000 + 96 * base_time) / 1000; 1305 1306 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = resp_timeout; 1307 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = rttv_timeout; 1308 saa_portp->saa_pt_sa_cap_mask = sa_cap_mask; 1309 1310 /* 1311 * cpi transaction is handled as a client, 1312 * decrement refcount; make sure it's the last 1313 * thing we do on this entry 1314 */ 1315 ASSERT(saa_portp->saa_pt_reference_count > 0); 1316 saa_portp->saa_pt_reference_count--; 1317 1318 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1319 ibmf_saa_impl_get_cpi_cb, IBMF_TNF_TRACE, "", 1320 "ibmf_saa_impl_get_cpi_cb: %s, subnet_timeout = 0x%x, " 1321 "resp_time_value = 0x%x\n", 1322 tnf_string, msg, "updated resp timeout", 1323 tnf_opaque, subnet_timeout, saa_portp->saa_pt_timeout, 1324 tnf_opaque, resp_time_value, resp_time_value); 1325 1326 mutex_exit(&saa_portp->saa_pt_mutex); 1327 1328 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_get_cpi_cb_end, 1329 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_cpi_cb() exit\n"); 1330 } 1331 1332 /* 1333 * ibmf_saa_impl_send_request: 1334 * Sends a request to the sa. Can be used for both classportinfo and record 1335 * requests. Will set up all data structures for using the multi-packet 1336 * protocol, create the mad, and send it. Returns SA_SUCCESS if msg transport 1337 * worked, meaning succesful send for the async case and a succesful send and 1338 * recv for the sync case. 1339 */ 1340 int 1341 ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info) 1342 { 1343 uint16_t attr_id; 1344 saa_client_data_t *client_data; 1345 saa_port_t *saa_portp; 1346 uint32_t transport_flags; 1347 ibmf_msg_cb_t ibmf_callback; 1348 void *ibmf_callback_arg; 1349 ibmf_msg_t *msgp; 1350 ibmf_retrans_t ibmf_retrans; 1351 uint16_t sa_cap_mask; 1352 boolean_t sleep_flag; 1353 int ibmf_status = IBMF_SUCCESS; 1354 int retry_count; 1355 uint16_t mad_status; 1356 boolean_t sa_is_redirected = B_FALSE; 1357 1358 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1359 ibmf_saa_impl_send_request_start, 1360 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() enter\n"); 1361 1362 attr_id = trans_info->si_trans_attr_id; 1363 client_data = trans_info->si_trans_client_data; 1364 saa_portp = trans_info->si_trans_port; 1365 1366 /* 1367 * don't send on invalid entry 1368 * Note that there is a window where it could become 1369 * invalid after this test is done, but we'd rely on ibmf errors... 1370 */ 1371 if (ibmf_saa_is_valid(saa_portp, B_FALSE) == B_FALSE) { 1372 1373 IBMF_TRACE_4(IBMF_TNF_NODEBUG, DPRINT_L1, 1374 ibmf_saa_impl_send_request, 1375 IBMF_TNF_ERROR, "", 1376 "ibmf_saa_impl_send_request: %s, hca_guid = %016" 1377 PRIx64 ", port_guid = %016" PRIx64 1378 ", number = %d\n", 1379 tnf_string, msg, "sending on invalid port", 1380 tnf_opaque, hca_guid, 1381 saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid, 1382 tnf_opaque, port_guid, 1383 saa_portp->saa_pt_port_guid, 1384 tnf_uint, port, 1385 saa_portp->saa_pt_ibmf_reginfo.ir_port_num); 1386 1387 ibmf_status = IBMF_REQ_INVALID; 1388 goto bail; 1389 } 1390 1391 /* check whether SA supports this attribute */ 1392 mutex_enter(&saa_portp->saa_pt_mutex); 1393 1394 sa_cap_mask = saa_portp->saa_pt_sa_cap_mask; 1395 sa_is_redirected = saa_portp->saa_pt_redirect_active; 1396 1397 mutex_exit(&saa_portp->saa_pt_mutex); 1398 1399 ibmf_status = ibmf_saa_impl_check_sa_support(sa_cap_mask, attr_id); 1400 1401 if (ibmf_status != IBMF_SUCCESS) { 1402 1403 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1404 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1405 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1406 tnf_string, msg, "SA does not support attribute", 1407 tnf_int, ibmf_status, ibmf_status); 1408 1409 goto bail; 1410 } 1411 1412 /* make only non-blocking calls if this is an async request */ 1413 if ((trans_info->si_trans_callback == NULL) && 1414 (trans_info->si_trans_sub_callback == NULL)) { 1415 ibmf_callback = NULL; 1416 ibmf_callback_arg = NULL; 1417 sleep_flag = B_TRUE; 1418 } else { 1419 ibmf_callback = ibmf_saa_async_cb; 1420 ibmf_callback_arg = (void *)trans_info; 1421 sleep_flag = B_FALSE; 1422 } 1423 1424 ibmf_status = ibmf_saa_impl_init_msg(trans_info, sleep_flag, &msgp, 1425 &transport_flags, &ibmf_retrans); 1426 if (ibmf_status != IBMF_SUCCESS) { 1427 1428 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1429 ibmf_saa_impl_send_request_err, IBMF_TNF_ERROR, "", 1430 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1431 tnf_string, msg, "init_msg() failed", 1432 tnf_int, ibmf_status, ibmf_status); 1433 1434 goto bail; 1435 } 1436 1437 mutex_enter(&saa_portp->saa_pt_mutex); 1438 1439 saa_portp->saa_pt_num_outstanding_trans++; 1440 1441 mutex_exit(&saa_portp->saa_pt_mutex); 1442 1443 /* 1444 * increment the number of outstanding transaction so 1445 * ibmf_close_sa_session() will wait. classportinfo requests 1446 * don't have associated clients so check for valid clientp 1447 */ 1448 if (client_data != NULL) { 1449 1450 mutex_enter(&client_data->saa_client_mutex); 1451 1452 client_data->saa_client_num_pending_trans++; 1453 1454 mutex_exit(&client_data->saa_client_mutex); 1455 } 1456 1457 /* 1458 * make the call to msg_transport. If synchronous and success, 1459 * check that the response mad isn't status busy. If so, repeat the 1460 * call 1461 */ 1462 retry_count = 0; 1463 1464 /* 1465 * set the send time here. We only set this once at the beginning of 1466 * the transaction. Retrying because of busys or mastersmlid changes 1467 * does not change the original send time. It is meant to be an 1468 * absolute time out value and will only be used if there are other 1469 * problems (i.e. a buggy SA) 1470 */ 1471 trans_info->si_trans_send_time = gethrtime(); 1472 1473 for (;;) { 1474 1475 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 1476 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 1477 ibmf_callback, ibmf_callback_arg, transport_flags); 1478 1479 if (ibmf_callback != NULL) 1480 break; 1481 1482 /* 1483 * stop here for non-sequenced transactions since they wouldn't 1484 * receive a timeout or busy response 1485 */ 1486 if (!(transport_flags & IBMF_MSG_TRANS_FLAG_SEQ)) 1487 break; 1488 1489 /* 1490 * if the transaction timed out and this was a synchronous 1491 * request there's a possiblity we were talking to the wrong 1492 * master smlid or that the SA has stopped responding on the 1493 * redirected desination (if redirect is active). 1494 * Check this and retry if necessary. 1495 */ 1496 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1497 (sleep_flag == B_TRUE)) { 1498 if (sa_is_redirected == B_TRUE) { 1499 ibmf_status = ibmf_saa_impl_revert_to_qp1( 1500 saa_portp, msgp, ibmf_callback, 1501 ibmf_callback_arg, transport_flags); 1502 } else { 1503 ibmf_status = ibmf_saa_impl_new_smlid_retry( 1504 saa_portp, msgp, ibmf_callback, 1505 ibmf_callback_arg, transport_flags); 1506 } 1507 } 1508 1509 /* 1510 * if the transaction timed out (and retrying with a new SM LID 1511 * didn't help) check how long it's been since we received an SA 1512 * packet. If it hasn't been max_wait_time then retry the 1513 * request. 1514 */ 1515 if ((ibmf_status == IBMF_TRANS_TIMEOUT) && 1516 (sleep_flag == B_TRUE)) { 1517 1518 ibmf_status = ibmf_saa_check_sa_and_retry( 1519 saa_portp, msgp, ibmf_callback, ibmf_callback_arg, 1520 trans_info->si_trans_send_time, transport_flags); 1521 } 1522 1523 if (ibmf_status != IBMF_SUCCESS) 1524 break; 1525 1526 if (retry_count >= IBMF_SAA_MAX_BUSY_RETRY_COUNT) 1527 break; 1528 1529 /* sync transaction with status SUCCESS should have response */ 1530 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 1531 1532 mad_status = b2h16(msgp->im_msgbufs_recv. 1533 im_bufs_mad_hdr->Status); 1534 1535 if ((mad_status != MAD_STATUS_BUSY) && 1536 (mad_status != MAD_STATUS_REDIRECT_REQUIRED)) 1537 break; 1538 1539 if (mad_status == MAD_STATUS_REDIRECT_REQUIRED) { 1540 1541 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1542 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1543 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1544 tnf_string, msg, 1545 "response returned redirect status", 1546 tnf_int, retry_count, retry_count); 1547 1548 /* update address info and copy it into msgp */ 1549 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 1550 } else { 1551 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1552 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1553 "ibmf_saa_impl_send_request: %s, retry_count %d\n", 1554 tnf_string, msg, "response returned busy status", 1555 tnf_int, retry_count, retry_count); 1556 } 1557 1558 retry_count++; 1559 1560 /* 1561 * since this is a blocking call, sleep for some time 1562 * to allow SA to transition from busy state (if busy) 1563 */ 1564 if (mad_status == MAD_STATUS_BUSY) 1565 delay(drv_usectohz( 1566 IBMF_SAA_BUSY_RETRY_SLEEP_SECS * 1000000)); 1567 } 1568 1569 if (ibmf_status != IBMF_SUCCESS) { 1570 1571 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 1572 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1573 "ibmf_saa_impl_send_request: %s, ibmf_status = %d\n", 1574 tnf_string, msg, "ibmf_msg_transport() failed", 1575 tnf_int, ibmf_status, ibmf_status); 1576 1577 ibmf_saa_impl_free_msg(saa_portp->saa_pt_ibmf_handle, msgp); 1578 1579 mutex_enter(&saa_portp->saa_pt_mutex); 1580 1581 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1582 saa_portp->saa_pt_num_outstanding_trans--; 1583 1584 mutex_exit(&saa_portp->saa_pt_mutex); 1585 1586 if (client_data != NULL) { 1587 1588 mutex_enter(&client_data->saa_client_mutex); 1589 1590 ASSERT(client_data->saa_client_num_pending_trans > 0); 1591 client_data->saa_client_num_pending_trans--; 1592 1593 if ((client_data->saa_client_num_pending_trans == 0) && 1594 (client_data->saa_client_state == 1595 SAA_CLIENT_STATE_WAITING)) 1596 cv_signal(&client_data->saa_client_state_cv); 1597 1598 mutex_exit(&client_data->saa_client_mutex); 1599 } 1600 1601 } else if (sleep_flag == B_TRUE) { 1602 1603 mutex_enter(&saa_portp->saa_pt_mutex); 1604 1605 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 1606 saa_portp->saa_pt_num_outstanding_trans--; 1607 1608 mutex_exit(&saa_portp->saa_pt_mutex); 1609 1610 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1611 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1612 "ibmf_saa_impl_send_request: %s\n", 1613 tnf_string, msg, "Message sent and received successfully"); 1614 1615 /* fill in response values and free the message */ 1616 ibmf_saa_impl_prepare_response(saa_portp->saa_pt_ibmf_handle, 1617 msgp, B_FALSE, &trans_info->si_trans_status, 1618 &trans_info->si_trans_result, 1619 &trans_info->si_trans_length, sleep_flag); 1620 1621 if (client_data != NULL) { 1622 mutex_enter(&client_data->saa_client_mutex); 1623 1624 ASSERT(client_data->saa_client_num_pending_trans > 0); 1625 client_data->saa_client_num_pending_trans--; 1626 1627 if ((client_data->saa_client_num_pending_trans == 0) && 1628 (client_data->saa_client_state == 1629 SAA_CLIENT_STATE_WAITING)) 1630 cv_signal(&client_data->saa_client_state_cv); 1631 1632 mutex_exit(&client_data->saa_client_mutex); 1633 } 1634 } else { 1635 1636 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1637 ibmf_saa_impl_send_request, IBMF_TNF_TRACE, "", 1638 "ibmf_saa_impl_send_request: %s\n", 1639 tnf_string, msg, "Message sent successfully"); 1640 } 1641 1642 bail: 1643 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1644 ibmf_saa_impl_send_request_end, 1645 IBMF_TNF_TRACE, "", "ibmf_saa_impl_send_request() exiting" 1646 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1647 1648 return (ibmf_status); 1649 } 1650 1651 /* 1652 * ibmf_saa_impl_init_msg: 1653 * Allocates an ibmf message and fills out the header fields and formatted data 1654 * fields. Also sets up the correct transport_flags and retrans argument for 1655 * the message transport call based on the request information. 1656 * 1657 * Input Arguments 1658 * trans_info saa_trans_info structure passed to send_request 1659 * sleep_flag B_TRUE if init_msg can sleep in function calls 1660 * 1661 * Output Arguments 1662 * msgp ibmf message that should be given to msg_transport 1663 * transport_flagsp transport flags that should be given to msg_transport 1664 * ibmf_retrans_t retrans parameter that should be given to msg_transport 1665 * 1666 * Returns 1667 * ibmf_status 1668 */ 1669 static int 1670 ibmf_saa_impl_init_msg(saa_impl_trans_info_t *trans_info, boolean_t sleep_flag, 1671 ibmf_msg_t **msgp, uint32_t *transport_flagsp, 1672 ibmf_retrans_t *ibmf_retransp) 1673 { 1674 int ibmf_status; 1675 ibmf_msg_bufs_t *req_mad; 1676 ib_mad_hdr_t *mad_hdr; 1677 int ibmf_sleep_flag, km_sleep_flag; 1678 int free_res; 1679 ib_sa_hdr_t sa_hdr; 1680 ibmf_msg_t *ibmf_msg; 1681 uint16_t attr_id, pack_attr_id; 1682 uint8_t method; 1683 saa_client_data_t *client_data; 1684 saa_port_t *saa_portp; 1685 sa_multipath_record_t *multipath_template; 1686 size_t payload_length; 1687 uint32_t transport_flags; 1688 1689 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1690 ibmf_saa_impl_init_msg_start, 1691 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() entering\n"); 1692 1693 attr_id = trans_info->si_trans_attr_id; 1694 method = trans_info->si_trans_method; 1695 client_data = trans_info->si_trans_client_data; 1696 saa_portp = trans_info->si_trans_port; 1697 1698 if (sleep_flag == B_TRUE) { 1699 ibmf_sleep_flag = IBMF_ALLOC_SLEEP; 1700 km_sleep_flag = KM_SLEEP; 1701 } else { 1702 ibmf_sleep_flag = IBMF_ALLOC_NOSLEEP; 1703 km_sleep_flag = KM_NOSLEEP; 1704 } 1705 1706 ibmf_status = ibmf_alloc_msg(saa_portp->saa_pt_ibmf_handle, 1707 ibmf_sleep_flag, &ibmf_msg); 1708 if (ibmf_status != IBMF_SUCCESS) { 1709 1710 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1711 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1712 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1713 tnf_string, msg, "Cannot allocate msg_buf.", 1714 tnf_int, ibmf_status, ibmf_status); 1715 1716 goto bail; 1717 } 1718 1719 req_mad = &ibmf_msg->im_msgbufs_send; 1720 1721 /* create a template (SA MAD) */ 1722 mad_hdr = kmem_zalloc(sizeof (ib_mad_hdr_t), km_sleep_flag); 1723 1724 if (mad_hdr == NULL) { 1725 1726 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1727 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1728 "ibmf_saa_impl_init_msg: %s\n", 1729 tnf_string, msg, "Cannot allocate mad header."); 1730 1731 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1732 &ibmf_msg); 1733 ASSERT(free_res == IBMF_SUCCESS); 1734 1735 ibmf_status = IBMF_NO_MEMORY; 1736 goto bail; 1737 } 1738 1739 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mad_hdr, 1740 *req_mad)) 1741 1742 bzero(mad_hdr, sizeof (ib_mad_hdr_t)); 1743 mad_hdr->BaseVersion = SAA_MAD_BASE_VERSION; 1744 mad_hdr->MgmtClass = MAD_MGMT_CLASS_SUBN_ADM; 1745 mad_hdr->ClassVersion = SAA_MAD_CLASS_VERSION; 1746 mad_hdr->R_Method = method; 1747 mad_hdr->AttributeID = h2b16(attr_id); 1748 1749 /* attribute modifier is all Fs since RIDs are no longer used */ 1750 mad_hdr->AttributeModifier = h2b32(0xffffffff); 1751 1752 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 1753 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1754 "ibmf_saa_impl_init_msg: %s, class = 0x%x, method = 0x%x," 1755 " attr_id = 0x%x\n", tnf_string, msg, "Sending MAD", 1756 tnf_opaque, class, mad_hdr->MgmtClass, 1757 tnf_opaque, method, mad_hdr->R_Method, 1758 tnf_opaque, attr_id, attr_id); 1759 1760 bzero(&sa_hdr, sizeof (ib_sa_hdr_t)); 1761 sa_hdr.ComponentMask = trans_info->si_trans_component_mask; 1762 1763 if (client_data != NULL) 1764 sa_hdr.SM_KEY = client_data->saa_client_sm_key; 1765 1766 /* 1767 * pack data for IB wire format; req_mad will have different pointers to 1768 * sa header and payload, mad_hdr will be the same 1769 */ 1770 req_mad->im_bufs_mad_hdr = mad_hdr; 1771 1772 ibmf_status = ibmf_saa_utils_pack_sa_hdr(&sa_hdr, 1773 &req_mad->im_bufs_cl_hdr, &req_mad->im_bufs_cl_hdr_len, 1774 km_sleep_flag); 1775 1776 if (ibmf_status != IBMF_SUCCESS) { 1777 1778 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1779 ibmf_saa_impl_init_msg, IBMF_TNF_ERROR, "", 1780 "ibmf_saa_impl_init_msg: %s, ibmf_status = %d\n", 1781 tnf_string, msg, "ibmf_saa_utils_pack_sa_hdr() failed", 1782 tnf_int, ibmf_status, ibmf_status); 1783 1784 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1785 1786 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1787 &ibmf_msg); 1788 ASSERT(free_res == IBMF_SUCCESS); 1789 1790 goto bail; 1791 } 1792 1793 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 1794 1795 multipath_template = 1796 (sa_multipath_record_t *)trans_info->si_trans_template; 1797 1798 payload_length = sizeof (sa_multipath_record_t) + 1799 ((multipath_template->SGIDCount + 1800 multipath_template->DGIDCount) * sizeof (ib_gid_t)); 1801 1802 pack_attr_id = attr_id; 1803 } else { 1804 1805 /* trace record template is a path record */ 1806 pack_attr_id = (attr_id == SA_TRACERECORD_ATTRID) ? 1807 SA_PATHRECORD_ATTRID : attr_id; 1808 1809 payload_length = ibmf_saa_impl_get_attr_id_length(pack_attr_id); 1810 1811 if (payload_length == 0) { 1812 payload_length = trans_info->si_trans_template_length; 1813 1814 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 1815 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1816 "ibmf_saa_impl_init_msg: %s, length = %d\n", 1817 tnf_string, msg, 1818 "Unknown attribute. Using user-defined length.", 1819 tnf_uint, length, payload_length) 1820 } 1821 } 1822 1823 /* transport type depends on method */ 1824 switch (method) { 1825 1826 case SA_SUBN_ADM_GET: 1827 case SA_SUBN_ADM_DELETE: 1828 case SA_SUBN_ADM_GET_TABLE: 1829 case SA_SUBN_ADM_GET_TRACE_TABLE: 1830 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1831 break; 1832 case SA_SUBN_ADM_SET: 1833 /* unsubscribes can be sequenced or unsequenced */ 1834 if (trans_info->si_trans_unseq_unsubscribe == B_TRUE) { 1835 transport_flags = 0; 1836 } else { 1837 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ; 1838 } 1839 break; 1840 case SA_SUBN_ADM_GET_MULTI: 1841 transport_flags = IBMF_MSG_TRANS_FLAG_SEQ | 1842 IBMF_MSG_TRANS_FLAG_RMPP; 1843 break; 1844 default : 1845 ibmf_status = IBMF_UNSUPP_METHOD; 1846 goto bail; 1847 } 1848 1849 trans_info->si_trans_transport_flags = transport_flags; 1850 1851 if (trans_info->si_trans_template != NULL) { 1852 1853 ibmf_status = ibmf_saa_utils_pack_payload( 1854 trans_info->si_trans_template, payload_length, pack_attr_id, 1855 &req_mad->im_bufs_cl_data, &req_mad->im_bufs_cl_data_len, 1856 km_sleep_flag); 1857 if (ibmf_status != IBMF_SUCCESS) { 1858 1859 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 1860 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1861 "ibmf_saa_impl_init_msg: %s, ibmf_status =" 1862 " %d\n", tnf_string, msg, 1863 "ibmf_saa_utils_pack_payload() failed", 1864 tnf_int, ibmf_status, ibmf_status); 1865 1866 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1867 1868 kmem_free(req_mad->im_bufs_cl_hdr, 1869 req_mad->im_bufs_cl_hdr_len); 1870 1871 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1872 &ibmf_msg); 1873 ASSERT(free_res == IBMF_SUCCESS); 1874 1875 goto bail; 1876 } 1877 1878 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 1879 ibmf_saa_impl_init_msg, IBMF_TNF_TRACE, "", 1880 "ibmf_saa_impl_init_msg: %s, attr_id = 0x%x, length =" 1881 " %d\n", tnf_string, msg, "Packed payload successfully", 1882 tnf_opaque, attr_id, attr_id, 1883 tnf_uint, length, req_mad->im_bufs_cl_data_len); 1884 1885 /* non-RMPP transactions have template size limit */ 1886 if (((transport_flags & IBMF_MSG_TRANS_FLAG_RMPP) == 0) && 1887 ((req_mad->im_bufs_cl_data_len + req_mad->im_bufs_cl_hdr_len 1888 + sizeof (ib_mad_hdr_t)) > IBMF_MAD_SIZE)) { 1889 1890 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 1891 ibmf_saa_impl_init_msg_err, IBMF_TNF_ERROR, "", 1892 "ibmf_saa_impl_init_msg: %s\n", tnf_string, msg, 1893 "Template too large to fit in single packet"); 1894 1895 kmem_free(mad_hdr, sizeof (ib_mad_hdr_t)); 1896 1897 kmem_free(req_mad->im_bufs_cl_hdr, 1898 req_mad->im_bufs_cl_hdr_len); 1899 1900 kmem_free(req_mad->im_bufs_cl_data, 1901 req_mad->im_bufs_cl_data_len); 1902 1903 free_res = ibmf_free_msg(saa_portp->saa_pt_ibmf_handle, 1904 &ibmf_msg); 1905 ASSERT(free_res == IBMF_SUCCESS); 1906 1907 ibmf_status = IBMF_REQ_INVALID; 1908 goto bail; 1909 } 1910 } 1911 1912 mutex_enter(&saa_portp->saa_pt_mutex); 1913 1914 mad_hdr->TransactionID = h2b64(saa_portp->saa_pt_current_tid++); 1915 1916 bcopy(&saa_portp->saa_pt_ibmf_retrans, ibmf_retransp, 1917 sizeof (ibmf_retrans_t)); 1918 1919 /* copy local addressing information to message */ 1920 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &ibmf_msg->im_local_addr, 1921 sizeof (ibmf_addr_info_t)); 1922 1923 /* copy global addressing information to message if in use */ 1924 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 1925 1926 ibmf_msg->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 1927 1928 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 1929 &ibmf_msg->im_global_addr, 1930 sizeof (ibmf_global_addr_info_t)); 1931 } else { 1932 ibmf_msg->im_msg_flags = 0; 1933 } 1934 1935 mutex_exit(&saa_portp->saa_pt_mutex); 1936 1937 *msgp = ibmf_msg; 1938 *transport_flagsp = transport_flags; 1939 bail: 1940 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 1941 ibmf_saa_impl_init_msg_end, 1942 IBMF_TNF_TRACE, "", "ibmf_saa_impl_init_msg() exiting" 1943 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 1944 1945 return (ibmf_status); 1946 1947 } 1948 1949 /* 1950 * ibmf_saa_impl_new_smlid_retry: 1951 * 1952 * It's possible for the MasterSMLID to change while ibmf_saa is running. The 1953 * MasterSMLID is set when we first register with ibmf_saa. If a request 1954 * timesout, this function should be called to check whether the SM LID changed. 1955 * If so, it will call msg_transport again with the request. 1956 * 1957 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 1958 * same values passed to the original ibmf_msg_transport that timedout. The 1959 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 1960 * 1961 * If the lid did not change then this function returns IBMF_TRANS_TIMEOUT. 1962 * That way, callers can simply return the result of this function. 1963 * 1964 * Input Arguments 1965 * saa_portp pointer to saa_port structure 1966 * msgp ibmf message that timedout 1967 * ibmf_callback callback that should be called by msg_transport 1968 * ibmf_callback_arg args for ibmf_callback 1969 * transport_flags flags for ibmf_msg_transport 1970 * 1971 * Output Arguments 1972 * none 1973 * 1974 * Returns 1975 * IBMF_SUCCESS if lid changed and request was resent successfully, 1976 * IBMF_TRANS_TIMEOUT if lid did not change, 1977 * same values as ibmf_msg_transport() if lid changed but request could not be 1978 * resent. 1979 */ 1980 static int 1981 ibmf_saa_impl_new_smlid_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 1982 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, int transport_flags) 1983 { 1984 ibt_hca_portinfo_t *ibt_portinfop; 1985 ib_lid_t master_sm_lid; 1986 int subnet_timeout; 1987 uint_t nports, size; 1988 ibmf_retrans_t ibmf_retrans; 1989 int ibmf_status; 1990 ibt_status_t ibt_status; 1991 1992 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 1993 ibmf_saa_impl_new_smlid_retry_start, 1994 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() enter\n"); 1995 1996 _NOTE(ASSUMING_PROTECTED(*msgp)) 1997 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 1998 1999 /* first query the portinfo to see if the lid changed */ 2000 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2001 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2002 2003 if (ibt_status != IBT_SUCCESS) { 2004 2005 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2006 ibmf_saa_impl_new_smlid_retry_err, IBMF_TNF_ERROR, "", 2007 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status =" 2008 " %d\n", tnf_string, msg, 2009 "ibt_query_hca_ports_byguid() failed", 2010 tnf_int, ibt_status, ibt_status); 2011 2012 ibmf_status = IBMF_TRANSPORT_FAILURE; 2013 2014 goto bail; 2015 } 2016 2017 master_sm_lid = ibt_portinfop->p_sm_lid; 2018 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2019 2020 ibt_free_portinfo(ibt_portinfop, size); 2021 2022 /* if master smlid is different than the remote lid we sent to */ 2023 if (master_sm_lid != msgp->im_local_addr.ia_remote_lid) { 2024 2025 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L2, 2026 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2027 "ibmf_saa_impl_new_smlid_retry: %s, new_lid 0x%x," 2028 " old_lid 0x%x\n", tnf_string, msg, 2029 "master smlid has changed. retrying msg_transport", 2030 tnf_opaque, new_lid, master_sm_lid, 2031 tnf_opaque, old_lid, msgp->im_local_addr.ia_remote_lid); 2032 2033 mutex_enter(&saa_portp->saa_pt_mutex); 2034 2035 /* update the master sm lid value in ibmf_saa */ 2036 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2037 master_sm_lid; 2038 2039 /* new tid needed */ 2040 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2041 h2b64(saa_portp->saa_pt_current_tid++); 2042 2043 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2044 sizeof (ibmf_retrans_t)); 2045 2046 /* update the subnet timeout since this may be a new sm/sa */ 2047 saa_portp->saa_pt_timeout = subnet_timeout; 2048 2049 /* place upper bound on subnet timeout in case of faulty SM */ 2050 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2051 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2052 2053 /* increment the reference count to account for the cpi call */ 2054 saa_portp->saa_pt_reference_count++; 2055 2056 mutex_exit(&saa_portp->saa_pt_mutex); 2057 2058 /* update the remote lid for this particular message */ 2059 msgp->im_local_addr.ia_remote_lid = master_sm_lid; 2060 2061 /* get the classportinfo again since this may be a new sm/sa */ 2062 ibmf_saa_impl_get_classportinfo(saa_portp); 2063 2064 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2065 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2066 ibmf_callback, ibmf_callback_arg, transport_flags); 2067 2068 if (ibmf_status != IBMF_SUCCESS) { 2069 2070 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2071 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2072 "ibmf_saa_impl_new_smlid_retry: %s, ibmf_status = " 2073 "%d\n", tnf_string, msg, 2074 "ibmf_msg_transport() failed", 2075 tnf_int, ibmf_status, ibmf_status); 2076 } 2077 2078 goto bail; 2079 } 2080 2081 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2082 ibmf_saa_impl_new_smlid_retry, IBMF_TNF_TRACE, "", 2083 "ibmf_saa_impl_new_smlid_retry: %s, master_smlid = 0x%x\n", 2084 tnf_string, msg, 2085 "master smlid did not change. returning failure", 2086 tnf_opaque, master_smlid, master_sm_lid); 2087 2088 /* mark status as timeout since that was original failure */ 2089 ibmf_status = IBMF_TRANS_TIMEOUT; 2090 2091 bail: 2092 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2093 ibmf_saa_impl_new_smlid_retry_end, 2094 IBMF_TNF_TRACE, "", "ibmf_saa_impl_new_smlid_retry() exiting" 2095 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2096 2097 return (ibmf_status); 2098 } 2099 2100 /* 2101 * ibmf_saa_impl_revert_to_qp1() 2102 * 2103 * The SA that we had contact with via redirect may fail to respond. If this 2104 * occurs SA should revert back to qp1 and the SMLID set in the port. 2105 * msg_transport for the message that timed out will be retried with 2106 * these new parameters. 2107 * 2108 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 2109 * same values passed to the original ibmf_msg_transport that timedout. The 2110 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 2111 * 2112 * Input Arguments 2113 * saa_portp pointer to saa_port structure 2114 * msgp ibmf message that timedout 2115 * ibmf_callback callback that should be called by msg_transport 2116 * ibmf_callback_arg args for ibmf_callback 2117 * transport_flags flags for ibmf_msg_transport 2118 * 2119 * Output Arguments 2120 * none 2121 * 2122 * Returns 2123 * none 2124 */ 2125 static int 2126 ibmf_saa_impl_revert_to_qp1(saa_port_t *saa_portp, ibmf_msg_t *msgp, 2127 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_args, int transport_flags) 2128 { 2129 ibt_hca_portinfo_t *ibt_portinfop; 2130 ib_lid_t master_sm_lid, base_lid; 2131 uint8_t sm_sl; 2132 int subnet_timeout; 2133 uint_t nports, size; 2134 ibmf_retrans_t ibmf_retrans; 2135 int ibmf_status; 2136 ibt_status_t ibt_status; 2137 2138 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 2139 ibmf_saa_impl_revert_to_qp1_start, 2140 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() enter\n"); 2141 2142 _NOTE(ASSUMING_PROTECTED(*msgp)) 2143 _NOTE(ASSUMING_PROTECTED(*msgp->im_msgbufs_send.im_bufs_mad_hdr)) 2144 2145 /* first query the portinfo to see if the lid changed */ 2146 ibt_status = ibt_query_hca_ports_byguid(saa_portp->saa_pt_node_guid, 2147 saa_portp->saa_pt_port_num, &ibt_portinfop, &nports, &size); 2148 2149 if (ibt_status != IBT_SUCCESS) { 2150 2151 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2152 ibmf_saa_impl_revert_to_qp1_err, IBMF_TNF_ERROR, "", 2153 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status =" 2154 " %d\n", tnf_string, msg, 2155 "ibt_query_hca_ports_byguid() failed", 2156 tnf_int, ibt_status, ibt_status); 2157 2158 ibmf_status = IBMF_TRANSPORT_FAILURE; 2159 2160 goto bail; 2161 } 2162 2163 master_sm_lid = ibt_portinfop->p_sm_lid; 2164 base_lid = ibt_portinfop->p_base_lid; 2165 sm_sl = ibt_portinfop->p_sm_sl; 2166 subnet_timeout = ibt_portinfop->p_subnet_timeout; 2167 2168 ibt_free_portinfo(ibt_portinfop, size); 2169 2170 2171 mutex_enter(&saa_portp->saa_pt_mutex); 2172 2173 saa_portp->saa_pt_redirect_active = B_FALSE; 2174 2175 /* update the address info in ibmf_saa */ 2176 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = base_lid; 2177 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = master_sm_lid; 2178 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = sm_sl; 2179 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 2180 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = IB_PKEY_DEFAULT_LIMITED; 2181 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 2182 saa_portp->saa_pt_ibmf_msg_flags = 0; 2183 2184 /* new tid needed */ 2185 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 2186 h2b64(saa_portp->saa_pt_current_tid++); 2187 2188 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2189 sizeof (ibmf_retrans_t)); 2190 2191 /* update the subnet timeout since this may be a new sm/sa */ 2192 saa_portp->saa_pt_timeout = subnet_timeout; 2193 2194 /* place upper bound on subnet timeout in case of faulty SM */ 2195 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 2196 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 2197 2198 /* increment the reference count to account for the cpi call */ 2199 saa_portp->saa_pt_reference_count++; 2200 2201 mutex_exit(&saa_portp->saa_pt_mutex); 2202 2203 /* update the address info for this particular message */ 2204 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 2205 sizeof (ibmf_addr_info_t)); 2206 msgp->im_msg_flags = 0; /* No GRH */ 2207 2208 /* get the classportinfo again since this may be a new sm/sa */ 2209 ibmf_saa_impl_get_classportinfo(saa_portp); 2210 2211 ibmf_status = ibmf_msg_transport(saa_portp->saa_pt_ibmf_handle, 2212 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2213 ibmf_callback, ibmf_callback_args, transport_flags); 2214 2215 if (ibmf_status != IBMF_SUCCESS) { 2216 2217 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2218 ibmf_saa_impl_revert_to_qp1, IBMF_TNF_TRACE, "", 2219 "ibmf_saa_impl_revert_to_qp1: %s, ibmf_status = " 2220 "%d\n", tnf_string, msg, 2221 "ibmf_msg_transport() failed", 2222 tnf_int, ibmf_status, ibmf_status); 2223 } 2224 2225 bail: 2226 2227 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2228 ibmf_saa_impl_revert_to_qp1_end, 2229 IBMF_TNF_TRACE, "", "ibmf_saa_impl_revert_to_qp1() exiting" 2230 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 2231 2232 return (ibmf_status); 2233 } 2234 2235 /* 2236 * ibmf_saa_impl_async_event_cb: 2237 * ibmf event callback, argument to ibmf_register 2238 * ibmf_handle is unused 2239 */ 2240 /* ARGSUSED */ 2241 static void 2242 ibmf_saa_impl_async_event_cb( 2243 ibmf_handle_t ibmf_handle, 2244 void *clnt_private, 2245 ibmf_async_event_t event_type) 2246 { 2247 saa_port_t *saa_portp; 2248 2249 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2250 ibmf_saa_impl_async_event_cb_start, IBMF_TNF_TRACE, "", 2251 "ibmf_saa_impl_async_event_cb: Handling event type 0x%x\n", 2252 tnf_opaque, event_type, event_type); 2253 2254 saa_portp = (saa_port_t *)clnt_private; 2255 ASSERT(saa_portp != NULL); 2256 2257 switch (event_type) { 2258 2259 case IBMF_CI_OFFLINE: 2260 ibmf_saa_impl_hca_detach(saa_portp); 2261 break; 2262 default: 2263 break; 2264 } 2265 2266 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 2267 ibmf_saa_impl_async_event_cb_end, 2268 IBMF_TNF_TRACE, "", "ibmf_saa_impl_async_event_cb() exit\n"); 2269 } 2270 2271 2272 /* 2273 * ibmf_saa_impl_ibt_async_handler: 2274 * MUST NOT BE STATIC (referred from within IBMF) 2275 */ 2276 void 2277 ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event) 2278 { 2279 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2280 ibmf_saa_impl_ibt_async_handler_start, IBMF_TNF_TRACE, "", 2281 "ibmf_saa_impl_ibt_async_handler: Handling event code 0x%x\n", 2282 tnf_opaque, code, code); 2283 2284 switch (code) { 2285 2286 case IBT_EVENT_PORT_UP: 2287 ibmf_saa_impl_port_up(event->ev_hca_guid, event->ev_port); 2288 break; 2289 case IBT_ERROR_PORT_DOWN: 2290 ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port); 2291 break; 2292 case IBT_PORT_CHANGE_EVENT: 2293 ibmf_saa_impl_port_chg(event); 2294 break; 2295 case IBT_CLNT_REREG_EVENT: 2296 ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port); 2297 break; 2298 default: 2299 break; 2300 } 2301 2302 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_async_handler_end, 2303 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibt_async_handler() exit\n"); 2304 } 2305 2306 /* 2307 * ibmf_saa_impl_port_chg: 2308 */ 2309 static void 2310 ibmf_saa_impl_port_chg(ibt_async_event_t *event) 2311 { 2312 saa_port_t *saa_portp = NULL; 2313 boolean_t is_ready = B_FALSE; 2314 ibt_hca_portinfo_t *ibt_portinfop; 2315 uint_t nports, size; 2316 ibt_status_t ibt_status; 2317 ib_guid_t ci_guid; 2318 int port_num; 2319 2320 ci_guid = event->ev_hca_guid; 2321 port_num = event->ev_port; 2322 2323 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start, 2324 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg" 2325 " guid %016" PRIx64 " port %d\n", 2326 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2327 2328 /* Get classportinfo of corresponding entry */ 2329 mutex_enter(&saa_statep->saa_port_list_mutex); 2330 2331 saa_portp = saa_statep->saa_port_list; 2332 while (saa_portp != NULL) { 2333 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2334 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2335 mutex_enter(&saa_portp->saa_pt_mutex); 2336 2337 is_ready = (saa_portp->saa_pt_state 2338 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2339 2340 /* 2341 * increment reference count to account for cpi and 2342 * informinfos. All 4 informinfo's sent are treated as 2343 * one port client reference 2344 */ 2345 if (is_ready) 2346 saa_portp->saa_pt_reference_count ++; 2347 2348 mutex_exit(&saa_portp->saa_pt_mutex); 2349 2350 if (is_ready) 2351 break; /* normally, only 1 port entry */ 2352 } 2353 saa_portp = saa_portp->next; 2354 } 2355 2356 mutex_exit(&saa_statep->saa_port_list_mutex); 2357 2358 if (saa_portp != NULL) { 2359 /* first query the portinfo to see if the lid changed */ 2360 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2361 &ibt_portinfop, &nports, &size); 2362 2363 if (ibt_status != IBT_SUCCESS) { 2364 2365 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2366 ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "", 2367 "ibmf_saa_impl_port_chg: %s, ibmf_status =" 2368 " %d\n", tnf_string, msg, 2369 "ibt_query_hca_ports_byguid() failed", 2370 tnf_int, ibt_status, ibt_status); 2371 2372 goto bail; 2373 } 2374 2375 mutex_enter(&saa_portp->saa_pt_mutex); 2376 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) { 2377 /* update the Master SM Lid value in ibmf_saa */ 2378 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2379 ibt_portinfop->p_sm_lid; 2380 } 2381 if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) { 2382 /* update the Master SM SL value in ibmf_saa */ 2383 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = 2384 ibt_portinfop->p_sm_sl; 2385 } 2386 if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) { 2387 /* update the Subnet timeout value in ibmf_saa */ 2388 saa_portp->saa_pt_timeout = 2389 ibt_portinfop->p_subnet_timeout; 2390 } 2391 mutex_exit(&saa_portp->saa_pt_mutex); 2392 2393 ibt_free_portinfo(ibt_portinfop, size); 2394 2395 /* get the classportinfo again */ 2396 ibmf_saa_impl_get_classportinfo(saa_portp); 2397 } 2398 bail: 2399 2400 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end, 2401 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n"); 2402 } 2403 /* 2404 * ibmf_saa_impl_client_rereg: 2405 */ 2406 static void 2407 ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num) 2408 { 2409 saa_port_t *saa_portp = NULL; 2410 boolean_t is_ready = B_FALSE; 2411 ibt_hca_portinfo_t *ibt_portinfop; 2412 ib_lid_t master_sm_lid; 2413 uint_t nports, size; 2414 ibt_status_t ibt_status; 2415 boolean_t event_subs = B_FALSE; 2416 2417 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start, 2418 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt " 2419 "rereg guid %016" PRIx64 " port %d\n", 2420 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2421 2422 /* Get classportinfo of corresponding entry */ 2423 mutex_enter(&saa_statep->saa_port_list_mutex); 2424 2425 saa_portp = saa_statep->saa_port_list; 2426 while (saa_portp != NULL) { 2427 2428 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2429 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2430 2431 mutex_enter(&saa_portp->saa_pt_mutex); 2432 2433 is_ready = (saa_portp->saa_pt_state 2434 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2435 2436 /* 2437 * increment reference count to account for cpi and 2438 * informinfos. All 4 informinfo's sent are treated as 2439 * one port client reference 2440 */ 2441 if (is_ready) 2442 saa_portp->saa_pt_reference_count += 2; 2443 2444 mutex_exit(&saa_portp->saa_pt_mutex); 2445 2446 if (is_ready) 2447 break; /* normally, only 1 port entry */ 2448 } 2449 saa_portp = saa_portp->next; 2450 } 2451 2452 mutex_exit(&saa_statep->saa_port_list_mutex); 2453 2454 if (saa_portp != NULL && is_ready == B_TRUE) { 2455 2456 /* verify whether master sm lid changed */ 2457 2458 /* first query the portinfo to see if the lid changed */ 2459 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2460 &ibt_portinfop, &nports, &size); 2461 2462 if (ibt_status != IBT_SUCCESS) { 2463 2464 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2465 ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "", 2466 "ibmf_saa_impl_client_rereg: %s, ibmf_status =" 2467 " %d\n", tnf_string, msg, 2468 "ibt_query_hca_ports_byguid() failed", 2469 tnf_int, ibt_status, ibt_status); 2470 2471 goto bail; 2472 } 2473 2474 master_sm_lid = ibt_portinfop->p_sm_lid; 2475 2476 ibt_free_portinfo(ibt_portinfop, size); 2477 2478 /* check whether we need to subscribe for events */ 2479 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2480 2481 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2482 B_TRUE : B_FALSE; 2483 2484 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2485 2486 /* update the master smlid */ 2487 mutex_enter(&saa_portp->saa_pt_mutex); 2488 2489 /* update the master sm lid value in ibmf_saa */ 2490 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2491 master_sm_lid; 2492 2493 /* if we're not subscribed for events, dec reference count */ 2494 if (event_subs == B_FALSE) 2495 saa_portp->saa_pt_reference_count--; 2496 2497 mutex_exit(&saa_portp->saa_pt_mutex); 2498 2499 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2500 ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "", 2501 "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n", 2502 tnf_string, msg, 2503 "port is up. Sending classportinfo request", 2504 tnf_opaque, master_sm_lid, master_sm_lid); 2505 2506 /* get the classportinfo again */ 2507 ibmf_saa_impl_get_classportinfo(saa_portp); 2508 2509 /* 2510 * resubscribe to events if there are subscribers since SA may 2511 * have removed our subscription records when the port went down 2512 */ 2513 if (event_subs == B_TRUE) 2514 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2515 } 2516 2517 bail: 2518 2519 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end, 2520 IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n"); 2521 } 2522 /* 2523 * ibmf_saa_impl_port_up: 2524 */ 2525 static void 2526 ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num) 2527 { 2528 saa_port_t *saa_portp = NULL; 2529 int is_ready; 2530 ibt_hca_portinfo_t *ibt_portinfop; 2531 ib_lid_t master_sm_lid; 2532 uint_t nports, size; 2533 ibt_status_t ibt_status; 2534 boolean_t event_subs = B_FALSE; 2535 2536 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_up_start, 2537 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up: Handling port up" 2538 " guid %016" PRIx64 " port %d\n", 2539 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2540 2541 /* Get classportinfo of corresponding entry */ 2542 mutex_enter(&saa_statep->saa_port_list_mutex); 2543 2544 saa_portp = saa_statep->saa_port_list; 2545 while (saa_portp != NULL) { 2546 2547 if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid && 2548 saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) { 2549 2550 mutex_enter(&saa_portp->saa_pt_mutex); 2551 2552 is_ready = (saa_portp->saa_pt_state 2553 == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE; 2554 2555 /* 2556 * increment reference count to account for cpi and 2557 * informinfos. All 4 informinfo's sent are treated as 2558 * one port client reference 2559 */ 2560 if (is_ready == B_TRUE) 2561 saa_portp->saa_pt_reference_count += 2; 2562 2563 mutex_exit(&saa_portp->saa_pt_mutex); 2564 2565 if (is_ready == B_TRUE) 2566 break; /* normally, only 1 port entry */ 2567 } 2568 saa_portp = saa_portp->next; 2569 } 2570 2571 mutex_exit(&saa_statep->saa_port_list_mutex); 2572 2573 if (saa_portp != NULL && is_ready == B_TRUE) { 2574 2575 /* verify whether master sm lid changed */ 2576 2577 /* first query the portinfo to see if the lid changed */ 2578 ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num, 2579 &ibt_portinfop, &nports, &size); 2580 2581 if (ibt_status != IBT_SUCCESS) { 2582 2583 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 2584 ibmf_saa_impl_port_up_err, IBMF_TNF_ERROR, "", 2585 "ibmf_saa_impl_port_up: %s, ibmf_status =" 2586 " %d\n", tnf_string, msg, 2587 "ibt_query_hca_ports_byguid() failed", 2588 tnf_int, ibt_status, ibt_status); 2589 2590 goto bail; 2591 } 2592 2593 master_sm_lid = ibt_portinfop->p_sm_lid; 2594 2595 ibt_free_portinfo(ibt_portinfop, size); 2596 2597 /* check whether we need to subscribe for events */ 2598 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2599 2600 event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2601 B_TRUE : B_FALSE; 2602 2603 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2604 2605 /* update the master smlid */ 2606 mutex_enter(&saa_portp->saa_pt_mutex); 2607 2608 /* update the master sm lid value in ibmf_saa */ 2609 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = 2610 master_sm_lid; 2611 2612 /* if we're not subscribed for events, dec reference count */ 2613 if (event_subs == B_FALSE) 2614 saa_portp->saa_pt_reference_count--; 2615 2616 mutex_exit(&saa_portp->saa_pt_mutex); 2617 2618 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 2619 ibmf_saa_impl_port_up, IBMF_TNF_TRACE, "", 2620 "ibmf_saa_impl_port_up: %s, master_sm_lid = 0x%x\n", 2621 tnf_string, msg, 2622 "port is up. Sending classportinfo request", 2623 tnf_opaque, master_sm_lid, master_sm_lid); 2624 2625 /* get the classportinfo again */ 2626 ibmf_saa_impl_get_classportinfo(saa_portp); 2627 2628 /* 2629 * resubscribe to events if there are subscribers since SA may 2630 * have removed our subscription records when the port went down 2631 */ 2632 if (event_subs == B_TRUE) 2633 ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE); 2634 } 2635 2636 bail: 2637 2638 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_up_end, 2639 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_up() exit\n"); 2640 } 2641 2642 /* 2643 * ibmf_saa_impl_port_down: 2644 */ 2645 static void 2646 ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num) 2647 { 2648 2649 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_down_start, 2650 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down: Handling port down" 2651 " guid %016" PRIx64 " port %d\n", 2652 tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num); 2653 2654 2655 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_down_end, 2656 IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_down() exit\n"); 2657 } 2658 2659 /* 2660 * ibmf_saa_impl_hca_detach: 2661 * find entry, unregister if there are no clients 2662 * have to unregister since ibmf needs to close the hca and will only do this if 2663 * no clients are registered 2664 */ 2665 static void 2666 ibmf_saa_impl_hca_detach(saa_port_t *saa_removed) 2667 { 2668 saa_port_t *saa_portp; 2669 boolean_t must_unreg, must_unsub; 2670 2671 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_start, 2672 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: Detaching" 2673 " entry %016" PRIx64 "\n", tnf_opaque, entry, saa_removed); 2674 2675 /* find this entry */ 2676 mutex_enter(&saa_statep->saa_port_list_mutex); 2677 2678 saa_portp = saa_statep->saa_port_list; 2679 while (saa_portp != NULL) { 2680 2681 if (saa_portp == saa_removed) 2682 break; 2683 2684 saa_portp = saa_portp->next; 2685 } 2686 mutex_exit(&saa_statep->saa_port_list_mutex); 2687 2688 ASSERT(saa_portp != NULL); 2689 2690 if (saa_portp == NULL) { 2691 2692 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2693 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2694 "ibmf_saa_impl_hca_detach: %s, entry %016" 2695 PRIx64 "\n", 2696 tnf_string, msg, 2697 "Port entry NOT found", 2698 tnf_opaque, entryp, saa_removed); 2699 2700 goto bail; 2701 } 2702 2703 /* if there are clients expecting Reports(), unsusbscribe */ 2704 mutex_enter(&saa_portp->saa_pt_event_sub_mutex); 2705 2706 must_unsub = (saa_portp->saa_pt_event_sub_client_list != NULL) ? 2707 B_TRUE : B_FALSE; 2708 2709 mutex_exit(&saa_portp->saa_pt_event_sub_mutex); 2710 2711 /* fail if outstanding transactions */ 2712 mutex_enter(&saa_portp->saa_pt_mutex); 2713 2714 if (saa_portp->saa_pt_num_outstanding_trans > 0) { 2715 2716 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L1, 2717 ibmf_saa_impl_fini_err, IBMF_TNF_TRACE, "", 2718 "ibmf_saa_impl_fini: %s, port = %016" PRIx64 2719 ", num transactions = %d\n", 2720 tnf_string, msg, "Detaching HCA." 2721 " Outstanding transactions on port.", 2722 tnf_opaque, port, 2723 saa_portp->saa_pt_port_guid, 2724 tnf_uint, outstanding_transactions, 2725 saa_portp->saa_pt_num_outstanding_trans); 2726 2727 mutex_exit(&saa_portp->saa_pt_mutex); 2728 2729 goto bail; 2730 } 2731 2732 2733 /* 2734 * increment reference count by one to account for unsubscribe requests 2735 * that are about to be sent. All four informinfo's are treated as one 2736 * port client reference. The count will be decremented by 2737 * subscribe_events() before the call returns. 2738 */ 2739 if (must_unsub == B_TRUE) 2740 saa_portp->saa_pt_reference_count++; 2741 2742 mutex_exit(&saa_portp->saa_pt_mutex); 2743 2744 /* 2745 * try and unsubscribe from SA. Generate synchronous, unsequenced 2746 * unsubscribe requests. 2747 */ 2748 if (must_unsub == B_TRUE) 2749 ibmf_saa_subscribe_events(saa_portp, B_FALSE, B_TRUE); 2750 2751 /* warning if registered clients */ 2752 mutex_enter(&saa_portp->saa_pt_mutex); 2753 2754 if (saa_portp->saa_pt_reference_count > 0) { 2755 2756 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 2757 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2758 "ibmf_saa_impl_hca_detach: %s, port %016" 2759 PRIx64 "\n", 2760 tnf_string, msg, 2761 "Detaching HCA for port with clients still" 2762 " registered", tnf_opaque, port, 2763 saa_portp->saa_pt_port_guid); 2764 } 2765 2766 /* synchronize on end of registration */ 2767 while (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_REGISTERING) { 2768 2769 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2770 ibmf_saa_impl_hca_detach, IBMF_TNF_TRACE, "", 2771 "ibmf_saa_impl_hca_detach: %s\n", 2772 tnf_string, msg, "someone is registering. waiting" 2773 " for them to finish"); 2774 2775 cv_wait(&saa_portp->saa_pt_ibmf_reg_cv, 2776 &saa_portp->saa_pt_mutex); 2777 2778 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L1, 2779 ibmf_saa_impl_hca_detach, 2780 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach: %s\n", 2781 tnf_string, msg, "done waiting"); 2782 } 2783 2784 /* unregister from ibmf */ 2785 if (saa_portp->saa_pt_state == IBMF_SAA_PORT_STATE_READY) { 2786 must_unreg = B_TRUE; 2787 } else 2788 must_unreg = B_FALSE; 2789 2790 ibmf_saa_impl_invalidate_port(saa_portp); 2791 2792 mutex_exit(&saa_portp->saa_pt_mutex); 2793 2794 if (must_unreg == B_TRUE) { 2795 if (ibmf_saa_impl_ibmf_unreg(saa_portp) != IBMF_SUCCESS) { 2796 mutex_enter(&saa_portp->saa_pt_mutex); 2797 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2798 (void) ibmf_saa_impl_init_kstats(saa_portp); 2799 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2800 saa_portp->saa_pt_state = IBMF_SAA_PORT_STATE_READY; 2801 if (must_unsub == B_TRUE) 2802 saa_portp->saa_pt_reference_count++; 2803 mutex_exit(&saa_portp->saa_pt_mutex); 2804 2805 if (must_unsub == B_TRUE) { 2806 ibmf_saa_subscribe_events(saa_portp, B_TRUE, 2807 B_FALSE); 2808 } 2809 } 2810 } 2811 bail: 2812 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_hca_detach_end, 2813 IBMF_TNF_TRACE, "", "ibmf_saa_impl_hca_detach() exit\n"); 2814 } 2815 2816 /* ARGSUSED */ 2817 void 2818 ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args) 2819 { 2820 saa_impl_trans_info_t *trans_info; 2821 int status; 2822 size_t length; 2823 void *result; 2824 saa_port_t *saa_portp; 2825 saa_client_data_t *client_data; 2826 int ibmf_status; 2827 boolean_t ignore_data; 2828 ibmf_retrans_t ibmf_retrans; 2829 boolean_t sa_is_redirected = B_FALSE; 2830 2831 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_start, 2832 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() enter\n"); 2833 2834 trans_info = (saa_impl_trans_info_t *)args; 2835 2836 client_data = trans_info->si_trans_client_data; 2837 saa_portp = trans_info->si_trans_port; 2838 2839 mutex_enter(&saa_portp->saa_pt_mutex); 2840 sa_is_redirected = saa_portp->saa_pt_redirect_active; 2841 mutex_exit(&saa_portp->saa_pt_mutex); 2842 2843 if ((msgp->im_msg_status == IBMF_TRANS_TIMEOUT) && 2844 (sa_is_redirected == B_TRUE)) { 2845 2846 /* 2847 * We should retry the request using SM_LID and QP1 if we 2848 * have been using redirect up until now 2849 */ 2850 ibmf_status = ibmf_saa_impl_revert_to_qp1( 2851 saa_portp, msgp, ibmf_saa_async_cb, args, 2852 trans_info->si_trans_transport_flags); 2853 2854 /* 2855 * If revert_to_qp1 returns success msg was resent. 2856 * Otherwise msg could not be resent. Continue normally 2857 */ 2858 if (ibmf_status == IBMF_SUCCESS) 2859 goto bail; 2860 2861 } else if (msgp->im_msg_status == IBMF_TRANS_TIMEOUT) { 2862 2863 2864 ibmf_status = ibmf_saa_impl_new_smlid_retry(saa_portp, msgp, 2865 ibmf_saa_async_cb, args, 2866 trans_info->si_trans_transport_flags); 2867 2868 /* 2869 * if smlid_retry() returns success sm lid changed and msg 2870 * was resent. Otherwise, lid did not change or msg could not 2871 * be resent. Continue normally. 2872 */ 2873 if (ibmf_status == IBMF_SUCCESS) 2874 goto bail; 2875 2876 /* 2877 * check whether we've received anything from the SA in a while. 2878 * If we have, this function will retry and return success. If 2879 * we haven't continue normally so that we return a timeout to 2880 * the client 2881 */ 2882 ibmf_status = ibmf_saa_check_sa_and_retry( 2883 saa_portp, msgp, ibmf_saa_async_cb, args, 2884 trans_info->si_trans_send_time, 2885 trans_info->si_trans_transport_flags); 2886 2887 if (ibmf_status == IBMF_SUCCESS) 2888 goto bail; 2889 } 2890 2891 /* 2892 * If SA returned success but mad status is busy, retry a few times. 2893 * If SA returned success but mad status says redirect is required, 2894 * update the address info and retry the request to the new SA address 2895 */ 2896 if (msgp->im_msg_status == IBMF_SUCCESS) { 2897 2898 ASSERT(msgp->im_msgbufs_recv.im_bufs_mad_hdr != NULL); 2899 2900 if ((b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) == 2901 MAD_STATUS_BUSY) && 2902 (trans_info->si_trans_retry_busy_count < 2903 IBMF_SAA_MAX_BUSY_RETRY_COUNT)) { 2904 2905 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 2906 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2907 "ibmf_saa_async_cb: %s, retry_count = %d\n", 2908 tnf_string, msg, 2909 "async response returned busy status", 2910 tnf_int, retry_count, 2911 trans_info->si_trans_retry_busy_count); 2912 2913 trans_info->si_trans_retry_busy_count++; 2914 2915 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2916 sizeof (ibmf_retrans_t)); 2917 2918 ibmf_status = ibmf_msg_transport( 2919 saa_portp->saa_pt_ibmf_handle, 2920 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2921 ibmf_saa_async_cb, args, 2922 trans_info->si_trans_transport_flags); 2923 2924 /* 2925 * if retry is successful, quit here since async_cb will 2926 * get called again; otherwise, let this function call 2927 * handle the cleanup 2928 */ 2929 if (ibmf_status == IBMF_SUCCESS) 2930 goto bail; 2931 } else if (b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->Status) 2932 == MAD_STATUS_REDIRECT_REQUIRED) { 2933 2934 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L2, 2935 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2936 "ibmf_saa_async_cb: " 2937 "async response returned redirect status\n"); 2938 2939 /* update address info and copy it into msgp */ 2940 ibmf_saa_impl_update_sa_address_info(saa_portp, msgp); 2941 2942 /* retry with new address info */ 2943 bcopy(&saa_portp->saa_pt_ibmf_retrans, &ibmf_retrans, 2944 sizeof (ibmf_retrans_t)); 2945 2946 ibmf_status = ibmf_msg_transport( 2947 saa_portp->saa_pt_ibmf_handle, 2948 saa_portp->saa_pt_qp_handle, msgp, &ibmf_retrans, 2949 ibmf_saa_async_cb, args, 2950 trans_info->si_trans_transport_flags); 2951 2952 /* 2953 * if retry is successful, quit here since async_cb will 2954 * get called again; otherwise, let this function call 2955 * handle the cleanup 2956 */ 2957 if (ibmf_status == IBMF_SUCCESS) 2958 goto bail; 2959 } 2960 } 2961 2962 mutex_enter(&saa_portp->saa_pt_mutex); 2963 2964 ASSERT(saa_portp->saa_pt_num_outstanding_trans > 0); 2965 saa_portp->saa_pt_num_outstanding_trans--; 2966 2967 mutex_exit(&saa_portp->saa_pt_mutex); 2968 2969 if ((trans_info->si_trans_callback == NULL) && 2970 (trans_info->si_trans_sub_callback == NULL)) 2971 ignore_data = B_TRUE; 2972 else 2973 ignore_data = B_FALSE; 2974 2975 ibmf_saa_impl_prepare_response(ibmf_handle, msgp, ignore_data, &status, 2976 &result, &length, B_FALSE); 2977 2978 mutex_enter(&saa_portp->saa_pt_kstat_mutex); 2979 2980 IBMF_SAA_SUB32_KSTATS(saa_portp, outstanding_requests, 1); 2981 2982 if (status != IBMF_SUCCESS) 2983 IBMF_SAA_ADD32_KSTATS(saa_portp, failed_requests, 1); 2984 2985 if (status == IBMF_TRANS_TIMEOUT) 2986 IBMF_SAA_ADD32_KSTATS(saa_portp, requests_timedout, 1); 2987 2988 mutex_exit(&saa_portp->saa_pt_kstat_mutex); 2989 2990 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 2991 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 2992 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 2993 "Calling ibmf_saa client's callback"); 2994 2995 /* 2996 * there are three classes or trans_info users: ibmf_saa clients and 2997 * classportinfo requests; informinfo subscribe requests, and report 2998 * responses. For the first two, call the correct callback. For report 2999 * responses there's no need to notify anyone. 3000 */ 3001 if (trans_info->si_trans_callback != NULL) { 3002 /* ibmf_saa client or classportinfo request */ 3003 trans_info->si_trans_callback(trans_info->si_trans_callback_arg, 3004 length, result, status); 3005 } else if (trans_info->si_trans_sub_callback != NULL) { 3006 /* informinfo subscribe request */ 3007 trans_info->si_trans_sub_callback( 3008 trans_info->si_trans_callback_arg, length, result, status, 3009 trans_info->si_trans_sub_producer_type); 3010 } 3011 3012 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3013 ibmf_saa_async_cb, IBMF_TNF_TRACE, "", 3014 "ibmf_saa_async_cb: %s\n", tnf_string, msg, 3015 "Returned from callback"); 3016 3017 if (client_data != NULL) { 3018 mutex_enter(&client_data->saa_client_mutex); 3019 3020 ASSERT(client_data->saa_client_num_pending_trans > 0); 3021 client_data->saa_client_num_pending_trans--; 3022 3023 if ((client_data->saa_client_num_pending_trans == 0) && 3024 (client_data->saa_client_state == SAA_CLIENT_STATE_WAITING)) 3025 cv_signal(&client_data->saa_client_state_cv); 3026 3027 mutex_exit(&client_data->saa_client_mutex); 3028 } 3029 3030 kmem_free(trans_info, sizeof (saa_impl_trans_info_t)); 3031 3032 bail: 3033 3034 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_async_cb_end, 3035 IBMF_TNF_TRACE, "", "ibmf_saa_async_cb() exit\n"); 3036 } 3037 3038 /* 3039 * ibmf_saa_check_sa_and_retry: 3040 * 3041 * If a particular transaction times out, we don't want to give up if we know 3042 * the SA is responding. Check the time since we last received a response. If 3043 * it's less than ibmf_saa_max_wait_time retry the request. 3044 * 3045 * msgp, ibmf_callback, ibmf_callback_arg, and transport flags should be the 3046 * same values passed to the original ibmf_msg_transport that timed out. The 3047 * ibmf_retrans parameter will be re-retrieved from the saa_portp structure. 3048 * 3049 * If max_wait_time seconds have passed, this function returns IBMF_TIMEOUT. 3050 * That way, callers can simply return the result of this function. 3051 * 3052 * Input Arguments 3053 * saa_portp pointer to saa_port structure 3054 * msgp ibmf message that timedout 3055 * ibmf_callback callback that should be called by msg_transport 3056 * ibmf_callback_arg args for ibmf_callback 3057 * transport_flags flags for ibmf_msg_transport 3058 * 3059 * Output Arguments 3060 * none 3061 * 3062 * Returns 3063 * IBMF_SUCCESS if we've recently received data from the SA and request was 3064 * resent. 3065 * IBMF_TRANS_TIMEOUT if no data has been received from the SA in max_wait_time 3066 * same values as ibmf_msg_transport() if data has been received but request 3067 * could not be resent. 3068 */ 3069 static int 3070 ibmf_saa_check_sa_and_retry(saa_port_t *saa_portp, ibmf_msg_t *msgp, 3071 ibmf_msg_cb_t ibmf_callback, void *ibmf_callback_arg, 3072 hrtime_t trans_send_time, int transport_flags) 3073 { 3074 hrtime_t curr_time, sa_uptime; 3075 ibmf_retrans_t ibmf_retrans; 3076 int ibmf_status; 3077 3078 do { 3079 3080 mutex_enter(&saa_portp->saa_pt_mutex); 3081 3082 sa_uptime = saa_portp->saa_pt_sa_uptime; 3083 3084 /* if nothing received from SA since we sent */ 3085 curr_time = gethrtime(); 3086 3087 /* 3088 * check if it's been a very long time since this 3089 * particular transaction was sent 3090 */ 3091 if (((curr_time - trans_send_time) / 1000000000) > 3092 ibmf_saa_trans_wait_time) { 3093 3094 mutex_exit(&saa_portp->saa_pt_mutex); 3095 3096 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3097 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3098 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3099 "%p sa_uptime = %" PRIu64 ", trans send time = %" 3100 PRIu64 ", curr_time = %" PRIu64 "\n", 3101 tnf_string, msg, 3102 "Nothing received for this transaction", 3103 tnf_opaque, msgp, msgp, 3104 tnf_long, sa_uptime, sa_uptime, 3105 tnf_long, trans_send_time, trans_send_time, 3106 tnf_long, curr_time, curr_time); 3107 3108 ibmf_status = IBMF_TRANS_TIMEOUT; 3109 3110 break; 3111 } 3112 3113 /* 3114 * check time since we received something, 3115 * and make sure that it hasn't been an extra long 3116 * time for this particular transaction 3117 */ 3118 if (((curr_time - sa_uptime) / 1000000000) < 3119 ibmf_saa_max_wait_time) { 3120 3121 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L2, 3122 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3123 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3124 "%p sa_uptime = %" PRIu64 " trans_send_time = %" 3125 PRIu64 " curr_time = %" PRIu64 "\n", 3126 tnf_string, msg, "Something received. Retrying", 3127 tnf_opaque, msgp, msgp, 3128 tnf_long, sa_uptime, sa_uptime, 3129 tnf_long, trans_send_time, trans_send_time, 3130 tnf_long, curr_time, curr_time); 3131 3132 /* 3133 * something received in WAIT_TIME_IN_SECS; 3134 * resend request 3135 */ 3136 3137 /* new tid needed */ 3138 msgp->im_msgbufs_send.im_bufs_mad_hdr->TransactionID = 3139 h2b64(saa_portp->saa_pt_current_tid++); 3140 3141 bcopy(&saa_portp->saa_pt_ibmf_retrans, 3142 &ibmf_retrans, sizeof (ibmf_retrans_t)); 3143 3144 mutex_exit(&saa_portp->saa_pt_mutex); 3145 3146 ibmf_status = ibmf_msg_transport( 3147 saa_portp->saa_pt_ibmf_handle, 3148 saa_portp->saa_pt_qp_handle, msgp, 3149 &ibmf_retrans, ibmf_callback, ibmf_callback_arg, 3150 transport_flags); 3151 3152 if (ibmf_status == IBMF_SUCCESS) 3153 goto bail; 3154 3155 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3156 ibmf_saa_check_sa_and_retry, IBMF_TNF_TRACE, "", 3157 "ibmf_saa_check_sa_and_retry: %s, ibmf_status = " 3158 "%d\n", tnf_string, msg, 3159 "ibmf_msg_transport() failed", 3160 tnf_int, ibmf_status, ibmf_status); 3161 } else { 3162 3163 mutex_exit(&saa_portp->saa_pt_mutex); 3164 3165 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L1, 3166 ibmf_saa_check_sa_and_retry_err, IBMF_TNF_ERROR, "", 3167 "ibmf_saa_check_sa_and_retry: %s, msgp = " 3168 "%p sa_uptime = %" PRIu64 " curr_time = %" 3169 PRIu64 "\n", tnf_string, msg, 3170 "Nothing received. Timing out", 3171 tnf_opaque, msgp, msgp, 3172 tnf_long, sa_uptime, sa_uptime, 3173 tnf_long, curr_time, curr_time); 3174 3175 ibmf_status = IBMF_TRANS_TIMEOUT; 3176 3177 break; 3178 } 3179 } while (ibmf_status == IBMF_TRANS_TIMEOUT); 3180 3181 bail: 3182 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3183 ibmf_saa_check_sa_and_retry_end, 3184 IBMF_TNF_TRACE, "", "ibmf_saa_check_sa_and_retry() exiting" 3185 " ibmf_status = %d\n", tnf_int, result, ibmf_status); 3186 3187 return (ibmf_status); 3188 } 3189 3190 3191 /* 3192 * ibmf_saa_impl_prepare_response: 3193 */ 3194 static void 3195 ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle, 3196 ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result, 3197 size_t *length, boolean_t sleep_flag) 3198 { 3199 ibmf_msg_bufs_t *resp_buf; 3200 uint16_t attr_id; 3201 uint8_t method; 3202 boolean_t is_get_resp; 3203 uint16_t mad_status; 3204 uint16_t attr_offset; 3205 ib_sa_hdr_t *sa_hdr; 3206 3207 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3208 ibmf_saa_impl_prepare_response_start, 3209 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() enter\n"); 3210 3211 _NOTE(ASSUMING_PROTECTED(*msgp)) 3212 3213 *result = NULL; 3214 *length = 0; 3215 sa_hdr = NULL; 3216 3217 resp_buf = &msgp->im_msgbufs_recv; 3218 3219 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*resp_buf)) 3220 3221 if (msgp->im_msg_status != IBMF_SUCCESS) { 3222 3223 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3224 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3225 "ibmf_saa_impl_prepare_response: %s, msg_status = %d\n", 3226 tnf_string, msg, "Bad ibmf status", 3227 tnf_int, msg_status, msgp->im_msg_status); 3228 3229 *status = msgp->im_msg_status; 3230 3231 goto exit; 3232 } 3233 3234 if (resp_buf->im_bufs_mad_hdr == NULL) { 3235 3236 /* 3237 * this was an unsequenced transaction (from an unsubscribe for 3238 * following a CI_OFFLINE event) 3239 */ 3240 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3241 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3242 "ibmf_saa_impl_prepare_response: %s\n", 3243 tnf_string, msg, "Unsequenced transaction callback"); 3244 3245 goto exit; 3246 } 3247 3248 if ((mad_status = b2h16(resp_buf->im_bufs_mad_hdr->Status)) != 3249 MAD_STATUS_NO_INVALID_FIELDS) { 3250 3251 /* convert mad packet status to IBMF status */ 3252 switch (mad_status) { 3253 3254 case SA_STATUS_ERR_NO_RESOURCES: 3255 *status = IBMF_NO_RESOURCES; 3256 break; 3257 case SA_STATUS_ERR_REQ_INVALID: 3258 *status = IBMF_REQ_INVALID; 3259 break; 3260 case SA_STATUS_ERR_NO_RECORDS: 3261 *status = IBMF_NO_RECORDS; 3262 break; 3263 case SA_STATUS_ERR_TOO_MANY_RECORDS: 3264 *status = IBMF_TOO_MANY_RECORDS; 3265 break; 3266 case SA_STATUS_ERR_REQ_INVALID_GID: 3267 *status = IBMF_INVALID_GID; 3268 break; 3269 case SA_STATUS_ERR_REQ_INSUFFICIENT_COMPONENTS: 3270 *status = IBMF_INSUFF_COMPS; 3271 break; 3272 case MAD_STATUS_UNSUPP_METHOD: 3273 *status = IBMF_UNSUPP_METHOD; 3274 break; 3275 case MAD_STATUS_UNSUPP_METHOD_ATTR: 3276 *status = IBMF_UNSUPP_METHOD_ATTR; 3277 break; 3278 case MAD_STATUS_INVALID_FIELD: 3279 *status = IBMF_INVALID_FIELD; 3280 break; 3281 default: 3282 *status = IBMF_REQ_INVALID; 3283 break; 3284 } 3285 3286 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L2, 3287 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3288 "ibmf_saa_impl_prepare_response: %s, mad_status = %x\n", 3289 tnf_string, msg, "Bad MAD status", 3290 tnf_int, mad_status, mad_status); 3291 3292 goto exit; 3293 } 3294 3295 attr_id = b2h16(resp_buf->im_bufs_mad_hdr->AttributeID); 3296 method = resp_buf->im_bufs_mad_hdr->R_Method; 3297 3298 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3299 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3300 "ibmf_saa_impl_prepare_response: attr_id = 0x%x, method = " 3301 "0x%x\n", 3302 tnf_opaque, attr_id, attr_id, 3303 tnf_opaque, method, method); 3304 3305 /* 3306 * ignore any data from deleteresp since there's no way to know whether 3307 * real data was returned; also ignore data if this was a Report 3308 * response 3309 */ 3310 if (method == SA_SUBN_ADM_DELETE_RESP) { 3311 3312 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3313 ibmf_saa_impl_prepare_response, IBMF_TNF_TRACE, "", 3314 "impf_saa_impl_prepare_response: %s\n", 3315 tnf_string, msg, 3316 "DeleteResp or NoticeResp returned. " 3317 "Ignoring response data"); 3318 3319 *status = IBMF_SUCCESS; 3320 3321 *length = 0; 3322 *result = NULL; 3323 3324 goto exit; 3325 } 3326 3327 if (attr_id == SA_MULTIPATHRECORD_ATTRID) { 3328 3329 /* 3330 * getmulti is only for requests; attribute should not 3331 * be returned from SA 3332 */ 3333 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L1, 3334 ibmf_saa_impl_prepare_response_err, IBMF_TNF_ERROR, 3335 "", "ibmf_saa_impl_prepare_response: %s\n", 3336 tnf_string, msg, "SA returned getmulti record"); 3337 3338 *status = IBMF_REQ_INVALID; 3339 3340 goto exit; 3341 } 3342 3343 /* if we are supposed to ignore data, stop here */ 3344 if (ignore_data == B_TRUE) { 3345 3346 *status = IBMF_SUCCESS; 3347 3348 goto exit; 3349 } 3350 3351 is_get_resp = resp_buf->im_bufs_mad_hdr->R_Method == 3352 SA_SUBN_ADM_GET_RESP ? B_TRUE: B_FALSE; 3353 3354 /* unpack the sa header to get the attribute offset */ 3355 *status = ibmf_saa_utils_unpack_sa_hdr(resp_buf->im_bufs_cl_hdr, 3356 resp_buf->im_bufs_cl_hdr_len, &sa_hdr, sleep_flag); 3357 if (*status != IBMF_SUCCESS) { 3358 3359 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3360 ibmf_saa_impl_prepare_response_err, 3361 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3362 " ibmf_status = %d\n", tnf_string, msg, 3363 "Could not unpack sa hdr", tnf_int, ibmf_status, *status); 3364 3365 goto exit; 3366 } 3367 3368 attr_offset = sa_hdr->AttributeOffset; 3369 3370 /* 3371 * unpack data payload; if unpack function doesn't return success 3372 * (because it could not allocate memory) forward this status to waiting 3373 * client 3374 */ 3375 *status = ibmf_saa_utils_unpack_payload(resp_buf->im_bufs_cl_data, 3376 resp_buf->im_bufs_cl_data_len, attr_id, result, length, 3377 attr_offset, is_get_resp, sleep_flag); 3378 if (*status == IBMF_SUCCESS) { 3379 3380 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3381 ibmf_saa_impl_prepare_response, 3382 IBMF_TNF_TRACE, "", 3383 "ibmf_saa_impl_prepare_response: attr_id = " 3384 "0x%x, attr_offset = %d, packed_payload_len = %d, " 3385 "unpacked_payload_len = %d\n", 3386 tnf_opaque, attr_id, attr_id, 3387 tnf_opaque, attr_offset, attr_offset, 3388 tnf_opaque, packed_payload_len, 3389 resp_buf->im_bufs_cl_data_len, 3390 tnf_opaque, unpacked_payload_len, *length); 3391 } else { 3392 3393 IBMF_TRACE_5(IBMF_TNF_DEBUG, DPRINT_L1, 3394 ibmf_saa_impl_prepare_response_err, 3395 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response: %s," 3396 "attr_id = 0x%x, attr_offset = %d, packed_payload_len = %d," 3397 "status = %d\n", 3398 tnf_string, msg, "Could not unpack payload", 3399 tnf_opaque, attr_id, attr_id, 3400 tnf_int, attr_offset, attr_offset, 3401 tnf_int, packed_payload_len, 3402 resp_buf->im_bufs_cl_data_len, 3403 tnf_int, status, *status); 3404 } 3405 exit: 3406 if (sa_hdr != NULL) 3407 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3408 3409 ibmf_saa_impl_free_msg(ibmf_handle, msgp); 3410 3411 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3412 ibmf_saa_impl_prepare_response_end, 3413 IBMF_TNF_TRACE, "", "ibmf_saa_impl_prepare_response() exit," 3414 " status = 0x%d\n", tnf_int, status, *status); 3415 } 3416 3417 3418 /* 3419 * ibmf_saa_impl_check_sa_support: 3420 * Checks the capability mask (returned from the SA classportinfo response) to 3421 * determine whether the sa supports the specified attribute ID. 3422 * 3423 * Input Arguments 3424 * cap_mask 16-bit capability mask returned in SA's classportinfo 3425 * attr_id attribute ID of current request 3426 * 3427 * Returns 3428 * IBMF_NOT_SUPPORTED if capability mask indicates SA does not support attribute 3429 * IBMF_SUCCESS otherwise 3430 */ 3431 static int 3432 ibmf_saa_impl_check_sa_support(uint16_t cap_mask, uint16_t attr_id) 3433 { 3434 boolean_t attr_supported = B_TRUE; 3435 3436 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3437 ibmf_saa_impl_check_sa_support, IBMF_TNF_TRACE, "", 3438 "ibmf_saa_impl_check_sa_support: cap_mask = 0x%x, " 3439 "attr_id = 0x%x\n", tnf_opaque, cap_mask, cap_mask, 3440 tnf_opaque, attr_id, attr_id); 3441 3442 switch (attr_id) { 3443 3444 case SA_SWITCHINFORECORD_ATTRID: 3445 case SA_LINEARFDBRECORD_ATTRID: 3446 case SA_RANDOMFDBRECORD_ATTRID: 3447 case SA_MULTICASTFDBRECORD_ATTRID: 3448 case SA_SMINFORECORD_ATTRID: 3449 case SA_INFORMINFORECORD_ATTRID: 3450 case SA_LINKRECORD_ATTRID: 3451 case SA_GUIDINFORECORD_ATTRID: 3452 case SA_TRACERECORD_ATTRID: 3453 case SA_SERVICEASSNRECORD_ATTRID: 3454 3455 if ((cap_mask & 3456 SA_CAPMASK_OPT_RECORDS_SUPPORTED) == 0) { 3457 3458 IBMF_TRACE_3(IBMF_TNF_NODEBUG, DPRINT_L1, 3459 ibmf_saa_impl_check_sa_support, 3460 IBMF_TNF_ERROR, "", 3461 "ibmf_saa_impl_check_sa_support: %s, " 3462 "cap_mask = 0x%x\n", tnf_string, msg, 3463 "SA does not support optional records", 3464 tnf_opaque, cap_mask, cap_mask, 3465 tnf_opaque, attr_id, attr_id); 3466 3467 attr_supported = B_FALSE; 3468 } 3469 break; 3470 3471 case SA_MULTIPATHRECORD_ATTRID: 3472 3473 if ((cap_mask & SA_CAPMASK_MULTIPATH_SUPPORTED) == 0) { 3474 3475 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3476 ibmf_saa_impl_check_sa_support, 3477 IBMF_TNF_ERROR, "", 3478 "ibmf_saa_impl_check_sa_support: %s, " 3479 "cap_mask = 0x%x\n", tnf_string, msg, 3480 "SA does not support multipath records", 3481 tnf_opaque, cap_mask, cap_mask); 3482 3483 attr_supported = B_FALSE; 3484 } 3485 break; 3486 3487 case SA_MCMEMBERRECORD_ATTRID: 3488 3489 if ((cap_mask & SA_CAPMASK_UD_MCAST_SUPPORTED) == 0) { 3490 3491 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 3492 ibmf_saa_impl_check_sa_support, 3493 IBMF_TNF_ERROR, "", 3494 "ibmf_saa_impl_check_sa_support: %s, " 3495 "cap_mask = 0x%x\n", tnf_string, msg, 3496 "SA does not support ud multicast", 3497 tnf_opaque, cap_mask, cap_mask); 3498 3499 attr_supported = B_FALSE; 3500 } 3501 break; 3502 3503 default: 3504 break; 3505 } /* switch */ 3506 3507 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3508 ibmf_saa_impl_check_sa_support_end, IBMF_TNF_TRACE, "", 3509 "ibmf_saa_impl_check_sa_support() exiting, attr_supported = %d\n", 3510 tnf_opaque, attr_supported, attr_supported); 3511 3512 if (attr_supported == B_FALSE) 3513 return (IBMF_UNSUPP_METHOD_ATTR); 3514 else 3515 return (IBMF_SUCCESS); 3516 } 3517 3518 /* 3519 * ibmf_saa_impl_get_attr_id_length: 3520 * 3521 * Returns the host size of the specified sa record. Returns 0 for unknown 3522 * attributes. multipath record size is a dynamic value given as a parameter 3523 * specified with the ibmf_sa_access() call. 3524 */ 3525 static uint_t 3526 ibmf_saa_impl_get_attr_id_length(uint16_t attr_id) 3527 { 3528 uint_t attr_length; 3529 3530 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3531 ibmf_saa_impl_get_attr_id_length_start, 3532 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length() enter\n"); 3533 3534 /* this function should not be used for multipath record */ 3535 ASSERT(attr_id != SA_MULTIPATHRECORD_ATTRID); 3536 3537 switch (attr_id) { 3538 case SA_CLASSPORTINFO_ATTRID: 3539 attr_length = sizeof (ib_mad_classportinfo_t); 3540 break; 3541 case SA_NOTICE_ATTRID: 3542 attr_length = sizeof (ib_mad_notice_t); 3543 break; 3544 case SA_INFORMINFO_ATTRID: 3545 attr_length = sizeof (ib_mad_informinfo_t); 3546 break; 3547 case SA_NODERECORD_ATTRID: 3548 attr_length = sizeof (sa_node_record_t); 3549 break; 3550 case SA_PORTINFORECORD_ATTRID: 3551 attr_length = sizeof (sa_portinfo_record_t); 3552 break; 3553 case SA_SLTOVLRECORD_ATTRID: 3554 attr_length = sizeof (sa_SLtoVLmapping_record_t); 3555 break; 3556 case SA_SWITCHINFORECORD_ATTRID: 3557 attr_length = sizeof (sa_switchinfo_record_t); 3558 break; 3559 case SA_LINEARFDBRECORD_ATTRID: 3560 attr_length = sizeof (sa_linearft_record_t); 3561 break; 3562 case SA_RANDOMFDBRECORD_ATTRID: 3563 attr_length = sizeof (sa_randomft_record_t); 3564 break; 3565 case SA_MULTICASTFDBRECORD_ATTRID: 3566 attr_length = sizeof (sa_multicastft_record_t); 3567 break; 3568 case SA_SMINFORECORD_ATTRID: 3569 attr_length = sizeof (sa_sminfo_record_t); 3570 break; 3571 case SA_INFORMINFORECORD_ATTRID: 3572 attr_length = sizeof (sa_informinfo_record_t); 3573 break; 3574 case SA_LINKRECORD_ATTRID: 3575 attr_length = sizeof (sa_link_record_t); 3576 break; 3577 case SA_GUIDINFORECORD_ATTRID: 3578 attr_length = sizeof (sa_guidinfo_record_t); 3579 break; 3580 case SA_SERVICERECORD_ATTRID: 3581 attr_length = sizeof (sa_service_record_t); 3582 break; 3583 case SA_PARTITIONRECORD_ATTRID: 3584 attr_length = sizeof (sa_pkey_table_record_t); 3585 break; 3586 case SA_PATHRECORD_ATTRID: 3587 attr_length = sizeof (sa_path_record_t); 3588 break; 3589 case SA_VLARBRECORD_ATTRID: 3590 attr_length = sizeof (sa_VLarb_table_record_t); 3591 break; 3592 case SA_MCMEMBERRECORD_ATTRID: 3593 attr_length = sizeof (sa_mcmember_record_t); 3594 break; 3595 case SA_TRACERECORD_ATTRID: 3596 attr_length = sizeof (sa_trace_record_t); 3597 break; 3598 case SA_SERVICEASSNRECORD_ATTRID: 3599 attr_length = sizeof (sa_service_assn_record_t); 3600 break; 3601 default: 3602 /* should only get the above type of packets */ 3603 attr_length = 0; 3604 break; 3605 } 3606 3607 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 3608 ibmf_saa_impl_get_attr_id_length_end, 3609 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_attr_id_length():" 3610 " attr_id: 0x%x size %d\n", 3611 tnf_opaque, attr_id, attr_id, tnf_uint, attr_length, attr_length); 3612 3613 return (attr_length); 3614 } 3615 3616 /* 3617 * ibmf_saa_impl_free_msg: 3618 * Takes a completed message and free memory associated with the message, 3619 * including the individual fields of the im_msgbufs_send. 3620 * ibmf_free_msg, called at the end of this function, takes a pointer to the 3621 * message pointer so that it can set the message pointer to NULL. This 3622 * function takes just the message pointer so the msgp will not be NULL after 3623 * this function returns. 3624 * 3625 * Input Arguments 3626 * ibmf_hdl ibmf handle used in ibmf_msg_alloc 3627 * msgp pointer to ibmf_msg_t to free 3628 * 3629 * Returns 3630 * void 3631 */ 3632 static void 3633 ibmf_saa_impl_free_msg(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msgp) 3634 { 3635 int res; 3636 3637 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3638 ibmf_saa_impl_free_msg_start, 3639 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() enter: msg %p\n", 3640 tnf_opaque, msg, msgp); 3641 3642 ASSERT(msgp != NULL); 3643 3644 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, 3645 sizeof (ib_mad_hdr_t)); 3646 3647 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_hdr, 3648 msgp->im_msgbufs_send.im_bufs_cl_hdr_len); 3649 3650 if (msgp->im_msgbufs_send.im_bufs_cl_data_len > 0) 3651 kmem_free(msgp->im_msgbufs_send.im_bufs_cl_data, 3652 msgp->im_msgbufs_send.im_bufs_cl_data_len); 3653 3654 res = ibmf_free_msg(ibmf_hdl, &msgp); 3655 ASSERT(res == IBMF_SUCCESS); 3656 3657 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3658 ibmf_saa_impl_free_msg_end, 3659 IBMF_TNF_TRACE, "", "ibmf_saa_impl_free_msg() exit\n"); 3660 } 3661 3662 /* 3663 * ibmf_saa_impl_get_port_guid: 3664 */ 3665 static int 3666 ibmf_saa_impl_get_port_guid(ibt_hca_portinfo_t *ibt_portinfop, 3667 ib_guid_t *guid_ret) 3668 { 3669 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3670 ibmf_saa_impl_get_port_guid_start, 3671 IBMF_TNF_TRACE, "", "ibmf_saa_impl_get_port_guid() enter\n"); 3672 3673 if (ibt_portinfop->p_linkstate != IBT_PORT_ACTIVE) { 3674 3675 return (IBMF_BAD_PORT_STATE); 3676 } 3677 3678 if (ibt_portinfop->p_sgid_tbl_sz == 0) { 3679 3680 IBMF_TRACE_1(IBMF_TNF_NODEBUG, DPRINT_L2, 3681 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3682 "ibmf_saa_impl_get_port_guid: %s\n", tnf_string, msg, 3683 "portinfo sgid table size is 0. Exiting.\n"); 3684 3685 return (IBMF_TRANSPORT_FAILURE); 3686 } 3687 3688 *guid_ret = ibt_portinfop->p_sgid_tbl[0].gid_guid; 3689 3690 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 3691 ibmf_saa_impl_get_port_guid_end, IBMF_TNF_TRACE, "", 3692 "ibmf_saa_impl_get_port_guid: Returning port_guid %016" PRIx64 "\n", 3693 tnf_opaque, port_guid, *guid_ret); 3694 3695 return (IBMF_SUCCESS); 3696 } 3697 3698 /* 3699 * ibmf_saa_impl_set_transaction_params: 3700 */ 3701 static void 3702 ibmf_saa_impl_set_transaction_params(saa_port_t *saa_portp, 3703 ibt_hca_portinfo_t *portinfop) 3704 { 3705 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3706 ibmf_saa_impl_set_transaction_params_start, 3707 IBMF_TNF_TRACE, "", 3708 "ibmf_saa_impl_set_transaction_params() enter\n"); 3709 3710 _NOTE(ASSUMING_PROTECTED(*saa_portp)) 3711 3712 saa_portp->saa_pt_ibmf_retrans.retrans_retries = 3713 IBMF_SAA_RETRANS_RETRIES; 3714 /* 3715 * For the first transaction (generally getting the 3716 * classportinfo) have ibmf pick our timeouts. It should be using the 3717 * default IB spec values. 3718 * Once we get the classportinfo we'll update the correct response time 3719 * value (rtv) and round-trip time (rttv). ibmf should always calculate 3720 * trans_to since it depends on the particular transaction's number of 3721 * packets. 3722 */ 3723 saa_portp->saa_pt_ibmf_retrans.retrans_rtv = 0; 3724 saa_portp->saa_pt_ibmf_retrans.retrans_rttv = 0; 3725 saa_portp->saa_pt_ibmf_retrans.retrans_trans_to = 0; 3726 3727 /* 3728 * Assume that the SA supports all optional records. If it 3729 * does not, the request will get returned with ERR_NOT_SUPP. When 3730 * the classportinfo response comes back we will update the cap mask 3731 * to prevent unnecessary unsupported requests. 3732 */ 3733 saa_portp->saa_pt_sa_cap_mask = 0xFFFF; 3734 3735 saa_portp->saa_pt_ibmf_msg_flags = 0; 3736 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = 1; 3737 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = 3738 IB_PKEY_DEFAULT_LIMITED; 3739 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = IB_GSI_QKEY; 3740 3741 /* 3742 * fill out addr information for MADs that will be sent 3743 * to SA on this port 3744 */ 3745 saa_portp->saa_pt_ibmf_addr_info.ia_local_lid = portinfop->p_base_lid; 3746 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = portinfop->p_sm_lid; 3747 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = portinfop->p_sm_sl; 3748 3749 /* place upper bound on subnet timeout in case of faulty SM */ 3750 saa_portp->saa_pt_timeout = portinfop->p_subnet_timeout; 3751 3752 if (saa_portp->saa_pt_timeout > ibmf_saa_max_subnet_timeout) 3753 saa_portp->saa_pt_timeout = ibmf_saa_max_subnet_timeout; 3754 3755 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L3, 3756 ibmf_saa_impl_set_transaction_params, 3757 IBMF_TNF_TRACE, "", 3758 "ibmf_saa_impl_set_transaction_params: local_lid = 0x%x, " 3759 "sm_lid = 0x%x, sm_sl = 0x%x, sn_timeout = 0x%x\n", 3760 tnf_opaque, local_lid, portinfop->p_base_lid, 3761 tnf_opaque, sm_lid, portinfop->p_sm_lid, 3762 tnf_opaque, sm_sl, portinfop->p_sm_sl, 3763 tnf_opaque, subnet_timeout, portinfop->p_subnet_timeout); 3764 3765 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3766 ibmf_saa_impl_set_transaction_params_end, 3767 IBMF_TNF_TRACE, "", 3768 "ibmf_saa_impl_set_transaction_params() exit\n"); 3769 } 3770 3771 3772 /* 3773 * ibmf_saa_impl_update_sa_address_info 3774 */ 3775 static void 3776 ibmf_saa_impl_update_sa_address_info(saa_port_t *saa_portp, ibmf_msg_t *msgp) 3777 { 3778 void *result; 3779 ib_sa_hdr_t *sa_hdr; 3780 int rv; 3781 size_t length; 3782 uint16_t attr_id; 3783 ib_mad_classportinfo_t *cpi; 3784 ibmf_global_addr_info_t *gaddrp = &saa_portp->saa_pt_ibmf_global_addr; 3785 ibt_hca_portinfo_t *ibt_pinfo; 3786 uint_t nports, size; 3787 ibt_status_t ibt_status; 3788 3789 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 3790 ibmf_saa_impl_update_sa_address_info, 3791 IBMF_TNF_TRACE, "", 3792 "ibmf_saa_impl_update_sa_address_info() enter\n"); 3793 3794 /* 3795 * decode the respons of msgp as a classportinfo attribute 3796 */ 3797 rv = ibmf_saa_utils_unpack_sa_hdr(msgp->im_msgbufs_recv.im_bufs_cl_hdr, 3798 msgp->im_msgbufs_recv.im_bufs_cl_hdr_len, &sa_hdr, KM_NOSLEEP); 3799 if (rv != IBMF_SUCCESS) { 3800 3801 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3802 ibmf_saa_impl_update_sa_address_err, 3803 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3804 "%s, ibmf_status = %d\n", tnf_string, msg, 3805 "Could not unpack sa hdr", tnf_int, ibmf_status, rv); 3806 3807 return; 3808 } 3809 3810 attr_id = b2h16(msgp->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID); 3811 if (attr_id != MAD_ATTR_ID_CLASSPORTINFO) { 3812 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3813 ibmf_saa_impl_update_sa_address_info_err, 3814 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3815 "%s, attrID = %x\n", tnf_string, msg, 3816 "Wrong attribute ID", tnf_int, ibmf_status, attr_id); 3817 3818 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3819 return; 3820 } 3821 rv = ibmf_saa_utils_unpack_payload( 3822 msgp->im_msgbufs_recv.im_bufs_cl_data, 3823 msgp->im_msgbufs_recv.im_bufs_cl_data_len, attr_id, &result, 3824 &length, sa_hdr->AttributeOffset, B_TRUE, KM_NOSLEEP); 3825 if (rv != IBMF_SUCCESS) { 3826 3827 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3828 ibmf_saa_impl_update_sa_address_err, 3829 IBMF_TNF_TRACE, "", "ibmf_saa_impl_update_sa_address_info: " 3830 "%s, ibmf_status = %d\n", tnf_string, msg, 3831 "Could not unpack payload", tnf_int, ibmf_status, rv); 3832 3833 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3834 return; 3835 } 3836 3837 kmem_free(sa_hdr, sizeof (ib_sa_hdr_t)); 3838 3839 /* 3840 * Use the classportinfo contents to update the SA address info 3841 */ 3842 cpi = (ib_mad_classportinfo_t *)result; 3843 mutex_enter(&saa_portp->saa_pt_mutex); 3844 saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid = cpi->RedirectLID; 3845 saa_portp->saa_pt_ibmf_addr_info.ia_remote_qno = cpi->RedirectQP; 3846 saa_portp->saa_pt_ibmf_addr_info.ia_p_key = cpi->RedirectP_Key; 3847 saa_portp->saa_pt_ibmf_addr_info.ia_q_key = cpi->RedirectQ_Key; 3848 saa_portp->saa_pt_ibmf_addr_info.ia_service_level = cpi->RedirectSL; 3849 3850 saa_portp->saa_pt_redirect_active = B_TRUE; 3851 3852 if ((cpi->RedirectGID_hi != 0) || (cpi->RedirectGID_lo != 0)) { 3853 3854 mutex_exit(&saa_portp->saa_pt_mutex); 3855 ibt_status = ibt_query_hca_ports_byguid( 3856 saa_portp->saa_pt_node_guid, saa_portp->saa_pt_port_num, 3857 &ibt_pinfo, &nports, &size); 3858 if (ibt_status != IBT_SUCCESS) { 3859 3860 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3861 ibmf_saa_impl_update_sa_address_err, IBMF_TNF_TRACE, 3862 "", "ibmf_saa_impl_update_sa_address_info: " 3863 "%s, ibt_status = %d\n", tnf_string, msg, 3864 "Could not query hca port", 3865 tnf_int, ibt_status, ibt_status); 3866 3867 kmem_free(result, length); 3868 return; 3869 } 3870 3871 mutex_enter(&saa_portp->saa_pt_mutex); 3872 /* 3873 * Fill in global address info parameters 3874 * 3875 * NOTE: The HopLimit value is not specified through the 3876 * contents of ClassPortInfo. It may be possible to find 3877 * out the proper value to use even for SA beeing redirected 3878 * to another subnet. But we do only support redirect within 3879 * our local subnet 3880 */ 3881 gaddrp->ig_sender_gid.gid_prefix = 3882 ibt_pinfo->p_sgid_tbl[0].gid_prefix; 3883 gaddrp->ig_sender_gid.gid_guid = saa_portp->saa_pt_port_guid; 3884 gaddrp->ig_recver_gid.gid_prefix = cpi->RedirectGID_hi; 3885 gaddrp->ig_recver_gid.gid_guid = cpi->RedirectGID_lo; 3886 gaddrp->ig_flow_label = cpi->RedirectFL; 3887 gaddrp->ig_tclass = cpi->RedirectTC; 3888 gaddrp->ig_hop_limit = 0; 3889 3890 saa_portp->saa_pt_ibmf_msg_flags = 3891 IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3892 3893 mutex_exit(&saa_portp->saa_pt_mutex); 3894 ibt_free_portinfo(ibt_pinfo, size); 3895 } else { 3896 saa_portp->saa_pt_ibmf_msg_flags = 0; 3897 mutex_exit(&saa_portp->saa_pt_mutex); 3898 } 3899 kmem_free(result, length); 3900 3901 /* 3902 * Update the address info of msgp with the new address parameters 3903 */ 3904 mutex_enter(&saa_portp->saa_pt_mutex); 3905 bcopy(&saa_portp->saa_pt_ibmf_addr_info, &msgp->im_local_addr, 3906 sizeof (ibmf_addr_info_t)); 3907 if (saa_portp->saa_pt_ibmf_msg_flags & IBMF_MSG_FLAGS_GLOBAL_ADDRESS) { 3908 3909 msgp->im_msg_flags = IBMF_MSG_FLAGS_GLOBAL_ADDRESS; 3910 3911 bcopy(&saa_portp->saa_pt_ibmf_global_addr, 3912 &msgp->im_global_addr, sizeof (ibmf_global_addr_info_t)); 3913 } else { 3914 msgp->im_msg_flags = 0; 3915 } 3916 mutex_exit(&saa_portp->saa_pt_mutex); 3917 } 3918 3919 /* 3920 * ibmf_saa_impl_ibmf_unreg: 3921 */ 3922 static int 3923 ibmf_saa_impl_ibmf_unreg(saa_port_t *saa_portp) 3924 { 3925 int ibmf_status; 3926 3927 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_start, 3928 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() enter\n"); 3929 3930 /* teardown async cb */ 3931 ibmf_status = ibmf_tear_down_async_cb(saa_portp->saa_pt_ibmf_handle, 3932 saa_portp->saa_pt_qp_handle, 0); 3933 if (ibmf_status != IBMF_SUCCESS) { 3934 3935 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3936 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3937 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3938 tnf_string, msg, "Could not tear down async cb", 3939 tnf_int, ibmf_status, ibmf_status); 3940 3941 goto bail; 3942 } 3943 3944 /* free qp */ 3945 ibmf_status = ibmf_free_qp(saa_portp->saa_pt_ibmf_handle, 3946 &saa_portp->saa_pt_qp_handle, 0); 3947 3948 if (ibmf_status != IBMF_SUCCESS) { 3949 3950 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3951 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3952 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3953 tnf_string, msg, "Could not free queue pair", 3954 tnf_int, ibmf_status, ibmf_status); 3955 3956 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 1); 3957 goto bail; 3958 } 3959 3960 ibmf_status = ibmf_unregister(&saa_portp->saa_pt_ibmf_handle, 0); 3961 3962 if (ibmf_status != IBMF_SUCCESS) { 3963 3964 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L1, 3965 ibmf_saa_impl_ibmf_unreg, IBMF_TNF_TRACE, "", 3966 "ibmf_saa_impl_ibmf_unreg: %s, ibmf_status = %d\n", 3967 tnf_string, msg, "ibmf_unregister() failed", 3968 tnf_int, ibmf_status, ibmf_status); 3969 3970 (void) ibmf_saa_impl_setup_qp_async_cb(saa_portp, 0); 3971 } 3972 3973 bail: 3974 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_ibmf_unreg_end, 3975 IBMF_TNF_TRACE, "", "ibmf_saa_impl_ibmf_unreg() exit\n"); 3976 3977 return (ibmf_status); 3978 } 3979