1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * ibcm_utils.c 28 * 29 * contains internal lookup functions of IB CM module 30 * along with some other miscellaneous stuff 31 * 32 * TBD: 33 * 1. Code needed to ensure that if any clients are using a service then 34 * don't de-register it. 35 */ 36 37 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 38 #include <sys/ddi.h> 39 40 41 /* statics */ 42 static vmem_t *ibcm_local_sid_arena; 43 static vmem_t *ibcm_ip_sid_arena; 44 static ib_svc_id_t ibcm_local_sid_seed; 45 static ib_com_id_t ibcm_local_cid_seed; 46 _NOTE(READ_ONLY_DATA({ibcm_local_sid_arena ibcm_local_sid_seed 47 ibcm_ip_sid_arena ibcm_local_cid_seed})) 48 static void ibcm_delete_state_from_avl(ibcm_state_data_t *statep); 49 static void ibcm_init_conn_trace(ibcm_state_data_t *statep); 50 static void ibcm_fini_conn_trace(ibcm_state_data_t *statep); 51 static void ibcm_dump_conn_trbuf(void *statep, char *line_prefix, 52 char *buf, int buf_size); 53 54 /* 55 * ibcm_lookup_msg: 56 * 57 * Retrieves an existing state structure or creates a new one if none found. 58 * This function is used during 59 * Passive connection side for INCOMING REQ/REJ/RTU/MRA/DREQ/DREP/LAP msgs 60 * Active connection side for INCOMING REP/REJ/MRA/DREQ/DREP/APR msgs 61 * Active side CM for outgoing REQ message. 62 * 63 * NOTE: Only return IBCM_LOOKUP_FAIL if lookup failed to find a match. 64 * 65 * Arguments are:- 66 * event_type - type of message 67 * incoming REQ, REP, REJ, MRA, RTU 68 * remote_qpn - Remote QP number 69 * comid - local/remote comid 70 * remote_hca_guid - Remote HCA GUID 71 * hcap - HCA entry ptr 72 * rstatep - return statep pointer 73 * 74 * Return Values: 75 * IBCM_LOOKUP_NEW - new statep allocated 76 * IBCM_LOOKUP_EXISTS - found an existing entry 77 * IBCM_LOOKUP_FAIL - No lookup entry found 78 * IBCM_MEMORY_FAILURE - Memory allocs failed 79 */ 80 ibcm_status_t 81 ibcm_lookup_msg(ibcm_event_type_t event_type, ib_com_id_t comid, 82 ib_qpn_t remote_qpn, ib_guid_t remote_hca_guid, ibcm_hca_info_t *hcap, 83 ibcm_state_data_t **rstatep) 84 { 85 avl_index_t where; 86 ibcm_state_data_t *sp; 87 88 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: event = 0x%x, comid = 0x%x", 89 event_type, comid); 90 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: rem_qpn = 0x%lX, " 91 "rem_hca_guid = 0x%llX", remote_qpn, remote_hca_guid); 92 93 ASSERT(rw_lock_held(&hcap->hca_state_rwlock)); 94 95 /* 96 * Lookup in "hca_passive_tree" for IBCM_INCOMING_REQ and 97 * IBCM_INCOMING_REP_STALE; 98 * 99 * Lookup in "hca_passive_comid_tree" for IBCM_INCOMING_REQ_STALE 100 * 101 * All other lookups in "hca_active_tree". 102 * 103 * NOTE: "hca_active_tree" lookups are based on the local comid. 104 * "hca_passive_state_tree" lookups are based on remote QPN 105 * and remote hca GUID. 106 * 107 * Call avl_find to lookup in the respective tree and save result in 108 * "sp". If "sp" is null it implies that no match was found. If so, 109 * allocate a new ibcm_state_data_t and insert it into the AVL tree(s). 110 */ 111 if ((event_type == IBCM_INCOMING_REQ) || 112 (event_type == IBCM_INCOMING_REP_STALE)) { 113 ibcm_passive_node_info_t info; 114 115 info.info_qpn = remote_qpn; 116 info.info_hca_guid = remote_hca_guid; 117 118 /* Lookup based on Remote QPN and Remote GUID in Passive Tree */ 119 sp = avl_find(&hcap->hca_passive_tree, &info, &where); 120 } else if ((event_type == IBCM_INCOMING_REQ_STALE) || 121 (event_type == IBCM_INCOMING_REJ_RCOMID)) { 122 ibcm_passive_comid_node_info_t info; 123 124 info.info_comid = comid; 125 info.info_hca_guid = remote_hca_guid; 126 127 /* Lookup based on Remote COMID in Passive Tree */ 128 sp = avl_find(&hcap->hca_passive_comid_tree, &info, &where); 129 } else { /* any other event including IBCM_OUTGOING_REQ */ 130 /* Lookup based on Local comid in Active Tree */ 131 sp = avl_find(&hcap->hca_active_tree, &comid, &where); 132 } 133 134 /* matching entry found !! */ 135 if (sp != NULL) { 136 IBTF_DPRINTF_L4(cmlog, "ibcm_lookup_msg: match found " 137 "statep = %p", sp); 138 if (event_type == IBCM_INCOMING_REQ) 139 kmem_free(*rstatep, sizeof (ibcm_state_data_t)); 140 *rstatep = sp; /* return the matched statep */ 141 142 mutex_enter(&(sp->state_mutex)); 143 IBCM_REF_CNT_INCR(sp); /* increment the ref count */ 144 mutex_exit(&(sp->state_mutex)); 145 146 return (IBCM_LOOKUP_EXISTS); 147 } 148 149 /* 150 * If we came here then it implies that CM didn't 151 * find a matching entry. We will create a new entry in avl tree, 152 * if event_type is INCOMING/OUTGOING REQ, REQ_STALE/REP_STALE. 153 * statep is created for INCOMING/OUTGOING REQ. 154 * For all other event_types we return lookup failure 155 */ 156 if (!((event_type == IBCM_INCOMING_REQ) || 157 (event_type == IBCM_INCOMING_REQ_STALE) || 158 (event_type == IBCM_INCOMING_REP_STALE) || 159 (event_type == IBCM_OUTGOING_REQ))) { 160 IBTF_DPRINTF_L2(cmlog, "ibcm_lookup_msg: failed for " 161 "event type %x remote_comid = 0x%x", 162 event_type, comid); 163 164 return (IBCM_LOOKUP_FAIL); 165 } 166 167 if ((event_type == IBCM_INCOMING_REQ) || 168 (event_type == IBCM_OUTGOING_REQ)) { 169 170 /* fill in the new ibcm_state_data */ 171 sp = *rstatep; 172 173 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sp)) 174 175 /* initialize statep */ 176 mutex_init(&sp->state_mutex, NULL, MUTEX_DEFAULT, NULL); 177 cv_init(&sp->block_client_cv, NULL, CV_DRIVER, NULL); 178 cv_init(&sp->block_mad_cv, NULL, CV_DRIVER, NULL); 179 180 sp->hcap = hcap; 181 IBCM_REF_CNT_INCR(sp); 182 sp->local_comid = comid; 183 184 if (ibcm_enable_trace != 0) 185 ibcm_init_conn_trace(sp); 186 187 if (event_type == IBCM_INCOMING_REQ) { /* Passive side */ 188 sp->state = IBCM_STATE_REQ_RCVD; 189 sp->clnt_proceed = IBCM_BLOCK; 190 sp->close_nocb_state = IBCM_UNBLOCK; 191 sp->remote_hca_guid = remote_hca_guid; 192 sp->remote_qpn = remote_qpn; 193 194 } else if (event_type == IBCM_OUTGOING_REQ) { /* Active side */ 195 sp->close_nocb_state = IBCM_UNBLOCK; 196 sp->state = IBCM_STATE_IDLE; 197 } 198 199 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sp)) 200 201 } else { 202 sp = *rstatep; /* for incoming REQ/REP STALE only */ 203 } 204 205 if ((event_type == IBCM_INCOMING_REQ) || 206 (event_type == IBCM_INCOMING_REP_STALE)) { 207 208 /* First, insert a new "sp" into "hca_passive_tree" @ "where" */ 209 avl_insert(&(hcap->hca_passive_tree), (void *)sp, where); 210 211 if (event_type == IBCM_INCOMING_REQ) { /* Only INCOMING_REQ */ 212 /* 213 * We have to do an avl_find() to figure out 214 * "where" to insert the statep into the active tree. 215 * 216 * CM doesn't care for avl_find's retval. 217 */ 218 (void) avl_find(&hcap->hca_active_tree, 219 &sp->local_comid, &where); 220 221 /* Next, insert the "sp" into "hca_active_tree" */ 222 avl_insert(&hcap->hca_active_tree, (void *)sp, where); 223 } 224 } else if (event_type == IBCM_INCOMING_REQ_STALE) { 225 avl_insert(&(hcap->hca_passive_comid_tree), (void *)sp, where); 226 } else { /* IBCM_OUTGOING_REQ */ 227 /* Insert the new sp only into "hca_active_tree", @ "where" */ 228 avl_insert(&(hcap->hca_active_tree), (void *)sp, where); 229 } 230 231 return (IBCM_LOOKUP_NEW); /* return new lookup */ 232 } 233 234 235 /* 236 * ibcm_active_node_compare: 237 * - AVL active tree node compare 238 * 239 * Arguments: 240 * p1 : pointer to local comid 241 * p2 : pointer to passed ibcm_state_data_t 242 * 243 * Return values: 244 * 0 : match found 245 * -1 : no match but insert to left side of the tree 246 * +1 : no match but insert to right side of the tree 247 */ 248 int 249 ibcm_active_node_compare(const void *p1, const void *p2) 250 { 251 ib_com_id_t *local_comid = (ib_com_id_t *)p1; 252 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2; 253 254 IBTF_DPRINTF_L5(cmlog, "ibcm_active_node_compare: " 255 "comid: 0x%x, statep: 0x%p", *local_comid, statep); 256 257 if (*local_comid > statep->local_comid) { 258 return (+1); 259 } else if (*local_comid < statep->local_comid) { 260 return (-1); 261 } else { 262 return (0); 263 } 264 } 265 266 267 /* 268 * ibcm_passive_node_compare: 269 * - AVL passive tree node compare (passive side) 270 * 271 * Arguments: 272 * p1 : pointer to ibcm_passive_node_info (remote qpn and remote guid) 273 * p2 : pointer to passed ibcm_state_data_t 274 * 275 * Return values: 276 * 0 : match found 277 * -1 : no match but insert to left side of the tree 278 * +1 : no match but insert to right side of the tree 279 */ 280 int 281 ibcm_passive_node_compare(const void *p1, const void *p2) 282 { 283 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2; 284 ibcm_passive_node_info_t *infop = (ibcm_passive_node_info_t *)p1; 285 286 IBTF_DPRINTF_L5(cmlog, "ibcm_passive_node_compare: " 287 "statep: 0x%p, p1: 0x%p", statep, p1); 288 289 /* 290 * PASSIVE SIDE: (REQ, REP, MRA, REJ) 291 * always search by active COMID 292 */ 293 if (infop->info_qpn > statep->remote_qpn) { 294 return (+1); 295 } else if (infop->info_qpn < statep->remote_qpn) { 296 return (-1); 297 } else { 298 if (infop->info_hca_guid < statep->remote_hca_guid) { 299 return (-1); 300 } else if (infop->info_hca_guid > statep->remote_hca_guid) { 301 return (+1); 302 } else { 303 return (0); 304 } 305 } 306 } 307 308 /* 309 * ibcm_passive_comid_node_compare: 310 * - AVL passive comid tree node compare (passive side) 311 * 312 * Arguments: 313 * p1 : pointer to ibcm_passive_comid_node_info 314 * (remote comid and remote guid) 315 * p2 : pointer to passed ibcm_state_data_t 316 * 317 * Return values: 318 * 0 : match found 319 * -1 : no match but insert to left side of the tree 320 * +1 : no match but insert to right side of the tree 321 */ 322 int 323 ibcm_passive_comid_node_compare(const void *p1, const void *p2) 324 { 325 ibcm_state_data_t *statep = (ibcm_state_data_t *)p2; 326 ibcm_passive_comid_node_info_t *infop = 327 (ibcm_passive_comid_node_info_t *)p1; 328 329 IBTF_DPRINTF_L5(cmlog, "ibcm_passive_comid_node_compare: " 330 "statep: 0x%p, p1: 0x%p", statep, p1); 331 332 if (infop->info_comid > statep->remote_comid) { 333 return (+1); 334 } else if (infop->info_comid < statep->remote_comid) { 335 return (-1); 336 } else { 337 if (infop->info_hca_guid < statep->remote_hca_guid) { 338 return (-1); 339 } else if (infop->info_hca_guid > statep->remote_hca_guid) { 340 return (+1); 341 } else { 342 return (0); 343 } 344 } 345 } 346 347 348 void 349 ibcm_delete_state_from_avl(ibcm_state_data_t *statep) 350 { 351 avl_index_t a_where = 0; 352 avl_index_t p_where = 0; 353 avl_index_t pcomid_where = 0; 354 ibcm_hca_info_t *hcap; 355 ibcm_state_data_t *active_nodep, *passive_nodep; 356 ibcm_state_data_t *passive_comid_nodep; 357 ibcm_passive_node_info_t info; 358 ibcm_passive_comid_node_info_t info_comid; 359 360 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_from_avl: statep 0x%p", 361 statep); 362 363 if (statep == NULL) { 364 IBTF_DPRINTF_L2(cmlog, "ibcm_delete_state_from_avl: statep" 365 " NULL"); 366 return; 367 } 368 369 hcap = statep->hcap; 370 371 /* 372 * Once the avl tree lock is acquired, no other thread can increment 373 * ref cnt, until tree lock is exit'ed. Since the statep is removed 374 * from the avl's after acquiring lock below, no other thread can 375 * increment the ref cnt after acquiring the lock below 376 */ 377 378 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 379 380 /* Lookup based on Local comid in the active tree */ 381 active_nodep = avl_find(&hcap->hca_active_tree, &(statep->local_comid), 382 &a_where); 383 384 /* Lookup based on Remote QPN and Remote GUID in the passive tree */ 385 info.info_qpn = statep->remote_qpn; 386 info.info_hca_guid = statep->remote_hca_guid; 387 passive_nodep = avl_find(&hcap->hca_passive_tree, &info, &p_where); 388 389 /* Lookup based on Remote Comid and Remote GUID in the passive tree */ 390 info_comid.info_comid = statep->remote_comid; 391 info_comid.info_hca_guid = statep->remote_hca_guid; 392 passive_comid_nodep = avl_find(&hcap->hca_passive_comid_tree, 393 &info_comid, &pcomid_where); 394 395 /* remove it from the tree, destroy record and the nodep */ 396 if (active_nodep == statep) { 397 avl_remove(&hcap->hca_active_tree, active_nodep); 398 } 399 400 if (passive_nodep == statep) { 401 avl_remove(&hcap->hca_passive_tree, passive_nodep); 402 } 403 404 if (passive_comid_nodep == statep) { 405 avl_remove(&hcap->hca_passive_comid_tree, passive_comid_nodep); 406 } 407 408 rw_exit(&hcap->hca_state_rwlock); 409 } 410 411 /* 412 * ibcm_dealloc_state_data: 413 * Deallocates all buffers and the memory of state structure 414 * This routine can be called on statep that has ref_cnt of 0, and that is 415 * already deleted from the avl tree's 416 * 417 * Arguments are:- 418 * statep - statep to be deleted 419 * 420 * Return Values: NONE 421 */ 422 void 423 ibcm_dealloc_state_data(ibcm_state_data_t *statep) 424 { 425 timeout_id_t timer_val; 426 int dump_trace; 427 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_state_data: statep 0x%p", statep); 428 429 if (statep == NULL) { 430 IBTF_DPRINTF_L2(cmlog, "ibcm_dealloc_state_data: statep NULL"); 431 return; 432 } 433 434 /* ref_cnt is 0 */ 435 /* If timer is running - expire it */ 436 mutex_enter(&statep->state_mutex); 437 timer_val = statep->timerid; 438 if (timer_val != 0) { 439 statep->timerid = 0; 440 mutex_exit(&statep->state_mutex); 441 (void) untimeout(timer_val); 442 } else 443 mutex_exit(&statep->state_mutex); 444 445 /* release the ref cnt on the associated ibmf qp */ 446 if (statep->stored_reply_addr.cm_qp_entry != NULL) 447 ibcm_release_qp(statep->stored_reply_addr.cm_qp_entry); 448 449 if (statep->stored_msg != NULL) 450 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, 451 &statep->stored_msg); 452 453 if (statep->dreq_msg != NULL) 454 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, 455 &statep->dreq_msg); 456 457 if (statep->drep_msg != NULL) 458 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, 459 &statep->drep_msg); 460 461 if (statep->mra_msg != NULL) 462 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, 463 &statep->mra_msg); 464 465 if (statep->lapr_msg != NULL) 466 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, 467 &statep->lapr_msg); 468 469 if (statep->defer_cm_msg != NULL) 470 kmem_free(statep->defer_cm_msg, IBCM_MSG_SIZE); 471 472 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_state_data: done for sp = 0x%p", 473 statep); 474 475 /* Ensure the thread doing ref cnt decr releases the mutex */ 476 mutex_enter(&statep->state_mutex); 477 dump_trace = statep->cm_retries > 0; 478 mutex_exit(&statep->state_mutex); 479 480 /* 481 * now call the mutex_destroy() and cv_destroy() 482 */ 483 mutex_destroy(&statep->state_mutex); 484 485 cv_destroy(&statep->block_client_cv); 486 cv_destroy(&statep->block_mad_cv); 487 488 /* free the comid */ 489 ibcm_free_comid(statep->hcap, statep->local_comid); 490 491 /* Decrement the resource on hcap */ 492 ibcm_dec_hca_res_cnt(statep->hcap); 493 494 /* dump the trace data into ibtf_debug_buf */ 495 if ((ibcm_enable_trace & 4) || dump_trace) 496 ibcm_dump_conn_trace(statep); 497 498 ibcm_fini_conn_trace(statep); 499 500 /* free the statep */ 501 kmem_free(statep, sizeof (ibcm_state_data_t)); 502 } 503 504 /* 505 * ibcm_delete_state_data: 506 * Deletes the state from avl trees, and tries to deallocate state 507 * 508 * Arguments are:- 509 * statep - statep to be deleted 510 * 511 * Return Values: NONE 512 */ 513 void 514 ibcm_delete_state_data(ibcm_state_data_t *statep) 515 { 516 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_data:"); 517 518 ibcm_delete_state_from_avl(statep); 519 520 /* Must acquire the state mutex to set delete_state_data */ 521 mutex_enter(&statep->state_mutex); 522 if (statep->ref_cnt > 0) { 523 statep->delete_state_data = B_TRUE; 524 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_state_data: statep 0x%p " 525 "ref_cnt = %x", statep, statep->ref_cnt); 526 mutex_exit(&statep->state_mutex); 527 return; 528 } 529 mutex_exit(&statep->state_mutex); 530 531 ibcm_dealloc_state_data(statep); 532 } 533 534 /* 535 * ibcm_find_sidr_entry: 536 * Routines for CM SIDR state structure list manipulation. 537 * Finds an entry based on lid, gid and grh exists fields 538 * 539 * INPUTS: 540 * lid: LID of incoming SIDR REQ 541 * gid: GID of incoming SIDR REQ 542 * grh_exists: TRUE if GRH exists in the incoming SIDR REQ 543 * req_id: Request ID 544 * hcap: CM State table to search for SIDR state structure 545 * statep: Returns a valid state structure, if one exists based 546 * on lid, gid and grh_exists fields 547 * flag: IBCM_FLAG_LOOKUP - just lookup 548 * IBCM_FLAG_LOOKUP_AND_ADD - if lookup fails, add it. 549 * Return Values: 550 * IBCM_LOOKUP_EXISTS - found an existing entry 551 * IBCM_LOOKUP_FAIL - failed to find an entry 552 * IBCM_LOOKUP_NEW - created a new entry 553 */ 554 ibcm_status_t 555 ibcm_find_sidr_entry(ibcm_sidr_srch_t *srch_param, ibcm_hca_info_t *hcap, 556 ibcm_ud_state_data_t **ud_statep, ibcm_lookup_flag_t flag) 557 { 558 ibcm_status_t status; 559 ibcm_ud_state_data_t *usp; 560 561 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: srch_params are:" 562 "lid=%x, (%llX, %llX), grh: %x, id: %x", 563 srch_param->srch_lid, srch_param->srch_gid.gid_prefix, 564 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists, 565 srch_param->srch_req_id); 566 567 if (flag == IBCM_FLAG_ADD) { 568 *ud_statep = ibcm_add_sidr_entry(srch_param, hcap); 569 return (IBCM_LOOKUP_NEW); 570 } 571 572 usp = hcap->hca_sidr_list; /* Point to the list */ 573 574 /* traverse the list for a matching entry */ 575 while (usp != NULL) { 576 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: " 577 "lid=%x, (%llX, %llX), grh: %x, id: %x", 578 usp->ud_sidr_req_lid, usp->ud_sidr_req_gid.gid_prefix, 579 usp->ud_sidr_req_gid.gid_guid, usp->ud_grh_exists, 580 usp->ud_req_id); 581 582 if ((usp->ud_sidr_req_lid == srch_param->srch_lid) && 583 ((srch_param->srch_gid.gid_prefix == 0) || 584 (srch_param->srch_gid.gid_prefix == 585 usp->ud_sidr_req_gid.gid_prefix)) && 586 ((srch_param->srch_gid.gid_guid == 0) || 587 (srch_param->srch_gid.gid_guid == 588 usp->ud_sidr_req_gid.gid_guid)) && 589 (srch_param->srch_req_id == usp->ud_req_id) && 590 (usp->ud_grh_exists == srch_param->srch_grh_exists) && 591 (usp->ud_mode == srch_param->srch_mode)) { /* found match */ 592 *ud_statep = usp; 593 IBTF_DPRINTF_L5(cmlog, "ibcm_find_sidr_entry: " 594 "found usp = %p", usp); 595 mutex_enter(&usp->ud_state_mutex); 596 IBCM_UD_REF_CNT_INCR(usp); 597 mutex_exit(&usp->ud_state_mutex); 598 599 return (IBCM_LOOKUP_EXISTS); 600 } 601 usp = usp->ud_nextp; 602 } 603 604 /* 605 * If code came here --> it couldn't find a match. 606 * OR 607 * the "hcap->hca_sidr_list" was NULL 608 */ 609 if (flag == IBCM_FLAG_LOOKUP) { 610 IBTF_DPRINTF_L3(cmlog, "ibcm_find_sidr_entry: no match found " 611 "lid=%x, (%llX, %llX), grh: %x, id: %x", 612 srch_param->srch_lid, srch_param->srch_gid.gid_prefix, 613 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists, 614 srch_param->srch_req_id); 615 status = IBCM_LOOKUP_FAIL; 616 } else { 617 *ud_statep = ibcm_add_sidr_entry(srch_param, hcap); 618 status = IBCM_LOOKUP_NEW; 619 } 620 621 return (status); 622 } 623 624 625 /* 626 * ibcm_add_sidr_entry: 627 * Adds a SIDR entry. Called *ONLY* from ibcm_find_sidr_entry() 628 * 629 * INPUTS: 630 * lid: LID of incoming SIDR REQ 631 * gid: GID of incoming SIDR REQ 632 * grh_exists: TRUE if GRH exists in the incoming SIDR REQ 633 * req_id: Request ID 634 * hcap: CM State table to search for SIDR state structure 635 * Return Values: NONE 636 */ 637 ibcm_ud_state_data_t * 638 ibcm_add_sidr_entry(ibcm_sidr_srch_t *srch_param, ibcm_hca_info_t *hcap) 639 { 640 ibcm_ud_state_data_t *ud_statep; 641 642 IBTF_DPRINTF_L5(cmlog, "ibcm_add_sidr_entry: lid=%x, guid=%llX, " 643 "grh = %x req_id = %x", srch_param->srch_lid, 644 srch_param->srch_gid.gid_guid, srch_param->srch_grh_exists, 645 srch_param->srch_req_id); 646 647 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 648 649 /* didn't find the entry - so create new */ 650 ud_statep = kmem_zalloc(sizeof (ibcm_ud_state_data_t), KM_SLEEP); 651 652 mutex_init(&ud_statep->ud_state_mutex, NULL, MUTEX_DEFAULT, NULL); 653 cv_init(&ud_statep->ud_block_client_cv, NULL, CV_DRIVER, NULL); 654 655 /* Initialize some ud_statep fields */ 656 mutex_enter(&ud_statep->ud_state_mutex); 657 ud_statep->ud_hcap = hcap; 658 ud_statep->ud_req_id = srch_param->srch_req_id; 659 ud_statep->ud_ref_cnt = 1; 660 ud_statep->ud_grh_exists = srch_param->srch_grh_exists; 661 ud_statep->ud_sidr_req_lid = srch_param->srch_lid; 662 ud_statep->ud_sidr_req_gid = srch_param->srch_gid; 663 ud_statep->ud_mode = srch_param->srch_mode; 664 ud_statep->ud_max_cm_retries = ibcm_max_retries; 665 mutex_exit(&ud_statep->ud_state_mutex); 666 667 /* Update the list */ 668 ud_statep->ud_nextp = hcap->hca_sidr_list; 669 hcap->hca_sidr_list = ud_statep; 670 671 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 672 673 return (ud_statep); 674 } 675 676 677 /* 678 * ibcm_delete_ud_state_data: 679 * Deletes a given state structure 680 * 681 * Arguments are:- 682 * statep - statep to be deleted 683 * 684 * Return Values: NONE 685 */ 686 void 687 ibcm_delete_ud_state_data(ibcm_ud_state_data_t *ud_statep) 688 { 689 ibcm_ud_state_data_t *prevp, *headp; 690 ibcm_hca_info_t *hcap; 691 692 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_ud_state_data: ud_statep 0x%p", 693 ud_statep); 694 695 if (ud_statep == NULL || ud_statep->ud_hcap == NULL) { 696 IBTF_DPRINTF_L2(cmlog, "ibcm_delete_ud_state_data: " 697 "ud_statep or hcap is NULL"); 698 return; 699 } 700 701 hcap = ud_statep->ud_hcap; 702 703 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 704 705 /* Next, remove this from the HCA SIDR list */ 706 if (hcap->hca_sidr_list != NULL) { 707 prevp = NULL; 708 headp = hcap->hca_sidr_list; 709 710 while (headp != NULL) { 711 /* delete the matching entry */ 712 if (headp == ud_statep) { 713 if (prevp) { 714 prevp->ud_nextp = headp->ud_nextp; 715 } else { 716 prevp = headp->ud_nextp; 717 hcap->hca_sidr_list = prevp; 718 } 719 break; 720 } 721 prevp = headp; 722 headp = headp->ud_nextp; 723 } 724 } 725 726 rw_exit(&hcap->hca_sidr_list_lock); 727 728 /* 729 * While ref_cnt > 0 730 * - implies someone else is accessing the statep (possibly in 731 * a timeout function handler etc.) 732 * - don't delete statep unless they are done otherwise potentially 733 * one could access released memory and panic. 734 */ 735 mutex_enter(&ud_statep->ud_state_mutex); 736 if (ud_statep->ud_ref_cnt > 0) { 737 ud_statep->ud_delete_state_data = B_TRUE; 738 IBTF_DPRINTF_L4(cmlog, "ibcm_delete_ud_state_data: " 739 "ud_statep 0x%p ud_ref_cnt = %x", ud_statep, 740 ud_statep->ud_ref_cnt); 741 mutex_exit(&ud_statep->ud_state_mutex); 742 return; 743 } 744 mutex_exit(&ud_statep->ud_state_mutex); 745 746 ibcm_dealloc_ud_state_data(ud_statep); 747 } 748 749 /* 750 * ibcm_ud_dealloc_state_data: 751 * Deallocates a given ud state structure 752 * 753 * Arguments are:- 754 * ud statep - ud statep to be deleted 755 * 756 * Return Values: NONE 757 */ 758 void 759 ibcm_dealloc_ud_state_data(ibcm_ud_state_data_t *ud_statep) 760 { 761 timeout_id_t timer_val; 762 763 IBTF_DPRINTF_L4(cmlog, "ibcm_dealloc_ud_state_data: ud_statep 0x%p", 764 ud_statep); 765 766 /* If timer is running - expire it */ 767 mutex_enter(&ud_statep->ud_state_mutex); 768 if (ud_statep->ud_timerid) { 769 timer_val = ud_statep->ud_timerid; 770 ud_statep->ud_timerid = 0; 771 mutex_exit(&ud_statep->ud_state_mutex); 772 (void) untimeout(timer_val); 773 IBTF_DPRINTF_L2(cmlog, "ibcm_dealloc_ud_state_data: " 774 "Unexpected timer id 0x%p ud_statep 0x%p", timer_val, 775 ud_statep); 776 } else 777 mutex_exit(&ud_statep->ud_state_mutex); 778 779 if (ud_statep->ud_stored_msg != NULL) { 780 (void) ibcm_free_out_msg( 781 ud_statep->ud_stored_reply_addr.ibmf_hdl, 782 &ud_statep->ud_stored_msg); 783 } 784 785 /* release the ref cnt on the associated ibmf qp */ 786 ASSERT(ud_statep->ud_stored_reply_addr.cm_qp_entry != NULL); 787 ibcm_release_qp(ud_statep->ud_stored_reply_addr.cm_qp_entry); 788 789 /* Ensure the thread doing ref cnt decr releases the mutex */ 790 mutex_enter(&ud_statep->ud_state_mutex); 791 mutex_exit(&ud_statep->ud_state_mutex); 792 793 /* now do the mutex_destroy() and cv_destroy() */ 794 mutex_destroy(&ud_statep->ud_state_mutex); 795 796 cv_destroy(&ud_statep->ud_block_client_cv); 797 798 /* free the req id on SIDR REQ sender side */ 799 if (ud_statep->ud_mode == IBCM_ACTIVE_MODE) 800 ibcm_free_reqid(ud_statep->ud_hcap, ud_statep->ud_req_id); 801 802 /* Decrement the resource on hcap */ 803 ibcm_dec_hca_res_cnt(ud_statep->ud_hcap); 804 805 /* free the statep */ 806 kmem_free(ud_statep, sizeof (ibcm_ud_state_data_t)); 807 } 808 809 810 /* 811 * ibcm_init_ids: 812 * Create the vmem arenas for the various global ids 813 * 814 * Arguments are:- 815 * NONE 816 * 817 * Return Values: ibcm_status_t 818 */ 819 820 ibcm_status_t 821 ibcm_init_ids(void) 822 { 823 timespec_t tv; 824 825 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_local_sid_arena)) 826 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_ip_sid_arena)) 827 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_local_sid_seed)) 828 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibcm_local_cid_seed)) 829 830 ibcm_local_sid_arena = vmem_create("ibcm_local_sid", 831 (void *)IBCM_INITIAL_SID, IBCM_MAX_LOCAL_SIDS, 1, NULL, NULL, NULL, 832 0, VM_SLEEP | VMC_IDENTIFIER); 833 834 if (!ibcm_local_sid_arena) 835 return (IBCM_FAILURE); 836 837 ibcm_ip_sid_arena = vmem_create("ibcm_ip_sid", (void *)IBCM_INITIAL_SID, 838 IBCM_MAX_IP_SIDS, 1, NULL, NULL, NULL, 0, 839 VM_SLEEP | VMC_IDENTIFIER); 840 841 if (!ibcm_ip_sid_arena) 842 return (IBCM_FAILURE); 843 844 /* create a random starting value for local service ids */ 845 gethrestime(&tv); 846 ibcm_local_sid_seed = ((uint64_t)tv.tv_sec << 20) & 0x007FFFFFFFF00000; 847 ASSERT((ibcm_local_sid_seed & IB_SID_AGN_MASK) == 0); 848 ibcm_local_sid_seed |= IB_SID_AGN_LOCAL; 849 850 ibcm_local_cid_seed = (ib_com_id_t)tv.tv_sec; 851 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_local_sid_arena)) 852 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_local_sid_seed)) 853 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_ip_sid_arena)) 854 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibcm_local_cid_seed)) 855 856 return (IBCM_SUCCESS); 857 } 858 859 860 /* 861 * ibcm_init_hca_ids: 862 * Create the vmem arenas for the various hca level ids 863 * 864 * Arguments are:- 865 * hcap pointer to ibcm_hca_info_t 866 * 867 * Return Values: ibcm_status_t 868 */ 869 ibcm_status_t 870 ibcm_init_hca_ids(ibcm_hca_info_t *hcap) 871 { 872 hcap->hca_comid_arena = vmem_create("ibcm_com_ids", 873 (void *)IBCM_INITIAL_COMID, IBCM_MAX_COMIDS, 874 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 875 876 if (!hcap->hca_comid_arena) 877 return (IBCM_FAILURE); 878 879 hcap->hca_reqid_arena = vmem_create("ibcm_req_ids", 880 (void *)IBCM_INITIAL_REQID, IBCM_MAX_REQIDS, 881 1, NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 882 883 if (!hcap->hca_reqid_arena) { 884 vmem_destroy(hcap->hca_comid_arena); 885 return (IBCM_FAILURE); 886 } 887 888 return (IBCM_SUCCESS); 889 } 890 891 /* 892 * ibcm_free_ids: 893 * Destroy the vmem arenas for the various ids 894 * 895 * Arguments are:- 896 * NONE 897 * 898 * Return Values: NONE 899 */ 900 void 901 ibcm_fini_ids(void) 902 { 903 /* All arenas shall be valid */ 904 vmem_destroy(ibcm_local_sid_arena); 905 vmem_destroy(ibcm_ip_sid_arena); 906 } 907 908 /* 909 * ibcm_free_hca_ids: 910 * Destroy the vmem arenas for the various ids 911 * 912 * Arguments are:- 913 * hcap pointer to ibcm_hca_info_t 914 * 915 * Return Values: NONE 916 */ 917 void 918 ibcm_fini_hca_ids(ibcm_hca_info_t *hcap) 919 { 920 /* All arenas shall be valid */ 921 vmem_destroy(hcap->hca_comid_arena); 922 vmem_destroy(hcap->hca_reqid_arena); 923 } 924 925 /* Communication id management routines ie., allocate, free up comids */ 926 927 /* 928 * ibcm_alloc_comid: 929 * Allocate a new communication id 930 * 931 * Arguments are:- 932 * hcap : pointer to ibcm_hca_info_t 933 * comid: pointer to the newly allocated communication id 934 * 935 * Return Values: ibt_status_t 936 */ 937 ibcm_status_t 938 ibcm_alloc_comid(ibcm_hca_info_t *hcap, ib_com_id_t *comidp) 939 { 940 ib_com_id_t comid; 941 942 /* Use next fit, so least recently used com id is allocated */ 943 comid = (ib_com_id_t)(uintptr_t)vmem_alloc(hcap->hca_comid_arena, 1, 944 VM_SLEEP | VM_NEXTFIT); 945 946 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_comid: hcap 0x%p comid 0x%lX", hcap, 947 comid); 948 949 /* 950 * As comid is 32 bits, and maximum connections possible are 2^24 951 * per hca, comid allocation would never fail 952 */ 953 *comidp = comid + ibcm_local_cid_seed; 954 if (comid == 0) { 955 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_comid: hcap 0x%p" 956 "no more comids available", hcap); 957 return (IBCM_FAILURE); 958 } 959 960 return (IBCM_SUCCESS); 961 } 962 963 /* 964 * ibcm_free_comid: 965 * Releases the given Communication Id 966 * 967 * Arguments are: 968 * hcap : pointer to ibcm_hca_info_t 969 * comid : Communication id to be free'd 970 * 971 * Return Values: NONE 972 */ 973 void 974 ibcm_free_comid(ibcm_hca_info_t *hcap, ib_com_id_t comid) 975 { 976 IBTF_DPRINTF_L4(cmlog, "ibcm_free_comid: hcap 0x%p" 977 "comid %x", hcap, comid); 978 comid -= ibcm_local_cid_seed; 979 vmem_free(hcap->hca_comid_arena, (void *)(uintptr_t)comid, 1); 980 } 981 982 /* Allocate and Free local service ids */ 983 984 /* 985 * ibcm_alloc_local_sids: 986 * Create and destroy the vmem arenas for the service ids 987 * 988 * Arguments are:- 989 * Number of contiguous SIDs needed 990 * 991 * Return Values: starting SID 992 */ 993 ib_svc_id_t 994 ibcm_alloc_local_sids(int num_sids) 995 { 996 ib_svc_id_t sid; 997 998 sid = (ib_svc_id_t)(uintptr_t)vmem_alloc(ibcm_local_sid_arena, 999 num_sids, VM_SLEEP | VM_NEXTFIT); 1000 1001 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_local_sids: ServiceID 0x%llX " 1002 "num_sids %d", sid, num_sids); 1003 if (sid == 0) { 1004 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_local_sids: " 1005 "no more local sids available"); 1006 } else { 1007 ASSERT((ibcm_local_sid_seed & IB_SID_AGN_MASK) == 1008 IB_SID_AGN_LOCAL); 1009 sid += ibcm_local_sid_seed; 1010 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_local_sids: Success: " 1011 "allocated 0x%llX:%d", sid, num_sids); 1012 } 1013 return (sid); 1014 } 1015 1016 /* 1017 * ibcm_free_local_sids: 1018 * Releases the given Local service id 1019 * 1020 * Arguments are: 1021 * num_sids: Number of local service id's to be free'd 1022 * service_id: Starting local service id that needs to be free'd 1023 * 1024 * Return Values: NONE 1025 */ 1026 void 1027 ibcm_free_local_sids(ib_svc_id_t service_id, int num_sids) 1028 { 1029 service_id -= ibcm_local_sid_seed; 1030 IBTF_DPRINTF_L4(cmlog, "ibcm_free_local_sids: " 1031 "service_id 0x%llX num_sids %d", service_id, num_sids); 1032 vmem_free(ibcm_local_sid_arena, 1033 (void *)(uintptr_t)service_id, num_sids); 1034 } 1035 1036 /* 1037 * ibcm_alloc_ip_sid: 1038 * Allocate a local IP SID. 1039 */ 1040 ib_svc_id_t 1041 ibcm_alloc_ip_sid() 1042 { 1043 ib_svc_id_t sid; 1044 1045 sid = (ib_svc_id_t)(uintptr_t)vmem_alloc(ibcm_ip_sid_arena, 1, 1046 VM_SLEEP | VM_NEXTFIT); 1047 if (sid == 0) { 1048 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_ip_sid: no more RDMA IP " 1049 "SIDs available"); 1050 } else { 1051 sid += IB_SID_IPADDR_PREFIX; 1052 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_ip_sid: Success: RDMA IP SID" 1053 " allocated : 0x%016llX", sid); 1054 } 1055 return (sid); 1056 } 1057 1058 /* 1059 * ibcm_free_ip_sid: 1060 * Releases the given IP Service ID 1061 */ 1062 void 1063 ibcm_free_ip_sid(ib_svc_id_t sid) 1064 { 1065 sid -= IB_SID_IPADDR_PREFIX; 1066 vmem_free(ibcm_ip_sid_arena, (void *)(uintptr_t)sid, 1); 1067 } 1068 1069 1070 /* Allocate and free request id routines for SIDR */ 1071 1072 /* 1073 * ibcm_alloc_reqid: 1074 * Allocate a new SIDR REQ request id 1075 * 1076 * Arguments are:- 1077 * hcap : pointer to ibcm_hca_info_t 1078 * *reqid : pointer to the new request id returned 1079 * 1080 * Return Values: ibcm_status_t 1081 */ 1082 ibcm_status_t 1083 ibcm_alloc_reqid(ibcm_hca_info_t *hcap, uint32_t *reqid) 1084 { 1085 /* Use next fit, so least recently used com id is allocated */ 1086 *reqid = (uint32_t)(uintptr_t)vmem_alloc(hcap->hca_reqid_arena, 1, 1087 VM_SLEEP | VM_NEXTFIT); 1088 1089 IBTF_DPRINTF_L4(cmlog, "ibcm_alloc_reqid: hcap 0x%p reqid %x", hcap, 1090 *reqid); 1091 if (!(*reqid)) { 1092 IBTF_DPRINTF_L2(cmlog, "ibcm_alloc_reqid: " 1093 "no more req ids available"); 1094 return (IBCM_FAILURE); 1095 } 1096 return (IBCM_SUCCESS); 1097 } 1098 1099 /* 1100 * ibcm_free_reqid: 1101 * Releases the given SIDR REQ request id 1102 * 1103 * Arguments are: 1104 * hcap : pointer to ibcm_hca_info_t 1105 * reqid : Request id to be free'd 1106 * 1107 * Return Values: NONE 1108 */ 1109 void 1110 ibcm_free_reqid(ibcm_hca_info_t *hcap, uint32_t reqid) 1111 { 1112 IBTF_DPRINTF_L4(cmlog, "ibcm_free_reqid: hcap 0x%p reqid %x", hcap, 1113 reqid); 1114 vmem_free(hcap->hca_reqid_arena, (void *)(uintptr_t)reqid, 1); 1115 } 1116 1117 /* 1118 * ibcm_generate_tranid: 1119 * Generate a new transaction id based on args 1120 * 1121 * Arguments are:- 1122 * event_type CM Message REQ/DREQ/LAP 1123 * id 32 bit identifier 1124 * cm_tran_priv CM private data to be filled in top 28 MSB bits of 1125 * tran id 1126 * 1127 * 1128 * Return Value: uint64_t 1129 */ 1130 uint64_t 1131 ibcm_generate_tranid(uint8_t event, uint32_t id, uint32_t cm_tran_priv) 1132 { 1133 /* 1134 * copy comid to bits 31-0 of tran id, 1135 * attr id to bits 35-32 of tran id, 1136 * cm_priv to bits 63-36 of tran id 1137 */ 1138 if (cm_tran_priv == 0) 1139 /* 1140 * The below ensures that no duplicate transaction id is 1141 * generated atleast for next 6 months. Calculations: 1142 * (2^28)/(1000 * 60 * 24 * 30) = 6 approx 1143 */ 1144 cm_tran_priv = gethrtime() >> 20; /* ~time in ms */ 1145 1146 return ((((uint64_t)cm_tran_priv << 36) | (uint64_t)event << 32) | id); 1147 } 1148 1149 #ifdef DEBUG 1150 1151 /* 1152 * ibcm_decode_tranid: 1153 * Decodes a given transaction id, assuming certain format. 1154 * 1155 * Arguments are:- 1156 * tran_id Transaction id to be decoded 1157 * cm_tran_priv CM private data retrieved from transaction id 1158 * 1159 * Return Value: None 1160 */ 1161 void 1162 ibcm_decode_tranid(uint64_t tran_id, uint32_t *cm_tran_priv) 1163 { 1164 ib_com_id_t id; 1165 ibcm_event_type_t event; 1166 1167 id = tran_id & 0xFFFFFFFF; 1168 event = (tran_id >> 32) & 0xF; 1169 1170 IBTF_DPRINTF_L5(cmlog, "ibcm_decode_tranid: id = 0x%x, event = %x", 1171 id, event); 1172 1173 if (cm_tran_priv) { 1174 *cm_tran_priv = tran_id >> 36; 1175 IBTF_DPRINTF_L5(cmlog, "ibcm_decode_tranid: " 1176 "cm_tran_priv = %x", *cm_tran_priv); 1177 } 1178 } 1179 1180 #endif 1181 1182 /* 1183 * Service ID entry create and lookup functions 1184 */ 1185 1186 /* 1187 * ibcm_svc_compare: 1188 * - AVL svc tree node compare 1189 * 1190 * Arguments: 1191 * p1 : pointer to local comid 1192 * p2 : pointer to passed ibcm_state_data_t 1193 * 1194 * Return values: 1195 * 0 : match found 1196 * -1 : no match but insert to left side of the tree 1197 * +1 : no match but insert to right side of the tree 1198 */ 1199 int 1200 ibcm_svc_compare(const void *p1, const void *p2) 1201 { 1202 ibcm_svc_lookup_t *sidp = (ibcm_svc_lookup_t *)p1; 1203 ibcm_svc_info_t *svcp = (ibcm_svc_info_t *)p2; 1204 ib_svc_id_t start_sid = sidp->sid; 1205 ib_svc_id_t end_sid = start_sid + sidp->num_sids - 1; 1206 1207 IBTF_DPRINTF_L5(cmlog, "ibcm_svc_compare: " 1208 "sid: 0x%llx, numsids: %d, node_sid: 0x%llx node_num_sids: %d", 1209 sidp->sid, sidp->num_sids, svcp->svc_id, svcp->svc_num_sids); 1210 1211 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 1212 1213 if (svcp->svc_id > end_sid) 1214 return (-1); 1215 if (svcp->svc_id + svcp->svc_num_sids - 1 < start_sid) 1216 return (+1); 1217 return (0); /* means there is some overlap of SIDs */ 1218 } 1219 1220 1221 /* 1222 * ibcm_create_svc_entry: 1223 * Make sure no conflicting entry exists, then allocate it. 1224 * Fill in the critical "look up" details that are provided 1225 * in the arguments before dropping the lock. 1226 * 1227 * Return values: 1228 * Pointer to ibcm_svc_info_t, if created, otherwise NULL. 1229 */ 1230 ibcm_svc_info_t * 1231 ibcm_create_svc_entry(ib_svc_id_t sid, int num_sids) 1232 { 1233 ibcm_svc_info_t *svcp; 1234 ibcm_svc_info_t *svcinfop; 1235 ibcm_svc_lookup_t svc; 1236 avl_index_t where = 0; 1237 1238 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 1239 1240 /* assume success, and avoid kmem while holding the writer lock */ 1241 svcinfop = kmem_zalloc(sizeof (*svcinfop), KM_SLEEP); 1242 svcinfop->svc_id = sid; 1243 svcinfop->svc_num_sids = num_sids; 1244 1245 svc.sid = sid; 1246 svc.num_sids = num_sids; 1247 1248 mutex_enter(&ibcm_svc_info_lock); 1249 #ifdef __lock_lint 1250 ibcm_svc_compare(NULL, NULL); 1251 #endif 1252 svcp = avl_find(&ibcm_svc_avl_tree, &svc, &where); 1253 if (svcp != NULL) { /* overlab exists */ 1254 mutex_exit(&ibcm_svc_info_lock); 1255 kmem_free(svcinfop, sizeof (*svcinfop)); 1256 return (NULL); 1257 } 1258 avl_insert(&ibcm_svc_avl_tree, (void *)svcinfop, where); 1259 mutex_exit(&ibcm_svc_info_lock); 1260 1261 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 1262 1263 return (svcinfop); 1264 } 1265 1266 /* 1267 * ibcm_find_svc_entry: 1268 * Finds a ibcm_svc_info_t entry into the CM's global table. 1269 * The search done here assumes the list is sorted by SID. 1270 * 1271 * Arguments are: 1272 * sid - Service ID to look up 1273 * 1274 * Return values: 1275 * Pointer to ibcm_svc_info_t, if found, otherwise NULL. 1276 */ 1277 ibcm_svc_info_t * 1278 ibcm_find_svc_entry(ib_svc_id_t sid) 1279 { 1280 ibcm_svc_info_t *svcp; 1281 ibcm_svc_lookup_t svc; 1282 1283 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: finding SID 0x%llX", sid); 1284 1285 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 1286 1287 svc.sid = sid; 1288 svc.num_sids = 1; 1289 #ifdef __lock_lint 1290 ibcm_svc_compare(NULL, NULL); 1291 #endif 1292 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 1293 if (svcp != NULL) { 1294 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: " 1295 "found SID = 0x%llX", sid); 1296 return (svcp); /* found it */ 1297 } 1298 IBTF_DPRINTF_L3(cmlog, "ibcm_find_svc_entry: SID %llX not found", sid); 1299 return (NULL); 1300 } 1301 1302 /* 1303 * ibcm_alloc_ibmf_msg: 1304 * Allocate an ibmf message structure and the additional memory required for 1305 * sending an outgoing CM mad. The ibmf message structure contains two 1306 * ibmf_msg_bufs_t fields, one for the incoming MAD and one for the outgoing 1307 * MAD. The CM must allocate the memory for the outgoing MAD. The msg_buf 1308 * field has three buffers: the mad header, the class header, and the class 1309 * data. To simplify the code and reduce the number of kmem_zalloc() calls, 1310 * ibcm_alloc_ibmf_msg will allocate one buffer and set the pointers to the 1311 * right offsets. No class header is needed so only the mad header and class 1312 * data fields are used. 1313 */ 1314 ibt_status_t 1315 ibcm_alloc_out_msg(ibmf_handle_t ibmf_handle, ibmf_msg_t **ibmf_msgpp, 1316 uint8_t method) 1317 { 1318 ib_mad_hdr_t *output_mad_hdr; 1319 int sa_retval; 1320 1321 if ((sa_retval = 1322 ibmf_alloc_msg(ibmf_handle, IBMF_ALLOC_SLEEP, ibmf_msgpp)) != 1323 IBMF_SUCCESS) { 1324 IBTF_DPRINTF_L1(cmlog, "ibcm_alloc_out_msg: " 1325 "ibmf_alloc_msg failed with IBMF_ALLOC_SLEEP"); 1326 return (ibcm_ibmf_analyze_error(sa_retval)); 1327 } 1328 1329 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr = kmem_zalloc( 1330 IBCM_MAD_SIZE, KM_SLEEP); 1331 1332 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_cl_data_len = IBCM_MSG_SIZE; 1333 (*ibmf_msgpp)->im_msgbufs_send.im_bufs_cl_data = 1334 (uchar_t *)((*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr) + 1335 IBCM_MAD_HDR_SIZE; 1336 1337 /* initialize generic CM MAD header fields */ 1338 output_mad_hdr = IBCM_OUT_HDRP((*ibmf_msgpp)); 1339 output_mad_hdr->BaseVersion = IBCM_MAD_BASE_VERSION; 1340 output_mad_hdr->MgmtClass = MAD_MGMT_CLASS_COMM_MGT; 1341 output_mad_hdr->ClassVersion = IBCM_MAD_CLASS_VERSION; 1342 output_mad_hdr->R_Method = method; 1343 1344 return (IBT_SUCCESS); 1345 } 1346 1347 /* 1348 * ibcm_free_ibmf_msg: 1349 * Frees the buffer and ibmf message associated with an outgoing CM message. 1350 * This function should only be used to free messages created by 1351 * ibcm_alloc_out_msg. Will return IBCM_FAILURE if the ibmf_free_msg() call 1352 * fails and IBCM_SUCCESS otherwise. 1353 */ 1354 ibcm_status_t 1355 ibcm_free_out_msg(ibmf_handle_t ibmf_handle, ibmf_msg_t **ibmf_msgpp) 1356 { 1357 int ibmf_status; 1358 1359 kmem_free((*ibmf_msgpp)->im_msgbufs_send.im_bufs_mad_hdr, 1360 IBCM_MAD_SIZE); 1361 1362 if ((ibmf_status = ibmf_free_msg(ibmf_handle, ibmf_msgpp)) != 1363 IBMF_SUCCESS) { 1364 IBTF_DPRINTF_L2(cmlog, "ibcm_free_out_msg: " 1365 "ibmf_free_msg failed %d", ibmf_status); 1366 return (IBCM_FAILURE); 1367 } else 1368 return (IBCM_SUCCESS); 1369 } 1370 1371 ibcm_qp_list_t * 1372 ibcm_find_qp(ibcm_hca_info_t *hcap, int port_no, ib_pkey_t pkey) 1373 { 1374 ibcm_qp_list_t *entry; 1375 ibmf_qp_handle_t ibmf_qp; 1376 int ibmf_status; 1377 1378 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry)) 1379 1380 mutex_enter(&ibcm_qp_list_lock); 1381 1382 /* 1383 * CM currently does not track port up and down status. If tracking of 1384 * " port status" is added in the future, then CM could be optimized to 1385 * re-use other ports on hcap, if the port associated with the above 1386 * port_no is down. But, the issue of "reachability" needs to be 1387 * handled, before selecting an alternative port different from above. 1388 */ 1389 entry = hcap->hca_port_info[port_no-1].port_qplist; 1390 while (entry != NULL) { 1391 if (entry->qp_pkey == pkey) { 1392 ++entry->qp_ref_cnt; 1393 mutex_exit(&ibcm_qp_list_lock); 1394 return (entry); 1395 } 1396 entry = entry->qp_next; 1397 } 1398 1399 /* 1400 * entry not found, attempt to alloc a qp 1401 * This may be optimized in the future, to allocate ibmf qp's 1402 * once the "CM mgmt pkeys" are precisely known. 1403 */ 1404 ibmf_status = ibmf_alloc_qp( 1405 hcap->hca_port_info[port_no-1].port_ibmf_hdl, pkey, IB_GSI_QKEY, 1406 IBMF_ALT_QP_MAD_NO_RMPP, &ibmf_qp); 1407 1408 if (ibmf_status != IBMF_SUCCESS) { 1409 mutex_exit(&ibcm_qp_list_lock); 1410 IBTF_DPRINTF_L2(cmlog, "ibcm_find_qp: failed to alloc IBMF QP" 1411 "for Pkey = %x port_no = %x status = %d hcaguid = %llXp", 1412 pkey, port_no, ibmf_status, hcap->hca_guid); 1413 /* 1414 * This may be optimized in the future, so as CM would attempt 1415 * to re-use other QP's whose ref cnt is 0 in the respective 1416 * port_qplist, by doing an ibmf_modify_qp with pkey above. 1417 */ 1418 return (NULL); 1419 } 1420 1421 entry = kmem_alloc(sizeof (ibcm_qp_list_t), KM_SLEEP); 1422 entry->qp_next = hcap->hca_port_info[port_no-1].port_qplist; 1423 hcap->hca_port_info[port_no-1].port_qplist = entry; 1424 entry->qp_cm = ibmf_qp; 1425 entry->qp_ref_cnt = 1; 1426 entry->qp_pkey = pkey; 1427 entry->qp_port = &(hcap->hca_port_info[port_no-1]); 1428 1429 mutex_exit(&ibcm_qp_list_lock); 1430 1431 /* set-up the handler */ 1432 ibmf_status = ibmf_setup_async_cb( 1433 hcap->hca_port_info[port_no-1].port_ibmf_hdl, ibmf_qp, 1434 ibcm_recv_cb, entry, 0); 1435 1436 ASSERT(ibmf_status == IBMF_SUCCESS); 1437 1438 #ifdef DEBUG 1439 ibcm_query_qp(hcap->hca_port_info[port_no-1].port_ibmf_hdl, ibmf_qp); 1440 #endif 1441 1442 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*entry)) 1443 1444 return (entry); 1445 } 1446 1447 void 1448 ibcm_release_qp(ibcm_qp_list_t *cm_qp_entry) 1449 { 1450 mutex_enter(&ibcm_qp_list_lock); 1451 --cm_qp_entry->qp_ref_cnt; 1452 ASSERT(cm_qp_entry->qp_ref_cnt >= 0); 1453 mutex_exit(&ibcm_qp_list_lock); 1454 } 1455 1456 1457 /* called holding the ibcm_qp_list_lock mutex */ 1458 ibcm_status_t 1459 ibcm_free_qp(ibcm_qp_list_t *cm_qp_entry) 1460 { 1461 int ibmf_status; 1462 1463 IBTF_DPRINTF_L5(cmlog, "ibcm_free_qp: qp_hdl %p ref_cnt %d pkey %x", 1464 cm_qp_entry->qp_cm, cm_qp_entry->qp_ref_cnt, cm_qp_entry->qp_pkey); 1465 1466 /* check, there are no users of this ibmf qp */ 1467 if (cm_qp_entry->qp_ref_cnt != 0) 1468 return (IBCM_FAILURE); 1469 1470 /* Tear down the receive callback */ 1471 ibmf_status = ibmf_tear_down_async_cb( 1472 cm_qp_entry->qp_port->port_ibmf_hdl, cm_qp_entry->qp_cm, 0); 1473 if (ibmf_status != IBMF_SUCCESS) { 1474 IBTF_DPRINTF_L2(cmlog, "ibcm_free_qp: " 1475 "ibmf_tear_down_async_cb failed %d port_num %d", 1476 ibmf_status, cm_qp_entry->qp_port->port_num); 1477 return (IBCM_FAILURE); 1478 } 1479 1480 ibmf_status = ibmf_free_qp(cm_qp_entry->qp_port->port_ibmf_hdl, 1481 &cm_qp_entry->qp_cm, 0); 1482 if (ibmf_status != IBMF_SUCCESS) { 1483 IBTF_DPRINTF_L2(cmlog, "ibcm_free_qp: ibmf_free_qp failed for" 1484 " ibmf_status %d qp hdl %p port_no %x", ibmf_status, 1485 cm_qp_entry->qp_cm, cm_qp_entry->qp_port->port_num); 1486 return (IBCM_FAILURE); 1487 } 1488 1489 return (IBCM_SUCCESS); 1490 } 1491 1492 ibcm_status_t 1493 ibcm_free_allqps(ibcm_hca_info_t *hcap, int port_no) 1494 { 1495 ibcm_qp_list_t *entry, *freed; 1496 ibcm_status_t ibcm_status = IBCM_SUCCESS; 1497 1498 IBTF_DPRINTF_L5(cmlog, "ibcm_free_allqps: hcap %p port_no %d", hcap, 1499 port_no); 1500 1501 mutex_enter(&ibcm_qp_list_lock); 1502 entry = hcap->hca_port_info[port_no-1].port_qplist; 1503 while ((entry != NULL) && 1504 ((ibcm_status = ibcm_free_qp(entry)) == IBCM_SUCCESS)) { 1505 freed = entry; 1506 entry = entry->qp_next; 1507 kmem_free(freed, sizeof (ibcm_qp_list_t)); 1508 } 1509 1510 if (ibcm_status != IBCM_SUCCESS) /* sanity the linked list */ 1511 hcap->hca_port_info[port_no-1].port_qplist = entry; 1512 else /* all ibmf qp's of port must have been free'd successfully */ 1513 hcap->hca_port_info[port_no-1].port_qplist = NULL; 1514 1515 mutex_exit(&ibcm_qp_list_lock); 1516 return (ibcm_status); 1517 } 1518 1519 /* 1520 * ibt_bind_service() and ibt_get_paths() needs the following helper function 1521 * to handle endianess in case of Service Data. 1522 */ 1523 void 1524 ibcm_swizzle_from_srv(ibt_srv_data_t *sb_data, uint8_t *service_bytes) 1525 { 1526 uint8_t *p8 = service_bytes; 1527 uint16_t *p16; 1528 uint32_t *p32; 1529 uint64_t *p64; 1530 int i; 1531 1532 for (i = 0; i < 16; i++) 1533 *p8++ = sb_data->s_data8[i]; 1534 1535 p16 = (uint16_t *)p8; 1536 for (i = 0; i < 8; i++) 1537 *p16++ = h2b16(sb_data->s_data16[i]); 1538 1539 p32 = (uint32_t *)p16; 1540 for (i = 0; i < 4; i++) 1541 *p32++ = h2b32(sb_data->s_data32[i]); 1542 1543 p64 = (uint64_t *)p32; 1544 for (i = 0; i < 2; i++) 1545 *p64++ = h2b64(sb_data->s_data64[i]); 1546 } 1547 1548 void 1549 ibcm_swizzle_to_srv(uint8_t *service_bytes, ibt_srv_data_t *sb_data) 1550 { 1551 uint8_t *p8 = service_bytes; 1552 uint16_t *p16; 1553 uint32_t *p32; 1554 uint64_t *p64; 1555 int i; 1556 1557 for (i = 0; i < 16; i++) 1558 sb_data->s_data8[i] = *p8++; 1559 1560 p16 = (uint16_t *)p8; 1561 for (i = 0; i < 8; i++) 1562 sb_data->s_data16[i] = h2b16(*p16++); 1563 1564 p32 = (uint32_t *)p16; 1565 for (i = 0; i < 4; i++) 1566 sb_data->s_data32[i] = h2b32(*p32++); 1567 p64 = (uint64_t *)p32; 1568 1569 for (i = 0; i < 2; i++) 1570 sb_data->s_data64[i] = h2b64(*p64++); 1571 } 1572 1573 /* Trace related functions */ 1574 1575 void 1576 ibcm_init_conn_trace(ibcm_state_data_t *sp) 1577 { 1578 IBTF_DPRINTF_L5(cmlog, "ibcm_init_conn_trace: statep %p", sp); 1579 1580 /* Initialize trace related fields */ 1581 1582 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sp->conn_trace)) 1583 sp->conn_trace = kmem_zalloc(sizeof (ibcm_conn_trace_t), KM_SLEEP); 1584 if ((ibcm_enable_trace & 1) == 0) 1585 sp->conn_trace->conn_base_tm = gethrtime(); 1586 sp->conn_trace->conn_allocated_trcnt = ibcm_conn_max_trcnt; 1587 sp->conn_trace->conn_trace_events = 1588 kmem_zalloc(sp->conn_trace->conn_allocated_trcnt, KM_SLEEP); 1589 sp->conn_trace->conn_trace_event_times = 1590 kmem_zalloc(sp->conn_trace->conn_allocated_trcnt * 1591 sizeof (tm_diff_type), KM_SLEEP); 1592 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sp->conn_trace)) 1593 } 1594 1595 void 1596 ibcm_fini_conn_trace(ibcm_state_data_t *statep) 1597 { 1598 IBTF_DPRINTF_L5(cmlog, "ibcm_fini_conn_trace: statep %p tracep %p", 1599 statep, statep->conn_trace); 1600 1601 /* free the trace data */ 1602 if (statep->conn_trace) { 1603 if (statep->conn_trace->conn_trace_events) 1604 kmem_free(statep->conn_trace->conn_trace_events, 1605 statep->conn_trace->conn_allocated_trcnt); 1606 if (statep->conn_trace->conn_trace_event_times) 1607 kmem_free(statep->conn_trace->conn_trace_event_times, 1608 statep->conn_trace->conn_allocated_trcnt * 1609 sizeof (tm_diff_type)); 1610 1611 kmem_free(statep->conn_trace, sizeof (ibcm_conn_trace_t)); 1612 } 1613 } 1614 1615 /* mostly used to profile connection establishment times with dtrace */ 1616 void 1617 ibcm_established(hrtime_t time_diff) 1618 { 1619 if (time_diff > 1000000000LL) /* 1 second */ 1620 IBTF_DPRINTF_L2(cmlog, "slow connection time (%d seconds)", 1621 (uint_t)(time_diff >> 30)); 1622 } 1623 1624 void 1625 ibcm_insert_trace(void *statep, ibcm_state_rc_trace_qualifier_t event_qualifier) 1626 { 1627 ibcm_conn_trace_t *conn_trace; 1628 uint8_t conn_trace_ind; 1629 hrtime_t time_diff; 1630 hrtime_t hrt; 1631 1632 if (!(((ibcm_state_data_t *)statep)->conn_trace)) 1633 return; 1634 1635 conn_trace = ((ibcm_state_data_t *)statep)->conn_trace; 1636 1637 if (!conn_trace->conn_trace_events) 1638 return; 1639 1640 IBTF_DPRINTF_L5(cmlog, "ibcm_insert_trace: statep %p event %d", 1641 statep, event_qualifier); 1642 1643 mutex_enter(&ibcm_trace_mutex); 1644 1645 /* No more trace memory available, hence return */ 1646 if (conn_trace->conn_trace_ind == conn_trace->conn_allocated_trcnt) { 1647 mutex_exit(&ibcm_trace_mutex); 1648 return; 1649 } else 1650 ++conn_trace->conn_trace_ind; 1651 1652 conn_trace_ind = conn_trace->conn_trace_ind - 1; 1653 1654 conn_trace->conn_trace_events[conn_trace_ind] = event_qualifier; 1655 1656 if ((ibcm_enable_trace & 1) == 0) { 1657 hrt = gethrtime(); 1658 time_diff = hrt - conn_trace->conn_base_tm; 1659 if (event_qualifier == IBCM_TRACE_CALLED_CONN_EST_EVENT) 1660 ibcm_established(time_diff); 1661 time_diff >>= 10; 1662 if (time_diff >= TM_DIFF_MAX) { 1663 /* RESET, future times are relative to new base time. */ 1664 conn_trace->conn_base_tm = hrt; 1665 time_diff = 0; 1666 } 1667 conn_trace->conn_trace_event_times[conn_trace_ind] = time_diff; 1668 } 1669 1670 mutex_exit(&ibcm_trace_mutex); 1671 1672 IBTF_DPRINTF_L5(cmlog, "ibcm_insert_trace: statep %p inserted event %d", 1673 statep, event_qualifier); 1674 } 1675 1676 void 1677 ibcm_dump_conn_trace(void *statep) 1678 { 1679 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_conn_trace: statep %p", 1680 statep); 1681 1682 mutex_enter(&ibcm_trace_print_mutex); 1683 ibcm_debug_buf[0] = '\0'; 1684 ibcm_dump_conn_trbuf(statep, "ibcm: ", ibcm_debug_buf, 1685 IBCM_DEBUG_BUF_SIZE); 1686 if (ibcm_debug_buf[0] != '\0') 1687 IBTF_DPRINTF_L2(cmlog, "\n%s", ibcm_debug_buf); 1688 1689 #ifdef DEBUG 1690 1691 if (ibcm_test_mode > 1) 1692 cmn_err(CE_CONT, "IBCM DEBUG TRACE:\n%s", ibcm_debug_buf); 1693 #endif 1694 1695 mutex_exit(&ibcm_trace_print_mutex); 1696 } 1697 1698 void 1699 ibcm_dump_conn_trbuf(void *statep, char *line_prefix, char *buf, int buf_size) 1700 { 1701 ibcm_conn_trace_t *conn_trace; 1702 int tr_ind; 1703 ibcm_state_data_t *sp; 1704 int cur_size = 0; /* size of item copied */ 1705 int rem_size; /* remaining size in trace buffer */ 1706 int next_data = 0; /* location where next item copied */ 1707 1708 if ((buf == NULL) || (buf_size <= 0)) 1709 return; 1710 1711 sp = (ibcm_state_data_t *)statep; 1712 1713 if (!sp->conn_trace) 1714 return; 1715 1716 conn_trace = sp->conn_trace; 1717 1718 if (!conn_trace->conn_trace_events) 1719 return; 1720 1721 rem_size = buf_size; 1722 1723 /* Print connection level global data */ 1724 1725 /* Print statep, local comid, local qpn */ 1726 cur_size = snprintf(&buf[next_data], rem_size, "%s%s0x%p\n%s%s0x%p\n" 1727 "%s%s0x%x/%llx/%d\n%s%s0x%x\n%s%s0x%x/%llx\n%s%s0x%x\n%s%s%llu\n", 1728 line_prefix, event_str[IBCM_DISPLAY_SID], (void *)sp, 1729 line_prefix, event_str[IBCM_DISPLAY_CHAN], (void *)sp->channel, 1730 line_prefix, event_str[IBCM_DISPLAY_LCID], sp->local_comid, 1731 (longlong_t)sp->local_hca_guid, sp->prim_port, 1732 line_prefix, event_str[IBCM_DISPLAY_LQPN], sp->local_qpn, 1733 line_prefix, event_str[IBCM_DISPLAY_RCID], sp->remote_comid, 1734 (longlong_t)sp->remote_hca_guid, 1735 line_prefix, event_str[IBCM_DISPLAY_RQPN], sp->remote_qpn, 1736 line_prefix, event_str[IBCM_DISPLAY_TM], conn_trace->conn_base_tm); 1737 1738 rem_size = rem_size - cur_size; 1739 if (rem_size <= 0) { 1740 buf[buf_size-1] = '\n'; 1741 return; 1742 } 1743 1744 next_data = next_data + cur_size; 1745 1746 for (tr_ind = 0; tr_ind < conn_trace->conn_trace_ind; tr_ind++) { 1747 cur_size = snprintf(&buf[next_data], rem_size, 1748 "%s%sTM_DIFF %u\n", line_prefix, 1749 event_str[conn_trace->conn_trace_events[tr_ind]], 1750 conn_trace->conn_trace_event_times[tr_ind]); 1751 rem_size = rem_size - cur_size; 1752 if (rem_size <= 0) { 1753 buf[buf_size-1] = '\n'; 1754 return; 1755 } 1756 next_data = next_data + cur_size; 1757 } 1758 1759 buf[next_data] = '\0'; 1760 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_conn_trbuf: statep %p " 1761 "debug buf size %d bytes", statep, next_data); 1762 } 1763 1764 1765 #ifdef DEBUG 1766 1767 void 1768 ibcm_query_qp(ibmf_handle_t ibmf_hdl, ibmf_qp_handle_t ibmf_qp) 1769 { 1770 uint8_t qp_port_num; 1771 ib_qpn_t qp_num; 1772 ib_pkey_t qp_pkey; 1773 ib_qkey_t qp_qkey; 1774 int ibmf_status; 1775 1776 if (ibmf_qp == IBMF_QP_HANDLE_DEFAULT) { 1777 IBTF_DPRINTF_L4(cmlog, "ibcm_query_qp: QP1"); 1778 return; 1779 } 1780 1781 ibmf_status = 1782 ibmf_query_qp(ibmf_hdl, ibmf_qp, &qp_num, &qp_pkey, &qp_qkey, 1783 &qp_port_num, 0); 1784 1785 ASSERT(ibmf_status == IBMF_SUCCESS); 1786 1787 IBTF_DPRINTF_L5(cmlog, "ibcm_query_qp: qpn %x qkey %x pkey %x port %d", 1788 qp_num, qp_qkey, qp_pkey, qp_port_num); 1789 } 1790 1791 /* 1792 * ibcm_dump_raw_message: 1793 * dumps 256 bytes of data of a raw message (REP/REQ/DREQ ...) 1794 * (can be called from the kernel debugger w/ the message pointer) 1795 * 1796 * Arguments: 1797 * msgp - the messages that needs to be dumped 1798 * 1799 * Return values: NONE 1800 */ 1801 void 1802 ibcm_dump_raw_message(uchar_t *c) 1803 { 1804 int i; 1805 1806 for (i = 0; i < IBCM_MAD_SIZE; i += 16) { 1807 /* print in batches of 16 chars at a time */ 1808 IBTF_DPRINTF_L4(cmlog, 1809 "%x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", 1810 c[i], c[i + 1], c[i + 2], c[i + 3], c[i + 4], c[i + 5], 1811 c[i + 6], c[i + 7], c[i + 8], c[i + 9], c[i + 10], 1812 c[i + 11], c[i + 12], c[i + 13], c[i + 14], c[i + 15]); 1813 } 1814 } 1815 1816 1817 /* 1818 * ibcm_dump_srv_rec: 1819 * Dumps Service Records. 1820 * 1821 * Arguments: 1822 * srv_rec - the pointer to sa_service_record_t struct. 1823 * 1824 * Return values: NONE 1825 */ 1826 void 1827 ibcm_dump_srvrec(sa_service_record_t *srv_rec) 1828 { 1829 uint8_t i; 1830 1831 IBTF_DPRINTF_L4(cmlog, "ibcm_dump_srvrec: Service Records"); 1832 IBTF_DPRINTF_L4(cmlog, "SID : 0x%016llX", srv_rec->ServiceID); 1833 IBTF_DPRINTF_L4(cmlog, "Svc GID : 0x%016llX:0x%016llX", 1834 srv_rec->ServiceGID.gid_prefix, srv_rec->ServiceGID.gid_guid); 1835 IBTF_DPRINTF_L4(cmlog, "Svc PKey : 0x%X", srv_rec->ServiceP_Key); 1836 1837 IBTF_DPRINTF_L4(cmlog, "Svc Lease : 0x%lX", srv_rec->ServiceLease); 1838 IBTF_DPRINTF_L4(cmlog, "Svc Key-hi: 0x%016llX", srv_rec->ServiceKey_hi); 1839 IBTF_DPRINTF_L4(cmlog, "Svc Key-lo: 0x%016llX", srv_rec->ServiceKey_lo); 1840 IBTF_DPRINTF_L4(cmlog, "Svc Name : %s", srv_rec->ServiceName); 1841 IBTF_DPRINTF_L4(cmlog, "Svc Data : "); 1842 for (i = 0; i < IB_SVC_DATA_LEN; i += 8) { 1843 IBTF_DPRINTF_L4(cmlog, 1844 "\t 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X, 0x%X", 1845 srv_rec->ServiceData[i], srv_rec->ServiceData[i+1], 1846 srv_rec->ServiceData[i+2], srv_rec->ServiceData[i+3], 1847 srv_rec->ServiceData[i+4], srv_rec->ServiceData[i+5], 1848 srv_rec->ServiceData[i+6], srv_rec->ServiceData[i+7]); 1849 } 1850 } 1851 1852 1853 /* 1854 * ibcm_dump_pathrec: 1855 * Dumps Path Records. 1856 * 1857 * Arguments: 1858 * path_rec - the pointer to sa_path_record_t struct. 1859 * 1860 * Return values: NONE 1861 */ 1862 void 1863 ibcm_dump_pathrec(sa_path_record_t *path_rec) 1864 { 1865 IBTF_DPRINTF_L5(cmlog, "Path Record:"); 1866 IBTF_DPRINTF_L5(cmlog, "SGID: (sn_prefix) %016llX", 1867 path_rec->SGID.gid_prefix); 1868 IBTF_DPRINTF_L5(cmlog, "SGID: (GUID) %016llX", 1869 path_rec->SGID.gid_guid); 1870 IBTF_DPRINTF_L5(cmlog, "DGID: (sn_prefix) %016llX", 1871 path_rec->DGID.gid_prefix); 1872 IBTF_DPRINTF_L5(cmlog, "DGID: (GUID) %016llX", 1873 path_rec->DGID.gid_guid); 1874 IBTF_DPRINTF_L5(cmlog, "SLID: %04X", path_rec->SLID); 1875 IBTF_DPRINTF_L5(cmlog, "DLID: %04X", path_rec->DLID); 1876 IBTF_DPRINTF_L5(cmlog, "Raw Traffic: %01X", path_rec->RawTraffic); 1877 IBTF_DPRINTF_L5(cmlog, "Flow Label: %05X", path_rec->FlowLabel); 1878 IBTF_DPRINTF_L5(cmlog, "Hop Limit: %02X", path_rec->HopLimit); 1879 IBTF_DPRINTF_L5(cmlog, "TClass: %02X", path_rec->TClass); 1880 IBTF_DPRINTF_L5(cmlog, "Reversible: %01X", path_rec->Reversible); 1881 IBTF_DPRINTF_L5(cmlog, "Numb Paths: %02d", path_rec->NumbPath); 1882 IBTF_DPRINTF_L5(cmlog, "P_Key: %04X", path_rec->P_Key); 1883 IBTF_DPRINTF_L5(cmlog, "SL: %02X", path_rec->SL); 1884 IBTF_DPRINTF_L5(cmlog, "Path MTU Selector: %01X", 1885 path_rec->MtuSelector); 1886 IBTF_DPRINTF_L5(cmlog, "Path MTU: %02X", path_rec->Mtu); 1887 IBTF_DPRINTF_L5(cmlog, "Path Rate Selector:%01X", 1888 path_rec->RateSelector); 1889 IBTF_DPRINTF_L5(cmlog, "Path Rate: %02X", path_rec->Rate); 1890 IBTF_DPRINTF_L5(cmlog, "Packet LT Selector:%01X", 1891 path_rec->PacketLifeTimeSelector); 1892 IBTF_DPRINTF_L5(cmlog, "Packet Life Time: %d (dec)", 1893 path_rec->PacketLifeTime); 1894 IBTF_DPRINTF_L5(cmlog, "Preference Bit: %02X", path_rec->Preference); 1895 } 1896 1897 1898 /* 1899 * ibcm_dump_node_rec: 1900 * Dumps Node Records. 1901 * 1902 * Arguments: 1903 * nrec - the pointer to sa_node_record_t struct. 1904 * 1905 * Return values: NONE 1906 */ 1907 void 1908 ibcm_dump_noderec(sa_node_record_t *nrec) 1909 { 1910 IBTF_DPRINTF_L5(cmlog, "ibcm_dump_noderec: Node Info Record"); 1911 IBTF_DPRINTF_L5(cmlog, "LID : %04X", nrec->LID); 1912 IBTF_DPRINTF_L5(cmlog, "Base Ver : %02X", nrec->NodeInfo.BaseVersion); 1913 IBTF_DPRINTF_L5(cmlog, "Class Ver : %02X", nrec->NodeInfo.ClassVersion); 1914 IBTF_DPRINTF_L5(cmlog, "Node Type : %02d", nrec->NodeInfo.NodeType); 1915 IBTF_DPRINTF_L5(cmlog, "Num Ports : %02X", nrec->NodeInfo.NumPorts); 1916 IBTF_DPRINTF_L5(cmlog, "SysImgGUID: %016llX", 1917 nrec->NodeInfo.SystemImageGUID); 1918 IBTF_DPRINTF_L5(cmlog, "NODE GUID : %016llX", nrec->NodeInfo.NodeGUID); 1919 IBTF_DPRINTF_L5(cmlog, "Port GUID : %016llX", nrec->NodeInfo.PortGUID); 1920 IBTF_DPRINTF_L5(cmlog, "PartionCap: %04X", nrec->NodeInfo.PartitionCap); 1921 IBTF_DPRINTF_L5(cmlog, "Device ID : %04X", nrec->NodeInfo.DeviceID); 1922 IBTF_DPRINTF_L5(cmlog, "Revision : %06X", nrec->NodeInfo.Revision); 1923 IBTF_DPRINTF_L5(cmlog, "LocalPort#: %02X", nrec->NodeInfo.LocalPortNum); 1924 IBTF_DPRINTF_L5(cmlog, "Vendor ID : %06X", nrec->NodeInfo.VendorID); 1925 IBTF_DPRINTF_L5(cmlog, "Description: %s", 1926 (char *)&nrec->NodeDescription); 1927 } 1928 #endif 1929 1930 1931 /* 1932 * ibcm_ibmf_analyze_error: 1933 * Checks IBMF status and determines appropriate ibt status. 1934 * 1935 * Arguments: 1936 * ibmf_status - IBMF Status 1937 * 1938 * Return values: 1939 * ibt_status_t 1940 */ 1941 ibt_status_t 1942 ibcm_ibmf_analyze_error(int ibmf_status) 1943 { 1944 if (ibt_check_failure(ibmf_status, NULL) != IBT_FAILURE_STANDARD) { 1945 /* 1946 * IBMF specific failure, return special error code 1947 * to the client so that it can retrieve any associated ENA. 1948 */ 1949 return (ibmf_status); 1950 } else if (ibmf_status == IBMF_TRANS_TIMEOUT) { 1951 return (IBT_IBMF_TIMEOUT); 1952 } else { 1953 /* 1954 * IBMF failed for some other reason, invalid arguments etc. 1955 * Analyze, log ENA with IBTF and obtain a special ibt_status_t 1956 * that indicates IBMF failure. 1957 */ 1958 if ((ibmf_status == IBMF_BAD_CLASS) || 1959 (ibmf_status == IBMF_BAD_HANDLE) || 1960 (ibmf_status == IBMF_BAD_QP_HANDLE) || 1961 (ibmf_status == IBMF_BAD_NODE) || 1962 (ibmf_status == IBMF_BAD_PORT) || 1963 (ibmf_status == IBMF_BAD_VERSION) || 1964 (ibmf_status == IBMF_BAD_FLAGS) || 1965 (ibmf_status == IBMF_BAD_SIZE) || 1966 (ibmf_status == IBMF_INVALID_GID) || 1967 (ibmf_status == IBMF_INVALID_ARG) || 1968 (ibmf_status == IBMF_INVALID_FIELD) || 1969 (ibmf_status == IBMF_UNSUPP_METHOD) || 1970 (ibmf_status == IBMF_UNSUPP_METHOD_ATTR)) { 1971 1972 /* 1973 * These errors, we should not see... 1974 * something really bad happened!. 1975 */ 1976 IBTF_DPRINTF_L2(cmlog, "ibcm_ibmf_analyze_error: " 1977 "Unexpected ERROR from IBMF - %d", ibmf_status); 1978 } 1979 return (ibt_get_module_failure(IBT_FAILURE_IBMF, 0)); 1980 } 1981 } 1982